diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000000000000000000000000000000000..85c8a0eca6e9fcf58ac9503bee1c6788e1a709cd --- /dev/null +++ b/.dockerignore @@ -0,0 +1,8 @@ +.git +.vscode + +/geometry +/input +/parameters +/build +/algo/external diff --git a/.gitignore b/.gitignore index b04e2e307cb6544c130e2b0f9c5d956a3124d6d6..fcc9f04b2a7a5a7db24b02214fed9cdf13aa97b6 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ /input /parameters /build +/algo/external # Comment some single files Dart*.cfg diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 784c7f305368c89855fa74e3b194f11d4562cdfc..64d2672d731fc2a329b7ff9a55b6d818b81699c3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,6 +2,7 @@ stages: - checkRepository - checkFormat - build + - package - finalise - documentation @@ -513,6 +514,52 @@ CbmRoot_Continuous: variables: <<: [*mergeVariables, *linux_variables] +BuildOnlineContainerMR: + stage: package + only: + - merge_requests + needs: + - CodeFormatCheck + - FileFormatCheck + - FileEndCheck + - FileLicenceCheck + tags: + - docker + image: + name: gcr.io/kaniko-project/executor:v1.16.0-debug + entrypoint: [""] + variables: + IMAGE_NAME: $CI_REGISTRY_IMAGE/cbm_online + IMAGE_TAG: mr$CI_MERGE_REQUEST_IID + DOCKERFILE: $CI_PROJECT_DIR/algo/containers/cbm_online/Dockerfile + script: + - mkdir -p /kaniko/.docker + - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json + - /kaniko/executor --context dir://$CI_PROJECT_DIR --dockerfile $DOCKERFILE --destination $IMAGE_NAME:$IMAGE_TAG + # Cache settings suggested by Copilot: --cache=true --cache-repo=${IMAGE_NAME} --cache-ttl=168h --cache-shared=true --cache-dir=/cache + # TODO: Test if these work + +# Create container for master and DC_* branches on merge +BuildOnlineContainerBranch: + stage: package + only: + - master@computing/cbmroot + - nightly_master@computing/cbmroot + - /^DC_.*$/@computing/cbmroot + tags: + - docker + image: + name: gcr.io/kaniko-project/executor:v1.16.0-debug + entrypoint: [""] + variables: + IMAGE_NAME: $CI_REGISTRY_IMAGE/cbm_online + IMAGE_TAG: $CI_COMMIT_BRANCH + DOCKERFILE: $CI_PROJECT_DIR/algo/containers/cbm_online/Dockerfile + script: + - mkdir -p /kaniko/.docker + - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json + - /kaniko/executor --context dir://$CI_PROJECT_DIR --dockerfile $DOCKERFILE --destination $IMAGE_NAME:$IMAGE_TAG + InformCodeOwners: stage: finalise tags: diff --git a/algo/CMakeLists.txt b/algo/CMakeLists.txt index cd2e1c91694315f70c56ce5b53715d4395beb66b..4a256fdd47012ee1cd6e0b8f49c559cd522204c6 100644 --- a/algo/CMakeLists.txt +++ b/algo/CMakeLists.txt @@ -1,3 +1,57 @@ +cmake_minimum_required(VERSION 3.14.0 FATAL_ERROR) +cmake_policy(VERSION 3.14...3.23) + +if (CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR) + set(CBM_ONLINE_STANDALONE ON) + project(CbmOnline) +else() + set(CBM_ONLINE_STANDALONE OFF) +endif() + +if (CBM_ONLINE_STANDALONE) + message(STATUS "Building online standalone") + set(CMAKE_EXPORT_COMPILE_COMMANDS ON) + set(CMAKE_CXX_STANDARD 17) + set(CMAKE_CXX_EXTENSIONS OFF) + + Set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib") + Set(EXECUTABLE_OUTPUT_PATH "${CMAKE_BINARY_DIR}/bin") + Set(INCLUDE_OUTPUT_DIRECTORY "${PROJECT_BINARY_DIR}/include") + set(CMAKE_INSTALL_LIBDIR lib) + + + # Set default build type + if (NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE "RelWithDebInfo" CACHE STRING "Build type" FORCE) + endif() + + #Fairsoft Modules Path, SIMPATH must always be set + if (NOT DEFINED SIMPATH) + message(FATAL_ERROR "SIMPATH not set") + endif() + list(PREPEND CMAKE_PREFIX_PATH ${SIMPATH}) + + find_package(OpenMP REQUIRED) + find_package(Boost REQUIRED COMPONENTS serialization regex filesystem log log_setup container program_options thread) + find_package(ROOT CONFIG REQUIRED GenVector) + find_package(Vc 1.4.1 CONFIG REQUIRED) + find_package(fmt CONFIG REQUIRED) + + include(ExternalProject) + + # cmake is symlinked to ../cmake by the install script in the container + # We use the symlink because some cmake macros use relative path from the project root + # to locate files (e.g. Gen_Exe_Script). + # Thus we need the same path if the project root changes. + set(CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake") + list(PREPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/modules") + include(CbmMacros) # for 'download_project_if_needed', 'Gen_Exe_Script' macro + include(../external/InstallGtest.cmake) + + add_subdirectory(../external external) +endif() + + add_subdirectory(log) add_subdirectory(data) add_subdirectory(test) @@ -100,7 +154,6 @@ target_link_libraries(Algo external::fles_logging external::fles_ipc external::fles_monitoring - libzmq cppzmq ) target_compile_definitions(Algo PUBLIC NO_ROOT) @@ -126,6 +179,11 @@ if (CMAKE_SYSTEM_NAME STREQUAL "Linux") endif() +if (CBM_ONLINE_STANDALONE) + # Add online binary + add_subdirectory(../reco/app/cbmreco cbmreco) +endif() + install(TARGETS Algo DESTINATION lib) install(DIRECTORY base/compat TYPE INCLUDE FILES_MATCHING PATTERN "*.h") install(DIRECTORY base/config TYPE INCLUDE FILES_MATCHING PATTERN "*.h") diff --git a/algo/containers/README b/algo/containers/README new file mode 100644 index 0000000000000000000000000000000000000000..60277c7e124844ddd6b60797e5c7c96f9a7c6b99 --- /dev/null +++ b/algo/containers/README @@ -0,0 +1,66 @@ +## Build container + +### Login + +Login via: `docker login hub.cbm.gsi.de` + +### Build Images + +Currently 3 images are build, each depending on the previous one(s): +1. `online_dev`: Contains all files to compile the CbmRoot online-code. This includes: + - Subset of FairSoft packages (boost, root, fmt) + - ROCm GPU compiler + hip development packages + - Various system libraries (+ header) +2. `online_runtime`: Minimal container that only contains libraries required to run + online reconstruction +3. `cbm_online`: Runnable container. Consists of `online_runtime` + cbm online binaries + +Each image is accompanied by a `build_and_push.sh` script, that builds the image +and pushes it to the container registry. + +To rebuild and publish the base images: +``` +cd online_dev +./build_and_push.sh +cd ../online_runtime +./build_and_push.sh +``` + +Building the online code and creating a new image, happens inside a container via kaniko. +To build new cbm_online-images locally, you first have to create a new build-image: +``` +cd <CbmRoot> +./algo/containers/cbm_online/make_build_image.sh <gitlab_user> <gitlab_password> +``` +**This will store your password in plain text in the container. Make sure to delete after you're done.** + +Then run the new container, to build a new cbm_online-image: +``` +docker run -it --rm cbm_online_builder +``` + +This will push a new container image `cbm_online:<gitlab_user>-debug` to the CbmRoot registry. + +You can use `test_run.sh` or `test_run_gpu.sh` scripts to test the image locally. +Make sure to pull the new image first with: `docker pull hub.cbm.gsi.de/computing/cbmroot/cbm_online:<gitlab_user>-debug` + +## Test on virgo + +### Login + +Login via: `apptainer remote login -u <user> docker://hub.cbm.gsi.de` + +`<user>` should be your gitlab-username. apptainer will ask for your password. + +### Pull new images + +`apptainer pull docker://hub.cbm.gsi.de/computing/cbmroot/cbm_online:<tag>` + +Use `master` as tag to pull the current CbmRoot master. Use `mr<N>` to pull the container for +MR `N`. + +### Run on debug partition + +Run the container on the virgo debug partition with GPUs: + +```srun --constraint=mi50 -p debug apptainer exec cbm_online.sif cbmreco -p /lustre/cbm/online/params -i /lustre/cbm/online/data/2391_node8_0_0000.tsa -d hip0 -n1``` diff --git a/algo/containers/cbm_online/Dockerfile b/algo/containers/cbm_online/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..963008a23a1e5f5f2a3c0b59e6f244ae76678552 --- /dev/null +++ b/algo/containers/cbm_online/Dockerfile @@ -0,0 +1,18 @@ +ARG REGISTRY=hub.cbm.gsi.de/computing/cbmroot + +FROM ${REGISTRY}/online_dev:latest as build + ARG SOURCE_DIR=/opt/cbm/src + WORKDIR ${SOURCE_DIR} + # Kaniko doesn't support RUN --mount, so we have to copy the source code instead + COPY . . + RUN algo/containers/cbm_online/scripts/install.sh + +FROM ${REGISTRY}/online_runtime:latest + ARG CBMROOT_PATH=/opt/cbm/cbmroot + ARG FAIRSOFT_PATH=/opt/cbm/fairsoft + COPY --from=build ${CBMROOT_PATH}/bin ${CBMROOT_PATH}/bin + COPY --from=build ${CBMROOT_PATH}/lib/*.so ${CBMROOT_PATH}/lib/ + + ENV LD_LIBRARY_PATH=${CBMROOT_PATH}/lib:${FAIRSOFT_PATH}/lib:/opt/rocm/lib:${LD_LIBRARY_PATH} + ENV PATH=${CBMROOT_PATH}/bin:${PATH} + CMD ["cbmreco"] diff --git a/algo/containers/cbm_online/DockerfileBuild b/algo/containers/cbm_online/DockerfileBuild new file mode 100644 index 0000000000000000000000000000000000000000..2efec38e0f0c087dd8dbd1576ab9f4c95eaa4791 --- /dev/null +++ b/algo/containers/cbm_online/DockerfileBuild @@ -0,0 +1,13 @@ +FROM gcr.io/kaniko-project/executor:v1.16.0-debug + ARG USER="username" + ARG PASSWORD="your_password" + ARG REGISTRY="hub.cbm.gsi.de" + ARG IMAGE="${REGISTRY}/computing/cbmroot/cbm_online" + ARG TAG="${USER}-debug" + RUN mkdir -p /kaniko/.docker + RUN echo "{\"auths\":{\"${REGISTRY}\":{\"username\":\"${USER}\",\"password\":\"${PASSWORD}\"}}}" > /kaniko/.docker/config.json + # TODO: Mount CbmRoot source instead of copying it into the container (-> same with authentification...) + COPY . /workspace + ENV IMAGE=${IMAGE} + ENV TAG=${TAG} + ENTRYPOINT ["/bin/sh" , "-xc", "/kaniko/executor --context dir:///workspace --dockerfile /workspace/algo/containers/cbm_online/Dockerfile --destination $IMAGE:$TAG"] diff --git a/algo/containers/cbm_online/make_build_image.sh b/algo/containers/cbm_online/make_build_image.sh new file mode 100755 index 0000000000000000000000000000000000000000..1e229720e0af2c8082d9c8d2b0816fec5296c429 --- /dev/null +++ b/algo/containers/cbm_online/make_build_image.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +set -e + +user="$1" +password="$2" + +script=$(readlink -f "$0") + +function check_arg() { + if [ -z "$1" ]; then + echo "Error: No $2 specified." + echo "Usage: $script <user> <password>" + exit 1 + fi +} + +check_arg "$user" "user" +check_arg "$password" "password" + +# This script must be run from the CbmRoot top level directory. +# Check for .git, .clang-format, and .gitlab-ci.yml +if [ ! -d .git ] || [ ! -f .clang-format ] || [ ! -f .gitlab-ci.yml ]; then + echo "Error: This script must be run from the CbmRoot top level directory." + exit 1 +fi + +registry="hub.cbm.gsi.de/computing/cbmroot" +image="cbm_online" + +build_args=" \ + --build-arg="USER=$user" \ + --build-arg="PASSWORD=$password" \ +" +dockerfile="algo/containers/cbm_online/DockerfileBuild" +docker build --progress plain $build_args -f $dockerfile -t cbm_online_builder . diff --git a/algo/containers/cbm_online/scripts/install.sh b/algo/containers/cbm_online/scripts/install.sh new file mode 100755 index 0000000000000000000000000000000000000000..18a28208de596e9b392d603e08d2f674f385c109 --- /dev/null +++ b/algo/containers/cbm_online/scripts/install.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +set -e + +srcdir=cbmroot +build_type=RelWithDebInfo +install_prefix=/opt/cbm/cbmroot +fairsoft_path=/opt/cbm/fairsoft +rocm_root=/opt/rocm +hip_archs="gfx906;gfx908" # MI50;MI100 +n_jobs=16 + +cmake_args="-DCMAKE_INSTALL_PREFIX=$install_prefix \ + -DCMAKE_BUILD_TYPE=$build_type \ + -DSIMPATH=$fairsoft_path \ + -DXPU_ENABLE_HIP=ON \ + -DXPU_ROCM_ROOT=$rocm_root \ + -DXPU_HIP_ARCH=$hip_archs \ +" + +ln -s ../cmake algo/cmake +cmake -S algo -B build $cmake_args +cmake --build build -j$n_jobs --target install diff --git a/algo/containers/cbm_online/test_run.sh b/algo/containers/cbm_online/test_run.sh new file mode 100755 index 0000000000000000000000000000000000000000..230149c3e3c5a8306b0050ebdaeb5259dd7767ba --- /dev/null +++ b/algo/containers/cbm_online/test_run.sh @@ -0,0 +1,22 @@ +#! /bin/bash + +set -e + +IMAGE=hub.cbm.gsi.de/fweig/containers/cbm_online:online-container +DOCKER_FLAGS=" \ + --rm \ + -it \ + --mount type=bind,source=$HOME/cbm/params,target=/opt/cbm/params \ + --mount type=bind,source=/home/cbmdata/mcbm22/2391,target=/opt/cbm/data \ +" + +CMD="cbmreco" +CMD_FLAGS=" \ + -p /opt/cbm/params \ + -i /opt/cbm/data/2391_node8_1_0000.tsa \ + -n 1 \ + --omp 1 \ +" + +docker run $DOCKER_FLAGS $IMAGE $CMD $CMD_FLAGS +# docker run $DOCKER_FLAGS $IMAGE ls -l /opt/cbm/data diff --git a/algo/containers/cbm_online/test_run_gpu.sh b/algo/containers/cbm_online/test_run_gpu.sh new file mode 100755 index 0000000000000000000000000000000000000000..c5559d63c0318ea189bda6964763fb4e52ecbb9c --- /dev/null +++ b/algo/containers/cbm_online/test_run_gpu.sh @@ -0,0 +1,30 @@ +#! /bin/bash + +set -e + +IMAGE=hub.cbm.gsi.de/fweig/containers/cbm_online:online-container +DOCKER_FLAGS=" \ + --rm \ + -it \ + --mount type=bind,source=$HOME/cbm/params,target=/opt/cbm/params \ + --mount type=bind,source=/home/cbmdata/mcbm22/2391,target=/opt/cbm/data \ + --device=/dev/kfd \ + --device=/dev/dri \ + --security-opt seccomp=unconfined \ + --group-add video +" + +CMD="cbmreco" +CMD_FLAGS=" \ + -p /opt/cbm/params \ + -i /opt/cbm/data/2391_node8_1_0000.tsa \ + -n 1 \ + --omp 1\ + -l info \ + -d hip1 \ +" +# CMD=xpuinfo +# CMD_FLAGS= + +docker run $DOCKER_FLAGS $IMAGE $CMD $CMD_FLAGS +# docker run $DOCKER_FLAGS $IMAGE ls -l /opt/cbm/data diff --git a/algo/containers/online_dev/Dockerfile b/algo/containers/online_dev/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..298903937feb59c9196de395f6a5d531b78b1cf5 --- /dev/null +++ b/algo/containers/online_dev/Dockerfile @@ -0,0 +1,10 @@ +FROM ubuntu:22.04 as rocm + RUN --mount=source=scripts,target=s s/install_rocm.sh + +FROM debian:12-slim + RUN --mount=source=scripts,target=s s/install_packages.sh + + ARG FAIRSOFT_VERSION=nov22p1 + RUN --mount=source=scripts,target=s s/install_fairsoft.sh ${FAIRSOFT_VERSION} + + COPY --from=rocm /opt/rocm /opt/rocm diff --git a/algo/containers/online_dev/build_and_push.sh b/algo/containers/online_dev/build_and_push.sh new file mode 100755 index 0000000000000000000000000000000000000000..28317a7d6b9697c1579d71e0c87f3a63eadb8b37 --- /dev/null +++ b/algo/containers/online_dev/build_and_push.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +#TODO: make ROCm & fairsoft version a parameter + +set -e + +registry="hub.cbm.gsi.de/computing/cbmroot" +image="online_dev" +tag="latest" + +docker build -t $registry/$image:$tag --push . diff --git a/algo/containers/online_dev/scripts/install_fairsoft.sh b/algo/containers/online_dev/scripts/install_fairsoft.sh new file mode 100755 index 0000000000000000000000000000000000000000..1ee11d8a5cc3be6571441366e59f09fe9a82a7ec --- /dev/null +++ b/algo/containers/online_dev/scripts/install_fairsoft.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +set -e + +version=$1 + +if [ -z "$version" ]; then + echo "Error: No version specified" + exit 1 +fi + +upstream="https://github.com/FairRootGroup/FairSoft.git" +cxx_std="17" +build_type="RelWithDebInfo" +packages="boost root fmt" +install_dir="/opt/cbm/fairsoft" + +git clone -b $version $upstream FairSoft +cd FairSoft + +cmake_defs=" \ + -DCMAKE_CXX_STANDARD=$cxx_std \ + -DCMAKE_BUILD_TYPE=$build_type \ + -DCMAKE_INSTALL_PREFIX=$install_dir \ +" + +cmake -S . -B build $cmake_defs +cmake --build build -j$(nproc) --target $packages + +# Cleanup +cd .. +rm -rf FairSoft diff --git a/algo/containers/online_dev/scripts/install_packages.sh b/algo/containers/online_dev/scripts/install_packages.sh new file mode 100755 index 0000000000000000000000000000000000000000..734ab7ba2bb0d4dabffb901195f598eb8aa0fee7 --- /dev/null +++ b/algo/containers/online_dev/scripts/install_packages.sh @@ -0,0 +1,29 @@ +#! /bin/bash + +set -e + +# System dependencies required by FairSoft +FAIRSOFT_DEPS="autoconf automake binutils \ + bison build-essential bzip2 ca-certificates coreutils \ + curl debianutils file findutils flex g++ gcc gfortran git gzip \ + hostname libbz2-dev libcurl4-openssl-dev libgsl-dev libicu-dev \ + libfftw3-dev \ + libgl1-mesa-dev libglu1-mesa-dev libgrpc++-dev \ + libncurses-dev libreadline-dev libsqlite3-dev libssl-dev libtool \ + libx11-dev libxerces-c-dev libxext-dev libxft-dev \ + libxml2-dev libxmu-dev libxpm-dev libyaml-cpp-dev lsb-release make patch cmake \ + python3-dev protobuf-compiler-grpc rsync sed subversion tar unzip wget xutils-dev xz-utils" + +# Additional system dependencies required by CBMROOT +CBMROOT_DEPS="libtbb-dev sqlite3 libsqlite3-dev libgsl-dev" + +# Additional packages (mostly for debugging) +ADDITIONAL_DEPS="gdb-minimal valgrind vim-tiny" + +export DEBIAN_FRONTEND=noninteractive +apt-get update +apt-get -y upgrade +apt-get -y install --no-install-recommends $FAIRSOFT_DEPS $CBMROOT_DEPS $ADDITIONAL_DEPS + +apt-get -y clean +rm -rf /var/lib/apt/lists/* diff --git a/algo/containers/online_dev/scripts/install_rocm.sh b/algo/containers/online_dev/scripts/install_rocm.sh new file mode 100755 index 0000000000000000000000000000000000000000..7e0aa8afdc5517d10fcd4b81f59c3a72b40ed33a --- /dev/null +++ b/algo/containers/online_dev/scripts/install_rocm.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +set -e + +ROCM_VERSION="5.4" +ROCM_PACKAGES="rocm-hip-runtime-dev5.4 hipcub-dev5.4" # Need to add version suffix, why?? + +export DEBIAN_FRONTEND=noninteractive + +apt-get update +apt-get -y upgrade + +apt-get -y install --no-install-recommends ca-certificates curl libnuma-dev gnupg +curl -sL https://repo.radeon.com/rocm/rocm.gpg.key | apt-key add - +printf "deb [arch=amd64] https://repo.radeon.com/rocm/apt/$ROCM_VERSION/ jammy main" | tee /etc/apt/sources.list.d/rocm.list +# printf "deb [arch=amd64] https://repo.radeon.com/amdgpu/$AMDGPU_VERSION/ubuntu jammy main" | tee /etc/apt/sources.list.d/amdgpu.list + +apt-get update +apt-get -y install --no-install-recommends $ROCM_PACKAGES + +apt-get -y clean +rm -rf /var/lib/apt/lists/* diff --git a/algo/containers/online_runtime/Dockerfile b/algo/containers/online_runtime/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..d7f24999d65dad869713f57ca7cfb9c79da2c98e --- /dev/null +++ b/algo/containers/online_runtime/Dockerfile @@ -0,0 +1,12 @@ +ARG RUNTIME_BASE_IMAGE=debian:12-slim +ARG DEV_IMAGE=hub.cbm.gsi.de/computing/cbmroot/online_dev:latest + +FROM ubuntu:22.04 as rocm_runtime + RUN --mount=source=scripts,target=s s/install_hip_runtime.sh + +FROM ${DEV_IMAGE} as online_dev +FROM ${RUNTIME_BASE_IMAGE} as online_runtime + RUN --mount=source=scripts,target=s s/install_packages.sh + + COPY --from=rocm_runtime /opt/rocm /opt/rocm + COPY --from=online_dev /opt/cbm/fairsoft/lib/libboost* /opt/cbm/fairsoft/lib/ diff --git a/algo/containers/online_runtime/build_and_push.sh b/algo/containers/online_runtime/build_and_push.sh new file mode 100755 index 0000000000000000000000000000000000000000..7c05888e00b27b00674ad87a2a82bcd9e7e69cb2 --- /dev/null +++ b/algo/containers/online_runtime/build_and_push.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +#TODO: make rocm version a parameter + +set -e + +registry="hub.cbm.gsi.de/computing/cbmroot" +image="online_runtime" +tag="latest" + +docker build -f Dockerfile -t $registry/$image:$tag --push . diff --git a/algo/containers/online_runtime/scripts/install_hip_runtime.sh b/algo/containers/online_runtime/scripts/install_hip_runtime.sh new file mode 100755 index 0000000000000000000000000000000000000000..6887db1ad10ff1607ef53dc5ad51a669d4e253aa --- /dev/null +++ b/algo/containers/online_runtime/scripts/install_hip_runtime.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +set -e + +ROCM_VERSION="5.4" +ROCM_PACKAGES="rocm-hip-runtime5.4" # Need to add version suffix, why?? + +export DEBIAN_FRONTEND=noninteractive + +apt-get update +apt-get -y upgrade + +apt-get -y install --no-install-recommends ca-certificates curl libnuma-dev gnupg +curl -sL https://repo.radeon.com/rocm/rocm.gpg.key | apt-key add - +printf "deb [arch=amd64] https://repo.radeon.com/rocm/apt/$ROCM_VERSION/ jammy main" | tee /etc/apt/sources.list.d/rocm.list +# printf "deb [arch=amd64] https://repo.radeon.com/amdgpu/$AMDGPU_VERSION/ubuntu jammy main" | tee /etc/apt/sources.list.d/amdgpu.list + +apt-get update +apt-get -y install --no-install-recommends $ROCM_PACKAGES + +apt-get -y clean +rm -rf /var/lib/apt/lists/* + +# Remove LLVM to save space +rm -rf /opt/rocm/llvm diff --git a/algo/containers/online_runtime/scripts/install_packages.sh b/algo/containers/online_runtime/scripts/install_packages.sh new file mode 100755 index 0000000000000000000000000000000000000000..2b6284ed6237ced6b0a388a2f797eb9add33a7f5 --- /dev/null +++ b/algo/containers/online_runtime/scripts/install_packages.sh @@ -0,0 +1,20 @@ +#! /bin/bash + +set -e + +# Runtime dependencies of CbmRoot (online only) +CBMROOT_DEPS="libstdc++6 libgomp1 libc6 libgcc-s1 libtbb12" + +# Runtime dependencies of ROCm +ROCM_DEPS="libnuma1 libdrm2 libdrm-amdgpu1" + +# Additional packages (mostly for debugging) +ADDITIONAL_DEPS="gdb-minimal valgrind vim-tiny" + +export DEBIAN_FRONTEND=noninteractive +apt-get update +apt-get -y upgrade +apt-get -y install --no-install-recommends $CBMROOT_DEPS $ROCM_DEPS $ADDITIONAL_DEPS + +apt-get -y clean +rm -rf /var/lib/apt/lists/* diff --git a/algo/data/CMakeLists.txt b/algo/data/CMakeLists.txt index eea39c14db2196d0152f4b17908fb5ec60182ce2..717ba53896c880649c4c07c1ff010fad8c14551b 100644 --- a/algo/data/CMakeLists.txt +++ b/algo/data/CMakeLists.txt @@ -2,54 +2,56 @@ # the array . # The extension is already found. Any number of sources could be listed here. +set(OFFLINE_DATA_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../core/data) + set(SRCS - ${CMAKE_SOURCE_DIR}/core/data/CbmDefs.cxx - ${CMAKE_SOURCE_DIR}/core/data/CbmAddress.cxx + ${OFFLINE_DATA_DIR}/CbmDefs.cxx + ${OFFLINE_DATA_DIR}/CbmAddress.cxx - ${CMAKE_SOURCE_DIR}/core/data/bmon/CbmBmonDigi.cxx + ${OFFLINE_DATA_DIR}/bmon/CbmBmonDigi.cxx - ${CMAKE_SOURCE_DIR}/core/data/sts/CbmStsAddress.cxx - ${CMAKE_SOURCE_DIR}/core/data/sts/CbmStsDigi.cxx + ${OFFLINE_DATA_DIR}/sts/CbmStsAddress.cxx + ${OFFLINE_DATA_DIR}/sts/CbmStsDigi.cxx - ${CMAKE_SOURCE_DIR}/core/data/rich/CbmRichDigi.cxx + ${OFFLINE_DATA_DIR}/rich/CbmRichDigi.cxx - ${CMAKE_SOURCE_DIR}/core/data/much/CbmMuchDigi.cxx - ${CMAKE_SOURCE_DIR}/core/data/much/CbmMuchAddress.cxx + ${OFFLINE_DATA_DIR}/much/CbmMuchDigi.cxx + ${OFFLINE_DATA_DIR}/much/CbmMuchAddress.cxx - ${CMAKE_SOURCE_DIR}/core/data/trd/CbmTrdDigi.cxx - ${CMAKE_SOURCE_DIR}/core/data/trd/CbmTrdRawMessageSpadic.cxx + ${OFFLINE_DATA_DIR}/trd/CbmTrdDigi.cxx + ${OFFLINE_DATA_DIR}/trd/CbmTrdRawMessageSpadic.cxx - ${CMAKE_SOURCE_DIR}/core/data/tof/CbmTofDigi.cxx - ${CMAKE_SOURCE_DIR}/core/data/tof/CbmTofAddress.cxx - ${CMAKE_SOURCE_DIR}/core/data/tof/CbmTofDetectorId.cxx - ${CMAKE_SOURCE_DIR}/core/data/tof/CbmTofDetectorId_v12b.cxx + ${OFFLINE_DATA_DIR}/tof/CbmTofDigi.cxx + ${OFFLINE_DATA_DIR}/tof/CbmTofAddress.cxx + ${OFFLINE_DATA_DIR}/tof/CbmTofDetectorId.cxx + ${OFFLINE_DATA_DIR}/tof/CbmTofDetectorId_v12b.cxx - ${CMAKE_SOURCE_DIR}/core/data/psd/CbmPsdDigi.cxx - ${CMAKE_SOURCE_DIR}/core/data/psd/CbmPsdAddress.cxx + ${OFFLINE_DATA_DIR}/psd/CbmPsdDigi.cxx + ${OFFLINE_DATA_DIR}/psd/CbmPsdAddress.cxx - ${CMAKE_SOURCE_DIR}/core/data/fsd/CbmFsdDigi.cxx - ${CMAKE_SOURCE_DIR}/core/data/fsd/CbmFsdAddress.cxx + ${OFFLINE_DATA_DIR}/fsd/CbmFsdDigi.cxx + ${OFFLINE_DATA_DIR}/fsd/CbmFsdAddress.cxx - ${CMAKE_SOURCE_DIR}/core/data/raw/CriGet4Mess001.cxx - ${CMAKE_SOURCE_DIR}/core/data/raw/StsXyterMessage.cxx + ${OFFLINE_DATA_DIR}/raw/CriGet4Mess001.cxx + ${OFFLINE_DATA_DIR}/raw/StsXyterMessage.cxx ) add_library(OnlineData SHARED ${SRCS}) target_include_directories(OnlineData - PUBLIC ${CMAKE_SOURCE_DIR}/core/data - PUBLIC ${CMAKE_SOURCE_DIR}/core/data/base - PUBLIC ${CMAKE_SOURCE_DIR}/core/data/bmon - PUBLIC ${CMAKE_SOURCE_DIR}/core/data/sts - PUBLIC ${CMAKE_SOURCE_DIR}/core/data/rich - PUBLIC ${CMAKE_SOURCE_DIR}/core/data/much - PUBLIC ${CMAKE_SOURCE_DIR}/core/data/trd - PUBLIC ${CMAKE_SOURCE_DIR}/core/data/tof - PUBLIC ${CMAKE_SOURCE_DIR}/core/data/psd - PUBLIC ${CMAKE_SOURCE_DIR}/core/data/fsd - PUBLIC ${CMAKE_SOURCE_DIR}/core/data/global - PUBLIC ${CMAKE_SOURCE_DIR}/core/data/raw + PUBLIC ${OFFLINE_DATA_DIR} + PUBLIC ${OFFLINE_DATA_DIR}/base + PUBLIC ${OFFLINE_DATA_DIR}/bmon + PUBLIC ${OFFLINE_DATA_DIR}/sts + PUBLIC ${OFFLINE_DATA_DIR}/rich + PUBLIC ${OFFLINE_DATA_DIR}/much + PUBLIC ${OFFLINE_DATA_DIR}/trd + PUBLIC ${OFFLINE_DATA_DIR}/tof + PUBLIC ${OFFLINE_DATA_DIR}/psd + PUBLIC ${OFFLINE_DATA_DIR}/fsd + PUBLIC ${OFFLINE_DATA_DIR}/global + PUBLIC ${OFFLINE_DATA_DIR}/raw ) target_compile_definitions(OnlineData PUBLIC NO_ROOT) diff --git a/external/CMakeLists.txt b/external/CMakeLists.txt index eea9de219dc9927cc175de5f9b396e4e774f14fa..2a8f0eedfa08643874fcb8c8415986ec1c322f8e 100644 --- a/external/CMakeLists.txt +++ b/external/CMakeLists.txt @@ -7,6 +7,10 @@ Option(DOWNLOAD_EXTERNALS "Download the code from the external repositories." ON) if(DOWNLOAD_EXTERNALS) + Include(InstallFlesnet.cmake) + Set(FLES_IPC_INCLUDE_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/flesnet/lib/fles_ipc PARENT_SCOPE) + Set(FLES_LOGGING_INCLUDE_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/flesnet/lib/logging PARENT_SCOPE) + download_project_if_needed(PROJECT cppzmq GIT_REPOSITORY "https://github.com/zeromq/cppzmq/" GIT_TAG "4f111562e7ce23d53bda53748d934ca523d650d7" @@ -16,14 +20,11 @@ if(DOWNLOAD_EXTERNALS) add_library(cppzmq INTERFACE) target_include_directories(cppzmq INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/cppzmq) + target_link_libraries(cppzmq INTERFACE external::zmq) # Install header files install(FILES cppzmq/zmq.hpp DESTINATION include) - Include(InstallFlesnet.cmake) - Set(FLES_IPC_INCLUDE_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/flesnet/lib/fles_ipc PARENT_SCOPE) - Set(FLES_LOGGING_INCLUDE_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/flesnet/lib/logging PARENT_SCOPE) - Add_Subdirectory(flib_dpb) download_project_if_needed(PROJECT gsl @@ -66,21 +67,23 @@ if(DOWNLOAD_EXTERNALS) Set(XPU_INCLUDE_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/xpu-dev/src PARENT_SCOPE) endif() + Include(InstallYamlCpp.cmake) - Include(InstallKFParticle.cmake) - Include(InstallNicaFemto.cmake) - Include(InstallAnalysisTree.cmake) - Include(InstallAnalysisTreeQA.cmake) + if (NOT CBM_ONLINE_STANDALONE) # Not required for online standalone + Include(InstallKFParticle.cmake) + Include(InstallNicaFemto.cmake) + Include(InstallAnalysisTree.cmake) + Include(InstallAnalysisTreeQA.cmake) - Include(InstallYamlCpp.cmake) + Include(InstallBBA.cmake) - Include(InstallBBA.cmake) + if(NOT BUILD_FOR_TIDY) + Include(InstallParameter.cmake) + Include(InstallInput.cmake) + Include(InstallGeometry.cmake) + endif() - if(NOT BUILD_FOR_TIDY) - Include(InstallParameter.cmake) - Include(InstallInput.cmake) - Include(InstallGeometry.cmake) endif() else() diff --git a/external/InstallFlesnet.cmake b/external/InstallFlesnet.cmake index 7a17ddf5e75c5400ac36b7aecec47de0c4a6b74f..65d774d077de35872480314601217976c102c4d7 100644 --- a/external/InstallFlesnet.cmake +++ b/external/InstallFlesnet.cmake @@ -8,6 +8,7 @@ set(FLESNET_VERSION e553a5fbd4d1ccc2c55ea11632c38f24516b8461) # 2023-10-12 set(FLESNET_SRC_URL "https://github.com/cbm-fles/flesnet") set(FLESNET_DESTDIR "${PROJECT_BINARY_DIR}/external/flesnet-prefix") +set(FLESNET_BIN_PREFIX "${FLESNET_DESTDIR}/src/flesnet-build") download_project_if_needed( PROJECT flesnet @@ -49,10 +50,22 @@ ExternalProject_Add( ${CMAKE_CURRENT_SOURCE_DIR}/flesnet BUILD_IN_SOURCE 0 LOG_DOWNLOAD 1 LOG_CONFIGURE 1 LOG_BUILD 1 LOG_INSTALL 1 - BUILD_COMMAND ${FLESNET_BUILD_COMMAND} logging monitoring fles_ipc - BUILD_BYPRODUCTS ${FLESNET_DESTDIR}/src/flesnet-build/lib/fles_ipc/${CMAKE_STATIC_LIBRARY_PREFIX}fles_ipc${CMAKE_STATIC_LIBRARY_SUFFIX} + BUILD_COMMAND ${FLESNET_BUILD_COMMAND} logging monitoring fles_ipc tsclient + BUILD_BYPRODUCTS + ${FLESNET_DESTDIR}/src/flesnet-build/lib/fles_ipc/${CMAKE_STATIC_LIBRARY_PREFIX}fles_ipc${CMAKE_STATIC_LIBRARY_SUFFIX} + ${FLESNET_BIN_PREFIX}/tsclient INSTALL_COMMAND "" ) +install(PROGRAMS ${FLESNET_BIN_PREFIX}/tsclient TYPE BIN) + +add_library(external::zmq STATIC IMPORTED GLOBAL) +add_dependencies(external::zmq flesnet) +set_target_properties(external::zmq PROPERTIES + IMPORTED_LOCATION ${FLESNET_DESTDIR}/src/flesnet-build/src/zeromq-build/lib/${CMAKE_STATIC_LIBRARY_PREFIX}zmq${CMAKE_STATIC_LIBRARY_SUFFIX} +) +target_include_directories(external::zmq INTERFACE + ${FLESNET_DESTDIR}/src/flesnet-build/src/zeromq/include +) add_library(external::fles_logging STATIC IMPORTED GLOBAL) add_dependencies(external::fles_logging flesnet)