diff --git a/algo/CMakeLists.txt b/algo/CMakeLists.txt index b57bfe43b8a7bb07a8638a580b70cc2244ef2677..19dacfe7d7988a80ee71cfd342b2228bc576a048 100644 --- a/algo/CMakeLists.txt +++ b/algo/CMakeLists.txt @@ -32,7 +32,7 @@ if (CBM_ONLINE_STANDALONE) list(PREPEND CMAKE_PREFIX_PATH ${SIMPATH}) find_package(OpenMP REQUIRED) - find_package(Boost REQUIRED COMPONENTS serialization regex filesystem log log_setup container program_options thread) + find_package(Boost REQUIRED COMPONENTS serialization regex filesystem log log_setup container program_options thread iostreams) find_package(ROOT CONFIG REQUIRED GenVector) find_package(Vc 1.4.1 CONFIG REQUIRED) find_package(fmt CONFIG REQUIRED) @@ -158,6 +158,7 @@ target_link_libraries(Algo fmt::fmt Boost::program_options Boost::filesystem + Boost::iostreams Boost::headers xpu external::yaml-cpp @@ -226,7 +227,7 @@ install( global/RecoResultsOutputArchive.h global/StorableRecoResults.h ca/TrackingChain.h - # NOTE: SZh 20.11.2023: + # NOTE: SZh 20.11.2023: # The ca/qa directory depends on the online qa classes, so for now it has to be a part of the Algo library. ca/qa/CaQaBuilder.h ca/qa/CaQaConfig.h diff --git a/algo/base/BuildInfo.h b/algo/base/BuildInfo.h index e96a9cb89c3d4ffa69058a13f983fca5c417276d..d044534a545f184c491114841cb10b052982d2ab 100644 --- a/algo/base/BuildInfo.h +++ b/algo/base/BuildInfo.h @@ -14,6 +14,11 @@ #define HAVE_OMP #endif +// Ensure we have the boost compression header AND flesnet is compiled with compression enabled +#if __has_include(<boost/iostreams/filter/zstd.hpp>) && defined(BOOST_IOS_HAS_ZSTD) +#define HAVE_ZSTD +#endif + namespace cbm::algo::BuildInfo { @@ -42,6 +47,13 @@ namespace cbm::algo::BuildInfo false; #endif + inline constexpr bool WITH_ZSTD = +#ifdef HAVE_ZSTD + true; +#else + false; +#endif + } // namespace cbm::algo::BuildInfo #endif // CBM_ALGO_BUILD_INFO_H diff --git a/algo/base/Options.cxx b/algo/base/Options.cxx index b69c98ee2f5d9219d1c459e62168e4fd0be4264d..536eb6fefd754479e439931ea14d647d47fe3507 100644 --- a/algo/base/Options.cxx +++ b/algo/base/Options.cxx @@ -74,6 +74,7 @@ Options::Options(int argc, char** argv) "write log messages to file") ("output-types,O", po::value(&fOutputTypes)->multitoken()->value_name("<types>"), "space seperated list of reconstruction output types (hit, digi, ...)") + ("compress-archive", po::bool_switch(&fCompressArchive)->default_value(false), "Enable compression for output archives") ("steps", po::value(&fRecoSteps)->multitoken()->default_value({Step::Unpack, Step::DigiTrigger, Step::LocalReco, Step::Tracking})->value_name("<steps>"), "space seperated list of reconstruction steps (unpack, digitrigger, localreco, ...)") ("systems,s", po::value(&fDetectors)->multitoken()->default_value({Subsystem::STS, Subsystem::TOF, Subsystem::BMON, Subsystem::MUCH, Subsystem::RICH, Subsystem::TRD, Subsystem::TRD2D})->value_name("<detectors>"), diff --git a/algo/base/Options.h b/algo/base/Options.h index 9eeaeec67df67a3f82be005b8dd0f153f6b6e996..c9959ffe28315032a024063e0f6b8405993ab1eb 100644 --- a/algo/base/Options.h +++ b/algo/base/Options.h @@ -50,6 +50,8 @@ namespace cbm::algo return std::find(fOutputTypes.begin(), fOutputTypes.end(), recoData) != fOutputTypes.end(); } + bool CompressArchive() const { return fCompressArchive; } + const std::vector<fles::Subsystem>& Detectors() const { return fDetectors; } bool HasDetector(fles::Subsystem detector) const { @@ -73,6 +75,7 @@ namespace cbm::algo int fNumOMPThreads = -1; std::vector<Step> fRecoSteps; std::vector<RecoData> fOutputTypes; + bool fCompressArchive = false; std::vector<fles::Subsystem> fDetectors; std::string fChildId = "00"; }; diff --git a/algo/global/Reco.cxx b/algo/global/Reco.cxx index 5ac8ae0479f62fffdfd5dcf1a4e7459ac303bd2c..4cde9b421791721816e4f39e032a3ae5e90825b3 100644 --- a/algo/global/Reco.cxx +++ b/algo/global/Reco.cxx @@ -3,6 +3,7 @@ Authors: Felix Weiglhofer [committer] */ #include "Reco.h" +#include "BuildInfo.h" #include "CbmDigiEvent.h" #include "Exceptions.h" #include "compat/OpenMP.h" @@ -36,6 +37,10 @@ void Reco::Validate(const Options& opts) if (hasOutputFile && !hasOutputType) { throw FatalError("Output file specified, but no output types given: -O <types> missing"); } + + if (!BuildInfo::WITH_ZSTD && opts.CompressArchive()) { + throw FatalError("Archive compression enabled but compiled without Zstd: Remove --archive-compression flag"); + } } void Reco::Init(const Options& opts) diff --git a/cmake/modules/FindBoostZstd.cmake b/cmake/modules/FindBoostZstd.cmake new file mode 100644 index 0000000000000000000000000000000000000000..4cb61659ece79fa5a65e3bc786af7c734142c748 --- /dev/null +++ b/cmake/modules/FindBoostZstd.cmake @@ -0,0 +1,24 @@ +# Copyright (C) 2024 FIAS Frankfurt Institute for Advanced Studies, Frankfurt / Main +# SPDX-License-Identifier: GPL-3.0-only +# Authors: Felix Weiglhofer [committer] + +set(CMAKE_REQUIRED_LIBRARIES Boost::iostreams) +check_cxx_source_compiles(" +#include <boost/iostreams/filter/zstd.hpp> +#include <boost/iostreams/filtering_stream.hpp> + +int main(int argc, char *argv[]) +{ + std::unique_ptr<boost::iostreams::filtering_istream> in_; + in_ = std::make_unique<boost::iostreams::filtering_istream>(); + in_->push(boost::iostreams::zstd_decompressor()); + + return 0; +}" BOOST_IOS_HAS_ZSTD) +unset(CMAKE_REQUIRED_LIBRARIES) + +if(BOOST_IOS_HAS_ZSTD) + message(STATUS "Boost::iostream with ZSTD filter found.") +else() + message(STATUS "Boost::iostream does not support ZSTD filter.") +endif() diff --git a/external/InstallFlesnet.cmake b/external/InstallFlesnet.cmake index 95b4c7004050a989bf769d2caaaa042ede8c4e10..9effb58bdb86fb81fb42b491243010b7d680407e 100644 --- a/external/InstallFlesnet.cmake +++ b/external/InstallFlesnet.cmake @@ -19,6 +19,8 @@ download_project_if_needed( TEST_FILE CMakeLists.txt ) +include(FindBoostZstd) + If(ProjectUpdated) File(REMOVE_RECURSE ${FLESNET_DESTDIR}) Message("flesnet source directory was changed so build directory was deleted") @@ -112,6 +114,9 @@ target_compile_definitions(external::fles_monitoring add_library(external::fles_ipc STATIC IMPORTED GLOBAL) add_dependencies(external::fles_ipc flesnet external::fles_logging) +if (BOOST_IOS_HAS_ZSTD) + target_compile_definitions(external::fles_ipc INTERFACE BOOST_IOS_HAS_ZSTD) +endif() set(dir_to_link ${FLESNET_DESTDIR}/src/flesnet-build/src/zeromq-build/lib/${CMAKE_STATIC_LIBRARY_PREFIX}zmq${CMAKE_STATIC_LIBRARY_SUFFIX} diff --git a/reco/app/cbmreco/main.cxx b/reco/app/cbmreco/main.cxx index 296671a49171fcdc317250eacf60933e9bcc605e..351a893907aaaae5cbb0a16909dc67742689f630 100644 --- a/reco/app/cbmreco/main.cxx +++ b/reco/app/cbmreco/main.cxx @@ -142,7 +142,7 @@ int main(int argc, char** argv) L_(info) << "CBMRECO buildType=" << BuildInfo::BUILD_TYPE << " gpuDebug=" << BuildInfo::GPU_DEBUG << " parallelSTL=" << BuildInfo::WITH_PARALLEL_ALGORITHM << " OMP=" << BuildInfo::WITH_OMP - << " commit=" << BuildInfo::GIT_HASH; + << " ZSTD=" << BuildInfo::WITH_ZSTD << " commit=" << BuildInfo::GIT_HASH; std::stringstream ss; for (int i = 0; i < argc; i++) { ss << argv[i] << " "; @@ -159,7 +159,11 @@ int main(int argc, char** argv) std::optional<RecoResultsOutputArchive> archive; if (!opts.OutputFile().empty()) { L_(info) << "Writing results to file: " << opts.OutputFile(); - archive.emplace(opts.OutputFile().string()); + fles::ArchiveCompression compression = fles::ArchiveCompression::None; + if (opts.CompressArchive()) { + compression = fles::ArchiveCompression::Zstd; + } + archive.emplace(opts.OutputFile().string(), compression); } int tsIdx = 0;