From bdbc058cf188ab641a3b76ad84d427475d02fdff Mon Sep 17 00:00:00 2001
From: Felix Weiglhofer <weiglhofer@fias.uni-frankfurt.de>
Date: Tue, 30 Jan 2024 12:55:09 +0000
Subject: [PATCH] cbmreco: Add flag to enable archive compression.
---
algo/CMakeLists.txt | 5 +++--
algo/base/BuildInfo.h | 12 ++++++++++++
algo/base/Options.cxx | 1 +
algo/base/Options.h | 3 +++
algo/global/Reco.cxx | 5 +++++
cmake/modules/FindBoostZstd.cmake | 24 ++++++++++++++++++++++++
external/InstallFlesnet.cmake | 5 +++++
reco/app/cbmreco/main.cxx | 8 ++++++--
8 files changed, 59 insertions(+), 4 deletions(-)
create mode 100644 cmake/modules/FindBoostZstd.cmake
diff --git a/algo/CMakeLists.txt b/algo/CMakeLists.txt
index b57bfe43b8..19dacfe7d7 100644
--- a/algo/CMakeLists.txt
+++ b/algo/CMakeLists.txt
@@ -32,7 +32,7 @@ if (CBM_ONLINE_STANDALONE)
list(PREPEND CMAKE_PREFIX_PATH ${SIMPATH})
find_package(OpenMP REQUIRED)
- find_package(Boost REQUIRED COMPONENTS serialization regex filesystem log log_setup container program_options thread)
+ find_package(Boost REQUIRED COMPONENTS serialization regex filesystem log log_setup container program_options thread iostreams)
find_package(ROOT CONFIG REQUIRED GenVector)
find_package(Vc 1.4.1 CONFIG REQUIRED)
find_package(fmt CONFIG REQUIRED)
@@ -158,6 +158,7 @@ target_link_libraries(Algo
fmt::fmt
Boost::program_options
Boost::filesystem
+ Boost::iostreams
Boost::headers
xpu
external::yaml-cpp
@@ -226,7 +227,7 @@ install(
global/RecoResultsOutputArchive.h
global/StorableRecoResults.h
ca/TrackingChain.h
- # NOTE: SZh 20.11.2023:
+ # NOTE: SZh 20.11.2023:
# The ca/qa directory depends on the online qa classes, so for now it has to be a part of the Algo library.
ca/qa/CaQaBuilder.h
ca/qa/CaQaConfig.h
diff --git a/algo/base/BuildInfo.h b/algo/base/BuildInfo.h
index e96a9cb89c..d044534a54 100644
--- a/algo/base/BuildInfo.h
+++ b/algo/base/BuildInfo.h
@@ -14,6 +14,11 @@
#define HAVE_OMP
#endif
+// Ensure we have the boost compression header AND flesnet is compiled with compression enabled
+#if __has_include(<boost/iostreams/filter/zstd.hpp>) && defined(BOOST_IOS_HAS_ZSTD)
+#define HAVE_ZSTD
+#endif
+
namespace cbm::algo::BuildInfo
{
@@ -42,6 +47,13 @@ namespace cbm::algo::BuildInfo
false;
#endif
+ inline constexpr bool WITH_ZSTD =
+#ifdef HAVE_ZSTD
+ true;
+#else
+ false;
+#endif
+
} // namespace cbm::algo::BuildInfo
#endif // CBM_ALGO_BUILD_INFO_H
diff --git a/algo/base/Options.cxx b/algo/base/Options.cxx
index b69c98ee2f..536eb6fefd 100644
--- a/algo/base/Options.cxx
+++ b/algo/base/Options.cxx
@@ -74,6 +74,7 @@ Options::Options(int argc, char** argv)
"write log messages to file")
("output-types,O", po::value(&fOutputTypes)->multitoken()->value_name("<types>"),
"space seperated list of reconstruction output types (hit, digi, ...)")
+ ("compress-archive", po::bool_switch(&fCompressArchive)->default_value(false), "Enable compression for output archives")
("steps", po::value(&fRecoSteps)->multitoken()->default_value({Step::Unpack, Step::DigiTrigger, Step::LocalReco, Step::Tracking})->value_name("<steps>"),
"space seperated list of reconstruction steps (unpack, digitrigger, localreco, ...)")
("systems,s", po::value(&fDetectors)->multitoken()->default_value({Subsystem::STS, Subsystem::TOF, Subsystem::BMON, Subsystem::MUCH, Subsystem::RICH, Subsystem::TRD, Subsystem::TRD2D})->value_name("<detectors>"),
diff --git a/algo/base/Options.h b/algo/base/Options.h
index 9eeaeec67d..c9959ffe28 100644
--- a/algo/base/Options.h
+++ b/algo/base/Options.h
@@ -50,6 +50,8 @@ namespace cbm::algo
return std::find(fOutputTypes.begin(), fOutputTypes.end(), recoData) != fOutputTypes.end();
}
+ bool CompressArchive() const { return fCompressArchive; }
+
const std::vector<fles::Subsystem>& Detectors() const { return fDetectors; }
bool HasDetector(fles::Subsystem detector) const
{
@@ -73,6 +75,7 @@ namespace cbm::algo
int fNumOMPThreads = -1;
std::vector<Step> fRecoSteps;
std::vector<RecoData> fOutputTypes;
+ bool fCompressArchive = false;
std::vector<fles::Subsystem> fDetectors;
std::string fChildId = "00";
};
diff --git a/algo/global/Reco.cxx b/algo/global/Reco.cxx
index 5ac8ae0479..4cde9b4217 100644
--- a/algo/global/Reco.cxx
+++ b/algo/global/Reco.cxx
@@ -3,6 +3,7 @@
Authors: Felix Weiglhofer [committer] */
#include "Reco.h"
+#include "BuildInfo.h"
#include "CbmDigiEvent.h"
#include "Exceptions.h"
#include "compat/OpenMP.h"
@@ -36,6 +37,10 @@ void Reco::Validate(const Options& opts)
if (hasOutputFile && !hasOutputType) {
throw FatalError("Output file specified, but no output types given: -O <types> missing");
}
+
+ if (!BuildInfo::WITH_ZSTD && opts.CompressArchive()) {
+ throw FatalError("Archive compression enabled but compiled without Zstd: Remove --archive-compression flag");
+ }
}
void Reco::Init(const Options& opts)
diff --git a/cmake/modules/FindBoostZstd.cmake b/cmake/modules/FindBoostZstd.cmake
new file mode 100644
index 0000000000..4cb61659ec
--- /dev/null
+++ b/cmake/modules/FindBoostZstd.cmake
@@ -0,0 +1,24 @@
+# Copyright (C) 2024 FIAS Frankfurt Institute for Advanced Studies, Frankfurt / Main
+# SPDX-License-Identifier: GPL-3.0-only
+# Authors: Felix Weiglhofer [committer]
+
+set(CMAKE_REQUIRED_LIBRARIES Boost::iostreams)
+check_cxx_source_compiles("
+#include <boost/iostreams/filter/zstd.hpp>
+#include <boost/iostreams/filtering_stream.hpp>
+
+int main(int argc, char *argv[])
+{
+ std::unique_ptr<boost::iostreams::filtering_istream> in_;
+ in_ = std::make_unique<boost::iostreams::filtering_istream>();
+ in_->push(boost::iostreams::zstd_decompressor());
+
+ return 0;
+}" BOOST_IOS_HAS_ZSTD)
+unset(CMAKE_REQUIRED_LIBRARIES)
+
+if(BOOST_IOS_HAS_ZSTD)
+ message(STATUS "Boost::iostream with ZSTD filter found.")
+else()
+ message(STATUS "Boost::iostream does not support ZSTD filter.")
+endif()
diff --git a/external/InstallFlesnet.cmake b/external/InstallFlesnet.cmake
index 95b4c70040..9effb58bdb 100644
--- a/external/InstallFlesnet.cmake
+++ b/external/InstallFlesnet.cmake
@@ -19,6 +19,8 @@ download_project_if_needed(
TEST_FILE CMakeLists.txt
)
+include(FindBoostZstd)
+
If(ProjectUpdated)
File(REMOVE_RECURSE ${FLESNET_DESTDIR})
Message("flesnet source directory was changed so build directory was deleted")
@@ -112,6 +114,9 @@ target_compile_definitions(external::fles_monitoring
add_library(external::fles_ipc STATIC IMPORTED GLOBAL)
add_dependencies(external::fles_ipc flesnet external::fles_logging)
+if (BOOST_IOS_HAS_ZSTD)
+ target_compile_definitions(external::fles_ipc INTERFACE BOOST_IOS_HAS_ZSTD)
+endif()
set(dir_to_link
${FLESNET_DESTDIR}/src/flesnet-build/src/zeromq-build/lib/${CMAKE_STATIC_LIBRARY_PREFIX}zmq${CMAKE_STATIC_LIBRARY_SUFFIX}
diff --git a/reco/app/cbmreco/main.cxx b/reco/app/cbmreco/main.cxx
index 296671a491..351a893907 100644
--- a/reco/app/cbmreco/main.cxx
+++ b/reco/app/cbmreco/main.cxx
@@ -142,7 +142,7 @@ int main(int argc, char** argv)
L_(info) << "CBMRECO buildType=" << BuildInfo::BUILD_TYPE << " gpuDebug=" << BuildInfo::GPU_DEBUG
<< " parallelSTL=" << BuildInfo::WITH_PARALLEL_ALGORITHM << " OMP=" << BuildInfo::WITH_OMP
- << " commit=" << BuildInfo::GIT_HASH;
+ << " ZSTD=" << BuildInfo::WITH_ZSTD << " commit=" << BuildInfo::GIT_HASH;
std::stringstream ss;
for (int i = 0; i < argc; i++) {
ss << argv[i] << " ";
@@ -159,7 +159,11 @@ int main(int argc, char** argv)
std::optional<RecoResultsOutputArchive> archive;
if (!opts.OutputFile().empty()) {
L_(info) << "Writing results to file: " << opts.OutputFile();
- archive.emplace(opts.OutputFile().string());
+ fles::ArchiveCompression compression = fles::ArchiveCompression::None;
+ if (opts.CompressArchive()) {
+ compression = fles::ArchiveCompression::Zstd;
+ }
+ archive.emplace(opts.OutputFile().string(), compression);
}
int tsIdx = 0;
--
GitLab