From c5ac3b1ec536efae4742eb3fd1f59e36875b3b92 Mon Sep 17 00:00:00 2001 From: Sandro Wenzel Date: Wed, 11 Nov 2026 11:39:51 +0100 Subject: [PATCH] Set the static TLS reserve for Geant4 inside the sim executables This fixes o2-sim on the GRID, where Geant4 cannot be loaded since it uses the initial-exec TLS model, and adds a unit test. - A Geant4 loaded by dlopen needs a glibc static TLS reserve, which the Geant4 environment sets in GLIBC_TUNABLES. - In GRID jobs the variable arrives truncated as glibc.rtld.optional_static_tls, without its value, and is ignored. - o2-sim now sets the reserve at start, so the workers it launches have it. - The executables that load Geant4 check the reserve first in main and, if it is missing or too small, restart themselves once with it set, since glibc reads tunables only at program start. - Other tunables are kept, and a larger reserve set by the user is left as is. https://its.cern.ch/jira/browse/O2-7305 Co-Authored-By: Claude Opus 5.5 --- Detectors/gconfig/CMakeLists.txt | 6 + .../include/SimSetup/StaticTLSReserve.h | 127 ++++++++++++++++++ .../gconfig/test/testStaticTLSReserve.cxx | 70 ++++++++++ run/O2SimDeviceRunner.cxx | 3 + run/o2sim.cxx | 3 + run/o2sim_evalmat.cxx | 3 + run/o2sim_parallel.cxx | 3 + 7 files changed, 215 insertions(+) create mode 100644 Detectors/gconfig/include/SimSetup/StaticTLSReserve.h create mode 100644 Detectors/gconfig/test/testStaticTLSReserve.cxx diff --git a/Detectors/gconfig/CMakeLists.txt b/Detectors/gconfig/CMakeLists.txt index d35bbed3deb36..cb4af087cef05 100644 --- a/Detectors/gconfig/CMakeLists.txt +++ b/Detectors/gconfig/CMakeLists.txt @@ -74,6 +74,12 @@ o2_target_root_dictionary(SimSetup include/SimSetup/MCReplayParam.h LINKDEF src/GConfLinkDef.h) +o2_add_test(StaticTLSReserve + SOURCES test/testStaticTLSReserve.cxx + COMPONENT_NAME SimSetup + PUBLIC_LINK_LIBRARIES O2::SimSetup + LABELS simsetup) + o2_add_test_root_macro(UserDecay.C PUBLIC_LINK_LIBRARIES O2::SimSetup LABELS simsetup) diff --git a/Detectors/gconfig/include/SimSetup/StaticTLSReserve.h b/Detectors/gconfig/include/SimSetup/StaticTLSReserve.h new file mode 100644 index 0000000000000..5c6c8c98a1961 --- /dev/null +++ b/Detectors/gconfig/include/SimSetup/StaticTLSReserve.h @@ -0,0 +1,127 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#ifndef O2_SIMSETUP_STATICTLSRESERVE_H +#define O2_SIMSETUP_STATICTLSRESERVE_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace o2 +{ + +/// Static TLS reserve in bytes for a Geant4 built with the initial-exec TLS model and loaded by dlopen +constexpr unsigned long Geant4StaticTLSReserve = 32768; + +/// The GLIBC_TUNABLES value with a static TLS reserve of at least `reserve` bytes, or an empty string if +/// `tunables` already has one. As in glibc, the last reserve counts and numbers may be hex (0x) or octal (0). +/// Other tunables are kept; entries without a value are dropped, since glibc then ignores the whole string. +inline std::string withStaticTLSReserve(const char* tunables, unsigned long reserve = Geant4StaticTLSReserve) +{ + constexpr std::string_view name = "glibc.rtld.optional_static_tls"; + std::string kept; + bool haveReserve = false; + bool malformed = false; + unsigned long current = 0; + std::string_view rest = tunables ? tunables : ""; + while (!rest.empty()) { + const auto colon = rest.find(':'); + const std::string_view entry = rest.substr(0, colon); + rest = colon == std::string_view::npos ? std::string_view{} : rest.substr(colon + 1); + const auto equal = entry.find('='); + if (equal == std::string_view::npos || equal == 0) { + malformed = true; // also the truncated reserve as it arrives in GRID jobs + continue; + } + if (entry.substr(0, equal) == name) { + std::string_view value = entry.substr(equal + 1); + int base = 10; + if (value.size() > 2 && value[0] == '0' && (value[1] == 'x' || value[1] == 'X')) { + base = 16; + value.remove_prefix(2); + } else if (value.size() > 1 && value[0] == '0') { + base = 8; + value.remove_prefix(1); + } + unsigned long bytes = 0; + const auto [end, ec] = std::from_chars(value.data(), value.data() + value.size(), bytes, base); + if (!value.empty() && ec == std::errc() && end == value.data() + value.size()) { + haveReserve = true; + current = bytes; + } else { + haveReserve = false; // a broken last value leaves no reserve + } + continue; // our own entry replaces all of them + } + kept += (kept.empty() ? "" : ":") + std::string(entry); + } + if (!malformed && haveReserve && current >= reserve) { + return {}; + } + return std::string(name) + "=" + std::to_string(reserve) + (kept.empty() ? "" : ":" + kept); +} + +/// Sets the static TLS reserve in the environment, for the programs started from here. Returns true if it changed. +inline bool setStaticTLSReserve(unsigned long reserve = Geant4StaticTLSReserve) +{ + const auto tunables = withStaticTLSReserve(getenv("GLIBC_TUNABLES"), reserve); + if (tunables.empty()) { + return false; + } + setenv("GLIBC_TUNABLES", tunables.c_str(), 1); + return true; +} + +/// Restarts the program once with the static TLS reserve if it lacks one. glibc reads GLIBC_TUNABLES only at +/// program start, so the reserve cannot be set later. Call this first in main, before any thread is started. +inline void ensureStaticTLSReserve([[maybe_unused]] char* argv[], + [[maybe_unused]] unsigned long reserve = Geant4StaticTLSReserve) +{ +#ifdef __linux__ + constexpr const char* guard = "O2_STATIC_TLS_REEXEC"; + if (getenv(guard)) { + unsetenv(guard); // restarted already; do not loop if glibc ignores the tunable + return; + } + if (withStaticTLSReserve(getenv("GLIBC_TUNABLES"), reserve).empty()) { + return; + } + // the real path keeps the process name; /proc/self/exe would rename it to "exe" + char self[PATH_MAX]; + const auto length = readlink("/proc/self/exe", self, sizeof(self) - 1); + if (length <= 0) { + return; + } + self[length] = '\0'; + const char* base = strrchr(self, '/'); + if (strncmp(base ? base + 1 : self, "ld-", 3) == 0) { + return; // started through the dynamic loader, which a restart would not reproduce + } + setStaticTLSReserve(reserve); + setenv(guard, "1", 1); + fprintf(stderr, "Restarting %s with GLIBC_TUNABLES=%s for the static TLS of Geant4\n", self, getenv("GLIBC_TUNABLES")); + execv(self, argv); + const int error = errno; + unsetenv(guard); + fprintf(stderr, "Restart of %s failed: %s\n", self, strerror(error)); +#endif +} + +} // namespace o2 + +#endif diff --git a/Detectors/gconfig/test/testStaticTLSReserve.cxx b/Detectors/gconfig/test/testStaticTLSReserve.cxx new file mode 100644 index 0000000000000..9d19e7a111392 --- /dev/null +++ b/Detectors/gconfig/test/testStaticTLSReserve.cxx @@ -0,0 +1,70 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#define BOOST_TEST_MODULE Test SimSetup StaticTLSReserve +#define BOOST_TEST_MAIN +#define BOOST_TEST_DYN_LINK +#include +#include "SimSetup/StaticTLSReserve.h" + +using o2::withStaticTLSReserve; + +BOOST_AUTO_TEST_CASE(missing_or_broken_reserve_is_added) +{ + const std::string reserve = "glibc.rtld.optional_static_tls=32768"; + BOOST_CHECK_EQUAL(withStaticTLSReserve(nullptr, 32768), reserve); + BOOST_CHECK_EQUAL(withStaticTLSReserve("", 32768), reserve); + // the value as it arrives in GRID jobs + BOOST_CHECK_EQUAL(withStaticTLSReserve("glibc.rtld.optional_static_tls", 32768), reserve); + BOOST_CHECK_EQUAL(withStaticTLSReserve("glibc.rtld.optional_static_tls=", 32768), reserve); + BOOST_CHECK_EQUAL(withStaticTLSReserve("glibc.rtld.optional_static_tls=512", 32768), reserve); + BOOST_CHECK_EQUAL(withStaticTLSReserve("glibc.rtld.optional_static_tls=-1", 32768), reserve); + BOOST_CHECK_EQUAL(withStaticTLSReserve("glibc.rtld.optional_static_tls=32768x", 32768), reserve); +} + +BOOST_AUTO_TEST_CASE(sufficient_reserve_is_kept) +{ + BOOST_CHECK_EQUAL(withStaticTLSReserve("glibc.rtld.optional_static_tls=32768", 32768), ""); + BOOST_CHECK_EQUAL(withStaticTLSReserve("glibc.rtld.optional_static_tls=1048576", 32768), ""); + BOOST_CHECK_EQUAL(withStaticTLSReserve("glibc.malloc.arena_max=4:glibc.rtld.optional_static_tls=65536", 32768), ""); + // glibc also takes hex and octal + BOOST_CHECK_EQUAL(withStaticTLSReserve("glibc.rtld.optional_static_tls=0x100000", 32768), ""); + BOOST_CHECK_EQUAL(withStaticTLSReserve("glibc.rtld.optional_static_tls=0100000", 32768), ""); + BOOST_CHECK_EQUAL(withStaticTLSReserve("glibc.rtld.optional_static_tls=0x200", 32768), "glibc.rtld.optional_static_tls=32768"); +} + +BOOST_AUTO_TEST_CASE(malformed_entries_are_dropped) +{ + // glibc ignores the whole string if one entry has no value, so a good reserve next to it does not count + BOOST_CHECK_EQUAL(withStaticTLSReserve("glibc.rtld.optional_static_tls=32768:glibc.malloc.arena_max", 32768), + "glibc.rtld.optional_static_tls=32768"); + BOOST_CHECK_EQUAL(withStaticTLSReserve("glibc.malloc.arena_max:glibc.malloc.tcache_count=0", 32768), + "glibc.rtld.optional_static_tls=32768:glibc.malloc.tcache_count=0"); +} + +BOOST_AUTO_TEST_CASE(other_tunables_are_kept) +{ + BOOST_CHECK_EQUAL(withStaticTLSReserve("glibc.malloc.arena_max=4", 32768), + "glibc.rtld.optional_static_tls=32768:glibc.malloc.arena_max=4"); + BOOST_CHECK_EQUAL(withStaticTLSReserve("glibc.rtld.optional_static_tls:glibc.malloc.arena_max=4", 32768), + "glibc.rtld.optional_static_tls=32768:glibc.malloc.arena_max=4"); + // a longer name is a different tunable + BOOST_CHECK_EQUAL(withStaticTLSReserve("glibc.rtld.optional_static_tlsx=1", 32768), + "glibc.rtld.optional_static_tls=32768:glibc.rtld.optional_static_tlsx=1"); +} + +BOOST_AUTO_TEST_CASE(last_reserve_counts) +{ + // glibc applies the entries in order, so a later small reserve wins over an earlier large one + BOOST_CHECK_EQUAL(withStaticTLSReserve("glibc.rtld.optional_static_tls=65536:glibc.rtld.optional_static_tls=512", 32768), + "glibc.rtld.optional_static_tls=32768"); + BOOST_CHECK_EQUAL(withStaticTLSReserve("glibc.rtld.optional_static_tls=512:glibc.rtld.optional_static_tls=65536", 32768), ""); +} diff --git a/run/O2SimDeviceRunner.cxx b/run/O2SimDeviceRunner.cxx index 11da3792d0ac5..96742d1a6976e 100644 --- a/run/O2SimDeviceRunner.cxx +++ b/run/O2SimDeviceRunner.cxx @@ -39,6 +39,7 @@ #include "rapidjson/document.h" #include "rapidjson/stringbuffer.h" #include "rapidjson/filereadstream.h" +#include namespace bpo = boost::program_options; std::vector gChildProcesses; // global vector of child pids @@ -267,6 +268,8 @@ bool waitForControlInput(int workerID) int main(int argc, char* argv[]) { + // Geant4 is loaded by dlopen and may need a static TLS reserve, which glibc only takes at program start + o2::ensureStaticTLSReserve(argv); struct sigaction act; memset(&act, 0, sizeof act); sigemptyset(&act.sa_mask); diff --git a/run/o2sim.cxx b/run/o2sim.cxx index 4a9cdbb4bdf90..c82df14b3ac45 100644 --- a/run/o2sim.cxx +++ b/run/o2sim.cxx @@ -13,9 +13,12 @@ #include #include #include +#include int main(int argc, char* argv[]) { + // Geant4 is loaded by dlopen and may need a static TLS reserve, which glibc only takes at program start + o2::ensureStaticTLSReserve(argv); TStopwatch timer; timer.Start(); auto& conf = o2::conf::SimConfig::Instance(); diff --git a/run/o2sim_evalmat.cxx b/run/o2sim_evalmat.cxx index 77ce4ace9ec84..dec95a1e87acf 100644 --- a/run/o2sim_evalmat.cxx +++ b/run/o2sim_evalmat.cxx @@ -14,9 +14,12 @@ #include #include #include +#include int main(int argc, char* argv[]) { + // Geant4 is loaded by dlopen and may need a static TLS reserve, which glibc only takes at program start + o2::ensureStaticTLSReserve(argv); TStopwatch timer; timer.Start(); diff --git a/run/o2sim_parallel.cxx b/run/o2sim_parallel.cxx index 08e3e24935d0c..2a44592df117a 100644 --- a/run/o2sim_parallel.cxx +++ b/run/o2sim_parallel.cxx @@ -48,6 +48,7 @@ #include "SimPublishChannelHelper.h" #include #include +#include std::string getServerLogName() { @@ -435,6 +436,8 @@ std::vector checkArgs(int argc, char* argv[]) // for parallel simulation int main(int argc, char* argv[]) { + // for the workers, which load Geant4 by dlopen; glibc takes the static TLS reserve only at program start + o2::setStaticTLSReserve(); LOG(info) << "This is o2-sim version " << o2::fullVersion() << " (" << o2::gitRevision() << ")"; LOG(info) << o2::getBuildInfo();