From c4411913da9d916e5add792e4cae9e6fbbb790a3 Mon Sep 17 00:00:00 2001 From: shahoian Date: Tue, 17 Nov 2026 16:38:15 +0100 Subject: [PATCH] Mutex-free TGeo material budget for multithreaded users GeometryManager::meanMaterialBudget(Ext) used the shared TGeo navigator under sTGMutex, serializing all callers (Propagator TGeo branch, LUT building). Add opt-in per-thread navigators: * GeometryManager::setMaxThreads(n): wraps TGeoManager::SetMaxThreads and enables per-thread navigation; returns the effective capacity. * GeometryManager::getThreadNavigator(): thread_local navigator, created on first use, removed at thread exit. nullptr if not enabled. Throws std::runtime_error if more threads than declared ask for one. * meanMaterialBudget / meanMaterialBudgetExt (and thus Propagator, unchanged) take the calling thread's navigator when enabled, otherwise the shared one under the mutex, as before. meanMaterialBudgetExt now navigates via a TGeoNavigator instead of the gGeoManager global API. * getOriginalMatrix / getOriginalMatrixFromPath navigate the same way (they also no longer re-lock sTGMutex when getOriginalMatrix(symname) falls back to getOriginalMatrixFromPath, which self-deadlocked). * rebuildVoxels(): the voxels, shared by all navigators and rebuilt lazily after TGeoPhysicalNode::Align, are rebuilt serially (also in setMaxThreads). * MatLayerCylSet LUT build uses the same mechanism and limits its parallelism to the enabled capacity. * Unit test testGeometryManagerMT. How to use in code calling Propagator (or meanMaterialBudget) from threads: 1. Load the geometry (and apply alignment). 2. Serially, BEFORE starting the worker threads, call once o2::base::GeometryManager::setMaxThreads(nThreads); with nThreads >= the maximum number of threads which will ever use TGeo in the process (workers other than the calling thread). 3. Call Propagator methods (USEMatCorrTGeo or TGeo fall-back without LUT) from the threads as usual: no further change needed. Without step 2 behaviour is unchanged (thread-safe but mutexed). MatCorrType::USEMatCorrLUT never touches TGeo. Corner cases to avoid (TGeoManager is not designed for them): * setMaxThreads can be called effectively once per geometry: TGeo would reset the thread slots of threads already navigating. Repeated calls with n <= capacity are no-ops, a larger n is refused with an error. * TGeo thread slots (TGeoManager::ThreadId) are never released: every distinct thread which navigated keeps its slot after exit. With thread churn (short-lived threads) use a thread pool, otherwise slots run out and getThreadNavigator throws. Never silently fall back to the shared navigator in multithread mode: it would index the per-volume thread data with the calling thread's slot, possibly out of range. * Direct use of the gGeoManager navigation API (cd, InitTrack, GetCurrentNode, ...) from a worker thread is safe only after that thread has called getThreadNavigator() (or called setMaxThreads): GetCurrentNavigator returns nullptr for threads without a navigator. Prefer calling the methods of the navigator returned by getThreadNavigator(). Any TGeo navigation from a new thread consumes a slot. * TGeoPhysicalNode::Align after setMaxThreads: call rebuildVoxels() while no thread navigates, otherwise the lazy rebuild races between the threads. * Replacing gGeoManager disables multithreading (detected via IsMultiThread(), not only the pointer) and the stale navigators are dropped. The slot map is static in TGeo and not reset: threads which navigated before must not be reused and enabling again may find slots already consumed (warning issued). * setMaxThreads, rebuildVoxels and geometry replacement need all other threads to be idle. Co-Authored-By: Claude Sonnet 5.5 --- Detectors/Base/CMakeLists.txt | 7 + .../include/DetectorsBase/GeometryManager.h | 39 +++- Detectors/Base/src/GeometryManager.cxx | 191 ++++++++++++++++-- Detectors/Base/src/MatLayerCylSet.cxx | 26 ++- Detectors/Base/test/testGeometryManagerMT.cxx | 183 +++++++++++++++++ 5 files changed, 409 insertions(+), 37 deletions(-) create mode 100644 Detectors/Base/test/testGeometryManagerMT.cxx diff --git a/Detectors/Base/CMakeLists.txt b/Detectors/Base/CMakeLists.txt index e225a2d975ef3..51047840e171e 100644 --- a/Detectors/Base/CMakeLists.txt +++ b/Detectors/Base/CMakeLists.txt @@ -105,6 +105,13 @@ o2_add_test( PUBLIC_LINK_LIBRARIES O2::DetectorsBase LABELS detectorsbase) +o2_add_test( + GeometryManagerMT + SOURCES test/testGeometryManagerMT.cxx + COMPONENT_NAME DetectorsBase + PUBLIC_LINK_LIBRARIES O2::DetectorsBase + LABELS detectorsbase) + o2_add_test( O2Tessellated SOURCES test/testO2Tessellated.cxx diff --git a/Detectors/Base/include/DetectorsBase/GeometryManager.h b/Detectors/Base/include/DetectorsBase/GeometryManager.h index 6c771aa4bc936..70aa47c126ec0 100644 --- a/Detectors/Base/include/DetectorsBase/GeometryManager.h +++ b/Detectors/Base/include/DetectorsBase/GeometryManager.h @@ -26,6 +26,7 @@ #include "GPUCommonLogger.h" // for LOG #include "MathUtils/Cartesian.h" #include "DetectorsBase/MatCell.h" +#include #include #include class TGeoHMatrix; // lines 11-11 @@ -105,8 +106,40 @@ class GeometryManager : public TObject ClassDefNV(MatBudgetExt, 1); }; + /// Multithreaded TGeo navigation (used by Propagator and MatLayerCylSet). + /// + /// setMaxThreads(n): enable per-thread navigation for up to n threads in addition to the calling one. + /// It wraps TGeoManager::SetMaxThreads, which is one-way and not re-entrant, hence: + /// * call it once, serially, after the geometry is loaded/aligned and BEFORE the threads start navigating, + /// with n >= the maximum number of threads which will ever call TGeo (the TGeo thread slots are never + /// released: a slot is consumed by every distinct thread which navigated, even after it exited); + /// * repeated calls with n <= current capacity are no-ops; a larger n is refused (error logged), since TGeo + /// would reset the thread slots of the threads already navigating; + /// * after TGeoPhysicalNode::Align, call rebuildVoxels() while no thread is navigating; + /// * replacing gGeoManager disables multithreading (threads that navigated before must not be reused). + /// Returns the effective capacity (0 if not enabled). + /// While not enabled, everything behaves as before: the shared TGeo navigator is used under a mutex. + /// Once enabled, a thread navigates through its own navigator, obtained by getThreadNavigator(). This is done + /// internally by meanMaterialBudget(Ext), getOriginalMatrix* and thus Propagator. Direct use of the gGeoManager + /// navigation API (cd, InitTrack, GetCurrentNode, ...) is safe only in a thread which has called + /// getThreadNavigator() before (or which called setMaxThreads): gGeoManager->GetCurrentNavigator() returns + /// nullptr for any other thread, and any TGeo navigation from a new thread consumes one of the thread slots. + /// Prefer calling the methods of the navigator returned by getThreadNavigator(). + static int setMaxThreads(int nThreads); + static int getMaxThreads(); // capacity set by setMaxThreads for the current geometry, 0 if not enabled + + /// Voxel structures are shared by all navigators and rebuilt lazily at the first navigation after + /// TGeoPhysicalNode::Align. Do it explicitly (serially, with no thread navigating). Called by setMaxThreads. + static void rebuildVoxels(); + + /// Navigator owned by the calling thread (created on the first call, released at the thread exit), to be + /// passed to meanMaterialBudget(..., nav). Returns nullptr if multithreading is not enabled (=> mutexed shared + /// navigator). Throws std::runtime_error if more threads than declared in setMaxThreads() ask for a navigator. + static TGeoNavigator* getThreadNavigator(); + /// Mean material budget between two points. Pass a navigator owned by the calling thread to - /// run lock-free from several threads; with nav = nullptr the shared navigator is used under + /// run lock-free from several threads. With nav = nullptr: the calling thread's own navigator if setMaxThreads() + /// was called, otherwise the shared navigator is used under /// a mutex, as before. static o2::base::MatBudget meanMaterialBudget(float x0, float y0, float z0, float x1, float y1, float z1, TGeoNavigator* nav = nullptr); @@ -182,6 +215,10 @@ class GeometryManager : public TObject static constexpr UInt_t sDetOffsetLarge = 17; /// detector identifier will start from this bit for detectors after the FOC static constexpr UInt_t sSensorMaskLarge = (0x1 << sDetOffsetLarge) - 1; /// mask=max sensitive volumes allowed per detector (131071) static std::mutex sTGMutex; + static std::atomic sMaxThreads; // TGeo multithreading capacity, 0: not enabled + static std::atomic sGeneration; // bumped when multithreading is (re)enabled or invalidated + static TGeoManager* sGeo; // TGeoManager multithreading was enabled for + static bool isMTActive(); // multithreading enabled for the current gGeoManager (resets stale state) ClassDefOverride(GeometryManager, 0); // Manager of geometry information for alignment }; diff --git a/Detectors/Base/src/GeometryManager.cxx b/Detectors/Base/src/GeometryManager.cxx index ecc279dab1db2..5b870d36e420a 100644 --- a/Detectors/Base/src/GeometryManager.cxx +++ b/Detectors/Base/src/GeometryManager.cxx @@ -19,6 +19,9 @@ #include // for TGeoNavigator #include // for TGeoNode #include // for TGeoPhysicalNode, TGeoPNEntry +#include +#include +#include #include #include #include // for NULL @@ -66,6 +69,126 @@ using namespace o2::base; /// the look-up table mapping unique volume indices to symbolic volume names. For that, it /// collects several static methods std::mutex GeometryManager::sTGMutex; +std::atomic GeometryManager::sMaxThreads{0}; +std::atomic GeometryManager::sGeneration{0}; +TGeoManager* GeometryManager::sGeo = nullptr; + +//_____________________________________________________________________________________ +bool GeometryManager::isMTActive() +{ + // A TGeoManager which was not put in multithread mode by setMaxThreads (e.g. a geometry loaded after + // the previous one was deleted, possibly at the same address) is detected by the IsMultiThread() check. + if (sMaxThreads.load() == 0) { + return false; + } + if (gGeoManager && gGeoManager == sGeo && gGeoManager->IsMultiThread()) { + return true; + } + std::lock_guard guard(sTGMutex); + if (sMaxThreads.load() != 0 && !(gGeoManager && gGeoManager == sGeo && gGeoManager->IsMultiThread())) { + LOG(warning) << "TGeoManager was replaced: per-thread TGeo navigation is disabled until GeometryManager::setMaxThreads is called again"; + sMaxThreads = 0; + sGeo = nullptr; + sGeneration++; // invalidates the navigators held by threads + } + return false; +} + +//_____________________________________________________________________________________ +int GeometryManager::getMaxThreads() +{ + return isMTActive() ? sMaxThreads.load() : 0; +} + +//_____________________________________________________________________________________ +void GeometryManager::rebuildVoxels() +{ + // TGeoVoxelFinder rebuilds its voxels lazily at the first navigation into a volume flagged by + // SetNeedRebuild() (e.g. mothers of TGeoPhysicalNode::Align-ed nodes). The voxel structures are shared + // between all navigators, so this has to be done serially, not concurrently from the worker threads. + if (!gGeoManager) { + LOG(fatal) << "geometry is not loaded"; + } + std::lock_guard guard(sTGMutex); + TIter next(gGeoManager->GetListOfVolumes()); + while (auto* vol = static_cast(next())) { + if (auto* voxels = vol->GetVoxels(); voxels && voxels->NeedRebuild()) { + voxels->Voxelize(); // resets the NeedRebuild flag + vol->FindOverlaps(); + } + } +} + +//_____________________________________________________________________________________ +int GeometryManager::setMaxThreads(int nThreads) +{ + if (!gGeoManager) { + LOG(fatal) << "geometry is not loaded"; + } + if (nThreads < 1) { + return getMaxThreads(); + } + if (isMTActive()) { + const int cur = sMaxThreads.load(); + if (nThreads > cur) { + // TGeoManager::SetMaxThreads would clear the thread slots (and the per-volume thread data) of the + // threads which already navigate, so the capacity cannot be raised once it is enabled. + LOG(error) << "GeometryManager::setMaxThreads(" << nThreads << ") refused: multithreaded navigation was already enabled for " + << cur << " threads and cannot be extended. Call setMaxThreads once with the maximum number of threads"; + } + return cur; + } + rebuildVoxels(); + std::lock_guard guard(sTGMutex); + gGeoManager->SetMaxThreads(nThreads); // one-way in TGeoManager + // The map of thread slots is static in TGeoManager (it is not reset with the geometry, and the threads cache + // their slot), so slots used with a previous geometry stay consumed. The calling thread takes a slot first. + const int ownerSlot = TGeoManager::ThreadId(); + if (ownerSlot > nThreads) { + LOG(warning) << "TGeo thread slots were already consumed (calling thread has slot " << ownerSlot + << ", capacity " << nThreads << "): per-thread navigators will not be available"; + } + sGeo = gGeoManager; + sGeneration++; + sMaxThreads = nThreads; + return nThreads; +} + +//_____________________________________________________________________________________ +TGeoNavigator* GeometryManager::getThreadNavigator() +{ + struct Holder { + TGeoNavigator* nav = nullptr; + int gen = -1; // value of sGeneration the navigator belongs to + ~Holder() + { + if (nav) { + std::lock_guard guard(sTGMutex); + if (gen == sGeneration.load() && gGeoManager && gGeoManager == sGeo) { // else the navigator died with its geometry + gGeoManager->RemoveNavigator(nav); // the TGeo thread slot of this thread stays consumed + } + } + } + }; + thread_local Holder h; + if (!isMTActive()) { + return nullptr; + } + if (h.nav && h.gen == sGeneration.load()) { + return h.nav; + } + h.nav = nullptr; // none yet, or the one from the previous geometry + std::lock_guard guard(sTGMutex); + h.gen = sGeneration.load(); + // The slot (TGeoManager::ThreadId) is what indexes the per-volume thread data: it must be within the + // reserved range, otherwise TGeo would access beyond its arrays. + if (TGeoManager::ThreadId() > gGeoManager->GetMaxThreads()) { + throw std::runtime_error("GeometryManager: more threads use TGeo than declared in GeometryManager::setMaxThreads(" + + std::to_string(sMaxThreads.load()) + ")"); + } + h.nav = gGeoManager->AddNavigator(); + return h.nav; +} //______________________________________________________________________ Bool_t GeometryManager::getOriginalMatrix(const char* symname, TGeoHMatrix& m) @@ -76,15 +199,21 @@ Bool_t GeometryManager::getOriginalMatrix(const char* symname, TGeoHMatrix& m) ; return kFALSE; } - std::lock_guard guard(sTGMutex); + // navigator of the calling thread if setMaxThreads was called, otherwise the shared one under the mutex + std::unique_lock guard(sTGMutex, std::defer_lock); + TGeoNavigator* nav = getThreadNavigator(); + if (!nav) { + guard.lock(); + nav = gGeoManager->GetCurrentNavigator(); + } if (!gGeoManager->GetListOfPhysicalNodes()) { LOG(warning) << "gGeoManager doesn't contain any aligned nodes!"; - if (!gGeoManager->cd(symname)) { + if (!nav->cd(symname)) { LOG(error) << "Volume path " << symname << " not valid!"; return kFALSE; } else { - m = *gGeoManager->GetCurrentMatrix(); + m = *nav->GetCurrentMatrix(); return kTRUE; } } @@ -101,6 +230,9 @@ Bool_t GeometryManager::getOriginalMatrix(const char* symname, TGeoHMatrix& m) path = symname; } + if (guard.owns_lock()) { // getOriginalMatrixFromPath takes the (non-recursive) mutex itself + guard.unlock(); + } return getOriginalMatrixFromPath(path, m); } @@ -113,19 +245,25 @@ Bool_t GeometryManager::getOriginalMatrixFromPath(const char* path, TGeoHMatrix& LOG(error) << "Can't get the original global matrix! gGeoManager doesn't exist or it is still opened!"; return kFALSE; } - std::lock_guard guard(sTGMutex); - if (!gGeoManager->CheckPath(path)) { + // navigator of the calling thread if setMaxThreads was called, otherwise the shared one under the mutex + std::unique_lock guard(sTGMutex, std::defer_lock); + TGeoNavigator* nav = getThreadNavigator(); + if (!nav) { + guard.lock(); + nav = gGeoManager->GetCurrentNavigator(); + } + if (!nav->CheckPath(path)) { LOG(error) << "Volume path " << path << " not valid!"; return kFALSE; } TIter next(gGeoManager->GetListOfPhysicalNodes()); - gGeoManager->cd(path); + nav->cd(path); - while (gGeoManager->GetLevel()) { + while (nav->GetLevel()) { TGeoPhysicalNode* physNode = nullptr; next.Reset(); - TGeoNode* node = gGeoManager->GetCurrentNode(); + TGeoNode* node = nav->GetCurrentNode(); while ((physNode = (TGeoPhysicalNode*)next())) { if (physNode->GetNode() == node) { @@ -145,7 +283,7 @@ Bool_t GeometryManager::getOriginalMatrixFromPath(const char* path, TGeoHMatrix& m.MultiplyLeft(lm); - gGeoManager->CdUp(); + nav->CdUp(); } return kTRUE; } @@ -375,9 +513,15 @@ GeometryManager::MatBudgetExt GeometryManager::meanMaterialBudgetExt(float x0, f for (int i = 3; i--;) { dir[i] *= invlen; } - std::lock_guard guard(sTGMutex); + // per-thread navigator if setMaxThreads was called, otherwise the shared one under the mutex + std::unique_lock guard(sTGMutex, std::defer_lock); + TGeoNavigator* nav = getThreadNavigator(); + if (!nav) { + guard.lock(); + nav = gGeoManager->GetCurrentNavigator(); + } // Initialize start point and direction - TGeoNode* currentnode = gGeoManager->InitTrack(startD, dir); + TGeoNode* currentnode = nav->InitTrack(startD, dir); if (!currentnode) { LOG(error) << "start point out of geometry: " << x0 << ':' << y0 << ':' << z0; return MatBudgetExt(); // return empty struct @@ -389,11 +533,11 @@ GeometryManager::MatBudgetExt GeometryManager::meanMaterialBudgetExt(float x0, f // Locate next boundary within length without computing safety. // Propagate either with length (if no boundary found) or just cross boundary - gGeoManager->FindNextBoundaryAndStep(length, kFALSE); + nav->FindNextBoundaryAndStep(length, kFALSE); Double_t stepTot = 0.0; // Step made - Double_t step = gGeoManager->GetStep(); + Double_t step = nav->GetStep(); // If no boundary within proposed length, return current step data - if (!gGeoManager->IsOnBoundary()) { + if (!nav->IsOnBoundary()) { budStep.meanX2X0 = budStep.length / budStep.meanX2X0; return MatBudgetExt(budStep); } @@ -408,7 +552,7 @@ GeometryManager::MatBudgetExt GeometryManager::meanMaterialBudgetExt(float x0, f if (nzero > 3) { // This means navigation has problems on one boundary // Try to cross by making a small step - const double* curPos = gGeoManager->GetCurrentPoint(); + const double* curPos = nav->GetCurrentPoint(); LOG(warning) << "Cannot cross boundary at (" << curPos[0] << ',' << curPos[1] << ',' << curPos[2] << ')'; budTotal.normalize(stepTot); budTotal.nCross = -1; // flag failed navigation @@ -426,14 +570,14 @@ GeometryManager::MatBudgetExt GeometryManager::meanMaterialBudgetExt(float x0, f if (step >= length) { break; } - currentnode = gGeoManager->GetCurrentNode(); + currentnode = nav->GetCurrentNode(); if (!currentnode) { break; } length -= step; accountMaterial(currentnode->GetVolume()->GetMedium()->GetMaterial(), budStep); - gGeoManager->FindNextBoundaryAndStep(length, kFALSE); - step = gGeoManager->GetStep(); + nav->FindNextBoundaryAndStep(length, kFALSE); + step = nav->GetStep(); } budTotal.normalize(stepTot); return MatBudgetExt(budTotal); @@ -470,11 +614,14 @@ o2::base::MatBudget GeometryManager::meanMaterialBudget(float x0, float y0, floa for (int i = 3; i--;) { dir[i] *= invlen; } - // A caller that passes its own navigator owns it exclusively, so no locking is needed. A caller - // that passes none shares gGeoManager's current navigator and must still serialize. Deciding - // this from the argument keeps the choice local: it does not depend on -- and cannot be broken - // by -- process-global state such as TGeoManager::GetMaxThreads(). + // A caller that passes its own navigator (or, with multithreading enabled by setMaxThreads, gets the one of + // its thread) owns it exclusively, so no locking is needed. Otherwise the shared navigator is used under a mutex. + // In multithread mode a thread without navigator must never touch the shared one (GetCurrentNavigator would + // return nullptr for it and the per-volume thread data is indexed by the thread slot). std::unique_lock guard(sTGMutex, std::defer_lock); + if (!nav) { + nav = getThreadNavigator(); + } if (!nav) { guard.lock(); nav = gGeoManager->GetCurrentNavigator(); diff --git a/Detectors/Base/src/MatLayerCylSet.cxx b/Detectors/Base/src/MatLayerCylSet.cxx index 2df1d694e4d3c..f52a0ab8ad75a 100644 --- a/Detectors/Base/src/MatLayerCylSet.cxx +++ b/Detectors/Base/src/MatLayerCylSet.cxx @@ -175,28 +175,26 @@ void MatLayerCylSet::populateFromTGeo(int ntrPerCell, int nThreads, MatbudGeomBa Clock::time_point tFillStart, tFillEnd; if (backend == MatbudGeomBackend::ROOT) { // TGeo has to be told that several threads will navigate it, and each thread needs its own - // navigator. SetMaxThreads() is one-way -- TGeoManager has no API to return to - // single-threaded mode -- so we do not pretend to restore it; that is harmless because - // meanMaterialBudget() decides whether to lock from its own argument, not from this global. - // The navigators we book are ours, though, so those we do give back. - gGeoManager->SetMaxThreads(nThreads); - - tbb::enumerable_thread_specific threadNavigators( - []() { return gGeoManager->AddNavigator(); }); + // navigator. Use the same per-thread navigator mechanism (and thread slots) as the Propagator: the + // navigators are thread_local and are released when their threads exit. The capacity can be set only + // once per geometry: if the caller has already enabled fewer threads, do not use more than that (the + // calling thread participates in the parallel_for in addition to the nCap workers). + const int nCap = GeometryManager::setMaxThreads(nThreads); + const int nUse = std::max(1, std::min(nThreads, nCap + 1)); + if (nUse < nThreads) { + LOG(warning) << "TGeo multithreading was enabled for " << nCap << " threads only, using " << nUse << " instead of " << nThreads; + } tFillStart = Clock::now(); { - tbb::global_control threadControl(tbb::global_control::max_allowed_parallelism, nThreads); + tbb::global_control threadControl(tbb::global_control::max_allowed_parallelism, nUse); tbb::parallel_for(tbb::blocked_range(0, totalCells), - [&fillRange, &threadNavigators](const tbb::blocked_range& range) { - fillRange(range, threadNavigators.local()); + [&fillRange](const tbb::blocked_range& range) { + fillRange(range, GeometryManager::getThreadNavigator()); }); } tFillEnd = Clock::now(); - for (TGeoNavigator* nav : threadNavigators) { - gGeoManager->RemoveNavigator(nav); - } } else { // VecGeom navigation needs no per-thread navigator bookkeeping. Trigger the lazy world // build/BVH init before tFillStart so it counts as "setup", not fill time. diff --git a/Detectors/Base/test/testGeometryManagerMT.cxx b/Detectors/Base/test/testGeometryManagerMT.cxx new file mode 100644 index 0000000000000..955ba71b25221 --- /dev/null +++ b/Detectors/Base/test/testGeometryManagerMT.cxx @@ -0,0 +1,183 @@ +// Copyright 2019-2026 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#define BOOST_TEST_MODULE Test GeometryManager multithreaded TGeo navigation +#define BOOST_TEST_MAIN +#define BOOST_TEST_DYN_LINK +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "DetectorsBase/GeometryManager.h" + +using o2::base::GeometryManager; + +namespace +{ +using Ray = std::array; + +void buildGeometry() +{ + delete gGeoManager; + auto* geo = new TGeoManager("testMT", "test geometry"); + auto* vac = new TGeoMaterial("Vac", 0, 0, 0, 1e9, 1e9); + auto* matA = new TGeoMaterial("A", 27., 13., 2.7, 8.9, 39.); + auto* matB = new TGeoMaterial("B", 56., 26., 7.9, 1.76, 16.8); + auto* medVac = new TGeoMedium("Vac", 1, vac); + auto* medA = new TGeoMedium("A", 2, matA); + auto* medB = new TGeoMedium("B", 3, matB); + auto* top = geo->MakeBox("TOP", medVac, 100., 100., 100.); + geo->SetTopVolume(top); + // a few daughters, so that rays cross several boundaries + for (int i = 0; i < 6; i++) { + auto* v = geo->MakeBox(Form("BOX%d", i), (i % 2) ? medA : medB, 8. + i, 10. + i, 12.); + top->AddNode(v, i, new TGeoTranslation(-60. + 24. * i, 5. * (i % 3) - 5., 3. * i - 9.)); + } + geo->CloseGeometry(); +} + +std::vector makeRays(int n) +{ + std::mt19937 gen(12345); + std::uniform_real_distribution u(-95.f, 95.f); + std::vector rays(n); + for (auto& r : rays) { + for (auto& x : r) { + x = u(gen); + } + } + return rays; +} + +struct Res { + float rho, x2x0, len; + double extRho, extX2X0; + int nCross; +}; + +Res calc(const Ray& r) +{ + auto b = GeometryManager::meanMaterialBudget(r[0], r[1], r[2], r[3], r[4], r[5]); // nav = nullptr + auto e = GeometryManager::meanMaterialBudgetExt(r[0], r[1], r[2], r[3], r[4], r[5]); + return {b.meanRho, b.meanX2X0, b.length, e.meanRho, e.meanX2X0, e.nCross}; +} + +bool same(const Res& a, const Res& b) +{ + auto eq = [](double x, double y) { return std::abs(x - y) <= 1e-5 * (std::abs(x) + std::abs(y)) + 1e-9; }; + return eq(a.rho, b.rho) && eq(a.x2x0, b.x2x0) && eq(a.len, b.len) && eq(a.extRho, b.extRho) && eq(a.extX2X0, b.extX2X0) && a.nCross == b.nCross; +} +} // namespace + +BOOST_AUTO_TEST_CASE(GeometryManagerMT) +{ + buildGeometry(); + const auto rays = makeRays(3000); + + // 1. not enabled: nothing changes, the shared navigator is used under the mutex + BOOST_CHECK_EQUAL(GeometryManager::getMaxThreads(), 0); + BOOST_CHECK(GeometryManager::getThreadNavigator() == nullptr); + std::vector ref; + for (const auto& r : rays) { + ref.push_back(calc(r)); + } + + // 2. enabling: idempotent for the same or smaller capacity, refused (capacity unchanged) for a larger one + constexpr int NThr = 4; + BOOST_CHECK_EQUAL(GeometryManager::setMaxThreads(NThr), NThr); + BOOST_CHECK_EQUAL(GeometryManager::setMaxThreads(NThr), NThr); + BOOST_CHECK_EQUAL(GeometryManager::setMaxThreads(2), NThr); + BOOST_CHECK_EQUAL(GeometryManager::setMaxThreads(2 * NThr), NThr); + BOOST_CHECK_EQUAL(GeometryManager::getMaxThreads(), NThr); + // the calling thread can still navigate (through its own navigator) + for (size_t i = 0; i < 100; i++) { + BOOST_CHECK(same(calc(rays[i]), ref[i])); + } + + // 3. NThr concurrent threads, all started before any finishes: results must agree with the serial reference + // and every thread must get its own navigator. The threads are kept alive after the work (their TGeo slots + // are consumed for good) to test the exhaustion below deterministically. + std::atomic nReady{0}, nBad{0}; + std::atomic go{false}, release{false}; + std::vector navs(NThr, nullptr); + std::vector workers; + for (int it = 0; it < NThr; it++) { + workers.emplace_back([&, it]() { + navs[it] = GeometryManager::getThreadNavigator(); + nReady++; + while (!go) { + std::this_thread::yield(); + } + { + // alignment-matrix lookup navigates too: it must use the navigator of this thread + TGeoHMatrix m; + if (!GeometryManager::getOriginalMatrix("/TOP_1/BOX3_3", m) || std::abs(m.GetTranslation()[0] - 12.) > 1e-9) { + nBad++; + } + } + for (size_t i = 0; i < rays.size(); i++) { + if (!same(calc(rays[(i + it * 97) % rays.size()]), ref[(i + it * 97) % rays.size()])) { + nBad++; + } + } + while (!release) { + std::this_thread::yield(); + } + }); + } + while (nReady < NThr) { + std::this_thread::yield(); + } + go = true; + // 4. more threads than declared: a clear error, no UB, no silent use of the shared navigator + // (only after the workers are done with the work, they are still alive) + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + bool thrown = false; + std::thread extra([&]() { + try { + GeometryManager::getThreadNavigator(); + } catch (const std::runtime_error&) { + thrown = true; + } + }); + extra.join(); + BOOST_CHECK(thrown); + release = true; + for (auto& w : workers) { + w.join(); + } + BOOST_CHECK_EQUAL(nBad.load(), 0); + std::set uniq(navs.begin(), navs.end()); + BOOST_CHECK(!uniq.count(nullptr)); + BOOST_CHECK_EQUAL(uniq.size(), size_t(NThr)); + + // 5. geometry replaced: multithreading is disabled for the new one, which works as before (mutexed) + buildGeometry(); + BOOST_CHECK_EQUAL(GeometryManager::getMaxThreads(), 0); + BOOST_CHECK(GeometryManager::getThreadNavigator() == nullptr); + for (size_t i = 0; i < 100; i++) { + BOOST_CHECK(same(calc(rays[i]), ref[i])); + } + // the threads of the previous geometry are gone: their navigators must not have been removed from the new one + std::thread late([&]() { BOOST_CHECK(same(calc(rays[0]), ref[0])); }); + late.join(); +}