From 661638f0fa3402b9a0b087d8282851299e9836a1 Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Mon, 16 Nov 2026 15:45:33 +0100 Subject: [PATCH 01/10] GPU: scan the pad baseline in time segments where the backend asks for it Needs review: shared TPC cluster finder code, selected per backend by the new parameter CF_CHECK_PAD_BASELINE_SEGMENTED, which is 1 only for Metal. CUDA, HIP and OpenCL keep the existing scan, since its speed there could not be measured here. GPUTPCCFCheckPadBaseline runs one block of 576 threads per row, and without the HIP tail filter only the up to 138 threads that own a pad do work, each walking all time bins of its pad. Without the filter the baseline needs, per pad, only the number of time bins with charge, the longest run of them and the largest charge, and these combine over consecutive segments of time. So with the new parameter all four threads per pad scan a quarter of the time bins each, straight from the charge map, and the quarters are joined at the end. The block size, its static_assert, the grid and the noisy-pad flags are unchanged; with the HIP tail filter on, the existing scan runs. On an M1 Max the kernel takes 1.6 ms per call instead of 3.7 (median in a Metal System Trace), and the 30-collision Pb-Pb time frame 1088 ms per event instead of 1182 (3 interleaved rounds of 10 runs, load average 4 to 6). The clusters are byte-identical to the CPU's, also with noisy-pad thresholds tightened until 6 to 24% of the clusters are dropped, and the CPU backend's own output is unchanged. --- .../Definitions/Parameters/GPUParameters.csv | 1 + .../GPUTPCCFCheckPadBaseline.cxx | 65 +++++++++++++++++++ .../GPUTPCCFCheckPadBaseline.h | 12 +++- 3 files changed, 77 insertions(+), 1 deletion(-) diff --git a/GPU/GPUTracking/Definitions/Parameters/GPUParameters.csv b/GPU/GPUTracking/Definitions/Parameters/GPUParameters.csv index 330c9a3fa7377..dad88d7ef873c 100644 --- a/GPU/GPUTracking/Definitions/Parameters/GPUParameters.csv +++ b/GPU/GPUTracking/Definitions/Parameters/GPUParameters.csv @@ -114,4 +114,5 @@ MERGER_INTERPOLATION_ERROR_TYPE,"""half""","""float""",,,,,,,,,,,,,,,,, COMP_GATHER_KERNEL,4,0,4,4,,,,,,4,4,4,4,,4,,4,4, COMP_GATHER_MODE,3,0,3,3,,,,,,3,3,3,3,,3,,3,3, CF_SCAN_WORKGROUP_SIZE,512,0,,,,,,,,224,,992,448,,1024,,,448, +CF_CHECK_PAD_BASELINE_SEGMENTED,0,0,,,,,,,,,,,,,,,,,1 MERGER_SPLIT_LOOP_INTERPOLATION,,,,,,,,,,,,,,,1,,,1, diff --git a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFCheckPadBaseline.cxx b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFCheckPadBaseline.cxx index ecd11250b239c..8d22281978dbb 100644 --- a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFCheckPadBaseline.cxx +++ b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFCheckPadBaseline.cxx @@ -238,6 +238,12 @@ GPUd() void GPUTPCCFCheckPadBaseline::CheckBaselineGPU(int32_t nBlocks, int32_t const CfFragment& frag = clusterer.mPmemory->frag; const bool hipFilterOn = clusterer.Param().rec.tpc.hipTailFilter; +#if GPUCA_PAR_CF_CHECK_PAD_BASELINE_SEGMENTED + if (!hipFilterOn) { + CheckBaselineGPUSegmented(iBlock, iThread, smem, clusterer); + return; + } +#endif const Charge hipTailThreshold = clusterer.Param().rec.tpc.hipTailFilterThreshold; const Charge hipTailFilterAlpha = clusterer.Param().rec.tpc.hipTailFilterAlpha; CfArray2D chargeMap(reinterpret_cast(clusterer.mPchargeMap)); @@ -406,6 +412,65 @@ GPUd() void GPUTPCCFCheckPadBaseline::CheckBaselineGPU(int32_t nBlocks, int32_t #endif } +// Without the HIP tail filter, a pad only needs the number of time bins with charge, the longest run of +// them and the largest charge. These combine over consecutive time segments, so the NThreads / MaxNPadsPerRow +// threads of a pad each scan one segment straight from the charge map, instead of one thread scanning all time +// bins of the pad, and the segments are joined at the end. The result is the same as that of the scan in time order. +GPUd() void GPUTPCCFCheckPadBaseline::CheckBaselineGPUSegmented(int32_t iBlock, int32_t iThread, GPUSharedMemory& smem, processorType& clusterer) +{ +#if defined(GPUCA_GPUCODE) && !defined(GPUCA_DEVICE_SINGLE_THREAD) + constexpr int32_t NSegments = NThreads / MaxNPadsPerRow; + constexpr GPUTPCGeometry geo; + const CfFragment& frag = clusterer.mPmemory->frag; + CfArray2D chargeMap(reinterpret_cast(clusterer.mPchargeMap)); + const auto iRow = iBlock; + const int16_t nPads = geo.NPads(iRow); + const CfChargePos basePos{(Row)iRow, 0, 0}; + const int16_t pad = iThread % MaxNPadsPerRow; + const int32_t segment = iThread / MaxNPadsPerRow; + const int32_t segmentLength = (frag.length + NSegments - 1) / NSegments; + const int32_t tStart = CAMath::Min(segment * segmentLength, frag.length); + const int32_t tEnd = CAMath::Min(tStart + segmentLength, frag.length); + + if (segment < NSegments) { + int32_t totalCharges = 0, consecCharges = 0, maxConsecCharges = 0, firstConsecCharges = 0; + bool leading = true; + Charge maxCharge = 0; + if (pad < nPads) { + for (int32_t t = tStart; t < tEnd; t++) { + const Charge q = chargeMap[basePos.delta({pad, (TPCFragmentTime)t})].unpack(); + totalCharges += q > 0; + consecCharges = q > 0 ? consecCharges + 1 : 0; + maxConsecCharges = CAMath::Max(consecCharges, maxConsecCharges); + maxCharge = CAMath::Max(q, maxCharge); + leading &= q > 0; + firstConsecCharges += leading; + } + } + smem.segments.totalCharges[iThread] = totalCharges; + smem.segments.maxConsecCharges[iThread] = maxConsecCharges; + smem.segments.firstConsecCharges[iThread] = firstConsecCharges; + smem.segments.lastConsecCharges[iThread] = consecCharges; + smem.segments.maxCharge[iThread] = maxCharge; + } + GPUbarrier(); + + if (segment == 0 && pad < nPads) { + int32_t totalCharges = 0, maxConsecCharges = 0, consecCharges = 0; + Charge maxCharge = 0; + for (int32_t i = 0; i < NSegments; i++) { + const int32_t j = i * MaxNPadsPerRow + pad; + const int32_t length = CAMath::Min(CAMath::Min(i * segmentLength, frag.length) + segmentLength, frag.length) - CAMath::Min(i * segmentLength, frag.length); + totalCharges += smem.segments.totalCharges[j]; + maxConsecCharges = CAMath::Max(maxConsecCharges, CAMath::Max(smem.segments.maxConsecCharges[j], consecCharges + smem.segments.firstConsecCharges[j])); + consecCharges = smem.segments.firstConsecCharges[j] == length ? consecCharges + length : smem.segments.lastConsecCharges[j]; + maxCharge = CAMath::Max(smem.segments.maxCharge[j], maxCharge); + } + updatePadBaseline(basePos.gpad + pad, clusterer, totalCharges, maxConsecCharges, maxCharge); + } +#endif +} + GPUd() void GPUTPCCFCheckPadBaseline::CheckBaselineCPU(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUSharedMemory& smem, processorType& clusterer) { #ifndef GPUCA_GPUCODE diff --git a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFCheckPadBaseline.h b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFCheckPadBaseline.h index 0609868dd3b2c..4b925544fbdd0 100644 --- a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFCheckPadBaseline.h +++ b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFCheckPadBaseline.h @@ -95,7 +95,16 @@ class GPUTPCCFCheckPadBaseline : public GPUKernelTemplate }; struct GPUSharedMemory : public GPUKernelTemplate::GPUSharedMemoryScan64 { - tpccf::Charge charges[NumOfCachedTBs][MaxNPadsPerRow]; + union { + tpccf::Charge charges[NumOfCachedTBs][MaxNPadsPerRow]; + struct { // Per-thread results of the segmented scan, see CheckBaselineGPUSegmented + int32_t totalCharges[NThreads]; + int32_t maxConsecCharges[NThreads]; + int32_t firstConsecCharges[NThreads]; + int32_t lastConsecCharges[NThreads]; + tpccf::Charge maxCharge[NThreads]; + } segments; + }; HipTailRange tails[MaxNPadsPerRow]; uint8_t tailsClosedPad[MaxNPadsPerRow]; HipTailRange tailsClosed[MaxNPadsPerRow]; @@ -139,6 +148,7 @@ class GPUTPCCFCheckPadBaseline : public GPUKernelTemplate private: GPUd() static void CheckBaselineGPU(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUSharedMemory& smem, processorType& clusterer); + GPUd() static void CheckBaselineGPUSegmented(int32_t iBlock, int32_t iThread, GPUSharedMemory& smem, processorType& clusterer); GPUd() static void CheckBaselineCPU(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUSharedMemory& smem, processorType& clusterer); GPUd() static void updatePadBaseline(int32_t pad, const GPUTPCClusterFinder&, int32_t totalCharges, int32_t consecCharges, tpccf::Charge maxCharge); From cc125bbd464b251bd431a283efc2416f05a8e3e1 Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Mon, 16 Nov 2026 15:45:33 +0100 Subject: [PATCH 02/10] GPU: refit the sector tracks of the merger on all streams The 36 GPUTPCGMMergerSectorRefit launches, with their UnpackSaveNumber and UnpackResetIds, all ran on stream 0. Distribute them over the streams, as MergeBorders does, with stream 0 waiting for the other streams before UnpackGlobal. Why this is safe: the sectors only share the atomic nUnpackedTracks. Each sector writes its own part of mTrackIDs and reads only its own sector tracker's output, which is complete after the SynchronizeGPU() before the merger. The other streams wait for stream 0's transfer of the merger memory, and UnpackGlobal, which reads every sector's mTrackIDs and the counter, runs after the join. The only thing lost is that the sector tracks are no longer stored sector by sector, so mSectorTrackInfoIndex [1, NSECTORS) are no longer meaningful. Only the deterministic sort and the merger debug dumps (SectorTrackInfoFirst / Last) read them; the production kernels use track.Sector() and SectorTrackInfoLocalTotal(). So deterministicGPUReconstruction, and with it the debugLevel 6 dumps, keep the serial order. The index order of the sector tracks was already not deterministic within a sector, because of the atomic. On an M1 Max with Metal (8 streams), pbpb30 standalone benchmark: - the refit block alone, between two synchronizations: 145.5 ms serial, 132 ms on the streams; the refit is mostly throughput-bound there; - per event, 10 interleaved rounds of --runs 10: 980 ms -> 969 ms (-1%); - FST TPC-only warm TF at TFDELAY=2: within the noise (3 rounds). The dumps of pbpb30 and sim5pp stay identical to the CPU's, sim5pp gives 19009 / 52 / 3027 in all runs, and the FST 1,355,432 clusters. GPUs with more compute units than the M1 Max should gain more from it. Untested with CUDA and HIP. --- .../Global/GPUChainTrackingMerger.cxx | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx index 528a848acf389..c8659561a1dfe 100644 --- a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx +++ b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx @@ -121,10 +121,23 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) if (GetProcessingSettings().deterministicGPUReconstruction) { runKernel(GetGridAuto(0, deviceType), 1); } + // Sectors only share the atomic track counter, so they can be refit concurrently, but then the sector track infos are not ordered by sector. + // Only the deterministic sort and the debug dumps need that order (SectorTrackInfoFirst / Last), so keep the serial order for them. + const uint32_t nRefitStreams = doGPU && !GetProcessingSettings().deterministicGPUReconstruction ? mRec->NStreams() : 1; + RecordMarker(&mEvents->single, 0); for (uint32_t i = 0; i < NSECTORS; i++) { - runKernel({{1, -WarpSize(), 0, deviceType}}, i); - runKernel(GetGridAuto(0, deviceType), i); - runKernel(GetGridAuto(0, deviceType), i); // TODO: Why all in stream 0? + const int32_t stream = i % nRefitStreams; + runKernel({{1, -WarpSize(), stream, deviceType}, krnlRunRangeNone, {nullptr, stream && i < nRefitStreams ? &mEvents->single : nullptr}}, i); + runKernel(GetGridAuto(stream, deviceType), i); + runKernel(GetGridAuto(stream, deviceType), i); + } + ReleaseEvent(mEvents->single); + if (nRefitStreams > 1) { // Stream 0 must wait for all streams before unpacking the extrapolated tracks + const int32_t ne = std::min(NSECTORS, nRefitStreams) - 1; + for (int32_t j = 0; j < ne; j++) { + RecordMarker(&mEvents->sector[j], j + 1); + } + StreamWaitForEvents(0, &mEvents->sector[0], ne); } if (GetProcessingSettings().deterministicGPUReconstruction) { runKernel({{1, -WarpSize(), 0, deviceType}}, NSECTORS); From 53b3d9eb45b3e92e84540df6ffa28108705badab Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Mon, 16 Nov 2026 15:45:33 +0100 Subject: [PATCH 03/10] GPU: put the orderings of the merger's sorts in a header Move the comparators of the TPC merger's sorts, and MergeLooperParam they sort, from GPUTPCGMMerger.cxx and GPUTPCGMO2Output.cxx into GPUTPCGMMergerSortComp.h, unchanged. The CUDA and HIP backends see them because they compile the kernel sources into one unit; a backend that runs these sorts in host code needs them from a header. --- GPU/GPUTracking/CMakeLists.txt | 1 + GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx | 79 +------------ .../Merger/GPUTPCGMMergerSortComp.h | 110 ++++++++++++++++++ GPU/GPUTracking/Merger/GPUTPCGMO2Output.cxx | 12 +- 4 files changed, 113 insertions(+), 89 deletions(-) create mode 100644 GPU/GPUTracking/Merger/GPUTPCGMMergerSortComp.h diff --git a/GPU/GPUTracking/CMakeLists.txt b/GPU/GPUTracking/CMakeLists.txt index c6a6906911680..e7199baa8a935 100644 --- a/GPU/GPUTracking/CMakeLists.txt +++ b/GPU/GPUTracking/CMakeLists.txt @@ -146,6 +146,7 @@ set(HDRS_INSTALL Global/GPUChainTrackingGetters.inc Global/GPUErrorCodes.h Merger/GPUTPCGMMergerTypes.h + Merger/GPUTPCGMMergerSortComp.h qa/GPUQAHelper.h qconfigoptions.h Refit/GPUTrackParamConvert.h diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx index cf8ea3c4f4675..57d04b4bf3c32 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx @@ -63,84 +63,7 @@ using namespace o2::gpu; using namespace o2::tpc; using namespace gputpcgmmergertypes; -namespace o2::gpu::internal -{ -struct MergeLooperParam { - float refz; - float x; - float y; - uint32_t id; -}; - -struct MergeBorderTracks_compMax { - GPUd() bool operator()(const GPUTPCGMBorderRange& a, const GPUTPCGMBorderRange& b) - { - return GPUCA_DETERMINISTIC_CODE((a.fMax != b.fMax) ? (a.fMax < b.fMax) : (a.fId < b.fId), a.fMax < b.fMax); - } -}; -struct MergeBorderTracks_compMin { - GPUd() bool operator()(const GPUTPCGMBorderRange& a, const GPUTPCGMBorderRange& b) - { - return GPUCA_DETERMINISTIC_CODE((a.fMin != b.fMin) ? (a.fMin < b.fMin) : (a.fId < b.fId), a.fMin < b.fMin); - } -}; - -struct GPUTPCGMMergerSortTracks_comp { - const GPUTPCGMMergedTrack* const mCmp; - GPUhd() GPUTPCGMMergerSortTracks_comp(GPUTPCGMMergedTrack* cmp) : mCmp(cmp) {} - GPUd() bool operator()(const int32_t aa, const int32_t bb) - { - const GPUTPCGMMergedTrack& GPUrestrict() a = mCmp[aa]; - const GPUTPCGMMergedTrack& GPUrestrict() b = mCmp[bb]; - if (a.OK() != b.OK()) { - return a.OK(); - } - if (a.CCE() != b.CCE()) { - return a.CCE() > b.CCE(); - } - GPUCA_DETERMINISTIC_CODE( // clang-format off - if (a.NClusters() != b.NClusters()) { - return a.NClusters() > b.NClusters(); - } if (CAMath::Abs(a.GetParam().GetQPt()) != CAMath::Abs(b.GetParam().GetQPt())) { - return CAMath::Abs(a.GetParam().GetQPt()) > CAMath::Abs(b.GetParam().GetQPt()); - } if (a.GetParam().GetY() != b.GetParam().GetY()) { - return a.GetParam().GetY() > b.GetParam().GetY(); - } - return aa > bb; - , // !GPUCA_DETERMINISTIC_CODE - return a.NClusters() > b.NClusters(); - ) // clang-format on - } -}; - -struct GPUTPCGMMergerSortTracksQPt_comp { - const GPUTPCGMMergedTrack* const mCmp; - GPUhd() GPUTPCGMMergerSortTracksQPt_comp(GPUTPCGMMergedTrack* cmp) : mCmp(cmp) {} - GPUd() bool operator()(const int32_t aa, const int32_t bb) - { - const GPUTPCGMMergedTrack& GPUrestrict() a = mCmp[aa]; - const GPUTPCGMMergedTrack& GPUrestrict() b = mCmp[bb]; - GPUCA_DETERMINISTIC_CODE( // clang-format off - if (CAMath::Abs(a.GetParam().GetQPt()) != CAMath::Abs(b.GetParam().GetQPt())) { - return CAMath::Abs(a.GetParam().GetQPt()) > CAMath::Abs(b.GetParam().GetQPt()); - } if (a.GetParam().GetY() != b.GetParam().GetY()) { - return a.GetParam().GetY() > b.GetParam().GetY(); - } - return a.GetParam().GetZ() > b.GetParam().GetZ(); - , // !GPUCA_DETERMINISTIC_CODE - return CAMath::Abs(a.GetParam().GetQPt()) > CAMath::Abs(b.GetParam().GetQPt()); - ) // clang-format on - } -}; - -struct GPUTPCGMMergerMergeLoopers_comp { - GPUd() bool operator()(const MergeLooperParam& a, const MergeLooperParam& b) - { - return GPUCA_DETERMINISTIC_CODE(CAMath::Abs(a.refz) != CAMath::Abs(b.refz) ? CAMath::Abs(a.refz) < CAMath::Abs(b.refz) : a.id < b.id, CAMath::Abs(a.refz) < CAMath::Abs(b.refz)); - } -}; - -} // namespace o2::gpu::internal +#include "GPUTPCGMMergerSortComp.h" using namespace o2::gpu::internal; diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerSortComp.h b/GPU/GPUTracking/Merger/GPUTPCGMMergerSortComp.h new file mode 100644 index 0000000000000..ea239a726a71a --- /dev/null +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerSortComp.h @@ -0,0 +1,110 @@ +// Copyright 2019-2025 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// \file GPUTPCGMMergerSortComp.h +/// \brief Orderings of the merger's sorts, for the kernels and for the backends that run these sorts themselves + +#ifndef GPUTPCGMMERGERSORTCOMP_H +#define GPUTPCGMMERGERSORTCOMP_H + +#include "GPUCommonDef.h" +#include "GPUCommonMath.h" +#include "GPUTPCGMMergedTrack.h" +#include "GPUTPCGMMergerTypes.h" +#include "GPUTPCGMMerger.h" + +namespace o2::gpu::internal +{ +struct MergeLooperParam { + float refz; + float x; + float y; + uint32_t id; +}; + +struct MergeBorderTracks_compMax { + GPUd() bool operator()(const gputpcgmmergertypes::GPUTPCGMBorderRange& a, const gputpcgmmergertypes::GPUTPCGMBorderRange& b) + { + return GPUCA_DETERMINISTIC_CODE((a.fMax != b.fMax) ? (a.fMax < b.fMax) : (a.fId < b.fId), a.fMax < b.fMax); + } +}; +struct MergeBorderTracks_compMin { + GPUd() bool operator()(const gputpcgmmergertypes::GPUTPCGMBorderRange& a, const gputpcgmmergertypes::GPUTPCGMBorderRange& b) + { + return GPUCA_DETERMINISTIC_CODE((a.fMin != b.fMin) ? (a.fMin < b.fMin) : (a.fId < b.fId), a.fMin < b.fMin); + } +}; + +struct GPUTPCGMMergerSortTracks_comp { + const GPUTPCGMMergedTrack* const mCmp; + GPUhd() GPUTPCGMMergerSortTracks_comp(GPUTPCGMMergedTrack* cmp) : mCmp(cmp) {} + GPUd() bool operator()(const int32_t aa, const int32_t bb) + { + const GPUTPCGMMergedTrack& GPUrestrict() a = mCmp[aa]; + const GPUTPCGMMergedTrack& GPUrestrict() b = mCmp[bb]; + if (a.OK() != b.OK()) { + return a.OK(); + } + if (a.CCE() != b.CCE()) { + return a.CCE() > b.CCE(); + } + GPUCA_DETERMINISTIC_CODE( // clang-format off + if (a.NClusters() != b.NClusters()) { + return a.NClusters() > b.NClusters(); + } if (CAMath::Abs(a.GetParam().GetQPt()) != CAMath::Abs(b.GetParam().GetQPt())) { + return CAMath::Abs(a.GetParam().GetQPt()) > CAMath::Abs(b.GetParam().GetQPt()); + } if (a.GetParam().GetY() != b.GetParam().GetY()) { + return a.GetParam().GetY() > b.GetParam().GetY(); + } + return aa > bb; + , // !GPUCA_DETERMINISTIC_CODE + return a.NClusters() > b.NClusters(); + ) // clang-format on + } +}; + +struct GPUTPCGMMergerSortTracksQPt_comp { + const GPUTPCGMMergedTrack* const mCmp; + GPUhd() GPUTPCGMMergerSortTracksQPt_comp(GPUTPCGMMergedTrack* cmp) : mCmp(cmp) {} + GPUd() bool operator()(const int32_t aa, const int32_t bb) + { + const GPUTPCGMMergedTrack& GPUrestrict() a = mCmp[aa]; + const GPUTPCGMMergedTrack& GPUrestrict() b = mCmp[bb]; + GPUCA_DETERMINISTIC_CODE( // clang-format off + if (CAMath::Abs(a.GetParam().GetQPt()) != CAMath::Abs(b.GetParam().GetQPt())) { + return CAMath::Abs(a.GetParam().GetQPt()) > CAMath::Abs(b.GetParam().GetQPt()); + } if (a.GetParam().GetY() != b.GetParam().GetY()) { + return a.GetParam().GetY() > b.GetParam().GetY(); + } + return a.GetParam().GetZ() > b.GetParam().GetZ(); + , // !GPUCA_DETERMINISTIC_CODE + return CAMath::Abs(a.GetParam().GetQPt()) > CAMath::Abs(b.GetParam().GetQPt()); + ) // clang-format on + } +}; + +struct GPUTPCGMMergerMergeLoopers_comp { + GPUd() bool operator()(const MergeLooperParam& a, const MergeLooperParam& b) + { + return GPUCA_DETERMINISTIC_CODE(CAMath::Abs(a.refz) != CAMath::Abs(b.refz) ? CAMath::Abs(a.refz) < CAMath::Abs(b.refz) : a.id < b.id, CAMath::Abs(a.refz) < CAMath::Abs(b.refz)); + } +}; + +struct GPUTPCGMO2OutputSort_comp { + GPUd() bool operator()(const GPUTPCGMMerger::tmpSort& a, const GPUTPCGMMerger::tmpSort& b) + { + return GPUCA_DETERMINISTIC_CODE(a.y != b.y ? a.y > b.y : a.x > b.x, a.y > b.y); + } +}; + +} // namespace o2::gpu::internal + +#endif diff --git a/GPU/GPUTracking/Merger/GPUTPCGMO2Output.cxx b/GPU/GPUTracking/Merger/GPUTPCGMO2Output.cxx index 4bcffd8a31c46..a649cfdaedfae 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMO2Output.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMO2Output.cxx @@ -33,17 +33,7 @@ using namespace o2::tpc::constants; GPUdi() static constexpr uint8_t getFlagsReject() { return GPUTPCGMMergedTrackHit::flagReject | GPUTPCGMMergedTrackHit::flagHighIncl; } -namespace o2::gpu::internal -{ - -struct GPUTPCGMO2OutputSort_comp { - GPUd() bool operator()(const GPUTPCGMMerger::tmpSort& a, const GPUTPCGMMerger::tmpSort& b) - { - return GPUCA_DETERMINISTIC_CODE(a.y != b.y ? a.y > b.y : a.x > b.x, a.y > b.y); - } -}; - -} // namespace o2::gpu::internal +#include "GPUTPCGMMergerSortComp.h" template <> GPUdii() void GPUTPCGMO2Output::Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& GPUrestrict() merger) From 687be8e1fdd9d6bb280c5f92994f542e476497c6 Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Mon, 16 Nov 2026 15:45:33 +0100 Subject: [PATCH 04/10] GPU TPC clusterizer: decode the dense-link ZS channels after the link headers on Metal GPUTPCCFDecodeZSDenseLink decodes a page with one warp, walking it timebin by timebin and link by link. For every link header, the warp also collected the channels with data, a thread per channel with three warp scans, inside a serial chain of dependent loads that is the latency limit of the kernel. On Metal the header loop now only walks the headers, and thread 0 stores each link's level-2 bitmask and the offset of its channel bitmask in shared memory (+64 bytes). The channels are then collected with one thread per byte of a link's channel bitmask: one warp scan per 32 bytes instead of about three per link. Same samples in the same order, same pointers into the page, including into the next one, same error paths. On an M1 Max the decode takes 23 to 39% less time per launch in normal multi-stream runs (pbpb30 4.9 -> 3.7 ms, 50 kHz TF0 6.7 -> 4.2 ms), about 1 to 2% of the full time frame. The cluster dumps are byte-identical: sim5pp against the CPU, 30 kHz (1 of 4) and 50 kHz TF0 before against after. pbpb30 in deterministic mode gives 22,183 tracks as before, and a 36 against 36 pass comparison at 50 kHz shows no difference in the track count. It is Metal-only for now, since it was measured only there: CUDA, HIP, OpenCL and the CPU compile the existing code unchanged. The chain it shortens is there on every backend, so it could be enabled for them once timed. --- .../TPCClusterFinder/GPUTPCCFDecodeZS.cxx | 42 +++++++++++++++++++ .../TPCClusterFinder/GPUTPCCFDecodeZS.h | 4 ++ 2 files changed, 46 insertions(+) diff --git a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFDecodeZS.cxx b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFDecodeZS.cxx index cd1b9b32494fd..f27fbb092d76e 100644 --- a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFDecodeZS.cxx +++ b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFDecodeZS.cxx @@ -670,6 +670,9 @@ GPUd() int16_t GPUTPCCFDecodeZSDenseLink::DecodeTB( int32_t timeBin = (linkBC + (uint64_t)(raw::RDHUtils::getHeartBeatOrbit(*rawDataHeader) - ctx.firstHBF) * o2::constants::lhc::LHCMaxBunches) / LHCBCPERTIMEBIN; int16_t nSamplesInTB = 0; +#ifdef __METAL__ + const uint8_t* const tbStart = ctx.page; +#endif // Read timebin link headers for (uint8_t iLink = 0; iLink < nLinksInTimebin; iLink++) { @@ -687,6 +690,15 @@ GPUd() int16_t GPUTPCCFDecodeZSDenseLink::DecodeTB( MAYBE_PAGE_OVERFLOW(ctx.page); } +#ifdef __METAL__ // Channels collected after this loop, see below. Measured on Metal only, not yet timed on CUDA and HIP + if (ctx.iThread == 0) { + smem.linkBitmaskL2[iLink] = bitmaskL2; + smem.linkBitmaskOffset[iLink] = ctx.page - tbStart; + } + + ConsumeBytes(ctx.page, CAMath::Popcount(bitmaskL2)); + MAYBE_PAGE_OVERFLOW(ctx.page); +#else int32_t nBytesBitmask = CAMath::Popcount(bitmaskL2); for (int32_t chan = ctx.iThread; chan < CAMath::nextMultipleOf(80); chan += NTHREADS) { @@ -713,9 +725,39 @@ GPUd() int16_t GPUTPCCFDecodeZSDenseLink::DecodeTB( if (ctx.iThread == 0) { smem.samplesPerLinkEnd[iLink] = nSamplesInTB; } +#endif } // for (uint8_t iLink = 0; iLink < nLinksInTimebin; iLink++) +#ifdef __METAL__ + GPUbarrierWarp(); // Ensure the link headers are in shared memory before reading them + + // Collect the channels with data, one byte of a link's channel bitmask per thread. A thread per + // channel would need 3 warp-wide scans for each link, in a chain that is long enough as it is. + for (int32_t slot = ctx.iThread; slot < CAMath::nextMultipleOf(nLinksInTimebin * 10); slot += NTHREADS) { + const int32_t iLink = slot / 10; + const int32_t chanL2Idx = slot % 10; + uint8_t bitmask = 0; + if (iLink < nLinksInTimebin) { + const uint16_t bitmaskL2 = smem.linkBitmaskL2[iLink]; + if (TEST_BIT(bitmaskL2, chanL2Idx)) { + const int32_t chanByteOffset = CAMath::Popcount(bitmaskL2) - 1 - CAMath::Popcount(bitmaskL2 >> (chanL2Idx + 1)); + bitmask = PEEK_OVERFLOW(tbStart + smem.linkBitmaskOffset[iLink], chanByteOffset); + } + } + const int32_t nSamplesThread = CAMath::Popcount(bitmask); + const int32_t threadSampleEnd = warp_scan_inclusive_add(nSamplesThread); + int32_t sampleOffset = nSamplesInTB + threadSampleEnd - nSamplesThread; + for (uint32_t bits = bitmask; bits; bits &= bits - 1) { + smem.rawFECChannels[sampleOffset++] = chanL2Idx * 8 + CAMath::Ctz(bits); + } + if (chanL2Idx == 9 && iLink < nLinksInTimebin) { + smem.samplesPerLinkEnd[iLink] = nSamplesInTB + threadSampleEnd; + } + nSamplesInTB += warp_broadcast(threadSampleEnd, NTHREADS - 1); + } +#endif + GPUbarrierWarp(); // Ensure all writes to shared memory are finished, before reading it if (nSamplesInTB > nSamplesLeftInPage) { diff --git a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFDecodeZS.h b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFDecodeZS.h index eed5e0d6f30ac..91d78618245a5 100644 --- a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFDecodeZS.h +++ b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFDecodeZS.h @@ -165,6 +165,10 @@ class GPUTPCCFDecodeZSDenseLink : public GPUTPCCFDecodeZSLinkBase // GPUCA_SHARED_STORAGE(uint32_t ZSPage[o2::tpc::TPCZSHDR::TPC_ZS_PAGE_SIZE / sizeof(uint32_t)]); uint16_t samplesPerLinkEnd[MaxNLinksPerTimebin]; // Offset from end of TB link header to first sample not in this link uint8_t linkIds[MaxNLinksPerTimebin]; +#ifdef __METAL__ + uint16_t linkBitmaskL2[MaxNLinksPerTimebin]; + uint16_t linkBitmaskOffset[MaxNLinksPerTimebin]; // Offset of the link's channel bitmask from the timebin header +#endif uint8_t rawFECChannels[MaxNLinksPerTimebin * 80]; }; From aa77dc3f5d8dd70f950c2de94997ca62466c8b19 Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Mon, 16 Nov 2026 15:45:33 +0100 Subject: [PATCH 05/10] GPU Metal: run the merger's sorts on the host The sorts of the merged tracks by quality and by q/pT, of the looper candidates and of the O2 output tracks, which CUDA and HIP run with thrust, ran on a single GPU thread otherwise: about 730 ms per event on a 30-collision Pb-Pb time frame, at up to 140 times the CPU's time. The GPU memory is shared with the host, so sort there once the stream is drained, with the std::sort of the CPU backend, which also orders the elements that compare equal as on the CPU. The number of elements is read from the device copy of the counters, as the host copy of the O2 output count is still being transferred when the sort is launched. --- .../Base/metal/GPUReconstructionMetal.h | 4 ++ ...PUReconstructionMetalKernelsSpecialize.inc | 53 +++++++++++++++++++ 2 files changed, 57 insertions(+) diff --git a/GPU/GPUTracking/Base/metal/GPUReconstructionMetal.h b/GPU/GPUTracking/Base/metal/GPUReconstructionMetal.h index 247ae6657c01e..091fbbb415804 100644 --- a/GPU/GPUTracking/Base/metal/GPUReconstructionMetal.h +++ b/GPU/GPUTracking/Base/metal/GPUReconstructionMetal.h @@ -64,6 +64,10 @@ class GPUReconstructionMetal : public GPUReconstructionProcessing::KernelInterfa void CommitStream(int32_t stream); void DrainStream(int32_t stream); uint64_t EventValue(deviceEvent ev, bool next); + template + T* HostPointer(T* p); + template + void SortOnHost(const krnlSetupTime& _xyz, T* begin, const N& count, const S& comp); private: int32_t AddKernels(); diff --git a/GPU/GPUTracking/Base/metal/GPUReconstructionMetalKernelsSpecialize.inc b/GPU/GPUTracking/Base/metal/GPUReconstructionMetalKernelsSpecialize.inc index 5f456a7ce06bb..869ba7790526b 100644 --- a/GPU/GPUTracking/Base/metal/GPUReconstructionMetalKernelsSpecialize.inc +++ b/GPU/GPUTracking/Base/metal/GPUReconstructionMetalKernelsSpecialize.inc @@ -11,6 +11,8 @@ /// \file GPUReconstructionMetalKernelsSpecialize.inc +#include "GPUTPCGMMergerSortComp.h" + template <> inline void GPUReconstructionMetal::runKernelBackend(const krnlSetupTime& _xyz, void* const& ptr, uint64_t const& size) { @@ -30,3 +32,54 @@ inline void GPUReconstructionMetal::runKernelBackend(const krn RecordMarker(_xyz.z.ev, _xyz.x.stream); } } + +// The sorts that CUDA and HIP run with thrust (GPUCA_SPECIALIZE_THRUST_SORTS) +// run on a single GPU thread otherwise. The memory is shared with the host, so +// sort there, with the same std::sort as the CPU backend, which also orders +// the elements that compare equal as on the CPU. The number of elements is +// read once the stream is drained, from the device copy of the counters, +// since the host copy may still be on its way. +template +inline T* GPUReconstructionMetal::HostPointer(T* p) +{ + const size_t offset = (const char*)p - (const char*)mDeviceMemoryBase; + return offset < mDeviceMemorySize ? (T*)((char*)mInternals->mem_gpu.contents + offset) : p; +} + +template +inline void GPUReconstructionMetal::SortOnHost(const krnlSetupTime& _xyz, T* begin, const N& count, const S& comp) +{ + if (_xyz.z.evList) { + StreamWaitForEvents(_xyz.x.stream, _xyz.z.evList, _xyz.z.nEvents); + } + DrainStream(_xyz.x.stream); + T* hostBegin = HostPointer(begin); + GPUCommonAlgorithm::sortDeviceDynamic(hostBegin, hostBegin + count(), comp); + if (_xyz.z.ev) { + RecordMarker(_xyz.z.ev, _xyz.x.stream); + } +} + +template <> +inline void GPUReconstructionMetal::runKernelBackend(const krnlSetupTime& _xyz) +{ + SortOnHost(_xyz, mProcessorsShadow->tpcMerger.TrackOrderProcess(), [&] { return HostPointer(mProcessorsShadow->tpcMerger.Memory())->nMergedTracks; }, internal::GPUTPCGMMergerSortTracks_comp(HostPointer(mProcessorsShadow->tpcMerger.MergedTracks()))); +} + +template <> +inline void GPUReconstructionMetal::runKernelBackend(const krnlSetupTime& _xyz) +{ + SortOnHost(_xyz, mProcessorsShadow->tpcMerger.TrackSort(), [&] { return HostPointer(mProcessorsShadow->tpcMerger.Memory())->nMergedTracks; }, internal::GPUTPCGMMergerSortTracksQPt_comp(HostPointer(mProcessorsShadow->tpcMerger.MergedTracks()))); +} + +template <> +inline void GPUReconstructionMetal::runKernelBackend(const krnlSetupTime& _xyz) +{ + SortOnHost(_xyz, mProcessorsShadow->tpcMerger.LooperCandidates(), [&] { return HostPointer(mProcessorsShadow->tpcMerger.Memory())->nLooperMatchCandidates; }, internal::GPUTPCGMMergerMergeLoopers_comp()); +} + +template <> +inline void GPUReconstructionMetal::runKernelBackend(const krnlSetupTime& _xyz) +{ + SortOnHost(_xyz, mProcessorsShadow->tpcMerger.TrackSortO2(), [&] { return HostPointer(mProcessorsShadow->tpcMerger.Memory())->nO2Tracks; }, internal::GPUTPCGMO2OutputSort_comp()); +} From e92998710edfbfec32a8d48e1b636728d1d0de82 Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Mon, 16 Nov 2026 15:45:33 +0100 Subject: [PATCH 06/10] GPU Metal: also run the border-range sorts on the host The border-range sorts of the track merging between sectors, 360 per time frame, took 312 ms per event on a 30-collision Pb-Pb time frame on a single GPU thread. On the host they take about 6 ms, plus 52 ms of waiting for the streams. --- .../metal/GPUReconstructionMetalKernelsSpecialize.inc | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/GPU/GPUTracking/Base/metal/GPUReconstructionMetalKernelsSpecialize.inc b/GPU/GPUTracking/Base/metal/GPUReconstructionMetalKernelsSpecialize.inc index 869ba7790526b..63b771565f80d 100644 --- a/GPU/GPUTracking/Base/metal/GPUReconstructionMetalKernelsSpecialize.inc +++ b/GPU/GPUTracking/Base/metal/GPUReconstructionMetalKernelsSpecialize.inc @@ -83,3 +83,13 @@ inline void GPUReconstructionMetal::runKernelBackendtpcMerger.TrackSortO2(), [&] { return HostPointer(mProcessorsShadow->tpcMerger.Memory())->nO2Tracks; }, internal::GPUTPCGMO2OutputSort_comp()); } + +template <> +inline void GPUReconstructionMetal::runKernelBackend(const krnlSetupTime& _xyz, gputpcgmmergertypes::GPUTPCGMBorderRange* const& range, int32_t const& N, int32_t const& cmpMax) +{ + if (cmpMax) { + SortOnHost(_xyz, range, [&] { return N; }, internal::MergeBorderTracks_compMax()); + } else { + SortOnHost(_xyz, range, [&] { return N; }, internal::MergeBorderTracks_compMin()); + } +} From 6c767cce7d1468815ff0a1332246babac44ca244 Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Mon, 16 Nov 2026 15:45:33 +0100 Subject: [PATCH 07/10] GPU Metal: run the second step of the border merging on the host The candidate search of the border track merging, 180 launches per time frame of a few thousand tracks each, took 46 ms per event (kernels serialized) on a 30-collision Pb-Pb time frame on the GPU, where each thread first walks the sorted ranges from the start. On the host, right after the range sorts which already drained the stream, it takes 4.6 ms, including pointing the processors at the host view of the shared memory and back for each launch. One host thread, so that the merging across the central electrode stays as on the CPU. Benchmark, 6 interleaved rounds of 10 events: -6 +- 2 ms per event. The kernel runs on a private copy of mProcessorsShadow: SetDevicePointers of every allocated resource points the processors at the host memory, they are copied, and the shadow is restored from a copy before the kernel runs, so the host threads never touch it. The copy is the processors' constant memory, with the parameters and calibration objects of the host. With --debug 1 and above, a change of the shadow while the kernel runs is fatal. --- .../Base/metal/GPUReconstructionMetal.h | 2 + ...PUReconstructionMetalKernelsSpecialize.inc | 65 +++++++++++++++++++ 2 files changed, 67 insertions(+) diff --git a/GPU/GPUTracking/Base/metal/GPUReconstructionMetal.h b/GPU/GPUTracking/Base/metal/GPUReconstructionMetal.h index 091fbbb415804..54e599302df7d 100644 --- a/GPU/GPUTracking/Base/metal/GPUReconstructionMetal.h +++ b/GPU/GPUTracking/Base/metal/GPUReconstructionMetal.h @@ -68,6 +68,8 @@ class GPUReconstructionMetal : public GPUReconstructionProcessing::KernelInterfa T* HostPointer(T* p); template void SortOnHost(const krnlSetupTime& _xyz, T* begin, const N& count, const S& comp); + template + void RunOnHost(const krnlSetupTime& _xyz, typename T::processorType* procs, uint32_t nProcs, uint32_t nBlocks, const Args&... args); private: int32_t AddKernels(); diff --git a/GPU/GPUTracking/Base/metal/GPUReconstructionMetalKernelsSpecialize.inc b/GPU/GPUTracking/Base/metal/GPUReconstructionMetalKernelsSpecialize.inc index 63b771565f80d..5d1b3a8283541 100644 --- a/GPU/GPUTracking/Base/metal/GPUReconstructionMetalKernelsSpecialize.inc +++ b/GPU/GPUTracking/Base/metal/GPUReconstructionMetalKernelsSpecialize.inc @@ -12,6 +12,8 @@ /// \file GPUReconstructionMetalKernelsSpecialize.inc #include "GPUTPCGMMergerSortComp.h" +#include +#include template <> inline void GPUReconstructionMetal::runKernelBackend(const krnlSetupTime& _xyz, void* const& ptr, uint64_t const& size) @@ -60,6 +62,63 @@ inline void GPUReconstructionMetal::SortOnHost(const krnlSetupTime& _xyz, T* beg } } +// Kernels that are a long serial chain per GPU thread, with little parallelism, +// run on the host instead, on the shared memory, as the CPU backend runs them. +// The processors in mProcessorsShadow hold GPU addresses: the kernel runs on a +// private copy of mProcessorsShadow, whose processors point at the host view of +// the same memory, with the host's parameters and calibration objects, and +// which is their constant memory. SetDevicePointers relocates the pointers in +// place, so they are copied and the shadow is restored before the kernel runs: +// the kernel, on many host threads, never touches the shadow. +struct GPUMetalHostProcessor : public GPUProcessor { + static const GPUConstantMem*& ConstantMem(GPUProcessor& p) { return p.*&GPUMetalHostProcessor::mConstantMem; } +}; + +template +inline void GPUReconstructionMetal::RunOnHost(const krnlSetupTime& _xyz, typename T::processorType* procs, uint32_t nProcs, uint32_t nBlocks, const Args&... args) +{ + if (_xyz.z.evList) { + StreamWaitForEvents(_xyz.x.stream, _xyz.z.evList, _xyz.z.nEvents); + } + DrainStream(_xyz.x.stream); + const auto start = std::chrono::steady_clock::now(); + const auto shadow = std::make_unique(sizeof(GPUConstantMem)); + static_assert(alignof(GPUConstantMem) <= __STDCPP_DEFAULT_NEW_ALIGNMENT__); + const auto hostMem = std::make_unique(sizeof(GPUConstantMem)); + GPUConstantMem* host = (GPUConstantMem*)hostMem.get(); + memcpy(shadow.get(), (void*)mProcessorsShadow, sizeof(GPUConstantMem)); + for (const auto& res : mMemoryResources) { + if (res.PtrDevice()) { + res.SetDevicePointers(HostPointer(res.PtrDevice())); + } + } + memcpy((void*)host, (void*)mProcessorsShadow, sizeof(GPUConstantMem)); + for (const auto& res : mMemoryResources) { + if (res.PtrDevice()) { + res.SetDevicePointers(res.PtrDevice()); + } + } + memcpy((void*)mProcessorsShadow, shadow.get(), sizeof(GPUConstantMem)); + host->param = param(); + host->calibObjects = processors()->calibObjects; + host->errorCodes.setMemory(HostPointer(const_cast(host->errorCodes.getErrorPtr()) - 1)); + auto* hostProcs = (typename T::processorType*)((char*)host + ((char*)procs - (char*)mProcessorsShadow)); + for (uint32_t i = 0; i < nProcs; i++) { + GPUMetalHostProcessor::ConstantMem(hostProcs[i]) = host; + } + tbb::parallel_for(0, nBlocks, [&](uint32_t iBlock) { + typename T::GPUSharedMemory smem; + T::template Thread(nBlocks, 1, iBlock, 0, smem, hostProcs[_xyz.y.index], args...); + }); + if (GetProcessingSettings().debugLevel >= 1 && memcmp(shadow.get(), (void*)mProcessorsShadow, sizeof(GPUConstantMem))) { + GPUFatal("Metal: mProcessorsShadow changed while %s ran on the host", GetKernelName()); + } + _xyz.t = std::chrono::duration(std::chrono::steady_clock::now() - start).count(); + if (_xyz.z.ev) { + RecordMarker(_xyz.z.ev, _xyz.x.stream); + } +} + template <> inline void GPUReconstructionMetal::runKernelBackend(const krnlSetupTime& _xyz) { @@ -93,3 +152,9 @@ inline void GPUReconstructionMetal::runKernelBackend +inline void GPUReconstructionMetal::runKernelBackend(const krnlSetupTime& _xyz, int32_t const& iSector, uint8_t const& mergeMode) +{ + RunOnHost(_xyz, &mProcessorsShadow->tpcMerger, 1, 1, iSector, mergeMode); +} From 31f0a89ecdbd119c35d3522381404cc2d883192c Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Mon, 16 Nov 2026 15:45:33 +0100 Subject: [PATCH 08/10] GPU Metal: run the extrapolation tracking on the host The extrapolation of the sector tracks into the neighbouring sectors is one thread per track, of which only those near a sector border work, on a long serial Kalman chain. It took 133 ms per event (kernels serialized) on a 30-collision Pb-Pb time frame on the GPU, against 7.8 ms on the host, run as the CPU backend does on the host threads. The stream is drained first, which includes the waits for the neighbouring sectors that the chain puts in it. Benchmark, 6 interleaved rounds of 10 events: -51 +- 2 ms per event with both border merging and extrapolation on the host, -15 +- 2 ms against a single host thread. --- .../Base/metal/GPUReconstructionMetalKernelsSpecialize.inc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/GPU/GPUTracking/Base/metal/GPUReconstructionMetalKernelsSpecialize.inc b/GPU/GPUTracking/Base/metal/GPUReconstructionMetalKernelsSpecialize.inc index 5d1b3a8283541..7e828340f9d59 100644 --- a/GPU/GPUTracking/Base/metal/GPUReconstructionMetalKernelsSpecialize.inc +++ b/GPU/GPUTracking/Base/metal/GPUReconstructionMetalKernelsSpecialize.inc @@ -158,3 +158,9 @@ inline void GPUReconstructionMetal::runKernelBackend(_xyz, &mProcessorsShadow->tpcMerger, 1, 1, iSector, mergeMode); } + +template <> +inline void GPUReconstructionMetal::runKernelBackend(const krnlSetupTime& _xyz) +{ + RunOnHost(_xyz, mProcessorsShadow->tpcTrackers, GPUTPCGeometry::NSECTORS, getNKernelHostThreads(false)); +} From 4ef5f4683bb9006d6051ced54679f8d6f49dea39 Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Mon, 16 Nov 2026 15:45:33 +0100 Subject: [PATCH 09/10] GPU Metal: fit the tracks grouped by sector within length buckets The merger fits one track per thread, in the order of GPUTPCGMMergerSortTracks, which sorts by the number of clusters so that the threads of a SIMD group run for as long as each other. On Metal the fit is bound by the address translation of its scattered reads (on an M1 Max during the fit: MMU limiter 48%, TLB miss rate 68%, ALU utilization 5%): the 32 threads of a group read clusters, transformation and calibration data of 32 unrelated sectors. The Metal backend, which runs this sort on the host, now sorts by the number of clusters in buckets of 16, and within a bucket by side and sector, so that the threads of a group mostly read the same pages. On a quarter of a 30 kHz Pb-Pb time frame the fit takes 13% less time (679 -> 593 ms, --debug 1, interleaved runs). Buckets of 1 (610 ms), 8, 32 and 64 clusters are slower; ordering by z within a sector does not help; no sort at all takes 950 ms. The order does not change which track is fitted how: the sim5pp cluster dump is byte-identical to the CPU, and pbpb30 in deterministic mode gives 22,183 tracks. --- ...PUReconstructionMetalKernelsSpecialize.inc | 27 ++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/GPU/GPUTracking/Base/metal/GPUReconstructionMetalKernelsSpecialize.inc b/GPU/GPUTracking/Base/metal/GPUReconstructionMetalKernelsSpecialize.inc index 7e828340f9d59..1626fe978b6cf 100644 --- a/GPU/GPUTracking/Base/metal/GPUReconstructionMetalKernelsSpecialize.inc +++ b/GPU/GPUTracking/Base/metal/GPUReconstructionMetalKernelsSpecialize.inc @@ -122,7 +122,32 @@ inline void GPUReconstructionMetal::RunOnHost(const krnlSetupTime& _xyz, typenam template <> inline void GPUReconstructionMetal::runKernelBackend(const krnlSetupTime& _xyz) { - SortOnHost(_xyz, mProcessorsShadow->tpcMerger.TrackOrderProcess(), [&] { return HostPointer(mProcessorsShadow->tpcMerger.Memory())->nMergedTracks; }, internal::GPUTPCGMMergerSortTracks_comp(HostPointer(mProcessorsShadow->tpcMerger.MergedTracks()))); + // The fit runs one track per thread, in this order. Tracks of similar length + // keep the threads of a SIMD group busy for as long as each other; within a + // length bucket of 16 clusters, tracks of the same sector and side then read + // the same pages of the cluster, transformation and calibration data, which + // the Metal GPU, bound by its address translation in the fit, needs more than + // the exact length order. + const GPUTPCGMMergedTrack* trk = HostPointer(mProcessorsShadow->tpcMerger.MergedTracks()); + auto comp = [trk](const int32_t aa, const int32_t bb) { + const GPUTPCGMMergedTrack& a = trk[aa]; + const GPUTPCGMMergedTrack& b = trk[bb]; + if (a.OK() != b.OK()) { + return a.OK(); + } + if (a.CCE() != b.CCE()) { + return a.CCE() > b.CCE(); + } + if (a.NClusters() / 16 != b.NClusters() / 16) { + return a.NClusters() / 16 > b.NClusters() / 16; + } + const bool sideA = a.GetParam().GetZ() < 0, sideB = b.GetParam().GetZ() < 0; + if (sideA != sideB || a.GetAlpha() != b.GetAlpha()) { + return sideA != sideB ? sideA < sideB : a.GetAlpha() < b.GetAlpha(); + } + return aa < bb; + }; + SortOnHost(_xyz, mProcessorsShadow->tpcMerger.TrackOrderProcess(), [&] { return HostPointer(mProcessorsShadow->tpcMerger.Memory())->nMergedTracks; }, comp); } template <> From 6d525863af8cc0119a4fe365409baf07bf7b519c Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Mon, 16 Nov 2026 15:45:33 +0100 Subject: [PATCH 10/10] GPU Metal: let more groups of the tracklet constructor fit on a core The tracklet constructor follows each tracklet row by row through a chain of dependent reads: hit data, links and three spline evaluations of the transformation per row, which take about 60% of its time. On an M1 Max it is bound by their latency, not by bandwidth, address translation or ALU (ALU utilization 16%, LLC limiter 22%, MMU limiter 5%, compute occupancy 10%), so it gets faster with more threads in flight, and two limits kept them few: the 12 KB of rows that each group copies to threadgroup memory, and registers allocated for groups of 128 threads. On Metal the rows are now read from device memory, and the pipeline is compiled as for 640 threads per group while it still runs groups of 128, as the minimum blocks of __launch_bounds__ do. Either change alone gains nothing. Together the kernel takes 14% less time on a quarter of a 30 kHz Pb-Pb time frame: 421 -> 362 ms (--debug 1, three interleaved rounds), with the same number of tracks. Compiling for 512 to 768 threads is within 4%, 1024 is slower; groups of 64 or 192 threads are slower than 128. The sim5pp cluster dump is byte-identical to the CPU, and pbpb30 in deterministic mode gives 22,183 tracks. Other backends compile the existing code unchanged. --- .../Base/metal/GPUReconstructionMetalKernels.mm | 7 +++++-- .../SectorTracker/GPUTPCTrackletConstructor.cxx | 17 +++++++++++++++-- .../SectorTracker/GPUTPCTrackletConstructor.h | 4 +++- 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/GPU/GPUTracking/Base/metal/GPUReconstructionMetalKernels.mm b/GPU/GPUTracking/Base/metal/GPUReconstructionMetalKernels.mm index b9c532485cba1..9d5862bcd62eb 100644 --- a/GPU/GPUTracking/Base/metal/GPUReconstructionMetalKernels.mm +++ b/GPU/GPUTracking/Base/metal/GPUReconstructionMetalKernels.mm @@ -102,10 +102,13 @@ const uint32_t tunedThreads = this->template getKernelProperties().nThreads; // Like __launch_bounds__: without it the register allocation may leave room - // for fewer threads than the kernel is tuned for. + // for fewer threads than the kernel is tuned for. The tracklet constructor + // runs in groups of 128 threads but gets the registers of a group of 640, as + // the minimum blocks of __launch_bounds__ would do, so that more groups fit + // on a core to hide the latency of its row-by-row reads. MTLComputePipelineDescriptor* desc = [MTLComputePipelineDescriptor new]; desc.computeFunction = krnl; - desc.maxTotalThreadsPerThreadgroup = tunedThreads; + desc.maxTotalThreadsPerThreadgroup = std::is_same_v ? 640 : tunedThreads; NSError* error = nil; id pso = [mInternals->device newComputePipelineStateWithDescriptor:desc options:MTLPipelineOptionNone reflection:nil error:&error]; [desc release]; diff --git a/GPU/GPUTracking/SectorTracker/GPUTPCTrackletConstructor.cxx b/GPU/GPUTracking/SectorTracker/GPUTPCTrackletConstructor.cxx index 33a3264a87ab3..09bd6cb09f858 100644 --- a/GPU/GPUTracking/SectorTracker/GPUTPCTrackletConstructor.cxx +++ b/GPU/GPUTracking/SectorTracker/GPUTPCTrackletConstructor.cxx @@ -29,6 +29,15 @@ using namespace o2::gpu; +// Metal reads the rows from device memory: their 12 KB of threadgroup memory +// limit how many groups fit on a core, and with more groups the GPU hides +// more of the latency of the row-by-row hit and transformation reads. +#ifdef __METAL__ +#define GPUCA_TRACKLET_CONSTRUCTOR_ROW(row, iRow, s) const GPUglobalref() GPUTPCRow& GPUrestrict() row = tracker.Row(iRow) +#else +#define GPUCA_TRACKLET_CONSTRUCTOR_ROW(row, iRow, s) GPUCA_MAKE_SHARED_REF(GPUTPCRow, row, tracker.Row(iRow), s.mRows[iRow]) +#endif + GPUdii() void GPUTPCTrackletConstructor::InitTracklet(GPUTPCTrackParam& GPUrestrict() tParam) { // Initialize Tracklet Parameters using default values @@ -96,7 +105,7 @@ GPUd() void GPUTPCTrackletConstructor::StoreTracklet(int32_t /*nBlocks*/, int32_ calink ih = rowHits[iRow]; tracker.TrackletRowHits()[hitout + (iRow - r.mFirstRow)] = ih; if (ih != CALINK_INVAL && ih != CALINK_DEAD_CHANNEL) { - GPUCA_MAKE_SHARED_REF(GPUTPCRow, row, tracker.Row(iRow), s.mRows[iRow]); + GPUCA_TRACKLET_CONSTRUCTOR_ROW(row, iRow, s); tracker.MaximizeHitWeight(row, ih, w); } } @@ -106,7 +115,7 @@ template GPUdic(2, 1) void GPUTPCTrackletConstructor::UpdateTracklet(int32_t /*nBlocks*/, int32_t /*nThreads*/, int32_t /*iBlock*/, int32_t /*iThread*/, GPUsharedref() T& s, GPUTPCThreadMemory& GPUrestrict() r, GPUconstantref() GPUTPCTracker& GPUrestrict() tracker, GPUTPCTrackParam& GPUrestrict() tParam, int32_t iRow, calink& rowHit, calink* rowHits) { // reconstruction of tracklets, tracklets update step - GPUCA_MAKE_SHARED_REF(GPUTPCRow, row, tracker.Row(iRow), s.mRows[iRow]); + GPUCA_TRACKLET_CONSTRUCTOR_ROW(row, iRow, s); float y0 = row.Grid().YMin(); float stepY = row.HstepY(); @@ -482,7 +491,9 @@ GPUdii() void GPUTPCTrackletConstructor::Thread(int32_t nBlocks, int32_t nThread if (get_local_id(0) == 0) { sMem.mNStartHits = *tracker.NStartHits(); } +#ifndef __METAL__ GPUCA_SHARED_CACHE(&sMem.mRows[0], tracker.TrackingDataRows(), GPUTPCGeometry::NROWS * sizeof(GPUTPCRow)); +#endif GPUbarrier(); GPUTPCThreadMemory rMem; @@ -509,3 +520,5 @@ GPUd() int32_t GPUTPCTrackletConstructor::GPUTPCTrackletConstructorExtrapolation } return (rMem.mNHits); } + +#undef GPUCA_TRACKLET_CONSTRUCTOR_ROW diff --git a/GPU/GPUTracking/SectorTracker/GPUTPCTrackletConstructor.h b/GPU/GPUTracking/SectorTracker/GPUTPCTrackletConstructor.h index ecef065a1df40..6c4191fb9f4fb 100644 --- a/GPU/GPUTracking/SectorTracker/GPUTPCTrackletConstructor.h +++ b/GPU/GPUTracking/SectorTracker/GPUTPCTrackletConstructor.h @@ -62,8 +62,10 @@ class GPUTPCTrackletConstructor : public GPUKernelTemplate }; struct GPUSharedMemory { +#ifndef __METAL__ // see GPUCA_TRACKLET_CONSTRUCTOR_ROW GPUCA_SHARED_STORAGE(GPUTPCRow mRows[GPUTPCGeometry::NROWS]); // rows - uint32_t mNStartHits; // Total number of start hits +#endif + uint32_t mNStartHits; // Total number of start hits #ifdef GPUCA_TRACKLET_CONSTRUCTOR_DO_PROFILE int32_t fMaxSync; // temporary shared variable during profile creation