Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions GPU/GPUTracking/Base/metal/GPUReconstructionMetal.h
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,12 @@ class GPUReconstructionMetal : public GPUReconstructionProcessing::KernelInterfa
void CommitStream(int32_t stream);
void DrainStream(int32_t stream);
uint64_t EventValue(deviceEvent ev, bool next);
template <class T>
T* HostPointer(T* p);
template <class T, class N, class S>
void SortOnHost(const krnlSetupTime& _xyz, T* begin, const N& count, const S& comp);
template <class T, int32_t I, typename... Args>
void RunOnHost(const krnlSetupTime& _xyz, typename T::processorType* procs, uint32_t nProcs, uint32_t nBlocks, const Args&... args);

private:
int32_t AddKernels();
Expand Down
7 changes: 5 additions & 2 deletions GPU/GPUTracking/Base/metal/GPUReconstructionMetalKernels.mm
Original file line number Diff line number Diff line change
Expand Up @@ -102,10 +102,13 @@
const uint32_t tunedThreads = this->template getKernelProperties<T, I>().nThreads;

// Like __launch_bounds__: without it the register allocation may leave room
// for fewer threads than the kernel is tuned for.
// for fewer threads than the kernel is tuned for. The tracklet constructor
// runs in groups of 128 threads but gets the registers of a group of 640, as
// the minimum blocks of __launch_bounds__ would do, so that more groups fit
// on a core to hide the latency of its row-by-row reads.
MTLComputePipelineDescriptor* desc = [MTLComputePipelineDescriptor new];
desc.computeFunction = krnl;
desc.maxTotalThreadsPerThreadgroup = tunedThreads;
desc.maxTotalThreadsPerThreadgroup = std::is_same_v<T, GPUTPCTrackletConstructor> ? 640 : tunedThreads;
NSError* error = nil;
id<MTLComputePipelineState> pso = [mInternals->device newComputePipelineStateWithDescriptor:desc options:MTLPipelineOptionNone reflection:nil error:&error];
[desc release];
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,10 @@

/// \file GPUReconstructionMetalKernelsSpecialize.inc

#include "GPUTPCGMMergerSortComp.h"
#include <tbb/parallel_for.h>
#include <chrono>

template <>
inline void GPUReconstructionMetal::runKernelBackend<GPUMemClean16, 0>(const krnlSetupTime& _xyz, void* const& ptr, uint64_t const& size)
{
Expand All @@ -30,3 +34,158 @@ inline void GPUReconstructionMetal::runKernelBackend<GPUMemClean16, 0>(const krn
RecordMarker(_xyz.z.ev, _xyz.x.stream);
}
}

// The sorts that CUDA and HIP run with thrust (GPUCA_SPECIALIZE_THRUST_SORTS)
// run on a single GPU thread otherwise. The memory is shared with the host, so
// sort there, with the same std::sort as the CPU backend, which also orders
// the elements that compare equal as on the CPU. The number of elements is
// read once the stream is drained, from the device copy of the counters,
// since the host copy may still be on its way.
template <class T>
inline T* GPUReconstructionMetal::HostPointer(T* p)
{
const size_t offset = (const char*)p - (const char*)mDeviceMemoryBase;
return offset < mDeviceMemorySize ? (T*)((char*)mInternals->mem_gpu.contents + offset) : p;
}

template <class T, class N, class S>
inline void GPUReconstructionMetal::SortOnHost(const krnlSetupTime& _xyz, T* begin, const N& count, const S& comp)
{
if (_xyz.z.evList) {
StreamWaitForEvents(_xyz.x.stream, _xyz.z.evList, _xyz.z.nEvents);
}
DrainStream(_xyz.x.stream);
T* hostBegin = HostPointer(begin);
GPUCommonAlgorithm::sortDeviceDynamic(hostBegin, hostBegin + count(), comp);
if (_xyz.z.ev) {
RecordMarker(_xyz.z.ev, _xyz.x.stream);
}
}

// Kernels that are a long serial chain per GPU thread, with little parallelism,
// run on the host instead, on the shared memory, as the CPU backend runs them.
// The processors in mProcessorsShadow hold GPU addresses: the kernel runs on a
// private copy of mProcessorsShadow, whose processors point at the host view of
// the same memory, with the host's parameters and calibration objects, and
// which is their constant memory. SetDevicePointers relocates the pointers in
// place, so they are copied and the shadow is restored before the kernel runs:
// the kernel, on many host threads, never touches the shadow.
struct GPUMetalHostProcessor : public GPUProcessor {
static const GPUConstantMem*& ConstantMem(GPUProcessor& p) { return p.*&GPUMetalHostProcessor::mConstantMem; }
};

template <class T, int32_t I, typename... Args>
inline void GPUReconstructionMetal::RunOnHost(const krnlSetupTime& _xyz, typename T::processorType* procs, uint32_t nProcs, uint32_t nBlocks, const Args&... args)
{
if (_xyz.z.evList) {
StreamWaitForEvents(_xyz.x.stream, _xyz.z.evList, _xyz.z.nEvents);
}
DrainStream(_xyz.x.stream);
const auto start = std::chrono::steady_clock::now();
const auto shadow = std::make_unique<char[]>(sizeof(GPUConstantMem));
static_assert(alignof(GPUConstantMem) <= __STDCPP_DEFAULT_NEW_ALIGNMENT__);
const auto hostMem = std::make_unique<char[]>(sizeof(GPUConstantMem));
GPUConstantMem* host = (GPUConstantMem*)hostMem.get();
memcpy(shadow.get(), (void*)mProcessorsShadow, sizeof(GPUConstantMem));
for (const auto& res : mMemoryResources) {
if (res.PtrDevice()) {
res.SetDevicePointers(HostPointer(res.PtrDevice()));
}
}
memcpy((void*)host, (void*)mProcessorsShadow, sizeof(GPUConstantMem));
for (const auto& res : mMemoryResources) {
if (res.PtrDevice()) {
res.SetDevicePointers(res.PtrDevice());
}
}
memcpy((void*)mProcessorsShadow, shadow.get(), sizeof(GPUConstantMem));
host->param = param();
host->calibObjects = processors()->calibObjects;
host->errorCodes.setMemory(HostPointer(const_cast<uint32_t*>(host->errorCodes.getErrorPtr()) - 1));
auto* hostProcs = (typename T::processorType*)((char*)host + ((char*)procs - (char*)mProcessorsShadow));
for (uint32_t i = 0; i < nProcs; i++) {
GPUMetalHostProcessor::ConstantMem(hostProcs[i]) = host;
}
tbb::parallel_for<uint32_t>(0, nBlocks, [&](uint32_t iBlock) {
typename T::GPUSharedMemory smem;
T::template Thread<I>(nBlocks, 1, iBlock, 0, smem, hostProcs[_xyz.y.index], args...);
});
if (GetProcessingSettings().debugLevel >= 1 && memcmp(shadow.get(), (void*)mProcessorsShadow, sizeof(GPUConstantMem))) {
GPUFatal("Metal: mProcessorsShadow changed while %s ran on the host", GetKernelName<T, I>());
}
_xyz.t = std::chrono::duration<double>(std::chrono::steady_clock::now() - start).count();
if (_xyz.z.ev) {
RecordMarker(_xyz.z.ev, _xyz.x.stream);
}
}

template <>
inline void GPUReconstructionMetal::runKernelBackend<GPUTPCGMMergerSortTracks, 0>(const krnlSetupTime& _xyz)
{
// The fit runs one track per thread, in this order. Tracks of similar length
// keep the threads of a SIMD group busy for as long as each other; within a
// length bucket of 16 clusters, tracks of the same sector and side then read
// the same pages of the cluster, transformation and calibration data, which
// the Metal GPU, bound by its address translation in the fit, needs more than
// the exact length order.
const GPUTPCGMMergedTrack* trk = HostPointer(mProcessorsShadow->tpcMerger.MergedTracks());
auto comp = [trk](const int32_t aa, const int32_t bb) {
const GPUTPCGMMergedTrack& a = trk[aa];
const GPUTPCGMMergedTrack& b = trk[bb];
if (a.OK() != b.OK()) {
return a.OK();
}
if (a.CCE() != b.CCE()) {
return a.CCE() > b.CCE();
}
if (a.NClusters() / 16 != b.NClusters() / 16) {
return a.NClusters() / 16 > b.NClusters() / 16;
}
const bool sideA = a.GetParam().GetZ() < 0, sideB = b.GetParam().GetZ() < 0;
if (sideA != sideB || a.GetAlpha() != b.GetAlpha()) {
return sideA != sideB ? sideA < sideB : a.GetAlpha() < b.GetAlpha();
}
return aa < bb;
};
SortOnHost(_xyz, mProcessorsShadow->tpcMerger.TrackOrderProcess(), [&] { return HostPointer(mProcessorsShadow->tpcMerger.Memory())->nMergedTracks; }, comp);
}

template <>
inline void GPUReconstructionMetal::runKernelBackend<GPUTPCGMMergerSortTracksQPt, 0>(const krnlSetupTime& _xyz)
{
SortOnHost(_xyz, mProcessorsShadow->tpcMerger.TrackSort(), [&] { return HostPointer(mProcessorsShadow->tpcMerger.Memory())->nMergedTracks; }, internal::GPUTPCGMMergerSortTracksQPt_comp(HostPointer(mProcessorsShadow->tpcMerger.MergedTracks())));
}

template <>
inline void GPUReconstructionMetal::runKernelBackend<GPUTPCGMMergerMergeLoopers, 1>(const krnlSetupTime& _xyz)
{
SortOnHost(_xyz, mProcessorsShadow->tpcMerger.LooperCandidates(), [&] { return HostPointer(mProcessorsShadow->tpcMerger.Memory())->nLooperMatchCandidates; }, internal::GPUTPCGMMergerMergeLoopers_comp());
}

template <>
inline void GPUReconstructionMetal::runKernelBackend<GPUTPCGMO2Output, GPUTPCGMO2Output::sort>(const krnlSetupTime& _xyz)
{
SortOnHost(_xyz, mProcessorsShadow->tpcMerger.TrackSortO2(), [&] { return HostPointer(mProcessorsShadow->tpcMerger.Memory())->nO2Tracks; }, internal::GPUTPCGMO2OutputSort_comp());
}

template <>
inline void GPUReconstructionMetal::runKernelBackend<GPUTPCGMMergerMergeBorders, 3>(const krnlSetupTime& _xyz, gputpcgmmergertypes::GPUTPCGMBorderRange* const& range, int32_t const& N, int32_t const& cmpMax)
{
if (cmpMax) {
SortOnHost(_xyz, range, [&] { return N; }, internal::MergeBorderTracks_compMax());
} else {
SortOnHost(_xyz, range, [&] { return N; }, internal::MergeBorderTracks_compMin());
}
}

template <>
inline void GPUReconstructionMetal::runKernelBackend<GPUTPCGMMergerMergeBorders, 2>(const krnlSetupTime& _xyz, int32_t const& iSector, uint8_t const& mergeMode)
{
RunOnHost<GPUTPCGMMergerMergeBorders, 2>(_xyz, &mProcessorsShadow->tpcMerger, 1, 1, iSector, mergeMode);
}

template <>
inline void GPUReconstructionMetal::runKernelBackend<GPUTPCExtrapolationTracking, 0>(const krnlSetupTime& _xyz)
{
RunOnHost<GPUTPCExtrapolationTracking, 0>(_xyz, mProcessorsShadow->tpcTrackers, GPUTPCGeometry::NSECTORS, getNKernelHostThreads(false));
}
1 change: 1 addition & 0 deletions GPU/GPUTracking/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,7 @@ set(HDRS_INSTALL
Global/GPUChainTrackingGetters.inc
Global/GPUErrorCodes.h
Merger/GPUTPCGMMergerTypes.h
Merger/GPUTPCGMMergerSortComp.h
qa/GPUQAHelper.h
qconfigoptions.h
Refit/GPUTrackParamConvert.h
Expand Down
1 change: 1 addition & 0 deletions GPU/GPUTracking/Definitions/Parameters/GPUParameters.csv
Original file line number Diff line number Diff line change
Expand Up @@ -114,4 +114,5 @@ MERGER_INTERPOLATION_ERROR_TYPE,"""half""","""float""",,,,,,,,,,,,,,,,,
COMP_GATHER_KERNEL,4,0,4,4,,,,,,4,4,4,4,,4,,4,4,
COMP_GATHER_MODE,3,0,3,3,,,,,,3,3,3,3,,3,,3,3,
CF_SCAN_WORKGROUP_SIZE,512,0,,,,,,,,224,,992,448,,1024,,,448,
CF_CHECK_PAD_BASELINE_SEGMENTED,0,0,,,,,,,,,,,,,,,,,1
MERGER_SPLIT_LOOP_INTERPOLATION,,,,,,,,,,,,,,,1,,,1,
19 changes: 16 additions & 3 deletions GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx
Original file line number Diff line number Diff line change
Expand Up @@ -121,10 +121,23 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput)
if (GetProcessingSettings().deterministicGPUReconstruction) {
runKernel<GPUTPCGlobalDebugSortKernels, GPUTPCGlobalDebugSortKernels::clearIds>(GetGridAuto(0, deviceType), 1);
}
// Sectors only share the atomic track counter, so they can be refit concurrently, but then the sector track infos are not ordered by sector.
// Only the deterministic sort and the debug dumps need that order (SectorTrackInfoFirst / Last), so keep the serial order for them.
const uint32_t nRefitStreams = doGPU && !GetProcessingSettings().deterministicGPUReconstruction ? mRec->NStreams() : 1;
RecordMarker(&mEvents->single, 0);
for (uint32_t i = 0; i < NSECTORS; i++) {
runKernel<GPUTPCGMMergerUnpackSaveNumber>({{1, -WarpSize(), 0, deviceType}}, i);
runKernel<GPUTPCGMMergerUnpackResetIds>(GetGridAuto(0, deviceType), i);
runKernel<GPUTPCGMMergerSectorRefit>(GetGridAuto(0, deviceType), i); // TODO: Why all in stream 0?
const int32_t stream = i % nRefitStreams;
runKernel<GPUTPCGMMergerUnpackSaveNumber>({{1, -WarpSize(), stream, deviceType}, krnlRunRangeNone, {nullptr, stream && i < nRefitStreams ? &mEvents->single : nullptr}}, i);
runKernel<GPUTPCGMMergerUnpackResetIds>(GetGridAuto(stream, deviceType), i);
runKernel<GPUTPCGMMergerSectorRefit>(GetGridAuto(stream, deviceType), i);
}
ReleaseEvent(mEvents->single);
if (nRefitStreams > 1) { // Stream 0 must wait for all streams before unpacking the extrapolated tracks
const int32_t ne = std::min<int32_t>(NSECTORS, nRefitStreams) - 1;
for (int32_t j = 0; j < ne; j++) {
RecordMarker(&mEvents->sector[j], j + 1);
}
StreamWaitForEvents(0, &mEvents->sector[0], ne);
}
if (GetProcessingSettings().deterministicGPUReconstruction) {
runKernel<GPUTPCGMMergerUnpackSaveNumber>({{1, -WarpSize(), 0, deviceType}}, NSECTORS);
Expand Down
79 changes: 1 addition & 78 deletions GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx
Original file line number Diff line number Diff line change
Expand Up @@ -63,84 +63,7 @@ using namespace o2::gpu;
using namespace o2::tpc;
using namespace gputpcgmmergertypes;

namespace o2::gpu::internal
{
struct MergeLooperParam {
float refz;
float x;
float y;
uint32_t id;
};

struct MergeBorderTracks_compMax {
GPUd() bool operator()(const GPUTPCGMBorderRange& a, const GPUTPCGMBorderRange& b)
{
return GPUCA_DETERMINISTIC_CODE((a.fMax != b.fMax) ? (a.fMax < b.fMax) : (a.fId < b.fId), a.fMax < b.fMax);
}
};
struct MergeBorderTracks_compMin {
GPUd() bool operator()(const GPUTPCGMBorderRange& a, const GPUTPCGMBorderRange& b)
{
return GPUCA_DETERMINISTIC_CODE((a.fMin != b.fMin) ? (a.fMin < b.fMin) : (a.fId < b.fId), a.fMin < b.fMin);
}
};

struct GPUTPCGMMergerSortTracks_comp {
const GPUTPCGMMergedTrack* const mCmp;
GPUhd() GPUTPCGMMergerSortTracks_comp(GPUTPCGMMergedTrack* cmp) : mCmp(cmp) {}
GPUd() bool operator()(const int32_t aa, const int32_t bb)
{
const GPUTPCGMMergedTrack& GPUrestrict() a = mCmp[aa];
const GPUTPCGMMergedTrack& GPUrestrict() b = mCmp[bb];
if (a.OK() != b.OK()) {
return a.OK();
}
if (a.CCE() != b.CCE()) {
return a.CCE() > b.CCE();
}
GPUCA_DETERMINISTIC_CODE( // clang-format off
if (a.NClusters() != b.NClusters()) {
return a.NClusters() > b.NClusters();
} if (CAMath::Abs(a.GetParam().GetQPt()) != CAMath::Abs(b.GetParam().GetQPt())) {
return CAMath::Abs(a.GetParam().GetQPt()) > CAMath::Abs(b.GetParam().GetQPt());
} if (a.GetParam().GetY() != b.GetParam().GetY()) {
return a.GetParam().GetY() > b.GetParam().GetY();
}
return aa > bb;
, // !GPUCA_DETERMINISTIC_CODE
return a.NClusters() > b.NClusters();
) // clang-format on
}
};

struct GPUTPCGMMergerSortTracksQPt_comp {
const GPUTPCGMMergedTrack* const mCmp;
GPUhd() GPUTPCGMMergerSortTracksQPt_comp(GPUTPCGMMergedTrack* cmp) : mCmp(cmp) {}
GPUd() bool operator()(const int32_t aa, const int32_t bb)
{
const GPUTPCGMMergedTrack& GPUrestrict() a = mCmp[aa];
const GPUTPCGMMergedTrack& GPUrestrict() b = mCmp[bb];
GPUCA_DETERMINISTIC_CODE( // clang-format off
if (CAMath::Abs(a.GetParam().GetQPt()) != CAMath::Abs(b.GetParam().GetQPt())) {
return CAMath::Abs(a.GetParam().GetQPt()) > CAMath::Abs(b.GetParam().GetQPt());
} if (a.GetParam().GetY() != b.GetParam().GetY()) {
return a.GetParam().GetY() > b.GetParam().GetY();
}
return a.GetParam().GetZ() > b.GetParam().GetZ();
, // !GPUCA_DETERMINISTIC_CODE
return CAMath::Abs(a.GetParam().GetQPt()) > CAMath::Abs(b.GetParam().GetQPt());
) // clang-format on
}
};

struct GPUTPCGMMergerMergeLoopers_comp {
GPUd() bool operator()(const MergeLooperParam& a, const MergeLooperParam& b)
{
return GPUCA_DETERMINISTIC_CODE(CAMath::Abs(a.refz) != CAMath::Abs(b.refz) ? CAMath::Abs(a.refz) < CAMath::Abs(b.refz) : a.id < b.id, CAMath::Abs(a.refz) < CAMath::Abs(b.refz));
}
};

} // namespace o2::gpu::internal
#include "GPUTPCGMMergerSortComp.h"

using namespace o2::gpu::internal;

Expand Down
Loading
Loading