From fa56a5800581b918e14b92b481add1596ab38243 Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Mon, 9 Nov 2026 20:29:30 +0100 Subject: [PATCH 1/5] GPU Metal: make command-buffer failures fatal A failed Metal command buffer (e.g. killed by the macOS GPU watchdog, kIOGPUCommandBufferCallbackErrorImpactingInteractivity / ...Timeout) was only logged when the stream was drained, and the reconstruction carried on without its work: on 20 kHz Pb-Pb time frames this gave silently wrong outputs (271,967-296,957 tracks vs 297,669 on CPU for the same TF). Worse, a failed buffer never signals the events it was to signal, so later buffers waiting for them on the GPU, and the host waiting for those, blocked forever: one such run hung at 0% CPU. Signal the stream-order event and the recorded event of a buffer from its completion handler when it fails, so everything queued behind it completes, and fail in DrainStream through GPUChkErr, like the CUDA and HIP backends do on a failed synchronisation ("GPU Backend Failure"). Failures seen while the device is torn down are only logged, so the destructor does not throw. pbpb30 (M1 Max, abp.sh, 3 rounds, --runs 10): 966.7 ms/ev before, 962.5 after; Metal dumps byte-identical before and after; sim5pp dump still identical to CPU. --- .../Base/metal/GPUReconstructionMetal.mm | 14 ++++++++++++-- .../metal/GPUReconstructionMetalIncludesHost.h | 13 +++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/GPU/GPUTracking/Base/metal/GPUReconstructionMetal.mm b/GPU/GPUTracking/Base/metal/GPUReconstructionMetal.mm index 1a23b201da209..7ead126108632 100644 --- a/GPU/GPUTracking/Base/metal/GPUReconstructionMetal.mm +++ b/GPU/GPUTracking/Base/metal/GPUReconstructionMetal.mm @@ -212,7 +212,8 @@ uint32_t metalGPUCoreCount() int32_t GPUReconstructionMetal::ExitDevice_Runtime() { - // Tear down Metal + // Tear down Metal, also after a failure, without failing again + mInternals->exiting = true; SynchronizeGPU(); for (uint32_t i = 0; i < mEvents.size(); i++) { @@ -380,7 +381,9 @@ MetalMemRef metalResolve(const GPUReconstructionMetalInternals* internals, const // Does not change the retain count, so it's important we manage // the lifetime of the events outside here. auto mtlEvent = (__bridge id)(ev->get()); - [cb encodeSignalEvent:mtlEvent value:EventValue(*ev, true)]; + const uint64_t value = EventValue(*ev, true); + [cb encodeSignalEvent:mtlEvent value:value]; + metalSignalOnError(cb, mtlEvent, value); CommitStream(stream); } @@ -392,6 +395,7 @@ MetalMemRef metalResolve(const GPUReconstructionMetalInternals* internals, const // in constant memory. So each buffer of a stream waits for the previous one. const uint64_t value = ++mInternals->streamOrderValue[stream]; [cb encodeSignalEvent:mInternals->streamOrder[stream] value:value]; + metalSignalOnError(cb, mInternals->streamOrder[stream], value); [cb commit]; // Hand the committed buffer over to the stream's submission list, which owns // it until it is waited for, and open a fresh one to encode the next work in. @@ -420,16 +424,22 @@ MetalMemRef metalResolve(const GPUReconstructionMetalInternals* internals, const const auto t0 = std::chrono::steady_clock::now(); while (!mInternals->submittedBuffers[stream].empty() && !(done = mInternals->streamOrder[stream].signaledValue >= mInternals->streamOrderValue[stream]) && std::chrono::steady_clock::now() - t0 < std::chrono::milliseconds(20)) { } + int64_t error = 0; for (id cb : mInternals->submittedBuffers[stream]) { if (!done) { [cb waitUntilCompleted]; } if (cb.status == MTLCommandBufferStatusError) { GPUError("Metal command buffer on stream %d failed: %s", stream, [[cb.error description] UTF8String]); + error = error ? error : cb.error.code; } [cb release]; } mInternals->submittedBuffers[stream].clear(); + // The work of a failed buffer is lost, so the results are wrong: fail like the other backends. + if (error && !mInternals->exiting) { + GPUChkErr(error); + } } void GPUReconstructionMetal::SynchronizeGPU() diff --git a/GPU/GPUTracking/Base/metal/GPUReconstructionMetalIncludesHost.h b/GPU/GPUTracking/Base/metal/GPUReconstructionMetalIncludesHost.h index 76303183ac04c..c39fa4e0d9c4b 100644 --- a/GPU/GPUTracking/Base/metal/GPUReconstructionMetalIncludesHost.h +++ b/GPU/GPUTracking/Base/metal/GPUReconstructionMetalIncludesHost.h @@ -66,6 +66,8 @@ struct GPUReconstructionMetalInternals { // collects the stream's work until something has to wait for it. std::array, constants::GPU_MAX_STREAMS> streamFence; std::array streamHasWork; + // Set while the device is torn down, when a failed command buffer is no longer fatal + bool exiting = false; std::vector> functions; std::vector> pipelines; @@ -90,6 +92,17 @@ inline void metalBeginEncoder(GPUReconstructionMetalInternals* internals, E enco { [encoder waitForFence:internals->streamFence[stream]]; } +// A failed command buffer does not signal its events, and the work waiting for +// them would wait forever: signal them from the host, so that everything +// completes and the failure is reported when the host synchronises. +inline void metalSignalOnError(id cb, id event, uint64_t value) +{ + [cb addCompletedHandler:^(id c) { + if (c.status == MTLCommandBufferStatusError && event.signaledValue < value) { + event.signaledValue = value; + } + }]; +} template inline void metalEndEncoder(GPUReconstructionMetalInternals* internals, E encoder, int32_t stream) { From 6826099b15e026402b81e4e5c73077254e6c2676 Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Mon, 9 Nov 2026 21:23:28 +0100 Subject: [PATCH 2/5] GPU Metal: optional command-buffer timeline With GPUCA_METAL_TIMELINE=, write one line per command buffer once it completes: stream, status, CPU commit time, GPUStartTime and GPUEndTime, the kernels (with their threadgroup counts), blits, and the events it waits for and signals; and one line per host drain that had to wait. Off by default, the only cost then is a pointer test per encoded command. This is what is needed to see how the streams overlap on the GPU and where they wait for each other, without a profiler attached. --- .../Base/metal/GPUReconstructionMetal.mm | 29 +++++++++++++++++++ .../GPUReconstructionMetalIncludesHost.h | 22 ++++++++++++++ .../metal/GPUReconstructionMetalKernels.mm | 4 +++ 3 files changed, 55 insertions(+) diff --git a/GPU/GPUTracking/Base/metal/GPUReconstructionMetal.mm b/GPU/GPUTracking/Base/metal/GPUReconstructionMetal.mm index 7ead126108632..8c31f7376b6e0 100644 --- a/GPU/GPUTracking/Base/metal/GPUReconstructionMetal.mm +++ b/GPU/GPUTracking/Base/metal/GPUReconstructionMetal.mm @@ -167,6 +167,10 @@ uint32_t metalGPUCoreCount() mInternals->streamOrderValue[i] = 0; mInternals->streamFence[i] = [mInternals->device newFence]; mInternals->streamHasWork[i] = false; + mInternals->timelineKernels[i] = 0; + } + if (const char* timeline = getenv("GPUCA_METAL_TIMELINE")) { + mInternals->timeline = fopen(timeline, "w"); } mInternals->mem_host = metalNewSharedBuffer(mInternals->device, mHostMemorySize); @@ -247,6 +251,9 @@ uint32_t metalGPUCoreCount() } [mInternals->mem_host release]; } + if (mInternals->timeline) { + fflush(mInternals->timeline); // not closed: a completion handler may still write to it + } [mInternals->library release]; [mInternals->device release]; @@ -313,6 +320,7 @@ MetalMemRef metalResolve(const GPUReconstructionMetalInternals* internals, const id cb = mInternals->commandBuffers[realStream]; id blit = [cb blitCommandEncoder]; metalBeginEncoder(mInternals, blit, realStream); + metalTimelineAdd(mInternals, realStream, "b", nullptr, sizeBytes); [blit copyFromBuffer:s.buffer sourceOffset:s.offset toBuffer:d.buffer @@ -346,6 +354,7 @@ MetalMemRef metalResolve(const GPUReconstructionMetalInternals* internals, const id cb = mInternals->commandBuffers[realStream]; id blit = [cb blitCommandEncoder]; metalBeginEncoder(mInternals, blit, realStream); + metalTimelineAdd(mInternals, realStream, "b", nullptr, size); [blit copyFromBuffer:s.buffer sourceOffset:s.offset toBuffer:mInternals->mem_constant @@ -384,6 +393,7 @@ MetalMemRef metalResolve(const GPUReconstructionMetalInternals* internals, const const uint64_t value = EventValue(*ev, true); [cb encodeSignalEvent:mtlEvent value:value]; metalSignalOnError(cb, mtlEvent, value); + metalTimelineAdd(mInternals, stream, "s", mtlEvent, value); CommitStream(stream); } @@ -396,6 +406,19 @@ MetalMemRef metalResolve(const GPUReconstructionMetalInternals* internals, const const uint64_t value = ++mInternals->streamOrderValue[stream]; [cb encodeSignalEvent:mInternals->streamOrder[stream] value:value]; metalSignalOnError(cb, mInternals->streamOrder[stream], value); + if (mInternals->timeline) { + metalTimelineAdd(mInternals, stream, "s", mInternals->streamOrder[stream], value); + FILE* f = mInternals->timeline; + const std::string ops = std::move(mInternals->timelineOps[stream]); + const uint32_t nKernels = mInternals->timelineKernels[stream]; + const uint64_t seq = mInternals->timelineBuffers++; + const double commitTime = metalHostTime(); + [cb addCompletedHandler:^(id c) { + fprintf(f, "CB %llu s%d st%d commit %.6f gpu %.6f %.6f nk %u:%s\n", (unsigned long long)seq, stream, (int)c.status, commitTime, c.GPUStartTime, c.GPUEndTime, nKernels, ops.c_str()); + }]; + mInternals->timelineOps[stream].clear(); + mInternals->timelineKernels[stream] = 0; + } [cb commit]; // Hand the committed buffer over to the stream's submission list, which owns // it until it is waited for, and open a fresh one to encode the next work in. @@ -403,6 +426,7 @@ MetalMemRef metalResolve(const GPUReconstructionMetalInternals* internals, const mInternals->streamHasWork[stream] = false; mInternals->commandBuffers[stream] = [[mInternals->commandQueues[stream] commandBuffer] retain]; [mInternals->commandBuffers[stream] encodeWaitForEvent:mInternals->streamOrder[stream] value:value]; + metalTimelineAdd(mInternals, stream, "w", mInternals->streamOrder[stream], value); } void GPUReconstructionMetal::DrainStream(int32_t stream) @@ -421,6 +445,7 @@ MetalMemRef metalResolve(const GPUReconstructionMetalInternals* internals, const // waitUntilCompleted, which sleeps until the completion has been processed. // Only short waits are worth a busy core, so give up after a while. bool done = false; + const double drainStart = mInternals->timeline && !mInternals->submittedBuffers[stream].empty() ? metalHostTime() : 0.; const auto t0 = std::chrono::steady_clock::now(); while (!mInternals->submittedBuffers[stream].empty() && !(done = mInternals->streamOrder[stream].signaledValue >= mInternals->streamOrderValue[stream]) && std::chrono::steady_clock::now() - t0 < std::chrono::milliseconds(20)) { } @@ -436,6 +461,9 @@ MetalMemRef metalResolve(const GPUReconstructionMetalInternals* internals, const [cb release]; } mInternals->submittedBuffers[stream].clear(); + if (drainStart) { + fprintf(mInternals->timeline, "DR s%d %.6f %.6f\n", stream, drainStart, metalHostTime()); + } // The work of a failed buffer is lost, so the results are wrong: fail like the other backends. if (error && !mInternals->exiting) { GPUChkErr(error); @@ -469,6 +497,7 @@ MetalMemRef metalResolve(const GPUReconstructionMetalInternals* internals, const auto mtlEvent = (__bridge id)(evList[ei].get()); if (const uint64_t value = EventValue(evList[ei], false)) { [cb encodeWaitForEvent:mtlEvent value:value]; + metalTimelineAdd(mInternals, stream, "w", mtlEvent, value); } } CommitStream(stream); diff --git a/GPU/GPUTracking/Base/metal/GPUReconstructionMetalIncludesHost.h b/GPU/GPUTracking/Base/metal/GPUReconstructionMetalIncludesHost.h index c39fa4e0d9c4b..a4b30032e972a 100644 --- a/GPU/GPUTracking/Base/metal/GPUReconstructionMetalIncludesHost.h +++ b/GPU/GPUTracking/Base/metal/GPUReconstructionMetalIncludesHost.h @@ -34,6 +34,8 @@ using half = _Float16; #include #include #include +#include +#include #include "GPULogging.h" #include "GPUReconstructionMetal.h" @@ -68,6 +70,12 @@ struct GPUReconstructionMetalInternals { std::array streamHasWork; // Set while the device is torn down, when a failed command buffer is no longer fatal bool exiting = false; + // Opt-in command-buffer timeline, GPUCA_METAL_TIMELINE=: what each open + // buffer contains (kernels, blits, waits, signals), written out once it completes + FILE* timeline = nullptr; + std::array timelineOps; + std::array timelineKernels; + uint64_t timelineBuffers = 0; std::vector> functions; std::vector> pipelines; @@ -103,6 +111,20 @@ inline void metalSignalOnError(id cb, id event } }]; } +// In the time base of the command buffers' GPUStartTime and GPUEndTime +inline double metalHostTime() +{ + static const double scale = [] { mach_timebase_info_data_t tb; mach_timebase_info(&tb); return 1e-9 * tb.numer / tb.denom; }(); + return mach_absolute_time() * scale; +} +inline void metalTimelineAdd(GPUReconstructionMetalInternals* internals, int32_t stream, const char* op, const void* id, uint64_t value) +{ + if (internals->timeline) { + char buf[96]; + snprintf(buf, sizeof(buf), " %s:%p=%llu", op, id, (unsigned long long)value); + internals->timelineOps[stream] += buf; + } +} template inline void metalEndEncoder(GPUReconstructionMetalInternals* internals, E encoder, int32_t stream) { diff --git a/GPU/GPUTracking/Base/metal/GPUReconstructionMetalKernels.mm b/GPU/GPUTracking/Base/metal/GPUReconstructionMetalKernels.mm index 7d536ef9f01ff..b7becb8456333 100644 --- a/GPU/GPUTracking/Base/metal/GPUReconstructionMetalKernels.mm +++ b/GPU/GPUTracking/Base/metal/GPUReconstructionMetalKernels.mm @@ -50,6 +50,10 @@ if (threadGroupSize > mInternals->maxThreadsPerGroup[num]) { threadGroupSize = mInternals->maxThreadsPerGroup[num]; } + if (mInternals->timeline) { + mInternals->timelineOps[kExec.stream] += std::string(" k:") + GetKernelName() + "/" + std::to_string(kExec.nBlocks); + mInternals->timelineKernels[kExec.stream]++; + } [computeEncoder dispatchThreadgroups:MTLSizeMake(kExec.nBlocks, 1, 1) threadsPerThreadgroup:MTLSizeMake(threadGroupSize, 1, 1)]; metalEndEncoder(mInternals, computeEncoder, kExec.stream); From 57490e0319fbdc70a70e6068d484264a086d0570 Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Mon, 9 Nov 2026 21:23:28 +0100 Subject: [PATCH 3/5] GPU Metal: implement the device kernel timers With --debug 1 and deviceTimers (the default), the kernel timers expect the backend to report the kernel's GPU time, and Metal reported none, so all kernel times read 0. Give the timed kernel a command buffer of its own and report its GPUEndTime - GPUStartTime. The times are only set once the completion is processed, so wait for it rather than for the stream's event. pbpb30 on M1 Max: 1012 ms of kernel time per event, vs 1218 ms on the CPU. --- .../Base/metal/GPUReconstructionMetalKernels.mm | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/GPU/GPUTracking/Base/metal/GPUReconstructionMetalKernels.mm b/GPU/GPUTracking/Base/metal/GPUReconstructionMetalKernels.mm index b7becb8456333..9eace2cb4f591 100644 --- a/GPU/GPUTracking/Base/metal/GPUReconstructionMetalKernels.mm +++ b/GPU/GPUTracking/Base/metal/GPUReconstructionMetalKernels.mm @@ -29,6 +29,11 @@ if (kEvents.evList) { StreamWaitForEvents(kExec.stream, kEvents.evList, kEvents.nEvents); } + // Device timers: the kernel gets a command buffer of its own, whose GPU time is the kernel's + const bool timed = GetProcessingSettings().deviceTimers && GetProcessingSettings().debugLevel > 0; + if (timed && mInternals->streamHasWork[kExec.stream]) { + CommitStream(kExec.stream); + } id pso = mInternals->pipelines[num]; id computeEncoder = [mInternals->commandBuffers[kExec.stream] computeCommandEncoder]; @@ -57,9 +62,19 @@ [computeEncoder dispatchThreadgroups:MTLSizeMake(kExec.nBlocks, 1, 1) threadsPerThreadgroup:MTLSizeMake(threadGroupSize, 1, 1)]; metalEndEncoder(mInternals, computeEncoder, kExec.stream); + id cb = [mInternals->commandBuffers[kExec.stream] retain]; if (kEvents.ev) { RecordMarker(kEvents.ev, kExec.stream); } + if (timed) { + if (mInternals->commandBuffers[kExec.stream] == cb) { + CommitStream(kExec.stream); + } + DrainStream(kExec.stream); + [cb waitUntilCompleted]; // the times are only set once the completion is processed + _xyz.t = cb.GPUEndTime - cb.GPUStartTime; + } + [cb release]; } template From 2b562baf8c827dfbe5141632ac8b684bae00fcbc Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Mon, 9 Nov 2026 21:23:38 +0100 Subject: [PATCH 4/5] GPU Metal: wait for other streams in a command buffer of its own StreamWaitForEvents encoded the waits into the stream's open command buffer, after whatever work had accumulated there. With the merger's SectorRefit on all streams, stream 0's buffer (its own 4 sectors of refit) then waited for the 7 other streams: on pbpb30 its GPU work ended 36 ms before the wait was satisfied, and the buffer stayed incomplete meanwhile. Commit the pending work first, so that every GPU-side wait sits at the start of a buffer with no work before it. The host drains the stream right after the waits anyway. pbpb30, M1 Max, 3 rounds of 10 events: 958.8 -> 959.7 ms/ev; 6 rounds of the same change as a runtime switch: 971.2 -> 961.3 ms/ev. Metal dumps byte-identical on pbpb30, sim5pp identical to the CPU. --- GPU/GPUTracking/Base/metal/GPUReconstructionMetal.mm | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/GPU/GPUTracking/Base/metal/GPUReconstructionMetal.mm b/GPU/GPUTracking/Base/metal/GPUReconstructionMetal.mm index 8c31f7376b6e0..1f9b0cf3218ad 100644 --- a/GPU/GPUTracking/Base/metal/GPUReconstructionMetal.mm +++ b/GPU/GPUTracking/Base/metal/GPUReconstructionMetal.mm @@ -491,6 +491,11 @@ MetalMemRef metalResolve(const GPUReconstructionMetalInternals* internals, const void GPUReconstructionMetal::StreamWaitForEvents(int32_t stream, deviceEvent* evList, int32_t nEvents) { + // Encoded after the stream's pending work, the waits would keep its command buffer + // from completing until the other streams are done: wait in a buffer of its own. + if (mInternals->streamHasWork[stream]) { + CommitStream(stream); + } // Encode commands to wait for all the events id cb = mInternals->commandBuffers[stream]; for (int32_t ei = 0; ei < nEvents; ei++) { From eb715df06bf212170150f83551ac72cbec3f0b60 Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Tue, 10 Nov 2026 11:36:30 +0100 Subject: [PATCH 5/5] GPU Metal: optionally limit the GPU work in flight At 20 kHz occupancy the macOS GPU watchdog killed command buffers, and reset the desktop once, while all 8 streams ran multi-kernel buffers of up to ~1 s at the same time. Host-serialised execution (serializeGPU 3) was the only mode that survived. Add two runtime limits, off by default: - GPUCA_METAL_MAX_KERNELS_PER_BUFFER=N commits a stream's command buffer after N kernels, so that long chains (sector tracking, SectorRefit, TrackFit) become short buffers. Buffers of a stream stay ordered by its order event, the events of the other streams are untouched. - GPUCA_METAL_MAX_STREAMS_IN_FLIGHT=M makes the host wait, before it commits a buffer on an idle stream, until fewer than M streams have committed buffers that are not complete. This cannot deadlock: a buffer only waits for events signalled by buffers committed before it, since RecordMarker commits the signal at once, so the work in flight completes without the host. pbpb30, M1 Max, 6 rounds of 10 events (timeline: longest buffer, streams running at once): default unchanged (931 ms/ev, -3 ms over two run orders); N=1 M=1 1416 ms/ev, 87 ms, 1 stream (serializeGPU 3: 1413 ms/ev); N=1 M=2 1100 ms/ev, 87 ms, 2; N=2 M=4 941 ms/ev, 123 ms, 4. The longest buffers left hold the TrackFit kernel (~85 ms alone). Metal dumps byte-identical on pbpb30 with and without the limits, sim5pp identical to the CPU. --- .../Base/metal/GPUReconstructionMetal.mm | 9 ++++++ .../GPUReconstructionMetalIncludesHost.h | 31 +++++++++++++++++++ .../metal/GPUReconstructionMetalKernels.mm | 2 ++ 3 files changed, 42 insertions(+) diff --git a/GPU/GPUTracking/Base/metal/GPUReconstructionMetal.mm b/GPU/GPUTracking/Base/metal/GPUReconstructionMetal.mm index 1f9b0cf3218ad..5292eb5b2f15e 100644 --- a/GPU/GPUTracking/Base/metal/GPUReconstructionMetal.mm +++ b/GPU/GPUTracking/Base/metal/GPUReconstructionMetal.mm @@ -172,6 +172,8 @@ uint32_t metalGPUCoreCount() if (const char* timeline = getenv("GPUCA_METAL_TIMELINE")) { mInternals->timeline = fopen(timeline, "w"); } + mInternals->maxKernelsPerBuffer = getenv("GPUCA_METAL_MAX_KERNELS_PER_BUFFER") ? atoi(getenv("GPUCA_METAL_MAX_KERNELS_PER_BUFFER")) : 0; + mInternals->maxStreamsInFlight = getenv("GPUCA_METAL_MAX_STREAMS_IN_FLIGHT") ? atoi(getenv("GPUCA_METAL_MAX_STREAMS_IN_FLIGHT")) : 0; mInternals->mem_host = metalNewSharedBuffer(mInternals->device, mHostMemorySize); if (mInternals->mem_host == nil) { @@ -251,6 +253,9 @@ uint32_t metalGPUCoreCount() } [mInternals->mem_host release]; } + // The completion handlers of the limit still use the internals + std::unique_lock lock(mInternals->inFlightMutex); + mInternals->inFlightDone.wait(lock, [this] { return mInternals->streamsInFlight == 0; }); if (mInternals->timeline) { fflush(mInternals->timeline); // not closed: a completion handler may still write to it } @@ -406,6 +411,9 @@ MetalMemRef metalResolve(const GPUReconstructionMetalInternals* internals, const const uint64_t value = ++mInternals->streamOrderValue[stream]; [cb encodeSignalEvent:mInternals->streamOrder[stream] value:value]; metalSignalOnError(cb, mInternals->streamOrder[stream], value); + if (mInternals->maxStreamsInFlight) { + metalLimitStreamsInFlight(mInternals, cb, stream); + } if (mInternals->timeline) { metalTimelineAdd(mInternals, stream, "s", mInternals->streamOrder[stream], value); FILE* f = mInternals->timeline; @@ -424,6 +432,7 @@ MetalMemRef metalResolve(const GPUReconstructionMetalInternals* internals, const // it until it is waited for, and open a fresh one to encode the next work in. mInternals->submittedBuffers[stream].emplace_back(cb); mInternals->streamHasWork[stream] = false; + mInternals->streamKernels[stream] = 0; mInternals->commandBuffers[stream] = [[mInternals->commandQueues[stream] commandBuffer] retain]; [mInternals->commandBuffers[stream] encodeWaitForEvent:mInternals->streamOrder[stream] value:value]; metalTimelineAdd(mInternals, stream, "w", mInternals->streamOrder[stream], value); diff --git a/GPU/GPUTracking/Base/metal/GPUReconstructionMetalIncludesHost.h b/GPU/GPUTracking/Base/metal/GPUReconstructionMetalIncludesHost.h index a4b30032e972a..e17275bdbbc2a 100644 --- a/GPU/GPUTracking/Base/metal/GPUReconstructionMetalIncludesHost.h +++ b/GPU/GPUTracking/Base/metal/GPUReconstructionMetalIncludesHost.h @@ -33,6 +33,7 @@ using half = _Float16; #include #include #include +#include #include #include #include @@ -76,6 +77,17 @@ struct GPUReconstructionMetalInternals { std::array timelineOps; std::array timelineKernels; uint64_t timelineBuffers = 0; + // Opt-in limits on the GPU work in flight, 0 = none, so that no stretch of GPU + // work runs long enough for the macOS GPU watchdog: GPUCA_METAL_MAX_KERNELS_PER_BUFFER + // commits a stream's buffer after that many kernels, GPUCA_METAL_MAX_STREAMS_IN_FLIGHT + // is how many streams may have committed buffers that are not complete yet. + uint32_t maxKernelsPerBuffer = 0; + uint32_t maxStreamsInFlight = 0; + std::array streamKernels{}; + std::array streamBuffersInFlight{}; + uint32_t streamsInFlight = 0; + std::mutex inFlightMutex; + std::condition_variable inFlightDone; std::vector> functions; std::vector> pipelines; @@ -111,6 +123,25 @@ inline void metalSignalOnError(id cb, id event } }]; } +// Before a buffer of an idle stream is committed, wait until fewer than the +// limit of streams have work in flight. This cannot deadlock: a buffer only waits +// for events that buffers committed before it signal (RecordMarker commits the +// signal at once), so the work in flight completes without the host's help. +inline void metalLimitStreamsInFlight(GPUReconstructionMetalInternals* internals, id cb, int32_t stream) +{ + std::unique_lock lock(internals->inFlightMutex); + if (internals->streamBuffersInFlight[stream]++ == 0) { + internals->inFlightDone.wait(lock, [internals] { return internals->streamsInFlight < internals->maxStreamsInFlight; }); + internals->streamsInFlight++; + } + [cb addCompletedHandler:^(id) { + std::lock_guard l(internals->inFlightMutex); + if (--internals->streamBuffersInFlight[stream] == 0) { + internals->streamsInFlight--; + internals->inFlightDone.notify_all(); + } + }]; +} // In the time base of the command buffers' GPUStartTime and GPUEndTime inline double metalHostTime() { diff --git a/GPU/GPUTracking/Base/metal/GPUReconstructionMetalKernels.mm b/GPU/GPUTracking/Base/metal/GPUReconstructionMetalKernels.mm index 9eace2cb4f591..09780ac104232 100644 --- a/GPU/GPUTracking/Base/metal/GPUReconstructionMetalKernels.mm +++ b/GPU/GPUTracking/Base/metal/GPUReconstructionMetalKernels.mm @@ -65,6 +65,8 @@ id cb = [mInternals->commandBuffers[kExec.stream] retain]; if (kEvents.ev) { RecordMarker(kEvents.ev, kExec.stream); + } else if (mInternals->maxKernelsPerBuffer && ++mInternals->streamKernels[kExec.stream] >= mInternals->maxKernelsPerBuffer) { + CommitStream(kExec.stream); } if (timed) { if (mInternals->commandBuffers[kExec.stream] == cb) {