diff --git a/Changes.md b/Changes.md index 26b06b8e..765e56fd 100644 --- a/Changes.md +++ b/Changes.md @@ -2,6 +2,7 @@ libhxcpp 6.0.0 (unreleased) ------------------------------------------------------------ +* Fixed a program on Linux with glibc staying as large as its largest burst of large allocations, such as the files a game reads while it loads: glibc keeps a freed buffer's memory in its heap unless it had mapped the buffer on its own, and once a buffer mapped on its own has been freed it stops mapping those of that size or less, up to 32 MB. The memory of the large buffers a collection freed then stayed the process's. The collector now trims the C heap once 16 MB has been freed to it, on the thread that frees the buffers. A program that dropped 384 MB of 4 MB buffers stayed at 396 MB resident and now returns to 12 MB * Fixed a native extension that includes hx/CFFI.h before hx/CFFIPrime.h, as SteamWrap does, failing to compile with hxs_encoding, hxs_utf16 and alloc_hxs_utf8 not declared: CFFI.h included first declares the CFFI API without HXCPP_PRIME, so without the Prime string functions that the ordinary wrapper of a DEFINE_PRIME function now uses. CFFIPrime.h declares them then * Made the collector's thread context on Linux a __thread pointer, read with one load from the thread pointer, instead of a call to pthread_getspecific each time an allocation leaves the inline path or a String or number is boxed. It was 3 to 4% of an OpenFL game's main thread. HXCPP_PTHREAD_FAST_TLS keeps pthread_getspecific (libhxcpp #429) * Fixed a call on a null Array in cppia ending the process under HXCPP_CHECK_POINTER: push, contains, remove, indexOf, lastIndexOf, join, toString, sort, insert and unshift used the array without checking it, interpreted and under the JIT. They throw Null Object Reference, as a typed array's calls and the other Array calls do diff --git a/src/hx/gc/Immix.cpp b/src/hx/gc/Immix.cpp index b530ae8d..05aeab12 100644 --- a/src/hx/gc/Immix.cpp +++ b/src/hx/gc/Immix.cpp @@ -32,6 +32,16 @@ #include #include +// glibc keeps the memory of a freed buffer in its heap, unless the buffer was +// mapped on its own, and it stops mapping buffers on their own up to the size +// of the largest one freed so far, 32 MB at most on 64 bits. The memory of the +// large buffers a collection frees then stays the process's, for the next +// allocation of a size that fits, which may never come +#if defined(__GLIBC__) && !defined(__EMSCRIPTEN__) + #include + #define HX_GC_TRIM_FREED +#endif + #if defined(__SANITIZE_ADDRESS__) #define HX_ADDRESS_SANITIZER #elif defined(__has_feature) @@ -3840,6 +3850,11 @@ class AllocCounter : public hx::VisitContext static const size_t sLargeAllocSpaceFloor = 40 << 20; +#ifdef HX_GC_TRIM_FREED +// The heap is trimmed once this much has been freed to it: a trim walks every +// free chunk of the heap, which is not worth it for less +static const size_t sTrimFreedBytes = 16 << 20; +#endif class GlobalAllocator *sGlobalAlloc = 0; @@ -4223,6 +4238,37 @@ class GlobalAllocator return bytes; } + // Counts memory handed to free(), for trimFreed + void noteFreed(size_t inBytes) + { + #ifdef HX_GC_TRIM_FREED + mFreedSinceTrim.fetch_add(inBytes, std::memory_order_relaxed); + #endif + } + + bool trimDue() + { + #ifdef HX_GC_TRIM_FREED + return mFreedSinceTrim.load(std::memory_order_relaxed) >= sTrimFreedBytes; + #else + return false; + #endif + } + + // Hands the free memory of the C heap back to the system once enough has + // been freed to it. Not called while the other threads wait: the trim + // takes time in the memory it gives back + void trimFreed() + { + #ifdef HX_GC_TRIM_FREED + if (trimDue()) + { + mFreedSinceTrim.store(0, std::memory_order_relaxed); + malloc_trim(0); + } + #endif + } + // A collection drops the recycle cache it found, and frees it once every // thread runs again: freeing tens of thousands of buffers took tens of // milliseconds while they all waited (libhxcpp #332). A thread of its @@ -4231,7 +4277,8 @@ class GlobalAllocator // before it returns void releaseDropped(RecycleLists &ioDropped, bool inNow) { - if (ioDropped.empty() && !inNow) + // With nothing to free, the releaser still trims what the sweep freed + if (ioDropped.empty() && !inNow && !trimDue()) return; #if !defined(HXCPP_SINGLE_THREADED_APP) && !defined(__EMSCRIPTEN__) if (!inNow) @@ -4264,21 +4311,31 @@ class GlobalAllocator } #endif - mDroppedBytes.fetch_sub(freeRecycle(ioDropped), std::memory_order_relaxed); + releaseBatch(ioDropped); #if !defined(HXCPP_SINGLE_THREADED_APP) && !defined(__EMSCRIPTEN__) - std::unique_lock lock(mReleaseLock); - while(!mReleaseQueue.empty()) { - RecycleLists waiting; - waiting.swap(mReleaseQueue.back()); - mReleaseQueue.pop_back(); - lock.unlock(); - mDroppedBytes.fetch_sub(freeRecycle(waiting), std::memory_order_relaxed); - lock.lock(); + std::unique_lock lock(mReleaseLock); + while(!mReleaseQueue.empty()) + { + RecycleLists waiting; + waiting.swap(mReleaseQueue.back()); + mReleaseQueue.pop_back(); + lock.unlock(); + releaseBatch(waiting); + lock.lock(); + } + mReleaseIdle.wait(lock, [this]() { return !mReleaseBusy; }); } - mReleaseIdle.wait(lock, [this]() { return !mReleaseBusy; }); #endif + trimFreed(); + } + + void releaseBatch(RecycleLists &ioBatch) + { + size_t bytes = freeRecycle(ioBatch); + mDroppedBytes.fetch_sub(bytes, std::memory_order_relaxed); + noteFreed(bytes); } #if !defined(HXCPP_SINGLE_THREADED_APP) && !defined(__EMSCRIPTEN__) @@ -4295,7 +4352,8 @@ class GlobalAllocator mReleaseQueue.pop_back(); mReleaseBusy = true; lock.unlock(); - mDroppedBytes.fetch_sub(freeRecycle(batch), std::memory_order_relaxed); + releaseBatch(batch); + trimFreed(); lock.lock(); mReleaseBusy = false; mReleaseIdle.notify_all(); @@ -5168,6 +5226,7 @@ class GlobalAllocator #endif HxFreeGCBlock(g.alloc); + noteFreed(groupBytes); g.alloc = 0; groups++; @@ -6754,6 +6813,7 @@ class GlobalAllocator else { HxFree(blob); + noteFreed(size); } qeraseLarge(idx); @@ -7628,6 +7688,10 @@ class GlobalAllocator std::atomic_size_t mRecycleBytes; // The data bytes of dropped caches not freed yet std::atomic_size_t mDroppedBytes; + #ifdef HX_GC_TRIM_FREED + // Bytes freed to the C heap since it was last trimmed + std::atomic_size_t mFreedSinceTrim{0}; + #endif #if !defined(HXCPP_SINGLE_THREADED_APP) && !defined(__EMSCRIPTEN__) std::mutex mReleaseLock; std::condition_variable mReleaseWake; diff --git a/test/regression/LargeFreedMemoryReturned/Main.hx b/test/regression/LargeFreedMemoryReturned/Main.hx new file mode 100644 index 00000000..a03c478c --- /dev/null +++ b/test/regression/LargeFreedMemoryReturned/Main.hx @@ -0,0 +1,77 @@ +import cpp.vm.Gc; +import haxe.io.Bytes; + +// glibc keeps a freed buffer's memory in its heap unless the buffer was mapped +// on its own, and it stops mapping buffers on their own once one that size has +// been freed. The collector hands the memory of the large buffers it frees back +// to the system. +class Main { + static inline var MB = 1024 * 1024; + + static var kept:Array = []; + static var temporary:Array; + + // Megabytes of the process in memory, or -1 where it cannot be measured: + // without /proc, and under ASan, whose allocator replaces glibc's and keeps + // freed memory in its quarantine + static function resident():Float { + #if (HXCPP_SANITIZE == "address") + return -1; + #else + if (Sys.systemName() != "Linux" || !sys.FileSystem.exists("/proc/self/status")) + return -1; + // VmRSS is in kB whatever the page size, which is 16 or 64 KB on some arm64 kernels + for (line in sys.io.File.getContent("/proc/self/status").split("\n")) { + if (StringTools.startsWith(line, "VmRSS:")) + return Std.parseFloat(StringTools.trim(line.substr(6)).split(" ")[0]) / 1024; + } + return -1; + #end + } + + // A small buffer that lives follows each large one, so that none of the + // freed memory is at the end of the heap, which glibc gives back by itself + static function burst(count:Int, size:Int) { + temporary = []; + for (i in 0...count) { + var bytes = Bytes.alloc(size); + bytes.fill(0, size, 1); + temporary.push(bytes); + kept.push(Bytes.alloc(64 * 1024)); + } + } + + static function release() { + temporary = null; + // The first collection keeps the dead buffers for reuse, the second frees them + Gc.run(true); + Gc.run(true); + } + + static function main() { + if (resident() < 0) { + Sys.println("the burst was in memory: true"); + Sys.println("freed large buffers go back to the system: true"); + return; + } + // Freeing buffers mapped on their own raises the size from which glibc maps them + burst(4, 8 * MB); + release(); + var before = resident(); + burst(96, 4 * MB); + var during = resident(); + release(); + // A thread of the collector's frees them + var after = resident(); + var waited = 0.0; + while (after - before > 96 && waited < 10) { + Sys.sleep(0.05); + waited += 0.05; + after = resident(); + } + Sys.println("the burst was in memory: " + (during - before > 300)); + Sys.println("freed large buffers go back to the system: " + (after - before <= 96)); + if (after - before > 96) + Sys.println("resident before " + Math.round(before) + " MB, during " + Math.round(during) + " MB, after " + Math.round(after) + " MB"); + } +} diff --git a/test/regression/LargeFreedMemoryReturned/build.hxml b/test/regression/LargeFreedMemoryReturned/build.hxml new file mode 100644 index 00000000..b484e67e --- /dev/null +++ b/test/regression/LargeFreedMemoryReturned/build.hxml @@ -0,0 +1,2 @@ +-m Main +--cpp bin diff --git a/test/regression/LargeFreedMemoryReturned/stdout.txt b/test/regression/LargeFreedMemoryReturned/stdout.txt new file mode 100644 index 00000000..fb2d4bb3 --- /dev/null +++ b/test/regression/LargeFreedMemoryReturned/stdout.txt @@ -0,0 +1,2 @@ +the burst was in memory: true +freed large buffers go back to the system: true