From d5991ca0bc278d1edec2e8d11464cf1827823367 Mon Sep 17 00:00:00 2001 From: Vansh Verma Date: Sat, 14 Nov 2026 00:12:03 -0800 Subject: [PATCH] arm64: one thread for small level-1 calls on Apple M4 with OpenMP axpy (#6250) stays on one thread on Apple M4 with OpenMP while the vectors are too short to pay for waking the rest of the OpenMP team, since libomp parks idle threads on Apple Silicon. zaxpy and the arm64 asum, sum, copy, dot, zdot, iamax and izamax kernels make the same one-thread-or-all choice above n = 10000 and pay the same cost: with the default 14 threads on an M4 Pro, sasum at n = 12000 takes 44 us instead of 0.42 us. Move the check from axpy.c into common_thread.h as apple_omp_level1_threads(), which counts the bytes a call reads and writes, and use it in all of these. About 384 KB per thread to be woken is at or just below the measured crossover for every routine; for axpy, which moves three vectors, it is the 128 KB per vector of #6250. Apply it to unit-stride calls only. Calls with other strides handle one element per iteration, 1.5 to 15 times slower per element than unit stride, so one thread loses to the team well inside the byte budget. #6250 got this wrong for real axpy: saxpy with n = 300000 and stride 2 takes 151 us in develop and 61 us with this change, because strided calls keep the team again, as before #6250. Only OpenMP builds for VORTEXM4, and DYNAMIC_ARCH OpenMP builds running on vortexm4, change behaviour; measured on an M4 Pro. --- common_thread.h | 21 +++++++++++++++++++++ interface/axpy.c | 22 +--------------------- interface/zaxpy.c | 1 + kernel/arm64/casum_thunderx2t99.c | 1 + kernel/arm64/copy_thunderx2t99.c | 1 + kernel/arm64/csum_thunderx2t99.c | 1 + kernel/arm64/dasum_thunderx2t99.c | 1 + kernel/arm64/dot.c | 1 + kernel/arm64/dsum_thunderx2t99.c | 1 + kernel/arm64/iamax_thunderx2t99.c | 1 + kernel/arm64/izamax_thunderx2t99.c | 1 + kernel/arm64/sasum_thunderx2t99.c | 1 + kernel/arm64/ssum_thunderx2t99.c | 1 + kernel/arm64/zasum_thunderx2t99.c | 1 + kernel/arm64/zdot_thunderx2t99.c | 1 + kernel/arm64/zsum_thunderx2t99.c | 1 + 16 files changed, 36 insertions(+), 21 deletions(-) diff --git a/common_thread.h b/common_thread.h index d4c3b630af..66767391f2 100644 --- a/common_thread.h +++ b/common_thread.h @@ -183,6 +183,27 @@ static __inline int sme_level3_threads(int nthreads) { #define sme_level3_threads(nthreads) (nthreads) #endif +#if defined(USE_OPENMP) && defined(ARCH_ARM64) && defined(OS_DARWIN) && (defined(VORTEXM4) || defined(DYNAMIC_ARCH)) +#ifdef DYNAMIC_ARCH +extern char *gotoblas_corename(void); +#endif +/* Apple M4 with OpenMP: libomp parks idle threads on Apple Silicon (KMP_BLOCKTIME + defaults to 0 there), and every threaded call wakes the rest of the OpenMP team, + about 3 us per thread on average. A unit-stride level-1 call that reads and writes + fewer than about 384 KB per thread that would be woken is faster on one thread; + bytes is the memory the call reads and writes. Calls with other strides run a + scalar loop and keep the team. Measured on M4 Pro only. */ +static __inline int apple_omp_level1_threads(size_t bytes, int unit_stride, int nthreads) { + if (nthreads <= 1 || !unit_stride) return nthreads; +#ifdef DYNAMIC_ARCH + if (strcmp(gotoblas_corename(), "vortexm4") != 0) return nthreads; +#endif + return bytes < (size_t)(nthreads - 1) * 384 * 1024 ? 1 : nthreads; +} +#else +#define apple_omp_level1_threads(bytes, unit_stride, nthreads) (nthreads) +#endif + static __inline int num_cpu_avail(int level) { #ifdef USE_OPENMP diff --git a/interface/axpy.c b/interface/axpy.c index b7e278d984..7b19199a1d 100644 --- a/interface/axpy.c +++ b/interface/axpy.c @@ -47,23 +47,6 @@ #define MULTI_THREAD_MINIMAL 10000 #endif -#if defined(SMP) && defined(USE_OPENMP) && defined(ARCH_ARM64) && defined(OS_DARWIN) && \ - (defined(DYNAMIC_ARCH) || defined(VORTEXM4)) -#ifdef DYNAMIC_ARCH -extern char *gotoblas_corename(void); -#endif -/* Apple M4, OpenMP: libomp parks idle threads on Apple Silicon (KMP_BLOCKTIME - defaults to 0 there), and every call wakes the rest of the OpenMP team, about - 3 us per thread on average. Stay on one thread until the vector takes longer - than that, about 128 KB per woken thread; measured on M4 Pro only. */ -static int apple_axpy_threads(BLASLONG n, int nthreads) { -#ifdef DYNAMIC_ARCH - if (strcmp(gotoblas_corename(), "vortexm4") != 0) return nthreads; -#endif - return (size_t)n * sizeof(FLOAT) < (size_t)(nthreads - 1) * 128 * 1024 ? 1 : nthreads; -} -#endif - OPENBLAS_EXPORT #ifndef CBLAS @@ -117,10 +100,7 @@ void CNAME(blasint n, FLOAT alpha, FLOAT *x, blasint incx, FLOAT *y, blasint inc else nthreads = num_cpu_avail(1); -#if defined(USE_OPENMP) && defined(ARCH_ARM64) && defined(OS_DARWIN) && \ - (defined(DYNAMIC_ARCH) || defined(VORTEXM4)) - if (nthreads > 1) nthreads = apple_axpy_threads(n, nthreads); -#endif + nthreads = apple_omp_level1_threads(3 * (size_t)n * sizeof(FLOAT), incx == 1 && incy == 1, nthreads); if (nthreads == 1) { #endif diff --git a/interface/zaxpy.c b/interface/zaxpy.c index 6c201bedc8..bdd8fc3f55 100644 --- a/interface/zaxpy.c +++ b/interface/zaxpy.c @@ -108,6 +108,7 @@ void CNAME(blasint n, FLOAT *ALPHA, FLOAT *x, blasint incx, FLOAT *y, blasint in nthreads = 1; else nthreads = num_cpu_avail(1); + nthreads = apple_omp_level1_threads(3 * (size_t)n * COMPSIZE * sizeof(FLOAT), incx == 1 && incy == 1, nthreads); if (nthreads == 1) { #endif diff --git a/kernel/arm64/casum_thunderx2t99.c b/kernel/arm64/casum_thunderx2t99.c index c6dbb3f772..d7a1104716 100644 --- a/kernel/arm64/casum_thunderx2t99.c +++ b/kernel/arm64/casum_thunderx2t99.c @@ -237,6 +237,7 @@ FLOAT CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x) nthreads = 1; else nthreads = num_cpu_avail(1); + nthreads = apple_omp_level1_threads((size_t)n * COMPSIZE * sizeof(FLOAT), inc_x == 1, nthreads); if (nthreads == 1) { asum = casum_compute(n, x, inc_x); diff --git a/kernel/arm64/copy_thunderx2t99.c b/kernel/arm64/copy_thunderx2t99.c index 263cc30130..90a04edf66 100644 --- a/kernel/arm64/copy_thunderx2t99.c +++ b/kernel/arm64/copy_thunderx2t99.c @@ -188,6 +188,7 @@ int CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y) nthreads = 1; else nthreads = num_cpu_avail(1); + nthreads = apple_omp_level1_threads(2 * (size_t)n * COMPSIZE * sizeof(FLOAT), inc_x == 1 && inc_y == 1, nthreads); if (nthreads == 1) { do_copy(n, x, inc_x, y, inc_y); diff --git a/kernel/arm64/csum_thunderx2t99.c b/kernel/arm64/csum_thunderx2t99.c index 874f4eb5ad..0ad79e22a8 100644 --- a/kernel/arm64/csum_thunderx2t99.c +++ b/kernel/arm64/csum_thunderx2t99.c @@ -219,6 +219,7 @@ FLOAT CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x) nthreads = 1; else nthreads = num_cpu_avail(1); + nthreads = apple_omp_level1_threads((size_t)n * COMPSIZE * sizeof(FLOAT), inc_x == 1, nthreads); if (nthreads == 1) { asum = casum_compute(n, x, inc_x); diff --git a/kernel/arm64/dasum_thunderx2t99.c b/kernel/arm64/dasum_thunderx2t99.c index b554f0a9b1..885bf861f1 100644 --- a/kernel/arm64/dasum_thunderx2t99.c +++ b/kernel/arm64/dasum_thunderx2t99.c @@ -234,6 +234,7 @@ FLOAT CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x) nthreads = 1; else nthreads = num_cpu_avail(1); + nthreads = apple_omp_level1_threads((size_t)n * COMPSIZE * sizeof(FLOAT), inc_x == 1, nthreads); if (nthreads == 1) { asum = dasum_compute(n, x, inc_x); diff --git a/kernel/arm64/dot.c b/kernel/arm64/dot.c index 1e49ffdd60..ae96dc38fd 100644 --- a/kernel/arm64/dot.c +++ b/kernel/arm64/dot.c @@ -171,6 +171,7 @@ RETURN_TYPE CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y nthreads = 1; else nthreads = get_dot_optimal_nthreads(n); + nthreads = apple_omp_level1_threads(2 * (size_t)n * COMPSIZE * sizeof(FLOAT), inc_x == 1 && inc_y == 1, nthreads); if (nthreads == 1) { dot = dot_compute(n, x, inc_x, y, inc_y); diff --git a/kernel/arm64/dsum_thunderx2t99.c b/kernel/arm64/dsum_thunderx2t99.c index d42ac8cc33..09d7f35b9d 100644 --- a/kernel/arm64/dsum_thunderx2t99.c +++ b/kernel/arm64/dsum_thunderx2t99.c @@ -216,6 +216,7 @@ FLOAT CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x) nthreads = 1; else nthreads = num_cpu_avail(1); + nthreads = apple_omp_level1_threads((size_t)n * COMPSIZE * sizeof(FLOAT), inc_x == 1, nthreads); if (nthreads == 1) { dsum = dsum_compute(n, x, inc_x); diff --git a/kernel/arm64/iamax_thunderx2t99.c b/kernel/arm64/iamax_thunderx2t99.c index e3bec4a204..e2aa083ad1 100644 --- a/kernel/arm64/iamax_thunderx2t99.c +++ b/kernel/arm64/iamax_thunderx2t99.c @@ -325,6 +325,7 @@ BLASLONG CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x) nthreads = 1; else nthreads = num_cpu_avail(1); + nthreads = apple_omp_level1_threads((size_t)n * COMPSIZE * sizeof(FLOAT), inc_x == 1, nthreads); if (nthreads == 1) { max_index = iamax_compute(n, x, inc_x); diff --git a/kernel/arm64/izamax_thunderx2t99.c b/kernel/arm64/izamax_thunderx2t99.c index b2e2828f0f..b601fc24d1 100644 --- a/kernel/arm64/izamax_thunderx2t99.c +++ b/kernel/arm64/izamax_thunderx2t99.c @@ -334,6 +334,7 @@ BLASLONG CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x) nthreads = 1; else nthreads = num_cpu_avail(1); + nthreads = apple_omp_level1_threads((size_t)n * COMPSIZE * sizeof(FLOAT), inc_x == 1, nthreads); if (nthreads == 1) { max_index = izamax_compute(n, x, inc_x); diff --git a/kernel/arm64/sasum_thunderx2t99.c b/kernel/arm64/sasum_thunderx2t99.c index 2db1e69e74..c4bc254b61 100644 --- a/kernel/arm64/sasum_thunderx2t99.c +++ b/kernel/arm64/sasum_thunderx2t99.c @@ -235,6 +235,7 @@ FLOAT CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x) nthreads = 1; else nthreads = num_cpu_avail(1); + nthreads = apple_omp_level1_threads((size_t)n * COMPSIZE * sizeof(FLOAT), inc_x == 1, nthreads); if (nthreads == 1) { asum = sasum_compute(n, x, inc_x); diff --git a/kernel/arm64/ssum_thunderx2t99.c b/kernel/arm64/ssum_thunderx2t99.c index 5285841109..edd0dacb6a 100644 --- a/kernel/arm64/ssum_thunderx2t99.c +++ b/kernel/arm64/ssum_thunderx2t99.c @@ -217,6 +217,7 @@ FLOAT CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x) nthreads = 1; else nthreads = num_cpu_avail(1); + nthreads = apple_omp_level1_threads((size_t)n * COMPSIZE * sizeof(FLOAT), inc_x == 1, nthreads); if (nthreads == 1) { ssum = ssum_compute(n, x, inc_x); diff --git a/kernel/arm64/zasum_thunderx2t99.c b/kernel/arm64/zasum_thunderx2t99.c index 4813574002..be0edb1363 100644 --- a/kernel/arm64/zasum_thunderx2t99.c +++ b/kernel/arm64/zasum_thunderx2t99.c @@ -235,6 +235,7 @@ FLOAT CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x) nthreads = 1; else nthreads = num_cpu_avail(1); + nthreads = apple_omp_level1_threads((size_t)n * COMPSIZE * sizeof(FLOAT), inc_x == 1, nthreads); if (nthreads == 1) { asum = zasum_compute(n, x, inc_x); diff --git a/kernel/arm64/zdot_thunderx2t99.c b/kernel/arm64/zdot_thunderx2t99.c index 2b28ed9e28..593aaac1d6 100644 --- a/kernel/arm64/zdot_thunderx2t99.c +++ b/kernel/arm64/zdot_thunderx2t99.c @@ -347,6 +347,7 @@ OPENBLAS_COMPLEX_FLOAT CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLA nthreads = 1; else nthreads = num_cpu_avail(1); + nthreads = apple_omp_level1_threads(2 * (size_t)n * COMPSIZE * sizeof(FLOAT), inc_x == 1 && inc_y == 1, nthreads); if (nthreads == 1) { zdot_compute(n, x, inc_x, y, inc_y, &zdot); diff --git a/kernel/arm64/zsum_thunderx2t99.c b/kernel/arm64/zsum_thunderx2t99.c index b8ebdbdec1..d2f8466983 100644 --- a/kernel/arm64/zsum_thunderx2t99.c +++ b/kernel/arm64/zsum_thunderx2t99.c @@ -219,6 +219,7 @@ FLOAT CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x) nthreads = 1; else nthreads = num_cpu_avail(1); + nthreads = apple_omp_level1_threads((size_t)n * COMPSIZE * sizeof(FLOAT), inc_x == 1, nthreads); if (nthreads == 1) { asum = zasum_compute(n, x, inc_x);