diff --git a/common_thread.h b/common_thread.h index d4c3b630af..66767391f2 100644 --- a/common_thread.h +++ b/common_thread.h @@ -183,6 +183,27 @@ static __inline int sme_level3_threads(int nthreads) { #define sme_level3_threads(nthreads) (nthreads) #endif +#if defined(USE_OPENMP) && defined(ARCH_ARM64) && defined(OS_DARWIN) && (defined(VORTEXM4) || defined(DYNAMIC_ARCH)) +#ifdef DYNAMIC_ARCH +extern char *gotoblas_corename(void); +#endif +/* Apple M4 with OpenMP: libomp parks idle threads on Apple Silicon (KMP_BLOCKTIME + defaults to 0 there), and every threaded call wakes the rest of the OpenMP team, + about 3 us per thread on average. A unit-stride level-1 call that reads and writes + fewer than about 384 KB per thread that would be woken is faster on one thread; + bytes is the memory the call reads and writes. Calls with other strides run a + scalar loop and keep the team. Measured on M4 Pro only. */ +static __inline int apple_omp_level1_threads(size_t bytes, int unit_stride, int nthreads) { + if (nthreads <= 1 || !unit_stride) return nthreads; +#ifdef DYNAMIC_ARCH + if (strcmp(gotoblas_corename(), "vortexm4") != 0) return nthreads; +#endif + return bytes < (size_t)(nthreads - 1) * 384 * 1024 ? 1 : nthreads; +} +#else +#define apple_omp_level1_threads(bytes, unit_stride, nthreads) (nthreads) +#endif + static __inline int num_cpu_avail(int level) { #ifdef USE_OPENMP diff --git a/interface/axpy.c b/interface/axpy.c index b7e278d984..7b19199a1d 100644 --- a/interface/axpy.c +++ b/interface/axpy.c @@ -47,23 +47,6 @@ #define MULTI_THREAD_MINIMAL 10000 #endif -#if defined(SMP) && defined(USE_OPENMP) && defined(ARCH_ARM64) && defined(OS_DARWIN) && \ - (defined(DYNAMIC_ARCH) || defined(VORTEXM4)) -#ifdef DYNAMIC_ARCH -extern char *gotoblas_corename(void); -#endif -/* Apple M4, OpenMP: libomp parks idle threads on Apple Silicon (KMP_BLOCKTIME - defaults to 0 there), and every call wakes the rest of the OpenMP team, about - 3 us per thread on average. Stay on one thread until the vector takes longer - than that, about 128 KB per woken thread; measured on M4 Pro only. */ -static int apple_axpy_threads(BLASLONG n, int nthreads) { -#ifdef DYNAMIC_ARCH - if (strcmp(gotoblas_corename(), "vortexm4") != 0) return nthreads; -#endif - return (size_t)n * sizeof(FLOAT) < (size_t)(nthreads - 1) * 128 * 1024 ? 1 : nthreads; -} -#endif - OPENBLAS_EXPORT #ifndef CBLAS @@ -117,10 +100,7 @@ void CNAME(blasint n, FLOAT alpha, FLOAT *x, blasint incx, FLOAT *y, blasint inc else nthreads = num_cpu_avail(1); -#if defined(USE_OPENMP) && defined(ARCH_ARM64) && defined(OS_DARWIN) && \ - (defined(DYNAMIC_ARCH) || defined(VORTEXM4)) - if (nthreads > 1) nthreads = apple_axpy_threads(n, nthreads); -#endif + nthreads = apple_omp_level1_threads(3 * (size_t)n * sizeof(FLOAT), incx == 1 && incy == 1, nthreads); if (nthreads == 1) { #endif diff --git a/interface/zaxpy.c b/interface/zaxpy.c index 6c201bedc8..bdd8fc3f55 100644 --- a/interface/zaxpy.c +++ b/interface/zaxpy.c @@ -108,6 +108,7 @@ void CNAME(blasint n, FLOAT *ALPHA, FLOAT *x, blasint incx, FLOAT *y, blasint in nthreads = 1; else nthreads = num_cpu_avail(1); + nthreads = apple_omp_level1_threads(3 * (size_t)n * COMPSIZE * sizeof(FLOAT), incx == 1 && incy == 1, nthreads); if (nthreads == 1) { #endif diff --git a/kernel/arm64/casum_thunderx2t99.c b/kernel/arm64/casum_thunderx2t99.c index c6dbb3f772..d7a1104716 100644 --- a/kernel/arm64/casum_thunderx2t99.c +++ b/kernel/arm64/casum_thunderx2t99.c @@ -237,6 +237,7 @@ FLOAT CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x) nthreads = 1; else nthreads = num_cpu_avail(1); + nthreads = apple_omp_level1_threads((size_t)n * COMPSIZE * sizeof(FLOAT), inc_x == 1, nthreads); if (nthreads == 1) { asum = casum_compute(n, x, inc_x); diff --git a/kernel/arm64/copy_thunderx2t99.c b/kernel/arm64/copy_thunderx2t99.c index 263cc30130..90a04edf66 100644 --- a/kernel/arm64/copy_thunderx2t99.c +++ b/kernel/arm64/copy_thunderx2t99.c @@ -188,6 +188,7 @@ int CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y) nthreads = 1; else nthreads = num_cpu_avail(1); + nthreads = apple_omp_level1_threads(2 * (size_t)n * COMPSIZE * sizeof(FLOAT), inc_x == 1 && inc_y == 1, nthreads); if (nthreads == 1) { do_copy(n, x, inc_x, y, inc_y); diff --git a/kernel/arm64/csum_thunderx2t99.c b/kernel/arm64/csum_thunderx2t99.c index 874f4eb5ad..0ad79e22a8 100644 --- a/kernel/arm64/csum_thunderx2t99.c +++ b/kernel/arm64/csum_thunderx2t99.c @@ -219,6 +219,7 @@ FLOAT CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x) nthreads = 1; else nthreads = num_cpu_avail(1); + nthreads = apple_omp_level1_threads((size_t)n * COMPSIZE * sizeof(FLOAT), inc_x == 1, nthreads); if (nthreads == 1) { asum = casum_compute(n, x, inc_x); diff --git a/kernel/arm64/dasum_thunderx2t99.c b/kernel/arm64/dasum_thunderx2t99.c index b554f0a9b1..885bf861f1 100644 --- a/kernel/arm64/dasum_thunderx2t99.c +++ b/kernel/arm64/dasum_thunderx2t99.c @@ -234,6 +234,7 @@ FLOAT CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x) nthreads = 1; else nthreads = num_cpu_avail(1); + nthreads = apple_omp_level1_threads((size_t)n * COMPSIZE * sizeof(FLOAT), inc_x == 1, nthreads); if (nthreads == 1) { asum = dasum_compute(n, x, inc_x); diff --git a/kernel/arm64/dot.c b/kernel/arm64/dot.c index 1e49ffdd60..ae96dc38fd 100644 --- a/kernel/arm64/dot.c +++ b/kernel/arm64/dot.c @@ -171,6 +171,7 @@ RETURN_TYPE CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y nthreads = 1; else nthreads = get_dot_optimal_nthreads(n); + nthreads = apple_omp_level1_threads(2 * (size_t)n * COMPSIZE * sizeof(FLOAT), inc_x == 1 && inc_y == 1, nthreads); if (nthreads == 1) { dot = dot_compute(n, x, inc_x, y, inc_y); diff --git a/kernel/arm64/dsum_thunderx2t99.c b/kernel/arm64/dsum_thunderx2t99.c index d42ac8cc33..09d7f35b9d 100644 --- a/kernel/arm64/dsum_thunderx2t99.c +++ b/kernel/arm64/dsum_thunderx2t99.c @@ -216,6 +216,7 @@ FLOAT CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x) nthreads = 1; else nthreads = num_cpu_avail(1); + nthreads = apple_omp_level1_threads((size_t)n * COMPSIZE * sizeof(FLOAT), inc_x == 1, nthreads); if (nthreads == 1) { dsum = dsum_compute(n, x, inc_x); diff --git a/kernel/arm64/iamax_thunderx2t99.c b/kernel/arm64/iamax_thunderx2t99.c index e3bec4a204..e2aa083ad1 100644 --- a/kernel/arm64/iamax_thunderx2t99.c +++ b/kernel/arm64/iamax_thunderx2t99.c @@ -325,6 +325,7 @@ BLASLONG CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x) nthreads = 1; else nthreads = num_cpu_avail(1); + nthreads = apple_omp_level1_threads((size_t)n * COMPSIZE * sizeof(FLOAT), inc_x == 1, nthreads); if (nthreads == 1) { max_index = iamax_compute(n, x, inc_x); diff --git a/kernel/arm64/izamax_thunderx2t99.c b/kernel/arm64/izamax_thunderx2t99.c index b2e2828f0f..b601fc24d1 100644 --- a/kernel/arm64/izamax_thunderx2t99.c +++ b/kernel/arm64/izamax_thunderx2t99.c @@ -334,6 +334,7 @@ BLASLONG CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x) nthreads = 1; else nthreads = num_cpu_avail(1); + nthreads = apple_omp_level1_threads((size_t)n * COMPSIZE * sizeof(FLOAT), inc_x == 1, nthreads); if (nthreads == 1) { max_index = izamax_compute(n, x, inc_x); diff --git a/kernel/arm64/sasum_thunderx2t99.c b/kernel/arm64/sasum_thunderx2t99.c index 2db1e69e74..c4bc254b61 100644 --- a/kernel/arm64/sasum_thunderx2t99.c +++ b/kernel/arm64/sasum_thunderx2t99.c @@ -235,6 +235,7 @@ FLOAT CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x) nthreads = 1; else nthreads = num_cpu_avail(1); + nthreads = apple_omp_level1_threads((size_t)n * COMPSIZE * sizeof(FLOAT), inc_x == 1, nthreads); if (nthreads == 1) { asum = sasum_compute(n, x, inc_x); diff --git a/kernel/arm64/ssum_thunderx2t99.c b/kernel/arm64/ssum_thunderx2t99.c index 5285841109..edd0dacb6a 100644 --- a/kernel/arm64/ssum_thunderx2t99.c +++ b/kernel/arm64/ssum_thunderx2t99.c @@ -217,6 +217,7 @@ FLOAT CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x) nthreads = 1; else nthreads = num_cpu_avail(1); + nthreads = apple_omp_level1_threads((size_t)n * COMPSIZE * sizeof(FLOAT), inc_x == 1, nthreads); if (nthreads == 1) { ssum = ssum_compute(n, x, inc_x); diff --git a/kernel/arm64/zasum_thunderx2t99.c b/kernel/arm64/zasum_thunderx2t99.c index 4813574002..be0edb1363 100644 --- a/kernel/arm64/zasum_thunderx2t99.c +++ b/kernel/arm64/zasum_thunderx2t99.c @@ -235,6 +235,7 @@ FLOAT CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x) nthreads = 1; else nthreads = num_cpu_avail(1); + nthreads = apple_omp_level1_threads((size_t)n * COMPSIZE * sizeof(FLOAT), inc_x == 1, nthreads); if (nthreads == 1) { asum = zasum_compute(n, x, inc_x); diff --git a/kernel/arm64/zdot_thunderx2t99.c b/kernel/arm64/zdot_thunderx2t99.c index 2b28ed9e28..593aaac1d6 100644 --- a/kernel/arm64/zdot_thunderx2t99.c +++ b/kernel/arm64/zdot_thunderx2t99.c @@ -347,6 +347,7 @@ OPENBLAS_COMPLEX_FLOAT CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLA nthreads = 1; else nthreads = num_cpu_avail(1); + nthreads = apple_omp_level1_threads(2 * (size_t)n * COMPSIZE * sizeof(FLOAT), inc_x == 1 && inc_y == 1, nthreads); if (nthreads == 1) { zdot_compute(n, x, inc_x, y, inc_y, &zdot); diff --git a/kernel/arm64/zsum_thunderx2t99.c b/kernel/arm64/zsum_thunderx2t99.c index b8ebdbdec1..d2f8466983 100644 --- a/kernel/arm64/zsum_thunderx2t99.c +++ b/kernel/arm64/zsum_thunderx2t99.c @@ -219,6 +219,7 @@ FLOAT CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x) nthreads = 1; else nthreads = num_cpu_avail(1); + nthreads = apple_omp_level1_threads((size_t)n * COMPSIZE * sizeof(FLOAT), inc_x == 1, nthreads); if (nthreads == 1) { asum = zasum_compute(n, x, inc_x);