From 3024fd6b26c169a8111cf89bb03016eb929e36a0 Mon Sep 17 00:00:00 2001 From: Kunnis Date: Wed, 8 May 2024 22:58:28 -0500 Subject: [PATCH 01/27] Just reordering some structs. --- ggml.c | 108 ++++++++++++++++++++++++++++----------------------------- 1 file changed, 53 insertions(+), 55 deletions(-) diff --git a/ggml.c b/ggml.c index 093d38d00ae35..fc30bd839be94 100644 --- a/ggml.c +++ b/ggml.c @@ -109,6 +109,8 @@ typedef void * thread_ret_t; #endif +typedef pthread_t ggml_thread_t; + #ifdef GGML_USE_CPU_HBM #include #endif @@ -1534,6 +1536,57 @@ static inline void __sse_f16x4_store(ggml_fp16_t *x, __m128 y) { #define GGML_F16_ARR (GGML_F16_STEP/GGML_F16_EPR) #endif +// +// ggml context +// + +struct ggml_context { + size_t mem_size; + void* mem_buffer; + bool mem_buffer_owned; + bool no_alloc; + bool no_alloc_save; // this is used to save the no_alloc state when using scratch buffers + + int n_objects; + + struct ggml_object* objects_begin; + struct ggml_object* objects_end; + + struct ggml_scratch scratch; + struct ggml_scratch scratch_save; +}; + +struct ggml_context_container { + bool used; + + struct ggml_context context; +}; + +struct ggml_compute_state_shared { + const struct ggml_cgraph* cgraph; + const struct ggml_cplan* cplan; + + int64_t perf_node_start_cycles; + int64_t perf_node_start_time_us; + + const int n_threads; + + // synchronization primitives + atomic_int n_active; // num active threads + atomic_int node_n; // active graph node + atomic_int node_task; // active graph node task phase + + ggml_abort_callback abort_callback; // abort ggml_graph_compute when true + void* abort_callback_data; +}; + +struct ggml_compute_state { + ggml_thread_t thrd; + int ith; + struct ggml_compute_state_shared* shared; + enum ggml_status ec; +}; + // // fundamental operations // @@ -2380,32 +2433,6 @@ static void ggml_setup_op_has_task_pass(void) { } } -// -// ggml context -// - -struct ggml_context { - size_t mem_size; - void * mem_buffer; - bool mem_buffer_owned; - bool no_alloc; - bool no_alloc_save; // this is used to save the no_alloc state when using scratch buffers - - int n_objects; - - struct ggml_object * objects_begin; - struct ggml_object * objects_end; - - struct ggml_scratch scratch; - struct ggml_scratch scratch_save; -}; - -struct ggml_context_container { - bool used; - - struct ggml_context context; -}; - // // NUMA support // @@ -19172,8 +19199,6 @@ typedef int ggml_lock_t; #define GGML_LOCK_INITIALIZER 0 -typedef pthread_t ggml_thread_t; - #define ggml_thread_create pthread_create #define ggml_thread_join pthread_join @@ -19199,8 +19224,6 @@ typedef int ggml_lock_t; #define GGML_LOCK_INITIALIZER 0 -typedef pthread_t ggml_thread_t; - #define ggml_thread_create pthread_create #define ggml_thread_join pthread_join @@ -19280,31 +19303,6 @@ static void set_numa_thread_affinity(int thread_n) { UNUSED(thread_n); } static void clear_numa_thread_affinity(void) {} #endif -struct ggml_compute_state_shared { - const struct ggml_cgraph * cgraph; - const struct ggml_cplan * cplan; - - int64_t perf_node_start_cycles; - int64_t perf_node_start_time_us; - - const int n_threads; - - // synchronization primitives - atomic_int n_active; // num active threads - atomic_int node_n; // active graph node - atomic_int node_task; // active graph node task phase - - ggml_abort_callback abort_callback; // abort ggml_graph_compute when true - void * abort_callback_data; -}; - -struct ggml_compute_state { - ggml_thread_t thrd; - int ith; - struct ggml_compute_state_shared * shared; - enum ggml_status ec; -}; - static void ggml_graph_compute_perf_stats_node(struct ggml_tensor * node, const struct ggml_compute_state_shared * st) { int64_t cycles_cur = ggml_perf_cycles() - st->perf_node_start_cycles; int64_t time_us_cur = ggml_perf_time_us() - st->perf_node_start_time_us; From 5978b6ebf080c24fa02f369d53412dfff44bbed9 Mon Sep 17 00:00:00 2001 From: Kunnis Date: Wed, 8 May 2024 23:02:20 -0500 Subject: [PATCH 02/27] Adding in the calls to mm_pause --- ggml.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ggml.c b/ggml.c index fc30bd839be94..8a5149c1bc657 100644 --- a/ggml.c +++ b/ggml.c @@ -19578,6 +19578,10 @@ static void ggml_graph_compute_thread_sync_node(int * node_n, struct ggml_comput * node_n = atomic_load(&state->shared->node_n); if (* node_n != last_node_n) break; +#if defined(__SSE3__) + //Tell the processor we're spinning. It's a processor hint for spinlocks. + _mm_pause(); +#endif } } @@ -19592,6 +19596,10 @@ static void ggml_graph_compute_thread_sync_task(int * task_phase, struct ggml_co * task_phase = atomic_load(&state->shared->node_task); if (* task_phase != last_task_phase) break; +#if defined(__SSE3__) + //Tell the processor we're spinning. It's a processor hint for spinlocks. + _mm_pause(); +#endif } } From e098171aa772c83e7b6d5ef5721e0761697f6b94 Mon Sep 17 00:00:00 2001 From: Kunnis Date: Wed, 8 May 2024 23:10:50 -0500 Subject: [PATCH 03/27] Passing around the state --- ggml.c | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/ggml.c b/ggml.c index 8a5149c1bc657..06d2c1bf6bbf7 100644 --- a/ggml.c +++ b/ggml.c @@ -11771,7 +11771,8 @@ static bool ggml_compute_forward_mul_mat_use_blas(struct ggml_tensor * dst) { static void ggml_compute_forward_mul_mat( const struct ggml_compute_params * params, - struct ggml_tensor * dst) { + struct ggml_tensor * dst, + struct ggml_compute_state* state) { const struct ggml_tensor * src0 = dst->src[0]; const struct ggml_tensor * src1 = dst->src[1]; @@ -17481,7 +17482,7 @@ static void ggml_compute_forward_cross_entropy_loss_back( ///////////////////////////////// -static void ggml_compute_forward(struct ggml_compute_params * params, struct ggml_tensor * tensor) { +static void ggml_compute_forward(struct ggml_compute_params * params, struct ggml_tensor * tensor, struct ggml_compute_state * state) { GGML_ASSERT(params); if (tensor->op == GGML_OP_NONE || ggml_is_empty(tensor)) { @@ -17579,7 +17580,7 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm } break; case GGML_OP_MUL_MAT: { - ggml_compute_forward_mul_mat(params, tensor); + ggml_compute_forward_mul_mat(params, tensor, state); } break; case GGML_OP_MUL_MAT_ID: { @@ -19639,7 +19640,7 @@ static thread_ret_t ggml_graph_compute_thread(void * data) { struct ggml_tensor * node = cgraph->nodes[node_n]; if (GGML_OP_HAS_FINALIZE[node->op]) { params.nth = ggml_get_n_tasks(node, n_threads, state->shared->n_threads); - ggml_compute_forward(¶ms, node); + ggml_compute_forward(¶ms, node, state); } ggml_graph_compute_perf_stats_node(node, state->shared); } @@ -19659,17 +19660,17 @@ static thread_ret_t ggml_graph_compute_thread(void * data) { /* INIT */ if (GGML_OP_HAS_INIT[node->op]) { params.type = GGML_TASK_TYPE_INIT; - ggml_compute_forward(¶ms, node); + ggml_compute_forward(¶ms, node, state); } // TODO: maybe push node_n to the atomic but if other threads see n_tasks is 1, // they do something more efficient than spinning (?) params.type = GGML_TASK_TYPE_COMPUTE; - ggml_compute_forward(¶ms, node); + ggml_compute_forward(¶ms, node, state); if (GGML_OP_HAS_FINALIZE[node->op]) { params.type = GGML_TASK_TYPE_FINALIZE; - ggml_compute_forward(¶ms, node); + ggml_compute_forward(¶ms, node, state); } ggml_graph_compute_perf_stats_node(node, state->shared); @@ -19708,7 +19709,7 @@ static thread_ret_t ggml_graph_compute_thread(void * data) { if (state->ith < n_tasks) { if (GGML_OP_HAS_INIT[node->op]) { - ggml_compute_forward(¶ms, node); + ggml_compute_forward(¶ms, node, state); } } @@ -19729,7 +19730,7 @@ static thread_ret_t ggml_graph_compute_thread(void * data) { if (state->ith < n_tasks) { params.type = GGML_TASK_TYPE_COMPUTE; - ggml_compute_forward(¶ms, node); + ggml_compute_forward(¶ms, node, state); } if (atomic_fetch_sub(&state->shared->n_active, 1) == 1) { From a968553c6f1d633f2cd3264220bd731c79e0a9fa Mon Sep 17 00:00:00 2001 From: Kunnis Date: Wed, 8 May 2024 23:43:43 -0500 Subject: [PATCH 04/27] Renaming and moving a bunch of variables around. --- ggml.c | 46 +++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/ggml.c b/ggml.c index 06d2c1bf6bbf7..89c6bc3323198 100644 --- a/ggml.c +++ b/ggml.c @@ -11983,44 +11983,44 @@ UseGgmlGemm2:; const int64_t dr0 = (nr0 + nth0 - 1)/nth0; const int64_t dr1 = (nr1 + nth1 - 1)/nth1; - const int64_t ir010 = dr0*ith0; - const int64_t ir011 = MIN(ir010 + dr0, nr0); + const int64_t ir0_start = dr0*ith0; + const int64_t ir0_end = MIN(ir0_start + dr0, nr0); - const int64_t ir110 = dr1*ith1; - const int64_t ir111 = MIN(ir110 + dr1, nr1); + const int64_t ir1_start = dr1*ith1; + const int64_t ir1_end = MIN(ir1_start + dr1, nr1); - //printf("ir010 = %6lld, ir011 = %6lld, ir110 = %6lld, ir111 = %6lld\n", ir010, ir011, ir110, ir111); + //printf("ir0_start = %6lld, ir0_end = %6lld, ir1_start = %6lld, ir1_end = %6lld\n", ir0_start, ir0_end, ir1_start, ir1_end); // threads with no work simply yield (not sure if it helps) - if (ir010 >= ir011 || ir110 >= ir111) { + if (ir0_start >= ir0_end || ir1_start >= ir1_end) { sched_yield(); return; } - assert(ne12 % ne02 == 0); - assert(ne13 % ne03 == 0); - - // block-tiling attempt - const int64_t blck_0 = 16; - const int64_t blck_1 = 16; - // dot kernels can handle 1 row and col at a time, but mmla kernels can process 2 rows and cols - int64_t nrc = vec_dot_num_rows; + int64_t num_rows_per_vec_dot = vec_dot_num_rows; // TODO: currently the mmla kernels support only even numbered rows/cols. // this check can be removed once they are extended to support odd numbered rows/cols too if ((nr0 % 2 != 0) || (ne11 % 2 != 0)) { - nrc = 1; + num_rows_per_vec_dot = 1; } + assert(ne12% ne02 == 0); + assert(ne13% ne03 == 0); + + // block-tiling attempt + const int64_t blck_0 = 16; + const int64_t blck_1 = 16; + const size_t src1_col_stride = src1_cont || src1->type != vec_dot_type ? row_size : nb11; // attempt to reduce false-sharing (does not seem to make a difference) // 16 * 2, accounting for mmla kernels float tmp[32]; - for (int64_t iir1 = ir110; iir1 < ir111; iir1 += blck_1) { - for (int64_t iir0 = ir010; iir0 < ir011; iir0 += blck_0) { - for (int64_t ir1 = iir1; ir1 < iir1 + blck_1 && ir1 < ir111; ir1 += nrc) { + for (int64_t iir1 = ir1_start; iir1 < ir1_end; iir1 += blck_1) { + for (int64_t iir0 = ir0_start; iir0 < ir0_end; iir0 += blck_0) { + for (int64_t ir1 = iir1; ir1 < iir1 + blck_1 && ir1 < ir1_end; ir1 += num_rows_per_vec_dot) { const int64_t i13 = (ir1/(ne12*ne1)); const int64_t i12 = (ir1 - i13*ne12*ne1)/ne1; const int64_t i11 = (ir1 - i13*ne12*ne1 - i12*ne1); @@ -12045,16 +12045,16 @@ UseGgmlGemm2:; : (i11*nb11 + i12*nb12 + i13*nb13)); float * dst_col = (float *) ((char *) dst->data + (i1*nb1 + i2*nb2 + i3*nb3)); - //for (int64_t ir0 = iir0; ir0 < iir0 + blck_0 && ir0 < ir011; ++ir0) { + //for (int64_t ir0 = iir0; ir0 < iir0 + blck_0 && ir0 < ir0_end; ++ir0) { // vec_dot(ne00, &dst_col[ir0], src0_row + ir0*nb01, src1_col); //} - for (int64_t ir0 = iir0; ir0 < iir0 + blck_0 && ir0 < ir011; ir0 += nrc) { - vec_dot(ne00, &tmp[ir0 - iir0], (nrc>1 ? 16 : 0), src0_row + ir0*nb01, (nrc>1 ? nb01 : 0), src1_col, (nrc>1 ? src1_col_stride : 0), nrc); + for (int64_t ir0 = iir0; ir0 < iir0 + blck_0 && ir0 < ir0_end; ir0 += num_rows_per_vec_dot) { + vec_dot(ne00, &tmp[ir0 - iir0], (num_rows_per_vec_dot >1 ? 16 : 0), src0_row + ir0*nb01, (num_rows_per_vec_dot >1 ? nb01 : 0), src1_col, (num_rows_per_vec_dot >1 ? src1_col_stride : 0), num_rows_per_vec_dot); } - for (int cn = 0; cn < nrc; ++cn) { - memcpy(&dst_col[iir0 + cn*nb1/nb0], tmp + (cn*16), (MIN(iir0 + blck_0, ir011) - iir0)*sizeof(float)); + for (int cn = 0; cn < num_rows_per_vec_dot; ++cn) { + memcpy(&dst_col[iir0 + cn*nb1/nb0], tmp + (cn*16), (MIN(iir0 + blck_0, ir0_end) - iir0)*sizeof(float)); } } } From 7b932e4908748c7fdf7dd2ac0dfc70049757aae5 Mon Sep 17 00:00:00 2001 From: Kunnis Date: Wed, 8 May 2024 23:58:59 -0500 Subject: [PATCH 05/27] Extracting the logic to it's own function. --- ggml.c | 136 ++++++++++++++++++++++++++++++++++----------------------- 1 file changed, 82 insertions(+), 54 deletions(-) diff --git a/ggml.c b/ggml.c index 89c6bc3323198..23ecc75355f0e 100644 --- a/ggml.c +++ b/ggml.c @@ -11769,6 +11769,87 @@ static bool ggml_compute_forward_mul_mat_use_blas(struct ggml_tensor * dst) { } #endif +static void ggml_compute_forward_mul_mat_one_chunk( + const struct ggml_compute_params* params, + struct ggml_tensor* dst, + const int64_t num_rows_per_vec_dot, + const int64_t ir0_start, + const int64_t ir0_end, + const int64_t ir1_start, + const int64_t ir1_end, + + const bool src1_cont, + const int64_t r2, + const int64_t r3, + enum ggml_type vec_dot_type, + const void* wdata, + const size_t row_size, + const ggml_vec_dot_t vec_dot +) { + + const struct ggml_tensor* src0 = dst->src[0]; + const struct ggml_tensor* src1 = dst->src[1]; + + GGML_TENSOR_BINARY_OP_LOCALS + + const enum ggml_type type = src0->type; + + assert(ne12 % ne02 == 0); + assert(ne13 % ne03 == 0); + + // block-tiling attempt + const int64_t blck_0 = 16; + const int64_t blck_1 = 16; + + const size_t src1_col_stride = src1_cont || src1->type != vec_dot_type ? row_size : nb11; + + // attempt to reduce false-sharing (does not seem to make a difference) + // 16 * 2, accounting for mmla kernels + float tmp[32]; + + for (int64_t iir1 = ir1_start; iir1 < ir1_end; iir1 += blck_1) { + for (int64_t iir0 = ir0_start; iir0 < ir0_end; iir0 += blck_0) { + for (int64_t ir1 = iir1; ir1 < iir1 + blck_1 && ir1 < ir1_end; ir1 += num_rows_per_vec_dot) { + const int64_t i13 = (ir1 / (ne12 * ne1)); + const int64_t i12 = (ir1 - i13 * ne12 * ne1) / ne1; + const int64_t i11 = (ir1 - i13 * ne12 * ne1 - i12 * ne1); + + // broadcast src0 into src1 + const int64_t i03 = i13 / r3; + const int64_t i02 = i12 / r2; + + const int64_t i1 = i11; + const int64_t i2 = i12; + const int64_t i3 = i13; + + const char* src0_row = (const char*)src0->data + (0 + i02 * nb02 + i03 * nb03); + + // desc: when src1 is not a contiguous memory block we have to calculate the offset using the strides + // if it is, then we have either copied the data to params->wdata and made it contiguous or we are using + // the original src1 data pointer, so we should index using the indices directly + // TODO: this is a bit of a hack, we should probably have a better way to handle this + const char* src1_col = (const char*)wdata + + (src1_cont || src1->type != vec_dot_type + ? (i11 + i12 * ne11 + i13 * ne12 * ne11) * row_size + : (i11 * nb11 + i12 * nb12 + i13 * nb13)); + float* dst_col = (float*)((char*)dst->data + (i1 * nb1 + i2 * nb2 + i3 * nb3)); + + //for (int64_t ir0 = iir0; ir0 < iir0 + blck_0 && ir0 < ir0_end; ++ir0) { + // vec_dot(ne00, &dst_col[ir0], src0_row + ir0*nb01, src1_col); + //} + + for (int64_t ir0 = iir0; ir0 < iir0 + blck_0 && ir0 < ir0_end; ir0 += num_rows_per_vec_dot) { + vec_dot(ne00, &tmp[ir0 - iir0], (num_rows_per_vec_dot > 1 ? 16 : 0), src0_row + ir0 * nb01, (num_rows_per_vec_dot > 1 ? nb01 : 0), src1_col, (num_rows_per_vec_dot > 1 ? src1_col_stride : 0), num_rows_per_vec_dot); + } + + for (int cn = 0; cn < num_rows_per_vec_dot; ++cn) { + memcpy(&dst_col[iir0 + cn * nb1 / nb0], tmp + (cn * 16), (MIN(iir0 + blck_0, ir0_end) - iir0) * sizeof(float)); + } + } + } + } +} + static void ggml_compute_forward_mul_mat( const struct ggml_compute_params * params, struct ggml_tensor * dst, @@ -12005,60 +12086,7 @@ UseGgmlGemm2:; num_rows_per_vec_dot = 1; } - assert(ne12% ne02 == 0); - assert(ne13% ne03 == 0); - - // block-tiling attempt - const int64_t blck_0 = 16; - const int64_t blck_1 = 16; - - const size_t src1_col_stride = src1_cont || src1->type != vec_dot_type ? row_size : nb11; - - // attempt to reduce false-sharing (does not seem to make a difference) - // 16 * 2, accounting for mmla kernels - float tmp[32]; - - for (int64_t iir1 = ir1_start; iir1 < ir1_end; iir1 += blck_1) { - for (int64_t iir0 = ir0_start; iir0 < ir0_end; iir0 += blck_0) { - for (int64_t ir1 = iir1; ir1 < iir1 + blck_1 && ir1 < ir1_end; ir1 += num_rows_per_vec_dot) { - const int64_t i13 = (ir1/(ne12*ne1)); - const int64_t i12 = (ir1 - i13*ne12*ne1)/ne1; - const int64_t i11 = (ir1 - i13*ne12*ne1 - i12*ne1); - - // broadcast src0 into src1 - const int64_t i03 = i13/r3; - const int64_t i02 = i12/r2; - - const int64_t i1 = i11; - const int64_t i2 = i12; - const int64_t i3 = i13; - - const char * src0_row = (const char *) src0->data + (0 + i02*nb02 + i03*nb03); - - // desc: when src1 is not a contiguous memory block we have to calculate the offset using the strides - // if it is, then we have either copied the data to params->wdata and made it contiguous or we are using - // the original src1 data pointer, so we should index using the indices directly - // TODO: this is a bit of a hack, we should probably have a better way to handle this - const char * src1_col = (const char *) wdata + - (src1_cont || src1->type != vec_dot_type - ? (i11 + i12*ne11 + i13*ne12*ne11)*row_size - : (i11*nb11 + i12*nb12 + i13*nb13)); - float * dst_col = (float *) ((char *) dst->data + (i1*nb1 + i2*nb2 + i3*nb3)); - - //for (int64_t ir0 = iir0; ir0 < iir0 + blck_0 && ir0 < ir0_end; ++ir0) { - // vec_dot(ne00, &dst_col[ir0], src0_row + ir0*nb01, src1_col); - //} - - for (int64_t ir0 = iir0; ir0 < iir0 + blck_0 && ir0 < ir0_end; ir0 += num_rows_per_vec_dot) { - vec_dot(ne00, &tmp[ir0 - iir0], (num_rows_per_vec_dot >1 ? 16 : 0), src0_row + ir0*nb01, (num_rows_per_vec_dot >1 ? nb01 : 0), src1_col, (num_rows_per_vec_dot >1 ? src1_col_stride : 0), num_rows_per_vec_dot); - } - - for (int cn = 0; cn < num_rows_per_vec_dot; ++cn) { - memcpy(&dst_col[iir0 + cn*nb1/nb0], tmp + (cn*16), (MIN(iir0 + blck_0, ir0_end) - iir0)*sizeof(float)); - } - } - } - } + ggml_compute_forward_mul_mat_one_chunk(params, dst, num_rows_per_vec_dot, ir0_start, ir0_end, ir1_start, ir1_end, src1_cont, r2, r3, vec_dot_type, wdata, row_size, vec_dot); } // ggml_compute_forward_mul_mat_id From 4f95478ea3d4ec370efada288c6e1345778d425d Mon Sep 17 00:00:00 2001 From: Kunnis Date: Thu, 9 May 2024 00:07:40 -0500 Subject: [PATCH 06/27] Moving some variable definitions into the chunk function. --- ggml.c | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/ggml.c b/ggml.c index 23ecc75355f0e..3b5e0719af395 100644 --- a/ggml.c +++ b/ggml.c @@ -11779,10 +11779,7 @@ static void ggml_compute_forward_mul_mat_one_chunk( const int64_t ir1_end, const bool src1_cont, - const int64_t r2, - const int64_t r3, enum ggml_type vec_dot_type, - const void* wdata, const size_t row_size, const ggml_vec_dot_t vec_dot ) { @@ -11794,6 +11791,14 @@ static void ggml_compute_forward_mul_mat_one_chunk( const enum ggml_type type = src0->type; + const void* wdata = (src1->type == vec_dot_type) ? src1->data : params->wdata; + + // broadcast factors + const int64_t r2 = ne12 / ne02; + const int64_t r3 = ne13 / ne03; + + //printf("ir0_start = %6lld, ir0_end = %6lld, ir1_start = %6lld, ir1_end = %6lld\n", ir0_start, ir0_end, ir1_start, ir1_end); + assert(ne12 % ne02 == 0); assert(ne13 % ne03 == 0); @@ -11891,8 +11896,8 @@ static void ggml_compute_forward_mul_mat( GGML_ASSERT(nb2 <= nb3); // broadcast factors - const int64_t r2 = ne12/ne02; - const int64_t r3 = ne13/ne03; + const int64_t r2 = ne12 / ne02; + const int64_t r3 = ne13 / ne03; // nb01 >= nb00 - src0 is not transposed // compute by src0 rows @@ -12070,8 +12075,6 @@ UseGgmlGemm2:; const int64_t ir1_start = dr1*ith1; const int64_t ir1_end = MIN(ir1_start + dr1, nr1); - //printf("ir0_start = %6lld, ir0_end = %6lld, ir1_start = %6lld, ir1_end = %6lld\n", ir0_start, ir0_end, ir1_start, ir1_end); - // threads with no work simply yield (not sure if it helps) if (ir0_start >= ir0_end || ir1_start >= ir1_end) { sched_yield(); @@ -12086,7 +12089,7 @@ UseGgmlGemm2:; num_rows_per_vec_dot = 1; } - ggml_compute_forward_mul_mat_one_chunk(params, dst, num_rows_per_vec_dot, ir0_start, ir0_end, ir1_start, ir1_end, src1_cont, r2, r3, vec_dot_type, wdata, row_size, vec_dot); + ggml_compute_forward_mul_mat_one_chunk(params, dst, num_rows_per_vec_dot, ir0_start, ir0_end, ir1_start, ir1_end, src1_cont, vec_dot_type, row_size, vec_dot); } // ggml_compute_forward_mul_mat_id From 086e5a8225949ca95de8828cc7df4ec0372203da Mon Sep 17 00:00:00 2001 From: Kunnis Date: Thu, 9 May 2024 21:49:00 -0500 Subject: [PATCH 07/27] Moving some variables around --- ggml.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ggml.c b/ggml.c index 3b5e0719af395..e0cca5854ff13 100644 --- a/ggml.c +++ b/ggml.c @@ -11779,9 +11779,7 @@ static void ggml_compute_forward_mul_mat_one_chunk( const int64_t ir1_end, const bool src1_cont, - enum ggml_type vec_dot_type, - const size_t row_size, - const ggml_vec_dot_t vec_dot + const size_t row_size ) { const struct ggml_tensor* src0 = dst->src[0]; @@ -11791,7 +11789,8 @@ static void ggml_compute_forward_mul_mat_one_chunk( const enum ggml_type type = src0->type; - const void* wdata = (src1->type == vec_dot_type) ? src1->data : params->wdata; + ggml_vec_dot_t const vec_dot = type_traits[type].vec_dot; + enum ggml_type const vec_dot_type = type_traits[type].vec_dot_type; // broadcast factors const int64_t r2 = ne12 / ne02; @@ -11799,6 +11798,8 @@ static void ggml_compute_forward_mul_mat_one_chunk( //printf("ir0_start = %6lld, ir0_end = %6lld, ir1_start = %6lld, ir1_end = %6lld\n", ir0_start, ir0_end, ir1_start, ir1_end); + const void* wdata = (src1->type == vec_dot_type) ? src1->data : params->wdata; + assert(ne12 % ne02 == 0); assert(ne13 % ne03 == 0); @@ -11875,7 +11876,6 @@ static void ggml_compute_forward_mul_mat( const bool src1_cont = ggml_is_contiguous(src1); - ggml_vec_dot_t const vec_dot = type_traits[type].vec_dot; enum ggml_type const vec_dot_type = type_traits[type].vec_dot_type; ggml_from_float_t const from_float_to_vec_dot = type_traits[vec_dot_type].from_float; int64_t const vec_dot_num_rows = type_traits[type].nrows; @@ -12089,7 +12089,7 @@ UseGgmlGemm2:; num_rows_per_vec_dot = 1; } - ggml_compute_forward_mul_mat_one_chunk(params, dst, num_rows_per_vec_dot, ir0_start, ir0_end, ir1_start, ir1_end, src1_cont, vec_dot_type, row_size, vec_dot); + ggml_compute_forward_mul_mat_one_chunk(params, dst, num_rows_per_vec_dot, ir0_start, ir0_end, ir1_start, ir1_end, src1_cont, row_size); } // ggml_compute_forward_mul_mat_id From 209922f5ac1828daa8f5ff181b469f7b352cc8b5 Mon Sep 17 00:00:00 2001 From: Kunnis Date: Thu, 9 May 2024 21:53:27 -0500 Subject: [PATCH 08/27] moving src1_cont inside --- ggml.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ggml.c b/ggml.c index e0cca5854ff13..0fc56d5e6f947 100644 --- a/ggml.c +++ b/ggml.c @@ -11778,7 +11778,6 @@ static void ggml_compute_forward_mul_mat_one_chunk( const int64_t ir1_start, const int64_t ir1_end, - const bool src1_cont, const size_t row_size ) { @@ -11789,6 +11788,8 @@ static void ggml_compute_forward_mul_mat_one_chunk( const enum ggml_type type = src0->type; + const bool src1_cont = ggml_is_contiguous(src1); + ggml_vec_dot_t const vec_dot = type_traits[type].vec_dot; enum ggml_type const vec_dot_type = type_traits[type].vec_dot_type; @@ -12089,7 +12090,7 @@ UseGgmlGemm2:; num_rows_per_vec_dot = 1; } - ggml_compute_forward_mul_mat_one_chunk(params, dst, num_rows_per_vec_dot, ir0_start, ir0_end, ir1_start, ir1_end, src1_cont, row_size); + ggml_compute_forward_mul_mat_one_chunk(params, dst, num_rows_per_vec_dot, ir0_start, ir0_end, ir1_start, ir1_end, row_size); } // ggml_compute_forward_mul_mat_id From bb1b1d0071ffd2a9a2491fe5b0c5e89b3294e764 Mon Sep 17 00:00:00 2001 From: Kunnis Date: Thu, 9 May 2024 21:58:36 -0500 Subject: [PATCH 09/27] Moving row_size --- ggml.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/ggml.c b/ggml.c index 0fc56d5e6f947..32eeb247fadaf 100644 --- a/ggml.c +++ b/ggml.c @@ -11776,10 +11776,7 @@ static void ggml_compute_forward_mul_mat_one_chunk( const int64_t ir0_start, const int64_t ir0_end, const int64_t ir1_start, - const int64_t ir1_end, - - const size_t row_size -) { + const int64_t ir1_end) { const struct ggml_tensor* src0 = dst->src[0]; const struct ggml_tensor* src1 = dst->src[1]; @@ -11800,6 +11797,7 @@ static void ggml_compute_forward_mul_mat_one_chunk( //printf("ir0_start = %6lld, ir0_end = %6lld, ir1_start = %6lld, ir1_end = %6lld\n", ir0_start, ir0_end, ir1_start, ir1_end); const void* wdata = (src1->type == vec_dot_type) ? src1->data : params->wdata; + const size_t row_size = ggml_row_size(vec_dot_type, ne10); assert(ne12 % ne02 == 0); assert(ne13 % ne03 == 0); @@ -12090,7 +12088,7 @@ UseGgmlGemm2:; num_rows_per_vec_dot = 1; } - ggml_compute_forward_mul_mat_one_chunk(params, dst, num_rows_per_vec_dot, ir0_start, ir0_end, ir1_start, ir1_end, row_size); + ggml_compute_forward_mul_mat_one_chunk(params, dst, num_rows_per_vec_dot, ir0_start, ir0_end, ir1_start, ir1_end); } // ggml_compute_forward_mul_mat_id From daa87b1813dec45c36635f8358719b2d2ba83893 Mon Sep 17 00:00:00 2001 From: Kunnis Date: Thu, 9 May 2024 22:12:20 -0500 Subject: [PATCH 10/27] adding the current_chunk --- ggml.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ggml.c b/ggml.c index 32eeb247fadaf..6d368185ae203 100644 --- a/ggml.c +++ b/ggml.c @@ -1578,6 +1578,8 @@ struct ggml_compute_state_shared { ggml_abort_callback abort_callback; // abort ggml_graph_compute when true void* abort_callback_data; + + atomic_int current_chunk; // currently processing chunk during Mat_Mul, shared between all the threads. }; struct ggml_compute_state { @@ -12003,6 +12005,8 @@ UseGgmlGemm1:; if (ith != 0) { return; } + //Every thread starts at ith, so the first unprocessed chunk is nth. This save a bit of coordination right at the start. + atomic_store(&state->shared->current_chunk, nth); if (src1->type != vec_dot_type) { char * wdata = params->wdata; const size_t row_size = ggml_row_size(vec_dot_type, ne10); From 700c782dc14972a34389fffcfc4783dd0db87ac5 Mon Sep 17 00:00:00 2001 From: Kunnis Date: Thu, 9 May 2024 22:18:26 -0500 Subject: [PATCH 11/27] Reorg the code. --- ggml.c | 35 +++++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/ggml.c b/ggml.c index 6d368185ae203..e316c2ba1d352 100644 --- a/ggml.c +++ b/ggml.c @@ -12056,8 +12056,24 @@ UseGgmlGemm1:; UseGgmlGemm2:; #endif - const int64_t nr0 = ne01; // src0 rows - const int64_t nr1 = ne1*ne12*ne13; // src1 rows +#ifdef GGML_PERF + int chunks_executed = 0; + UNUSED(chunks_executed); +#endif + + //This is the size of the first dimension of the result, so we can iterate that way. (see the ASSERT above, these are the same numbers) + const int64_t nr0 = ne0; + + //This is the size of the rest of the dimensions of the result + const int64_t nr1 = ne1 * ne2 * ne3; + + // dot kernels can handle 1 row and col at a time, but mmla kernels can process 2 rows and cols + int64_t num_rows_per_vec_dot = vec_dot_num_rows; + // TODO: currently the mmla kernels support only even numbered rows/cols. + // this check can be removed once they are extended to support odd numbered rows/cols too + if ((nr0 % 2 != 0) || (ne11 % 2 != 0)) { + num_rows_per_vec_dot = 1; + } //printf("nr0 = %lld, nr1 = %lld\n", nr0, nr1); @@ -12084,15 +12100,14 @@ UseGgmlGemm2:; return; } - // dot kernels can handle 1 row and col at a time, but mmla kernels can process 2 rows and cols - int64_t num_rows_per_vec_dot = vec_dot_num_rows; - // TODO: currently the mmla kernels support only even numbered rows/cols. - // this check can be removed once they are extended to support odd numbered rows/cols too - if ((nr0 % 2 != 0) || (ne11 % 2 != 0)) { - num_rows_per_vec_dot = 1; - } - ggml_compute_forward_mul_mat_one_chunk(params, dst, num_rows_per_vec_dot, ir0_start, ir0_end, ir1_start, ir1_end); + +#ifdef GGML_PERF + // These numbers are useful when trying to measure how well the threading scheduling works. + //int64_t workSize = (ne01 * ne11 * ne12 * ne13 * ne00) / nchunk0 / nchunk1; + //float time = (ggml_perf_time_us() - t0); + //printf("MUL_MAT = %f ms, [%d, %d, %d, %d] x [%d, %d, %d, %d] = %I64u, %f ops/usec in %d chunks.\n", time / 1000.0, ne00, ne01, ne02, ne03, ne10, ne11, ne12, ne13, workSize, (float)workSize/time, chunks_executed); +#endif } // ggml_compute_forward_mul_mat_id From 891d58371166a835a72c2970238c32ef63eb5aa3 Mon Sep 17 00:00:00 2001 From: Kunnis Date: Thu, 9 May 2024 22:25:20 -0500 Subject: [PATCH 12/27] Formatting to match the orig patch --- ggml.c | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/ggml.c b/ggml.c index e316c2ba1d352..645b11e14f0e5 100644 --- a/ggml.c +++ b/ggml.c @@ -2435,6 +2435,7 @@ static void ggml_setup_op_has_task_pass(void) { } } + // // NUMA support // @@ -11798,6 +11799,11 @@ static void ggml_compute_forward_mul_mat_one_chunk( //printf("ir0_start = %6lld, ir0_end = %6lld, ir1_start = %6lld, ir1_end = %6lld\n", ir0_start, ir0_end, ir1_start, ir1_end); + // threads with no work simply yield (not sure if it helps) + if (ir0_start >= ir0_end || ir1_start >= ir1_end) { + return; + } + const void* wdata = (src1->type == vec_dot_type) ? src1->data : params->wdata; const size_t row_size = ggml_row_size(vec_dot_type, ne10); @@ -11860,7 +11866,7 @@ static void ggml_compute_forward_mul_mat_one_chunk( static void ggml_compute_forward_mul_mat( const struct ggml_compute_params * params, struct ggml_tensor * dst, - struct ggml_compute_state* state) { + struct ggml_compute_state * state) { const struct ggml_tensor * src0 = dst->src[0]; const struct ggml_tensor * src1 = dst->src[1]; @@ -11897,8 +11903,8 @@ static void ggml_compute_forward_mul_mat( GGML_ASSERT(nb2 <= nb3); // broadcast factors - const int64_t r2 = ne12 / ne02; - const int64_t r3 = ne13 / ne03; + const int64_t r2 = ne12/ne02; + const int64_t r3 = ne13/ne03; // nb01 >= nb00 - src0 is not transposed // compute by src0 rows @@ -19617,17 +19623,17 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads, int n_cur_ return n_tasks; } -static void ggml_graph_compute_thread_sync_node(int * node_n, struct ggml_compute_state * state, const bool do_yield) { +static void ggml_graph_compute_thread_sync_node(int* node_n, struct ggml_compute_state* state, const bool do_yield) { // wait for other threads to finish - const int last_node_n = * node_n; + const int last_node_n = *node_n; while (true) { if (do_yield) { sched_yield(); } - * node_n = atomic_load(&state->shared->node_n); - if (* node_n != last_node_n) break; + *node_n = atomic_load(&state->shared->node_n); + if (*node_n != last_node_n) break; #if defined(__SSE3__) //Tell the processor we're spinning. It's a processor hint for spinlocks. _mm_pause(); @@ -19635,17 +19641,17 @@ static void ggml_graph_compute_thread_sync_node(int * node_n, struct ggml_comput } } -static void ggml_graph_compute_thread_sync_task(int * task_phase, struct ggml_compute_state * state, const bool do_yield) { +static void ggml_graph_compute_thread_sync_task(int* task_phase, struct ggml_compute_state* state, const bool do_yield) { // wait for other threads to finish - const int last_task_phase = * task_phase; + const int last_task_phase = *task_phase; while (true) { if (do_yield) { sched_yield(); } - * task_phase = atomic_load(&state->shared->node_task); - if (* task_phase != last_task_phase) break; + *task_phase = atomic_load(&state->shared->node_task); + if (*task_phase != last_task_phase) break; #if defined(__SSE3__) //Tell the processor we're spinning. It's a processor hint for spinlocks. _mm_pause(); @@ -20030,6 +20036,7 @@ enum ggml_status ggml_graph_compute(struct ggml_cgraph * cgraph, struct ggml_cpl /*.node_task =*/ GGML_TASK_TYPE_FINALIZE, /*.abort_callback =*/ NULL, /*.abort_callback_data =*/ NULL, + /*.current_chunk; =*/ 0, }; struct ggml_compute_state * workers = alloca(sizeof(struct ggml_compute_state)*n_threads); From 9acaec58bd24425c80d99e09b75fae3a026fb7dc Mon Sep 17 00:00:00 2001 From: Kunnis Date: Thu, 9 May 2024 22:31:43 -0500 Subject: [PATCH 13/27] starting to setup the chunking variables --- ggml.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ggml.c b/ggml.c index 645b11e14f0e5..dc4aed7ea3ab7 100644 --- a/ggml.c +++ b/ggml.c @@ -12062,6 +12062,9 @@ UseGgmlGemm1:; UseGgmlGemm2:; #endif + int chunk_size = 16; + UNUSED(chunk_size); + #ifdef GGML_PERF int chunks_executed = 0; UNUSED(chunks_executed); From c0557fa20a937b429db0e33c4b96f5d5c381b446 Mon Sep 17 00:00:00 2001 From: Kunnis Date: Thu, 9 May 2024 23:22:12 -0500 Subject: [PATCH 14/27] Starting the buildup of the loop --- ggml.c | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/ggml.c b/ggml.c index dc4aed7ea3ab7..f53ca24eb19ad 100644 --- a/ggml.c +++ b/ggml.c @@ -12088,20 +12088,23 @@ UseGgmlGemm2:; // distribute the thread work across the inner or outer loop based on which one is larger - const int64_t nth0 = nr0 > nr1 ? nth : 1; // parallelize by src0 rows - const int64_t nth1 = nr0 > nr1 ? 1 : nth; // parallelize by src1 rows + const int64_t nchunk0 = nr0 > nr1 ? nth : 1; // parallelize by src0 rows + const int64_t nchunk1 = nr0 > nr1 ? 1 : nth; // parallelize by src1 rows - const int64_t ith0 = ith % nth0; - const int64_t ith1 = ith / nth0; + int current_chunk = ith; - const int64_t dr0 = (nr0 + nth0 - 1)/nth0; - const int64_t dr1 = (nr1 + nth1 - 1)/nth1; + { + const int64_t ith0 = current_chunk % nchunk0; + const int64_t ith1 = current_chunk / nchunk0; + + const int64_t dr0 = (nr0 + nchunk0 - 1) / nchunk0; + const int64_t dr1 = (nr1 + nchunk1 - 1) / nchunk1; - const int64_t ir0_start = dr0*ith0; - const int64_t ir0_end = MIN(ir0_start + dr0, nr0); + const int64_t ir0_start = dr0 * ith0; + const int64_t ir0_end = MIN(ir0_start + dr0, nr0); - const int64_t ir1_start = dr1*ith1; - const int64_t ir1_end = MIN(ir1_start + dr1, nr1); + const int64_t ir1_start = dr1 * ith1; + const int64_t ir1_end = MIN(ir1_start + dr1, nr1); // threads with no work simply yield (not sure if it helps) if (ir0_start >= ir0_end || ir1_start >= ir1_end) { @@ -12109,7 +12112,13 @@ UseGgmlGemm2:; return; } - ggml_compute_forward_mul_mat_one_chunk(params, dst, num_rows_per_vec_dot, ir0_start, ir0_end, ir1_start, ir1_end); + ggml_compute_forward_mul_mat_one_chunk(params, dst, num_rows_per_vec_dot, ir0_start, ir0_end, ir1_start, ir1_end); + +#ifdef GGML_PERF + chunks_executed++; +#endif + + } #ifdef GGML_PERF // These numbers are useful when trying to measure how well the threading scheduling works. From 4762d79d3d8af7e8babdd594ff1205a0e8dc3e53 Mon Sep 17 00:00:00 2001 From: Kunnis Date: Thu, 9 May 2024 23:23:13 -0500 Subject: [PATCH 15/27] The yield shouldn't be necessary. --- ggml.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/ggml.c b/ggml.c index f53ca24eb19ad..896f540ea448a 100644 --- a/ggml.c +++ b/ggml.c @@ -12106,12 +12106,6 @@ UseGgmlGemm2:; const int64_t ir1_start = dr1 * ith1; const int64_t ir1_end = MIN(ir1_start + dr1, nr1); - // threads with no work simply yield (not sure if it helps) - if (ir0_start >= ir0_end || ir1_start >= ir1_end) { - sched_yield(); - return; - } - ggml_compute_forward_mul_mat_one_chunk(params, dst, num_rows_per_vec_dot, ir0_start, ir0_end, ir1_start, ir1_end); #ifdef GGML_PERF From fc7dc515f187e96ecd0c6e3ac906c6c5faf357c2 Mon Sep 17 00:00:00 2001 From: Kunnis Date: Thu, 9 May 2024 23:29:49 -0500 Subject: [PATCH 16/27] adding the looping structure based on the chunk configuration. --- ggml.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ggml.c b/ggml.c index 896f540ea448a..f04a11b691f57 100644 --- a/ggml.c +++ b/ggml.c @@ -12091,8 +12091,10 @@ UseGgmlGemm2:; const int64_t nchunk0 = nr0 > nr1 ? nth : 1; // parallelize by src0 rows const int64_t nchunk1 = nr0 > nr1 ? 1 : nth; // parallelize by src1 rows + //The first chunk comes from our thread_id, the rest will get auto-assigned. int current_chunk = ith; + while (current_chunk < nchunk0 * nchunk1) { const int64_t ith0 = current_chunk % nchunk0; const int64_t ith1 = current_chunk / nchunk0; @@ -12112,6 +12114,10 @@ UseGgmlGemm2:; chunks_executed++; #endif + if (nth >= nchunk0 * nchunk1) + break; + + current_chunk = atomic_fetch_add(&state->shared->current_chunk, 1); } #ifdef GGML_PERF From 807c8252ced16419e623c27299055abb2dea9fe7 Mon Sep 17 00:00:00 2001 From: Kunnis Date: Thu, 9 May 2024 23:50:37 -0500 Subject: [PATCH 17/27] Add in the re-chunking code. --- ggml.c | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/ggml.c b/ggml.c index f04a11b691f57..b6f90300aa5eb 100644 --- a/ggml.c +++ b/ggml.c @@ -12062,9 +12062,6 @@ UseGgmlGemm1:; UseGgmlGemm2:; #endif - int chunk_size = 16; - UNUSED(chunk_size); - #ifdef GGML_PERF int chunks_executed = 0; UNUSED(chunks_executed); @@ -12084,12 +12081,32 @@ UseGgmlGemm2:; num_rows_per_vec_dot = 1; } + //Now select a reasonable chunk size. + int chunk_size = 16; + + //We need to step up the size if it's small + if (nr0 == 1 || nr1 == 1) + chunk_size = 64; + + // distribute the work across the inner or outer loop based on which one is larger + //The number of chunks in the 0/1 dim. + //CEIL(nr0/chunk_size) + int64_t nchunk0 = (nr0 + chunk_size - 1) / chunk_size; + int64_t nchunk1 = (nr1 + chunk_size - 1) / chunk_size; + //printf("nr0 = %lld, nr1 = %lld\n", nr0, nr1); - // distribute the thread work across the inner or outer loop based on which one is larger + //If the chunking is poor for the number of threads on this setup, scrap the whole plan. Re-chunk it by thread. + if (nchunk0 * nchunk1 < nth * 4) + { + // distribute the thread work across the inner or outer loop based on which one is larger + nchunk0 = nr0 > nr1 ? nth : 1; // parallelize by src0 rows + nchunk1 = nr0 > nr1 ? 1 : nth; // parallelize by src1 rows + } - const int64_t nchunk0 = nr0 > nr1 ? nth : 1; // parallelize by src0 rows - const int64_t nchunk1 = nr0 > nr1 ? 1 : nth; // parallelize by src1 rows + //The number of elements in each chunk + const int64_t dr0 = (nr0 + nchunk0 - 1) / nchunk0; + const int64_t dr1 = (nr1 + nchunk1 - 1) / nchunk1; //The first chunk comes from our thread_id, the rest will get auto-assigned. int current_chunk = ith; @@ -12099,9 +12116,6 @@ UseGgmlGemm2:; const int64_t ith0 = current_chunk % nchunk0; const int64_t ith1 = current_chunk / nchunk0; - const int64_t dr0 = (nr0 + nchunk0 - 1) / nchunk0; - const int64_t dr1 = (nr1 + nchunk1 - 1) / nchunk1; - const int64_t ir0_start = dr0 * ith0; const int64_t ir0_end = MIN(ir0_start + dr0, nr0); From 974e43be2547873e5da63f0aec4a2ff7e0b02041 Mon Sep 17 00:00:00 2001 From: Kunnis Date: Fri, 10 May 2024 15:17:33 -0500 Subject: [PATCH 18/27] Making it much more likely to rechunk. --- ggml.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ggml.c b/ggml.c index b6f90300aa5eb..bcca5162ddd9c 100644 --- a/ggml.c +++ b/ggml.c @@ -12097,8 +12097,10 @@ UseGgmlGemm2:; //printf("nr0 = %lld, nr1 = %lld\n", nr0, nr1); //If the chunking is poor for the number of threads on this setup, scrap the whole plan. Re-chunk it by thread. - if (nchunk0 * nchunk1 < nth * 4) + if (nchunk0 * nchunk1 < nth * 400) { + //if (ith == 0) + // printf("rechunked"); // distribute the thread work across the inner or outer loop based on which one is larger nchunk0 = nr0 > nr1 ? nth : 1; // parallelize by src0 rows nchunk1 = nr0 > nr1 ? 1 : nth; // parallelize by src1 rows @@ -12108,6 +12110,9 @@ UseGgmlGemm2:; const int64_t dr0 = (nr0 + nchunk0 - 1) / nchunk0; const int64_t dr1 = (nr1 + nchunk1 - 1) / nchunk1; + //if (ith == 0) + // printf("MUL_MAT = [%d, %d, %d, %d] x [%d, %d, %d, %d] = %d x %d = %d. Fp/Ch %d\n", ne00, ne01, ne02, ne03, ne10, ne11, ne12, ne13, nchunk0, nchunk1, nchunk0 * nchunk1, ne00 * nr0 * nr1 / nchunk0 / nchunk1); + //The first chunk comes from our thread_id, the rest will get auto-assigned. int current_chunk = ith; From 1c68ea8d9f01169a82a18c03c0a8e3cad7911631 Mon Sep 17 00:00:00 2001 From: Kunnis Date: Fri, 10 May 2024 16:05:07 -0500 Subject: [PATCH 19/27] disable resizing if numa is enabled. --- ggml.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ggml.c b/ggml.c index bcca5162ddd9c..6182dbd2d5028 100644 --- a/ggml.c +++ b/ggml.c @@ -12097,7 +12097,7 @@ UseGgmlGemm2:; //printf("nr0 = %lld, nr1 = %lld\n", nr0, nr1); //If the chunking is poor for the number of threads on this setup, scrap the whole plan. Re-chunk it by thread. - if (nchunk0 * nchunk1 < nth * 400) + if (nchunk0 * nchunk1 < nth * 4 || ggml_is_numa()) { //if (ith == 0) // printf("rechunked"); From bd80601ea8eb4bc7d6135dfa4214d3c67e6e7f87 Mon Sep 17 00:00:00 2001 From: Kunnis Date: Fri, 10 May 2024 17:30:37 -0500 Subject: [PATCH 20/27] Updating comments with what we've learned. --- ggml.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/ggml.c b/ggml.c index 6182dbd2d5028..1a6ab13cda6f0 100644 --- a/ggml.c +++ b/ggml.c @@ -12094,13 +12094,11 @@ UseGgmlGemm2:; int64_t nchunk0 = (nr0 + chunk_size - 1) / chunk_size; int64_t nchunk1 = (nr1 + chunk_size - 1) / chunk_size; - //printf("nr0 = %lld, nr1 = %lld\n", nr0, nr1); - //If the chunking is poor for the number of threads on this setup, scrap the whole plan. Re-chunk it by thread. + // Also, chunking by thread was measured to have perform better on NUMA systems. See https://github.com/ggerganov/llama.cpp/pull/6915 + // In theory, chunking should be just as useful on NUMA and non NUMA systems, but testing disagreed with that. if (nchunk0 * nchunk1 < nth * 4 || ggml_is_numa()) { - //if (ith == 0) - // printf("rechunked"); // distribute the thread work across the inner or outer loop based on which one is larger nchunk0 = nr0 > nr1 ? nth : 1; // parallelize by src0 rows nchunk1 = nr0 > nr1 ? 1 : nth; // parallelize by src1 rows @@ -12111,7 +12109,7 @@ UseGgmlGemm2:; const int64_t dr1 = (nr1 + nchunk1 - 1) / nchunk1; //if (ith == 0) - // printf("MUL_MAT = [%d, %d, %d, %d] x [%d, %d, %d, %d] = %d x %d = %d. Fp/Ch %d\n", ne00, ne01, ne02, ne03, ne10, ne11, ne12, ne13, nchunk0, nchunk1, nchunk0 * nchunk1, ne00 * nr0 * nr1 / nchunk0 / nchunk1); + // printf("MUL_MAT = [%d, %d, %d, %d] x [%d, %d, %d, %d] = %d x %d = %d. Fp Ops/Ch %d\n", ne00, ne01, ne02, ne03, ne10, ne11, ne12, ne13, nchunk0, nchunk1, nchunk0 * nchunk1, ne00 * nr0 * nr1 / nchunk0 / nchunk1); //The first chunk comes from our thread_id, the rest will get auto-assigned. int current_chunk = ith; From d9ba30a2046615e04dc9286e31d54652c6cd6ad7 Mon Sep 17 00:00:00 2001 From: Kunnis Date: Tue, 14 May 2024 17:15:47 -0500 Subject: [PATCH 21/27] Fix formatting --- ggml.c | 55 +++++++++++++++++++++++++++---------------------------- 1 file changed, 27 insertions(+), 28 deletions(-) diff --git a/ggml.c b/ggml.c index 1a6ab13cda6f0..c0d498014b752 100644 --- a/ggml.c +++ b/ggml.c @@ -2435,7 +2435,6 @@ static void ggml_setup_op_has_task_pass(void) { } } - // // NUMA support // @@ -11773,16 +11772,16 @@ static bool ggml_compute_forward_mul_mat_use_blas(struct ggml_tensor * dst) { #endif static void ggml_compute_forward_mul_mat_one_chunk( - const struct ggml_compute_params* params, - struct ggml_tensor* dst, + const struct ggml_compute_params * params, + struct ggml_tensor * dst, const int64_t num_rows_per_vec_dot, const int64_t ir0_start, const int64_t ir0_end, const int64_t ir1_start, const int64_t ir1_end) { - const struct ggml_tensor* src0 = dst->src[0]; - const struct ggml_tensor* src1 = dst->src[1]; + const struct ggml_tensor * src0 = dst->src[0]; + const struct ggml_tensor * src1 = dst->src[1]; GGML_TENSOR_BINARY_OP_LOCALS @@ -11804,7 +11803,7 @@ static void ggml_compute_forward_mul_mat_one_chunk( return; } - const void* wdata = (src1->type == vec_dot_type) ? src1->data : params->wdata; + const void * wdata = (src1->type == vec_dot_type) ? src1->data : params->wdata; const size_t row_size = ggml_row_size(vec_dot_type, ne10); assert(ne12 % ne02 == 0); @@ -12011,7 +12010,7 @@ UseGgmlGemm1:; if (ith != 0) { return; } - //Every thread starts at ith, so the first unprocessed chunk is nth. This save a bit of coordination right at the start. + // Every thread starts at ith, so the first unprocessed chunk is nth. This save a bit of coordination right at the start. atomic_store(&state->shared->current_chunk, nth); if (src1->type != vec_dot_type) { char * wdata = params->wdata; @@ -12067,10 +12066,10 @@ UseGgmlGemm2:; UNUSED(chunks_executed); #endif - //This is the size of the first dimension of the result, so we can iterate that way. (see the ASSERT above, these are the same numbers) + // This is the size of the first dimension of the result, so we can iterate that way. (see the ASSERT above, these are the same numbers) const int64_t nr0 = ne0; - //This is the size of the rest of the dimensions of the result + // This is the size of the rest of the dimensions of the result const int64_t nr1 = ne1 * ne2 * ne3; // dot kernels can handle 1 row and col at a time, but mmla kernels can process 2 rows and cols @@ -12081,24 +12080,24 @@ UseGgmlGemm2:; num_rows_per_vec_dot = 1; } - //Now select a reasonable chunk size. + // Now select a reasonable chunk size. int chunk_size = 16; - //We need to step up the size if it's small - if (nr0 == 1 || nr1 == 1) + // We need to step up the size if it's small + if (nr0 == 1 || nr1 == 1) { chunk_size = 64; + } // distribute the work across the inner or outer loop based on which one is larger - //The number of chunks in the 0/1 dim. - //CEIL(nr0/chunk_size) + // The number of chunks in the 0/1 dim. + // CEIL(nr0/chunk_size) int64_t nchunk0 = (nr0 + chunk_size - 1) / chunk_size; int64_t nchunk1 = (nr1 + chunk_size - 1) / chunk_size; - //If the chunking is poor for the number of threads on this setup, scrap the whole plan. Re-chunk it by thread. + // If the chunking is poor for the number of threads on this setup, scrap the whole plan. Re-chunk it by thread. // Also, chunking by thread was measured to have perform better on NUMA systems. See https://github.com/ggerganov/llama.cpp/pull/6915 // In theory, chunking should be just as useful on NUMA and non NUMA systems, but testing disagreed with that. - if (nchunk0 * nchunk1 < nth * 4 || ggml_is_numa()) - { + if (nchunk0 * nchunk1 < nth * 4 || ggml_is_numa()) { // distribute the thread work across the inner or outer loop based on which one is larger nchunk0 = nr0 > nr1 ? nth : 1; // parallelize by src0 rows nchunk1 = nr0 > nr1 ? 1 : nth; // parallelize by src1 rows @@ -12114,8 +12113,7 @@ UseGgmlGemm2:; //The first chunk comes from our thread_id, the rest will get auto-assigned. int current_chunk = ith; - while (current_chunk < nchunk0 * nchunk1) - { + while (current_chunk < nchunk0 * nchunk1) { const int64_t ith0 = current_chunk % nchunk0; const int64_t ith1 = current_chunk / nchunk0; @@ -12131,8 +12129,9 @@ UseGgmlGemm2:; chunks_executed++; #endif - if (nth >= nchunk0 * nchunk1) + if (nth >= nchunk0 * nchunk1) { break; + } current_chunk = atomic_fetch_add(&state->shared->current_chunk, 1); } @@ -19652,17 +19651,17 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads, int n_cur_ return n_tasks; } -static void ggml_graph_compute_thread_sync_node(int* node_n, struct ggml_compute_state* state, const bool do_yield) { +static void ggml_graph_compute_thread_sync_node(int * node_n, struct ggml_compute_state * state, const bool do_yield) { // wait for other threads to finish - const int last_node_n = *node_n; + const int last_node_n = * node_n; while (true) { if (do_yield) { sched_yield(); } - *node_n = atomic_load(&state->shared->node_n); - if (*node_n != last_node_n) break; + * node_n = atomic_load(&state->shared->node_n); + if (* node_n != last_node_n) break; #if defined(__SSE3__) //Tell the processor we're spinning. It's a processor hint for spinlocks. _mm_pause(); @@ -19670,17 +19669,17 @@ static void ggml_graph_compute_thread_sync_node(int* node_n, struct ggml_compute } } -static void ggml_graph_compute_thread_sync_task(int* task_phase, struct ggml_compute_state* state, const bool do_yield) { +static void ggml_graph_compute_thread_sync_task(int * task_phase, struct ggml_compute_state * state, const bool do_yield) { // wait for other threads to finish - const int last_task_phase = *task_phase; + const int last_task_phase = * task_phase; while (true) { if (do_yield) { sched_yield(); } - *task_phase = atomic_load(&state->shared->node_task); - if (*task_phase != last_task_phase) break; + * task_phase = atomic_load(&state->shared->node_task); + if (* task_phase != last_task_phase) break; #if defined(__SSE3__) //Tell the processor we're spinning. It's a processor hint for spinlocks. _mm_pause(); From 163dbfdd2821634ec2813a8a39faa91b1a160d9a Mon Sep 17 00:00:00 2001 From: Kunnis Date: Tue, 14 May 2024 17:19:34 -0500 Subject: [PATCH 22/27] Couple more formatting fixes. --- ggml.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ggml.c b/ggml.c index c0d498014b752..78bc6c3c43a49 100644 --- a/ggml.c +++ b/ggml.c @@ -11834,17 +11834,17 @@ static void ggml_compute_forward_mul_mat_one_chunk( const int64_t i2 = i12; const int64_t i3 = i13; - const char* src0_row = (const char*)src0->data + (0 + i02 * nb02 + i03 * nb03); + const char * src0_row = (const char*)src0->data + (0 + i02 * nb02 + i03 * nb03); // desc: when src1 is not a contiguous memory block we have to calculate the offset using the strides // if it is, then we have either copied the data to params->wdata and made it contiguous or we are using // the original src1 data pointer, so we should index using the indices directly // TODO: this is a bit of a hack, we should probably have a better way to handle this - const char* src1_col = (const char*)wdata + + const char * src1_col = (const char*)wdata + (src1_cont || src1->type != vec_dot_type ? (i11 + i12 * ne11 + i13 * ne12 * ne11) * row_size : (i11 * nb11 + i12 * nb12 + i13 * nb13)); - float* dst_col = (float*)((char*)dst->data + (i1 * nb1 + i2 * nb2 + i3 * nb3)); + float * dst_col = (float*)((char*)dst->data + (i1 * nb1 + i2 * nb2 + i3 * nb3)); //for (int64_t ir0 = iir0; ir0 < iir0 + blck_0 && ir0 < ir0_end; ++ir0) { // vec_dot(ne00, &dst_col[ir0], src0_row + ir0*nb01, src1_col); From 6b0c90fc713a19e1e6f5d3ec0d7a824c71d6266d Mon Sep 17 00:00:00 2001 From: Kunnis Date: Tue, 14 May 2024 17:32:23 -0500 Subject: [PATCH 23/27] More style fixes. --- ggml.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ggml.c b/ggml.c index 78bc6c3c43a49..2effeca4918cd 100644 --- a/ggml.c +++ b/ggml.c @@ -12103,14 +12103,14 @@ UseGgmlGemm2:; nchunk1 = nr0 > nr1 ? 1 : nth; // parallelize by src1 rows } - //The number of elements in each chunk + // The number of elements in each chunk const int64_t dr0 = (nr0 + nchunk0 - 1) / nchunk0; const int64_t dr1 = (nr1 + nchunk1 - 1) / nchunk1; //if (ith == 0) // printf("MUL_MAT = [%d, %d, %d, %d] x [%d, %d, %d, %d] = %d x %d = %d. Fp Ops/Ch %d\n", ne00, ne01, ne02, ne03, ne10, ne11, ne12, ne13, nchunk0, nchunk1, nchunk0 * nchunk1, ne00 * nr0 * nr1 / nchunk0 / nchunk1); - //The first chunk comes from our thread_id, the rest will get auto-assigned. + // The first chunk comes from our thread_id, the rest will get auto-assigned. int current_chunk = ith; while (current_chunk < nchunk0 * nchunk1) { From 741a1981b8501efbcfe1650969c342413433ccdd Mon Sep 17 00:00:00 2001 From: Kunnis Date: Tue, 14 May 2024 18:32:52 -0500 Subject: [PATCH 24/27] Fix Warnings --- ggml.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/ggml.c b/ggml.c index 2effeca4918cd..a683c2dc43fb6 100644 --- a/ggml.c +++ b/ggml.c @@ -11880,8 +11880,6 @@ static void ggml_compute_forward_mul_mat( const enum ggml_type type = src0->type; - const bool src1_cont = ggml_is_contiguous(src1); - enum ggml_type const vec_dot_type = type_traits[type].vec_dot_type; ggml_from_float_t const from_float_to_vec_dot = type_traits[vec_dot_type].from_float; int64_t const vec_dot_num_rows = type_traits[type].nrows; @@ -11901,10 +11899,6 @@ static void ggml_compute_forward_mul_mat( GGML_ASSERT(nb1 <= nb2); GGML_ASSERT(nb2 <= nb3); - // broadcast factors - const int64_t r2 = ne12/ne02; - const int64_t r3 = ne13/ne03; - // nb01 >= nb00 - src0 is not transposed // compute by src0 rows @@ -11985,6 +11979,12 @@ static void ggml_compute_forward_mul_mat( #endif #if GGML_USE_LLAMAFILE + const bool src1_cont = ggml_is_contiguous(src1); + + // broadcast factors + const int64_t r2 = ne12 / ne02; + const int64_t r3 = ne13 / ne03; + if (src1_cont) { for (int64_t i13 = 0; i13 < ne13; i13++) for (int64_t i12 = 0; i12 < ne12; i12++) @@ -12036,11 +12036,11 @@ UseGgmlGemm1:; return; } - const void * wdata = (src1->type == vec_dot_type) ? src1->data : params->wdata; - const size_t row_size = ggml_row_size(vec_dot_type, ne10); - #if GGML_USE_LLAMAFILE if (src1->type != vec_dot_type) { + const void* wdata = (src1->type == vec_dot_type) ? src1->data : params->wdata; + const size_t row_size = ggml_row_size(vec_dot_type, ne10); + for (int64_t i13 = 0; i13 < ne13; i13++) for (int64_t i12 = 0; i12 < ne12; i12++) if (!llamafile_sgemm(ne01, ne11, ne00/ggml_blck_size(src0->type), From 2dd9f017e82545ff54ed0493da61ebb988bdf445 Mon Sep 17 00:00:00 2001 From: Kunnis Date: Tue, 14 May 2024 20:51:47 -0500 Subject: [PATCH 25/27] Going with unused because there's conditional logic that needs it. --- ggml.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/ggml.c b/ggml.c index a683c2dc43fb6..8dfcbec34ed59 100644 --- a/ggml.c +++ b/ggml.c @@ -11899,6 +11899,12 @@ static void ggml_compute_forward_mul_mat( GGML_ASSERT(nb1 <= nb2); GGML_ASSERT(nb2 <= nb3); + // broadcast factors + const int64_t r2 = ne12 / ne02; + const int64_t r3 = ne13 / ne03; + UNUSED(r2); + UNUSED(r3); + // nb01 >= nb00 - src0 is not transposed // compute by src0 rows @@ -11981,10 +11987,6 @@ static void ggml_compute_forward_mul_mat( #if GGML_USE_LLAMAFILE const bool src1_cont = ggml_is_contiguous(src1); - // broadcast factors - const int64_t r2 = ne12 / ne02; - const int64_t r3 = ne13 / ne03; - if (src1_cont) { for (int64_t i13 = 0; i13 < ne13; i13++) for (int64_t i12 = 0; i12 < ne12; i12++) From f2aabab436b9bc8e13aaf2c9c2c142e4d8195738 Mon Sep 17 00:00:00 2001 From: slaren Date: Wed, 15 May 2024 18:45:54 +0200 Subject: [PATCH 26/27] Update ggml.c --- ggml.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ggml.c b/ggml.c index 8dfcbec34ed59..8a6f9352c727f 100644 --- a/ggml.c +++ b/ggml.c @@ -19683,7 +19683,7 @@ static void ggml_graph_compute_thread_sync_task(int * task_phase, struct ggml_co * task_phase = atomic_load(&state->shared->node_task); if (* task_phase != last_task_phase) break; #if defined(__SSE3__) - //Tell the processor we're spinning. It's a processor hint for spinlocks. + // Tell the processor we're spinning. It's a processor hint for spinlocks. _mm_pause(); #endif } From 14c104d1668716e1049fb99df121e9d60efb1926 Mon Sep 17 00:00:00 2001 From: slaren Date: Wed, 15 May 2024 18:46:01 +0200 Subject: [PATCH 27/27] Update ggml.c --- ggml.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ggml.c b/ggml.c index 8a6f9352c727f..dacf50217f8b1 100644 --- a/ggml.c +++ b/ggml.c @@ -19665,7 +19665,7 @@ static void ggml_graph_compute_thread_sync_node(int * node_n, struct ggml_comput * node_n = atomic_load(&state->shared->node_n); if (* node_n != last_node_n) break; #if defined(__SSE3__) - //Tell the processor we're spinning. It's a processor hint for spinlocks. + // Tell the processor we're spinning. It's a processor hint for spinlocks. _mm_pause(); #endif }