948ef47106
Some checks failed
Bindings Tests (Go) / ubuntu-22 (push) Failing after 31s
CI (sycl) / ubuntu-22-cmake-sycl (linux/arm/v7, icx, icpx, ON) (push) Failing after 3s
CI (sycl) / ubuntu-22-cmake-sycl (linux/ppc64le, icx, icpx, ON) (push) Failing after 2s
CI (sycl) / ubuntu-22-cmake-sycl-fp16 (linux/amd64, icx, icpx, ON) (push) Failing after 3s
CI (sycl) / ubuntu-22-cmake-sycl-fp16 (linux/arm/v7, icx, icpx, ON) (push) Failing after 3s
CI (sycl) / ubuntu-22-cmake-sycl-fp16 (linux/arm64, icx, icpx, ON) (push) Failing after 3s
CI (sycl) / ubuntu-22-cmake-sycl-fp16 (linux/ppc64le, icx, icpx, ON) (push) Failing after 3s
CI (vad) / vad (push) Failing after 3s
CI (wasm) / emscripten (Release) (push) Failing after 4s
Bindings Tests (Ruby) / ubuntu-22 (push) Failing after 37s
CI (android) / android (push) Failing after 16s
CI (android) / android_java (push) Failing after 3s
CI (clang) / ubuntu-22-clang (linux/amd64, Debug) (push) Failing after 3s
CI (clang) / ubuntu-22-clang (linux/amd64, Release) (push) Failing after 2s
CI (clang) / ubuntu-22-clang (linux/ppc64le, Debug) (push) Failing after 3s
CI (clang) / ubuntu-22-clang (linux/ppc64le, Release) (push) Failing after 3s
CI (cpu) / ggml-ci-x64-cpu-low-perf (push) Failing after 3s
CI (cpu) / ggml-ci-x64-cpu-high-perf (push) Failing after 2s
CI (gcc) / ubuntu-22-gcc (linux/amd64, Debug) (push) Failing after 3s
CI (gcc) / ubuntu-22-gcc (linux/amd64, Release) (push) Failing after 3s
CI (gcc) / ubuntu-22-gcc (linux/ppc64le, Debug) (push) Failing after 3s
CI (gcc) / ubuntu-22-gcc (linux/ppc64le, Release) (push) Failing after 4s
CI (gcc) / ubuntu-22-gcc-arm-v7 (linux/arm/v7, Debug) (push) Failing after 3s
CI (gcc) / ubuntu-22-gcc-arm-v7 (linux/arm/v7, Release) (push) Failing after 3s
CI (quantize) / quantize (push) Failing after 3s
CI (sanitize) / ubuntu-22-gcc-sanitized (ADDRESS) (push) Failing after 3s
CI (sanitize) / ubuntu-22-gcc-sanitized (THREAD) (push) Failing after 3s
CI (sanitize) / ubuntu-22-gcc-sanitized (UNDEFINED) (push) Failing after 2s
CI (sycl) / ubuntu-22-cmake-sycl (linux/amd64, icx, icpx, ON) (push) Failing after 3s
CI (sycl) / ubuntu-22-cmake-sycl (linux/arm64, icx, icpx, ON) (push) Failing after 3s
Examples WASM / deploy-wasm-github-pages (push) Failing after 3s
Examples Tests / addon_node-ubuntu-22 (20.x) (push) Failing after 3s
Examples Tests / addon_node-ubuntu-22 (22.x) (push) Failing after 2s
CI (coreml) / coreml-base-en (push) Has been cancelled
CI (self-hosted) / gpu-vulkan (push) Has been cancelled
CI (self-hosted) / npu-amd-windows (push) Has been cancelled
CI (self-hosted) / gpu-vulkan-nvidia-cm2 (push) Has been cancelled
CI (clang) / ubuntu-22-clang-arm64 (Debug) (push) Has been cancelled
CI (clang) / ubuntu-22-clang-arm64 (Release) (push) Has been cancelled
CI (self-hosted) / gpu-cuda (push) Has been cancelled
CI (self-hosted) / gpu-vulkan-nvidia-cm (push) Has been cancelled
CI (cpu) / ggml-ci-arm64-cpu-low-perf (push) Has been cancelled
CI (cpu) / ggml-ci-arm64-cpu-high-perf (push) Has been cancelled
CI (cpu) / ggml-ci-arm64-cpu-high-perf-sve (push) Has been cancelled
CI (freebsd) / freeBSD-latest (push) Has been cancelled
CI (self-hosted) / gpu-metal (push) Has been cancelled
CI (gcc) / ubuntu-22-gcc-arm64 (Debug) (push) Has been cancelled
CI (gcc) / ubuntu-22-gcc-arm64 (Release) (push) Has been cancelled
CI (macOS) / macOS-latest (generic/platform=iOS) (push) Has been cancelled
CI (macOS) / macOS-latest (generic/platform=macOS) (push) Has been cancelled
CI (macOS) / macOS-latest (generic/platform=tvOS) (push) Has been cancelled
CI (self-hosted) / npu-amd-linux (push) Has been cancelled
CI (windows) / windows-msys2 (Release, clang, clang-x86_64, CLANG64) (push) Has been cancelled
CI (windows) / windows-msys2 (Release, gcc, ucrt-x86_64, UCRT64) (push) Has been cancelled
Publish Docker image / Push Docker image to Docker Hub (map[dockerfile:.devops/main-cuda.Dockerfile platform:linux/amd64 runs_on:ubuntu-24.04 tag:main-cuda]) (push) Has been cancelled
Publish Docker image / Push Docker image to Docker Hub (map[dockerfile:.devops/main-intel.Dockerfile platform:linux/amd64 runs_on:ubuntu-24.04 tag:main-intel]) (push) Has been cancelled
Publish Docker image / Push Docker image to Docker Hub (map[dockerfile:.devops/main-musa.Dockerfile platform:linux/amd64 runs_on:ubuntu-24.04 tag:main-musa]) (push) Has been cancelled
Publish Docker image / Push Docker image to Docker Hub (map[dockerfile:.devops/main-vulkan.Dockerfile platform:linux/amd64 runs_on:ubuntu-24.04 tag:main-vulkan]) (push) Has been cancelled
Publish Docker image / Push Docker image to Docker Hub (map[dockerfile:.devops/main-vulkan.Dockerfile platform:linux/arm64 runs_on:ubuntu-24.04-arm tag:main-vulkan-arm64]) (push) Has been cancelled
Publish Docker image / Push Docker image to Docker Hub (map[dockerfile:.devops/main.Dockerfile platform:linux/amd64 runs_on:ubuntu-24.04 tag:main]) (push) Has been cancelled
Publish Docker image / Push Docker image to Docker Hub (map[dockerfile:.devops/main.Dockerfile platform:linux/arm64 runs_on:ubuntu-24.04-arm tag:main-arm64]) (push) Has been cancelled
Mark inactive issues stale / mark-issues (push) Failing after 1s
261 lines
18 KiB
C
261 lines
18 KiB
C
#pragma clang diagnostic ignored "-Wunused-variable"
|
|
#pragma clang diagnostic ignored "-Wunused-function"
|
|
#pragma clang diagnostic ignored "-Wunused-but-set-variable"
|
|
|
|
#include <HAP_farf.h>
|
|
#include <HAP_perf.h>
|
|
|
|
#include <math.h>
|
|
#include <string.h>
|
|
|
|
#include "dma-queue.h"
|
|
#include "work-queue.h"
|
|
#include "hvx-utils.h"
|
|
#include "hex-utils.h"
|
|
#include "hvx-copy.h"
|
|
#include "hvx-quant.h"
|
|
|
|
#define GGML_COMMON_DECL_C
|
|
#include "ggml-common.h"
|
|
|
|
#include "hex-common.h"
|
|
#include "htp-ctx.h"
|
|
#include "htp-ops.h"
|
|
#include "htp-tensor.h"
|
|
#include "htp/set-rows-ops.h"
|
|
|
|
#define set_rows_preamble \
|
|
const uint32_t ne00 = octx->src[0]->ne[0]; \
|
|
const uint32_t ne01 = octx->src[0]->ne[1]; \
|
|
const uint32_t ne02 = octx->src[0]->ne[2]; \
|
|
const uint32_t ne03 = octx->src[0]->ne[3]; \
|
|
\
|
|
const uint32_t ne10 = octx->src[1]->ne[0]; \
|
|
const uint32_t ne11 = octx->src[1]->ne[1]; \
|
|
const uint32_t ne12 = octx->src[1]->ne[2]; \
|
|
const uint32_t ne13 = octx->src[1]->ne[3]; \
|
|
\
|
|
const uint32_t nb01 = octx->src[0]->nb[1]; \
|
|
const uint32_t nb02 = octx->src[0]->nb[2]; \
|
|
const uint32_t nb03 = octx->src[0]->nb[3]; \
|
|
\
|
|
const uint32_t nb10 = octx->src[1]->nb[0]; \
|
|
const uint32_t nb11 = octx->src[1]->nb[1]; \
|
|
const uint32_t nb12 = octx->src[1]->nb[2]; \
|
|
\
|
|
const uint32_t nb1 = octx->dst->nb[1]; \
|
|
const uint32_t nb2 = octx->dst->nb[2]; \
|
|
const uint32_t nb3 = octx->dst->nb[3]; \
|
|
\
|
|
const uint32_t ne0 = octx->dst->ne[0]; \
|
|
const uint32_t ne1 = octx->dst->ne[1]; \
|
|
const uint32_t ne2 = octx->dst->ne[2]; \
|
|
const uint32_t ne3 = octx->dst->ne[3]; \
|
|
\
|
|
const uint32_t nr = ne01;
|
|
|
|
struct set_rows_context {
|
|
struct htp_ops_context * octx;
|
|
const struct htp_set_rows_kernel_params * kparams;
|
|
struct htp_set_rows_vtcm_layout vtcm_layout;
|
|
uint8_t * vtcm_base;
|
|
uint32_t task_start;
|
|
uint32_t tasks;
|
|
uint32_t tasks_per_thread;
|
|
};
|
|
|
|
#define SET_ROWS_THREAD_DMA_FN(TYPE_NAME, IDX_TYPE, COMPUTE_EXPR) \
|
|
static void set_rows_thread_dma_##TYPE_NAME##_##IDX_TYPE(unsigned int nth, unsigned int ith, void *data) { \
|
|
struct set_rows_context * srctx = (struct set_rows_context *)data; \
|
|
struct htp_ops_context * octx = srctx->octx; \
|
|
const struct htp_set_rows_kernel_params * kparams = srctx->kparams; \
|
|
set_rows_preamble; \
|
|
struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \
|
|
const uint32_t dr = srctx->tasks_per_thread; \
|
|
const uint32_t ir0 = srctx->task_start + dr * ith; \
|
|
if (ir0 >= srctx->task_start + srctx->tasks) { \
|
|
return; \
|
|
} \
|
|
const uint32_t ir1 = MIN(ir0 + dr, srctx->task_start + srctx->tasks); \
|
|
dma_queue * dma_queue = octx->ctx->dma[ith]; \
|
|
const struct htp_set_rows_vtcm_layout * vtcm_layout = &srctx->vtcm_layout; \
|
|
uint8_t * vtcm_src0 = srctx->vtcm_base + vtcm_layout->off_src0 + ith * vtcm_layout->src0_bytes_per_thread; \
|
|
uint8_t * vtcm_dst = srctx->vtcm_base + vtcm_layout->off_dst + ith * vtcm_layout->dst_bytes_per_thread; \
|
|
const uint32_t src0_row_size = ne00 * sizeof(float); \
|
|
const uint32_t dst_row_size = htp_tensor_get_row_size(octx->dst->type, ne00); \
|
|
const uint32_t nrows_per_thread = ir1 - ir0; \
|
|
const uint32_t total_steps = ne03 * ne02 * nrows_per_thread; \
|
|
uint32_t pi_step = 0; \
|
|
uint32_t pi02 = 0; \
|
|
uint32_t pi03 = 0; \
|
|
for (uint32_t step = 0, spad_idx = 0; step < total_steps && spad_idx < 2; ++step, spad_idx++) { \
|
|
uint32_t i = ir0 + pi_step; \
|
|
const uintptr_t src0_ptr = octx->src[0]->data + i*nb01 + pi02*nb02 + pi03*nb03; \
|
|
dma_queue_push(dma_queue, \
|
|
dma_make_ptr((void *)octx->dst->data, \
|
|
vtcm_dst + spad_idx * vtcm_layout->dst_spad_half_size), \
|
|
dst_row_size, vtcm_layout->dst_spad_half_size, dst_row_size, 0); \
|
|
dma_queue_push(dma_queue, \
|
|
dma_make_ptr((void *)(vtcm_src0 + spad_idx * vtcm_layout->src0_spad_half_size), \
|
|
(const void *)src0_ptr), \
|
|
vtcm_layout->src0_spad_half_size, src0_row_size, src0_row_size, 1); \
|
|
pi_step++; \
|
|
if (pi_step == nrows_per_thread) { \
|
|
pi_step = 0; \
|
|
pi02++; \
|
|
if (pi02 == ne02) { \
|
|
pi02 = 0; \
|
|
pi03++; \
|
|
} \
|
|
} \
|
|
} \
|
|
uint32_t ci_step = 0; \
|
|
uint32_t ci02 = 0; \
|
|
uint32_t ci03 = 0; \
|
|
uint32_t ci11_base = 0; \
|
|
uint32_t ci12_base = 0; \
|
|
for (uint32_t step = 0; step < total_steps; ++step) { \
|
|
void * dst_spad = (void *) dma_queue_pop(dma_queue).src; \
|
|
void * src_spad = (void *) dma_queue_pop(dma_queue).dst; \
|
|
uint32_t i = ir0 + ci_step; \
|
|
const uintptr_t src1_addr = octx->src[1]->data + i*nb10 + ci11_base*nb11 + ci12_base*nb12; \
|
|
const IDX_TYPE i1 = *(const IDX_TYPE *)src1_addr; \
|
|
const bool valid_i1 = ((uint64_t)i1 < (uint64_t)ne1); \
|
|
const uint32_t target_i1 = (uint32_t)i1; \
|
|
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, step); \
|
|
if (valid_i1) { \
|
|
COMPUTE_EXPR; \
|
|
} \
|
|
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, step); \
|
|
if (valid_i1) { \
|
|
const uintptr_t dst_ptr = octx->dst->data + target_i1*nb1 + ci02*nb2 + ci03*nb3; \
|
|
dma_queue_push(dma_queue, \
|
|
dma_make_ptr((void *)dst_ptr, (const void *)dst_spad), \
|
|
dst_row_size, vtcm_layout->dst_spad_half_size, dst_row_size, 1); \
|
|
} else { \
|
|
dma_queue_push(dma_queue, \
|
|
dma_make_ptr((void *)octx->dst->data, (const void *)dst_spad), \
|
|
dst_row_size, vtcm_layout->dst_spad_half_size, dst_row_size, 0); \
|
|
} \
|
|
const uint32_t next_step = step + 2; \
|
|
if (next_step < total_steps) { \
|
|
uint32_t ni = ir0 + pi_step; \
|
|
const uintptr_t psrc0_ptr = octx->src[0]->data + ni*nb01 + pi02*nb02 + pi03*nb03; \
|
|
dma_queue_push(dma_queue, \
|
|
dma_make_ptr((void *)src_spad, (const void *)psrc0_ptr), \
|
|
vtcm_layout->src0_spad_half_size, src0_row_size, src0_row_size, 1); \
|
|
pi_step++; \
|
|
if (pi_step == nrows_per_thread) { \
|
|
pi_step = 0; \
|
|
pi02++; \
|
|
if (pi02 == ne02) { \
|
|
pi02 = 0; \
|
|
pi03++; \
|
|
} \
|
|
} \
|
|
} \
|
|
ci_step++; \
|
|
if (ci_step == nrows_per_thread) { \
|
|
ci_step = 0; \
|
|
ci02++; \
|
|
ci11_base++; \
|
|
if (ci11_base == ne11) { \
|
|
ci11_base = 0; \
|
|
} \
|
|
if (ci02 == ne02) { \
|
|
ci02 = 0; \
|
|
ci03++; \
|
|
ci12_base++; \
|
|
if (ci12_base == ne12) { \
|
|
ci12_base = 0; \
|
|
} \
|
|
} \
|
|
} \
|
|
} \
|
|
dma_queue_flush(dma_queue); \
|
|
}
|
|
|
|
SET_ROWS_THREAD_DMA_FN(f32, int32_t, { hvx_copy_f32_uu((uint8_t *)dst_spad, (const uint8_t *)src_spad, ne00); })
|
|
SET_ROWS_THREAD_DMA_FN(f32, int64_t, { hvx_copy_f32_uu((uint8_t *)dst_spad, (const uint8_t *)src_spad, ne00); })
|
|
|
|
SET_ROWS_THREAD_DMA_FN(f16, int32_t, { hvx_copy_f16_f32_uu((uint8_t *)dst_spad, (const uint8_t *)src_spad, ne00); })
|
|
SET_ROWS_THREAD_DMA_FN(f16, int64_t, { hvx_copy_f16_f32_uu((uint8_t *)dst_spad, (const uint8_t *)src_spad, ne00); })
|
|
|
|
SET_ROWS_THREAD_DMA_FN(q8_0, int32_t, { hvx_quantize_row_q8_0_f32(dst_spad, (const float *)src_spad, ne00); })
|
|
SET_ROWS_THREAD_DMA_FN(q8_0, int64_t, { hvx_quantize_row_q8_0_f32(dst_spad, (const float *)src_spad, ne00); })
|
|
|
|
int op_set_rows(struct htp_ops_context * octx) {
|
|
const struct htp_set_rows_kernel_params * kparams = (const struct htp_set_rows_kernel_params *)octx->kernel_params;
|
|
set_rows_preamble;
|
|
|
|
if (octx->src[0]->type != HTP_TYPE_F32) {
|
|
return HTP_STATUS_NO_SUPPORT;
|
|
}
|
|
|
|
if (octx->dst->type != HTP_TYPE_F32 && octx->dst->type != HTP_TYPE_F16 && octx->dst->type != HTP_TYPE_Q8_0) {
|
|
return HTP_STATUS_NO_SUPPORT;
|
|
}
|
|
|
|
if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) {
|
|
return HTP_STATUS_OK;
|
|
}
|
|
|
|
const struct htp_tensor * dst = octx->dst;
|
|
const uint32_t total_tasks = kparams->total_tasks;
|
|
|
|
uint32_t task_start = 0;
|
|
uint32_t tasks = total_tasks;
|
|
|
|
if (octx->ctx->mdev.count > 1) {
|
|
const bool can_split = htp_tensor_mdev_data_aligned(dst) && (dst->nb[1] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) == 0 && !htp_tensor_is_permuted(dst);
|
|
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_tasks, can_split ? 1 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
|
|
task_start = range.start;
|
|
tasks = range.count;
|
|
}
|
|
|
|
if (tasks == 0) {
|
|
return HTP_STATUS_OK;
|
|
}
|
|
|
|
if (!htp_ops_context_set_n_threads(octx, (uint32_t) kparams->n_threads)) {
|
|
return HTP_STATUS_INVAL_PARAMS;
|
|
}
|
|
|
|
const uint32_t n_threads = octx->n_threads;
|
|
|
|
// l2fetch the src1 (indices) tensor in the main thread
|
|
hex_l2fetch_block((const void *)octx->src[1]->data, octx->src[1]->ne[3] * octx->src[1]->nb[3]);
|
|
|
|
struct set_rows_context srctx;
|
|
srctx.octx = octx;
|
|
srctx.kparams = kparams;
|
|
srctx.task_start = task_start;
|
|
srctx.tasks = tasks;
|
|
srctx.tasks_per_thread = fastdiv(tasks + n_threads - 1, &octx->n_threads_div);
|
|
|
|
htp_set_rows_vtcm_layout_build(&srctx.vtcm_layout, octx->dst->type, ne00, n_threads);
|
|
srctx.vtcm_base = (uint8_t *)octx->ctx->vtcm_base;
|
|
|
|
work_queue_func_t q_func = NULL;
|
|
const bool is_i32 = (octx->src[1]->type == HTP_TYPE_I32);
|
|
|
|
switch (octx->dst->type) {
|
|
case HTP_TYPE_F32: q_func = is_i32 ? set_rows_thread_dma_f32_int32_t : set_rows_thread_dma_f32_int64_t; break;
|
|
case HTP_TYPE_F16: q_func = is_i32 ? set_rows_thread_dma_f16_int32_t : set_rows_thread_dma_f16_int64_t; break;
|
|
case HTP_TYPE_Q8_0: q_func = is_i32 ? set_rows_thread_dma_q8_0_int32_t : set_rows_thread_dma_q8_0_int64_t; break;
|
|
default: return HTP_STATUS_NO_SUPPORT;
|
|
}
|
|
|
|
FARF(HIGH, "set-rows: (%ux%ux%ux%u) x (%ux%ux%ux%u) -> (%ux%ux%ux%u) : src0-vtcm-size %zu dst-vtcm-size %zu n-threads %d\n",
|
|
octx->src[0]->ne[0], octx->src[0]->ne[1], octx->src[0]->ne[2], octx->src[0]->ne[3],
|
|
octx->src[1]->ne[0], octx->src[1]->ne[1], octx->src[1]->ne[2], octx->src[1]->ne[3],
|
|
octx->dst->ne[0], octx->dst->ne[1], octx->dst->ne[2], octx->dst->ne[3],
|
|
srctx.vtcm_layout.src0_bytes_per_thread * n_threads,
|
|
srctx.vtcm_layout.dst_bytes_per_thread * n_threads,
|
|
n_threads);
|
|
|
|
work_queue_run(octx->ctx->work_queue, q_func, &srctx, n_threads);
|
|
|
|
return HTP_STATUS_OK;
|
|
}
|