shahondin1624/llama.cpp
1
0
Fork 0
Code Issues 2 Pull Requests Actions 3 Packages Projects Releases Wiki Activity

ggml-hexagon: add PAD op HVX kernel (#23078)

Browse Source 
* ggml-hexagon: add PAD op HVX kernel

Implements GGML_OP_PAD on the Hexagon HTP backend using HVX vectorized
kernels. Supports zero-padding and circular padding across all 4 tensor
dimensions.

* hex-ggml: remove duplicate op cases (merge conflict)

* hex-pad: fix editorconfig checks and macro alignment

---------

Co-authored-by: Max Krasnyansky <maxk@qti.qualcomm.com>
This commit is contained in:
Pranav Dhinakar
2026-05-18 13:39:36 -07:00
committed by GitHub
parent 5cbaa5e69e
commit b7340443d4
6 changed files with 569 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ggml/src/ggml-hexagon/ggml-hexagon.cpp
+18
View File
@@ -2744,6 +2744,18 @@ static bool ggml_hexagon_supported_ssm_conv(const struct ggml_hexagon_session *
return true;
}
static bool ggml_hexagon_supported_pad(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
const struct ggml_tensor * src0 = op->src[0];
const struct ggml_tensor * dst = op;
if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) {
return false;
}
GGML_UNUSED(sess);
return true;
}
static bool ggml_hexagon_supported_cumsum(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
const struct ggml_tensor * src0 = op->src[0];
const struct ggml_tensor * dst = op;
@@ -2857,6 +2869,8 @@ static htp_op_code op_remap_to_htp(const ggml_tensor * t) {
case GGML_OP_FILL: return HTP_OP_FILL;
case GGML_OP_DIAG: return HTP_OP_DIAG;
case GGML_OP_SOLVE_TRI: return HTP_OP_SOLVE_TRI;
case GGML_OP_PAD: return HTP_OP_PAD;
case GGML_OP_UNARY:
switch (ggml_get_unary_op(t)) {
case GGML_UNARY_OP_SILU: return HTP_OP_UNARY_SILU;
@@ -3416,6 +3430,10 @@ static bool ggml_backend_hexagon_device_supports_op(ggml_backend_dev_t dev, cons
supp = ggml_hexagon_supported_solve_tri(sess, op);
break;
case GGML_OP_PAD:
supp = ggml_hexagon_supported_pad(sess, op);
break;
default:
break;
}
ggml/src/ggml-hexagon/htp/CMakeLists.txt
+1
View File
@@ -38,6 +38,7 @@ add_library(${HTP_LIB} SHARED
diag-ops.c
solve-tri-ops.c
gated-delta-net-ops.c
pad-ops.c
)
target_compile_definitions(${HTP_LIB} PRIVATE
ggml/src/ggml-hexagon/htp/htp-ctx.h
+1
View File
@@ -107,5 +107,6 @@ int op_fill(struct htp_ops_context * octx);
int op_diag(struct htp_ops_context * octx);
int op_solve_tri(struct htp_ops_context * octx);
int op_gated_delta_net(struct htp_ops_context * octx);
int op_pad(struct htp_ops_context * octx);
#endif /* HTP_CTX_H */
ggml/src/ggml-hexagon/htp/htp-ops.h
+1
View File
@@ -86,6 +86,7 @@ enum htp_op_code {
HTP_OP_SOLVE_TRI,
HTP_OP_L2_NORM,
HTP_OP_GATED_DELTA_NET,
HTP_OP_PAD,
HTP_OP_INVALID
};
ggml/src/ggml-hexagon/htp/main.c
+3
View File
@@ -595,6 +595,9 @@ static int execute_op(struct htp_ops_context * octx) {
case HTP_OP_SOLVE_TRI:
return op_solve_tri(octx);
case HTP_OP_PAD:
return op_pad(octx);
case HTP_OP_GATED_DELTA_NET:
return op_gated_delta_net(octx);
ggml/src/ggml-hexagon/htp/pad-ops.c
+545
View File
@@ -0,0 +1,545 @@
#pragma clang diagnostic ignored "-Wunused-variable"
#pragma clang diagnostic ignored "-Wunused-function"
#pragma clang diagnostic ignored "-Wunused-but-set-variable"
#include <HAP_farf.h>
#include <HAP_perf.h>
#include <string.h>
#include "hex-dma.h"
#include "hvx-utils.h"
#define GGML_COMMON_DECL_C
#include "ggml-common.h"
#include "htp-ctx.h"
#include "htp-ops.h"
/* Circular wrap: maps any integer x into [0, n) */
static inline uint32_t wrap_around(int32_t x, uint32_t n) {
return (uint32_t)(((x % (int32_t)n) + (int32_t)n) % (int32_t)n);
}
/* Decompose a flat dst row index into (i1, i2, i3) */
static inline void pad_decompose_row(uint32_t ir, uint32_t ne1, uint32_t ne2,
uint32_t *i1, uint32_t *i2, uint32_t *i3) {
*i1 = ir % ne1;
*i2 = (ir / ne1) % ne2;
*i3 = ir / (ne1 * ne2);
}
/* Return non-zero if row (i1,i2,i3) falls in the non-padded interior */
static inline int pad_is_interior(uint32_t i1, uint32_t i2, uint32_t i3,
int32_t lp1, int32_t rp1, uint32_t ne1,
int32_t lp2, int32_t rp2, uint32_t ne2,
int32_t lp3, int32_t rp3, uint32_t ne3) {
return ((int32_t)i1 >= lp1 && (int32_t)i1 < (int32_t)ne1 - rp1) &&
((int32_t)i2 >= lp2 && (int32_t)i2 < (int32_t)ne2 - rp2) &&
((int32_t)i3 >= lp3 && (int32_t)i3 < (int32_t)ne3 - rp3);
}
/* Compute the DDR src row pointer for a zero-pad interior row */
static inline const uint8_t * pad_src_row_ptr(const struct htp_tensor * src,
uint32_t i1, uint32_t i2, uint32_t i3,
int32_t lp1, int32_t lp2, int32_t lp3) {
return (const uint8_t *) src->data
+ (i1 - (uint32_t)lp1) * src->nb[1]
+ (i2 - (uint32_t)lp2) * src->nb[2]
+ (i3 - (uint32_t)lp3) * src->nb[3];
}
/* Compute the DDR src row pointer for a circular row (wrap-around indexing) */
static inline const uint8_t * pad_circ_src_row_ptr(const struct htp_tensor * src,
uint32_t i1, uint32_t i2, uint32_t i3,
int32_t lp1, int32_t lp2, int32_t lp3) {
return (const uint8_t *) src->data
+ wrap_around((int32_t)i1 - lp1, src->ne[1]) * src->nb[1]
+ wrap_around((int32_t)i2 - lp2, src->ne[2]) * src->nb[2]
+ wrap_around((int32_t)i3 - lp3, src->ne[3]) * src->nb[3];
}
struct htp_pad_context {
struct htp_ops_context * octx;
int32_t lp0, rp0;
int32_t lp1, rp1;
int32_t lp2, rp2;
int32_t lp3, rp3;
uint32_t nrows_per_thread;
uint32_t total_dst_rows;
size_t type_size;
// Row sizes for DMA kernel (populated when VTCM is available)
size_t src_row_size;
size_t src_row_size_aligned;
size_t dst_row_size;
size_t dst_row_size_aligned;
};
#define htp_pad_preamble \
const struct htp_tensor * src = octx->src[0]; \
const struct htp_tensor * dst = octx->dst; \
\
const uint32_t ne00 = src->ne[0]; \
const uint32_t nb00 = src->nb[0]; \
\
const uint32_t ne0 = dst->ne[0]; \
const uint32_t ne1 = dst->ne[1]; \
const uint32_t ne2 = dst->ne[2]; \
const uint32_t ne3 = dst->ne[3]; \
\
const uint32_t nb1 = dst->nb[1]; \
const uint32_t nb2 = dst->nb[2]; \
const uint32_t nb3 = dst->nb[3]; \
\
const int32_t lp0 = pctx->lp0, rp0 = pctx->rp0; \
const int32_t lp1 = pctx->lp1, rp1 = pctx->rp1; \
const int32_t lp2 = pctx->lp2, rp2 = pctx->rp2; \
const int32_t lp3 = pctx->lp3, rp3 = pctx->rp3; \
\
const size_t type_size = pctx->type_size; \
\
const uint32_t row_start = pctx->nrows_per_thread * ith; \
const uint32_t row_end = MIN(row_start + pctx->nrows_per_thread, pctx->total_dst_rows);
#define htp_pad_dma_preamble \
const size_t src_row_size = pctx->src_row_size; \
const size_t src_row_size_aligned = pctx->src_row_size_aligned; \
const size_t dst_row_size = pctx->dst_row_size; \
const size_t dst_row_size_aligned = pctx->dst_row_size_aligned; \
\
uint8_t * src_spad_base = octx->src0_spad.data + ith * octx->src0_spad.size_per_thread; \
uint8_t * dst_spad_base = octx->dst_spad.data + ith * octx->dst_spad.size_per_thread; \
\
dma_queue * dma = octx->ctx->dma[ith];
// ---------------------------------------------------------------------------
// HVX vectorized PAD kernel
// ---------------------------------------------------------------------------
static void pad_job_per_thread_hvx(unsigned int nth, unsigned int ith, void * data) {
const struct htp_pad_context * pctx = (const struct htp_pad_context *) data;
struct htp_ops_context * octx = pctx->octx;
htp_pad_preamble;
uint64_t t1, t2;
t1 = HAP_perf_get_qtimer_count();
for (uint32_t dst_row = row_start; dst_row < row_end; dst_row++) {
uint32_t i1, i2, i3;
pad_decompose_row(dst_row, ne1, ne2, &i1, &i2, &i3);
uint8_t * dst_ptr = (uint8_t *) dst->data + i1 * nb1 + i2 * nb2 + i3 * nb3;
const int interior = pad_is_interior(i1, i2, i3,
lp1, rp1, ne1,
lp2, rp2, ne2,
lp3, rp3, ne3);
if (!interior) {
hvx_splat_f32_u(dst_ptr, 0.0f, ne0);
} else {
const uint8_t * src_ptr = pad_src_row_ptr(src, i1, i2, i3, lp1, lp2, lp3);
if (lp0 > 0) {
hvx_splat_f32_u(dst_ptr, 0.0f, (uint32_t)lp0);
}
uint8_t * dst_row_start = dst_ptr + (size_t)lp0 * type_size;
if (nb00 == type_size) {
hvx_copy_f32_uu(dst_row_start, src_ptr, ne00);
} else {
for (uint32_t i = 0; i < ne00; i++) {
memcpy(dst_row_start + i * type_size,
src_ptr + (size_t)i * nb00,
type_size);
}
}
if (rp0 > 0) {
hvx_splat_f32_u(dst_ptr + ((size_t)lp0 + ne00) * type_size, 0.0f, (uint32_t)rp0);
}
}
}
t2 = HAP_perf_get_qtimer_count();
FARF(HIGH, "pad-hvx %d/%d: (%ux%ux%ux%u) -> (%ux%ux%ux%u) rows %u:%u usec %u\n",
ith, nth,
src->ne[0], src->ne[1], src->ne[2], src->ne[3],
dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3],
row_start, row_end,
(unsigned) HAP_perf_qtimer_count_to_us(t2 - t1));
}
// ---------------------------------------------------------------------------
// HVX + DMA PAD kernel — aligned, double-buffered
// ---------------------------------------------------------------------------
static void pad_job_per_thread_hvx_dma(unsigned int nth, unsigned int ith, void * data) {
const struct htp_pad_context * pctx = (const struct htp_pad_context *) data;
struct htp_ops_context * octx = pctx->octx;
htp_pad_preamble;
htp_pad_dma_preamble;
uint64_t t1, t2;
t1 = HAP_perf_get_qtimer_count();
// -----------------------------------------------------------------------
// Priming phase: push 2 pairs of (dummy_dst_DMA, src_DMA) to seed the
// double-buffer pipeline before the main loop begins.
// -----------------------------------------------------------------------
for (uint32_t ir = row_start, spad_idx = 0; ir < row_end && spad_idx < 2; ir++, spad_idx++) {
uint8_t * src_spad_cur = src_spad_base + spad_idx * src_row_size_aligned;
uint8_t * dst_spad_cur = dst_spad_base + spad_idx * dst_row_size_aligned;
dma_queue_push_vtcm_to_ddr(dma,
dma_make_ptr((uint8_t *)dst->data, dst_spad_cur),
dst_row_size, dst_row_size_aligned, 0);
uint32_t i1, i2, i3;
pad_decompose_row(ir, ne1, ne2, &i1, &i2, &i3);
const int interior = pad_is_interior(i1, i2, i3,
lp1, rp1, ne1,
lp2, rp2, ne2,
lp3, rp3, ne3);
const uint8_t * src_ptr = interior
? pad_src_row_ptr(src, i1, i2, i3, lp1, lp2, lp3) : NULL;
// Interior row: real DMA (1 row) from DDR to VTCM.
// Border row: null DMA (nrows=0)
dma_queue_push_ddr_to_vtcm(dma,
dma_make_ptr(src_spad_cur,
src_ptr ? src_ptr : (const uint8_t *)src_spad_cur),
src_row_size_aligned, src_row_size, src_ptr ? 1 : 0);
}
// -----------------------------------------------------------------------
// Main loop: pop completed DMAs, compute in VTCM with aligned HVX ops,
// push dst DMA and prefetch src for the next+1 row.
// -----------------------------------------------------------------------
for (uint32_t ir = row_start; ir < row_end; ir++) {
uint8_t * dst_spad_cur = (uint8_t *) dma_queue_pop(dma).src;
uint8_t * src_spad_cur = (uint8_t *) dma_queue_pop(dma).dst;
uint32_t i1, i2, i3;
pad_decompose_row(ir, ne1, ne2, &i1, &i2, &i3);
uint8_t * dst_ptr = (uint8_t *) dst->data + i1 * nb1 + i2 * nb2 + i3 * nb3;
const int interior = pad_is_interior(i1, i2, i3,
lp1, rp1, ne1,
lp2, rp2, ne2,
lp3, rp3, ne3);
if (!interior) {
hvx_splat_f32_a(dst_spad_cur, 0.0f, ne0);
} else {
hvx_splat_f32_a(dst_spad_cur, 0.0f, ne0);
uint8_t * dst_interior = dst_spad_cur + (size_t)lp0 * type_size;
if ((uintptr_t)dst_interior % VLEN == 0) {
hvx_copy_f32_aa(dst_interior, src_spad_cur, ne00);
} else {
hvx_copy_f32_ua(dst_interior, src_spad_cur, ne00);
}
}
dma_queue_push_vtcm_to_ddr(dma,
dma_make_ptr(dst_ptr, dst_spad_cur),
dst_row_size, dst_row_size_aligned, 1);
const uint32_t next_row = ir + 2;
if (next_row < row_end) {
uint32_t ni1, ni2, ni3;
pad_decompose_row(next_row, ne1, ne2, &ni1, &ni2, &ni3);
const int next_interior = pad_is_interior(ni1, ni2, ni3,
lp1, rp1, ne1,
lp2, rp2, ne2,
lp3, rp3, ne3);
const uint8_t * next_src_ptr = next_interior
? pad_src_row_ptr(src, ni1, ni2, ni3, lp1, lp2, lp3) : NULL;
dma_queue_push_ddr_to_vtcm(dma,
dma_make_ptr(src_spad_cur,
next_src_ptr ? next_src_ptr : (const uint8_t *)src_spad_cur),
src_row_size_aligned, src_row_size, next_src_ptr ? 1 : 0);
}
}
dma_queue_flush(dma);
t2 = HAP_perf_get_qtimer_count();
FARF(HIGH, "pad-hvx-dma %d/%d: (%ux%ux%ux%u) -> (%ux%ux%ux%u) rows %u:%u usec %u\n",
ith, nth,
src->ne[0], src->ne[1], src->ne[2], src->ne[3],
dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3],
row_start, row_end,
(unsigned) HAP_perf_qtimer_count_to_us(t2 - t1));
}
// ---------------------------------------------------------------------------
// HVX circular PAD kernel
// ---------------------------------------------------------------------------
static void pad_job_per_thread_hvx_circular(unsigned int nth, unsigned int ith, void * data) {
const struct htp_pad_context * pctx = (const struct htp_pad_context *) data;
struct htp_ops_context * octx = pctx->octx;
htp_pad_preamble;
uint64_t t1, t2;
t1 = HAP_perf_get_qtimer_count();
for (uint32_t dst_row = row_start; dst_row < row_end; dst_row++) {
uint32_t i1, i2, i3;
pad_decompose_row(dst_row, ne1, ne2, &i1, &i2, &i3);
uint8_t * dst_ptr = (uint8_t *) dst->data + i1 * nb1 + i2 * nb2 + i3 * nb3;
const uint8_t * src_row = pad_circ_src_row_ptr(src, i1, i2, i3, lp1, lp2, lp3);
if (nb00 == type_size) {
if (lp0 > 0) {
if ((uint32_t)lp0 < 32) {
memcpy(dst_ptr,
src_row + (size_t)(ne00 - (uint32_t)lp0) * type_size,
(size_t)lp0 * type_size);
} else {
hvx_copy_f32_uu(dst_ptr,
src_row + (size_t)(ne00 - (uint32_t)lp0) * type_size,
(uint32_t)lp0);
}
}
hvx_copy_f32_uu(dst_ptr + (size_t)lp0 * type_size, src_row, ne00);
if (rp0 > 0) {
if ((uint32_t)rp0 < 32) {
memcpy(dst_ptr + ((size_t)lp0 + ne00) * type_size,
src_row,
(size_t)rp0 * type_size);
} else {
hvx_copy_f32_uu(dst_ptr + ((size_t)lp0 + ne00) * type_size,
src_row,
(uint32_t)rp0);
}
}
} else {
for (uint32_t i = 0; i < (uint32_t)lp0; i++) {
*(float *)(dst_ptr + i * type_size) =
*(const float *)(src_row + (size_t)(ne00 - (uint32_t)lp0 + i) * nb00);
}
for (uint32_t i = 0; i < ne00; i++) {
*(float *)(dst_ptr + ((size_t)lp0 + i) * type_size) =
*(const float *)(src_row + (size_t)i * nb00);
}
for (uint32_t i = 0; i < (uint32_t)rp0; i++) {
*(float *)(dst_ptr + ((size_t)lp0 + ne00 + i) * type_size) =
*(const float *)(src_row + (size_t)i * nb00);
}
}
}
t2 = HAP_perf_get_qtimer_count();
FARF(HIGH, "pad-hvx-circ %d/%d: (%ux%ux%ux%u) -> (%ux%ux%ux%u) rows %u:%u usec %u\n",
ith, nth,
src->ne[0], src->ne[1], src->ne[2], src->ne[3],
dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3],
row_start, row_end,
(unsigned) HAP_perf_qtimer_count_to_us(t2 - t1));
}
// ---------------------------------------------------------------------------
// HVX + DMA circular PAD kernel — aligned, double-buffered
// ---------------------------------------------------------------------------
static void pad_job_per_thread_hvx_circular_dma(unsigned int nth, unsigned int ith, void * data) {
const struct htp_pad_context * pctx = (const struct htp_pad_context *) data;
struct htp_ops_context * octx = pctx->octx;
htp_pad_preamble;
htp_pad_dma_preamble;
uint64_t t1, t2;
t1 = HAP_perf_get_qtimer_count();
// -----------------------------------------------------------------------
// Priming phase: push 2 pairs of (dummy_dst_DMA, src_DMA) to seed the
// double-buffer pipeline. Every row is a real src DMA (no null DMAs).
// -----------------------------------------------------------------------
for (uint32_t ir = row_start, spad_idx = 0; ir < row_end && spad_idx < 2; ir++, spad_idx++) {
uint8_t * src_spad_cur = src_spad_base + spad_idx * src_row_size_aligned;
uint8_t * dst_spad_cur = dst_spad_base + spad_idx * dst_row_size_aligned;
dma_queue_push_vtcm_to_ddr(dma,
dma_make_ptr((uint8_t *)dst->data, dst_spad_cur),
dst_row_size, dst_row_size_aligned, 0);
uint32_t pi1, pi2, pi3;
pad_decompose_row(ir, ne1, ne2, &pi1, &pi2, &pi3);
dma_queue_push_ddr_to_vtcm(dma,
dma_make_ptr(src_spad_cur, pad_circ_src_row_ptr(src, pi1, pi2, pi3, lp1, lp2, lp3)),
src_row_size_aligned, src_row_size, 1);
}
// -----------------------------------------------------------------------
// Main loop: pop completed DMAs, assemble circular row in VTCM with
// aligned HVX ops, push dst DMA and prefetch src for the next+1 row.
// -----------------------------------------------------------------------
for (uint32_t ir = row_start; ir < row_end; ir++) {
uint8_t * dst_spad_cur = (uint8_t *) dma_queue_pop(dma).src;
uint8_t * src_spad_cur = (uint8_t *) dma_queue_pop(dma).dst;
uint32_t i1, i2, i3;
pad_decompose_row(ir, ne1, ne2, &i1, &i2, &i3);
uint8_t * dst_ptr = (uint8_t *) dst->data + i1 * nb1 + i2 * nb2 + i3 * nb3;
if (lp0 > 0) {
uint8_t * dst_left = dst_spad_cur;
const uint8_t * src_left = src_spad_cur + (size_t)(ne00 - (uint32_t)lp0) * type_size;
if ((uint32_t)lp0 < 32) {
memcpy(dst_left, src_left, (size_t)lp0 * type_size);
} else {
hvx_copy_f32_uu(dst_left, src_left, (uint32_t)lp0);
}
}
{
uint8_t * dst_mid = dst_spad_cur + (size_t)lp0 * type_size;
if ((uintptr_t)dst_mid % VLEN == 0) {
hvx_copy_f32_aa(dst_mid, src_spad_cur, ne00);
} else {
hvx_copy_f32_ua(dst_mid, src_spad_cur, ne00);
}
}
if (rp0 > 0) {
uint8_t * dst_right = dst_spad_cur + ((size_t)lp0 + ne00) * type_size;
if ((uint32_t)rp0 < 32) {
memcpy(dst_right, src_spad_cur, (size_t)rp0 * type_size);
} else {
if ((uintptr_t)dst_right % VLEN == 0) {
hvx_copy_f32_aa(dst_right, src_spad_cur, (uint32_t)rp0);
} else {
hvx_copy_f32_ua(dst_right, src_spad_cur, (uint32_t)rp0);
}
}
}
dma_queue_push_vtcm_to_ddr(dma,
dma_make_ptr(dst_ptr, dst_spad_cur),
dst_row_size, dst_row_size_aligned, 1);
const uint32_t next_row = ir + 2;
if (next_row < row_end) {
uint32_t nri1, nri2, nri3;
pad_decompose_row(next_row, ne1, ne2, &nri1, &nri2, &nri3);
dma_queue_push_ddr_to_vtcm(dma,
dma_make_ptr(src_spad_cur,
pad_circ_src_row_ptr(src, nri1, nri2, nri3, lp1, lp2, lp3)),
src_row_size_aligned, src_row_size, 1);
}
}
dma_queue_flush(dma);
t2 = HAP_perf_get_qtimer_count();
FARF(HIGH, "pad-hvx-circ-dma %d/%d: (%ux%ux%ux%u) -> (%ux%ux%ux%u) rows %u:%u usec %u\n",
ith, nth,
src->ne[0], src->ne[1], src->ne[2], src->ne[3],
dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3],
row_start, row_end,
(unsigned) HAP_perf_qtimer_count_to_us(t2 - t1));
}
int op_pad(struct htp_ops_context * octx) {
const struct htp_tensor * src0 = octx->src[0];
const struct htp_tensor * dst = octx->dst;
// Only F32 supported
size_t type_size;
switch (src0->type) {
case HTP_TYPE_F32: type_size = 4; break;
default:
FARF(ERROR, "pad-hvx: unsupported type %u\n", src0->type);
return HTP_STATUS_NO_SUPPORT;
}
if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) {
return HTP_STATUS_OK;
}
const int32_t lp0 = octx->op_params[0];
const int32_t rp0 = octx->op_params[1];
const int32_t lp1 = octx->op_params[2];
const int32_t rp1 = octx->op_params[3];
const int32_t lp2 = octx->op_params[4];
const int32_t rp2 = octx->op_params[5];
const int32_t lp3 = octx->op_params[6];
const int32_t rp3 = octx->op_params[7];
const int32_t circular = octx->op_params[8];
const uint32_t ne0 = dst->ne[0];
const uint32_t ne00 = src0->ne[0];
const uint32_t total_dst_rows = dst->ne[1] * dst->ne[2] * dst->ne[3];
const uint32_t n_threads = MIN(octx->n_threads, total_dst_rows > 0 ? total_dst_rows : 1);
const size_t src_row_size = (size_t)ne00 * type_size;
const size_t dst_row_size = (size_t)ne0 * type_size;
const size_t src_row_size_aligned = hex_round_up(src_row_size, VLEN);
const size_t dst_row_size_aligned = hex_round_up(dst_row_size, VLEN);
// Total VTCM needed: 2 buffers (ping+pong) for src and dst, per thread
const size_t vtcm_needed = (size_t)n_threads * 2 * (src_row_size_aligned + dst_row_size_aligned);
const int use_dma = (src0->nb[0] == (uint32_t)type_size) &&
(ne00 >= 512) &&
(octx->ctx->vtcm_base != NULL) &&
(octx->ctx->vtcm_size >= vtcm_needed);
if (use_dma) {
octx->src0_spad.size_per_thread = 2 * src_row_size_aligned;
octx->dst_spad.size_per_thread = 2 * dst_row_size_aligned;
octx->src0_spad.size = n_threads * octx->src0_spad.size_per_thread;
octx->dst_spad.size = n_threads * octx->dst_spad.size_per_thread;
octx->src0_spad.data = octx->ctx->vtcm_base;
octx->dst_spad.data = octx->src0_spad.data + octx->src0_spad.size;
}
struct htp_pad_context pctx = {
.octx = octx,
.lp0 = lp0, .rp0 = rp0,
.lp1 = lp1, .rp1 = rp1,
.lp2 = lp2, .rp2 = rp2,
.lp3 = lp3, .rp3 = rp3,
.nrows_per_thread = (total_dst_rows + n_threads - 1) / n_threads,
.total_dst_rows = total_dst_rows,
.type_size = type_size,
.src_row_size = src_row_size,
.src_row_size_aligned = src_row_size_aligned,
.dst_row_size = dst_row_size,
.dst_row_size_aligned = dst_row_size_aligned,
};
FARF(HIGH, "pad-hvx%s%s: (%ux%ux%ux%u) -> (%ux%ux%ux%u) pads=(%d,%d,%d,%d,%d,%d,%d,%d)\n",
circular ? "-circ" : "",
use_dma ? "-dma" : "",
src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3],
dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3],
lp0, rp0, lp1, rp1, lp2, rp2, lp3, rp3);
if (circular && use_dma) { worker_pool_run_func(octx->ctx->worker_pool, pad_job_per_thread_hvx_circular_dma, &pctx, n_threads); }
else if (circular) { worker_pool_run_func(octx->ctx->worker_pool, pad_job_per_thread_hvx_circular, &pctx, n_threads); }
else if (use_dma) { worker_pool_run_func(octx->ctx->worker_pool, pad_job_per_thread_hvx_dma, &pctx, n_threads); }
else { worker_pool_run_func(octx->ctx->worker_pool, pad_job_per_thread_hvx, &pctx, n_threads); }
return HTP_STATUS_OK;
}