/*
* Copyright (C) 2012 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "rsCpuIntrinsic.h"
#include "rsCpuIntrinsicInlines.h"
#include "rsCpuBLASDispatch.h"
#include "eight_bit_int_gemm.h"
namespace android {
namespace renderscript {
class RsdCpuScriptIntrinsicBLAS : public RsdCpuScriptIntrinsic {
public:
void invokeForEach(uint32_t slot,
const Allocation ** ain,
uint32_t inLen,
Allocation * aout,
const void * usr,
uint32_t usrLen,
const RsScriptCall *sc) override;
void populateScript(Script *) override;
~RsdCpuScriptIntrinsicBLAS() override;
RsdCpuScriptIntrinsicBLAS(RsdCpuReferenceImpl *ctx, const Script *s);
protected:
uint8_t a_offset = 0;
uint8_t b_offset = 0;
uint8_t c_offset = 0;
#ifdef RS_COMPATIBILITY_LIB
bool isBlasLibInitialized = false;
#endif
static void kernelBNNM(size_t m, size_t n, size_t k,
const uint8_t* a, uint8_t a_offset, size_t lda,
const uint8_t* b, uint8_t b_offset, size_t ldb,
uint8_t* c, int32_t c_offset, size_t ldc,
int32_t c_mult_int);
};
void RsdCpuScriptIntrinsicBLAS::populateScript(Script *s) {
s->mHal.info.exportedVariableCount = 0;
}
static void initABC(const Allocation ** ain,
size_t size,
void** A,
void** B,
void** C,
int* lda,
int* ldb,
int* ldc)
{
if (ain[0]) {
*A = ain[0]->mHal.drvState.lod[0].mallocPtr;
*lda = (int)(ain[0]->mHal.drvState.lod[0].stride/size);
}
if (ain[1]) {
*B = ain[1]->mHal.drvState.lod[0].mallocPtr;
*ldb = (int)(ain[1]->mHal.drvState.lod[0].stride/size);
}
if (ain[2]) {
*C = ain[2]->mHal.drvState.lod[0].mallocPtr;
*ldc = (int)(ain[2]->mHal.drvState.lod[0].stride/size);
}
}
// Routine to setup LaunchStruct for GEMM callback.
static void setupGEMM(MTLaunchStructForEachBlas *mtls, const Allocation **ain, RsBlasCall* call,
RsdCpuReferenceImpl *ctx) {
uint32_t mm, nn, kk;
mm = call->M;
nn = call->N;
kk = call->K;
memset(mtls, 0, sizeof(MTLaunchStructForEachBlas));
mtls->rs = ctx;
mtls->sc = call;
mtls->dimPtr = &mtls->fep.dim;
mtls->fep.dim.x = nn;
mtls->fep.dim.y = mm;
mtls->fep.dim.z = kk;
if (ain) {
memcpy(mtls->ains, ain, 3 * sizeof(ain[0]));
}
uint32_t elementBytes = 4;
if (ain[0]) {
elementBytes = ain[0]->getType()->getElement()->getSizeBytes();
}
const uint32_t MIN_SIZE_TO_TILE = 64 * 1024 / elementBytes;
const uint32_t MAX_WORK_PER_THREAD = 512 / elementBytes;
const uint32_t THREAD_COUNT = ctx->getThreadCount();
uint32_t tileSizeN = 0;
uint32_t tileSizeM = 0;
// Do not tile the matrix if:
// 1. It is too small comparing to the other matrix.
// 2. It is too small comparing to MIN_SIZE_TO_TILE .
if (nn * kk > MIN_SIZE_TO_TILE && nn * THREAD_COUNT > mm) {
tileSizeN = rsMin(nn / THREAD_COUNT, MAX_WORK_PER_THREAD);
}
if (mm * kk > MIN_SIZE_TO_TILE && mm * THREAD_COUNT > nn) {
tileSizeM = rsMin(mm / THREAD_COUNT, MAX_WORK_PER_THREAD);
}
mtls->numTileM = 1;
mtls->numTileN = 1;
mtls->tileSizeM = mm;
mtls->tileSizeN = nn;
// If tiling is needed, compute the number of slices for A & B.
mtls->isThreadable = (tileSizeM > 0 || tileSizeN > 0);
if (tileSizeM) {
mtls->numTileM += (mm - 1) / tileSizeM;
mtls->tileSizeM = tileSizeM;
}
if (tileSizeN) {
mtls->numTileN += (nn - 1) / tileSizeN;
mtls->tileSizeN = tileSizeN;
}
mtls->mSliceNum = 0;
}
// Generic GEMM callback routine.
template <typename T_data, typename T_param, typename Func>
static void walk_tiled_gemm(Func blasFunc, T_param alpha, T_param beta, int vecSize,
RsBlasCall* call, const MTLaunchStructForEachBlas *mtls) {
// setup BLAS enum args
enum CBLAS_TRANSPOSE TransA = (enum CBLAS_TRANSPOSE)call->transA;
enum CBLAS_TRANSPOSE TransB = (enum CBLAS_TRANSPOSE)call->transB;
void *A = nullptr;
void *B = nullptr;
void *C = nullptr;
int lda = 0, ldb = 0, ldc = 0;
const Allocation *ain[RS_KERNEL_INPUT_LIMIT];
ain[0] = mtls->ains[0];
ain[1] = mtls->ains[1];
ain[2] = mtls->ains[2];
initABC(ain, sizeof(T_data) * vecSize, &A, &B, &C, &lda, &ldb, &ldc);
// Determin the stride of the tiled matrices.
int mStride = (TransA == CblasNoTrans) ? lda : 1;
int nStride = (TransB == CblasNoTrans) ? 1 : ldb;
while (1) {
uint32_t slice = (uint32_t)__sync_fetch_and_add(&mtls->mSliceNum, 1);
uint32_t mStart = (slice % mtls->numTileM) * mtls->tileSizeM;
uint32_t mEnd = mStart + mtls->tileSizeM;
mEnd = rsMin(mEnd, (uint32_t)call->M);
if (mEnd <= mStart) {
return;
}
uint32_t nStart = (slice / mtls->numTileM) * mtls->tileSizeN;
uint32_t nEnd = nStart + mtls->tileSizeN;
nEnd = rsMin(nEnd, (uint32_t)call->N);
if (nEnd <= nStart) {
return;
}
blasFunc(CblasRowMajor, TransA, TransB,
mEnd - mStart, nEnd - nStart, call->K, alpha,
(T_data *)A + mStart * mStride * vecSize, lda,
(T_data *)B + nStart * nStride * vecSize, ldb, beta,
(T_data *)C + (mStart * ldc + nStart) * vecSize, ldc);
}
}
// SGEMM callback
static void walk_2d_sgemm(void *usr, uint32_t idx) {
const MTLaunchStructForEachBlas *mtls = (const MTLaunchStructForEachBlas *)usr;
RsBlasCall* call = (RsBlasCall*) mtls->sc;
float alpha = call->alpha.f;
float beta = call->beta.f;
walk_tiled_gemm<float, float, FnPtr_cblas_sgemm>(cblas_sgemm, alpha, beta, 1, call, mtls);
}
// DGEMM callback
static void walk_2d_dgemm(void *usr, uint32_t idx) {
const MTLaunchStructForEachBlas *mtls = (const MTLaunchStructForEachBlas *)usr;
RsBlasCall* call = (RsBlasCall*) mtls->sc;
double alpha = call->alpha.d;
double beta = call->beta.d;
walk_tiled_gemm<double, double, FnPtr_cblas_dgemm>(cblas_dgemm, alpha, beta, 1, call, mtls);
}
// CGEMM callback
static void walk_2d_cgemm(void *usr, uint32_t idx) {
const MTLaunchStructForEachBlas *mtls = (const MTLaunchStructForEachBlas *)usr;
RsBlasCall* call = (RsBlasCall*) mtls->sc;
void * alpha = (void *)&call->alpha.c;
void * beta = (void *)&call->beta.c;
walk_tiled_gemm<float, void *, FnPtr_cblas_cgemm>(cblas_cgemm, alpha, beta, 2, call, mtls);
}
// ZGEMM callback
static void walk_2d_zgemm(void *usr, uint32_t idx) {
const MTLaunchStructForEachBlas *mtls = (const MTLaunchStructForEachBlas *)usr;
RsBlasCall* call = (RsBlasCall*) mtls->sc;
void * alpha = (void *)&call->alpha.z;
void * beta = (void *)&call->beta.z;
walk_tiled_gemm<double, void *, FnPtr_cblas_zgemm>(cblas_zgemm, alpha, beta, 2, call, mtls);
}
void RsdCpuScriptIntrinsicBLAS::invokeForEach(uint32_t slot,
const Allocation ** ain,
uint32_t inLen,
Allocation * aout,
const void * usr,
uint32_t usrLen,
const RsScriptCall *sc) {
RsBlasCall* call = (RsBlasCall*) usr;
// setup BLAS enum args
enum CBLAS_TRANSPOSE TransA = (enum CBLAS_TRANSPOSE)call->transA;
enum CBLAS_TRANSPOSE TransB = (enum CBLAS_TRANSPOSE)call->transB;
enum CBLAS_UPLO Uplo = (enum CBLAS_UPLO)call->uplo;
enum CBLAS_DIAG Diag = (enum CBLAS_DIAG)call->diag;
enum CBLAS_SIDE Side = (enum CBLAS_SIDE)call->side;
void *A = nullptr;
void *B = nullptr;
void *C = nullptr;
void *X = nullptr;
void *Y = nullptr;
int lda = 0, ldb = 0, ldc = 0;
MTLaunchStructForEachBlas mtls;
#ifdef RS_COMPATIBILITY_LIB
// Allow BNNM even without libblas
if (call->func != RsBlas_bnnm && !isBlasLibInitialized) {
if (!loadBLASLib()) {
ALOGE("Failed to load the BLAS lib, IntrinsicBLAS NOT supported!\n");
return;
}
isBlasLibInitialized = true;
}
#endif
switch (call->func) {
// Level 1 BLAS: returns into a 1D Allocation
// Level 2 BLAS
case (RsBlas_sgemv):
initABC(ain, sizeof(float), &A, &X, &Y, &lda, &ldb, &ldc);
cblas_sgemv(CblasRowMajor, TransA, call->M, call->N, call->alpha.f, (float*)A,
lda, (float*)X, call->incX, call->beta.f, (float*)Y, call->incY);
break;
case (RsBlas_sgbmv):
initABC(ain, sizeof(float), &A, &X, &Y, &lda, &ldb, &ldc);
cblas_sgbmv(CblasRowMajor, TransA, call->M, call->N, call->KL, call->KU,
call->alpha.f, (float*)A, lda, (float*)X, call->incX,
call->beta.f, (float*)Y, call->incY);
break;
case (RsBlas_strmv):
initABC(ain, sizeof(float), &A, &X, nullptr, &lda, &ldb, nullptr);
cblas_strmv(CblasRowMajor, Uplo, TransA, Diag, call->N, (float*)A,
lda, (float*)X, call->incX);
break;
case (RsBlas_stbmv):
initABC(ain, sizeof(float), &A, &X, nullptr, &lda, &ldb, nullptr);
cblas_stbmv(CblasRowMajor, Uplo, TransA, Diag, call->N, call->K, (float*)A,
lda, (float*)X, call->incX);
break;
// stpmv takes a packed 1D Allocation only
case (RsBlas_stpmv):
initABC(ain, sizeof(float), &A, &X, nullptr, &lda, &ldb, nullptr);
cblas_stpmv(CblasRowMajor, Uplo, TransA, Diag, call->N, (float*)A,
(float*)X, call->incX);
break;
case (RsBlas_strsv):
initABC(ain, sizeof(float), &A, &X, nullptr, &lda, &ldb, nullptr);
cblas_strsv(CblasRowMajor, Uplo, TransA, Diag, call->N, (float*)A, lda,
(float*)X, call->incX);
break;
case (RsBlas_stbsv):
initABC(ain, sizeof(float), &A, &X, nullptr, &lda, &ldb, nullptr);
cblas_stbsv(CblasRowMajor, Uplo, TransA, Diag, call->N, call->K, (float*)A,
lda, (float*)X, call->incX);
break;
case (RsBlas_stpsv):
initABC(ain, sizeof(float), &A, &X, nullptr, &lda, &ldb, nullptr);
cblas_stpsv(CblasRowMajor, Uplo, TransA, Diag, call->N, (float*)A,
(float*)X, call->incX);
break;
case (RsBlas_dgemv):
initABC(ain, sizeof(double), &A, &X, &Y, &lda, &ldb, &ldc);
cblas_dgemv(CblasRowMajor, TransA, call->M, call->N, call->alpha.d, (double*)A,
lda, (double*)X, call->incX, call->beta.d, (double*)Y, call->incY);
break;
case (RsBlas_dgbmv):
initABC(ain, sizeof(double), &A, &X, &Y, &lda, &ldb, &ldc);
cblas_dgbmv(CblasRowMajor, TransA, call->M, call->N, call->KL, call->KU,
call->alpha.d, (double*)A, lda, (double*)X, call->incX,
call->beta.d, (double*)Y, call->incY);
break;
case (RsBlas_dtrmv):
initABC(ain, sizeof(double), &A, &X, nullptr, &lda, &ldb, nullptr);
cblas_dtrmv(CblasRowMajor, Uplo, TransA, Diag, call->N, (double*)A,
lda, (double*)X, call->incX);
break;
case (RsBlas_dtbmv):
initABC(ain, sizeof(double), &A, &X, nullptr, &lda, &ldb, nullptr);
cblas_dtbmv(CblasRowMajor, Uplo, TransA, Diag, call->N, call->K, (double*)A,
lda, (double*)X, call->incX);
break;
// stpmv takes a packed 1D Allocation only
case (RsBlas_dtpmv):
initABC(ain, sizeof(double), &A, &X, nullptr, &lda, &ldb, nullptr);
cblas_dtpmv(CblasRowMajor, Uplo, TransA, Diag, call->N, (double*)A,
(double*)X, call->incX);
break;
case (RsBlas_dtrsv):
initABC(ain, sizeof(double), &A, &X, nullptr, &lda, &ldb, nullptr);
cblas_dtrsv(CblasRowMajor, Uplo, TransA, Diag, call->N, (double*)A, lda,
(double*)X, call->incX);
break;
case (RsBlas_dtbsv):
initABC(ain, sizeof(double), &A, &X, nullptr, &lda, &ldb, nullptr);
cblas_dtbsv(CblasRowMajor, Uplo, TransA, Diag, call->N, call->K, (double*)A,
lda, (double*)X, call->incX);
break;
case (RsBlas_dtpsv):
initABC(ain, sizeof(double), &A, &X, nullptr, &lda, &ldb, nullptr);
cblas_dtpsv(CblasRowMajor, Uplo, TransA, Diag, call->N, (double*)A,
(double*)X, call->incX);
break;
case (RsBlas_cgemv):
initABC(ain, sizeof(float)*2, &A, &X, &Y, &lda, &ldb, &ldc);
cblas_cgemv(CblasRowMajor, TransA, call->M, call->N, (void*)&call->alpha.c, (void*)A,
lda, (void*)X, call->incX, (void*)&call->beta.c, (void*)Y, call->incY);
break;
case (RsBlas_cgbmv):
initABC(ain, sizeof(float)*2, &A, &X, &Y, &lda, &ldb, &ldc);
cblas_cgbmv(CblasRowMajor, TransA, call->M, call->N, call->KL, call->KU,
(void*)&call->alpha.c, (void*)A, lda, (void*)X, call->incX,
(void*)&call->beta.c, (void*)Y, call->incY);
break;
case (RsBlas_ctrmv):
initABC(ain, sizeof(float)*2, &A, &X, nullptr, &lda, &ldb, nullptr);
cblas_ctrmv(CblasRowMajor, Uplo, TransA, Diag, call->N, (void*)A,
lda, (void*)X, call->incX);
break;
case (RsBlas_ctbmv):
initABC(ain, sizeof(float)*2, &A, &X, nullptr, &lda, &ldb, nullptr);
cblas_ctbmv(CblasRowMajor, Uplo, TransA, Diag, call->N, call->K, (void*)A,
lda, (void*)X, call->incX);
break;
// stpmv takes a packed 1D Allocation only
case (RsBlas_ctpmv):
initABC(ain, sizeof(float)*2, &A, &X, nullptr, &lda, &ldb, nullptr);
cblas_ctpmv(CblasRowMajor, Uplo, TransA, Diag, call->N, (void*)A,
(void*)X, call->incX);
break;
case (RsBlas_ctrsv):
initABC(ain, sizeof(float)*2, &A, &X, nullptr, &lda, &ldb, nullptr);
cblas_ctrsv(CblasRowMajor, Uplo, TransA, Diag, call->N, (void*)A, lda,
(void*)X, call->incX);
break;
case (RsBlas_ctbsv):
initABC(ain, sizeof(float)*2, &A, &X, nullptr, &lda, &ldb, nullptr);
cblas_ctbsv(CblasRowMajor, Uplo, TransA, Diag, call->N, call->K, (void*)A,
lda, (void*)X, call->incX);
break;
case (RsBlas_ctpsv):
initABC(ain, sizeof(float)*2, &A, &X, nullptr, &lda, &ldb, nullptr);
cblas_ctpsv(CblasRowMajor, Uplo, TransA, Diag, call->N, (void*)A,
(void*)X, call->incX);
break;
case (RsBlas_zgemv):
initABC(ain, sizeof(double)*2, &A, &X, &Y, &lda, &ldb, &ldc);
cblas_zgemv(CblasRowMajor, TransA, call->M, call->N, (void*)&call->alpha.z, (void*)A,
lda, (void*)X, call->incX, (void*)&call->beta.z, (void*)Y, call->incY);
break;
case (RsBlas_zgbmv):
initABC(ain, sizeof(double)*2, &A, &X, &Y, &lda, &ldb, &ldc);
cblas_zgbmv(CblasRowMajor, TransA, call->M, call->N, call->KL, call->KU,
(void*)&call->alpha.z, (void*)A, lda, (void*)X, call->incX,
(void*)&call->beta.z, (void*)Y, call->incY);
break;
case (RsBlas_ztrmv):
initABC(ain, sizeof(double)*2, &A, &X, nullptr, &lda, &ldb, nullptr);
cblas_ztrmv(CblasRowMajor, Uplo, TransA, Diag, call->N, (void*)A,
lda, (void*)X, call->incX);
break;
case (RsBlas_ztbmv):
initABC(ain, sizeof(double)*2, &A, &X, nullptr, &lda, &ldb, nullptr);
cblas_ztbmv(CblasRowMajor, Uplo, TransA, Diag, call->N, call->K, (void*)A,
lda, (void*)X, call->incX);
break;
// stpmv takes a packed 1D Allocation only
case (RsBlas_ztpmv):
initABC(ain, sizeof(double)*2, &A, &X, nullptr, &lda, &ldb, nullptr);
cblas_ztpmv(CblasRowMajor, Uplo, TransA, Diag, call->N, (void*)A,
(void*)X, call->incX);
break;
case (RsBlas_ztrsv):
initABC(ain, sizeof(double)*2, &A, &X, nullptr, &lda, &ldb, nullptr);
cblas_ztrsv(CblasRowMajor, Uplo, TransA, Diag, call->N, (void*)A, lda,
(void*)X, call->incX);
break;
case (RsBlas_ztbsv):
initABC(ain, sizeof(double)*2, &A, &X, nullptr, &lda, &ldb, nullptr);
cblas_ztbsv(CblasRowMajor, Uplo, TransA, Diag, call->N, call->K, (void*)A,
lda, (void*)X, call->incX);
break;
case (RsBlas_ztpsv):
initABC(ain, sizeof(double)*2, &A, &X, nullptr, &lda, &ldb, nullptr);
cblas_ztpsv(CblasRowMajor, Uplo, TransA, Diag, call->N, (void*)A,
(void*)X, call->incX);
break;
// S and D only
case (RsBlas_ssymv):
initABC(ain, sizeof(float), &A, &X, &Y, &lda, &ldb, &ldc);
cblas_ssymv(CblasRowMajor, Uplo, call->N, call->alpha.f, (float*)A, lda,
(float*)X, call->incX, call->beta.f, (float*)Y, call->incY);
break;
case (RsBlas_ssbmv):
initABC(ain, sizeof(float), &A, &X, &Y, &lda, &ldb, &ldc);
cblas_ssbmv(CblasRowMajor, Uplo, call->N, call->K, call->alpha.f,
(float*)A, lda, (float*)X, call->incX, call->beta.f,
(float*)Y, call->incY);
break;
//sspmv requires a packed 1D Allocation
case (RsBlas_sspmv):
initABC(ain, sizeof(float), &A, &X, &Y, &lda, &ldb, &ldc);
cblas_sspmv(CblasRowMajor, Uplo, call->N, call->alpha.f, (float*)A,
(float*)X, call->incX, call->beta.f, (float*)Y, call->incY);
break;
// following calls have init reordered because A is output matrix
case (RsBlas_sger):
initABC(ain, sizeof(float), &X, &Y, &A, &ldb, &ldc, &lda);
cblas_sger(CblasRowMajor, call->M, call->N, call->alpha.f, (float*)X,
call->incX, (float*)Y, call->incY, (float*)A, lda);
break;
case (RsBlas_ssyr):
initABC(ain, sizeof(float), &X, &A, nullptr, &ldb, &lda, nullptr);
cblas_ssyr(CblasRowMajor, Uplo, call->N, call->alpha.f, (float*)X, call->incX,
(float*)A, lda);
break;
// sspr is packed 1D Allocation A only
case (RsBlas_sspr):
initABC(ain, sizeof(float), &X, &A, nullptr, &ldb, &lda, nullptr);
cblas_sspr(CblasRowMajor, Uplo, call->N, call->alpha.f, (float*)X, call->incX,
(float*)A);
break;
case (RsBlas_ssyr2):
initABC(ain, sizeof(float), &X, &Y, &A, &ldb, &ldc, &lda);
cblas_ssyr2(CblasRowMajor, Uplo, call->N, call->alpha.f, (float*)X, call->incX,
(float*)Y, call->incY, (float*)A, lda);
break;
// sspr2 is packed 1D Allocation A only
case (RsBlas_sspr2):
initABC(ain, sizeof(float), &X, &Y, &A, &ldb, &ldc, &lda);
cblas_sspr2(CblasRowMajor, Uplo, call->N, call->alpha.f, (float*)X, call->incX,
(float*)Y, call->incY, (float*)A);
break;
case (RsBlas_dsymv):
initABC(ain, sizeof(double), &A, &X, &Y, &lda, &ldb, &ldc);
cblas_dsymv(CblasRowMajor, Uplo, call->N, call->alpha.d, (double*)A, lda,
(double*)X, call->incX, call->beta.d, (double*)Y, call->incY);
break;
case (RsBlas_dsbmv):
initABC(ain, sizeof(double), &A, &X, &Y, &lda, &ldb, &ldc);
cblas_dsbmv(CblasRowMajor, Uplo, call->N, call->K, call->alpha.d,
(double*)A, lda, (double*)X, call->incX, call->beta.d,
(double*)Y, call->incY);
break;
// dspmv requires a packed 1D Allocation
case (RsBlas_dspmv):
initABC(ain, sizeof(double), &A, &X, &Y, &lda, &ldb, &ldc);
cblas_dspmv(CblasRowMajor, Uplo, call->N, call->alpha.d, (double*)A,
(double*)X, call->incX, call->beta.d, (double*)Y, call->incY);
break;
// following calls have init reordered because A is output matrix
case (RsBlas_dger):
initABC(ain, sizeof(double), &X, &Y, &A, &ldb, &ldc, &lda);
cblas_dger(CblasRowMajor, call->M, call->N, call->alpha.d, (double*)X,
call->incX, (double*)Y, call->incY, (double*)A, lda);
break;
case (RsBlas_dsyr):
initABC(ain, sizeof(double), &X, &A, nullptr, &ldb, &lda, nullptr);
cblas_dsyr(CblasRowMajor, Uplo, call->N, call->alpha.d, (double*)X, call->incX,
(double*)A, lda);
break;
// dspr is packed 1D Allocation A only
case (RsBlas_dspr):
initABC(ain, sizeof(double), &X, &A, nullptr, &ldb, &lda, nullptr);
cblas_dspr(CblasRowMajor, Uplo, call->N, call->alpha.d, (double*)X, call->incX,
(double*)A);
break;
case (RsBlas_dsyr2):
initABC(ain, sizeof(double), &X, &Y, &A, &ldb, &ldc, &lda);
cblas_dsyr2(CblasRowMajor, Uplo, call->N, call->alpha.d, (double*)X, call->incX,
(double*)Y, call->incY, (double*)A, lda);
break;
// dspr2 is packed 1D Allocation A only
case (RsBlas_dspr2):
initABC(ain, sizeof(double), &X, &Y, &A, &ldb, &ldc, &lda);
cblas_dspr2(CblasRowMajor, Uplo, call->N, call->alpha.d, (double*)X, call->incX,
(double*)Y, call->incY, (double*)A);
break;
// C and Z only
case (RsBlas_chemv):
initABC(ain, sizeof(float)*2, &A, &X, &Y, &lda, &ldb, &ldc);
cblas_chemv(CblasRowMajor, Uplo, call->N, (void*)&call->alpha.c, A, lda,
X, call->incX, (void*)&call->beta.c, Y, call->incY);
break;
case (RsBlas_chbmv):
initABC(ain, sizeof(float)*2, &A, &X, &Y, &lda, &ldb, &ldc);
cblas_chbmv(CblasRowMajor, Uplo, call->N, call->K, (void*)&call->alpha.c,
A, lda, X, call->incX, (void*)&call->beta.c, Y, call->incY);
break;
case (RsBlas_chpmv):
initABC(ain, sizeof(float)*2, &A, &X, &Y, &lda, &ldb, &ldc);
cblas_chpmv(CblasRowMajor, Uplo, call->N, (void*)&call->alpha.c, A,
X, call->incX, (void*)&call->beta.c, Y, call->incY);
break;
case (RsBlas_cgeru):
initABC(ain, sizeof(float)*2, &X, &Y, &A, &ldb, &ldc, &lda);
cblas_cgeru(CblasRowMajor, call->M, call->N, (void*)&call->alpha.c,
X, call->incX, Y, call->incY, A, lda);
break;
case (RsBlas_cgerc):
initABC(ain, sizeof(float)*2, &X, &Y, &A, &ldb, &ldc, &lda);
cblas_cgerc(CblasRowMajor, call->M, call->N, (void*)&call->alpha.c,
X, call->incX, Y, call->incY, A, lda);
break;
case (RsBlas_cher):
initABC(ain, sizeof(float)*2, &X, nullptr, &A, &ldb, nullptr, &lda);
cblas_cher(CblasRowMajor, Uplo, call->N, call->alpha.f,
X, call->incX, A, lda);
break;
// packed 1D Allocations only
case (RsBlas_chpr):
initABC(ain, sizeof(float)*2, &X, nullptr, &A, &ldb, nullptr, &lda);
cblas_chpr(CblasRowMajor, Uplo, call->N, call->alpha.f, X,
call->incX, A);
break;
case (RsBlas_cher2):
initABC(ain, sizeof(float)*2, &X, &Y, &A, &ldb, &ldc, &lda);
cblas_cher2(CblasRowMajor, Uplo, call->N, (void*)&call->alpha.c,
X, call->incX, Y, call->incY, A, lda);
break;
// packed 1D Allocations only
case (RsBlas_chpr2):
initABC(ain, sizeof(float)*2, &X, &Y, &A, &ldb, &ldc, &lda);
cblas_chpr2(CblasRowMajor, Uplo, call->N, (void*)&call->alpha.c, X,
call->incX, Y, call->incY, A);
break;
case (RsBlas_zhemv):
initABC(ain, sizeof(double)*2, &A, &X, &Y, &lda, &ldb, &ldc);
cblas_zhemv(CblasRowMajor, Uplo, call->N, (void*)&call->alpha.z, A, lda,
X, call->incX, (void*)&call->beta.z, Y, call->incY);
break;
case (RsBlas_zhbmv):
initABC(ain, sizeof(double)*2, &A, &X, &Y, &lda, &ldb, &ldc);
cblas_zhbmv(CblasRowMajor, Uplo, call->N, call->K, (void*)&call->alpha.z,
A, lda, X, call->incX, (void*)&call->beta.z, Y, call->incY);
break;
case (RsBlas_zhpmv):
initABC(ain, sizeof(double)*2, &A, &X, &Y, &lda, &ldb, &ldc);
cblas_zhpmv(CblasRowMajor, Uplo, call->N, (void*)&call->alpha.z, A,
X, call->incX, (void*)&call->beta.z, Y, call->incY);
break;
case (RsBlas_zgeru):
initABC(ain, sizeof(double)*2, &X, &Y, &A, &ldb, &ldc, &lda);
cblas_zgeru(CblasRowMajor, call->M, call->N, (void*)&call->alpha.z,
X, call->incX, Y, call->incY, A, lda);
break;
case (RsBlas_zgerc):
initABC(ain, sizeof(double)*2, &X, &Y, &A, &ldb, &ldc, &lda);
cblas_zgerc(CblasRowMajor, call->M, call->N, (void*)&call->alpha.z,
X, call->incX, Y, call->incY, A, lda);
break;
case (RsBlas_zher):
initABC(ain, sizeof(double)*2, &X, nullptr, &A, &ldb, nullptr, &lda);
cblas_zher(CblasRowMajor, Uplo, call->N, call->alpha.d,
X, call->incX, A, lda);
break;
// packed 1D Allocations only
case (RsBlas_zhpr):
initABC(ain, sizeof(double)*2, &X, nullptr, &A, &ldb, nullptr, &lda);
cblas_zhpr(CblasRowMajor, Uplo, call->N, call->alpha.d, X,
call->incX, A);
break;
case (RsBlas_zher2):
initABC(ain, sizeof(double)*2, &X, &Y, &A, &ldb, &ldc, &lda);
cblas_zher2(CblasRowMajor, Uplo, call->N, (void*)&call->alpha.z,
X, call->incX, Y, call->incY, A, lda);
break;
// packed 1D Allocations only
case (RsBlas_zhpr2):
initABC(ain, sizeof(double)*2, &X, &Y, &A, &ldb, &ldc, &lda);
cblas_zhpr2(CblasRowMajor, Uplo, call->N, (void*)&call->alpha.z, X,
call->incX, Y, call->incY, A);
break;
// Level 3 BLAS
case (RsBlas_sgemm):
setupGEMM(&mtls, ain, call, mCtx);
if (mtls.isThreadable) {
mCtx->launchThreads(walk_2d_sgemm, &mtls);
} else {
initABC(ain, sizeof(float), &A, &B, &C, &lda, &ldb, &ldc);
cblas_sgemm(CblasRowMajor, TransA, TransB, call->M, call->N, call->K, call->alpha.f,
(float*)A, lda, (float*)B, ldb, call->beta.f, (float*)C, ldc);
}
break;
case (RsBlas_ssymm):
initABC(ain, sizeof(float), &A, &B, &C, &lda, &ldb, &ldc);
cblas_ssymm(CblasRowMajor, Side, Uplo, call->M, call->N, call->alpha.f, (float*)A,
lda, (float*)B, ldb, call->beta.f, (float*)C, ldc);
break;
case (RsBlas_ssyrk):
initABC(ain, sizeof(float), &A, nullptr, &C, &lda, nullptr, &ldc);
cblas_ssyrk(CblasRowMajor, Uplo, TransA, call->N, call->K, call->alpha.f, (float*)A,
lda, call->beta.f, (float*)C, ldc);
break;
case (RsBlas_ssyr2k):
initABC(ain, sizeof(float), &A, &B, &C, &lda, &ldb, &ldc);
cblas_ssyr2k(CblasRowMajor, Uplo, TransA, call->N, call->K, call->alpha.f, (float*)A,
lda, (float*)B, ldb, call->beta.f, (float*)C, ldc);
break;
case (RsBlas_strmm):
initABC(ain, sizeof(float), &A, &B, nullptr, &lda, &ldb, nullptr);
cblas_strmm(CblasRowMajor, Side, Uplo, TransA, Diag, call->M, call->N, call->alpha.f,
(float*)A, lda, (float*)B, ldb);
break;
case (RsBlas_strsm):
initABC(ain, sizeof(float), &A, &B, nullptr, &lda, &ldb, nullptr);
cblas_strsm(CblasRowMajor, Side, Uplo, TransA, Diag, call->M, call->N, call->alpha.f,
(float*)A, lda, (float*)B, ldb);
break;
case (RsBlas_dgemm):
setupGEMM(&mtls, ain, call, mCtx);
if (mtls.isThreadable) {
mCtx->launchThreads(walk_2d_dgemm, &mtls);
} else {
initABC(ain, sizeof(double), &A, &B, &C, &lda, &ldb, &ldc);
cblas_dgemm(CblasRowMajor, TransA, TransB, call->M, call->N, call->K, call->alpha.d,
(double*)A, lda, (double*)B, ldb, call->beta.d, (double*)C, ldc);
}
break;
case (RsBlas_dsymm):
initABC(ain, sizeof(double), &A, &B, &C, &lda, &ldb, &ldc);
cblas_dsymm(CblasRowMajor, Side, Uplo, call->M, call->N, call->alpha.d, (double*)A,
lda, (double*)B, ldb, call->beta.d, (double*)C, ldc);
break;
case (RsBlas_dsyrk):
initABC(ain, sizeof(double), &A, nullptr, &C, &lda, nullptr, &ldc);
cblas_dsyrk(CblasRowMajor, Uplo, TransA, call->N, call->K, call->alpha.d, (double*)A,
lda, call->beta.d, (double*)C, ldc);
break;
case (RsBlas_dsyr2k):
initABC(ain, sizeof(double), &A, &B, &C, &lda, &ldb, &ldc);
cblas_dsyr2k(CblasRowMajor, Uplo, TransA, call->N, call->K, call->alpha.d, (double*)A,
lda, (double*)B, ldb, call->beta.d, (double*)C, ldc);
break;
case (RsBlas_dtrmm):
initABC(ain, sizeof(double), &A, &B, nullptr, &lda, &ldb, nullptr);
cblas_dtrmm(CblasRowMajor, Side, Uplo, TransA, Diag, call->M, call->N, call->alpha.d,
(double*)A, lda, (double*)B, ldb);
break;
case (RsBlas_dtrsm):
initABC(ain, sizeof(double), &A, &B, nullptr, &lda, &ldb, nullptr);
cblas_dtrsm(CblasRowMajor, Side, Uplo, TransA, Diag, call->M, call->N, call->alpha.d,
(double*)A, lda, (double*)B, ldb);
break;
case (RsBlas_cgemm):
setupGEMM(&mtls, ain, call, mCtx);
if (mtls.isThreadable) {
mCtx->launchThreads(walk_2d_cgemm, &mtls);
} else {
initABC(ain, sizeof(float)*2, &A, &B, &C, &lda, &ldb, &ldc);
cblas_cgemm(CblasRowMajor, TransA, TransB, call->M, call->N, call->K, (void*)&call->alpha.c,
A, lda, B, ldb, (void*)&call->beta.c, C, ldc);
}
break;
case (RsBlas_csymm):
initABC(ain, sizeof(float)*2, &A, &B, &C, &lda, &ldb, &ldc);
cblas_csymm(CblasRowMajor, Side, Uplo, call->M, call->N, (void*)&call->alpha.c, A,
lda, B, ldb, (void*)&call->beta.c, C, ldc);
break;
case (RsBlas_csyrk):
initABC(ain, sizeof(float)*2, &A, nullptr, &C, &lda, nullptr, &ldc);
cblas_csyrk(CblasRowMajor, Uplo, TransA, call->N, call->K, (void*)&call->alpha.c, A,
lda, (void*)&call->beta.c, C, ldc);
break;
case (RsBlas_csyr2k):
initABC(ain, sizeof(float)*2, &A, &B, &C, &lda, &ldb, &ldc);
cblas_csyr2k(CblasRowMajor, Uplo, TransA, call->N, call->K, (void*)&call->alpha.c, A,
lda, B, ldb, (void*)&call->beta.c, C, ldc);
break;
case (RsBlas_ctrmm):
initABC(ain, sizeof(float)*2, &A, &B, nullptr, &lda, &ldb, nullptr);
cblas_ctrmm(CblasRowMajor, Side, Uplo, TransA, Diag, call->M, call->N, (void*)&call->alpha.c,
A, lda, B, ldb);
break;
case (RsBlas_ctrsm):
initABC(ain, sizeof(float)*2, &A, &B, nullptr, &lda, &ldb, nullptr);
cblas_ctrsm(CblasRowMajor, Side, Uplo, TransA, Diag, call->M, call->N, (void*)&call->alpha.c,
A, lda, B, ldb);
break;
case (RsBlas_zgemm):
setupGEMM(&mtls, ain, call, mCtx);
if (mtls.isThreadable) {
mCtx->launchThreads(walk_2d_zgemm, &mtls);
} else {
initABC(ain, sizeof(double)*2, &A, &B, &C, &lda, &ldb, &ldc);
cblas_zgemm(CblasRowMajor, TransA, TransB, call->M, call->N, call->K, (void*)&call->alpha.z,
A, lda, B, ldb, (void*)&call->beta.z, C, ldc);
}
break;
case (RsBlas_zsymm):
initABC(ain, sizeof(double)*2, &A, &B, &C, &lda, &ldb, &ldc);
cblas_zsymm(CblasRowMajor, Side, Uplo, call->M, call->N, (void*)&call->alpha.z, A,
lda, B, ldb, (void*)&call->beta.z, C, ldc);
break;
case (RsBlas_zsyrk):
initABC(ain, sizeof(double)*2, &A, nullptr, &C, &lda, nullptr, &ldc);
cblas_zsyrk(CblasRowMajor, Uplo, TransA, call->N, call->K, (void*)&call->alpha.z, A,
lda, (void*)&call->beta.z, C, ldc);
break;
case (RsBlas_zsyr2k):
initABC(ain, sizeof(double)*2, &A, &B, &C, &lda, &ldb, &ldc);
cblas_zsyr2k(CblasRowMajor, Uplo, TransA, call->N, call->K, (void*)&call->alpha.z, A,
lda, B, ldb, (void*)&call->beta.z, C, ldc);
break;
case (RsBlas_ztrmm):
initABC(ain, sizeof(double)*2, &A, &B, nullptr, &lda, &ldb, nullptr);
cblas_ztrmm(CblasRowMajor, Side, Uplo, TransA, Diag, call->M, call->N, (void*)&call->alpha.z,
A, lda, B, ldb);
break;
case (RsBlas_ztrsm):
initABC(ain, sizeof(double)*2, &A, &B, nullptr, &lda, &ldb, nullptr);
cblas_ztrsm(CblasRowMajor, Side, Uplo, TransA, Diag, call->M, call->N, (void*)&call->alpha.z,
A, lda, B, ldb);
break;
// Level 3 C and Z only
case (RsBlas_chemm):
initABC(ain, sizeof(float)*2, &A, &B, &C, &lda, &ldb, &ldc);
cblas_chemm(CblasRowMajor, Side, Uplo, call->M, call->N, (void*)&call->alpha.c, A, lda,
B, ldb, (void*)&call->beta.c, C, ldc);
break;
case (RsBlas_cherk):
initABC(ain, sizeof(float)*2, &A, nullptr, &C, &lda, nullptr, &ldc);
cblas_cherk(CblasRowMajor, Uplo, TransA, call->N, call->K, call->alpha.f, A, lda,
call->beta.f, C, ldc);
break;
case (RsBlas_cher2k):
initABC(ain, sizeof(float)*2, &A, &B, &C, &lda, &ldb, &ldc);
cblas_cher2k(CblasRowMajor, Uplo, TransA, call->N, call->K, (void*)&call->alpha.c, A, lda,
B, ldb, call->beta.f, C, ldc);
break;
case (RsBlas_zhemm):
initABC(ain, sizeof(double)*2, &A, &B, &C, &lda, &ldb, &ldc);
cblas_zhemm(CblasRowMajor, Side, Uplo, call->M, call->N, (void*)&call->alpha.z, A, lda,
B, ldb, (void*)&call->beta.z, C, ldc);
break;
case (RsBlas_zherk):
initABC(ain, sizeof(double)*2, &A, nullptr, &C, &lda, nullptr, &ldc);
cblas_zherk(CblasRowMajor, Uplo, TransA, call->N, call->K, call->alpha.d, A, lda,
call->beta.d, C, ldc);
break;
case (RsBlas_zher2k):
initABC(ain, sizeof(double)*2, &A, &B, &C, &lda, &ldb, &ldc);
cblas_zher2k(CblasRowMajor, Uplo, TransA, call->N, call->K, (void*)&call->alpha.z, A, lda,
B, ldb, call->beta.d, C, ldc);
break;
case (RsBlas_bnnm):
initABC(ain, sizeof(uint8_t), &A, &B, &C, &lda, &ldb, &ldc);
kernelBNNM(call->M, call->N, call->K,
(const uint8_t*)A, call->a_offset, lda,
(const uint8_t*)B, call->b_offset, ldb,
(uint8_t*)C, call->c_offset, ldc,
call->c_mult_int);
break;
default:
ALOGE("unimplemented\n");
}
}
void RsdCpuScriptIntrinsicBLAS::kernelBNNM(size_t m, size_t n, size_t k,
const uint8_t* a, uint8_t a_offset, size_t lda,
const uint8_t* b, uint8_t b_offset, size_t ldb,
uint8_t* c, int32_t c_offset, size_t ldc,
int32_t c_mult_int) {
const int c_shift = 21;
#if defined(ARCH_ARM_HAVE_VFP) || defined(ARCH_ARM_USE_INTRINSICS)
// Non-optimized path for ARMv7 devices without SIMD instructions.
if (!gArchUseSIMD) {
/*
* Calculations are done in 1.10.21 fixed-point format for the final output,
* just before there's a shift down to drop the fractional parts. The output
* values are gated to 0 to 255 to fit in a byte, but the 10-bit format
* gives some headroom to avoid wrapping around on small overflows.
*/
size_t i = 0, j = 0, l = 0;
for (j = 0; j < n; j++) {
for (i = 0; i < m; i++) {
int32_t total = 0;
for (l = 0; l < k; l++) {
const int a_index = ((i * lda) + l);
const uint8_t a_as_byte = a[a_index];
const int32_t a_as_int = (((int32_t)(a_as_byte)) - a_offset);
const int b_index = ((j * ldb) + l);
const uint8_t b_as_byte = b[b_index];
const int32_t b_as_int = (((int32_t)(b_as_byte)) - b_offset);
const int32_t mult_as_int = (a_as_int * b_as_int);
total += mult_as_int;
}
const int c_index = ((ldc * i) + j);
int32_t output =
((((total + c_offset) * c_mult_int) + (1 << (c_shift - 1)))
>> c_shift);
if (output > 255) {
output = 255;
}
if (output < 0) {
output = 0;
}
c[c_index] = (uint8_t)(output);
}
}
return;
}
#endif
// Using gemmlowp to calculate the low precision 8 bit GEMM.
// Set MaxNumThreads to 0. The value 0 lets the implementation query
// the system to determine the number of hardware threads
gemmlowp::eight_bit_int_gemm::SetMaxNumThreads(0);
bool transpose_a = true;
bool transpose_b = false;
bool transpose_c = true;
gemmlowp::eight_bit_int_gemm::EightBitIntGemm(transpose_a, transpose_b, transpose_c,
m, n, k, a, -a_offset, lda,
b, -b_offset, ldb, c, c_offset,
c_mult_int, c_shift, ldc,
gemmlowp::eight_bit_int_gemm::BitDepthSetting::A8B8);
}
RsdCpuScriptIntrinsicBLAS::RsdCpuScriptIntrinsicBLAS(RsdCpuReferenceImpl *ctx,
const Script *s)
: RsdCpuScriptIntrinsic(ctx, s, nullptr, RS_SCRIPT_INTRINSIC_ID_BLAS) {
}
RsdCpuScriptIntrinsicBLAS::~RsdCpuScriptIntrinsicBLAS() {
}
RsdCpuScriptImpl * rsdIntrinsic_BLAS(RsdCpuReferenceImpl *ctx,
const Script *s, const Element *e) {
return new RsdCpuScriptIntrinsicBLAS(ctx, s);
}
} // namespace renderscript
} // namespace android