/*
* Copyright (C) 2012 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <sys/mman.h>
#include <unistd.h>
#include "rsCpuIntrinsic.h"
#include "rsCpuIntrinsicInlines.h"
#include <sys/mman.h>
#include <stddef.h>
#include <stdint.h>
#include <stdlib.h>
//#include <utils/StopWatch.h>
/* uint kernel
* Q0 D0: Load slot for R
* D1: Load slot for G
* Q1 D2: Load slot for B
* D3: Load slot for A
* Q2 D4: Matrix
* D5: =
* Q3 D6: =
* D7: =
* Q4 D8: Add R
* D9:
* Q5 D10: Add G
* D11:
* Q6 D12: Add B
* D13:
* Q7 D14: Add A
* D15:
* Q8 D16: I32: R Sum
* D17:
* Q9 D18: I32: G Sum
* D19:
* Q10 D20: I32: B Sum
* D21:
* Q11 D22: I32: A Sum
* D23:
* Q12 D24: U16: expanded R
* D25:
* Q13 D26: U16: expanded G
* D27:
* Q14 D28: U16: expanded B
* D29:
* Q15 D30: U16: expanded A
* D31:
*
*/
/* float kernel
* Q0 D0: Load slot for R
* D1: =
* Q1 D2: Load slot for G
* D3: =
* Q2 D4: Load slot for B
* D5: =
* Q3 D6: Load slot for A
* D7: =
* Q4 D8: Matrix
* D9: =
* Q5 D10: =
* D11: =
* Q6 D12: =
* D13: =
* Q7 D14: =
* D15: =
* Q8 D16: Add R
* D17: =
* Q9 D18: Add G
* D19: =
* Q10 D20: Add B
* D21: =
* Q11 D22: Add A
* D23: =
* Q12 D24: Sum R
* D25: =
* Q13 D26: Sum G
* D27: =
* Q14 D28: Sum B
* D29: =
* Q15 D30: Sum A
* D31: =
*
*/
using namespace android;
using namespace android::renderscript;
namespace android {
namespace renderscript {
typedef union {
uint64_t key;
struct {
uint32_t inVecSize :2; // [0 - 1]
uint32_t outVecSize :2; // [2 - 3]
uint32_t inType :4; // [4 - 7]
uint32_t outType :4; // [8 - 11]
uint32_t dot :1; // [12]
uint32_t _unused1 :1; // [13]
uint32_t copyAlpha :1; // [14]
uint32_t _unused2 :1; // [15]
uint32_t coeffMask :16; // [16-31]
uint32_t addMask :4; // [32-35]
} u;
} Key_t;
//Re-enable when intrinsic is fixed
#if defined(ARCH_ARM64_USE_INTRINSICS)
typedef struct {
void (*column[4])(void);
void (*store)(void);
void (*load)(void);
void (*store_end)(void);
void (*load_end)(void);
} FunctionTab_t;
extern "C" void rsdIntrinsicColorMatrix_int_K(
void *out, void const *in, size_t count,
FunctionTab_t const *fns,
int16_t const *mult, int32_t const *add);
extern "C" void rsdIntrinsicColorMatrix_float_K(
void *out, void const *in, size_t count,
FunctionTab_t const *fns,
float const *mult, float const *add);
/* The setup functions fill in function tables to be used by above functions;
* this code also eliminates jump-to-another-jump cases by short-circuiting
* empty functions. While it's not performance critical, it works out easier
* to write the set-up code in assembly than to try to expose the same symbols
* and write the code in C.
*/
extern "C" void rsdIntrinsicColorMatrixSetup_int_K(
FunctionTab_t *fns,
uint32_t mask, int dt, int st);
extern "C" void rsdIntrinsicColorMatrixSetup_float_K(
FunctionTab_t *fns,
uint32_t mask, int dt, int st);
#endif
class RsdCpuScriptIntrinsicColorMatrix : public RsdCpuScriptIntrinsic {
public:
void populateScript(Script *) override;
void setGlobalVar(uint32_t slot, const void *data, size_t dataLength) override;
~RsdCpuScriptIntrinsicColorMatrix() override;
RsdCpuScriptIntrinsicColorMatrix(RsdCpuReferenceImpl *ctx, const Script *s, const Element *e);
void preLaunch(uint32_t slot, const Allocation ** ains,
uint32_t inLen, Allocation * aout, const void * usr,
uint32_t usrLen, const RsScriptCall *sc) override;
protected:
float fp[16];
float fpa[4];
// The following four fields are read as constants
// by the SIMD assembly code.
short ip[16];
int ipa[4];
float tmpFp[16];
float tmpFpa[4];
#if defined(ARCH_ARM64_USE_INTRINSICS)
FunctionTab_t mFnTab;
#endif
static void kernel(const RsExpandKernelDriverInfo *info,
uint32_t xstart, uint32_t xend,
uint32_t outstep);
void updateCoeffCache(float fpMul, float addMul);
Key_t mLastKey;
unsigned char *mBuf;
size_t mBufSize;
Key_t computeKey(const Element *ein, const Element *eout);
bool build(Key_t key);
void (*mOptKernel)(void *dst, const void *src, const short *coef, uint32_t count);
};
}
}
Key_t RsdCpuScriptIntrinsicColorMatrix::computeKey(
const Element *ein, const Element *eout) {
Key_t key;
key.key = 0;
// Compute a unique code key for this operation
// Add to the key the input and output types
bool hasFloat = false;
if (ein->getType() == RS_TYPE_FLOAT_32) {
hasFloat = true;
key.u.inType = RS_TYPE_FLOAT_32;
rsAssert(key.u.inType == RS_TYPE_FLOAT_32);
}
if (eout->getType() == RS_TYPE_FLOAT_32) {
hasFloat = true;
key.u.outType = RS_TYPE_FLOAT_32;
rsAssert(key.u.outType == RS_TYPE_FLOAT_32);
}
// Mask in the bits indicating which coefficients in the
// color matrix are needed.
if (hasFloat) {
for (uint32_t i=0; i < 16; i++) {
if (fabs(fp[i]) != 0.f) {
key.u.coeffMask |= 1 << i;
}
}
if (fabs(fpa[0]) != 0.f) key.u.addMask |= 0x1;
if (fabs(fpa[1]) != 0.f) key.u.addMask |= 0x2;
if (fabs(fpa[2]) != 0.f) key.u.addMask |= 0x4;
if (fabs(fpa[3]) != 0.f) key.u.addMask |= 0x8;
} else {
for (uint32_t i=0; i < 16; i++) {
if (ip[i] != 0) {
key.u.coeffMask |= 1 << i;
}
}
if (ipa[0] != 0) key.u.addMask |= 0x1;
if (ipa[1] != 0) key.u.addMask |= 0x2;
if (ipa[2] != 0) key.u.addMask |= 0x4;
if (ipa[3] != 0) key.u.addMask |= 0x8;
}
// Look for a dot product where the r,g,b colums are the same
if ((ip[0] == ip[1]) && (ip[0] == ip[2]) &&
(ip[4] == ip[5]) && (ip[4] == ip[6]) &&
(ip[8] == ip[9]) && (ip[8] == ip[10]) &&
(ip[12] == ip[13]) && (ip[12] == ip[14])) {
if (!key.u.addMask) key.u.dot = 1;
}
// Is alpha a simple copy
if (!(key.u.coeffMask & 0x0888) && (ip[15] == 256) && !(key.u.addMask & 0x8)) {
key.u.copyAlpha = !(key.u.inType || key.u.outType);
}
//ALOGE("build key %08x, %08x", (int32_t)(key.key >> 32), (int32_t)key.key);
switch (ein->getVectorSize()) {
case 4:
key.u.inVecSize = 3;
break;
case 3:
key.u.inVecSize = 2;
key.u.coeffMask &= ~0xF000;
break;
case 2:
key.u.inVecSize = 1;
key.u.coeffMask &= ~0xFF00;
break;
default:
key.u.coeffMask &= ~0xFFF0;
break;
}
switch (eout->getVectorSize()) {
case 4:
key.u.outVecSize = 3;
break;
case 3:
key.u.outVecSize = 2;
key.u.coeffMask &= ~0x8888;
key.u.addMask &= 7;
break;
case 2:
key.u.outVecSize = 1;
key.u.coeffMask &= ~0xCCCC;
key.u.addMask &= 3;
break;
default:
key.u.coeffMask &= ~0xEEEE;
key.u.addMask &= 1;
break;
}
if (key.u.inType && !key.u.outType) {
key.u.addMask |= 1;
if (key.u.outVecSize > 0) key.u.addMask |= 2;
if (key.u.outVecSize > 1) key.u.addMask |= 4;
if (key.u.outVecSize > 2) key.u.addMask |= 8;
}
//ALOGE("build key %08x, %08x", (int32_t)(key.key >> 32), (int32_t)key.key);
return key;
}
#if defined(ARCH_ARM_USE_INTRINSICS) && !defined(ARCH_ARM64_USE_INTRINSICS)
#define DEF_SYM(x) \
extern "C" uint32_t _N_ColorMatrix_##x; \
extern "C" uint32_t _N_ColorMatrix_##x##_end; \
extern "C" uint32_t _N_ColorMatrix_##x##_len;
DEF_SYM(prefix_i)
DEF_SYM(prefix_f)
DEF_SYM(postfix1)
DEF_SYM(postfix2)
DEF_SYM(load_u8_4)
DEF_SYM(load_u8_3)
DEF_SYM(load_u8_2)
DEF_SYM(load_u8_1)
DEF_SYM(load_u8f_4)
DEF_SYM(load_u8f_3)
DEF_SYM(load_u8f_2)
DEF_SYM(load_u8f_1)
DEF_SYM(load_f32_4)
DEF_SYM(load_f32_3)
DEF_SYM(load_f32_2)
DEF_SYM(load_f32_1)
DEF_SYM(store_u8_4)
DEF_SYM(store_u8_2)
DEF_SYM(store_u8_1)
DEF_SYM(store_f32_4)
DEF_SYM(store_f32_3)
DEF_SYM(store_f32_2)
DEF_SYM(store_f32_1)
DEF_SYM(store_f32u_4)
DEF_SYM(store_f32u_2)
DEF_SYM(store_f32u_1)
DEF_SYM(unpack_u8_4)
DEF_SYM(unpack_u8_3)
DEF_SYM(unpack_u8_2)
DEF_SYM(unpack_u8_1)
DEF_SYM(pack_u8_4)
DEF_SYM(pack_u8_3)
DEF_SYM(pack_u8_2)
DEF_SYM(pack_u8_1)
DEF_SYM(dot)
DEF_SYM(add_0_u8)
DEF_SYM(add_1_u8)
DEF_SYM(add_2_u8)
DEF_SYM(add_3_u8)
#define ADD_CHUNK(x) \
memcpy(buf, &_N_ColorMatrix_##x, _N_ColorMatrix_##x##_len); \
buf += _N_ColorMatrix_##x##_len
static uint8_t * addBranch(uint8_t *buf, const uint8_t *target, uint32_t condition) {
size_t off = (target - buf - 8) >> 2;
rsAssert(((off & 0xff000000) == 0) ||
((off & 0xff000000) == 0xff000000));
uint32_t op = (condition << 28);
op |= 0xa << 24; // branch
op |= 0xffffff & off;
((uint32_t *)buf)[0] = op;
return buf + 4;
}
static uint32_t encodeSIMDRegs(uint32_t vd, uint32_t vn, uint32_t vm) {
rsAssert(vd < 32);
rsAssert(vm < 32);
rsAssert(vn < 32);
uint32_t op = ((vd & 0xf) << 12) | (((vd & 0x10) >> 4) << 22);
op |= (vm & 0xf) | (((vm & 0x10) >> 4) << 5);
op |= ((vn & 0xf) << 16) | (((vn & 0x10) >> 4) << 7);
return op;
}
static uint8_t * addVMLAL_S16(uint8_t *buf, uint32_t dest_q, uint32_t src_d1, uint32_t src_d2, uint32_t src_d2_s) {
//vmlal.s16 Q#1, D#1, D#2[#]
uint32_t op = 0xf2900240 | encodeSIMDRegs(dest_q << 1, src_d1, src_d2 | (src_d2_s << 3));
((uint32_t *)buf)[0] = op;
return buf + 4;
}
static uint8_t * addVMULL_S16(uint8_t *buf, uint32_t dest_q, uint32_t src_d1, uint32_t src_d2, uint32_t src_d2_s) {
//vmull.s16 Q#1, D#1, D#2[#]
uint32_t op = 0xf2900A40 | encodeSIMDRegs(dest_q << 1, src_d1, src_d2 | (src_d2_s << 3));
((uint32_t *)buf)[0] = op;
return buf + 4;
}
static uint8_t * addVQADD_S32(uint8_t *buf, uint32_t dest_q, uint32_t src_q1, uint32_t src_q2) {
//vqadd.s32 Q#1, Q#1, Q#2
uint32_t op = 0xf2200050 | encodeSIMDRegs(dest_q << 1, src_q1 << 1, src_q2 << 1);
((uint32_t *)buf)[0] = op;
return buf + 4;
}
static uint8_t * addVMLAL_F32(uint8_t *buf, uint32_t dest_q, uint32_t src_d1, uint32_t src_d2, uint32_t src_d2_s) {
//vmlal.f32 Q#1, D#1, D#2[#]
uint32_t op = 0xf3a00140 | encodeSIMDRegs(dest_q << 1, src_d1, src_d2 | (src_d2_s << 4));
((uint32_t *)buf)[0] = op;
return buf + 4;
}
static uint8_t * addVMULL_F32(uint8_t *buf, uint32_t dest_q, uint32_t src_d1, uint32_t src_d2, uint32_t src_d2_s) {
//vmull.f32 Q#1, D#1, D#2[#]
uint32_t op = 0xf3a00940 | encodeSIMDRegs(dest_q << 1, src_d1, src_d2 | (src_d2_s << 4));
((uint32_t *)buf)[0] = op;
return buf + 4;
}
static uint8_t * addVORR_32(uint8_t *buf, uint32_t dest_q, uint32_t src_q1, uint32_t src_q2) {
//vadd.f32 Q#1, D#1, D#2
uint32_t op = 0xf2200150 | encodeSIMDRegs(dest_q << 1, src_q1 << 1, src_q2 << 1);
((uint32_t *)buf)[0] = op;
return buf + 4;
}
static uint8_t * addVMOV_32(uint8_t *buf, uint32_t dest_q, uint32_t imm) {
//vmov.32 Q#1, #imm
rsAssert(imm == 0);
uint32_t op = 0xf2800050 | encodeSIMDRegs(dest_q << 1, 0, 0);
((uint32_t *)buf)[0] = op;
return buf + 4;
}
static uint8_t * addVADD_F32(uint8_t *buf, uint32_t dest_q, uint32_t src_q1, uint32_t src_q2) {
//vadd.f32 Q#1, D#1, D#2
uint32_t op = 0xf2000d40 | encodeSIMDRegs(dest_q << 1, src_q1 << 1, src_q2 << 1);
((uint32_t *)buf)[0] = op;
return buf + 4;
}
#endif
#if defined(ARCH_X86_HAVE_SSSE3)
extern void rsdIntrinsicColorMatrixDot_K(void *dst, const void *src,
const short *coef, uint32_t count);
extern void rsdIntrinsicColorMatrix3x3_K(void *dst, const void *src,
const short *coef, uint32_t count);
extern void rsdIntrinsicColorMatrix4x4_K(void *dst, const void *src,
const short *coef, uint32_t count);
void * selectKernel(Key_t key)
{
void * kernel = nullptr;
// inType, outType float if nonzero
if (!(key.u.inType || key.u.outType)) {
if (key.u.dot)
kernel = (void *)rsdIntrinsicColorMatrixDot_K;
else if (key.u.copyAlpha)
kernel = (void *)rsdIntrinsicColorMatrix3x3_K;
else
kernel = (void *)rsdIntrinsicColorMatrix4x4_K;
}
return kernel;
}
#endif
bool RsdCpuScriptIntrinsicColorMatrix::build(Key_t key) {
#if defined(ARCH_ARM_USE_INTRINSICS) && !defined(ARCH_ARM64_USE_INTRINSICS)
mBufSize = 4096;
//StopWatch build_time("rs cm: build time");
mBuf = (uint8_t *)mmap(0, mBufSize, PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANON, -1, 0);
if (mBuf == MAP_FAILED) {
mBuf = NULL;
return false;
}
uint8_t *buf = mBuf;
uint8_t *buf2 = nullptr;
int ops[5][4]; // 0=unused, 1 = set, 2 = accumulate, 3 = final
int opInit[4] = {0, 0, 0, 0};
memset(ops, 0, sizeof(ops));
for (int i=0; i < 4; i++) {
if (key.u.coeffMask & (1 << (i*4))) {
ops[i][0] = 0x2 | opInit[0];
opInit[0] = 1;
}
if (!key.u.dot) {
if (key.u.coeffMask & (1 << (1 + i*4))) {
ops[i][1] = 0x2 | opInit[1];
opInit[1] = 1;
}
if (key.u.coeffMask & (1 << (2 + i*4))) {
ops[i][2] = 0x2 | opInit[2];
opInit[2] = 1;
}
}
if (!key.u.copyAlpha) {
if (key.u.coeffMask & (1 << (3 + i*4))) {
ops[i][3] = 0x2 | opInit[3];
opInit[3] = 1;
}
}
}
if (key.u.inType || key.u.outType) {
key.u.copyAlpha = 0;
ADD_CHUNK(prefix_f);
buf2 = buf;
// Load the incoming r,g,b,a as needed
if (key.u.inType) {
switch(key.u.inVecSize) {
case 3:
ADD_CHUNK(load_f32_4);
break;
case 2:
ADD_CHUNK(load_f32_3);
break;
case 1:
ADD_CHUNK(load_f32_2);
break;
case 0:
ADD_CHUNK(load_f32_1);
break;
}
} else {
switch(key.u.inVecSize) {
case 3:
ADD_CHUNK(load_u8f_4);
break;
case 2:
ADD_CHUNK(load_u8f_3);
break;
case 1:
ADD_CHUNK(load_u8f_2);
break;
case 0:
ADD_CHUNK(load_u8f_1);
break;
}
}
for (int i=0; i < 4; i++) {
for (int j=0; j < 4; j++) {
switch(ops[i][j]) {
case 0:
break;
case 2:
buf = addVMULL_F32(buf, 12+j, i*2, 8+i*2 + (j >> 1), j & 1);
break;
case 3:
buf = addVMLAL_F32(buf, 12+j, i*2, 8+i*2 + (j >> 1), j & 1);
break;
}
}
}
for (int j=0; j < 4; j++) {
if (opInit[j]) {
if (key.u.addMask & (1 << j)) {
buf = addVADD_F32(buf, j, 12+j, 8+j);
} else {
buf = addVORR_32(buf, j, 12+j, 12+j);
}
} else {
if (key.u.addMask & (1 << j)) {
buf = addVORR_32(buf, j, 8+j, 8+j);
} else {
buf = addVMOV_32(buf, j, 0);
}
}
}
if (key.u.outType) {
switch(key.u.outVecSize) {
case 3:
ADD_CHUNK(store_f32_4);
break;
case 2:
ADD_CHUNK(store_f32_3);
break;
case 1:
ADD_CHUNK(store_f32_2);
break;
case 0:
ADD_CHUNK(store_f32_1);
break;
}
} else {
switch(key.u.outVecSize) {
case 3:
case 2:
ADD_CHUNK(store_f32u_4);
break;
case 1:
ADD_CHUNK(store_f32u_2);
break;
case 0:
ADD_CHUNK(store_f32u_1);
break;
}
}
} else {
// Add the function prefix
// Store the address for the loop return
ADD_CHUNK(prefix_i);
buf2 = buf;
// Load the incoming r,g,b,a as needed
switch(key.u.inVecSize) {
case 3:
ADD_CHUNK(load_u8_4);
if (key.u.copyAlpha) {
ADD_CHUNK(unpack_u8_3);
} else {
ADD_CHUNK(unpack_u8_4);
}
break;
case 2:
ADD_CHUNK(load_u8_3);
ADD_CHUNK(unpack_u8_3);
break;
case 1:
ADD_CHUNK(load_u8_2);
ADD_CHUNK(unpack_u8_2);
break;
case 0:
ADD_CHUNK(load_u8_1);
ADD_CHUNK(unpack_u8_1);
break;
}
// Add multiply and accumulate
// use MULL to init the output register,
// use MLAL from there
for (int i=0; i < 4; i++) {
for (int j=0; j < 4; j++) {
switch(ops[i][j]) {
case 0:
break;
case 2:
buf = addVMULL_S16(buf, 8+j, 24+i*2, 4+i, j);
break;
case 3:
buf = addVMLAL_S16(buf, 8+j, 24+i*2, 4+i, j);
break;
}
}
}
for (int j=0; j < 4; j++) {
if (opInit[j]) {
if (key.u.addMask & (1 << j)) {
buf = addVQADD_S32(buf, 8+j, 8+j, 4+j);
}
} else {
if (key.u.addMask & (1 << j)) {
buf = addVORR_32(buf, 8+j, 4+j, 4+j);
}
}
}
// If we have a dot product, perform the special pack.
if (key.u.dot) {
ADD_CHUNK(pack_u8_1);
ADD_CHUNK(dot);
} else {
switch(key.u.outVecSize) {
case 3:
if (key.u.copyAlpha) {
ADD_CHUNK(pack_u8_3);
} else {
ADD_CHUNK(pack_u8_4);
}
break;
case 2:
ADD_CHUNK(pack_u8_3);
break;
case 1:
ADD_CHUNK(pack_u8_2);
break;
case 0:
ADD_CHUNK(pack_u8_1);
break;
}
}
// Write out result
switch(key.u.outVecSize) {
case 3:
case 2:
ADD_CHUNK(store_u8_4);
break;
case 1:
ADD_CHUNK(store_u8_2);
break;
case 0:
ADD_CHUNK(store_u8_1);
break;
}
}
if (key.u.inType != key.u.outType) {
key.u.copyAlpha = 0;
key.u.dot = 0;
}
// Loop, branch, and cleanup
ADD_CHUNK(postfix1);
buf = addBranch(buf, buf2, 0x01);
ADD_CHUNK(postfix2);
int ret = mprotect(mBuf, mBufSize, PROT_READ | PROT_EXEC);
if (ret == -1) {
ALOGE("mprotect error %i", ret);
return false;
}
__builtin___clear_cache((char *) mBuf, (char*) mBuf + mBufSize);
return true;
#else
return false;
#endif
}
void RsdCpuScriptIntrinsicColorMatrix::updateCoeffCache(float fpMul, float addMul) {
for(int ct=0; ct < 16; ct++) {
ip[ct] = (short)(fp[ct] * 256.f + 0.5f);
tmpFp[ct] = fp[ct] * fpMul;
//ALOGE("mat %i %f %f", ct, fp[ct], tmpFp[ct]);
}
float add = 0.f;
if (fpMul > 254.f) add = 0.5f;
for(int ct=0; ct < 4; ct++) {
tmpFpa[ct] = fpa[ct] * addMul + add;
//ALOGE("fpa %i %f %f", ct, fpa[ct], tmpFpa[ct * 4 + 0]);
}
for(int ct=0; ct < 4; ct++) {
ipa[ct] = (int)(fpa[ct] * 65536.f + 0.5f);
}
}
void RsdCpuScriptIntrinsicColorMatrix::setGlobalVar(uint32_t slot, const void *data,
size_t dataLength) {
switch(slot) {
case 0:
memcpy (fp, data, sizeof(fp));
break;
case 1:
memcpy (fpa, data, sizeof(fpa));
break;
default:
rsAssert(0);
break;
}
mRootPtr = &kernel;
}
static void One(const RsExpandKernelDriverInfo *info, void *out,
const void *py, const float* coeff, const float *add,
uint32_t vsin, uint32_t vsout, bool fin, bool fout) {
float4 f = 0.f;
if (fin) {
switch(vsin) {
case 3:
f = ((const float4 *)py)[0];
break;
case 2:
f = ((const float4 *)py)[0];
f.w = 0.f;
break;
case 1:
f.xy = ((const float2 *)py)[0];
break;
case 0:
f.x = ((const float *)py)[0];
break;
}
} else {
switch(vsin) {
case 3:
f = convert_float4(((const uchar4 *)py)[0]);
break;
case 2:
f = convert_float4(((const uchar4 *)py)[0]);
f.w = 0.f;
break;
case 1:
f.xy = convert_float2(((const uchar2 *)py)[0]);
break;
case 0:
f.x = (float)(((const uchar *)py)[0]);
break;
}
}
//ALOGE("f1 %f %f %f %f", f.x, f.y, f.z, f.w);
float4 sum;
sum.x = f.x * coeff[0] +
f.y * coeff[4] +
f.z * coeff[8] +
f.w * coeff[12];
sum.y = f.x * coeff[1] +
f.y * coeff[5] +
f.z * coeff[9] +
f.w * coeff[13];
sum.z = f.x * coeff[2] +
f.y * coeff[6] +
f.z * coeff[10] +
f.w * coeff[14];
sum.w = f.x * coeff[3] +
f.y * coeff[7] +
f.z * coeff[11] +
f.w * coeff[15];
//ALOGE("f2 %f %f %f %f", sum.x, sum.y, sum.z, sum.w);
sum.x += add[0];
sum.y += add[1];
sum.z += add[2];
sum.w += add[3];
//ALOGE("fout %i vs %i, sum %f %f %f %f", fout, vsout, sum.x, sum.y, sum.z, sum.w);
if (fout) {
switch(vsout) {
case 3:
case 2:
((float4 *)out)[0] = sum;
break;
case 1:
((float2 *)out)[0] = sum.xy;
break;
case 0:
((float *)out)[0] = sum.x;
break;
}
} else {
sum.x = sum.x < 0 ? 0 : (sum.x > 255.5 ? 255.5 : sum.x);
sum.y = sum.y < 0 ? 0 : (sum.y > 255.5 ? 255.5 : sum.y);
sum.z = sum.z < 0 ? 0 : (sum.z > 255.5 ? 255.5 : sum.z);
sum.w = sum.w < 0 ? 0 : (sum.w > 255.5 ? 255.5 : sum.w);
switch(vsout) {
case 3:
case 2:
((uchar4 *)out)[0] = convert_uchar4(sum);
break;
case 1:
((uchar2 *)out)[0] = convert_uchar2(sum.xy);
break;
case 0:
((uchar *)out)[0] = sum.x;
break;
}
}
//ALOGE("out %p %f %f %f %f", out, ((float *)out)[0], ((float *)out)[1], ((float *)out)[2], ((float *)out)[3]);
}
void RsdCpuScriptIntrinsicColorMatrix::kernel(const RsExpandKernelDriverInfo *info,
uint32_t xstart, uint32_t xend,
uint32_t outstep) {
RsdCpuScriptIntrinsicColorMatrix *cp = (RsdCpuScriptIntrinsicColorMatrix *)info->usr;
uint32_t instep = info->inStride[0];
uchar *out = (uchar *)info->outPtr[0];
uchar *in = (uchar *)info->inPtr[0];
uint32_t x1 = xstart;
uint32_t x2 = xend;
uint32_t vsin = cp->mLastKey.u.inVecSize;
uint32_t vsout = cp->mLastKey.u.outVecSize;
bool floatIn = !!cp->mLastKey.u.inType;
bool floatOut = !!cp->mLastKey.u.outType;
//if (!info->current.y) ALOGE("steps %i %i %i %i", instep, outstep, vsin, vsout);
if(x2 > x1) {
int32_t len = x2 - x1;
if (gArchUseSIMD) {
if((cp->mOptKernel != nullptr) && (len >= 4)) {
// The optimized kernel processes 4 pixels at once
// and requires a minimum of 1 chunk of 4
cp->mOptKernel(out, in, cp->ip, len >> 2);
// Update the len and pointers so the generic code can
// finish any leftover pixels
len &= ~3;
x1 += len;
out += outstep * len;
in += instep * len;
}
#if defined(ARCH_ARM64_USE_INTRINSICS)
else {
if (cp->mLastKey.u.inType == RS_TYPE_FLOAT_32 || cp->mLastKey.u.outType == RS_TYPE_FLOAT_32) {
// Currently this generates off by one errors.
//rsdIntrinsicColorMatrix_float_K(out, in, len, &cp->mFnTab, cp->tmpFp, cp->tmpFpa);
//x1 += len;
//out += outstep * len;
//in += instep * len;
} else {
rsdIntrinsicColorMatrix_int_K(out, in, len, &cp->mFnTab, cp->ip, cp->ipa);
x1 += len;
out += outstep * len;
in += instep * len;
}
}
#endif
}
while(x1 != x2) {
One(info, out, in, cp->tmpFp, cp->tmpFpa, vsin, vsout, floatIn, floatOut);
out += outstep;
in += instep;
x1++;
}
}
}
void RsdCpuScriptIntrinsicColorMatrix::preLaunch(uint32_t slot,
const Allocation ** ains,
uint32_t inLen,
Allocation * aout,
const void * usr,
uint32_t usrLen,
const RsScriptCall *sc) {
const Element *ein = ains[0]->mHal.state.type->getElement();
const Element *eout = aout->mHal.state.type->getElement();
if (ein->getType() == eout->getType()) {
if (eout->getType() == RS_TYPE_UNSIGNED_8) {
updateCoeffCache(1.f, 255.f);
} else {
updateCoeffCache(1.f, 1.f);
}
} else {
if (eout->getType() == RS_TYPE_UNSIGNED_8) {
updateCoeffCache(255.f, 255.f);
} else {
updateCoeffCache(1.f / 255.f, 1.f);
}
}
Key_t key = computeKey(ein, eout);
#if defined(ARCH_X86_HAVE_SSSE3)
if ((mOptKernel == nullptr) || (mLastKey.key != key.key)) {
// FIXME: Disable mOptKernel to pass RS color matrix CTS cases
// mOptKernel = (void (*)(void *, const void *, const short *, uint32_t)) selectKernel(key);
mLastKey = key;
}
#else //if !defined(ARCH_X86_HAVE_SSSE3)
if ((mOptKernel == nullptr) || (mLastKey.key != key.key)) {
if (mBuf) munmap(mBuf, mBufSize);
mBuf = nullptr;
mOptKernel = nullptr;
if (build(key)) {
mOptKernel = (void (*)(void *, const void *, const short *, uint32_t)) mBuf;
}
#if defined(ARCH_ARM64_USE_INTRINSICS)
else {
int dt = key.u.outVecSize + (key.u.outType == RS_TYPE_FLOAT_32 ? 4 : 0);
int st = key.u.inVecSize + (key.u.inType == RS_TYPE_FLOAT_32 ? 4 : 0);
uint32_t mm = 0;
int i;
for (i = 0; i < 4; i++)
{
uint32_t m = (key.u.coeffMask >> i) & 0x1111;
m = ((m * 0x249) >> 9) & 15;
m |= ((key.u.addMask >> i) & 1) << 4;
mm |= m << (i * 5);
}
if (key.u.inType == RS_TYPE_FLOAT_32 || key.u.outType == RS_TYPE_FLOAT_32) {
rsdIntrinsicColorMatrixSetup_float_K(&mFnTab, mm, dt, st);
} else {
rsdIntrinsicColorMatrixSetup_int_K(&mFnTab, mm, dt, st);
}
}
#endif
mLastKey = key;
}
#endif //if !defined(ARCH_X86_HAVE_SSSE3)
}
RsdCpuScriptIntrinsicColorMatrix::RsdCpuScriptIntrinsicColorMatrix(
RsdCpuReferenceImpl *ctx, const Script *s, const Element *e)
: RsdCpuScriptIntrinsic(ctx, s, e, RS_SCRIPT_INTRINSIC_ID_COLOR_MATRIX) {
mLastKey.key = 0;
mBuf = nullptr;
mBufSize = 0;
mOptKernel = nullptr;
const static float defaultMatrix[] = {
1.f, 0.f, 0.f, 0.f,
0.f, 1.f, 0.f, 0.f,
0.f, 0.f, 1.f, 0.f,
0.f, 0.f, 0.f, 1.f
};
const static float defaultAdd[] = {0.f, 0.f, 0.f, 0.f};
setGlobalVar(0, defaultMatrix, sizeof(defaultMatrix));
setGlobalVar(1, defaultAdd, sizeof(defaultAdd));
}
RsdCpuScriptIntrinsicColorMatrix::~RsdCpuScriptIntrinsicColorMatrix() {
if (mBuf) munmap(mBuf, mBufSize);
mBuf = nullptr;
mOptKernel = nullptr;
}
void RsdCpuScriptIntrinsicColorMatrix::populateScript(Script *s) {
s->mHal.info.exportedVariableCount = 2;
}
RsdCpuScriptImpl * rsdIntrinsic_ColorMatrix(RsdCpuReferenceImpl *ctx,
const Script *s, const Element *e) {
return new RsdCpuScriptIntrinsicColorMatrix(ctx, s, e);
}