/******************************************************************************
*
* Copyright (C) 2015 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at:
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*****************************************************************************
* Originally developed and contributed by Ittiam Systems Pvt. Ltd, Bangalore
*/
/**
*******************************************************************************
* @file
* ih264_resi_trans_quant_sse42.c
*
* @brief
* Contains function definitions single stage forward transform for H.264
* It will calculate the residue, do the cf and then do quantization
*
* @author
* Mohit [100664]
*
* @par List of Functions:
* - ih264_resi_trans_quant_4x4_sse42()
* - ih264_resi_trans_quant_chroma_4x4_sse42()
*
* @remarks
* None
*
*******************************************************************************
*/
/* System include files */
#include <stddef.h>
/* User include files */
#include "ih264_typedefs.h"
#include "ih264_defs.h"
#include "ih264_size_defs.h"
#include "ih264_macros.h"
#include "ih264_trans_macros.h"
#include "ih264_trans_data.h"
#include "ih264_structs.h"
#include "ih264_trans_quant_itrans_iquant.h"
#include <immintrin.h>
/**
*******************************************************************************
*
* @brief
* This function performs forward transform and quantization on a 4*4 block
*
* @par Description:
* The function accepts source buffer and estimation buffer. From these, it
* computes the residue. This is residue is then transformed and quantized.
* The transform and quantization are in placed computed. They use the residue
* buffer for this.
*
* @param[in] pu1_src
* Pointer to source sub-block
*
* @param[in] pu1_pred
* Pointer to prediction sub-block
*
* @param[in] pi2_out
* Pointer to residual sub-block
*
* @param[in] src_strd
* Source stride
*
* @param[in] pred_strd
* Prediction stride
*
* @param[in] dst_strd
* Destination stride
*
* @param[in] u4_qbits
* QP_BITS_h264_4x4 + floor(QP/6)
*
* @param[in] pu2_threshold_matrix
* Pointer to Forward Quant Threshold Matrix
*
* @param[in] pu2_scale_matrix
* Pointer to Forward Quant Scale Matrix
*
* @param[in] u4_round_factor
* Quantization Round factor
*
* @param[out] pu1_nnz
* Total non-zero coefficients in the current sub-block
*
* @returns
*
* @remarks
* None
*
*******************************************************************************
*/
void ih264_resi_trans_quant_4x4_sse42(UWORD8 *pu1_src, UWORD8 *pu1_pred,
WORD16 *pi2_out, WORD32 src_strd, WORD32 pred_strd,
const UWORD16 *pu2_scale_matrix, const UWORD16 *pu2_threshold_matrix,
UWORD32 u4_qbits, UWORD32 u4_round_factor, UWORD8 *pu1_nnz,
WORD16 *pi2_alt_dc_addr)
{
WORD32 tmp_dc, u4_zero_coeff, u4_nonzero_coeff = 0;
WORD32 mask0, mask1;
__m128i sum0, sum1, sum2, cmp0, cmp1;
__m128i rnd_fact = _mm_set1_epi32(u4_round_factor);
__m128i temp_2 = _mm_set1_epi16(2);
__m128i temp_1 = _mm_set1_epi16(1);
__m128i src_r0, src_r1, src_r2, src_r3;
__m128i pred_r0, pred_r1, pred_r2, pred_r3;
__m128i temp0, temp1, temp2, temp3;
__m128i zero_8x16b = _mm_setzero_si128(); // all bits reset to zero
__m128i sign_reg0, sign_reg2;
__m128i scalemat_r0_r1, scalemat_r2_r3;
UNUSED (pu2_threshold_matrix);
scalemat_r0_r1 = _mm_loadu_si128((__m128i *) (pu2_scale_matrix)); //b00 b01 b02 b03 b10 b11 b12 b13 -- the scaling matrix 0th,1st row
scalemat_r2_r3 = _mm_loadu_si128((__m128i *) (pu2_scale_matrix + 8)); //b20 b21 b22 b23 b30 b31 b32 b33 -- the scaling matrix 2nd,3rd row
src_r0 = _mm_loadl_epi64((__m128i *) (&pu1_src[0])); //a00 a01 a02 a03 0 0 0 0 0 0 0 0 -- all 8 bits
src_r1 = _mm_loadl_epi64((__m128i *) (&pu1_src[src_strd])); //a10 a11 a12 a13 0 0 0 0 0 0 0 0 -- all 8 bits
src_r2 = _mm_loadl_epi64((__m128i *) (&pu1_src[2 * src_strd])); //a20 a21 a22 a23 0 0 0 0 0 0 0 0 -- all 8 bits
src_r3 = _mm_loadl_epi64((__m128i *) (&pu1_src[3 * src_strd])); //a30 a31 a32 a33 0 0 0 0 0 0 0 0 -- all 8 bits
src_r0 = _mm_cvtepu8_epi16(src_r0);
src_r1 = _mm_cvtepu8_epi16(src_r1);
src_r2 = _mm_cvtepu8_epi16(src_r2);
src_r3 = _mm_cvtepu8_epi16(src_r3);
pred_r0 = _mm_loadl_epi64((__m128i *) (&pu1_pred[0])); //p00 p01 p02 p03 0 0 0 0 0 0 0 0 -- all 8 bits
pred_r1 = _mm_loadl_epi64((__m128i *) (&pu1_pred[pred_strd])); //p10 p11 p12 p13 0 0 0 0 0 0 0 0 -- all 8 bits
pred_r2 = _mm_loadl_epi64((__m128i *) (&pu1_pred[2 * pred_strd])); //p20 p21 p22 p23 0 0 0 0 0 0 0 0 -- all 8 bits
pred_r3 = _mm_loadl_epi64((__m128i *) (&pu1_pred[3 * pred_strd])); //p30 p31 p32 p33 0 0 0 0 0 0 0 0 -- all 8 bits
pred_r0 = _mm_cvtepu8_epi16(pred_r0); //p00 p01 p02 p03 -- all 16 bits
pred_r1 = _mm_cvtepu8_epi16(pred_r1); //p10 p11 p12 p13 -- all 16 bits
pred_r2 = _mm_cvtepu8_epi16(pred_r2); //p20 p21 p22 p23 -- all 16 bits
pred_r3 = _mm_cvtepu8_epi16(pred_r3); //p30 p31 p32 p33 -- all 16 bits
src_r0 = _mm_sub_epi16(src_r0, pred_r0);
src_r1 = _mm_sub_epi16(src_r1, pred_r1);
src_r2 = _mm_sub_epi16(src_r2, pred_r2);
src_r3 = _mm_sub_epi16(src_r3, pred_r3);
/* Perform Forward transform */
/*-------------------------------------------------------------*/
/* DCT [ Horizontal transformation ] */
/*-------------------------------------------------------------*/
// Matrix transpose
/*
* a0 a1 a2 a3
* b0 b1 b2 b3
* c0 c1 c2 c3
* d0 d1 d2 d3
*/
temp0 = _mm_unpacklo_epi16(src_r0, src_r1); //a0 b0 a1 b1 a2 b2 a3 b3
temp2 = _mm_unpacklo_epi16(src_r2, src_r3); //c0 d0 c1 d1 c2 d2 c3 d3
temp1 = _mm_unpacklo_epi32(temp0, temp2); //a0 b0 c0 d0 a1 b1 c1 d1
temp3 = _mm_unpackhi_epi32(temp0, temp2); //a2 b2 c2 d2 a3 b3 c3 d3
src_r0 = _mm_unpacklo_epi64(temp1, zero_8x16b); //a0 b0 c0 d0
src_r1 = _mm_unpackhi_epi64(temp1, zero_8x16b); //a1 b1 c1 d1
src_r2 = _mm_unpacklo_epi64(temp3, zero_8x16b); //a2 b2 c2 d2
src_r3 = _mm_unpackhi_epi64(temp3, zero_8x16b); //a3 b3 c3 d3
/*----------------------------------------------------------*/
/* x0 = z0 + z3 */
temp0 = _mm_add_epi16(src_r0, src_r3);
/* x1 = z1 + z2 */
temp1 = _mm_add_epi16(src_r1, src_r2);
/* x2 = z1 - z2 */
temp2 = _mm_sub_epi16(src_r1, src_r2);
/* x3 = z0 - z3 */
temp3 = _mm_sub_epi16(src_r0, src_r3);
/* z0 = x0 + x1 */
src_r0 = _mm_add_epi16(temp0, temp1);
/* z1 = (x3 << 1) + x2 */
src_r1 = _mm_slli_epi16(temp3, 1); //(x3<<1)
src_r1 = _mm_add_epi16(src_r1, temp2);
/* z2 = x0 - x1 */
src_r2 = _mm_sub_epi16(temp0, temp1);
/* z3 = x3 - (x2 << 1) */
src_r3 = _mm_slli_epi16(temp2, 1); //(x2<<1)
src_r3 = _mm_sub_epi16(temp3, src_r3);
// Matrix transpose
/*
* a0 b0 c0 d0
* a1 b1 c1 d1
* a2 b2 c2 d2
* a3 b3 c3 d3
*/
temp0 = _mm_unpacklo_epi16(src_r0, src_r1); //a0 a1 b0 b1 c0 c1 d0 d1
temp2 = _mm_unpacklo_epi16(src_r2, src_r3); //a2 a3 b2 b3 c2 c3 d2 d3
temp1 = _mm_unpacklo_epi32(temp0, temp2); //a0 a1 a2 a3 b0 b1 b2 b3
temp3 = _mm_unpackhi_epi32(temp0, temp2); //c0 c1 c2 c3 d0 d1 d2 d3
src_r0 = _mm_unpacklo_epi64(temp1, zero_8x16b); //a0 a1 a2 a3
src_r1 = _mm_unpackhi_epi64(temp1, zero_8x16b); //b0 b1 b2 b3
src_r2 = _mm_unpacklo_epi64(temp3, zero_8x16b); //c0 c1 c2 c3
src_r3 = _mm_unpackhi_epi64(temp3, zero_8x16b); //d0 d1 d2 d3
/*----------------------------------------------------------*/
/* x0 = z0 + z3 */
temp0 = _mm_add_epi16(src_r0, src_r3);
/* x1 = z1 + z2 */
temp1 = _mm_add_epi16(src_r1, src_r2);
/* x2 = z1 - z2 */
temp2 = _mm_sub_epi16(src_r1, src_r2);
/* x3 = z0 - z3 */
temp3 = _mm_sub_epi16(src_r0, src_r3);
/* z0 = x0 + x1 */
src_r0 = _mm_add_epi16(temp0, temp1);
/* z1 = (x3 << 1) + x2 */
src_r1 = _mm_slli_epi16(temp3, 1); //(x3<<1)
src_r1 = _mm_add_epi16(src_r1, temp2);
/* z2 = x0 - x1 */
src_r2 = _mm_sub_epi16(temp0, temp1);
/* z3 = x3 - (x2 << 1) */
src_r3 = _mm_slli_epi16(temp2, 1); //(x2<<1)
src_r3 = _mm_sub_epi16(temp3, src_r3);
tmp_dc = _mm_extract_epi16(src_r0,0); //a0
*pi2_alt_dc_addr = tmp_dc;
src_r0 = _mm_unpacklo_epi64(src_r0, src_r1); //a0 a1 a2 a3 b0 b1 b2 b3
src_r2 = _mm_unpacklo_epi64(src_r2, src_r3); //c0 c1 c2 c3 d0 d1 d2 d3
sign_reg0 = _mm_cmpgt_epi16(zero_8x16b,src_r0);
sign_reg2 = _mm_cmpgt_epi16(zero_8x16b,src_r2);
sign_reg0 = _mm_mullo_epi16(temp_2,sign_reg0);
sign_reg2 = _mm_mullo_epi16(temp_2,sign_reg2);
sign_reg0 = _mm_add_epi16(temp_1,sign_reg0);
sign_reg2 = _mm_add_epi16(temp_1,sign_reg2);
src_r0 = _mm_abs_epi16(src_r0);
src_r2 = _mm_abs_epi16(src_r2);
src_r1 = _mm_srli_si128(src_r0, 8);
src_r0 = _mm_cvtepu16_epi32(src_r0);
src_r1 = _mm_cvtepu16_epi32(src_r1);
src_r3 = _mm_srli_si128(src_r2, 8);
src_r2 = _mm_cvtepu16_epi32(src_r2);
src_r3 = _mm_cvtepu16_epi32(src_r3);
temp0 = _mm_cvtepu16_epi32(scalemat_r0_r1);
scalemat_r0_r1 = _mm_srli_si128(scalemat_r0_r1, 8);
temp2 = _mm_cvtepu16_epi32(scalemat_r2_r3);
scalemat_r2_r3 = _mm_srli_si128(scalemat_r2_r3, 8);
temp1 = _mm_cvtepu16_epi32(scalemat_r0_r1);
temp3 = _mm_cvtepu16_epi32(scalemat_r2_r3);
temp0 = _mm_mullo_epi32(temp0, src_r0);
temp1 = _mm_mullo_epi32(temp1, src_r1);
temp2 = _mm_mullo_epi32(temp2, src_r2);
temp3 = _mm_mullo_epi32(temp3, src_r3);
temp0 = _mm_add_epi32(temp0,rnd_fact);
temp1 = _mm_add_epi32(temp1,rnd_fact);
temp2 = _mm_add_epi32(temp2,rnd_fact);
temp3 = _mm_add_epi32(temp3,rnd_fact);
temp0 = _mm_srli_epi32(temp0,u4_qbits);
temp1 = _mm_srli_epi32(temp1,u4_qbits);
temp2 = _mm_srli_epi32(temp2,u4_qbits);
temp3 = _mm_srli_epi32(temp3,u4_qbits);
temp0 = _mm_packs_epi32 (temp0,temp1);
temp2 = _mm_packs_epi32 (temp2,temp3);
temp0 = _mm_sign_epi16(temp0, sign_reg0);
temp2 = _mm_sign_epi16(temp2, sign_reg2);
_mm_storeu_si128((__m128i *) (&pi2_out[0]), temp0);
_mm_storeu_si128((__m128i *) (&pi2_out[8]), temp2);
cmp0 = _mm_cmpeq_epi16(temp0, zero_8x16b);
cmp1 = _mm_cmpeq_epi16(temp2, zero_8x16b);
mask0 = _mm_movemask_epi8(cmp0);
mask1 = _mm_movemask_epi8(cmp1);
u4_zero_coeff = 0;
if(mask0)
{
if(mask0 == 0xffff)
u4_zero_coeff+=8;
else
{
cmp0 = _mm_and_si128(temp_1, cmp0);
sum0 = _mm_hadd_epi16(cmp0, zero_8x16b);
sum1 = _mm_hadd_epi16(sum0, zero_8x16b);
sum2 = _mm_hadd_epi16(sum1, zero_8x16b);
u4_zero_coeff += _mm_cvtsi128_si32(sum2);
}
}
if(mask1)
{
if(mask1 == 0xffff)
u4_zero_coeff+=8;
else
{
cmp1 = _mm_and_si128(temp_1, cmp1);
sum0 = _mm_hadd_epi16(cmp1, zero_8x16b);
sum1 = _mm_hadd_epi16(sum0, zero_8x16b);
sum2 = _mm_hadd_epi16(sum1, zero_8x16b);
u4_zero_coeff += _mm_cvtsi128_si32(sum2);
}
}
/* Return total nonzero coefficients in the current sub block */
u4_nonzero_coeff = 16 - u4_zero_coeff;
*pu1_nnz = u4_nonzero_coeff;
}
/**
*******************************************************************************
*
* @brief
* This function performs forward transform and quantization on a 4*4 chroma block
*
* @par Description:
* The function accepts source buffer and estimation buffer. From these, it
* computes the residue. This is residue is then transformed and quantized.
* The transform and quantization are in placed computed. They use the residue
* buffer for this.
*
* @param[in] pu1_src
* Pointer to source sub-block
*
* @param[in] pu1_pred
* Pointer to prediction sub-block
*
* @param[in] pi2_out
* Pointer to residual sub-block
*
* @param[in] src_strd
* Source stride
*
* @param[in] pred_strd
* Prediction stride
*
* @param[in] dst_strd
* Destination stride
*
* @param[in] u4_qbits
* QP_BITS_h264_4x4 + floor(QP/6)
*
* @param[in] pu2_threshold_matrix
* Pointer to Forward Quant Threshold Matrix
*
* @param[in] pu2_scale_matrix
* Pointer to Forward Quant Scale Matrix
*
* @param[in] u4_round_factor
* Quantization Round factor
*
* @param[out] pu1_nnz
* Total non-zero coefficients in the current sub-block
*
* @returns
*
* @remarks
* None
*
*******************************************************************************
*/
void ih264_resi_trans_quant_chroma_4x4_sse42(UWORD8 *pu1_src,UWORD8 *pu1_pred,WORD16 *pi2_out,
WORD32 src_strd,WORD32 pred_strd,
const UWORD16 *pu2_scale_matrix,
const UWORD16 *pu2_threshold_matrix,
UWORD32 u4_qbits,UWORD32 u4_round_factor,
UWORD8 *pu1_nnz, WORD16 *pi2_alt_dc_addr)
{
WORD32 tmp_dc, u4_zero_coeff, u4_nonzero_coeff = 0;
WORD32 mask0, mask1;
__m128i cmp0, cmp1, sum0, sum1, sum2;
__m128i rnd_fact = _mm_set1_epi32(u4_round_factor);
__m128i temp_2 = _mm_set1_epi16(2);
__m128i temp_1 = _mm_set1_epi16(1);
__m128i src_r0, src_r1, src_r2, src_r3;
__m128i pred_r0, pred_r1, pred_r2, pred_r3;
__m128i temp0, temp1, temp2, temp3;
__m128i zero_8x16b = _mm_setzero_si128(); // all bits reset to zero
__m128i sign_reg0, sign_reg2;
__m128i scalemat_r0_r1, scalemat_r2_r3;
__m128i chroma_mask = _mm_set1_epi16 (0xFF);
UNUSED (pu2_threshold_matrix);
scalemat_r0_r1 = _mm_loadu_si128((__m128i *) (pu2_scale_matrix)); //b00 b01 b02 b03 b10 b11 b12 b13 -- the scaling matrix 0th,1st row
scalemat_r2_r3 = _mm_loadu_si128((__m128i *) (pu2_scale_matrix + 8)); //b20 b21 b22 b23 b30 b31 b32 b33 -- the scaling matrix 2nd,3rd row
src_r0 = _mm_loadl_epi64((__m128i *) (&pu1_src[0])); //a00 a01 a02 a03 0 0 0 0 0 0 0 0 -- all 8 bits
src_r1 = _mm_loadl_epi64((__m128i *) (&pu1_src[src_strd])); //a10 a11 a12 a13 0 0 0 0 0 0 0 0 -- all 8 bits
src_r2 = _mm_loadl_epi64((__m128i *) (&pu1_src[2 * src_strd])); //a20 a21 a22 a23 0 0 0 0 0 0 0 0 -- all 8 bits
src_r3 = _mm_loadl_epi64((__m128i *) (&pu1_src[3 * src_strd])); //a30 a31 a32 a33 0 0 0 0 0 0 0 0 -- all 8 bits
src_r0 = _mm_and_si128(src_r0, chroma_mask);
src_r1 = _mm_and_si128(src_r1, chroma_mask);
src_r2 = _mm_and_si128(src_r2, chroma_mask);
src_r3 = _mm_and_si128(src_r3, chroma_mask);
// src_r0 = _mm_cvtepu8_epi16(src_r0);
// src_r1 = _mm_cvtepu8_epi16(src_r1);
// src_r2 = _mm_cvtepu8_epi16(src_r2);
// src_r3 = _mm_cvtepu8_epi16(src_r3);
pred_r0 = _mm_loadl_epi64((__m128i *) (&pu1_pred[0])); //p00 p01 p02 p03 0 0 0 0 0 0 0 0 -- all 8 bits
pred_r1 = _mm_loadl_epi64((__m128i *) (&pu1_pred[pred_strd])); //p10 p11 p12 p13 0 0 0 0 0 0 0 0 -- all 8 bits
pred_r2 = _mm_loadl_epi64((__m128i *) (&pu1_pred[2 * pred_strd])); //p20 p21 p22 p23 0 0 0 0 0 0 0 0 -- all 8 bits
pred_r3 = _mm_loadl_epi64((__m128i *) (&pu1_pred[3 * pred_strd])); //p30 p31 p32 p33 0 0 0 0 0 0 0 0 -- all 8 bits
pred_r0 = _mm_and_si128(pred_r0, chroma_mask);
pred_r1 = _mm_and_si128(pred_r1, chroma_mask);
pred_r2 = _mm_and_si128(pred_r2, chroma_mask);
pred_r3 = _mm_and_si128(pred_r3, chroma_mask);
// pred_r0 = _mm_cvtepu8_epi16(pred_r0); //p00 p01 p02 p03 -- all 16 bits
// pred_r1 = _mm_cvtepu8_epi16(pred_r1); //p10 p11 p12 p13 -- all 16 bits
// pred_r2 = _mm_cvtepu8_epi16(pred_r2); //p20 p21 p22 p23 -- all 16 bits
// pred_r3 = _mm_cvtepu8_epi16(pred_r3); //p30 p31 p32 p33 -- all 16 bits
src_r0 = _mm_sub_epi16(src_r0, pred_r0);
src_r1 = _mm_sub_epi16(src_r1, pred_r1);
src_r2 = _mm_sub_epi16(src_r2, pred_r2);
src_r3 = _mm_sub_epi16(src_r3, pred_r3);
/* Perform Forward transform */
/*-------------------------------------------------------------*/
/* DCT [ Horizontal transformation ] */
/*-------------------------------------------------------------*/
// Matrix transpose
/*
* a0 a1 a2 a3
* b0 b1 b2 b3
* c0 c1 c2 c3
* d0 d1 d2 d3
*/
temp0 = _mm_unpacklo_epi16(src_r0, src_r1); //a0 b0 a1 b1 a2 b2 a3 b3
temp2 = _mm_unpacklo_epi16(src_r2, src_r3); //c0 d0 c1 d1 c2 d2 c3 d3
temp1 = _mm_unpacklo_epi32(temp0, temp2); //a0 b0 c0 d0 a1 b1 c1 d1
temp3 = _mm_unpackhi_epi32(temp0, temp2); //a2 b2 c2 d2 a3 b3 c3 d3
src_r0 = _mm_unpacklo_epi64(temp1, zero_8x16b); //a0 b0 c0 d0
src_r1 = _mm_unpackhi_epi64(temp1, zero_8x16b); //a1 b1 c1 d1
src_r2 = _mm_unpacklo_epi64(temp3, zero_8x16b); //a2 b2 c2 d2
src_r3 = _mm_unpackhi_epi64(temp3, zero_8x16b); //a3 b3 c3 d3
/*----------------------------------------------------------*/
/* x0 = z0 + z3 */
temp0 = _mm_add_epi16(src_r0, src_r3);
/* x1 = z1 + z2 */
temp1 = _mm_add_epi16(src_r1, src_r2);
/* x2 = z1 - z2 */
temp2 = _mm_sub_epi16(src_r1, src_r2);
/* x3 = z0 - z3 */
temp3 = _mm_sub_epi16(src_r0, src_r3);
/* z0 = x0 + x1 */
src_r0 = _mm_add_epi16(temp0, temp1);
/* z1 = (x3 << 1) + x2 */
src_r1 = _mm_slli_epi16(temp3, 1); //(x3<<1)
src_r1 = _mm_add_epi16(src_r1, temp2);
/* z2 = x0 - x1 */
src_r2 = _mm_sub_epi16(temp0, temp1);
/* z3 = x3 - (x2 << 1) */
src_r3 = _mm_slli_epi16(temp2, 1); //(x2<<1)
src_r3 = _mm_sub_epi16(temp3, src_r3);
// Matrix transpose
/*
* a0 b0 c0 d0
* a1 b1 c1 d1
* a2 b2 c2 d2
* a3 b3 c3 d3
*/
temp0 = _mm_unpacklo_epi16(src_r0, src_r1); //a0 a1 b0 b1 c0 c1 d0 d1
temp2 = _mm_unpacklo_epi16(src_r2, src_r3); //a2 a3 b2 b3 c2 c3 d2 d3
temp1 = _mm_unpacklo_epi32(temp0, temp2); //a0 a1 a2 a3 b0 b1 b2 b3
temp3 = _mm_unpackhi_epi32(temp0, temp2); //c0 c1 c2 c3 d0 d1 d2 d3
src_r0 = _mm_unpacklo_epi64(temp1, zero_8x16b); //a0 a1 a2 a3
src_r1 = _mm_unpackhi_epi64(temp1, zero_8x16b); //b0 b1 b2 b3
src_r2 = _mm_unpacklo_epi64(temp3, zero_8x16b); //c0 c1 c2 c3
src_r3 = _mm_unpackhi_epi64(temp3, zero_8x16b); //d0 d1 d2 d3
/*----------------------------------------------------------*/
/* x0 = z0 + z3 */
temp0 = _mm_add_epi16(src_r0, src_r3);
/* x1 = z1 + z2 */
temp1 = _mm_add_epi16(src_r1, src_r2);
/* x2 = z1 - z2 */
temp2 = _mm_sub_epi16(src_r1, src_r2);
/* x3 = z0 - z3 */
temp3 = _mm_sub_epi16(src_r0, src_r3);
/* z0 = x0 + x1 */
src_r0 = _mm_add_epi16(temp0, temp1);
/* z1 = (x3 << 1) + x2 */
src_r1 = _mm_slli_epi16(temp3, 1); //(x3<<1)
src_r1 = _mm_add_epi16(src_r1, temp2);
/* z2 = x0 - x1 */
src_r2 = _mm_sub_epi16(temp0, temp1);
/* z3 = x3 - (x2 << 1) */
src_r3 = _mm_slli_epi16(temp2, 1); //(x2<<1)
src_r3 = _mm_sub_epi16(temp3, src_r3);
tmp_dc = _mm_extract_epi16(src_r0,0); //a0
*pi2_alt_dc_addr = tmp_dc;
src_r0 = _mm_unpacklo_epi64(src_r0, src_r1); //a0 a1 a2 a3 b0 b1 b2 b3
src_r2 = _mm_unpacklo_epi64(src_r2, src_r3); //c0 c1 c2 c3 d0 d1 d2 d3
sign_reg0 = _mm_cmpgt_epi16(zero_8x16b,src_r0);
sign_reg2 = _mm_cmpgt_epi16(zero_8x16b,src_r2);
sign_reg0 = _mm_mullo_epi16(temp_2,sign_reg0);
sign_reg2 = _mm_mullo_epi16(temp_2,sign_reg2);
sign_reg0 = _mm_add_epi16(temp_1,sign_reg0);
sign_reg2 = _mm_add_epi16(temp_1,sign_reg2);
src_r0 = _mm_abs_epi16(src_r0);
src_r2 = _mm_abs_epi16(src_r2);
src_r1 = _mm_srli_si128(src_r0, 8);
src_r0 = _mm_cvtepu16_epi32(src_r0);
src_r1 = _mm_cvtepu16_epi32(src_r1);
src_r3 = _mm_srli_si128(src_r2, 8);
src_r2 = _mm_cvtepu16_epi32(src_r2);
src_r3 = _mm_cvtepu16_epi32(src_r3);
temp0 = _mm_cvtepu16_epi32(scalemat_r0_r1);
scalemat_r0_r1 = _mm_srli_si128(scalemat_r0_r1, 8);
temp2 = _mm_cvtepu16_epi32(scalemat_r2_r3);
scalemat_r2_r3 = _mm_srli_si128(scalemat_r2_r3, 8);
temp1 = _mm_cvtepu16_epi32(scalemat_r0_r1);
temp3 = _mm_cvtepu16_epi32(scalemat_r2_r3);
temp0 = _mm_mullo_epi32(temp0, src_r0);
temp1 = _mm_mullo_epi32(temp1, src_r1);
temp2 = _mm_mullo_epi32(temp2, src_r2);
temp3 = _mm_mullo_epi32(temp3, src_r3);
temp0 = _mm_add_epi32(temp0,rnd_fact);
temp1 = _mm_add_epi32(temp1,rnd_fact);
temp2 = _mm_add_epi32(temp2,rnd_fact);
temp3 = _mm_add_epi32(temp3,rnd_fact);
temp0 = _mm_srli_epi32(temp0,u4_qbits);
temp1 = _mm_srli_epi32(temp1,u4_qbits);
temp2 = _mm_srli_epi32(temp2,u4_qbits);
temp3 = _mm_srli_epi32(temp3,u4_qbits);
temp0 = _mm_packs_epi32 (temp0,temp1);
temp2 = _mm_packs_epi32 (temp2,temp3);
temp0 = _mm_sign_epi16(temp0, sign_reg0);
temp2 = _mm_sign_epi16(temp2, sign_reg2);
//temp0 = _mm_insert_epi16(temp0, tmp_dc, 0);
_mm_storeu_si128((__m128i *) (&pi2_out[0]), temp0);
_mm_storeu_si128((__m128i *) (&pi2_out[8]), temp2);
cmp0 = _mm_cmpeq_epi16(temp0, zero_8x16b);
cmp1 = _mm_cmpeq_epi16(temp2, zero_8x16b);
mask0 = _mm_movemask_epi8(cmp0);
mask1 = _mm_movemask_epi8(cmp1);
u4_zero_coeff = 0;
if(mask0)
{
if(mask0 == 0xffff)
u4_zero_coeff+=8;
else
{
cmp0 = _mm_and_si128(temp_1, cmp0);
sum0 = _mm_hadd_epi16(cmp0, zero_8x16b);
sum1 = _mm_hadd_epi16(sum0, zero_8x16b);
sum2 = _mm_hadd_epi16(sum1, zero_8x16b);
u4_zero_coeff += _mm_cvtsi128_si32(sum2);
}
}
if(mask1)
{
if(mask1 == 0xffff)
u4_zero_coeff+=8;
else
{
cmp1 = _mm_and_si128(temp_1, cmp1);
sum0 = _mm_hadd_epi16(cmp1, zero_8x16b);
sum1 = _mm_hadd_epi16(sum0, zero_8x16b);
sum2 = _mm_hadd_epi16(sum1, zero_8x16b);
u4_zero_coeff += _mm_cvtsi128_si32(sum2);
}
}
/* Return total nonzero coefficients in the current sub block */
u4_nonzero_coeff = 16 - u4_zero_coeff;
*pu1_nnz = u4_nonzero_coeff;
}
/**
*******************************************************************************
*
* @brief
* This function performs forward hadamard transform and quantization on a 4*4 block
*
* @par Description:
* The function accepts source buffer and estimation buffer. From these, it
* computes the residue. This is residue is then transformed and quantized.
* The transform and quantization are in placed computed. They use the residue
* buffer for this.
*
* @param[in] pu1_src
* Pointer to source sub-block
*
* @param[in] pu1_pred
* Pointer to prediction sub-block
*
* @param[in] pi2_out
* Pointer to residual sub-block
*
* @param[in] src_strd
* Source stride
*
* @param[in] pred_strd
* Prediction stride
*
* @param[in] dst_strd
* Destination stride
*
* @param[in] u4_qbits
* QP_BITS_h264_4x4 + floor(QP/6)
*
* @param[in] pu2_threshold_matrix
* Pointer to Forward Quant Threshold Matrix
*
* @param[in] pu2_scale_matrix
* Pointer to Forward Quant Scale Matrix
*
* @param[in] u4_round_factor
* Quantization Round factor
*
* @param[out] pu1_nnz
* Total non-zero coefficients in the current sub-block
*
* @returns
*
* @remarks
* None
*
*/
void ih264_hadamard_quant_4x4_sse42(WORD16 *pi2_src, WORD16 *pi2_dst,
const UWORD16 *pu2_scale_matrix,
const UWORD16 *pu2_threshold_matrix, UWORD32 u4_qbits,
UWORD32 u4_round_factor,UWORD8 *pu1_nnz
)
{
WORD32 u4_zero_coeff,u4_nonzero_coeff=0;
__m128i cmp0, cmp1, sum0, sum1, sum2;
WORD32 mask0, mask1;
__m128i src_r0_r1, src_r2_r3, sign_reg;
__m128i src_r0, src_r1, src_r2, src_r3;
__m128i zero_8x16b = _mm_setzero_si128();
__m128i temp0, temp1, temp2, temp3;
__m128i sign_reg0, sign_reg1, sign_reg2, sign_reg3;
__m128i temp_1 = _mm_set1_epi16(1);
__m128i rnd_fact = _mm_set1_epi32(u4_round_factor);
__m128i scale_val = _mm_set1_epi32(pu2_scale_matrix[0]);
UNUSED (pu2_threshold_matrix);
src_r0_r1 = _mm_loadu_si128((__m128i *) (pi2_src)); //a00 a01 a02 a03 a10 a11 a12 a13 -- the source matrix 0th,1st row
src_r2_r3 = _mm_loadu_si128((__m128i *) (pi2_src + 8)); //a20 a21 a22 a23 a30 a31 a32 a33 -- the source matrix 2nd,3rd row
sign_reg = _mm_cmpgt_epi16(zero_8x16b, src_r0_r1);
src_r0 = _mm_unpacklo_epi16(src_r0_r1, sign_reg); //a0 a1 a2 a3
src_r1 = _mm_unpackhi_epi16(src_r0_r1, sign_reg); //b0 b1 b2 b3
sign_reg = _mm_cmpgt_epi16(zero_8x16b, src_r2_r3);
src_r2 = _mm_unpacklo_epi16(src_r2_r3, sign_reg); //c0 c1 c2 c3
src_r3 = _mm_unpackhi_epi16(src_r2_r3, sign_reg); //d0 d1 d2 d3
/* Perform Inverse transform */
/*-------------------------------------------------------------*/
/* Forward DC transform [ Horizontal transformation ] */
/*-------------------------------------------------------------*/
// Matrix transpose
/*
* a0 a1 a2 a3
* b0 b1 b2 b3
* c0 c1 c2 c3
* d0 d1 d2 d3
*/
temp0 = _mm_unpacklo_epi32(src_r0, src_r1); //a0 b0 a1 b1
temp2 = _mm_unpacklo_epi32(src_r2, src_r3); //c0 d0 c1 d1
temp1 = _mm_unpackhi_epi32(src_r0, src_r1); //a2 b2 a3 b3
temp3 = _mm_unpackhi_epi32(src_r2, src_r3); //c2 d2 c3 d3
src_r0 = _mm_unpacklo_epi64(temp0, temp2); //a0 b0 c0 d0
src_r1 = _mm_unpackhi_epi64(temp0, temp2); //a1 b1 c1 d1
src_r2 = _mm_unpacklo_epi64(temp1, temp3); //a2 b2 c2 d2
src_r3 = _mm_unpackhi_epi64(temp1, temp3); //a3 b3 c3 d3
temp0 = _mm_add_epi32(src_r0, src_r3);
temp1 = _mm_add_epi32(src_r1, src_r2);
temp2 = _mm_sub_epi32(src_r1, src_r2);
temp3 = _mm_sub_epi32(src_r0, src_r3);
src_r0 = _mm_add_epi32(temp0, temp1);
src_r1 = _mm_add_epi32(temp2, temp3);
src_r2 = _mm_sub_epi32(temp0, temp1);
src_r3 = _mm_sub_epi32(temp3, temp2);
/*-------------------------------------------------------------*/
/* Forward DC transform [ Vertical transformation ] */
/*-------------------------------------------------------------*/
// Matrix transpose
/*
* a0 b0 c0 d0
* a1 b1 c1 d1
* a2 b2 c2 d2
* a3 b3 c3 d3
*/
temp0 = _mm_unpacklo_epi32(src_r0, src_r1); //a0 a1 b0 b1
temp2 = _mm_unpacklo_epi32(src_r2, src_r3); //a2 a3 b2 b3
temp1 = _mm_unpackhi_epi32(src_r0, src_r1); //c0 c1 d0 d1
temp3 = _mm_unpackhi_epi32(src_r2, src_r3); //c2 c3 d2 d3
src_r0 = _mm_unpacklo_epi64(temp0, temp2); //a0 a1 a2 a3
src_r1 = _mm_unpackhi_epi64(temp0, temp2); //b0 b1 b2 b3
src_r2 = _mm_unpacklo_epi64(temp1, temp3); //c0 c1 c2 c3
src_r3 = _mm_unpackhi_epi64(temp1, temp3); //d0 d1 d2 d3
temp0 = _mm_add_epi32(src_r0, src_r3);
temp1 = _mm_add_epi32(src_r1, src_r2);
temp2 = _mm_sub_epi32(src_r1, src_r2);
temp3 = _mm_sub_epi32(src_r0, src_r3);
src_r0 = _mm_add_epi32(temp0, temp1);
src_r1 = _mm_add_epi32(temp2, temp3);
src_r2 = _mm_sub_epi32(temp0, temp1);
src_r3 = _mm_sub_epi32(temp3, temp2);
src_r0 = _mm_srai_epi32(src_r0, 1);
src_r1 = _mm_srai_epi32(src_r1, 1);
src_r2 = _mm_srai_epi32(src_r2, 1);
src_r3 = _mm_srai_epi32(src_r3, 1);
// Quantization
sign_reg0 = _mm_cmpgt_epi32(zero_8x16b, src_r0); //Find sign of each value for later restoration
sign_reg1 = _mm_cmpgt_epi32(zero_8x16b, src_r1);
sign_reg2 = _mm_cmpgt_epi32(zero_8x16b, src_r2);
sign_reg3 = _mm_cmpgt_epi32(zero_8x16b, src_r3);
sign_reg0 = _mm_packs_epi32(sign_reg0, sign_reg1); //Sign = -1 or 0 depending on <0 or >0 respectively
sign_reg2 = _mm_packs_epi32(sign_reg2, sign_reg3);
sign_reg0 = _mm_slli_epi16(sign_reg0, 1); //Sign = -2 or 0 depending on <0 or >0 respectively
sign_reg2 = _mm_slli_epi16(sign_reg2, 1);
sign_reg0 = _mm_add_epi16(temp_1,sign_reg0); //Sign = -1 or 1 depending on <0 or >0 respectively
sign_reg2 = _mm_add_epi16(temp_1,sign_reg2);
src_r0 = _mm_abs_epi32(src_r0); //Absolute values
src_r1 = _mm_abs_epi32(src_r1);
src_r2 = _mm_abs_epi32(src_r2);
src_r3 = _mm_abs_epi32(src_r3);
temp0 = _mm_mullo_epi32(scale_val, src_r0); //multiply by pu2_scale_matrix[0]
temp1 = _mm_mullo_epi32(scale_val, src_r1);
temp2 = _mm_mullo_epi32(scale_val, src_r2);
temp3 = _mm_mullo_epi32(scale_val, src_r3);
temp0 = _mm_add_epi32(temp0,rnd_fact); //Add round factor
temp1 = _mm_add_epi32(temp1,rnd_fact);
temp2 = _mm_add_epi32(temp2,rnd_fact);
temp3 = _mm_add_epi32(temp3,rnd_fact);
temp0 = _mm_srli_epi32(temp0,u4_qbits); //RIght shift by qbits, unsigned variable, so shift right immediate works
temp1 = _mm_srli_epi32(temp1,u4_qbits);
temp2 = _mm_srli_epi32(temp2,u4_qbits);
temp3 = _mm_srli_epi32(temp3,u4_qbits);
temp0 = _mm_packs_epi32 (temp0,temp1); //Final values are 16-bits only.
temp2 = _mm_packs_epi32 (temp2,temp3);
temp0 = _mm_sign_epi16(temp0, sign_reg0); //Sign restoration
temp2 = _mm_sign_epi16(temp2, sign_reg2);
_mm_storeu_si128((__m128i *) (&pi2_dst[0]), temp0);
_mm_storeu_si128((__m128i *) (&pi2_dst[8]), temp2);
cmp0 = _mm_cmpeq_epi16(temp0, zero_8x16b);
cmp1 = _mm_cmpeq_epi16(temp2, zero_8x16b);
mask0 = _mm_movemask_epi8(cmp0);
mask1 = _mm_movemask_epi8(cmp1);
u4_zero_coeff = 0;
if(mask0)
{
if(mask0 == 0xffff)
u4_zero_coeff+=8;
else
{
cmp0 = _mm_and_si128(temp_1, cmp0);
sum0 = _mm_hadd_epi16(cmp0, zero_8x16b);
sum1 = _mm_hadd_epi16(sum0, zero_8x16b);
sum2 = _mm_hadd_epi16(sum1, zero_8x16b);
u4_zero_coeff += _mm_cvtsi128_si32(sum2);
}
}
if(mask1)
{
if(mask1 == 0xffff)
u4_zero_coeff+=8;
else
{
cmp1 = _mm_and_si128(temp_1, cmp1);
sum0 = _mm_hadd_epi16(cmp1, zero_8x16b);
sum1 = _mm_hadd_epi16(sum0, zero_8x16b);
sum2 = _mm_hadd_epi16(sum1, zero_8x16b);
u4_zero_coeff += _mm_cvtsi128_si32(sum2);
}
}
/* Return total nonzero coefficients in the current sub block */
u4_nonzero_coeff = 16 - u4_zero_coeff;
pu1_nnz[0] = u4_nonzero_coeff;
}
/**
*******************************************************************************
*
* @brief
* This function performs forward hadamard transform and quantization on a 2*2 block
* for both U and V planes
*
* @par Description:
* The function accepts source buffer and estimation buffer. From these, it
* computes the residue. This is residue is then transformed and quantized.
* The transform and quantization are in placed computed. They use the residue
* buffer for this.
*
* @param[in] pu1_src
* Pointer to source sub-block
*
* @param[in] pu1_pred
* Pointer to prediction sub-block
*
* @param[in] pi2_out
* Pointer to residual sub-block
*
* @param[in] src_strd
* Source stride
*
* @param[in] pred_strd
* Prediction stride
*
* @param[in] dst_strd
* Destination stride
*
* @param[in] u4_qbits
* QP_BITS_h264_4x4 + floor(QP/6)
*
* @param[in] pu2_threshold_matrix
* Pointer to Forward Quant Threshold Matrix
*
* @param[in] pu2_scale_matrix
* Pointer to Forward Quant Scale Matrix
*
* @param[in] u4_round_factor
* Quantization Round factor
*
* @param[out] pu1_nnz
* Total non-zero coefficients in the current sub-block
*
* @returns
*
* @remarks
* NNZ for dc is populated at 0 and 5th position of pu1_nnz
*
*/
void ih264_hadamard_quant_2x2_uv_sse42(WORD16 *pi2_src, WORD16 *pi2_dst,
const UWORD16 *pu2_scale_matrix,
const UWORD16 *pu2_threshold_matrix, UWORD32 u4_qbits,
UWORD32 u4_round_factor,UWORD8 *pu1_nnz)
{
WORD32 val, nonzero_coeff_0=0, nonzero_coeff_1=0;
__m128i cmp, cmp0, cmp1;
__m128i sum0, sum1;
WORD32 mask, mask0, mask1;
__m128i src, plane_0, plane_1, temp0, temp1, sign_reg;
__m128i zero_8x16b = _mm_setzero_si128();
__m128i scale_val = _mm_set1_epi32(pu2_scale_matrix[0]);
__m128i sign_reg0, sign_reg1;
__m128i temp_1 = _mm_set1_epi16(1);
__m128i rnd_fact = _mm_set1_epi32(u4_round_factor);
UNUSED (pu2_threshold_matrix);
src = _mm_loadu_si128((__m128i *)pi2_src); //a0 a1 a2 a3 b0 b1 b2 b3
sign_reg = _mm_cmpgt_epi16(zero_8x16b, src);
plane_0 = _mm_unpacklo_epi16(src, sign_reg); //a0 a1 a2 a3 -- 32 bits
plane_1 = _mm_unpackhi_epi16(src, sign_reg); //b0 b1 b2 b3 -- 32 bits
temp0 = _mm_hadd_epi32(plane_0, plane_1); //a0+a1 a2+a3 b0+b1 b2+b3
temp1 = _mm_hsub_epi32(plane_0, plane_1); //a0-a1 a2-a3 b0-b1 b2-b3
plane_0 = _mm_hadd_epi32(temp0, temp1); //a0+a1+a2+a3 b0+b1+b2+b3 a0-a1+a2-a3 b0-b1+b2-b3
plane_1 = _mm_hsub_epi32(temp0, temp1); //a0+a1-a2-a3 b0+b1-b2-b3 a0-a1-a2+a3 b0-b1-b2+b3
temp0 = _mm_unpacklo_epi32(plane_0, plane_1); //a0+a1+a2+a3 a0+a1-a2-a3 b0+b1+b2+b3 b0+b1-b2-b3
temp1 = _mm_unpackhi_epi32(plane_0, plane_1); //a0-a1+a2-a3 a0-a1-a2+a3 b0-b1+b2-b3 b0-b1-b2+b3
plane_0 = _mm_unpacklo_epi64(temp0, temp1); //a0+a1+a2+a3 a0+a1-a2-a3 a0-a1+a2-a3 a0-a1-a2+a3
plane_1 = _mm_unpackhi_epi64(temp0, temp1); //b0+b1+b2+b3 b0+b1-b2-b3 b0-b1+b2-b3 b0-b1-b2+b3
plane_0 = _mm_shuffle_epi32(plane_0, 0xd8); //a0+a1+a2+a3 a0-a1+a2-a3 a0+a1-a2-a3 a0-a1-a2+a3
plane_1 = _mm_shuffle_epi32(plane_1, 0xd8); //b0+b1+b2+b3 b0-b1+b2-b3 b0+b1-b2-b3 b0-b1-b2+b3
// Quantization
sign_reg0 = _mm_cmpgt_epi32(zero_8x16b, plane_0); //Find sign of each value for later restoration
sign_reg1 = _mm_cmpgt_epi32(zero_8x16b, plane_1);
sign_reg0 = _mm_packs_epi32(sign_reg0, sign_reg1); //Sign = -1 or 0 depending on <0 or >0 respectively
sign_reg0 = _mm_slli_epi16(sign_reg0, 1); //Sign = -2 or 0 depending on <0 or >0 respectively
sign_reg0 = _mm_add_epi16(temp_1,sign_reg0); //Sign = -1 or 1 depending on <0 or >0 respectively
plane_0 = _mm_abs_epi32(plane_0); //Absolute values
plane_1 = _mm_abs_epi32(plane_1);
temp0 = _mm_mullo_epi32(scale_val, plane_0); //multiply by pu2_scale_matrix[0]
temp1 = _mm_mullo_epi32(scale_val, plane_1); //multiply by pu2_scale_matrix[0]
temp0 = _mm_add_epi32(temp0,rnd_fact); //Add round factor
temp1 = _mm_add_epi32(temp1,rnd_fact);
temp0 = _mm_srli_epi32(temp0,u4_qbits); //RIght shift by qbits, unsigned variable, so shift right immediate works
temp1 = _mm_srli_epi32(temp1,u4_qbits);
temp0 = _mm_packs_epi32 (temp0,temp1); //Final values are 16-bits only.
temp0 = _mm_sign_epi16(temp0, sign_reg0); //Sign restoration
_mm_storeu_si128((__m128i *) (&pi2_dst[0]), temp0);
cmp = _mm_cmpeq_epi16(temp0, zero_8x16b);
mask = _mm_movemask_epi8(cmp);
mask0 = mask & 0xff;
mask1 = mask>>8;
if(mask0)
{
if(mask0 == 0xff)
nonzero_coeff_0 += 4;
else
{
cmp0 = _mm_and_si128(temp_1, cmp);
sum0 = _mm_hadd_epi16(cmp0, zero_8x16b);
sum1 = _mm_hadd_epi16(sum0, zero_8x16b);
val = _mm_cvtsi128_si32(sum1);
val = val & 0xffff;
nonzero_coeff_0 += val;
}
}
if(mask1)
{
if(mask1 == 0xff)
nonzero_coeff_1 += 4;
else
{
cmp1 = _mm_srli_si128(cmp, 8);
cmp1 = _mm_and_si128(temp_1, cmp1);
sum0 = _mm_hadd_epi16(cmp1, zero_8x16b);
sum1 = _mm_hadd_epi16(sum0, zero_8x16b);
nonzero_coeff_1 += _mm_cvtsi128_si32(sum1);
}
}
pu1_nnz[0] = 4 - nonzero_coeff_0;
pu1_nnz[1] = 4 - nonzero_coeff_1;
}