/******************************************************************************
*
* Copyright (C) 2012 Ittiam Systems Pvt Ltd, Bangalore
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at:
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
******************************************************************************/
/**
*******************************************************************************
* @file
* ihevc_sao_atom_intr.c
*
* @brief
* Contains function definitions for Sample adaptive offset(SAO) used in-loop
* filtering
*
* @author
* 100592
*
* @par List of Functions:
* - ihevc_sao_band_offset_luma_ssse3()
* - ihevc_sao_band_offset_chroma_ssse3()
* - ihevc_sao_edge_offset_class0_ssse3()
* - ihevc_sao_edge_offset_class0_chroma_ssse3()
* - ihevc_sao_edge_offset_class1_ssse3()
* - ihevc_sao_edge_offset_class1_chroma_ssse3()
* - ihevc_sao_edge_offset_class2_ssse3()
* - ihevc_sao_edge_offset_class2_chroma_ssse3()
* - ihevc_sao_edge_offset_class3_ssse3()
* - ihevc_sao_edge_offset_class3_chroma_ssse3()
*
* @remarks
* None
*
*******************************************************************************
*/
/*****************************************************************************/
/* File Includes */
/*****************************************************************************/
#include <stdio.h>
#include "ihevc_typedefs.h"
#include "ihevc_platform_macros.h"
#include "ihevc_macros.h"
#include "ihevc_func_selector.h"
#include "ihevc_defs.h"
#include "ihevc_tables_x86_intr.h"
#include "ihevc_common_tables.h"
#include "ihevc_sao.h"
#include <immintrin.h>
#define NUM_BAND_TABLE 32
/**
*******************************************************************************
*
* @brief
* Has two sets of functions : band offset and edge offset both for luma and chroma
* edge offset has horizontal ,vertical, 135 degree and 45 degree
*
* @par Description:
*
*
* @param[in-out] pu1_src
* Pointer to the source
*
* @param[in] src_strd
* Source stride
*
* @param[in-out] pu1_src_left
* source left boundary
*
* @param[in-out] pu1_src_top
* Source top boundary
*
* @param[in-out] pu1_src_top_left
* Source top left boundary
*
* @param[in] pu1_src_top_right
* Source top right boundary
*
* @param[in] pu1_src_bot_left
* Source bottom left boundary
*
* @param[in] pu1_avail
* boundary availability flags
*
* @param[in] pi1_sao_offset_u
* Chroma U sao offset values
*
* @param[in] pi1_sao_offset_v
* Chroma V sao offset values
*
* @param[in] pi1_sao_offset
* Luma sao offset values
*
* @param[in] wd
* width of the source
* @param[in] ht
* height of the source
* @returns
*
* @remarks
* None
*
*******************************************************************************
*/
void ihevc_sao_band_offset_luma_ssse3(UWORD8 *pu1_src,
WORD32 src_strd,
UWORD8 *pu1_src_left,
UWORD8 *pu1_src_top,
UWORD8 *pu1_src_top_left,
WORD32 sao_band_pos,
WORD8 *pi1_sao_offset,
WORD32 wd,
WORD32 ht)
{
WORD32 row, col;
UWORD8 *pu1_src_cpy;
WORD32 wd_rem;
WORD8 offset = 0;
__m128i src_temp0_8x16b, src_temp1_8x16b, src_temp2_8x16b, src_temp3_8x16b;
__m128i band_table0_8x16b, band_table1_8x16b, band_table2_8x16b, band_table3_8x16b;
__m128i tmp_set_128i_1, tmp_set_128i_2, tmp_set_128i_3, tmp_set_128i_4;
__m128i band_pos_16x8b;
__m128i sao_offset;
__m128i cmp_mask, cmp_store;
/* Updating left and top-left and top */
for(row = 0; row < ht; row++)
{
pu1_src_left[row] = pu1_src[row * src_strd + (wd - 1)];
}
pu1_src_top_left[0] = pu1_src_top[wd - 1];
for(col = 0; col < wd; col += 8)
{
tmp_set_128i_1 = _mm_loadl_epi64((__m128i *)(pu1_src + (ht - 1) * src_strd + offset));
_mm_storel_epi64((__m128i *)(pu1_src_top + offset), tmp_set_128i_1);
offset += 8;
}
//replicating sao_band_pos as 8 bit value 16 times
band_pos_16x8b = _mm_set1_epi16((WORD16)(sao_band_pos << 3));
//value set for sao_offset extraction
tmp_set_128i_1 = _mm_set_epi8(-128, 1, -128, 1, -128, 1, -128, 1, -128, 1, -128, 1, -128, 1, -128, 1);
tmp_set_128i_2 = _mm_set_epi8(-128, 2, -128, 2, -128, 2, -128, 2, -128, 2, -128, 2, -128, 2, -128, 2);
tmp_set_128i_3 = _mm_set_epi8(-128, 3, -128, 3, -128, 3, -128, 3, -128, 3, -128, 3, -128, 3, -128, 3);
tmp_set_128i_4 = _mm_set_epi8(-128, 4, -128, 4, -128, 4, -128, 4, -128, 4, -128, 4, -128, 4, -128, 4);
//loaded sao offset values
sao_offset = _mm_loadl_epi64((__m128i *)pi1_sao_offset);
//loading 16bit 32values of gu2_table_band_idx consecutively in 4 registers
band_table0_8x16b = _mm_load_si128((__m128i *)(gu2_table_band_idx));
band_table1_8x16b = _mm_load_si128((__m128i *)(gu2_table_band_idx + 8));
band_table2_8x16b = _mm_load_si128((__m128i *)(gu2_table_band_idx + 16));
band_table3_8x16b = _mm_load_si128((__m128i *)(gu2_table_band_idx + 24));
//band_position addition
band_table0_8x16b = _mm_add_epi16(band_table0_8x16b, band_pos_16x8b);
band_table1_8x16b = _mm_add_epi16(band_table1_8x16b, band_pos_16x8b);
band_table2_8x16b = _mm_add_epi16(band_table2_8x16b, band_pos_16x8b);
band_table3_8x16b = _mm_add_epi16(band_table3_8x16b, band_pos_16x8b);
//sao_offset duplication
tmp_set_128i_1 = _mm_shuffle_epi8(sao_offset, tmp_set_128i_1);
tmp_set_128i_2 = _mm_shuffle_epi8(sao_offset, tmp_set_128i_2);
tmp_set_128i_3 = _mm_shuffle_epi8(sao_offset, tmp_set_128i_3);
tmp_set_128i_4 = _mm_shuffle_epi8(sao_offset, tmp_set_128i_4);
//settng for comparision
cmp_mask = _mm_set1_epi16(16);
cmp_store = _mm_set1_epi16(0x00ff);
//sao_offset addition
band_table0_8x16b = _mm_add_epi16(band_table0_8x16b, tmp_set_128i_1);
band_table1_8x16b = _mm_add_epi16(band_table1_8x16b, tmp_set_128i_2);
band_table2_8x16b = _mm_add_epi16(band_table2_8x16b, tmp_set_128i_3);
band_table3_8x16b = _mm_add_epi16(band_table3_8x16b, tmp_set_128i_4);
//masking upper 8bit values of each 16 bit band table value
band_table0_8x16b = _mm_and_si128(band_table0_8x16b, cmp_store);
band_table1_8x16b = _mm_and_si128(band_table1_8x16b, cmp_store);
band_table2_8x16b = _mm_and_si128(band_table2_8x16b, cmp_store);
band_table3_8x16b = _mm_and_si128(band_table3_8x16b, cmp_store);
switch(sao_band_pos)
{
case 0:
tmp_set_128i_2 = _mm_cmpgt_epi16(cmp_mask, band_table0_8x16b);
band_table0_8x16b = _mm_and_si128(band_table0_8x16b, tmp_set_128i_2);
break;
case 28:
tmp_set_128i_2 = _mm_cmpgt_epi16(cmp_mask, band_table3_8x16b);
band_table3_8x16b = _mm_or_si128(band_table3_8x16b, tmp_set_128i_2);
break;
case 29:
tmp_set_128i_2 = _mm_cmpgt_epi16(cmp_mask, band_table2_8x16b);
band_table2_8x16b = _mm_or_si128(band_table2_8x16b, tmp_set_128i_2);
tmp_set_128i_2 = _mm_cmpgt_epi16(cmp_mask, band_table3_8x16b);
band_table3_8x16b = _mm_and_si128(band_table3_8x16b, tmp_set_128i_2);
break;
case 30:
tmp_set_128i_2 = _mm_cmpgt_epi16(cmp_mask, band_table1_8x16b);
band_table1_8x16b = _mm_or_si128(band_table1_8x16b, tmp_set_128i_2);
tmp_set_128i_2 = _mm_cmpgt_epi16(cmp_mask, band_table2_8x16b);
band_table2_8x16b = _mm_and_si128(band_table2_8x16b, tmp_set_128i_2);
break;
case 31:
tmp_set_128i_2 = _mm_cmpgt_epi16(cmp_mask, band_table0_8x16b);
band_table0_8x16b = _mm_or_si128(band_table0_8x16b, tmp_set_128i_2);
tmp_set_128i_2 = _mm_cmpgt_epi16(cmp_mask, band_table1_8x16b);
band_table1_8x16b = _mm_and_si128(band_table1_8x16b, tmp_set_128i_2);
break;
default:
break;
}
//sao_offset is reused for zero cmp mask.
sao_offset = _mm_setzero_si128();
tmp_set_128i_1 = _mm_set1_epi8(1);
//tmp_set_128i_2 = _mm_set_epi8 (128,7,128,6,128,5,128,4,128,3,128,2,128,1,128,0);
cmp_mask = _mm_packus_epi16(cmp_mask, cmp_mask); //cmp_msk=dup16(16);
//masking upper 8bit values of each 16 bit band table value
band_table0_8x16b = _mm_and_si128(band_table0_8x16b, cmp_store);
band_table1_8x16b = _mm_and_si128(band_table1_8x16b, cmp_store);
band_table2_8x16b = _mm_and_si128(band_table2_8x16b, cmp_store);
band_table3_8x16b = _mm_and_si128(band_table3_8x16b, cmp_store);
//band table 8x16 four registers are packed into 16x8 two registers: band_table0_8x16b and band_table2_8x16b
band_table0_8x16b = _mm_packus_epi16(band_table0_8x16b, band_table1_8x16b);
band_table2_8x16b = _mm_packus_epi16(band_table2_8x16b, band_table3_8x16b);
band_table3_8x16b = _mm_slli_epi16(cmp_mask, 1); // to compare if value is greater than 31
band_pos_16x8b = _mm_packus_epi16(band_pos_16x8b, band_pos_16x8b); //band_pos is now 8 bit aligned
band_table3_8x16b = _mm_sub_epi8(band_table3_8x16b, tmp_set_128i_1); // to compare if value is greater than 31
cmp_mask = _mm_sub_epi8(cmp_mask, tmp_set_128i_1);
// band_pos_16x8b = _mm_or_si128(band_pos_16x8b,cmp_store);
for(col = wd; col >= 16; col -= 16)
{
pu1_src_cpy = pu1_src;
for(row = ht; row > 0; row -= 2)
{
//row = 0 load 8 pixel values from 7:0 pos. relative to cur. pos.
src_temp0_8x16b = _mm_loadu_si128((__m128i *)(pu1_src_cpy));
// row = 1
src_temp2_8x16b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + src_strd));
//saturated substract 8 bit
tmp_set_128i_1 = _mm_sub_epi8(src_temp0_8x16b, band_pos_16x8b);
tmp_set_128i_3 = _mm_sub_epi8(src_temp2_8x16b, band_pos_16x8b);
//if the values less than 0 put ff
tmp_set_128i_2 = _mm_cmpgt_epi8(sao_offset, tmp_set_128i_1);
tmp_set_128i_4 = _mm_cmpgt_epi8(sao_offset, tmp_set_128i_3);
tmp_set_128i_1 = _mm_or_si128(tmp_set_128i_1, tmp_set_128i_2);
tmp_set_128i_3 = _mm_or_si128(tmp_set_128i_3, tmp_set_128i_4);
//if the values gret=ater than 31 put ff
tmp_set_128i_2 = _mm_cmpgt_epi8(tmp_set_128i_1, band_table3_8x16b);
tmp_set_128i_4 = _mm_cmpgt_epi8(tmp_set_128i_3, band_table3_8x16b);
tmp_set_128i_1 = _mm_or_si128(tmp_set_128i_1, tmp_set_128i_2);
tmp_set_128i_3 = _mm_or_si128(tmp_set_128i_3, tmp_set_128i_4);
//row 0 and row1
//if the values >16 then put ff ,cmp_mask = dup16(15)
cmp_store = _mm_cmpgt_epi8(tmp_set_128i_1, cmp_mask);
//values 16 to 31 for row 0 & 1 but values <16 ==0
tmp_set_128i_2 = _mm_and_si128(tmp_set_128i_1, cmp_store);
// values 0 to 15 for row 0 & 1
tmp_set_128i_1 = _mm_or_si128(tmp_set_128i_1, cmp_store);
//values 16 to 31 for row 0 & 1 but values <16 masked to ff
cmp_store = _mm_cmpeq_epi8(tmp_set_128i_2, sao_offset);
tmp_set_128i_2 = _mm_or_si128(tmp_set_128i_2, cmp_store);
//row 2 and row 3
//if the values >16 then put ff ,cmp_mask = dup16(15)
cmp_store = _mm_cmpgt_epi8(tmp_set_128i_3, cmp_mask);
//values 16 to 31 for row 2 & 3 but values <16 ==0
tmp_set_128i_4 = _mm_and_si128(tmp_set_128i_3, cmp_store);
// values 0 to 15 for row 2 & 3
tmp_set_128i_3 = _mm_or_si128(tmp_set_128i_3, cmp_store);
//values 16 to 31 for row 2 & 3 but values <16 masked to ff
cmp_store = _mm_cmpeq_epi8(tmp_set_128i_4, sao_offset);
tmp_set_128i_4 = _mm_or_si128(tmp_set_128i_4, cmp_store);
//row 0 and row 1
//to preserve pixel values in which no offset needs to be added.
cmp_store = _mm_cmpeq_epi8(tmp_set_128i_1, tmp_set_128i_2);
src_temp0_8x16b = _mm_and_si128(src_temp0_8x16b, cmp_store);
//row 2 and row 3
//to preserve pixel values in which no offset needs to be added.
cmp_store = _mm_cmpeq_epi8(tmp_set_128i_3, tmp_set_128i_4);
src_temp2_8x16b = _mm_and_si128(src_temp2_8x16b, cmp_store);
//indexing 0 - 15 bandtable indexes
tmp_set_128i_1 = _mm_shuffle_epi8(band_table0_8x16b, tmp_set_128i_1);
tmp_set_128i_3 = _mm_shuffle_epi8(band_table0_8x16b, tmp_set_128i_3);
tmp_set_128i_2 = _mm_shuffle_epi8(band_table2_8x16b, tmp_set_128i_2);
tmp_set_128i_4 = _mm_shuffle_epi8(band_table2_8x16b, tmp_set_128i_4);
// combining all offsets results
tmp_set_128i_1 = _mm_or_si128(tmp_set_128i_1, tmp_set_128i_2);
tmp_set_128i_3 = _mm_or_si128(tmp_set_128i_3, tmp_set_128i_4);
// combing results woth the pixel values
src_temp0_8x16b = _mm_or_si128(src_temp0_8x16b, tmp_set_128i_1);
src_temp2_8x16b = _mm_or_si128(src_temp2_8x16b, tmp_set_128i_3);
//row = 0 store 8 pixel values from 7:0 pos. relative to cur. pos.
_mm_storeu_si128((__m128i *)(pu1_src_cpy), src_temp0_8x16b);
// row = 1
_mm_storeu_si128((__m128i *)(pu1_src_cpy + src_strd), src_temp2_8x16b);
pu1_src_cpy += (src_strd << 1);
}
pu1_src += 16;
}
wd_rem = wd & 0xF;
if(wd_rem)
{pu1_src_cpy = pu1_src;
for(row = ht; row > 0; row -= 4)
{
//row = 0 load 8 pixel values from 7:0 pos. relative to cur. pos.
src_temp0_8x16b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy));
// row = 1
src_temp1_8x16b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy + src_strd));
// row = 2
src_temp2_8x16b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy + 2 * src_strd));
// row = 3
src_temp3_8x16b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy + 3 * src_strd));
//row0 and row1 packed and row2 and row3 packed
src_temp0_8x16b = _mm_unpacklo_epi64(src_temp0_8x16b, src_temp1_8x16b);
src_temp2_8x16b = _mm_unpacklo_epi64(src_temp2_8x16b, src_temp3_8x16b);
//saturated substract 8 bit
tmp_set_128i_1 = _mm_sub_epi8(src_temp0_8x16b, band_pos_16x8b);
tmp_set_128i_3 = _mm_sub_epi8(src_temp2_8x16b, band_pos_16x8b);
//if the values less than 0 put ff
tmp_set_128i_2 = _mm_cmpgt_epi8(sao_offset, tmp_set_128i_1);
tmp_set_128i_4 = _mm_cmpgt_epi8(sao_offset, tmp_set_128i_3);
tmp_set_128i_1 = _mm_or_si128(tmp_set_128i_1, tmp_set_128i_2);
tmp_set_128i_3 = _mm_or_si128(tmp_set_128i_3, tmp_set_128i_4);
//if the values gret=ater than 31 put ff
tmp_set_128i_2 = _mm_cmpgt_epi8(tmp_set_128i_1, band_table3_8x16b);
tmp_set_128i_4 = _mm_cmpgt_epi8(tmp_set_128i_3, band_table3_8x16b);
tmp_set_128i_1 = _mm_or_si128(tmp_set_128i_1, tmp_set_128i_2);
tmp_set_128i_3 = _mm_or_si128(tmp_set_128i_3, tmp_set_128i_4);
//row 0 and row1
//if the values >16 then put ff ,cmp_mask = dup16(15)
cmp_store = _mm_cmpgt_epi8(tmp_set_128i_1, cmp_mask);
//values 16 to 31 for row 0 & 1 but values <16 ==0
tmp_set_128i_2 = _mm_and_si128(tmp_set_128i_1, cmp_store);
// values 0 to 15 for row 0 & 1
tmp_set_128i_1 = _mm_or_si128(tmp_set_128i_1, cmp_store);
//values 16 to 31 for row 0 & 1 but values <16 masked to ff
cmp_store = _mm_cmpeq_epi8(tmp_set_128i_2, sao_offset);
tmp_set_128i_2 = _mm_or_si128(tmp_set_128i_2, cmp_store);
//row 2 and row 3
//if the values >16 then put ff ,cmp_mask = dup16(15)
cmp_store = _mm_cmpgt_epi8(tmp_set_128i_3, cmp_mask);
//values 16 to 31 for row 2 & 3 but values <16 ==0
tmp_set_128i_4 = _mm_and_si128(tmp_set_128i_3, cmp_store);
// values 0 to 15 for row 2 & 3
tmp_set_128i_3 = _mm_or_si128(tmp_set_128i_3, cmp_store);
//values 16 to 31 for row 2 & 3 but values <16 masked to ff
cmp_store = _mm_cmpeq_epi8(tmp_set_128i_4, sao_offset);
tmp_set_128i_4 = _mm_or_si128(tmp_set_128i_4, cmp_store);
//row 0 and row 1
//to preserve pixel values in which no offset needs to be added.
cmp_store = _mm_cmpeq_epi8(tmp_set_128i_1, tmp_set_128i_2);
src_temp0_8x16b = _mm_and_si128(src_temp0_8x16b, cmp_store);
//row 2 and row 3
//to preserve pixel values in which no offset needs to be added.
cmp_store = _mm_cmpeq_epi8(tmp_set_128i_3, tmp_set_128i_4);
src_temp2_8x16b = _mm_and_si128(src_temp2_8x16b, cmp_store);
//indexing 0 - 15 bandtable indexes
tmp_set_128i_1 = _mm_shuffle_epi8(band_table0_8x16b, tmp_set_128i_1);
tmp_set_128i_3 = _mm_shuffle_epi8(band_table0_8x16b, tmp_set_128i_3);
tmp_set_128i_2 = _mm_shuffle_epi8(band_table2_8x16b, tmp_set_128i_2);
tmp_set_128i_4 = _mm_shuffle_epi8(band_table2_8x16b, tmp_set_128i_4);
// combining all offsets results
tmp_set_128i_1 = _mm_or_si128(tmp_set_128i_1, tmp_set_128i_2);
tmp_set_128i_3 = _mm_or_si128(tmp_set_128i_3, tmp_set_128i_4);
// combing results woth the pixel values
src_temp0_8x16b = _mm_or_si128(src_temp0_8x16b, tmp_set_128i_1);
src_temp2_8x16b = _mm_or_si128(src_temp2_8x16b, tmp_set_128i_3);
//Getting row1 separately
src_temp1_8x16b = _mm_srli_si128(src_temp0_8x16b, 8);
//Getting row3 separately
src_temp3_8x16b = _mm_srli_si128(src_temp2_8x16b, 8);
//row = 0 store 8 pixel values from 7:0 pos. relative to cur. pos.
_mm_storel_epi64((__m128i *)(pu1_src_cpy), src_temp0_8x16b);
// row = 1
_mm_storel_epi64((__m128i *)(pu1_src_cpy + src_strd), src_temp1_8x16b);
// row = 2
_mm_storel_epi64((__m128i *)(pu1_src_cpy + 2 * src_strd), src_temp2_8x16b);
// row = 3
_mm_storel_epi64((__m128i *)(pu1_src_cpy + 3 * src_strd), src_temp3_8x16b);
pu1_src_cpy += (src_strd << 2);
}
pu1_src += 8;
}
}
void ihevc_sao_band_offset_chroma_ssse3(UWORD8 *pu1_src,
WORD32 src_strd,
UWORD8 *pu1_src_left,
UWORD8 *pu1_src_top,
UWORD8 *pu1_src_top_left,
WORD32 sao_band_pos_u,
WORD32 sao_band_pos_v,
WORD8 *pi1_sao_offset_u,
WORD8 *pi1_sao_offset_v,
WORD32 wd,
WORD32 ht)
{
WORD32 row, col;
WORD8 offset = 0;
__m128i src_temp0_8x16b, src_temp1_8x16b, src_temp2_8x16b, src_temp3_8x16b;
__m128i cmp_msk2;
__m128i band_table0_16x8b, band_table1_16x8b, band_table2_16x8b, band_table3_16x8b;
__m128i tmp_set_128i_1, tmp_set_128i_2, tmp_set_128i_3, tmp_set_128i_4;
__m128i band_pos_u_16x8b, band_pos_v_16x8b;
__m128i sao_offset;
__m128i cmp_mask;
/* Updating left and top and top-left */
for(row = 0; row < ht; row++)
{
pu1_src_left[2 * row] = pu1_src[row * src_strd + (wd - 2)];
pu1_src_left[2 * row + 1] = pu1_src[row * src_strd + (wd - 1)];
}
pu1_src_top_left[0] = pu1_src_top[wd - 2];
pu1_src_top_left[1] = pu1_src_top[wd - 1];
for(col = 0; col < wd; col += 8)
{
tmp_set_128i_1 = _mm_loadl_epi64((__m128i *)(pu1_src + (ht - 1) * src_strd + offset));
_mm_storel_epi64((__m128i *)(pu1_src_top + offset), tmp_set_128i_1);
offset += 8;
}
{ // band _table creation
__m128i temp0_8x16b, temp1_8x16b, temp2_8x16b, temp3_8x16b;
// Band table for U component : band_table0_16x8b and band_table2_16x8b
//replicating sao_band_pos as 8 bit value 16 times
band_pos_u_16x8b = _mm_set1_epi16((WORD16)(sao_band_pos_u << 3));
//value set for sao_offset extraction
tmp_set_128i_1 = _mm_set_epi8(-128, 1, -128, 1, -128, 1, -128, 1, -128, 1, -128, 1, -128, 1, -128, 1);
tmp_set_128i_2 = _mm_set_epi8(-128, 2, -128, 2, -128, 2, -128, 2, -128, 2, -128, 2, -128, 2, -128, 2);
tmp_set_128i_3 = _mm_set_epi8(-128, 3, -128, 3, -128, 3, -128, 3, -128, 3, -128, 3, -128, 3, -128, 3);
tmp_set_128i_4 = _mm_set_epi8(-128, 4, -128, 4, -128, 4, -128, 4, -128, 4, -128, 4, -128, 4, -128, 4);
//loaded sao offset values
sao_offset = _mm_loadl_epi64((__m128i *)pi1_sao_offset_u);
//loading 16bit 32values of gu2_table_band_idx consecutively in 4 registers
band_table0_16x8b = _mm_load_si128((__m128i *)(gu2_table_band_idx));
band_table1_16x8b = _mm_load_si128((__m128i *)(gu2_table_band_idx + 8));
band_table2_16x8b = _mm_load_si128((__m128i *)(gu2_table_band_idx + 16));
band_table3_16x8b = _mm_load_si128((__m128i *)(gu2_table_band_idx + 24));
//band_position addition
band_table0_16x8b = _mm_add_epi16(band_table0_16x8b, band_pos_u_16x8b);
band_table1_16x8b = _mm_add_epi16(band_table1_16x8b, band_pos_u_16x8b);
band_table2_16x8b = _mm_add_epi16(band_table2_16x8b, band_pos_u_16x8b);
band_table3_16x8b = _mm_add_epi16(band_table3_16x8b, band_pos_u_16x8b);
//sao_offset duplication
temp0_8x16b = _mm_shuffle_epi8(sao_offset, tmp_set_128i_1);
temp1_8x16b = _mm_shuffle_epi8(sao_offset, tmp_set_128i_2);
temp2_8x16b = _mm_shuffle_epi8(sao_offset, tmp_set_128i_3);
temp3_8x16b = _mm_shuffle_epi8(sao_offset, tmp_set_128i_4);
//sao_offset addition
band_table0_16x8b = _mm_add_epi16(band_table0_16x8b, temp0_8x16b);
band_table1_16x8b = _mm_add_epi16(band_table1_16x8b, temp1_8x16b);
band_table2_16x8b = _mm_add_epi16(band_table2_16x8b, temp2_8x16b);
band_table3_16x8b = _mm_add_epi16(band_table3_16x8b, temp3_8x16b);
//reuse for clipping
temp1_8x16b = _mm_set1_epi16(0x00ff);
//settng for comparision
cmp_mask = _mm_set1_epi16(16);
//masking upper 8bit values of each 16 bit band table value
band_table0_16x8b = _mm_and_si128(band_table0_16x8b, temp1_8x16b);
band_table1_16x8b = _mm_and_si128(band_table1_16x8b, temp1_8x16b);
band_table2_16x8b = _mm_and_si128(band_table2_16x8b, temp1_8x16b);
band_table3_16x8b = _mm_and_si128(band_table3_16x8b, temp1_8x16b);
//temp1_8x16b reuse for compare storage
switch(sao_band_pos_u)
{
case 0:
temp3_8x16b = _mm_cmpgt_epi16(cmp_mask, band_table0_16x8b);
band_table0_16x8b = _mm_and_si128(band_table0_16x8b, temp3_8x16b);
break;
case 28:
temp3_8x16b = _mm_cmpgt_epi16(cmp_mask, band_table3_16x8b);
band_table3_16x8b = _mm_or_si128(band_table3_16x8b, temp3_8x16b);
break;
case 29:
temp3_8x16b = _mm_cmpgt_epi16(cmp_mask, band_table2_16x8b);
band_table2_16x8b = _mm_or_si128(band_table2_16x8b, temp3_8x16b);
temp3_8x16b = _mm_cmpgt_epi16(cmp_mask, band_table3_16x8b);
band_table3_16x8b = _mm_and_si128(band_table3_16x8b, temp3_8x16b);
break;
case 30:
temp3_8x16b = _mm_cmpgt_epi16(cmp_mask, band_table1_16x8b);
band_table1_16x8b = _mm_or_si128(band_table1_16x8b, temp3_8x16b);
temp3_8x16b = _mm_cmpgt_epi16(cmp_mask, band_table2_16x8b);
band_table2_16x8b = _mm_and_si128(band_table2_16x8b, temp3_8x16b);
break;
case 31:
temp3_8x16b = _mm_cmpgt_epi16(cmp_mask, band_table0_16x8b);
band_table0_16x8b = _mm_or_si128(band_table0_16x8b, temp3_8x16b);
temp3_8x16b = _mm_cmpgt_epi16(cmp_mask, band_table1_16x8b);
band_table1_16x8b = _mm_and_si128(band_table1_16x8b, temp3_8x16b);
break;
default:
break;
}
//masking upper 8bit values of each 16 bit band table value
band_table0_16x8b = _mm_and_si128(band_table0_16x8b, temp1_8x16b);
band_table1_16x8b = _mm_and_si128(band_table1_16x8b, temp1_8x16b);
band_table2_16x8b = _mm_and_si128(band_table2_16x8b, temp1_8x16b);
band_table3_16x8b = _mm_and_si128(band_table3_16x8b, temp1_8x16b);
//band table 8x16 four registers are packed into 16x8 two registers: band_table0_8x16b and band_table2_8x16b
band_table0_16x8b = _mm_packus_epi16(band_table0_16x8b, band_table1_16x8b);
band_table2_16x8b = _mm_packus_epi16(band_table2_16x8b, band_table3_16x8b);
// Band table for U component over
// Band table for V component : band_table1_16x8b and band_table3_16x8b
// replicating sao_band_pos as 8 bit value 16 times
band_pos_v_16x8b = _mm_set1_epi16((WORD16)(sao_band_pos_v << 3));
//loaded sao offset values
sao_offset = _mm_loadl_epi64((__m128i *)pi1_sao_offset_v);
//loading 16bit 32values of gu2_table_band_idx consecutively in 4 registers
temp0_8x16b = _mm_load_si128((__m128i *)(gu2_table_band_idx));
band_table1_16x8b = _mm_load_si128((__m128i *)(gu2_table_band_idx + 8));
temp2_8x16b = _mm_load_si128((__m128i *)(gu2_table_band_idx + 16));
band_table3_16x8b = _mm_load_si128((__m128i *)(gu2_table_band_idx + 24));
//band_position addition
temp0_8x16b = _mm_add_epi16(temp0_8x16b, band_pos_v_16x8b);
band_table1_16x8b = _mm_add_epi16(band_table1_16x8b, band_pos_v_16x8b);
temp2_8x16b = _mm_add_epi16(temp2_8x16b, band_pos_v_16x8b);
band_table3_16x8b = _mm_add_epi16(band_table3_16x8b, band_pos_v_16x8b);
//sao_offset duplication
tmp_set_128i_1 = _mm_shuffle_epi8(sao_offset, tmp_set_128i_1);
tmp_set_128i_2 = _mm_shuffle_epi8(sao_offset, tmp_set_128i_2);
tmp_set_128i_3 = _mm_shuffle_epi8(sao_offset, tmp_set_128i_3);
tmp_set_128i_4 = _mm_shuffle_epi8(sao_offset, tmp_set_128i_4);
//sao_offset addition
temp0_8x16b = _mm_add_epi16(temp0_8x16b, tmp_set_128i_1);
band_table1_16x8b = _mm_add_epi16(band_table1_16x8b, tmp_set_128i_2);
temp2_8x16b = _mm_add_epi16(temp2_8x16b, tmp_set_128i_3);
band_table3_16x8b = _mm_add_epi16(band_table3_16x8b, tmp_set_128i_4);
//masking upper 8bit values of 16 bit band table value
temp0_8x16b = _mm_and_si128(temp0_8x16b, temp1_8x16b);
band_table1_16x8b = _mm_and_si128(band_table1_16x8b, temp1_8x16b);
temp2_8x16b = _mm_and_si128(temp2_8x16b, temp1_8x16b);
band_table3_16x8b = _mm_and_si128(band_table3_16x8b, temp1_8x16b);
//temp1_8x16b reuse for compare storage
switch(sao_band_pos_v)
{
case 0:
temp3_8x16b = _mm_cmpgt_epi16(cmp_mask, temp0_8x16b);
temp0_8x16b = _mm_and_si128(temp0_8x16b, temp3_8x16b);
break;
case 28:
temp3_8x16b = _mm_cmpgt_epi16(cmp_mask, band_table3_16x8b);
band_table3_16x8b = _mm_or_si128(band_table3_16x8b, temp3_8x16b);
break;
case 29:
temp3_8x16b = _mm_cmpgt_epi16(cmp_mask, temp2_8x16b);
temp2_8x16b = _mm_or_si128(temp2_8x16b, temp3_8x16b);
temp3_8x16b = _mm_cmpgt_epi16(cmp_mask, band_table3_16x8b);
band_table3_16x8b = _mm_and_si128(band_table3_16x8b, temp3_8x16b);
break;
case 30:
temp3_8x16b = _mm_cmpgt_epi16(cmp_mask, band_table1_16x8b);
band_table1_16x8b = _mm_or_si128(band_table1_16x8b, temp3_8x16b);
temp3_8x16b = _mm_cmpgt_epi16(cmp_mask, temp2_8x16b);
temp2_8x16b = _mm_and_si128(temp2_8x16b, temp3_8x16b);
break;
case 31:
temp3_8x16b = _mm_cmpgt_epi16(cmp_mask, temp0_8x16b);
temp0_8x16b = _mm_or_si128(temp0_8x16b, temp3_8x16b);
temp3_8x16b = _mm_cmpgt_epi16(cmp_mask, band_table1_16x8b);
band_table1_16x8b = _mm_and_si128(band_table1_16x8b, temp3_8x16b);
break;
default:
break;
}
//masking upper 8bit values of each 16 bit band table value
temp0_8x16b = _mm_and_si128(temp0_8x16b, temp1_8x16b);
band_table1_16x8b = _mm_and_si128(band_table1_16x8b, temp1_8x16b);
temp2_8x16b = _mm_and_si128(temp2_8x16b, temp1_8x16b);
band_table3_16x8b = _mm_and_si128(band_table3_16x8b, temp1_8x16b);
//band table 8x16 four registers are packed into 16x8 two registers: band_table0_8x16b and band_table2_8x16b
band_table1_16x8b = _mm_packus_epi16(temp0_8x16b, band_table1_16x8b);
band_table3_16x8b = _mm_packus_epi16(temp2_8x16b, band_table3_16x8b);
//band table for u and v created
}
{
UWORD8 *pu1_src_cpy;
WORD32 wd_rem;
//sao_offset is reused for zero cmp mask.
sao_offset = _mm_setzero_si128();
tmp_set_128i_1 = _mm_set1_epi8(1);
//tmp_set_128i_2 = _mm_set_epi8 (128,7,128,6,128,5,128,4,128,3,128,2,128,1,128,0);
cmp_mask = _mm_packus_epi16(cmp_mask, cmp_mask); //cmp_msk=dup16(16);
//to avoid ffff to be saturated to 0 instead it should be to ff
cmp_msk2 = _mm_slli_epi16(cmp_mask, 1); // to compare if value is greater than 31
band_pos_u_16x8b = _mm_packus_epi16(band_pos_u_16x8b, band_pos_u_16x8b); //band_pos_u is now 8 bit aligned
band_pos_v_16x8b = _mm_packus_epi16(band_pos_v_16x8b, band_pos_v_16x8b); //band_pos_v is now 8 bit aligned
cmp_msk2 = _mm_sub_epi8(cmp_msk2, tmp_set_128i_1); // to compare if value is greater than 31
cmp_mask = _mm_sub_epi8(cmp_mask, tmp_set_128i_1);
for(col = wd; col >= 16; col -= 16)
{
pu1_src_cpy = pu1_src;
for(row = ht; row > 0; row -= 2)
{
//row = 0 load 8 pixel values from 7:0 pos. relative to cur. pos.
src_temp0_8x16b = _mm_loadu_si128((__m128i *)(pu1_src_cpy));
// row = 1
src_temp3_8x16b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + src_strd));
//odd values
src_temp1_8x16b = _mm_srli_epi16(src_temp0_8x16b, 8);
src_temp2_8x16b = _mm_srli_epi16(src_temp3_8x16b, 8);
//even values
src_temp0_8x16b = _mm_slli_epi16(src_temp0_8x16b, 8);
src_temp3_8x16b = _mm_slli_epi16(src_temp3_8x16b, 8);
src_temp0_8x16b = _mm_srli_epi16(src_temp0_8x16b, 8);
src_temp3_8x16b = _mm_srli_epi16(src_temp3_8x16b, 8);
//combining odd values
src_temp2_8x16b = _mm_packus_epi16(src_temp1_8x16b, src_temp2_8x16b);
//combining even values
src_temp0_8x16b = _mm_packus_epi16(src_temp0_8x16b, src_temp3_8x16b);
//saturated substract 8 bit
tmp_set_128i_1 = _mm_sub_epi8(src_temp0_8x16b, band_pos_u_16x8b);
tmp_set_128i_3 = _mm_sub_epi8(src_temp2_8x16b, band_pos_v_16x8b);
//if the values less than 0 put ff
tmp_set_128i_2 = _mm_cmpgt_epi8(sao_offset, tmp_set_128i_1);
tmp_set_128i_4 = _mm_cmpgt_epi8(sao_offset, tmp_set_128i_3);
tmp_set_128i_1 = _mm_or_si128(tmp_set_128i_1, tmp_set_128i_2);
tmp_set_128i_3 = _mm_or_si128(tmp_set_128i_3, tmp_set_128i_4);
//if the values greater than 31 put ff
tmp_set_128i_2 = _mm_cmpgt_epi8(tmp_set_128i_1, cmp_msk2);
tmp_set_128i_4 = _mm_cmpgt_epi8(tmp_set_128i_3, cmp_msk2);
tmp_set_128i_1 = _mm_or_si128(tmp_set_128i_1, tmp_set_128i_2);
tmp_set_128i_3 = _mm_or_si128(tmp_set_128i_3, tmp_set_128i_4);
// registers reused to increase performance
//if the values >16 then put ff ,cmp_mask = dup16(15) row 0 and row1
src_temp1_8x16b = _mm_cmpgt_epi8(tmp_set_128i_1, cmp_mask);
//if the values >16 then put ff ,cmp_mask = dup16(15) row 2 and row 3
src_temp3_8x16b = _mm_cmpgt_epi8(tmp_set_128i_3, cmp_mask);
//values 16 to 31 for row 0 & 1 but values <16 ==0
tmp_set_128i_2 = _mm_and_si128(tmp_set_128i_1, src_temp1_8x16b);
// values 0 to 15 for row 0 & 1
tmp_set_128i_1 = _mm_or_si128(tmp_set_128i_1, src_temp1_8x16b);
//values 16 to 31 for row 2 & 3 but values <16 ==0
tmp_set_128i_4 = _mm_and_si128(tmp_set_128i_3, src_temp3_8x16b);
// values 0 to 15 for row 2 & 3
tmp_set_128i_3 = _mm_or_si128(tmp_set_128i_3, src_temp3_8x16b);
//values 16 to 31 for row 0 & 1 but values <16 masked to ff row 0 and row1
src_temp1_8x16b = _mm_cmpeq_epi8(tmp_set_128i_2, sao_offset);
//values 16 to 31 for row 0 & 1 but values <16 masked to ff row 2 and row 3
src_temp3_8x16b = _mm_cmpeq_epi8(tmp_set_128i_4, sao_offset);
tmp_set_128i_2 = _mm_or_si128(tmp_set_128i_2, src_temp1_8x16b);
tmp_set_128i_4 = _mm_or_si128(tmp_set_128i_4, src_temp3_8x16b);
//to choose which pixel values to preserve in row 0 and row 1
src_temp1_8x16b = _mm_cmpeq_epi8(tmp_set_128i_1, tmp_set_128i_2);
//to choose which pixel values to preserve in row 2 and row 3
src_temp3_8x16b = _mm_cmpeq_epi8(tmp_set_128i_3, tmp_set_128i_4);
//values of all rows to which no offset needs to be added preserved.
src_temp0_8x16b = _mm_and_si128(src_temp0_8x16b, src_temp1_8x16b);
src_temp2_8x16b = _mm_and_si128(src_temp2_8x16b, src_temp3_8x16b);
//indexing 0 - 15 bandtable indexes
tmp_set_128i_1 = _mm_shuffle_epi8(band_table0_16x8b, tmp_set_128i_1); //U low
tmp_set_128i_3 = _mm_shuffle_epi8(band_table1_16x8b, tmp_set_128i_3); //V low
//indexing 16 -31 bandtable indexes
tmp_set_128i_2 = _mm_shuffle_epi8(band_table2_16x8b, tmp_set_128i_2); //U high
tmp_set_128i_4 = _mm_shuffle_epi8(band_table3_16x8b, tmp_set_128i_4); //V high
// combining all offsets results
tmp_set_128i_1 = _mm_or_si128(tmp_set_128i_1, tmp_set_128i_2); //U
tmp_set_128i_3 = _mm_or_si128(tmp_set_128i_3, tmp_set_128i_4); //V
// combing results with the pixel values
src_temp0_8x16b = _mm_or_si128(src_temp0_8x16b, tmp_set_128i_1);
src_temp2_8x16b = _mm_or_si128(src_temp2_8x16b, tmp_set_128i_3);
//reorganising even and odd values
src_temp1_8x16b = _mm_unpacklo_epi8(src_temp0_8x16b, src_temp2_8x16b);
src_temp3_8x16b = _mm_unpackhi_epi8(src_temp0_8x16b, src_temp2_8x16b);
//row = 0 store 8 pixel values from 7:0 pos. relative to cur. pos.
_mm_storeu_si128((__m128i *)(pu1_src_cpy), src_temp1_8x16b);
// row = 1
_mm_storeu_si128((__m128i *)(pu1_src_cpy + src_strd), src_temp3_8x16b);
pu1_src_cpy += (src_strd << 1);
}
pu1_src += 16;
}
wd_rem = wd & 0xF;
if(wd_rem)
{
pu1_src_cpy = pu1_src;
for(row = ht; row > 0; row -= 4)
{
//row = 0 load 8 pixel values from 7:0 pos. relative to cur. pos.
src_temp0_8x16b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy));
// row = 1
src_temp1_8x16b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy + src_strd));
// row = 2
src_temp2_8x16b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy + 2 * src_strd));
// row = 3
src_temp3_8x16b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy + 3 * src_strd));
//row0 and row1 packed and row2 and row3 packed
src_temp0_8x16b = _mm_unpacklo_epi64(src_temp0_8x16b, src_temp1_8x16b);
src_temp3_8x16b = _mm_unpacklo_epi64(src_temp2_8x16b, src_temp3_8x16b);
//odd values
src_temp1_8x16b = _mm_srli_epi16(src_temp0_8x16b, 8);
src_temp2_8x16b = _mm_srli_epi16(src_temp3_8x16b, 8);
//even values
src_temp0_8x16b = _mm_slli_epi16(src_temp0_8x16b, 8);
src_temp3_8x16b = _mm_slli_epi16(src_temp3_8x16b, 8);
src_temp0_8x16b = _mm_srli_epi16(src_temp0_8x16b, 8);
src_temp3_8x16b = _mm_srli_epi16(src_temp3_8x16b, 8);
//combining odd values
src_temp2_8x16b = _mm_packus_epi16(src_temp1_8x16b, src_temp2_8x16b);
//combining even values
src_temp0_8x16b = _mm_packus_epi16(src_temp0_8x16b, src_temp3_8x16b);
//saturated substract 8 bit
tmp_set_128i_1 = _mm_sub_epi8(src_temp0_8x16b, band_pos_u_16x8b);
tmp_set_128i_3 = _mm_sub_epi8(src_temp2_8x16b, band_pos_v_16x8b);
//if the values less than 0 put ff
tmp_set_128i_2 = _mm_cmpgt_epi8(sao_offset, tmp_set_128i_1);
tmp_set_128i_4 = _mm_cmpgt_epi8(sao_offset, tmp_set_128i_3);
tmp_set_128i_1 = _mm_or_si128(tmp_set_128i_1, tmp_set_128i_2);
tmp_set_128i_3 = _mm_or_si128(tmp_set_128i_3, tmp_set_128i_4);
//if the values greater than 31 put ff
tmp_set_128i_2 = _mm_cmpgt_epi8(tmp_set_128i_1, cmp_msk2);
tmp_set_128i_4 = _mm_cmpgt_epi8(tmp_set_128i_3, cmp_msk2);
tmp_set_128i_1 = _mm_or_si128(tmp_set_128i_1, tmp_set_128i_2);
tmp_set_128i_3 = _mm_or_si128(tmp_set_128i_3, tmp_set_128i_4);
// registers reused to increase performance
//if the values >16 then put ff ,cmp_mask = dup16(15) row 0 and row1
src_temp1_8x16b = _mm_cmpgt_epi8(tmp_set_128i_1, cmp_mask);
//if the values >16 then put ff ,cmp_mask = dup16(15) row 2 and row 3
src_temp3_8x16b = _mm_cmpgt_epi8(tmp_set_128i_3, cmp_mask);
//values 16 to 31 for row 0 & 1 but values <16 ==0
tmp_set_128i_2 = _mm_and_si128(tmp_set_128i_1, src_temp1_8x16b);
// values 0 to 15 for row 0 & 1
tmp_set_128i_1 = _mm_or_si128(tmp_set_128i_1, src_temp1_8x16b);
//values 16 to 31 for row 2 & 3 but values <16 ==0
tmp_set_128i_4 = _mm_and_si128(tmp_set_128i_3, src_temp3_8x16b);
// values 0 to 15 for row 2 & 3
tmp_set_128i_3 = _mm_or_si128(tmp_set_128i_3, src_temp3_8x16b);
//values 16 to 31 for row 0 & 1 but values <16 masked to ff row 0 and row1
src_temp1_8x16b = _mm_cmpeq_epi8(tmp_set_128i_2, sao_offset);
//values 16 to 31 for row 0 & 1 but values <16 masked to ff row 2 and row 3
src_temp3_8x16b = _mm_cmpeq_epi8(tmp_set_128i_4, sao_offset);
tmp_set_128i_2 = _mm_or_si128(tmp_set_128i_2, src_temp1_8x16b);
tmp_set_128i_4 = _mm_or_si128(tmp_set_128i_4, src_temp3_8x16b);
//to choose which pixel values to preserve in row 0 and row 1
src_temp1_8x16b = _mm_cmpeq_epi8(tmp_set_128i_1, tmp_set_128i_2);
//to choose which pixel values to preserve in row 2 and row 3
src_temp3_8x16b = _mm_cmpeq_epi8(tmp_set_128i_3, tmp_set_128i_4);
//values of all rows to which no offset needs to be added preserved.
src_temp0_8x16b = _mm_and_si128(src_temp0_8x16b, src_temp1_8x16b);
src_temp2_8x16b = _mm_and_si128(src_temp2_8x16b, src_temp3_8x16b);
//indexing 0 - 15 bandtable indexes
tmp_set_128i_1 = _mm_shuffle_epi8(band_table0_16x8b, tmp_set_128i_1); //U low
tmp_set_128i_3 = _mm_shuffle_epi8(band_table1_16x8b, tmp_set_128i_3); //V low
//indexing 16 -31 bandtable indexes
tmp_set_128i_2 = _mm_shuffle_epi8(band_table2_16x8b, tmp_set_128i_2); //U high
tmp_set_128i_4 = _mm_shuffle_epi8(band_table3_16x8b, tmp_set_128i_4); //V high
// combining all offsets results
tmp_set_128i_1 = _mm_or_si128(tmp_set_128i_1, tmp_set_128i_2); //U
tmp_set_128i_3 = _mm_or_si128(tmp_set_128i_3, tmp_set_128i_4); //V
// combing results with the pixel values
src_temp0_8x16b = _mm_or_si128(src_temp0_8x16b, tmp_set_128i_1);
src_temp2_8x16b = _mm_or_si128(src_temp2_8x16b, tmp_set_128i_3);
//reorganising even and odd values
src_temp1_8x16b = _mm_unpacklo_epi8(src_temp0_8x16b, src_temp2_8x16b);
src_temp3_8x16b = _mm_unpackhi_epi8(src_temp0_8x16b, src_temp2_8x16b);
//Getting row1 separately
src_temp0_8x16b = _mm_srli_si128(src_temp1_8x16b, 8);
//Getting row3 separately
src_temp2_8x16b = _mm_srli_si128(src_temp3_8x16b, 8);
//row = 0 store 8 pixel values from 7:0 pos. relative to cur. pos.
_mm_storel_epi64((__m128i *)(pu1_src_cpy), src_temp1_8x16b);
// row = 1
_mm_storel_epi64((__m128i *)(pu1_src_cpy + src_strd), src_temp0_8x16b);
// row = 2
_mm_storel_epi64((__m128i *)(pu1_src_cpy + 2 * src_strd), src_temp3_8x16b);
// row = 3
_mm_storel_epi64((__m128i *)(pu1_src_cpy + 3 * src_strd), src_temp2_8x16b);
pu1_src_cpy += (src_strd << 2);
}
pu1_src += 16;
}
}
}
void ihevc_sao_edge_offset_class0_ssse3(UWORD8 *pu1_src,
WORD32 src_strd,
UWORD8 *pu1_src_left,
UWORD8 *pu1_src_top,
UWORD8 *pu1_src_top_left,
UWORD8 *pu1_src_top_right,
UWORD8 *pu1_src_bot_left,
UWORD8 *pu1_avail,
WORD8 *pi1_sao_offset,
WORD32 wd,
WORD32 ht)
{
WORD32 row, col;
UWORD8 *pu1_src_cpy, *pu1_src_left_cpy, *pu1_src_left_str, *pu1_left_tmp;
UWORD8 au1_mask[MAX_CTB_SIZE], *au1_mask_cpy;
UWORD8 au1_src_left_tmp[MAX_CTB_SIZE + 8];
UWORD8 au1_src_left_tmp1[MAX_CTB_SIZE + 8];
UWORD8 u1_avail0, u1_avail1;
WORD32 wd_rem;
WORD32 offset = 0;
__m128i src_temp0_16x8b, src_temp1_16x8b;
__m128i left0_16x8b, left1_16x8b;
__m128i cmp_gt0_16x8b, cmp_lt0_16x8b, cmp_gt1_16x8b, cmp_lt1_16x8b;
__m128i edge0_16x8b, edge1_16x8b;
__m128i au1_mask8x16b;
__m128i edge_idx_8x16b, sao_offset_8x16b;
__m128i const2_16x8b, const0_16x8b;
__m128i left_store_16x8b;
UNUSED(pu1_src_top_right);
UNUSED(pu1_src_bot_left);
au1_mask8x16b = _mm_set1_epi8(0xff);
/* Update top and top-left arrays */
*pu1_src_top_left = pu1_src_top[wd - 1];
for(col = wd; col >= 16; col -= 16)
{
const0_16x8b = _mm_loadu_si128((__m128i *)(pu1_src + offset + (ht - 1) * src_strd));
_mm_storeu_si128((__m128i *)(pu1_src_top + offset), const0_16x8b);
offset += 16;
}
//setting availability mask to ff size MAX_CTB_SIZE
for(col = 0; col < MAX_CTB_SIZE; col += 16)
_mm_storeu_si128((__m128i *)(au1_mask + col), au1_mask8x16b);
for(row = 0; row < ht; row++)
{
au1_src_left_tmp[row] = pu1_src_left[row];
}
edge_idx_8x16b = _mm_loadl_epi64((__m128i *)gi1_table_edge_idx);
sao_offset_8x16b = _mm_loadl_epi64((__m128i *)pi1_sao_offset);
//availability mask creation
u1_avail0 = pu1_avail[0];
u1_avail1 = pu1_avail[1];
au1_mask[0] = u1_avail0;
au1_mask[wd - 1] = u1_avail1;
const2_16x8b = _mm_set1_epi8(2);
const0_16x8b = _mm_setzero_si128();
pu1_src_left_cpy = au1_src_left_tmp;
pu1_src_left_str = au1_src_left_tmp1;
{
au1_mask_cpy = au1_mask;
for(col = wd; col >= 16; col -= 16)
{
pu1_src_cpy = pu1_src;
au1_mask8x16b = _mm_loadu_si128((__m128i *)au1_mask_cpy);
//pu1_src_left_cpy =au1_src_left_tmp;
for(row = ht; row > 0; row -= 2)
{
left_store_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_left_cpy));
//row = 0 load 8 pixel values from 7:0 pos. relative to cur. pos.
src_temp0_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy));
// row = 1
src_temp1_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + src_strd));
left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, left_store_16x8b, 2);
//row 1 left
left1_16x8b = _mm_alignr_epi8(src_temp1_16x8b, left_store_16x8b, 15);
left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, src_temp1_16x8b, 15);
//row 0 left
left0_16x8b = _mm_alignr_epi8(src_temp0_16x8b, left_store_16x8b, 15);
left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, src_temp0_16x8b, 15);
//separating +ve and and -ve values.
cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, left0_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(left0_16x8b, src_temp0_16x8b);
cmp_gt1_16x8b = _mm_subs_epu8(src_temp1_16x8b, left1_16x8b);
cmp_lt1_16x8b = _mm_subs_epu8(left1_16x8b, src_temp1_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
cmp_gt1_16x8b = _mm_cmpeq_epi8(cmp_gt1_16x8b, const0_16x8b);
cmp_lt1_16x8b = _mm_cmpeq_epi8(cmp_lt1_16x8b, const0_16x8b);
//combining the appropriate sign change
left0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b);
left1_16x8b = _mm_sub_epi8(cmp_gt1_16x8b, cmp_lt1_16x8b);
//row = 0 right
edge0_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + 1));
// row = 1 right
edge1_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + src_strd + 1));
//separating +ve and and -ve values.
cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, edge0_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(edge0_16x8b, src_temp0_16x8b);
cmp_gt1_16x8b = _mm_subs_epu8(src_temp1_16x8b, edge1_16x8b);
cmp_lt1_16x8b = _mm_subs_epu8(edge1_16x8b, src_temp1_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
cmp_gt1_16x8b = _mm_cmpeq_epi8(cmp_gt1_16x8b, const0_16x8b);
cmp_lt1_16x8b = _mm_cmpeq_epi8(cmp_lt1_16x8b, const0_16x8b);
//combining the appropriate sign change
edge0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b);
edge1_16x8b = _mm_sub_epi8(cmp_gt1_16x8b, cmp_lt1_16x8b);
//combining sign-left and sign_right
edge0_16x8b = _mm_add_epi8(edge0_16x8b, left0_16x8b);
edge1_16x8b = _mm_add_epi8(edge1_16x8b, left1_16x8b);
//adding constant 2
edge0_16x8b = _mm_add_epi8(edge0_16x8b, const2_16x8b);
edge1_16x8b = _mm_add_epi8(edge1_16x8b, const2_16x8b);
//shuffle to get sao index
edge0_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge0_16x8b);
edge1_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge1_16x8b);
//using availability mask
edge0_16x8b = _mm_and_si128(edge0_16x8b, au1_mask8x16b);
edge1_16x8b = _mm_and_si128(edge1_16x8b, au1_mask8x16b);
//shuffle to get sao offset
edge0_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge0_16x8b);
edge1_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge1_16x8b);
//cnvert to 16 bit then add and then saturated pack
left0_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge0_16x8b);
cmp_gt0_16x8b = _mm_unpacklo_epi8(src_temp0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_unpacklo_epi8(edge0_16x8b, left0_16x8b);
src_temp0_16x8b = _mm_unpackhi_epi8(src_temp0_16x8b, const0_16x8b);
cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b);
cmp_lt0_16x8b = _mm_unpackhi_epi8(edge0_16x8b, left0_16x8b);
src_temp0_16x8b = _mm_add_epi16(src_temp0_16x8b, cmp_lt0_16x8b);
src_temp0_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_temp0_16x8b);
left0_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge1_16x8b);
cmp_gt0_16x8b = _mm_unpacklo_epi8(src_temp1_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_unpacklo_epi8(edge1_16x8b, left0_16x8b);
src_temp1_16x8b = _mm_unpackhi_epi8(src_temp1_16x8b, const0_16x8b);
cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b);
cmp_lt0_16x8b = _mm_unpackhi_epi8(edge1_16x8b, left0_16x8b);
src_temp1_16x8b = _mm_add_epi16(src_temp1_16x8b, cmp_lt0_16x8b);
src_temp1_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_temp1_16x8b);
_mm_storel_epi64((__m128i *)(pu1_src_left_str), left_store_16x8b);
//row = 0 store 8 pixel values from 7:0 pos. relative to cur. pos.
_mm_storeu_si128((__m128i *)(pu1_src_cpy), src_temp0_16x8b);
// row = 1
_mm_storeu_si128((__m128i *)(pu1_src_cpy + src_strd), src_temp1_16x8b);
pu1_src_cpy += (src_strd << 1);
pu1_src_left_cpy += 2;
pu1_src_left_str += 2;
}
au1_mask_cpy += 16;
pu1_src += 16;
pu1_src_left_cpy -= ht;
pu1_src_left_str -= ht;
pu1_left_tmp = pu1_src_left_cpy;
pu1_src_left_cpy = pu1_src_left_str;
pu1_src_left_str = pu1_left_tmp;
}
wd_rem = wd & 0xF;
if(wd_rem)
{
cmp_gt1_16x8b = _mm_loadu_si128((__m128i *)(pu1_src + (ht - 1) * src_strd));
_mm_storel_epi64((__m128i *)(pu1_src_top + offset), cmp_gt1_16x8b);
au1_mask8x16b = _mm_loadl_epi64((__m128i *)au1_mask_cpy);
pu1_src_cpy = pu1_src;
au1_mask8x16b = _mm_unpacklo_epi64(au1_mask8x16b, au1_mask8x16b);
//pu1_src_left_cpy =au1_src_left_tmp;
for(row = ht; row > 0; row -= 4)
{
left_store_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_left_cpy));
//row = 0 load 8 pixel values from 7:0 pos. relative to cur. pos.
src_temp0_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy));
// row = 1
cmp_gt0_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy + src_strd));
// row = 2
src_temp1_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy + 2 * src_strd));
// row = 3
cmp_gt1_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy + 3 * src_strd));
left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, left_store_16x8b, 4);
//row 3 left
edge0_16x8b = _mm_slli_si128(cmp_gt1_16x8b, 8);
cmp_lt1_16x8b = _mm_alignr_epi8(cmp_gt1_16x8b, left_store_16x8b, 15);
left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, edge0_16x8b, 15);
//row 2 left
edge0_16x8b = _mm_slli_si128(src_temp1_16x8b, 8);
left1_16x8b = _mm_alignr_epi8(src_temp1_16x8b, left_store_16x8b, 15);
left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, edge0_16x8b, 15);
//row 1 left
edge0_16x8b = _mm_slli_si128(cmp_gt0_16x8b, 8);
cmp_lt0_16x8b = _mm_alignr_epi8(cmp_gt0_16x8b, left_store_16x8b, 15);
left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, edge0_16x8b, 15);
//row 0 left
edge0_16x8b = _mm_slli_si128(src_temp0_16x8b, 8);
left0_16x8b = _mm_alignr_epi8(src_temp0_16x8b, left_store_16x8b, 15);
left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, edge0_16x8b, 15);
// packing rows together for 16 SIMD operations
src_temp0_16x8b = _mm_unpacklo_epi64(src_temp0_16x8b, cmp_gt0_16x8b);
src_temp1_16x8b = _mm_unpacklo_epi64(src_temp1_16x8b, cmp_gt1_16x8b);
// packing rows together for 16 SIMD operations
left0_16x8b = _mm_unpacklo_epi64(left0_16x8b, cmp_lt0_16x8b);
left1_16x8b = _mm_unpacklo_epi64(left1_16x8b, cmp_lt1_16x8b);
//separating +ve and and -ve values.
cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, left0_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(left0_16x8b, src_temp0_16x8b);
cmp_gt1_16x8b = _mm_subs_epu8(src_temp1_16x8b, left1_16x8b);
cmp_lt1_16x8b = _mm_subs_epu8(left1_16x8b, src_temp1_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
cmp_gt1_16x8b = _mm_cmpeq_epi8(cmp_gt1_16x8b, const0_16x8b);
cmp_lt1_16x8b = _mm_cmpeq_epi8(cmp_lt1_16x8b, const0_16x8b);
//combining the appropriate sign change
left0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b);
left1_16x8b = _mm_sub_epi8(cmp_gt1_16x8b, cmp_lt1_16x8b);
//row = 0 right
edge0_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy + 1));
// row = 1 right
cmp_gt0_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy + src_strd + 1));
// row = 2 right
edge1_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy + 2 * src_strd + 1));
// row = 3 right
cmp_gt1_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy + 3 * src_strd + 1));
// packing rows together for 16 SIMD operations
edge0_16x8b = _mm_unpacklo_epi64(edge0_16x8b, cmp_gt0_16x8b);
edge1_16x8b = _mm_unpacklo_epi64(edge1_16x8b, cmp_gt1_16x8b);
//separating +ve and and -ve values.
cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, edge0_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(edge0_16x8b, src_temp0_16x8b);
cmp_gt1_16x8b = _mm_subs_epu8(src_temp1_16x8b, edge1_16x8b);
cmp_lt1_16x8b = _mm_subs_epu8(edge1_16x8b, src_temp1_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
cmp_gt1_16x8b = _mm_cmpeq_epi8(cmp_gt1_16x8b, const0_16x8b);
cmp_lt1_16x8b = _mm_cmpeq_epi8(cmp_lt1_16x8b, const0_16x8b);
//combining the appropriate sign change
edge0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b);
edge1_16x8b = _mm_sub_epi8(cmp_gt1_16x8b, cmp_lt1_16x8b);
//combining sign-left and sign_right
edge0_16x8b = _mm_add_epi8(edge0_16x8b, left0_16x8b);
edge1_16x8b = _mm_add_epi8(edge1_16x8b, left1_16x8b);
//adding constant 2
edge0_16x8b = _mm_add_epi8(edge0_16x8b, const2_16x8b);
edge1_16x8b = _mm_add_epi8(edge1_16x8b, const2_16x8b);
//shuffle to get sao index
edge0_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge0_16x8b);
edge1_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge1_16x8b);
//shuffle to get sao offset
//using availability mask
edge0_16x8b = _mm_and_si128(edge0_16x8b, au1_mask8x16b);
edge1_16x8b = _mm_and_si128(edge1_16x8b, au1_mask8x16b);
edge0_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge0_16x8b);
edge1_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge1_16x8b);
//cnvert to 16 bit then add and then saturated pack
left0_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge0_16x8b);
cmp_gt0_16x8b = _mm_unpacklo_epi8(src_temp0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_unpacklo_epi8(edge0_16x8b, left0_16x8b);
src_temp0_16x8b = _mm_unpackhi_epi8(src_temp0_16x8b, const0_16x8b);
cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b);
cmp_lt0_16x8b = _mm_unpackhi_epi8(edge0_16x8b, left0_16x8b);
src_temp0_16x8b = _mm_add_epi16(src_temp0_16x8b, cmp_lt0_16x8b);
src_temp0_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_temp0_16x8b);
left0_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge1_16x8b);
cmp_gt0_16x8b = _mm_unpacklo_epi8(src_temp1_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_unpacklo_epi8(edge1_16x8b, left0_16x8b);
src_temp1_16x8b = _mm_unpackhi_epi8(src_temp1_16x8b, const0_16x8b);
cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b);
cmp_lt0_16x8b = _mm_unpackhi_epi8(edge1_16x8b, left0_16x8b);
src_temp1_16x8b = _mm_add_epi16(src_temp1_16x8b, cmp_lt0_16x8b);
src_temp1_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_temp1_16x8b);
//separting row 1 and row 3
cmp_lt0_16x8b = _mm_srli_si128(src_temp0_16x8b, 8);
cmp_lt1_16x8b = _mm_srli_si128(src_temp1_16x8b, 8);
_mm_storel_epi64((__m128i *)(pu1_src_left_str), left_store_16x8b);
//row = 0 store 8 pixel values from 7:0 pos. relative to cur. pos.
_mm_storel_epi64((__m128i *)(pu1_src_cpy), src_temp0_16x8b);
// row = 1
_mm_storel_epi64((__m128i *)(pu1_src_cpy + src_strd), cmp_lt0_16x8b);
// row = 2
_mm_storel_epi64((__m128i *)(pu1_src_cpy + 2 * src_strd), src_temp1_16x8b);
// row = 3
_mm_storel_epi64((__m128i *)(pu1_src_cpy + 3 * src_strd), cmp_lt1_16x8b);
pu1_src_cpy += (src_strd << 2);
pu1_src_left_cpy += 4;
pu1_src_left_str += 4;
}
pu1_src += wd;
pu1_src_left_cpy -= ht;
pu1_src_left_str -= ht;
pu1_left_tmp = pu1_src_left_cpy;
pu1_src_left_cpy = pu1_src_left_str;
pu1_src_left_str = pu1_left_tmp;
}
for(row = 0; row < ht; row++)
{
pu1_src_left[row] = pu1_src_left_cpy[row];
}
}
}
void ihevc_sao_edge_offset_class0_chroma_ssse3(UWORD8 *pu1_src,
WORD32 src_strd,
UWORD8 *pu1_src_left,
UWORD8 *pu1_src_top,
UWORD8 *pu1_src_top_left,
UWORD8 *pu1_src_top_right,
UWORD8 *pu1_src_bot_left,
UWORD8 *pu1_avail,
WORD8 *pi1_sao_offset_u,
WORD8 *pi1_sao_offset_v,
WORD32 wd,
WORD32 ht)
{
WORD32 row, col;
UWORD8 *pu1_src_cpy, *pu1_src_left_cpy, *pu1_src_left_str, *pu1_left_tmp;
UWORD8 au1_mask[MAX_CTB_SIZE], *au1_mask_cpy;
UWORD8 au1_src_left_tmp[2 * (MAX_CTB_SIZE + 8)];
UWORD8 au1_src_left_tmp1[2 * (MAX_CTB_SIZE + 8)];
UWORD8 u1_avail0, u1_avail1;
WORD32 wd_rem;
WORD32 offset = 0;
__m128i src_temp0_16x8b, src_temp1_16x8b;
__m128i left0_16x8b, left1_16x8b;
__m128i cmp_gt0_16x8b, cmp_lt0_16x8b;
__m128i edge0_16x8b, edge1_16x8b;
__m128i au1_mask8x16b;
__m128i edge_idx_8x16b, sao_offset_8x16b;
__m128i const2_16x8b, const0_16x8b;
__m128i left_store_16x8b;
__m128i chroma_offset_8x16b;
UNUSED(pu1_src_top_right);
UNUSED(pu1_src_bot_left);
au1_mask8x16b = _mm_set1_epi8(0xff);
/* Update top and top-left arrays */
pu1_src_top_left[0] = pu1_src_top[wd - 2];
pu1_src_top_left[1] = pu1_src_top[wd - 1];;
for(col = wd; col >= 16; col -= 16)
{
const0_16x8b = _mm_loadu_si128((__m128i *)(pu1_src + offset + (ht - 1) * src_strd));
_mm_storeu_si128((__m128i *)(pu1_src_top + offset), const0_16x8b);
offset += 16;
}
for(row = 0; row < 2 * ht; row++)
{
au1_src_left_tmp[row] = pu1_src_left[row];
}
//setting availability mask to ff size MAX_CTB_SIZE
for(col = 0; col < MAX_CTB_SIZE; col += 16)
_mm_storeu_si128((__m128i *)(au1_mask + col), au1_mask8x16b);
edge_idx_8x16b = _mm_loadl_epi64((__m128i *)gi1_table_edge_idx);
sao_offset_8x16b = _mm_loadl_epi64((__m128i *)pi1_sao_offset_u);
const0_16x8b = _mm_loadl_epi64((__m128i *)pi1_sao_offset_v);
chroma_offset_8x16b = _mm_set1_epi16(0x0800);
//availability mask creation
u1_avail0 = pu1_avail[0];
u1_avail1 = pu1_avail[1];
au1_mask[0] = u1_avail0;
au1_mask[1] = u1_avail0;
au1_mask[wd - 1] = u1_avail1;
au1_mask[wd - 2] = u1_avail1;
sao_offset_8x16b = _mm_unpacklo_epi64(sao_offset_8x16b, const0_16x8b);
const2_16x8b = _mm_set1_epi8(2);
const0_16x8b = _mm_setzero_si128();
{
pu1_src_left_cpy = au1_src_left_tmp;
pu1_src_left_str = au1_src_left_tmp1;
au1_mask_cpy = au1_mask;
for(col = wd; col >= 16; col -= 16)
{
pu1_src_cpy = pu1_src;
au1_mask8x16b = _mm_loadu_si128((__m128i *)au1_mask_cpy);
for(row = ht; row > 0; row -= 2)
{
left_store_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_left_cpy));
//row = 0 load 8 pixel values from 7:0 pos. relative to cur. pos.
src_temp0_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy));
// row = 1
src_temp1_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + src_strd));
left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, left_store_16x8b, 4);
//row 1 left
left1_16x8b = _mm_alignr_epi8(src_temp1_16x8b, left_store_16x8b, 14);
left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, src_temp1_16x8b, 14);
//row 0 left
left0_16x8b = _mm_alignr_epi8(src_temp0_16x8b, left_store_16x8b, 14);
left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, src_temp0_16x8b, 14);
//separating +ve and and -ve values.row 0 left
cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, left0_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(left0_16x8b, src_temp0_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//combining the appropriate sign change
left0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b);
//separating +ve and and -ve values.row 1 left
cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, left1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(left1_16x8b, src_temp1_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//combining the appropriate sign change
left1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b);
//row = 0 right
edge0_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + 2));
// row = 1 right
edge1_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + src_strd + 2));
//separating +ve and and -ve values.row 0 right
cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, edge0_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(edge0_16x8b, src_temp0_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//combining the appropriate sign change
edge0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b);
//separating +ve and and -ve values.row 1 right
cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, edge1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(edge1_16x8b, src_temp1_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//combining the appropriate sign change
edge1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b);
//combining sign-left and sign_right
edge0_16x8b = _mm_add_epi8(edge0_16x8b, left0_16x8b);
edge1_16x8b = _mm_add_epi8(edge1_16x8b, left1_16x8b);
//adding constant 2
edge0_16x8b = _mm_add_epi8(edge0_16x8b, const2_16x8b);
edge1_16x8b = _mm_add_epi8(edge1_16x8b, const2_16x8b);
//shuffle to get sao index
edge0_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge0_16x8b);
edge1_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge1_16x8b);
//using availability mask
edge0_16x8b = _mm_and_si128(edge0_16x8b, au1_mask8x16b);
edge1_16x8b = _mm_and_si128(edge1_16x8b, au1_mask8x16b);
//adding chroma offset to access U and V
edge0_16x8b = _mm_add_epi8(edge0_16x8b, chroma_offset_8x16b);
edge1_16x8b = _mm_add_epi8(edge1_16x8b, chroma_offset_8x16b);
//shuffle to get sao offset
edge0_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge0_16x8b);
edge1_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge1_16x8b);
//cnvert to 16 bit then add and then saturated pack
left0_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge0_16x8b);
cmp_gt0_16x8b = _mm_unpacklo_epi8(src_temp0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_unpacklo_epi8(edge0_16x8b, left0_16x8b);
src_temp0_16x8b = _mm_unpackhi_epi8(src_temp0_16x8b, const0_16x8b);
edge0_16x8b = _mm_unpackhi_epi8(edge0_16x8b, left0_16x8b);
cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b);
src_temp0_16x8b = _mm_add_epi16(src_temp0_16x8b, edge0_16x8b);
src_temp0_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_temp0_16x8b);
left0_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge1_16x8b);
cmp_gt0_16x8b = _mm_unpacklo_epi8(src_temp1_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_unpacklo_epi8(edge1_16x8b, left0_16x8b);
src_temp1_16x8b = _mm_unpackhi_epi8(src_temp1_16x8b, const0_16x8b);
edge1_16x8b = _mm_unpackhi_epi8(edge1_16x8b, left0_16x8b);
cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b);
src_temp1_16x8b = _mm_add_epi16(src_temp1_16x8b, edge1_16x8b);
src_temp1_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_temp1_16x8b);
_mm_storel_epi64((__m128i *)(pu1_src_left_str), left_store_16x8b);
//row = 0 store 8 pixel values from 7:0 pos. relative to cur. pos.
_mm_storeu_si128((__m128i *)(pu1_src_cpy), src_temp0_16x8b);
// row = 1
_mm_storeu_si128((__m128i *)(pu1_src_cpy + src_strd), src_temp1_16x8b);
pu1_src_cpy += (src_strd << 1);
pu1_src_left_cpy += 4;
pu1_src_left_str += 4;
}
au1_mask_cpy += 16;
pu1_src += 16;
pu1_src_left_cpy -= 2 * ht;
pu1_src_left_str -= 2 * ht;
pu1_left_tmp = pu1_src_left_cpy;
pu1_src_left_cpy = pu1_src_left_str;
pu1_src_left_str = pu1_left_tmp;
}
wd_rem = wd & 0xF;
if(wd_rem)
{
cmp_gt0_16x8b = _mm_loadu_si128((__m128i *)(pu1_src + (ht - 1) * src_strd));
_mm_storel_epi64((__m128i *)(pu1_src_top + offset), cmp_gt0_16x8b);
au1_mask8x16b = _mm_loadl_epi64((__m128i *)au1_mask_cpy);
pu1_src_cpy = pu1_src;
au1_mask8x16b = _mm_unpacklo_epi64(au1_mask8x16b, au1_mask8x16b);
for(row = ht; row > 0; row -= 4)
{
left_store_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_left_cpy));
//row = 0 load 8 pixel values from 7:0 pos. relative to cur. pos.
src_temp0_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy));
// row = 1
cmp_gt0_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy + src_strd));
// row = 2
src_temp1_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy + 2 * src_strd));
// row = 3
cmp_lt0_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy + 3 * src_strd));
left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, left_store_16x8b, 8);
//row 3 left
edge0_16x8b = _mm_slli_si128(cmp_lt0_16x8b, 8);
left0_16x8b = _mm_alignr_epi8(cmp_lt0_16x8b, left_store_16x8b, 14);
left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, edge0_16x8b, 14);
//row 2 left
edge0_16x8b = _mm_slli_si128(src_temp1_16x8b, 8);
left1_16x8b = _mm_alignr_epi8(src_temp1_16x8b, left_store_16x8b, 14);
left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, edge0_16x8b, 14);
// packing rows together for 16 SIMD operations
src_temp1_16x8b = _mm_unpacklo_epi64(src_temp1_16x8b, cmp_lt0_16x8b);
left1_16x8b = _mm_unpacklo_epi64(left1_16x8b, left0_16x8b);
//row 1 left
edge0_16x8b = _mm_slli_si128(cmp_gt0_16x8b, 8);
edge1_16x8b = _mm_alignr_epi8(cmp_gt0_16x8b, left_store_16x8b, 14);
left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, edge0_16x8b, 14);
//row 0 left
edge0_16x8b = _mm_slli_si128(src_temp0_16x8b, 8);
left0_16x8b = _mm_alignr_epi8(src_temp0_16x8b, left_store_16x8b, 14);
left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, edge0_16x8b, 14);
// packing rows together for 16 SIMD operations
src_temp0_16x8b = _mm_unpacklo_epi64(src_temp0_16x8b, cmp_gt0_16x8b);
left0_16x8b = _mm_unpacklo_epi64(left0_16x8b, edge1_16x8b);
//separating +ve and and -ve values.for row 2 and row 3
cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, left1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(left1_16x8b, src_temp1_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//combining the appropriate sign change
left1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b);
//separating +ve and and -ve values.
cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, left0_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(left0_16x8b, src_temp0_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//combining the appropriate sign change
left0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b);
//row = 0 right
edge0_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy + 2));
// row = 1 right
cmp_gt0_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy + src_strd + 2));
// row = 2 right
edge1_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy + 2 * src_strd + 2));
// row = 3 right
cmp_lt0_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy + 3 * src_strd + 2));
// packing rows together for 16 SIMD operations
edge0_16x8b = _mm_unpacklo_epi64(edge0_16x8b, cmp_gt0_16x8b);
edge1_16x8b = _mm_unpacklo_epi64(edge1_16x8b, cmp_lt0_16x8b);
//separating +ve and and -ve values.
cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, edge0_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(edge0_16x8b, src_temp0_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//combining the appropriate sign change
edge0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b);
cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, edge1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(edge1_16x8b, src_temp1_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//combining the appropriate sign change
edge1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b);
//combining sign-left and sign_right
edge0_16x8b = _mm_add_epi8(edge0_16x8b, left0_16x8b);
edge1_16x8b = _mm_add_epi8(edge1_16x8b, left1_16x8b);
//adding constant 2
edge0_16x8b = _mm_add_epi8(edge0_16x8b, const2_16x8b);
edge1_16x8b = _mm_add_epi8(edge1_16x8b, const2_16x8b);
//shuffle to get sao index
edge0_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge0_16x8b);
edge1_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge1_16x8b);
//shuffle to get sao offset
//using availability mask
edge0_16x8b = _mm_and_si128(edge0_16x8b, au1_mask8x16b);
edge1_16x8b = _mm_and_si128(edge1_16x8b, au1_mask8x16b);
//adding chroma offset to access U and V
edge0_16x8b = _mm_add_epi8(edge0_16x8b, chroma_offset_8x16b);
edge1_16x8b = _mm_add_epi8(edge1_16x8b, chroma_offset_8x16b);
edge0_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge0_16x8b);
edge1_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge1_16x8b);
//cnvert to 16 bit then add and then saturated pack
left0_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge0_16x8b);
cmp_gt0_16x8b = _mm_unpacklo_epi8(src_temp0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_unpacklo_epi8(edge0_16x8b, left0_16x8b);
src_temp0_16x8b = _mm_unpackhi_epi8(src_temp0_16x8b, const0_16x8b);
edge0_16x8b = _mm_unpackhi_epi8(edge0_16x8b, left0_16x8b);
cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b);
src_temp0_16x8b = _mm_add_epi16(src_temp0_16x8b, edge0_16x8b);
src_temp0_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_temp0_16x8b);
left0_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge1_16x8b);
cmp_gt0_16x8b = _mm_unpacklo_epi8(src_temp1_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_unpacklo_epi8(edge1_16x8b, left0_16x8b);
src_temp1_16x8b = _mm_unpackhi_epi8(src_temp1_16x8b, const0_16x8b);
edge1_16x8b = _mm_unpackhi_epi8(edge1_16x8b, left0_16x8b);
cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b);
src_temp1_16x8b = _mm_add_epi16(src_temp1_16x8b, edge1_16x8b);
src_temp1_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_temp1_16x8b);
//seaprting row 1 and row 3
cmp_gt0_16x8b = _mm_srli_si128(src_temp0_16x8b, 8);
cmp_lt0_16x8b = _mm_srli_si128(src_temp1_16x8b, 8);
_mm_storel_epi64((__m128i *)(pu1_src_left_str), left_store_16x8b);
//row = 0 store 8 pixel values from 7:0 pos. relative to cur. pos.
_mm_storel_epi64((__m128i *)(pu1_src_cpy), src_temp0_16x8b);
// row = 1
_mm_storel_epi64((__m128i *)(pu1_src_cpy + src_strd), cmp_gt0_16x8b);
// row = 2
_mm_storel_epi64((__m128i *)(pu1_src_cpy + 2 * src_strd), src_temp1_16x8b);
// row = 3
_mm_storel_epi64((__m128i *)(pu1_src_cpy + 3 * src_strd), cmp_lt0_16x8b);
pu1_src_cpy += (src_strd << 2);
pu1_src_left_cpy += 8;
pu1_src_left_str += 8;
}
pu1_src += wd;
pu1_src_left_cpy -= 2 * ht;
pu1_src_left_str -= 2 * ht;
pu1_left_tmp = pu1_src_left_cpy;
pu1_src_left_cpy = pu1_src_left_str;
pu1_src_left_str = pu1_left_tmp;
}
for(row = 0; row < 2 * ht; row++)
{
pu1_src_left[row] = pu1_src_left_cpy[row];
}
}
}
void ihevc_sao_edge_offset_class1_ssse3(UWORD8 *pu1_src,
WORD32 src_strd,
UWORD8 *pu1_src_left,
UWORD8 *pu1_src_top,
UWORD8 *pu1_src_top_left,
UWORD8 *pu1_src_top_right,
UWORD8 *pu1_src_bot_left,
UWORD8 *pu1_avail,
WORD8 *pi1_sao_offset,
WORD32 wd,
WORD32 ht)
{
WORD32 row, col;
UWORD8 *pu1_src_top_cpy;
UWORD8 *pu1_src_cpy;
WORD32 wd_rem;
__m128i src_top_16x8b, src_bottom_16x8b;
__m128i src_temp0_16x8b, src_temp1_16x8b;
__m128i signup0_16x8b, signdwn1_16x8b;
__m128i cmp_gt0_16x8b, cmp_lt0_16x8b;
__m128i edge0_16x8b, edge1_16x8b;
__m128i edge_idx_8x16b, sao_offset_8x16b;
__m128i const2_16x8b, const0_16x8b;
UNUSED(pu1_src_top_right);
UNUSED(pu1_src_bot_left);
/* Updating left and top-left */
for(row = 0; row < ht; row++)
{
pu1_src_left[row] = pu1_src[row * src_strd + (wd - 1)];
}
*pu1_src_top_left = pu1_src_top[wd - 1];
pu1_src_top_cpy = pu1_src_top;
edge_idx_8x16b = _mm_loadl_epi64((__m128i *)gi1_table_edge_idx);
sao_offset_8x16b = _mm_loadl_epi64((__m128i *)pi1_sao_offset);
/* Update height and source pointers based on the availability flags */
if(0 == pu1_avail[2])
{
pu1_src_top_cpy = pu1_src;
pu1_src += src_strd;
ht--;
}
if(0 == pu1_avail[3])
{
ht--;
}
const2_16x8b = _mm_set1_epi8(2);
const0_16x8b = _mm_setzero_si128();
{
WORD32 ht_rem;
for(col = wd; col >= 16; col -= 16)
{
pu1_src_cpy = pu1_src;
src_top_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_top_cpy + wd - col));
//row = 0
src_temp0_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy));
//separating +ve and and -ve values.
cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, src_top_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(src_top_16x8b, src_temp0_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//combining the appropriate sign change
signup0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b);
for(row = ht; row >= 2; row -= 2)
{
//row = 1 load 8 pixel values from 7:0 pos. relative to cur. pos.
src_temp1_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + src_strd));
// row = 2
src_bottom_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + 2 * src_strd));
//row 0 -row1
//separating +ve and and -ve values.
cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, src_temp1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, src_temp0_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//combining the appropriate sign change
edge0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b);
//row1-row0
edge1_16x8b = _mm_sub_epi8(cmp_lt0_16x8b, cmp_gt0_16x8b);
//row1 -bottom
cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, src_bottom_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, src_temp1_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//combining the appropriate sign change
signdwn1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b);
//combining sign-left and sign_right
edge0_16x8b = _mm_add_epi8(edge0_16x8b, signup0_16x8b);
edge1_16x8b = _mm_add_epi8(edge1_16x8b, signdwn1_16x8b);
//for the next iteration signup0_16x8b = -signdwn1_16x8b
signup0_16x8b = _mm_sub_epi8(cmp_lt0_16x8b, cmp_gt0_16x8b);
//adding constant 2
edge0_16x8b = _mm_add_epi8(edge0_16x8b, const2_16x8b);
edge1_16x8b = _mm_add_epi8(edge1_16x8b, const2_16x8b);
//shuffle to get sao index
edge0_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge0_16x8b);
edge1_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge1_16x8b);
//shuffle to get sao offset
edge0_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge0_16x8b);
edge1_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge1_16x8b);
//copying the next top
src_top_16x8b = src_temp1_16x8b;
//cnvert to 16 bit then add and then saturated pack
signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge0_16x8b);
cmp_gt0_16x8b = _mm_unpacklo_epi8(src_temp0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_unpacklo_epi8(edge0_16x8b, signdwn1_16x8b);
src_temp0_16x8b = _mm_unpackhi_epi8(src_temp0_16x8b, const0_16x8b);
cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b);
cmp_lt0_16x8b = _mm_unpackhi_epi8(edge0_16x8b, signdwn1_16x8b);
src_temp0_16x8b = _mm_add_epi16(src_temp0_16x8b, cmp_lt0_16x8b);
src_temp0_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_temp0_16x8b);
signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge1_16x8b);
cmp_gt0_16x8b = _mm_unpacklo_epi8(src_temp1_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_unpacklo_epi8(edge1_16x8b, signdwn1_16x8b);
src_temp1_16x8b = _mm_unpackhi_epi8(src_temp1_16x8b, const0_16x8b);
cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b);
cmp_lt0_16x8b = _mm_unpackhi_epi8(edge1_16x8b, signdwn1_16x8b);
src_temp1_16x8b = _mm_add_epi16(src_temp1_16x8b, cmp_lt0_16x8b);
src_temp1_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_temp1_16x8b);
//row = 0 store 8 pixel values from 7:0 pos. relative to cur. pos.
_mm_storeu_si128((__m128i *)(pu1_src_cpy), src_temp0_16x8b);
// row = 1
_mm_storeu_si128((__m128i *)(pu1_src_cpy + src_strd), src_temp1_16x8b);
src_temp0_16x8b = src_bottom_16x8b;
pu1_src_cpy += (src_strd << 1);
}
ht_rem = ht & 0x1;
if(ht_rem)
{
src_bottom_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + src_strd));
//current row -next row
//separating +ve and and -ve values.
cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, src_bottom_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, src_temp0_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//combining the appropriate sign change
edge0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b);
//adding top and botton and constant 2
edge0_16x8b = _mm_add_epi8(edge0_16x8b, signup0_16x8b);
edge0_16x8b = _mm_add_epi8(edge0_16x8b, const2_16x8b);
edge0_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge0_16x8b);
edge0_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge0_16x8b);
//copying the next top
src_top_16x8b = src_temp0_16x8b;
//cnvert to 16 bit then add and then saturated pack
signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge0_16x8b);
cmp_gt0_16x8b = _mm_unpacklo_epi8(src_temp0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_unpacklo_epi8(edge0_16x8b, signdwn1_16x8b);
src_temp0_16x8b = _mm_unpackhi_epi8(src_temp0_16x8b, const0_16x8b);
cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b);
cmp_lt0_16x8b = _mm_unpackhi_epi8(edge0_16x8b, signdwn1_16x8b);
src_temp0_16x8b = _mm_add_epi16(src_temp0_16x8b, cmp_lt0_16x8b);
src_temp0_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_temp0_16x8b);
_mm_storeu_si128((__m128i *)(pu1_src_cpy), src_temp0_16x8b);
}
if(0 == pu1_avail[3])
{
src_top_16x8b = src_bottom_16x8b;
}
//updating top flag
_mm_storeu_si128((__m128i *)(pu1_src_top + wd - col), src_top_16x8b);
pu1_src += 16;
}
wd_rem = wd & 0xF;
if(wd_rem)
{
pu1_src_cpy = pu1_src;
src_top_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_top_cpy + wd - col));
//row = 0
src_temp0_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy));
//separating +ve and and -ve values.
cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, src_top_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(src_top_16x8b, src_temp0_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//combining the appropriate sign change
signup0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b);
signup0_16x8b = _mm_slli_si128(signup0_16x8b, 8);
for(row = ht; row >= 4; row -= 4)
{
//row = 1 load 8 pixel values from 7:0 pos. relative to cur. pos.
src_temp1_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy + src_strd));
// row = 2
src_bottom_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy + 2 * src_strd));
//row 0 -row1
//separating +ve and and -ve values.
cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, src_temp1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, src_temp0_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//combining the appropriate sign change
edge0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b);
//row1-row0
edge1_16x8b = _mm_sub_epi8(cmp_lt0_16x8b, cmp_gt0_16x8b);
edge0_16x8b = _mm_slli_si128(edge0_16x8b, 8); //aligned left (0-1)
signup0_16x8b = _mm_alignr_epi8(edge1_16x8b, signup0_16x8b, 8); //(1-0),(0-top)
//row1 -row2
cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, src_bottom_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, src_temp1_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//combining the appropriate sign change
signdwn1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(1-2)
edge0_16x8b = _mm_alignr_epi8(signdwn1_16x8b, edge0_16x8b, 8); //(1-2),(0-1)
//packing row 0 n row 1
src_temp0_16x8b = _mm_unpacklo_epi64(src_temp0_16x8b, src_temp1_16x8b);
//row = 3
src_top_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy + 3 * src_strd));
// row = 4
src_temp1_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy + 4 * src_strd));
edge0_16x8b = _mm_add_epi8(edge0_16x8b, signup0_16x8b); //(1,0) sign_up empty
signdwn1_16x8b = _mm_slli_si128(signdwn1_16x8b, 8); //allign left (1-2)
//separating +ve and and -ve values.(2,3)
cmp_gt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, src_top_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(src_top_16x8b, src_bottom_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//combining the appropriate sign change
edge1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(2-3)
signup0_16x8b = _mm_alignr_epi8(edge1_16x8b, signdwn1_16x8b, 8); //(2-3) ,(1-2) (substract with down)
edge1_16x8b = _mm_slli_si128(edge1_16x8b, 8);
//separating +ve and and -ve values.(3,4)
cmp_gt0_16x8b = _mm_subs_epu8(src_top_16x8b, src_temp1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, src_top_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
signdwn1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(3-4)
//combining sign-left and sign_right
edge1_16x8b = _mm_alignr_epi8(signdwn1_16x8b, edge1_16x8b, 8); //(3-4),(2-3)
edge1_16x8b = _mm_sub_epi8(edge1_16x8b, signup0_16x8b); //(3,2)
//packing row 2 n row 3
src_bottom_16x8b = _mm_unpacklo_epi64(src_bottom_16x8b, src_top_16x8b);
//for the next iteration signup0_16x8b = -signdwn1_16x8b
signup0_16x8b = _mm_sub_epi8(cmp_lt0_16x8b, cmp_gt0_16x8b); //(4-3)
//adding constant 2
edge0_16x8b = _mm_add_epi8(edge0_16x8b, const2_16x8b);
edge1_16x8b = _mm_add_epi8(edge1_16x8b, const2_16x8b);
//shuffle to get sao index
edge0_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge0_16x8b);
edge1_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge1_16x8b);
//shuffle to get sao offset
edge0_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge0_16x8b);
edge1_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge1_16x8b);
//the next top already in src_top_16x8b
//src_top_16x8b = src_temp1_16x8b;
//cnvert to 16 bit then add and then saturated pack
signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge0_16x8b);
cmp_gt0_16x8b = _mm_unpacklo_epi8(src_temp0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_unpacklo_epi8(edge0_16x8b, signdwn1_16x8b);
src_temp0_16x8b = _mm_unpackhi_epi8(src_temp0_16x8b, const0_16x8b);
cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b);
cmp_lt0_16x8b = _mm_unpackhi_epi8(edge0_16x8b, signdwn1_16x8b);
src_temp0_16x8b = _mm_add_epi16(src_temp0_16x8b, cmp_lt0_16x8b);
src_temp0_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_temp0_16x8b);
signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge1_16x8b);
cmp_gt0_16x8b = _mm_unpacklo_epi8(src_bottom_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_unpacklo_epi8(edge1_16x8b, signdwn1_16x8b);
src_bottom_16x8b = _mm_unpackhi_epi8(src_bottom_16x8b, const0_16x8b);
cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b);
cmp_lt0_16x8b = _mm_unpackhi_epi8(edge1_16x8b, signdwn1_16x8b);
src_bottom_16x8b = _mm_add_epi16(src_bottom_16x8b, cmp_lt0_16x8b);
src_bottom_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_bottom_16x8b);
cmp_gt0_16x8b = _mm_srli_si128(src_temp0_16x8b, 8);
cmp_lt0_16x8b = _mm_srli_si128(src_bottom_16x8b, 8);
//row = 0 store 8 pixel values from 7:0 pos. relative to cur. pos.
_mm_storel_epi64((__m128i *)(pu1_src_cpy), src_temp0_16x8b);
// row = 1
_mm_storel_epi64((__m128i *)(pu1_src_cpy + src_strd), cmp_gt0_16x8b);
//row = 2
_mm_storel_epi64((__m128i *)(pu1_src_cpy + 2 * src_strd), src_bottom_16x8b);
// row = 3
_mm_storel_epi64((__m128i *)(pu1_src_cpy + 3 * src_strd), cmp_lt0_16x8b);
src_temp0_16x8b = src_temp1_16x8b;
signup0_16x8b = _mm_slli_si128(signup0_16x8b, 8);
pu1_src_cpy += (src_strd << 2);
}
ht_rem = ht & 0x2;
if(ht_rem)
{
//row = 1 load 8 pixel values from 7:0 pos. relative to cur. pos.
src_temp1_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy + src_strd));
// row = 2
src_bottom_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy + 2 * src_strd));
//row 0 -row1
//separating +ve and and -ve values.
cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, src_temp1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, src_temp0_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//combining the appropriate sign change
edge0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b);
//row1-row0
edge1_16x8b = _mm_sub_epi8(cmp_lt0_16x8b, cmp_gt0_16x8b);
edge0_16x8b = _mm_slli_si128(edge0_16x8b, 8); //aligned left (0-1)
signup0_16x8b = _mm_alignr_epi8(edge1_16x8b, signup0_16x8b, 8); //(1-0),(0-top)
//row1 -row2
cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, src_bottom_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, src_temp1_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//combining the appropriate sign change
signdwn1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(1-2)
edge0_16x8b = _mm_alignr_epi8(signdwn1_16x8b, edge0_16x8b, 8); //(1-2),(0-1)
//adding top and down substraction
edge0_16x8b = _mm_add_epi8(edge0_16x8b, signup0_16x8b); //(1,0) sign_up empty
//for the next iteration signup0_16x8b = -signdwn1_16x8b
signup0_16x8b = _mm_sub_epi8(cmp_lt0_16x8b, cmp_gt0_16x8b); //(2-1) for next
src_top_16x8b = src_temp1_16x8b;
//adding constant 2
edge0_16x8b = _mm_add_epi8(edge0_16x8b, const2_16x8b);
//shuffle to get sao index
edge0_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge0_16x8b);
//shuffle to get sao offset
edge0_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge0_16x8b);
//the next top already in src_top_16x8b
//cnvert to 16 bit then add and then saturated pack
signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge0_16x8b);
src_temp0_16x8b = _mm_unpacklo_epi8(src_temp0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_unpacklo_epi8(edge0_16x8b, signdwn1_16x8b);
src_temp1_16x8b = _mm_unpacklo_epi8(src_temp1_16x8b, const0_16x8b);
src_temp0_16x8b = _mm_add_epi16(src_temp0_16x8b, cmp_lt0_16x8b);
cmp_lt0_16x8b = _mm_unpackhi_epi8(edge0_16x8b, signdwn1_16x8b);
src_temp1_16x8b = _mm_add_epi16(src_temp1_16x8b, cmp_lt0_16x8b);
src_temp0_16x8b = _mm_packus_epi16(src_temp0_16x8b, src_temp1_16x8b);
cmp_gt0_16x8b = _mm_srli_si128(src_temp0_16x8b, 8);
//row = 0 store 8 pixel values from 7:0 pos. relative to cur. pos.
_mm_storel_epi64((__m128i *)(pu1_src_cpy), src_temp0_16x8b);
// row = 1
_mm_storel_epi64((__m128i *)(pu1_src_cpy + src_strd), cmp_gt0_16x8b);
src_temp0_16x8b = src_bottom_16x8b;
pu1_src_cpy += (src_strd << 1);
}
ht_rem = ht & 0x1;
if(ht_rem)
{
//row = 1 load 8 pixel values from 7:0 pos. relative to cur. pos.
src_bottom_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy + src_strd));
//row 0 -row1
//separating +ve and and -ve values.
cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, src_bottom_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, src_temp0_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//combining the appropriate sign change
edge0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b);
//adding top and down substraction
edge0_16x8b = _mm_add_epi8(edge0_16x8b, signup0_16x8b);
//adding constant 2
edge0_16x8b = _mm_add_epi8(edge0_16x8b, const2_16x8b);
edge0_16x8b = _mm_slli_si128(edge0_16x8b, 8);
edge0_16x8b = _mm_srli_si128(edge0_16x8b, 8);
//shuffle to get sao index
edge0_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge0_16x8b);
//shuffle to get sao offset
edge0_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge0_16x8b);
src_top_16x8b = src_temp0_16x8b;
//cnvert to 16 bit then add and then saturated pack
signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge0_16x8b);
src_temp0_16x8b = _mm_unpacklo_epi8(src_temp0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_unpacklo_epi8(edge0_16x8b, signdwn1_16x8b);
src_temp0_16x8b = _mm_add_epi16(src_temp0_16x8b, cmp_lt0_16x8b);
src_temp0_16x8b = _mm_packus_epi16(src_temp0_16x8b, const0_16x8b);
//row = 0 store 8 pixel values from 7:0 pos. relative to cur. pos.
_mm_storel_epi64((__m128i *)(pu1_src_cpy), src_temp0_16x8b);
pu1_src_cpy += (src_strd);
}
if(0 == pu1_avail[3])
{
src_top_16x8b = src_bottom_16x8b;
}
_mm_storel_epi64((__m128i *)(pu1_src_top + wd - col), src_top_16x8b);
pu1_src += 8;
}
}
}
void ihevc_sao_edge_offset_class1_chroma_ssse3(UWORD8 *pu1_src,
WORD32 src_strd,
UWORD8 *pu1_src_left,
UWORD8 *pu1_src_top,
UWORD8 *pu1_src_top_left,
UWORD8 *pu1_src_top_right,
UWORD8 *pu1_src_bot_left,
UWORD8 *pu1_avail,
WORD8 *pi1_sao_offset_u,
WORD8 *pi1_sao_offset_v,
WORD32 wd,
WORD32 ht)
{
WORD32 row, col;
UWORD8 *pu1_src_top_cpy;
UWORD8 *pu1_src_cpy;
WORD32 wd_rem;
__m128i src_top_16x8b, src_bottom_16x8b;
__m128i src_temp0_16x8b, src_temp1_16x8b;
__m128i signup0_16x8b, signdwn1_16x8b;
__m128i cmp_gt0_16x8b, cmp_lt0_16x8b;
__m128i edge0_16x8b, edge1_16x8b;
__m128i edge_idx_8x16b, sao_offset_8x16b;
__m128i const2_16x8b, const0_16x8b;
__m128i chroma_offset_8x16b;
UNUSED(pu1_src_top_right);
UNUSED(pu1_src_bot_left);
/* Updating left and top and top-left */
for(row = 0; row < ht; row++)
{
pu1_src_left[2 * row] = pu1_src[row * src_strd + (wd - 2)];
pu1_src_left[2 * row + 1] = pu1_src[row * src_strd + (wd - 1)];
}
pu1_src_top_left[0] = pu1_src_top[wd - 2];
pu1_src_top_left[1] = pu1_src_top[wd - 1];
pu1_src_top_cpy = pu1_src_top;
edge_idx_8x16b = _mm_loadl_epi64((__m128i *)gi1_table_edge_idx);
sao_offset_8x16b = _mm_loadl_epi64((__m128i *)pi1_sao_offset_u);
const0_16x8b = _mm_loadl_epi64((__m128i *)pi1_sao_offset_v);
chroma_offset_8x16b = _mm_set1_epi16(0x0800);
/* Update height and source pointers based on the availability flags */
if(0 == pu1_avail[2])
{
pu1_src_top_cpy = pu1_src;
pu1_src += src_strd;
ht--;
}
if(0 == pu1_avail[3])
{
ht--;
}
sao_offset_8x16b = _mm_unpacklo_epi64(sao_offset_8x16b, const0_16x8b);
const2_16x8b = _mm_set1_epi8(2);
const0_16x8b = _mm_setzero_si128();
{
WORD32 ht_rem;
for(col = wd; col >= 16; col -= 16)
{
pu1_src_cpy = pu1_src;
src_top_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_top_cpy + wd - col));
//row = 0
src_temp0_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy));
//separating +ve and and -ve values.
cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, src_top_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(src_top_16x8b, src_temp0_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//combining the appropriate sign change
signup0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b);
for(row = ht; row >= 2; row -= 2)
{
//row = 1 load 8 pixel values from 7:0 pos. relative to cur. pos.
src_temp1_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + src_strd));
// row = 2
src_bottom_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + 2 * src_strd));
//row 0 -row1
//separating +ve and and -ve values.
cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, src_temp1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, src_temp0_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//combining the appropriate sign change
edge0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b);
//row1-row0
edge1_16x8b = _mm_sub_epi8(cmp_lt0_16x8b, cmp_gt0_16x8b);
//row1 -bottom
cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, src_bottom_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, src_temp1_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//combining the appropriate sign change
signdwn1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b);
//combining sign-left and sign_right
edge0_16x8b = _mm_add_epi8(edge0_16x8b, signup0_16x8b);
edge1_16x8b = _mm_add_epi8(edge1_16x8b, signdwn1_16x8b);
//for the next iteration signup0_16x8b = -signdwn1_16x8b
signup0_16x8b = _mm_sub_epi8(cmp_lt0_16x8b, cmp_gt0_16x8b);
//adding constant 2
edge0_16x8b = _mm_add_epi8(edge0_16x8b, const2_16x8b);
edge1_16x8b = _mm_add_epi8(edge1_16x8b, const2_16x8b);
//copying the next top
src_top_16x8b = src_temp1_16x8b;
//shuffle to get sao index
edge0_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge0_16x8b);
edge1_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge1_16x8b);
//adding chroma offset to access U and V
edge0_16x8b = _mm_add_epi8(edge0_16x8b, chroma_offset_8x16b);
edge1_16x8b = _mm_add_epi8(edge1_16x8b, chroma_offset_8x16b);
//shuffle to get sao offset
edge0_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge0_16x8b);
edge1_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge1_16x8b);
//cnvert to 16 bit then add and then saturated pack
signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge0_16x8b);
cmp_gt0_16x8b = _mm_unpacklo_epi8(src_temp0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_unpacklo_epi8(edge0_16x8b, signdwn1_16x8b);
src_temp0_16x8b = _mm_unpackhi_epi8(src_temp0_16x8b, const0_16x8b);
edge0_16x8b = _mm_unpackhi_epi8(edge0_16x8b, signdwn1_16x8b);
cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b);
src_temp0_16x8b = _mm_add_epi16(src_temp0_16x8b, edge0_16x8b);
src_temp0_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_temp0_16x8b);
signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge1_16x8b);
cmp_gt0_16x8b = _mm_unpacklo_epi8(src_temp1_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_unpacklo_epi8(edge1_16x8b, signdwn1_16x8b);
src_temp1_16x8b = _mm_unpackhi_epi8(src_temp1_16x8b, const0_16x8b);
edge1_16x8b = _mm_unpackhi_epi8(edge1_16x8b, signdwn1_16x8b);
cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b);
src_temp1_16x8b = _mm_add_epi16(src_temp1_16x8b, edge1_16x8b);
src_temp1_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_temp1_16x8b);
//row = 0 store 8 pixel values from 7:0 pos. relative to cur. pos.
_mm_storeu_si128((__m128i *)(pu1_src_cpy), src_temp0_16x8b);
// row = 1
_mm_storeu_si128((__m128i *)(pu1_src_cpy + src_strd), src_temp1_16x8b);
src_temp0_16x8b = src_bottom_16x8b;
pu1_src_cpy += (src_strd << 1);
}
ht_rem = ht & 0x1;
if(ht_rem)
{
src_bottom_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + src_strd));
//current row -next row
//separating +ve and and -ve values.
cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, src_bottom_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, src_temp0_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//combining the appropriate sign change
edge0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b);
//adding top and botton and constant 2
edge0_16x8b = _mm_add_epi8(edge0_16x8b, signup0_16x8b);
edge0_16x8b = _mm_add_epi8(edge0_16x8b, const2_16x8b);
//copying the next top
src_top_16x8b = src_temp0_16x8b;
edge0_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge0_16x8b);
//adding chroma offset to access U and V
edge0_16x8b = _mm_add_epi8(edge0_16x8b, chroma_offset_8x16b);
edge0_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge0_16x8b);
//cnvert to 16 bit then add and then saturated pack
signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge0_16x8b);
cmp_gt0_16x8b = _mm_unpacklo_epi8(src_temp0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_unpacklo_epi8(edge0_16x8b, signdwn1_16x8b);
src_temp0_16x8b = _mm_unpackhi_epi8(src_temp0_16x8b, const0_16x8b);
edge0_16x8b = _mm_unpackhi_epi8(edge0_16x8b, signdwn1_16x8b);
cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b);
src_temp0_16x8b = _mm_add_epi16(src_temp0_16x8b, edge0_16x8b);
src_temp0_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_temp0_16x8b);
_mm_storeu_si128((__m128i *)(pu1_src_cpy), src_temp0_16x8b);
}
if(0 == pu1_avail[3])
{
src_top_16x8b = src_bottom_16x8b;
}
//updating top flag
_mm_storeu_si128((__m128i *)(pu1_src_top + wd - col), src_top_16x8b);
pu1_src += 16;
}
wd_rem = wd & 0xF;
if(wd_rem)
{
pu1_src_cpy = pu1_src;
src_top_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_top_cpy + wd - col));
//row = 0
src_temp0_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy));
//separating +ve and and -ve values.
cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, src_top_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(src_top_16x8b, src_temp0_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//combining the appropriate sign change
signup0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b);
signup0_16x8b = _mm_slli_si128(signup0_16x8b, 8);
for(row = ht; row >= 4; row -= 4)
{
//row = 1 load 8 pixel values from 7:0 pos. relative to cur. pos.
src_temp1_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy + src_strd));
// row = 2
src_bottom_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy + 2 * src_strd));
//row 0 -row1
//separating +ve and and -ve values.
cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, src_temp1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, src_temp0_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//combining the appropriate sign change
edge0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b);
//row1-row0
edge1_16x8b = _mm_sub_epi8(cmp_lt0_16x8b, cmp_gt0_16x8b);
edge0_16x8b = _mm_slli_si128(edge0_16x8b, 8); //aligned left (0-1)
signup0_16x8b = _mm_alignr_epi8(edge1_16x8b, signup0_16x8b, 8); //(1-0),(0-top)
//row1 -row2
cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, src_bottom_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, src_temp1_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//combining the appropriate sign change
signdwn1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(1-2)
edge0_16x8b = _mm_alignr_epi8(signdwn1_16x8b, edge0_16x8b, 8); //(1-2),(0-1)
//packing row 0 n row 1
src_temp0_16x8b = _mm_unpacklo_epi64(src_temp0_16x8b, src_temp1_16x8b);
//row = 3
src_top_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy + 3 * src_strd));
// row = 4
src_temp1_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy + 4 * src_strd));
edge0_16x8b = _mm_add_epi8(edge0_16x8b, signup0_16x8b); //(1,0) sign_up empty
signdwn1_16x8b = _mm_slli_si128(signdwn1_16x8b, 8); //allign left (1-2)
//separating +ve and and -ve values.(2,3)
cmp_gt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, src_top_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(src_top_16x8b, src_bottom_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//combining the appropriate sign change
edge1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(2-3)
signup0_16x8b = _mm_alignr_epi8(edge1_16x8b, signdwn1_16x8b, 8); //(2-3) ,(1-2) (substract with down)
edge1_16x8b = _mm_slli_si128(edge1_16x8b, 8);
//separating +ve and and -ve values.(3,4)
cmp_gt0_16x8b = _mm_subs_epu8(src_top_16x8b, src_temp1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, src_top_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
signdwn1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(3-4)
//combining sign-left and sign_right
edge1_16x8b = _mm_alignr_epi8(signdwn1_16x8b, edge1_16x8b, 8); //(3-4),(2-3)
edge1_16x8b = _mm_sub_epi8(edge1_16x8b, signup0_16x8b); //(3,2)
//packing row 2 n row 3
src_bottom_16x8b = _mm_unpacklo_epi64(src_bottom_16x8b, src_top_16x8b);
//for the next iteration signup0_16x8b = -signdwn1_16x8b
signup0_16x8b = _mm_sub_epi8(cmp_lt0_16x8b, cmp_gt0_16x8b); //(4-3)
//adding constant 2
edge0_16x8b = _mm_add_epi8(edge0_16x8b, const2_16x8b);
edge1_16x8b = _mm_add_epi8(edge1_16x8b, const2_16x8b);
//shuffle to get sao index
edge0_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge0_16x8b);
edge1_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge1_16x8b);
//adding chroma offset to access U and V
edge0_16x8b = _mm_add_epi8(edge0_16x8b, chroma_offset_8x16b);
edge1_16x8b = _mm_add_epi8(edge1_16x8b, chroma_offset_8x16b);
//shuffle to get sao offset
edge0_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge0_16x8b);
edge1_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge1_16x8b);
//the next top already in src_top_16x8b
//cnvert to 16 bit then add and then saturated pack
signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge0_16x8b);
cmp_gt0_16x8b = _mm_unpacklo_epi8(src_temp0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_unpacklo_epi8(edge0_16x8b, signdwn1_16x8b);
src_temp0_16x8b = _mm_unpackhi_epi8(src_temp0_16x8b, const0_16x8b);
edge0_16x8b = _mm_unpackhi_epi8(edge0_16x8b, signdwn1_16x8b);
cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b);
src_temp0_16x8b = _mm_add_epi16(src_temp0_16x8b, edge0_16x8b);
src_temp0_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_temp0_16x8b);
signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge1_16x8b);
cmp_gt0_16x8b = _mm_unpacklo_epi8(src_bottom_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_unpacklo_epi8(edge1_16x8b, signdwn1_16x8b);
src_bottom_16x8b = _mm_unpackhi_epi8(src_bottom_16x8b, const0_16x8b);
edge1_16x8b = _mm_unpackhi_epi8(edge1_16x8b, signdwn1_16x8b);
cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b);
src_bottom_16x8b = _mm_add_epi16(src_bottom_16x8b, edge1_16x8b);
src_bottom_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_bottom_16x8b);
cmp_gt0_16x8b = _mm_srli_si128(src_temp0_16x8b, 8);
cmp_lt0_16x8b = _mm_srli_si128(src_bottom_16x8b, 8);
//row = 0 store 8 pixel values from 7:0 pos. relative to cur. pos.
_mm_storel_epi64((__m128i *)(pu1_src_cpy), src_temp0_16x8b);
// row = 1
_mm_storel_epi64((__m128i *)(pu1_src_cpy + src_strd), cmp_gt0_16x8b);
//row = 2
_mm_storel_epi64((__m128i *)(pu1_src_cpy + 2 * src_strd), src_bottom_16x8b);
// row = 3
_mm_storel_epi64((__m128i *)(pu1_src_cpy + 3 * src_strd), cmp_lt0_16x8b);
src_temp0_16x8b = src_temp1_16x8b;
signup0_16x8b = _mm_slli_si128(signup0_16x8b, 8);
pu1_src_cpy += (src_strd << 2);
}
ht_rem = ht & 0x2;
if(ht_rem)
{
//row = 1 load 8 pixel values from 7:0 pos. relative to cur. pos.
src_temp1_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy + src_strd));
// row = 2
src_bottom_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy + 2 * src_strd));
//row 0 -row1
//separating +ve and and -ve values.
cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, src_temp1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, src_temp0_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//combining the appropriate sign change
edge0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b);
//row1-row0
edge1_16x8b = _mm_sub_epi8(cmp_lt0_16x8b, cmp_gt0_16x8b);
edge0_16x8b = _mm_slli_si128(edge0_16x8b, 8); //aligned left (0-1)
signup0_16x8b = _mm_alignr_epi8(edge1_16x8b, signup0_16x8b, 8); //(1-0),(0-top)
//row1 -row2
cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, src_bottom_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, src_temp1_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//combining the appropriate sign change
signdwn1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(1-2)
edge0_16x8b = _mm_alignr_epi8(signdwn1_16x8b, edge0_16x8b, 8); //(1-2),(0-1)
//adding top and down substraction
edge0_16x8b = _mm_add_epi8(edge0_16x8b, signup0_16x8b); //(1,0) sign_up empty
//for the next iteration signup0_16x8b = -signdwn1_16x8b
signup0_16x8b = _mm_sub_epi8(cmp_lt0_16x8b, cmp_gt0_16x8b); //(2-1) for next
src_top_16x8b = src_temp1_16x8b;
//adding constant 2
edge0_16x8b = _mm_add_epi8(edge0_16x8b, const2_16x8b);
//shuffle to get sao index
edge0_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge0_16x8b);
//adding chroma offset to access U and V
edge0_16x8b = _mm_add_epi8(edge0_16x8b, chroma_offset_8x16b);
//shuffle to get sao offset
edge0_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge0_16x8b);
//the next top already in src_top_16x8b
//cnvert to 16 bit then add and then saturated pack
signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge0_16x8b);
src_temp0_16x8b = _mm_unpacklo_epi8(src_temp0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_unpacklo_epi8(edge0_16x8b, signdwn1_16x8b);
src_temp1_16x8b = _mm_unpacklo_epi8(src_temp1_16x8b, const0_16x8b);
edge0_16x8b = _mm_unpackhi_epi8(edge0_16x8b, signdwn1_16x8b);
src_temp0_16x8b = _mm_add_epi16(src_temp0_16x8b, cmp_lt0_16x8b);
src_temp1_16x8b = _mm_add_epi16(src_temp1_16x8b, edge0_16x8b);
src_temp0_16x8b = _mm_packus_epi16(src_temp0_16x8b, src_temp1_16x8b);
cmp_gt0_16x8b = _mm_srli_si128(src_temp0_16x8b, 8);
//row = 0 store 8 pixel values from 7:0 pos. relative to cur. pos.
_mm_storel_epi64((__m128i *)(pu1_src_cpy), src_temp0_16x8b);
// row = 1
_mm_storel_epi64((__m128i *)(pu1_src_cpy + src_strd), cmp_gt0_16x8b);
src_temp0_16x8b = src_bottom_16x8b;
pu1_src_cpy += (src_strd << 1);
}
ht_rem = ht & 0x1;
if(ht_rem)
{
//row = 1 load 8 pixel values from 7:0 pos. relative to cur. pos.
src_bottom_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy + src_strd));
//row 0 -row1
//separating +ve and and -ve values.
cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, src_bottom_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, src_temp0_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//combining the appropriate sign change
edge0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b);
//adding top and down substraction
edge0_16x8b = _mm_add_epi8(edge0_16x8b, signup0_16x8b);
//adding constant 2
edge0_16x8b = _mm_add_epi8(edge0_16x8b, const2_16x8b);
src_top_16x8b = src_temp0_16x8b;
edge0_16x8b = _mm_slli_si128(edge0_16x8b, 8);
edge0_16x8b = _mm_srli_si128(edge0_16x8b, 8);
//shuffle to get sao index
edge0_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge0_16x8b);
//adding chroma offset to access U and V
edge0_16x8b = _mm_add_epi8(edge0_16x8b, chroma_offset_8x16b);
//shuffle to get sao offset
edge0_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge0_16x8b);
//cnvert to 16 bit then add and then saturated pack
signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge0_16x8b);
src_temp0_16x8b = _mm_unpacklo_epi8(src_temp0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_unpacklo_epi8(edge0_16x8b, signdwn1_16x8b);
src_temp0_16x8b = _mm_add_epi16(src_temp0_16x8b, cmp_lt0_16x8b);
src_temp0_16x8b = _mm_packus_epi16(src_temp0_16x8b, const0_16x8b);
//row = 0 store 8 pixel values from 7:0 pos. relative to cur. pos.
_mm_storel_epi64((__m128i *)(pu1_src_cpy), src_temp0_16x8b);
pu1_src_cpy += (src_strd);
}
if(0 == pu1_avail[3])
{
src_top_16x8b = src_bottom_16x8b;
}
_mm_storel_epi64((__m128i *)(pu1_src_top + wd - col), src_top_16x8b);
pu1_src += 8;
}
}
}
/* 135 degree filtering */
void ihevc_sao_edge_offset_class2_ssse3(UWORD8 *pu1_src,
WORD32 src_strd,
UWORD8 *pu1_src_left,
UWORD8 *pu1_src_top,
UWORD8 *pu1_src_top_left,
UWORD8 *pu1_src_top_right,
UWORD8 *pu1_src_bot_left,
UWORD8 *pu1_avail,
WORD8 *pi1_sao_offset,
WORD32 wd,
WORD32 ht)
{
WORD32 row, col;
UWORD8 *pu1_src_top_cpy, *pu1_src_left_cpy, *pu1_src_left_cpy2;
UWORD8 *pu1_left_tmp, *pu1_src_left_str, *pu1_src_left_str2;
UWORD8 *pu1_firstleft;
UWORD8 *pu1_src_cpy, *pu1_src_org;
UWORD8 au1_mask[MAX_CTB_SIZE], *au1_mask_cpy;
UWORD8 au1_src_left_tmp[MAX_CTB_SIZE + 8];
UWORD8 au1_src_left_tmp1[MAX_CTB_SIZE + 8];
WORD32 wd_rem;
UWORD8 u1_pos_0_0_tmp, u1_pos_wd_ht_tmp;
WORD32 ht_tmp, ht_0;
WORD32 bit_depth;
UWORD8 u1_avail0, u1_avail1;
__m128i src_top_16x8b, src_bottom_16x8b;
__m128i src_temp0_16x8b, src_temp1_16x8b;
__m128i signup0_16x8b, signdwn1_16x8b;
__m128i cmp_gt0_16x8b, cmp_lt0_16x8b;
__m128i edge0_16x8b, edge1_16x8b;
__m128i au1_mask8x16b;
__m128i edge_idx_8x16b, sao_offset_8x16b;
__m128i const2_16x8b, const0_16x8b;
__m128i left_store_16x8b;
UNUSED(pu1_src_top_right);
UNUSED(pu1_src_bot_left);
ht_0 = ht; ht_tmp = ht;
au1_mask8x16b = _mm_set1_epi8(0xff);
//setting availability mask to ff size MAX_CTB_SIZE
for(col = 0; col < MAX_CTB_SIZE; col += 16)
_mm_storeu_si128((__m128i *)(au1_mask + col), au1_mask8x16b);
for(row = 0; row < ht; row++)
{
au1_src_left_tmp[row] = pu1_src_left[row];
}
bit_depth = BIT_DEPTH_LUMA;
pu1_src_org = pu1_src;
pu1_src_top_cpy = pu1_src_top;
pu1_src_left_cpy2 = au1_src_left_tmp;
pu1_src_left_cpy = au1_src_left_tmp;
pu1_src_left_str2 = au1_src_left_tmp1;
pu1_src_left_str = au1_src_left_tmp1;
edge_idx_8x16b = _mm_loadl_epi64((__m128i *)gi1_table_edge_idx);
sao_offset_8x16b = _mm_loadl_epi64((__m128i *)pi1_sao_offset);
/* If top-left is available, process separately */
if(0 != pu1_avail[4])
{
WORD8 edge_idx;
edge_idx = 2 + SIGN(pu1_src[0] - pu1_src_top_left[0]) +
SIGN(pu1_src[0] - pu1_src[1 + src_strd]);
edge_idx = gi1_table_edge_idx[edge_idx];
if(0 != edge_idx)
{
u1_pos_0_0_tmp = CLIP3(pu1_src[0] + pi1_sao_offset[edge_idx], 0, (1 << bit_depth) - 1);
}
else
{
u1_pos_0_0_tmp = pu1_src[0];
}
}
else
{
u1_pos_0_0_tmp = pu1_src[0];
}
/* If bottom-right is available, process separately */
if(0 != pu1_avail[7])
{
WORD8 edge_idx;
edge_idx = 2 + SIGN(pu1_src[wd - 1 + (ht - 1) * src_strd] - pu1_src[wd - 1 + (ht - 1) * src_strd - 1 - src_strd]) +
SIGN(pu1_src[wd - 1 + (ht - 1) * src_strd] - pu1_src[wd - 1 + (ht - 1) * src_strd + 1 + src_strd]);
edge_idx = gi1_table_edge_idx[edge_idx];
if(0 != edge_idx)
{
u1_pos_wd_ht_tmp = CLIP3(pu1_src[wd - 1 + (ht - 1) * src_strd] + pi1_sao_offset[edge_idx], 0, (1 << bit_depth) - 1);
}
else
{
u1_pos_wd_ht_tmp = pu1_src[wd - 1 + (ht - 1) * src_strd];
}
}
else
{
u1_pos_wd_ht_tmp = pu1_src[wd - 1 + (ht - 1) * src_strd];
}
pu1_firstleft = pu1_src_top_left;
/* Update height and source pointers based on the availability flags */
if(0 == pu1_avail[2])
{
pu1_firstleft = pu1_src_left_cpy2;
pu1_src_left_cpy2++;
pu1_src_left_str2++;
pu1_src_top_cpy = pu1_src;
pu1_src += src_strd;
ht--;
}
if(0 == pu1_avail[3])
{
ht--;
ht_0--;
}
//storing top left in a mmx register
left_store_16x8b = _mm_loadl_epi64((__m128i *)pu1_firstleft);
const2_16x8b = _mm_set1_epi8(2);
const0_16x8b = _mm_setzero_si128();
left_store_16x8b = _mm_slli_si128(left_store_16x8b, 15);
//update top -left
*pu1_src_top_left = pu1_src_top[wd - 1];
//availability mask creation
u1_avail0 = pu1_avail[0];
u1_avail1 = pu1_avail[1];
au1_mask[0] = u1_avail0;
au1_mask[wd - 1] = u1_avail1;
{
WORD32 ht_rem;
pu1_src_left_cpy = pu1_src_left_cpy2;
pu1_src_left_str = pu1_src_left_str2;
au1_mask_cpy = au1_mask;
for(col = wd; col >= 16; col -= 16)
{
pu1_src_cpy = pu1_src;
src_top_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_top_cpy + wd - col));
//row = 0
src_temp0_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy));
src_top_16x8b = _mm_alignr_epi8(src_top_16x8b, left_store_16x8b, 15);
//loading the mask
au1_mask8x16b = _mm_loadu_si128((__m128i *)au1_mask_cpy);
//separating +ve and and -ve values.
cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, src_top_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(src_top_16x8b, src_temp0_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//combining the appropriate sign change
signup0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b);
for(row = ht; row >= 2; row -= 2)
{
left_store_16x8b = _mm_loadl_epi64((__m128i *)pu1_src_left_cpy);
//row = 1
src_temp1_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + src_strd));
// row = 1 right
src_bottom_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + src_strd + 1));
//to insert left in row 0
signdwn1_16x8b = _mm_slli_si128(left_store_16x8b, 15);
//row 0 -row1
//separating +ve and and -ve values.
cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, src_bottom_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, src_temp0_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//manipulation for row 1 - row 0
signdwn1_16x8b = _mm_alignr_epi8(src_temp0_16x8b, signdwn1_16x8b, 15);
//combining the appropriate sign change
edge0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(0-1)
//row1-row0
//separating +ve and and -ve values.
cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, signdwn1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp1_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
// row = 2 right
src_bottom_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + 2 * src_strd + 1));
edge1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(1-0)
//row1 -bottom
cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, src_bottom_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, src_temp1_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//combining the appropriate sign change
signdwn1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b);
// row = 2
src_bottom_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + 2 * src_strd));
//combining sign-left and sign_right
edge0_16x8b = _mm_add_epi8(edge0_16x8b, signup0_16x8b);
//storing the row 1 left for next row.
signup0_16x8b = _mm_slli_si128(left_store_16x8b, 14);
//combining sign-left and sign_right
edge1_16x8b = _mm_add_epi8(edge1_16x8b, signdwn1_16x8b);
//manipulation for bottom - row 1
signup0_16x8b = _mm_alignr_epi8(src_temp1_16x8b, signup0_16x8b, 15);
//eliminating old left for row 0 and row 1
left_store_16x8b = _mm_srli_si128(left_store_16x8b, 2);
//bottom - row1
cmp_gt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, signup0_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(signup0_16x8b, src_bottom_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//for the next iteration bottom -row1
signup0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b);
//row1 getting it right for left of next block
left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, src_temp1_16x8b, 15);
//adding constant 2
edge0_16x8b = _mm_add_epi8(edge0_16x8b, const2_16x8b);
edge1_16x8b = _mm_add_epi8(edge1_16x8b, const2_16x8b);
//shuffle to get sao index
edge0_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge0_16x8b);
edge1_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge1_16x8b);
//using availability mask
edge0_16x8b = _mm_and_si128(edge0_16x8b, au1_mask8x16b);
edge1_16x8b = _mm_and_si128(edge1_16x8b, au1_mask8x16b);
//shuffle to get sao offset
edge0_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge0_16x8b);
edge1_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge1_16x8b);
//row0 getting it right for left of next block
left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, src_temp0_16x8b, 15);
//copying the next top
src_top_16x8b = src_temp1_16x8b;
//cnvert to 16 bit then add and then saturated pack
signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge0_16x8b);
cmp_gt0_16x8b = _mm_unpacklo_epi8(src_temp0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_unpacklo_epi8(edge0_16x8b, signdwn1_16x8b);
src_temp0_16x8b = _mm_unpackhi_epi8(src_temp0_16x8b, const0_16x8b);
cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b);
cmp_lt0_16x8b = _mm_unpackhi_epi8(edge0_16x8b, signdwn1_16x8b);
src_temp0_16x8b = _mm_add_epi16(src_temp0_16x8b, cmp_lt0_16x8b);
src_temp0_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_temp0_16x8b);
signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge1_16x8b);
cmp_gt0_16x8b = _mm_unpacklo_epi8(src_temp1_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_unpacklo_epi8(edge1_16x8b, signdwn1_16x8b);
src_temp1_16x8b = _mm_unpackhi_epi8(src_temp1_16x8b, const0_16x8b);
cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b);
cmp_lt0_16x8b = _mm_unpackhi_epi8(edge1_16x8b, signdwn1_16x8b);
src_temp1_16x8b = _mm_add_epi16(src_temp1_16x8b, cmp_lt0_16x8b);
src_temp1_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_temp1_16x8b);
//store left boundary
_mm_storel_epi64((__m128i *)(pu1_src_left_str), left_store_16x8b);
//row = 0 store 8 pixel values from 7:0 pos. relative to cur. pos.
_mm_storeu_si128((__m128i *)(pu1_src_cpy), src_temp0_16x8b);
// row = 1
_mm_storeu_si128((__m128i *)(pu1_src_cpy + src_strd), src_temp1_16x8b);
src_temp0_16x8b = src_bottom_16x8b;
pu1_src_cpy += (src_strd << 1);
pu1_src_left_cpy += 2;
pu1_src_left_str += 2;
}
ht_rem = ht & 0x1;
if(ht_rem)
{
left_store_16x8b = _mm_loadl_epi64((__m128i *)pu1_src_left_cpy);
src_bottom_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + src_strd + 1));
//current row -next row
//separating +ve and and -ve values.
cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, src_bottom_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, src_temp0_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//combining the appropriate sign change
edge0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b);
//adding top and botton and constant 2
edge0_16x8b = _mm_add_epi8(edge0_16x8b, signup0_16x8b);
edge0_16x8b = _mm_add_epi8(edge0_16x8b, const2_16x8b);
//eliminating old left for row 0 and row 1
left_store_16x8b = _mm_srli_si128(left_store_16x8b, 1);
edge0_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge0_16x8b);
//using availability mask
edge0_16x8b = _mm_and_si128(edge0_16x8b, au1_mask8x16b);
edge0_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge0_16x8b);
//row0 getting it right for left of next block
left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, src_temp0_16x8b, 15);
//copying the next top
src_top_16x8b = src_temp0_16x8b;
//cnvert to 16 bit then add and then saturated pack
signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge0_16x8b);
cmp_gt0_16x8b = _mm_unpacklo_epi8(src_temp0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_unpacklo_epi8(edge0_16x8b, signdwn1_16x8b);
src_temp0_16x8b = _mm_unpackhi_epi8(src_temp0_16x8b, const0_16x8b);
cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b);
cmp_lt0_16x8b = _mm_unpackhi_epi8(edge0_16x8b, signdwn1_16x8b);
src_temp0_16x8b = _mm_add_epi16(src_temp0_16x8b, cmp_lt0_16x8b);
src_temp0_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_temp0_16x8b);
//store left boundary
_mm_storel_epi64((__m128i *)(pu1_src_left_str), left_store_16x8b);
_mm_storeu_si128((__m128i *)(pu1_src_cpy), src_temp0_16x8b);
pu1_src_cpy += (src_strd);
pu1_src_left_cpy += 1;
pu1_src_left_str += 1;
}
if(0 == pu1_avail[3])
{
src_top_16x8b = src_bottom_16x8b;
pu1_src_left_str[0] = pu1_src_cpy[15];
}
if(0 == pu1_avail[2])
{
pu1_src_left_str[-ht_0] = pu1_src[15 - src_strd];
}
//for the top left of next part of the block
left_store_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_top_cpy + wd - col));
//updating top flag
_mm_storeu_si128((__m128i *)(pu1_src_top + wd - col), src_top_16x8b);
pu1_src += 16;
au1_mask_cpy += 16;
pu1_left_tmp = pu1_src_left_cpy2;
pu1_src_left_cpy2 = pu1_src_left_str2;
pu1_src_left_str2 = pu1_left_tmp;
pu1_src_left_cpy = pu1_src_left_cpy2;
pu1_src_left_str = pu1_src_left_str2;
}
wd_rem = wd & 0xF;
if(wd_rem)
{
pu1_src_left_cpy = pu1_src_left_cpy2;
pu1_src_left_str = pu1_src_left_str2;
pu1_src_cpy = pu1_src;
src_top_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_top_cpy + wd - col));
//row = 0
src_temp0_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy));
src_top_16x8b = _mm_alignr_epi8(src_top_16x8b, left_store_16x8b, 15);
au1_mask8x16b = _mm_loadl_epi64((__m128i *)au1_mask_cpy); //????
//separating +ve and and -ve values.
cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, src_top_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(src_top_16x8b, src_temp0_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//preparing au1_mask
au1_mask8x16b = _mm_unpacklo_epi64(au1_mask8x16b, au1_mask8x16b);
//combining the appropriate sign change
signup0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b);
signup0_16x8b = _mm_slli_si128(signup0_16x8b, 8);
for(row = ht; row >= 4; row -= 4)
{
left_store_16x8b = _mm_loadl_epi64((__m128i *)pu1_src_left_cpy);
//row = 1 load 8 pixel values from 7:0 pos. relative to cur. pos.
src_temp1_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + src_strd));
// row = 2
src_bottom_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + 2 * src_strd));
//right row1
signdwn1_16x8b = _mm_srli_si128(src_temp1_16x8b, 1);
//row 0 -row1
//separating +ve and and -ve values.
cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, signdwn1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp0_16x8b);
//manipulation for row 1 -row 0
signdwn1_16x8b = _mm_slli_si128(left_store_16x8b, 15);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//row 0 left
signdwn1_16x8b = _mm_alignr_epi8(src_temp0_16x8b, signdwn1_16x8b, 15);
//combining the appropriate sign change
edge0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b);
//row 1 -row0
//separating +ve and and -ve values.
cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, signdwn1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp1_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//row1-row0
edge1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b);
edge0_16x8b = _mm_slli_si128(edge0_16x8b, 8); //aligned left (0-1)
signup0_16x8b = _mm_alignr_epi8(edge1_16x8b, signup0_16x8b, 8); //(1-0),(0-top)
//right row2
signdwn1_16x8b = _mm_srli_si128(src_bottom_16x8b, 1);
//packing row 0 n row 1
src_temp0_16x8b = _mm_unpacklo_epi64(src_temp0_16x8b, src_temp1_16x8b);
//row1 -row2
cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, signdwn1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp1_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//combining the appropriate sign change
signdwn1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(1-2)
edge0_16x8b = _mm_alignr_epi8(signdwn1_16x8b, edge0_16x8b, 8); //(1-2),(0-1)
//manipulation for row 2 -row 1
signdwn1_16x8b = _mm_slli_si128(left_store_16x8b, 14);
//row 1 left
signdwn1_16x8b = _mm_alignr_epi8(src_temp1_16x8b, signdwn1_16x8b, 15);
//row = 3
src_top_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + 3 * src_strd));
// row = 4
src_temp1_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + 4 * src_strd));
edge0_16x8b = _mm_add_epi8(edge0_16x8b, signup0_16x8b); //(1,0) sign_up empty
//separating +ve and and -ve values.(2,1)
cmp_gt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, signdwn1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_bottom_16x8b);
//manipulation for row 3 -row 2
signdwn1_16x8b = _mm_slli_si128(left_store_16x8b, 13);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//row 2 left
signdwn1_16x8b = _mm_alignr_epi8(src_bottom_16x8b, signdwn1_16x8b, 15);
//combining the appropriate sign change
signup0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(2-1)
//separating +ve and and -ve values.(3,2)
cmp_gt0_16x8b = _mm_subs_epu8(src_top_16x8b, signdwn1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_top_16x8b);
signup0_16x8b = _mm_slli_si128(signup0_16x8b, 8); //aligned left (2-1)
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//right row3
signdwn1_16x8b = _mm_srli_si128(src_top_16x8b, 1);
//combining the appropriate sign change
edge1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(3-2)
signup0_16x8b = _mm_alignr_epi8(edge1_16x8b, signup0_16x8b, 8); //(3-2) ,(2-1)
//separating +ve and and -ve values.(2,3)
cmp_gt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, signdwn1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_bottom_16x8b);
//right row 4
signdwn1_16x8b = _mm_srli_si128(src_temp1_16x8b, 1);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//combining the appropriate sign change
edge1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(2-3)
//separating +ve and and -ve values.(3,bottom)
cmp_gt0_16x8b = _mm_subs_epu8(src_top_16x8b, signdwn1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_top_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
edge1_16x8b = _mm_slli_si128(edge1_16x8b, 8); //aligned left (2-3)
//combining the appropriate sign change
signdwn1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(3-bottom)
edge1_16x8b = _mm_alignr_epi8(signdwn1_16x8b, edge1_16x8b, 8); //(3-bottom),(2-3)
//manipulation for bottom -row 3
signdwn1_16x8b = _mm_slli_si128(left_store_16x8b, 12);
//eliminating old left for row 0,1,2,3
left_store_16x8b = _mm_srli_si128(left_store_16x8b, 4);
//packing row 2 n row 3
src_bottom_16x8b = _mm_unpacklo_epi64(src_bottom_16x8b, src_top_16x8b);
//row 3 left
signdwn1_16x8b = _mm_alignr_epi8(src_top_16x8b, signdwn1_16x8b, 15);
//loading row 3 right into left
left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, src_bottom_16x8b, 15);
//adding bottom and top values of row 2 and row 3
edge1_16x8b = _mm_add_epi8(edge1_16x8b, signup0_16x8b); //(3,2)
//separating +ve and and -ve values.(botttom,3)
cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, signdwn1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp1_16x8b);
//to store right of row 2
signdwn1_16x8b = _mm_slli_si128(src_bottom_16x8b, 8);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
signup0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(bottom -3) for next iteration
//storing right of row 2into left
left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, signdwn1_16x8b, 15);
//to store right of row 0
signdwn1_16x8b = _mm_slli_si128(src_temp0_16x8b, 8);
//storing right of row 1 into left
left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, src_temp0_16x8b, 15);
//adding constant 2
edge0_16x8b = _mm_add_epi8(edge0_16x8b, const2_16x8b);
edge1_16x8b = _mm_add_epi8(edge1_16x8b, const2_16x8b);
//shuffle to get sao index
edge0_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge0_16x8b);
edge1_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge1_16x8b);
//using availability mask
edge0_16x8b = _mm_and_si128(edge0_16x8b, au1_mask8x16b);
edge1_16x8b = _mm_and_si128(edge1_16x8b, au1_mask8x16b);
//shuffle to get sao offset
edge0_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge0_16x8b);
edge1_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge1_16x8b);
//storing right of row 0 into left
left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, signdwn1_16x8b, 15);
//cnvert to 16 bit then add and then saturated pack
signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge0_16x8b);
cmp_gt0_16x8b = _mm_unpacklo_epi8(src_temp0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_unpacklo_epi8(edge0_16x8b, signdwn1_16x8b);
src_temp0_16x8b = _mm_unpackhi_epi8(src_temp0_16x8b, const0_16x8b);
cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b);
cmp_lt0_16x8b = _mm_unpackhi_epi8(edge0_16x8b, signdwn1_16x8b);
src_temp0_16x8b = _mm_add_epi16(src_temp0_16x8b, cmp_lt0_16x8b);
src_temp0_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_temp0_16x8b);
signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge1_16x8b);
cmp_gt0_16x8b = _mm_unpacklo_epi8(src_bottom_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_unpacklo_epi8(edge1_16x8b, signdwn1_16x8b);
src_bottom_16x8b = _mm_unpackhi_epi8(src_bottom_16x8b, const0_16x8b);
cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b);
cmp_lt0_16x8b = _mm_unpackhi_epi8(edge1_16x8b, signdwn1_16x8b);
src_bottom_16x8b = _mm_add_epi16(src_bottom_16x8b, cmp_lt0_16x8b);
src_bottom_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_bottom_16x8b);
cmp_gt0_16x8b = _mm_srli_si128(src_temp0_16x8b, 8);
cmp_lt0_16x8b = _mm_srli_si128(src_bottom_16x8b, 8);
_mm_storel_epi64((__m128i *)(pu1_src_left_str), left_store_16x8b);
//row = 0 store 8 pixel values from 7:0 pos. relative to cur. pos.
_mm_storel_epi64((__m128i *)(pu1_src_cpy), src_temp0_16x8b);
// row = 1
_mm_storel_epi64((__m128i *)(pu1_src_cpy + src_strd), cmp_gt0_16x8b);
//row = 2
_mm_storel_epi64((__m128i *)(pu1_src_cpy + 2 * src_strd), src_bottom_16x8b);
// row = 3
_mm_storel_epi64((__m128i *)(pu1_src_cpy + 3 * src_strd), cmp_lt0_16x8b);
src_temp0_16x8b = src_temp1_16x8b;
signup0_16x8b = _mm_slli_si128(signup0_16x8b, 8);
pu1_src_cpy += (src_strd << 2);
pu1_src_left_cpy += 4;
pu1_src_left_str += 4;
}
ht_rem = ht & 0x2;
if(ht_rem)
{
left_store_16x8b = _mm_loadl_epi64((__m128i *)pu1_src_left_cpy);
//row = 1 load 8 pixel values from 7:0 pos. relative to cur. pos.
src_temp1_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + src_strd));
// row = 2
src_bottom_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + 2 * src_strd));
//row 0 -row 1
signdwn1_16x8b = _mm_srli_si128(src_temp1_16x8b, 1);
//separating +ve and and -ve values.
cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, signdwn1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp0_16x8b);
//manipulation for row 1 -row 0
signdwn1_16x8b = _mm_slli_si128(left_store_16x8b, 15);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//manipulation for row 1 - row 0
signdwn1_16x8b = _mm_alignr_epi8(src_temp0_16x8b, signdwn1_16x8b, 15);
//combining the appropriate sign change
edge0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b);
//row1-row0
//separating +ve and and -ve values.
cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, signdwn1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp1_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//combining the appropriate sign chang
edge1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b);
//row 1 -bottom
signdwn1_16x8b = _mm_srli_si128(src_bottom_16x8b, 1);
edge0_16x8b = _mm_slli_si128(edge0_16x8b, 8); //aligned left (0-1)
signup0_16x8b = _mm_alignr_epi8(edge1_16x8b, signup0_16x8b, 8); //(1-0),(0-top)
//row1 -bottom
cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, signdwn1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp1_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//combining the appropriate sign change
signdwn1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(1-2)
edge0_16x8b = _mm_alignr_epi8(signdwn1_16x8b, edge0_16x8b, 8); //(1-2),(0-1)
//manipulation for bottom -row1
signdwn1_16x8b = _mm_slli_si128(left_store_16x8b, 14);
//manipulation for bottom- row 1
signdwn1_16x8b = _mm_alignr_epi8(src_temp1_16x8b, signdwn1_16x8b, 15);
//adding top and down substraction
edge0_16x8b = _mm_add_epi8(edge0_16x8b, signup0_16x8b); //(1,0) sign_up empty
//bottom - row 1
cmp_gt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, signdwn1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_bottom_16x8b);
//eliminating old left for row 0,1
left_store_16x8b = _mm_srli_si128(left_store_16x8b, 2);
signdwn1_16x8b = _mm_slli_si128(src_temp1_16x8b, 8);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//for the next iteration signup0_16x8b
signup0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(2-1) for next
//storing right of row 1 into left
left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, signdwn1_16x8b, 15);
//for storing right of row 1
signdwn1_16x8b = _mm_slli_si128(src_temp0_16x8b, 8);
src_top_16x8b = src_temp1_16x8b;
//storing right of row 0 into left
left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, signdwn1_16x8b, 15);
//adding constant 2
edge0_16x8b = _mm_add_epi8(edge0_16x8b, const2_16x8b);
//shuffle to get sao index
edge0_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge0_16x8b);
//using availability mask
edge0_16x8b = _mm_and_si128(edge0_16x8b, au1_mask8x16b);
//shuffle to get sao offset
edge0_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge0_16x8b);
//the next top already in src_top_16x8b
//cnvert to 16 bit then add and then saturated pack
signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge0_16x8b);
src_temp0_16x8b = _mm_unpacklo_epi8(src_temp0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_unpacklo_epi8(edge0_16x8b, signdwn1_16x8b);
src_temp1_16x8b = _mm_unpacklo_epi8(src_temp1_16x8b, const0_16x8b);
src_temp0_16x8b = _mm_add_epi16(src_temp0_16x8b, cmp_lt0_16x8b);
cmp_lt0_16x8b = _mm_unpackhi_epi8(edge0_16x8b, signdwn1_16x8b);
src_temp1_16x8b = _mm_add_epi16(src_temp1_16x8b, cmp_lt0_16x8b);
src_temp0_16x8b = _mm_packus_epi16(src_temp0_16x8b, src_temp1_16x8b);
cmp_gt0_16x8b = _mm_srli_si128(src_temp0_16x8b, 8);
_mm_storel_epi64((__m128i *)(pu1_src_left_str), left_store_16x8b);
//row = 0 store 8 pixel values from 7:0 pos. relative to cur. pos.
_mm_storel_epi64((__m128i *)(pu1_src_cpy), src_temp0_16x8b);
// row = 1
_mm_storel_epi64((__m128i *)(pu1_src_cpy + src_strd), cmp_gt0_16x8b);
src_temp0_16x8b = src_bottom_16x8b;
pu1_src_cpy += (src_strd << 1);
pu1_src_left_cpy += 2;
pu1_src_left_str += 2;
}
ht_rem = ht & 0x1;
if(ht_rem)
{
left_store_16x8b = _mm_loadl_epi64((__m128i *)pu1_src_left_cpy);
//row = 1 load 8 pixel values from 7:0 pos. relative to cur. pos.
src_bottom_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + src_strd));
//left store manipulation 1
left_store_16x8b = _mm_srli_si128(left_store_16x8b, 1);
//row 0 -row1
signdwn1_16x8b = _mm_srli_si128(src_bottom_16x8b, 1);
//separating +ve and and -ve values.
cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, signdwn1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp0_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//combining the appropriate sign change
edge0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b);
//adding top and down substraction
edge0_16x8b = _mm_add_epi8(edge0_16x8b, signup0_16x8b);
//for row 0 right to put into left store
signdwn1_16x8b = _mm_slli_si128(src_temp0_16x8b, 8);
//adding constant 2
edge0_16x8b = _mm_add_epi8(edge0_16x8b, const2_16x8b);
edge0_16x8b = _mm_slli_si128(edge0_16x8b, 8);
edge0_16x8b = _mm_srli_si128(edge0_16x8b, 8);
//filling the left boundary value
left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, signdwn1_16x8b, 15);
//shuffle to get sao index
edge0_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge0_16x8b);
//using availability mask
edge0_16x8b = _mm_and_si128(edge0_16x8b, au1_mask8x16b);
//shuffle to get sao offset
edge0_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge0_16x8b);
src_top_16x8b = src_temp0_16x8b;
//cnvert to 16 bit then add and then saturated pack
signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge0_16x8b);
src_temp0_16x8b = _mm_unpacklo_epi8(src_temp0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_unpacklo_epi8(edge0_16x8b, signdwn1_16x8b);
src_temp0_16x8b = _mm_add_epi16(src_temp0_16x8b, cmp_lt0_16x8b);
src_temp0_16x8b = _mm_packus_epi16(src_temp0_16x8b, const0_16x8b);
_mm_storel_epi64((__m128i *)(pu1_src_left_str), left_store_16x8b);
//row = 0 store 8 pixel values from 7:0 pos. relative to cur. pos.
_mm_storel_epi64((__m128i *)(pu1_src_cpy), src_temp0_16x8b);
pu1_src_cpy += (src_strd);
pu1_src_left_cpy += 1;
pu1_src_left_str += 1;
}
if(0 == pu1_avail[3])
{
src_top_16x8b = src_bottom_16x8b;
pu1_src_left_str[0] = pu1_src_cpy[7];
}
if(0 == pu1_avail[2])
{
pu1_src_left_str[-ht_0] = pu1_src[7 - src_strd];
}
_mm_storel_epi64((__m128i *)(pu1_src_top + wd - col), src_top_16x8b);
pu1_src += 8;
au1_mask_cpy += 16;
pu1_left_tmp = pu1_src_left_cpy2;
pu1_src_left_cpy2 = pu1_src_left_str2;
pu1_src_left_str2 = pu1_left_tmp;
pu1_src_left_cpy = pu1_src_left_cpy2;
pu1_src_left_str = pu1_src_left_str2;
}
pu1_src_org[0] = u1_pos_0_0_tmp;
pu1_src_org[wd - 1 + (ht_tmp - 1) * src_strd] = u1_pos_wd_ht_tmp;
pu1_src_left_cpy = (0 == pu1_avail[2]) ? (pu1_src_left_cpy - 1) : pu1_src_left_cpy;
for(row = 0; row < ht_tmp; row++)
{
pu1_src_left[row] = pu1_src_left_cpy[row];
}
}
}
/* 135 degree filtering */
void ihevc_sao_edge_offset_class2_chroma_ssse3(UWORD8 *pu1_src,
WORD32 src_strd,
UWORD8 *pu1_src_left,
UWORD8 *pu1_src_top,
UWORD8 *pu1_src_top_left,
UWORD8 *pu1_src_top_right,
UWORD8 *pu1_src_bot_left,
UWORD8 *pu1_avail,
WORD8 *pi1_sao_offset_u,
WORD8 *pi1_sao_offset_v,
WORD32 wd,
WORD32 ht)
{
WORD32 row, col;
UWORD8 *pu1_src_top_cpy, *pu1_src_left_cpy, *pu1_src_left_cpy2;
UWORD8 *pu1_left_tmp, *pu1_src_left_str, *pu1_src_left_str2;
UWORD8 *pu1_firstleft;
UWORD8 *pu1_src_cpy, *pu1_src_org;
UWORD8 au1_mask[MAX_CTB_SIZE], *au1_mask_cpy;
UWORD8 au1_src_left_tmp[2 * (MAX_CTB_SIZE + 8)];
UWORD8 au1_src_left_tmp1[2 * (MAX_CTB_SIZE + 8)];
WORD32 wd_rem;
UWORD8 u1_pos_0_0_tmp_u, u1_pos_0_0_tmp_v, u1_pos_wd_ht_tmp_u, u1_pos_wd_ht_tmp_v;
WORD32 ht_tmp;
WORD32 ht_0;
WORD32 bit_depth;
UWORD8 u1_avail0, u1_avail1;
__m128i src_temp0_16x8b, src_temp1_16x8b;
__m128i signup0_16x8b, signdwn1_16x8b;
__m128i cmp_gt0_16x8b, cmp_lt0_16x8b;
__m128i edge0_16x8b, edge1_16x8b;
__m128i src_top_16x8b, src_bottom_16x8b;
__m128i au1_mask8x16b;
__m128i edge_idx_8x16b, sao_offset_8x16b;
__m128i const2_16x8b, const0_16x8b;
__m128i left_store_16x8b;
__m128i chroma_offset_8x16b;
UNUSED(pu1_src_top_right);
UNUSED(pu1_src_bot_left);
ht_0 = ht; ht_tmp = ht;
au1_mask8x16b = _mm_set1_epi8(0xff);
/* Updating left and top-left */
for(row = 0; row < 2 * ht; row++)
{
au1_src_left_tmp[row] = pu1_src_left[row];
}
//setting availability mask to ff size MAX_CTB_SIZE
for(col = 0; col < MAX_CTB_SIZE; col += 16)
_mm_storeu_si128((__m128i *)(au1_mask + col), au1_mask8x16b);
bit_depth = BIT_DEPTH_LUMA;
pu1_src_org = pu1_src;
pu1_src_top_cpy = pu1_src_top;
pu1_src_left_cpy2 = au1_src_left_tmp;
pu1_src_left_cpy = au1_src_left_tmp;
pu1_src_left_str2 = au1_src_left_tmp1;
pu1_src_left_str = au1_src_left_tmp1;
edge_idx_8x16b = _mm_loadl_epi64((__m128i *)gi1_table_edge_idx);
sao_offset_8x16b = _mm_loadl_epi64((__m128i *)pi1_sao_offset_u);
const0_16x8b = _mm_loadl_epi64((__m128i *)pi1_sao_offset_v);
chroma_offset_8x16b = _mm_set1_epi16(0x0800);
/* If top-left is available, process separately */
if(0 != pu1_avail[4])
{
WORD32 edge_idx;
/* U */
edge_idx = 2 + SIGN(pu1_src[0] - pu1_src_top_left[0]) +
SIGN(pu1_src[0] - pu1_src[2 + src_strd]);
edge_idx = gi1_table_edge_idx[edge_idx];
if(0 != edge_idx)
{
u1_pos_0_0_tmp_u = CLIP3(pu1_src[0] + pi1_sao_offset_u[edge_idx], 0, (1 << bit_depth) - 1);
}
else
{
u1_pos_0_0_tmp_u = pu1_src[0];
}
/* V */
edge_idx = 2 + SIGN(pu1_src[1] - pu1_src_top_left[1]) +
SIGN(pu1_src[1] - pu1_src[1 + 2 + src_strd]);
edge_idx = gi1_table_edge_idx[edge_idx];
if(0 != edge_idx)
{
u1_pos_0_0_tmp_v = CLIP3(pu1_src[1] + pi1_sao_offset_v[edge_idx], 0, (1 << bit_depth) - 1);
}
else
{
u1_pos_0_0_tmp_v = pu1_src[1];
}
}
else
{
u1_pos_0_0_tmp_u = pu1_src[0];
u1_pos_0_0_tmp_v = pu1_src[1];
}
/* If bottom-right is available, process separately */
if(0 != pu1_avail[7])
{
WORD32 edge_idx;
/* U */
edge_idx = 2 + SIGN(pu1_src[wd - 2 + (ht - 1) * src_strd] - pu1_src[wd - 2 + (ht - 1) * src_strd - 2 - src_strd]) +
SIGN(pu1_src[wd - 2 + (ht - 1) * src_strd] - pu1_src[wd - 2 + (ht - 1) * src_strd + 2 + src_strd]);
edge_idx = gi1_table_edge_idx[edge_idx];
if(0 != edge_idx)
{
u1_pos_wd_ht_tmp_u = CLIP3(pu1_src[wd - 2 + (ht - 1) * src_strd] + pi1_sao_offset_u[edge_idx], 0, (1 << bit_depth) - 1);
}
else
{
u1_pos_wd_ht_tmp_u = pu1_src[wd - 2 + (ht - 1) * src_strd];
}
/* V */
edge_idx = 2 + SIGN(pu1_src[wd - 1 + (ht - 1) * src_strd] - pu1_src[wd - 1 + (ht - 1) * src_strd - 2 - src_strd]) +
SIGN(pu1_src[wd - 1 + (ht - 1) * src_strd] - pu1_src[wd - 1 + (ht - 1) * src_strd + 2 + src_strd]);
edge_idx = gi1_table_edge_idx[edge_idx];
if(0 != edge_idx)
{
u1_pos_wd_ht_tmp_v = CLIP3(pu1_src[wd - 1 + (ht - 1) * src_strd] + pi1_sao_offset_v[edge_idx], 0, (1 << bit_depth) - 1);
}
else
{
u1_pos_wd_ht_tmp_v = pu1_src[wd - 1 + (ht - 1) * src_strd];
}
}
else
{
u1_pos_wd_ht_tmp_u = pu1_src[wd - 2 + (ht - 1) * src_strd];
u1_pos_wd_ht_tmp_v = pu1_src[wd - 1 + (ht - 1) * src_strd];
}
pu1_firstleft = pu1_src_top_left;
/* Update height and source pointers based on the availability flags */
if(0 == pu1_avail[2])
{
pu1_firstleft = pu1_src_left_cpy2;
pu1_src_left_cpy2 += 2;
pu1_src_left_str2 += 2;
pu1_src_top_cpy = pu1_src;
pu1_src += src_strd;
ht--;
}
if(0 == pu1_avail[3])
{
ht--;
ht_0--;
}
//storing top left in a mmx register
left_store_16x8b = _mm_loadl_epi64((__m128i *)pu1_firstleft);
sao_offset_8x16b = _mm_unpacklo_epi64(sao_offset_8x16b, const0_16x8b);
const2_16x8b = _mm_set1_epi8(2);
const0_16x8b = _mm_setzero_si128();
left_store_16x8b = _mm_slli_si128(left_store_16x8b, 14);
//availability mask creation
u1_avail0 = pu1_avail[0];
u1_avail1 = pu1_avail[1];
au1_mask[0] = u1_avail0;
au1_mask[1] = u1_avail0;
au1_mask[wd - 1] = u1_avail1;
au1_mask[wd - 2] = u1_avail1;
/* top-left arrays */
pu1_src_top_left[0] = pu1_src_top[wd - 2];
pu1_src_top_left[1] = pu1_src_top[wd - 1];
{
WORD32 ht_rem;
au1_mask_cpy = au1_mask;
pu1_src_left_cpy = pu1_src_left_cpy2;
pu1_src_left_str = pu1_src_left_str2;
for(col = wd; col >= 16; col -= 16)
{
pu1_src_cpy = pu1_src;
src_top_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_top_cpy + wd - col));
//row = 0
src_temp0_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy));
src_top_16x8b = _mm_alignr_epi8(src_top_16x8b, left_store_16x8b, 14);
//loading the mask
au1_mask8x16b = _mm_loadu_si128((__m128i *)au1_mask_cpy);
//separating +ve and and -ve values.
cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, src_top_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(src_top_16x8b, src_temp0_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//combining the appropriate sign change
signup0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b);
for(row = ht; row >= 2; row -= 2)
{
left_store_16x8b = _mm_loadl_epi64((__m128i *)pu1_src_left_cpy);
//row = 1
src_temp1_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + src_strd));
// row = 1 right
src_bottom_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + src_strd + 2));
//to insert left in row 0
signdwn1_16x8b = _mm_slli_si128(left_store_16x8b, 14);
//row 0 -row1
//separating +ve and and -ve values.
cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, src_bottom_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, src_temp0_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//manipulation for row 1 - row 0
signdwn1_16x8b = _mm_alignr_epi8(src_temp0_16x8b, signdwn1_16x8b, 14);
//combining the appropriate sign change
edge0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(0-1)
//row1-row0
//separating +ve and and -ve values.
cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, signdwn1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp1_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
// row = 2 right
src_bottom_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + 2 * src_strd + 2));
edge1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(1-0)
//row1 -bottom
cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, src_bottom_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, src_temp1_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//combining the appropriate sign change
signdwn1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b);
// row = 2
src_bottom_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + 2 * src_strd));
//combining sign-left and sign_right
edge0_16x8b = _mm_add_epi8(edge0_16x8b, signup0_16x8b);
//storing the row 1 left for next row.
signup0_16x8b = _mm_slli_si128(left_store_16x8b, 12);
//combining sign-left and sign_right
edge1_16x8b = _mm_add_epi8(edge1_16x8b, signdwn1_16x8b);
//manipulation for bottom - row 1
signup0_16x8b = _mm_alignr_epi8(src_temp1_16x8b, signup0_16x8b, 14);
//eliminating old left for row 0 and row 1
left_store_16x8b = _mm_srli_si128(left_store_16x8b, 4);
//bottom - row1
cmp_gt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, signup0_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(signup0_16x8b, src_bottom_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//for the next iteration bottom -row1
signup0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b);
//row1 getting it right for left of next iteration
left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, src_temp1_16x8b, 14);
//copying the next top
src_top_16x8b = src_temp1_16x8b;
//row0 getting its right for left of next iteration.
left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, src_temp0_16x8b, 14);
//adding constant 2
edge0_16x8b = _mm_add_epi8(edge0_16x8b, const2_16x8b);
edge1_16x8b = _mm_add_epi8(edge1_16x8b, const2_16x8b);
//shuffle to get sao index
edge0_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge0_16x8b);
edge1_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge1_16x8b);
//using availability mask
edge0_16x8b = _mm_and_si128(edge0_16x8b, au1_mask8x16b);
edge1_16x8b = _mm_and_si128(edge1_16x8b, au1_mask8x16b);
//adding chroma offset to access U and V
edge0_16x8b = _mm_add_epi8(edge0_16x8b, chroma_offset_8x16b);
edge1_16x8b = _mm_add_epi8(edge1_16x8b, chroma_offset_8x16b);
//shuffle to get sao offset
edge0_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge0_16x8b);
edge1_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge1_16x8b);
//cnvert to 16 bit then add and then saturated pack
signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge0_16x8b);
cmp_gt0_16x8b = _mm_unpacklo_epi8(src_temp0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_unpacklo_epi8(edge0_16x8b, signdwn1_16x8b);
src_temp0_16x8b = _mm_unpackhi_epi8(src_temp0_16x8b, const0_16x8b);
edge0_16x8b = _mm_unpackhi_epi8(edge0_16x8b, signdwn1_16x8b);
cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b);
src_temp0_16x8b = _mm_add_epi16(src_temp0_16x8b, edge0_16x8b);
src_temp0_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_temp0_16x8b);
signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge1_16x8b);
cmp_gt0_16x8b = _mm_unpacklo_epi8(src_temp1_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_unpacklo_epi8(edge1_16x8b, signdwn1_16x8b);
src_temp1_16x8b = _mm_unpackhi_epi8(src_temp1_16x8b, const0_16x8b);
edge1_16x8b = _mm_unpackhi_epi8(edge1_16x8b, signdwn1_16x8b);
cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b);
src_temp1_16x8b = _mm_add_epi16(src_temp1_16x8b, edge1_16x8b);
src_temp1_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_temp1_16x8b);
//store left boundary
_mm_storel_epi64((__m128i *)(pu1_src_left_str), left_store_16x8b);
//row = 0 store 8 pixel values from 7:0 pos. relative to cur. pos.
_mm_storeu_si128((__m128i *)(pu1_src_cpy), src_temp0_16x8b);
// row = 1
_mm_storeu_si128((__m128i *)(pu1_src_cpy + src_strd), src_temp1_16x8b);
src_temp0_16x8b = src_bottom_16x8b;
pu1_src_cpy += (src_strd << 1);
pu1_src_left_cpy += 4;
pu1_src_left_str += 4;
}
ht_rem = ht & 0x1;
if(ht_rem)
{
left_store_16x8b = _mm_loadl_epi64((__m128i *)pu1_src_left_cpy);
src_bottom_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + src_strd + 2));
//current row -next row
//separating +ve and and -ve values.
cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, src_bottom_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, src_temp0_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//combining the appropriate sign change
edge0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b);
//adding top and botton and constant 2
edge0_16x8b = _mm_add_epi8(edge0_16x8b, signup0_16x8b);
edge0_16x8b = _mm_add_epi8(edge0_16x8b, const2_16x8b);
//eliminating old left for row 0 and row 1
left_store_16x8b = _mm_srli_si128(left_store_16x8b, 2);
//copying the next top
src_top_16x8b = src_temp0_16x8b;
//row0 getting it right for left of next block
left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, src_temp0_16x8b, 14);
edge0_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge0_16x8b);
//using availability mask
edge0_16x8b = _mm_and_si128(edge0_16x8b, au1_mask8x16b);
//adding chroma offset to access U and V
edge0_16x8b = _mm_add_epi8(edge0_16x8b, chroma_offset_8x16b);
edge0_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge0_16x8b);
//cnvert to 16 bit then add and then saturated pack
signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge0_16x8b);
cmp_gt0_16x8b = _mm_unpacklo_epi8(src_temp0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_unpacklo_epi8(edge0_16x8b, signdwn1_16x8b);
src_temp0_16x8b = _mm_unpackhi_epi8(src_temp0_16x8b, const0_16x8b);
edge0_16x8b = _mm_unpackhi_epi8(edge0_16x8b, signdwn1_16x8b);
cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b);
src_temp0_16x8b = _mm_add_epi16(src_temp0_16x8b, edge0_16x8b);
src_temp0_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_temp0_16x8b);
_mm_storel_epi64((__m128i *)(pu1_src_left_str), left_store_16x8b);
_mm_storeu_si128((__m128i *)(pu1_src_cpy), src_temp0_16x8b);
pu1_src_cpy += (src_strd);
pu1_src_left_cpy += 2;
pu1_src_left_str += 2;
}
if(0 == pu1_avail[3])
{
src_top_16x8b = src_bottom_16x8b;
pu1_src_left_str[1] = pu1_src_cpy[15];
pu1_src_left_str[0] = pu1_src_cpy[14];
}
if(0 == pu1_avail[2])
{
pu1_src_left_str[-2 * ht_0] = pu1_src[14 - src_strd];
pu1_src_left_str[-2 * ht_0 + 1] = pu1_src[15 - src_strd];
}
//for the top left of next part of the block
left_store_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_top_cpy + wd - col));
//updating top flag
_mm_storeu_si128((__m128i *)(pu1_src_top + wd - col), src_top_16x8b);
pu1_src += 16;
au1_mask_cpy += 16;
pu1_left_tmp = pu1_src_left_cpy2;
pu1_src_left_cpy2 = pu1_src_left_str2;
pu1_src_left_str2 = pu1_left_tmp;
pu1_src_left_cpy = pu1_src_left_cpy2;
pu1_src_left_str = pu1_src_left_str2;
}
wd_rem = wd & 0xF;
if(wd_rem)
{
pu1_src_left_cpy = pu1_src_left_cpy2;
pu1_src_left_str = pu1_src_left_str2;
pu1_src_cpy = pu1_src;
src_top_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_top_cpy + wd - col));
//row = 0
src_temp0_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy));
src_top_16x8b = _mm_alignr_epi8(src_top_16x8b, left_store_16x8b, 14);
au1_mask8x16b = _mm_loadl_epi64((__m128i *)au1_mask_cpy); //????
//separating +ve and and -ve values.
cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, src_top_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(src_top_16x8b, src_temp0_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//preparing au1_mask
au1_mask8x16b = _mm_unpacklo_epi64(au1_mask8x16b, au1_mask8x16b);
//combining the appropriate sign change
signup0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b);
signup0_16x8b = _mm_slli_si128(signup0_16x8b, 8);
for(row = ht; row >= 4; row -= 4)
{
left_store_16x8b = _mm_loadl_epi64((__m128i *)pu1_src_left_cpy);
//row = 1 load 8 pixel values from 7:0 pos. relative to cur. pos.
src_temp1_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + src_strd));
// row = 2
src_bottom_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + 2 * src_strd));
//right row1
signdwn1_16x8b = _mm_srli_si128(src_temp1_16x8b, 2);
//row 0 -row1
//separating +ve and and -ve values.
cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, signdwn1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp0_16x8b);
//manipulation for row 1 -row 0
signdwn1_16x8b = _mm_slli_si128(left_store_16x8b, 14);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//row 0 left
signdwn1_16x8b = _mm_alignr_epi8(src_temp0_16x8b, signdwn1_16x8b, 14);
//combining the appropriate sign change
edge0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b);
//row 1 -row0
//separating +ve and and -ve values.
cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, signdwn1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp1_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//row1-row0
edge1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b);
edge0_16x8b = _mm_slli_si128(edge0_16x8b, 8); //aligned left (0-1)
signup0_16x8b = _mm_alignr_epi8(edge1_16x8b, signup0_16x8b, 8); //(1-0),(0-top)
//right row2
signdwn1_16x8b = _mm_srli_si128(src_bottom_16x8b, 2);
//packing row 0 n row 1
src_temp0_16x8b = _mm_unpacklo_epi64(src_temp0_16x8b, src_temp1_16x8b);
//row1 -row2
cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, signdwn1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp1_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//combining the appropriate sign change
signdwn1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(1-2)
edge0_16x8b = _mm_alignr_epi8(signdwn1_16x8b, edge0_16x8b, 8); //(1-2),(0-1)
//manipulation for row 2 -row 1
signdwn1_16x8b = _mm_slli_si128(left_store_16x8b, 12);
//row 1 left
signdwn1_16x8b = _mm_alignr_epi8(src_temp1_16x8b, signdwn1_16x8b, 14);
//row = 3
src_top_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + 3 * src_strd));
// row = 4
src_temp1_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + 4 * src_strd));
edge0_16x8b = _mm_add_epi8(edge0_16x8b, signup0_16x8b); //(1,0) sign_up empty
//separating +ve and and -ve values.(2,1)
cmp_gt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, signdwn1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_bottom_16x8b);
//manipulation for row 3 -row 2
signdwn1_16x8b = _mm_slli_si128(left_store_16x8b, 10);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//row 2 left
signdwn1_16x8b = _mm_alignr_epi8(src_bottom_16x8b, signdwn1_16x8b, 14);
//combining the appropriate sign change
signup0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(2-1)
//separating +ve and and -ve values.(3,2)
cmp_gt0_16x8b = _mm_subs_epu8(src_top_16x8b, signdwn1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_top_16x8b);
signup0_16x8b = _mm_slli_si128(signup0_16x8b, 8); //aligned left (2-1)
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//right row3
signdwn1_16x8b = _mm_srli_si128(src_top_16x8b, 2);
//combining the appropriate sign change
edge1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(3-2)
signup0_16x8b = _mm_alignr_epi8(edge1_16x8b, signup0_16x8b, 8); //(3-2) ,(2-1)
//separating +ve and and -ve values.(2,3)
cmp_gt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, signdwn1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_bottom_16x8b);
//right row 4
signdwn1_16x8b = _mm_srli_si128(src_temp1_16x8b, 2);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//combining the appropriate sign change
edge1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(2-3)
//separating +ve and and -ve values.(3,bottom)
cmp_gt0_16x8b = _mm_subs_epu8(src_top_16x8b, signdwn1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_top_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
edge1_16x8b = _mm_slli_si128(edge1_16x8b, 8); //aligned left (2-3)
//combining the appropriate sign change
signdwn1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(3-bottom)
edge1_16x8b = _mm_alignr_epi8(signdwn1_16x8b, edge1_16x8b, 8); //(3-bottom),(2-3)
//manipulation for bottom -row 3
signdwn1_16x8b = _mm_slli_si128(left_store_16x8b, 8);
//eliminating old left for row 0,1,2,3
left_store_16x8b = _mm_srli_si128(left_store_16x8b, 8);
//packing row 2 n row 3
src_bottom_16x8b = _mm_unpacklo_epi64(src_bottom_16x8b, src_top_16x8b);
//row 3 left
signdwn1_16x8b = _mm_alignr_epi8(src_top_16x8b, signdwn1_16x8b, 14);
//adding bottom and top values of row 2 and row 3
edge1_16x8b = _mm_add_epi8(edge1_16x8b, signup0_16x8b); //(3,2)
//separating +ve and and -ve values.(botttom,3)
cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, signdwn1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp1_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
signup0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(bottom -3) for next iteration
//to store right of row 2
signdwn1_16x8b = _mm_slli_si128(src_bottom_16x8b, 8);
//loading row 3 right into left
left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, src_bottom_16x8b, 14);
//storing right of row 2into left
left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, signdwn1_16x8b, 14);
//to store right of row 0
signdwn1_16x8b = _mm_slli_si128(src_temp0_16x8b, 8);
//storing right of row 1 into left
left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, src_temp0_16x8b, 14);
//storing right of row 0 into left
left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, signdwn1_16x8b, 14);
//adding constant 2
edge0_16x8b = _mm_add_epi8(edge0_16x8b, const2_16x8b);
edge1_16x8b = _mm_add_epi8(edge1_16x8b, const2_16x8b);
//shuffle to get sao index
edge0_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge0_16x8b);
edge1_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge1_16x8b);
//using availability mask
edge0_16x8b = _mm_and_si128(edge0_16x8b, au1_mask8x16b);
edge1_16x8b = _mm_and_si128(edge1_16x8b, au1_mask8x16b);
//adding chroma offset to access U and V
edge0_16x8b = _mm_add_epi8(edge0_16x8b, chroma_offset_8x16b);
edge1_16x8b = _mm_add_epi8(edge1_16x8b, chroma_offset_8x16b);
//shuffle to get sao offset
edge0_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge0_16x8b);
edge1_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge1_16x8b);
//cnvert to 16 bit then add and then saturated pack
signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge0_16x8b);
cmp_gt0_16x8b = _mm_unpacklo_epi8(src_temp0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_unpacklo_epi8(edge0_16x8b, signdwn1_16x8b);
src_temp0_16x8b = _mm_unpackhi_epi8(src_temp0_16x8b, const0_16x8b);
edge0_16x8b = _mm_unpackhi_epi8(edge0_16x8b, signdwn1_16x8b);
cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b);
src_temp0_16x8b = _mm_add_epi16(src_temp0_16x8b, edge0_16x8b);
src_temp0_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_temp0_16x8b);
signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge1_16x8b);
cmp_gt0_16x8b = _mm_unpacklo_epi8(src_bottom_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_unpacklo_epi8(edge1_16x8b, signdwn1_16x8b);
src_bottom_16x8b = _mm_unpackhi_epi8(src_bottom_16x8b, const0_16x8b);
edge1_16x8b = _mm_unpackhi_epi8(edge1_16x8b, signdwn1_16x8b);
cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b);
src_bottom_16x8b = _mm_add_epi16(src_bottom_16x8b, edge1_16x8b);
src_bottom_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_bottom_16x8b);
cmp_gt0_16x8b = _mm_srli_si128(src_temp0_16x8b, 8);
cmp_lt0_16x8b = _mm_srli_si128(src_bottom_16x8b, 8);
_mm_storel_epi64((__m128i *)(pu1_src_left_str), left_store_16x8b);
//row = 0 store 8 pixel values from 7:0 pos. relative to cur. pos.
_mm_storel_epi64((__m128i *)(pu1_src_cpy), src_temp0_16x8b);
// row = 1
_mm_storel_epi64((__m128i *)(pu1_src_cpy + src_strd), cmp_gt0_16x8b);
//row = 2
_mm_storel_epi64((__m128i *)(pu1_src_cpy + 2 * src_strd), src_bottom_16x8b);
// row = 3
_mm_storel_epi64((__m128i *)(pu1_src_cpy + 3 * src_strd), cmp_lt0_16x8b);
src_temp0_16x8b = src_temp1_16x8b;
signup0_16x8b = _mm_slli_si128(signup0_16x8b, 8);
pu1_src_cpy += (src_strd << 2);
pu1_src_left_cpy += 8;
pu1_src_left_str += 8;
}
ht_rem = ht & 0x2;
if(ht_rem)
{
left_store_16x8b = _mm_loadl_epi64((__m128i *)pu1_src_left_cpy);
//row = 1 load 8 pixel values from 7:0 pos. relative to cur. pos.
src_temp1_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + src_strd));
// row = 2
src_bottom_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + 2 * src_strd));
//row 0 -row 1
signdwn1_16x8b = _mm_srli_si128(src_temp1_16x8b, 2);
//separating +ve and and -ve values.
cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, signdwn1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp0_16x8b);
//manipulation for row 1 -row 0
signdwn1_16x8b = _mm_slli_si128(left_store_16x8b, 14);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//manipulation for row 1 - row 0
signdwn1_16x8b = _mm_alignr_epi8(src_temp0_16x8b, signdwn1_16x8b, 14);
//combining the appropriate sign change
edge0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b);
//row1-row0
//separating +ve and and -ve values.
cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, signdwn1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp1_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//combining the appropriate sign chang
edge1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b);
//row 1 -bottom
signdwn1_16x8b = _mm_srli_si128(src_bottom_16x8b, 2);
edge0_16x8b = _mm_slli_si128(edge0_16x8b, 8); //aligned left (0-1)
signup0_16x8b = _mm_alignr_epi8(edge1_16x8b, signup0_16x8b, 8); //(1-0),(0-top)
//row1 -bottom
cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, signdwn1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp1_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//combining the appropriate sign change
signdwn1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(1-2)
edge0_16x8b = _mm_alignr_epi8(signdwn1_16x8b, edge0_16x8b, 8); //(1-2),(0-1)
//manipulation for bottom -row1
signdwn1_16x8b = _mm_slli_si128(left_store_16x8b, 12);
//eliminating old left for row 0,1
left_store_16x8b = _mm_srli_si128(left_store_16x8b, 4);
//manipulation for bottom- row 1
signdwn1_16x8b = _mm_alignr_epi8(src_temp1_16x8b, signdwn1_16x8b, 14);
//adding top and down substraction
edge0_16x8b = _mm_add_epi8(edge0_16x8b, signup0_16x8b); //(1,0) sign_up empty
//bottom - row 1
cmp_gt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, signdwn1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_bottom_16x8b);
//shifting row 1
signdwn1_16x8b = _mm_slli_si128(src_temp1_16x8b, 8);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//for the next iteration signup0_16x8b
signup0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(2-1) for next
//storing right of row 1 into left
left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, signdwn1_16x8b, 14); //for storing right of row 0
signdwn1_16x8b = _mm_slli_si128(src_temp0_16x8b, 8);
//the next top in src_top_16x8b
src_top_16x8b = src_temp1_16x8b;
//storing right of row 0 into left
left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, signdwn1_16x8b, 14);
//adding constant 2
edge0_16x8b = _mm_add_epi8(edge0_16x8b, const2_16x8b);
//shuffle to get sao index
edge0_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge0_16x8b);
//using availability mask
edge0_16x8b = _mm_and_si128(edge0_16x8b, au1_mask8x16b);
//adding chroma offset to access U and V
edge0_16x8b = _mm_add_epi8(edge0_16x8b, chroma_offset_8x16b);
//shuffle to get sao offset
edge0_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge0_16x8b);
//the next top already in src_top_16x8b
//cnvert to 16 bit then add and then saturated pack
signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge0_16x8b);
src_temp0_16x8b = _mm_unpacklo_epi8(src_temp0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_unpacklo_epi8(edge0_16x8b, signdwn1_16x8b);
src_temp1_16x8b = _mm_unpacklo_epi8(src_temp1_16x8b, const0_16x8b);
edge0_16x8b = _mm_unpackhi_epi8(edge0_16x8b, signdwn1_16x8b);
src_temp0_16x8b = _mm_add_epi16(src_temp0_16x8b, cmp_lt0_16x8b);
src_temp1_16x8b = _mm_add_epi16(src_temp1_16x8b, edge0_16x8b);
src_temp0_16x8b = _mm_packus_epi16(src_temp0_16x8b, src_temp1_16x8b);
cmp_gt0_16x8b = _mm_srli_si128(src_temp0_16x8b, 8);
_mm_storel_epi64((__m128i *)(pu1_src_left_str), left_store_16x8b);
//row = 0 store 8 pixel values from 7:0 pos. relative to cur. pos.
_mm_storel_epi64((__m128i *)(pu1_src_cpy), src_temp0_16x8b);
// row = 1
_mm_storel_epi64((__m128i *)(pu1_src_cpy + src_strd), cmp_gt0_16x8b);
src_temp0_16x8b = src_bottom_16x8b;
pu1_src_cpy += (src_strd << 1);
pu1_src_left_cpy += 4;
pu1_src_left_str += 4;
}
ht_rem = ht & 0x1;
if(ht_rem)
{
left_store_16x8b = _mm_loadl_epi64((__m128i *)pu1_src_left_cpy);
//row = 1 load 8 pixel values from 7:0 pos. relative to cur. pos.
src_bottom_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + src_strd));
//row 0 -row1
signdwn1_16x8b = _mm_srli_si128(src_bottom_16x8b, 2);
//separating +ve and and -ve values.
cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, signdwn1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp0_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//combining the appropriate sign change
edge0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b);
//adding top and down substraction
edge0_16x8b = _mm_add_epi8(edge0_16x8b, signup0_16x8b);
//for row 0 right to put into left store
signdwn1_16x8b = _mm_slli_si128(src_temp0_16x8b, 8);
//left store manipulation 1
left_store_16x8b = _mm_srli_si128(left_store_16x8b, 2);
src_top_16x8b = src_temp0_16x8b;
//filling the left boundary value
left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, signdwn1_16x8b, 14);
//adding constant 2
edge0_16x8b = _mm_add_epi8(edge0_16x8b, const2_16x8b);
edge0_16x8b = _mm_slli_si128(edge0_16x8b, 8);
edge0_16x8b = _mm_srli_si128(edge0_16x8b, 8);
//shuffle to get sao index
edge0_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge0_16x8b);
//using availability mask
edge0_16x8b = _mm_and_si128(edge0_16x8b, au1_mask8x16b);
//adding chroma offset to access U and V
edge0_16x8b = _mm_add_epi8(edge0_16x8b, chroma_offset_8x16b);
//shuffle to get sao offset
edge0_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge0_16x8b);
//cnvert to 16 bit then add and then saturated pack
signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge0_16x8b);
src_temp0_16x8b = _mm_unpacklo_epi8(src_temp0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_unpacklo_epi8(edge0_16x8b, signdwn1_16x8b);
src_temp0_16x8b = _mm_add_epi16(src_temp0_16x8b, cmp_lt0_16x8b);
src_temp0_16x8b = _mm_packus_epi16(src_temp0_16x8b, const0_16x8b);
_mm_storel_epi64((__m128i *)(pu1_src_left_str), left_store_16x8b);
//row = 0 store 8 pixel values from 7:0 pos. relative to cur. pos.
_mm_storel_epi64((__m128i *)(pu1_src_cpy), src_temp0_16x8b);
pu1_src_cpy += (src_strd);
pu1_src_left_cpy += 2;
pu1_src_left_str += 2;
}
if(0 == pu1_avail[3])
{
src_top_16x8b = src_bottom_16x8b;
pu1_src_left_str[1] = pu1_src_cpy[7];
pu1_src_left_str[0] = pu1_src_cpy[6];
}
if(0 == pu1_avail[2])
{
pu1_src_left_str[-2 * ht_0] = pu1_src[6 - src_strd];
pu1_src_left_str[-2 * ht_0 + 1] = pu1_src[7 - src_strd];
}
_mm_storel_epi64((__m128i *)(pu1_src_top + wd - col), src_top_16x8b);
pu1_src += 8;
pu1_left_tmp = pu1_src_left_cpy2;
pu1_src_left_cpy2 = pu1_src_left_str2;
pu1_src_left_str2 = pu1_left_tmp;
pu1_src_left_cpy = pu1_src_left_cpy2;
pu1_src_left_str = pu1_src_left_str2;
}
pu1_src_org[0] = u1_pos_0_0_tmp_u;
pu1_src_org[1] = u1_pos_0_0_tmp_v;
pu1_src_org[wd - 2 + (ht_tmp - 1) * src_strd] = u1_pos_wd_ht_tmp_u;
pu1_src_org[wd - 1 + (ht_tmp - 1) * src_strd] = u1_pos_wd_ht_tmp_v;
pu1_src_left_cpy = (0 == pu1_avail[2]) ? (pu1_src_left_cpy - 2) : pu1_src_left_cpy;
for(row = 0; row < 2 * ht_tmp; row++)
{
pu1_src_left[row] = pu1_src_left_cpy[row];
}
}
}
void ihevc_sao_edge_offset_class3_ssse3(UWORD8 *pu1_src,
WORD32 src_strd,
UWORD8 *pu1_src_left,
UWORD8 *pu1_src_top,
UWORD8 *pu1_src_top_left,
UWORD8 *pu1_src_top_right,
UWORD8 *pu1_src_bot_left,
UWORD8 *pu1_avail,
WORD8 *pi1_sao_offset,
WORD32 wd,
WORD32 ht)
{
WORD32 row, col;
UWORD8 *pu1_src_top_cpy, *pu1_src_left_cpy, *pu1_src_left_cpy2;
UWORD8 *pu1_left_tmp, *pu1_src_left_str, *pu1_src_left_str2;
UWORD8 *pu1_src_cpy, *pu1_src_org;
UWORD8 au1_src_left_tmp[MAX_CTB_SIZE + 8];
UWORD8 au1_src_left_tmp1[MAX_CTB_SIZE + 8];
UWORD8 au1_mask[MAX_CTB_SIZE], *au1_mask_cpy;
WORD32 wd_rem;
UWORD8 u1_pos_wd_0_tmp, u1_pos_0_ht_tmp;
WORD32 ht_tmp;
WORD32 bit_depth;
UWORD8 u1_avail0, u1_avail1;
__m128i src_top_16x8b, src_bottom_16x8b;
__m128i src_temp0_16x8b, src_temp1_16x8b;
__m128i signup0_16x8b, signdwn1_16x8b;
__m128i cmp_gt0_16x8b, cmp_lt0_16x8b;
__m128i edge0_16x8b, edge1_16x8b;
__m128i au1_mask8x16b;
__m128i edge_idx_8x16b, sao_offset_8x16b;
__m128i const2_16x8b, const0_16x8b;
__m128i left_store_16x8b;
ht_tmp = ht;
au1_mask8x16b = _mm_set1_epi8(0xff);
au1_src_left_tmp[0] = pu1_src[(wd - 1)];
//manipulation for bottom left
for(row = 1; row < ht; row++)
{
au1_src_left_tmp[row] = pu1_src_left[row];
}
au1_src_left_tmp[ht] = pu1_src_bot_left[0];
*pu1_src_top_left = pu1_src_top[wd - 1];
//setting availability mask to ff size MAX_CTB_SIZE
for(col = 0; col < MAX_CTB_SIZE; col += 16)
_mm_storeu_si128((__m128i *)(au1_mask + col), au1_mask8x16b);
bit_depth = BIT_DEPTH_LUMA;
pu1_src_org = pu1_src;
pu1_src_top_cpy = pu1_src_top;
pu1_src_left_cpy2 = au1_src_left_tmp;
pu1_src_left_cpy = au1_src_left_tmp;
pu1_src_left_str2 = au1_src_left_tmp1;
pu1_src_left_str = au1_src_left_tmp1;
edge_idx_8x16b = _mm_loadl_epi64((__m128i *)gi1_table_edge_idx);
sao_offset_8x16b = _mm_loadl_epi64((__m128i *)pi1_sao_offset);
/* If top-right is available, process separately */
if(0 != pu1_avail[5])
{
WORD32 edge_idx;
edge_idx = 2 + SIGN(pu1_src[wd - 1] - pu1_src_top_right[0]) +
SIGN(pu1_src[wd - 1] - pu1_src[wd - 1 - 1 + src_strd]);
edge_idx = gi1_table_edge_idx[edge_idx];
if(0 != edge_idx)
{
u1_pos_wd_0_tmp = CLIP3(pu1_src[wd - 1] + pi1_sao_offset[edge_idx], 0, (1 << bit_depth) - 1);
}
else
{
u1_pos_wd_0_tmp = pu1_src[wd - 1];
}
}
else
{
u1_pos_wd_0_tmp = pu1_src[wd - 1];
}
/* If bottom-left is available, process separately */
if(0 != pu1_avail[6])
{
WORD32 edge_idx;
edge_idx = 2 + SIGN(pu1_src[(ht - 1) * src_strd] - pu1_src[(ht - 1) * src_strd + 1 - src_strd]) +
SIGN(pu1_src[(ht - 1) * src_strd] - pu1_src_bot_left[0]);
edge_idx = gi1_table_edge_idx[edge_idx];
if(0 != edge_idx)
{
u1_pos_0_ht_tmp = CLIP3(pu1_src[(ht - 1) * src_strd] + pi1_sao_offset[edge_idx], 0, (1 << bit_depth) - 1);
}
else
{
u1_pos_0_ht_tmp = pu1_src[(ht - 1) * src_strd];
}
}
else
{
u1_pos_0_ht_tmp = pu1_src[(ht - 1) * src_strd];
}
/* Update height and source pointers based on the availability flags */
if(0 == pu1_avail[2])
{
pu1_src_left_cpy2++;
pu1_src_left_str2++;
pu1_src_top_cpy = pu1_src;
pu1_src += src_strd;
ht--;
}
if(0 == pu1_avail[3])
{
ht--;
}
const2_16x8b = _mm_set1_epi8(2);
const0_16x8b = _mm_setzero_si128();
//availability mask creation
u1_avail0 = pu1_avail[0];
u1_avail1 = pu1_avail[1];
au1_mask[0] = u1_avail0;
au1_mask[wd - 1] = u1_avail1;
{
WORD32 ht_rem;
pu1_src_left_cpy = pu1_src_left_cpy2;
pu1_src_left_str = pu1_src_left_str2;
au1_mask_cpy = au1_mask;
for(col = wd; col >= 16; col -= 16)
{
pu1_src_cpy = pu1_src;
src_top_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_top_cpy + wd - col + 1));
//row = 0
src_temp0_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy));
//loading the mask
au1_mask8x16b = _mm_loadu_si128((__m128i *)au1_mask_cpy);
//separating +ve and and -ve values.
cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, src_top_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(src_top_16x8b, src_temp0_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//combining the appropriate sign change
signup0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b);
for(row = ht; row >= 2; row -= 2)
{
left_store_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_left_cpy));
//row = 1
src_temp1_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + src_strd));
//to insert left in row 1
signdwn1_16x8b = _mm_slli_si128(left_store_16x8b, 14);
// row = 0 right
src_bottom_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + 1));
//manipulation for row 1 - row 0
signdwn1_16x8b = _mm_alignr_epi8(src_temp1_16x8b, signdwn1_16x8b, 15);
//row 0 -row1
//separating +ve and and -ve values.
cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, signdwn1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp0_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//combining the appropriate sign change
edge0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(0-1)
//combining sign-left and sign_right
edge0_16x8b = _mm_add_epi8(edge0_16x8b, signup0_16x8b);
//row1-row0
//separating +ve and and -ve values.
cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, src_bottom_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, src_temp1_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
// row = 2
src_bottom_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + 2 * src_strd));
// row = 1 right
signdwn1_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + src_strd + 1));
edge1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(1-0)
//bottom - row1
cmp_gt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, signdwn1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_bottom_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//for the next iteration bottom -row1
signup0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b);
//to insert left in row 1
signdwn1_16x8b = _mm_slli_si128(left_store_16x8b, 13);
//manipulation for row 1 - bottom
signdwn1_16x8b = _mm_alignr_epi8(src_bottom_16x8b, signdwn1_16x8b, 15);
//row1 -bottom
cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, signdwn1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp1_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//combining the appropriate sign change
signdwn1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b);
//combining sign-left and sign_right
edge1_16x8b = _mm_add_epi8(edge1_16x8b, signdwn1_16x8b);
//eliminating old left for row 0 and row 1
left_store_16x8b = _mm_srli_si128(left_store_16x8b, 2);
//row1 getting it right for left of next block
left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, src_temp1_16x8b, 15);
//adding constant 2
edge0_16x8b = _mm_add_epi8(edge0_16x8b, const2_16x8b);
edge1_16x8b = _mm_add_epi8(edge1_16x8b, const2_16x8b);
//shuffle to get sao index
edge0_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge0_16x8b);
edge1_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge1_16x8b);
//using availability mask
edge0_16x8b = _mm_and_si128(edge0_16x8b, au1_mask8x16b);
edge1_16x8b = _mm_and_si128(edge1_16x8b, au1_mask8x16b);
//shuffle to get sao offset
edge0_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge0_16x8b);
edge1_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge1_16x8b);
//row0 getting it right for left of next block
left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, src_temp0_16x8b, 15);
//copying the next top
src_top_16x8b = src_temp1_16x8b;
//cnvert to 16 bit then add and then saturated pack
signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge0_16x8b);
cmp_gt0_16x8b = _mm_unpacklo_epi8(src_temp0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_unpacklo_epi8(edge0_16x8b, signdwn1_16x8b);
src_temp0_16x8b = _mm_unpackhi_epi8(src_temp0_16x8b, const0_16x8b);
cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b);
cmp_lt0_16x8b = _mm_unpackhi_epi8(edge0_16x8b, signdwn1_16x8b);
src_temp0_16x8b = _mm_add_epi16(src_temp0_16x8b, cmp_lt0_16x8b);
src_temp0_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_temp0_16x8b);
signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge1_16x8b);
cmp_gt0_16x8b = _mm_unpacklo_epi8(src_temp1_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_unpacklo_epi8(edge1_16x8b, signdwn1_16x8b);
src_temp1_16x8b = _mm_unpackhi_epi8(src_temp1_16x8b, const0_16x8b);
cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b);
cmp_lt0_16x8b = _mm_unpackhi_epi8(edge1_16x8b, signdwn1_16x8b);
src_temp1_16x8b = _mm_add_epi16(src_temp1_16x8b, cmp_lt0_16x8b);
src_temp1_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_temp1_16x8b);
//store left boundary
_mm_storel_epi64((__m128i *)(pu1_src_left_str), left_store_16x8b);
//row = 0 store 8 pixel values from 7:0 pos. relative to cur. pos.
_mm_storeu_si128((__m128i *)(pu1_src_cpy), src_temp0_16x8b);
// row = 1
_mm_storeu_si128((__m128i *)(pu1_src_cpy + src_strd), src_temp1_16x8b);
src_temp0_16x8b = src_bottom_16x8b;
pu1_src_cpy += (src_strd << 1);
pu1_src_left_cpy += 2;
pu1_src_left_str += 2;
}
ht_rem = ht & 0x1;
if(ht_rem)
{
left_store_16x8b = _mm_loadl_epi64((__m128i *)pu1_src_left_cpy);
src_bottom_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + src_strd));
//to insert left in row 1
signdwn1_16x8b = _mm_slli_si128(left_store_16x8b, 14);
//manipulation for row 1 - row 0
signdwn1_16x8b = _mm_alignr_epi8(src_bottom_16x8b, signdwn1_16x8b, 15);
//current row -next row
//separating +ve and and -ve values.
cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, signdwn1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp0_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//combining the appropriate sign change
edge0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b);
//adding top and bottom and constant 2
edge0_16x8b = _mm_add_epi8(edge0_16x8b, signup0_16x8b);
edge0_16x8b = _mm_add_epi8(edge0_16x8b, const2_16x8b);
//eliminating old left for row 0 and row 1
left_store_16x8b = _mm_srli_si128(left_store_16x8b, 1);
edge0_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge0_16x8b);
//using availability mask
edge0_16x8b = _mm_and_si128(edge0_16x8b, au1_mask8x16b);
edge0_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge0_16x8b);
//row0 getting it right for left of next block
left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, src_temp0_16x8b, 15);
//copying the next top
src_top_16x8b = src_temp0_16x8b;
//cnvert to 16 bit then add and then saturated pack
signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge0_16x8b);
cmp_gt0_16x8b = _mm_unpacklo_epi8(src_temp0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_unpacklo_epi8(edge0_16x8b, signdwn1_16x8b);
src_temp0_16x8b = _mm_unpackhi_epi8(src_temp0_16x8b, const0_16x8b);
cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b);
cmp_lt0_16x8b = _mm_unpackhi_epi8(edge0_16x8b, signdwn1_16x8b);
src_temp0_16x8b = _mm_add_epi16(src_temp0_16x8b, cmp_lt0_16x8b);
src_temp0_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_temp0_16x8b);
//store left boundary
_mm_storel_epi64((__m128i *)(pu1_src_left_str), left_store_16x8b);
_mm_storeu_si128((__m128i *)(pu1_src_cpy), src_temp0_16x8b);
pu1_src_cpy += (src_strd);
src_temp0_16x8b = src_bottom_16x8b;
pu1_src_left_cpy++;
pu1_src_left_str++;
}
{ //for bottom right
left_store_16x8b = _mm_loadl_epi64((__m128i *)pu1_src_left_cpy);
left_store_16x8b = _mm_srli_si128(left_store_16x8b, 1);
left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, src_temp0_16x8b, 15);
_mm_storel_epi64((__m128i *)(pu1_src_left_str), left_store_16x8b);
}
if(0 == pu1_avail[3])
{
src_top_16x8b = src_bottom_16x8b;
}
//for the top left of next part of the block
left_store_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_top_cpy + wd - col));
//updating top flag
_mm_storeu_si128((__m128i *)(pu1_src_top + wd - col), src_top_16x8b);
pu1_src += 16;
au1_mask_cpy += 16;
pu1_left_tmp = pu1_src_left_cpy2;
pu1_src_left_cpy2 = pu1_src_left_str2;
pu1_src_left_str2 = pu1_left_tmp;
pu1_src_left_cpy = pu1_src_left_cpy2;
pu1_src_left_str = pu1_src_left_str2;
}
wd_rem = wd & 0xF;
if(wd_rem)
{
pu1_src_cpy = pu1_src;
pu1_src_left_cpy = pu1_src_left_cpy2;
pu1_src_left_str = pu1_src_left_str2;
src_top_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_top_cpy + wd - col + 1));
//row = 0
src_temp0_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy));
au1_mask8x16b = _mm_loadl_epi64((__m128i *)au1_mask_cpy); //????
//separating +ve and and -ve values.
cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, src_top_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(src_top_16x8b, src_temp0_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//preparing au1_mask
au1_mask8x16b = _mm_unpacklo_epi64(au1_mask8x16b, au1_mask8x16b);
//combining the appropriate sign change
signup0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b);
signup0_16x8b = _mm_slli_si128(signup0_16x8b, 8);
for(row = ht; row >= 4; row -= 4)
{
left_store_16x8b = _mm_loadl_epi64((__m128i *)pu1_src_left_cpy);
//row = 1 load 8 pixel values from 7:0 pos. relative to cur. pos.
src_temp1_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + src_strd));
// row = 2
src_bottom_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + 2 * src_strd));
//manipulation for row 0 -row 1
signdwn1_16x8b = _mm_slli_si128(left_store_16x8b, 14);
//row 1 left
signdwn1_16x8b = _mm_alignr_epi8(src_temp1_16x8b, signdwn1_16x8b, 15);
//row 0 -row1
//separating +ve and and -ve values.
cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, signdwn1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp0_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//manipulatiing for row 1 -row 0
signdwn1_16x8b = _mm_srli_si128(src_temp0_16x8b, 1);
//combining the appropriate sign change
edge0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b);
//row 1 -row0
//separating +ve and and -ve values.
cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, signdwn1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp1_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//row1-row0
edge1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b);
edge0_16x8b = _mm_slli_si128(edge0_16x8b, 8); //aligned left (0-1)
signup0_16x8b = _mm_alignr_epi8(edge1_16x8b, signup0_16x8b, 8); //(1-0),(0-top)
//manipulation for row 1 -row 2
signdwn1_16x8b = _mm_slli_si128(left_store_16x8b, 13);
//row 2 left
signdwn1_16x8b = _mm_alignr_epi8(src_bottom_16x8b, signdwn1_16x8b, 15);
//packing row 0 n row 1
src_temp0_16x8b = _mm_unpacklo_epi64(src_temp0_16x8b, src_temp1_16x8b);
//row1 -row2
cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, signdwn1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp1_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//combining the appropriate sign change
signdwn1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(1-2)
edge0_16x8b = _mm_alignr_epi8(signdwn1_16x8b, edge0_16x8b, 8); //(1-2),(0-1)
//row 1 right
signdwn1_16x8b = _mm_srli_si128(src_temp1_16x8b, 1);
//row = 3
src_top_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + 3 * src_strd));
// row = 4
src_temp1_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + 4 * src_strd));
edge0_16x8b = _mm_add_epi8(edge0_16x8b, signup0_16x8b); //(1,0) sign_up empty
//separating +ve and and -ve values.(2,1)
cmp_gt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, signdwn1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_bottom_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//row 2 right
signdwn1_16x8b = _mm_srli_si128(src_bottom_16x8b, 1);
//combining the appropriate sign change
signup0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(2-1)
//separating +ve and and -ve values.(3,2)
cmp_gt0_16x8b = _mm_subs_epu8(src_top_16x8b, signdwn1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_top_16x8b);
signup0_16x8b = _mm_slli_si128(signup0_16x8b, 8); //aligned left (2-1)
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//manipulation for row 2 -row 3
signdwn1_16x8b = _mm_slli_si128(left_store_16x8b, 12);
//row 3 left
signdwn1_16x8b = _mm_alignr_epi8(src_top_16x8b, signdwn1_16x8b, 15);
//combining the appropriate sign change
edge1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(3-2)
signup0_16x8b = _mm_alignr_epi8(edge1_16x8b, signup0_16x8b, 8); //(3-2) ,(2-1)
//separating +ve and and -ve values.(2,3)
cmp_gt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, signdwn1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_bottom_16x8b);
//manipulation for row 3 -bottom
signdwn1_16x8b = _mm_slli_si128(left_store_16x8b, 11);
//bottom left
signdwn1_16x8b = _mm_alignr_epi8(src_temp1_16x8b, signdwn1_16x8b, 15);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//combining the appropriate sign change
edge1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(2-3)
//separating +ve and and -ve values.(3,bottom)
cmp_gt0_16x8b = _mm_subs_epu8(src_top_16x8b, signdwn1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_top_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
edge1_16x8b = _mm_slli_si128(edge1_16x8b, 8); //aligned left (2-3)
//combining the appropriate sign change
signdwn1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(3-bottom)
edge1_16x8b = _mm_alignr_epi8(signdwn1_16x8b, edge1_16x8b, 8); //(3-bottom),(2-3)
//eliminating old left for row 0,1,2,3
left_store_16x8b = _mm_srli_si128(left_store_16x8b, 4);
//packing row 2 n row 3
src_bottom_16x8b = _mm_unpacklo_epi64(src_bottom_16x8b, src_top_16x8b);
//row 3 right
signdwn1_16x8b = _mm_srli_si128(src_top_16x8b, 1);
//loading row 3 right into left
left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, src_bottom_16x8b, 15);
//adding bottom and top values of row 2 and row 3
edge1_16x8b = _mm_add_epi8(edge1_16x8b, signup0_16x8b); //(3,2)
//separating +ve and and -ve values.(botttom,3)
cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, signdwn1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp1_16x8b);
//to store right of row 2
signdwn1_16x8b = _mm_slli_si128(src_bottom_16x8b, 8);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
signup0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(bottom -3) for next iteration
//storing right of row 2into left
left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, signdwn1_16x8b, 15);
//to store right of row 0
signdwn1_16x8b = _mm_slli_si128(src_temp0_16x8b, 8);
//storing right of row 1 into left
left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, src_temp0_16x8b, 15);
//adding constant 2
edge0_16x8b = _mm_add_epi8(edge0_16x8b, const2_16x8b);
edge1_16x8b = _mm_add_epi8(edge1_16x8b, const2_16x8b);
//shuffle to get sao index
edge0_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge0_16x8b);
edge1_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge1_16x8b);
//using availability mask
edge0_16x8b = _mm_and_si128(edge0_16x8b, au1_mask8x16b);
edge1_16x8b = _mm_and_si128(edge1_16x8b, au1_mask8x16b);
//shuffle to get sao offset
edge0_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge0_16x8b);
edge1_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge1_16x8b);
//storing right of row 0 into left
left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, signdwn1_16x8b, 15);
//cnvert to 16 bit then add and then saturated pack
signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge0_16x8b);
cmp_gt0_16x8b = _mm_unpacklo_epi8(src_temp0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_unpacklo_epi8(edge0_16x8b, signdwn1_16x8b);
src_temp0_16x8b = _mm_unpackhi_epi8(src_temp0_16x8b, const0_16x8b);
cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b);
cmp_lt0_16x8b = _mm_unpackhi_epi8(edge0_16x8b, signdwn1_16x8b);
src_temp0_16x8b = _mm_add_epi16(src_temp0_16x8b, cmp_lt0_16x8b);
src_temp0_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_temp0_16x8b);
signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge1_16x8b);
cmp_gt0_16x8b = _mm_unpacklo_epi8(src_bottom_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_unpacklo_epi8(edge1_16x8b, signdwn1_16x8b);
src_bottom_16x8b = _mm_unpackhi_epi8(src_bottom_16x8b, const0_16x8b);
cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b);
cmp_lt0_16x8b = _mm_unpackhi_epi8(edge1_16x8b, signdwn1_16x8b);
src_bottom_16x8b = _mm_add_epi16(src_bottom_16x8b, cmp_lt0_16x8b);
src_bottom_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_bottom_16x8b);
cmp_gt0_16x8b = _mm_srli_si128(src_temp0_16x8b, 8);
cmp_lt0_16x8b = _mm_srli_si128(src_bottom_16x8b, 8);
_mm_storel_epi64((__m128i *)(pu1_src_left_str), left_store_16x8b);
//row = 0 store 8 pixel values from 7:0 pos. relative to cur. pos.
_mm_storel_epi64((__m128i *)(pu1_src_cpy), src_temp0_16x8b);
// row = 1
_mm_storel_epi64((__m128i *)(pu1_src_cpy + src_strd), cmp_gt0_16x8b);
//row = 2
_mm_storel_epi64((__m128i *)(pu1_src_cpy + 2 * src_strd), src_bottom_16x8b);
// row = 3
_mm_storel_epi64((__m128i *)(pu1_src_cpy + 3 * src_strd), cmp_lt0_16x8b);
src_temp0_16x8b = src_temp1_16x8b;
signup0_16x8b = _mm_slli_si128(signup0_16x8b, 8);
pu1_src_cpy += (src_strd << 2);
pu1_src_left_cpy += 4;
pu1_src_left_str += 4;
}
ht_rem = ht & 0x2;
if(ht_rem)
{
left_store_16x8b = _mm_loadl_epi64((__m128i *)pu1_src_left_cpy);
//row = 1 load 8 pixel values from 7:0 pos. relative to cur. pos.
src_temp1_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + src_strd));
// row = 2
src_bottom_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + 2 * src_strd));
//manipulation for row 0 -row 1
signdwn1_16x8b = _mm_slli_si128(left_store_16x8b, 14);
//bottom left
signdwn1_16x8b = _mm_alignr_epi8(src_temp1_16x8b, signdwn1_16x8b, 15);
//separating +ve and and -ve values.
cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, signdwn1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp0_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//manipulation for row 1 - row 0
signdwn1_16x8b = _mm_srli_si128(src_temp0_16x8b, 1);
//combining the appropriate sign change
edge0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b);
//row1-row0
//separating +ve and and -ve values.
cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, signdwn1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp1_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//combining the appropriate sign chang
edge1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b);
//manipulation for row 1 -bottom
signdwn1_16x8b = _mm_slli_si128(left_store_16x8b, 13);
//bottom left
signdwn1_16x8b = _mm_alignr_epi8(src_bottom_16x8b, signdwn1_16x8b, 15);
edge0_16x8b = _mm_slli_si128(edge0_16x8b, 8); //aligned left (0-1)
signup0_16x8b = _mm_alignr_epi8(edge1_16x8b, signup0_16x8b, 8); //(1-0),(0-top)
//row1 -bottom
cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, signdwn1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp1_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//combining the appropriate sign change
signdwn1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(1-2)
edge0_16x8b = _mm_alignr_epi8(signdwn1_16x8b, edge0_16x8b, 8); //(1-2),(0-1)
//manipulation for bottom- row 1 (row 1 right)
signdwn1_16x8b = _mm_srli_si128(src_temp1_16x8b, 1);
//adding top and down substraction
edge0_16x8b = _mm_add_epi8(edge0_16x8b, signup0_16x8b); //(1,0) sign_up empty
//bottom - row 1
cmp_gt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, signdwn1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_bottom_16x8b);
//eliminating old left for row 0,1
left_store_16x8b = _mm_srli_si128(left_store_16x8b, 2);
signdwn1_16x8b = _mm_slli_si128(src_temp1_16x8b, 8);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//for the next iteration signup0_16x8b
signup0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(2-1) for next
//storing right of row 1 into left
left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, signdwn1_16x8b, 15);
//for storing right of row 1
signdwn1_16x8b = _mm_slli_si128(src_temp0_16x8b, 8);
src_top_16x8b = src_temp1_16x8b;
//storing right of row 0 into left
left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, signdwn1_16x8b, 15);
//adding constant 2
edge0_16x8b = _mm_add_epi8(edge0_16x8b, const2_16x8b);
//shuffle to get sao index
edge0_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge0_16x8b);
//using availability mask
edge0_16x8b = _mm_and_si128(edge0_16x8b, au1_mask8x16b);
//shuffle to get sao offset
edge0_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge0_16x8b);
//the next top already in src_top_16x8b
//cnvert to 16 bit then add and then saturated pack
signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge0_16x8b);
src_temp0_16x8b = _mm_unpacklo_epi8(src_temp0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_unpacklo_epi8(edge0_16x8b, signdwn1_16x8b);
src_temp1_16x8b = _mm_unpacklo_epi8(src_temp1_16x8b, const0_16x8b);
src_temp0_16x8b = _mm_add_epi16(src_temp0_16x8b, cmp_lt0_16x8b);
cmp_lt0_16x8b = _mm_unpackhi_epi8(edge0_16x8b, signdwn1_16x8b);
src_temp1_16x8b = _mm_add_epi16(src_temp1_16x8b, cmp_lt0_16x8b);
src_temp0_16x8b = _mm_packus_epi16(src_temp0_16x8b, src_temp1_16x8b);
cmp_gt0_16x8b = _mm_srli_si128(src_temp0_16x8b, 8);
_mm_storel_epi64((__m128i *)(pu1_src_left_str), left_store_16x8b);
//row = 0 store 8 pixel values from 7:0 pos. relative to cur. pos.
_mm_storel_epi64((__m128i *)(pu1_src_cpy), src_temp0_16x8b);
// row = 1
_mm_storel_epi64((__m128i *)(pu1_src_cpy + src_strd), cmp_gt0_16x8b);
src_temp0_16x8b = src_bottom_16x8b;
pu1_src_cpy += (src_strd << 1);
pu1_src_left_cpy += 2;
pu1_src_left_str += 2;
}
ht_rem = ht & 0x1;
if(ht_rem)
{
left_store_16x8b = _mm_loadl_epi64((__m128i *)pu1_src_left_cpy);
//row = 1 load 8 pixel values from 7:0 pos. relative to cur. pos.
src_bottom_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + src_strd));
//manipulation for row 0 -bottom
signdwn1_16x8b = _mm_slli_si128(left_store_16x8b, 14);
//bottom left
signdwn1_16x8b = _mm_alignr_epi8(src_bottom_16x8b, signdwn1_16x8b, 15);
//separating +ve and and -ve values.
cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, signdwn1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp0_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//combining the appropriate sign change
edge0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b);
//adding top and down substraction
edge0_16x8b = _mm_add_epi8(edge0_16x8b, signup0_16x8b);
//for row 0 right to put into left store
signdwn1_16x8b = _mm_slli_si128(src_temp0_16x8b, 8);
//adding constant 2
edge0_16x8b = _mm_add_epi8(edge0_16x8b, const2_16x8b);
edge0_16x8b = _mm_slli_si128(edge0_16x8b, 8);
edge0_16x8b = _mm_srli_si128(edge0_16x8b, 8);
//left store manipulation 1
left_store_16x8b = _mm_srli_si128(left_store_16x8b, 1);
//filling the left boundary value
left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, signdwn1_16x8b, 15);
//shuffle to get sao index
edge0_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge0_16x8b);
//using availability mask
edge0_16x8b = _mm_and_si128(edge0_16x8b, au1_mask8x16b);
//shuffle to get sao offset
edge0_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge0_16x8b);
src_top_16x8b = src_temp0_16x8b;
//cnvert to 16 bit then add and then saturated pack
signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge0_16x8b);
src_temp0_16x8b = _mm_unpacklo_epi8(src_temp0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_unpacklo_epi8(edge0_16x8b, signdwn1_16x8b);
src_temp0_16x8b = _mm_add_epi16(src_temp0_16x8b, cmp_lt0_16x8b);
src_temp0_16x8b = _mm_packus_epi16(src_temp0_16x8b, const0_16x8b);
_mm_storel_epi64((__m128i *)(pu1_src_left_str), left_store_16x8b);
//row = 0 store 8 pixel values from 7:0 pos. relative to cur. pos.
_mm_storel_epi64((__m128i *)(pu1_src_cpy), src_temp0_16x8b);
pu1_src_cpy += (src_strd);
src_temp0_16x8b = src_bottom_16x8b;
pu1_src_left_cpy++;
pu1_src_left_str++;
}
{ //for bottom right
left_store_16x8b = _mm_loadl_epi64((__m128i *)pu1_src_left_cpy);
left_store_16x8b = _mm_srli_si128(left_store_16x8b, 1);
src_temp0_16x8b = _mm_slli_si128(src_temp0_16x8b, 8);
left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, src_temp0_16x8b, 15);
_mm_storel_epi64((__m128i *)(pu1_src_left_str), left_store_16x8b);
}
if(0 == pu1_avail[3])
{
src_top_16x8b = src_bottom_16x8b;
}
_mm_storel_epi64((__m128i *)(pu1_src_top + wd - col), src_top_16x8b);
pu1_src += 8;
pu1_left_tmp = pu1_src_left_cpy2;
pu1_src_left_cpy2 = pu1_src_left_str2;
pu1_src_left_str2 = pu1_left_tmp;
pu1_src_left_cpy = pu1_src_left_cpy2;
pu1_src_left_str = pu1_src_left_str2;
}
pu1_src_org[wd - 1] = u1_pos_wd_0_tmp;
pu1_src_org[(ht_tmp - 1) * src_strd] = u1_pos_0_ht_tmp;
pu1_src_left_cpy = (0 == pu1_avail[2]) ? (pu1_src_left_cpy - 1) : pu1_src_left_cpy;
pu1_src_left[0] = au1_src_left_tmp[0];
for(row = 1; row < ht_tmp; row++)
{
pu1_src_left[row] = pu1_src_left_cpy[row];
}
}
}
void ihevc_sao_edge_offset_class3_chroma_ssse3(UWORD8 *pu1_src,
WORD32 src_strd,
UWORD8 *pu1_src_left,
UWORD8 *pu1_src_top,
UWORD8 *pu1_src_top_left,
UWORD8 *pu1_src_top_right,
UWORD8 *pu1_src_bot_left,
UWORD8 *pu1_avail,
WORD8 *pi1_sao_offset_u,
WORD8 *pi1_sao_offset_v,
WORD32 wd,
WORD32 ht)
{
WORD32 row, col;
UWORD8 *pu1_src_top_cpy, *pu1_src_left_cpy, *pu1_src_left_cpy2;
UWORD8 *pu1_src_cpy, *pu1_src_org;
UWORD8 au1_src_left_tmp[2 * (MAX_CTB_SIZE + 8)];
UWORD8 au1_mask[MAX_CTB_SIZE], *au1_mask_cpy;
WORD32 wd_rem;
UWORD8 u1_pos_wd_0_tmp_u, u1_pos_wd_0_tmp_v, u1_pos_0_ht_tmp_u, u1_pos_0_ht_tmp_v;
WORD32 ht_tmp;
WORD32 bit_depth;
UWORD8 u1_avail0, u1_avail1;
__m128i src_top_16x8b, src_bottom_16x8b;
__m128i src_temp0_16x8b, src_temp1_16x8b;
__m128i signup0_16x8b, signdwn1_16x8b;
__m128i cmp_gt0_16x8b, cmp_lt0_16x8b;
__m128i edge0_16x8b, edge1_16x8b;
__m128i au1_mask8x16b;
__m128i edge_idx_8x16b, sao_offset_8x16b;
__m128i left_store_16x8b;
__m128i const0_16x8b, const2_16x8b;
__m128i chroma_offset_8x16b;
ht_tmp = ht;
au1_mask8x16b = _mm_set1_epi8(0xff);
au1_src_left_tmp[0] = pu1_src[(wd - 2)];
au1_src_left_tmp[1] = pu1_src[(wd - 1)];
//manipulation for bottom left
for(row = 2; row < 2 * ht; row++)
{
au1_src_left_tmp[row] = pu1_src_left[row];
}
au1_src_left_tmp[2 * ht] = pu1_src_bot_left[0];
au1_src_left_tmp[2 * ht + 1] = pu1_src_bot_left[1];
pu1_src_top_left[0] = pu1_src_top[wd - 2];
pu1_src_top_left[1] = pu1_src_top[wd - 1];
//setting availability mask to ff size MAX_CTB_SIZE
for(col = 0; col < MAX_CTB_SIZE; col += 16)
_mm_storeu_si128((__m128i *)(au1_mask + col), au1_mask8x16b);
bit_depth = BIT_DEPTH_LUMA;
pu1_src_org = pu1_src;
pu1_src_top_cpy = pu1_src_top;
pu1_src_left_cpy2 = au1_src_left_tmp;
pu1_src_left_cpy = au1_src_left_tmp;
edge_idx_8x16b = _mm_loadl_epi64((__m128i *)gi1_table_edge_idx);
sao_offset_8x16b = _mm_loadl_epi64((__m128i *)pi1_sao_offset_u);
const0_16x8b = _mm_loadl_epi64((__m128i *)pi1_sao_offset_v);
chroma_offset_8x16b = _mm_set1_epi16(0x0800);
/* If top-right is available, process separately */
if(0 != pu1_avail[5])
{
WORD32 edge_idx;
/* U */
edge_idx = 2 + SIGN(pu1_src[wd - 2] - pu1_src_top_right[0]) +
SIGN(pu1_src[wd - 2] - pu1_src[wd - 2 - 2 + src_strd]);
edge_idx = gi1_table_edge_idx[edge_idx];
if(0 != edge_idx)
{
u1_pos_wd_0_tmp_u = CLIP3(pu1_src[wd - 2] + pi1_sao_offset_u[edge_idx], 0, (1 << bit_depth) - 1);
}
else
{
u1_pos_wd_0_tmp_u = pu1_src[wd - 2];
}
/* V */
edge_idx = 2 + SIGN(pu1_src[wd - 1] - pu1_src_top_right[1]) +
SIGN(pu1_src[wd - 1] - pu1_src[wd - 1 - 2 + src_strd]);
edge_idx = gi1_table_edge_idx[edge_idx];
if(0 != edge_idx)
{
u1_pos_wd_0_tmp_v = CLIP3(pu1_src[wd - 1] + pi1_sao_offset_v[edge_idx], 0, (1 << bit_depth) - 1);
}
else
{
u1_pos_wd_0_tmp_v = pu1_src[wd - 1];
}
}
else
{
u1_pos_wd_0_tmp_u = pu1_src[wd - 2];
u1_pos_wd_0_tmp_v = pu1_src[wd - 1];
}
/* If bottom-left is available, process separately */
if(0 != pu1_avail[6])
{
WORD32 edge_idx;
/* U */
edge_idx = 2 + SIGN(pu1_src[(ht - 1) * src_strd] - pu1_src[(ht - 1) * src_strd + 2 - src_strd]) +
SIGN(pu1_src[(ht - 1) * src_strd] - pu1_src_bot_left[0]);
edge_idx = gi1_table_edge_idx[edge_idx];
if(0 != edge_idx)
{
u1_pos_0_ht_tmp_u = CLIP3(pu1_src[(ht - 1) * src_strd] + pi1_sao_offset_u[edge_idx], 0, (1 << bit_depth) - 1);
}
else
{
u1_pos_0_ht_tmp_u = pu1_src[(ht - 1) * src_strd];
}
/* V */
edge_idx = 2 + SIGN(pu1_src[(ht - 1) * src_strd + 1] - pu1_src[(ht - 1) * src_strd + 1 + 2 - src_strd]) +
SIGN(pu1_src[(ht - 1) * src_strd + 1] - pu1_src_bot_left[1]);
edge_idx = gi1_table_edge_idx[edge_idx];
if(0 != edge_idx)
{
u1_pos_0_ht_tmp_v = CLIP3(pu1_src[(ht - 1) * src_strd + 1] + pi1_sao_offset_v[edge_idx], 0, (1 << bit_depth) - 1);
}
else
{
u1_pos_0_ht_tmp_v = pu1_src[(ht - 1) * src_strd + 1];
}
}
else
{
u1_pos_0_ht_tmp_u = pu1_src[(ht - 1) * src_strd];
u1_pos_0_ht_tmp_v = pu1_src[(ht - 1) * src_strd + 1];
}
/* Update height and source pointers based on the availability flags */
if(0 == pu1_avail[2])
{
pu1_src_left_cpy2 += 2;
pu1_src_top_cpy = pu1_src;
pu1_src += src_strd;
ht--;
}
if(0 == pu1_avail[3])
{
ht--;
}
sao_offset_8x16b = _mm_unpacklo_epi64(sao_offset_8x16b, const0_16x8b);
const2_16x8b = _mm_set1_epi8(2);
const0_16x8b = _mm_setzero_si128();
//availability mask creation
u1_avail0 = pu1_avail[0];
u1_avail1 = pu1_avail[1];
au1_mask[0] = u1_avail0;
au1_mask[1] = u1_avail0;
au1_mask[wd - 1] = u1_avail1;
au1_mask[wd - 2] = u1_avail1;
{
WORD32 ht_rem;
au1_mask_cpy = au1_mask;
for(col = wd; col >= 16; col -= 16)
{
pu1_src_cpy = pu1_src;
src_top_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_top_cpy + wd - col + 2));
//row = 0
src_temp0_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy));
//loading the mask
au1_mask8x16b = _mm_loadu_si128((__m128i *)au1_mask_cpy);
//separating +ve and and -ve values.
cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, src_top_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(src_top_16x8b, src_temp0_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//combining the appropriate sign change
signup0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b);
pu1_src_left_cpy = pu1_src_left_cpy2;
for(row = ht; row >= 2; row -= 2)
{
left_store_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_left_cpy));
//row = 1
src_temp1_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + src_strd));
//to insert left in row 1
signdwn1_16x8b = _mm_slli_si128(left_store_16x8b, 12);
// row = 0 right
src_bottom_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + 2));
//manipulation for row 1 - row 0
signdwn1_16x8b = _mm_alignr_epi8(src_temp1_16x8b, signdwn1_16x8b, 14);
//row 0 -row1
//separating +ve and and -ve values.
cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, signdwn1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp0_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//combining the appropriate sign change
edge0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(0-1)
//combining sign-left and sign_right
edge0_16x8b = _mm_add_epi8(edge0_16x8b, signup0_16x8b);
//row1-row0
//separating +ve and and -ve values.
cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, src_bottom_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, src_temp1_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
// row = 2
src_bottom_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + 2 * src_strd));
// row = 1 right
signdwn1_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + src_strd + 2));
edge1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(1-0)
//bottom - row1
cmp_gt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, signdwn1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_bottom_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//for the next iteration bottom -row1
signup0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b);
//to insert left in row 1
signdwn1_16x8b = _mm_slli_si128(left_store_16x8b, 10);
//manipulation for row 1 - bottom
signdwn1_16x8b = _mm_alignr_epi8(src_bottom_16x8b, signdwn1_16x8b, 14);
//row1 -bottom
cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, signdwn1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp1_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//combining the appropriate sign change
signdwn1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b);
//combining sign-left and sign_right
edge1_16x8b = _mm_add_epi8(edge1_16x8b, signdwn1_16x8b);
//eliminating old left for row 0 and row 1
left_store_16x8b = _mm_srli_si128(left_store_16x8b, 4);
//row1 getting it right for left of next block
left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, src_temp1_16x8b, 14);
//row0 getting it right for left of next block
left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, src_temp0_16x8b, 14);
//copying the next top
src_top_16x8b = src_temp1_16x8b;
//adding constant 2
edge0_16x8b = _mm_add_epi8(edge0_16x8b, const2_16x8b);
edge1_16x8b = _mm_add_epi8(edge1_16x8b, const2_16x8b);
//shuffle to get sao index
edge0_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge0_16x8b);
edge1_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge1_16x8b);
//using availability mask
edge0_16x8b = _mm_and_si128(edge0_16x8b, au1_mask8x16b);
edge1_16x8b = _mm_and_si128(edge1_16x8b, au1_mask8x16b);
//adding chroma offset to access U and V
edge0_16x8b = _mm_add_epi8(edge0_16x8b, chroma_offset_8x16b);
edge1_16x8b = _mm_add_epi8(edge1_16x8b, chroma_offset_8x16b);
//shuffle to get sao offset
edge0_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge0_16x8b);
edge1_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge1_16x8b);
//cnvert to 16 bit then add and then saturated pack
signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge0_16x8b);
cmp_gt0_16x8b = _mm_unpacklo_epi8(src_temp0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_unpacklo_epi8(edge0_16x8b, signdwn1_16x8b);
src_temp0_16x8b = _mm_unpackhi_epi8(src_temp0_16x8b, const0_16x8b);
edge0_16x8b = _mm_unpackhi_epi8(edge0_16x8b, signdwn1_16x8b);
cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b);
src_temp0_16x8b = _mm_add_epi16(src_temp0_16x8b, edge0_16x8b);
src_temp0_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_temp0_16x8b);
signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge1_16x8b);
cmp_gt0_16x8b = _mm_unpacklo_epi8(src_temp1_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_unpacklo_epi8(edge1_16x8b, signdwn1_16x8b);
src_temp1_16x8b = _mm_unpackhi_epi8(src_temp1_16x8b, const0_16x8b);
edge1_16x8b = _mm_unpackhi_epi8(edge1_16x8b, signdwn1_16x8b);
cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b);
src_temp1_16x8b = _mm_add_epi16(src_temp1_16x8b, edge1_16x8b);
src_temp1_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_temp1_16x8b);
//store left boundary
_mm_storel_epi64((__m128i *)(pu1_src_left_cpy), left_store_16x8b);
//row = 0 store 8 pixel values from 7:0 pos. relative to cur. pos.
_mm_storeu_si128((__m128i *)(pu1_src_cpy), src_temp0_16x8b);
// row = 1
_mm_storeu_si128((__m128i *)(pu1_src_cpy + src_strd), src_temp1_16x8b);
src_temp0_16x8b = src_bottom_16x8b;
pu1_src_cpy += (src_strd << 1);
pu1_src_left_cpy += 4;
}
ht_rem = ht & 0x1;
if(ht_rem)
{
left_store_16x8b = _mm_loadl_epi64((__m128i *)pu1_src_left_cpy);
src_bottom_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + src_strd));
//to insert left in row 1
signdwn1_16x8b = _mm_slli_si128(left_store_16x8b, 12);
//manipulation for row 1 - row 0
signdwn1_16x8b = _mm_alignr_epi8(src_bottom_16x8b, signdwn1_16x8b, 14);
//current row -next row
//separating +ve and and -ve values.
cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, signdwn1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp0_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//combining the appropriate sign change
edge0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b);
//adding top and bottom and constant 2
edge0_16x8b = _mm_add_epi8(edge0_16x8b, signup0_16x8b);
edge0_16x8b = _mm_add_epi8(edge0_16x8b, const2_16x8b);
//eliminating old left for row 0 and row 1
left_store_16x8b = _mm_srli_si128(left_store_16x8b, 2);
//row0 getting it right for left of next block
left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, src_temp0_16x8b, 14);
//copying the next top
src_top_16x8b = src_temp0_16x8b;
edge0_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge0_16x8b);
//using availability mask
edge0_16x8b = _mm_and_si128(edge0_16x8b, au1_mask8x16b);
//adding chroma offset to access U and V
edge0_16x8b = _mm_add_epi8(edge0_16x8b, chroma_offset_8x16b);
edge0_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge0_16x8b);
//cnvert to 16 bit then add and then saturated pack
signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge0_16x8b);
cmp_gt0_16x8b = _mm_unpacklo_epi8(src_temp0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_unpacklo_epi8(edge0_16x8b, signdwn1_16x8b);
src_temp0_16x8b = _mm_unpackhi_epi8(src_temp0_16x8b, const0_16x8b);
edge0_16x8b = _mm_unpackhi_epi8(edge0_16x8b, signdwn1_16x8b);
cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b);
src_temp0_16x8b = _mm_add_epi16(src_temp0_16x8b, edge0_16x8b);
src_temp0_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_temp0_16x8b);
//store left boundary
_mm_storel_epi64((__m128i *)(pu1_src_left_cpy), left_store_16x8b);
_mm_storeu_si128((__m128i *)(pu1_src_cpy), src_temp0_16x8b);
pu1_src_cpy += (src_strd);
src_temp0_16x8b = src_bottom_16x8b;
pu1_src_left_cpy += 2;
}
{ //for bottom right
left_store_16x8b = _mm_loadl_epi64((__m128i *)pu1_src_left_cpy);
left_store_16x8b = _mm_srli_si128(left_store_16x8b, 2);
left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, src_temp0_16x8b, 14);
_mm_storel_epi64((__m128i *)(pu1_src_left_cpy), left_store_16x8b);
}
if(0 == pu1_avail[3])
{
src_top_16x8b = src_bottom_16x8b;
}
//for the top left of next part of the block
left_store_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_top_cpy + wd - col));
//updating top flag
_mm_storeu_si128((__m128i *)(pu1_src_top + wd - col), src_top_16x8b);
pu1_src += 16;
au1_mask_cpy += 16;
}
pu1_src_left_cpy = pu1_src_left_cpy2;
wd_rem = wd & 0xF;
if(wd_rem)
{
pu1_src_cpy = pu1_src;
src_top_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_top_cpy + wd - col + 2));
//row = 0
src_temp0_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy));
au1_mask8x16b = _mm_loadl_epi64((__m128i *)au1_mask_cpy); //????
//separating +ve and and -ve values.
cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, src_top_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(src_top_16x8b, src_temp0_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//preparing au1_mask
au1_mask8x16b = _mm_unpacklo_epi64(au1_mask8x16b, au1_mask8x16b);
//combining the appropriate sign change
signup0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b);
signup0_16x8b = _mm_slli_si128(signup0_16x8b, 8);
pu1_src_left_cpy = pu1_src_left_cpy2;
for(row = ht; row >= 4; row -= 4)
{
left_store_16x8b = _mm_loadu_si128((__m128i *)pu1_src_left_cpy);
//row = 1 load 8 pixel values from 7:0 pos. relative to cur. pos.
src_temp1_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + src_strd));
// row = 2
src_bottom_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + 2 * src_strd));
//manipulation for row 0 -row 1
signdwn1_16x8b = _mm_slli_si128(left_store_16x8b, 12);
//row 1 left
signdwn1_16x8b = _mm_alignr_epi8(src_temp1_16x8b, signdwn1_16x8b, 14);
//row 0 -row1
//separating +ve and and -ve values.
cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, signdwn1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp0_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//manipulatiing for row 1 -row 0
signdwn1_16x8b = _mm_srli_si128(src_temp0_16x8b, 2);
//combining the appropriate sign change
edge0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b);
//row 1 -row0
//separating +ve and and -ve values.
cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, signdwn1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp1_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//row1-row0
edge1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b);
edge0_16x8b = _mm_slli_si128(edge0_16x8b, 8); //aligned left (0-1)
signup0_16x8b = _mm_alignr_epi8(edge1_16x8b, signup0_16x8b, 8); //(1-0),(0-top)
//manipulation for row 1 -row 2
signdwn1_16x8b = _mm_slli_si128(left_store_16x8b, 10);
//row 2 left
signdwn1_16x8b = _mm_alignr_epi8(src_bottom_16x8b, signdwn1_16x8b, 14);
//packing row 0 n row 1
src_temp0_16x8b = _mm_unpacklo_epi64(src_temp0_16x8b, src_temp1_16x8b);
//row1 -row2
cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, signdwn1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp1_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//combining the appropriate sign change
signdwn1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(1-2)
edge0_16x8b = _mm_alignr_epi8(signdwn1_16x8b, edge0_16x8b, 8); //(1-2),(0-1)
//row 1 right
signdwn1_16x8b = _mm_srli_si128(src_temp1_16x8b, 2);
//row = 3
src_top_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + 3 * src_strd));
// row = 4
src_temp1_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + 4 * src_strd));
edge0_16x8b = _mm_add_epi8(edge0_16x8b, signup0_16x8b); //(1,0) sign_up empty
//separating +ve and and -ve values.(2,1)
cmp_gt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, signdwn1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_bottom_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//row 2 right
signdwn1_16x8b = _mm_srli_si128(src_bottom_16x8b, 2);
//combining the appropriate sign change
signup0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(2-1)
//separating +ve and and -ve values.(3,2)
cmp_gt0_16x8b = _mm_subs_epu8(src_top_16x8b, signdwn1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_top_16x8b);
signup0_16x8b = _mm_slli_si128(signup0_16x8b, 8); //aligned left (2-1)
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//manipulation for row 2 -row 3
signdwn1_16x8b = _mm_slli_si128(left_store_16x8b, 8);
//row 3 left
signdwn1_16x8b = _mm_alignr_epi8(src_top_16x8b, signdwn1_16x8b, 14);
//combining the appropriate sign change
edge1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(3-2)
signup0_16x8b = _mm_alignr_epi8(edge1_16x8b, signup0_16x8b, 8); //(3-2) ,(2-1)
//separating +ve and and -ve values.(2,3)
cmp_gt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, signdwn1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_bottom_16x8b);
//manipulation for row 3 -bottom
signdwn1_16x8b = _mm_slli_si128(left_store_16x8b, 6);
//bottom left
signdwn1_16x8b = _mm_alignr_epi8(src_temp1_16x8b, signdwn1_16x8b, 14);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//combining the appropriate sign change
edge1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(2-3)
//separating +ve and and -ve values.(3,bottom)
cmp_gt0_16x8b = _mm_subs_epu8(src_top_16x8b, signdwn1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_top_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
edge1_16x8b = _mm_slli_si128(edge1_16x8b, 8); //aligned left (2-3)
//combining the appropriate sign change
signdwn1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(3-bottom)
edge1_16x8b = _mm_alignr_epi8(signdwn1_16x8b, edge1_16x8b, 8); //(3-bottom),(2-3)
//eliminating old left for row 0,1,2,3
left_store_16x8b = _mm_srli_si128(left_store_16x8b, 8);
//packing row 2 n row 3
src_bottom_16x8b = _mm_unpacklo_epi64(src_bottom_16x8b, src_top_16x8b);
//row 3 right
signdwn1_16x8b = _mm_srli_si128(src_top_16x8b, 2);
//loading row 3 right into left
left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, src_bottom_16x8b, 14);
//adding bottom and top values of row 2 and row 3
edge1_16x8b = _mm_add_epi8(edge1_16x8b, signup0_16x8b); //(3,2)
//separating +ve and and -ve values.(botttom,3)
cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, signdwn1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp1_16x8b);
//to store right of row 2
signdwn1_16x8b = _mm_slli_si128(src_bottom_16x8b, 8);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
signup0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(bottom -3) for next iteration
//storing right of row 2into left
left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, signdwn1_16x8b, 14);
//to store right of row 0
signdwn1_16x8b = _mm_slli_si128(src_temp0_16x8b, 8);
//storing right of row 1 into left
left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, src_temp0_16x8b, 14);
//storing right of row 0 into left
left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, signdwn1_16x8b, 14);
//adding constant 2
edge0_16x8b = _mm_add_epi8(edge0_16x8b, const2_16x8b);
edge1_16x8b = _mm_add_epi8(edge1_16x8b, const2_16x8b);
//shuffle to get sao index
edge0_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge0_16x8b);
edge1_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge1_16x8b);
//using availability mask
edge0_16x8b = _mm_and_si128(edge0_16x8b, au1_mask8x16b);
edge1_16x8b = _mm_and_si128(edge1_16x8b, au1_mask8x16b);
//adding chroma offset to access U and V
edge0_16x8b = _mm_add_epi8(edge0_16x8b, chroma_offset_8x16b);
edge1_16x8b = _mm_add_epi8(edge1_16x8b, chroma_offset_8x16b);
//shuffle to get sao offset
edge0_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge0_16x8b);
edge1_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge1_16x8b);
//cnvert to 16 bit then add and then saturated pack
signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge0_16x8b);
cmp_gt0_16x8b = _mm_unpacklo_epi8(src_temp0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_unpacklo_epi8(edge0_16x8b, signdwn1_16x8b);
src_temp0_16x8b = _mm_unpackhi_epi8(src_temp0_16x8b, const0_16x8b);
edge0_16x8b = _mm_unpackhi_epi8(edge0_16x8b, signdwn1_16x8b);
cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b);
src_temp0_16x8b = _mm_add_epi16(src_temp0_16x8b, edge0_16x8b);
src_temp0_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_temp0_16x8b);
signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge1_16x8b);
cmp_gt0_16x8b = _mm_unpacklo_epi8(src_bottom_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_unpacklo_epi8(edge1_16x8b, signdwn1_16x8b);
src_bottom_16x8b = _mm_unpackhi_epi8(src_bottom_16x8b, const0_16x8b);
edge1_16x8b = _mm_unpackhi_epi8(edge1_16x8b, signdwn1_16x8b);
cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b);
src_bottom_16x8b = _mm_add_epi16(src_bottom_16x8b, edge1_16x8b);
src_bottom_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_bottom_16x8b);
cmp_gt0_16x8b = _mm_srli_si128(src_temp0_16x8b, 8);
cmp_lt0_16x8b = _mm_srli_si128(src_bottom_16x8b, 8);
_mm_storel_epi64((__m128i *)(pu1_src_left_cpy), left_store_16x8b);
//row = 0 store 8 pixel values from 7:0 pos. relative to cur. pos.
_mm_storel_epi64((__m128i *)(pu1_src_cpy), src_temp0_16x8b);
// row = 1
_mm_storel_epi64((__m128i *)(pu1_src_cpy + src_strd), cmp_gt0_16x8b);
//row = 2
_mm_storel_epi64((__m128i *)(pu1_src_cpy + 2 * src_strd), src_bottom_16x8b);
// row = 3
_mm_storel_epi64((__m128i *)(pu1_src_cpy + 3 * src_strd), cmp_lt0_16x8b);
src_temp0_16x8b = src_temp1_16x8b;
signup0_16x8b = _mm_slli_si128(signup0_16x8b, 8);
pu1_src_cpy += (src_strd << 2);
pu1_src_left_cpy += 8;
}
ht_rem = ht & 0x2;
if(ht_rem)
{
left_store_16x8b = _mm_loadl_epi64((__m128i *)pu1_src_left_cpy);
//row = 1 load 8 pixel values from 7:0 pos. relative to cur. pos.
src_temp1_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + src_strd));
// row = 2
src_bottom_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + 2 * src_strd));
//manipulation for row 0 -row 1
signdwn1_16x8b = _mm_slli_si128(left_store_16x8b, 12);
//bottom left
signdwn1_16x8b = _mm_alignr_epi8(src_temp1_16x8b, signdwn1_16x8b, 14);
//separating +ve and and -ve values.
cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, signdwn1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp0_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//manipulation for row 1 - row 0
signdwn1_16x8b = _mm_srli_si128(src_temp0_16x8b, 2);
//combining the appropriate sign change
edge0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b);
//row1-row0
//separating +ve and and -ve values.
cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, signdwn1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp1_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//combining the appropriate sign chang
edge1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b);
//manipulation for row 1 -bottom
signdwn1_16x8b = _mm_slli_si128(left_store_16x8b, 10);
//bottom left
signdwn1_16x8b = _mm_alignr_epi8(src_bottom_16x8b, signdwn1_16x8b, 14);
edge0_16x8b = _mm_slli_si128(edge0_16x8b, 8); //aligned left (0-1)
signup0_16x8b = _mm_alignr_epi8(edge1_16x8b, signup0_16x8b, 8); //(1-0),(0-top)
//row1 -bottom
cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, signdwn1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp1_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//combining the appropriate sign change
signdwn1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(1-2)
edge0_16x8b = _mm_alignr_epi8(signdwn1_16x8b, edge0_16x8b, 8); //(1-2),(0-1)
//manipulation for bottom- row 1 (row 1 right)
signdwn1_16x8b = _mm_srli_si128(src_temp1_16x8b, 2);
//adding top and down substraction
edge0_16x8b = _mm_add_epi8(edge0_16x8b, signup0_16x8b); //(1,0) sign_up empty
//bottom - row 1
cmp_gt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, signdwn1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_bottom_16x8b);
//eliminating old left for row 0,1
left_store_16x8b = _mm_srli_si128(left_store_16x8b, 4);
signdwn1_16x8b = _mm_slli_si128(src_temp1_16x8b, 8);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//for the next iteration signup0_16x8b
signup0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(2-1) for next
//storing right of row 1 into left
left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, signdwn1_16x8b, 14);
//for storing right of row 1
signdwn1_16x8b = _mm_slli_si128(src_temp0_16x8b, 8);
src_top_16x8b = src_temp1_16x8b;
//storing right of row 0 into left
left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, signdwn1_16x8b, 14);
//adding constant 2
edge0_16x8b = _mm_add_epi8(edge0_16x8b, const2_16x8b);
//shuffle to get sao index
edge0_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge0_16x8b);
//using availability mask
edge0_16x8b = _mm_and_si128(edge0_16x8b, au1_mask8x16b);
//adding chroma offset to access U and V
edge0_16x8b = _mm_add_epi8(edge0_16x8b, chroma_offset_8x16b);
//shuffle to get sao offset
edge0_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge0_16x8b);
//the next top already in src_top_16x8b
//cnvert to 16 bit then add and then saturated pack
signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge0_16x8b);
src_temp0_16x8b = _mm_unpacklo_epi8(src_temp0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_unpacklo_epi8(edge0_16x8b, signdwn1_16x8b);
src_temp1_16x8b = _mm_unpacklo_epi8(src_temp1_16x8b, const0_16x8b);
edge0_16x8b = _mm_unpackhi_epi8(edge0_16x8b, signdwn1_16x8b);
src_temp0_16x8b = _mm_add_epi16(src_temp0_16x8b, cmp_lt0_16x8b);
src_temp1_16x8b = _mm_add_epi16(src_temp1_16x8b, edge0_16x8b);
src_temp0_16x8b = _mm_packus_epi16(src_temp0_16x8b, src_temp1_16x8b);
cmp_gt0_16x8b = _mm_srli_si128(src_temp0_16x8b, 8);
_mm_storel_epi64((__m128i *)(pu1_src_left_cpy), left_store_16x8b);
//row = 0 store 8 pixel values from 7:0 pos. relative to cur. pos.
_mm_storel_epi64((__m128i *)(pu1_src_cpy), src_temp0_16x8b);
// row = 1
_mm_storel_epi64((__m128i *)(pu1_src_cpy + src_strd), cmp_gt0_16x8b);
src_temp0_16x8b = src_bottom_16x8b;
pu1_src_cpy += (src_strd << 1);
pu1_src_left_cpy += 4;
}
ht_rem = ht & 0x1;
if(ht_rem)
{
left_store_16x8b = _mm_loadl_epi64((__m128i *)pu1_src_left_cpy);
//row = 1 load 8 pixel values from 7:0 pos. relative to cur. pos.
src_bottom_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + src_strd));
//manipulation for row 0 -bottom
signdwn1_16x8b = _mm_slli_si128(left_store_16x8b, 12);
//bottom left
signdwn1_16x8b = _mm_alignr_epi8(src_bottom_16x8b, signdwn1_16x8b, 14);
//separating +ve and and -ve values.
cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, signdwn1_16x8b);
cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp0_16x8b);
//creating mask 00 for +ve and -ve values and FF for zero.
cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b);
//combining the appropriate sign change
edge0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b);
//adding top and down substraction
edge0_16x8b = _mm_add_epi8(edge0_16x8b, signup0_16x8b);
//for row 0 right to put into left store
signdwn1_16x8b = _mm_slli_si128(src_temp0_16x8b, 8);
//adding constant 2
edge0_16x8b = _mm_add_epi8(edge0_16x8b, const2_16x8b);
edge0_16x8b = _mm_slli_si128(edge0_16x8b, 8);
edge0_16x8b = _mm_srli_si128(edge0_16x8b, 8);
//left store manipulation 1
left_store_16x8b = _mm_srli_si128(left_store_16x8b, 2);
//filling the left boundary value
left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, signdwn1_16x8b, 14);
src_top_16x8b = src_temp0_16x8b;
//shuffle to get sao index
edge0_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge0_16x8b);
//using availability mask
edge0_16x8b = _mm_and_si128(edge0_16x8b, au1_mask8x16b);
//adding chroma offset to access U and V
edge0_16x8b = _mm_add_epi8(edge0_16x8b, chroma_offset_8x16b);
//shuffle to get sao offset
edge0_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge0_16x8b);
//cnvert to 16 bit then add and then saturated pack
signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge0_16x8b);
src_temp0_16x8b = _mm_unpacklo_epi8(src_temp0_16x8b, const0_16x8b);
cmp_lt0_16x8b = _mm_unpacklo_epi8(edge0_16x8b, signdwn1_16x8b);
src_temp0_16x8b = _mm_add_epi16(src_temp0_16x8b, cmp_lt0_16x8b);
src_temp0_16x8b = _mm_packus_epi16(src_temp0_16x8b, const0_16x8b);
_mm_storel_epi64((__m128i *)(pu1_src_left_cpy), left_store_16x8b);
//row = 0 store 8 pixel values from 7:0 pos. relative to cur. pos.
_mm_storel_epi64((__m128i *)(pu1_src_cpy), src_temp0_16x8b);
pu1_src_cpy += (src_strd);
src_temp0_16x8b = src_bottom_16x8b;
pu1_src_left_cpy += 2;
}
{ //for bottom right
left_store_16x8b = _mm_loadl_epi64((__m128i *)pu1_src_left_cpy);
left_store_16x8b = _mm_srli_si128(left_store_16x8b, 2);
src_temp0_16x8b = _mm_slli_si128(src_temp0_16x8b, 8);
left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, src_temp0_16x8b, 14);
_mm_storel_epi64((__m128i *)(pu1_src_left_cpy), left_store_16x8b);
}
if(0 == pu1_avail[3])
{
src_top_16x8b = src_bottom_16x8b;
}
_mm_storel_epi64((__m128i *)(pu1_src_top + wd - col), src_top_16x8b);
pu1_src += 8;
}
pu1_src_org[wd - 2] = u1_pos_wd_0_tmp_u;
pu1_src_org[wd - 1] = u1_pos_wd_0_tmp_v;
pu1_src_org[(ht_tmp - 1) * src_strd] = u1_pos_0_ht_tmp_u;
pu1_src_org[(ht_tmp - 1) * src_strd + 1] = u1_pos_0_ht_tmp_v;
for(row = 0; row < 2 * ht_tmp; row++)
{
pu1_src_left[row] = au1_src_left_tmp[row];
}
}
}