;
; Copyright (c) 2010 The WebM project authors. All Rights Reserved.
;
; Use of this source code is governed by a BSD-style license
; that can be found in the LICENSE file in the root of the source
; tree. An additional intellectual property rights grant can be found
; in the file PATENTS. All contributing project authors may
; be found in the AUTHORS file in the root of the source tree.
;
%include "vpx_ports/x86_abi_support.asm"
%idefine QWORD
%macro PROCESS_16X2X3 1
%if %1
movdqa xmm0, [rsi]
lddqu xmm5, [rdi]
lddqu xmm6, [rdi+1]
lddqu xmm7, [rdi+2]
psadbw xmm5, xmm0
psadbw xmm6, xmm0
psadbw xmm7, xmm0
%else
movdqa xmm0, [rsi]
lddqu xmm1, [rdi]
lddqu xmm2, [rdi+1]
lddqu xmm3, [rdi+2]
psadbw xmm1, xmm0
psadbw xmm2, xmm0
psadbw xmm3, xmm0
paddw xmm5, xmm1
paddw xmm6, xmm2
paddw xmm7, xmm3
%endif
movdqa xmm0, QWORD PTR [rsi+rax]
lddqu xmm1, QWORD PTR [rdi+rdx]
lddqu xmm2, QWORD PTR [rdi+rdx+1]
lddqu xmm3, QWORD PTR [rdi+rdx+2]
lea rsi, [rsi+rax*2]
lea rdi, [rdi+rdx*2]
psadbw xmm1, xmm0
psadbw xmm2, xmm0
psadbw xmm3, xmm0
paddw xmm5, xmm1
paddw xmm6, xmm2
paddw xmm7, xmm3
%endmacro
%macro PROCESS_8X2X3 1
%if %1
movq mm0, [rsi]
movq mm5, [rdi]
movq mm6, [rdi+1]
movq mm7, [rdi+2]
psadbw mm5, mm0
psadbw mm6, mm0
psadbw mm7, mm0
%else
movq mm0, [rsi]
movq mm1, [rdi]
movq mm2, [rdi+1]
movq mm3, [rdi+2]
psadbw mm1, mm0
psadbw mm2, mm0
psadbw mm3, mm0
paddw mm5, mm1
paddw mm6, mm2
paddw mm7, mm3
%endif
movq mm0, QWORD PTR [rsi+rax]
movq mm1, QWORD PTR [rdi+rdx]
movq mm2, QWORD PTR [rdi+rdx+1]
movq mm3, QWORD PTR [rdi+rdx+2]
lea rsi, [rsi+rax*2]
lea rdi, [rdi+rdx*2]
psadbw mm1, mm0
psadbw mm2, mm0
psadbw mm3, mm0
paddw mm5, mm1
paddw mm6, mm2
paddw mm7, mm3
%endmacro
%macro LOAD_X4_ADDRESSES 5
mov %2, [%1+REG_SZ_BYTES*0]
mov %3, [%1+REG_SZ_BYTES*1]
mov %4, [%1+REG_SZ_BYTES*2]
mov %5, [%1+REG_SZ_BYTES*3]
%endmacro
%macro PROCESS_16X2X4 1
%if %1
movdqa xmm0, [rsi]
lddqu xmm4, [rcx]
lddqu xmm5, [rdx]
lddqu xmm6, [rbx]
lddqu xmm7, [rdi]
psadbw xmm4, xmm0
psadbw xmm5, xmm0
psadbw xmm6, xmm0
psadbw xmm7, xmm0
%else
movdqa xmm0, [rsi]
lddqu xmm1, [rcx]
lddqu xmm2, [rdx]
lddqu xmm3, [rbx]
psadbw xmm1, xmm0
psadbw xmm2, xmm0
psadbw xmm3, xmm0
paddw xmm4, xmm1
lddqu xmm1, [rdi]
paddw xmm5, xmm2
paddw xmm6, xmm3
psadbw xmm1, xmm0
paddw xmm7, xmm1
%endif
movdqa xmm0, QWORD PTR [rsi+rax]
lddqu xmm1, QWORD PTR [rcx+rbp]
lddqu xmm2, QWORD PTR [rdx+rbp]
lddqu xmm3, QWORD PTR [rbx+rbp]
psadbw xmm1, xmm0
psadbw xmm2, xmm0
psadbw xmm3, xmm0
paddw xmm4, xmm1
lddqu xmm1, QWORD PTR [rdi+rbp]
paddw xmm5, xmm2
paddw xmm6, xmm3
lea rsi, [rsi+rax*2]
lea rcx, [rcx+rbp*2]
lea rdx, [rdx+rbp*2]
lea rbx, [rbx+rbp*2]
lea rdi, [rdi+rbp*2]
psadbw xmm1, xmm0
paddw xmm7, xmm1
%endmacro
%macro PROCESS_8X2X4 1
%if %1
movq mm0, [rsi]
movq mm4, [rcx]
movq mm5, [rdx]
movq mm6, [rbx]
movq mm7, [rdi]
psadbw mm4, mm0
psadbw mm5, mm0
psadbw mm6, mm0
psadbw mm7, mm0
%else
movq mm0, [rsi]
movq mm1, [rcx]
movq mm2, [rdx]
movq mm3, [rbx]
psadbw mm1, mm0
psadbw mm2, mm0
psadbw mm3, mm0
paddw mm4, mm1
movq mm1, [rdi]
paddw mm5, mm2
paddw mm6, mm3
psadbw mm1, mm0
paddw mm7, mm1
%endif
movq mm0, QWORD PTR [rsi+rax]
movq mm1, QWORD PTR [rcx+rbp]
movq mm2, QWORD PTR [rdx+rbp]
movq mm3, QWORD PTR [rbx+rbp]
psadbw mm1, mm0
psadbw mm2, mm0
psadbw mm3, mm0
paddw mm4, mm1
movq mm1, QWORD PTR [rdi+rbp]
paddw mm5, mm2
paddw mm6, mm3
lea rsi, [rsi+rax*2]
lea rcx, [rcx+rbp*2]
lea rdx, [rdx+rbp*2]
lea rbx, [rbx+rbp*2]
lea rdi, [rdi+rbp*2]
psadbw mm1, mm0
paddw mm7, mm1
%endmacro
;void int vp8_sad16x16x3_sse3(
; unsigned char *src_ptr,
; int src_stride,
; unsigned char *ref_ptr,
; int ref_stride,
; int *results)
global sym(vp8_sad16x16x3_sse3)
sym(vp8_sad16x16x3_sse3):
push rbp
mov rbp, rsp
SHADOW_ARGS_TO_STACK 5
push rsi
push rdi
; end prolog
mov rsi, arg(0) ;src_ptr
mov rdi, arg(2) ;ref_ptr
movsxd rax, dword ptr arg(1) ;src_stride
movsxd rdx, dword ptr arg(3) ;ref_stride
PROCESS_16X2X3 1
PROCESS_16X2X3 0
PROCESS_16X2X3 0
PROCESS_16X2X3 0
PROCESS_16X2X3 0
PROCESS_16X2X3 0
PROCESS_16X2X3 0
PROCESS_16X2X3 0
mov rdi, arg(4) ;Results
movq xmm0, xmm5
psrldq xmm5, 8
paddw xmm0, xmm5
movd [rdi], xmm0
;-
movq xmm0, xmm6
psrldq xmm6, 8
paddw xmm0, xmm6
movd [rdi+4], xmm0
;-
movq xmm0, xmm7
psrldq xmm7, 8
paddw xmm0, xmm7
movd [rdi+8], xmm0
; begin epilog
pop rdi
pop rsi
UNSHADOW_ARGS
pop rbp
ret
;void int vp8_sad16x8x3_sse3(
; unsigned char *src_ptr,
; int src_stride,
; unsigned char *ref_ptr,
; int ref_stride,
; int *results)
global sym(vp8_sad16x8x3_sse3)
sym(vp8_sad16x8x3_sse3):
push rbp
mov rbp, rsp
SHADOW_ARGS_TO_STACK 5
push rsi
push rdi
; end prolog
mov rsi, arg(0) ;src_ptr
mov rdi, arg(2) ;ref_ptr
movsxd rax, dword ptr arg(1) ;src_stride
movsxd rdx, dword ptr arg(3) ;ref_stride
PROCESS_16X2X3 1
PROCESS_16X2X3 0
PROCESS_16X2X3 0
PROCESS_16X2X3 0
mov rdi, arg(4) ;Results
movq xmm0, xmm5
psrldq xmm5, 8
paddw xmm0, xmm5
movd [rdi], xmm0
;-
movq xmm0, xmm6
psrldq xmm6, 8
paddw xmm0, xmm6
movd [rdi+4], xmm0
;-
movq xmm0, xmm7
psrldq xmm7, 8
paddw xmm0, xmm7
movd [rdi+8], xmm0
; begin epilog
pop rdi
pop rsi
UNSHADOW_ARGS
pop rbp
ret
;void int vp8_sad8x16x3_sse3(
; unsigned char *src_ptr,
; int src_stride,
; unsigned char *ref_ptr,
; int ref_stride,
; int *results)
global sym(vp8_sad8x16x3_sse3)
sym(vp8_sad8x16x3_sse3):
push rbp
mov rbp, rsp
SHADOW_ARGS_TO_STACK 5
push rsi
push rdi
; end prolog
mov rsi, arg(0) ;src_ptr
mov rdi, arg(2) ;ref_ptr
movsxd rax, dword ptr arg(1) ;src_stride
movsxd rdx, dword ptr arg(3) ;ref_stride
PROCESS_8X2X3 1
PROCESS_8X2X3 0
PROCESS_8X2X3 0
PROCESS_8X2X3 0
PROCESS_8X2X3 0
PROCESS_8X2X3 0
PROCESS_8X2X3 0
PROCESS_8X2X3 0
mov rdi, arg(4) ;Results
movd [rdi], mm5
movd [rdi+4], mm6
movd [rdi+8], mm7
; begin epilog
pop rdi
pop rsi
UNSHADOW_ARGS
pop rbp
ret
;void int vp8_sad8x8x3_sse3(
; unsigned char *src_ptr,
; int src_stride,
; unsigned char *ref_ptr,
; int ref_stride,
; int *results)
global sym(vp8_sad8x8x3_sse3)
sym(vp8_sad8x8x3_sse3):
push rbp
mov rbp, rsp
SHADOW_ARGS_TO_STACK 5
push rsi
push rdi
; end prolog
mov rsi, arg(0) ;src_ptr
mov rdi, arg(2) ;ref_ptr
movsxd rax, dword ptr arg(1) ;src_stride
movsxd rdx, dword ptr arg(3) ;ref_stride
PROCESS_8X2X3 1
PROCESS_8X2X3 0
PROCESS_8X2X3 0
PROCESS_8X2X3 0
mov rdi, arg(4) ;Results
movd [rdi], mm5
movd [rdi+4], mm6
movd [rdi+8], mm7
; begin epilog
pop rdi
pop rsi
UNSHADOW_ARGS
pop rbp
ret
;void int vp8_sad4x4x3_sse3(
; unsigned char *src_ptr,
; int src_stride,
; unsigned char *ref_ptr,
; int ref_stride,
; int *results)
global sym(vp8_sad4x4x3_sse3)
sym(vp8_sad4x4x3_sse3):
push rbp
mov rbp, rsp
SHADOW_ARGS_TO_STACK 5
push rsi
push rdi
; end prolog
mov rsi, arg(0) ;src_ptr
mov rdi, arg(2) ;ref_ptr
movsxd rax, dword ptr arg(1) ;src_stride
movsxd rdx, dword ptr arg(3) ;ref_stride
movd mm0, QWORD PTR [rsi]
movd mm1, QWORD PTR [rdi]
movd mm2, QWORD PTR [rsi+rax]
movd mm3, QWORD PTR [rdi+rdx]
punpcklbw mm0, mm2
punpcklbw mm1, mm3
movd mm4, QWORD PTR [rdi+1]
movd mm5, QWORD PTR [rdi+2]
movd mm2, QWORD PTR [rdi+rdx+1]
movd mm3, QWORD PTR [rdi+rdx+2]
psadbw mm1, mm0
punpcklbw mm4, mm2
punpcklbw mm5, mm3
psadbw mm4, mm0
psadbw mm5, mm0
lea rsi, [rsi+rax*2]
lea rdi, [rdi+rdx*2]
movd mm0, QWORD PTR [rsi]
movd mm2, QWORD PTR [rdi]
movd mm3, QWORD PTR [rsi+rax]
movd mm6, QWORD PTR [rdi+rdx]
punpcklbw mm0, mm3
punpcklbw mm2, mm6
movd mm3, QWORD PTR [rdi+1]
movd mm7, QWORD PTR [rdi+2]
psadbw mm2, mm0
paddw mm1, mm2
movd mm2, QWORD PTR [rdi+rdx+1]
movd mm6, QWORD PTR [rdi+rdx+2]
punpcklbw mm3, mm2
punpcklbw mm7, mm6
psadbw mm3, mm0
psadbw mm7, mm0
paddw mm3, mm4
paddw mm7, mm5
mov rdi, arg(4) ;Results
movd [rdi], mm1
movd [rdi+4], mm3
movd [rdi+8], mm7
; begin epilog
pop rdi
pop rsi
UNSHADOW_ARGS
pop rbp
ret
;unsigned int vp8_sad16x16_sse3(
; unsigned char *src_ptr,
; int src_stride,
; unsigned char *ref_ptr,
; int ref_stride,
; int max_err)
;%define lddqu movdqu
global sym(vp8_sad16x16_sse3)
sym(vp8_sad16x16_sse3):
push rbp
mov rbp, rsp
SHADOW_ARGS_TO_STACK 5
push rbx
push rsi
push rdi
; end prolog
mov rsi, arg(0) ;src_ptr
mov rdi, arg(2) ;ref_ptr
movsxd rbx, dword ptr arg(1) ;src_stride
movsxd rdx, dword ptr arg(3) ;ref_stride
lea rcx, [rsi+rbx*8]
lea rcx, [rcx+rbx*8]
pxor mm7, mm7
vp8_sad16x16_sse3_loop:
movd rax, mm7
cmp rax, arg(4)
jg vp8_sad16x16_early_exit
movq mm0, QWORD PTR [rsi]
movq mm2, QWORD PTR [rsi+8]
movq mm1, QWORD PTR [rdi]
movq mm3, QWORD PTR [rdi+8]
movq mm4, QWORD PTR [rsi+rbx]
movq mm5, QWORD PTR [rdi+rdx]
psadbw mm0, mm1
psadbw mm2, mm3
movq mm1, QWORD PTR [rsi+rbx+8]
movq mm3, QWORD PTR [rdi+rdx+8]
psadbw mm4, mm5
psadbw mm1, mm3
lea rsi, [rsi+rbx*2]
lea rdi, [rdi+rdx*2]
paddw mm0, mm2
paddw mm4, mm1
paddw mm7, mm0
paddw mm7, mm4
cmp rsi, rcx
jne vp8_sad16x16_sse3_loop
movd rax, mm7
vp8_sad16x16_early_exit:
; begin epilog
pop rdi
pop rsi
pop rbx
UNSHADOW_ARGS
pop rbp
ret
;void vp8_sad16x16x4d_sse3(
; unsigned char *src_ptr,
; int src_stride,
; unsigned char *ref_ptr_base,
; int ref_stride,
; int *results)
global sym(vp8_sad16x16x4d_sse3)
sym(vp8_sad16x16x4d_sse3):
push rbp
mov rbp, rsp
SHADOW_ARGS_TO_STACK 5
push rsi
push rdi
push rbx
; end prolog
push rbp
mov rdi, arg(2) ; ref_ptr_base
LOAD_X4_ADDRESSES rdi, rcx, rdx, rax, rdi
mov rsi, arg(0) ;src_ptr
movsxd rbx, dword ptr arg(1) ;src_stride
movsxd rbp, dword ptr arg(3) ;ref_stride
xchg rbx, rax
PROCESS_16X2X4 1
PROCESS_16X2X4 0
PROCESS_16X2X4 0
PROCESS_16X2X4 0
PROCESS_16X2X4 0
PROCESS_16X2X4 0
PROCESS_16X2X4 0
PROCESS_16X2X4 0
pop rbp
mov rdi, arg(4) ;Results
movq xmm0, xmm4
psrldq xmm4, 8
paddw xmm0, xmm4
movd [rdi], xmm0
;-
movq xmm0, xmm5
psrldq xmm5, 8
paddw xmm0, xmm5
movd [rdi+4], xmm0
;-
movq xmm0, xmm6
psrldq xmm6, 8
paddw xmm0, xmm6
movd [rdi+8], xmm0
;-
movq xmm0, xmm7
psrldq xmm7, 8
paddw xmm0, xmm7
movd [rdi+12], xmm0
; begin epilog
pop rbx
pop rdi
pop rsi
UNSHADOW_ARGS
pop rbp
ret
;void vp8_sad16x8x4d_sse3(
; unsigned char *src_ptr,
; int src_stride,
; unsigned char *ref_ptr_base,
; int ref_stride,
; int *results)
global sym(vp8_sad16x8x4d_sse3)
sym(vp8_sad16x8x4d_sse3):
push rbp
mov rbp, rsp
SHADOW_ARGS_TO_STACK 5
push rsi
push rdi
push rbx
; end prolog
push rbp
mov rdi, arg(2) ; ref_ptr_base
LOAD_X4_ADDRESSES rdi, rcx, rdx, rax, rdi
mov rsi, arg(0) ;src_ptr
movsxd rbx, dword ptr arg(1) ;src_stride
movsxd rbp, dword ptr arg(3) ;ref_stride
xchg rbx, rax
PROCESS_16X2X4 1
PROCESS_16X2X4 0
PROCESS_16X2X4 0
PROCESS_16X2X4 0
pop rbp
mov rdi, arg(4) ;Results
movq xmm0, xmm4
psrldq xmm4, 8
paddw xmm0, xmm4
movd [rdi], xmm0
;-
movq xmm0, xmm5
psrldq xmm5, 8
paddw xmm0, xmm5
movd [rdi+4], xmm0
;-
movq xmm0, xmm6
psrldq xmm6, 8
paddw xmm0, xmm6
movd [rdi+8], xmm0
;-
movq xmm0, xmm7
psrldq xmm7, 8
paddw xmm0, xmm7
movd [rdi+12], xmm0
; begin epilog
pop rbx
pop rdi
pop rsi
UNSHADOW_ARGS
pop rbp
ret
;void int vp8_sad8x16x4d_sse3(
; unsigned char *src_ptr,
; int src_stride,
; unsigned char *ref_ptr,
; int ref_stride,
; int *results)
global sym(vp8_sad8x16x4d_sse3)
sym(vp8_sad8x16x4d_sse3):
push rbp
mov rbp, rsp
SHADOW_ARGS_TO_STACK 5
push rsi
push rdi
push rbx
; end prolog
push rbp
mov rdi, arg(2) ; ref_ptr_base
LOAD_X4_ADDRESSES rdi, rcx, rdx, rax, rdi
mov rsi, arg(0) ;src_ptr
movsxd rbx, dword ptr arg(1) ;src_stride
movsxd rbp, dword ptr arg(3) ;ref_stride
xchg rbx, rax
PROCESS_8X2X4 1
PROCESS_8X2X4 0
PROCESS_8X2X4 0
PROCESS_8X2X4 0
PROCESS_8X2X4 0
PROCESS_8X2X4 0
PROCESS_8X2X4 0
PROCESS_8X2X4 0
pop rbp
mov rdi, arg(4) ;Results
movd [rdi], mm4
movd [rdi+4], mm5
movd [rdi+8], mm6
movd [rdi+12], mm7
; begin epilog
pop rbx
pop rdi
pop rsi
UNSHADOW_ARGS
pop rbp
ret
;void int vp8_sad8x8x4d_sse3(
; unsigned char *src_ptr,
; int src_stride,
; unsigned char *ref_ptr,
; int ref_stride,
; int *results)
global sym(vp8_sad8x8x4d_sse3)
sym(vp8_sad8x8x4d_sse3):
push rbp
mov rbp, rsp
SHADOW_ARGS_TO_STACK 5
push rsi
push rdi
push rbx
; end prolog
push rbp
mov rdi, arg(2) ; ref_ptr_base
LOAD_X4_ADDRESSES rdi, rcx, rdx, rax, rdi
mov rsi, arg(0) ;src_ptr
movsxd rbx, dword ptr arg(1) ;src_stride
movsxd rbp, dword ptr arg(3) ;ref_stride
xchg rbx, rax
PROCESS_8X2X4 1
PROCESS_8X2X4 0
PROCESS_8X2X4 0
PROCESS_8X2X4 0
pop rbp
mov rdi, arg(4) ;Results
movd [rdi], mm4
movd [rdi+4], mm5
movd [rdi+8], mm6
movd [rdi+12], mm7
; begin epilog
pop rbx
pop rdi
pop rsi
UNSHADOW_ARGS
pop rbp
ret
;void int vp8_sad4x4x4d_sse3(
; unsigned char *src_ptr,
; int src_stride,
; unsigned char *ref_ptr,
; int ref_stride,
; int *results)
global sym(vp8_sad4x4x4d_sse3)
sym(vp8_sad4x4x4d_sse3):
push rbp
mov rbp, rsp
SHADOW_ARGS_TO_STACK 5
push rsi
push rdi
push rbx
; end prolog
push rbp
mov rdi, arg(2) ; ref_ptr_base
LOAD_X4_ADDRESSES rdi, rcx, rdx, rax, rdi
mov rsi, arg(0) ;src_ptr
movsxd rbx, dword ptr arg(1) ;src_stride
movsxd rbp, dword ptr arg(3) ;ref_stride
xchg rbx, rax
movd mm0, QWORD PTR [rsi]
movd mm1, QWORD PTR [rcx]
movd mm2, QWORD PTR [rsi+rax]
movd mm3, QWORD PTR [rcx+rbp]
punpcklbw mm0, mm2
punpcklbw mm1, mm3
movd mm4, QWORD PTR [rdx]
movd mm5, QWORD PTR [rbx]
movd mm6, QWORD PTR [rdi]
movd mm2, QWORD PTR [rdx+rbp]
movd mm3, QWORD PTR [rbx+rbp]
movd mm7, QWORD PTR [rdi+rbp]
psadbw mm1, mm0
punpcklbw mm4, mm2
punpcklbw mm5, mm3
punpcklbw mm6, mm7
psadbw mm4, mm0
psadbw mm5, mm0
psadbw mm6, mm0
lea rsi, [rsi+rax*2]
lea rcx, [rcx+rbp*2]
lea rdx, [rdx+rbp*2]
lea rbx, [rbx+rbp*2]
lea rdi, [rdi+rbp*2]
movd mm0, QWORD PTR [rsi]
movd mm2, QWORD PTR [rcx]
movd mm3, QWORD PTR [rsi+rax]
movd mm7, QWORD PTR [rcx+rbp]
punpcklbw mm0, mm3
punpcklbw mm2, mm7
movd mm3, QWORD PTR [rdx]
movd mm7, QWORD PTR [rbx]
psadbw mm2, mm0
mov rax, rbp
pop rbp
mov rsi, arg(4) ;Results
paddw mm1, mm2
movd [rsi], mm1
movd mm2, QWORD PTR [rdx+rax]
movd mm1, QWORD PTR [rbx+rax]
punpcklbw mm3, mm2
punpcklbw mm7, mm1
psadbw mm3, mm0
psadbw mm7, mm0
movd mm2, QWORD PTR [rdi]
movd mm1, QWORD PTR [rdi+rax]
paddw mm3, mm4
paddw mm7, mm5
movd [rsi+4], mm3
punpcklbw mm2, mm1
movd [rsi+8], mm7
psadbw mm2, mm0
paddw mm2, mm6
movd [rsi+12], mm2
; begin epilog
pop rbx
pop rdi
pop rsi
UNSHADOW_ARGS
pop rbp
ret