;*!
;* \copy
;* Copyright (c) 2009-2013, Cisco Systems
;* All rights reserved.
;*
;* Redistribution and use in source and binary forms, with or without
;* modification, are permitted provided that the following conditions
;* are met:
;*
;* * Redistributions of source code must retain the above copyright
;* notice, this list of conditions and the following disclaimer.
;*
;* * Redistributions in binary form must reproduce the above copyright
;* notice, this list of conditions and the following disclaimer in
;* the documentation and/or other materials provided with the
;* distribution.
;*
;* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
;* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
;* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
;* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
;* COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
;* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
;* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
;* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
;* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
;* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
;* ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
;* POSSIBILITY OF SUCH DAMAGE.
;*
;*************************************************************************/
%include "asm_inc.asm"
;***********************************************************************
; Local Data (Read Only)
;***********************************************************************
%ifdef X86_32_PICASM
SECTION .text align=16
%else
SECTION .rodata align=16
%endif
ALIGN 16
mv_x_inc_x4 dw 0x10, 0x10, 0x10, 0x10
mv_y_inc_x4 dw 0x04, 0x04, 0x04, 0x04
mx_x_offset_x4 dw 0x00, 0x04, 0x08, 0x0C
SECTION .text
%ifdef X86_32
;**********************************************************************************************************************
;void SumOf8x8BlockOfFrame_sse2(uint8_t* pRefPicture, const int32_t kiWidth, const int32_t kiHeight, const int32_t kiRefStride,
; uint16_t* pFeatureOfBlock, uint32_t pTimesOfFeatureValue[]);
;*********************************************************************************************************************
WELS_EXTERN SumOf8x8BlockOfFrame_sse2
%define pushsize 16
%define localsize 4
%define ref esp + pushsize + localsize + 4
%define sum_ref esp + pushsize + localsize + 20
%define times_of_sum esp + pushsize + localsize + 24
%define width esp + pushsize + localsize + 8
%define height esp + pushsize + localsize + 12
%define linesize esp + pushsize + localsize + 16
%define tmp_width esp + 0
push ebx
push ebp
push esi
push edi
sub esp, localsize
pxor xmm0, xmm0
mov esi, [ref]
mov edi, [sum_ref]
mov edx, [times_of_sum]
mov ebx, [linesize]
mov eax, [width]
lea ecx, [ebx+ebx*2] ; 3*linesize
mov [tmp_width], eax
lea ebp, [esi+ebx*4]
FIRST_ROW:
movq xmm1, [esi]
movq xmm2, [esi+ebx]
movq xmm3, [esi+ebx*2]
movq xmm4, [esi+ecx]
shufps xmm1, xmm2, 01000100b
shufps xmm3, xmm4, 01000100b
psadbw xmm1, xmm0
psadbw xmm3, xmm0
paddd xmm1, xmm3
movq xmm2, [ebp]
movq xmm3, [ebp+ebx]
movq xmm4, [ebp+ebx*2]
movq xmm5, [ebp+ecx]
shufps xmm2, xmm3, 01000100b
shufps xmm4, xmm5, 01000100b
psadbw xmm2, xmm0
psadbw xmm4, xmm0
paddd xmm2, xmm4
paddd xmm1, xmm2
pshufd xmm2, xmm1, 00001110b
paddd xmm1, xmm2
movd eax, xmm1
mov [edi], ax
inc dword [edx+eax*4]
inc esi
inc ebp
add edi, 2
dec dword [tmp_width]
jg FIRST_ROW
mov esi, [ref]
mov edi, [sum_ref]
mov ebp, [width]
dec dword [height]
HEIGHT_LOOP:
mov [tmp_width], ebp
WIDTH_LOOP:
movq xmm1, [esi+ebx*8]
movq xmm2, [esi]
psadbw xmm1, xmm0
psadbw xmm2, xmm0
psubd xmm1, xmm2
movd eax, xmm1
mov cx, [edi]
add eax, ecx
mov [edi+ebp*2], ax
inc dword [edx+eax*4]
inc esi
add edi, 2
dec dword [tmp_width]
jg WIDTH_LOOP
add esi, ebx
sub esi, ebp
dec dword [height]
jg HEIGHT_LOOP
add esp, localsize
pop edi
pop esi
pop ebp
pop ebx
%undef pushsize
%undef localsize
%undef ref
%undef sum_ref
%undef times_of_sum
%undef width
%undef height
%undef linesize
%undef tmp_width
ret
%macro COUNT_SUM 3
%define xmm_reg %1
%define tmp_reg %2
movd tmp_reg, xmm_reg
inc dword [edx+tmp_reg*4]
%if %3 == 1
psrldq xmm_reg, 4
%endif
%endmacro
;-----------------------------------------------------------------------------
; requires: width % 8 == 0 && height > 1
;-----------------------------------------------------------------------------
;void SumOf8x8BlockOfFrame_sse4(uint8_t* pRefPicture, const int32_t kiWidth, const int32_t kiHeight, const int32_t kiRefStride,
; uint16_t* pFeatureOfBlock, uint32_t pTimesOfFeatureValue[]);
;-----------------------------------------------------------------------------
; read extra (16 - (width % 8) ) mod 16 bytes of every line
; write extra (16 - (width % 8)*2 ) mod 16 bytes in the end of sum_ref
WELS_EXTERN SumOf8x8BlockOfFrame_sse4
%define pushsize 16
%define localsize 4
%define ref esp + pushsize + localsize + 4
%define sum_ref esp + pushsize + localsize + 20
%define times_of_sum esp + pushsize + localsize + 24
%define width esp + pushsize + localsize + 8
%define height esp + pushsize + localsize + 12
%define linesize esp + pushsize + localsize + 16
%define tmp_width esp + 0
push ebx
push ebp
push esi
push edi
sub esp, localsize
pxor xmm0, xmm0
mov esi, [ref]
mov edi, [sum_ref]
mov edx, [times_of_sum]
mov ebx, [linesize]
mov eax, [width]
lea ecx, [ebx+ebx*2] ; 3*linesize
mov [tmp_width], eax
lea ebp, [esi+ebx*4]
FIRST_ROW_SSE4:
movdqu xmm1, [esi]
movdqu xmm3, [esi+ebx]
movdqu xmm5, [esi+ebx*2]
movdqu xmm7, [esi+ecx]
movdqa xmm2, xmm1
mpsadbw xmm1, xmm0, 000b
mpsadbw xmm2, xmm0, 100b
paddw xmm1, xmm2 ; 8 sums of line1
movdqa xmm4, xmm3
mpsadbw xmm3, xmm0, 000b
mpsadbw xmm4, xmm0, 100b
paddw xmm3, xmm4 ; 8 sums of line2
movdqa xmm2, xmm5
mpsadbw xmm5, xmm0, 000b
mpsadbw xmm2, xmm0, 100b
paddw xmm5, xmm2 ; 8 sums of line3
movdqa xmm4, xmm7
mpsadbw xmm7, xmm0, 000b
mpsadbw xmm4, xmm0, 100b
paddw xmm7, xmm4 ; 8 sums of line4
paddw xmm1, xmm3
paddw xmm5, xmm7
paddw xmm1, xmm5 ; sum the upper 4 lines first
movdqu xmm2, [ebp]
movdqu xmm3, [ebp+ebx]
movdqu xmm4, [ebp+ebx*2]
movdqu xmm5, [ebp+ecx]
movdqa xmm6, xmm2
mpsadbw xmm2, xmm0, 000b
mpsadbw xmm6, xmm0, 100b
paddw xmm2, xmm6
movdqa xmm7, xmm3
mpsadbw xmm3, xmm0, 000b
mpsadbw xmm7, xmm0, 100b
paddw xmm3, xmm7
movdqa xmm6, xmm4
mpsadbw xmm4, xmm0, 000b
mpsadbw xmm6, xmm0, 100b
paddw xmm4, xmm6
movdqa xmm7, xmm5
mpsadbw xmm5, xmm0, 000b
mpsadbw xmm7, xmm0, 100b
paddw xmm5, xmm7
paddw xmm2, xmm3
paddw xmm4, xmm5
paddw xmm1, xmm2
paddw xmm1, xmm4 ; sum of lines 1- 8
movdqu [edi], xmm1
movdqa xmm2, xmm1
punpcklwd xmm1, xmm0
punpckhwd xmm2, xmm0
COUNT_SUM xmm1, eax, 1
COUNT_SUM xmm1, eax, 1
COUNT_SUM xmm1, eax, 1
COUNT_SUM xmm1, eax, 0
COUNT_SUM xmm2, eax, 1
COUNT_SUM xmm2, eax, 1
COUNT_SUM xmm2, eax, 1
COUNT_SUM xmm2, eax, 0
lea esi, [esi+8]
lea ebp, [ebp+8]
lea edi, [edi+16] ; element size is 2
sub dword [tmp_width], 8
jg near FIRST_ROW_SSE4
mov esi, [ref]
mov edi, [sum_ref]
mov ebp, [width]
dec dword [height]
HEIGHT_LOOP_SSE4:
mov ecx, ebp
WIDTH_LOOP_SSE4:
movdqu xmm1, [esi+ebx*8]
movdqu xmm2, [esi]
movdqu xmm7, [edi]
movdqa xmm3, xmm1
mpsadbw xmm1, xmm0, 000b
mpsadbw xmm3, xmm0, 100b
paddw xmm1, xmm3
movdqa xmm4, xmm2
mpsadbw xmm2, xmm0, 000b
mpsadbw xmm4, xmm0, 100b
paddw xmm2, xmm4
paddw xmm7, xmm1
psubw xmm7, xmm2
movdqu [edi+ebp*2], xmm7
movdqa xmm6, xmm7
punpcklwd xmm7, xmm0
punpckhwd xmm6, xmm0
COUNT_SUM xmm7, eax, 1
COUNT_SUM xmm7, eax, 1
COUNT_SUM xmm7, eax, 1
COUNT_SUM xmm7, eax, 0
COUNT_SUM xmm6, eax, 1
COUNT_SUM xmm6, eax, 1
COUNT_SUM xmm6, eax, 1
COUNT_SUM xmm6, eax, 0
lea esi, [esi+8]
lea edi, [edi+16]
sub ecx, 8
jg near WIDTH_LOOP_SSE4
lea esi, [esi+ebx]
sub esi, ebp
dec dword [height]
jg near HEIGHT_LOOP_SSE4
add esp, localsize
pop edi
pop esi
pop ebp
pop ebx
%undef pushsize
%undef localsize
%undef ref
%undef sum_ref
%undef times_of_sum
%undef width
%undef height
%undef linesize
%undef tmp_width
ret
;****************************************************************************************************************************************************
;void SumOf16x16BlockOfFrame_sse2(uint8_t* pRefPicture, const int32_t kiWidth, const int32_t kiHeight, const int32_t kiRefStride,
; uint16_t* pFeatureOfBlock, uint32_t pTimesOfFeatureValue[]);
;****************************************************************************************************************************************************
WELS_EXTERN SumOf16x16BlockOfFrame_sse2
%define pushsize 16
%define localsize 4
%define ref esp + pushsize + localsize + 4
%define sum_ref esp + pushsize + localsize + 20
%define times_of_sum esp + pushsize + localsize + 24
%define width esp + pushsize + localsize + 8
%define height esp + pushsize + localsize + 12
%define linesize esp + pushsize + localsize + 16
%define tmp_width esp
push ebx
push ebp
push esi
push edi
sub esp, localsize
pxor xmm0, xmm0
mov esi, [ref]
mov edi, [sum_ref]
mov edx, [times_of_sum]
mov ebx, [linesize]
mov eax, [width]
lea ecx, [ebx+ebx*2]
mov [tmp_width], eax
FIRST_ROW_X16H:
movdqu xmm1, [esi]
movdqu xmm2, [esi+ebx]
movdqu xmm3, [esi+ebx*2]
movdqu xmm4, [esi+ecx]
psadbw xmm1, xmm0
psadbw xmm2, xmm0
psadbw xmm3, xmm0
psadbw xmm4, xmm0
paddw xmm1, xmm2
paddw xmm3, xmm4
paddw xmm1, xmm3
lea ebp, [esi+ebx*4]
movdqu xmm2, [ebp]
movdqu xmm3, [ebp+ebx]
movdqu xmm4, [ebp+ebx*2]
movdqu xmm5, [ebp+ecx]
psadbw xmm2, xmm0
psadbw xmm3, xmm0
psadbw xmm4, xmm0
psadbw xmm5, xmm0
paddw xmm2, xmm3
paddw xmm4, xmm5
paddw xmm2, xmm4
paddw xmm1, xmm2
lea ebp, [ebp+ebx*4]
movdqu xmm2, [ebp]
movdqu xmm3, [ebp+ebx]
movdqu xmm4, [ebp+ebx*2]
movdqu xmm5, [ebp+ecx]
psadbw xmm2, xmm0
psadbw xmm3, xmm0
psadbw xmm4, xmm0
psadbw xmm5, xmm0
paddw xmm2, xmm3
paddw xmm4, xmm5
paddw xmm2, xmm4
paddw xmm1, xmm2
lea ebp, [ebp+ebx*4]
movdqu xmm2, [ebp]
movdqu xmm3, [ebp+ebx]
movdqu xmm4, [ebp+ebx*2]
movdqu xmm5, [ebp+ecx]
psadbw xmm2, xmm0
psadbw xmm3, xmm0
psadbw xmm4, xmm0
psadbw xmm5, xmm0
paddw xmm2, xmm3
paddw xmm4, xmm5
paddw xmm2, xmm4
paddw xmm1, xmm2
movdqa xmm2, xmm1
punpckhwd xmm2, xmm0
paddw xmm1, xmm2
movd eax, xmm1
mov [edi], ax
inc dword [edx+eax*4]
inc esi
lea edi, [edi+2]
dec dword [tmp_width]
jg near FIRST_ROW_X16H
mov esi, [ref]
mov edi, [sum_ref]
mov ebp, [width]
dec dword [height]
mov ecx, ebx
sal ecx, 4 ; succeeded 16th line
HEIGHT_LOOP_X16:
mov [tmp_width], ebp
WIDTH_LOOP_X16:
movdqu xmm1, [esi+ecx]
movdqu xmm2, [esi]
psadbw xmm1, xmm0
psadbw xmm2, xmm0
psubw xmm1, xmm2
movdqa xmm2, xmm1
punpckhwd xmm2, xmm0
paddw xmm1, xmm2
movd eax, xmm1
add ax, word [edi]
mov [edi+ebp*2], ax
inc dword [edx+eax*4]
inc esi
add edi, 2
dec dword [tmp_width]
jg near WIDTH_LOOP_X16
add esi, ebx
sub esi, ebp
dec dword [height]
jg near HEIGHT_LOOP_X16
add esp, localsize
pop edi
pop esi
pop ebp
pop ebx
%undef pushsize
%undef localsize
%undef ref
%undef sum_ref
%undef times_of_sum
%undef width
%undef height
%undef linesize
%undef tmp_width
ret
; requires: width % 16 == 0 && height > 1
;-----------------------------------------------------------------------------------------------------------------------------
;void SumOf16x16BlockOfFrame_sse4(uint8_t* pRefPicture, const int32_t kiWidth, const int32_t kiHeight, const int32_t kiRefStride,
; uint16_t* pFeatureOfBlock, uint32_t pTimesOfFeatureValue[]);
;-----------------------------------------------------------------------------------------------------------------------------
; try 8 mv via offset
%macro SUM_LINE_X16_SSE41 5 ; ref, dst0, dst1, tmp0, tmp1
movdqu %2, [%1]
movdqu %3, [%1+8h]
movdqa %4, %2
movdqa %5, %3
mpsadbw %2, xmm0, 0 ; 000 B
mpsadbw %4, xmm0, 5 ; 101 B
mpsadbw %3, xmm0, 2 ; 010 B
mpsadbw %5, xmm0, 7 ; 111 B
paddw %2, %4
paddw %3, %5
paddw %2, %3 ; accumulate cost
%endmacro ; end of SAD_16x16_LINE_SSE41
WELS_EXTERN SumOf16x16BlockOfFrame_sse4
%define pushsize 16
%define localsize 4
%define ref esp + pushsize + localsize + 4
%define sum_ref esp + pushsize + localsize + 20
%define times_of_sum esp + pushsize + localsize + 24
%define width esp + pushsize + localsize + 8
%define height esp + pushsize + localsize + 12
%define linesize esp + pushsize + localsize + 16
%define tmp_width esp
push ebx
push ebp
push esi
push edi
sub esp, localsize
pxor xmm0, xmm0
mov esi, [ref]
mov edi, [sum_ref]
mov edx, [times_of_sum]
mov ebx, [linesize]
mov eax, [width]
lea ecx, [ebx+ebx*2]
mov [tmp_width], eax
FIRST_ROW_X16_SSE4:
SUM_LINE_X16_SSE41 esi, xmm1, xmm2, xmm3, xmm4
SUM_LINE_X16_SSE41 esi+ebx, xmm2, xmm3, xmm4, xmm5
SUM_LINE_X16_SSE41 esi+ebx*2, xmm3, xmm4, xmm5, xmm6
SUM_LINE_X16_SSE41 esi+ecx, xmm4, xmm5, xmm6, xmm7
paddw xmm1, xmm2
paddw xmm3, xmm4
paddw xmm1, xmm3
lea ebp, [esi+ebx*4]
SUM_LINE_X16_SSE41 ebp, xmm2, xmm3, xmm4, xmm5
paddw xmm1, xmm2
SUM_LINE_X16_SSE41 ebp+ebx, xmm2, xmm3, xmm4, xmm5
paddw xmm1, xmm2
SUM_LINE_X16_SSE41 ebp+ebx*2, xmm2, xmm3, xmm4, xmm5
paddw xmm1, xmm2
SUM_LINE_X16_SSE41 ebp+ecx, xmm2, xmm3, xmm4, xmm5
paddw xmm1, xmm2
lea ebp, [ebp+ebx*4]
SUM_LINE_X16_SSE41 ebp, xmm2, xmm3, xmm4, xmm5
paddw xmm1, xmm2
SUM_LINE_X16_SSE41 ebp+ebx, xmm2, xmm3, xmm4, xmm5
paddw xmm1, xmm2
SUM_LINE_X16_SSE41 ebp+ebx*2, xmm2, xmm3, xmm4, xmm5
paddw xmm1, xmm2
SUM_LINE_X16_SSE41 ebp+ecx, xmm2, xmm3, xmm4, xmm5
paddw xmm1, xmm2
lea ebp, [ebp+ebx*4]
SUM_LINE_X16_SSE41 ebp, xmm2, xmm3, xmm4, xmm5
paddw xmm1, xmm2
SUM_LINE_X16_SSE41 ebp+ebx, xmm2, xmm3, xmm4, xmm5
paddw xmm1, xmm2
SUM_LINE_X16_SSE41 ebp+ebx*2, xmm2, xmm3, xmm4, xmm5
paddw xmm1, xmm2
SUM_LINE_X16_SSE41 ebp+ecx, xmm2, xmm3, xmm4, xmm5
paddw xmm1, xmm2
movdqa [edi], xmm1
movdqa xmm2, xmm1
punpcklwd xmm1, xmm0
punpckhwd xmm2, xmm0
COUNT_SUM xmm1, eax, 1
COUNT_SUM xmm1, eax, 1
COUNT_SUM xmm1, eax, 1
COUNT_SUM xmm1, eax, 0
COUNT_SUM xmm2, eax, 1
COUNT_SUM xmm2, eax, 1
COUNT_SUM xmm2, eax, 1
COUNT_SUM xmm2, eax, 0
lea esi, [esi+8]
lea edi, [edi+16] ; element size is 2
sub dword [tmp_width], 8
jg near FIRST_ROW_X16_SSE4
mov esi, [ref]
mov edi, [sum_ref]
mov ebp, [width]
dec dword [height]
mov ecx, ebx
sal ecx, 4 ; succeeded 16th line
HEIGHT_LOOP_X16_SSE4:
mov [tmp_width], ebp
WIDTH_LOOP_X16_SSE4:
movdqa xmm7, [edi]
SUM_LINE_X16_SSE41 esi+ecx, xmm1, xmm2, xmm3, xmm4
SUM_LINE_X16_SSE41 esi, xmm2, xmm3, xmm4, xmm5
paddw xmm7, xmm1
psubw xmm7, xmm2
movdqa [edi+ebp*2], xmm7
movdqa xmm6, xmm7
punpcklwd xmm7, xmm0
punpckhwd xmm6, xmm0
COUNT_SUM xmm7, eax, 1
COUNT_SUM xmm7, eax, 1
COUNT_SUM xmm7, eax, 1
COUNT_SUM xmm7, eax, 0
COUNT_SUM xmm6, eax, 1
COUNT_SUM xmm6, eax, 1
COUNT_SUM xmm6, eax, 1
COUNT_SUM xmm6, eax, 0
lea esi, [esi+8]
lea edi, [edi+16]
sub dword [tmp_width], 8
jg near WIDTH_LOOP_X16_SSE4
add esi, ebx
sub esi, ebp
dec dword [height]
jg near HEIGHT_LOOP_X16_SSE4
add esp, localsize
pop edi
pop esi
pop ebp
pop ebx
%undef pushsize
%undef localsize
%undef ref
%undef sum_ref
%undef times_of_sum
%undef width
%undef height
%undef linesize
%undef tmp_width
ret
;-----------------------------------------------------------------------------------------------------------------------------
; void FillQpelLocationByFeatureValue_sse2(uint16_t* pFeatureOfBlock, const int32_t kiWidth, const int32_t kiHeight, uint16_t** pFeatureValuePointerList)
;-----------------------------------------------------------------------------------------------------------------------------
WELS_EXTERN FillQpelLocationByFeatureValue_sse2
push esi
push edi
push ebx
push ebp
%define _ps 16 ; push size
%define _ls 4 ; local size
%define sum_ref esp+_ps+_ls+4
%define pos_list esp+_ps+_ls+16
%define width esp+_ps+_ls+8
%define height esp+_ps+_ls+12
%define i_height esp
sub esp, _ls
mov esi, [sum_ref]
mov edi, [pos_list]
mov ebp, [width]
mov ebx, [height]
mov [i_height], ebx
%assign push_num 5
INIT_X86_32_PIC_NOPRESERVE ecx
movq xmm7, [pic(mv_x_inc_x4)] ; x_qpel inc
movq xmm6, [pic(mv_y_inc_x4)] ; y_qpel inc
movq xmm5, [pic(mx_x_offset_x4)] ; x_qpel vector
DEINIT_X86_32_PIC
pxor xmm4, xmm4
pxor xmm3, xmm3 ; y_qpel vector
HASH_HEIGHT_LOOP_SSE2:
movdqa xmm2, xmm5 ; x_qpel vector
mov ecx, ebp
HASH_WIDTH_LOOP_SSE2:
movq xmm0, [esi] ; load x8 sum
punpcklwd xmm0, xmm4
movdqa xmm1, xmm2
punpcklwd xmm1, xmm3
%rep 3
movd edx, xmm0
lea ebx, [edi+edx*4]
mov eax, [ebx]
movd [eax], xmm1
mov edx, [eax+4] ; explictly load eax+4 due cache miss from vtune observation
lea eax, [eax+4]
mov [ebx], eax
psrldq xmm1, 4
psrldq xmm0, 4
%endrep
movd edx, xmm0
lea ebx, [edi+edx*4]
mov eax, [ebx]
movd [eax], xmm1
mov edx, [eax+4] ; explictly load eax+4 due cache miss from vtune observation
lea eax, [eax+4]
mov [ebx], eax
paddw xmm2, xmm7
lea esi, [esi+8]
sub ecx, 4
jnz near HASH_WIDTH_LOOP_SSE2
paddw xmm3, xmm6
dec dword [i_height]
jnz near HASH_HEIGHT_LOOP_SSE2
add esp, _ls
%undef _ps
%undef _ls
%undef sum_ref
%undef pos_list
%undef width
%undef height
%undef i_height
pop ebp
pop ebx
pop edi
pop esi
ret
;---------------------------------------------------------------------------------------------------------------------------------------------------
; void InitializeHashforFeature_sse2( uint32_t* pTimesOfFeatureValue, uint16_t* pBuf, const int32_t kiListSize,
; uint16_t** pLocationOfFeature, uint16_t** pFeatureValuePointerList )
;---------------------------------------------------------------------------------------------------------------------------------------------------
WELS_EXTERN InitializeHashforFeature_sse2
push ebx
push esi
push edi
push ebp
%define _ps 16 ; push size
mov edi, [esp+_ps+16] ; pPositionOfSum
mov ebp, [esp+_ps+20] ; sum_idx_list
mov esi, [esp+_ps+4] ; pTimesOfSum
mov ebx, [esp+_ps+8] ; pBuf
mov edx, [esp+_ps+12] ; list_sz
sar edx, 2
mov ecx, 0
pxor xmm7, xmm7
hash_assign_loop_x4_sse2:
movdqa xmm0, [esi+ecx]
pslld xmm0, 2
movdqa xmm1, xmm0
pcmpeqd xmm1, xmm7
movmskps eax, xmm1
cmp eax, 0x0f
je near hash_assign_with_copy_sse2
%assign x 0
%rep 4
lea eax, [edi+ecx+x]
mov [eax], ebx
lea eax, [ebp+ecx+x]
mov [eax], ebx
movd eax, xmm0
add ebx, eax
psrldq xmm0, 4
%assign x x+4
%endrep
jmp near assign_next_sse2
hash_assign_with_copy_sse2:
movd xmm1, ebx
pshufd xmm2, xmm1, 0
movdqa [edi+ecx], xmm2
movdqa [ebp+ecx], xmm2
assign_next_sse2:
add ecx, 16
dec edx
jnz near hash_assign_loop_x4_sse2
mov edx, [esp+_ps+12] ; list_sz
and edx, 3
jz near hash_assign_no_rem_sse2
hash_assign_loop_x4_rem_sse2:
lea eax, [edi+ecx]
mov [eax], ebx
lea eax, [ebp+ecx]
mov [eax], ebx
mov eax, [esi+ecx]
sal eax, 2
add ebx, eax
add ecx, 4
dec edx
jnz near hash_assign_loop_x4_rem_sse2
hash_assign_no_rem_sse2:
%undef _ps
pop ebp
pop edi
pop esi
pop ebx
ret
%else
;**********************************************************************************************************************
;void SumOf8x8BlockOfFrame_sse2(uint8_t* pRefPicture, const int32_t kiWidth, const int32_t kiHeight, const int32_t kiRefStride,
; uint16_t* pFeatureOfBlock, uint32_t pTimesOfFeatureValue[]);
;*********************************************************************************************************************
WELS_EXTERN SumOf8x8BlockOfFrame_sse2
%assign push_num 0
LOAD_6_PARA
PUSH_XMM 6
SIGN_EXTENSION r1, r1d
SIGN_EXTENSION r2, r2d
SIGN_EXTENSION r3, r3d
push r12
push r13
push r0
push r2
push r4
pxor xmm0, xmm0
lea r6, [r3+r3*2]
mov r12, r1 ;r12:tmp_width
lea r13, [r0+r3*4] ;rbp:r13
FIRST_ROW:
movq xmm1, [r0]
movq xmm2, [r0+r3]
movq xmm3, [r0+r3*2]
movq xmm4, [r0+r6]
shufps xmm1, xmm2, 01000100b
shufps xmm3, xmm4, 01000100b
psadbw xmm1, xmm0
psadbw xmm3, xmm0
paddd xmm1, xmm3
movq xmm2, [r13]
movq xmm3, [r13+r3]
movq xmm4, [r13+r3*2]
movq xmm5, [r13+r6]
shufps xmm2, xmm3, 01000100b
shufps xmm4, xmm5, 01000100b
psadbw xmm2, xmm0
psadbw xmm4, xmm0
paddd xmm2, xmm4
paddd xmm1, xmm2
pshufd xmm2, xmm1, 00001110b
paddd xmm1, xmm2
movd r2d, xmm1
mov [r4], r2w
inc dword [r5+r2*4]
inc r0
inc r13
add r4, 2
dec r12
jg FIRST_ROW
pop r4
pop r2
pop r0
mov r13, r2
dec r13
HEIGHT_LOOP:
mov r12, r1
WIDTH_LOOP:
movq xmm1, [r0+r3*8]
movq xmm2, [r0]
psadbw xmm1, xmm0
psadbw xmm2, xmm0
psubd xmm1, xmm2
movd r2d, xmm1
mov r6w, [r4]
add r2d, r6d
mov [r4+r1*2], r2w
inc dword [r5+r2*4]
inc r0
add r4, 2
dec r12
jg WIDTH_LOOP
add r0, r3
sub r0, r1
dec r13
jg HEIGHT_LOOP
pop r13
pop r12
POP_XMM
LOAD_6_PARA_POP
ret
%macro COUNT_SUM 4
%define xmm_reg %1
%define tmp_dreg %2
%define tmp_qreg %3
movd tmp_dreg, xmm_reg
inc dword [r5+tmp_qreg*4]
%if %4 == 1
psrldq xmm_reg, 4
%endif
%endmacro
;-----------------------------------------------------------------------------
; requires: width % 8 == 0 && height > 1
;-----------------------------------------------------------------------------
;void SumOf8x8BlockOfFrame_sse4(uint8_t* pRefPicture, const int32_t kiWidth, const int32_t kiHeight, const int32_t kiRefStride,
; uint16_t* pFeatureOfBlock, uint32_t pTimesOfFeatureValue[]);
;-----------------------------------------------------------------------------
; read extra (16 - (width % 8) ) mod 16 bytes of every line
; write extra (16 - (width % 8)*2 ) mod 16 bytes in the end of sum_ref
WELS_EXTERN SumOf8x8BlockOfFrame_sse4
%assign push_num 0
LOAD_6_PARA
PUSH_XMM 8
SIGN_EXTENSION r1, r1d
SIGN_EXTENSION r2, r2d
SIGN_EXTENSION r3, r3d
push r12
push r13
push r0
push r2
push r4
pxor xmm0, xmm0
lea r6, [r3+r3*2]
mov r12, r1 ;r12:tmp_width
lea r13, [r0+r3*4] ;rbp:r13
FIRST_ROW_SSE4:
movdqu xmm1, [r0]
movdqu xmm3, [r0+r3]
movdqu xmm5, [r0+r3*2]
movdqu xmm7, [r0+r6]
movdqa xmm2, xmm1
mpsadbw xmm1, xmm0, 000b
mpsadbw xmm2, xmm0, 100b
paddw xmm1, xmm2 ; 8 sums of line1
movdqa xmm4, xmm3
mpsadbw xmm3, xmm0, 000b
mpsadbw xmm4, xmm0, 100b
paddw xmm3, xmm4 ; 8 sums of line2
movdqa xmm2, xmm5
mpsadbw xmm5, xmm0, 000b
mpsadbw xmm2, xmm0, 100b
paddw xmm5, xmm2 ; 8 sums of line3
movdqa xmm4, xmm7
mpsadbw xmm7, xmm0, 000b
mpsadbw xmm4, xmm0, 100b
paddw xmm7, xmm4 ; 8 sums of line4
paddw xmm1, xmm3
paddw xmm5, xmm7
paddw xmm1, xmm5 ; sum the upper 4 lines first
movdqu xmm2, [r13]
movdqu xmm3, [r13+r3]
movdqu xmm4, [r13+r3*2]
movdqu xmm5, [r13+r6]
movdqa xmm6, xmm2
mpsadbw xmm2, xmm0, 000b
mpsadbw xmm6, xmm0, 100b
paddw xmm2, xmm6
movdqa xmm7, xmm3
mpsadbw xmm3, xmm0, 000b
mpsadbw xmm7, xmm0, 100b
paddw xmm3, xmm7
movdqa xmm6, xmm4
mpsadbw xmm4, xmm0, 000b
mpsadbw xmm6, xmm0, 100b
paddw xmm4, xmm6
movdqa xmm7, xmm5
mpsadbw xmm5, xmm0, 000b
mpsadbw xmm7, xmm0, 100b
paddw xmm5, xmm7
paddw xmm2, xmm3
paddw xmm4, xmm5
paddw xmm1, xmm2
paddw xmm1, xmm4 ; sum of lines 1- 8
movdqu [r4], xmm1
movdqa xmm2, xmm1
punpcklwd xmm1, xmm0
punpckhwd xmm2, xmm0
COUNT_SUM xmm1, r2d, r2, 1
COUNT_SUM xmm1, r2d, r2, 1
COUNT_SUM xmm1, r2d, r2, 1
COUNT_SUM xmm1, r2d, r2, 0
COUNT_SUM xmm2, r2d, r2 ,1
COUNT_SUM xmm2, r2d, r2 ,1
COUNT_SUM xmm2, r2d, r2 ,1
COUNT_SUM xmm2, r2d, r2 ,0
lea r0, [r0+8]
lea r13, [r13+8]
lea r4, [r4+16] ; element size is 2
sub r12, 8
jg near FIRST_ROW_SSE4
pop r4
pop r2
pop r0
mov r13, r2
dec r13
HEIGHT_LOOP_SSE4:
mov r12, r1
WIDTH_LOOP_SSE4:
movdqu xmm1, [r0+r3*8]
movdqu xmm2, [r0]
movdqu xmm7, [r4]
movdqa xmm3, xmm1
mpsadbw xmm1, xmm0, 000b
mpsadbw xmm3, xmm0, 100b
paddw xmm1, xmm3
movdqa xmm4, xmm2
mpsadbw xmm2, xmm0, 000b
mpsadbw xmm4, xmm0, 100b
paddw xmm2, xmm4
paddw xmm7, xmm1
psubw xmm7, xmm2
movdqu [r4+r1*2], xmm7
movdqa xmm6, xmm7
punpcklwd xmm7, xmm0
punpckhwd xmm6, xmm0
COUNT_SUM xmm7, r2d, r2, 1
COUNT_SUM xmm7, r2d, r2, 1
COUNT_SUM xmm7, r2d, r2, 1
COUNT_SUM xmm7, r2d, r2, 0
COUNT_SUM xmm6, r2d, r2, 1
COUNT_SUM xmm6, r2d, r2, 1
COUNT_SUM xmm6, r2d, r2, 1
COUNT_SUM xmm6, r2d, r2, 0
lea r0, [r0+8]
lea r4, [r4+16]
sub r12, 8
jg near WIDTH_LOOP_SSE4
lea r0, [r0+r3]
sub r0, r1
dec r13
jg near HEIGHT_LOOP_SSE4
pop r13
pop r12
POP_XMM
LOAD_6_PARA_POP
ret
;****************************************************************************************************************************************************
;void SumOf16x16BlockOfFrame_sse2(uint8_t* pRefPicture, const int32_t kiWidth, const int32_t kiHeight, const int32_t kiRefStride,
; uint16_t* pFeatureOfBlock, uint32_t pTimesOfFeatureValue[]);
;****************************************************************************************************************************************************
WELS_EXTERN SumOf16x16BlockOfFrame_sse2
%assign push_num 0
LOAD_6_PARA
PUSH_XMM 6
SIGN_EXTENSION r1, r1d
SIGN_EXTENSION r2, r2d
SIGN_EXTENSION r3, r3d
push r12
push r13
push r0
push r2
push r4
pxor xmm0, xmm0
lea r6, [r3+r3*2]
mov r12, r1 ;r12:tmp_width
FIRST_ROW_X16H:
movdqu xmm1, [r0]
movdqu xmm2, [r0+r3]
movdqu xmm3, [r0+r3*2]
movdqu xmm4, [r0+r6]
psadbw xmm1, xmm0
psadbw xmm2, xmm0
psadbw xmm3, xmm0
psadbw xmm4, xmm0
paddw xmm1, xmm2
paddw xmm3, xmm4
paddw xmm1, xmm3
lea r13, [r0+r3*4] ;ebp:r13
movdqu xmm2, [r13]
movdqu xmm3, [r13+r3]
movdqu xmm4, [r13+r3*2]
movdqu xmm5, [r13+r6]
psadbw xmm2, xmm0
psadbw xmm3, xmm0
psadbw xmm4, xmm0
psadbw xmm5, xmm0
paddw xmm2, xmm3
paddw xmm4, xmm5
paddw xmm2, xmm4
paddw xmm1, xmm2
lea r13, [r13+r3*4]
movdqu xmm2, [r13]
movdqu xmm3, [r13+r3]
movdqu xmm4, [r13+r3*2]
movdqu xmm5, [r13+r6]
psadbw xmm2, xmm0
psadbw xmm3, xmm0
psadbw xmm4, xmm0
psadbw xmm5, xmm0
paddw xmm2, xmm3
paddw xmm4, xmm5
paddw xmm2, xmm4
paddw xmm1, xmm2
lea r13, [r13+r3*4]
movdqu xmm2, [r13]
movdqu xmm3, [r13+r3]
movdqu xmm4, [r13+r3*2]
movdqu xmm5, [r13+r6]
psadbw xmm2, xmm0
psadbw xmm3, xmm0
psadbw xmm4, xmm0
psadbw xmm5, xmm0
paddw xmm2, xmm3
paddw xmm4, xmm5
paddw xmm2, xmm4
paddw xmm1, xmm2
movdqa xmm2, xmm1
punpckhwd xmm2, xmm0
paddw xmm1, xmm2
movd r2d, xmm1
mov [r4], r2w
inc dword [r5+r2*4]
inc r0
lea r4, [r4+2]
dec r12
jg near FIRST_ROW_X16H
pop r4
pop r2
pop r0
mov r13, r2
dec r13
mov r6, r3
sal r6, 4 ; succeeded 16th line
HEIGHT_LOOP_X16:
mov r12, r1
WIDTH_LOOP_X16:
movdqu xmm1, [r0+r6]
movdqu xmm2, [r0]
psadbw xmm1, xmm0
psadbw xmm2, xmm0
psubw xmm1, xmm2
movdqa xmm2, xmm1
punpckhwd xmm2, xmm0
paddw xmm1, xmm2
movd r2d, xmm1
add r2w, word [r4]
mov [r4+r1*2], r2w
inc dword [r5+r2*4]
inc r0
add r4, 2
dec r12
jg near WIDTH_LOOP_X16
add r0, r3
sub r0, r1
dec r13
jg near HEIGHT_LOOP_X16
pop r13
pop r12
POP_XMM
LOAD_6_PARA_POP
ret
; requires: width % 16 == 0 && height > 1
;-----------------------------------------------------------------------------------------------------------------------------
;void SumOf16x16BlockOfFrame_sse4(uint8_t* pRefPicture, const int32_t kiWidth, const int32_t kiHeight, const int32_t kiRefStride,
; uint16_t* pFeatureOfBlock, uint32_t pTimesOfFeatureValue[]);
;-----------------------------------------------------------------------------------------------------------------------------
; try 8 mv via offset
%macro SUM_LINE_X16_SSE41 5 ; ref, dst0, dst1, tmp0, tmp1
movdqu %2, [%1]
movdqu %3, [%1+8h]
movdqa %4, %2
movdqa %5, %3
mpsadbw %2, xmm0, 0 ; 000 B
mpsadbw %4, xmm0, 5 ; 101 B
mpsadbw %3, xmm0, 2 ; 010 B
mpsadbw %5, xmm0, 7 ; 111 B
paddw %2, %4
paddw %3, %5
paddw %2, %3 ; accumulate cost
%endmacro ; end of SAD_16x16_LINE_SSE41
WELS_EXTERN SumOf16x16BlockOfFrame_sse4
%assign push_num 0
LOAD_6_PARA
PUSH_XMM 8
SIGN_EXTENSION r1, r1d
SIGN_EXTENSION r2, r2d
SIGN_EXTENSION r3, r3d
push r12
push r13
push r0
push r2
push r4
pxor xmm0, xmm0
lea r6, [r3+r3*2]
mov r12, r1 ;r12:tmp_width
FIRST_ROW_X16_SSE4:
SUM_LINE_X16_SSE41 r0, xmm1, xmm2, xmm3, xmm4
SUM_LINE_X16_SSE41 r0+r3, xmm2, xmm3, xmm4, xmm5
SUM_LINE_X16_SSE41 r0+r3*2,xmm3, xmm4, xmm5, xmm6
SUM_LINE_X16_SSE41 r0+r6, xmm4, xmm5, xmm6, xmm7
paddw xmm1, xmm2
paddw xmm3, xmm4
paddw xmm1, xmm3
lea r13, [r0+r3*4]
SUM_LINE_X16_SSE41 r13, xmm2, xmm3, xmm4, xmm5
paddw xmm1, xmm2
SUM_LINE_X16_SSE41 r13+r3, xmm2, xmm3, xmm4, xmm5
paddw xmm1, xmm2
SUM_LINE_X16_SSE41 r13+r3*2, xmm2, xmm3, xmm4, xmm5
paddw xmm1, xmm2
SUM_LINE_X16_SSE41 r13+r6, xmm2, xmm3, xmm4, xmm5
paddw xmm1, xmm2
lea r13, [r13+r3*4]
SUM_LINE_X16_SSE41 r13, xmm2, xmm3, xmm4, xmm5
paddw xmm1, xmm2
SUM_LINE_X16_SSE41 r13+r3, xmm2, xmm3, xmm4, xmm5
paddw xmm1, xmm2
SUM_LINE_X16_SSE41 r13+r3*2, xmm2, xmm3, xmm4, xmm5
paddw xmm1, xmm2
SUM_LINE_X16_SSE41 r13+r6, xmm2, xmm3, xmm4, xmm5
paddw xmm1, xmm2
lea r13, [r13+r3*4]
SUM_LINE_X16_SSE41 r13, xmm2, xmm3, xmm4, xmm5
paddw xmm1, xmm2
SUM_LINE_X16_SSE41 r13+r3, xmm2, xmm3, xmm4, xmm5
paddw xmm1, xmm2
SUM_LINE_X16_SSE41 r13+r3*2, xmm2, xmm3, xmm4, xmm5
paddw xmm1, xmm2
SUM_LINE_X16_SSE41 r13+r6, xmm2, xmm3, xmm4, xmm5
paddw xmm1, xmm2
movdqa [r4], xmm1
movdqa xmm2, xmm1
punpcklwd xmm1, xmm0
punpckhwd xmm2, xmm0
COUNT_SUM xmm1, r2d, r2, 1
COUNT_SUM xmm1, r2d, r2, 1
COUNT_SUM xmm1, r2d, r2, 1
COUNT_SUM xmm1, r2d, r2, 0
COUNT_SUM xmm2, r2d, r2, 1
COUNT_SUM xmm2, r2d, r2, 1
COUNT_SUM xmm2, r2d, r2, 1
COUNT_SUM xmm2, r2d, r2, 0
lea r0, [r0+8]
lea r4, [r4+16] ; element size is 2
sub r12, 8
jg near FIRST_ROW_X16_SSE4
pop r4
pop r2
pop r0
mov r13, r2
dec r13
mov r6, r3
sal r6, 4 ; succeeded 16th line
HEIGHT_LOOP_X16_SSE4:
mov r12, r1
WIDTH_LOOP_X16_SSE4:
movdqa xmm7, [r4]
SUM_LINE_X16_SSE41 r0+r6, xmm1, xmm2, xmm3, xmm4
SUM_LINE_X16_SSE41 r0, xmm2, xmm3, xmm4, xmm5
paddw xmm7, xmm1
psubw xmm7, xmm2
movdqa [r4+r1*2], xmm7
movdqa xmm6, xmm7
punpcklwd xmm7, xmm0
punpckhwd xmm6, xmm0
COUNT_SUM xmm7, r2d, r2, 1
COUNT_SUM xmm7, r2d, r2, 1
COUNT_SUM xmm7, r2d, r2, 1
COUNT_SUM xmm7, r2d, r2, 0
COUNT_SUM xmm6, r2d, r2, 1
COUNT_SUM xmm6, r2d, r2, 1
COUNT_SUM xmm6, r2d, r2, 1
COUNT_SUM xmm6, r2d, r2, 0
lea r0, [r0+8]
lea r4, [r4+16]
sub r12, 8
jg near WIDTH_LOOP_X16_SSE4
add r0, r3
sub r0, r1
dec r13
jg near HEIGHT_LOOP_X16_SSE4
pop r13
pop r12
POP_XMM
LOAD_6_PARA_POP
ret
;-----------------------------------------------------------------------------------------------------------------------------
; void FillQpelLocationByFeatureValue_sse2(uint16_t* pFeatureOfBlock, const int32_t kiWidth, const int32_t kiHeight, uint16_t** pFeatureValuePointerList)
;-----------------------------------------------------------------------------------------------------------------------------
WELS_EXTERN FillQpelLocationByFeatureValue_sse2
%assign push_num 0
LOAD_4_PARA
PUSH_XMM 8
SIGN_EXTENSION r1, r1d
SIGN_EXTENSION r2, r2d
push r12
push r13
mov r12, r2
movq xmm7, [mv_x_inc_x4] ; x_qpel inc
movq xmm6, [mv_y_inc_x4] ; y_qpel inc
movq xmm5, [mx_x_offset_x4] ; x_qpel vector
pxor xmm4, xmm4
pxor xmm3, xmm3 ; y_qpel vector
HASH_HEIGHT_LOOP_SSE2:
movdqa xmm2, xmm5 ; x_qpel vector
mov r4, r1
HASH_WIDTH_LOOP_SSE2:
movq xmm0, [r0] ; load x8 sum
punpcklwd xmm0, xmm4
movdqa xmm1, xmm2
punpcklwd xmm1, xmm3
%rep 3
movd r2d, xmm0 ;edx:r3
lea r5, [r3+r2*8] ;ebx:r5
mov r6, [r5] ;eax:r6
movd [r6], xmm1
mov r13, [r6+4] ; explictly load eax+4 due cache miss from vtune observation
lea r6, [r6+4]
mov [r5], r6
psrldq xmm1, 4
psrldq xmm0, 4
%endrep
movd r2d, xmm0
lea r5, [r3+r2*8] ;ebx:r5
mov r6, [r5] ;eax:r6
movd [r6], xmm1
mov r13, [r6+4] ; explictly load eax+4 due cache miss from vtune observation
lea r6, [r6+4]
mov [r5], r6
paddw xmm2, xmm7
lea r0, [r0+8]
sub r4, 4
jnz near HASH_WIDTH_LOOP_SSE2
paddw xmm3, xmm6
dec r12
jnz near HASH_HEIGHT_LOOP_SSE2
pop r13
pop r12
POP_XMM
ret
;---------------------------------------------------------------------------------------------------------------------------------------------------
; void InitializeHashforFeature_sse2( uint32_t* pTimesOfFeatureValue, uint16_t* pBuf, const int32_t kiListSize,
; uint16_t** pLocationOfFeature, uint16_t** pFeatureValuePointerList);
;uint16_t** pPositionOfSum, uint16_t** sum_idx_list, uint32_t* pTimesOfSum, uint16_t* pBuf, const int32_t list_sz )
;---------------------------------------------------------------------------------------------------------------------------------------------------
WELS_EXTERN InitializeHashforFeature_sse2
%assign push_num 0
LOAD_5_PARA
SIGN_EXTENSION r2, r2d
push r12
push r13
mov r12, r2
sar r2, 2
mov r5, 0 ;r5:ecx
xor r6, r6
pxor xmm3, xmm3
hash_assign_loop_x4_sse2:
movdqa xmm0, [r0+r5]
pslld xmm0, 2
movdqa xmm1, xmm0
pcmpeqd xmm1, xmm3
movmskps r6, xmm1
cmp r6, 0x0f
jz near hash_assign_with_copy_sse2
%assign x 0
%rep 4
lea r13, [r3+r5*2+x]
mov [r13], r1
lea r13, [r4+r5*2+x]
mov [r13], r1
movd r6d, xmm0
add r1, r6
psrldq xmm0, 4
%assign x x+8
%endrep
jmp near assign_next_sse2
hash_assign_with_copy_sse2:
movq xmm1, r1
pshufd xmm2, xmm1, 01000100b
movdqa [r3+r5*2], xmm2
movdqa [r4+r5*2], xmm2
movdqa [r3+r5*2+16], xmm2
movdqa [r4+r5*2+16], xmm2
assign_next_sse2:
add r5, 16
dec r2
jnz near hash_assign_loop_x4_sse2
and r12, 3
jz near hash_assign_no_rem_sse2
hash_assign_loop_x4_rem_sse2:
lea r13, [r3+r5*2]
mov [r13], r1
lea r13, [r4+r5*2]
mov [r13], r1
mov r6d, [r0+r5]
sal r6, 2
add r1, r6
add r5, 4
dec r12
jnz near hash_assign_loop_x4_rem_sse2
hash_assign_no_rem_sse2:
pop r13
pop r12
ret
%endif
;**********************************************************************************************************************************
; int32_t SumOf8x8SingleBlock_sse2(uint8_t* ref0, int32_t linesize)
;**********************************************************************************************************************************
WELS_EXTERN SumOf8x8SingleBlock_sse2
%assign push_num 0
LOAD_2_PARA
SIGN_EXTENSION r1, r1d
pxor xmm0, xmm0
movq xmm1, [r0]
movhps xmm1, [r0+r1]
lea r0, [r0+2*r1]
movq xmm2, [r0]
movhps xmm2, [r0+r1]
lea r0, [r0+2*r1]
movq xmm3, [r0]
movhps xmm3, [r0+r1]
lea r0, [r0+2*r1]
movq xmm4, [r0]
movhps xmm4, [r0+r1]
psadbw xmm1, xmm0
psadbw xmm2, xmm0
psadbw xmm3, xmm0
psadbw xmm4, xmm0
paddw xmm1, xmm2
paddw xmm3, xmm4
paddw xmm1, xmm3
movdqa xmm2, xmm1
punpckhwd xmm2, xmm0
paddw xmm1, xmm2
movd retrd, xmm1
ret
;**********************************************************************************************************************************
; int32_t SumOf16x16SingleBlock_sse2(uint8_t* ref0, int32_t linesize)
;**********************************************************************************************************************************
WELS_EXTERN SumOf16x16SingleBlock_sse2
%assign push_num 0
LOAD_2_PARA
PUSH_XMM 6
SIGN_EXTENSION r1, r1d
pxor xmm0, xmm0
movdqa xmm1, [r0]
movdqa xmm2, [r0+r1]
lea r0, [r0+2*r1]
movdqa xmm3, [r0]
movdqa xmm4, [r0+r1]
psadbw xmm1, xmm0
psadbw xmm2, xmm0
psadbw xmm3, xmm0
psadbw xmm4, xmm0
paddw xmm1, xmm2
paddw xmm3, xmm4
paddw xmm1, xmm3
lea r0, [r0+2*r1]
movdqa xmm2, [r0]
movdqa xmm3, [r0+r1]
lea r0, [r0+2*r1]
movdqa xmm4, [r0]
movdqa xmm5, [r0+r1]
psadbw xmm2, xmm0
psadbw xmm3, xmm0
psadbw xmm4, xmm0
psadbw xmm5, xmm0
paddw xmm2, xmm3
paddw xmm4, xmm5
paddw xmm2, xmm4
paddw xmm1, xmm2
lea r0, [r0+2*r1]
movdqa xmm2, [r0]
movdqa xmm3, [r0+r1]
lea r0, [r0+2*r1]
movdqa xmm4, [r0]
movdqa xmm5, [r0+r1]
psadbw xmm2, xmm0
psadbw xmm3, xmm0
psadbw xmm4, xmm0
psadbw xmm5, xmm0
paddw xmm2, xmm3
paddw xmm4, xmm5
paddw xmm2, xmm4
paddw xmm1, xmm2
lea r0, [r0+2*r1]
movdqa xmm2, [r0]
movdqa xmm3, [r0+r1]
lea r0, [r0+2*r1]
movdqa xmm4, [r0]
movdqa xmm5, [r0+r1]
psadbw xmm2, xmm0
psadbw xmm3, xmm0
psadbw xmm4, xmm0
psadbw xmm5, xmm0
paddw xmm2, xmm3
paddw xmm4, xmm5
paddw xmm2, xmm4
paddw xmm1, xmm2
movdqa xmm2, xmm1
punpckhwd xmm2, xmm0
paddw xmm1, xmm2
movd retrd, xmm1
POP_XMM
ret
;**********************************************************************************************************************************
;
; uint32_t SampleSad16x16Hor8_sse41( uint8_t *src, int32_t stride_src, uint8_t *ref, int32_t stride_ref, uint16 base_cost[8], int32_t *index_min_cost )
;
; \note:
; src need align with 16 bytes, ref is optional
; \return value:
; return minimal SAD cost, according index carried by index_min_cost
;**********************************************************************************************************************************
; try 8 mv via offset
; xmm7 store sad costs
%macro SAD_16x16_LINE_SSE41 4 ; src, ref, stride_src, stride_ref
movdqa xmm0, [%1]
movdqu xmm1, [%2]
movdqu xmm2, [%2+8h]
movdqa xmm3, xmm1
movdqa xmm4, xmm2
mpsadbw xmm1, xmm0, 0 ; 000 B
paddw xmm7, xmm1 ; accumulate cost
mpsadbw xmm3, xmm0, 5 ; 101 B
paddw xmm7, xmm3 ; accumulate cost
mpsadbw xmm2, xmm0, 2 ; 010 B
paddw xmm7, xmm2 ; accumulate cost
mpsadbw xmm4, xmm0, 7 ; 111 B
paddw xmm7, xmm4 ; accumulate cost
add %1, %3
add %2, %4
%endmacro ; end of SAD_16x16_LINE_SSE41
%macro SAD_16x16_LINE_SSE41E 4 ; src, ref, stride_src, stride_ref
movdqa xmm0, [%1]
movdqu xmm1, [%2]
movdqu xmm2, [%2+8h]
movdqa xmm3, xmm1
movdqa xmm4, xmm2
mpsadbw xmm1, xmm0, 0 ; 000 B
paddw xmm7, xmm1 ; accumulate cost
mpsadbw xmm3, xmm0, 5 ; 101 B
paddw xmm7, xmm3 ; accumulate cost
mpsadbw xmm2, xmm0, 2 ; 010 B
paddw xmm7, xmm2 ; accumulate cost
mpsadbw xmm4, xmm0, 7 ; 111 B
paddw xmm7, xmm4 ; accumulate cost
%endmacro ; end of SAD_16x16_LINE_SSE41E
WELS_EXTERN SampleSad16x16Hor8_sse41
;push ebx
;push esi
;mov eax, [esp+12] ; src
;mov ecx, [esp+16] ; stride_src
;mov ebx, [esp+20] ; ref
;mov edx, [esp+24] ; stride_ref
;mov esi, [esp+28] ; base_cost
%assign push_num 0
LOAD_6_PARA
PUSH_XMM 8
SIGN_EXTENSION r1, r1d
SIGN_EXTENSION r3, r3d
pxor xmm7, xmm7
SAD_16x16_LINE_SSE41 r0, r2, r1, r3
SAD_16x16_LINE_SSE41 r0, r2, r1, r3
SAD_16x16_LINE_SSE41 r0, r2, r1, r3
SAD_16x16_LINE_SSE41 r0, r2, r1, r3
SAD_16x16_LINE_SSE41 r0, r2, r1, r3
SAD_16x16_LINE_SSE41 r0, r2, r1, r3
SAD_16x16_LINE_SSE41 r0, r2, r1, r3
SAD_16x16_LINE_SSE41 r0, r2, r1, r3
SAD_16x16_LINE_SSE41 r0, r2, r1, r3
SAD_16x16_LINE_SSE41 r0, r2, r1, r3
SAD_16x16_LINE_SSE41 r0, r2, r1, r3
SAD_16x16_LINE_SSE41 r0, r2, r1, r3
SAD_16x16_LINE_SSE41 r0, r2, r1, r3
SAD_16x16_LINE_SSE41 r0, r2, r1, r3
SAD_16x16_LINE_SSE41 r0, r2, r1, r3
SAD_16x16_LINE_SSE41E r0, r2, r1, r3
pxor xmm0, xmm0
movdqa xmm6, xmm7
punpcklwd xmm6, xmm0
punpckhwd xmm7, xmm0
movdqa xmm5, [r4]
movdqa xmm4, xmm5
punpcklwd xmm4, xmm0
punpckhwd xmm5, xmm0
paddd xmm4, xmm6
paddd xmm5, xmm7
movdqa xmm3, xmm4
pminud xmm3, xmm5
pshufd xmm2, xmm3, 01001110B
pminud xmm2, xmm3
pshufd xmm3, xmm2, 10110001B
pminud xmm2, xmm3
movd retrd, xmm2
pcmpeqd xmm4, xmm2
movmskps r2d, xmm4
bsf r1d, r2d
jnz near WRITE_INDEX
pcmpeqd xmm5, xmm2
movmskps r2d, xmm5
bsf r1d, r2d
add r1d, 4
WRITE_INDEX:
mov [r5], r1d
POP_XMM
LOAD_6_PARA_POP
ret
;**********************************************************************************************************************************
;
; uint32_t SampleSad8x8Hor8_sse41( uint8_t *src, int32_t stride_src, uint8_t *ref, int32_t stride_ref, uint16_t base_cost[8], int32_t *index_min_cost )
;
; \note:
; src and ref is optional to align with 16 due inter 8x8
; \return value:
; return minimal SAD cost, according index carried by index_min_cost
;
;**********************************************************************************************************************************
; try 8 mv via offset
; xmm7 store sad costs
%macro SAD_8x8_LINE_SSE41 4 ; src, ref, stride_src, stride_ref
movdqu xmm0, [%1]
movdqu xmm1, [%2]
movdqa xmm2, xmm1
mpsadbw xmm1, xmm0, 0 ; 000 B
paddw xmm7, xmm1 ; accumulate cost
mpsadbw xmm2, xmm0, 5 ; 101 B
paddw xmm7, xmm2 ; accumulate cost
add %1, %3
add %2, %4
%endmacro ; end of SAD_8x8_LINE_SSE41
%macro SAD_8x8_LINE_SSE41E 4 ; src, ref, stride_src, stride_ref
movdqu xmm0, [%1]
movdqu xmm1, [%2]
movdqa xmm2, xmm1
mpsadbw xmm1, xmm0, 0 ; 000 B
paddw xmm7, xmm1 ; accumulate cost
mpsadbw xmm2, xmm0, 5 ; 101 B
paddw xmm7, xmm2 ; accumulate cost
%endmacro ; end of SAD_8x8_LINE_SSE41E
WELS_EXTERN SampleSad8x8Hor8_sse41
%assign push_num 0
LOAD_6_PARA
PUSH_XMM 8
SIGN_EXTENSION r1, r1d
SIGN_EXTENSION r3, r3d
movdqa xmm7, [r4] ; load base cost list
SAD_8x8_LINE_SSE41 r0, r2, r1, r3
SAD_8x8_LINE_SSE41 r0, r2, r1, r3
SAD_8x8_LINE_SSE41 r0, r2, r1, r3
SAD_8x8_LINE_SSE41 r0, r2, r1, r3
SAD_8x8_LINE_SSE41 r0, r2, r1, r3
SAD_8x8_LINE_SSE41 r0, r2, r1, r3
SAD_8x8_LINE_SSE41 r0, r2, r1, r3
SAD_8x8_LINE_SSE41E r0, r2, r1, r3
phminposuw xmm0, xmm7 ; horizon search the minimal sad cost and its index
movd retrd, xmm0 ; for return: DEST[15:0] <- MIN, DEST[31:16] <- INDEX
mov r1d, retrd
and retrd, 0xFFFF
sar r1d, 16
mov [r5], r1d
POP_XMM
LOAD_6_PARA_POP
ret