www.pudn.com > x264_2007.rar > pixel-sse2.asm
;*****************************************************************************
;* pixel-sse2.asm: h264 encoder library
;*****************************************************************************
;* Copyright (C) 2005 x264 project
;*
;* Authors: Alex Izvorski
;* Loren Merritt
;*
;* This program is free software; you can redistribute it and/or modify
;* it under the terms of the GNU General Public License as published by
;* the Free Software Foundation; either version 2 of the License, or
;* (at your option) any later version.
;*
;* This program is distributed in the hope that it will be useful,
;* but WITHOUT ANY WARRANTY; without even the implied warranty of
;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
;* GNU General Public License for more details.
;*
;* You should have received a copy of the GNU General Public License
;* along with this program; if not, write to the Free Software
;* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111, USA.
;*****************************************************************************
BITS 32
;=============================================================================
; Macros and other preprocessor constants
;=============================================================================
%include "i386inc.asm"
SECTION_RODATA
pw_1: times 8 dw 1
ssim_c1: times 4 dd 416 ; .01*.01*255*255*64
ssim_c2: times 4 dd 235963 ; .03*.03*255*255*64*63
mask_ff: times 16 db 0xff
times 16 db 0
SECTION .text
%macro HADDW 2 ; sum junk
; ebx is no longer used at this point, so no push needed
picgetgot ebx
pmaddwd %1, [pw_1 GOT_ebx]
movhlps %2, %1
paddd %1, %2
pshuflw %2, %1, 0xE
paddd %1, %2
%endmacro
%macro SAD_INC_4x16P_SSE2 0
movdqu xmm1, [ecx]
movdqu xmm2, [ecx+edx]
lea ecx, [ecx+2*edx]
movdqu xmm3, [ecx]
movdqu xmm4, [ecx+edx]
psadbw xmm1, [eax]
psadbw xmm2, [eax+ebx]
lea eax, [eax+2*ebx]
psadbw xmm3, [eax]
psadbw xmm4, [eax+ebx]
lea eax, [eax+2*ebx]
lea ecx, [ecx+2*edx]
paddw xmm1, xmm2
paddw xmm3, xmm4
paddw xmm0, xmm1
paddw xmm0, xmm3
%endmacro
%macro SAD_START_SSE2 0
push ebx
mov eax, [esp+ 8] ; pix1
mov ebx, [esp+12] ; stride1
mov ecx, [esp+16] ; pix2
mov edx, [esp+20] ; stride2
%endmacro
%macro SAD_END_SSE2 0
movdqa xmm1, xmm0
psrldq xmm0, 8
paddw xmm0, xmm1
movd eax, xmm0
pop ebx
ret
%endmacro
;-----------------------------------------------------------------------------
; int __cdecl x264_pixel_sad_16x16_sse2 (uint8_t *, int, uint8_t *, int )
;-----------------------------------------------------------------------------
cglobal x264_pixel_sad_16x16_sse2
SAD_START_SSE2
movdqu xmm0, [ecx]
movdqu xmm1, [ecx+edx]
lea ecx, [ecx+2*edx]
movdqu xmm2, [ecx]
movdqu xmm3, [ecx+edx]
lea ecx, [ecx+2*edx]
psadbw xmm0, [eax]
psadbw xmm1, [eax+ebx]
lea eax, [eax+2*ebx]
movdqu xmm4, [ecx]
paddw xmm0, xmm1
psadbw xmm2, [eax]
psadbw xmm3, [eax+ebx]
lea eax, [eax+2*ebx]
movdqu xmm5, [ecx+edx]
lea ecx, [ecx+2*edx]
paddw xmm2, xmm3
movdqu xmm6, [ecx]
movdqu xmm7, [ecx+edx]
lea ecx, [ecx+2*edx]
paddw xmm0, xmm2
psadbw xmm4, [eax]
psadbw xmm5, [eax+ebx]
lea eax, [eax+2*ebx]
movdqu xmm1, [ecx]
paddw xmm4, xmm5
psadbw xmm6, [eax]
psadbw xmm7, [eax+ebx]
lea eax, [eax+2*ebx]
movdqu xmm2, [ecx+edx]
lea ecx, [ecx+2*edx]
paddw xmm6, xmm7
movdqu xmm3, [ecx]
paddw xmm0, xmm4
movdqu xmm4, [ecx+edx]
lea ecx, [ecx+2*edx]
paddw xmm0, xmm6
psadbw xmm1, [eax]
psadbw xmm2, [eax+ebx]
lea eax, [eax+2*ebx]
movdqu xmm5, [ecx]
paddw xmm1, xmm2
psadbw xmm3, [eax]
psadbw xmm4, [eax+ebx]
lea eax, [eax+2*ebx]
movdqu xmm6, [ecx+edx]
lea ecx, [ecx+2*edx]
paddw xmm3, xmm4
movdqu xmm7, [ecx]
paddw xmm0, xmm1
movdqu xmm1, [ecx+edx]
paddw xmm0, xmm3
psadbw xmm5, [eax]
psadbw xmm6, [eax+ebx]
lea eax, [eax+2*ebx]
paddw xmm5, xmm6
psadbw xmm7, [eax]
psadbw xmm1, [eax+ebx]
paddw xmm7, xmm1
paddw xmm0, xmm5
paddw xmm0, xmm7
SAD_END_SSE2
;-----------------------------------------------------------------------------
; int __cdecl x264_pixel_sad_16x8_sse2 (uint8_t *, int, uint8_t *, int )
;-----------------------------------------------------------------------------
cglobal x264_pixel_sad_16x8_sse2
SAD_START_SSE2
pxor xmm0, xmm0
SAD_INC_4x16P_SSE2
SAD_INC_4x16P_SSE2
SAD_END_SSE2
%macro SAD_X3_START_1x16P 0
push edi
push esi
mov edi, [esp+12]
mov eax, [esp+16]
mov ecx, [esp+20]
mov edx, [esp+24]
mov esi, [esp+28]
movdqa xmm3, [edi]
movdqu xmm0, [eax]
movdqu xmm1, [ecx]
movdqu xmm2, [edx]
psadbw xmm0, xmm3
psadbw xmm1, xmm3
psadbw xmm2, xmm3
%endmacro
%macro SAD_X3_1x16P 2
movdqa xmm3, [edi+%1]
movdqu xmm4, [eax+%2]
movdqu xmm5, [ecx+%2]
movdqu xmm6, [edx+%2]
psadbw xmm4, xmm3
psadbw xmm5, xmm3
psadbw xmm6, xmm3
paddw xmm0, xmm4
paddw xmm1, xmm5
paddw xmm2, xmm6
%endmacro
%macro SAD_X3_2x16P 1
%if %1
SAD_X3_START_1x16P
%else
SAD_X3_1x16P 0, 0
%endif
SAD_X3_1x16P FENC_STRIDE, esi
add edi, 2*FENC_STRIDE
lea eax, [eax+2*esi]
lea ecx, [ecx+2*esi]
lea edx, [edx+2*esi]
%endmacro
%macro SAD_X4_START_1x16P 0
push edi
push esi
push ebx
mov edi, [esp+16]
mov eax, [esp+20]
mov ebx, [esp+24]
mov ecx, [esp+28]
mov edx, [esp+32]
mov esi, [esp+36]
movdqa xmm7, [edi]
movdqu xmm0, [eax]
movdqu xmm1, [ebx]
movdqu xmm2, [ecx]
movdqu xmm3, [edx]
psadbw xmm0, xmm7
psadbw xmm1, xmm7
psadbw xmm2, xmm7
psadbw xmm3, xmm7
%endmacro
%macro SAD_X4_1x16P 2
movdqa xmm7, [edi+%1]
movdqu xmm4, [eax+%2]
movdqu xmm5, [ebx+%2]
movdqu xmm6, [ecx+%2]
psadbw xmm4, xmm7
psadbw xmm5, xmm7
paddw xmm0, xmm4
psadbw xmm6, xmm7
movdqu xmm4, [edx+%2]
paddw xmm1, xmm5
psadbw xmm4, xmm7
paddw xmm2, xmm6
paddw xmm3, xmm4
%endmacro
%macro SAD_X4_2x16P 1
%if %1
SAD_X4_START_1x16P
%else
SAD_X4_1x16P 0, 0
%endif
SAD_X4_1x16P FENC_STRIDE, esi
add edi, 2*FENC_STRIDE
lea eax, [eax+2*esi]
lea ebx, [ebx+2*esi]
lea ecx, [ecx+2*esi]
lea edx, [edx+2*esi]
%endmacro
%macro SAD_X3_END 0
mov eax, [esp+32]
pshufd xmm4, xmm0, 2
pshufd xmm5, xmm1, 2
pshufd xmm6, xmm2, 2
paddw xmm0, xmm4
paddw xmm1, xmm5
paddw xmm2, xmm6
movd [eax+0], xmm0
movd [eax+4], xmm1
movd [eax+8], xmm2
pop esi
pop edi
ret
%endmacro
%macro SAD_X4_END 0
mov eax, [esp+40]
pshufd xmm4, xmm0, 2
pshufd xmm5, xmm1, 2
pshufd xmm6, xmm2, 2
pshufd xmm7, xmm3, 2
paddw xmm0, xmm4
paddw xmm1, xmm5
paddw xmm2, xmm6
paddw xmm3, xmm7
movd [eax+0], xmm0
movd [eax+4], xmm1
movd [eax+8], xmm2
movd [eax+12], xmm3
pop ebx
pop esi
pop edi
ret
%endmacro
;-----------------------------------------------------------------------------
; void x264_pixel_sad_x3_16x16_sse2( uint8_t *fenc, uint8_t *pix0, uint8_t *pix1,
; uint8_t *pix2, int i_stride, int scores[3] )
;-----------------------------------------------------------------------------
%macro SAD_X 3
cglobal x264_pixel_sad_x%1_%2x%3_sse2
SAD_X%1_2x%2P 1
%rep %3/2-1
SAD_X%1_2x%2P 0
%endrep
SAD_X%1_END
%endmacro
SAD_X 3, 16, 16
SAD_X 3, 16, 8
SAD_X 4, 16, 16
SAD_X 4, 16, 8
%macro SSD_INC_2x16P_SSE2 0
movdqu xmm1, [eax]
movdqu xmm2, [ecx]
movdqu xmm3, [eax+ebx]
movdqu xmm4, [ecx+edx]
movdqa xmm5, xmm1
movdqa xmm6, xmm3
psubusb xmm1, xmm2
psubusb xmm3, xmm4
psubusb xmm2, xmm5
psubusb xmm4, xmm6
por xmm1, xmm2
por xmm3, xmm4
movdqa xmm2, xmm1
movdqa xmm4, xmm3
punpcklbw xmm1, xmm7
punpckhbw xmm2, xmm7
punpcklbw xmm3, xmm7
punpckhbw xmm4, xmm7
pmaddwd xmm1, xmm1
pmaddwd xmm2, xmm2
pmaddwd xmm3, xmm3
pmaddwd xmm4, xmm4
lea eax, [eax+2*ebx]
lea ecx, [ecx+2*edx]
paddd xmm1, xmm2
paddd xmm3, xmm4
paddd xmm0, xmm1
paddd xmm0, xmm3
%endmacro
%macro SSD_START_SSE2 0
push ebx
mov eax, [esp+ 8] ; pix1
mov ebx, [esp+12] ; stride1
mov ecx, [esp+16] ; pix2
mov edx, [esp+20] ; stride2
pxor xmm7, xmm7 ; zero
pxor xmm0, xmm0 ; mm0 holds the sum
%endmacro
%macro SSD_END_SSE2 0
movdqa xmm1, xmm0
psrldq xmm1, 8
paddd xmm0, xmm1
movdqa xmm1, xmm0
psrldq xmm1, 4
paddd xmm0, xmm1
movd eax, xmm0
pop ebx
ret
%endmacro
;-----------------------------------------------------------------------------
; int __cdecl x264_pixel_ssd_16x16_sse2 (uint8_t *, int, uint8_t *, int )
;-----------------------------------------------------------------------------
cglobal x264_pixel_ssd_16x16_sse2
SSD_START_SSE2
%rep 8
SSD_INC_2x16P_SSE2
%endrep
SSD_END_SSE2
;-----------------------------------------------------------------------------
; int __cdecl x264_pixel_ssd_16x8_sse2 (uint8_t *, int, uint8_t *, int )
;-----------------------------------------------------------------------------
cglobal x264_pixel_ssd_16x8_sse2
SSD_START_SSE2
%rep 4
SSD_INC_2x16P_SSE2
%endrep
SSD_END_SSE2
%macro SUMSUB_BADC 4
paddw %1, %2
paddw %3, %4
paddw %2, %2
paddw %4, %4
psubw %2, %1
psubw %4, %3
%endmacro
%macro HADAMARD1x4 4
SUMSUB_BADC %1, %2, %3, %4
SUMSUB_BADC %1, %3, %2, %4
%endmacro
%macro SBUTTERFLY 5
mov%1 %5, %3
punpckl%2 %3, %4
punpckh%2 %5, %4
%endmacro
%macro SBUTTERFLY2 5 ; not really needed, but allows transpose4x4 to not shuffle registers
mov%1 %5, %3
punpckh%2 %3, %4
punpckl%2 %5, %4
%endmacro
%macro TRANSPOSE4x4D 5 ; ABCD-T -> ADTC
SBUTTERFLY dqa, dq, %1, %2, %5
SBUTTERFLY dqa, dq, %3, %4, %2
SBUTTERFLY dqa, qdq, %1, %3, %4
SBUTTERFLY dqa, qdq, %5, %2, %3
%endmacro
%macro TRANSPOSE2x4x4W 5 ; ABCD-T -> ABCD
SBUTTERFLY dqa, wd, %1, %2, %5
SBUTTERFLY dqa, wd, %3, %4, %2
SBUTTERFLY dqa, dq, %1, %3, %4
SBUTTERFLY2 dqa, dq, %5, %2, %3
SBUTTERFLY dqa, qdq, %1, %3, %2
SBUTTERFLY2 dqa, qdq, %4, %5, %3
%endmacro
%macro LOAD_DIFF_8P 4 ; MMP, MMT, [pix1], [pix2]
movq %1, %3
movq %2, %4
punpcklbw %1, %2
punpcklbw %2, %2
psubw %1, %2
%endmacro
%macro SUM4x4_SSE2 4 ; 02 13 junk sum
pxor %3, %3
psubw %3, %1
pmaxsw %1, %3
pxor %3, %3
psubw %3, %2
pmaxsw %2, %3
paddusw %4, %1
paddusw %4, %2
%endmacro
%macro SUM8x4_SSE2 7 ; a02 a13 junk1 b02 b13 junk2 (1=4 2=5 3=6) sum
pxor %3, %3
pxor %6, %6
psubw %3, %1
psubw %6, %4
pmaxsw %1, %3
pmaxsw %4, %6
pxor %3, %3
pxor %6, %6
psubw %3, %2
psubw %6, %5
pmaxsw %2, %3
pmaxsw %5, %6
paddusw %1, %2
paddusw %4, %5
paddusw %7, %1
paddusw %7, %4
%endmacro
%macro SUM8x4_SSSE3 7 ; a02 a13 . b02 b13 . sum
pabsw %1, %1
pabsw %2, %2
pabsw %4, %4
pabsw %5, %5
paddusw %1, %2
paddusw %4, %5
paddusw %7, %1
paddusw %7, %4
%endmacro
%macro SATD_TWO_SSE2 0
LOAD_DIFF_8P xmm0, xmm4, [eax], [ecx]
LOAD_DIFF_8P xmm1, xmm5, [eax+ebx], [ecx+edx]
lea eax, [eax+2*ebx]
lea ecx, [ecx+2*edx]
LOAD_DIFF_8P xmm2, xmm4, [eax], [ecx]
LOAD_DIFF_8P xmm3, xmm5, [eax+ebx], [ecx+edx]
lea eax, [eax+2*ebx]
lea ecx, [ecx+2*edx]
HADAMARD1x4 xmm0, xmm1, xmm2, xmm3
TRANSPOSE2x4x4W xmm0, xmm1, xmm2, xmm3, xmm4
HADAMARD1x4 xmm0, xmm1, xmm2, xmm3
SUM8x4 xmm0, xmm1, xmm4, xmm2, xmm3, xmm5, xmm6
%endmacro
%macro SATD_START 0
push ebx
mov eax, [esp+ 8] ; pix1
mov ebx, [esp+12] ; stride1
mov ecx, [esp+16] ; pix2
mov edx, [esp+20] ; stride2
pxor xmm6, xmm6
%endmacro
%macro SATD_END 0
; each column sum of SATD is necessarily even, so we don't lose any precision by shifting first.
psrlw xmm6, 1
HADDW xmm6, xmm7
movd eax, xmm6
pop ebx
ret
%endmacro
%macro SATDS 1
;-----------------------------------------------------------------------------
; int __cdecl x264_pixel_satd_16x16_sse2 (uint8_t *, int, uint8_t *, int )
;-----------------------------------------------------------------------------
cglobal x264_pixel_satd_16x16_%1
SATD_START
SATD_TWO_SSE2
SATD_TWO_SSE2
SATD_TWO_SSE2
SATD_TWO_SSE2
mov eax, [esp+ 8]
mov ecx, [esp+16]
add eax, 8
add ecx, 8
SATD_TWO_SSE2
SATD_TWO_SSE2
SATD_TWO_SSE2
SATD_TWO_SSE2
SATD_END
;-----------------------------------------------------------------------------
; int __cdecl x264_pixel_satd_8x16_sse2 (uint8_t *, int, uint8_t *, int )
;-----------------------------------------------------------------------------
cglobal x264_pixel_satd_8x16_%1
SATD_START
SATD_TWO_SSE2
SATD_TWO_SSE2
SATD_TWO_SSE2
SATD_TWO_SSE2
SATD_END
;-----------------------------------------------------------------------------
; int __cdecl x264_pixel_satd_16x8_sse2 (uint8_t *, int, uint8_t *, int )
;-----------------------------------------------------------------------------
cglobal x264_pixel_satd_16x8_%1
SATD_START
SATD_TWO_SSE2
SATD_TWO_SSE2
mov eax, [esp+ 8]
mov ecx, [esp+16]
add eax, 8
add ecx, 8
SATD_TWO_SSE2
SATD_TWO_SSE2
SATD_END
;-----------------------------------------------------------------------------
; int __cdecl x264_pixel_satd_8x8_sse2 (uint8_t *, int, uint8_t *, int )
;-----------------------------------------------------------------------------
cglobal x264_pixel_satd_8x8_%1
SATD_START
SATD_TWO_SSE2
SATD_TWO_SSE2
SATD_END
;-----------------------------------------------------------------------------
; int __cdecl x264_pixel_satd_8x4_sse2 (uint8_t *, int, uint8_t *, int )
;-----------------------------------------------------------------------------
cglobal x264_pixel_satd_8x4_%1
SATD_START
SATD_TWO_SSE2
SATD_END
%endmacro ; SATDS
%define SUM8x4 SUM8x4_SSE2
SATDS sse2
%ifdef HAVE_SSE3
%define SUM8x4 SUM8x4_SSSE3
SATDS ssse3
%endif
;-----------------------------------------------------------------------------
; void x264_pixel_ssim_4x4x2_core_sse2( const uint8_t *pix1, int stride1,
; const uint8_t *pix2, int stride2, int sums[2][4] )
;-----------------------------------------------------------------------------
cglobal x264_pixel_ssim_4x4x2_core_sse2
push ebx
mov eax, [esp+ 8]
mov ebx, [esp+12]
mov ecx, [esp+16]
mov edx, [esp+20]
pxor xmm0, xmm0
pxor xmm1, xmm1
pxor xmm2, xmm2
pxor xmm3, xmm3
pxor xmm4, xmm4
%rep 4
movq xmm5, [eax]
movq xmm6, [ecx]
punpcklbw xmm5, xmm0
punpcklbw xmm6, xmm0
paddw xmm1, xmm5
paddw xmm2, xmm6
movdqa xmm7, xmm5
pmaddwd xmm5, xmm5
pmaddwd xmm7, xmm6
pmaddwd xmm6, xmm6
paddd xmm3, xmm5
paddd xmm4, xmm7
paddd xmm3, xmm6
add eax, ebx
add ecx, edx
%endrep
; PHADDW xmm1, xmm2
; PHADDD xmm3, xmm4
mov eax, [esp+24]
picgetgot ebx
movdqa xmm7, [pw_1 GOT_ebx]
pshufd xmm5, xmm3, 0xB1
pmaddwd xmm1, xmm7
pmaddwd xmm2, xmm7
pshufd xmm6, xmm4, 0xB1
packssdw xmm1, xmm2
paddd xmm3, xmm5
pmaddwd xmm1, xmm7
paddd xmm4, xmm6
pshufd xmm1, xmm1, 0xD8
movdqa xmm5, xmm3
punpckldq xmm3, xmm4
punpckhdq xmm5, xmm4
movq [eax+ 0], xmm1
movq [eax+ 8], xmm3
psrldq xmm1, 8
movq [eax+16], xmm1
movq [eax+24], xmm5
pop ebx
ret
;-----------------------------------------------------------------------------
; float x264_pixel_ssim_end_sse2( int sum0[5][4], int sum1[5][4], int width )
;-----------------------------------------------------------------------------
cglobal x264_pixel_ssim_end4_sse2
mov eax, [esp+ 4]
mov ecx, [esp+ 8]
mov edx, [esp+12]
picpush ebx
picgetgot ebx
movdqa xmm0, [eax+ 0]
movdqa xmm1, [eax+16]
movdqa xmm2, [eax+32]
movdqa xmm3, [eax+48]
movdqa xmm4, [eax+64]
paddd xmm0, [ecx+ 0]
paddd xmm1, [ecx+16]
paddd xmm2, [ecx+32]
paddd xmm3, [ecx+48]
paddd xmm4, [ecx+64]
paddd xmm0, xmm1
paddd xmm1, xmm2
paddd xmm2, xmm3
paddd xmm3, xmm4
movdqa xmm5, [ssim_c1 GOT_ebx]
movdqa xmm6, [ssim_c2 GOT_ebx]
TRANSPOSE4x4D xmm0, xmm1, xmm2, xmm3, xmm4
; s1=mm0, s2=mm3, ss=mm4, s12=mm2
movdqa xmm1, xmm3
pslld xmm3, 16
pmaddwd xmm1, xmm0 ; s1*s2
por xmm0, xmm3
pmaddwd xmm0, xmm0 ; s1*s1 + s2*s2
pslld xmm1, 1
pslld xmm2, 7
pslld xmm4, 6
psubd xmm2, xmm1 ; covar*2
psubd xmm4, xmm0 ; vars
paddd xmm0, xmm5
paddd xmm1, xmm5
paddd xmm2, xmm6
paddd xmm4, xmm6
cvtdq2ps xmm0, xmm0 ; (float)(s1*s1 + s2*s2 + ssim_c1)
cvtdq2ps xmm1, xmm1 ; (float)(s1*s2*2 + ssim_c1)
cvtdq2ps xmm2, xmm2 ; (float)(covar*2 + ssim_c2)
cvtdq2ps xmm4, xmm4 ; (float)(vars + ssim_c2)
mulps xmm1, xmm2
mulps xmm0, xmm4
divps xmm1, xmm0 ; ssim
neg edx
movdqu xmm3, [mask_ff + edx*4 + 16 GOT_ebx]
pand xmm1, xmm3
movhlps xmm0, xmm1
addps xmm0, xmm1
pshuflw xmm1, xmm0, 0xE
addss xmm0, xmm1
movd [picesp+4], xmm0
fld dword [picesp+4]
picpop ebx
ret