wuggy/Dactyloidae
wuggy/Dactyloidae
1
0
Fork 0
mirror of https://repo.dactyloidae.xyz/Dactyloidae/UXP.git synced 2026-09-22 08:27:31 +09:00
Code Issues Projects Releases Packages Wiki Activity

ffvpx: update ffmpeg to 3.4.4 with AVC/AAC/HEVC/FLAC added.

Browse source
This commit is contained in:
Roy Tam 2019-02-22 21:56:49 +08:00
commit 113a86e60a
377 changed files with 133192 additions and 291 deletions
Download patch file Download diff file Expand all files Collapse all files

487
media/ffvpx/libavcodec/x86/aacpsdsp.asm Normal file
View file

@ -0,0 +1,487 @@
;******************************************************************************
;* SIMD optimized MPEG-4 Parametric Stereo decoding functions
;*
;* Copyright (C) 2015 James Almer
;*
;* This file is part of FFmpeg.
;*
;* FFmpeg is free software; you can redistribute it and/or
;* modify it under the terms of the GNU Lesser General Public
;* License as published by the Free Software Foundation; either
;* version 2.1 of the License, or (at your option) any later version.
;*
;* FFmpeg is distributed in the hope that it will be useful,
;* but WITHOUT ANY WARRANTY; without even the implied warranty of
;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
;* Lesser General Public License for more details.
;*
;* You should have received a copy of the GNU Lesser General Public
;* License along with FFmpeg; if not, write to the Free Software
;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
;******************************************************************************
%include "libavutil/x86/x86util.asm"
SECTION_RODATA
ps_p1m1p1m1: dd 0, 0x80000000, 0, 0x80000000
SECTION .text
;*************************************************************************
;void ff_ps_add_squares_<opt>(float *dst, const float (*src)[2], int n);
;*************************************************************************
%macro PS_ADD_SQUARES 1
cglobal ps_add_squares, 3, 3, %1, dst, src, n
shl nd, 3
add srcq, nq
neg nq
align 16
.loop:
movaps m0, [srcq+nq]
movaps m1, [srcq+nq+mmsize]
mulps m0, m0
mulps m1, m1
HADDPS m0, m1, m2
addps m0, [dstq]
movaps [dstq], m0
add dstq, mmsize
add nq, mmsize*2
jl .loop
REP_RET
%endmacro
INIT_XMM sse
PS_ADD_SQUARES 2
INIT_XMM sse3
PS_ADD_SQUARES 3
;*******************************************************************
;void ff_ps_mul_pair_single_sse(float (*dst)[2], float (*src0)[2],
; float *src1, int n);
;*******************************************************************
INIT_XMM sse
cglobal ps_mul_pair_single, 4, 4, 4, dst, src1, src2, n
shl nd, 3
add src1q, nq
add dstq, nq
neg nq
align 16
.loop:
movu m0, [src1q+nq]
movu m1, [src1q+nq+mmsize]
mova m2, [src2q]
mova m3, m2
unpcklps m2, m2
unpckhps m3, m3
mulps m0, m2
mulps m1, m3
mova [dstq+nq], m0
mova [dstq+nq+mmsize], m1
add src2q, mmsize
add nq, mmsize*2
jl .loop
REP_RET
;***********************************************************************
;void ff_ps_stereo_interpolate_sse3(float (*l)[2], float (*r)[2],
; float h[2][4], float h_step[2][4],
; int len);
;***********************************************************************
INIT_XMM sse3
cglobal ps_stereo_interpolate, 5, 5, 6, l, r, h, h_step, n
movaps m0, [hq]
movaps m1, [h_stepq]
unpcklps m4, m0, m0
unpckhps m0, m0
unpcklps m5, m1, m1
unpckhps m1, m1
shl nd, 3
add lq, nq
add rq, nq
neg nq
align 16
.loop:
addps m4, m5
addps m0, m1
movddup m2, [lq+nq]
movddup m3, [rq+nq]
mulps m2, m4
mulps m3, m0
addps m2, m3
movsd [lq+nq], m2
movhps [rq+nq], m2
add nq, 8
jl .loop
REP_RET
;***************************************************************************
;void ps_stereo_interpolate_ipdopd_sse3(float (*l)[2], float (*r)[2],
; float h[2][4], float h_step[2][4],
; int len);
;***************************************************************************
INIT_XMM sse3
cglobal ps_stereo_interpolate_ipdopd, 5, 5, 10, l, r, h, h_step, n
movaps m0, [hq]
movaps m1, [hq+mmsize]
%if ARCH_X86_64
movaps m8, [h_stepq]
movaps m9, [h_stepq+mmsize]
%define H_STEP0 m8
%define H_STEP1 m9
%else
%define H_STEP0 [h_stepq]
%define H_STEP1 [h_stepq+mmsize]
%endif
shl nd, 3
add lq, nq
add rq, nq
neg nq
align 16
.loop:
addps m0, H_STEP0
addps m1, H_STEP1
movddup m2, [lq+nq]
movddup m3, [rq+nq]
shufps m4, m2, m2, q2301
shufps m5, m3, m3, q2301
unpcklps m6, m0, m0
unpckhps m7, m0, m0
mulps m2, m6
mulps m3, m7
unpcklps m6, m1, m1
unpckhps m7, m1, m1
mulps m4, m6
mulps m5, m7
addps m2, m3
addsubps m2, m4
addsubps m2, m5
movsd [lq+nq], m2
movhps [rq+nq], m2
add nq, 8
jl .loop
REP_RET
;**********************************************************
;void ps_hybrid_analysis_ileave_sse(float out[2][38][64],
; float (*in)[32][2],
; int i, int len)
;**********************************************************
INIT_XMM sse
cglobal ps_hybrid_analysis_ileave, 3, 7, 5, out, in, i, len, in0, in1, tmp
movsxdifnidn iq, id
mov lend, 32 << 3
lea inq, [inq+iq*4]
mov tmpd, id
shl tmpd, 8
add outq, tmpq
mov tmpd, 64
sub tmpd, id
mov id, tmpd
test id, 1
jne .loop4
test id, 2
jne .loop8
align 16
.loop16:
mov in0q, inq
mov in1q, 38*64*4
add in1q, in0q
mov tmpd, lend
.inner_loop16:
movaps m0, [in0q]
movaps m1, [in1q]
movaps m2, [in0q+lenq]
movaps m3, [in1q+lenq]
TRANSPOSE4x4PS 0, 1, 2, 3, 4
movaps [outq], m0
movaps [outq+lenq], m1
movaps [outq+lenq*2], m2
movaps [outq+3*32*2*4], m3
lea in0q, [in0q+lenq*2]
lea in1q, [in1q+lenq*2]
add outq, mmsize
sub tmpd, mmsize
jg .inner_loop16
add inq, 16
add outq, 3*32*2*4
sub id, 4
jg .loop16
RET
align 16
.loop8:
mov in0q, inq
mov in1q, 38*64*4
add in1q, in0q
mov tmpd, lend
.inner_loop8:
movlps m0, [in0q]
movlps m1, [in1q]
movhps m0, [in0q+lenq]
movhps m1, [in1q+lenq]
SBUTTERFLYPS 0, 1, 2
SBUTTERFLYPD 0, 1, 2
movaps [outq], m0
movaps [outq+lenq], m1
lea in0q, [in0q+lenq*2]
lea in1q, [in1q+lenq*2]
add outq, mmsize
sub tmpd, mmsize
jg .inner_loop8
add inq, 8
add outq, lenq
sub id, 2
jg .loop16
RET
align 16
.loop4:
mov in0q, inq
mov in1q, 38*64*4
add in1q, in0q
mov tmpd, lend
.inner_loop4:
movss m0, [in0q]
movss m1, [in1q]
movss m2, [in0q+lenq]
movss m3, [in1q+lenq]
movlhps m0, m1
movlhps m2, m3
shufps m0, m2, q2020
movaps [outq], m0
lea in0q, [in0q+lenq*2]
lea in1q, [in1q+lenq*2]
add outq, mmsize
sub tmpd, mmsize
jg .inner_loop4
add inq, 4
sub id, 1
test id, 2
jne .loop8
cmp id, 4
jge .loop16
RET
;***********************************************************
;void ps_hybrid_synthesis_deint_sse4(float out[2][38][64],
; float (*in)[32][2],
; int i, int len)
;***********************************************************
%macro HYBRID_SYNTHESIS_DEINT 0
cglobal ps_hybrid_synthesis_deint, 3, 7, 5, out, in, i, len, out0, out1, tmp
%if cpuflag(sse4)
%define MOVH movsd
%else
%define MOVH movlps
%endif
movsxdifnidn iq, id
mov lend, 32 << 3
lea outq, [outq+iq*4]
mov tmpd, id
shl tmpd, 8
add inq, tmpq
mov tmpd, 64
sub tmpd, id
mov id, tmpd
test id, 1
jne .loop4
test id, 2
jne .loop8
align 16
.loop16:
mov out0q, outq
mov out1q, 38*64*4
add out1q, out0q
mov tmpd, lend
.inner_loop16:
movaps m0, [inq]
movaps m1, [inq+lenq]
movaps m2, [inq+lenq*2]
movaps m3, [inq+3*32*2*4]
TRANSPOSE4x4PS 0, 1, 2, 3, 4
movaps [out0q], m0
movaps [out1q], m1
movaps [out0q+lenq], m2
movaps [out1q+lenq], m3
lea out0q, [out0q+lenq*2]
lea out1q, [out1q+lenq*2]
add inq, mmsize
sub tmpd, mmsize
jg .inner_loop16
add outq, 16
add inq, 3*32*2*4
sub id, 4
jg .loop16
RET
align 16
.loop8:
mov out0q, outq
mov out1q, 38*64*4
add out1q, out0q
mov tmpd, lend
.inner_loop8:
movaps m0, [inq]
movaps m1, [inq+lenq]
SBUTTERFLYPS 0, 1, 2
SBUTTERFLYPD 0, 1, 2
MOVH [out0q], m0
MOVH [out1q], m1
movhps [out0q+lenq], m0
movhps [out1q+lenq], m1
lea out0q, [out0q+lenq*2]
lea out1q, [out1q+lenq*2]
add inq, mmsize
sub tmpd, mmsize
jg .inner_loop8
add outq, 8
add inq, lenq
sub id, 2
jg .loop16
RET
align 16
.loop4:
mov out0q, outq
mov out1q, 38*64*4
add out1q, out0q
mov tmpd, lend
.inner_loop4:
movaps m0, [inq]
movss [out0q], m0
%if cpuflag(sse4)
extractps [out1q], m0, 1
extractps [out0q+lenq], m0, 2
extractps [out1q+lenq], m0, 3
%else
movhlps m1, m0
movss [out0q+lenq], m1
shufps m0, m0, 0xb1
movss [out1q], m0
movhlps m1, m0
movss [out1q+lenq], m1
%endif
lea out0q, [out0q+lenq*2]
lea out1q, [out1q+lenq*2]
add inq, mmsize
sub tmpd, mmsize
jg .inner_loop4
add outq, 4
sub id, 1
test id, 2
jne .loop8
cmp id, 4
jge .loop16
RET
%endmacro
INIT_XMM sse
HYBRID_SYNTHESIS_DEINT
INIT_XMM sse4
HYBRID_SYNTHESIS_DEINT
;*******************************************************************
;void ff_ps_hybrid_analysis_<opt>(float (*out)[2], float (*in)[2],
; const float (*filter)[8][2],
; ptrdiff_t stride, int n);
;*******************************************************************
%macro PS_HYBRID_ANALYSIS_LOOP 3
movu %1, [inq+mmsize*%3]
movu m1, [inq+mmsize*(5-%3)+8]
%if cpuflag(sse3)
pshufd %2, %1, q2301
pshufd m4, m1, q0123
pshufd m1, m1, q1032
pshufd m2, [filterq+nq+mmsize*%3], q2301
addsubps %2, m4
addsubps %1, m1
%else
mova m2, [filterq+nq+mmsize*%3]
mova %2, %1
mova m4, m1
shufps %2, %2, q2301
shufps m4, m4, q0123
shufps m1, m1, q1032
shufps m2, m2, q2301
xorps m4, m7
xorps m1, m7
subps %2, m4
subps %1, m1
%endif
mulps %2, m2
mulps %1, m2
%if %3
addps m3, %2
addps m0, %1
%endif
%endmacro
%macro PS_HYBRID_ANALYSIS 0
cglobal ps_hybrid_analysis, 5, 5, 8, out, in, filter, stride, n
%if cpuflag(sse3)
%define MOVH movsd
%else
%define MOVH movlps
%endif
shl strideq, 3
shl nd, 6
add filterq, nq
neg nq
mova m7, [ps_p1m1p1m1]
align 16
.loop:
PS_HYBRID_ANALYSIS_LOOP m0, m3, 0
PS_HYBRID_ANALYSIS_LOOP m5, m6, 1
PS_HYBRID_ANALYSIS_LOOP m5, m6, 2
%if cpuflag(sse3)
pshufd m3, m3, q2301
xorps m0, m7
hsubps m3, m0
pshufd m1, m3, q0020
pshufd m3, m3, q0031
addps m1, m3
movsd m2, [inq+6*8]
%else
mova m1, m3
mova m2, m0
shufps m1, m1, q2301
shufps m2, m2, q2301
subps m1, m3
addps m2, m0
unpcklps m3, m1, m2
unpckhps m1, m2
addps m1, m3
movu m2, [inq+6*8] ; faster than movlps and no risk of overread
%endif
movss m3, [filterq+nq+8*6]
SPLATD m3
mulps m2, m3
addps m1, m2
MOVH [outq], m1
add outq, strideq
add nq, 64
jl .loop
REP_RET
%endmacro
INIT_XMM sse
PS_HYBRID_ANALYSIS
INIT_XMM sse3
PS_HYBRID_ANALYSIS

72
media/ffvpx/libavcodec/x86/aacpsdsp_init.c Normal file
View file

@ -0,0 +1,72 @@
/*
* SIMD optimized MPEG-4 Parametric Stereo decoding functions
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include <stddef.h>
#include "config.h"
#include "libavutil/x86/cpu.h"
#include "libavutil/attributes.h"
#include "libavcodec/aacpsdsp.h"
void ff_ps_add_squares_sse (float *dst, const float (*src)[2], int n);
void ff_ps_add_squares_sse3 (float *dst, const float (*src)[2], int n);
void ff_ps_mul_pair_single_sse (float (*dst)[2], float (*src0)[2],
float *src1, int n);
void ff_ps_hybrid_analysis_sse (float (*out)[2], float (*in)[2],
const float (*filter)[8][2],
ptrdiff_t stride, int n);
void ff_ps_hybrid_analysis_sse3(float (*out)[2], float (*in)[2],
const float (*filter)[8][2],
ptrdiff_t stride, int n);
void ff_ps_stereo_interpolate_sse3(float (*l)[2], float (*r)[2],
float h[2][4], float h_step[2][4],
int len);
void ff_ps_stereo_interpolate_ipdopd_sse3(float (*l)[2], float (*r)[2],
float h[2][4], float h_step[2][4],
int len);
void ff_ps_hybrid_synthesis_deint_sse(float out[2][38][64], float (*in)[32][2],
int i, int len);
void ff_ps_hybrid_synthesis_deint_sse4(float out[2][38][64], float (*in)[32][2],
int i, int len);
void ff_ps_hybrid_analysis_ileave_sse(float (*out)[32][2], float L[2][38][64],
int i, int len);
av_cold void ff_psdsp_init_x86(PSDSPContext *s)
{
int cpu_flags = av_get_cpu_flags();
if (EXTERNAL_SSE(cpu_flags)) {
s->add_squares = ff_ps_add_squares_sse;
s->mul_pair_single = ff_ps_mul_pair_single_sse;
s->hybrid_analysis_ileave = ff_ps_hybrid_analysis_ileave_sse;
s->hybrid_synthesis_deint = ff_ps_hybrid_synthesis_deint_sse;
s->hybrid_analysis = ff_ps_hybrid_analysis_sse;
}
if (EXTERNAL_SSE3(cpu_flags)) {
s->add_squares = ff_ps_add_squares_sse3;
s->stereo_interpolate[0] = ff_ps_stereo_interpolate_sse3;
s->stereo_interpolate[1] = ff_ps_stereo_interpolate_ipdopd_sse3;
s->hybrid_analysis = ff_ps_hybrid_analysis_sse3;
}
if (EXTERNAL_SSE4(cpu_flags)) {
s->hybrid_synthesis_deint = ff_ps_hybrid_synthesis_deint_sse4;
}
}

139
media/ffvpx/libavcodec/x86/bswapdsp.asm Normal file
View file

@ -0,0 +1,139 @@
;******************************************************************************
;* optimized bswap buffer functions
;* Copyright (c) 2008 Loren Merritt
;* Copyright (c) 2003-2013 Michael Niedermayer
;* Copyright (c) 2013 Daniel Kang
;*
;* This file is part of FFmpeg.
;*
;* FFmpeg is free software; you can redistribute it and/or
;* modify it under the terms of the GNU Lesser General Public
;* License as published by the Free Software Foundation; either
;* version 2.1 of the License, or (at your option) any later version.
;*
;* FFmpeg is distributed in the hope that it will be useful,
;* but WITHOUT ANY WARRANTY; without even the implied warranty of
;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
;* Lesser General Public License for more details.
;*
;* You should have received a copy of the GNU Lesser General Public
;* License along with FFmpeg; if not, write to the Free Software
;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
;******************************************************************************
%include "libavutil/x86/x86util.asm"
SECTION_RODATA
pb_bswap32: db 3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12
cextern pb_80
SECTION .text
; %1 = aligned/unaligned
%macro BSWAP_LOOPS 1
mov r3d, r2d
sar r2d, 3
jz .left4_%1
.loop8_%1:
mov%1 m0, [r1 + 0]
mov%1 m1, [r1 + 16]
%if cpuflag(ssse3)
pshufb m0, m2
pshufb m1, m2
mov%1 [r0 + 0], m0
mov%1 [r0 + 16], m1
%else
pshuflw m0, m0, 10110001b
pshuflw m1, m1, 10110001b
pshufhw m0, m0, 10110001b
pshufhw m1, m1, 10110001b
mova m2, m0
mova m3, m1
psllw m0, 8
psllw m1, 8
psrlw m2, 8
psrlw m3, 8
por m2, m0
por m3, m1
mov%1 [r0 + 0], m2
mov%1 [r0 + 16], m3
%endif
add r0, 32
add r1, 32
dec r2d
jnz .loop8_%1
.left4_%1:
mov r2d, r3d
test r3d, 4
jz .left
mov%1 m0, [r1]
%if cpuflag(ssse3)
pshufb m0, m2
mov%1 [r0], m0
%else
pshuflw m0, m0, 10110001b
pshufhw m0, m0, 10110001b
mova m2, m0
psllw m0, 8
psrlw m2, 8
por m2, m0
mov%1 [r0], m2
%endif
add r1, 16
add r0, 16
%endmacro
; void ff_bswap_buf(uint32_t *dst, const uint32_t *src, int w);
%macro BSWAP32_BUF 0
%if cpuflag(ssse3)
cglobal bswap32_buf, 3,4,3
mov r3, r1
mova m2, [pb_bswap32]
%else
cglobal bswap32_buf, 3,4,5
mov r3, r1
%endif
or r3, r0
test r3, 15
jz .start_align
BSWAP_LOOPS u
jmp .left
.start_align:
BSWAP_LOOPS a
.left:
%if cpuflag(ssse3)
test r2d, 2
jz .left1
movq m0, [r1]
pshufb m0, m2
movq [r0], m0
add r1, 8
add r0, 8
.left1:
test r2d, 1
jz .end
mov r2d, [r1]
bswap r2d
mov [r0], r2d
%else
and r2d, 3
jz .end
.loop2:
mov r3d, [r1]
bswap r3d
mov [r0], r3d
add r1, 4
add r0, 4
dec r2d
jnz .loop2
%endif
.end:
RET
%endmacro
INIT_XMM sse2
BSWAP32_BUF
INIT_XMM ssse3
BSWAP32_BUF

37
media/ffvpx/libavcodec/x86/bswapdsp_init.c Normal file
View file

@ -0,0 +1,37 @@
/*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include <stdint.h>
#include "libavutil/attributes.h"
#include "libavutil/cpu.h"
#include "libavutil/x86/cpu.h"
#include "libavcodec/bswapdsp.h"
void ff_bswap32_buf_sse2(uint32_t *dst, const uint32_t *src, int w);
void ff_bswap32_buf_ssse3(uint32_t *dst, const uint32_t *src, int w);
av_cold void ff_bswapdsp_init_x86(BswapDSPContext *c)
{
int cpu_flags = av_get_cpu_flags();
if (EXTERNAL_SSE2(cpu_flags))
c->bswap_buf = ff_bswap32_buf_sse2;
if (EXTERNAL_SSSE3(cpu_flags))
c->bswap_buf = ff_bswap32_buf_ssse3;
}

301
media/ffvpx/libavcodec/x86/cabac.h Normal file
View file

@ -0,0 +1,301 @@
/*
* Copyright (c) 2003 Michael Niedermayer <michaelni@gmx.at>
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#ifndef AVCODEC_X86_CABAC_H
#define AVCODEC_X86_CABAC_H
#include "libavcodec/cabac.h"
#include "libavutil/attributes.h"
#include "libavutil/macros.h"
#include "libavutil/x86/asm.h"
#include "config.h"
#if (defined(__i386) && defined(__clang__) && (__clang_major__<2 || (__clang_major__==2 && __clang_minor__<10)))\
|| ( !defined(__clang__) && defined(__llvm__) && __GNUC__==4 && __GNUC_MINOR__==2 && __GNUC_PATCHLEVEL__<=1)\
|| (defined(__INTEL_COMPILER) && defined(_MSC_VER))
# define BROKEN_COMPILER 1
#else
# define BROKEN_COMPILER 0
#endif
#if HAVE_INLINE_ASM
#ifndef UNCHECKED_BITSTREAM_READER
#define UNCHECKED_BITSTREAM_READER !CONFIG_SAFE_BITSTREAM_READER
#endif
#if UNCHECKED_BITSTREAM_READER
#define END_CHECK(end) ""
#else
#define END_CHECK(end) \
"cmp "end" , %%"FF_REG_c" \n\t"\
"jge 1f \n\t"
#endif
#ifdef BROKEN_RELOCATIONS
#define TABLES_ARG , "r"(tables)
#if HAVE_FAST_CMOV
#define BRANCHLESS_GET_CABAC_UPDATE(ret, retq, low, range, tmp) \
"cmp "low" , "tmp" \n\t"\
"cmova %%ecx , "range" \n\t"\
"sbb %%rcx , %%rcx \n\t"\
"and %%ecx , "tmp" \n\t"\
"xor %%rcx , "retq" \n\t"\
"sub "tmp" , "low" \n\t"
#else /* HAVE_FAST_CMOV */
#define BRANCHLESS_GET_CABAC_UPDATE(ret, retq, low, range, tmp) \
/* P4 Prescott has crappy cmov,sbb,64-bit shift so avoid them */ \
"sub "low" , "tmp" \n\t"\
"sar $31 , "tmp" \n\t"\
"sub %%ecx , "range" \n\t"\
"and "tmp" , "range" \n\t"\
"add %%ecx , "range" \n\t"\
"shl $17 , %%ecx \n\t"\
"and "tmp" , %%ecx \n\t"\
"sub %%ecx , "low" \n\t"\
"xor "tmp" , "ret" \n\t"\
"movslq "ret" , "retq" \n\t"
#endif /* HAVE_FAST_CMOV */
#define BRANCHLESS_GET_CABAC(ret, retq, statep, low, lowword, range, rangeq, tmp, tmpbyte, byte, end, norm_off, lps_off, mlps_off, tables) \
"movzbl "statep" , "ret" \n\t"\
"mov "range" , "tmp" \n\t"\
"and $0xC0 , "range" \n\t"\
"lea ("ret", "range", 2), %%ecx \n\t"\
"movzbl "lps_off"("tables", %%rcx), "range" \n\t"\
"sub "range" , "tmp" \n\t"\
"mov "tmp" , %%ecx \n\t"\
"shl $17 , "tmp" \n\t"\
BRANCHLESS_GET_CABAC_UPDATE(ret, retq, low, range, tmp) \
"movzbl "norm_off"("tables", "rangeq"), %%ecx \n\t"\
"shl %%cl , "range" \n\t"\
"movzbl "mlps_off"+128("tables", "retq"), "tmp" \n\t"\
"shl %%cl , "low" \n\t"\
"mov "tmpbyte" , "statep" \n\t"\
"test "lowword" , "lowword" \n\t"\
"jnz 2f \n\t"\
"mov "byte" , %%"FF_REG_c" \n\t"\
END_CHECK(end)\
"add"FF_OPSIZE" $2 , "byte" \n\t"\
"1: \n\t"\
"movzwl (%%"FF_REG_c") , "tmp" \n\t"\
"lea -1("low") , %%ecx \n\t"\
"xor "low" , %%ecx \n\t"\
"shr $15 , %%ecx \n\t"\
"bswap "tmp" \n\t"\
"shr $15 , "tmp" \n\t"\
"movzbl "norm_off"("tables", %%rcx), %%ecx \n\t"\
"sub $0xFFFF , "tmp" \n\t"\
"neg %%ecx \n\t"\
"add $7 , %%ecx \n\t"\
"shl %%cl , "tmp" \n\t"\
"add "tmp" , "low" \n\t"\
"2: \n\t"
#else /* BROKEN_RELOCATIONS */
#define TABLES_ARG NAMED_CONSTRAINTS_ARRAY_ADD(ff_h264_cabac_tables)
#define RIP_ARG
#if HAVE_FAST_CMOV
#define BRANCHLESS_GET_CABAC_UPDATE(ret, low, range, tmp)\
"mov "tmp" , %%ecx \n\t"\
"shl $17 , "tmp" \n\t"\
"cmp "low" , "tmp" \n\t"\
"cmova %%ecx , "range" \n\t"\
"sbb %%ecx , %%ecx \n\t"\
"and %%ecx , "tmp" \n\t"\
"xor %%ecx , "ret" \n\t"\
"sub "tmp" , "low" \n\t"
#else /* HAVE_FAST_CMOV */
#define BRANCHLESS_GET_CABAC_UPDATE(ret, low, range, tmp)\
"mov "tmp" , %%ecx \n\t"\
"shl $17 , "tmp" \n\t"\
"sub "low" , "tmp" \n\t"\
"sar $31 , "tmp" \n\t" /*lps_mask*/\
"sub %%ecx , "range" \n\t" /*RangeLPS - range*/\
"and "tmp" , "range" \n\t" /*(RangeLPS - range)&lps_mask*/\
"add %%ecx , "range" \n\t" /*new range*/\
"shl $17 , %%ecx \n\t"\
"and "tmp" , %%ecx \n\t"\
"sub %%ecx , "low" \n\t"\
"xor "tmp" , "ret" \n\t"
#endif /* HAVE_FAST_CMOV */
#define BRANCHLESS_GET_CABAC(ret, retq, statep, low, lowword, range, rangeq, tmp, tmpbyte, byte, end, norm_off, lps_off, mlps_off, tables) \
"movzbl "statep" , "ret" \n\t"\
"mov "range" , "tmp" \n\t"\
"and $0xC0 , "range" \n\t"\
"movzbl "MANGLE(ff_h264_cabac_tables)"+"lps_off"("ret", "range", 2), "range" \n\t"\
"sub "range" , "tmp" \n\t"\
BRANCHLESS_GET_CABAC_UPDATE(ret, low, range, tmp) \
"movzbl "MANGLE(ff_h264_cabac_tables)"+"norm_off"("range"), %%ecx \n\t"\
"shl %%cl , "range" \n\t"\
"movzbl "MANGLE(ff_h264_cabac_tables)"+"mlps_off"+128("ret"), "tmp" \n\t"\
"shl %%cl , "low" \n\t"\
"mov "tmpbyte" , "statep" \n\t"\
"test "lowword" , "lowword" \n\t"\
" jnz 2f \n\t"\
"mov "byte" , %%"FF_REG_c" \n\t"\
END_CHECK(end)\
"add"FF_OPSIZE" $2 , "byte" \n\t"\
"1: \n\t"\
"movzwl (%%"FF_REG_c") , "tmp" \n\t"\
"lea -1("low") , %%ecx \n\t"\
"xor "low" , %%ecx \n\t"\
"shr $15 , %%ecx \n\t"\
"bswap "tmp" \n\t"\
"shr $15 , "tmp" \n\t"\
"movzbl "MANGLE(ff_h264_cabac_tables)"+"norm_off"(%%ecx), %%ecx \n\t"\
"sub $0xFFFF , "tmp" \n\t"\
"neg %%ecx \n\t"\
"add $7 , %%ecx \n\t"\
"shl %%cl , "tmp" \n\t"\
"add "tmp" , "low" \n\t"\
"2: \n\t"
#endif /* BROKEN_RELOCATIONS */
#if HAVE_7REGS && !BROKEN_COMPILER
#define get_cabac_inline get_cabac_inline_x86
static av_always_inline int get_cabac_inline_x86(CABACContext *c,
uint8_t *const state)
{
int bit, tmp;
#ifdef BROKEN_RELOCATIONS
void *tables;
__asm__ volatile(
"lea "MANGLE(ff_h264_cabac_tables)", %0 \n\t"
: "=&r"(tables)
: NAMED_CONSTRAINTS_ARRAY(ff_h264_cabac_tables)
);
#endif
__asm__ volatile(
BRANCHLESS_GET_CABAC("%0", "%q0", "(%4)", "%1", "%w1",
"%2", "%q2", "%3", "%b3",
"%c6(%5)", "%c7(%5)",
AV_STRINGIFY(H264_NORM_SHIFT_OFFSET),
AV_STRINGIFY(H264_LPS_RANGE_OFFSET),
AV_STRINGIFY(H264_MLPS_STATE_OFFSET),
"%8")
: "=&r"(bit), "=&r"(c->low), "=&r"(c->range), "=&q"(tmp)
: "r"(state), "r"(c),
"i"(offsetof(CABACContext, bytestream)),
"i"(offsetof(CABACContext, bytestream_end))
TABLES_ARG
,"1"(c->low), "2"(c->range)
: "%"FF_REG_c, "memory"
);
return bit & 1;
}
#endif /* HAVE_7REGS && !BROKEN_COMPILER */
#if !BROKEN_COMPILER
#define get_cabac_bypass_sign get_cabac_bypass_sign_x86
static av_always_inline int get_cabac_bypass_sign_x86(CABACContext *c, int val)
{
x86_reg tmp;
__asm__ volatile(
"movl %c6(%2), %k1 \n\t"
"movl %c3(%2), %%eax \n\t"
"shl $17, %k1 \n\t"
"add %%eax, %%eax \n\t"
"sub %k1, %%eax \n\t"
"cdq \n\t"
"and %%edx, %k1 \n\t"
"add %k1, %%eax \n\t"
"xor %%edx, %%ecx \n\t"
"sub %%edx, %%ecx \n\t"
"test %%ax, %%ax \n\t"
"jnz 1f \n\t"
"mov %c4(%2), %1 \n\t"
"subl $0xFFFF, %%eax \n\t"
"movzwl (%1), %%edx \n\t"
"bswap %%edx \n\t"
"shrl $15, %%edx \n\t"
#if UNCHECKED_BITSTREAM_READER
"add $2, %1 \n\t"
"addl %%edx, %%eax \n\t"
"mov %1, %c4(%2) \n\t"
#else
"addl %%edx, %%eax \n\t"
"cmp %c5(%2), %1 \n\t"
"jge 1f \n\t"
"add"FF_OPSIZE" $2, %c4(%2) \n\t"
#endif
"1: \n\t"
"movl %%eax, %c3(%2) \n\t"
: "+c"(val), "=&r"(tmp)
: "r"(c),
"i"(offsetof(CABACContext, low)),
"i"(offsetof(CABACContext, bytestream)),
"i"(offsetof(CABACContext, bytestream_end)),
"i"(offsetof(CABACContext, range))
: "%eax", "%edx", "memory"
);
return val;
}
#define get_cabac_bypass get_cabac_bypass_x86
static av_always_inline int get_cabac_bypass_x86(CABACContext *c)
{
x86_reg tmp;
int res;
__asm__ volatile(
"movl %c6(%2), %k1 \n\t"
"movl %c3(%2), %%eax \n\t"
"shl $17, %k1 \n\t"
"add %%eax, %%eax \n\t"
"sub %k1, %%eax \n\t"
"cdq \n\t"
"and %%edx, %k1 \n\t"
"add %k1, %%eax \n\t"
"inc %%edx \n\t"
"test %%ax, %%ax \n\t"
"jnz 1f \n\t"
"mov %c4(%2), %1 \n\t"
"subl $0xFFFF, %%eax \n\t"
"movzwl (%1), %%ecx \n\t"
"bswap %%ecx \n\t"
"shrl $15, %%ecx \n\t"
"addl %%ecx, %%eax \n\t"
"cmp %c5(%2), %1 \n\t"
"jge 1f \n\t"
"add"FF_OPSIZE" $2, %c4(%2) \n\t"
"1: \n\t"
"movl %%eax, %c3(%2) \n\t"
: "=&d"(res), "=&r"(tmp)
: "r"(c),
"i"(offsetof(CABACContext, low)),
"i"(offsetof(CABACContext, bytestream)),
"i"(offsetof(CABACContext, bytestream_end)),
"i"(offsetof(CABACContext, range))
: "%eax", "%ecx", "memory"
);
return res;
}
#endif /* !BROKEN_COMPILER */
#endif /* HAVE_INLINE_ASM */
#endif /* AVCODEC_X86_CABAC_H */

493
media/ffvpx/libavcodec/x86/dct32.asm Normal file
View file

@ -0,0 +1,493 @@
;******************************************************************************
;* 32 point SSE-optimized DCT transform
;* Copyright (c) 2010 Vitor Sessak
;*
;* This file is part of FFmpeg.
;*
;* FFmpeg is free software; you can redistribute it and/or
;* modify it under the terms of the GNU Lesser General Public
;* License as published by the Free Software Foundation; either
;* version 2.1 of the License, or (at your option) any later version.
;*
;* FFmpeg is distributed in the hope that it will be useful,
;* but WITHOUT ANY WARRANTY; without even the implied warranty of
;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
;* Lesser General Public License for more details.
;*
;* You should have received a copy of the GNU Lesser General Public
;* License along with FFmpeg; if not, write to the Free Software
;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
;******************************************************************************
%include "libavutil/x86/x86util.asm"
SECTION_RODATA 32
align 32
ps_cos_vec: dd 0.500603, 0.505471, 0.515447, 0.531043
dd 0.553104, 0.582935, 0.622504, 0.674808
dd -10.190008, -3.407609, -2.057781, -1.484165
dd -1.169440, -0.972568, -0.839350, -0.744536
dd 0.502419, 0.522499, 0.566944, 0.646822
dd 0.788155, 1.060678, 1.722447, 5.101149
dd 0.509796, 0.601345, 0.899976, 2.562916
dd 0.509796, 0.601345, 0.899976, 2.562916
dd 1.000000, 1.000000, 1.306563, 0.541196
dd 1.000000, 1.000000, 1.306563, 0.541196
dd 1.000000, 0.707107, 1.000000, -0.707107
dd 1.000000, 0.707107, 1.000000, -0.707107
dd 0.707107, 0.707107, 0.707107, 0.707107
align 32
ps_p1p1m1m1: dd 0, 0, 0x80000000, 0x80000000, 0, 0, 0x80000000, 0x80000000
%macro BUTTERFLY 4
subps %4, %1, %2
addps %2, %2, %1
mulps %1, %4, %3
%endmacro
%macro BUTTERFLY0 5
%if cpuflag(sse2) && notcpuflag(avx)
pshufd %4, %1, %5
xorps %1, %2
addps %1, %4
mulps %1, %3
%else
shufps %4, %1, %1, %5
xorps %1, %1, %2
addps %4, %4, %1
mulps %1, %4, %3
%endif
%endmacro
%macro BUTTERFLY2 4
BUTTERFLY0 %1, %2, %3, %4, 0x1b
%endmacro
%macro BUTTERFLY3 4
BUTTERFLY0 %1, %2, %3, %4, 0xb1
%endmacro
%macro BUTTERFLY3V 5
movaps m%5, m%1
addps m%1, m%2
subps m%5, m%2
SWAP %2, %5
mulps m%2, [ps_cos_vec+192]
movaps m%5, m%3
addps m%3, m%4
subps m%4, m%5
mulps m%4, [ps_cos_vec+192]
%endmacro
%macro PASS6_AND_PERMUTE 0
mov tmpd, [outq+4]
movss m7, [outq+72]
addss m7, [outq+76]
movss m3, [outq+56]
addss m3, [outq+60]
addss m4, m3
movss m2, [outq+52]
addss m2, m3
movss m3, [outq+104]
addss m3, [outq+108]
addss m1, m3
addss m5, m4
movss [outq+ 16], m1
movss m1, [outq+100]
addss m1, m3
movss m3, [outq+40]
movss [outq+ 48], m1
addss m3, [outq+44]
movss m1, [outq+100]
addss m4, m3
addss m3, m2
addss m1, [outq+108]
movss [outq+ 40], m3
addss m2, [outq+36]
movss m3, [outq+8]
movss [outq+ 56], m2
addss m3, [outq+12]
movss [outq+ 32], m3
movss m3, [outq+80]
movss [outq+ 8], m5
movss [outq+ 80], m1
movss m2, [outq+52]
movss m5, [outq+120]
addss m5, [outq+124]
movss m1, [outq+64]
addss m2, [outq+60]
addss m0, m5
addss m5, [outq+116]
mov [outq+64], tmpd
addss m6, m0
addss m1, m6
mov tmpd, [outq+12]
mov [outq+ 96], tmpd
movss [outq+ 4], m1
movss m1, [outq+24]
movss [outq+ 24], m4
movss m4, [outq+88]
addss m4, [outq+92]
addss m3, m4
addss m4, [outq+84]
mov tmpd, [outq+108]
addss m1, [outq+28]
addss m0, m1
addss m1, m5
addss m6, m3
addss m3, m0
addss m0, m7
addss m5, [outq+20]
addss m7, m1
movss [outq+ 12], m6
mov [outq+112], tmpd
movss m6, [outq+28]
movss [outq+ 28], m0
movss m0, [outq+36]
movss [outq+ 36], m7
addss m1, m4
movss m7, [outq+116]
addss m0, m2
addss m7, [outq+124]
movss [outq+ 72], m0
movss m0, [outq+44]
addss m2, m0
movss [outq+ 44], m1
movss [outq+ 88], m2
addss m0, [outq+60]
mov tmpd, [outq+60]
mov [outq+120], tmpd
movss [outq+104], m0
addss m4, m5
addss m5, [outq+68]
movss [outq+52], m4
movss [outq+60], m5
movss m4, [outq+68]
movss m5, [outq+20]
movss [outq+ 20], m3
addss m5, m7
addss m7, m6
addss m4, m5
movss m2, [outq+84]
addss m2, [outq+92]
addss m5, m2
movss [outq+ 68], m4
addss m2, m7
movss m4, [outq+76]
movss [outq+ 84], m2
movss [outq+ 76], m5
addss m7, m4
addss m6, [outq+124]
addss m4, m6
addss m6, [outq+92]
movss [outq+100], m4
movss [outq+108], m6
movss m6, [outq+92]
movss [outq+92], m7
addss m6, [outq+124]
movss [outq+116], m6
%endmacro
INIT_YMM avx
SECTION .text
%if HAVE_AVX_EXTERNAL
; void ff_dct32_float_avx(FFTSample *out, const FFTSample *in)
cglobal dct32_float, 2,3,8, out, in, tmp
; pass 1
vmovaps m4, [inq+0]
vinsertf128 m5, m5, [inq+96], 1
vinsertf128 m5, m5, [inq+112], 0
vshufps m5, m5, m5, 0x1b
BUTTERFLY m4, m5, [ps_cos_vec], m6
vmovaps m2, [inq+64]
vinsertf128 m6, m6, [inq+32], 1
vinsertf128 m6, m6, [inq+48], 0
vshufps m6, m6, m6, 0x1b
BUTTERFLY m2, m6, [ps_cos_vec+32], m0
; pass 2
BUTTERFLY m5, m6, [ps_cos_vec+64], m0
BUTTERFLY m4, m2, [ps_cos_vec+64], m7
; pass 3
vperm2f128 m3, m6, m4, 0x31
vperm2f128 m1, m6, m4, 0x20
vshufps m3, m3, m3, 0x1b
BUTTERFLY m1, m3, [ps_cos_vec+96], m6
vperm2f128 m4, m5, m2, 0x20
vperm2f128 m5, m5, m2, 0x31
vshufps m5, m5, m5, 0x1b
BUTTERFLY m4, m5, [ps_cos_vec+96], m6
; pass 4
vmovaps m6, [ps_p1p1m1m1+0]
vmovaps m2, [ps_cos_vec+128]
BUTTERFLY2 m5, m6, m2, m7
BUTTERFLY2 m4, m6, m2, m7
BUTTERFLY2 m1, m6, m2, m7
BUTTERFLY2 m3, m6, m2, m7
; pass 5
vshufps m6, m6, m6, 0xcc
vmovaps m2, [ps_cos_vec+160]
BUTTERFLY3 m5, m6, m2, m7
BUTTERFLY3 m4, m6, m2, m7
BUTTERFLY3 m1, m6, m2, m7
BUTTERFLY3 m3, m6, m2, m7
vperm2f128 m6, m3, m3, 0x31
vmovaps [outq], m3
vextractf128 [outq+64], m5, 1
vextractf128 [outq+32], m5, 0
vextractf128 [outq+80], m4, 1
vextractf128 [outq+48], m4, 0
vperm2f128 m0, m1, m1, 0x31
vmovaps [outq+96], m1
vzeroupper
; pass 6, no SIMD...
INIT_XMM
PASS6_AND_PERMUTE
RET
%endif
%if ARCH_X86_64
%define SPILL SWAP
%define UNSPILL SWAP
%macro PASS5 0
nop ; FIXME code alignment
SWAP 5, 8
SWAP 4, 12
SWAP 6, 14
SWAP 7, 13
SWAP 0, 15
PERMUTE 9,10, 10,12, 11,14, 12,9, 13,11, 14,13
TRANSPOSE4x4PS 8, 9, 10, 11, 0
BUTTERFLY3V 8, 9, 10, 11, 0
addps m10, m11
TRANSPOSE4x4PS 12, 13, 14, 15, 0
BUTTERFLY3V 12, 13, 14, 15, 0
addps m14, m15
addps m12, m14
addps m14, m13
addps m13, m15
%endmacro
%macro PASS6 0
SWAP 9, 12
SWAP 11, 14
movss [outq+0x00], m8
pshuflw m0, m8, 0xe
movss [outq+0x10], m9
pshuflw m1, m9, 0xe
movss [outq+0x20], m10
pshuflw m2, m10, 0xe
movss [outq+0x30], m11
pshuflw m3, m11, 0xe
movss [outq+0x40], m12
pshuflw m4, m12, 0xe
movss [outq+0x50], m13
pshuflw m5, m13, 0xe
movss [outq+0x60], m14
pshuflw m6, m14, 0xe
movaps [outq+0x70], m15
pshuflw m7, m15, 0xe
addss m0, m1
addss m1, m2
movss [outq+0x08], m0
addss m2, m3
movss [outq+0x18], m1
addss m3, m4
movss [outq+0x28], m2
addss m4, m5
movss [outq+0x38], m3
addss m5, m6
movss [outq+0x48], m4
addss m6, m7
movss [outq+0x58], m5
movss [outq+0x68], m6
movss [outq+0x78], m7
PERMUTE 1,8, 3,9, 5,10, 7,11, 9,12, 11,13, 13,14, 8,1, 10,3, 12,5, 14,7
movhlps m0, m1
pshufd m1, m1, 3
SWAP 0, 2, 4, 6, 8, 10, 12, 14
SWAP 1, 3, 5, 7, 9, 11, 13, 15
%rep 7
movhlps m0, m1
pshufd m1, m1, 3
addss m15, m1
SWAP 0, 2, 4, 6, 8, 10, 12, 14
SWAP 1, 3, 5, 7, 9, 11, 13, 15
%endrep
%assign i 4
%rep 15
addss m0, m1
movss [outq+i], m0
SWAP 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
%assign i i+8
%endrep
%endmacro
%else ; ARCH_X86_32
%macro SPILL 2 ; xmm#, mempos
movaps [outq+(%2-8)*16], m%1
%endmacro
%macro UNSPILL 2
movaps m%1, [outq+(%2-8)*16]
%endmacro
%define PASS6 PASS6_AND_PERMUTE
%macro PASS5 0
movaps m2, [ps_cos_vec+160]
shufps m3, m3, 0xcc
BUTTERFLY3 m5, m3, m2, m1
SPILL 5, 8
UNSPILL 1, 9
BUTTERFLY3 m1, m3, m2, m5
SPILL 1, 14
BUTTERFLY3 m4, m3, m2, m5
SPILL 4, 12
BUTTERFLY3 m7, m3, m2, m5
SPILL 7, 13
UNSPILL 5, 10
BUTTERFLY3 m5, m3, m2, m7
SPILL 5, 10
UNSPILL 4, 11
BUTTERFLY3 m4, m3, m2, m7
SPILL 4, 11
BUTTERFLY3 m6, m3, m2, m7
SPILL 6, 9
BUTTERFLY3 m0, m3, m2, m7
SPILL 0, 15
%endmacro
%endif
; void ff_dct32_float_sse(FFTSample *out, const FFTSample *in)
%macro DCT32_FUNC 0
cglobal dct32_float, 2, 3, 16, out, in, tmp
; pass 1
movaps m0, [inq+0]
LOAD_INV m1, [inq+112]
BUTTERFLY m0, m1, [ps_cos_vec], m3
movaps m7, [inq+64]
LOAD_INV m4, [inq+48]
BUTTERFLY m7, m4, [ps_cos_vec+32], m3
; pass 2
movaps m2, [ps_cos_vec+64]
BUTTERFLY m1, m4, m2, m3
SPILL 1, 11
SPILL 4, 8
; pass 1
movaps m1, [inq+16]
LOAD_INV m6, [inq+96]
BUTTERFLY m1, m6, [ps_cos_vec+16], m3
movaps m4, [inq+80]
LOAD_INV m5, [inq+32]
BUTTERFLY m4, m5, [ps_cos_vec+48], m3
; pass 2
BUTTERFLY m0, m7, m2, m3
movaps m2, [ps_cos_vec+80]
BUTTERFLY m6, m5, m2, m3
BUTTERFLY m1, m4, m2, m3
; pass 3
movaps m2, [ps_cos_vec+96]
shufps m1, m1, 0x1b
BUTTERFLY m0, m1, m2, m3
SPILL 0, 15
SPILL 1, 14
UNSPILL 0, 8
shufps m5, m5, 0x1b
BUTTERFLY m0, m5, m2, m3
UNSPILL 1, 11
shufps m6, m6, 0x1b
BUTTERFLY m1, m6, m2, m3
SPILL 1, 11
shufps m4, m4, 0x1b
BUTTERFLY m7, m4, m2, m3
; pass 4
movaps m3, [ps_p1p1m1m1+0]
movaps m2, [ps_cos_vec+128]
BUTTERFLY2 m5, m3, m2, m1
BUTTERFLY2 m0, m3, m2, m1
SPILL 0, 9
BUTTERFLY2 m6, m3, m2, m1
SPILL 6, 10
UNSPILL 0, 11
BUTTERFLY2 m0, m3, m2, m1
SPILL 0, 11
BUTTERFLY2 m4, m3, m2, m1
BUTTERFLY2 m7, m3, m2, m1
UNSPILL 6, 14
BUTTERFLY2 m6, m3, m2, m1
UNSPILL 0, 15
BUTTERFLY2 m0, m3, m2, m1
PASS5
PASS6
RET
%endmacro
%macro LOAD_INV 2
%if cpuflag(sse2)
pshufd %1, %2, 0x1b
%elif cpuflag(sse)
movaps %1, %2
shufps %1, %1, 0x1b
%endif
%endmacro
%if ARCH_X86_32
INIT_XMM sse
DCT32_FUNC
%endif
INIT_XMM sse2
DCT32_FUNC

41
media/ffvpx/libavcodec/x86/dct_init.c Normal file
View file

@ -0,0 +1,41 @@
/*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "config.h"
#include "libavutil/attributes.h"
#include "libavutil/cpu.h"
#include "libavutil/x86/cpu.h"
#include "libavcodec/dct.h"
void ff_dct32_float_sse(FFTSample *out, const FFTSample *in);
void ff_dct32_float_sse2(FFTSample *out, const FFTSample *in);
void ff_dct32_float_avx(FFTSample *out, const FFTSample *in);
av_cold void ff_dct_init_x86(DCTContext *s)
{
int cpu_flags = av_get_cpu_flags();
#if ARCH_X86_32
if (EXTERNAL_SSE(cpu_flags))
s->dct32 = ff_dct32_float_sse;
#endif
if (EXTERNAL_SSE2(cpu_flags))
s->dct32 = ff_dct32_float_sse2;
if (EXTERNAL_AVX_FAST(cpu_flags))
s->dct32 = ff_dct32_float_avx;
}

594
media/ffvpx/libavcodec/x86/fdct.c Normal file
View file

@ -0,0 +1,594 @@
/*
* SIMD-optimized forward DCT
* The gcc porting is Copyright (c) 2001 Fabrice Bellard.
* cleanup/optimizations are Copyright (c) 2002-2004 Michael Niedermayer <michaelni@gmx.at>
* SSE2 optimization is Copyright (c) 2004 Denes Balatoni.
*
* from fdctam32.c - AP922 MMX(3D-Now) forward-DCT
*
* Intel Application Note AP-922 - fast, precise implementation of DCT
* http://developer.intel.com/vtune/cbts/appnotes.htm
*
* Also of inspiration:
* a page about fdct at http://www.geocities.com/ssavekar/dct.htm
* Skal's fdct at http://skal.planet-d.net/coding/dct.html
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "libavutil/common.h"
#include "libavutil/x86/asm.h"
#include "fdct.h"
#if HAVE_MMX_INLINE
//////////////////////////////////////////////////////////////////////
//
// constants for the forward DCT
// -----------------------------
//
// Be sure to check that your compiler is aligning all constants to QWORD
// (8-byte) memory boundaries! Otherwise the unaligned memory access will
// severely stall MMX execution.
//
//////////////////////////////////////////////////////////////////////
#define BITS_FRW_ACC 3 //; 2 or 3 for accuracy
#define SHIFT_FRW_COL BITS_FRW_ACC
#define SHIFT_FRW_ROW (BITS_FRW_ACC + 17 - 3)
#define RND_FRW_ROW (1 << (SHIFT_FRW_ROW-1))
//#define RND_FRW_COL (1 << (SHIFT_FRW_COL-1))
#define X8(x) x,x,x,x,x,x,x,x
//concatenated table, for forward DCT transformation
DECLARE_ALIGNED(16, static const int16_t, fdct_tg_all_16)[24] = {
X8(13036), // tg * (2<<16) + 0.5
X8(27146), // tg * (2<<16) + 0.5
X8(-21746) // tg * (2<<16) + 0.5
};
DECLARE_ALIGNED(16, static const int16_t, ocos_4_16)[8] = {
X8(23170) //cos * (2<<15) + 0.5
};
DECLARE_ALIGNED(16, static const int16_t, fdct_one_corr)[8] = { X8(1) };
DECLARE_ALIGNED(8, static const int32_t, fdct_r_row)[2] = {RND_FRW_ROW, RND_FRW_ROW };
static const struct
{
DECLARE_ALIGNED(16, const int32_t, fdct_r_row_sse2)[4];
} fdct_r_row_sse2 =
{{
RND_FRW_ROW, RND_FRW_ROW, RND_FRW_ROW, RND_FRW_ROW
}};
//DECLARE_ALIGNED(16, static const long, fdct_r_row_sse2)[4] = {RND_FRW_ROW, RND_FRW_ROW, RND_FRW_ROW, RND_FRW_ROW};
DECLARE_ALIGNED(8, static const int16_t, tab_frw_01234567)[] = { // forward_dct coeff table
16384, 16384, 22725, 19266,
16384, 16384, 12873, 4520,
21407, 8867, 19266, -4520,
-8867, -21407, -22725, -12873,
16384, -16384, 12873, -22725,
-16384, 16384, 4520, 19266,
8867, -21407, 4520, -12873,
21407, -8867, 19266, -22725,
22725, 22725, 31521, 26722,
22725, 22725, 17855, 6270,
29692, 12299, 26722, -6270,
-12299, -29692, -31521, -17855,
22725, -22725, 17855, -31521,
-22725, 22725, 6270, 26722,
12299, -29692, 6270, -17855,
29692, -12299, 26722, -31521,
21407, 21407, 29692, 25172,
21407, 21407, 16819, 5906,
27969, 11585, 25172, -5906,
-11585, -27969, -29692, -16819,
21407, -21407, 16819, -29692,
-21407, 21407, 5906, 25172,
11585, -27969, 5906, -16819,
27969, -11585, 25172, -29692,
19266, 19266, 26722, 22654,
19266, 19266, 15137, 5315,
25172, 10426, 22654, -5315,
-10426, -25172, -26722, -15137,
19266, -19266, 15137, -26722,
-19266, 19266, 5315, 22654,
10426, -25172, 5315, -15137,
25172, -10426, 22654, -26722,
16384, 16384, 22725, 19266,
16384, 16384, 12873, 4520,
21407, 8867, 19266, -4520,
-8867, -21407, -22725, -12873,
16384, -16384, 12873, -22725,
-16384, 16384, 4520, 19266,
8867, -21407, 4520, -12873,
21407, -8867, 19266, -22725,
19266, 19266, 26722, 22654,
19266, 19266, 15137, 5315,
25172, 10426, 22654, -5315,
-10426, -25172, -26722, -15137,
19266, -19266, 15137, -26722,
-19266, 19266, 5315, 22654,
10426, -25172, 5315, -15137,
25172, -10426, 22654, -26722,
21407, 21407, 29692, 25172,
21407, 21407, 16819, 5906,
27969, 11585, 25172, -5906,
-11585, -27969, -29692, -16819,
21407, -21407, 16819, -29692,
-21407, 21407, 5906, 25172,
11585, -27969, 5906, -16819,
27969, -11585, 25172, -29692,
22725, 22725, 31521, 26722,
22725, 22725, 17855, 6270,
29692, 12299, 26722, -6270,
-12299, -29692, -31521, -17855,
22725, -22725, 17855, -31521,
-22725, 22725, 6270, 26722,
12299, -29692, 6270, -17855,
29692, -12299, 26722, -31521,
};
static const struct
{
DECLARE_ALIGNED(16, const int16_t, tab_frw_01234567_sse2)[256];
} tab_frw_01234567_sse2 =
{{
//DECLARE_ALIGNED(16, static const int16_t, tab_frw_01234567_sse2)[] = { // forward_dct coeff table
#define TABLE_SSE2 C4, C4, C1, C3, -C6, -C2, -C1, -C5, \
C4, C4, C5, C7, C2, C6, C3, -C7, \
-C4, C4, C7, C3, C6, -C2, C7, -C5, \
C4, -C4, C5, -C1, C2, -C6, C3, -C1,
// c1..c7 * cos(pi/4) * 2^15
#define C1 22725
#define C2 21407
#define C3 19266
#define C4 16384
#define C5 12873
#define C6 8867
#define C7 4520
TABLE_SSE2
#undef C1
#undef C2
#undef C3
#undef C4
#undef C5
#undef C6
#undef C7
#define C1 31521
#define C2 29692
#define C3 26722
#define C4 22725
#define C5 17855
#define C6 12299
#define C7 6270
TABLE_SSE2
#undef C1
#undef C2
#undef C3
#undef C4
#undef C5
#undef C6
#undef C7
#define C1 29692
#define C2 27969
#define C3 25172
#define C4 21407
#define C5 16819
#define C6 11585
#define C7 5906
TABLE_SSE2
#undef C1
#undef C2
#undef C3
#undef C4
#undef C5
#undef C6
#undef C7
#define C1 26722
#define C2 25172
#define C3 22654
#define C4 19266
#define C5 15137
#define C6 10426
#define C7 5315
TABLE_SSE2
#undef C1
#undef C2
#undef C3
#undef C4
#undef C5
#undef C6
#undef C7
#define C1 22725
#define C2 21407
#define C3 19266
#define C4 16384
#define C5 12873
#define C6 8867
#define C7 4520
TABLE_SSE2
#undef C1
#undef C2
#undef C3
#undef C4
#undef C5
#undef C6
#undef C7
#define C1 26722
#define C2 25172
#define C3 22654
#define C4 19266
#define C5 15137
#define C6 10426
#define C7 5315
TABLE_SSE2
#undef C1
#undef C2
#undef C3
#undef C4
#undef C5
#undef C6
#undef C7
#define C1 29692
#define C2 27969
#define C3 25172
#define C4 21407
#define C5 16819
#define C6 11585
#define C7 5906
TABLE_SSE2
#undef C1
#undef C2
#undef C3
#undef C4
#undef C5
#undef C6
#undef C7
#define C1 31521
#define C2 29692
#define C3 26722
#define C4 22725
#define C5 17855
#define C6 12299
#define C7 6270
TABLE_SSE2
}};
#define S(s) AV_TOSTRING(s) //AV_STRINGIFY is too long
#define FDCT_COL(cpu, mm, mov)\
static av_always_inline void fdct_col_##cpu(const int16_t *in, int16_t *out, int offset)\
{\
__asm__ volatile (\
#mov" 16(%0), %%"#mm"0 \n\t" \
#mov" 96(%0), %%"#mm"1 \n\t" \
#mov" %%"#mm"0, %%"#mm"2 \n\t" \
#mov" 32(%0), %%"#mm"3 \n\t" \
"paddsw %%"#mm"1, %%"#mm"0 \n\t" \
#mov" 80(%0), %%"#mm"4 \n\t" \
"psllw $"S(SHIFT_FRW_COL)", %%"#mm"0 \n\t" \
#mov" (%0), %%"#mm"5 \n\t" \
"paddsw %%"#mm"3, %%"#mm"4 \n\t" \
"paddsw 112(%0), %%"#mm"5 \n\t" \
"psllw $"S(SHIFT_FRW_COL)", %%"#mm"4 \n\t" \
#mov" %%"#mm"0, %%"#mm"6 \n\t" \
"psubsw %%"#mm"1, %%"#mm"2 \n\t" \
#mov" 16(%1), %%"#mm"1 \n\t" \
"psubsw %%"#mm"4, %%"#mm"0 \n\t" \
#mov" 48(%0), %%"#mm"7 \n\t" \
"pmulhw %%"#mm"0, %%"#mm"1 \n\t" \
"paddsw 64(%0), %%"#mm"7 \n\t" \
"psllw $"S(SHIFT_FRW_COL)", %%"#mm"5 \n\t" \
"paddsw %%"#mm"4, %%"#mm"6 \n\t" \
"psllw $"S(SHIFT_FRW_COL)", %%"#mm"7 \n\t" \
#mov" %%"#mm"5, %%"#mm"4 \n\t" \
"psubsw %%"#mm"7, %%"#mm"5 \n\t" \
"paddsw %%"#mm"5, %%"#mm"1 \n\t" \
"paddsw %%"#mm"7, %%"#mm"4 \n\t" \
"por (%2), %%"#mm"1 \n\t" \
"psllw $"S(SHIFT_FRW_COL)"+1, %%"#mm"2 \n\t" \
"pmulhw 16(%1), %%"#mm"5 \n\t" \
#mov" %%"#mm"4, %%"#mm"7 \n\t" \
"psubsw 80(%0), %%"#mm"3 \n\t" \
"psubsw %%"#mm"6, %%"#mm"4 \n\t" \
#mov" %%"#mm"1, 32(%3) \n\t" \
"paddsw %%"#mm"6, %%"#mm"7 \n\t" \
#mov" 48(%0), %%"#mm"1 \n\t" \
"psllw $"S(SHIFT_FRW_COL)"+1, %%"#mm"3 \n\t" \
"psubsw 64(%0), %%"#mm"1 \n\t" \
#mov" %%"#mm"2, %%"#mm"6 \n\t" \
#mov" %%"#mm"4, 64(%3) \n\t" \
"paddsw %%"#mm"3, %%"#mm"2 \n\t" \
"pmulhw (%4), %%"#mm"2 \n\t" \
"psubsw %%"#mm"3, %%"#mm"6 \n\t" \
"pmulhw (%4), %%"#mm"6 \n\t" \
"psubsw %%"#mm"0, %%"#mm"5 \n\t" \
"por (%2), %%"#mm"5 \n\t" \
"psllw $"S(SHIFT_FRW_COL)", %%"#mm"1 \n\t" \
"por (%2), %%"#mm"2 \n\t" \
#mov" %%"#mm"1, %%"#mm"4 \n\t" \
#mov" (%0), %%"#mm"3 \n\t" \
"paddsw %%"#mm"6, %%"#mm"1 \n\t" \
"psubsw 112(%0), %%"#mm"3 \n\t" \
"psubsw %%"#mm"6, %%"#mm"4 \n\t" \
#mov" (%1), %%"#mm"0 \n\t" \
"psllw $"S(SHIFT_FRW_COL)", %%"#mm"3 \n\t" \
#mov" 32(%1), %%"#mm"6 \n\t" \
"pmulhw %%"#mm"1, %%"#mm"0 \n\t" \
#mov" %%"#mm"7, (%3) \n\t" \
"pmulhw %%"#mm"4, %%"#mm"6 \n\t" \
#mov" %%"#mm"5, 96(%3) \n\t" \
#mov" %%"#mm"3, %%"#mm"7 \n\t" \
#mov" 32(%1), %%"#mm"5 \n\t" \
"psubsw %%"#mm"2, %%"#mm"7 \n\t" \
"paddsw %%"#mm"2, %%"#mm"3 \n\t" \
"pmulhw %%"#mm"7, %%"#mm"5 \n\t" \
"paddsw %%"#mm"3, %%"#mm"0 \n\t" \
"paddsw %%"#mm"4, %%"#mm"6 \n\t" \
"pmulhw (%1), %%"#mm"3 \n\t" \
"por (%2), %%"#mm"0 \n\t" \
"paddsw %%"#mm"7, %%"#mm"5 \n\t" \
"psubsw %%"#mm"6, %%"#mm"7 \n\t" \
#mov" %%"#mm"0, 16(%3) \n\t" \
"paddsw %%"#mm"4, %%"#mm"5 \n\t" \
#mov" %%"#mm"7, 48(%3) \n\t" \
"psubsw %%"#mm"1, %%"#mm"3 \n\t" \
#mov" %%"#mm"5, 80(%3) \n\t" \
#mov" %%"#mm"3, 112(%3) \n\t" \
: \
: "r" (in + offset), "r" (fdct_tg_all_16), "r" (fdct_one_corr), \
"r" (out + offset), "r" (ocos_4_16)); \
}
FDCT_COL(mmx, mm, movq)
FDCT_COL(sse2, xmm, movdqa)
static av_always_inline void fdct_row_sse2(const int16_t *in, int16_t *out)
{
__asm__ volatile(
#define FDCT_ROW_SSE2_H1(i,t) \
"movq " #i "(%0), %%xmm2 \n\t" \
"movq " #i "+8(%0), %%xmm0 \n\t" \
"movdqa " #t "+32(%1), %%xmm3 \n\t" \
"movdqa " #t "+48(%1), %%xmm7 \n\t" \
"movdqa " #t "(%1), %%xmm4 \n\t" \
"movdqa " #t "+16(%1), %%xmm5 \n\t"
#define FDCT_ROW_SSE2_H2(i,t) \
"movq " #i "(%0), %%xmm2 \n\t" \
"movq " #i "+8(%0), %%xmm0 \n\t" \
"movdqa " #t "+32(%1), %%xmm3 \n\t" \
"movdqa " #t "+48(%1), %%xmm7 \n\t"
#define FDCT_ROW_SSE2(i) \
"movq %%xmm2, %%xmm1 \n\t" \
"pshuflw $27, %%xmm0, %%xmm0 \n\t" \
"paddsw %%xmm0, %%xmm1 \n\t" \
"psubsw %%xmm0, %%xmm2 \n\t" \
"punpckldq %%xmm2, %%xmm1 \n\t" \
"pshufd $78, %%xmm1, %%xmm2 \n\t" \
"pmaddwd %%xmm2, %%xmm3 \n\t" \
"pmaddwd %%xmm1, %%xmm7 \n\t" \
"pmaddwd %%xmm5, %%xmm2 \n\t" \
"pmaddwd %%xmm4, %%xmm1 \n\t" \
"paddd %%xmm7, %%xmm3 \n\t" \
"paddd %%xmm2, %%xmm1 \n\t" \
"paddd %%xmm6, %%xmm3 \n\t" \
"paddd %%xmm6, %%xmm1 \n\t" \
"psrad %3, %%xmm3 \n\t" \
"psrad %3, %%xmm1 \n\t" \
"packssdw %%xmm3, %%xmm1 \n\t" \
"movdqa %%xmm1, " #i "(%4) \n\t"
"movdqa (%2), %%xmm6 \n\t"
FDCT_ROW_SSE2_H1(0,0)
FDCT_ROW_SSE2(0)
FDCT_ROW_SSE2_H2(64,0)
FDCT_ROW_SSE2(64)
FDCT_ROW_SSE2_H1(16,64)
FDCT_ROW_SSE2(16)
FDCT_ROW_SSE2_H2(112,64)
FDCT_ROW_SSE2(112)
FDCT_ROW_SSE2_H1(32,128)
FDCT_ROW_SSE2(32)
FDCT_ROW_SSE2_H2(96,128)
FDCT_ROW_SSE2(96)
FDCT_ROW_SSE2_H1(48,192)
FDCT_ROW_SSE2(48)
FDCT_ROW_SSE2_H2(80,192)
FDCT_ROW_SSE2(80)
:
: "r" (in), "r" (tab_frw_01234567_sse2.tab_frw_01234567_sse2),
"r" (fdct_r_row_sse2.fdct_r_row_sse2), "i" (SHIFT_FRW_ROW), "r" (out)
XMM_CLOBBERS_ONLY("%xmm0", "%xmm1", "%xmm2", "%xmm3",
"%xmm4", "%xmm5", "%xmm6", "%xmm7")
);
}
static av_always_inline void fdct_row_mmxext(const int16_t *in, int16_t *out,
const int16_t *table)
{
__asm__ volatile (
"pshufw $0x1B, 8(%0), %%mm5 \n\t"
"movq (%0), %%mm0 \n\t"
"movq %%mm0, %%mm1 \n\t"
"paddsw %%mm5, %%mm0 \n\t"
"psubsw %%mm5, %%mm1 \n\t"
"movq %%mm0, %%mm2 \n\t"
"punpckldq %%mm1, %%mm0 \n\t"
"punpckhdq %%mm1, %%mm2 \n\t"
"movq (%1), %%mm1 \n\t"
"movq 8(%1), %%mm3 \n\t"
"movq 16(%1), %%mm4 \n\t"
"movq 24(%1), %%mm5 \n\t"
"movq 32(%1), %%mm6 \n\t"
"movq 40(%1), %%mm7 \n\t"
"pmaddwd %%mm0, %%mm1 \n\t"
"pmaddwd %%mm2, %%mm3 \n\t"
"pmaddwd %%mm0, %%mm4 \n\t"
"pmaddwd %%mm2, %%mm5 \n\t"
"pmaddwd %%mm0, %%mm6 \n\t"
"pmaddwd %%mm2, %%mm7 \n\t"
"pmaddwd 48(%1), %%mm0 \n\t"
"pmaddwd 56(%1), %%mm2 \n\t"
"paddd %%mm1, %%mm3 \n\t"
"paddd %%mm4, %%mm5 \n\t"
"paddd %%mm6, %%mm7 \n\t"
"paddd %%mm0, %%mm2 \n\t"
"movq (%2), %%mm0 \n\t"
"paddd %%mm0, %%mm3 \n\t"
"paddd %%mm0, %%mm5 \n\t"
"paddd %%mm0, %%mm7 \n\t"
"paddd %%mm0, %%mm2 \n\t"
"psrad $"S(SHIFT_FRW_ROW)", %%mm3 \n\t"
"psrad $"S(SHIFT_FRW_ROW)", %%mm5 \n\t"
"psrad $"S(SHIFT_FRW_ROW)", %%mm7 \n\t"
"psrad $"S(SHIFT_FRW_ROW)", %%mm2 \n\t"
"packssdw %%mm5, %%mm3 \n\t"
"packssdw %%mm2, %%mm7 \n\t"
"movq %%mm3, (%3) \n\t"
"movq %%mm7, 8(%3) \n\t"
:
: "r" (in), "r" (table), "r" (fdct_r_row), "r" (out));
}
static av_always_inline void fdct_row_mmx(const int16_t *in, int16_t *out, const int16_t *table)
{
//FIXME reorder (I do not have an old MMX-only CPU here to benchmark ...)
__asm__ volatile(
"movd 12(%0), %%mm1 \n\t"
"punpcklwd 8(%0), %%mm1 \n\t"
"movq %%mm1, %%mm2 \n\t"
"psrlq $0x20, %%mm1 \n\t"
"movq 0(%0), %%mm0 \n\t"
"punpcklwd %%mm2, %%mm1 \n\t"
"movq %%mm0, %%mm5 \n\t"
"paddsw %%mm1, %%mm0 \n\t"
"psubsw %%mm1, %%mm5 \n\t"
"movq %%mm0, %%mm2 \n\t"
"punpckldq %%mm5, %%mm0 \n\t"
"punpckhdq %%mm5, %%mm2 \n\t"
"movq 0(%1), %%mm1 \n\t"
"movq 8(%1), %%mm3 \n\t"
"movq 16(%1), %%mm4 \n\t"
"movq 24(%1), %%mm5 \n\t"
"movq 32(%1), %%mm6 \n\t"
"movq 40(%1), %%mm7 \n\t"
"pmaddwd %%mm0, %%mm1 \n\t"
"pmaddwd %%mm2, %%mm3 \n\t"
"pmaddwd %%mm0, %%mm4 \n\t"
"pmaddwd %%mm2, %%mm5 \n\t"
"pmaddwd %%mm0, %%mm6 \n\t"
"pmaddwd %%mm2, %%mm7 \n\t"
"pmaddwd 48(%1), %%mm0 \n\t"
"pmaddwd 56(%1), %%mm2 \n\t"
"paddd %%mm1, %%mm3 \n\t"
"paddd %%mm4, %%mm5 \n\t"
"paddd %%mm6, %%mm7 \n\t"
"paddd %%mm0, %%mm2 \n\t"
"movq (%2), %%mm0 \n\t"
"paddd %%mm0, %%mm3 \n\t"
"paddd %%mm0, %%mm5 \n\t"
"paddd %%mm0, %%mm7 \n\t"
"paddd %%mm0, %%mm2 \n\t"
"psrad $"S(SHIFT_FRW_ROW)", %%mm3 \n\t"
"psrad $"S(SHIFT_FRW_ROW)", %%mm5 \n\t"
"psrad $"S(SHIFT_FRW_ROW)", %%mm7 \n\t"
"psrad $"S(SHIFT_FRW_ROW)", %%mm2 \n\t"
"packssdw %%mm5, %%mm3 \n\t"
"packssdw %%mm2, %%mm7 \n\t"
"movq %%mm3, 0(%3) \n\t"
"movq %%mm7, 8(%3) \n\t"
:
: "r" (in), "r" (table), "r" (fdct_r_row), "r" (out));
}
void ff_fdct_mmx(int16_t *block)
{
DECLARE_ALIGNED(8, int64_t, align_tmp)[16];
int16_t * block1= (int16_t*)align_tmp;
const int16_t *table= tab_frw_01234567;
int i;
fdct_col_mmx(block, block1, 0);
fdct_col_mmx(block, block1, 4);
for(i=8;i>0;i--) {
fdct_row_mmx(block1, block, table);
block1 += 8;
table += 32;
block += 8;
}
}
#endif /* HAVE_MMX_INLINE */
#if HAVE_MMXEXT_INLINE
void ff_fdct_mmxext(int16_t *block)
{
DECLARE_ALIGNED(8, int64_t, align_tmp)[16];
int16_t *block1= (int16_t*)align_tmp;
const int16_t *table= tab_frw_01234567;
int i;
fdct_col_mmx(block, block1, 0);
fdct_col_mmx(block, block1, 4);
for(i=8;i>0;i--) {
fdct_row_mmxext(block1, block, table);
block1 += 8;
table += 32;
block += 8;
}
}
#endif /* HAVE_MMXEXT_INLINE */
#if HAVE_SSE2_INLINE
void ff_fdct_sse2(int16_t *block)
{
DECLARE_ALIGNED(16, int64_t, align_tmp)[16];
int16_t * const block1= (int16_t*)align_tmp;
fdct_col_sse2(block, block1, 0);
fdct_row_sse2(block1, block);
}
#endif /* HAVE_SSE2_INLINE */

28
media/ffvpx/libavcodec/x86/fdct.h Normal file
View file

@ -0,0 +1,28 @@
/*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#ifndef AVCODEC_X86_FDCT_H
#define AVCODEC_X86_FDCT_H
#include <stdint.h>
void ff_fdct_mmx(int16_t *block);
void ff_fdct_mmxext(int16_t *block);
void ff_fdct_sse2(int16_t *block);
#endif /* AVCODEC_X86_FDCT_H */

44
media/ffvpx/libavcodec/x86/fdctdsp_init.c Normal file
View file

@ -0,0 +1,44 @@
/*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "libavutil/attributes.h"
#include "libavutil/cpu.h"
#include "libavutil/x86/cpu.h"
#include "libavcodec/avcodec.h"
#include "libavcodec/fdctdsp.h"
#include "fdct.h"
av_cold void ff_fdctdsp_init_x86(FDCTDSPContext *c, AVCodecContext *avctx,
unsigned high_bit_depth)
{
int cpu_flags = av_get_cpu_flags();
const int dct_algo = avctx->dct_algo;
if (!high_bit_depth) {
if ((dct_algo == FF_DCT_AUTO || dct_algo == FF_DCT_MMX)) {
if (INLINE_MMX(cpu_flags))
c->fdct = ff_fdct_mmx;
if (INLINE_MMXEXT(cpu_flags))
c->fdct = ff_fdct_mmxext;
if (INLINE_SSE2(cpu_flags))
c->fdct = ff_fdct_sse2;
}
}
}

1093
media/ffvpx/libavcodec/x86/fft.asm Normal file
View file

File diff suppressed because it is too large Load diff

38
media/ffvpx/libavcodec/x86/fft.h Normal file
View file

@ -0,0 +1,38 @@
/*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#ifndef AVCODEC_X86_FFT_H
#define AVCODEC_X86_FFT_H
#include "libavcodec/fft.h"
void ff_fft_permute_sse(FFTContext *s, FFTComplex *z);
void ff_fft_calc_avx(FFTContext *s, FFTComplex *z);
void ff_fft_calc_sse(FFTContext *s, FFTComplex *z);
void ff_fft_calc_3dnow(FFTContext *s, FFTComplex *z);
void ff_fft_calc_3dnowext(FFTContext *s, FFTComplex *z);
void ff_imdct_calc_3dnow(FFTContext *s, FFTSample *output, const FFTSample *input);
void ff_imdct_half_3dnow(FFTContext *s, FFTSample *output, const FFTSample *input);
void ff_imdct_calc_3dnowext(FFTContext *s, FFTSample *output, const FFTSample *input);
void ff_imdct_half_3dnowext(FFTContext *s, FFTSample *output, const FFTSample *input);
void ff_imdct_calc_sse(FFTContext *s, FFTSample *output, const FFTSample *input);
void ff_imdct_half_sse(FFTContext *s, FFTSample *output, const FFTSample *input);
void ff_imdct_half_avx(FFTContext *s, FFTSample *output, const FFTSample *input);
#endif /* AVCODEC_X86_FFT_H */

61
media/ffvpx/libavcodec/x86/fft_init.c Normal file
View file

@ -0,0 +1,61 @@
/*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "config.h"
#include "libavutil/attributes.h"
#include "libavutil/cpu.h"
#include "libavutil/x86/cpu.h"
#include "fft.h"
av_cold void ff_fft_init_x86(FFTContext *s)
{
int cpu_flags = av_get_cpu_flags();
if (s->nbits > 16)
return;
#if ARCH_X86_32
if (EXTERNAL_AMD3DNOW(cpu_flags)) {
s->imdct_calc = ff_imdct_calc_3dnow;
s->imdct_half = ff_imdct_half_3dnow;
s->fft_calc = ff_fft_calc_3dnow;
}
if (EXTERNAL_AMD3DNOWEXT(cpu_flags)) {
s->imdct_calc = ff_imdct_calc_3dnowext;
s->imdct_half = ff_imdct_half_3dnowext;
s->fft_calc = ff_fft_calc_3dnowext;
}
#endif /* ARCH_X86_32 */
if (EXTERNAL_SSE(cpu_flags)) {
s->imdct_calc = ff_imdct_calc_sse;
s->imdct_half = ff_imdct_half_sse;
s->fft_permute = ff_fft_permute_sse;
s->fft_calc = ff_fft_calc_sse;
s->fft_permutation = FF_FFT_PERM_SWAP_LSBS;
}
if (EXTERNAL_AVX_FAST(cpu_flags) && s->nbits >= 5) {
s->imdct_half = ff_imdct_half_avx;
s->fft_calc = ff_fft_calc_avx;
s->fft_permutation = FF_FFT_PERM_AVX;
}
}

106
media/ffvpx/libavcodec/x86/fpel.asm Normal file
View file

@ -0,0 +1,106 @@
;******************************************************************************
;* SIMD-optimized fullpel functions
;* Copyright (c) 2008 Loren Merritt
;* Copyright (c) 2003-2013 Michael Niedermayer
;* Copyright (c) 2013 Daniel Kang
;*
;* This file is part of FFmpeg.
;*
;* FFmpeg is free software; you can redistribute it and/or
;* modify it under the terms of the GNU Lesser General Public
;* License as published by the Free Software Foundation; either
;* version 2.1 of the License, or (at your option) any later version.
;*
;* FFmpeg is distributed in the hope that it will be useful,
;* but WITHOUT ANY WARRANTY; without even the implied warranty of
;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
;* Lesser General Public License for more details.
;*
;* You should have received a copy of the GNU Lesser General Public
;* License along with FFmpeg; if not, write to the Free Software
;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
;******************************************************************************
%include "libavutil/x86/x86util.asm"
SECTION .text
%macro PAVGB_MMX 4
LOAD %3, %1
por %3, %2
pxor %2, %1
pand %2, %4
psrlq %2, 1
psubb %3, %2
SWAP %2, %3
%endmacro
; void ff_put/avg_pixels(uint8_t *block, const uint8_t *pixels,
; ptrdiff_t line_size, int h)
%macro OP_PIXELS 2
%if %2 == mmsize/2
%define LOAD movh
%define SAVE movh
%define LEN mmsize
%else
%define LOAD movu
%define SAVE mova
%define LEN %2
%endif
cglobal %1_pixels%2, 4,5,4
lea r4, [r2*3]
%ifidn %1, avg
%if notcpuflag(mmxext)
pcmpeqd m6, m6
paddb m6, m6
%endif
%endif
.loop:
%assign %%i 0
%rep LEN/mmsize
LOAD m0, [r1 + %%i]
LOAD m1, [r1+r2 + %%i]
LOAD m2, [r1+r2*2 + %%i]
LOAD m3, [r1+r4 + %%i]
%ifidn %1, avg
%if notcpuflag(mmxext)
PAVGB_MMX [r0 + %%i], m0, m4, m6
PAVGB_MMX [r0+r2 + %%i], m1, m5, m6
PAVGB_MMX [r0+r2*2 + %%i], m2, m4, m6
PAVGB_MMX [r0+r4 + %%i], m3, m5, m6
%else
pavgb m0, [r0 + %%i]
pavgb m1, [r0+r2 + %%i]
pavgb m2, [r0+r2*2 + %%i]
pavgb m3, [r0+r4 + %%i]
%endif
%endif
SAVE [r0 + %%i], m0
SAVE [r0+r2 + %%i], m1
SAVE [r0+r2*2 + %%i], m2
SAVE [r0+r4 + %%i], m3
%assign %%i %%i+mmsize
%endrep
sub r3d, 4
lea r1, [r1+r2*4]
lea r0, [r0+r2*4]
jne .loop
RET
%endmacro
INIT_MMX mmx
OP_PIXELS put, 4
OP_PIXELS avg, 4
OP_PIXELS put, 8
OP_PIXELS avg, 8
OP_PIXELS put, 16
OP_PIXELS avg, 16
INIT_MMX mmxext
OP_PIXELS avg, 4
OP_PIXELS avg, 8
OP_PIXELS avg, 16
INIT_XMM sse2
OP_PIXELS put, 16
OP_PIXELS avg, 16

49
media/ffvpx/libavcodec/x86/fpel.h Normal file
View file

@ -0,0 +1,49 @@
/*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#ifndef AVCODEC_X86_FPEL_H
#define AVCODEC_X86_FPEL_H
#include <stddef.h>
#include <stdint.h>
void ff_avg_pixels4_mmx(uint8_t *block, const uint8_t *pixels,
ptrdiff_t line_size, int h);
void ff_avg_pixels4_mmxext(uint8_t *block, const uint8_t *pixels,
ptrdiff_t line_size, int h);
void ff_avg_pixels8_mmx(uint8_t *block, const uint8_t *pixels,
ptrdiff_t line_size, int h);
void ff_avg_pixels8_mmxext(uint8_t *block, const uint8_t *pixels,
ptrdiff_t line_size, int h);
void ff_avg_pixels16_mmx(uint8_t *block, const uint8_t *pixels,
ptrdiff_t line_size, int h);
void ff_avg_pixels16_mmxext(uint8_t *block, const uint8_t *pixels,
ptrdiff_t line_size, int h);
void ff_avg_pixels16_sse2(uint8_t *block, const uint8_t *pixels,
ptrdiff_t line_size, int h);
void ff_put_pixels4_mmx(uint8_t *block, const uint8_t *pixels,
ptrdiff_t line_size, int h);
void ff_put_pixels8_mmx(uint8_t *block, const uint8_t *pixels,
ptrdiff_t line_size, int h);
void ff_put_pixels16_mmx(uint8_t *block, const uint8_t *pixels,
ptrdiff_t line_size, int h);
void ff_put_pixels16_sse2(uint8_t *block, const uint8_t *pixels,
ptrdiff_t line_size, int h);
#endif /* AVCODEC_X86_FPEL_H */

208
media/ffvpx/libavcodec/x86/h264_cabac.c Normal file
View file

@ -0,0 +1,208 @@
/*
* H.26L/H.264/AVC/JVT/14496-10/... encoder/decoder
* Copyright (c) 2003 Michael Niedermayer <michaelni@gmx.at>
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
/**
* @file
* H.264 / AVC / MPEG-4 part10 codec.
* non-SIMD x86-specific optimizations for H.264
* @author Michael Niedermayer <michaelni@gmx.at>
*/
#include <stddef.h>
#include "libavcodec/cabac.h"
#include "cabac.h"
#if HAVE_INLINE_ASM
#if ARCH_X86_64
#define REG64 "r"
#else
#define REG64 "m"
#endif
//FIXME use some macros to avoid duplicating get_cabac (cannot be done yet
//as that would make optimization work hard)
#if HAVE_7REGS && !BROKEN_COMPILER
#define decode_significance decode_significance_x86
static int decode_significance_x86(CABACContext *c, int max_coeff,
uint8_t *significant_coeff_ctx_base,
int *index, x86_reg last_off){
void *end= significant_coeff_ctx_base + max_coeff - 1;
int minusstart= -(intptr_t)significant_coeff_ctx_base;
int minusindex= 4-(intptr_t)index;
int bit;
x86_reg coeff_count;
#ifdef BROKEN_RELOCATIONS
void *tables;
__asm__ volatile(
"lea "MANGLE(ff_h264_cabac_tables)", %0 \n\t"
: "=&r"(tables)
: NAMED_CONSTRAINTS_ARRAY(ff_h264_cabac_tables)
);
#endif
__asm__ volatile(
"3: \n\t"
BRANCHLESS_GET_CABAC("%4", "%q4", "(%1)", "%3", "%w3",
"%5", "%q5", "%k0", "%b0",
"%c11(%6)", "%c12(%6)",
AV_STRINGIFY(H264_NORM_SHIFT_OFFSET),
AV_STRINGIFY(H264_LPS_RANGE_OFFSET),
AV_STRINGIFY(H264_MLPS_STATE_OFFSET),
"%13")
"test $1, %4 \n\t"
" jz 4f \n\t"
"add %10, %1 \n\t"
BRANCHLESS_GET_CABAC("%4", "%q4", "(%1)", "%3", "%w3",
"%5", "%q5", "%k0", "%b0",
"%c11(%6)", "%c12(%6)",
AV_STRINGIFY(H264_NORM_SHIFT_OFFSET),
AV_STRINGIFY(H264_LPS_RANGE_OFFSET),
AV_STRINGIFY(H264_MLPS_STATE_OFFSET),
"%13")
"sub %10, %1 \n\t"
"mov %2, %0 \n\t"
"movl %7, %%ecx \n\t"
"add %1, %%"FF_REG_c" \n\t"
"movl %%ecx, (%0) \n\t"
"test $1, %4 \n\t"
" jnz 5f \n\t"
"add"FF_OPSIZE" $4, %2 \n\t"
"4: \n\t"
"add $1, %1 \n\t"
"cmp %8, %1 \n\t"
" jb 3b \n\t"
"mov %2, %0 \n\t"
"movl %7, %%ecx \n\t"
"add %1, %%"FF_REG_c" \n\t"
"movl %%ecx, (%0) \n\t"
"5: \n\t"
"add %9, %k0 \n\t"
"shr $2, %k0 \n\t"
: "=&q"(coeff_count), "+r"(significant_coeff_ctx_base), "+m"(index),
"+&r"(c->low), "=&r"(bit), "+&r"(c->range)
: "r"(c), "m"(minusstart), "m"(end), "m"(minusindex), "m"(last_off),
"i"(offsetof(CABACContext, bytestream)),
"i"(offsetof(CABACContext, bytestream_end))
TABLES_ARG
: "%"FF_REG_c, "memory"
);
return coeff_count;
}
#define decode_significance_8x8 decode_significance_8x8_x86
static int decode_significance_8x8_x86(CABACContext *c,
uint8_t *significant_coeff_ctx_base,
int *index, uint8_t *last_coeff_ctx_base, const uint8_t *sig_off){
int minusindex= 4-(intptr_t)index;
int bit;
x86_reg coeff_count;
x86_reg last=0;
x86_reg state;
#ifdef BROKEN_RELOCATIONS
void *tables;
__asm__ volatile(
"lea "MANGLE(ff_h264_cabac_tables)", %0 \n\t"
: "=&r"(tables)
: NAMED_CONSTRAINTS_ARRAY(ff_h264_cabac_tables)
);
#endif
__asm__ volatile(
"mov %1, %6 \n\t"
"3: \n\t"
"mov %10, %0 \n\t"
"movzb (%0, %6), %6 \n\t"
"add %9, %6 \n\t"
BRANCHLESS_GET_CABAC("%4", "%q4", "(%6)", "%3", "%w3",
"%5", "%q5", "%k0", "%b0",
"%c12(%7)", "%c13(%7)",
AV_STRINGIFY(H264_NORM_SHIFT_OFFSET),
AV_STRINGIFY(H264_LPS_RANGE_OFFSET),
AV_STRINGIFY(H264_MLPS_STATE_OFFSET),
"%15")
"mov %1, %6 \n\t"
"test $1, %4 \n\t"
" jz 4f \n\t"
#ifdef BROKEN_RELOCATIONS
"movzb %c14(%15, %q6), %6\n\t"
#else
"movzb "MANGLE(ff_h264_cabac_tables)"+%c14(%6), %6\n\t"
#endif
"add %11, %6 \n\t"
BRANCHLESS_GET_CABAC("%4", "%q4", "(%6)", "%3", "%w3",
"%5", "%q5", "%k0", "%b0",
"%c12(%7)", "%c13(%7)",
AV_STRINGIFY(H264_NORM_SHIFT_OFFSET),
AV_STRINGIFY(H264_LPS_RANGE_OFFSET),
AV_STRINGIFY(H264_MLPS_STATE_OFFSET),
"%15")
"mov %2, %0 \n\t"
"mov %1, %6 \n\t"
"mov %k6, (%0) \n\t"
"test $1, %4 \n\t"
" jnz 5f \n\t"
"add"FF_OPSIZE" $4, %2 \n\t"
"4: \n\t"
"add $1, %6 \n\t"
"mov %6, %1 \n\t"
"cmp $63, %6 \n\t"
" jb 3b \n\t"
"mov %2, %0 \n\t"
"mov %k6, (%0) \n\t"
"5: \n\t"
"addl %8, %k0 \n\t"
"shr $2, %k0 \n\t"
: "=&q"(coeff_count), "+"REG64(last), "+"REG64(index), "+&r"(c->low),
"=&r"(bit), "+&r"(c->range), "=&r"(state)
: "r"(c), "m"(minusindex), "m"(significant_coeff_ctx_base),
REG64(sig_off), REG64(last_coeff_ctx_base),
"i"(offsetof(CABACContext, bytestream)),
"i"(offsetof(CABACContext, bytestream_end)),
"i"(H264_LAST_COEFF_FLAG_OFFSET_8x8_OFFSET) TABLES_ARG
: "%"FF_REG_c, "memory"
);
return coeff_count;
}
#endif /* HAVE_7REGS && BROKEN_COMPILER */
#endif /* HAVE_INLINE_ASM */

663
media/ffvpx/libavcodec/x86/h264_chromamc.asm Normal file
View file

@ -0,0 +1,663 @@
;******************************************************************************
;* MMX/SSSE3-optimized functions for H.264 chroma MC
;* Copyright (c) 2005 Zoltan Hidvegi <hzoli -a- hzoli -d- com>,
;* 2005-2008 Loren Merritt
;*
;* This file is part of FFmpeg.
;*
;* FFmpeg is free software; you can redistribute it and/or
;* modify it under the terms of the GNU Lesser General Public
;* License as published by the Free Software Foundation; either
;* version 2.1 of the License, or (at your option) any later version.
;*
;* FFmpeg is distributed in the hope that it will be useful,
;* but WITHOUT ANY WARRANTY; without even the implied warranty of
;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
;* Lesser General Public License for more details.
;*
;* You should have received a copy of the GNU Lesser General Public
;* License along with FFmpeg; if not, write to the Free Software
;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
;******************************************************************************
%include "libavutil/x86/x86util.asm"
SECTION_RODATA
rnd_rv40_2d_tbl: times 4 dw 0
times 4 dw 16
times 4 dw 32
times 4 dw 16
times 4 dw 32
times 4 dw 28
times 4 dw 32
times 4 dw 28
times 4 dw 0
times 4 dw 32
times 4 dw 16
times 4 dw 32
times 4 dw 32
times 4 dw 28
times 4 dw 32
times 4 dw 28
rnd_rv40_1d_tbl: times 4 dw 0
times 4 dw 2
times 4 dw 4
times 4 dw 2
times 4 dw 4
times 4 dw 3
times 4 dw 4
times 4 dw 3
times 4 dw 0
times 4 dw 4
times 4 dw 2
times 4 dw 4
times 4 dw 4
times 4 dw 3
times 4 dw 4
times 4 dw 3
cextern pw_3
cextern pw_4
cextern pw_8
pw_28: times 8 dw 28
cextern pw_32
cextern pw_64
SECTION .text
%macro mv0_pixels_mc8 0
lea r4, [r2*2 ]
.next4rows:
movq mm0, [r1 ]
movq mm1, [r1+r2]
add r1, r4
CHROMAMC_AVG mm0, [r0 ]
CHROMAMC_AVG mm1, [r0+r2]
movq [r0 ], mm0
movq [r0+r2], mm1
add r0, r4
movq mm0, [r1 ]
movq mm1, [r1+r2]
add r1, r4
CHROMAMC_AVG mm0, [r0 ]
CHROMAMC_AVG mm1, [r0+r2]
movq [r0 ], mm0
movq [r0+r2], mm1
add r0, r4
sub r3d, 4
jne .next4rows
%endmacro
%macro chroma_mc8_mmx_func 2-3
%ifidn %2, rv40
%ifdef PIC
%define rnd_1d_rv40 r8
%define rnd_2d_rv40 r8
%define extra_regs 2
%else ; no-PIC
%define rnd_1d_rv40 rnd_rv40_1d_tbl
%define rnd_2d_rv40 rnd_rv40_2d_tbl
%define extra_regs 1
%endif ; PIC
%else
%define extra_regs 0
%endif ; rv40
; void ff_put/avg_h264_chroma_mc8_*(uint8_t *dst /* align 8 */,
; uint8_t *src /* align 1 */,
; ptrdiff_t stride, int h, int mx, int my)
cglobal %1_%2_chroma_mc8%3, 6, 7 + extra_regs, 0
mov r6d, r5d
or r6d, r4d
jne .at_least_one_non_zero
; mx == 0 AND my == 0 - no filter needed
mv0_pixels_mc8
REP_RET
.at_least_one_non_zero:
%ifidn %2, rv40
%if ARCH_X86_64
mov r7, r5
and r7, 6 ; &~1 for mx/my=[0,7]
lea r7, [r7*4+r4]
sar r7d, 1
%define rnd_bias r7
%define dest_reg r0
%else ; x86-32
mov r0, r5
and r0, 6 ; &~1 for mx/my=[0,7]
lea r0, [r0*4+r4]
sar r0d, 1
%define rnd_bias r0
%define dest_reg r5
%endif
%else ; vc1, h264
%define rnd_bias 0
%define dest_reg r0
%endif
test r5d, r5d
mov r6, 1
je .my_is_zero
test r4d, r4d
mov r6, r2 ; dxy = x ? 1 : stride
jne .both_non_zero
.my_is_zero:
; mx == 0 XOR my == 0 - 1 dimensional filter only
or r4d, r5d ; x + y
%ifidn %2, rv40
%ifdef PIC
lea r8, [rnd_rv40_1d_tbl]
%endif
%if ARCH_X86_64 == 0
mov r5, r0m
%endif
%endif
movd m5, r4d
movq m4, [pw_8]
movq m6, [rnd_1d_%2+rnd_bias*8] ; mm6 = rnd >> 3
punpcklwd m5, m5
punpckldq m5, m5 ; mm5 = B = x
pxor m7, m7
psubw m4, m5 ; mm4 = A = 8-x
.next1drow:
movq m0, [r1 ] ; mm0 = src[0..7]
movq m2, [r1+r6] ; mm1 = src[1..8]
movq m1, m0
movq m3, m2
punpcklbw m0, m7
punpckhbw m1, m7
punpcklbw m2, m7
punpckhbw m3, m7
pmullw m0, m4 ; [mm0,mm1] = A * src[0..7]
pmullw m1, m4
pmullw m2, m5 ; [mm2,mm3] = B * src[1..8]
pmullw m3, m5
paddw m0, m6
paddw m1, m6
paddw m0, m2
paddw m1, m3
psrlw m0, 3
psrlw m1, 3
packuswb m0, m1
CHROMAMC_AVG m0, [dest_reg]
movq [dest_reg], m0 ; dst[0..7] = (A * src[0..7] + B * src[1..8] + (rnd >> 3)) >> 3
add dest_reg, r2
add r1, r2
dec r3d
jne .next1drow
REP_RET
.both_non_zero: ; general case, bilinear
movd m4, r4d ; x
movd m6, r5d ; y
%ifidn %2, rv40
%ifdef PIC
lea r8, [rnd_rv40_2d_tbl]
%endif
%if ARCH_X86_64 == 0
mov r5, r0m
%endif
%endif
mov r6, rsp ; backup stack pointer
and rsp, ~(mmsize-1) ; align stack
sub rsp, 16 ; AA and DD
punpcklwd m4, m4
punpcklwd m6, m6
punpckldq m4, m4 ; mm4 = x words
punpckldq m6, m6 ; mm6 = y words
movq m5, m4
pmullw m4, m6 ; mm4 = x * y
psllw m5, 3
psllw m6, 3
movq m7, m5
paddw m7, m6
movq [rsp+8], m4 ; DD = x * y
psubw m5, m4 ; mm5 = B = 8x - xy
psubw m6, m4 ; mm6 = C = 8y - xy
paddw m4, [pw_64]
psubw m4, m7 ; mm4 = A = xy - (8x+8y) + 64
pxor m7, m7
movq [rsp ], m4
movq m0, [r1 ] ; mm0 = src[0..7]
movq m1, [r1+1] ; mm1 = src[1..8]
.next2drow:
add r1, r2
movq m2, m0
movq m3, m1
punpckhbw m0, m7
punpcklbw m1, m7
punpcklbw m2, m7
punpckhbw m3, m7
pmullw m0, [rsp]
pmullw m2, [rsp]
pmullw m1, m5
pmullw m3, m5
paddw m2, m1 ; mm2 = A * src[0..3] + B * src[1..4]
paddw m3, m0 ; mm3 = A * src[4..7] + B * src[5..8]
movq m0, [r1]
movq m1, m0
punpcklbw m0, m7
punpckhbw m1, m7
pmullw m0, m6
pmullw m1, m6
paddw m2, m0
paddw m3, m1 ; [mm2,mm3] += C * src[0..7]
movq m1, [r1+1]
movq m0, m1
movq m4, m1
punpcklbw m0, m7
punpckhbw m4, m7
pmullw m0, [rsp+8]
pmullw m4, [rsp+8]
paddw m2, m0
paddw m3, m4 ; [mm2,mm3] += D * src[1..8]
movq m0, [r1]
paddw m2, [rnd_2d_%2+rnd_bias*8]
paddw m3, [rnd_2d_%2+rnd_bias*8]
psrlw m2, 6
psrlw m3, 6
packuswb m2, m3
CHROMAMC_AVG m2, [dest_reg]
movq [dest_reg], m2 ; dst[0..7] = ([mm2,mm3] + rnd) >> 6
add dest_reg, r2
dec r3d
jne .next2drow
mov rsp, r6 ; restore stack pointer
RET
%endmacro
%macro chroma_mc4_mmx_func 2
%define extra_regs 0
%ifidn %2, rv40
%ifdef PIC
%define extra_regs 1
%endif ; PIC
%endif ; rv40
cglobal %1_%2_chroma_mc4, 6, 6 + extra_regs, 0
pxor m7, m7
movd m2, r4d ; x
movd m3, r5d ; y
movq m4, [pw_8]
movq m5, [pw_8]
punpcklwd m2, m2
punpcklwd m3, m3
punpcklwd m2, m2
punpcklwd m3, m3
psubw m4, m2
psubw m5, m3
%ifidn %2, rv40
%ifdef PIC
lea r6, [rnd_rv40_2d_tbl]
%define rnd_2d_rv40 r6
%else
%define rnd_2d_rv40 rnd_rv40_2d_tbl
%endif
and r5, 6 ; &~1 for mx/my=[0,7]
lea r5, [r5*4+r4]
sar r5d, 1
%define rnd_bias r5
%else ; vc1, h264
%define rnd_bias 0
%endif
movd m0, [r1 ]
movd m6, [r1+1]
add r1, r2
punpcklbw m0, m7
punpcklbw m6, m7
pmullw m0, m4
pmullw m6, m2
paddw m6, m0
.next2rows:
movd m0, [r1 ]
movd m1, [r1+1]
add r1, r2
punpcklbw m0, m7
punpcklbw m1, m7
pmullw m0, m4
pmullw m1, m2
paddw m1, m0
movq m0, m1
pmullw m6, m5
pmullw m1, m3
paddw m6, [rnd_2d_%2+rnd_bias*8]
paddw m1, m6
psrlw m1, 6
packuswb m1, m1
CHROMAMC_AVG4 m1, m6, [r0]
movd [r0], m1
add r0, r2
movd m6, [r1 ]
movd m1, [r1+1]
add r1, r2
punpcklbw m6, m7
punpcklbw m1, m7
pmullw m6, m4
pmullw m1, m2
paddw m1, m6
movq m6, m1
pmullw m0, m5
pmullw m1, m3
paddw m0, [rnd_2d_%2+rnd_bias*8]
paddw m1, m0
psrlw m1, 6
packuswb m1, m1
CHROMAMC_AVG4 m1, m0, [r0]
movd [r0], m1
add r0, r2
sub r3d, 2
jnz .next2rows
REP_RET
%endmacro
%macro chroma_mc2_mmx_func 2
cglobal %1_%2_chroma_mc2, 6, 7, 0
mov r6d, r4d
shl r4d, 16
sub r4d, r6d
add r4d, 8
imul r5d, r4d ; x*y<<16 | y*(8-x)
shl r4d, 3
sub r4d, r5d ; x*(8-y)<<16 | (8-x)*(8-y)
movd m5, r4d
movd m6, r5d
punpckldq m5, m5 ; mm5 = {A,B,A,B}
punpckldq m6, m6 ; mm6 = {C,D,C,D}
pxor m7, m7
movd m2, [r1]
punpcklbw m2, m7
pshufw m2, m2, 0x94 ; mm0 = src[0,1,1,2]
.nextrow:
add r1, r2
movq m1, m2
pmaddwd m1, m5 ; mm1 = A * src[0,1] + B * src[1,2]
movd m0, [r1]
punpcklbw m0, m7
pshufw m0, m0, 0x94 ; mm0 = src[0,1,1,2]
movq m2, m0
pmaddwd m0, m6
paddw m1, [rnd_2d_%2]
paddw m1, m0 ; mm1 += C * src[0,1] + D * src[1,2]
psrlw m1, 6
packssdw m1, m7
packuswb m1, m7
CHROMAMC_AVG4 m1, m3, [r0]
movd r5d, m1
mov [r0], r5w
add r0, r2
sub r3d, 1
jnz .nextrow
REP_RET
%endmacro
%define rnd_1d_h264 pw_4
%define rnd_2d_h264 pw_32
%define rnd_1d_vc1 pw_3
%define rnd_2d_vc1 pw_28
%macro NOTHING 2-3
%endmacro
%macro DIRECT_AVG 2
PAVGB %1, %2
%endmacro
%macro COPY_AVG 3
movd %2, %3
PAVGB %1, %2
%endmacro
INIT_MMX mmx
%define CHROMAMC_AVG NOTHING
%define CHROMAMC_AVG4 NOTHING
chroma_mc8_mmx_func put, h264, _rnd
chroma_mc8_mmx_func put, vc1, _nornd
chroma_mc8_mmx_func put, rv40
chroma_mc4_mmx_func put, h264
chroma_mc4_mmx_func put, rv40
INIT_MMX mmxext
chroma_mc2_mmx_func put, h264
%define CHROMAMC_AVG DIRECT_AVG
%define CHROMAMC_AVG4 COPY_AVG
chroma_mc8_mmx_func avg, h264, _rnd
chroma_mc8_mmx_func avg, vc1, _nornd
chroma_mc8_mmx_func avg, rv40
chroma_mc4_mmx_func avg, h264
chroma_mc4_mmx_func avg, rv40
chroma_mc2_mmx_func avg, h264
INIT_MMX 3dnow
chroma_mc8_mmx_func avg, h264, _rnd
chroma_mc8_mmx_func avg, vc1, _nornd
chroma_mc8_mmx_func avg, rv40
chroma_mc4_mmx_func avg, h264
chroma_mc4_mmx_func avg, rv40
%macro chroma_mc8_ssse3_func 2-3
cglobal %1_%2_chroma_mc8%3, 6, 7, 8
mov r6d, r5d
or r6d, r4d
jne .at_least_one_non_zero
; mx == 0 AND my == 0 - no filter needed
mv0_pixels_mc8
REP_RET
.at_least_one_non_zero:
test r5d, r5d
je .my_is_zero
test r4d, r4d
je .mx_is_zero
; general case, bilinear
mov r6d, r4d
shl r4d, 8
sub r4, r6
mov r6, 8
add r4, 8 ; x*288+8 = x<<8 | (8-x)
sub r6d, r5d
imul r6, r4 ; (8-y)*(x*255+8) = (8-y)*x<<8 | (8-y)*(8-x)
imul r4d, r5d ; y *(x*255+8) = y *x<<8 | y *(8-x)
movd m7, r6d
movd m6, r4d
movdqa m5, [rnd_2d_%2]
movq m0, [r1 ]
movq m1, [r1+1]
pshuflw m7, m7, 0
pshuflw m6, m6, 0
punpcklbw m0, m1
movlhps m7, m7
movlhps m6, m6
.next2rows:
movq m1, [r1+r2*1 ]
movq m2, [r1+r2*1+1]
movq m3, [r1+r2*2 ]
movq m4, [r1+r2*2+1]
lea r1, [r1+r2*2]
punpcklbw m1, m2
movdqa m2, m1
punpcklbw m3, m4
movdqa m4, m3
pmaddubsw m0, m7
pmaddubsw m1, m6
pmaddubsw m2, m7
pmaddubsw m3, m6
paddw m0, m5
paddw m2, m5
paddw m1, m0
paddw m3, m2
psrlw m1, 6
movdqa m0, m4
psrlw m3, 6
%ifidn %1, avg
movq m2, [r0 ]
movhps m2, [r0+r2]
%endif
packuswb m1, m3
CHROMAMC_AVG m1, m2
movq [r0 ], m1
movhps [r0+r2], m1
sub r3d, 2
lea r0, [r0+r2*2]
jg .next2rows
REP_RET
.my_is_zero:
mov r5d, r4d
shl r4d, 8
add r4, 8
sub r4, r5 ; 255*x+8 = x<<8 | (8-x)
movd m7, r4d
movdqa m6, [rnd_1d_%2]
pshuflw m7, m7, 0
movlhps m7, m7
.next2xrows:
movq m0, [r1 ]
movq m1, [r1 +1]
movq m2, [r1+r2 ]
movq m3, [r1+r2+1]
punpcklbw m0, m1
punpcklbw m2, m3
pmaddubsw m0, m7
pmaddubsw m2, m7
%ifidn %1, avg
movq m4, [r0 ]
movhps m4, [r0+r2]
%endif
paddw m0, m6
paddw m2, m6
psrlw m0, 3
psrlw m2, 3
packuswb m0, m2
CHROMAMC_AVG m0, m4
movq [r0 ], m0
movhps [r0+r2], m0
sub r3d, 2
lea r0, [r0+r2*2]
lea r1, [r1+r2*2]
jg .next2xrows
REP_RET
.mx_is_zero:
mov r4d, r5d
shl r5d, 8
add r5, 8
sub r5, r4 ; 255*y+8 = y<<8 | (8-y)
movd m7, r5d
movdqa m6, [rnd_1d_%2]
pshuflw m7, m7, 0
movlhps m7, m7
.next2yrows:
movq m0, [r1 ]
movq m1, [r1+r2 ]
movdqa m2, m1
movq m3, [r1+r2*2]
lea r1, [r1+r2*2]
punpcklbw m0, m1
punpcklbw m2, m3
pmaddubsw m0, m7
pmaddubsw m2, m7
%ifidn %1, avg
movq m4, [r0 ]
movhps m4, [r0+r2]
%endif
paddw m0, m6
paddw m2, m6
psrlw m0, 3
psrlw m2, 3
packuswb m0, m2
CHROMAMC_AVG m0, m4
movq [r0 ], m0
movhps [r0+r2], m0
sub r3d, 2
lea r0, [r0+r2*2]
jg .next2yrows
REP_RET
%endmacro
%macro chroma_mc4_ssse3_func 2
cglobal %1_%2_chroma_mc4, 6, 7, 0
mov r6, r4
shl r4d, 8
sub r4d, r6d
mov r6, 8
add r4d, 8 ; x*288+8
sub r6d, r5d
imul r6d, r4d ; (8-y)*(x*255+8) = (8-y)*x<<8 | (8-y)*(8-x)
imul r4d, r5d ; y *(x*255+8) = y *x<<8 | y *(8-x)
movd m7, r6d
movd m6, r4d
movq m5, [pw_32]
movd m0, [r1 ]
pshufw m7, m7, 0
punpcklbw m0, [r1+1]
pshufw m6, m6, 0
.next2rows:
movd m1, [r1+r2*1 ]
movd m3, [r1+r2*2 ]
punpcklbw m1, [r1+r2*1+1]
punpcklbw m3, [r1+r2*2+1]
lea r1, [r1+r2*2]
movq m2, m1
movq m4, m3
pmaddubsw m0, m7
pmaddubsw m1, m6
pmaddubsw m2, m7
pmaddubsw m3, m6
paddw m0, m5
paddw m2, m5
paddw m1, m0
paddw m3, m2
psrlw m1, 6
movq m0, m4
psrlw m3, 6
packuswb m1, m1
packuswb m3, m3
CHROMAMC_AVG m1, [r0 ]
CHROMAMC_AVG m3, [r0+r2]
movd [r0 ], m1
movd [r0+r2], m3
sub r3d, 2
lea r0, [r0+r2*2]
jg .next2rows
REP_RET
%endmacro
%define CHROMAMC_AVG NOTHING
INIT_XMM ssse3
chroma_mc8_ssse3_func put, h264, _rnd
chroma_mc8_ssse3_func put, vc1, _nornd
INIT_MMX ssse3
chroma_mc4_ssse3_func put, h264
%define CHROMAMC_AVG DIRECT_AVG
INIT_XMM ssse3
chroma_mc8_ssse3_func avg, h264, _rnd
chroma_mc8_ssse3_func avg, vc1, _nornd
INIT_MMX ssse3
chroma_mc4_ssse3_func avg, h264

269
media/ffvpx/libavcodec/x86/h264_chromamc_10bit.asm Normal file
View file

@ -0,0 +1,269 @@
;*****************************************************************************
;* MMX/SSE2/AVX-optimized 10-bit H.264 chroma MC code
;*****************************************************************************
;* Copyright (C) 2005-2011 x264 project
;*
;* Authors: Daniel Kang <daniel.d.kang@gmail.com>
;*
;* This file is part of FFmpeg.
;*
;* FFmpeg is free software; you can redistribute it and/or
;* modify it under the terms of the GNU Lesser General Public
;* License as published by the Free Software Foundation; either
;* version 2.1 of the License, or (at your option) any later version.
;*
;* FFmpeg is distributed in the hope that it will be useful,
;* but WITHOUT ANY WARRANTY; without even the implied warranty of
;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
;* Lesser General Public License for more details.
;*
;* You should have received a copy of the GNU Lesser General Public
;* License along with FFmpeg; if not, write to the Free Software
;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
;******************************************************************************
%include "libavutil/x86/x86util.asm"
SECTION_RODATA
cextern pw_4
cextern pw_8
cextern pw_32
cextern pw_64
SECTION .text
%macro MV0_PIXELS_MC8 0
lea r4, [r2*3 ]
lea r5, [r2*4 ]
.next4rows:
movu m0, [r1 ]
movu m1, [r1+r2 ]
CHROMAMC_AVG m0, [r0 ]
CHROMAMC_AVG m1, [r0+r2 ]
mova [r0 ], m0
mova [r0+r2 ], m1
movu m0, [r1+r2*2]
movu m1, [r1+r4 ]
CHROMAMC_AVG m0, [r0+r2*2]
CHROMAMC_AVG m1, [r0+r4 ]
mova [r0+r2*2], m0
mova [r0+r4 ], m1
add r1, r5
add r0, r5
sub r3d, 4
jne .next4rows
%endmacro
;-----------------------------------------------------------------------------
; void ff_put/avg_h264_chroma_mc8(pixel *dst, pixel *src, ptrdiff_t stride,
; int h, int mx, int my)
;-----------------------------------------------------------------------------
%macro CHROMA_MC8 1
cglobal %1_h264_chroma_mc8_10, 6,7,8
mov r6d, r5d
or r6d, r4d
jne .at_least_one_non_zero
; mx == 0 AND my == 0 - no filter needed
MV0_PIXELS_MC8
REP_RET
.at_least_one_non_zero:
mov r6d, 2
test r5d, r5d
je .x_interpolation
mov r6, r2 ; dxy = x ? 1 : stride
test r4d, r4d
jne .xy_interpolation
.x_interpolation:
; mx == 0 XOR my == 0 - 1 dimensional filter only
or r4d, r5d ; x + y
movd m5, r4d
mova m4, [pw_8]
mova m6, [pw_4] ; mm6 = rnd >> 3
SPLATW m5, m5 ; mm5 = B = x
psubw m4, m5 ; mm4 = A = 8-x
.next1drow:
movu m0, [r1 ] ; mm0 = src[0..7]
movu m2, [r1+r6] ; mm2 = src[1..8]
pmullw m0, m4 ; mm0 = A * src[0..7]
pmullw m2, m5 ; mm2 = B * src[1..8]
paddw m0, m6
paddw m0, m2
psrlw m0, 3
CHROMAMC_AVG m0, [r0]
mova [r0], m0 ; dst[0..7] = (A * src[0..7] + B * src[1..8] + (rnd >> 3)) >> 3
add r0, r2
add r1, r2
dec r3d
jne .next1drow
REP_RET
.xy_interpolation: ; general case, bilinear
movd m4, r4m ; x
movd m6, r5m ; y
SPLATW m4, m4 ; mm4 = x words
SPLATW m6, m6 ; mm6 = y words
psllw m5, m4, 3 ; mm5 = 8x
pmullw m4, m6 ; mm4 = x * y
psllw m6, 3 ; mm6 = 8y
paddw m1, m5, m6 ; mm7 = 8x+8y
mova m7, m4 ; DD = x * y
psubw m5, m4 ; mm5 = B = 8x - xy
psubw m6, m4 ; mm6 = C = 8y - xy
paddw m4, [pw_64]
psubw m4, m1 ; mm4 = A = xy - (8x+8y) + 64
movu m0, [r1 ] ; mm0 = src[0..7]
movu m1, [r1+2] ; mm1 = src[1..8]
.next2drow:
add r1, r2
pmullw m2, m0, m4
pmullw m1, m5
paddw m2, m1 ; mm2 = A * src[0..7] + B * src[1..8]
movu m0, [r1]
movu m1, [r1+2]
pmullw m3, m0, m6
paddw m2, m3 ; mm2 += C * src[0..7+strde]
pmullw m3, m1, m7
paddw m2, m3 ; mm2 += D * src[1..8+strde]
paddw m2, [pw_32]
psrlw m2, 6
CHROMAMC_AVG m2, [r0]
mova [r0], m2 ; dst[0..7] = (mm2 + 32) >> 6
add r0, r2
dec r3d
jne .next2drow
REP_RET
%endmacro
;-----------------------------------------------------------------------------
; void ff_put/avg_h264_chroma_mc4(pixel *dst, pixel *src, ptrdiff_t stride,
; int h, int mx, int my)
;-----------------------------------------------------------------------------
;TODO: xmm mc4
%macro MC4_OP 2
movq %1, [r1 ]
movq m1, [r1+2]
add r1, r2
pmullw %1, m4
pmullw m1, m2
paddw m1, %1
mova %1, m1
pmullw %2, m5
pmullw m1, m3
paddw %2, [pw_32]
paddw m1, %2
psrlw m1, 6
CHROMAMC_AVG m1, %2, [r0]
movq [r0], m1
add r0, r2
%endmacro
%macro CHROMA_MC4 1
cglobal %1_h264_chroma_mc4_10, 6,6,7
movd m2, r4m ; x
movd m3, r5m ; y
mova m4, [pw_8]
mova m5, m4
SPLATW m2, m2
SPLATW m3, m3
psubw m4, m2
psubw m5, m3
movq m0, [r1 ]
movq m6, [r1+2]
add r1, r2
pmullw m0, m4
pmullw m6, m2
paddw m6, m0
.next2rows:
MC4_OP m0, m6
MC4_OP m6, m0
sub r3d, 2
jnz .next2rows
REP_RET
%endmacro
;-----------------------------------------------------------------------------
; void ff_put/avg_h264_chroma_mc2(pixel *dst, pixel *src, ptrdiff_t stride,
; int h, int mx, int my)
;-----------------------------------------------------------------------------
%macro CHROMA_MC2 1
cglobal %1_h264_chroma_mc2_10, 6,7
mov r6d, r4d
shl r4d, 16
sub r4d, r6d
add r4d, 8
imul r5d, r4d ; x*y<<16 | y*(8-x)
shl r4d, 3
sub r4d, r5d ; x*(8-y)<<16 | (8-x)*(8-y)
movd m5, r4d
movd m6, r5d
punpckldq m5, m5 ; mm5 = {A,B,A,B}
punpckldq m6, m6 ; mm6 = {C,D,C,D}
pxor m7, m7
pshufw m2, [r1], 0x94 ; mm0 = src[0,1,1,2]
.nextrow:
add r1, r2
movq m1, m2
pmaddwd m1, m5 ; mm1 = A * src[0,1] + B * src[1,2]
pshufw m0, [r1], 0x94 ; mm0 = src[0,1,1,2]
movq m2, m0
pmaddwd m0, m6
paddw m1, [pw_32]
paddw m1, m0 ; mm1 += C * src[0,1] + D * src[1,2]
psrlw m1, 6
packssdw m1, m7
CHROMAMC_AVG m1, m3, [r0]
movd [r0], m1
add r0, r2
dec r3d
jnz .nextrow
REP_RET
%endmacro
%macro NOTHING 2-3
%endmacro
%macro AVG 2-3
%if %0==3
movq %2, %3
%endif
pavgw %1, %2
%endmacro
%define CHROMAMC_AVG NOTHING
INIT_XMM sse2
CHROMA_MC8 put
%if HAVE_AVX_EXTERNAL
INIT_XMM avx
CHROMA_MC8 put
%endif
INIT_MMX mmxext
CHROMA_MC4 put
CHROMA_MC2 put
%define CHROMAMC_AVG AVG
INIT_XMM sse2
CHROMA_MC8 avg
%if HAVE_AVX_EXTERNAL
INIT_XMM avx
CHROMA_MC8 avg
%endif
INIT_MMX mmxext
CHROMA_MC4 avg
CHROMA_MC2 avg

1420
media/ffvpx/libavcodec/x86/h264_deblock.asm Normal file
View file

File diff suppressed because it is too large Load diff

1080
media/ffvpx/libavcodec/x86/h264_deblock_10bit.asm Normal file
View file

File diff suppressed because it is too large Load diff

1191
media/ffvpx/libavcodec/x86/h264_idct.asm Normal file
View file

File diff suppressed because it is too large Load diff

657
media/ffvpx/libavcodec/x86/h264_idct_10bit.asm Normal file
View file

@ -0,0 +1,657 @@
;*****************************************************************************
;* MMX/SSE2/AVX-optimized 10-bit H.264 iDCT code
;*****************************************************************************
;* Copyright (C) 2005-2011 x264 project
;*
;* Authors: Daniel Kang <daniel.d.kang@gmail.com>
;*
;* This file is part of FFmpeg.
;*
;* FFmpeg is free software; you can redistribute it and/or
;* modify it under the terms of the GNU Lesser General Public
;* License as published by the Free Software Foundation; either
;* version 2.1 of the License, or (at your option) any later version.
;*
;* FFmpeg is distributed in the hope that it will be useful,
;* but WITHOUT ANY WARRANTY; without even the implied warranty of
;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
;* Lesser General Public License for more details.
;*
;* You should have received a copy of the GNU Lesser General Public
;* License along with FFmpeg; if not, write to the Free Software
;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
;******************************************************************************
%include "libavutil/x86/x86util.asm"
SECTION .text
cextern pw_1023
%define pw_pixel_max pw_1023
cextern pd_32
;-----------------------------------------------------------------------------
; void ff_h264_idct_add_10(pixel *dst, int16_t *block, int stride)
;-----------------------------------------------------------------------------
%macro STORE_DIFFx2 6
psrad %1, 6
psrad %2, 6
packssdw %1, %2
movq %3, [%5]
movhps %3, [%5+%6]
paddsw %1, %3
CLIPW %1, %4, [pw_pixel_max]
movq [%5], %1
movhps [%5+%6], %1
%endmacro
%macro STORE_DIFF16 5
psrad %1, 6
psrad %2, 6
packssdw %1, %2
paddsw %1, [%5]
CLIPW %1, %3, %4
mova [%5], %1
%endmacro
;dst, in, stride
%macro IDCT4_ADD_10 3
mova m0, [%2+ 0]
mova m1, [%2+16]
mova m2, [%2+32]
mova m3, [%2+48]
IDCT4_1D d,0,1,2,3,4,5
TRANSPOSE4x4D 0,1,2,3,4
paddd m0, [pd_32]
IDCT4_1D d,0,1,2,3,4,5
pxor m5, m5
mova [%2+ 0], m5
mova [%2+16], m5
mova [%2+32], m5
mova [%2+48], m5
STORE_DIFFx2 m0, m1, m4, m5, %1, %3
lea %1, [%1+%3*2]
STORE_DIFFx2 m2, m3, m4, m5, %1, %3
%endmacro
%macro IDCT_ADD_10 0
cglobal h264_idct_add_10, 3,3
movsxdifnidn r2, r2d
IDCT4_ADD_10 r0, r1, r2
RET
%endmacro
INIT_XMM sse2
IDCT_ADD_10
%if HAVE_AVX_EXTERNAL
INIT_XMM avx
IDCT_ADD_10
%endif
;-----------------------------------------------------------------------------
; void ff_h264_idct_add16_10(pixel *dst, const int *block_offset,
; int16_t *block, int stride,
; const uint8_t nnzc[6*8])
;-----------------------------------------------------------------------------
;;;;;;; NO FATE SAMPLES TRIGGER THIS
%macro ADD4x4IDCT 0
add4x4_idct %+ SUFFIX:
add r5, r0
mova m0, [r2+ 0]
mova m1, [r2+16]
mova m2, [r2+32]
mova m3, [r2+48]
IDCT4_1D d,0,1,2,3,4,5
TRANSPOSE4x4D 0,1,2,3,4
paddd m0, [pd_32]
IDCT4_1D d,0,1,2,3,4,5
pxor m5, m5
mova [r2+ 0], m5
mova [r2+16], m5
mova [r2+32], m5
mova [r2+48], m5
STORE_DIFFx2 m0, m1, m4, m5, r5, r3
lea r5, [r5+r3*2]
STORE_DIFFx2 m2, m3, m4, m5, r5, r3
ret
%endmacro
INIT_XMM sse2
ALIGN 16
ADD4x4IDCT
%if HAVE_AVX_EXTERNAL
INIT_XMM avx
ALIGN 16
ADD4x4IDCT
%endif
%macro ADD16_OP 2
cmp byte [r4+%2], 0
jz .skipblock%1
mov r5d, [r1+%1*4]
call add4x4_idct %+ SUFFIX
.skipblock%1:
%if %1<15
add r2, 64
%endif
%endmacro
%macro IDCT_ADD16_10 0
cglobal h264_idct_add16_10, 5,6
movsxdifnidn r3, r3d
ADD16_OP 0, 4+1*8
ADD16_OP 1, 5+1*8
ADD16_OP 2, 4+2*8
ADD16_OP 3, 5+2*8
ADD16_OP 4, 6+1*8
ADD16_OP 5, 7+1*8
ADD16_OP 6, 6+2*8
ADD16_OP 7, 7+2*8
ADD16_OP 8, 4+3*8
ADD16_OP 9, 5+3*8
ADD16_OP 10, 4+4*8
ADD16_OP 11, 5+4*8
ADD16_OP 12, 6+3*8
ADD16_OP 13, 7+3*8
ADD16_OP 14, 6+4*8
ADD16_OP 15, 7+4*8
REP_RET
%endmacro
INIT_XMM sse2
IDCT_ADD16_10
%if HAVE_AVX_EXTERNAL
INIT_XMM avx
IDCT_ADD16_10
%endif
;-----------------------------------------------------------------------------
; void ff_h264_idct_dc_add_10(pixel *dst, int16_t *block, int stride)
;-----------------------------------------------------------------------------
%macro IDCT_DC_ADD_OP_10 3
pxor m5, m5
%if avx_enabled
paddw m1, m0, [%1+0 ]
paddw m2, m0, [%1+%2 ]
paddw m3, m0, [%1+%2*2]
paddw m4, m0, [%1+%3 ]
%else
mova m1, [%1+0 ]
mova m2, [%1+%2 ]
mova m3, [%1+%2*2]
mova m4, [%1+%3 ]
paddw m1, m0
paddw m2, m0
paddw m3, m0
paddw m4, m0
%endif
CLIPW m1, m5, m6
CLIPW m2, m5, m6
CLIPW m3, m5, m6
CLIPW m4, m5, m6
mova [%1+0 ], m1
mova [%1+%2 ], m2
mova [%1+%2*2], m3
mova [%1+%3 ], m4
%endmacro
INIT_MMX mmxext
cglobal h264_idct_dc_add_10,3,3
movsxdifnidn r2, r2d
movd m0, [r1]
mov dword [r1], 0
paddd m0, [pd_32]
psrad m0, 6
lea r1, [r2*3]
pshufw m0, m0, 0
mova m6, [pw_pixel_max]
IDCT_DC_ADD_OP_10 r0, r2, r1
RET
;-----------------------------------------------------------------------------
; void ff_h264_idct8_dc_add_10(pixel *dst, int16_t *block, int stride)
;-----------------------------------------------------------------------------
%macro IDCT8_DC_ADD 0
cglobal h264_idct8_dc_add_10,3,4,7
movsxdifnidn r2, r2d
movd m0, [r1]
mov dword[r1], 0
paddd m0, [pd_32]
psrad m0, 6
lea r1, [r2*3]
SPLATW m0, m0, 0
mova m6, [pw_pixel_max]
IDCT_DC_ADD_OP_10 r0, r2, r1
lea r0, [r0+r2*4]
IDCT_DC_ADD_OP_10 r0, r2, r1
RET
%endmacro
INIT_XMM sse2
IDCT8_DC_ADD
%if HAVE_AVX_EXTERNAL
INIT_XMM avx
IDCT8_DC_ADD
%endif
;-----------------------------------------------------------------------------
; void ff_h264_idct_add16intra_10(pixel *dst, const int *block_offset,
; int16_t *block, int stride,
; const uint8_t nnzc[6*8])
;-----------------------------------------------------------------------------
%macro AC 1
.ac%1:
mov r5d, [r1+(%1+0)*4]
call add4x4_idct %+ SUFFIX
mov r5d, [r1+(%1+1)*4]
add r2, 64
call add4x4_idct %+ SUFFIX
add r2, 64
jmp .skipadd%1
%endmacro
%assign last_block 16
%macro ADD16_OP_INTRA 2
cmp word [r4+%2], 0
jnz .ac%1
mov r5d, [r2+ 0]
or r5d, [r2+64]
jz .skipblock%1
mov r5d, [r1+(%1+0)*4]
call idct_dc_add %+ SUFFIX
.skipblock%1:
%if %1<last_block-2
add r2, 128
%endif
.skipadd%1:
%endmacro
%macro IDCT_ADD16INTRA_10 0
idct_dc_add %+ SUFFIX:
add r5, r0
movq m0, [r2+ 0]
movhps m0, [r2+64]
mov dword [r2+ 0], 0
mov dword [r2+64], 0
paddd m0, [pd_32]
psrad m0, 6
pshufhw m0, m0, 0
pshuflw m0, m0, 0
lea r6, [r3*3]
mova m6, [pw_pixel_max]
IDCT_DC_ADD_OP_10 r5, r3, r6
ret
cglobal h264_idct_add16intra_10,5,7,8
movsxdifnidn r3, r3d
ADD16_OP_INTRA 0, 4+1*8
ADD16_OP_INTRA 2, 4+2*8
ADD16_OP_INTRA 4, 6+1*8
ADD16_OP_INTRA 6, 6+2*8
ADD16_OP_INTRA 8, 4+3*8
ADD16_OP_INTRA 10, 4+4*8
ADD16_OP_INTRA 12, 6+3*8
ADD16_OP_INTRA 14, 6+4*8
REP_RET
AC 8
AC 10
AC 12
AC 14
AC 0
AC 2
AC 4
AC 6
%endmacro
INIT_XMM sse2
IDCT_ADD16INTRA_10
%if HAVE_AVX_EXTERNAL
INIT_XMM avx
IDCT_ADD16INTRA_10
%endif
%assign last_block 36
;-----------------------------------------------------------------------------
; void ff_h264_idct_add8_10(pixel **dst, const int *block_offset,
; int16_t *block, int stride,
; const uint8_t nnzc[6*8])
;-----------------------------------------------------------------------------
%macro IDCT_ADD8 0
cglobal h264_idct_add8_10,5,8,7
movsxdifnidn r3, r3d
%if ARCH_X86_64
mov r7, r0
%endif
add r2, 1024
mov r0, [r0]
ADD16_OP_INTRA 16, 4+ 6*8
ADD16_OP_INTRA 18, 4+ 7*8
add r2, 1024-128*2
%if ARCH_X86_64
mov r0, [r7+gprsize]
%else
mov r0, r0m
mov r0, [r0+gprsize]
%endif
ADD16_OP_INTRA 32, 4+11*8
ADD16_OP_INTRA 34, 4+12*8
REP_RET
AC 16
AC 18
AC 32
AC 34
%endmacro ; IDCT_ADD8
INIT_XMM sse2
IDCT_ADD8
%if HAVE_AVX_EXTERNAL
INIT_XMM avx
IDCT_ADD8
%endif
;-----------------------------------------------------------------------------
; void ff_h264_idct_add8_422_10(pixel **dst, const int *block_offset,
; int16_t *block, int stride,
; const uint8_t nnzc[6*8])
;-----------------------------------------------------------------------------
%assign last_block 44
%macro IDCT_ADD8_422 0
cglobal h264_idct_add8_422_10, 5, 8, 7
movsxdifnidn r3, r3d
%if ARCH_X86_64
mov r7, r0
%endif
add r2, 1024
mov r0, [r0]
ADD16_OP_INTRA 16, 4+ 6*8
ADD16_OP_INTRA 18, 4+ 7*8
ADD16_OP_INTRA 24, 4+ 8*8 ; i+4
ADD16_OP_INTRA 26, 4+ 9*8 ; i+4
add r2, 1024-128*4
%if ARCH_X86_64
mov r0, [r7+gprsize]
%else
mov r0, r0m
mov r0, [r0+gprsize]
%endif
ADD16_OP_INTRA 32, 4+11*8
ADD16_OP_INTRA 34, 4+12*8
ADD16_OP_INTRA 40, 4+13*8 ; i+4
ADD16_OP_INTRA 42, 4+14*8 ; i+4
REP_RET
AC 16
AC 18
AC 24 ; i+4
AC 26 ; i+4
AC 32
AC 34
AC 40 ; i+4
AC 42 ; i+4
%endmacro
INIT_XMM sse2
IDCT_ADD8_422
%if HAVE_AVX_EXTERNAL
INIT_XMM avx
IDCT_ADD8_422
%endif
;-----------------------------------------------------------------------------
; void ff_h264_idct8_add_10(pixel *dst, int16_t *block, int stride)
;-----------------------------------------------------------------------------
%macro IDCT8_1D 2
SWAP 0, 1
psrad m4, m5, 1
psrad m1, m0, 1
paddd m4, m5
paddd m1, m0
paddd m4, m7
paddd m1, m5
psubd m4, m0
paddd m1, m3
psubd m0, m3
psubd m5, m3
paddd m0, m7
psubd m5, m7
psrad m3, 1
psrad m7, 1
psubd m0, m3
psubd m5, m7
SWAP 1, 7
psrad m1, m7, 2
psrad m3, m4, 2
paddd m3, m0
psrad m0, 2
paddd m1, m5
psrad m5, 2
psubd m0, m4
psubd m7, m5
SWAP 5, 6
psrad m4, m2, 1
psrad m6, m5, 1
psubd m4, m5
paddd m6, m2
mova m2, %1
mova m5, %2
SUMSUB_BA d, 5, 2
SUMSUB_BA d, 6, 5
SUMSUB_BA d, 4, 2
SUMSUB_BA d, 7, 6
SUMSUB_BA d, 0, 4
SUMSUB_BA d, 3, 2
SUMSUB_BA d, 1, 5
SWAP 7, 6, 4, 5, 2, 3, 1, 0 ; 70315246 -> 01234567
%endmacro
%macro IDCT8_1D_FULL 1
mova m7, [%1+112*2]
mova m6, [%1+ 96*2]
mova m5, [%1+ 80*2]
mova m3, [%1+ 48*2]
mova m2, [%1+ 32*2]
mova m1, [%1+ 16*2]
IDCT8_1D [%1], [%1+ 64*2]
%endmacro
; %1=int16_t *block, %2=int16_t *dstblock
%macro IDCT8_ADD_SSE_START 2
IDCT8_1D_FULL %1
%if ARCH_X86_64
TRANSPOSE4x4D 0,1,2,3,8
mova [%2 ], m0
TRANSPOSE4x4D 4,5,6,7,8
mova [%2+8*2], m4
%else
mova [%1], m7
TRANSPOSE4x4D 0,1,2,3,7
mova m7, [%1]
mova [%2 ], m0
mova [%2+16*2], m1
mova [%2+32*2], m2
mova [%2+48*2], m3
TRANSPOSE4x4D 4,5,6,7,3
mova [%2+ 8*2], m4
mova [%2+24*2], m5
mova [%2+40*2], m6
mova [%2+56*2], m7
%endif
%endmacro
; %1=uint8_t *dst, %2=int16_t *block, %3=int stride
%macro IDCT8_ADD_SSE_END 3
IDCT8_1D_FULL %2
mova [%2 ], m6
mova [%2+16*2], m7
pxor m7, m7
STORE_DIFFx2 m0, m1, m6, m7, %1, %3
lea %1, [%1+%3*2]
STORE_DIFFx2 m2, m3, m6, m7, %1, %3
mova m0, [%2 ]
mova m1, [%2+16*2]
lea %1, [%1+%3*2]
STORE_DIFFx2 m4, m5, m6, m7, %1, %3
lea %1, [%1+%3*2]
STORE_DIFFx2 m0, m1, m6, m7, %1, %3
%endmacro
%macro IDCT8_ADD 0
cglobal h264_idct8_add_10, 3,4,16
movsxdifnidn r2, r2d
%if UNIX64 == 0
%assign pad 16-gprsize-(stack_offset&15)
sub rsp, pad
call h264_idct8_add1_10 %+ SUFFIX
add rsp, pad
RET
%endif
ALIGN 16
; TODO: does not need to use stack
h264_idct8_add1_10 %+ SUFFIX:
%assign pad 256+16-gprsize
sub rsp, pad
add dword [r1], 32
%if ARCH_X86_64
IDCT8_ADD_SSE_START r1, rsp
SWAP 1, 9
SWAP 2, 10
SWAP 3, 11
SWAP 5, 13
SWAP 6, 14
SWAP 7, 15
IDCT8_ADD_SSE_START r1+16, rsp+128
PERMUTE 1,9, 2,10, 3,11, 5,1, 6,2, 7,3, 9,13, 10,14, 11,15, 13,5, 14,6, 15,7
IDCT8_1D [rsp], [rsp+128]
SWAP 0, 8
SWAP 1, 9
SWAP 2, 10
SWAP 3, 11
SWAP 4, 12
SWAP 5, 13
SWAP 6, 14
SWAP 7, 15
IDCT8_1D [rsp+16], [rsp+144]
psrad m8, 6
psrad m0, 6
packssdw m8, m0
paddsw m8, [r0]
pxor m0, m0
mova [r1+ 0], m0
mova [r1+ 16], m0
mova [r1+ 32], m0
mova [r1+ 48], m0
mova [r1+ 64], m0
mova [r1+ 80], m0
mova [r1+ 96], m0
mova [r1+112], m0
mova [r1+128], m0
mova [r1+144], m0
mova [r1+160], m0
mova [r1+176], m0
mova [r1+192], m0
mova [r1+208], m0
mova [r1+224], m0
mova [r1+240], m0
CLIPW m8, m0, [pw_pixel_max]
mova [r0], m8
mova m8, [pw_pixel_max]
STORE_DIFF16 m9, m1, m0, m8, r0+r2
lea r0, [r0+r2*2]
STORE_DIFF16 m10, m2, m0, m8, r0
STORE_DIFF16 m11, m3, m0, m8, r0+r2
lea r0, [r0+r2*2]
STORE_DIFF16 m12, m4, m0, m8, r0
STORE_DIFF16 m13, m5, m0, m8, r0+r2
lea r0, [r0+r2*2]
STORE_DIFF16 m14, m6, m0, m8, r0
STORE_DIFF16 m15, m7, m0, m8, r0+r2
%else
IDCT8_ADD_SSE_START r1, rsp
IDCT8_ADD_SSE_START r1+16, rsp+128
lea r3, [r0+8]
IDCT8_ADD_SSE_END r0, rsp, r2
IDCT8_ADD_SSE_END r3, rsp+16, r2
mova [r1+ 0], m7
mova [r1+ 16], m7
mova [r1+ 32], m7
mova [r1+ 48], m7
mova [r1+ 64], m7
mova [r1+ 80], m7
mova [r1+ 96], m7
mova [r1+112], m7
mova [r1+128], m7
mova [r1+144], m7
mova [r1+160], m7
mova [r1+176], m7
mova [r1+192], m7
mova [r1+208], m7
mova [r1+224], m7
mova [r1+240], m7
%endif ; ARCH_X86_64
add rsp, pad
ret
%endmacro
INIT_XMM sse2
IDCT8_ADD
%if HAVE_AVX_EXTERNAL
INIT_XMM avx
IDCT8_ADD
%endif
;-----------------------------------------------------------------------------
; void ff_h264_idct8_add4_10(pixel **dst, const int *block_offset,
; int16_t *block, int stride,
; const uint8_t nnzc[6*8])
;-----------------------------------------------------------------------------
;;;;;;; NO FATE SAMPLES TRIGGER THIS
%macro IDCT8_ADD4_OP 2
cmp byte [r4+%2], 0
jz .skipblock%1
mov r0d, [r6+%1*4]
add r0, r5
call h264_idct8_add1_10 %+ SUFFIX
.skipblock%1:
%if %1<12
add r1, 256
%endif
%endmacro
%macro IDCT8_ADD4 0
cglobal h264_idct8_add4_10, 0,7,16
movsxdifnidn r3, r3d
%assign pad 16-gprsize-(stack_offset&15)
SUB rsp, pad
mov r5, r0mp
mov r6, r1mp
mov r1, r2mp
mov r2d, r3m
movifnidn r4, r4mp
IDCT8_ADD4_OP 0, 4+1*8
IDCT8_ADD4_OP 4, 6+1*8
IDCT8_ADD4_OP 8, 4+3*8
IDCT8_ADD4_OP 12, 6+3*8
ADD rsp, pad
RET
%endmacro ; IDCT8_ADD4
INIT_XMM sse2
IDCT8_ADD4
%if HAVE_AVX_EXTERNAL
INIT_XMM avx
IDCT8_ADD4
%endif

634
media/ffvpx/libavcodec/x86/h264_qpel.c Normal file
View file

@ -0,0 +1,634 @@
/*
* Copyright (c) 2004-2005 Michael Niedermayer, Loren Merritt
* Copyright (c) 2011 Daniel Kang
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "libavutil/attributes.h"
#include "libavutil/cpu.h"
#include "libavutil/x86/asm.h"
#include "libavutil/x86/cpu.h"
#include "libavcodec/h264dec.h"
#include "libavcodec/h264qpel.h"
#include "libavcodec/pixels.h"
#include "fpel.h"
#if HAVE_X86ASM
void ff_put_pixels4_l2_mmxext(uint8_t *dst, const uint8_t *src1, const uint8_t *src2,
int dstStride, int src1Stride, int h);
void ff_avg_pixels4_l2_mmxext(uint8_t *dst, const uint8_t *src1, const uint8_t *src2,
int dstStride, int src1Stride, int h);
void ff_put_pixels8_l2_mmxext(uint8_t *dst, const uint8_t *src1, const uint8_t *src2,
int dstStride, int src1Stride, int h);
void ff_avg_pixels8_l2_mmxext(uint8_t *dst, const uint8_t *src1, const uint8_t *src2,
int dstStride, int src1Stride, int h);
void ff_put_pixels16_l2_mmxext(uint8_t *dst, const uint8_t *src1, const uint8_t *src2,
int dstStride, int src1Stride, int h);
void ff_avg_pixels16_l2_mmxext(uint8_t *dst, const uint8_t *src1, const uint8_t *src2,
int dstStride, int src1Stride, int h);
#define ff_put_pixels8_l2_sse2 ff_put_pixels8_l2_mmxext
#define ff_avg_pixels8_l2_sse2 ff_avg_pixels8_l2_mmxext
#define ff_put_pixels16_l2_sse2 ff_put_pixels16_l2_mmxext
#define ff_avg_pixels16_l2_sse2 ff_avg_pixels16_l2_mmxext
#define ff_put_pixels16_mmxext ff_put_pixels16_mmx
#define ff_put_pixels8_mmxext ff_put_pixels8_mmx
#define ff_put_pixels4_mmxext ff_put_pixels4_mmx
#define DEF_QPEL(OPNAME)\
void ff_ ## OPNAME ## _h264_qpel4_h_lowpass_mmxext(uint8_t *dst, const uint8_t *src, int dstStride, int srcStride);\
void ff_ ## OPNAME ## _h264_qpel8_h_lowpass_mmxext(uint8_t *dst, const uint8_t *src, int dstStride, int srcStride);\
void ff_ ## OPNAME ## _h264_qpel8_h_lowpass_ssse3(uint8_t *dst, const uint8_t *src, int dstStride, int srcStride);\
void ff_ ## OPNAME ## _h264_qpel4_h_lowpass_l2_mmxext(uint8_t *dst, const uint8_t *src, const uint8_t *src2, int dstStride, int src2Stride);\
void ff_ ## OPNAME ## _h264_qpel8_h_lowpass_l2_mmxext(uint8_t *dst, const uint8_t *src, const uint8_t *src2, int dstStride, int src2Stride);\
void ff_ ## OPNAME ## _h264_qpel8_h_lowpass_l2_ssse3(uint8_t *dst, const uint8_t *src, const uint8_t *src2, int dstStride, int src2Stride);\
void ff_ ## OPNAME ## _h264_qpel4_v_lowpass_mmxext(uint8_t *dst, const uint8_t *src, int dstStride, int srcStride);\
void ff_ ## OPNAME ## _h264_qpel8or16_v_lowpass_op_mmxext(uint8_t *dst, const uint8_t *src, int dstStride, int srcStride, int h);\
void ff_ ## OPNAME ## _h264_qpel8or16_v_lowpass_sse2(uint8_t *dst, const uint8_t *src, int dstStride, int srcStride, int h);\
void ff_ ## OPNAME ## _h264_qpel4_hv_lowpass_v_mmxext(const uint8_t *src, int16_t *tmp, int srcStride);\
void ff_ ## OPNAME ## _h264_qpel4_hv_lowpass_h_mmxext(int16_t *tmp, uint8_t *dst, int dstStride);\
void ff_ ## OPNAME ## _h264_qpel8or16_hv1_lowpass_op_mmxext(const uint8_t *src, int16_t *tmp, int srcStride, int size);\
void ff_ ## OPNAME ## _h264_qpel8or16_hv1_lowpass_op_sse2(const uint8_t *src, int16_t *tmp, int srcStride, int size);\
void ff_ ## OPNAME ## _h264_qpel8or16_hv2_lowpass_op_mmxext(uint8_t *dst, int16_t *tmp, int dstStride, int unused, int h);\
void ff_ ## OPNAME ## _h264_qpel8or16_hv2_lowpass_ssse3(uint8_t *dst, int16_t *tmp, int dstStride, int tmpStride, int size);\
void ff_ ## OPNAME ## _pixels4_l2_shift5_mmxext(uint8_t *dst, const int16_t *src16, const uint8_t *src8, int dstStride, int src8Stride, int h);\
void ff_ ## OPNAME ## _pixels8_l2_shift5_mmxext(uint8_t *dst, const int16_t *src16, const uint8_t *src8, int dstStride, int src8Stride, int h);
DEF_QPEL(avg)
DEF_QPEL(put)
static av_always_inline void ff_put_h264_qpel8or16_hv1_lowpass_mmxext(int16_t *tmp, const uint8_t *src, int tmpStride, int srcStride, int size)
{
int w = (size + 8) >> 2;
src -= 2 * srcStride + 2;
while (w--) {
ff_put_h264_qpel8or16_hv1_lowpass_op_mmxext(src, tmp, srcStride, size);
tmp += 4;
src += 4;
}
}
#define QPEL_H264(OPNAME, OP, MMX)\
static av_always_inline void ff_ ## OPNAME ## h264_qpel4_hv_lowpass_ ## MMX(uint8_t *dst, int16_t *tmp, const uint8_t *src, int dstStride, int tmpStride, int srcStride){\
int w=3;\
src -= 2*srcStride+2;\
while(w--){\
ff_ ## OPNAME ## h264_qpel4_hv_lowpass_v_mmxext(src, tmp, srcStride);\
tmp += 4;\
src += 4;\
}\
tmp -= 3*4;\
ff_ ## OPNAME ## h264_qpel4_hv_lowpass_h_mmxext(tmp, dst, dstStride);\
}\
\
static av_always_inline void ff_ ## OPNAME ## h264_qpel8or16_v_lowpass_ ## MMX(uint8_t *dst, const uint8_t *src, int dstStride, int srcStride, int h){\
src -= 2*srcStride;\
ff_ ## OPNAME ## h264_qpel8or16_v_lowpass_op_mmxext(dst, src, dstStride, srcStride, h);\
src += 4;\
dst += 4;\
ff_ ## OPNAME ## h264_qpel8or16_v_lowpass_op_mmxext(dst, src, dstStride, srcStride, h);\
}\
static av_always_inline void ff_ ## OPNAME ## h264_qpel8or16_hv2_lowpass_ ## MMX(uint8_t *dst, int16_t *tmp, int dstStride, int tmpStride, int size){\
int w = size>>4;\
do{\
ff_ ## OPNAME ## h264_qpel8or16_hv2_lowpass_op_mmxext(dst, tmp, dstStride, 0, size);\
tmp += 8;\
dst += 8;\
}while(w--);\
}\
\
static av_always_inline void ff_ ## OPNAME ## h264_qpel8_v_lowpass_ ## MMX(uint8_t *dst, const uint8_t *src, int dstStride, int srcStride){\
ff_ ## OPNAME ## h264_qpel8or16_v_lowpass_ ## MMX(dst , src , dstStride, srcStride, 8);\
}\
static av_always_inline void ff_ ## OPNAME ## h264_qpel16_v_lowpass_ ## MMX(uint8_t *dst, const uint8_t *src, int dstStride, int srcStride){\
ff_ ## OPNAME ## h264_qpel8or16_v_lowpass_ ## MMX(dst , src , dstStride, srcStride, 16);\
ff_ ## OPNAME ## h264_qpel8or16_v_lowpass_ ## MMX(dst+8, src+8, dstStride, srcStride, 16);\
}\
\
static av_always_inline void ff_ ## OPNAME ## h264_qpel16_h_lowpass_ ## MMX(uint8_t *dst, const uint8_t *src, int dstStride, int srcStride){\
ff_ ## OPNAME ## h264_qpel8_h_lowpass_ ## MMX(dst , src , dstStride, srcStride);\
ff_ ## OPNAME ## h264_qpel8_h_lowpass_ ## MMX(dst+8, src+8, dstStride, srcStride);\
src += 8*srcStride;\
dst += 8*dstStride;\
ff_ ## OPNAME ## h264_qpel8_h_lowpass_ ## MMX(dst , src , dstStride, srcStride);\
ff_ ## OPNAME ## h264_qpel8_h_lowpass_ ## MMX(dst+8, src+8, dstStride, srcStride);\
}\
\
static av_always_inline void ff_ ## OPNAME ## h264_qpel16_h_lowpass_l2_ ## MMX(uint8_t *dst, const uint8_t *src, const uint8_t *src2, int dstStride, int src2Stride){\
ff_ ## OPNAME ## h264_qpel8_h_lowpass_l2_ ## MMX(dst , src , src2 , dstStride, src2Stride);\
ff_ ## OPNAME ## h264_qpel8_h_lowpass_l2_ ## MMX(dst+8, src+8, src2+8, dstStride, src2Stride);\
src += 8*dstStride;\
dst += 8*dstStride;\
src2 += 8*src2Stride;\
ff_ ## OPNAME ## h264_qpel8_h_lowpass_l2_ ## MMX(dst , src , src2 , dstStride, src2Stride);\
ff_ ## OPNAME ## h264_qpel8_h_lowpass_l2_ ## MMX(dst+8, src+8, src2+8, dstStride, src2Stride);\
}\
\
static av_always_inline void ff_ ## OPNAME ## h264_qpel8or16_hv_lowpass_ ## MMX(uint8_t *dst, int16_t *tmp, const uint8_t *src, int dstStride, int tmpStride, int srcStride, int size){\
ff_put_h264_qpel8or16_hv1_lowpass_ ## MMX(tmp, src, tmpStride, srcStride, size);\
ff_ ## OPNAME ## h264_qpel8or16_hv2_lowpass_ ## MMX(dst, tmp, dstStride, tmpStride, size);\
}\
static av_always_inline void ff_ ## OPNAME ## h264_qpel8_hv_lowpass_ ## MMX(uint8_t *dst, int16_t *tmp, const uint8_t *src, int dstStride, int tmpStride, int srcStride){\
ff_ ## OPNAME ## h264_qpel8or16_hv_lowpass_ ## MMX(dst , tmp , src , dstStride, tmpStride, srcStride, 8);\
}\
\
static av_always_inline void ff_ ## OPNAME ## h264_qpel16_hv_lowpass_ ## MMX(uint8_t *dst, int16_t *tmp, const uint8_t *src, int dstStride, int tmpStride, int srcStride){\
ff_ ## OPNAME ## h264_qpel8or16_hv_lowpass_ ## MMX(dst , tmp , src , dstStride, tmpStride, srcStride, 16);\
}\
\
static av_always_inline void ff_ ## OPNAME ## pixels16_l2_shift5_ ## MMX(uint8_t *dst, const int16_t *src16, const uint8_t *src8, int dstStride, int src8Stride, int h)\
{\
ff_ ## OPNAME ## pixels8_l2_shift5_ ## MMX(dst , src16 , src8 , dstStride, src8Stride, h);\
ff_ ## OPNAME ## pixels8_l2_shift5_ ## MMX(dst+8, src16+8, src8+8, dstStride, src8Stride, h);\
}\
#if ARCH_X86_64
#define QPEL_H264_H16_XMM(OPNAME, OP, MMX)\
void ff_avg_h264_qpel16_h_lowpass_l2_ssse3(uint8_t *dst, const uint8_t *src, const uint8_t *src2, int dstStride, int src2Stride);
void ff_put_h264_qpel16_h_lowpass_l2_ssse3(uint8_t *dst, const uint8_t *src, const uint8_t *src2, int dstStride, int src2Stride);
#else // ARCH_X86_64
#define QPEL_H264_H16_XMM(OPNAME, OP, MMX)\
static av_always_inline void ff_ ## OPNAME ## h264_qpel16_h_lowpass_l2_ ## MMX(uint8_t *dst, const uint8_t *src, const uint8_t *src2, int dstStride, int src2Stride){\
ff_ ## OPNAME ## h264_qpel8_h_lowpass_l2_ ## MMX(dst , src , src2 , dstStride, src2Stride);\
ff_ ## OPNAME ## h264_qpel8_h_lowpass_l2_ ## MMX(dst+8, src+8, src2+8, dstStride, src2Stride);\
src += 8*dstStride;\
dst += 8*dstStride;\
src2 += 8*src2Stride;\
ff_ ## OPNAME ## h264_qpel8_h_lowpass_l2_ ## MMX(dst , src , src2 , dstStride, src2Stride);\
ff_ ## OPNAME ## h264_qpel8_h_lowpass_l2_ ## MMX(dst+8, src+8, src2+8, dstStride, src2Stride);\
}
#endif // ARCH_X86_64
#define QPEL_H264_H_XMM(OPNAME, OP, MMX)\
QPEL_H264_H16_XMM(OPNAME, OP, MMX)\
static av_always_inline void ff_ ## OPNAME ## h264_qpel16_h_lowpass_ ## MMX(uint8_t *dst, const uint8_t *src, int dstStride, int srcStride){\
ff_ ## OPNAME ## h264_qpel8_h_lowpass_ ## MMX(dst , src , dstStride, srcStride);\
ff_ ## OPNAME ## h264_qpel8_h_lowpass_ ## MMX(dst+8, src+8, dstStride, srcStride);\
src += 8*srcStride;\
dst += 8*dstStride;\
ff_ ## OPNAME ## h264_qpel8_h_lowpass_ ## MMX(dst , src , dstStride, srcStride);\
ff_ ## OPNAME ## h264_qpel8_h_lowpass_ ## MMX(dst+8, src+8, dstStride, srcStride);\
}\
#define QPEL_H264_V_XMM(OPNAME, OP, MMX)\
static av_always_inline void ff_ ## OPNAME ## h264_qpel8_v_lowpass_ ## MMX(uint8_t *dst, const uint8_t *src, int dstStride, int srcStride){\
ff_ ## OPNAME ## h264_qpel8or16_v_lowpass_ ## MMX(dst , src , dstStride, srcStride, 8);\
}\
static av_always_inline void ff_ ## OPNAME ## h264_qpel16_v_lowpass_ ## MMX(uint8_t *dst, const uint8_t *src, int dstStride, int srcStride){\
ff_ ## OPNAME ## h264_qpel8or16_v_lowpass_ ## MMX(dst , src , dstStride, srcStride, 16);\
ff_ ## OPNAME ## h264_qpel8or16_v_lowpass_ ## MMX(dst+8, src+8, dstStride, srcStride, 16);\
}
static av_always_inline void put_h264_qpel8or16_hv1_lowpass_sse2(int16_t *tmp,
const uint8_t *src,
int tmpStride,
int srcStride,
int size)
{
int w = (size+8)>>3;
src -= 2*srcStride+2;
while(w--){
ff_put_h264_qpel8or16_hv1_lowpass_op_sse2(src, tmp, srcStride, size);
tmp += 8;
src += 8;
}
}
#define QPEL_H264_HV_XMM(OPNAME, OP, MMX)\
static av_always_inline void ff_ ## OPNAME ## h264_qpel8or16_hv_lowpass_ ## MMX(uint8_t *dst, int16_t *tmp, const uint8_t *src, int dstStride, int tmpStride, int srcStride, int size){\
put_h264_qpel8or16_hv1_lowpass_sse2(tmp, src, tmpStride, srcStride, size);\
ff_ ## OPNAME ## h264_qpel8or16_hv2_lowpass_ ## MMX(dst, tmp, dstStride, tmpStride, size);\
}\
static av_always_inline void ff_ ## OPNAME ## h264_qpel8_hv_lowpass_ ## MMX(uint8_t *dst, int16_t *tmp, const uint8_t *src, int dstStride, int tmpStride, int srcStride){\
ff_ ## OPNAME ## h264_qpel8or16_hv_lowpass_ ## MMX(dst, tmp, src, dstStride, tmpStride, srcStride, 8);\
}\
static av_always_inline void ff_ ## OPNAME ## h264_qpel16_hv_lowpass_ ## MMX(uint8_t *dst, int16_t *tmp, const uint8_t *src, int dstStride, int tmpStride, int srcStride){\
ff_ ## OPNAME ## h264_qpel8or16_hv_lowpass_ ## MMX(dst, tmp, src, dstStride, tmpStride, srcStride, 16);\
}\
#define ff_put_h264_qpel8_h_lowpass_l2_sse2 ff_put_h264_qpel8_h_lowpass_l2_mmxext
#define ff_avg_h264_qpel8_h_lowpass_l2_sse2 ff_avg_h264_qpel8_h_lowpass_l2_mmxext
#define ff_put_h264_qpel16_h_lowpass_l2_sse2 ff_put_h264_qpel16_h_lowpass_l2_mmxext
#define ff_avg_h264_qpel16_h_lowpass_l2_sse2 ff_avg_h264_qpel16_h_lowpass_l2_mmxext
#define ff_put_h264_qpel8_v_lowpass_ssse3 ff_put_h264_qpel8_v_lowpass_sse2
#define ff_avg_h264_qpel8_v_lowpass_ssse3 ff_avg_h264_qpel8_v_lowpass_sse2
#define ff_put_h264_qpel16_v_lowpass_ssse3 ff_put_h264_qpel16_v_lowpass_sse2
#define ff_avg_h264_qpel16_v_lowpass_ssse3 ff_avg_h264_qpel16_v_lowpass_sse2
#define ff_put_h264_qpel8or16_hv2_lowpass_sse2 ff_put_h264_qpel8or16_hv2_lowpass_mmxext
#define ff_avg_h264_qpel8or16_hv2_lowpass_sse2 ff_avg_h264_qpel8or16_hv2_lowpass_mmxext
#define H264_MC(OPNAME, SIZE, MMX, ALIGN) \
H264_MC_C(OPNAME, SIZE, MMX, ALIGN)\
H264_MC_V(OPNAME, SIZE, MMX, ALIGN)\
H264_MC_H(OPNAME, SIZE, MMX, ALIGN)\
H264_MC_HV(OPNAME, SIZE, MMX, ALIGN)\
static void put_h264_qpel16_mc00_sse2 (uint8_t *dst, const uint8_t *src,
ptrdiff_t stride)
{
ff_put_pixels16_sse2(dst, src, stride, 16);
}
static void avg_h264_qpel16_mc00_sse2 (uint8_t *dst, const uint8_t *src,
ptrdiff_t stride)
{
ff_avg_pixels16_sse2(dst, src, stride, 16);
}
#define put_h264_qpel8_mc00_sse2 put_h264_qpel8_mc00_mmxext
#define avg_h264_qpel8_mc00_sse2 avg_h264_qpel8_mc00_mmxext
#define H264_MC_C(OPNAME, SIZE, MMX, ALIGN) \
static void OPNAME ## h264_qpel ## SIZE ## _mc00_ ## MMX (uint8_t *dst, const uint8_t *src, ptrdiff_t stride)\
{\
ff_ ## OPNAME ## pixels ## SIZE ## _ ## MMX(dst, src, stride, SIZE);\
}\
#define H264_MC_H(OPNAME, SIZE, MMX, ALIGN) \
static void OPNAME ## h264_qpel ## SIZE ## _mc10_ ## MMX(uint8_t *dst, const uint8_t *src, ptrdiff_t stride)\
{\
ff_ ## OPNAME ## h264_qpel ## SIZE ## _h_lowpass_l2_ ## MMX(dst, src, src, stride, stride);\
}\
\
static void OPNAME ## h264_qpel ## SIZE ## _mc20_ ## MMX(uint8_t *dst, const uint8_t *src, ptrdiff_t stride)\
{\
ff_ ## OPNAME ## h264_qpel ## SIZE ## _h_lowpass_ ## MMX(dst, src, stride, stride);\
}\
\
static void OPNAME ## h264_qpel ## SIZE ## _mc30_ ## MMX(uint8_t *dst, const uint8_t *src, ptrdiff_t stride)\
{\
ff_ ## OPNAME ## h264_qpel ## SIZE ## _h_lowpass_l2_ ## MMX(dst, src, src+1, stride, stride);\
}\
#define H264_MC_V(OPNAME, SIZE, MMX, ALIGN) \
static void OPNAME ## h264_qpel ## SIZE ## _mc01_ ## MMX(uint8_t *dst, const uint8_t *src, ptrdiff_t stride)\
{\
LOCAL_ALIGNED(ALIGN, uint8_t, temp, [SIZE*SIZE]);\
ff_put_h264_qpel ## SIZE ## _v_lowpass_ ## MMX(temp, src, SIZE, stride);\
ff_ ## OPNAME ## pixels ## SIZE ## _l2_ ## MMX(dst, src, temp, stride, stride, SIZE);\
}\
\
static void OPNAME ## h264_qpel ## SIZE ## _mc02_ ## MMX(uint8_t *dst, const uint8_t *src, ptrdiff_t stride)\
{\
ff_ ## OPNAME ## h264_qpel ## SIZE ## _v_lowpass_ ## MMX(dst, src, stride, stride);\
}\
\
static void OPNAME ## h264_qpel ## SIZE ## _mc03_ ## MMX(uint8_t *dst, const uint8_t *src, ptrdiff_t stride)\
{\
LOCAL_ALIGNED(ALIGN, uint8_t, temp, [SIZE*SIZE]);\
ff_put_h264_qpel ## SIZE ## _v_lowpass_ ## MMX(temp, src, SIZE, stride);\
ff_ ## OPNAME ## pixels ## SIZE ## _l2_ ## MMX(dst, src+stride, temp, stride, stride, SIZE);\
}\
#define H264_MC_HV(OPNAME, SIZE, MMX, ALIGN) \
static void OPNAME ## h264_qpel ## SIZE ## _mc11_ ## MMX(uint8_t *dst, const uint8_t *src, ptrdiff_t stride)\
{\
LOCAL_ALIGNED(ALIGN, uint8_t, temp, [SIZE*SIZE]);\
ff_put_h264_qpel ## SIZE ## _v_lowpass_ ## MMX(temp, src, SIZE, stride);\
ff_ ## OPNAME ## h264_qpel ## SIZE ## _h_lowpass_l2_ ## MMX(dst, src, temp, stride, SIZE);\
}\
\
static void OPNAME ## h264_qpel ## SIZE ## _mc31_ ## MMX(uint8_t *dst, const uint8_t *src, ptrdiff_t stride)\
{\
LOCAL_ALIGNED(ALIGN, uint8_t, temp, [SIZE*SIZE]);\
ff_put_h264_qpel ## SIZE ## _v_lowpass_ ## MMX(temp, src+1, SIZE, stride);\
ff_ ## OPNAME ## h264_qpel ## SIZE ## _h_lowpass_l2_ ## MMX(dst, src, temp, stride, SIZE);\
}\
\
static void OPNAME ## h264_qpel ## SIZE ## _mc13_ ## MMX(uint8_t *dst, const uint8_t *src, ptrdiff_t stride)\
{\
LOCAL_ALIGNED(ALIGN, uint8_t, temp, [SIZE*SIZE]);\
ff_put_h264_qpel ## SIZE ## _v_lowpass_ ## MMX(temp, src, SIZE, stride);\
ff_ ## OPNAME ## h264_qpel ## SIZE ## _h_lowpass_l2_ ## MMX(dst, src+stride, temp, stride, SIZE);\
}\
\
static void OPNAME ## h264_qpel ## SIZE ## _mc33_ ## MMX(uint8_t *dst, const uint8_t *src, ptrdiff_t stride)\
{\
LOCAL_ALIGNED(ALIGN, uint8_t, temp, [SIZE*SIZE]);\
ff_put_h264_qpel ## SIZE ## _v_lowpass_ ## MMX(temp, src+1, SIZE, stride);\
ff_ ## OPNAME ## h264_qpel ## SIZE ## _h_lowpass_l2_ ## MMX(dst, src+stride, temp, stride, SIZE);\
}\
\
static void OPNAME ## h264_qpel ## SIZE ## _mc22_ ## MMX(uint8_t *dst, const uint8_t *src, ptrdiff_t stride)\
{\
LOCAL_ALIGNED(ALIGN, uint16_t, temp, [SIZE*(SIZE<8?12:24)]);\
ff_ ## OPNAME ## h264_qpel ## SIZE ## _hv_lowpass_ ## MMX(dst, temp, src, stride, SIZE, stride);\
}\
\
static void OPNAME ## h264_qpel ## SIZE ## _mc21_ ## MMX(uint8_t *dst, const uint8_t *src, ptrdiff_t stride)\
{\
LOCAL_ALIGNED(ALIGN, uint8_t, temp, [SIZE*(SIZE<8?12:24)*2 + SIZE*SIZE]);\
uint8_t * const halfHV= temp;\
int16_t * const halfV= (int16_t*)(temp + SIZE*SIZE);\
av_assert2(((int)temp & 7) == 0);\
ff_put_h264_qpel ## SIZE ## _hv_lowpass_ ## MMX(halfHV, halfV, src, SIZE, SIZE, stride);\
ff_ ## OPNAME ## h264_qpel ## SIZE ## _h_lowpass_l2_ ## MMX(dst, src, halfHV, stride, SIZE);\
}\
\
static void OPNAME ## h264_qpel ## SIZE ## _mc23_ ## MMX(uint8_t *dst, const uint8_t *src, ptrdiff_t stride)\
{\
LOCAL_ALIGNED(ALIGN, uint8_t, temp, [SIZE*(SIZE<8?12:24)*2 + SIZE*SIZE]);\
uint8_t * const halfHV= temp;\
int16_t * const halfV= (int16_t*)(temp + SIZE*SIZE);\
av_assert2(((int)temp & 7) == 0);\
ff_put_h264_qpel ## SIZE ## _hv_lowpass_ ## MMX(halfHV, halfV, src, SIZE, SIZE, stride);\
ff_ ## OPNAME ## h264_qpel ## SIZE ## _h_lowpass_l2_ ## MMX(dst, src+stride, halfHV, stride, SIZE);\
}\
\
static void OPNAME ## h264_qpel ## SIZE ## _mc12_ ## MMX(uint8_t *dst, const uint8_t *src, ptrdiff_t stride)\
{\
LOCAL_ALIGNED(ALIGN, uint8_t, temp, [SIZE*(SIZE<8?12:24)*2 + SIZE*SIZE]);\
uint8_t * const halfHV= temp;\
int16_t * const halfV= (int16_t*)(temp + SIZE*SIZE);\
av_assert2(((int)temp & 7) == 0);\
ff_put_h264_qpel ## SIZE ## _hv_lowpass_ ## MMX(halfHV, halfV, src, SIZE, SIZE, stride);\
ff_ ## OPNAME ## pixels ## SIZE ## _l2_shift5_mmxext(dst, halfV+2, halfHV, stride, SIZE, SIZE);\
}\
\
static void OPNAME ## h264_qpel ## SIZE ## _mc32_ ## MMX(uint8_t *dst, const uint8_t *src, ptrdiff_t stride)\
{\
LOCAL_ALIGNED(ALIGN, uint8_t, temp, [SIZE*(SIZE<8?12:24)*2 + SIZE*SIZE]);\
uint8_t * const halfHV= temp;\
int16_t * const halfV= (int16_t*)(temp + SIZE*SIZE);\
av_assert2(((int)temp & 7) == 0);\
ff_put_h264_qpel ## SIZE ## _hv_lowpass_ ## MMX(halfHV, halfV, src, SIZE, SIZE, stride);\
ff_ ## OPNAME ## pixels ## SIZE ## _l2_shift5_mmxext(dst, halfV+3, halfHV, stride, SIZE, SIZE);\
}\
#define H264_MC_4816(MMX)\
H264_MC(put_, 4, MMX, 8)\
H264_MC(put_, 8, MMX, 8)\
H264_MC(put_, 16,MMX, 8)\
H264_MC(avg_, 4, MMX, 8)\
H264_MC(avg_, 8, MMX, 8)\
H264_MC(avg_, 16,MMX, 8)\
#define H264_MC_816(QPEL, XMM)\
QPEL(put_, 8, XMM, 16)\
QPEL(put_, 16,XMM, 16)\
QPEL(avg_, 8, XMM, 16)\
QPEL(avg_, 16,XMM, 16)\
QPEL_H264(put_, PUT_OP, mmxext)
QPEL_H264(avg_, AVG_MMXEXT_OP, mmxext)
QPEL_H264_V_XMM(put_, PUT_OP, sse2)
QPEL_H264_V_XMM(avg_,AVG_MMXEXT_OP, sse2)
QPEL_H264_HV_XMM(put_, PUT_OP, sse2)
QPEL_H264_HV_XMM(avg_,AVG_MMXEXT_OP, sse2)
QPEL_H264_H_XMM(put_, PUT_OP, ssse3)
QPEL_H264_H_XMM(avg_,AVG_MMXEXT_OP, ssse3)
QPEL_H264_HV_XMM(put_, PUT_OP, ssse3)
QPEL_H264_HV_XMM(avg_,AVG_MMXEXT_OP, ssse3)
H264_MC_4816(mmxext)
H264_MC_816(H264_MC_V, sse2)
H264_MC_816(H264_MC_HV, sse2)
H264_MC_816(H264_MC_H, ssse3)
H264_MC_816(H264_MC_HV, ssse3)
//10bit
#define LUMA_MC_OP(OP, NUM, DEPTH, TYPE, OPT) \
void ff_ ## OP ## _h264_qpel ## NUM ## _ ## TYPE ## _ ## DEPTH ## _ ## OPT \
(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
#define LUMA_MC_ALL(DEPTH, TYPE, OPT) \
LUMA_MC_OP(put, 4, DEPTH, TYPE, OPT) \
LUMA_MC_OP(avg, 4, DEPTH, TYPE, OPT) \
LUMA_MC_OP(put, 8, DEPTH, TYPE, OPT) \
LUMA_MC_OP(avg, 8, DEPTH, TYPE, OPT) \
LUMA_MC_OP(put, 16, DEPTH, TYPE, OPT) \
LUMA_MC_OP(avg, 16, DEPTH, TYPE, OPT)
#define LUMA_MC_816(DEPTH, TYPE, OPT) \
LUMA_MC_OP(put, 8, DEPTH, TYPE, OPT) \
LUMA_MC_OP(avg, 8, DEPTH, TYPE, OPT) \
LUMA_MC_OP(put, 16, DEPTH, TYPE, OPT) \
LUMA_MC_OP(avg, 16, DEPTH, TYPE, OPT)
LUMA_MC_ALL(10, mc00, mmxext)
LUMA_MC_ALL(10, mc10, mmxext)
LUMA_MC_ALL(10, mc20, mmxext)
LUMA_MC_ALL(10, mc30, mmxext)
LUMA_MC_ALL(10, mc01, mmxext)
LUMA_MC_ALL(10, mc11, mmxext)
LUMA_MC_ALL(10, mc21, mmxext)
LUMA_MC_ALL(10, mc31, mmxext)
LUMA_MC_ALL(10, mc02, mmxext)
LUMA_MC_ALL(10, mc12, mmxext)
LUMA_MC_ALL(10, mc22, mmxext)
LUMA_MC_ALL(10, mc32, mmxext)
LUMA_MC_ALL(10, mc03, mmxext)
LUMA_MC_ALL(10, mc13, mmxext)
LUMA_MC_ALL(10, mc23, mmxext)
LUMA_MC_ALL(10, mc33, mmxext)
LUMA_MC_816(10, mc00, sse2)
LUMA_MC_816(10, mc10, sse2)
LUMA_MC_816(10, mc10, sse2_cache64)
LUMA_MC_816(10, mc10, ssse3_cache64)
LUMA_MC_816(10, mc20, sse2)
LUMA_MC_816(10, mc20, sse2_cache64)
LUMA_MC_816(10, mc20, ssse3_cache64)
LUMA_MC_816(10, mc30, sse2)
LUMA_MC_816(10, mc30, sse2_cache64)
LUMA_MC_816(10, mc30, ssse3_cache64)
LUMA_MC_816(10, mc01, sse2)
LUMA_MC_816(10, mc11, sse2)
LUMA_MC_816(10, mc21, sse2)
LUMA_MC_816(10, mc31, sse2)
LUMA_MC_816(10, mc02, sse2)
LUMA_MC_816(10, mc12, sse2)
LUMA_MC_816(10, mc22, sse2)
LUMA_MC_816(10, mc32, sse2)
LUMA_MC_816(10, mc03, sse2)
LUMA_MC_816(10, mc13, sse2)
LUMA_MC_816(10, mc23, sse2)
LUMA_MC_816(10, mc33, sse2)
#define QPEL16_OPMC(OP, MC, MMX)\
void ff_ ## OP ## _h264_qpel16_ ## MC ## _10_ ## MMX(uint8_t *dst, const uint8_t *src, ptrdiff_t stride){\
ff_ ## OP ## _h264_qpel8_ ## MC ## _10_ ## MMX(dst , src , stride);\
ff_ ## OP ## _h264_qpel8_ ## MC ## _10_ ## MMX(dst+16, src+16, stride);\
src += 8*stride;\
dst += 8*stride;\
ff_ ## OP ## _h264_qpel8_ ## MC ## _10_ ## MMX(dst , src , stride);\
ff_ ## OP ## _h264_qpel8_ ## MC ## _10_ ## MMX(dst+16, src+16, stride);\
}
#define QPEL16_OP(MC, MMX)\
QPEL16_OPMC(put, MC, MMX)\
QPEL16_OPMC(avg, MC, MMX)
#define QPEL16(MMX)\
QPEL16_OP(mc00, MMX)\
QPEL16_OP(mc01, MMX)\
QPEL16_OP(mc02, MMX)\
QPEL16_OP(mc03, MMX)\
QPEL16_OP(mc10, MMX)\
QPEL16_OP(mc11, MMX)\
QPEL16_OP(mc12, MMX)\
QPEL16_OP(mc13, MMX)\
QPEL16_OP(mc20, MMX)\
QPEL16_OP(mc21, MMX)\
QPEL16_OP(mc22, MMX)\
QPEL16_OP(mc23, MMX)\
QPEL16_OP(mc30, MMX)\
QPEL16_OP(mc31, MMX)\
QPEL16_OP(mc32, MMX)\
QPEL16_OP(mc33, MMX)
#if ARCH_X86_32 // ARCH_X86_64 implies SSE2+
QPEL16(mmxext)
#endif
#endif /* HAVE_X86ASM */
#define SET_QPEL_FUNCS(PFX, IDX, SIZE, CPU, PREFIX) \
do { \
c->PFX ## _pixels_tab[IDX][ 0] = PREFIX ## PFX ## SIZE ## _mc00_ ## CPU; \
c->PFX ## _pixels_tab[IDX][ 1] = PREFIX ## PFX ## SIZE ## _mc10_ ## CPU; \
c->PFX ## _pixels_tab[IDX][ 2] = PREFIX ## PFX ## SIZE ## _mc20_ ## CPU; \
c->PFX ## _pixels_tab[IDX][ 3] = PREFIX ## PFX ## SIZE ## _mc30_ ## CPU; \
c->PFX ## _pixels_tab[IDX][ 4] = PREFIX ## PFX ## SIZE ## _mc01_ ## CPU; \
c->PFX ## _pixels_tab[IDX][ 5] = PREFIX ## PFX ## SIZE ## _mc11_ ## CPU; \
c->PFX ## _pixels_tab[IDX][ 6] = PREFIX ## PFX ## SIZE ## _mc21_ ## CPU; \
c->PFX ## _pixels_tab[IDX][ 7] = PREFIX ## PFX ## SIZE ## _mc31_ ## CPU; \
c->PFX ## _pixels_tab[IDX][ 8] = PREFIX ## PFX ## SIZE ## _mc02_ ## CPU; \
c->PFX ## _pixels_tab[IDX][ 9] = PREFIX ## PFX ## SIZE ## _mc12_ ## CPU; \
c->PFX ## _pixels_tab[IDX][10] = PREFIX ## PFX ## SIZE ## _mc22_ ## CPU; \
c->PFX ## _pixels_tab[IDX][11] = PREFIX ## PFX ## SIZE ## _mc32_ ## CPU; \
c->PFX ## _pixels_tab[IDX][12] = PREFIX ## PFX ## SIZE ## _mc03_ ## CPU; \
c->PFX ## _pixels_tab[IDX][13] = PREFIX ## PFX ## SIZE ## _mc13_ ## CPU; \
c->PFX ## _pixels_tab[IDX][14] = PREFIX ## PFX ## SIZE ## _mc23_ ## CPU; \
c->PFX ## _pixels_tab[IDX][15] = PREFIX ## PFX ## SIZE ## _mc33_ ## CPU; \
} while (0)
#define H264_QPEL_FUNCS(x, y, CPU) \
do { \
c->put_h264_qpel_pixels_tab[0][x + y * 4] = put_h264_qpel16_mc ## x ## y ## _ ## CPU; \
c->put_h264_qpel_pixels_tab[1][x + y * 4] = put_h264_qpel8_mc ## x ## y ## _ ## CPU; \
c->avg_h264_qpel_pixels_tab[0][x + y * 4] = avg_h264_qpel16_mc ## x ## y ## _ ## CPU; \
c->avg_h264_qpel_pixels_tab[1][x + y * 4] = avg_h264_qpel8_mc ## x ## y ## _ ## CPU; \
} while (0)
#define H264_QPEL_FUNCS_10(x, y, CPU) \
do { \
c->put_h264_qpel_pixels_tab[0][x + y * 4] = ff_put_h264_qpel16_mc ## x ## y ## _10_ ## CPU; \
c->put_h264_qpel_pixels_tab[1][x + y * 4] = ff_put_h264_qpel8_mc ## x ## y ## _10_ ## CPU; \
c->avg_h264_qpel_pixels_tab[0][x + y * 4] = ff_avg_h264_qpel16_mc ## x ## y ## _10_ ## CPU; \
c->avg_h264_qpel_pixels_tab[1][x + y * 4] = ff_avg_h264_qpel8_mc ## x ## y ## _10_ ## CPU; \
} while (0)
av_cold void ff_h264qpel_init_x86(H264QpelContext *c, int bit_depth)
{
#if HAVE_X86ASM
int high_bit_depth = bit_depth > 8;
int cpu_flags = av_get_cpu_flags();
if (EXTERNAL_MMXEXT(cpu_flags)) {
if (!high_bit_depth) {
SET_QPEL_FUNCS(put_h264_qpel, 0, 16, mmxext, );
SET_QPEL_FUNCS(put_h264_qpel, 1, 8, mmxext, );
SET_QPEL_FUNCS(put_h264_qpel, 2, 4, mmxext, );
SET_QPEL_FUNCS(avg_h264_qpel, 0, 16, mmxext, );
SET_QPEL_FUNCS(avg_h264_qpel, 1, 8, mmxext, );
SET_QPEL_FUNCS(avg_h264_qpel, 2, 4, mmxext, );
} else if (bit_depth == 10) {
#if ARCH_X86_32
SET_QPEL_FUNCS(avg_h264_qpel, 0, 16, 10_mmxext, ff_);
SET_QPEL_FUNCS(put_h264_qpel, 0, 16, 10_mmxext, ff_);
SET_QPEL_FUNCS(put_h264_qpel, 1, 8, 10_mmxext, ff_);
SET_QPEL_FUNCS(avg_h264_qpel, 1, 8, 10_mmxext, ff_);
#endif
SET_QPEL_FUNCS(put_h264_qpel, 2, 4, 10_mmxext, ff_);
SET_QPEL_FUNCS(avg_h264_qpel, 2, 4, 10_mmxext, ff_);
}
}
if (EXTERNAL_SSE2(cpu_flags)) {
if (!high_bit_depth) {
H264_QPEL_FUNCS(0, 1, sse2);
H264_QPEL_FUNCS(0, 2, sse2);
H264_QPEL_FUNCS(0, 3, sse2);
H264_QPEL_FUNCS(1, 1, sse2);
H264_QPEL_FUNCS(1, 2, sse2);
H264_QPEL_FUNCS(1, 3, sse2);
H264_QPEL_FUNCS(2, 1, sse2);
H264_QPEL_FUNCS(2, 2, sse2);
H264_QPEL_FUNCS(2, 3, sse2);
H264_QPEL_FUNCS(3, 1, sse2);
H264_QPEL_FUNCS(3, 2, sse2);
H264_QPEL_FUNCS(3, 3, sse2);
}
if (bit_depth == 10) {
SET_QPEL_FUNCS(put_h264_qpel, 0, 16, 10_sse2, ff_);
SET_QPEL_FUNCS(put_h264_qpel, 1, 8, 10_sse2, ff_);
SET_QPEL_FUNCS(avg_h264_qpel, 0, 16, 10_sse2, ff_);
SET_QPEL_FUNCS(avg_h264_qpel, 1, 8, 10_sse2, ff_);
H264_QPEL_FUNCS_10(1, 0, sse2_cache64);
H264_QPEL_FUNCS_10(2, 0, sse2_cache64);
H264_QPEL_FUNCS_10(3, 0, sse2_cache64);
}
}
if (EXTERNAL_SSE2_FAST(cpu_flags)) {
if (!high_bit_depth) {
H264_QPEL_FUNCS(0, 0, sse2);
}
}
if (EXTERNAL_SSSE3(cpu_flags)) {
if (!high_bit_depth) {
H264_QPEL_FUNCS(1, 0, ssse3);
H264_QPEL_FUNCS(1, 1, ssse3);
H264_QPEL_FUNCS(1, 2, ssse3);
H264_QPEL_FUNCS(1, 3, ssse3);
H264_QPEL_FUNCS(2, 0, ssse3);
H264_QPEL_FUNCS(2, 1, ssse3);
H264_QPEL_FUNCS(2, 2, ssse3);
H264_QPEL_FUNCS(2, 3, ssse3);
H264_QPEL_FUNCS(3, 0, ssse3);
H264_QPEL_FUNCS(3, 1, ssse3);
H264_QPEL_FUNCS(3, 2, ssse3);
H264_QPEL_FUNCS(3, 3, ssse3);
}
if (bit_depth == 10) {
H264_QPEL_FUNCS_10(1, 0, ssse3_cache64);
H264_QPEL_FUNCS_10(2, 0, ssse3_cache64);
H264_QPEL_FUNCS_10(3, 0, ssse3_cache64);
}
}
if (EXTERNAL_AVX(cpu_flags)) {
/* AVX implies 64 byte cache lines without the need to avoid unaligned
* memory accesses that cross the boundary between two cache lines.
* TODO: Port X264_CPU_CACHELINE_32/64 detection from x264 to avoid
* having to treat SSE2 functions with such properties as AVX. */
if (bit_depth == 10) {
H264_QPEL_FUNCS_10(1, 0, sse2);
H264_QPEL_FUNCS_10(2, 0, sse2);
H264_QPEL_FUNCS_10(3, 0, sse2);
}
}
#endif
}

884
media/ffvpx/libavcodec/x86/h264_qpel_10bit.asm Normal file
View file

@ -0,0 +1,884 @@
;*****************************************************************************
;* MMX/SSE2/AVX-optimized 10-bit H.264 qpel code
;*****************************************************************************
;* Copyright (C) 2011 x264 project
;*
;* Authors: Daniel Kang <daniel.d.kang@gmail.com>
;*
;* This file is part of FFmpeg.
;*
;* FFmpeg is free software; you can redistribute it and/or
;* modify it under the terms of the GNU Lesser General Public
;* License as published by the Free Software Foundation; either
;* version 2.1 of the License, or (at your option) any later version.
;*
;* FFmpeg is distributed in the hope that it will be useful,
;* but WITHOUT ANY WARRANTY; without even the implied warranty of
;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
;* Lesser General Public License for more details.
;*
;* You should have received a copy of the GNU Lesser General Public
;* License along with FFmpeg; if not, write to the Free Software
;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
;******************************************************************************
%include "libavutil/x86/x86util.asm"
SECTION_RODATA 32
cextern pd_65535
cextern pw_1023
%define pw_pixel_max pw_1023
cextern pw_16
cextern pw_1
cextern pb_0
pad10: times 8 dw 10*1023
pad20: times 8 dw 20*1023
pad30: times 8 dw 30*1023
depad: times 4 dd 32*20*1023 + 512
depad2: times 8 dw 20*1023 + 16*1022 + 16
unpad: times 8 dw 16*1022/32 ; needs to be mod 16
tap1: times 4 dw 1, -5
tap2: times 4 dw 20, 20
tap3: times 4 dw -5, 1
SECTION .text
%macro AVG_MOV 2
pavgw %2, %1
mova %1, %2
%endmacro
%macro ADDW 3
%if mmsize == 8
paddw %1, %2
%else
movu %3, %2
paddw %1, %3
%endif
%endmacro
%macro FILT_H 4
paddw %1, %4
psubw %1, %2 ; a-b
psraw %1, 2 ; (a-b)/4
psubw %1, %2 ; (a-b)/4-b
paddw %1, %3 ; (a-b)/4-b+c
psraw %1, 2 ; ((a-b)/4-b+c)/4
paddw %1, %3 ; ((a-b)/4-b+c)/4+c = (a-5*b+20*c)/16
%endmacro
%macro PRELOAD_V 0
lea r3, [r2*3]
sub r1, r3
movu m0, [r1+r2]
movu m1, [r1+r2*2]
add r1, r3
movu m2, [r1]
movu m3, [r1+r2]
movu m4, [r1+r2*2]
add r1, r3
%endmacro
%macro FILT_V 8
movu %6, [r1]
paddw %1, %6
mova %7, %2
paddw %7, %5
mova %8, %3
paddw %8, %4
FILT_H %1, %7, %8, [pw_16]
psraw %1, 1
CLIPW %1, [pb_0], [pw_pixel_max]
%endmacro
%macro MC 1
%define OP_MOV mova
INIT_MMX mmxext
%1 put, 4
INIT_XMM sse2
%1 put, 8
%define OP_MOV AVG_MOV
INIT_MMX mmxext
%1 avg, 4
INIT_XMM sse2
%1 avg, 8
%endmacro
%macro MCAxA_OP 7
%if ARCH_X86_32
cglobal %1_h264_qpel%4_%2_10, %5,%6,%7
call stub_%1_h264_qpel%3_%2_10 %+ SUFFIX
mov r0, r0m
mov r1, r1m
add r0, %3*2
add r1, %3*2
call stub_%1_h264_qpel%3_%2_10 %+ SUFFIX
mov r0, r0m
mov r1, r1m
lea r0, [r0+r2*%3]
lea r1, [r1+r2*%3]
call stub_%1_h264_qpel%3_%2_10 %+ SUFFIX
mov r0, r0m
mov r1, r1m
lea r0, [r0+r2*%3+%3*2]
lea r1, [r1+r2*%3+%3*2]
call stub_%1_h264_qpel%3_%2_10 %+ SUFFIX
RET
%else ; ARCH_X86_64
cglobal %1_h264_qpel%4_%2_10, %5,%6 + 2,%7
mov r%6, r0
%assign p1 %6+1
mov r %+ p1, r1
call stub_%1_h264_qpel%3_%2_10 %+ SUFFIX
lea r0, [r%6+%3*2]
lea r1, [r %+ p1+%3*2]
call stub_%1_h264_qpel%3_%2_10 %+ SUFFIX
lea r0, [r%6+r2*%3]
lea r1, [r %+ p1+r2*%3]
call stub_%1_h264_qpel%3_%2_10 %+ SUFFIX
lea r0, [r%6+r2*%3+%3*2]
lea r1, [r %+ p1+r2*%3+%3*2]
%if UNIX64 == 0 ; fall through to function
call stub_%1_h264_qpel%3_%2_10 %+ SUFFIX
RET
%endif
%endif
%endmacro
;cpu, put/avg, mc, 4/8, ...
%macro cglobal_mc 6
%assign i %3*2
%if ARCH_X86_32 || cpuflag(sse2)
MCAxA_OP %1, %2, %3, i, %4,%5,%6
%endif
cglobal %1_h264_qpel%3_%2_10, %4,%5,%6
%if UNIX64 == 0 ; no prologue or epilogue for UNIX64
call stub_%1_h264_qpel%3_%2_10 %+ SUFFIX
RET
%endif
stub_%1_h264_qpel%3_%2_10 %+ SUFFIX:
%endmacro
;-----------------------------------------------------------------------------
; void ff_h264_qpel_mc00(uint8_t *dst, uint8_t *src, int stride)
;-----------------------------------------------------------------------------
%macro COPY4 0
movu m0, [r1 ]
OP_MOV [r0 ], m0
movu m0, [r1+r2 ]
OP_MOV [r0+r2 ], m0
movu m0, [r1+r2*2]
OP_MOV [r0+r2*2], m0
movu m0, [r1+r3 ]
OP_MOV [r0+r3 ], m0
%endmacro
%macro MC00 1
INIT_MMX mmxext
cglobal_mc %1, mc00, 4, 3,4,0
lea r3, [r2*3]
COPY4
ret
INIT_XMM sse2
cglobal %1_h264_qpel8_mc00_10, 3,4
lea r3, [r2*3]
COPY4
lea r0, [r0+r2*4]
lea r1, [r1+r2*4]
COPY4
RET
cglobal %1_h264_qpel16_mc00_10, 3,4
mov r3d, 8
.loop:
movu m0, [r1 ]
movu m1, [r1 +16]
OP_MOV [r0 ], m0
OP_MOV [r0 +16], m1
movu m0, [r1+r2 ]
movu m1, [r1+r2+16]
OP_MOV [r0+r2 ], m0
OP_MOV [r0+r2+16], m1
lea r0, [r0+r2*2]
lea r1, [r1+r2*2]
dec r3d
jg .loop
REP_RET
%endmacro
%define OP_MOV mova
MC00 put
%define OP_MOV AVG_MOV
MC00 avg
;-----------------------------------------------------------------------------
; void ff_h264_qpel_mc20(uint8_t *dst, uint8_t *src, int stride)
;-----------------------------------------------------------------------------
%macro MC_CACHE 1
%define OP_MOV mova
INIT_MMX mmxext
%1 put, 4
INIT_XMM sse2, cache64
%1 put, 8
INIT_XMM ssse3, cache64
%1 put, 8
INIT_XMM sse2
%1 put, 8
%define OP_MOV AVG_MOV
INIT_MMX mmxext
%1 avg, 4
INIT_XMM sse2, cache64
%1 avg, 8
INIT_XMM ssse3, cache64
%1 avg, 8
INIT_XMM sse2
%1 avg, 8
%endmacro
%macro MC20 2
cglobal_mc %1, mc20, %2, 3,4,9
mov r3d, %2
mova m1, [pw_pixel_max]
%if num_mmregs > 8
mova m8, [pw_16]
%define p16 m8
%else
%define p16 [pw_16]
%endif
.nextrow:
%if %0 == 4
movu m2, [r1-4]
movu m3, [r1-2]
movu m4, [r1+0]
ADDW m2, [r1+6], m5
ADDW m3, [r1+4], m5
ADDW m4, [r1+2], m5
%else ; movu is slow on these processors
%if mmsize==16
movu m2, [r1-4]
movu m0, [r1+6]
mova m6, m0
psrldq m0, 6
paddw m6, m2
PALIGNR m3, m0, m2, 2, m5
PALIGNR m7, m0, m2, 8, m5
paddw m3, m7
PALIGNR m4, m0, m2, 4, m5
PALIGNR m7, m0, m2, 6, m5
paddw m4, m7
SWAP 2, 6
%else
movu m2, [r1-4]
movu m6, [r1+4]
PALIGNR m3, m6, m2, 2, m5
paddw m3, m6
PALIGNR m4, m6, m2, 4, m5
PALIGNR m7, m6, m2, 6, m5
paddw m4, m7
paddw m2, [r1+6]
%endif
%endif
FILT_H m2, m3, m4, p16
psraw m2, 1
pxor m0, m0
CLIPW m2, m0, m1
OP_MOV [r0], m2
add r0, r2
add r1, r2
dec r3d
jg .nextrow
rep ret
%endmacro
MC_CACHE MC20
;-----------------------------------------------------------------------------
; void ff_h264_qpel_mc30(uint8_t *dst, uint8_t *src, int stride)
;-----------------------------------------------------------------------------
%macro MC30 2
cglobal_mc %1, mc30, %2, 3,5,9
lea r4, [r1+2]
jmp stub_%1_h264_qpel%2_mc10_10 %+ SUFFIX %+ .body
%endmacro
MC_CACHE MC30
;-----------------------------------------------------------------------------
; void ff_h264_qpel_mc10(uint8_t *dst, uint8_t *src, int stride)
;-----------------------------------------------------------------------------
%macro MC10 2
cglobal_mc %1, mc10, %2, 3,5,9
mov r4, r1
.body:
mov r3d, %2
mova m1, [pw_pixel_max]
%if num_mmregs > 8
mova m8, [pw_16]
%define p16 m8
%else
%define p16 [pw_16]
%endif
.nextrow:
%if %0 == 4
movu m2, [r1-4]
movu m3, [r1-2]
movu m4, [r1+0]
ADDW m2, [r1+6], m5
ADDW m3, [r1+4], m5
ADDW m4, [r1+2], m5
%else ; movu is slow on these processors
%if mmsize==16
movu m2, [r1-4]
movu m0, [r1+6]
mova m6, m0
psrldq m0, 6
paddw m6, m2
PALIGNR m3, m0, m2, 2, m5
PALIGNR m7, m0, m2, 8, m5
paddw m3, m7
PALIGNR m4, m0, m2, 4, m5
PALIGNR m7, m0, m2, 6, m5
paddw m4, m7
SWAP 2, 6
%else
movu m2, [r1-4]
movu m6, [r1+4]
PALIGNR m3, m6, m2, 2, m5
paddw m3, m6
PALIGNR m4, m6, m2, 4, m5
PALIGNR m7, m6, m2, 6, m5
paddw m4, m7
paddw m2, [r1+6]
%endif
%endif
FILT_H m2, m3, m4, p16
psraw m2, 1
pxor m0, m0
CLIPW m2, m0, m1
movu m3, [r4]
pavgw m2, m3
OP_MOV [r0], m2
add r0, r2
add r1, r2
add r4, r2
dec r3d
jg .nextrow
rep ret
%endmacro
MC_CACHE MC10
;-----------------------------------------------------------------------------
; void ff_h264_qpel_mc02(uint8_t *dst, uint8_t *src, int stride)
;-----------------------------------------------------------------------------
%macro V_FILT 10
v_filt%9_%10_10:
add r4, r2
.no_addr4:
FILT_V m0, m1, m2, m3, m4, m5, m6, m7
add r1, r2
add r0, r2
ret
%endmacro
INIT_MMX mmxext
RESET_MM_PERMUTATION
%assign i 0
%rep 4
V_FILT m0, m1, m2, m3, m4, m5, m6, m7, 4, i
SWAP 0,1,2,3,4,5
%assign i i+1
%endrep
INIT_XMM sse2
RESET_MM_PERMUTATION
%assign i 0
%rep 6
V_FILT m0, m1, m2, m3, m4, m5, m6, m7, 8, i
SWAP 0,1,2,3,4,5
%assign i i+1
%endrep
%macro MC02 2
cglobal_mc %1, mc02, %2, 3,4,8
PRELOAD_V
sub r0, r2
%assign j 0
%rep %2
%assign i (j % 6)
call v_filt%2_ %+ i %+ _10.no_addr4
OP_MOV [r0], m0
SWAP 0,1,2,3,4,5
%assign j j+1
%endrep
ret
%endmacro
MC MC02
;-----------------------------------------------------------------------------
; void ff_h264_qpel_mc01(uint8_t *dst, uint8_t *src, int stride)
;-----------------------------------------------------------------------------
%macro MC01 2
cglobal_mc %1, mc01, %2, 3,5,8
mov r4, r1
.body:
PRELOAD_V
sub r4, r2
sub r0, r2
%assign j 0
%rep %2
%assign i (j % 6)
call v_filt%2_ %+ i %+ _10
movu m7, [r4]
pavgw m0, m7
OP_MOV [r0], m0
SWAP 0,1,2,3,4,5
%assign j j+1
%endrep
ret
%endmacro
MC MC01
;-----------------------------------------------------------------------------
; void ff_h264_qpel_mc03(uint8_t *dst, uint8_t *src, int stride)
;-----------------------------------------------------------------------------
%macro MC03 2
cglobal_mc %1, mc03, %2, 3,5,8
lea r4, [r1+r2]
jmp stub_%1_h264_qpel%2_mc01_10 %+ SUFFIX %+ .body
%endmacro
MC MC03
;-----------------------------------------------------------------------------
; void ff_h264_qpel_mc11(uint8_t *dst, uint8_t *src, int stride)
;-----------------------------------------------------------------------------
%macro H_FILT_AVG 2-3
h_filt%1_%2_10:
;FILT_H with fewer registers and averaged with the FILT_V result
;m6,m7 are tmp registers, m0 is the FILT_V result, the rest are to be used next in the next iteration
;unfortunately I need three registers, so m5 will have to be re-read from memory
movu m5, [r4-4]
ADDW m5, [r4+6], m7
movu m6, [r4-2]
ADDW m6, [r4+4], m7
paddw m5, [pw_16]
psubw m5, m6 ; a-b
psraw m5, 2 ; (a-b)/4
psubw m5, m6 ; (a-b)/4-b
movu m6, [r4+0]
ADDW m6, [r4+2], m7
paddw m5, m6 ; (a-b)/4-b+c
psraw m5, 2 ; ((a-b)/4-b+c)/4
paddw m5, m6 ; ((a-b)/4-b+c)/4+c = (a-5*b+20*c)/16
psraw m5, 1
CLIPW m5, [pb_0], [pw_pixel_max]
;avg FILT_V, FILT_H
pavgw m0, m5
%if %0!=4
movu m5, [r1+r5]
%endif
ret
%endmacro
INIT_MMX mmxext
RESET_MM_PERMUTATION
%assign i 0
%rep 3
H_FILT_AVG 4, i
SWAP 0,1,2,3,4,5
%assign i i+1
%endrep
H_FILT_AVG 4, i, 0
INIT_XMM sse2
RESET_MM_PERMUTATION
%assign i 0
%rep 6
%if i==1
H_FILT_AVG 8, i, 0
%else
H_FILT_AVG 8, i
%endif
SWAP 0,1,2,3,4,5
%assign i i+1
%endrep
%macro MC11 2
; this REALLY needs x86_64
cglobal_mc %1, mc11, %2, 3,6,8
mov r4, r1
.body:
PRELOAD_V
sub r0, r2
sub r4, r2
mov r5, r2
neg r5
%assign j 0
%rep %2
%assign i (j % 6)
call v_filt%2_ %+ i %+ _10
call h_filt%2_ %+ i %+ _10
%if %2==8 && i==1
movu m5, [r1+r5]
%endif
OP_MOV [r0], m0
SWAP 0,1,2,3,4,5
%assign j j+1
%endrep
ret
%endmacro
MC MC11
;-----------------------------------------------------------------------------
; void ff_h264_qpel_mc31(uint8_t *dst, uint8_t *src, int stride)
;-----------------------------------------------------------------------------
%macro MC31 2
cglobal_mc %1, mc31, %2, 3,6,8
mov r4, r1
add r1, 2
jmp stub_%1_h264_qpel%2_mc11_10 %+ SUFFIX %+ .body
%endmacro
MC MC31
;-----------------------------------------------------------------------------
; void ff_h264_qpel_mc13(uint8_t *dst, uint8_t *src, int stride)
;-----------------------------------------------------------------------------
%macro MC13 2
cglobal_mc %1, mc13, %2, 3,7,12
lea r4, [r1+r2]
jmp stub_%1_h264_qpel%2_mc11_10 %+ SUFFIX %+ .body
%endmacro
MC MC13
;-----------------------------------------------------------------------------
; void ff_h264_qpel_mc33(uint8_t *dst, uint8_t *src, int stride)
;-----------------------------------------------------------------------------
%macro MC33 2
cglobal_mc %1, mc33, %2, 3,6,8
lea r4, [r1+r2]
add r1, 2
jmp stub_%1_h264_qpel%2_mc11_10 %+ SUFFIX %+ .body
%endmacro
MC MC33
;-----------------------------------------------------------------------------
; void ff_h264_qpel_mc22(uint8_t *dst, uint8_t *src, int stride)
;-----------------------------------------------------------------------------
%macro FILT_H2 3
psubw %1, %2 ; a-b
psubw %2, %3 ; b-c
psllw %2, 2
psubw %1, %2 ; a-5*b+4*c
psllw %3, 4
paddw %1, %3 ; a-5*b+20*c
%endmacro
%macro FILT_VNRD 8
movu %6, [r1]
paddw %1, %6
mova %7, %2
paddw %7, %5
mova %8, %3
paddw %8, %4
FILT_H2 %1, %7, %8
%endmacro
%macro HV 1
%if mmsize==16
%define PAD 12
%define COUNT 2
%else
%define PAD 4
%define COUNT 3
%endif
put_hv%1_10:
neg r2 ; This actually saves instructions
lea r1, [r1+r2*2-mmsize+PAD]
lea r4, [rsp+PAD+gprsize]
mov r3d, COUNT
.v_loop:
movu m0, [r1]
sub r1, r2
movu m1, [r1]
sub r1, r2
movu m2, [r1]
sub r1, r2
movu m3, [r1]
sub r1, r2
movu m4, [r1]
sub r1, r2
%assign i 0
%rep %1-1
FILT_VNRD m0, m1, m2, m3, m4, m5, m6, m7
psubw m0, [pad20]
movu [r4+i*mmsize*3], m0
sub r1, r2
SWAP 0,1,2,3,4,5
%assign i i+1
%endrep
FILT_VNRD m0, m1, m2, m3, m4, m5, m6, m7
psubw m0, [pad20]
movu [r4+i*mmsize*3], m0
add r4, mmsize
lea r1, [r1+r2*8+mmsize]
%if %1==8
lea r1, [r1+r2*4]
%endif
dec r3d
jg .v_loop
neg r2
ret
%endmacro
INIT_MMX mmxext
HV 4
INIT_XMM sse2
HV 8
%macro H_LOOP 1
%if num_mmregs > 8
%define s1 m8
%define s2 m9
%define s3 m10
%define d1 m11
%else
%define s1 [tap1]
%define s2 [tap2]
%define s3 [tap3]
%define d1 [depad]
%endif
h%1_loop_op:
movu m1, [r1+mmsize-4]
movu m2, [r1+mmsize-2]
mova m3, [r1+mmsize+0]
movu m4, [r1+mmsize+2]
movu m5, [r1+mmsize+4]
movu m6, [r1+mmsize+6]
%if num_mmregs > 8
pmaddwd m1, s1
pmaddwd m2, s1
pmaddwd m3, s2
pmaddwd m4, s2
pmaddwd m5, s3
pmaddwd m6, s3
paddd m1, d1
paddd m2, d1
%else
mova m0, s1
pmaddwd m1, m0
pmaddwd m2, m0
mova m0, s2
pmaddwd m3, m0
pmaddwd m4, m0
mova m0, s3
pmaddwd m5, m0
pmaddwd m6, m0
mova m0, d1
paddd m1, m0
paddd m2, m0
%endif
paddd m3, m5
paddd m4, m6
paddd m1, m3
paddd m2, m4
psrad m1, 10
psrad m2, 10
pslld m2, 16
pand m1, [pd_65535]
por m1, m2
%if num_mmregs <= 8
pxor m0, m0
%endif
CLIPW m1, m0, m7
add r1, mmsize*3
ret
%endmacro
INIT_MMX mmxext
H_LOOP 4
INIT_XMM sse2
H_LOOP 8
%macro MC22 2
cglobal_mc %1, mc22, %2, 3,7,12
%define PAD mmsize*8*4*2 ; SIZE*16*4*sizeof(pixel)
mov r6, rsp ; backup stack pointer
and rsp, ~(mmsize-1) ; align stack
sub rsp, PAD
call put_hv%2_10
mov r3d, %2
mova m7, [pw_pixel_max]
%if num_mmregs > 8
pxor m0, m0
mova m8, [tap1]
mova m9, [tap2]
mova m10, [tap3]
mova m11, [depad]
%endif
mov r1, rsp
.h_loop:
call h%2_loop_op
OP_MOV [r0], m1
add r0, r2
dec r3d
jg .h_loop
mov rsp, r6 ; restore stack pointer
ret
%endmacro
MC MC22
;-----------------------------------------------------------------------------
; void ff_h264_qpel_mc12(uint8_t *dst, uint8_t *src, int stride)
;-----------------------------------------------------------------------------
%macro MC12 2
cglobal_mc %1, mc12, %2, 3,7,12
%define PAD mmsize*8*4*2 ; SIZE*16*4*sizeof(pixel)
mov r6, rsp ; backup stack pointer
and rsp, ~(mmsize-1) ; align stack
sub rsp, PAD
call put_hv%2_10
xor r4d, r4d
.body:
mov r3d, %2
pxor m0, m0
mova m7, [pw_pixel_max]
%if num_mmregs > 8
mova m8, [tap1]
mova m9, [tap2]
mova m10, [tap3]
mova m11, [depad]
%endif
mov r1, rsp
.h_loop:
call h%2_loop_op
movu m3, [r1+r4-2*mmsize] ; movu needed for mc32, etc
paddw m3, [depad2]
psrlw m3, 5
psubw m3, [unpad]
CLIPW m3, m0, m7
pavgw m1, m3
OP_MOV [r0], m1
add r0, r2
dec r3d
jg .h_loop
mov rsp, r6 ; restore stack pointer
ret
%endmacro
MC MC12
;-----------------------------------------------------------------------------
; void ff_h264_qpel_mc32(uint8_t *dst, uint8_t *src, int stride)
;-----------------------------------------------------------------------------
%macro MC32 2
cglobal_mc %1, mc32, %2, 3,7,12
%define PAD mmsize*8*3*2 ; SIZE*16*4*sizeof(pixel)
mov r6, rsp ; backup stack pointer
and rsp, ~(mmsize-1) ; align stack
sub rsp, PAD
call put_hv%2_10
mov r4d, 2 ; sizeof(pixel)
jmp stub_%1_h264_qpel%2_mc12_10 %+ SUFFIX %+ .body
%endmacro
MC MC32
;-----------------------------------------------------------------------------
; void ff_h264_qpel_mc21(uint8_t *dst, uint8_t *src, int stride)
;-----------------------------------------------------------------------------
%macro H_NRD 1
put_h%1_10:
add rsp, gprsize
mov r3d, %1
xor r4d, r4d
mova m6, [pad20]
.nextrow:
movu m2, [r5-4]
movu m3, [r5-2]
movu m4, [r5+0]
ADDW m2, [r5+6], m5
ADDW m3, [r5+4], m5
ADDW m4, [r5+2], m5
FILT_H2 m2, m3, m4
psubw m2, m6
mova [rsp+r4], m2
add r4d, mmsize*3
add r5, r2
dec r3d
jg .nextrow
sub rsp, gprsize
ret
%endmacro
INIT_MMX mmxext
H_NRD 4
INIT_XMM sse2
H_NRD 8
%macro MC21 2
cglobal_mc %1, mc21, %2, 3,7,12
mov r5, r1
.body:
%define PAD mmsize*8*3*2 ; SIZE*16*4*sizeof(pixel)
mov r6, rsp ; backup stack pointer
and rsp, ~(mmsize-1) ; align stack
sub rsp, PAD
call put_h%2_10
sub rsp, PAD
call put_hv%2_10
mov r4d, PAD-mmsize ; H buffer
jmp stub_%1_h264_qpel%2_mc12_10 %+ SUFFIX %+ .body
%endmacro
MC MC21
;-----------------------------------------------------------------------------
; void ff_h264_qpel_mc23(uint8_t *dst, uint8_t *src, int stride)
;-----------------------------------------------------------------------------
%macro MC23 2
cglobal_mc %1, mc23, %2, 3,7,12
lea r5, [r1+r2]
jmp stub_%1_h264_qpel%2_mc21_10 %+ SUFFIX %+ .body
%endmacro
MC MC23

862
media/ffvpx/libavcodec/x86/h264_qpel_8bit.asm Normal file
View file

@ -0,0 +1,862 @@
;*****************************************************************************
;* MMX/SSE2/SSSE3-optimized H.264 QPEL code
;*****************************************************************************
;* Copyright (c) 2004-2005 Michael Niedermayer, Loren Merritt
;* Copyright (C) 2012 Daniel Kang
;*
;* Authors: Daniel Kang <daniel.d.kang@gmail.com>
;*
;* This file is part of FFmpeg.
;*
;* FFmpeg is free software; you can redistribute it and/or
;* modify it under the terms of the GNU Lesser General Public
;* License as published by the Free Software Foundation; either
;* version 2.1 of the License, or (at your option) any later version.
;*
;* FFmpeg is distributed in the hope that it will be useful,
;* but WITHOUT ANY WARRANTY; without even the implied warranty of
;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
;* Lesser General Public License for more details.
;*
;* You should have received a copy of the GNU Lesser General Public
;* License along with FFmpeg; if not, write to the Free Software
;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
;******************************************************************************
%include "libavutil/x86/x86util.asm"
SECTION_RODATA 32
cextern pw_16
cextern pw_5
cextern pb_0
SECTION .text
%macro op_avgh 3
movh %3, %2
pavgb %1, %3
movh %2, %1
%endmacro
%macro op_avg 2-3
pavgb %1, %2
mova %2, %1
%endmacro
%macro op_puth 2-3
movh %2, %1
%endmacro
%macro op_put 2-3
mova %2, %1
%endmacro
%macro QPEL4_H_LOWPASS_OP 1
cglobal %1_h264_qpel4_h_lowpass, 4,5 ; dst, src, dstStride, srcStride
movsxdifnidn r2, r2d
movsxdifnidn r3, r3d
pxor m7, m7
mova m4, [pw_5]
mova m5, [pw_16]
mov r4d, 4
.loop:
movh m1, [r1-1]
movh m2, [r1+0]
movh m3, [r1+1]
movh m0, [r1+2]
punpcklbw m1, m7
punpcklbw m2, m7
punpcklbw m3, m7
punpcklbw m0, m7
paddw m1, m0
paddw m2, m3
movh m0, [r1-2]
movh m3, [r1+3]
punpcklbw m0, m7
punpcklbw m3, m7
paddw m0, m3
psllw m2, 2
psubw m2, m1
pmullw m2, m4
paddw m0, m5
paddw m0, m2
psraw m0, 5
packuswb m0, m0
op_%1h m0, [r0], m6
add r0, r2
add r1, r3
dec r4d
jg .loop
REP_RET
%endmacro
INIT_MMX mmxext
QPEL4_H_LOWPASS_OP put
QPEL4_H_LOWPASS_OP avg
%macro QPEL8_H_LOWPASS_OP 1
cglobal %1_h264_qpel8_h_lowpass, 4,5 ; dst, src, dstStride, srcStride
movsxdifnidn r2, r2d
movsxdifnidn r3, r3d
mov r4d, 8
pxor m7, m7
mova m6, [pw_5]
.loop:
mova m0, [r1]
mova m2, [r1+1]
mova m1, m0
mova m3, m2
punpcklbw m0, m7
punpckhbw m1, m7
punpcklbw m2, m7
punpckhbw m3, m7
paddw m0, m2
paddw m1, m3
psllw m0, 2
psllw m1, 2
mova m2, [r1-1]
mova m4, [r1+2]
mova m3, m2
mova m5, m4
punpcklbw m2, m7
punpckhbw m3, m7
punpcklbw m4, m7
punpckhbw m5, m7
paddw m2, m4
paddw m5, m3
psubw m0, m2
psubw m1, m5
pmullw m0, m6
pmullw m1, m6
movd m2, [r1-2]
movd m5, [r1+7]
punpcklbw m2, m7
punpcklbw m5, m7
paddw m2, m3
paddw m4, m5
mova m5, [pw_16]
paddw m2, m5
paddw m4, m5
paddw m0, m2
paddw m1, m4
psraw m0, 5
psraw m1, 5
packuswb m0, m1
op_%1 m0, [r0], m4
add r0, r2
add r1, r3
dec r4d
jg .loop
REP_RET
%endmacro
INIT_MMX mmxext
QPEL8_H_LOWPASS_OP put
QPEL8_H_LOWPASS_OP avg
%macro QPEL8_H_LOWPASS_OP_XMM 1
cglobal %1_h264_qpel8_h_lowpass, 4,5,8 ; dst, src, dstStride, srcStride
movsxdifnidn r2, r2d
movsxdifnidn r3, r3d
mov r4d, 8
pxor m7, m7
mova m6, [pw_5]
.loop:
movu m1, [r1-2]
mova m0, m1
punpckhbw m1, m7
punpcklbw m0, m7
mova m2, m1
mova m3, m1
mova m4, m1
mova m5, m1
palignr m4, m0, 2
palignr m3, m0, 4
palignr m2, m0, 6
palignr m1, m0, 8
palignr m5, m0, 10
paddw m0, m5
paddw m2, m3
paddw m1, m4
psllw m2, 2
psubw m2, m1
paddw m0, [pw_16]
pmullw m2, m6
paddw m2, m0
psraw m2, 5
packuswb m2, m2
op_%1h m2, [r0], m4
add r1, r3
add r0, r2
dec r4d
jne .loop
REP_RET
%endmacro
INIT_XMM ssse3
QPEL8_H_LOWPASS_OP_XMM put
QPEL8_H_LOWPASS_OP_XMM avg
%macro QPEL4_H_LOWPASS_L2_OP 1
cglobal %1_h264_qpel4_h_lowpass_l2, 5,6 ; dst, src, src2, dstStride, srcStride
movsxdifnidn r3, r3d
movsxdifnidn r4, r4d
pxor m7, m7
mova m4, [pw_5]
mova m5, [pw_16]
mov r5d, 4
.loop:
movh m1, [r1-1]
movh m2, [r1+0]
movh m3, [r1+1]
movh m0, [r1+2]
punpcklbw m1, m7
punpcklbw m2, m7
punpcklbw m3, m7
punpcklbw m0, m7
paddw m1, m0
paddw m2, m3
movh m0, [r1-2]
movh m3, [r1+3]
punpcklbw m0, m7
punpcklbw m3, m7
paddw m0, m3
psllw m2, 2
psubw m2, m1
pmullw m2, m4
paddw m0, m5
paddw m0, m2
movh m3, [r2]
psraw m0, 5
packuswb m0, m0
pavgb m0, m3
op_%1h m0, [r0], m6
add r0, r3
add r1, r3
add r2, r4
dec r5d
jg .loop
REP_RET
%endmacro
INIT_MMX mmxext
QPEL4_H_LOWPASS_L2_OP put
QPEL4_H_LOWPASS_L2_OP avg
%macro QPEL8_H_LOWPASS_L2_OP 1
cglobal %1_h264_qpel8_h_lowpass_l2, 5,6 ; dst, src, src2, dstStride, srcStride
movsxdifnidn r3, r3d
movsxdifnidn r4, r4d
mov r5d, 8
pxor m7, m7
mova m6, [pw_5]
.loop:
mova m0, [r1]
mova m2, [r1+1]
mova m1, m0
mova m3, m2
punpcklbw m0, m7
punpckhbw m1, m7
punpcklbw m2, m7
punpckhbw m3, m7
paddw m0, m2
paddw m1, m3
psllw m0, 2
psllw m1, 2
mova m2, [r1-1]
mova m4, [r1+2]
mova m3, m2
mova m5, m4
punpcklbw m2, m7
punpckhbw m3, m7
punpcklbw m4, m7
punpckhbw m5, m7
paddw m2, m4
paddw m5, m3
psubw m0, m2
psubw m1, m5
pmullw m0, m6
pmullw m1, m6
movd m2, [r1-2]
movd m5, [r1+7]
punpcklbw m2, m7
punpcklbw m5, m7
paddw m2, m3
paddw m4, m5
mova m5, [pw_16]
paddw m2, m5
paddw m4, m5
paddw m0, m2
paddw m1, m4
psraw m0, 5
psraw m1, 5
mova m4, [r2]
packuswb m0, m1
pavgb m0, m4
op_%1 m0, [r0], m4
add r0, r3
add r1, r3
add r2, r4
dec r5d
jg .loop
REP_RET
%endmacro
INIT_MMX mmxext
QPEL8_H_LOWPASS_L2_OP put
QPEL8_H_LOWPASS_L2_OP avg
%macro QPEL8_H_LOWPASS_L2_OP_XMM 1
cglobal %1_h264_qpel8_h_lowpass_l2, 5,6,8 ; dst, src, src2, dstStride, src2Stride
movsxdifnidn r3, r3d
movsxdifnidn r4, r4d
mov r5d, 8
pxor m7, m7
mova m6, [pw_5]
.loop:
lddqu m1, [r1-2]
mova m0, m1
punpckhbw m1, m7
punpcklbw m0, m7
mova m2, m1
mova m3, m1
mova m4, m1
mova m5, m1
palignr m4, m0, 2
palignr m3, m0, 4
palignr m2, m0, 6
palignr m1, m0, 8
palignr m5, m0, 10
paddw m0, m5
paddw m2, m3
paddw m1, m4
psllw m2, 2
movh m3, [r2]
psubw m2, m1
paddw m0, [pw_16]
pmullw m2, m6
paddw m2, m0
psraw m2, 5
packuswb m2, m2
pavgb m2, m3
op_%1h m2, [r0], m4
add r1, r3
add r0, r3
add r2, r4
dec r5d
jg .loop
REP_RET
%endmacro
INIT_XMM ssse3
QPEL8_H_LOWPASS_L2_OP_XMM put
QPEL8_H_LOWPASS_L2_OP_XMM avg
; All functions that call this are required to have function arguments of
; dst, src, dstStride, srcStride
%macro FILT_V 1
mova m6, m2
movh m5, [r1]
paddw m6, m3
psllw m6, 2
psubw m6, m1
psubw m6, m4
punpcklbw m5, m7
pmullw m6, [pw_5]
paddw m0, [pw_16]
add r1, r3
paddw m0, m5
paddw m6, m0
psraw m6, 5
packuswb m6, m6
op_%1h m6, [r0], m0 ; 1
add r0, r2
SWAP 0, 1, 2, 3, 4, 5
%endmacro
%macro QPEL4_V_LOWPASS_OP 1
cglobal %1_h264_qpel4_v_lowpass, 4,4 ; dst, src, dstStride, srcStride
movsxdifnidn r2, r2d
movsxdifnidn r3, r3d
sub r1, r3
sub r1, r3
pxor m7, m7
movh m0, [r1]
movh m1, [r1+r3]
lea r1, [r1+2*r3]
movh m2, [r1]
movh m3, [r1+r3]
lea r1, [r1+2*r3]
movh m4, [r1]
add r1, r3
punpcklbw m0, m7
punpcklbw m1, m7
punpcklbw m2, m7
punpcklbw m3, m7
punpcklbw m4, m7
FILT_V %1
FILT_V %1
FILT_V %1
FILT_V %1
RET
%endmacro
INIT_MMX mmxext
QPEL4_V_LOWPASS_OP put
QPEL4_V_LOWPASS_OP avg
%macro QPEL8OR16_V_LOWPASS_OP 1
%if cpuflag(sse2)
cglobal %1_h264_qpel8or16_v_lowpass, 5,5,8 ; dst, src, dstStride, srcStride, h
movsxdifnidn r2, r2d
movsxdifnidn r3, r3d
sub r1, r3
sub r1, r3
%else
cglobal %1_h264_qpel8or16_v_lowpass_op, 5,5,8 ; dst, src, dstStride, srcStride, h
movsxdifnidn r2, r2d
movsxdifnidn r3, r3d
%endif
pxor m7, m7
movh m0, [r1]
movh m1, [r1+r3]
lea r1, [r1+2*r3]
movh m2, [r1]
movh m3, [r1+r3]
lea r1, [r1+2*r3]
movh m4, [r1]
add r1, r3
punpcklbw m0, m7
punpcklbw m1, m7
punpcklbw m2, m7
punpcklbw m3, m7
punpcklbw m4, m7
FILT_V %1
FILT_V %1
FILT_V %1
FILT_V %1
FILT_V %1
FILT_V %1
FILT_V %1
FILT_V %1
cmp r4d, 16
jne .end
FILT_V %1
FILT_V %1
FILT_V %1
FILT_V %1
FILT_V %1
FILT_V %1
FILT_V %1
FILT_V %1
.end:
REP_RET
%endmacro
INIT_MMX mmxext
QPEL8OR16_V_LOWPASS_OP put
QPEL8OR16_V_LOWPASS_OP avg
INIT_XMM sse2
QPEL8OR16_V_LOWPASS_OP put
QPEL8OR16_V_LOWPASS_OP avg
; All functions that use this are required to have args:
; src, tmp, srcSize
%macro FILT_HV 1 ; offset
mova m6, m2
movh m5, [r0]
paddw m6, m3
psllw m6, 2
paddw m0, [pw_16]
psubw m6, m1
psubw m6, m4
punpcklbw m5, m7
pmullw m6, [pw_5]
paddw m0, m5
add r0, r2
paddw m6, m0
mova [r1+%1], m6
SWAP 0, 1, 2, 3, 4, 5
%endmacro
%macro QPEL4_HV1_LOWPASS_OP 1
cglobal %1_h264_qpel4_hv_lowpass_v, 3,3 ; src, tmp, srcStride
movsxdifnidn r2, r2d
pxor m7, m7
movh m0, [r0]
movh m1, [r0+r2]
lea r0, [r0+2*r2]
movh m2, [r0]
movh m3, [r0+r2]
lea r0, [r0+2*r2]
movh m4, [r0]
add r0, r2
punpcklbw m0, m7
punpcklbw m1, m7
punpcklbw m2, m7
punpcklbw m3, m7
punpcklbw m4, m7
FILT_HV 0*24
FILT_HV 1*24
FILT_HV 2*24
FILT_HV 3*24
RET
cglobal %1_h264_qpel4_hv_lowpass_h, 3,4 ; tmp, dst, dstStride
movsxdifnidn r2, r2d
mov r3d, 4
.loop:
mova m0, [r0]
paddw m0, [r0+10]
mova m1, [r0+2]
paddw m1, [r0+8]
mova m2, [r0+4]
paddw m2, [r0+6]
psubw m0, m1
psraw m0, 2
psubw m0, m1
paddsw m0, m2
psraw m0, 2
paddw m0, m2
psraw m0, 6
packuswb m0, m0
op_%1h m0, [r1], m7
add r0, 24
add r1, r2
dec r3d
jnz .loop
REP_RET
%endmacro
INIT_MMX mmxext
QPEL4_HV1_LOWPASS_OP put
QPEL4_HV1_LOWPASS_OP avg
%macro QPEL8OR16_HV1_LOWPASS_OP 1
cglobal %1_h264_qpel8or16_hv1_lowpass_op, 4,4,8 ; src, tmp, srcStride, size
movsxdifnidn r2, r2d
pxor m7, m7
movh m0, [r0]
movh m1, [r0+r2]
lea r0, [r0+2*r2]
movh m2, [r0]
movh m3, [r0+r2]
lea r0, [r0+2*r2]
movh m4, [r0]
add r0, r2
punpcklbw m0, m7
punpcklbw m1, m7
punpcklbw m2, m7
punpcklbw m3, m7
punpcklbw m4, m7
FILT_HV 0*48
FILT_HV 1*48
FILT_HV 2*48
FILT_HV 3*48
FILT_HV 4*48
FILT_HV 5*48
FILT_HV 6*48
FILT_HV 7*48
cmp r3d, 16
jne .end
FILT_HV 8*48
FILT_HV 9*48
FILT_HV 10*48
FILT_HV 11*48
FILT_HV 12*48
FILT_HV 13*48
FILT_HV 14*48
FILT_HV 15*48
.end:
REP_RET
%endmacro
INIT_MMX mmxext
QPEL8OR16_HV1_LOWPASS_OP put
QPEL8OR16_HV1_LOWPASS_OP avg
INIT_XMM sse2
QPEL8OR16_HV1_LOWPASS_OP put
%macro QPEL8OR16_HV2_LOWPASS_OP 1
; unused is to match ssse3 and mmxext args
cglobal %1_h264_qpel8or16_hv2_lowpass_op, 5,5 ; dst, tmp, dstStride, unused, h
movsxdifnidn r2, r2d
.loop:
mova m0, [r1]
mova m3, [r1+8]
mova m1, [r1+2]
mova m4, [r1+10]
paddw m0, m4
paddw m1, m3
paddw m3, [r1+18]
paddw m4, [r1+16]
mova m2, [r1+4]
mova m5, [r1+12]
paddw m2, [r1+6]
paddw m5, [r1+14]
psubw m0, m1
psubw m3, m4
psraw m0, 2
psraw m3, 2
psubw m0, m1
psubw m3, m4
paddsw m0, m2
paddsw m3, m5
psraw m0, 2
psraw m3, 2
paddw m0, m2
paddw m3, m5
psraw m0, 6
psraw m3, 6
packuswb m0, m3
op_%1 m0, [r0], m7
add r1, 48
add r0, r2
dec r4d
jne .loop
REP_RET
%endmacro
INIT_MMX mmxext
QPEL8OR16_HV2_LOWPASS_OP put
QPEL8OR16_HV2_LOWPASS_OP avg
%macro QPEL8OR16_HV2_LOWPASS_OP_XMM 1
cglobal %1_h264_qpel8or16_hv2_lowpass, 5,5,8 ; dst, tmp, dstStride, tmpStride, size
movsxdifnidn r2, r2d
movsxdifnidn r3, r3d
cmp r4d, 16
je .op16
.loop8:
mova m1, [r1+16]
mova m0, [r1]
mova m2, m1
mova m3, m1
mova m4, m1
mova m5, m1
palignr m5, m0, 10
palignr m4, m0, 8
palignr m3, m0, 6
palignr m2, m0, 4
palignr m1, m0, 2
paddw m0, m5
paddw m1, m4
paddw m2, m3
psubw m0, m1
psraw m0, 2
psubw m0, m1
paddw m0, m2
psraw m0, 2
paddw m0, m2
psraw m0, 6
packuswb m0, m0
op_%1h m0, [r0], m7
add r1, 48
add r0, r2
dec r4d
jne .loop8
jmp .done
.op16:
mova m4, [r1+32]
mova m5, [r1+16]
mova m7, [r1]
mova m3, m4
mova m2, m4
mova m1, m4
mova m0, m4
palignr m0, m5, 10
palignr m1, m5, 8
palignr m2, m5, 6
palignr m3, m5, 4
palignr m4, m5, 2
paddw m0, m5
paddw m1, m4
paddw m2, m3
mova m6, m5
mova m4, m5
mova m3, m5
palignr m4, m7, 8
palignr m6, m7, 2
palignr m3, m7, 10
paddw m4, m6
mova m6, m5
palignr m5, m7, 6
palignr m6, m7, 4
paddw m3, m7
paddw m5, m6
psubw m0, m1
psubw m3, m4
psraw m0, 2
psraw m3, 2
psubw m0, m1
psubw m3, m4
paddw m0, m2
paddw m3, m5
psraw m0, 2
psraw m3, 2
paddw m0, m2
paddw m3, m5
psraw m0, 6
psraw m3, 6
packuswb m3, m0
op_%1 m3, [r0], m7
add r1, 48
add r0, r2
dec r4d
jne .op16
.done:
REP_RET
%endmacro
INIT_XMM ssse3
QPEL8OR16_HV2_LOWPASS_OP_XMM put
QPEL8OR16_HV2_LOWPASS_OP_XMM avg
%macro PIXELS4_L2_SHIFT5 1
cglobal %1_pixels4_l2_shift5,6,6 ; dst, src16, src8, dstStride, src8Stride, h
movsxdifnidn r3, r3d
movsxdifnidn r4, r4d
mova m0, [r1]
mova m1, [r1+24]
psraw m0, 5
psraw m1, 5
packuswb m0, m0
packuswb m1, m1
pavgb m0, [r2]
pavgb m1, [r2+r4]
op_%1h m0, [r0], m4
op_%1h m1, [r0+r3], m5
lea r2, [r2+r4*2]
lea r0, [r0+r3*2]
mova m0, [r1+48]
mova m1, [r1+72]
psraw m0, 5
psraw m1, 5
packuswb m0, m0
packuswb m1, m1
pavgb m0, [r2]
pavgb m1, [r2+r4]
op_%1h m0, [r0], m4
op_%1h m1, [r0+r3], m5
RET
%endmacro
INIT_MMX mmxext
PIXELS4_L2_SHIFT5 put
PIXELS4_L2_SHIFT5 avg
%macro PIXELS8_L2_SHIFT5 1
cglobal %1_pixels8_l2_shift5, 6, 6 ; dst, src16, src8, dstStride, src8Stride, h
movsxdifnidn r3, r3d
movsxdifnidn r4, r4d
.loop:
mova m0, [r1]
mova m1, [r1+8]
mova m2, [r1+48]
mova m3, [r1+48+8]
psraw m0, 5
psraw m1, 5
psraw m2, 5
psraw m3, 5
packuswb m0, m1
packuswb m2, m3
pavgb m0, [r2]
pavgb m2, [r2+r4]
op_%1 m0, [r0], m4
op_%1 m2, [r0+r3], m5
lea r2, [r2+2*r4]
add r1, 48*2
lea r0, [r0+2*r3]
sub r5d, 2
jne .loop
REP_RET
%endmacro
INIT_MMX mmxext
PIXELS8_L2_SHIFT5 put
PIXELS8_L2_SHIFT5 avg
%if ARCH_X86_64
%macro QPEL16_H_LOWPASS_L2_OP 1
cglobal %1_h264_qpel16_h_lowpass_l2, 5, 6, 16 ; dst, src, src2, dstStride, src2Stride
movsxdifnidn r3, r3d
movsxdifnidn r4, r4d
mov r5d, 16
pxor m15, m15
mova m14, [pw_5]
mova m13, [pw_16]
.loop:
lddqu m1, [r1+6]
lddqu m7, [r1-2]
mova m0, m1
punpckhbw m1, m15
punpcklbw m0, m15
punpcklbw m7, m15
mova m2, m1
mova m6, m0
mova m3, m1
mova m8, m0
mova m4, m1
mova m9, m0
mova m12, m0
mova m11, m1
palignr m11, m0, 10
palignr m12, m7, 10
palignr m4, m0, 2
palignr m9, m7, 2
palignr m3, m0, 4
palignr m8, m7, 4
palignr m2, m0, 6
palignr m6, m7, 6
paddw m11, m0
palignr m1, m0, 8
palignr m0, m7, 8
paddw m7, m12
paddw m2, m3
paddw m6, m8
paddw m1, m4
paddw m0, m9
psllw m2, 2
psllw m6, 2
psubw m2, m1
psubw m6, m0
paddw m11, m13
paddw m7, m13
pmullw m2, m14
pmullw m6, m14
lddqu m3, [r2]
paddw m2, m11
paddw m6, m7
psraw m2, 5
psraw m6, 5
packuswb m6, m2
pavgb m6, m3
op_%1 m6, [r0], m11
add r1, r3
add r0, r3
add r2, r4
dec r5d
jg .loop
REP_RET
%endmacro
INIT_XMM ssse3
QPEL16_H_LOWPASS_L2_OP put
QPEL16_H_LOWPASS_L2_OP avg
%endif

320
media/ffvpx/libavcodec/x86/h264_weight.asm Normal file
View file

@ -0,0 +1,320 @@
;*****************************************************************************
;* SSE2-optimized weighted prediction code
;*****************************************************************************
;* Copyright (c) 2004-2005 Michael Niedermayer, Loren Merritt
;* Copyright (C) 2010 Eli Friedman <eli.friedman@gmail.com>
;*
;* This file is part of FFmpeg.
;*
;* FFmpeg is free software; you can redistribute it and/or
;* modify it under the terms of the GNU Lesser General Public
;* License as published by the Free Software Foundation; either
;* version 2.1 of the License, or (at your option) any later version.
;*
;* FFmpeg is distributed in the hope that it will be useful,
;* but WITHOUT ANY WARRANTY; without even the implied warranty of
;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
;* Lesser General Public License for more details.
;*
;* You should have received a copy of the GNU Lesser General Public
;* License along with FFmpeg; if not, write to the Free Software
;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
;******************************************************************************
%include "libavutil/x86/x86util.asm"
SECTION .text
;-----------------------------------------------------------------------------
; biweight pred:
;
; void ff_h264_biweight_16_sse2(uint8_t *dst, uint8_t *src, int stride,
; int height, int log2_denom, int weightd,
; int weights, int offset);
; and
; void ff_h264_weight_16_sse2(uint8_t *dst, int stride, int height,
; int log2_denom, int weight, int offset);
;-----------------------------------------------------------------------------
%macro WEIGHT_SETUP 0
add r5, r5
inc r5
movd m3, r4d
movd m5, r5d
movd m6, r3d
pslld m5, m6
psrld m5, 1
%if mmsize == 16
pshuflw m3, m3, 0
pshuflw m5, m5, 0
punpcklqdq m3, m3
punpcklqdq m5, m5
%else
pshufw m3, m3, 0
pshufw m5, m5, 0
%endif
pxor m7, m7
%endmacro
%macro WEIGHT_OP 2
movh m0, [r0+%1]
movh m1, [r0+%2]
punpcklbw m0, m7
punpcklbw m1, m7
pmullw m0, m3
pmullw m1, m3
paddsw m0, m5
paddsw m1, m5
psraw m0, m6
psraw m1, m6
packuswb m0, m1
%endmacro
INIT_MMX mmxext
cglobal h264_weight_16, 6, 6, 0
WEIGHT_SETUP
.nextrow:
WEIGHT_OP 0, 4
mova [r0 ], m0
WEIGHT_OP 8, 12
mova [r0+8], m0
add r0, r1
dec r2d
jnz .nextrow
REP_RET
%macro WEIGHT_FUNC_MM 2
cglobal h264_weight_%1, 6, 6, %2
WEIGHT_SETUP
.nextrow:
WEIGHT_OP 0, mmsize/2
mova [r0], m0
add r0, r1
dec r2d
jnz .nextrow
REP_RET
%endmacro
INIT_MMX mmxext
WEIGHT_FUNC_MM 8, 0
INIT_XMM sse2
WEIGHT_FUNC_MM 16, 8
%macro WEIGHT_FUNC_HALF_MM 2
cglobal h264_weight_%1, 6, 6, %2
WEIGHT_SETUP
sar r2d, 1
lea r3, [r1*2]
.nextrow:
WEIGHT_OP 0, r1
movh [r0], m0
%if mmsize == 16
movhps [r0+r1], m0
%else
psrlq m0, 32
movh [r0+r1], m0
%endif
add r0, r3
dec r2d
jnz .nextrow
REP_RET
%endmacro
INIT_MMX mmxext
WEIGHT_FUNC_HALF_MM 4, 0
INIT_XMM sse2
WEIGHT_FUNC_HALF_MM 8, 8
%macro BIWEIGHT_SETUP 0
%if ARCH_X86_64
%define off_regd r7d
%else
%define off_regd r3d
%endif
mov off_regd, r7m
add off_regd, 1
or off_regd, 1
add r4d, 1
cmp r6d, 128
je .nonnormal
cmp r5d, 128
jne .normal
.nonnormal:
sar r5d, 1
sar r6d, 1
sar off_regd, 1
sub r4d, 1
.normal:
%if cpuflag(ssse3)
movd m4, r5d
movd m0, r6d
%else
movd m3, r5d
movd m4, r6d
%endif
movd m5, off_regd
movd m6, r4d
pslld m5, m6
psrld m5, 1
%if cpuflag(ssse3)
punpcklbw m4, m0
pshuflw m4, m4, 0
pshuflw m5, m5, 0
punpcklqdq m4, m4
punpcklqdq m5, m5
%else
%if mmsize == 16
pshuflw m3, m3, 0
pshuflw m4, m4, 0
pshuflw m5, m5, 0
punpcklqdq m3, m3
punpcklqdq m4, m4
punpcklqdq m5, m5
%else
pshufw m3, m3, 0
pshufw m4, m4, 0
pshufw m5, m5, 0
%endif
pxor m7, m7
%endif
%endmacro
%macro BIWEIGHT_STEPA 3
movh m%1, [r0+%3]
movh m%2, [r1+%3]
punpcklbw m%1, m7
punpcklbw m%2, m7
pmullw m%1, m3
pmullw m%2, m4
paddsw m%1, m%2
%endmacro
%macro BIWEIGHT_STEPB 0
paddsw m0, m5
paddsw m1, m5
psraw m0, m6
psraw m1, m6
packuswb m0, m1
%endmacro
INIT_MMX mmxext
cglobal h264_biweight_16, 7, 8, 0
BIWEIGHT_SETUP
movifnidn r3d, r3m
.nextrow:
BIWEIGHT_STEPA 0, 1, 0
BIWEIGHT_STEPA 1, 2, 4
BIWEIGHT_STEPB
mova [r0], m0
BIWEIGHT_STEPA 0, 1, 8
BIWEIGHT_STEPA 1, 2, 12
BIWEIGHT_STEPB
mova [r0+8], m0
add r0, r2
add r1, r2
dec r3d
jnz .nextrow
REP_RET
%macro BIWEIGHT_FUNC_MM 2
cglobal h264_biweight_%1, 7, 8, %2
BIWEIGHT_SETUP
movifnidn r3d, r3m
.nextrow:
BIWEIGHT_STEPA 0, 1, 0
BIWEIGHT_STEPA 1, 2, mmsize/2
BIWEIGHT_STEPB
mova [r0], m0
add r0, r2
add r1, r2
dec r3d
jnz .nextrow
REP_RET
%endmacro
INIT_MMX mmxext
BIWEIGHT_FUNC_MM 8, 0
INIT_XMM sse2
BIWEIGHT_FUNC_MM 16, 8
%macro BIWEIGHT_FUNC_HALF_MM 2
cglobal h264_biweight_%1, 7, 8, %2
BIWEIGHT_SETUP
movifnidn r3d, r3m
sar r3, 1
lea r4, [r2*2]
.nextrow:
BIWEIGHT_STEPA 0, 1, 0
BIWEIGHT_STEPA 1, 2, r2
BIWEIGHT_STEPB
movh [r0], m0
%if mmsize == 16
movhps [r0+r2], m0
%else
psrlq m0, 32
movh [r0+r2], m0
%endif
add r0, r4
add r1, r4
dec r3d
jnz .nextrow
REP_RET
%endmacro
INIT_MMX mmxext
BIWEIGHT_FUNC_HALF_MM 4, 0
INIT_XMM sse2
BIWEIGHT_FUNC_HALF_MM 8, 8
%macro BIWEIGHT_SSSE3_OP 0
pmaddubsw m0, m4
pmaddubsw m2, m4
paddsw m0, m5
paddsw m2, m5
psraw m0, m6
psraw m2, m6
packuswb m0, m2
%endmacro
INIT_XMM ssse3
cglobal h264_biweight_16, 7, 8, 8
BIWEIGHT_SETUP
movifnidn r3d, r3m
.nextrow:
movh m0, [r0]
movh m2, [r0+8]
movh m3, [r1+8]
punpcklbw m0, [r1]
punpcklbw m2, m3
BIWEIGHT_SSSE3_OP
mova [r0], m0
add r0, r2
add r1, r2
dec r3d
jnz .nextrow
REP_RET
INIT_XMM ssse3
cglobal h264_biweight_8, 7, 8, 8
BIWEIGHT_SETUP
movifnidn r3d, r3m
sar r3, 1
lea r4, [r2*2]
.nextrow:
movh m0, [r0]
movh m1, [r1]
movh m2, [r0+r2]
movh m3, [r1+r2]
punpcklbw m0, m1
punpcklbw m2, m3
BIWEIGHT_SSSE3_OP
movh [r0], m0
movhps [r0+r2], m0
add r0, r4
add r1, r4
dec r3d
jnz .nextrow
REP_RET

284
media/ffvpx/libavcodec/x86/h264_weight_10bit.asm Normal file
View file

@ -0,0 +1,284 @@
;*****************************************************************************
;* MMX/SSE2/AVX-optimized 10-bit H.264 weighted prediction code
;*****************************************************************************
;* Copyright (C) 2005-2011 x264 project
;*
;* Authors: Daniel Kang <daniel.d.kang@gmail.com>
;*
;* This file is part of FFmpeg.
;*
;* FFmpeg is free software; you can redistribute it and/or
;* modify it under the terms of the GNU Lesser General Public
;* License as published by the Free Software Foundation; either
;* version 2.1 of the License, or (at your option) any later version.
;*
;* FFmpeg is distributed in the hope that it will be useful,
;* but WITHOUT ANY WARRANTY; without even the implied warranty of
;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
;* Lesser General Public License for more details.
;*
;* You should have received a copy of the GNU Lesser General Public
;* License along with FFmpeg; if not, write to the Free Software
;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
;******************************************************************************
%include "libavutil/x86/x86util.asm"
SECTION_RODATA 32
sq_1: dq 1
dq 0
cextern pw_1
cextern pw_1023
%define pw_pixel_max pw_1023
SECTION .text
;-----------------------------------------------------------------------------
; void ff_h264_weight_16_10(uint8_t *dst, int stride, int height,
; int log2_denom, int weight, int offset);
;-----------------------------------------------------------------------------
%macro WEIGHT_PROLOGUE 0
.prologue:
PROLOGUE 0,6,8
movifnidn r0, r0mp
movifnidn r1d, r1m
movifnidn r2d, r2m
movifnidn r4d, r4m
movifnidn r5d, r5m
%endmacro
%macro WEIGHT_SETUP 0
mova m0, [pw_1]
movd m2, r3m
pslld m0, m2 ; 1<<log2_denom
SPLATW m0, m0
shl r5, 19 ; *8, move to upper half of dword
lea r5, [r5+r4*2+0x10000]
movd m3, r5d ; weight<<1 | 1+(offset<<(3))
pshufd m3, m3, 0
mova m4, [pw_pixel_max]
paddw m2, [sq_1] ; log2_denom+1
%if notcpuflag(sse4)
pxor m7, m7
%endif
%endmacro
%macro WEIGHT_OP 1-2
%if %0==1
mova m5, [r0+%1]
punpckhwd m6, m5, m0
punpcklwd m5, m0
%else
movq m5, [r0+%1]
movq m6, [r0+%2]
punpcklwd m5, m0
punpcklwd m6, m0
%endif
pmaddwd m5, m3
pmaddwd m6, m3
psrad m5, m2
psrad m6, m2
%if cpuflag(sse4)
packusdw m5, m6
pminsw m5, m4
%else
packssdw m5, m6
CLIPW m5, m7, m4
%endif
%endmacro
%macro WEIGHT_FUNC_DBL 0
cglobal h264_weight_16_10
WEIGHT_PROLOGUE
WEIGHT_SETUP
.nextrow:
WEIGHT_OP 0
mova [r0 ], m5
WEIGHT_OP 16
mova [r0+16], m5
add r0, r1
dec r2d
jnz .nextrow
REP_RET
%endmacro
INIT_XMM sse2
WEIGHT_FUNC_DBL
INIT_XMM sse4
WEIGHT_FUNC_DBL
%macro WEIGHT_FUNC_MM 0
cglobal h264_weight_8_10
WEIGHT_PROLOGUE
WEIGHT_SETUP
.nextrow:
WEIGHT_OP 0
mova [r0], m5
add r0, r1
dec r2d
jnz .nextrow
REP_RET
%endmacro
INIT_XMM sse2
WEIGHT_FUNC_MM
INIT_XMM sse4
WEIGHT_FUNC_MM
%macro WEIGHT_FUNC_HALF_MM 0
cglobal h264_weight_4_10
WEIGHT_PROLOGUE
sar r2d, 1
WEIGHT_SETUP
lea r3, [r1*2]
.nextrow:
WEIGHT_OP 0, r1
movh [r0], m5
movhps [r0+r1], m5
add r0, r3
dec r2d
jnz .nextrow
REP_RET
%endmacro
INIT_XMM sse2
WEIGHT_FUNC_HALF_MM
INIT_XMM sse4
WEIGHT_FUNC_HALF_MM
;-----------------------------------------------------------------------------
; void ff_h264_biweight_16_10(uint8_t *dst, uint8_t *src, int stride,
; int height, int log2_denom, int weightd,
; int weights, int offset);
;-----------------------------------------------------------------------------
%if ARCH_X86_32
DECLARE_REG_TMP 3
%else
DECLARE_REG_TMP 7
%endif
%macro BIWEIGHT_PROLOGUE 0
.prologue:
PROLOGUE 0,8,8
movifnidn r0, r0mp
movifnidn r1, r1mp
movifnidn r2d, r2m
movifnidn r5d, r5m
movifnidn r6d, r6m
movifnidn t0d, r7m
%endmacro
%macro BIWEIGHT_SETUP 0
lea t0, [t0*4+1] ; (offset<<2)+1
or t0, 1
shl r6, 16
or r5, r6
movd m4, r5d ; weightd | weights
movd m5, t0d ; (offset+1)|1
movd m6, r4m ; log2_denom
pslld m5, m6 ; (((offset<<2)+1)|1)<<log2_denom
paddd m6, [sq_1]
pshufd m4, m4, 0
pshufd m5, m5, 0
mova m3, [pw_pixel_max]
movifnidn r3d, r3m
%if notcpuflag(sse4)
pxor m7, m7
%endif
%endmacro
%macro BIWEIGHT 1-2
%if %0==1
mova m0, [r0+%1]
mova m1, [r1+%1]
punpckhwd m2, m0, m1
punpcklwd m0, m1
%else
movq m0, [r0+%1]
movq m1, [r1+%1]
punpcklwd m0, m1
movq m2, [r0+%2]
movq m1, [r1+%2]
punpcklwd m2, m1
%endif
pmaddwd m0, m4
pmaddwd m2, m4
paddd m0, m5
paddd m2, m5
psrad m0, m6
psrad m2, m6
%if cpuflag(sse4)
packusdw m0, m2
pminsw m0, m3
%else
packssdw m0, m2
CLIPW m0, m7, m3
%endif
%endmacro
%macro BIWEIGHT_FUNC_DBL 0
cglobal h264_biweight_16_10
BIWEIGHT_PROLOGUE
BIWEIGHT_SETUP
.nextrow:
BIWEIGHT 0
mova [r0 ], m0
BIWEIGHT 16
mova [r0+16], m0
add r0, r2
add r1, r2
dec r3d
jnz .nextrow
REP_RET
%endmacro
INIT_XMM sse2
BIWEIGHT_FUNC_DBL
INIT_XMM sse4
BIWEIGHT_FUNC_DBL
%macro BIWEIGHT_FUNC 0
cglobal h264_biweight_8_10
BIWEIGHT_PROLOGUE
BIWEIGHT_SETUP
.nextrow:
BIWEIGHT 0
mova [r0], m0
add r0, r2
add r1, r2
dec r3d
jnz .nextrow
REP_RET
%endmacro
INIT_XMM sse2
BIWEIGHT_FUNC
INIT_XMM sse4
BIWEIGHT_FUNC
%macro BIWEIGHT_FUNC_HALF 0
cglobal h264_biweight_4_10
BIWEIGHT_PROLOGUE
BIWEIGHT_SETUP
sar r3d, 1
lea r4, [r2*2]
.nextrow:
BIWEIGHT 0, r2
movh [r0 ], m0
movhps [r0+r2], m0
add r0, r4
add r1, r4
dec r3d
jnz .nextrow
REP_RET
%endmacro
INIT_XMM sse2
BIWEIGHT_FUNC_HALF
INIT_XMM sse4
BIWEIGHT_FUNC_HALF

117
media/ffvpx/libavcodec/x86/h264chroma_init.c Normal file
View file

@ -0,0 +1,117 @@
/*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include <stdint.h>
#include "config.h"
#include "libavutil/attributes.h"
#include "libavutil/cpu.h"
#include "libavutil/x86/cpu.h"
#include "libavcodec/h264chroma.h"
void ff_put_h264_chroma_mc8_rnd_mmx (uint8_t *dst, uint8_t *src,
ptrdiff_t stride, int h, int x, int y);
void ff_avg_h264_chroma_mc8_rnd_mmxext(uint8_t *dst, uint8_t *src,
ptrdiff_t stride, int h, int x, int y);
void ff_avg_h264_chroma_mc8_rnd_3dnow(uint8_t *dst, uint8_t *src,
ptrdiff_t stride, int h, int x, int y);
void ff_put_h264_chroma_mc4_mmx (uint8_t *dst, uint8_t *src,
ptrdiff_t stride, int h, int x, int y);
void ff_avg_h264_chroma_mc4_mmxext (uint8_t *dst, uint8_t *src,
ptrdiff_t stride, int h, int x, int y);
void ff_avg_h264_chroma_mc4_3dnow (uint8_t *dst, uint8_t *src,
ptrdiff_t stride, int h, int x, int y);
void ff_put_h264_chroma_mc2_mmxext (uint8_t *dst, uint8_t *src,
ptrdiff_t stride, int h, int x, int y);
void ff_avg_h264_chroma_mc2_mmxext (uint8_t *dst, uint8_t *src,
ptrdiff_t stride, int h, int x, int y);
void ff_put_h264_chroma_mc8_rnd_ssse3(uint8_t *dst, uint8_t *src,
ptrdiff_t stride, int h, int x, int y);
void ff_put_h264_chroma_mc4_ssse3 (uint8_t *dst, uint8_t *src,
ptrdiff_t stride, int h, int x, int y);
void ff_avg_h264_chroma_mc8_rnd_ssse3(uint8_t *dst, uint8_t *src,
ptrdiff_t stride, int h, int x, int y);
void ff_avg_h264_chroma_mc4_ssse3 (uint8_t *dst, uint8_t *src,
ptrdiff_t stride, int h, int x, int y);
#define CHROMA_MC(OP, NUM, DEPTH, OPT) \
void ff_ ## OP ## _h264_chroma_mc ## NUM ## _ ## DEPTH ## _ ## OPT \
(uint8_t *dst, uint8_t *src, \
ptrdiff_t stride, int h, int x, int y);
CHROMA_MC(put, 2, 10, mmxext)
CHROMA_MC(avg, 2, 10, mmxext)
CHROMA_MC(put, 4, 10, mmxext)
CHROMA_MC(avg, 4, 10, mmxext)
CHROMA_MC(put, 8, 10, sse2)
CHROMA_MC(avg, 8, 10, sse2)
CHROMA_MC(put, 8, 10, avx)
CHROMA_MC(avg, 8, 10, avx)
av_cold void ff_h264chroma_init_x86(H264ChromaContext *c, int bit_depth)
{
int high_bit_depth = bit_depth > 8;
int cpu_flags = av_get_cpu_flags();
if (EXTERNAL_MMX(cpu_flags) && !high_bit_depth) {
c->put_h264_chroma_pixels_tab[0] = ff_put_h264_chroma_mc8_rnd_mmx;
c->put_h264_chroma_pixels_tab[1] = ff_put_h264_chroma_mc4_mmx;
}
if (EXTERNAL_AMD3DNOW(cpu_flags) && !high_bit_depth) {
c->avg_h264_chroma_pixels_tab[0] = ff_avg_h264_chroma_mc8_rnd_3dnow;
c->avg_h264_chroma_pixels_tab[1] = ff_avg_h264_chroma_mc4_3dnow;
}
if (EXTERNAL_MMXEXT(cpu_flags) && !high_bit_depth) {
c->avg_h264_chroma_pixels_tab[0] = ff_avg_h264_chroma_mc8_rnd_mmxext;
c->avg_h264_chroma_pixels_tab[1] = ff_avg_h264_chroma_mc4_mmxext;
c->avg_h264_chroma_pixels_tab[2] = ff_avg_h264_chroma_mc2_mmxext;
c->put_h264_chroma_pixels_tab[2] = ff_put_h264_chroma_mc2_mmxext;
}
if (EXTERNAL_MMXEXT(cpu_flags) && bit_depth > 8 && bit_depth <= 10) {
c->put_h264_chroma_pixels_tab[2] = ff_put_h264_chroma_mc2_10_mmxext;
c->avg_h264_chroma_pixels_tab[2] = ff_avg_h264_chroma_mc2_10_mmxext;
c->put_h264_chroma_pixels_tab[1] = ff_put_h264_chroma_mc4_10_mmxext;
c->avg_h264_chroma_pixels_tab[1] = ff_avg_h264_chroma_mc4_10_mmxext;
}
if (EXTERNAL_SSE2(cpu_flags) && bit_depth > 8 && bit_depth <= 10) {
c->put_h264_chroma_pixels_tab[0] = ff_put_h264_chroma_mc8_10_sse2;
c->avg_h264_chroma_pixels_tab[0] = ff_avg_h264_chroma_mc8_10_sse2;
}
if (EXTERNAL_SSSE3(cpu_flags) && !high_bit_depth) {
c->put_h264_chroma_pixels_tab[0] = ff_put_h264_chroma_mc8_rnd_ssse3;
c->avg_h264_chroma_pixels_tab[0] = ff_avg_h264_chroma_mc8_rnd_ssse3;
c->put_h264_chroma_pixels_tab[1] = ff_put_h264_chroma_mc4_ssse3;
c->avg_h264_chroma_pixels_tab[1] = ff_avg_h264_chroma_mc4_ssse3;
}
if (EXTERNAL_AVX(cpu_flags) && bit_depth > 8 && bit_depth <= 10) {
// AVX implies !cache64.
// TODO: Port cache(32|64) detection from x264.
c->put_h264_chroma_pixels_tab[0] = ff_put_h264_chroma_mc8_10_avx;
c->avg_h264_chroma_pixels_tab[0] = ff_avg_h264_chroma_mc8_10_avx;
}
}

448
media/ffvpx/libavcodec/x86/h264dsp_init.c Normal file
View file

@ -0,0 +1,448 @@
/*
* Copyright (c) 2004-2005 Michael Niedermayer, Loren Merritt
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "libavutil/attributes.h"
#include "libavutil/cpu.h"
#include "libavutil/x86/asm.h"
#include "libavutil/x86/cpu.h"
#include "libavcodec/h264dsp.h"
/***********************************/
/* IDCT */
#define IDCT_ADD_FUNC(NUM, DEPTH, OPT) \
void ff_h264_idct ## NUM ## _add_ ## DEPTH ## _ ## OPT(uint8_t *dst, \
int16_t *block, \
int stride);
IDCT_ADD_FUNC(, 8, mmx)
IDCT_ADD_FUNC(, 8, sse2)
IDCT_ADD_FUNC(, 8, avx)
IDCT_ADD_FUNC(, 10, sse2)
IDCT_ADD_FUNC(_dc, 8, mmxext)
IDCT_ADD_FUNC(_dc, 8, sse2)
IDCT_ADD_FUNC(_dc, 8, avx)
IDCT_ADD_FUNC(_dc, 10, mmxext)
IDCT_ADD_FUNC(8_dc, 8, mmxext)
IDCT_ADD_FUNC(8_dc, 10, sse2)
IDCT_ADD_FUNC(8, 8, mmx)
IDCT_ADD_FUNC(8, 8, sse2)
IDCT_ADD_FUNC(8, 10, sse2)
IDCT_ADD_FUNC(, 10, avx)
IDCT_ADD_FUNC(8_dc, 10, avx)
IDCT_ADD_FUNC(8, 10, avx)
#define IDCT_ADD_REP_FUNC(NUM, REP, DEPTH, OPT) \
void ff_h264_idct ## NUM ## _add ## REP ## _ ## DEPTH ## _ ## OPT \
(uint8_t *dst, const int *block_offset, \
int16_t *block, int stride, const uint8_t nnzc[6 * 8]);
IDCT_ADD_REP_FUNC(8, 4, 8, mmx)
IDCT_ADD_REP_FUNC(8, 4, 8, mmxext)
IDCT_ADD_REP_FUNC(8, 4, 8, sse2)
IDCT_ADD_REP_FUNC(8, 4, 10, sse2)
IDCT_ADD_REP_FUNC(8, 4, 10, avx)
IDCT_ADD_REP_FUNC(, 16, 8, mmx)
IDCT_ADD_REP_FUNC(, 16, 8, mmxext)
IDCT_ADD_REP_FUNC(, 16, 8, sse2)
IDCT_ADD_REP_FUNC(, 16, 10, sse2)
IDCT_ADD_REP_FUNC(, 16intra, 8, mmx)
IDCT_ADD_REP_FUNC(, 16intra, 8, mmxext)
IDCT_ADD_REP_FUNC(, 16intra, 8, sse2)
IDCT_ADD_REP_FUNC(, 16intra, 10, sse2)
IDCT_ADD_REP_FUNC(, 16, 10, avx)
IDCT_ADD_REP_FUNC(, 16intra, 10, avx)
#define IDCT_ADD_REP_FUNC2(NUM, REP, DEPTH, OPT) \
void ff_h264_idct ## NUM ## _add ## REP ## _ ## DEPTH ## _ ## OPT \
(uint8_t **dst, const int *block_offset, \
int16_t *block, int stride, const uint8_t nnzc[6 * 8]);
IDCT_ADD_REP_FUNC2(, 8, 8, mmx)
IDCT_ADD_REP_FUNC2(, 8, 8, mmxext)
IDCT_ADD_REP_FUNC2(, 8, 8, sse2)
IDCT_ADD_REP_FUNC2(, 8, 10, sse2)
IDCT_ADD_REP_FUNC2(, 8, 10, avx)
IDCT_ADD_REP_FUNC2(, 8_422, 8, mmx)
IDCT_ADD_REP_FUNC2(, 8_422, 10, sse2)
IDCT_ADD_REP_FUNC2(, 8_422, 10, avx)
void ff_h264_luma_dc_dequant_idct_mmx(int16_t *output, int16_t *input, int qmul);
void ff_h264_luma_dc_dequant_idct_sse2(int16_t *output, int16_t *input, int qmul);
/***********************************/
/* deblocking */
void ff_h264_loop_filter_strength_mmxext(int16_t bS[2][4][4], uint8_t nnz[40],
int8_t ref[2][40],
int16_t mv[2][40][2],
int bidir, int edges, int step,
int mask_mv0, int mask_mv1, int field);
#define LF_FUNC(DIR, TYPE, DEPTH, OPT) \
void ff_deblock_ ## DIR ## _ ## TYPE ## _ ## DEPTH ## _ ## OPT(uint8_t *pix, \
int stride, \
int alpha, \
int beta, \
int8_t *tc0);
#define LF_IFUNC(DIR, TYPE, DEPTH, OPT) \
void ff_deblock_ ## DIR ## _ ## TYPE ## _ ## DEPTH ## _ ## OPT(uint8_t *pix, \
int stride, \
int alpha, \
int beta);
#define LF_FUNCS(type, depth) \
LF_FUNC(h, chroma, depth, mmxext) \
LF_IFUNC(h, chroma_intra, depth, mmxext) \
LF_FUNC(h, chroma422, depth, mmxext) \
LF_IFUNC(h, chroma422_intra, depth, mmxext) \
LF_FUNC(v, chroma, depth, mmxext) \
LF_IFUNC(v, chroma_intra, depth, mmxext) \
LF_FUNC(h, luma, depth, mmxext) \
LF_IFUNC(h, luma_intra, depth, mmxext) \
LF_FUNC(h, luma, depth, sse2) \
LF_IFUNC(h, luma_intra, depth, sse2) \
LF_FUNC(v, luma, depth, sse2) \
LF_IFUNC(v, luma_intra, depth, sse2) \
LF_FUNC(h, chroma, depth, sse2) \
LF_IFUNC(h, chroma_intra, depth, sse2) \
LF_FUNC(h, chroma422, depth, sse2) \
LF_IFUNC(h, chroma422_intra, depth, sse2) \
LF_FUNC(v, chroma, depth, sse2) \
LF_IFUNC(v, chroma_intra, depth, sse2) \
LF_FUNC(h, luma, depth, avx) \
LF_IFUNC(h, luma_intra, depth, avx) \
LF_FUNC(v, luma, depth, avx) \
LF_IFUNC(v, luma_intra, depth, avx) \
LF_FUNC(h, chroma, depth, avx) \
LF_IFUNC(h, chroma_intra, depth, avx) \
LF_FUNC(h, chroma422, depth, avx) \
LF_IFUNC(h, chroma422_intra, depth, avx) \
LF_FUNC(v, chroma, depth, avx) \
LF_IFUNC(v, chroma_intra, depth, avx)
LF_FUNC(h, luma_mbaff, 8, sse2)
LF_FUNC(h, luma_mbaff, 8, avx)
LF_FUNCS(uint8_t, 8)
LF_FUNCS(uint16_t, 10)
#if ARCH_X86_32 && HAVE_MMXEXT_EXTERNAL
LF_FUNC(v8, luma, 8, mmxext)
static void deblock_v_luma_8_mmxext(uint8_t *pix, int stride, int alpha,
int beta, int8_t *tc0)
{
if ((tc0[0] & tc0[1]) >= 0)
ff_deblock_v8_luma_8_mmxext(pix + 0, stride, alpha, beta, tc0);
if ((tc0[2] & tc0[3]) >= 0)
ff_deblock_v8_luma_8_mmxext(pix + 8, stride, alpha, beta, tc0 + 2);
}
LF_IFUNC(v8, luma_intra, 8, mmxext)
static void deblock_v_luma_intra_8_mmxext(uint8_t *pix, int stride,
int alpha, int beta)
{
ff_deblock_v8_luma_intra_8_mmxext(pix + 0, stride, alpha, beta);
ff_deblock_v8_luma_intra_8_mmxext(pix + 8, stride, alpha, beta);
}
#endif /* ARCH_X86_32 && HAVE_MMXEXT_EXTERNAL */
LF_FUNC(v, luma, 10, mmxext)
LF_IFUNC(v, luma_intra, 10, mmxext)
/***********************************/
/* weighted prediction */
#define H264_WEIGHT(W, OPT) \
void ff_h264_weight_ ## W ## _ ## OPT(uint8_t *dst, ptrdiff_t stride, \
int height, int log2_denom, \
int weight, int offset);
#define H264_BIWEIGHT(W, OPT) \
void ff_h264_biweight_ ## W ## _ ## OPT(uint8_t *dst, uint8_t *src, \
ptrdiff_t stride, int height, \
int log2_denom, int weightd, \
int weights, int offset);
#define H264_BIWEIGHT_MMX(W) \
H264_WEIGHT(W, mmxext) \
H264_BIWEIGHT(W, mmxext)
#define H264_BIWEIGHT_MMX_SSE(W) \
H264_BIWEIGHT_MMX(W) \
H264_WEIGHT(W, sse2) \
H264_BIWEIGHT(W, sse2) \
H264_BIWEIGHT(W, ssse3)
H264_BIWEIGHT_MMX_SSE(16)
H264_BIWEIGHT_MMX_SSE(8)
H264_BIWEIGHT_MMX(4)
#define H264_WEIGHT_10(W, DEPTH, OPT) \
void ff_h264_weight_ ## W ## _ ## DEPTH ## _ ## OPT(uint8_t *dst, \
ptrdiff_t stride, \
int height, \
int log2_denom, \
int weight, \
int offset);
#define H264_BIWEIGHT_10(W, DEPTH, OPT) \
void ff_h264_biweight_ ## W ## _ ## DEPTH ## _ ## OPT(uint8_t *dst, \
uint8_t *src, \
ptrdiff_t stride, \
int height, \
int log2_denom, \
int weightd, \
int weights, \
int offset);
#define H264_BIWEIGHT_10_SSE(W, DEPTH) \
H264_WEIGHT_10(W, DEPTH, sse2) \
H264_WEIGHT_10(W, DEPTH, sse4) \
H264_BIWEIGHT_10(W, DEPTH, sse2) \
H264_BIWEIGHT_10(W, DEPTH, sse4)
H264_BIWEIGHT_10_SSE(16, 10)
H264_BIWEIGHT_10_SSE(8, 10)
H264_BIWEIGHT_10_SSE(4, 10)
av_cold void ff_h264dsp_init_x86(H264DSPContext *c, const int bit_depth,
const int chroma_format_idc)
{
#if HAVE_X86ASM
int cpu_flags = av_get_cpu_flags();
if (EXTERNAL_MMXEXT(cpu_flags) && chroma_format_idc <= 1)
c->h264_loop_filter_strength = ff_h264_loop_filter_strength_mmxext;
if (bit_depth == 8) {
if (EXTERNAL_MMX(cpu_flags)) {
c->h264_idct_dc_add =
c->h264_idct_add = ff_h264_idct_add_8_mmx;
c->h264_idct8_dc_add =
c->h264_idct8_add = ff_h264_idct8_add_8_mmx;
c->h264_idct_add16 = ff_h264_idct_add16_8_mmx;
c->h264_idct8_add4 = ff_h264_idct8_add4_8_mmx;
if (chroma_format_idc <= 1) {
c->h264_idct_add8 = ff_h264_idct_add8_8_mmx;
} else {
c->h264_idct_add8 = ff_h264_idct_add8_422_8_mmx;
}
c->h264_idct_add16intra = ff_h264_idct_add16intra_8_mmx;
if (cpu_flags & AV_CPU_FLAG_CMOV)
c->h264_luma_dc_dequant_idct = ff_h264_luma_dc_dequant_idct_mmx;
}
if (EXTERNAL_MMXEXT(cpu_flags)) {
c->h264_idct_dc_add = ff_h264_idct_dc_add_8_mmxext;
c->h264_idct8_dc_add = ff_h264_idct8_dc_add_8_mmxext;
c->h264_idct_add16 = ff_h264_idct_add16_8_mmxext;
c->h264_idct8_add4 = ff_h264_idct8_add4_8_mmxext;
if (chroma_format_idc <= 1)
c->h264_idct_add8 = ff_h264_idct_add8_8_mmxext;
c->h264_idct_add16intra = ff_h264_idct_add16intra_8_mmxext;
c->h264_v_loop_filter_chroma = ff_deblock_v_chroma_8_mmxext;
c->h264_v_loop_filter_chroma_intra = ff_deblock_v_chroma_intra_8_mmxext;
if (chroma_format_idc <= 1) {
c->h264_h_loop_filter_chroma = ff_deblock_h_chroma_8_mmxext;
c->h264_h_loop_filter_chroma_intra = ff_deblock_h_chroma_intra_8_mmxext;
} else {
c->h264_h_loop_filter_chroma = ff_deblock_h_chroma422_8_mmxext;
c->h264_h_loop_filter_chroma_intra = ff_deblock_h_chroma422_intra_8_mmxext;
}
#if ARCH_X86_32 && HAVE_MMXEXT_EXTERNAL
c->h264_v_loop_filter_luma = deblock_v_luma_8_mmxext;
c->h264_h_loop_filter_luma = ff_deblock_h_luma_8_mmxext;
c->h264_v_loop_filter_luma_intra = deblock_v_luma_intra_8_mmxext;
c->h264_h_loop_filter_luma_intra = ff_deblock_h_luma_intra_8_mmxext;
#endif /* ARCH_X86_32 && HAVE_MMXEXT_EXTERNAL */
c->weight_h264_pixels_tab[0] = ff_h264_weight_16_mmxext;
c->weight_h264_pixels_tab[1] = ff_h264_weight_8_mmxext;
c->weight_h264_pixels_tab[2] = ff_h264_weight_4_mmxext;
c->biweight_h264_pixels_tab[0] = ff_h264_biweight_16_mmxext;
c->biweight_h264_pixels_tab[1] = ff_h264_biweight_8_mmxext;
c->biweight_h264_pixels_tab[2] = ff_h264_biweight_4_mmxext;
}
if (EXTERNAL_SSE2(cpu_flags)) {
c->h264_idct8_add = ff_h264_idct8_add_8_sse2;
c->h264_idct_add16 = ff_h264_idct_add16_8_sse2;
c->h264_idct8_add4 = ff_h264_idct8_add4_8_sse2;
if (chroma_format_idc <= 1)
c->h264_idct_add8 = ff_h264_idct_add8_8_sse2;
c->h264_idct_add16intra = ff_h264_idct_add16intra_8_sse2;
c->h264_luma_dc_dequant_idct = ff_h264_luma_dc_dequant_idct_sse2;
c->weight_h264_pixels_tab[0] = ff_h264_weight_16_sse2;
c->weight_h264_pixels_tab[1] = ff_h264_weight_8_sse2;
c->biweight_h264_pixels_tab[0] = ff_h264_biweight_16_sse2;
c->biweight_h264_pixels_tab[1] = ff_h264_biweight_8_sse2;
c->h264_v_loop_filter_luma = ff_deblock_v_luma_8_sse2;
c->h264_h_loop_filter_luma = ff_deblock_h_luma_8_sse2;
c->h264_v_loop_filter_luma_intra = ff_deblock_v_luma_intra_8_sse2;
c->h264_h_loop_filter_luma_intra = ff_deblock_h_luma_intra_8_sse2;
#if ARCH_X86_64
c->h264_h_loop_filter_luma_mbaff = ff_deblock_h_luma_mbaff_8_sse2;
#endif
c->h264_v_loop_filter_chroma = ff_deblock_v_chroma_8_sse2;
c->h264_v_loop_filter_chroma_intra = ff_deblock_v_chroma_intra_8_sse2;
if (chroma_format_idc <= 1) {
c->h264_h_loop_filter_chroma = ff_deblock_h_chroma_8_sse2;
c->h264_h_loop_filter_chroma_intra = ff_deblock_h_chroma_intra_8_sse2;
} else {
c->h264_h_loop_filter_chroma = ff_deblock_h_chroma422_8_sse2;
c->h264_h_loop_filter_chroma_intra = ff_deblock_h_chroma422_intra_8_sse2;
}
c->h264_idct_add = ff_h264_idct_add_8_sse2;
c->h264_idct_dc_add = ff_h264_idct_dc_add_8_sse2;
}
if (EXTERNAL_SSSE3(cpu_flags)) {
c->biweight_h264_pixels_tab[0] = ff_h264_biweight_16_ssse3;
c->biweight_h264_pixels_tab[1] = ff_h264_biweight_8_ssse3;
}
if (EXTERNAL_AVX(cpu_flags)) {
c->h264_v_loop_filter_luma = ff_deblock_v_luma_8_avx;
c->h264_h_loop_filter_luma = ff_deblock_h_luma_8_avx;
c->h264_v_loop_filter_luma_intra = ff_deblock_v_luma_intra_8_avx;
c->h264_h_loop_filter_luma_intra = ff_deblock_h_luma_intra_8_avx;
#if ARCH_X86_64
c->h264_h_loop_filter_luma_mbaff = ff_deblock_h_luma_mbaff_8_avx;
#endif
c->h264_v_loop_filter_chroma = ff_deblock_v_chroma_8_avx;
c->h264_v_loop_filter_chroma_intra = ff_deblock_v_chroma_intra_8_avx;
if (chroma_format_idc <= 1) {
c->h264_h_loop_filter_chroma = ff_deblock_h_chroma_8_avx;
c->h264_h_loop_filter_chroma_intra = ff_deblock_h_chroma_intra_8_avx;
} else {
c->h264_h_loop_filter_chroma = ff_deblock_h_chroma422_8_avx;
c->h264_h_loop_filter_chroma_intra = ff_deblock_h_chroma422_intra_8_avx;
}
c->h264_idct_add = ff_h264_idct_add_8_avx;
c->h264_idct_dc_add = ff_h264_idct_dc_add_8_avx;
}
} else if (bit_depth == 10) {
if (EXTERNAL_MMXEXT(cpu_flags)) {
#if ARCH_X86_32
c->h264_v_loop_filter_chroma = ff_deblock_v_chroma_10_mmxext;
c->h264_v_loop_filter_chroma_intra = ff_deblock_v_chroma_intra_10_mmxext;
if (chroma_format_idc <= 1) {
c->h264_h_loop_filter_chroma = ff_deblock_h_chroma_10_mmxext;
} else {
c->h264_h_loop_filter_chroma = ff_deblock_h_chroma422_10_mmxext;
}
c->h264_v_loop_filter_luma = ff_deblock_v_luma_10_mmxext;
c->h264_h_loop_filter_luma = ff_deblock_h_luma_10_mmxext;
c->h264_v_loop_filter_luma_intra = ff_deblock_v_luma_intra_10_mmxext;
c->h264_h_loop_filter_luma_intra = ff_deblock_h_luma_intra_10_mmxext;
#endif /* ARCH_X86_32 */
c->h264_idct_dc_add = ff_h264_idct_dc_add_10_mmxext;
}
if (EXTERNAL_SSE2(cpu_flags)) {
c->h264_idct_add = ff_h264_idct_add_10_sse2;
c->h264_idct8_dc_add = ff_h264_idct8_dc_add_10_sse2;
c->h264_idct_add16 = ff_h264_idct_add16_10_sse2;
if (chroma_format_idc <= 1) {
c->h264_idct_add8 = ff_h264_idct_add8_10_sse2;
} else {
c->h264_idct_add8 = ff_h264_idct_add8_422_10_sse2;
}
c->h264_idct_add16intra = ff_h264_idct_add16intra_10_sse2;
#if HAVE_ALIGNED_STACK
c->h264_idct8_add = ff_h264_idct8_add_10_sse2;
c->h264_idct8_add4 = ff_h264_idct8_add4_10_sse2;
#endif /* HAVE_ALIGNED_STACK */
c->weight_h264_pixels_tab[0] = ff_h264_weight_16_10_sse2;
c->weight_h264_pixels_tab[1] = ff_h264_weight_8_10_sse2;
c->weight_h264_pixels_tab[2] = ff_h264_weight_4_10_sse2;
c->biweight_h264_pixels_tab[0] = ff_h264_biweight_16_10_sse2;
c->biweight_h264_pixels_tab[1] = ff_h264_biweight_8_10_sse2;
c->biweight_h264_pixels_tab[2] = ff_h264_biweight_4_10_sse2;
c->h264_v_loop_filter_chroma = ff_deblock_v_chroma_10_sse2;
c->h264_v_loop_filter_chroma_intra = ff_deblock_v_chroma_intra_10_sse2;
if (chroma_format_idc <= 1) {
c->h264_h_loop_filter_chroma = ff_deblock_h_chroma_10_sse2;
} else {
c->h264_h_loop_filter_chroma = ff_deblock_h_chroma422_10_sse2;
}
#if HAVE_ALIGNED_STACK
c->h264_v_loop_filter_luma = ff_deblock_v_luma_10_sse2;
c->h264_h_loop_filter_luma = ff_deblock_h_luma_10_sse2;
c->h264_v_loop_filter_luma_intra = ff_deblock_v_luma_intra_10_sse2;
c->h264_h_loop_filter_luma_intra = ff_deblock_h_luma_intra_10_sse2;
#endif /* HAVE_ALIGNED_STACK */
}
if (EXTERNAL_SSE4(cpu_flags)) {
c->weight_h264_pixels_tab[0] = ff_h264_weight_16_10_sse4;
c->weight_h264_pixels_tab[1] = ff_h264_weight_8_10_sse4;
c->weight_h264_pixels_tab[2] = ff_h264_weight_4_10_sse4;
c->biweight_h264_pixels_tab[0] = ff_h264_biweight_16_10_sse4;
c->biweight_h264_pixels_tab[1] = ff_h264_biweight_8_10_sse4;
c->biweight_h264_pixels_tab[2] = ff_h264_biweight_4_10_sse4;
}
if (EXTERNAL_AVX(cpu_flags)) {
c->h264_idct_dc_add =
c->h264_idct_add = ff_h264_idct_add_10_avx;
c->h264_idct8_dc_add = ff_h264_idct8_dc_add_10_avx;
c->h264_idct_add16 = ff_h264_idct_add16_10_avx;
if (chroma_format_idc <= 1) {
c->h264_idct_add8 = ff_h264_idct_add8_10_avx;
} else {
c->h264_idct_add8 = ff_h264_idct_add8_422_10_avx;
}
c->h264_idct_add16intra = ff_h264_idct_add16intra_10_avx;
#if HAVE_ALIGNED_STACK
c->h264_idct8_add = ff_h264_idct8_add_10_avx;
c->h264_idct8_add4 = ff_h264_idct8_add4_10_avx;
#endif /* HAVE_ALIGNED_STACK */
c->h264_v_loop_filter_chroma = ff_deblock_v_chroma_10_avx;
c->h264_v_loop_filter_chroma_intra = ff_deblock_v_chroma_intra_10_avx;
if (chroma_format_idc <= 1) {
c->h264_h_loop_filter_chroma = ff_deblock_h_chroma_10_avx;
} else {
c->h264_h_loop_filter_chroma = ff_deblock_h_chroma422_10_avx;
}
#if HAVE_ALIGNED_STACK
c->h264_v_loop_filter_luma = ff_deblock_v_luma_10_avx;
c->h264_h_loop_filter_luma = ff_deblock_h_luma_10_avx;
c->h264_v_loop_filter_luma_intra = ff_deblock_v_luma_intra_10_avx;
c->h264_h_loop_filter_luma_intra = ff_deblock_h_luma_intra_10_avx;
#endif /* HAVE_ALIGNED_STACK */
}
}
#endif
}

369
media/ffvpx/libavcodec/x86/hevc_add_res.asm Normal file
View file

@ -0,0 +1,369 @@
; *****************************************************************************
; * Provide SIMD optimizations for add_residual functions for HEVC decoding
; * Copyright (c) 2014 Pierre-Edouard LEPERE
; *
; * This file is part of FFmpeg.
; *
; * FFmpeg is free software; you can redistribute it and/or
; * modify it under the terms of the GNU Lesser General Public
; * License as published by the Free Software Foundation; either
; * version 2.1 of the License, or (at your option) any later version.
; *
; * FFmpeg is distributed in the hope that it will be useful,
; * but WITHOUT ANY WARRANTY; without even the implied warranty of
; * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
; * Lesser General Public License for more details.
; *
; * You should have received a copy of the GNU Lesser General Public
; * License along with FFmpeg; if not, write to the Free Software
; * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
; ******************************************************************************
%include "libavutil/x86/x86util.asm"
SECTION .text
cextern pw_1023
%define max_pixels_10 pw_1023
; the add_res macros and functions were largely inspired by h264_idct.asm from the x264 project
%macro ADD_RES_MMX_4_8 0
mova m0, [r1]
mova m2, [r1+8]
pxor m1, m1
pxor m3, m3
psubw m1, m0
psubw m3, m2
packuswb m0, m2
packuswb m1, m3
movd m2, [r0]
movd m3, [r0+r2]
punpckldq m2, m3
paddusb m0, m2
psubusb m0, m1
movd [r0], m0
psrlq m0, 32
movd [r0+r2], m0
%endmacro
INIT_MMX mmxext
; void ff_hevc_add_residual_4_8_mmxext(uint8_t *dst, int16_t *res, ptrdiff_t stride)
cglobal hevc_add_residual_4_8, 3, 3, 6
ADD_RES_MMX_4_8
add r1, 16
lea r0, [r0+r2*2]
ADD_RES_MMX_4_8
RET
%macro ADD_RES_SSE_8_8 0
pxor m3, m3
mova m4, [r1]
mova m6, [r1+16]
mova m0, [r1+32]
mova m2, [r1+48]
psubw m5, m3, m4
psubw m7, m3, m6
psubw m1, m3, m0
packuswb m4, m0
packuswb m5, m1
psubw m3, m2
packuswb m6, m2
packuswb m7, m3
movq m0, [r0]
movq m1, [r0+r2]
movhps m0, [r0+r2*2]
movhps m1, [r0+r3]
paddusb m0, m4
paddusb m1, m6
psubusb m0, m5
psubusb m1, m7
movq [r0], m0
movq [r0+r2], m1
movhps [r0+2*r2], m0
movhps [r0+r3], m1
%endmacro
%macro ADD_RES_SSE_16_32_8 3
mova xm2, [r1+%1]
mova xm6, [r1+%1+16]
%if cpuflag(avx2)
vinserti128 m2, m2, [r1+%1+32], 1
vinserti128 m6, m6, [r1+%1+48], 1
%endif
psubw m1, m0, m2
psubw m5, m0, m6
packuswb m2, m6
packuswb m1, m5
mova xm4, [r1+%1+mmsize*2]
mova xm6, [r1+%1+mmsize*2+16]
%if cpuflag(avx2)
vinserti128 m4, m4, [r1+%1+96 ], 1
vinserti128 m6, m6, [r1+%1+112], 1
%endif
psubw m3, m0, m4
psubw m5, m0, m6
packuswb m4, m6
packuswb m3, m5
paddusb m2, [%2]
paddusb m4, [%3]
psubusb m2, m1
psubusb m4, m3
mova [%2], m2
mova [%3], m4
%endmacro
%macro TRANSFORM_ADD_8 0
; void ff_hevc_add_residual_8_8_<opt>(uint8_t *dst, int16_t *res, ptrdiff_t stride)
cglobal hevc_add_residual_8_8, 3, 4, 8
lea r3, [r2*3]
ADD_RES_SSE_8_8
add r1, 64
lea r0, [r0+r2*4]
ADD_RES_SSE_8_8
RET
; void ff_hevc_add_residual_16_8_<opt>(uint8_t *dst, int16_t *res, ptrdiff_t stride)
cglobal hevc_add_residual_16_8, 3, 5, 7
pxor m0, m0
lea r3, [r2*3]
mov r4d, 4
.loop:
ADD_RES_SSE_16_32_8 0, r0, r0+r2
ADD_RES_SSE_16_32_8 64, r0+r2*2, r0+r3
add r1, 128
lea r0, [r0+r2*4]
dec r4d
jg .loop
RET
; void ff_hevc_add_residual_32_8_<opt>(uint8_t *dst, int16_t *res, ptrdiff_t stride)
cglobal hevc_add_residual_32_8, 3, 5, 7
pxor m0, m0
mov r4d, 16
.loop:
ADD_RES_SSE_16_32_8 0, r0, r0+16
ADD_RES_SSE_16_32_8 64, r0+r2, r0+r2+16
add r1, 128
lea r0, [r0+r2*2]
dec r4d
jg .loop
RET
%endmacro
INIT_XMM sse2
TRANSFORM_ADD_8
INIT_XMM avx
TRANSFORM_ADD_8
%if HAVE_AVX2_EXTERNAL
INIT_YMM avx2
; void ff_hevc_add_residual_32_8_avx2(uint8_t *dst, int16_t *res, ptrdiff_t stride)
cglobal hevc_add_residual_32_8, 3, 5, 7
pxor m0, m0
lea r3, [r2*3]
mov r4d, 8
.loop:
ADD_RES_SSE_16_32_8 0, r0, r0+r2
ADD_RES_SSE_16_32_8 128, r0+r2*2, r0+r3
add r1, 256
lea r0, [r0+r2*4]
dec r4d
jg .loop
RET
%endif ;HAVE_AVX2_EXTERNAL
%macro ADD_RES_SSE_8_10 4
mova m0, [%4]
mova m1, [%4+16]
mova m2, [%4+32]
mova m3, [%4+48]
paddw m0, [%1+0]
paddw m1, [%1+%2]
paddw m2, [%1+%2*2]
paddw m3, [%1+%3]
CLIPW m0, m4, m5
CLIPW m1, m4, m5
CLIPW m2, m4, m5
CLIPW m3, m4, m5
mova [%1+0], m0
mova [%1+%2], m1
mova [%1+%2*2], m2
mova [%1+%3], m3
%endmacro
%macro ADD_RES_MMX_4_10 3
mova m0, [%1+0]
mova m1, [%1+%2]
paddw m0, [%3]
paddw m1, [%3+8]
CLIPW m0, m2, m3
CLIPW m1, m2, m3
mova [%1+0], m0
mova [%1+%2], m1
%endmacro
%macro ADD_RES_SSE_16_10 3
mova m0, [%3]
mova m1, [%3+16]
mova m2, [%3+32]
mova m3, [%3+48]
paddw m0, [%1]
paddw m1, [%1+16]
paddw m2, [%1+%2]
paddw m3, [%1+%2+16]
CLIPW m0, m4, m5
CLIPW m1, m4, m5
CLIPW m2, m4, m5
CLIPW m3, m4, m5
mova [%1], m0
mova [%1+16], m1
mova [%1+%2], m2
mova [%1+%2+16], m3
%endmacro
%macro ADD_RES_SSE_32_10 2
mova m0, [%2]
mova m1, [%2+16]
mova m2, [%2+32]
mova m3, [%2+48]
paddw m0, [%1]
paddw m1, [%1+16]
paddw m2, [%1+32]
paddw m3, [%1+48]
CLIPW m0, m4, m5
CLIPW m1, m4, m5
CLIPW m2, m4, m5
CLIPW m3, m4, m5
mova [%1], m0
mova [%1+16], m1
mova [%1+32], m2
mova [%1+48], m3
%endmacro
%macro ADD_RES_AVX2_16_10 4
mova m0, [%4]
mova m1, [%4+32]
mova m2, [%4+64]
mova m3, [%4+96]
paddw m0, [%1+0]
paddw m1, [%1+%2]
paddw m2, [%1+%2*2]
paddw m3, [%1+%3]
CLIPW m0, m4, m5
CLIPW m1, m4, m5
CLIPW m2, m4, m5
CLIPW m3, m4, m5
mova [%1+0], m0
mova [%1+%2], m1
mova [%1+%2*2], m2
mova [%1+%3], m3
%endmacro
%macro ADD_RES_AVX2_32_10 3
mova m0, [%3]
mova m1, [%3+32]
mova m2, [%3+64]
mova m3, [%3+96]
paddw m0, [%1]
paddw m1, [%1+32]
paddw m2, [%1+%2]
paddw m3, [%1+%2+32]
CLIPW m0, m4, m5
CLIPW m1, m4, m5
CLIPW m2, m4, m5
CLIPW m3, m4, m5
mova [%1], m0
mova [%1+32], m1
mova [%1+%2], m2
mova [%1+%2+32], m3
%endmacro
; void ff_hevc_add_residual_<4|8|16|32>_10(pixel *dst, int16_t *block, ptrdiff_t stride)
INIT_MMX mmxext
cglobal hevc_add_residual_4_10, 3, 3, 6
pxor m2, m2
mova m3, [max_pixels_10]
ADD_RES_MMX_4_10 r0, r2, r1
add r1, 16
lea r0, [r0+2*r2]
ADD_RES_MMX_4_10 r0, r2, r1
RET
INIT_XMM sse2
cglobal hevc_add_residual_8_10, 3, 4, 6
pxor m4, m4
mova m5, [max_pixels_10]
lea r3, [r2*3]
ADD_RES_SSE_8_10 r0, r2, r3, r1
lea r0, [r0+r2*4]
add r1, 64
ADD_RES_SSE_8_10 r0, r2, r3, r1
RET
cglobal hevc_add_residual_16_10, 3, 5, 6
pxor m4, m4
mova m5, [max_pixels_10]
mov r4d, 8
.loop:
ADD_RES_SSE_16_10 r0, r2, r1
lea r0, [r0+r2*2]
add r1, 64
dec r4d
jg .loop
RET
cglobal hevc_add_residual_32_10, 3, 5, 6
pxor m4, m4
mova m5, [max_pixels_10]
mov r4d, 32
.loop:
ADD_RES_SSE_32_10 r0, r1
lea r0, [r0+r2]
add r1, 64
dec r4d
jg .loop
RET
%if HAVE_AVX2_EXTERNAL
INIT_YMM avx2
cglobal hevc_add_residual_16_10, 3, 5, 6
pxor m4, m4
mova m5, [max_pixels_10]
lea r3, [r2*3]
mov r4d, 4
.loop:
ADD_RES_AVX2_16_10 r0, r2, r3, r1
lea r0, [r0+r2*4]
add r1, 128
dec r4d
jg .loop
RET
cglobal hevc_add_residual_32_10, 3, 5, 6
pxor m4, m4
mova m5, [max_pixels_10]
mov r4d, 16
.loop:
ADD_RES_AVX2_32_10 r0, r2, r1
lea r0, [r0+r2*2]
add r1, 128
dec r4d
jg .loop
RET
%endif ;HAVE_AVX2_EXTERNAL

871
media/ffvpx/libavcodec/x86/hevc_deblock.asm Normal file
View file

@ -0,0 +1,871 @@
;*****************************************************************************
;* SSE2-optimized HEVC deblocking code
;*****************************************************************************
;* Copyright (C) 2013 VTT
;*
;* Authors: Seppo Tomperi <seppo.tomperi@vtt.fi>
;*
;* This file is part of FFmpeg.
;*
;* FFmpeg is free software; you can redistribute it and/or
;* modify it under the terms of the GNU Lesser General Public
;* License as published by the Free Software Foundation; either
;* version 2.1 of the License, or (at your option) any later version.
;*
;* FFmpeg is distributed in the hope that it will be useful,
;* but WITHOUT ANY WARRANTY; without even the implied warranty of
;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
;* Lesser General Public License for more details.
;*
;* You should have received a copy of the GNU Lesser General Public
;* License along with FFmpeg; if not, write to the Free Software
;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
;******************************************************************************
%include "libavutil/x86/x86util.asm"
SECTION_RODATA
cextern pw_1023
%define pw_pixel_max_10 pw_1023
pw_pixel_max_12: times 8 dw ((1 << 12)-1)
pw_m2: times 8 dw -2
pd_1 : times 4 dd 1
cextern pw_4
cextern pw_8
cextern pw_m1
SECTION .text
INIT_XMM sse2
; in: 8 rows of 4 bytes in %4..%11
; out: 4 rows of 8 words in m0..m3
%macro TRANSPOSE4x8B_LOAD 8
movd m0, %1
movd m2, %2
movd m1, %3
movd m3, %4
punpcklbw m0, m2
punpcklbw m1, m3
punpcklwd m0, m1
movd m4, %5
movd m6, %6
movd m5, %7
movd m3, %8
punpcklbw m4, m6
punpcklbw m5, m3
punpcklwd m4, m5
punpckhdq m2, m0, m4
punpckldq m0, m4
pxor m5, m5
punpckhbw m1, m0, m5
punpcklbw m0, m5
punpckhbw m3, m2, m5
punpcklbw m2, m5
%endmacro
; in: 4 rows of 8 words in m0..m3
; out: 8 rows of 4 bytes in %1..%8
%macro TRANSPOSE8x4B_STORE 8
packuswb m0, m2
packuswb m1, m3
SBUTTERFLY bw, 0, 1, 2
SBUTTERFLY wd, 0, 1, 2
movd %1, m0
pshufd m0, m0, 0x39
movd %2, m0
pshufd m0, m0, 0x39
movd %3, m0
pshufd m0, m0, 0x39
movd %4, m0
movd %5, m1
pshufd m1, m1, 0x39
movd %6, m1
pshufd m1, m1, 0x39
movd %7, m1
pshufd m1, m1, 0x39
movd %8, m1
%endmacro
; in: 8 rows of 4 words in %4..%11
; out: 4 rows of 8 words in m0..m3
%macro TRANSPOSE4x8W_LOAD 8
movq m0, %1
movq m2, %2
movq m1, %3
movq m3, %4
punpcklwd m0, m2
punpcklwd m1, m3
punpckhdq m2, m0, m1
punpckldq m0, m1
movq m4, %5
movq m6, %6
movq m5, %7
movq m3, %8
punpcklwd m4, m6
punpcklwd m5, m3
punpckhdq m6, m4, m5
punpckldq m4, m5
punpckhqdq m1, m0, m4
punpcklqdq m0, m4
punpckhqdq m3, m2, m6
punpcklqdq m2, m6
%endmacro
; in: 4 rows of 8 words in m0..m3
; out: 8 rows of 4 words in %1..%8
%macro TRANSPOSE8x4W_STORE 9
TRANSPOSE4x4W 0, 1, 2, 3, 4
pxor m5, m5; zeros reg
CLIPW m0, m5, %9
CLIPW m1, m5, %9
CLIPW m2, m5, %9
CLIPW m3, m5, %9
movq %1, m0
movhps %2, m0
movq %3, m1
movhps %4, m1
movq %5, m2
movhps %6, m2
movq %7, m3
movhps %8, m3
%endmacro
; in: 8 rows of 8 bytes in %1..%8
; out: 8 rows of 8 words in m0..m7
%macro TRANSPOSE8x8B_LOAD 8
movq m7, %1
movq m2, %2
movq m1, %3
movq m3, %4
punpcklbw m7, m2
punpcklbw m1, m3
punpcklwd m3, m7, m1
punpckhwd m7, m1
movq m4, %5
movq m6, %6
movq m5, %7
movq m15, %8
punpcklbw m4, m6
punpcklbw m5, m15
punpcklwd m9, m4, m5
punpckhwd m4, m5
punpckldq m1, m3, m9; 0, 1
punpckhdq m3, m9; 2, 3
punpckldq m5, m7, m4; 4, 5
punpckhdq m7, m4; 6, 7
pxor m13, m13
punpcklbw m0, m1, m13; 0 in 16 bit
punpckhbw m1, m13; 1 in 16 bit
punpcklbw m2, m3, m13; 2
punpckhbw m3, m13; 3
punpcklbw m4, m5, m13; 4
punpckhbw m5, m13; 5
punpcklbw m6, m7, m13; 6
punpckhbw m7, m13; 7
%endmacro
; in: 8 rows of 8 words in m0..m8
; out: 8 rows of 8 bytes in %1..%8
%macro TRANSPOSE8x8B_STORE 8
packuswb m0, m4
packuswb m1, m5
packuswb m2, m6
packuswb m3, m7
TRANSPOSE2x4x4B 0, 1, 2, 3, 4
movq %1, m0
movhps %2, m0
movq %3, m1
movhps %4, m1
movq %5, m2
movhps %6, m2
movq %7, m3
movhps %8, m3
%endmacro
; in: 8 rows of 8 words in %1..%8
; out: 8 rows of 8 words in m0..m7
%macro TRANSPOSE8x8W_LOAD 8
movdqu m0, %1
movdqu m1, %2
movdqu m2, %3
movdqu m3, %4
movdqu m4, %5
movdqu m5, %6
movdqu m6, %7
movdqu m7, %8
TRANSPOSE8x8W 0, 1, 2, 3, 4, 5, 6, 7, 8
%endmacro
; in: 8 rows of 8 words in m0..m8
; out: 8 rows of 8 words in %1..%8
%macro TRANSPOSE8x8W_STORE 9
TRANSPOSE8x8W 0, 1, 2, 3, 4, 5, 6, 7, 8
pxor m8, m8
CLIPW m0, m8, %9
CLIPW m1, m8, %9
CLIPW m2, m8, %9
CLIPW m3, m8, %9
CLIPW m4, m8, %9
CLIPW m5, m8, %9
CLIPW m6, m8, %9
CLIPW m7, m8, %9
movdqu %1, m0
movdqu %2, m1
movdqu %3, m2
movdqu %4, m3
movdqu %5, m4
movdqu %6, m5
movdqu %7, m6
movdqu %8, m7
%endmacro
; in: %2 clobbered
; out: %1
; mask in m11
; clobbers m10
%macro MASKED_COPY 2
pand %2, m11 ; and mask
pandn m10, m11, %1; and -mask
por %2, m10
mova %1, %2
%endmacro
; in: %2 clobbered
; out: %1
; mask in %3, will be clobbered
%macro MASKED_COPY2 3
pand %2, %3 ; and mask
pandn %3, %1; and -mask
por %2, %3
mova %1, %2
%endmacro
ALIGN 16
; input in m0 ... m3 and tcs in r2. Output in m1 and m2
%macro CHROMA_DEBLOCK_BODY 1
psubw m4, m2, m1; q0 - p0
psubw m5, m0, m3; p1 - q1
psllw m4, 2; << 2
paddw m5, m4;
;tc calculations
movq m6, [tcq]; tc0
punpcklwd m6, m6
pshufd m6, m6, 0xA0; tc0, tc1
%if cpuflag(ssse3)
psignw m4, m6, [pw_m1]; -tc0, -tc1
%else
pmullw m4, m6, [pw_m1]; -tc0, -tc1
%endif
;end tc calculations
paddw m5, [pw_4]; +4
psraw m5, 3; >> 3
%if %1 > 8
psllw m4, %1-8; << (BIT_DEPTH - 8)
psllw m6, %1-8; << (BIT_DEPTH - 8)
%endif
pmaxsw m5, m4
pminsw m5, m6
paddw m1, m5; p0 + delta0
psubw m2, m5; q0 - delta0
%endmacro
; input in m0 ... m7, beta in r2 tcs in r3. Output in m1...m6
%macro LUMA_DEBLOCK_BODY 2
psllw m9, m2, 1; *2
psubw m10, m1, m9
paddw m10, m3
ABS1 m10, m11 ; 0dp0, 0dp3 , 1dp0, 1dp3
psllw m9, m5, 1; *2
psubw m11, m6, m9
paddw m11, m4
ABS1 m11, m13 ; 0dq0, 0dq3 , 1dq0, 1dq3
;beta calculations
%if %1 > 8
shl betaq, %1 - 8
%endif
movd m13, betad
SPLATW m13, m13, 0
;end beta calculations
paddw m9, m10, m11; 0d0, 0d3 , 1d0, 1d3
pshufhw m14, m9, 0x0f ;0b00001111; 0d3 0d3 0d0 0d0 in high
pshuflw m14, m14, 0x0f ;0b00001111; 1d3 1d3 1d0 1d0 in low
pshufhw m9, m9, 0xf0 ;0b11110000; 0d0 0d0 0d3 0d3
pshuflw m9, m9, 0xf0 ;0b11110000; 1d0 1d0 1d3 1d3
paddw m14, m9; 0d0+0d3, 1d0+1d3
;compare
pcmpgtw m15, m13, m14
movmskps r13, m15 ;filtering mask 0d0 + 0d3 < beta0 (bit 2 or 3) , 1d0 + 1d3 < beta1 (bit 0 or 1)
test r13, r13
je .bypassluma
;weak / strong decision compare to beta_2
psraw m15, m13, 2; beta >> 2
psllw m8, m9, 1;
pcmpgtw m15, m8; (d0 << 1) < beta_2, (d3 << 1) < beta_2
movmskps r6, m15;
;end weak / strong decision
; weak filter nd_p/q calculation
pshufd m8, m10, 0x31
psrld m8, 16
paddw m8, m10
movd r7d, m8
pshufd m8, m8, 0x4E
movd r8d, m8
pshufd m8, m11, 0x31
psrld m8, 16
paddw m8, m11
movd r9d, m8
pshufd m8, m8, 0x4E
movd r10d, m8
; end calc for weak filter
; filtering mask
mov r11, r13
shr r11, 3
movd m15, r11d
and r13, 1
movd m11, r13d
shufps m11, m15, 0
shl r11, 1
or r13, r11
pcmpeqd m11, [pd_1]; filtering mask
;decide between strong and weak filtering
;tc25 calculations
mov r11d, [tcq];
%if %1 > 8
shl r11, %1 - 8
%endif
movd m8, r11d; tc0
mov r3d, [tcq+4];
%if %1 > 8
shl r3, %1 - 8
%endif
add r11d, r3d; tc0 + tc1
jz .bypassluma
movd m9, r3d; tc1
punpcklwd m8, m8
punpcklwd m9, m9
shufps m8, m9, 0; tc0, tc1
mova m9, m8
psllw m8, 2; tc << 2
pavgw m8, m9; tc25 = ((tc * 5 + 1) >> 1)
;end tc25 calculations
;----beta_3 comparison-----
psubw m12, m0, m3; p3 - p0
ABS1 m12, m14; abs(p3 - p0)
psubw m15, m7, m4; q3 - q0
ABS1 m15, m14; abs(q3 - q0)
paddw m12, m15; abs(p3 - p0) + abs(q3 - q0)
pshufhw m12, m12, 0xf0 ;0b11110000;
pshuflw m12, m12, 0xf0 ;0b11110000;
psraw m13, 3; beta >> 3
pcmpgtw m13, m12;
movmskps r11, m13;
and r6, r11; strong mask , beta_2 and beta_3 comparisons
;----beta_3 comparison end-----
;----tc25 comparison---
psubw m12, m3, m4; p0 - q0
ABS1 m12, m14; abs(p0 - q0)
pshufhw m12, m12, 0xf0 ;0b11110000;
pshuflw m12, m12, 0xf0 ;0b11110000;
pcmpgtw m8, m12; tc25 comparisons
movmskps r11, m8;
and r6, r11; strong mask, beta_2, beta_3 and tc25 comparisons
;----tc25 comparison end---
mov r11, r6;
shr r11, 1;
and r6, r11; strong mask, bits 2 and 0
pmullw m14, m9, [pw_m2]; -tc * 2
paddw m9, m9
and r6, 5; 0b101
mov r11, r6; strong mask
shr r6, 2;
movd m12, r6d; store to xmm for mask generation
shl r6, 1
and r11, 1
movd m10, r11d; store to xmm for mask generation
or r6, r11; final strong mask, bits 1 and 0
jz .weakfilter
shufps m10, m12, 0
pcmpeqd m10, [pd_1]; strong mask
mova m13, [pw_4]; 4 in every cell
pand m11, m10; combine filtering mask and strong mask
paddw m12, m2, m3; p1 + p0
paddw m12, m4; p1 + p0 + q0
mova m10, m12; copy
paddw m12, m12; 2*p1 + 2*p0 + 2*q0
paddw m12, m1; p2 + 2*p1 + 2*p0 + 2*q0
paddw m12, m5; p2 + 2*p1 + 2*p0 + 2*q0 + q1
paddw m12, m13; p2 + 2*p1 + 2*p0 + 2*q0 + q1 + 4
psraw m12, 3; ((p2 + 2*p1 + 2*p0 + 2*q0 + q1 + 4) >> 3)
psubw m12, m3; ((p2 + 2*p1 + 2*p0 + 2*q0 + q1 + 4) >> 3) - p0
pmaxsw m12, m14
pminsw m12, m9; av_clip( , -2 * tc, 2 * tc)
paddw m12, m3; p0'
paddw m15, m1, m10; p2 + p1 + p0 + q0
psrlw m13, 1; 2 in every cell
paddw m15, m13; p2 + p1 + p0 + q0 + 2
psraw m15, 2; (p2 + p1 + p0 + q0 + 2) >> 2
psubw m15, m2;((p2 + p1 + p0 + q0 + 2) >> 2) - p1
pmaxsw m15, m14
pminsw m15, m9; av_clip( , -2 * tc, 2 * tc)
paddw m15, m2; p1'
paddw m8, m1, m0; p3 + p2
paddw m8, m8; 2*p3 + 2*p2
paddw m8, m1; 2*p3 + 3*p2
paddw m8, m10; 2*p3 + 3*p2 + p1 + p0 + q0
paddw m13, m13
paddw m8, m13; 2*p3 + 3*p2 + p1 + p0 + q0 + 4
psraw m8, 3; (2*p3 + 3*p2 + p1 + p0 + q0 + 4) >> 3
psubw m8, m1; ((2*p3 + 3*p2 + p1 + p0 + q0 + 4) >> 3) - p2
pmaxsw m8, m14
pminsw m8, m9; av_clip( , -2 * tc, 2 * tc)
paddw m8, m1; p2'
MASKED_COPY m1, m8
paddw m8, m3, m4; p0 + q0
paddw m8, m5; p0 + q0 + q1
paddw m8, m8; 2*p0 + 2*q0 + 2*q1
paddw m8, m2; p1 + 2*p0 + 2*q0 + 2*q1
paddw m8, m6; p1 + 2*p0 + 2*q0 + 2*q1 + q2
paddw m8, m13; p1 + 2*p0 + 2*q0 + 2*q1 + q2 + 4
psraw m8, 3; (p1 + 2*p0 + 2*q0 + 2*q1 + q2 + 4) >>3
psubw m8, m4;
pmaxsw m8, m14
pminsw m8, m9; av_clip( , -2 * tc, 2 * tc)
paddw m8, m4; q0'
MASKED_COPY m2, m15
paddw m15, m3, m4; p0 + q0
paddw m15, m5; p0 + q0 + q1
mova m10, m15;
paddw m15, m6; p0 + q0 + q1 + q2
psrlw m13, 1; 2 in every cell
paddw m15, m13; p0 + q0 + q1 + q2 + 2
psraw m15, 2; (p0 + q0 + q1 + q2 + 2) >> 2
psubw m15, m5; ((p0 + q0 + q1 + q2 + 2) >> 2) - q1
pmaxsw m15, m14
pminsw m15, m9; av_clip( , -2 * tc, 2 * tc)
paddw m15, m5; q1'
paddw m13, m7; q3 + 2
paddw m13, m6; q3 + q2 + 2
paddw m13, m13; 2*q3 + 2*q2 + 4
paddw m13, m6; 2*q3 + 3*q2 + 4
paddw m13, m10; 2*q3 + 3*q2 + q1 + q0 + p0 + 4
psraw m13, 3; (2*q3 + 3*q2 + q1 + q0 + p0 + 4) >> 3
psubw m13, m6; ((2*q3 + 3*q2 + q1 + q0 + p0 + 4) >> 3) - q2
pmaxsw m13, m14
pminsw m13, m9; av_clip( , -2 * tc, 2 * tc)
paddw m13, m6; q2'
MASKED_COPY m6, m13
MASKED_COPY m5, m15
MASKED_COPY m4, m8
MASKED_COPY m3, m12
.weakfilter:
not r6; strong mask -> weak mask
and r6, r13; final weak filtering mask, bits 0 and 1
jz .store
; weak filtering mask
mov r11, r6
shr r11, 1
movd m12, r11d
and r6, 1
movd m11, r6d
shufps m11, m12, 0
pcmpeqd m11, [pd_1]; filtering mask
mov r13, betaq
shr r13, 1;
add betaq, r13
shr betaq, 3; ((beta + (beta >> 1)) >> 3))
mova m13, [pw_8]
psubw m12, m4, m3 ; q0 - p0
psllw m10, m12, 3; 8 * (q0 - p0)
paddw m12, m10 ; 9 * (q0 - p0)
psubw m10, m5, m2 ; q1 - p1
psllw m8, m10, 1; 2 * ( q1 - p1 )
paddw m10, m8; 3 * ( q1 - p1 )
psubw m12, m10; 9 * (q0 - p0) - 3 * ( q1 - p1 )
paddw m12, m13; + 8
psraw m12, 4; >> 4 , delta0
PABSW m13, m12; abs(delta0)
psllw m10, m9, 2; 8 * tc
paddw m10, m9; 10 * tc
pcmpgtw m10, m13
pand m11, m10
psraw m9, 1; tc * 2 -> tc
psraw m14, 1; -tc * 2 -> -tc
pmaxsw m12, m14
pminsw m12, m9; av_clip(delta0, -tc, tc)
psraw m9, 1; tc -> tc / 2
%if cpuflag(ssse3)
psignw m14, m9, [pw_m1]; -tc / 2
%else
pmullw m14, m9, [pw_m1]; -tc / 2
%endif
pavgw m15, m1, m3; (p2 + p0 + 1) >> 1
psubw m15, m2; ((p2 + p0 + 1) >> 1) - p1
paddw m15, m12; ((p2 + p0 + 1) >> 1) - p1 + delta0
psraw m15, 1; (((p2 + p0 + 1) >> 1) - p1 + delta0) >> 1
pmaxsw m15, m14
pminsw m15, m9; av_clip(deltap1, -tc/2, tc/2)
paddw m15, m2; p1'
;beta calculations
movd m10, betad
SPLATW m10, m10, 0
movd m13, r7d; 1dp0 + 1dp3
movd m8, r8d; 0dp0 + 0dp3
punpcklwd m8, m8
punpcklwd m13, m13
shufps m13, m8, 0;
pcmpgtw m8, m10, m13
pand m8, m11
;end beta calculations
MASKED_COPY2 m2, m15, m8; write p1'
pavgw m8, m6, m4; (q2 + q0 + 1) >> 1
psubw m8, m5; ((q2 + q0 + 1) >> 1) - q1
psubw m8, m12; ((q2 + q0 + 1) >> 1) - q1 - delta0)
psraw m8, 1; ((q2 + q0 + 1) >> 1) - q1 - delta0) >> 1
pmaxsw m8, m14
pminsw m8, m9; av_clip(deltaq1, -tc/2, tc/2)
paddw m8, m5; q1'
movd m13, r9d;
movd m15, r10d;
punpcklwd m15, m15
punpcklwd m13, m13
shufps m13, m15, 0; dq0 + dq3
pcmpgtw m10, m13; compare to ((beta+(beta>>1))>>3)
pand m10, m11
MASKED_COPY2 m5, m8, m10; write q1'
paddw m15, m3, m12 ; p0 + delta0
MASKED_COPY m3, m15
psubw m8, m4, m12 ; q0 - delta0
MASKED_COPY m4, m8
%endmacro
;-----------------------------------------------------------------------------
; void ff_hevc_v_loop_filter_chroma(uint8_t *_pix, ptrdiff_t _stride, int32_t *tc,
; uint8_t *_no_p, uint8_t *_no_q);
;-----------------------------------------------------------------------------
%macro LOOP_FILTER_CHROMA 0
cglobal hevc_v_loop_filter_chroma_8, 3, 5, 7, pix, stride, tc, pix0, r3stride
sub pixq, 2
lea r3strideq, [3*strideq]
mov pix0q, pixq
add pixq, r3strideq
TRANSPOSE4x8B_LOAD PASS8ROWS(pix0q, pixq, strideq, r3strideq)
CHROMA_DEBLOCK_BODY 8
TRANSPOSE8x4B_STORE PASS8ROWS(pix0q, pixq, strideq, r3strideq)
RET
cglobal hevc_v_loop_filter_chroma_10, 3, 5, 7, pix, stride, tc, pix0, r3stride
sub pixq, 4
lea r3strideq, [3*strideq]
mov pix0q, pixq
add pixq, r3strideq
TRANSPOSE4x8W_LOAD PASS8ROWS(pix0q, pixq, strideq, r3strideq)
CHROMA_DEBLOCK_BODY 10
TRANSPOSE8x4W_STORE PASS8ROWS(pix0q, pixq, strideq, r3strideq), [pw_pixel_max_10]
RET
cglobal hevc_v_loop_filter_chroma_12, 3, 5, 7, pix, stride, tc, pix0, r3stride
sub pixq, 4
lea r3strideq, [3*strideq]
mov pix0q, pixq
add pixq, r3strideq
TRANSPOSE4x8W_LOAD PASS8ROWS(pix0q, pixq, strideq, r3strideq)
CHROMA_DEBLOCK_BODY 12
TRANSPOSE8x4W_STORE PASS8ROWS(pix0q, pixq, strideq, r3strideq), [pw_pixel_max_12]
RET
;-----------------------------------------------------------------------------
; void ff_hevc_h_loop_filter_chroma(uint8_t *_pix, ptrdiff_t _stride, int32_t *tc,
; uint8_t *_no_p, uint8_t *_no_q);
;-----------------------------------------------------------------------------
cglobal hevc_h_loop_filter_chroma_8, 3, 4, 7, pix, stride, tc, pix0
mov pix0q, pixq
sub pix0q, strideq
sub pix0q, strideq
movq m0, [pix0q]; p1
movq m1, [pix0q+strideq]; p0
movq m2, [pixq]; q0
movq m3, [pixq+strideq]; q1
pxor m5, m5; zeros reg
punpcklbw m0, m5
punpcklbw m1, m5
punpcklbw m2, m5
punpcklbw m3, m5
CHROMA_DEBLOCK_BODY 8
packuswb m1, m2
movh[pix0q+strideq], m1
movhps [pixq], m1
RET
cglobal hevc_h_loop_filter_chroma_10, 3, 4, 7, pix, stride, tc, pix0
mov pix0q, pixq
sub pix0q, strideq
sub pix0q, strideq
movu m0, [pix0q]; p1
movu m1, [pix0q+strideq]; p0
movu m2, [pixq]; q0
movu m3, [pixq+strideq]; q1
CHROMA_DEBLOCK_BODY 10
pxor m5, m5; zeros reg
CLIPW m1, m5, [pw_pixel_max_10]
CLIPW m2, m5, [pw_pixel_max_10]
movu [pix0q+strideq], m1
movu [pixq], m2
RET
cglobal hevc_h_loop_filter_chroma_12, 3, 4, 7, pix, stride, tc, pix0
mov pix0q, pixq
sub pix0q, strideq
sub pix0q, strideq
movu m0, [pix0q]; p1
movu m1, [pix0q+strideq]; p0
movu m2, [pixq]; q0
movu m3, [pixq+strideq]; q1
CHROMA_DEBLOCK_BODY 12
pxor m5, m5; zeros reg
CLIPW m1, m5, [pw_pixel_max_12]
CLIPW m2, m5, [pw_pixel_max_12]
movu [pix0q+strideq], m1
movu [pixq], m2
RET
%endmacro
INIT_XMM sse2
LOOP_FILTER_CHROMA
INIT_XMM avx
LOOP_FILTER_CHROMA
%if ARCH_X86_64
%macro LOOP_FILTER_LUMA 0
;-----------------------------------------------------------------------------
; void ff_hevc_v_loop_filter_luma(uint8_t *_pix, ptrdiff_t _stride, int beta,
; int32_t *tc, uint8_t *_no_p, uint8_t *_no_q);
;-----------------------------------------------------------------------------
cglobal hevc_v_loop_filter_luma_8, 4, 14, 16, pix, stride, beta, tc, pix0, src3stride
sub pixq, 4
lea pix0q, [3 * r1]
mov src3strideq, pixq
add pixq, pix0q
TRANSPOSE8x8B_LOAD PASS8ROWS(src3strideq, pixq, r1, pix0q)
LUMA_DEBLOCK_BODY 8, v
.store:
TRANSPOSE8x8B_STORE PASS8ROWS(src3strideq, pixq, r1, pix0q)
.bypassluma:
RET
cglobal hevc_v_loop_filter_luma_10, 4, 14, 16, pix, stride, beta, tc, pix0, src3stride
sub pixq, 8
lea pix0q, [3 * strideq]
mov src3strideq, pixq
add pixq, pix0q
TRANSPOSE8x8W_LOAD PASS8ROWS(src3strideq, pixq, strideq, pix0q)
LUMA_DEBLOCK_BODY 10, v
.store:
TRANSPOSE8x8W_STORE PASS8ROWS(src3strideq, pixq, r1, pix0q), [pw_pixel_max_10]
.bypassluma:
RET
cglobal hevc_v_loop_filter_luma_12, 4, 14, 16, pix, stride, beta, tc, pix0, src3stride
sub pixq, 8
lea pix0q, [3 * strideq]
mov src3strideq, pixq
add pixq, pix0q
TRANSPOSE8x8W_LOAD PASS8ROWS(src3strideq, pixq, strideq, pix0q)
LUMA_DEBLOCK_BODY 12, v
.store:
TRANSPOSE8x8W_STORE PASS8ROWS(src3strideq, pixq, r1, pix0q), [pw_pixel_max_12]
.bypassluma:
RET
;-----------------------------------------------------------------------------
; void ff_hevc_h_loop_filter_luma(uint8_t *_pix, ptrdiff_t _stride, int beta,
; int32_t *tc, uint8_t *_no_p, uint8_t *_no_q);
;-----------------------------------------------------------------------------
cglobal hevc_h_loop_filter_luma_8, 4, 14, 16, pix, stride, beta, tc, pix0, src3stride
lea src3strideq, [3 * strideq]
mov pix0q, pixq
sub pix0q, src3strideq
sub pix0q, strideq
movq m0, [pix0q]; p3
movq m1, [pix0q + strideq]; p2
movq m2, [pix0q + 2 * strideq]; p1
movq m3, [pix0q + src3strideq]; p0
movq m4, [pixq]; q0
movq m5, [pixq + strideq]; q1
movq m6, [pixq + 2 * strideq]; q2
movq m7, [pixq + src3strideq]; q3
pxor m8, m8
punpcklbw m0, m8
punpcklbw m1, m8
punpcklbw m2, m8
punpcklbw m3, m8
punpcklbw m4, m8
punpcklbw m5, m8
punpcklbw m6, m8
punpcklbw m7, m8
LUMA_DEBLOCK_BODY 8, h
.store:
packuswb m1, m2
packuswb m3, m4
packuswb m5, m6
movh [pix0q + strideq], m1
movhps [pix0q + 2 * strideq], m1
movh [pix0q + src3strideq], m3
movhps [pixq ], m3
movh [pixq + strideq], m5
movhps [pixq + 2 * strideq], m5
.bypassluma:
RET
cglobal hevc_h_loop_filter_luma_10, 4, 14, 16, pix, stride, beta, tc, pix0, src3stride
lea src3strideq, [3 * strideq]
mov pix0q, pixq
sub pix0q, src3strideq
sub pix0q, strideq
movdqu m0, [pix0q]; p3
movdqu m1, [pix0q + strideq]; p2
movdqu m2, [pix0q + 2 * strideq]; p1
movdqu m3, [pix0q + src3strideq]; p0
movdqu m4, [pixq]; q0
movdqu m5, [pixq + strideq]; q1
movdqu m6, [pixq + 2 * strideq]; q2
movdqu m7, [pixq + src3strideq]; q3
LUMA_DEBLOCK_BODY 10, h
.store:
pxor m8, m8; zeros reg
CLIPW m1, m8, [pw_pixel_max_10]
CLIPW m2, m8, [pw_pixel_max_10]
CLIPW m3, m8, [pw_pixel_max_10]
CLIPW m4, m8, [pw_pixel_max_10]
CLIPW m5, m8, [pw_pixel_max_10]
CLIPW m6, m8, [pw_pixel_max_10]
movdqu [pix0q + strideq], m1; p2
movdqu [pix0q + 2 * strideq], m2; p1
movdqu [pix0q + src3strideq], m3; p0
movdqu [pixq ], m4; q0
movdqu [pixq + strideq], m5; q1
movdqu [pixq + 2 * strideq], m6; q2
.bypassluma:
RET
cglobal hevc_h_loop_filter_luma_12, 4, 14, 16, pix, stride, beta, tc, pix0, src3stride
lea src3strideq, [3 * strideq]
mov pix0q, pixq
sub pix0q, src3strideq
sub pix0q, strideq
movdqu m0, [pix0q]; p3
movdqu m1, [pix0q + strideq]; p2
movdqu m2, [pix0q + 2 * strideq]; p1
movdqu m3, [pix0q + src3strideq]; p0
movdqu m4, [pixq]; q0
movdqu m5, [pixq + strideq]; q1
movdqu m6, [pixq + 2 * strideq]; q2
movdqu m7, [pixq + src3strideq]; q3
LUMA_DEBLOCK_BODY 12, h
.store:
pxor m8, m8; zeros reg
CLIPW m1, m8, [pw_pixel_max_12]
CLIPW m2, m8, [pw_pixel_max_12]
CLIPW m3, m8, [pw_pixel_max_12]
CLIPW m4, m8, [pw_pixel_max_12]
CLIPW m5, m8, [pw_pixel_max_12]
CLIPW m6, m8, [pw_pixel_max_12]
movdqu [pix0q + strideq], m1; p2
movdqu [pix0q + 2 * strideq], m2; p1
movdqu [pix0q + src3strideq], m3; p0
movdqu [pixq ], m4; q0
movdqu [pixq + strideq], m5; q1
movdqu [pixq + 2 * strideq], m6; q2
.bypassluma:
RET
%endmacro
INIT_XMM sse2
LOOP_FILTER_LUMA
INIT_XMM ssse3
LOOP_FILTER_LUMA
INIT_XMM avx
LOOP_FILTER_LUMA
%endif

853
media/ffvpx/libavcodec/x86/hevc_idct.asm Normal file
View file

@ -0,0 +1,853 @@
;*******************************************************************************
;* SIMD-optimized IDCT functions for HEVC decoding
;* Copyright (c) 2014 Pierre-Edouard LEPERE
;* Copyright (c) 2014 James Almer
;* Copyright (c) 2016 Alexandra Hájková
;*
;* This file is part of FFmpeg.
;*
;* FFmpeg is free software; you can redistribute it and/or
;* modify it under the terms of the GNU Lesser General Public
;* License as published by the Free Software Foundation; either
;* version 2.1 of the License, or (at your option) any later version.
;*
;* FFmpeg is distributed in the hope that it will be useful,
;* but WITHOUT ANY WARRANTY; without even the implied warranty of
;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
;* Lesser General Public License for more details.
;*
;* You should have received a copy of the GNU Lesser General Public
;* License along with FFmpeg; if not, write to the Free Software
;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
;******************************************************************************
%include "libavutil/x86/x86util.asm"
SECTION_RODATA
pd_64: times 4 dd 64
pd_2048: times 4 dd 2048
pd_512: times 4 dd 512
; 4x4 transform coeffs
cextern pw_64
pw_64_m64: times 4 dw 64, -64
pw_83_36: times 4 dw 83, 36
pw_36_m83: times 4 dw 36, -83
; 8x8 transform coeffs
pw_89_75: times 4 dw 89, 75
pw_50_18: times 4 dw 50, 18
pw_75_m18: times 4 dw 75, -18
pw_m89_m50: times 4 dw -89, -50
pw_50_m89: times 4 dw 50, -89
pw_18_75: times 4 dw 18, 75
pw_18_m50: times 4 dw 18, -50
pw_75_m89: times 4 dw 75, -89
; 16x16 transformation coeffs
trans_coeffs16: times 4 dw 90, 87
times 4 dw 80, 70
times 4 dw 57, 43
times 4 dw 25, 9
times 4 dw 87, 57
times 4 dw 9, -43
times 4 dw -80, -90
times 4 dw -70, -25
times 4 dw 80, 9
times 4 dw -70, -87
times 4 dw -25, 57
times 4 dw 90, 43
times 4 dw 70, -43
times 4 dw -87, 9
times 4 dw 90, 25
times 4 dw -80, -57
times 4 dw 57, -80
times 4 dw -25, 90
times 4 dw -9, -87
times 4 dw 43, 70
times 4 dw 43, -90
times 4 dw 57, 25
times 4 dw -87, 70
times 4 dw 9, -80
times 4 dw 25, -70
times 4 dw 90, -80
times 4 dw 43, 9
times 4 dw -57, 87
times 4 dw 9, -25
times 4 dw 43, -57
times 4 dw 70, -80
times 4 dw 87, -90
; 32x32 transform coeffs
trans_coeff32: times 8 dw 90
times 4 dw 88, 85
times 4 dw 82, 78
times 4 dw 73, 67
times 4 dw 61, 54
times 4 dw 46, 38
times 4 dw 31, 22
times 4 dw 13, 4
times 4 dw 90, 82
times 4 dw 67, 46
times 4 dw 22, -4
times 4 dw -31, -54
times 4 dw -73, -85
times 4 dw -90, -88
times 4 dw -78, -61
times 4 dw -38, -13
times 4 dw 88, 67
times 4 dw 31, -13
times 4 dw -54, -82
times 4 dw -90, -78
times 4 dw -46, -4
times 4 dw 38, 73
times 4 dw 90, 85
times 4 dw 61, 22
times 4 dw 85, 46
times 4 dw -13, -67
times 4 dw -90, -73
times 4 dw -22, 38
times 4 dw 82, 88
times 4 dw 54, -4
times 4 dw -61, -90
times 4 dw -78, -31
times 4 dw 82, 22
times 4 dw -54, -90
times 4 dw -61, 13
times 4 dw 78, 85
times 4 dw 31, -46
times 4 dw -90, -67
times 4 dw 4, 73
times 4 dw 88, 38
times 4 dw 78, -4
times 4 dw -82, -73
times 4 dw 13, 85
times 4 dw 67, -22
times 4 dw -88, -61
times 4 dw 31, 90
times 4 dw 54, -38
times 4 dw -90, -46
times 4 dw 73, -31
times 4 dw -90, -22
times 4 dw 78, 67
times 4 dw -38, -90
times 4 dw -13, 82
times 4 dw 61, -46
times 4 dw -88, -4
times 4 dw 85, 54
times 4 dw 67, -54
times 4 dw -78, 38
times 4 dw 85, -22
times 4 dw -90, 4
times 4 dw 90, 13
times 4 dw -88, -31
times 4 dw 82, 46
times 4 dw -73, -61
times 4 dw 61, -73
times 4 dw -46, 82
times 4 dw 31, -88
times 4 dw -13, 90
times 4 dw -4, -90
times 4 dw 22, 85
times 4 dw -38, -78
times 4 dw 54, 67
times 4 dw 54, -85
times 4 dw -4, 88
times 4 dw -46, -61
times 4 dw 82, 13
times 4 dw -90, 38
times 4 dw 67, -78
times 4 dw -22, 90
times 4 dw -31, -73
times 4 dw 46, -90
times 4 dw 38, 54
times 4 dw -90, 31
times 4 dw 61, -88
times 4 dw 22, 67
times 4 dw -85, 13
times 4 dw 73, -82
times 4 dw 4, 78
times 4 dw 38, -88
times 4 dw 73, -4
times 4 dw -67, 90
times 4 dw -46, -31
times 4 dw 85, -78
times 4 dw 13, 61
times 4 dw -90, 54
times 4 dw 22, -82
times 4 dw 31, -78
times 4 dw 90, -61
times 4 dw 4, 54
times 4 dw -88, 82
times 4 dw -38, -22
times 4 dw 73, -90
times 4 dw 67, -13
times 4 dw -46, 85
times 4 dw 22, -61
times 4 dw 85, -90
times 4 dw 73, -38
times 4 dw -4, 46
times 4 dw -78, 90
times 4 dw -82, 54
times 4 dw -13, -31
times 4 dw 67, -88
times 4 dw 13, -38
times 4 dw 61, -78
times 4 dw 88, -90
times 4 dw 85, -73
times 4 dw 54, -31
times 4 dw 4, 22
times 4 dw -46, 67
times 4 dw -82, 90
times 4 dw 4, -13
times 4 dw 22, -31
times 4 dw 38, -46
times 4 dw 54, -61
times 4 dw 67, -73
times 4 dw 78, -82
times 4 dw 85, -88
times 4 dw 90, -90
SECTION .text
; void ff_hevc_idct_HxW_dc_{8,10}_<opt>(int16_t *coeffs)
; %1 = HxW
; %2 = number of loops
; %3 = bitdepth
%macro IDCT_DC 3
cglobal hevc_idct_%1x%1_dc_%3, 1, 2, 1, coeff, tmp
movsx tmpd, word [coeffq]
add tmpd, (1 << (14 - %3)) + 1
sar tmpd, (15 - %3)
movd xm0, tmpd
SPLATW m0, xm0
DEFINE_ARGS coeff, cnt
mov cntd, %2
.loop:
mova [coeffq+mmsize*0], m0
mova [coeffq+mmsize*1], m0
mova [coeffq+mmsize*2], m0
mova [coeffq+mmsize*3], m0
add coeffq, mmsize*8
mova [coeffq+mmsize*-4], m0
mova [coeffq+mmsize*-3], m0
mova [coeffq+mmsize*-2], m0
mova [coeffq+mmsize*-1], m0
dec cntd
jg .loop
RET
%endmacro
; %1 = HxW
; %2 = bitdepth
%macro IDCT_DC_NL 2 ; No loop
cglobal hevc_idct_%1x%1_dc_%2, 1, 2, 1, coeff, tmp
movsx tmpd, word [coeffq]
add tmpd, (1 << (14 - %2)) + 1
sar tmpd, (15 - %2)
movd m0, tmpd
SPLATW m0, xm0
mova [coeffq+mmsize*0], m0
mova [coeffq+mmsize*1], m0
mova [coeffq+mmsize*2], m0
mova [coeffq+mmsize*3], m0
%if mmsize == 16
mova [coeffq+mmsize*4], m0
mova [coeffq+mmsize*5], m0
mova [coeffq+mmsize*6], m0
mova [coeffq+mmsize*7], m0
%endif
RET
%endmacro
; IDCT 4x4, expects input in m0, m1
; %1 - shift
; %2 - 1/0 - SCALE and Transpose or not
; %3 - 1/0 add constant or not
%macro TR_4x4 3
; interleaves src0 with src2 to m0
; and src1 with scr3 to m2
; src0: 00 01 02 03 m0: 00 20 01 21 02 22 03 23
; src1: 10 11 12 13 -->
; src2: 20 21 22 23 m1: 10 30 11 31 12 32 13 33
; src3: 30 31 32 33
SBUTTERFLY wd, 0, 1, 2
pmaddwd m2, m0, [pw_64] ; e0
pmaddwd m3, m1, [pw_83_36] ; o0
pmaddwd m0, [pw_64_m64] ; e1
pmaddwd m1, [pw_36_m83] ; o1
%if %3 == 1
%assign %%add 1 << (%1 - 1)
mova m4, [pd_ %+ %%add]
paddd m2, m4
paddd m0, m4
%endif
SUMSUB_BADC d, 3, 2, 1, 0, 4
%if %2 == 1
psrad m3, %1 ; e0 + o0
psrad m1, %1 ; e1 + o1
psrad m2, %1 ; e0 - o0
psrad m0, %1 ; e1 - o1
;clip16
packssdw m3, m1
packssdw m0, m2
; Transpose
SBUTTERFLY wd, 3, 0, 1
SBUTTERFLY wd, 3, 0, 1
SWAP 3, 1, 0
%else
SWAP 3, 2, 0
%endif
%endmacro
%macro DEFINE_BIAS 1
%assign shift (20 - %1)
%assign c_add (1 << (shift - 1))
%define arr_add pd_ %+ c_add
%endmacro
; %1 - bit_depth
; %2 - register add constant
; is loaded to
; shift = 20 - bit_depth
%macro LOAD_BIAS 2
DEFINE_BIAS %1
mova %2, [arr_add]
%endmacro
; %1, %2 - registers to load packed 16 bit values to
; %3, %4, %5, %6 - vertical offsets
; %7 - horizontal offset
%macro LOAD_BLOCK 7
movq %1, [r0 + %3 + %7]
movhps %1, [r0 + %5 + %7]
movq %2, [r0 + %4 + %7]
movhps %2, [r0 + %6 + %7]
%endmacro
; void ff_hevc_idct_4x4__{8,10}_<opt>(int16_t *coeffs, int col_limit)
; %1 = bitdepth
%macro IDCT_4x4 1
cglobal hevc_idct_4x4_%1, 1, 1, 5, coeffs
mova m0, [coeffsq]
mova m1, [coeffsq + 16]
TR_4x4 7, 1, 1
TR_4x4 20 - %1, 1, 1
mova [coeffsq], m0
mova [coeffsq + 16], m1
RET
%endmacro
; scale, pack (clip16) and store the residuals 0 e8[0] + o8[0] --> + %1
; 4 at one time (4 columns) 1 e8[1] + o8[1]
; from %5: e8/16 + o8/16, with %1 offset ...
; and %3: e8/16 - o8/16, with %2 offset 6 e8[1] - o8[1]
; %4 - shift 7 e8[0] - o8[0] --> + %2
%macro STORE_8 7
psrad %5, %4
psrad %3, %4
packssdw %5, %3
movq [coeffsq + %1], %5
movhps [coeffsq + %2], %5
%endmacro
; %1 - horizontal offset
; %2 - shift
; %3, %4 - transform coeffs
; %5 - vertical offset for e8 + o8
; %6 - vertical offset for e8 - o8
; %7 - register with e8 inside
; %8 - block_size
; %9 - register to store e8 +o8
; %10 - register to store e8 - o8
%macro E8_O8 10
pmaddwd m6, m4, %3
pmaddwd m7, m5, %4
paddd m6, m7
paddd m7, m6, %7 ; o8 + e8
psubd %7, m6 ; e8 - o8
%if %8 == 8
STORE_8 %5 + %1, %6 + %1, %7, %2, m7, 0, 0
%else
SWAP m7, %9
SWAP %7, %10
%endif
%endmacro
; 8x4 residuals are processed and stored
; %1 - horizontal offset
; %2 - shift
; %3 - offset of the even row
; %4 - step: 1 for 8x8, 2 for 16x16, 4 for 32x32
; %5 - offset of the odd row
; %6 - block size
; %7 - 1/0 add a constant in TR_4x4 or not
; I want to add a constant for 8x8 transform but not for 16x16 and 32x32
%macro TR_8x4 7
; load 4 columns of even rows
LOAD_BLOCK m0, m1, 0, 2 * %4 * %3, %4 * %3, 3 * %4 * %3, %1
TR_4x4 %2, 0, %7 ; e8: m0, m1, m2, m3, for 4 columns only
; load 4 columns of odd rows
LOAD_BLOCK m4, m5, %4 * %5, 3 * %4 * %5, 5 * %4 * %5, 7 * %4 * %5, %1
; 00 01 02 03
; 10 11 12 13 m4: 10 30 11 31 12 32 13 33
; ... -- >
; m5: 50 70 51 71 52 72 53 73
; 70 71 72 73
SBUTTERFLY wd, 4, 5, 6
E8_O8 %1, %2, [pw_89_75], [pw_50_18], 0, %5 * 7, m0, %6, m8, m15
E8_O8 %1, %2, [pw_75_m18], [pw_m89_m50], %5, %5 * 6, m1, %6, m9, m14
E8_O8 %1, %2, [pw_50_m89], [pw_18_75], %5 * 2, %5 * 5, m2, %6, m10, m13
E8_O8 %1, %2, [pw_18_m50], [pw_75_m89], %5 * 3, %5 * 4, m3, %6, m11, m12
%endmacro
%macro STORE_PACKED 7
movq [r0 + %3 + %7], %1
movhps [r0 + %4 + %7], %1
movq [r0 + %5 + %7], %2
movhps [r0 + %6 + %7], %2
%endmacro
; transpose 4x4 block packed
; in %1 and %2 registers
; %3 - temporary register
%macro TRANSPOSE_4x4 3
SBUTTERFLY wd, %1, %2, %3
SBUTTERFLY dq, %1, %2, %3
%endmacro
; %1 - horizontal offset of the block i
; %2 - vertical offset of the block i
; %3 - width in bytes
; %4 - vertical offset for the block j
; %5 - horizontal offset for the block j
%macro SWAP_BLOCKS 5
; M_j
LOAD_BLOCK m4, m5, %4, %4 + %3, %4 + 2 * %3, %4 + 3 * %3, %5
TRANSPOSE_4x4 4, 5, 6
; M_i
LOAD_BLOCK m6, m7, %2, %2 + %3, %2 + 2 * %3, %2 + 3 * %3, %1
STORE_PACKED m4, m5, %2, %2 + %3, %2 + 2 * %3, %2 + 3 * %3, %1
; transpose and store M_i
SWAP m6, m4
SWAP m7, m5
TRANSPOSE_4x4 4, 5, 6
STORE_PACKED m4, m5, %4, %4 + %3, %4 + 2 * %3, %4 + 3 * %3, %5
%endmacro
; %1 - horizontal offset
; %2 - vertical offset of the block
; %3 - width in bytes
%macro TRANSPOSE_BLOCK 3
LOAD_BLOCK m4, m5, %2, %2 + %3, %2 + 2 * %3, %2 + 3 * %3, %1
TRANSPOSE_4x4 4, 5, 6
STORE_PACKED m4, m5, %2, %2 + %3, %2 + 2 * %3, %2 + 3 * %3, %1
%endmacro
%macro TRANSPOSE_8x8 0
cglobal hevc_idct_transpose_8x8, 0, 0, 0
; M1 M2 ^T = M1^t M3^t
; M3 M4 M2^t M4^t
; M1 4x4 block
TRANSPOSE_BLOCK 0, 0, 16
; M2 and M3
SWAP_BLOCKS 0, 64, 16, 0, 8
; M4
TRANSPOSE_BLOCK 8, 64, 16
ret
%endmacro
; void ff_hevc_idct_8x8_{8,10}_<opt>(int16_t *coeffs, int col_limit)
; %1 = bitdepth
%macro IDCT_8x8 1
cglobal hevc_idct_8x8_%1, 1, 1, 8, coeffs
TR_8x4 0, 7, 32, 1, 16, 8, 1
TR_8x4 8, 7, 32, 1, 16, 8, 1
call hevc_idct_transpose_8x8_ %+ cpuname
DEFINE_BIAS %1
TR_8x4 0, shift, 32, 1, 16, 8, 1
TR_8x4 8, shift, 32, 1, 16, 8, 1
TAIL_CALL hevc_idct_transpose_8x8_ %+ cpuname, 1
%endmacro
; store intermedite e32 coeffs on stack
; as 16x4 matrix
; from m10: e8 + o8, with %6 offset
; and %3: e8 - o8, with %7 offset
; %4 - shift, unused here
%macro STORE_16 7
mova [rsp + %6], %5
mova [rsp + %7], %3
%endmacro
; %1, %2 - transform constants
; %3, %4 - regs with interleaved coeffs
; %5 - 1/0 SWAP or add
; %6, %7 - registers for intermidiate sums
; %8 - accumulator register
%macro ADD_ROWS 8
pmaddwd %6, %3, %1
pmaddwd %7, %4, %2
paddd %6, %7
%if %5 == 1
SWAP %6, %8
%else
paddd %8, %6
%endif
%endmacro
; %1 - transform coeffs
; %2, %3 offsets for storing e+o/e-o back to coeffsq
; %4 - shift
; %5 - add
; %6 - block_size
; %7 - register with e16
; %8, %9 - stack offsets for storing e+o/e-o
%macro E16_O16 9
ADD_ROWS [%1], [%1 + 16], m0, m1, 1, m5, m6, m7
ADD_ROWS [%1 + 2 * 16], [%1 + 3 * 16], m2, m3, 0, m5, m6, m7
%if %6 == 8
paddd %7, %5
%endif
paddd m4, m7, %7 ; o16 + e16
psubd %7, m7 ; e16 - o16
STORE_%6 %2, %3, %7, %4, m4, %8, %9
%endmacro
%macro TR_16x4 10
; produce 8x4 matrix of e16 coeffs
; for 4 first rows and store it on stack (128 bytes)
TR_8x4 %1, 7, %4, %5, %6, %8, 0
; load 8 even rows
LOAD_BLOCK m0, m1, %9 * %6, %9 * 3 * %6, %9 * 5 * %6, %9 * 7 * %6, %1
LOAD_BLOCK m2, m3, %9 * 9 * %6, %9 * 11 * %6, %9 * 13 * %6, %9 * 15 * %6, %1
SBUTTERFLY wd, 0, 1, 4
SBUTTERFLY wd, 2, 3, 4
E16_O16 trans_coeffs16, 0 + %1, 15 * %6 + %1, %2, %3, %7, m8, 0, 15 * 16
mova m8, %3
E16_O16 trans_coeffs16 + 64, %6 + %1, 14 * %6 + %1, %2, m8, %7, m9, 16, 14 * 16
E16_O16 trans_coeffs16 + 2 * 64, 2 * %6 + %1, 13 * %6 + %1, %2, m8, %7, m10, 2 * 16, 13 * 16
E16_O16 trans_coeffs16 + 3 * 64, 3 * %6 + %1, 12 * %6 + %1, %2, m8, %7, m11, 3 * 16, 12 * 16
E16_O16 trans_coeffs16 + 4 * 64, 4 * %6 + %1, 11 * %6 + %1, %2, m8, %7, m12, 4 * 16, 11 * 16
E16_O16 trans_coeffs16 + 5 * 64, 5 * %6 + %1, 10 * %6 + %1, %2, m8, %7, m13, 5 * 16, 10 * 16
E16_O16 trans_coeffs16 + 6 * 64, 6 * %6 + %1, 9 * %6 + %1, %2, m8, %7, m14, 6 * 16, 9 * 16
E16_O16 trans_coeffs16 + 7 * 64, 7 * %6 + %1, 8 * %6 + %1, %2, m8, %7, m15, 7 * 16, 8 * 16
%endmacro
%macro TRANSPOSE_16x16 0
cglobal hevc_idct_transpose_16x16, 0, 0, 0
; M1 M2 M3 M4 ^T m1 m5 m9 m13 M_i^T = m_i
; M5 M6 M7 M8 --> m2 m6 m10 m14
; M9 M10 M11 M12 m3 m7 m11 m15
; M13 M14 M15 M16 m4 m8 m12 m16
; M1 4x4 block
TRANSPOSE_BLOCK 0, 0, 32
; M5, M2
SWAP_BLOCKS 0, 128, 32, 0, 8
; M9, M3
SWAP_BLOCKS 0, 256, 32, 0, 16
; M13, M4
SWAP_BLOCKS 0, 384, 32, 0, 24
;M6
TRANSPOSE_BLOCK 8, 128, 32
; M10, M7
SWAP_BLOCKS 8, 256, 32, 128, 16
; M14, M8
SWAP_BLOCKS 8, 384, 32, 128, 24
;M11
TRANSPOSE_BLOCK 16, 256, 32
; M15, M12
SWAP_BLOCKS 16, 384, 32, 256, 24
;M16
TRANSPOSE_BLOCK 24, 384, 32
ret
%endmacro
; void ff_hevc_idct_16x16_{8,10}_<opt>(int16_t *coeffs, int col_limit)
; %1 = bitdepth
%macro IDCT_16x16 1
cglobal hevc_idct_16x16_%1, 1, 2, 16, coeffs
mov r1d, 3
.loop16:
TR_16x4 8 * r1, 7, [pd_64], 64, 2, 32, 8, 16, 1, 0
dec r1d
jge .loop16
call hevc_idct_transpose_16x16_ %+ cpuname
DEFINE_BIAS %1
mov r1d, 3
.loop16_2:
TR_16x4 8 * r1, shift, [arr_add], 64, 2, 32, 8, 16, 1, 1
dec r1d
jge .loop16_2
TAIL_CALL hevc_idct_transpose_16x16_ %+ cpuname, 1
%endmacro
; scale, pack (clip16) and store the residuals 0 e32[0] + o32[0] --> %1
; 4 at one time (4 columns) 1 e32[1] + o32[1]
; %1 - address to store e32 + o32
; %2 - address to store e32 - e32
; %5 - reg with e32 + o32 ...
; %3 - reg with e32 - o32 30 e32[1] - o32[1]
; %4 - shift 31 e32[0] - o32[0] --> %2
%macro STORE_32 5
psrad %5, %4
psrad %3, %4
packssdw %5, %3
movq [%1], %5
movhps [%2], %5
%endmacro
; %1 - transform coeffs
; %2 - stack offset for e32
; %2, %3 offsets for storing e+o/e-o back to coeffsq
; %4 - shift
; %5 - stack offset of e32
%macro E32_O32 5
ADD_ROWS [%1], [%1 + 16], m0, m1, 1, m8, m9, m10
ADD_ROWS [%1 + 2 * 16], [%1 + 3 * 16], m2, m3, 0, m8, m9, m10
ADD_ROWS [%1 + 4 * 16], [%1 + 5 * 16], m4, m5, 0, m8, m9, m10
ADD_ROWS [%1 + 6 * 16], [%1 + 7 * 16], m6, m7, 0, m8, m9, m10
paddd m11, m14, [rsp + %5]
paddd m12, m10, m11 ; o32 + e32
psubd m11, m10 ; e32 - o32
STORE_32 %2, %3, m11, %4, m12
%endmacro
; %1 - horizontal offset
; %2 - bitdepth
%macro TR_32x4 3
TR_16x4 %1, 7, [pd_64], 128, 4, 64, 16, 16, 2, 0
LOAD_BLOCK m0, m1, 64, 3 * 64, 5 * 64, 7 * 64, %1
LOAD_BLOCK m2, m3, 9 * 64, 11 * 64, 13 * 64, 15 * 64, %1
LOAD_BLOCK m4, m5, 17 * 64, 19 * 64, 21 * 64, 23 * 64, %1
LOAD_BLOCK m6, m7, 25 * 64, 27 * 64, 29 * 64, 31 * 64, %1
SBUTTERFLY wd, 0, 1, 8
SBUTTERFLY wd, 2, 3, 8
SBUTTERFLY wd, 4, 5, 8
SBUTTERFLY wd, 6, 7, 8
%if %3 == 1
%assign shift 7
mova m14, [pd_64]
%else
LOAD_BIAS %2, m14
%endif
lea r2, [trans_coeff32 + 15 * 128]
lea r3, [coeffsq + %1]
lea r4, [r3 + 16 * 64]
mov r5d, 15 * 16
%%loop:
E32_O32 r2, r3 + r5 * 4, r4, shift, r5
sub r2, 128
add r4, 64
sub r5d, 16
jge %%loop
%endmacro
%macro TRANSPOSE_32x32 0
cglobal hevc_idct_transpose_32x32, 0, 0, 0
; M0 M1 ... M7
; M8 M15
;
; ...
;
; M56 M63
TRANSPOSE_BLOCK 0, 0, 64 ; M1
mov r1d, 7
mov r2d, 7 * 256
.loop_transpose:
SWAP_BLOCKS 0, r2, 64, 0, r1 * 8
sub r2d, 256
dec r1d
jg .loop_transpose
TRANSPOSE_BLOCK 8, 256, 64 ; M9
mov r1d, 6
mov r2d, 512
mov r3d, 16
.loop_transpose2:
SWAP_BLOCKS 8, r2, 64, 256, r3
add r3d, 8
add r2d, 256
dec r1d
jg .loop_transpose2
TRANSPOSE_BLOCK 2 * 8, 2 * 256, 64 ; M9
mov r1d, 5
mov r2d, 768
mov r3d, 24
.loop_transpose3:
SWAP_BLOCKS 2 * 8, r2, 64, 2 * 256, r3
add r3d, 8
add r2d, 256
dec r1d
jg .loop_transpose3
TRANSPOSE_BLOCK 3 * 8, 3 * 256, 64 ; M27
mov r1d, 4
mov r2d, 1024
mov r3d, 32
.loop_transpose4:
SWAP_BLOCKS 3 * 8, r2, 64, 3 * 256, r3
add r3d, 8
add r2d, 256
dec r1d
jg .loop_transpose4
TRANSPOSE_BLOCK 4 * 8, 4 * 256, 64 ; M36
mov r1d, 3
mov r2d, 1280
mov r3d, 40
.loop_transpose5:
SWAP_BLOCKS 4 * 8, r2, 64, 4 * 256, r3
add r3d, 8
add r2d, 256
dec r1d
jg .loop_transpose5
TRANSPOSE_BLOCK 5 * 8, 5 * 256, 64 ; M45
SWAP_BLOCKS 5 * 8, 6 * 256, 64, 5 * 256, 6 * 8
SWAP_BLOCKS 5 * 8, 7 * 256, 64, 5 * 256, 7 * 8
TRANSPOSE_BLOCK 6 * 8, 6 * 256, 64 ; M54
SWAP_BLOCKS 6 * 8, 7 * 256, 64, 6 * 256, 7 * 8
TRANSPOSE_BLOCK 7 * 8, 7 * 256, 64 ; M63
ret
%endmacro
; void ff_hevc_idct_32x32_{8,10}_<opt>(int16_t *coeffs, int col_limit)
; %1 = bitdepth
%macro IDCT_32x32 1
cglobal hevc_idct_32x32_%1, 1, 6, 16, 256, coeffs
mov r1d, 7
.loop32:
TR_32x4 8 * r1, %1, 1
dec r1d
jge .loop32
call hevc_idct_transpose_32x32_ %+ cpuname
mov r1d, 7
.loop32_2:
TR_32x4 8 * r1, %1, 0
dec r1d
jge .loop32_2
TAIL_CALL hevc_idct_transpose_32x32_ %+ cpuname, 1
%endmacro
%macro INIT_IDCT_DC 1
INIT_MMX mmxext
IDCT_DC_NL 4, %1
IDCT_DC 8, 2, %1
INIT_XMM sse2
IDCT_DC_NL 8, %1
IDCT_DC 16, 4, %1
IDCT_DC 32, 16, %1
%if HAVE_AVX2_EXTERNAL
INIT_YMM avx2
IDCT_DC 16, 2, %1
IDCT_DC 32, 8, %1
%endif ;HAVE_AVX2_EXTERNAL
%endmacro
%macro INIT_IDCT 2
INIT_XMM %2
%if %1 == 8
TRANSPOSE_8x8
%if ARCH_X86_64
TRANSPOSE_16x16
TRANSPOSE_32x32
%endif
%endif
%if ARCH_X86_64
IDCT_32x32 %1
IDCT_16x16 %1
%endif
IDCT_8x8 %1
IDCT_4x4 %1
%endmacro
INIT_IDCT_DC 8
INIT_IDCT_DC 10
INIT_IDCT_DC 12
INIT_IDCT 8, sse2
INIT_IDCT 8, avx
INIT_IDCT 10, sse2
INIT_IDCT 10, avx
;INIT_IDCT 12, sse2
;INIT_IDCT 12, avx

1672
media/ffvpx/libavcodec/x86/hevc_mc.asm Normal file
View file

File diff suppressed because it is too large Load diff

340
media/ffvpx/libavcodec/x86/hevc_sao.asm Normal file
View file

@ -0,0 +1,340 @@
;******************************************************************************
;* SIMD optimized SAO functions for HEVC 8bit decoding
;*
;* Copyright (c) 2013 Pierre-Edouard LEPERE
;* Copyright (c) 2014 James Almer
;*
;* This file is part of FFmpeg.
;*
;* FFmpeg is free software; you can redistribute it and/or
;* modify it under the terms of the GNU Lesser General Public
;* License as published by the Free Software Foundation; either
;* version 2.1 of the License, or (at your option) any later version.
;*
;* FFmpeg is distributed in the hope that it will be useful,
;* but WITHOUT ANY WARRANTY; without even the implied warranty of
;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
;* Lesser General Public License for more details.
;*
;* You should have received a copy of the GNU Lesser General Public
;* License along with FFmpeg; if not, write to the Free Software
;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
;******************************************************************************
%include "libavutil/x86/x86util.asm"
SECTION_RODATA 32
pb_edge_shuffle: times 2 db 1, 2, 0, 3, 4, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1
pb_eo: db -1, 0, 1, 0, 0, -1, 0, 1, -1, -1, 1, 1, 1, -1, -1, 1
cextern pb_1
cextern pb_2
SECTION .text
;******************************************************************************
;SAO Band Filter
;******************************************************************************
%macro HEVC_SAO_BAND_FILTER_INIT 0
and leftq, 31
movd xm0, leftd
add leftq, 1
and leftq, 31
movd xm1, leftd
add leftq, 1
and leftq, 31
movd xm2, leftd
add leftq, 1
and leftq, 31
movd xm3, leftd
SPLATW m0, xm0
SPLATW m1, xm1
SPLATW m2, xm2
SPLATW m3, xm3
%if mmsize > 16
SPLATW m4, [offsetq + 2]
SPLATW m5, [offsetq + 4]
SPLATW m6, [offsetq + 6]
SPLATW m7, [offsetq + 8]
%else
movq m7, [offsetq + 2]
SPLATW m4, m7, 0
SPLATW m5, m7, 1
SPLATW m6, m7, 2
SPLATW m7, m7, 3
%endif
%if ARCH_X86_64
pxor m14, m14
%else ; ARCH_X86_32
mova [rsp+mmsize*0], m0
mova [rsp+mmsize*1], m1
mova [rsp+mmsize*2], m2
mova [rsp+mmsize*3], m3
mova [rsp+mmsize*4], m4
mova [rsp+mmsize*5], m5
mova [rsp+mmsize*6], m6
pxor m0, m0
%assign MMSIZE mmsize
%define m14 m0
%define m13 m1
%define m9 m2
%define m8 m3
%endif ; ARCH
DEFINE_ARGS dst, src, dststride, srcstride, offset, height
mov heightd, r7m
%endmacro
%macro HEVC_SAO_BAND_FILTER_COMPUTE 2
psraw %1, %2, 3
%if ARCH_X86_64
pcmpeqw m10, %1, m0
pcmpeqw m11, %1, m1
pcmpeqw m12, %1, m2
pcmpeqw %1, m3
pand m10, m4
pand m11, m5
pand m12, m6
pand %1, m7
por m10, m11
por m12, %1
por m10, m12
paddw %2, m10
%else ; ARCH_X86_32
pcmpeqw m4, %1, [rsp+MMSIZE*0]
pcmpeqw m5, %1, [rsp+MMSIZE*1]
pcmpeqw m6, %1, [rsp+MMSIZE*2]
pcmpeqw %1, [rsp+MMSIZE*3]
pand m4, [rsp+MMSIZE*4]
pand m5, [rsp+MMSIZE*5]
pand m6, [rsp+MMSIZE*6]
pand %1, m7
por m4, m5
por m6, %1
por m4, m6
paddw %2, m4
%endif ; ARCH
%endmacro
;void ff_hevc_sao_band_filter_<width>_8_<opt>(uint8_t *_dst, uint8_t *_src, ptrdiff_t _stride_dst, ptrdiff_t _stride_src,
; int16_t *sao_offset_val, int sao_left_class, int width, int height);
%macro HEVC_SAO_BAND_FILTER 2
cglobal hevc_sao_band_filter_%1_8, 6, 6, 15, 7*mmsize*ARCH_X86_32, dst, src, dststride, srcstride, offset, left
HEVC_SAO_BAND_FILTER_INIT
align 16
.loop:
%if %1 == 8
movq m8, [srcq]
punpcklbw m8, m14
HEVC_SAO_BAND_FILTER_COMPUTE m9, m8
packuswb m8, m14
movq [dstq], m8
%endif ; %1 == 8
%assign i 0
%rep %2
mova m13, [srcq + i]
punpcklbw m8, m13, m14
HEVC_SAO_BAND_FILTER_COMPUTE m9, m8
punpckhbw m13, m14
HEVC_SAO_BAND_FILTER_COMPUTE m9, m13
packuswb m8, m13
mova [dstq + i], m8
%assign i i+mmsize
%endrep
%if %1 == 48
INIT_XMM cpuname
mova m13, [srcq + i]
punpcklbw m8, m13, m14
HEVC_SAO_BAND_FILTER_COMPUTE m9, m8
punpckhbw m13, m14
HEVC_SAO_BAND_FILTER_COMPUTE m9, m13
packuswb m8, m13
mova [dstq + i], m8
%if cpuflag(avx2)
INIT_YMM cpuname
%endif
%endif ; %1 == 48
add dstq, dststrideq ; dst += dststride
add srcq, srcstrideq ; src += srcstride
dec heightd ; cmp height
jnz .loop ; height loop
REP_RET
%endmacro
%macro HEVC_SAO_BAND_FILTER_FUNCS 0
HEVC_SAO_BAND_FILTER 8, 0
HEVC_SAO_BAND_FILTER 16, 1
HEVC_SAO_BAND_FILTER 32, 2
HEVC_SAO_BAND_FILTER 48, 2
HEVC_SAO_BAND_FILTER 64, 4
%endmacro
INIT_XMM sse2
HEVC_SAO_BAND_FILTER_FUNCS
INIT_XMM avx
HEVC_SAO_BAND_FILTER_FUNCS
%if HAVE_AVX2_EXTERNAL
INIT_XMM avx2
HEVC_SAO_BAND_FILTER 8, 0
HEVC_SAO_BAND_FILTER 16, 1
INIT_YMM avx2
HEVC_SAO_BAND_FILTER 32, 1
HEVC_SAO_BAND_FILTER 48, 1
HEVC_SAO_BAND_FILTER 64, 2
%endif
;******************************************************************************
;SAO Edge Filter
;******************************************************************************
%define MAX_PB_SIZE 64
%define PADDING_SIZE 32 ; AV_INPUT_BUFFER_PADDING_SIZE
%define EDGE_SRCSTRIDE 2 * MAX_PB_SIZE + PADDING_SIZE
%macro HEVC_SAO_EDGE_FILTER_INIT 0
%if WIN64
movsxd eoq, dword eom
%elif ARCH_X86_64
movsxd eoq, eod
%else
mov eoq, r4m
%endif
lea tmp2q, [pb_eo]
movsx a_strideq, byte [tmp2q+eoq*4+1]
movsx b_strideq, byte [tmp2q+eoq*4+3]
imul a_strideq, EDGE_SRCSTRIDE
imul b_strideq, EDGE_SRCSTRIDE
movsx tmpq, byte [tmp2q+eoq*4]
add a_strideq, tmpq
movsx tmpq, byte [tmp2q+eoq*4+2]
add b_strideq, tmpq
%endmacro
%macro HEVC_SAO_EDGE_FILTER_COMPUTE 1
pminub m4, m1, m2
pminub m5, m1, m3
pcmpeqb m2, m4
pcmpeqb m3, m5
pcmpeqb m4, m1
pcmpeqb m5, m1
psubb m4, m2
psubb m5, m3
paddb m4, m6
paddb m4, m5
pshufb m2, m0, m4
%if %1 > 8
punpckhbw m5, m7, m1
punpckhbw m4, m2, m7
punpcklbw m3, m7, m1
punpcklbw m2, m7
pmaddubsw m5, m4
pmaddubsw m3, m2
packuswb m3, m5
%else
punpcklbw m3, m7, m1
punpcklbw m2, m7
pmaddubsw m3, m2
packuswb m3, m3
%endif
%endmacro
;void ff_hevc_sao_edge_filter_<width>_8_<opt>(uint8_t *_dst, uint8_t *_src, ptrdiff_t stride_dst, int16_t *sao_offset_val,
; int eo, int width, int height);
%macro HEVC_SAO_EDGE_FILTER 2-3
%if ARCH_X86_64
cglobal hevc_sao_edge_filter_%1_8, 4, 9, 8, dst, src, dststride, offset, eo, a_stride, b_stride, height, tmp
%define tmp2q heightq
HEVC_SAO_EDGE_FILTER_INIT
mov heightd, r6m
%else ; ARCH_X86_32
cglobal hevc_sao_edge_filter_%1_8, 1, 6, 8, dst, src, dststride, a_stride, b_stride, height
%define eoq srcq
%define tmpq heightq
%define tmp2q dststrideq
%define offsetq heightq
HEVC_SAO_EDGE_FILTER_INIT
mov srcq, srcm
mov offsetq, r3m
mov dststrideq, dststridem
%endif ; ARCH
%if mmsize > 16
vbroadcasti128 m0, [offsetq]
%else
movu m0, [offsetq]
%endif
mova m1, [pb_edge_shuffle]
packsswb m0, m0
mova m7, [pb_1]
pshufb m0, m1
mova m6, [pb_2]
%if ARCH_X86_32
mov heightd, r6m
%endif
align 16
.loop:
%if %1 == 8
movq m1, [srcq]
movq m2, [srcq + a_strideq]
movq m3, [srcq + b_strideq]
HEVC_SAO_EDGE_FILTER_COMPUTE %1
movq [dstq], m3
%endif
%assign i 0
%rep %2
mova m1, [srcq + i]
movu m2, [srcq + a_strideq + i]
movu m3, [srcq + b_strideq + i]
HEVC_SAO_EDGE_FILTER_COMPUTE %1
mov%3 [dstq + i], m3
%assign i i+mmsize
%endrep
%if %1 == 48
INIT_XMM cpuname
mova m1, [srcq + i]
movu m2, [srcq + a_strideq + i]
movu m3, [srcq + b_strideq + i]
HEVC_SAO_EDGE_FILTER_COMPUTE %1
mova [dstq + i], m3
%if cpuflag(avx2)
INIT_YMM cpuname
%endif
%endif
add dstq, dststrideq
add srcq, EDGE_SRCSTRIDE
dec heightd
jg .loop
RET
%endmacro
INIT_XMM ssse3
HEVC_SAO_EDGE_FILTER 8, 0
HEVC_SAO_EDGE_FILTER 16, 1, a
HEVC_SAO_EDGE_FILTER 32, 2, a
HEVC_SAO_EDGE_FILTER 48, 2, a
HEVC_SAO_EDGE_FILTER 64, 4, a
%if HAVE_AVX2_EXTERNAL
INIT_YMM avx2
HEVC_SAO_EDGE_FILTER 32, 1, a
HEVC_SAO_EDGE_FILTER 48, 1, u
HEVC_SAO_EDGE_FILTER 64, 2, a
%endif

370
media/ffvpx/libavcodec/x86/hevc_sao_10bit.asm Normal file
View file

@ -0,0 +1,370 @@
;******************************************************************************
;* SIMD optimized SAO functions for HEVC 10/12bit decoding
;*
;* Copyright (c) 2013 Pierre-Edouard LEPERE
;* Copyright (c) 2014 James Almer
;*
;* This file is part of FFmpeg.
;*
;* FFmpeg is free software; you can redistribute it and/or
;* modify it under the terms of the GNU Lesser General Public
;* License as published by the Free Software Foundation; either
;* version 2.1 of the License, or (at your option) any later version.
;*
;* FFmpeg is distributed in the hope that it will be useful,
;* but WITHOUT ANY WARRANTY; without even the implied warranty of
;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
;* Lesser General Public License for more details.
;*
;* You should have received a copy of the GNU Lesser General Public
;* License along with FFmpeg; if not, write to the Free Software
;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
;******************************************************************************
%include "libavutil/x86/x86util.asm"
SECTION_RODATA 32
pw_m2: times 16 dw -2
pw_mask10: times 16 dw 0x03FF
pw_mask12: times 16 dw 0x0FFF
pb_eo: db -1, 0, 1, 0, 0, -1, 0, 1, -1, -1, 1, 1, 1, -1, -1, 1
cextern pw_m1
cextern pw_1
cextern pw_2
SECTION .text
;******************************************************************************
;SAO Band Filter
;******************************************************************************
%macro HEVC_SAO_BAND_FILTER_INIT 1
and leftq, 31
movd xm0, leftd
add leftq, 1
and leftq, 31
movd xm1, leftd
add leftq, 1
and leftq, 31
movd xm2, leftd
add leftq, 1
and leftq, 31
movd xm3, leftd
SPLATW m0, xm0
SPLATW m1, xm1
SPLATW m2, xm2
SPLATW m3, xm3
%if mmsize > 16
SPLATW m4, [offsetq + 2]
SPLATW m5, [offsetq + 4]
SPLATW m6, [offsetq + 6]
SPLATW m7, [offsetq + 8]
%else
movq m7, [offsetq + 2]
SPLATW m4, m7, 0
SPLATW m5, m7, 1
SPLATW m6, m7, 2
SPLATW m7, m7, 3
%endif
%if ARCH_X86_64
mova m13, [pw_mask %+ %1]
pxor m14, m14
%else ; ARCH_X86_32
mova [rsp+mmsize*0], m0
mova [rsp+mmsize*1], m1
mova [rsp+mmsize*2], m2
mova [rsp+mmsize*3], m3
mova [rsp+mmsize*4], m4
mova [rsp+mmsize*5], m5
mova [rsp+mmsize*6], m6
mova m1, [pw_mask %+ %1]
pxor m0, m0
%define m14 m0
%define m13 m1
%define m9 m2
%define m8 m3
%endif ; ARCH
DEFINE_ARGS dst, src, dststride, srcstride, offset, height
mov heightd, r7m
%endmacro
;void ff_hevc_sao_band_filter_<width>_<depth>_<opt>(uint8_t *_dst, uint8_t *_src, ptrdiff_t _stride_dst, ptrdiff_t _stride_src,
; int16_t *sao_offset_val, int sao_left_class, int width, int height);
%macro HEVC_SAO_BAND_FILTER 3
cglobal hevc_sao_band_filter_%2_%1, 6, 6, 15, 7*mmsize*ARCH_X86_32, dst, src, dststride, srcstride, offset, left
HEVC_SAO_BAND_FILTER_INIT %1
align 16
.loop:
%assign i 0
%assign j 0
%rep %3
%assign k 8+(j&1)
%assign l 9-(j&1)
mova m %+ k, [srcq + i]
psraw m %+ l, m %+ k, %1-5
%if ARCH_X86_64
pcmpeqw m10, m %+ l, m0
pcmpeqw m11, m %+ l, m1
pcmpeqw m12, m %+ l, m2
pcmpeqw m %+ l, m3
pand m10, m4
pand m11, m5
pand m12, m6
pand m %+ l, m7
por m10, m11
por m12, m %+ l
por m10, m12
paddw m %+ k, m10
%else ; ARCH_X86_32
pcmpeqw m4, m %+ l, [rsp+mmsize*0]
pcmpeqw m5, m %+ l, [rsp+mmsize*1]
pcmpeqw m6, m %+ l, [rsp+mmsize*2]
pcmpeqw m %+ l, [rsp+mmsize*3]
pand m4, [rsp+mmsize*4]
pand m5, [rsp+mmsize*5]
pand m6, [rsp+mmsize*6]
pand m %+ l, m7
por m4, m5
por m6, m %+ l
por m4, m6
paddw m %+ k, m4
%endif ; ARCH
CLIPW m %+ k, m14, m13
mova [dstq + i], m %+ k
%assign i i+mmsize
%assign j j+1
%endrep
add dstq, dststrideq
add srcq, srcstrideq
dec heightd
jg .loop
REP_RET
%endmacro
%macro HEVC_SAO_BAND_FILTER_FUNCS 0
HEVC_SAO_BAND_FILTER 10, 8, 1
HEVC_SAO_BAND_FILTER 10, 16, 2
HEVC_SAO_BAND_FILTER 10, 32, 4
HEVC_SAO_BAND_FILTER 10, 48, 6
HEVC_SAO_BAND_FILTER 10, 64, 8
HEVC_SAO_BAND_FILTER 12, 8, 1
HEVC_SAO_BAND_FILTER 12, 16, 2
HEVC_SAO_BAND_FILTER 12, 32, 4
HEVC_SAO_BAND_FILTER 12, 48, 6
HEVC_SAO_BAND_FILTER 12, 64, 8
%endmacro
INIT_XMM sse2
HEVC_SAO_BAND_FILTER_FUNCS
INIT_XMM avx
HEVC_SAO_BAND_FILTER_FUNCS
%if HAVE_AVX2_EXTERNAL
INIT_XMM avx2
HEVC_SAO_BAND_FILTER 10, 8, 1
INIT_YMM avx2
HEVC_SAO_BAND_FILTER 10, 16, 1
HEVC_SAO_BAND_FILTER 10, 32, 2
HEVC_SAO_BAND_FILTER 10, 48, 3
HEVC_SAO_BAND_FILTER 10, 64, 4
INIT_XMM avx2
HEVC_SAO_BAND_FILTER 12, 8, 1
INIT_YMM avx2
HEVC_SAO_BAND_FILTER 12, 16, 1
HEVC_SAO_BAND_FILTER 12, 32, 2
HEVC_SAO_BAND_FILTER 12, 48, 3
HEVC_SAO_BAND_FILTER 12, 64, 4
%endif
;******************************************************************************
;SAO Edge Filter
;******************************************************************************
%define MAX_PB_SIZE 64
%define PADDING_SIZE 32 ; AV_INPUT_BUFFER_PADDING_SIZE
%define EDGE_SRCSTRIDE 2 * MAX_PB_SIZE + PADDING_SIZE
%macro PMINUW 4
%if cpuflag(sse4)
pminuw %1, %2, %3
%else
psubusw %4, %2, %3
psubw %1, %2, %4
%endif
%endmacro
%macro HEVC_SAO_EDGE_FILTER_INIT 0
%if WIN64
movsxd eoq, dword eom
%elif ARCH_X86_64
movsxd eoq, eod
%else
mov eoq, r4m
%endif
lea tmp2q, [pb_eo]
movsx a_strideq, byte [tmp2q+eoq*4+1]
movsx b_strideq, byte [tmp2q+eoq*4+3]
imul a_strideq, EDGE_SRCSTRIDE >> 1
imul b_strideq, EDGE_SRCSTRIDE >> 1
movsx tmpq, byte [tmp2q+eoq*4]
add a_strideq, tmpq
movsx tmpq, byte [tmp2q+eoq*4+2]
add b_strideq, tmpq
%endmacro
;void ff_hevc_sao_edge_filter_<width>_<depth>_<opt>(uint8_t *_dst, uint8_t *_src, ptrdiff_t stride_dst, int16_t *sao_offset_val,
; int eo, int width, int height);
%macro HEVC_SAO_EDGE_FILTER 3
%if ARCH_X86_64
cglobal hevc_sao_edge_filter_%2_%1, 4, 9, 16, dst, src, dststride, offset, eo, a_stride, b_stride, height, tmp
%define tmp2q heightq
HEVC_SAO_EDGE_FILTER_INIT
mov heightd, r6m
add a_strideq, a_strideq
add b_strideq, b_strideq
%else ; ARCH_X86_32
cglobal hevc_sao_edge_filter_%2_%1, 1, 6, 8, 5*mmsize, dst, src, dststride, a_stride, b_stride, height
%define eoq srcq
%define tmpq heightq
%define tmp2q dststrideq
%define offsetq heightq
%define m8 m1
%define m9 m2
%define m10 m3
%define m11 m4
%define m12 m5
HEVC_SAO_EDGE_FILTER_INIT
mov srcq, srcm
mov offsetq, r3m
mov dststrideq, dststridem
add a_strideq, a_strideq
add b_strideq, b_strideq
%endif ; ARCH
%if mmsize > 16
SPLATW m8, [offsetq+2]
SPLATW m9, [offsetq+4]
SPLATW m10, [offsetq+0]
SPLATW m11, [offsetq+6]
SPLATW m12, [offsetq+8]
%else
movq m10, [offsetq+0]
movd m12, [offsetq+6]
SPLATW m8, xm10, 1
SPLATW m9, xm10, 2
SPLATW m10, xm10, 0
SPLATW m11, xm12, 0
SPLATW m12, xm12, 1
%endif
pxor m0, m0
%if ARCH_X86_64
mova m13, [pw_m1]
mova m14, [pw_1]
mova m15, [pw_2]
%else
mov heightd, r6m
mova [rsp+mmsize*0], m8
mova [rsp+mmsize*1], m9
mova [rsp+mmsize*2], m10
mova [rsp+mmsize*3], m11
mova [rsp+mmsize*4], m12
%endif
align 16
.loop:
%assign i 0
%rep %3
mova m1, [srcq + i]
movu m2, [srcq+a_strideq + i]
movu m3, [srcq+b_strideq + i]
PMINUW m4, m1, m2, m6
PMINUW m5, m1, m3, m7
pcmpeqw m2, m4
pcmpeqw m3, m5
pcmpeqw m4, m1
pcmpeqw m5, m1
psubw m4, m2
psubw m5, m3
paddw m4, m5
pcmpeqw m2, m4, [pw_m2]
%if ARCH_X86_64
pcmpeqw m3, m4, m13
pcmpeqw m5, m4, m0
pcmpeqw m6, m4, m14
pcmpeqw m7, m4, m15
pand m2, m8
pand m3, m9
pand m5, m10
pand m6, m11
pand m7, m12
%else
pcmpeqw m3, m4, [pw_m1]
pcmpeqw m5, m4, m0
pcmpeqw m6, m4, [pw_1]
pcmpeqw m7, m4, [pw_2]
pand m2, [rsp+mmsize*0]
pand m3, [rsp+mmsize*1]
pand m5, [rsp+mmsize*2]
pand m6, [rsp+mmsize*3]
pand m7, [rsp+mmsize*4]
%endif
paddw m2, m3
paddw m5, m6
paddw m2, m7
paddw m2, m1
paddw m2, m5
CLIPW m2, m0, [pw_mask %+ %1]
mova [dstq + i], m2
%assign i i+mmsize
%endrep
add dstq, dststrideq
add srcq, EDGE_SRCSTRIDE
dec heightd
jg .loop
RET
%endmacro
INIT_XMM sse2
HEVC_SAO_EDGE_FILTER 10, 8, 1
HEVC_SAO_EDGE_FILTER 10, 16, 2
HEVC_SAO_EDGE_FILTER 10, 32, 4
HEVC_SAO_EDGE_FILTER 10, 48, 6
HEVC_SAO_EDGE_FILTER 10, 64, 8
HEVC_SAO_EDGE_FILTER 12, 8, 1
HEVC_SAO_EDGE_FILTER 12, 16, 2
HEVC_SAO_EDGE_FILTER 12, 32, 4
HEVC_SAO_EDGE_FILTER 12, 48, 6
HEVC_SAO_EDGE_FILTER 12, 64, 8
%if HAVE_AVX2_EXTERNAL
INIT_XMM avx2
HEVC_SAO_EDGE_FILTER 10, 8, 1
INIT_YMM avx2
HEVC_SAO_EDGE_FILTER 10, 16, 1
HEVC_SAO_EDGE_FILTER 10, 32, 2
HEVC_SAO_EDGE_FILTER 10, 48, 3
HEVC_SAO_EDGE_FILTER 10, 64, 4
INIT_XMM avx2
HEVC_SAO_EDGE_FILTER 12, 8, 1
INIT_YMM avx2
HEVC_SAO_EDGE_FILTER 12, 16, 1
HEVC_SAO_EDGE_FILTER 12, 32, 2
HEVC_SAO_EDGE_FILTER 12, 48, 3
HEVC_SAO_EDGE_FILTER 12, 64, 4
%endif

259
media/ffvpx/libavcodec/x86/hevcdsp.h Normal file
View file

@ -0,0 +1,259 @@
/*
* HEVC video decoder
*
* Copyright (C) 2012 - 2013 Guillaume Martres
* Copyright (C) 2013 - 2014 Pierre-Edouard Lepere
*
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#ifndef AVCODEC_X86_HEVCDSP_H
#define AVCODEC_X86_HEVCDSP_H
#include <stddef.h>
#include <stdint.h>
#define PEL_LINK(dst, idx1, idx2, idx3, name, D, opt) \
dst[idx1][idx2][idx3] = ff_hevc_put_hevc_ ## name ## _ ## D ## _##opt; \
dst ## _bi[idx1][idx2][idx3] = ff_hevc_put_hevc_bi_ ## name ## _ ## D ## _##opt; \
dst ## _uni[idx1][idx2][idx3] = ff_hevc_put_hevc_uni_ ## name ## _ ## D ## _##opt; \
dst ## _uni_w[idx1][idx2][idx3] = ff_hevc_put_hevc_uni_w_ ## name ## _ ## D ## _##opt; \
dst ## _bi_w[idx1][idx2][idx3] = ff_hevc_put_hevc_bi_w_ ## name ## _ ## D ## _##opt
#define PEL_PROTOTYPE(name, D, opt) \
void ff_hevc_put_hevc_ ## name ## _ ## D ## _##opt(int16_t *dst, uint8_t *_src, ptrdiff_t _srcstride, int height, intptr_t mx, intptr_t my,int width); \
void ff_hevc_put_hevc_bi_ ## name ## _ ## D ## _##opt(uint8_t *_dst, ptrdiff_t _dststride, uint8_t *_src, ptrdiff_t _srcstride, int16_t *src2, int height, intptr_t mx, intptr_t my, int width); \
void ff_hevc_put_hevc_uni_ ## name ## _ ## D ## _##opt(uint8_t *_dst, ptrdiff_t _dststride, uint8_t *_src, ptrdiff_t _srcstride, int height, intptr_t mx, intptr_t my, int width); \
void ff_hevc_put_hevc_uni_w_ ## name ## _ ## D ## _##opt(uint8_t *_dst, ptrdiff_t _dststride, uint8_t *_src, ptrdiff_t _srcstride, int height, int denom, int wx, int ox, intptr_t mx, intptr_t my, int width); \
void ff_hevc_put_hevc_bi_w_ ## name ## _ ## D ## _##opt(uint8_t *_dst, ptrdiff_t _dststride, uint8_t *_src, ptrdiff_t _srcstride, int16_t *src2, int height, int denom, int wx0, int wx1, int ox0, int ox1, intptr_t mx, intptr_t my, int width)
///////////////////////////////////////////////////////////////////////////////
// MC functions
///////////////////////////////////////////////////////////////////////////////
#define EPEL_PROTOTYPES(fname, bitd, opt) \
PEL_PROTOTYPE(fname##4, bitd, opt); \
PEL_PROTOTYPE(fname##6, bitd, opt); \
PEL_PROTOTYPE(fname##8, bitd, opt); \
PEL_PROTOTYPE(fname##12, bitd, opt); \
PEL_PROTOTYPE(fname##16, bitd, opt); \
PEL_PROTOTYPE(fname##24, bitd, opt); \
PEL_PROTOTYPE(fname##32, bitd, opt); \
PEL_PROTOTYPE(fname##48, bitd, opt); \
PEL_PROTOTYPE(fname##64, bitd, opt)
#define QPEL_PROTOTYPES(fname, bitd, opt) \
PEL_PROTOTYPE(fname##4, bitd, opt); \
PEL_PROTOTYPE(fname##8, bitd, opt); \
PEL_PROTOTYPE(fname##12, bitd, opt); \
PEL_PROTOTYPE(fname##16, bitd, opt); \
PEL_PROTOTYPE(fname##24, bitd, opt); \
PEL_PROTOTYPE(fname##32, bitd, opt); \
PEL_PROTOTYPE(fname##48, bitd, opt); \
PEL_PROTOTYPE(fname##64, bitd, opt)
#define WEIGHTING_PROTOTYPE(width, bitd, opt) \
void ff_hevc_put_hevc_uni_w##width##_##bitd##_##opt(uint8_t *dst, ptrdiff_t dststride, int16_t *_src, int height, int denom, int _wx, int _ox); \
void ff_hevc_put_hevc_bi_w##width##_##bitd##_##opt(uint8_t *dst, ptrdiff_t dststride, int16_t *_src, int16_t *_src2, int height, int denom, int _wx0, int _wx1, int _ox0, int _ox1)
#define WEIGHTING_PROTOTYPES(bitd, opt) \
WEIGHTING_PROTOTYPE(2, bitd, opt); \
WEIGHTING_PROTOTYPE(4, bitd, opt); \
WEIGHTING_PROTOTYPE(6, bitd, opt); \
WEIGHTING_PROTOTYPE(8, bitd, opt); \
WEIGHTING_PROTOTYPE(12, bitd, opt); \
WEIGHTING_PROTOTYPE(16, bitd, opt); \
WEIGHTING_PROTOTYPE(24, bitd, opt); \
WEIGHTING_PROTOTYPE(32, bitd, opt); \
WEIGHTING_PROTOTYPE(48, bitd, opt); \
WEIGHTING_PROTOTYPE(64, bitd, opt)
///////////////////////////////////////////////////////////////////////////////
// QPEL_PIXELS EPEL_PIXELS
///////////////////////////////////////////////////////////////////////////////
EPEL_PROTOTYPES(pel_pixels , 8, sse4);
EPEL_PROTOTYPES(pel_pixels , 10, sse4);
EPEL_PROTOTYPES(pel_pixels , 12, sse4);
void ff_hevc_put_hevc_pel_pixels16_8_avx2(int16_t *dst, uint8_t *_src, ptrdiff_t _srcstride, int height, intptr_t mx, intptr_t my,int width);
void ff_hevc_put_hevc_pel_pixels24_8_avx2(int16_t *dst, uint8_t *_src, ptrdiff_t _srcstride, int height, intptr_t mx, intptr_t my,int width);
void ff_hevc_put_hevc_pel_pixels32_8_avx2(int16_t *dst, uint8_t *_src, ptrdiff_t _srcstride, int height, intptr_t mx, intptr_t my,int width);
void ff_hevc_put_hevc_pel_pixels48_8_avx2(int16_t *dst, uint8_t *_src, ptrdiff_t _srcstride, int height, intptr_t mx, intptr_t my,int width);
void ff_hevc_put_hevc_pel_pixels64_8_avx2(int16_t *dst, uint8_t *_src, ptrdiff_t _srcstride, int height, intptr_t mx, intptr_t my,int width);
void ff_hevc_put_hevc_pel_pixels16_10_avx2(int16_t *dst, uint8_t *_src, ptrdiff_t _srcstride, int height, intptr_t mx, intptr_t my,int width);
void ff_hevc_put_hevc_pel_pixels24_10_avx2(int16_t *dst, uint8_t *_src, ptrdiff_t _srcstride, int height, intptr_t mx, intptr_t my,int width);
void ff_hevc_put_hevc_pel_pixels32_10_avx2(int16_t *dst, uint8_t *_src, ptrdiff_t _srcstride, int height, intptr_t mx, intptr_t my,int width);
void ff_hevc_put_hevc_pel_pixels48_10_avx2(int16_t *dst, uint8_t *_src, ptrdiff_t _srcstride, int height, intptr_t mx, intptr_t my,int width);
void ff_hevc_put_hevc_pel_pixels64_10_avx2(int16_t *dst, uint8_t *_src, ptrdiff_t _srcstride, int height, intptr_t mx, intptr_t my,int width);
void ff_hevc_put_hevc_uni_pel_pixels32_8_avx2(uint8_t *dst, ptrdiff_t dststride,uint8_t *_src, ptrdiff_t _srcstride, int height, intptr_t mx, intptr_t my,int width);
void ff_hevc_put_hevc_uni_pel_pixels48_8_avx2(uint8_t *dst, ptrdiff_t dststride,uint8_t *_src, ptrdiff_t _srcstride, int height, intptr_t mx, intptr_t my,int width);
void ff_hevc_put_hevc_uni_pel_pixels64_8_avx2(uint8_t *dst, ptrdiff_t dststride,uint8_t *_src, ptrdiff_t _srcstride, int height, intptr_t mx, intptr_t my,int width);
void ff_hevc_put_hevc_uni_pel_pixels96_8_avx2(uint8_t *dst, ptrdiff_t dststride,uint8_t *_src, ptrdiff_t _srcstride, int height, intptr_t mx, intptr_t my,int width); //used for 10bit
void ff_hevc_put_hevc_uni_pel_pixels128_8_avx2(uint8_t *dst, ptrdiff_t dststride,uint8_t *_src, ptrdiff_t _srcstride, int height, intptr_t mx, intptr_t my,int width);//used for 10bit
void ff_hevc_put_hevc_bi_pel_pixels16_8_avx2(uint8_t *_dst, ptrdiff_t _dststride, uint8_t *_src, ptrdiff_t _srcstride, int16_t *src2, int height, intptr_t mx, intptr_t my, int width);
void ff_hevc_put_hevc_bi_pel_pixels24_8_avx2(uint8_t *_dst, ptrdiff_t _dststride, uint8_t *_src, ptrdiff_t _srcstride, int16_t *src2, int height, intptr_t mx, intptr_t my, int width);
void ff_hevc_put_hevc_bi_pel_pixels32_8_avx2(uint8_t *_dst, ptrdiff_t _dststride, uint8_t *_src, ptrdiff_t _srcstride, int16_t *src2, int height, intptr_t mx, intptr_t my, int width);
void ff_hevc_put_hevc_bi_pel_pixels48_8_avx2(uint8_t *_dst, ptrdiff_t _dststride, uint8_t *_src, ptrdiff_t _srcstride, int16_t *src2, int height, intptr_t mx, intptr_t my, int width);
void ff_hevc_put_hevc_bi_pel_pixels64_8_avx2(uint8_t *_dst, ptrdiff_t _dststride, uint8_t *_src, ptrdiff_t _srcstride, int16_t *src2, int height, intptr_t mx, intptr_t my, int width);
void ff_hevc_put_hevc_bi_pel_pixels16_10_avx2(uint8_t *_dst, ptrdiff_t _dststride, uint8_t *_src, ptrdiff_t _srcstride, int16_t *src2, int height, intptr_t mx, intptr_t my, int width);
void ff_hevc_put_hevc_bi_pel_pixels24_10_avx2(uint8_t *_dst, ptrdiff_t _dststride, uint8_t *_src, ptrdiff_t _srcstride, int16_t *src2, int height, intptr_t mx, intptr_t my, int width);
void ff_hevc_put_hevc_bi_pel_pixels32_10_avx2(uint8_t *_dst, ptrdiff_t _dststride, uint8_t *_src, ptrdiff_t _srcstride, int16_t *src2, int height, intptr_t mx, intptr_t my, int width);
void ff_hevc_put_hevc_bi_pel_pixels48_10_avx2(uint8_t *_dst, ptrdiff_t _dststride, uint8_t *_src, ptrdiff_t _srcstride, int16_t *src2, int height, intptr_t mx, intptr_t my, int width);
void ff_hevc_put_hevc_bi_pel_pixels64_10_avx2(uint8_t *_dst, ptrdiff_t _dststride, uint8_t *_src, ptrdiff_t _srcstride, int16_t *src2, int height, intptr_t mx, intptr_t my, int width);
///////////////////////////////////////////////////////////////////////////////
// EPEL
///////////////////////////////////////////////////////////////////////////////
EPEL_PROTOTYPES(epel_h , 8, sse4);
EPEL_PROTOTYPES(epel_h , 10, sse4);
EPEL_PROTOTYPES(epel_h , 12, sse4);
EPEL_PROTOTYPES(epel_v , 8, sse4);
EPEL_PROTOTYPES(epel_v , 10, sse4);
EPEL_PROTOTYPES(epel_v , 12, sse4);
EPEL_PROTOTYPES(epel_hv , 8, sse4);
EPEL_PROTOTYPES(epel_hv , 10, sse4);
EPEL_PROTOTYPES(epel_hv , 12, sse4);
PEL_PROTOTYPE(epel_h16, 8, avx2);
PEL_PROTOTYPE(epel_h24, 8, avx2);
PEL_PROTOTYPE(epel_h32, 8, avx2);
PEL_PROTOTYPE(epel_h48, 8, avx2);
PEL_PROTOTYPE(epel_h64, 8, avx2);
PEL_PROTOTYPE(epel_h16,10, avx2);
PEL_PROTOTYPE(epel_h24,10, avx2);
PEL_PROTOTYPE(epel_h32,10, avx2);
PEL_PROTOTYPE(epel_h48,10, avx2);
PEL_PROTOTYPE(epel_h64,10, avx2);
PEL_PROTOTYPE(epel_v16, 8, avx2);
PEL_PROTOTYPE(epel_v24, 8, avx2);
PEL_PROTOTYPE(epel_v32, 8, avx2);
PEL_PROTOTYPE(epel_v48, 8, avx2);
PEL_PROTOTYPE(epel_v64, 8, avx2);
PEL_PROTOTYPE(epel_v16,10, avx2);
PEL_PROTOTYPE(epel_v24,10, avx2);
PEL_PROTOTYPE(epel_v32,10, avx2);
PEL_PROTOTYPE(epel_v48,10, avx2);
PEL_PROTOTYPE(epel_v64,10, avx2);
PEL_PROTOTYPE(epel_hv16, 8, avx2);
PEL_PROTOTYPE(epel_hv24, 8, avx2);
PEL_PROTOTYPE(epel_hv32, 8, avx2);
PEL_PROTOTYPE(epel_hv48, 8, avx2);
PEL_PROTOTYPE(epel_hv64, 8, avx2);
PEL_PROTOTYPE(epel_hv16,10, avx2);
PEL_PROTOTYPE(epel_hv24,10, avx2);
PEL_PROTOTYPE(epel_hv32,10, avx2);
PEL_PROTOTYPE(epel_hv48,10, avx2);
PEL_PROTOTYPE(epel_hv64,10, avx2);
///////////////////////////////////////////////////////////////////////////////
// QPEL
///////////////////////////////////////////////////////////////////////////////
QPEL_PROTOTYPES(qpel_h , 8, sse4);
QPEL_PROTOTYPES(qpel_h , 10, sse4);
QPEL_PROTOTYPES(qpel_h , 12, sse4);
QPEL_PROTOTYPES(qpel_v, 8, sse4);
QPEL_PROTOTYPES(qpel_v, 10, sse4);
QPEL_PROTOTYPES(qpel_v, 12, sse4);
QPEL_PROTOTYPES(qpel_hv, 8, sse4);
QPEL_PROTOTYPES(qpel_hv, 10, sse4);
QPEL_PROTOTYPES(qpel_hv, 12, sse4);
PEL_PROTOTYPE(qpel_h16, 8, avx2);
PEL_PROTOTYPE(qpel_h24, 8, avx2);
PEL_PROTOTYPE(qpel_h32, 8, avx2);
PEL_PROTOTYPE(qpel_h48, 8, avx2);
PEL_PROTOTYPE(qpel_h64, 8, avx2);
PEL_PROTOTYPE(qpel_h16,10, avx2);
PEL_PROTOTYPE(qpel_h24,10, avx2);
PEL_PROTOTYPE(qpel_h32,10, avx2);
PEL_PROTOTYPE(qpel_h48,10, avx2);
PEL_PROTOTYPE(qpel_h64,10, avx2);
PEL_PROTOTYPE(qpel_v16, 8, avx2);
PEL_PROTOTYPE(qpel_v24, 8, avx2);
PEL_PROTOTYPE(qpel_v32, 8, avx2);
PEL_PROTOTYPE(qpel_v48, 8, avx2);
PEL_PROTOTYPE(qpel_v64, 8, avx2);
PEL_PROTOTYPE(qpel_v16,10, avx2);
PEL_PROTOTYPE(qpel_v24,10, avx2);
PEL_PROTOTYPE(qpel_v32,10, avx2);
PEL_PROTOTYPE(qpel_v48,10, avx2);
PEL_PROTOTYPE(qpel_v64,10, avx2);
PEL_PROTOTYPE(qpel_hv16, 8, avx2);
PEL_PROTOTYPE(qpel_hv24, 8, avx2);
PEL_PROTOTYPE(qpel_hv32, 8, avx2);
PEL_PROTOTYPE(qpel_hv48, 8, avx2);
PEL_PROTOTYPE(qpel_hv64, 8, avx2);
PEL_PROTOTYPE(qpel_hv16,10, avx2);
PEL_PROTOTYPE(qpel_hv24,10, avx2);
PEL_PROTOTYPE(qpel_hv32,10, avx2);
PEL_PROTOTYPE(qpel_hv48,10, avx2);
PEL_PROTOTYPE(qpel_hv64,10, avx2);
WEIGHTING_PROTOTYPES(8, sse4);
WEIGHTING_PROTOTYPES(10, sse4);
WEIGHTING_PROTOTYPES(12, sse4);
///////////////////////////////////////////////////////////////////////////////
// TRANSFORM_ADD
///////////////////////////////////////////////////////////////////////////////
void ff_hevc_add_residual_4_8_mmxext(uint8_t *dst, int16_t *res, ptrdiff_t stride);
void ff_hevc_add_residual_8_8_sse2(uint8_t *dst, int16_t *res, ptrdiff_t stride);
void ff_hevc_add_residual_16_8_sse2(uint8_t *dst, int16_t *res, ptrdiff_t stride);
void ff_hevc_add_residual_32_8_sse2(uint8_t *dst, int16_t *res, ptrdiff_t stride);
void ff_hevc_add_residual_8_8_avx(uint8_t *dst, int16_t *res, ptrdiff_t stride);
void ff_hevc_add_residual_16_8_avx(uint8_t *dst, int16_t *res, ptrdiff_t stride);
void ff_hevc_add_residual_32_8_avx(uint8_t *dst, int16_t *res, ptrdiff_t stride);
void ff_hevc_add_residual_32_8_avx2(uint8_t *dst, int16_t *res, ptrdiff_t stride);
void ff_hevc_add_residual_4_10_mmxext(uint8_t *dst, int16_t *res, ptrdiff_t stride);
void ff_hevc_add_residual_8_10_sse2(uint8_t *dst, int16_t *res, ptrdiff_t stride);
void ff_hevc_add_residual_16_10_sse2(uint8_t *dst, int16_t *res, ptrdiff_t stride);
void ff_hevc_add_residual_32_10_sse2(uint8_t *dst, int16_t *res, ptrdiff_t stride);
void ff_hevc_add_residual_16_10_avx2(uint8_t *dst, int16_t *res, ptrdiff_t stride);
void ff_hevc_add_residual_32_10_avx2(uint8_t *dst, int16_t *res, ptrdiff_t stride);
#endif // AVCODEC_X86_HEVCDSP_H

1151
media/ffvpx/libavcodec/x86/hevcdsp_init.c Normal file
View file

File diff suppressed because it is too large Load diff

183
media/ffvpx/libavcodec/x86/idctdsp.asm Normal file
View file

@ -0,0 +1,183 @@
;******************************************************************************
;* SIMD-optimized IDCT-related routines
;* Copyright (c) 2008 Loren Merritt
;* Copyright (c) 2003-2013 Michael Niedermayer
;* Copyright (c) 2013 Daniel Kang
;*
;* This file is part of FFmpeg.
;*
;* FFmpeg is free software; you can redistribute it and/or
;* modify it under the terms of the GNU Lesser General Public
;* License as published by the Free Software Foundation; either
;* version 2.1 of the License, or (at your option) any later version.
;*
;* FFmpeg is distributed in the hope that it will be useful,
;* but WITHOUT ANY WARRANTY; without even the implied warranty of
;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
;* Lesser General Public License for more details.
;*
;* You should have received a copy of the GNU Lesser General Public
;* License along with FFmpeg; if not, write to the Free Software
;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
;******************************************************************************
%include "libavutil/x86/x86util.asm"
SECTION_RODATA
cextern pb_80
SECTION .text
;--------------------------------------------------------------------------
;void ff_put_signed_pixels_clamped(const int16_t *block, uint8_t *pixels,
; ptrdiff_t line_size)
;--------------------------------------------------------------------------
%macro PUT_SIGNED_PIXELS_CLAMPED_HALF 1
mova m1, [blockq+mmsize*0+%1]
mova m2, [blockq+mmsize*2+%1]
%if mmsize == 8
mova m3, [blockq+mmsize*4+%1]
mova m4, [blockq+mmsize*6+%1]
%endif
packsswb m1, [blockq+mmsize*1+%1]
packsswb m2, [blockq+mmsize*3+%1]
%if mmsize == 8
packsswb m3, [blockq+mmsize*5+%1]
packsswb m4, [blockq+mmsize*7+%1]
%endif
paddb m1, m0
paddb m2, m0
%if mmsize == 8
paddb m3, m0
paddb m4, m0
movq [pixelsq+lsizeq*0], m1
movq [pixelsq+lsizeq*1], m2
movq [pixelsq+lsizeq*2], m3
movq [pixelsq+lsize3q ], m4
%else
movq [pixelsq+lsizeq*0], m1
movhps [pixelsq+lsizeq*1], m1
movq [pixelsq+lsizeq*2], m2
movhps [pixelsq+lsize3q ], m2
%endif
%endmacro
%macro PUT_SIGNED_PIXELS_CLAMPED 1
cglobal put_signed_pixels_clamped, 3, 4, %1, block, pixels, lsize, lsize3
mova m0, [pb_80]
lea lsize3q, [lsizeq*3]
PUT_SIGNED_PIXELS_CLAMPED_HALF 0
lea pixelsq, [pixelsq+lsizeq*4]
PUT_SIGNED_PIXELS_CLAMPED_HALF 64
RET
%endmacro
INIT_MMX mmx
PUT_SIGNED_PIXELS_CLAMPED 0
INIT_XMM sse2
PUT_SIGNED_PIXELS_CLAMPED 3
;--------------------------------------------------------------------------
; void ff_put_pixels_clamped(const int16_t *block, uint8_t *pixels,
; ptrdiff_t line_size);
;--------------------------------------------------------------------------
; %1 = block offset
%macro PUT_PIXELS_CLAMPED_HALF 1
mova m0, [blockq+mmsize*0+%1]
mova m1, [blockq+mmsize*2+%1]
%if mmsize == 8
mova m2, [blockq+mmsize*4+%1]
mova m3, [blockq+mmsize*6+%1]
%endif
packuswb m0, [blockq+mmsize*1+%1]
packuswb m1, [blockq+mmsize*3+%1]
%if mmsize == 8
packuswb m2, [blockq+mmsize*5+%1]
packuswb m3, [blockq+mmsize*7+%1]
movq [pixelsq], m0
movq [lsizeq+pixelsq], m1
movq [2*lsizeq+pixelsq], m2
movq [lsize3q+pixelsq], m3
%else
movq [pixelsq], m0
movhps [lsizeq+pixelsq], m0
movq [2*lsizeq+pixelsq], m1
movhps [lsize3q+pixelsq], m1
%endif
%endmacro
%macro PUT_PIXELS_CLAMPED 0
cglobal put_pixels_clamped, 3, 4, 2, block, pixels, lsize, lsize3
lea lsize3q, [lsizeq*3]
PUT_PIXELS_CLAMPED_HALF 0
lea pixelsq, [pixelsq+lsizeq*4]
PUT_PIXELS_CLAMPED_HALF 64
RET
%endmacro
INIT_MMX mmx
PUT_PIXELS_CLAMPED
INIT_XMM sse2
PUT_PIXELS_CLAMPED
;--------------------------------------------------------------------------
; void ff_add_pixels_clamped(const int16_t *block, uint8_t *pixels,
; ptrdiff_t line_size);
;--------------------------------------------------------------------------
; %1 = block offset
%macro ADD_PIXELS_CLAMPED 1
mova m0, [blockq+mmsize*0+%1]
mova m1, [blockq+mmsize*1+%1]
%if mmsize == 8
mova m5, [blockq+mmsize*2+%1]
mova m6, [blockq+mmsize*3+%1]
%endif
movq m2, [pixelsq]
movq m3, [pixelsq+lsizeq]
%if mmsize == 8
mova m7, m2
punpcklbw m2, m4
punpckhbw m7, m4
paddsw m0, m2
paddsw m1, m7
mova m7, m3
punpcklbw m3, m4
punpckhbw m7, m4
paddsw m5, m3
paddsw m6, m7
%else
punpcklbw m2, m4
punpcklbw m3, m4
paddsw m0, m2
paddsw m1, m3
%endif
packuswb m0, m1
%if mmsize == 8
packuswb m5, m6
movq [pixelsq], m0
movq [pixelsq+lsizeq], m5
%else
movq [pixelsq], m0
movhps [pixelsq+lsizeq], m0
%endif
%endmacro
%macro ADD_PIXELS_CLAMPED 0
cglobal add_pixels_clamped, 3, 3, 5, block, pixels, lsize
pxor m4, m4
ADD_PIXELS_CLAMPED 0
lea pixelsq, [pixelsq+lsizeq*2]
ADD_PIXELS_CLAMPED 32
lea pixelsq, [pixelsq+lsizeq*2]
ADD_PIXELS_CLAMPED 64
lea pixelsq, [pixelsq+lsizeq*2]
ADD_PIXELS_CLAMPED 96
RET
%endmacro
INIT_MMX mmx
ADD_PIXELS_CLAMPED
INIT_XMM sse2
ADD_PIXELS_CLAMPED

39
media/ffvpx/libavcodec/x86/idctdsp.h Normal file
View file

@ -0,0 +1,39 @@
/*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#ifndef AVCODEC_X86_IDCTDSP_H
#define AVCODEC_X86_IDCTDSP_H
#include <stddef.h>
#include <stdint.h>
void ff_add_pixels_clamped_mmx(const int16_t *block, uint8_t *pixels,
ptrdiff_t line_size);
void ff_add_pixels_clamped_sse2(const int16_t *block, uint8_t *pixels,
ptrdiff_t line_size);
void ff_put_pixels_clamped_mmx(const int16_t *block, uint8_t *pixels,
ptrdiff_t line_size);
void ff_put_pixels_clamped_sse2(const int16_t *block, uint8_t *pixels,
ptrdiff_t line_size);
void ff_put_signed_pixels_clamped_mmx(const int16_t *block, uint8_t *pixels,
ptrdiff_t line_size);
void ff_put_signed_pixels_clamped_sse2(const int16_t *block, uint8_t *pixels,
ptrdiff_t line_size);
#endif /* AVCODEC_X86_IDCTDSP_H */

161
media/ffvpx/libavcodec/x86/idctdsp_init.c Normal file
View file

@ -0,0 +1,161 @@
/*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "config.h"
#include "libavutil/attributes.h"
#include "libavutil/cpu.h"
#include "libavutil/x86/cpu.h"
#include "libavcodec/avcodec.h"
#include "libavcodec/idctdsp.h"
#include "idctdsp.h"
#include "simple_idct.h"
/* Input permutation for the simple_idct_mmx */
static const uint8_t simple_mmx_permutation[64] = {
0x00, 0x08, 0x04, 0x09, 0x01, 0x0C, 0x05, 0x0D,
0x10, 0x18, 0x14, 0x19, 0x11, 0x1C, 0x15, 0x1D,
0x20, 0x28, 0x24, 0x29, 0x21, 0x2C, 0x25, 0x2D,
0x12, 0x1A, 0x16, 0x1B, 0x13, 0x1E, 0x17, 0x1F,
0x02, 0x0A, 0x06, 0x0B, 0x03, 0x0E, 0x07, 0x0F,
0x30, 0x38, 0x34, 0x39, 0x31, 0x3C, 0x35, 0x3D,
0x22, 0x2A, 0x26, 0x2B, 0x23, 0x2E, 0x27, 0x2F,
0x32, 0x3A, 0x36, 0x3B, 0x33, 0x3E, 0x37, 0x3F,
};
static const uint8_t idct_sse2_row_perm[8] = { 0, 4, 1, 5, 2, 6, 3, 7 };
av_cold int ff_init_scantable_permutation_x86(uint8_t *idct_permutation,
enum idct_permutation_type perm_type)
{
int i;
switch (perm_type) {
case FF_IDCT_PERM_SIMPLE:
for (i = 0; i < 64; i++)
idct_permutation[i] = simple_mmx_permutation[i];
return 1;
case FF_IDCT_PERM_SSE2:
for (i = 0; i < 64; i++)
idct_permutation[i] = (i & 0x38) | idct_sse2_row_perm[i & 7];
return 1;
}
return 0;
}
av_cold void ff_idctdsp_init_x86(IDCTDSPContext *c, AVCodecContext *avctx,
unsigned high_bit_depth)
{
int cpu_flags = av_get_cpu_flags();
if (EXTERNAL_MMX(cpu_flags)) {
c->put_signed_pixels_clamped = ff_put_signed_pixels_clamped_mmx;
c->put_pixels_clamped = ff_put_pixels_clamped_mmx;
c->add_pixels_clamped = ff_add_pixels_clamped_mmx;
if (!high_bit_depth &&
avctx->lowres == 0 &&
(avctx->idct_algo == FF_IDCT_AUTO ||
avctx->idct_algo == FF_IDCT_SIMPLEAUTO ||
avctx->idct_algo == FF_IDCT_SIMPLEMMX)) {
c->idct_put = ff_simple_idct_put_mmx;
c->idct_add = ff_simple_idct_add_mmx;
c->idct = ff_simple_idct_mmx;
c->perm_type = FF_IDCT_PERM_SIMPLE;
}
}
if (EXTERNAL_SSE2(cpu_flags)) {
c->put_signed_pixels_clamped = ff_put_signed_pixels_clamped_sse2;
c->put_pixels_clamped = ff_put_pixels_clamped_sse2;
c->add_pixels_clamped = ff_add_pixels_clamped_sse2;
if (!high_bit_depth &&
avctx->lowres == 0 &&
(avctx->idct_algo == FF_IDCT_AUTO ||
avctx->idct_algo == FF_IDCT_SIMPLEAUTO ||
avctx->idct_algo == FF_IDCT_SIMPLEMMX)) {
c->idct_put = ff_simple_idct_put_sse2;
c->idct_add = ff_simple_idct_add_sse2;
c->perm_type = FF_IDCT_PERM_SIMPLE;
}
if (ARCH_X86_64 &&
!high_bit_depth &&
avctx->lowres == 0 &&
(avctx->idct_algo == FF_IDCT_AUTO ||
avctx->idct_algo == FF_IDCT_SIMPLEAUTO ||
avctx->idct_algo == FF_IDCT_SIMPLEMMX ||
avctx->idct_algo == FF_IDCT_SIMPLE)) {
c->idct = ff_simple_idct8_sse2;
c->idct_put = ff_simple_idct8_put_sse2;
c->idct_add = ff_simple_idct8_add_sse2;
c->perm_type = FF_IDCT_PERM_TRANSPOSE;
}
}
if (ARCH_X86_64 && avctx->lowres == 0) {
if (EXTERNAL_AVX(cpu_flags) &&
!high_bit_depth &&
(avctx->idct_algo == FF_IDCT_AUTO ||
avctx->idct_algo == FF_IDCT_SIMPLEAUTO ||
avctx->idct_algo == FF_IDCT_SIMPLEMMX ||
avctx->idct_algo == FF_IDCT_SIMPLE)) {
c->idct = ff_simple_idct8_avx;
c->idct_put = ff_simple_idct8_put_avx;
c->idct_add = ff_simple_idct8_add_avx;
c->perm_type = FF_IDCT_PERM_TRANSPOSE;
}
if (avctx->bits_per_raw_sample == 10 &&
(avctx->idct_algo == FF_IDCT_AUTO ||
avctx->idct_algo == FF_IDCT_SIMPLEAUTO ||
avctx->idct_algo == FF_IDCT_SIMPLE)) {
if (EXTERNAL_SSE2(cpu_flags)) {
c->idct_put = ff_simple_idct10_put_sse2;
c->idct_add = NULL;
c->idct = ff_simple_idct10_sse2;
c->perm_type = FF_IDCT_PERM_TRANSPOSE;
}
if (EXTERNAL_AVX(cpu_flags)) {
c->idct_put = ff_simple_idct10_put_avx;
c->idct_add = NULL;
c->idct = ff_simple_idct10_avx;
c->perm_type = FF_IDCT_PERM_TRANSPOSE;
}
}
if (avctx->bits_per_raw_sample == 12 &&
(avctx->idct_algo == FF_IDCT_AUTO ||
avctx->idct_algo == FF_IDCT_SIMPLEMMX)) {
if (EXTERNAL_SSE2(cpu_flags)) {
c->idct_put = ff_simple_idct12_put_sse2;
c->idct_add = NULL;
c->idct = ff_simple_idct12_sse2;
c->perm_type = FF_IDCT_PERM_TRANSPOSE;
}
if (EXTERNAL_AVX(cpu_flags)) {
c->idct_put = ff_simple_idct12_put_avx;
c->idct_add = NULL;
c->idct = ff_simple_idct12_avx;
c->perm_type = FF_IDCT_PERM_TRANSPOSE;
}
}
}
}

742
media/ffvpx/libavcodec/x86/imdct36.asm Normal file
View file

@ -0,0 +1,742 @@
;******************************************************************************
;* 36 point SSE-optimized IMDCT transform
;* Copyright (c) 2011 Vitor Sessak
;*
;* This file is part of FFmpeg.
;*
;* FFmpeg is free software; you can redistribute it and/or
;* modify it under the terms of the GNU Lesser General Public
;* License as published by the Free Software Foundation; either
;* version 2.1 of the License, or (at your option) any later version.
;*
;* FFmpeg is distributed in the hope that it will be useful,
;* but WITHOUT ANY WARRANTY; without even the implied warranty of
;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
;* Lesser General Public License for more details.
;*
;* You should have received a copy of the GNU Lesser General Public
;* License along with FFmpeg; if not, write to the Free Software
;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
;******************************************************************************
%include "libavutil/x86/x86util.asm"
SECTION_RODATA
align 16
ps_mask: dd 0, ~0, ~0, ~0
ps_mask2: dd 0, ~0, 0, ~0
ps_mask3: dd 0, 0, 0, ~0
ps_mask4: dd 0, ~0, 0, 0
ps_val1: dd -0.5, -0.5, -0.8660254038, -0.8660254038
ps_val2: dd 1.0, 1.0, 0.8660254038, 0.8660254038
ps_val3: dd 0.1736481777, 0.1736481777, 0.3420201433, 0.3420201433
ps_val4: dd -0.7660444431, -0.7660444431, 0.8660254038, 0.8660254038
ps_val5: dd -0.9396926208, -0.9396926208, -0.9848077530, -0.9848077530
ps_val6: dd 0.5, 0.5, -0.6427876097, -0.6427876097
ps_val7: dd 1.0, 1.0, -0.6427876097, -0.6427876097
ps_p1p1m1m1: dd 0, 0, 0x80000000, 0x80000000
ps_p1m1p1m1: dd 0, 0x80000000, 0, 0x80000000
ps_cosh: dd 1.0, 0.50190991877167369479, 1.0, 5.73685662283492756461
dd 1.0, 0.51763809020504152469, 1.0, 1.93185165257813657349
dd 1.0, 0.55168895948124587824, -1.0, -1.18310079157624925896
dd 1.0, 0.61038729438072803416, -1.0, -0.87172339781054900991
dd 1.0, 0.70710678118654752439, 0.0, 0.0
ps_cosh_sse3: dd 1.0, -0.50190991877167369479, 1.0, -5.73685662283492756461
dd 1.0, -0.51763809020504152469, 1.0, -1.93185165257813657349
dd 1.0, -0.55168895948124587824, -1.0, 1.18310079157624925896
dd 1.0, -0.61038729438072803416, -1.0, 0.87172339781054900991
dd 1.0, -0.70710678118654752439, 0.0, 0.0
costabs: times 4 dd 0.98480773
times 4 dd 0.93969262
times 4 dd 0.86602539
times 4 dd -0.76604444
times 4 dd -0.64278764
times 4 dd 0.50000000
times 4 dd -0.50000000
times 4 dd -0.34202015
times 4 dd -0.17364818
times 4 dd 0.50190992
times 4 dd 0.51763808
times 4 dd 0.55168896
times 4 dd 0.61038726
times 4 dd 0.70710677
times 4 dd 0.87172341
times 4 dd 1.18310082
times 4 dd 1.93185163
times 4 dd 5.73685646
%define SBLIMIT 32
SECTION .text
%macro PSHUFD 3
%if cpuflag(sse2) && notcpuflag(avx)
pshufd %1, %2, %3
%else
shufps %1, %2, %2, %3
%endif
%endmacro
; input %2={x1,x2,x3,x4}, %3={y1,y2,y3,y4}
; output %1={x3,x4,y1,y2}
%macro BUILDINVHIGHLOW 3
%if cpuflag(avx)
shufps %1, %2, %3, 0x4e
%else
movlhps %1, %3
movhlps %1, %2
%endif
%endmacro
; input %2={x1,x2,x3,x4}, %3={y1,y2,y3,y4}
; output %1={x4,y1,y2,y3}
%macro ROTLEFT 3
%if cpuflag(ssse3)
palignr %1, %3, %2, 12
%else
BUILDINVHIGHLOW %1, %2, %3
shufps %1, %1, %3, 0x99
%endif
%endmacro
%macro INVERTHL 2
%if cpuflag(sse2)
PSHUFD %1, %2, 0x4e
%else
movhlps %1, %2
movlhps %1, %2
%endif
%endmacro
%macro BUTTERF 3
INVERTHL %2, %1
xorps %1, [ps_p1p1m1m1]
addps %1, %2
%if cpuflag(sse3)
mulps %1, %1, [ps_cosh_sse3 + %3]
PSHUFD %2, %1, 0xb1
addsubps %1, %1, %2
%else
mulps %1, [ps_cosh + %3]
PSHUFD %2, %1, 0xb1
xorps %1, [ps_p1m1p1m1]
addps %1, %2
%endif
%endmacro
%macro BUTTERF2 3
%if cpuflag(sse3)
mulps %1, %1, [ps_cosh_sse3 + %3]
PSHUFD %2, %1, 0xe1
addsubps %1, %1, %2
%else
mulps %1, [ps_cosh + %3]
PSHUFD %2, %1, 0xe1
xorps %1, [ps_p1m1p1m1]
addps %1, %2
%endif
%endmacro
%macro STORE 4
%if cpuflag(sse4)
movss [%3 ], %1
extractps dword [%3 + %4], %1, 1
extractps dword [%3 + 2*%4], %1, 2
extractps dword [%3 + 3*%4], %1, 3
%else
movhlps %2, %1
movss [%3 ], %1
movss [%3 + 2*%4], %2
shufps %1, %1, 0xb1
movss [%3 + %4], %1
movhlps %2, %1
movss [%3 + 3*%4], %2
%endif
%endmacro
%macro LOAD 4
movlps %1, [%3 ]
movhps %1, [%3 + %4]
movlps %2, [%3 + 2*%4]
movhps %2, [%3 + 3*%4]
shufps %1, %2, 0x88
%endmacro
%macro LOADA64 2
%if cpuflag(avx)
movu %1, [%2]
%else
movlps %1, [%2]
movhps %1, [%2 + 8]
%endif
%endmacro
%macro DEFINE_IMDCT 0
cglobal imdct36_float, 4,4,9, out, buf, in, win
; for(i=17;i>=1;i--) in[i] += in[i-1];
LOADA64 m0, inq
LOADA64 m1, inq + 16
ROTLEFT m5, m0, m1
PSHUFD m6, m0, 0x93
andps m6, m6, [ps_mask]
addps m0, m0, m6
LOADA64 m2, inq + 32
ROTLEFT m7, m1, m2
addps m1, m1, m5
LOADA64 m3, inq + 48
ROTLEFT m5, m2, m3
xorps m4, m4, m4
movlps m4, [inq+64]
BUILDINVHIGHLOW m6, m3, m4
shufps m6, m6, m4, 0xa9
addps m4, m4, m6
addps m2, m2, m7
addps m3, m3, m5
; for(i=17;i>=3;i-=2) in[i] += in[i-2];
movlhps m5, m5, m0
andps m5, m5, [ps_mask3]
BUILDINVHIGHLOW m7, m0, m1
andps m7, m7, [ps_mask2]
addps m0, m0, m5
BUILDINVHIGHLOW m6, m1, m2
andps m6, m6, [ps_mask2]
addps m1, m1, m7
BUILDINVHIGHLOW m7, m2, m3
andps m7, m7, [ps_mask2]
addps m2, m2, m6
movhlps m6, m6, m3
andps m6, m6, [ps_mask4]
addps m3, m3, m7
addps m4, m4, m6
; Populate tmp[]
movlhps m6, m1, m5 ; zero out high values
subps m6, m6, m4
subps m5, m0, m3
%if ARCH_X86_64
SWAP m5, m8
%endif
mulps m7, m2, [ps_val1]
%if ARCH_X86_64
mulps m5, m8, [ps_val2]
%else
mulps m5, m5, [ps_val2]
%endif
addps m7, m7, m5
mulps m5, m6, [ps_val1]
subps m7, m7, m5
%if ARCH_X86_64
SWAP m5, m8
%else
subps m5, m0, m3
%endif
subps m5, m5, m6
addps m5, m5, m2
shufps m6, m4, m3, 0xe4
subps m6, m6, m2
mulps m6, m6, [ps_val3]
addps m4, m4, m1
mulps m4, m4, [ps_val4]
shufps m1, m1, m0, 0xe4
addps m1, m1, m2
mulps m1, m1, [ps_val5]
mulps m3, m3, [ps_val6]
mulps m0, m0, [ps_val7]
addps m0, m0, m3
xorps m2, m1, [ps_p1p1m1m1]
subps m2, m2, m4
addps m2, m2, m0
addps m3, m4, m0
subps m3, m3, m6
xorps m3, m3, [ps_p1p1m1m1]
shufps m0, m0, m4, 0xe4
subps m0, m0, m1
addps m0, m0, m6
BUILDINVHIGHLOW m4, m2, m3
shufps m3, m3, m2, 0x4e
; we have tmp = {SwAPLH(m0), SwAPLH(m7), m3, m4, m5}
BUTTERF m0, m1, 0
BUTTERF m7, m2, 16
BUTTERF m3, m6, 32
BUTTERF m4, m1, 48
BUTTERF2 m5, m1, 64
; permutates:
; m0 0 1 2 3 => 2 6 10 14 m1
; m7 4 5 6 7 => 3 7 11 15 m2
; m3 8 9 10 11 => 17 13 9 5 m3
; m4 12 13 14 15 => 16 12 8 4 m5
; m5 16 17 xx xx => 0 1 xx xx m0
unpckhps m1, m0, m7
unpckhps m6, m3, m4
movhlps m2, m6, m1
movlhps m1, m1, m6
unpcklps m5, m5, m4
unpcklps m3, m3, m7
movhlps m4, m3, m5
movlhps m5, m5, m3
SWAP m4, m3
; permutation done
PSHUFD m6, m2, 0xb1
movss m4, [bufq + 4*68]
movss m7, [bufq + 4*64]
unpcklps m7, m7, m4
mulps m6, m6, [winq + 16*4]
addps m6, m6, m7
movss [outq + 64*SBLIMIT], m6
shufps m6, m6, m6, 0xb1
movss [outq + 68*SBLIMIT], m6
mulps m6, m3, [winq + 4*4]
LOAD m4, m7, bufq + 4*16, 16
addps m6, m6, m4
STORE m6, m7, outq + 16*SBLIMIT, 4*SBLIMIT
shufps m4, m0, m3, 0xb5
mulps m4, m4, [winq + 8*4]
LOAD m7, m6, bufq + 4*32, 16
addps m4, m4, m7
STORE m4, m6, outq + 32*SBLIMIT, 4*SBLIMIT
shufps m3, m3, m2, 0xb1
mulps m3, m3, [winq + 12*4]
LOAD m7, m6, bufq + 4*48, 16
addps m3, m3, m7
STORE m3, m7, outq + 48*SBLIMIT, 4*SBLIMIT
mulps m2, m2, [winq]
LOAD m6, m7, bufq, 16
addps m2, m2, m6
STORE m2, m7, outq, 4*SBLIMIT
mulps m4, m1, [winq + 20*4]
STORE m4, m7, bufq, 16
mulps m3, m5, [winq + 24*4]
STORE m3, m7, bufq + 4*16, 16
shufps m0, m0, m5, 0xb0
mulps m0, m0, [winq + 28*4]
STORE m0, m7, bufq + 4*32, 16
shufps m5, m5, m1, 0xb1
mulps m5, m5, [winq + 32*4]
STORE m5, m7, bufq + 4*48, 16
shufps m1, m1, m1, 0xb1
mulps m1, m1, [winq + 36*4]
movss [bufq + 4*64], m1
shufps m1, m1, 0xb1
movss [bufq + 4*68], m1
RET
%endmacro
%if ARCH_X86_32
INIT_XMM sse
DEFINE_IMDCT
%endif
INIT_XMM sse2
DEFINE_IMDCT
INIT_XMM sse3
DEFINE_IMDCT
INIT_XMM ssse3
DEFINE_IMDCT
%if HAVE_AVX_EXTERNAL
INIT_XMM avx
DEFINE_IMDCT
%endif
INIT_XMM sse
%if ARCH_X86_64
%define SPILL SWAP
%define UNSPILL SWAP
%define SPILLED(x) m %+ x
%else
%define SPILLED(x) [tmpq+(x-8)*16 + 32*4]
%macro SPILL 2 ; xmm#, mempos
movaps SPILLED(%2), m%1
%endmacro
%macro UNSPILL 2
movaps m%1, SPILLED(%2)
%endmacro
%endif
%macro DEFINE_FOUR_IMDCT 0
cglobal four_imdct36_float, 5,5,16, out, buf, in, win, tmp
movlps m0, [inq+64]
movhps m0, [inq+64 + 72]
movlps m3, [inq+64 + 2*72]
movhps m3, [inq+64 + 3*72]
shufps m5, m0, m3, 0xdd
shufps m0, m0, m3, 0x88
mova m1, [inq+48]
movu m6, [inq+48 + 72]
mova m7, [inq+48 + 2*72]
movu m3, [inq+48 + 3*72]
TRANSPOSE4x4PS 1, 6, 7, 3, 4
addps m4, m6, m7
mova [tmpq+4*28], m4
addps m7, m3
addps m6, m1
addps m3, m0
addps m0, m5
addps m0, m7
addps m7, m6
mova [tmpq+4*12], m7
SPILL 3, 12
mova m4, [inq+32]
movu m5, [inq+32 + 72]
mova m2, [inq+32 + 2*72]
movu m7, [inq+32 + 3*72]
TRANSPOSE4x4PS 4, 5, 2, 7, 3
addps m1, m7
SPILL 1, 11
addps m3, m5, m2
SPILL 3, 13
addps m7, m2
addps m5, m4
addps m6, m7
mova [tmpq], m6
addps m7, m5
mova [tmpq+4*16], m7
mova m2, [inq+16]
movu m7, [inq+16 + 72]
mova m1, [inq+16 + 2*72]
movu m6, [inq+16 + 3*72]
TRANSPOSE4x4PS 2, 7, 1, 6, 3
addps m4, m6
addps m6, m1
addps m1, m7
addps m7, m2
addps m5, m6
SPILL 5, 15
addps m6, m7
mulps m6, [costabs + 16*2]
mova [tmpq+4*8], m6
SPILL 1, 10
SPILL 0, 14
mova m1, [inq]
movu m6, [inq + 72]
mova m3, [inq + 2*72]
movu m5, [inq + 3*72]
TRANSPOSE4x4PS 1, 6, 3, 5, 0
addps m2, m5
addps m5, m3
addps m7, m5
addps m3, m6
addps m6, m1
SPILL 7, 8
addps m5, m6
SPILL 6, 9
addps m6, m4, SPILLED(12)
subps m6, m2
UNSPILL 7, 11
SPILL 5, 11
subps m5, m1, m7
mulps m7, [costabs + 16*5]
addps m7, m1
mulps m0, m6, [costabs + 16*6]
addps m0, m5
mova [tmpq+4*24], m0
addps m6, m5
mova [tmpq+4*4], m6
addps m6, m4, m2
mulps m6, [costabs + 16*1]
subps m4, SPILLED(12)
mulps m4, [costabs + 16*8]
addps m2, SPILLED(12)
mulps m2, [costabs + 16*3]
subps m5, m7, m6
subps m5, m2
addps m6, m7
addps m6, m4
addps m7, m2
subps m7, m4
mova [tmpq+4*20], m7
mova m2, [tmpq+4*28]
mova [tmpq+4*28], m5
UNSPILL 7, 13
subps m5, m7, m2
mulps m5, [costabs + 16*7]
UNSPILL 1, 10
mulps m1, [costabs + 16*2]
addps m4, m3, m2
mulps m4, [costabs + 16*4]
addps m2, m7
addps m7, m3
mulps m7, [costabs]
subps m3, m2
mulps m3, [costabs + 16*2]
addps m2, m7, m5
addps m2, m1
SPILL 2, 10
addps m7, m4
subps m7, m1
SPILL 7, 12
subps m5, m4
subps m5, m1
UNSPILL 0, 14
SPILL 5, 13
addps m1, m0, SPILLED(15)
subps m1, SPILLED(8)
mova m4, [costabs + 16*5]
mulps m4, [tmpq]
UNSPILL 2, 9
addps m4, m2
subps m2, [tmpq]
mulps m5, m1, [costabs + 16*6]
addps m5, m2
SPILL 5, 9
addps m2, m1
SPILL 2, 14
UNSPILL 5, 15
subps m7, m5, m0
addps m5, SPILLED(8)
mulps m5, [costabs + 16*1]
mulps m7, [costabs + 16*8]
addps m0, SPILLED(8)
mulps m0, [costabs + 16*3]
subps m2, m4, m5
subps m2, m0
SPILL 2, 15
addps m5, m4
addps m5, m7
addps m4, m0
subps m4, m7
SPILL 4, 8
mova m7, [tmpq+4*16]
mova m2, [tmpq+4*12]
addps m0, m7, m2
subps m0, SPILLED(11)
mulps m0, [costabs + 16*2]
addps m4, m7, SPILLED(11)
mulps m4, [costabs]
subps m7, m2
mulps m7, [costabs + 16*7]
addps m2, SPILLED(11)
mulps m2, [costabs + 16*4]
addps m1, m7, [tmpq+4*8]
addps m1, m4
addps m4, m2
subps m4, [tmpq+4*8]
SPILL 4, 11
subps m7, m2
subps m7, [tmpq+4*8]
addps m4, m6, SPILLED(10)
subps m6, SPILLED(10)
addps m2, m5, m1
mulps m2, [costabs + 16*9]
subps m5, m1
mulps m5, [costabs + 16*17]
subps m1, m4, m2
addps m4, m2
mulps m2, m1, [winq+4*36]
addps m2, [bufq+4*36]
mova [outq+1152], m2
mulps m1, [winq+4*32]
addps m1, [bufq+4*32]
mova [outq+1024], m1
mulps m1, m4, [winq+4*116]
mova [bufq+4*36], m1
mulps m4, [winq+4*112]
mova [bufq+4*32], m4
addps m2, m6, m5
subps m6, m5
mulps m1, m6, [winq+4*68]
addps m1, [bufq+4*68]
mova [outq+2176], m1
mulps m6, [winq]
addps m6, [bufq]
mova [outq], m6
mulps m1, m2, [winq+4*148]
mova [bufq+4*68], m1
mulps m2, [winq+4*80]
mova [bufq], m2
addps m5, m3, [tmpq+4*24]
mova m2, [tmpq+4*24]
subps m2, m3
mova m1, SPILLED(9)
subps m1, m0
mulps m1, [costabs + 16*10]
addps m0, SPILLED(9)
mulps m0, [costabs + 16*16]
addps m6, m5, m1
subps m5, m1
mulps m3, m5, [winq+4*40]
addps m3, [bufq+4*40]
mova [outq+1280], m3
mulps m5, [winq+4*28]
addps m5, [bufq+4*28]
mova [outq+896], m5
mulps m1, m6, [winq+4*120]
mova [bufq+4*40], m1
mulps m6, [winq+4*108]
mova [bufq+4*28], m6
addps m1, m2, m0
subps m2, m0
mulps m5, m2, [winq+4*64]
addps m5, [bufq+4*64]
mova [outq+2048], m5
mulps m2, [winq+4*4]
addps m2, [bufq+4*4]
mova [outq+128], m2
mulps m0, m1, [winq+4*144]
mova [bufq+4*64], m0
mulps m1, [winq+4*84]
mova [bufq+4*4], m1
mova m1, [tmpq+4*28]
mova m5, m1
addps m1, SPILLED(13)
subps m5, SPILLED(13)
UNSPILL 3, 15
addps m2, m7, m3
mulps m2, [costabs + 16*11]
subps m3, m7
mulps m3, [costabs + 16*15]
addps m0, m2, m1
subps m1, m2
SWAP m0, m2
mulps m6, m1, [winq+4*44]
addps m6, [bufq+4*44]
mova [outq+1408], m6
mulps m1, [winq+4*24]
addps m1, [bufq+4*24]
mova [outq+768], m1
mulps m0, m2, [winq+4*124]
mova [bufq+4*44], m0
mulps m2, [winq+4*104]
mova [bufq+4*24], m2
addps m0, m5, m3
subps m5, m3
mulps m1, m5, [winq+4*60]
addps m1, [bufq+4*60]
mova [outq+1920], m1
mulps m5, [winq+4*8]
addps m5, [bufq+4*8]
mova [outq+256], m5
mulps m1, m0, [winq+4*140]
mova [bufq+4*60], m1
mulps m0, [winq+4*88]
mova [bufq+4*8], m0
mova m1, [tmpq+4*20]
addps m1, SPILLED(12)
mova m2, [tmpq+4*20]
subps m2, SPILLED(12)
UNSPILL 7, 8
subps m0, m7, SPILLED(11)
addps m7, SPILLED(11)
mulps m4, m7, [costabs + 16*12]
mulps m0, [costabs + 16*14]
addps m5, m1, m4
subps m1, m4
mulps m7, m1, [winq+4*48]
addps m7, [bufq+4*48]
mova [outq+1536], m7
mulps m1, [winq+4*20]
addps m1, [bufq+4*20]
mova [outq+640], m1
mulps m1, m5, [winq+4*128]
mova [bufq+4*48], m1
mulps m5, [winq+4*100]
mova [bufq+4*20], m5
addps m6, m2, m0
subps m2, m0
mulps m1, m2, [winq+4*56]
addps m1, [bufq+4*56]
mova [outq+1792], m1
mulps m2, [winq+4*12]
addps m2, [bufq+4*12]
mova [outq+384], m2
mulps m0, m6, [winq+4*136]
mova [bufq+4*56], m0
mulps m6, [winq+4*92]
mova [bufq+4*12], m6
UNSPILL 0, 14
mulps m0, [costabs + 16*13]
mova m3, [tmpq+4*4]
addps m2, m0, m3
subps m3, m0
mulps m0, m3, [winq+4*52]
addps m0, [bufq+4*52]
mova [outq+1664], m0
mulps m3, [winq+4*16]
addps m3, [bufq+4*16]
mova [outq+512], m3
mulps m0, m2, [winq+4*132]
mova [bufq+4*52], m0
mulps m2, [winq+4*96]
mova [bufq+4*16], m2
RET
%endmacro
INIT_XMM sse
DEFINE_FOUR_IMDCT
%if HAVE_AVX_EXTERNAL
INIT_XMM avx
DEFINE_FOUR_IMDCT
%endif

224
media/ffvpx/libavcodec/x86/mdct15.asm Normal file
View file

@ -0,0 +1,224 @@
;******************************************************************************
;* SIMD optimized non-power-of-two MDCT functions
;*
;* Copyright (C) 2017 Rostislav Pehlivanov <atomnuker@gmail.com>
;*
;* This file is part of FFmpeg.
;*
;* FFmpeg is free software; you can redistribute it and/or
;* modify it under the terms of the GNU Lesser General Public
;* License as published by the Free Software Foundation; either
;* version 2.1 of the License, or (at your option) any later version.
;*
;* FFmpeg is distributed in the hope that it will be useful,
;* but WITHOUT ANY WARRANTY; without even the implied warranty of
;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
;* Lesser General Public License for more details.
;*
;* You should have received a copy of the GNU Lesser General Public
;* License along with FFmpeg; if not, write to the Free Software
;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
;******************************************************************************
%include "libavutil/x86/x86util.asm"
SECTION_RODATA 32
perm_neg: dd 2, 5, 3, 4, 6, 1, 7, 0
perm_pos: dd 0, 7, 1, 6, 4, 3, 5, 2
sign_adjust_r: times 4 dd 0x80000000, 0x00000000
sign_adjust_5: dd 0x00000000, 0x80000000, 0x80000000, 0x00000000
SECTION .text
%if ARCH_X86_64
;*****************************************************************************************
;void ff_fft15_avx(FFTComplex *out, FFTComplex *in, FFTComplex *exptab, ptrdiff_t stride);
;*****************************************************************************************
%macro FFT5 3 ; %1 - in_offset, %2 - dst1 (64bit used), %3 - dst2
VBROADCASTSD m0, [inq + %1] ; in[ 0].re, in[ 0].im, in[ 0].re, in[ 0].im
movsd xm1, [inq + 1*16 + 8 + %1] ; in[ 3].re, in[ 3].im, 0, 0
movsd xm4, [inq + 6*16 + 0 + %1] ; in[12].re, in[12].im, 0, 0
movhps xm1, [inq + 3*16 + 0 + %1] ; in[ 3].re, in[ 3].im, in[ 6].re, in[ 6].im
movhps xm4, [inq + 4*16 + 8 + %1] ; in[12].re, in[12].im, in[ 9].re, in[ 9].im
subps xm2, xm1, xm4 ; t[2].im, t[2].re, t[3].im, t[3].re
addps xm1, xm4 ; t[0].re, t[0].im, t[1].re, t[1].im
movhlps %2, xm1 ; t[0].re, t[1].re, t[0].im, t[1].im
addps %2, xm1
addps %2, xm0 ; DC[0].re, DC[0].im, junk...
movlhps %2, %2 ; DC[0].re, DC[0].im, DC[0].re, DC[0].im
shufps xm3, xm1, xm2, q0110 ; t[0].re, t[0].im, t[2].re, t[2].im
shufps xm1, xm2, q2332 ; t[1].re, t[1].im, t[3].re, t[3].im
mulps xm%3, xm1, xm5
mulps xm4, xm3, xm6
mulps xm1, xm6
xorps xm1, xm7
mulps xm3, xm5
addsubps xm3, xm1 ; t[0].re, t[0].im, t[2].re, t[2].im
subps xm%3, xm4 ; t[4].re, t[4].im, t[5].re, t[5].im
movhlps xm2, xm%3, xm3 ; t[2].re, t[2].im, t[5].re, t[5].im
movlhps xm3, xm%3 ; t[0].re, t[0].im, t[4].re, t[4].im
xorps xm2, xm7
addps xm%3, xm2, xm3
subps xm3, xm2
shufps xm3, xm3, q1032
vinsertf128 m%3, m%3, xm3, 1 ; All ACs (tmp[1] through to tmp[4])
addps m%3, m%3, m0 ; Finally offset with DCs
%endmacro
%macro BUTTERFLIES_DC 2 ; %1 - exptab_offset, %2 - out
mulps xm0, xm9, [exptabq + %1 + 16*0]
mulps xm1, xm10, [exptabq + %1 + 16*1]
haddps xm0, xm1
movhlps xm1, xm0 ; t[0].re, t[1].re, t[0].im, t[1].im
addps xm0, xm1
addps xm0, xm8
movsd [%2q], xm0
%endmacro
%macro BUTTERFLIES_AC 2 ; exptab, exptab_offset, src1, src2, src3, out (uses m0-m3)
mulps m0, m12, [exptabq + 64*0 + 0*mmsize + %1]
mulps m1, m12, [exptabq + 64*0 + 1*mmsize + %1]
mulps m2, m13, [exptabq + 64*1 + 0*mmsize + %1]
mulps m3, m13, [exptabq + 64*1 + 1*mmsize + %1]
addps m0, m0, m2
addps m1, m1, m3
addps m0, m0, m11
shufps m1, m1, m1, q2301
addps m0, m0, m1
vextractf128 xm1, m0, 1
movlps [%2q + strideq*1], xm0
movhps [%2q + strideq*2], xm0
movlps [%2q + stride3q], xm1
movhps [%2q + strideq*4], xm1
%endmacro
INIT_YMM avx
cglobal fft15, 4, 6, 14, out, in, exptab, stride, stride3, stride5
%define out0q inq
shl strideq, 3
movaps xm5, [exptabq + 480 + 16*0]
movaps xm6, [exptabq + 480 + 16*1]
movaps xm7, [sign_adjust_5]
FFT5 0, xm8, 11
FFT5 8, xm9, 12
FFT5 16, xm10, 13
lea stride3q, [strideq + strideq*2]
lea stride5q, [strideq + strideq*4]
mov out0q, outq
BUTTERFLIES_DC (8*6 + 4*0)*2*4, out0
lea outq, [out0q + stride5q*1]
BUTTERFLIES_DC (8*6 + 4*1)*2*4, out
lea outq, [out0q + stride5q*2]
BUTTERFLIES_DC (8*6 + 4*2)*2*4, out
BUTTERFLIES_AC (8*0)*2*4, out0
lea outq, [out0q + stride5q*1]
BUTTERFLIES_AC (8*2)*2*4, out
lea outq, [out0q + stride5q*2]
BUTTERFLIES_AC (8*4)*2*4, out
RET
%endif ; ARCH_X86_64
;*******************************************************************************************************
;void ff_mdct15_postreindex(FFTComplex *out, FFTComplex *in, FFTComplex *exp, int *lut, ptrdiff_t len8);
;*******************************************************************************************************
%macro LUT_LOAD_4D 3
mov r4d, [lutq + %3q*4 + 0]
movsd xmm%1, [inq + r4q*8]
mov r4d, [lutq + %3q*4 + 4]
movhps xmm%1, [inq + r4q*8]
%if cpuflag(avx2)
mov r4d, [lutq + %3q*4 + 8]
movsd %2, [inq + r4q*8]
mov r4d, [lutq + %3q*4 + 12]
movhps %2, [inq + r4q*8]
vinsertf128 %1, %1, %2, 1
%endif
%endmacro
%macro POSTROTATE_FN 1
cglobal mdct15_postreindex, 5, 7, 8 + cpuflag(avx2)*2, out, in, exp, lut, len8, offset_p, offset_n
xor offset_nq, offset_nq
lea offset_pq, [len8q*2 - %1]
movaps m7, [sign_adjust_r]
%if cpuflag(avx2)
movaps m8, [perm_pos]
movaps m9, [perm_neg]
%endif
.loop:
movups m0, [expq + offset_pq*8] ; exp[p0].re, exp[p0].im, exp[p1].re, exp[p1].im, exp[p2].re, exp[p2].im, exp[p3].re, exp[p3].im
movups m1, [expq + offset_nq*8] ; exp[n3].re, exp[n3].im, exp[n2].re, exp[n2].im, exp[n1].re, exp[n1].im, exp[n0].re, exp[n0].im
LUT_LOAD_4D m3, xm4, offset_p ; in[p0].re, in[p0].im, in[p1].re, in[p1].im, in[p2].re, in[p2].im, in[p3].re, in[p3].im
LUT_LOAD_4D m4, xm5, offset_n ; in[n3].re, in[n3].im, in[n2].re, in[n2].im, in[n1].re, in[n1].im, in[n0].re, in[n0].im
mulps m5, m3, m0 ; in[p].reim * exp[p].reim
mulps m6, m4, m1 ; in[n].reim * exp[n].reim
xorps m5, m7 ; in[p].re *= -1, in[p].im *= 1
xorps m6, m7 ; in[n].re *= -1, in[n].im *= 1
shufps m3, m3, m3, q2301 ; in[p].imre
shufps m4, m4, m4, q2301 ; in[n].imre
mulps m3, m0 ; in[p].imre * exp[p].reim
mulps m4, m1 ; in[n].imre * exp[n].reim
haddps m3, m6 ; out[n0].im, out[n1].im, out[n3].re, out[n2].re, out[n2].im, out[n3].im, out[n1].re, out[n0].re
haddps m5, m4 ; out[p0].re, out[p1].re, out[p3].im, out[p2].im, out[p2].re, out[p3].re, out[p1].im, out[p0].im
%if cpuflag(avx2)
vpermps m3, m9, m3 ; out[n3].im, out[n3].re, out[n2].im, out[n2].re, out[n1].im, out[n1].re, out[n0].im, out[n0].re
vpermps m5, m8, m5 ; out[p0].re, out[p0].im, out[p1].re, out[p1].im, out[p2].re, out[p2].im, out[p3].re, out[p3].im
%else
shufps m3, m3, m3, q0312
shufps m5, m5, m5, q2130
%endif
movups [outq + offset_nq*8], m3
movups [outq + offset_pq*8], m5
sub offset_pq, %1
add offset_nq, %1
cmp offset_nq, offset_pq
jle .loop
REP_RET
%endmacro
INIT_XMM sse3
POSTROTATE_FN 2
%if ARCH_X86_64 && HAVE_AVX2_EXTERNAL
INIT_YMM avx2
POSTROTATE_FN 4
%endif

104
media/ffvpx/libavcodec/x86/mdct15_init.c Normal file
View file

@ -0,0 +1,104 @@
/*
* SIMD optimized non-power-of-two MDCT functions
*
* Copyright (C) 2017 Rostislav Pehlivanov <atomnuker@gmail.com>
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "config.h"
#include "libavutil/x86/cpu.h"
#include "libavcodec/mdct15.h"
void ff_mdct15_postreindex_sse3(FFTComplex *out, FFTComplex *in, FFTComplex *exp, int *lut, ptrdiff_t len8);
void ff_mdct15_postreindex_avx2(FFTComplex *out, FFTComplex *in, FFTComplex *exp, int *lut, ptrdiff_t len8);
void ff_fft15_avx(FFTComplex *out, FFTComplex *in, FFTComplex *exptab, ptrdiff_t stride);
static void perm_twiddles(MDCT15Context *s)
{
int k;
FFTComplex exp_5point[4];
FFTComplex tmp[21], tmp2[30];
memcpy(tmp, s->exptab, sizeof(FFTComplex)*21);
/* 15-point FFT twiddles */
for (k = 0; k < 5; k++) {
tmp2[6*k + 0] = tmp[k + 0];
tmp2[6*k + 2] = tmp[k + 5];
tmp2[6*k + 4] = tmp[k + 10];
tmp2[6*k + 1] = tmp[2 * (k + 0)];
tmp2[6*k + 3] = tmp[2 * (k + 5)];
tmp2[6*k + 5] = tmp[2 * k + 5 ];
}
for (k = 0; k < 6; k++) {
FFTComplex ac_exp[] = {
{ tmp2[6*1 + k].re, tmp2[6*1 + k].re },
{ tmp2[6*2 + k].re, tmp2[6*2 + k].re },
{ tmp2[6*3 + k].re, tmp2[6*3 + k].re },
{ tmp2[6*4 + k].re, tmp2[6*4 + k].re },
{ tmp2[6*1 + k].im, -tmp2[6*1 + k].im },
{ tmp2[6*2 + k].im, -tmp2[6*2 + k].im },
{ tmp2[6*3 + k].im, -tmp2[6*3 + k].im },
{ tmp2[6*4 + k].im, -tmp2[6*4 + k].im },
};
memcpy(s->exptab + 8*k, ac_exp, 8*sizeof(FFTComplex));
}
/* Specialcase when k = 0 */
for (k = 0; k < 3; k++) {
FFTComplex dc_exp[] = {
{ tmp2[2*k + 0].re, -tmp2[2*k + 0].im },
{ tmp2[2*k + 0].im, tmp2[2*k + 0].re },
{ tmp2[2*k + 1].re, -tmp2[2*k + 1].im },
{ tmp2[2*k + 1].im, tmp2[2*k + 1].re },
};
memcpy(s->exptab + 8*6 + 4*k, dc_exp, 4*sizeof(FFTComplex));
}
/* 5-point FFT twiddles */
exp_5point[0].re = exp_5point[0].im = tmp[19].re;
exp_5point[1].re = exp_5point[1].im = tmp[19].im;
exp_5point[2].re = exp_5point[2].im = tmp[20].re;
exp_5point[3].re = exp_5point[3].im = tmp[20].im;
memcpy(s->exptab + 8*6 + 4*3, exp_5point, 4*sizeof(FFTComplex));
}
av_cold void ff_mdct15_init_x86(MDCT15Context *s)
{
int adjust_twiddles = 0;
int cpu_flags = av_get_cpu_flags();
if (EXTERNAL_SSE3(cpu_flags))
s->postreindex = ff_mdct15_postreindex_sse3;
if (ARCH_X86_64 && EXTERNAL_AVX(cpu_flags)) {
s->fft15 = ff_fft15_avx;
adjust_twiddles = 1;
}
if (ARCH_X86_64 && EXTERNAL_AVX2_FAST(cpu_flags))
s->postreindex = ff_mdct15_postreindex_avx2;
if (adjust_twiddles)
perm_twiddles(s);
}

934
media/ffvpx/libavcodec/x86/me_cmp.asm Normal file
View file

@ -0,0 +1,934 @@
;*****************************************************************************
;* SIMD-optimized motion compensation estimation
;*****************************************************************************
;* Copyright (c) 2000, 2001 Fabrice Bellard
;* Copyright (c) 2002-2004 Michael Niedermayer <michaelni@gmx.at>
;*
;* This file is part of FFmpeg.
;*
;* FFmpeg is free software; you can redistribute it and/or
;* modify it under the terms of the GNU Lesser General Public
;* License as published by the Free Software Foundation; either
;* version 2.1 of the License, or (at your option) any later version.
;*
;* FFmpeg is distributed in the hope that it will be useful,
;* but WITHOUT ANY WARRANTY; without even the implied warranty of
;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
;* Lesser General Public License for more details.
;*
;* You should have received a copy of the GNU Lesser General Public
;* License along with FFmpeg; if not, write to the Free Software
;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
;*****************************************************************************
%include "libavutil/x86/x86util.asm"
SECTION_RODATA
cextern pb_1
cextern pb_80
SECTION .text
%macro DIFF_PIXELS_1 4
movh %1, %3
movh %2, %4
punpcklbw %2, %1
punpcklbw %1, %1
psubw %1, %2
%endmacro
; %1=uint8_t *pix1, %2=uint8_t *pix2, %3=static offset, %4=stride, %5=stride*3
; %6=temporary storage location
; this macro requires $mmsize stack space (aligned) on %6 (except on SSE+x86-64)
%macro DIFF_PIXELS_8 6
DIFF_PIXELS_1 m0, m7, [%1 +%3], [%2 +%3]
DIFF_PIXELS_1 m1, m7, [%1+%4 +%3], [%2+%4 +%3]
DIFF_PIXELS_1 m2, m7, [%1+%4*2+%3], [%2+%4*2+%3]
add %1, %5
add %2, %5
DIFF_PIXELS_1 m3, m7, [%1 +%3], [%2 +%3]
DIFF_PIXELS_1 m4, m7, [%1+%4 +%3], [%2+%4 +%3]
DIFF_PIXELS_1 m5, m7, [%1+%4*2+%3], [%2+%4*2+%3]
DIFF_PIXELS_1 m6, m7, [%1+%5 +%3], [%2+%5 +%3]
%ifdef m8
DIFF_PIXELS_1 m7, m8, [%1+%4*4+%3], [%2+%4*4+%3]
%else
mova [%6], m0
DIFF_PIXELS_1 m7, m0, [%1+%4*4+%3], [%2+%4*4+%3]
mova m0, [%6]
%endif
sub %1, %5
sub %2, %5
%endmacro
%macro HADAMARD8 0
SUMSUB_BADC w, 0, 1, 2, 3
SUMSUB_BADC w, 4, 5, 6, 7
SUMSUB_BADC w, 0, 2, 1, 3
SUMSUB_BADC w, 4, 6, 5, 7
SUMSUB_BADC w, 0, 4, 1, 5
SUMSUB_BADC w, 2, 6, 3, 7
%endmacro
%macro ABS1_SUM 3
ABS1 %1, %2
paddusw %3, %1
%endmacro
%macro ABS2_SUM 6
ABS2 %1, %2, %3, %4
paddusw %5, %1
paddusw %6, %2
%endmacro
%macro ABS_SUM_8x8_64 1
ABS2 m0, m1, m8, m9
ABS2_SUM m2, m3, m8, m9, m0, m1
ABS2_SUM m4, m5, m8, m9, m0, m1
ABS2_SUM m6, m7, m8, m9, m0, m1
paddusw m0, m1
%endmacro
%macro ABS_SUM_8x8_32 1
mova [%1], m7
ABS1 m0, m7
ABS1 m1, m7
ABS1_SUM m2, m7, m0
ABS1_SUM m3, m7, m1
ABS1_SUM m4, m7, m0
ABS1_SUM m5, m7, m1
ABS1_SUM m6, m7, m0
mova m2, [%1]
ABS1_SUM m2, m7, m1
paddusw m0, m1
%endmacro
; FIXME: HSUM saturates at 64k, while an 8x8 hadamard or dct block can get up to
; about 100k on extreme inputs. But that's very unlikely to occur in natural video,
; and it's even more unlikely to not have any alternative mvs/modes with lower cost.
%macro HSUM 3
%if cpuflag(sse2)
movhlps %2, %1
paddusw %1, %2
pshuflw %2, %1, 0xE
paddusw %1, %2
pshuflw %2, %1, 0x1
paddusw %1, %2
movd %3, %1
%elif cpuflag(mmxext)
pshufw %2, %1, 0xE
paddusw %1, %2
pshufw %2, %1, 0x1
paddusw %1, %2
movd %3, %1
%elif cpuflag(mmx)
mova %2, %1
psrlq %1, 32
paddusw %1, %2
mova %2, %1
psrlq %1, 16
paddusw %1, %2
movd %3, %1
%endif
%endmacro
%macro STORE4 5
mova [%1+mmsize*0], %2
mova [%1+mmsize*1], %3
mova [%1+mmsize*2], %4
mova [%1+mmsize*3], %5
%endmacro
%macro LOAD4 5
mova %2, [%1+mmsize*0]
mova %3, [%1+mmsize*1]
mova %4, [%1+mmsize*2]
mova %5, [%1+mmsize*3]
%endmacro
%macro hadamard8_16_wrapper 2
cglobal hadamard8_diff, 4, 4, %1
%ifndef m8
%assign pad %2*mmsize-(4+stack_offset&(mmsize-1))
SUB rsp, pad
%endif
call hadamard8x8_diff %+ SUFFIX
%ifndef m8
ADD rsp, pad
%endif
RET
cglobal hadamard8_diff16, 5, 6, %1
%ifndef m8
%assign pad %2*mmsize-(4+stack_offset&(mmsize-1))
SUB rsp, pad
%endif
call hadamard8x8_diff %+ SUFFIX
mov r5d, eax
add r1, 8
add r2, 8
call hadamard8x8_diff %+ SUFFIX
add r5d, eax
cmp r4d, 16
jne .done
lea r1, [r1+r3*8-8]
lea r2, [r2+r3*8-8]
call hadamard8x8_diff %+ SUFFIX
add r5d, eax
add r1, 8
add r2, 8
call hadamard8x8_diff %+ SUFFIX
add r5d, eax
.done:
mov eax, r5d
%ifndef m8
ADD rsp, pad
%endif
RET
%endmacro
%macro HADAMARD8_DIFF 0-1
%if cpuflag(sse2)
hadamard8x8_diff %+ SUFFIX:
lea r0, [r3*3]
DIFF_PIXELS_8 r1, r2, 0, r3, r0, rsp+gprsize
HADAMARD8
%if ARCH_X86_64
TRANSPOSE8x8W 0, 1, 2, 3, 4, 5, 6, 7, 8
%else
TRANSPOSE8x8W 0, 1, 2, 3, 4, 5, 6, 7, [rsp+gprsize], [rsp+mmsize+gprsize]
%endif
HADAMARD8
ABS_SUM_8x8 rsp+gprsize
HSUM m0, m1, eax
and eax, 0xFFFF
ret
hadamard8_16_wrapper %1, 3
%elif cpuflag(mmx)
ALIGN 16
; int ff_hadamard8_diff_ ## cpu(MpegEncContext *s, uint8_t *src1,
; uint8_t *src2, ptrdiff_t stride, int h)
; r0 = void *s = unused, int h = unused (always 8)
; note how r1, r2 and r3 are not clobbered in this function, so 16x16
; can simply call this 2x2x (and that's why we access rsp+gprsize
; everywhere, which is rsp of calling func
hadamard8x8_diff %+ SUFFIX:
lea r0, [r3*3]
; first 4x8 pixels
DIFF_PIXELS_8 r1, r2, 0, r3, r0, rsp+gprsize+0x60
HADAMARD8
mova [rsp+gprsize+0x60], m7
TRANSPOSE4x4W 0, 1, 2, 3, 7
STORE4 rsp+gprsize, m0, m1, m2, m3
mova m7, [rsp+gprsize+0x60]
TRANSPOSE4x4W 4, 5, 6, 7, 0
STORE4 rsp+gprsize+0x40, m4, m5, m6, m7
; second 4x8 pixels
DIFF_PIXELS_8 r1, r2, 4, r3, r0, rsp+gprsize+0x60
HADAMARD8
mova [rsp+gprsize+0x60], m7
TRANSPOSE4x4W 0, 1, 2, 3, 7
STORE4 rsp+gprsize+0x20, m0, m1, m2, m3
mova m7, [rsp+gprsize+0x60]
TRANSPOSE4x4W 4, 5, 6, 7, 0
LOAD4 rsp+gprsize+0x40, m0, m1, m2, m3
HADAMARD8
ABS_SUM_8x8_32 rsp+gprsize+0x60
mova [rsp+gprsize+0x60], m0
LOAD4 rsp+gprsize , m0, m1, m2, m3
LOAD4 rsp+gprsize+0x20, m4, m5, m6, m7
HADAMARD8
ABS_SUM_8x8_32 rsp+gprsize
paddusw m0, [rsp+gprsize+0x60]
HSUM m0, m1, eax
and rax, 0xFFFF
ret
hadamard8_16_wrapper 0, 14
%endif
%endmacro
INIT_MMX mmx
HADAMARD8_DIFF
INIT_MMX mmxext
HADAMARD8_DIFF
INIT_XMM sse2
%if ARCH_X86_64
%define ABS_SUM_8x8 ABS_SUM_8x8_64
%else
%define ABS_SUM_8x8 ABS_SUM_8x8_32
%endif
HADAMARD8_DIFF 10
INIT_XMM ssse3
%define ABS_SUM_8x8 ABS_SUM_8x8_64
HADAMARD8_DIFF 9
; int ff_sse*_*(MpegEncContext *v, uint8_t *pix1, uint8_t *pix2,
; ptrdiff_t line_size, int h)
%macro SUM_SQUARED_ERRORS 1
cglobal sse%1, 5,5,8, v, pix1, pix2, lsize, h
%if %1 == mmsize
shr hd, 1
%endif
pxor m0, m0 ; mm0 = 0
pxor m7, m7 ; mm7 holds the sum
.next2lines: ; FIXME why are these unaligned movs? pix1[] is aligned
movu m1, [pix1q] ; m1 = pix1[0][0-15], [0-7] for mmx
movu m2, [pix2q] ; m2 = pix2[0][0-15], [0-7] for mmx
%if %1 == mmsize
movu m3, [pix1q+lsizeq] ; m3 = pix1[1][0-15], [0-7] for mmx
movu m4, [pix2q+lsizeq] ; m4 = pix2[1][0-15], [0-7] for mmx
%else ; %1 / 2 == mmsize; mmx only
mova m3, [pix1q+8] ; m3 = pix1[0][8-15]
mova m4, [pix2q+8] ; m4 = pix2[0][8-15]
%endif
; todo: mm1-mm2, mm3-mm4
; algo: subtract mm1 from mm2 with saturation and vice versa
; OR the result to get the absolute difference
mova m5, m1
mova m6, m3
psubusb m1, m2
psubusb m3, m4
psubusb m2, m5
psubusb m4, m6
por m2, m1
por m4, m3
; now convert to 16-bit vectors so we can square them
mova m1, m2
mova m3, m4
punpckhbw m2, m0
punpckhbw m4, m0
punpcklbw m1, m0 ; mm1 not spread over (mm1,mm2)
punpcklbw m3, m0 ; mm4 not spread over (mm3,mm4)
pmaddwd m2, m2
pmaddwd m4, m4
pmaddwd m1, m1
pmaddwd m3, m3
paddd m1, m2
paddd m3, m4
paddd m7, m1
paddd m7, m3
%if %1 == mmsize
lea pix1q, [pix1q + 2*lsizeq]
lea pix2q, [pix2q + 2*lsizeq]
%else
add pix1q, lsizeq
add pix2q, lsizeq
%endif
dec hd
jnz .next2lines
HADDD m7, m1
movd eax, m7 ; return value
RET
%endmacro
INIT_MMX mmx
SUM_SQUARED_ERRORS 8
INIT_MMX mmx
SUM_SQUARED_ERRORS 16
INIT_XMM sse2
SUM_SQUARED_ERRORS 16
;-----------------------------------------------
;int ff_sum_abs_dctelem(int16_t *block)
;-----------------------------------------------
; %1 = number of xmm registers used
; %2 = number of inline loops
%macro SUM_ABS_DCTELEM 2
cglobal sum_abs_dctelem, 1, 1, %1, block
pxor m0, m0
pxor m1, m1
%assign %%i 0
%rep %2
mova m2, [blockq+mmsize*(0+%%i)]
mova m3, [blockq+mmsize*(1+%%i)]
mova m4, [blockq+mmsize*(2+%%i)]
mova m5, [blockq+mmsize*(3+%%i)]
ABS1_SUM m2, m6, m0
ABS1_SUM m3, m6, m1
ABS1_SUM m4, m6, m0
ABS1_SUM m5, m6, m1
%assign %%i %%i+4
%endrep
paddusw m0, m1
HSUM m0, m1, eax
and eax, 0xFFFF
RET
%endmacro
INIT_MMX mmx
SUM_ABS_DCTELEM 0, 4
INIT_MMX mmxext
SUM_ABS_DCTELEM 0, 4
INIT_XMM sse2
SUM_ABS_DCTELEM 7, 2
INIT_XMM ssse3
SUM_ABS_DCTELEM 6, 2
;------------------------------------------------------------------------------
; int ff_hf_noise*_mmx(uint8_t *pix1, ptrdiff_t lsize, int h)
;------------------------------------------------------------------------------
; %1 = 8/16. %2-5=m#
%macro HF_NOISE_PART1 5
mova m%2, [pix1q]
%if %1 == 8
mova m%3, m%2
psllq m%2, 8
psrlq m%3, 8
psrlq m%2, 8
%else
mova m%3, [pix1q+1]
%endif
mova m%4, m%2
mova m%5, m%3
punpcklbw m%2, m7
punpcklbw m%3, m7
punpckhbw m%4, m7
punpckhbw m%5, m7
psubw m%2, m%3
psubw m%4, m%5
%endmacro
; %1-2 = m#
%macro HF_NOISE_PART2 4
psubw m%1, m%3
psubw m%2, m%4
pxor m3, m3
pxor m1, m1
pcmpgtw m3, m%1
pcmpgtw m1, m%2
pxor m%1, m3
pxor m%2, m1
psubw m%1, m3
psubw m%2, m1
paddw m%2, m%1
paddw m6, m%2
%endmacro
; %1 = 8/16
%macro HF_NOISE 1
cglobal hf_noise%1, 3,3,0, pix1, lsize, h
sub hd, 2
pxor m7, m7
pxor m6, m6
HF_NOISE_PART1 %1, 0, 1, 2, 3
add pix1q, lsizeq
HF_NOISE_PART1 %1, 4, 1, 5, 3
HF_NOISE_PART2 0, 2, 4, 5
add pix1q, lsizeq
.loop:
HF_NOISE_PART1 %1, 0, 1, 2, 3
HF_NOISE_PART2 4, 5, 0, 2
add pix1q, lsizeq
HF_NOISE_PART1 %1, 4, 1, 5, 3
HF_NOISE_PART2 0, 2, 4, 5
add pix1q, lsizeq
sub hd, 2
jne .loop
mova m0, m6
punpcklwd m0, m7
punpckhwd m6, m7
paddd m6, m0
mova m0, m6
psrlq m6, 32
paddd m0, m6
movd eax, m0 ; eax = result of hf_noise8;
REP_RET ; return eax;
%endmacro
INIT_MMX mmx
HF_NOISE 8
HF_NOISE 16
;---------------------------------------------------------------------------------------
;int ff_sad_<opt>(MpegEncContext *v, uint8_t *pix1, uint8_t *pix2, ptrdiff_t stride, int h);
;---------------------------------------------------------------------------------------
;%1 = 8/16
%macro SAD 1
cglobal sad%1, 5, 5, 3, v, pix1, pix2, stride, h
movu m2, [pix2q]
movu m1, [pix2q+strideq]
psadbw m2, [pix1q]
psadbw m1, [pix1q+strideq]
paddw m2, m1
%if %1 != mmsize
movu m0, [pix2q+8]
movu m1, [pix2q+strideq+8]
psadbw m0, [pix1q+8]
psadbw m1, [pix1q+strideq+8]
paddw m2, m0
paddw m2, m1
%endif
sub hd, 2
align 16
.loop:
lea pix1q, [pix1q+strideq*2]
lea pix2q, [pix2q+strideq*2]
movu m0, [pix2q]
movu m1, [pix2q+strideq]
psadbw m0, [pix1q]
psadbw m1, [pix1q+strideq]
paddw m2, m0
paddw m2, m1
%if %1 != mmsize
movu m0, [pix2q+8]
movu m1, [pix2q+strideq+8]
psadbw m0, [pix1q+8]
psadbw m1, [pix1q+strideq+8]
paddw m2, m0
paddw m2, m1
%endif
sub hd, 2
jg .loop
%if mmsize == 16
movhlps m0, m2
paddw m2, m0
%endif
movd eax, m2
RET
%endmacro
INIT_MMX mmxext
SAD 8
SAD 16
INIT_XMM sse2
SAD 16
;------------------------------------------------------------------------------------------
;int ff_sad_x2_<opt>(MpegEncContext *v, uint8_t *pix1, uint8_t *pix2, ptrdiff_t stride, int h);
;------------------------------------------------------------------------------------------
;%1 = 8/16
%macro SAD_X2 1
cglobal sad%1_x2, 5, 5, 5, v, pix1, pix2, stride, h
movu m0, [pix2q]
movu m2, [pix2q+strideq]
%if mmsize == 16
movu m3, [pix2q+1]
movu m4, [pix2q+strideq+1]
pavgb m0, m3
pavgb m2, m4
%else
pavgb m0, [pix2q+1]
pavgb m2, [pix2q+strideq+1]
%endif
psadbw m0, [pix1q]
psadbw m2, [pix1q+strideq]
paddw m0, m2
%if %1 != mmsize
movu m1, [pix2q+8]
movu m2, [pix2q+strideq+8]
pavgb m1, [pix2q+9]
pavgb m2, [pix2q+strideq+9]
psadbw m1, [pix1q+8]
psadbw m2, [pix1q+strideq+8]
paddw m0, m1
paddw m0, m2
%endif
sub hd, 2
align 16
.loop:
lea pix1q, [pix1q+2*strideq]
lea pix2q, [pix2q+2*strideq]
movu m1, [pix2q]
movu m2, [pix2q+strideq]
%if mmsize == 16
movu m3, [pix2q+1]
movu m4, [pix2q+strideq+1]
pavgb m1, m3
pavgb m2, m4
%else
pavgb m1, [pix2q+1]
pavgb m2, [pix2q+strideq+1]
%endif
psadbw m1, [pix1q]
psadbw m2, [pix1q+strideq]
paddw m0, m1
paddw m0, m2
%if %1 != mmsize
movu m1, [pix2q+8]
movu m2, [pix2q+strideq+8]
pavgb m1, [pix2q+9]
pavgb m2, [pix2q+strideq+9]
psadbw m1, [pix1q+8]
psadbw m2, [pix1q+strideq+8]
paddw m0, m1
paddw m0, m2
%endif
sub hd, 2
jg .loop
%if mmsize == 16
movhlps m1, m0
paddw m0, m1
%endif
movd eax, m0
RET
%endmacro
INIT_MMX mmxext
SAD_X2 8
SAD_X2 16
INIT_XMM sse2
SAD_X2 16
;------------------------------------------------------------------------------------------
;int ff_sad_y2_<opt>(MpegEncContext *v, uint8_t *pix1, uint8_t *pix2, ptrdiff_t stride, int h);
;------------------------------------------------------------------------------------------
;%1 = 8/16
%macro SAD_Y2 1
cglobal sad%1_y2, 5, 5, 4, v, pix1, pix2, stride, h
movu m1, [pix2q]
movu m0, [pix2q+strideq]
movu m3, [pix2q+2*strideq]
pavgb m1, m0
pavgb m0, m3
psadbw m1, [pix1q]
psadbw m0, [pix1q+strideq]
paddw m0, m1
mova m1, m3
%if %1 != mmsize
movu m4, [pix2q+8]
movu m5, [pix2q+strideq+8]
movu m6, [pix2q+2*strideq+8]
pavgb m4, m5
pavgb m5, m6
psadbw m4, [pix1q+8]
psadbw m5, [pix1q+strideq+8]
paddw m0, m4
paddw m0, m5
mova m4, m6
%endif
add pix2q, strideq
sub hd, 2
align 16
.loop:
lea pix1q, [pix1q+2*strideq]
lea pix2q, [pix2q+2*strideq]
movu m2, [pix2q]
movu m3, [pix2q+strideq]
pavgb m1, m2
pavgb m2, m3
psadbw m1, [pix1q]
psadbw m2, [pix1q+strideq]
paddw m0, m1
paddw m0, m2
mova m1, m3
%if %1 != mmsize
movu m5, [pix2q+8]
movu m6, [pix2q+strideq+8]
pavgb m4, m5
pavgb m5, m6
psadbw m4, [pix1q+8]
psadbw m5, [pix1q+strideq+8]
paddw m0, m4
paddw m0, m5
mova m4, m6
%endif
sub hd, 2
jg .loop
%if mmsize == 16
movhlps m1, m0
paddw m0, m1
%endif
movd eax, m0
RET
%endmacro
INIT_MMX mmxext
SAD_Y2 8
SAD_Y2 16
INIT_XMM sse2
SAD_Y2 16
;-------------------------------------------------------------------------------------------
;int ff_sad_approx_xy2_<opt>(MpegEncContext *v, uint8_t *pix1, uint8_t *pix2, ptrdiff_t stride, int h);
;-------------------------------------------------------------------------------------------
;%1 = 8/16
%macro SAD_APPROX_XY2 1
cglobal sad%1_approx_xy2, 5, 5, 7, v, pix1, pix2, stride, h
mova m4, [pb_1]
movu m1, [pix2q]
movu m0, [pix2q+strideq]
movu m3, [pix2q+2*strideq]
%if mmsize == 16
movu m5, [pix2q+1]
movu m6, [pix2q+strideq+1]
movu m2, [pix2q+2*strideq+1]
pavgb m1, m5
pavgb m0, m6
pavgb m3, m2
%else
pavgb m1, [pix2q+1]
pavgb m0, [pix2q+strideq+1]
pavgb m3, [pix2q+2*strideq+1]
%endif
psubusb m0, m4
pavgb m1, m0
pavgb m0, m3
psadbw m1, [pix1q]
psadbw m0, [pix1q+strideq]
paddw m0, m1
mova m1, m3
%if %1 != mmsize
movu m5, [pix2q+8]
movu m6, [pix2q+strideq+8]
movu m7, [pix2q+2*strideq+8]
pavgb m5, [pix2q+1+8]
pavgb m6, [pix2q+strideq+1+8]
pavgb m7, [pix2q+2*strideq+1+8]
psubusb m6, m4
pavgb m5, m6
pavgb m6, m7
psadbw m5, [pix1q+8]
psadbw m6, [pix1q+strideq+8]
paddw m0, m5
paddw m0, m6
mova m5, m7
%endif
add pix2q, strideq
sub hd, 2
align 16
.loop:
lea pix1q, [pix1q+2*strideq]
lea pix2q, [pix2q+2*strideq]
movu m2, [pix2q]
movu m3, [pix2q+strideq]
%if mmsize == 16
movu m5, [pix2q+1]
movu m6, [pix2q+strideq+1]
pavgb m2, m5
pavgb m3, m6
%else
pavgb m2, [pix2q+1]
pavgb m3, [pix2q+strideq+1]
%endif
psubusb m2, m4
pavgb m1, m2
pavgb m2, m3
psadbw m1, [pix1q]
psadbw m2, [pix1q+strideq]
paddw m0, m1
paddw m0, m2
mova m1, m3
%if %1 != mmsize
movu m6, [pix2q+8]
movu m7, [pix2q+strideq+8]
pavgb m6, [pix2q+8+1]
pavgb m7, [pix2q+strideq+8+1]
psubusb m6, m4
pavgb m5, m6
pavgb m6, m7
psadbw m5, [pix1q+8]
psadbw m6, [pix1q+strideq+8]
paddw m0, m5
paddw m0, m6
mova m5, m7
%endif
sub hd, 2
jg .loop
%if mmsize == 16
movhlps m1, m0
paddw m0, m1
%endif
movd eax, m0
RET
%endmacro
INIT_MMX mmxext
SAD_APPROX_XY2 8
SAD_APPROX_XY2 16
INIT_XMM sse2
SAD_APPROX_XY2 16
;--------------------------------------------------------------------
;int ff_vsad_intra(MpegEncContext *v, uint8_t *pix1, uint8_t *pix2,
; ptrdiff_t line_size, int h);
;--------------------------------------------------------------------
; %1 = 8/16
%macro VSAD_INTRA 1
cglobal vsad_intra%1, 5, 5, 3, v, pix1, pix2, lsize, h
mova m0, [pix1q]
%if %1 == mmsize
mova m2, [pix1q+lsizeq]
psadbw m0, m2
%else
mova m2, [pix1q+lsizeq]
mova m3, [pix1q+8]
mova m4, [pix1q+lsizeq+8]
psadbw m0, m2
psadbw m3, m4
paddw m0, m3
%endif
sub hd, 2
.loop:
lea pix1q, [pix1q + 2*lsizeq]
%if %1 == mmsize
mova m1, [pix1q]
psadbw m2, m1
paddw m0, m2
mova m2, [pix1q+lsizeq]
psadbw m1, m2
paddw m0, m1
%else
mova m1, [pix1q]
mova m3, [pix1q+8]
psadbw m2, m1
psadbw m4, m3
paddw m0, m2
paddw m0, m4
mova m2, [pix1q+lsizeq]
mova m4, [pix1q+lsizeq+8]
psadbw m1, m2
psadbw m3, m4
paddw m0, m1
paddw m0, m3
%endif
sub hd, 2
jg .loop
%if mmsize == 16
pshufd m1, m0, 0xe
paddd m0, m1
%endif
movd eax, m0
RET
%endmacro
INIT_MMX mmxext
VSAD_INTRA 8
VSAD_INTRA 16
INIT_XMM sse2
VSAD_INTRA 16
;---------------------------------------------------------------------
;int ff_vsad_approx(MpegEncContext *v, uint8_t *pix1, uint8_t *pix2,
; ptrdiff_t line_size, int h);
;---------------------------------------------------------------------
; %1 = 8/16
%macro VSAD_APPROX 1
cglobal vsad%1_approx, 5, 5, 5, v, pix1, pix2, lsize, h
mova m1, [pb_80]
mova m0, [pix1q]
%if %1 == mmsize ; vsad8_mmxext, vsad16_sse2
mova m4, [pix1q+lsizeq]
%if mmsize == 16
movu m3, [pix2q]
movu m2, [pix2q+lsizeq]
psubb m0, m3
psubb m4, m2
%else
psubb m0, [pix2q]
psubb m4, [pix2q+lsizeq]
%endif
pxor m0, m1
pxor m4, m1
psadbw m0, m4
%else ; vsad16_mmxext
mova m3, [pix1q+8]
psubb m0, [pix2q]
psubb m3, [pix2q+8]
pxor m0, m1
pxor m3, m1
mova m4, [pix1q+lsizeq]
mova m5, [pix1q+lsizeq+8]
psubb m4, [pix2q+lsizeq]
psubb m5, [pix2q+lsizeq+8]
pxor m4, m1
pxor m5, m1
psadbw m0, m4
psadbw m3, m5
paddw m0, m3
%endif
sub hd, 2
.loop:
lea pix1q, [pix1q + 2*lsizeq]
lea pix2q, [pix2q + 2*lsizeq]
mova m2, [pix1q]
%if %1 == mmsize ; vsad8_mmxext, vsad16_sse2
%if mmsize == 16
movu m3, [pix2q]
psubb m2, m3
%else
psubb m2, [pix2q]
%endif
pxor m2, m1
psadbw m4, m2
paddw m0, m4
mova m4, [pix1q+lsizeq]
movu m3, [pix2q+lsizeq]
psubb m4, m3
pxor m4, m1
psadbw m2, m4
paddw m0, m2
%else ; vsad16_mmxext
mova m3, [pix1q+8]
psubb m2, [pix2q]
psubb m3, [pix2q+8]
pxor m2, m1
pxor m3, m1
psadbw m4, m2
psadbw m5, m3
paddw m0, m4
paddw m0, m5
mova m4, [pix1q+lsizeq]
mova m5, [pix1q+lsizeq+8]
psubb m4, [pix2q+lsizeq]
psubb m5, [pix2q+lsizeq+8]
pxor m4, m1
pxor m5, m1
psadbw m2, m4
psadbw m3, m5
paddw m0, m2
paddw m0, m3
%endif
sub hd, 2
jg .loop
%if mmsize == 16
pshufd m1, m0, 0xe
paddd m0, m1
%endif
movd eax, m0
RET
%endmacro
INIT_MMX mmxext
VSAD_APPROX 8
VSAD_APPROX 16
INIT_XMM sse2
VSAD_APPROX 16

651
media/ffvpx/libavcodec/x86/me_cmp_init.c Normal file
View file

@ -0,0 +1,651 @@
/*
* SIMD-optimized motion estimation
* Copyright (c) 2000, 2001 Fabrice Bellard
* Copyright (c) 2002-2004 Michael Niedermayer <michaelni@gmx.at>
*
* MMX optimization by Nick Kurshev <nickols_k@mail.ru>
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "libavutil/attributes.h"
#include "libavutil/cpu.h"
#include "libavutil/x86/asm.h"
#include "libavutil/x86/cpu.h"
#include "libavcodec/me_cmp.h"
#include "libavcodec/mpegvideo.h"
int ff_sum_abs_dctelem_mmx(int16_t *block);
int ff_sum_abs_dctelem_mmxext(int16_t *block);
int ff_sum_abs_dctelem_sse2(int16_t *block);
int ff_sum_abs_dctelem_ssse3(int16_t *block);
int ff_sse8_mmx(MpegEncContext *v, uint8_t *pix1, uint8_t *pix2,
ptrdiff_t stride, int h);
int ff_sse16_mmx(MpegEncContext *v, uint8_t *pix1, uint8_t *pix2,
ptrdiff_t stride, int h);
int ff_sse16_sse2(MpegEncContext *v, uint8_t *pix1, uint8_t *pix2,
ptrdiff_t stride, int h);
int ff_hf_noise8_mmx(uint8_t *pix1, ptrdiff_t stride, int h);
int ff_hf_noise16_mmx(uint8_t *pix1, ptrdiff_t stride, int h);
int ff_sad8_mmxext(MpegEncContext *v, uint8_t *pix1, uint8_t *pix2,
ptrdiff_t stride, int h);
int ff_sad16_mmxext(MpegEncContext *v, uint8_t *pix1, uint8_t *pix2,
ptrdiff_t stride, int h);
int ff_sad16_sse2(MpegEncContext *v, uint8_t *pix1, uint8_t *pix2,
ptrdiff_t stride, int h);
int ff_sad8_x2_mmxext(MpegEncContext *v, uint8_t *pix1, uint8_t *pix2,
ptrdiff_t stride, int h);
int ff_sad16_x2_mmxext(MpegEncContext *v, uint8_t *pix1, uint8_t *pix2,
ptrdiff_t stride, int h);
int ff_sad16_x2_sse2(MpegEncContext *v, uint8_t *pix1, uint8_t *pix2,
ptrdiff_t stride, int h);
int ff_sad8_y2_mmxext(MpegEncContext *v, uint8_t *pix1, uint8_t *pix2,
ptrdiff_t stride, int h);
int ff_sad16_y2_mmxext(MpegEncContext *v, uint8_t *pix1, uint8_t *pix2,
ptrdiff_t stride, int h);
int ff_sad16_y2_sse2(MpegEncContext *v, uint8_t *pix1, uint8_t *pix2,
ptrdiff_t stride, int h);
int ff_sad8_approx_xy2_mmxext(MpegEncContext *v, uint8_t *pix1, uint8_t *pix2,
ptrdiff_t stride, int h);
int ff_sad16_approx_xy2_mmxext(MpegEncContext *v, uint8_t *pix1, uint8_t *pix2,
ptrdiff_t stride, int h);
int ff_sad16_approx_xy2_sse2(MpegEncContext *v, uint8_t *pix1, uint8_t *pix2,
ptrdiff_t stride, int h);
int ff_vsad_intra8_mmxext(MpegEncContext *v, uint8_t *pix1, uint8_t *pix2,
ptrdiff_t stride, int h);
int ff_vsad_intra16_mmxext(MpegEncContext *v, uint8_t *pix1, uint8_t *pix2,
ptrdiff_t stride, int h);
int ff_vsad_intra16_sse2(MpegEncContext *v, uint8_t *pix1, uint8_t *pix2,
ptrdiff_t stride, int h);
int ff_vsad8_approx_mmxext(MpegEncContext *v, uint8_t *pix1, uint8_t *pix2,
ptrdiff_t stride, int h);
int ff_vsad16_approx_mmxext(MpegEncContext *v, uint8_t *pix1, uint8_t *pix2,
ptrdiff_t stride, int h);
int ff_vsad16_approx_sse2(MpegEncContext *v, uint8_t *pix1, uint8_t *pix2,
ptrdiff_t stride, int h);
#define hadamard_func(cpu) \
int ff_hadamard8_diff_ ## cpu(MpegEncContext *s, uint8_t *src1, \
uint8_t *src2, ptrdiff_t stride, int h); \
int ff_hadamard8_diff16_ ## cpu(MpegEncContext *s, uint8_t *src1, \
uint8_t *src2, ptrdiff_t stride, int h);
hadamard_func(mmx)
hadamard_func(mmxext)
hadamard_func(sse2)
hadamard_func(ssse3)
#if HAVE_X86ASM
static int nsse16_mmx(MpegEncContext *c, uint8_t *pix1, uint8_t *pix2,
ptrdiff_t stride, int h)
{
int score1, score2;
if (c)
score1 = c->mecc.sse[0](c, pix1, pix2, stride, h);
else
score1 = ff_sse16_mmx(c, pix1, pix2, stride, h);
score2 = ff_hf_noise16_mmx(pix1, stride, h) + ff_hf_noise8_mmx(pix1+8, stride, h)
- ff_hf_noise16_mmx(pix2, stride, h) - ff_hf_noise8_mmx(pix2+8, stride, h);
if (c)
return score1 + FFABS(score2) * c->avctx->nsse_weight;
else
return score1 + FFABS(score2) * 8;
}
static int nsse8_mmx(MpegEncContext *c, uint8_t *pix1, uint8_t *pix2,
ptrdiff_t stride, int h)
{
int score1 = ff_sse8_mmx(c, pix1, pix2, stride, h);
int score2 = ff_hf_noise8_mmx(pix1, stride, h) -
ff_hf_noise8_mmx(pix2, stride, h);
if (c)
return score1 + FFABS(score2) * c->avctx->nsse_weight;
else
return score1 + FFABS(score2) * 8;
}
#endif /* HAVE_X86ASM */
#if HAVE_INLINE_ASM
static int vsad_intra16_mmx(MpegEncContext *v, uint8_t *pix, uint8_t *dummy,
ptrdiff_t stride, int h)
{
int tmp;
av_assert2((((int) pix) & 7) == 0);
av_assert2((stride & 7) == 0);
#define SUM(in0, in1, out0, out1) \
"movq (%0), %%mm2\n" \
"movq 8(%0), %%mm3\n" \
"add %2,%0\n" \
"movq %%mm2, " #out0 "\n" \
"movq %%mm3, " #out1 "\n" \
"psubusb " #in0 ", %%mm2\n" \
"psubusb " #in1 ", %%mm3\n" \
"psubusb " #out0 ", " #in0 "\n" \
"psubusb " #out1 ", " #in1 "\n" \
"por %%mm2, " #in0 "\n" \
"por %%mm3, " #in1 "\n" \
"movq " #in0 ", %%mm2\n" \
"movq " #in1 ", %%mm3\n" \
"punpcklbw %%mm7, " #in0 "\n" \
"punpcklbw %%mm7, " #in1 "\n" \
"punpckhbw %%mm7, %%mm2\n" \
"punpckhbw %%mm7, %%mm3\n" \
"paddw " #in1 ", " #in0 "\n" \
"paddw %%mm3, %%mm2\n" \
"paddw %%mm2, " #in0 "\n" \
"paddw " #in0 ", %%mm6\n"
__asm__ volatile (
"movl %3, %%ecx\n"
"pxor %%mm6, %%mm6\n"
"pxor %%mm7, %%mm7\n"
"movq (%0), %%mm0\n"
"movq 8(%0), %%mm1\n"
"add %2, %0\n"
"jmp 2f\n"
"1:\n"
SUM(%%mm4, %%mm5, %%mm0, %%mm1)
"2:\n"
SUM(%%mm0, %%mm1, %%mm4, %%mm5)
"subl $2, %%ecx\n"
"jnz 1b\n"
"movq %%mm6, %%mm0\n"
"psrlq $32, %%mm6\n"
"paddw %%mm6, %%mm0\n"
"movq %%mm0, %%mm6\n"
"psrlq $16, %%mm0\n"
"paddw %%mm6, %%mm0\n"
"movd %%mm0, %1\n"
: "+r" (pix), "=r" (tmp)
: "r" (stride), "m" (h)
: "%ecx");
return tmp & 0xFFFF;
}
#undef SUM
static int vsad16_mmx(MpegEncContext *v, uint8_t *pix1, uint8_t *pix2,
ptrdiff_t stride, int h)
{
int tmp;
av_assert2((((int) pix1) & 7) == 0);
av_assert2((((int) pix2) & 7) == 0);
av_assert2((stride & 7) == 0);
#define SUM(in0, in1, out0, out1) \
"movq (%0), %%mm2\n" \
"movq (%1), " #out0 "\n" \
"movq 8(%0), %%mm3\n" \
"movq 8(%1), " #out1 "\n" \
"add %3, %0\n" \
"add %3, %1\n" \
"psubb " #out0 ", %%mm2\n" \
"psubb " #out1 ", %%mm3\n" \
"pxor %%mm7, %%mm2\n" \
"pxor %%mm7, %%mm3\n" \
"movq %%mm2, " #out0 "\n" \
"movq %%mm3, " #out1 "\n" \
"psubusb " #in0 ", %%mm2\n" \
"psubusb " #in1 ", %%mm3\n" \
"psubusb " #out0 ", " #in0 "\n" \
"psubusb " #out1 ", " #in1 "\n" \
"por %%mm2, " #in0 "\n" \
"por %%mm3, " #in1 "\n" \
"movq " #in0 ", %%mm2\n" \
"movq " #in1 ", %%mm3\n" \
"punpcklbw %%mm7, " #in0 "\n" \
"punpcklbw %%mm7, " #in1 "\n" \
"punpckhbw %%mm7, %%mm2\n" \
"punpckhbw %%mm7, %%mm3\n" \
"paddw " #in1 ", " #in0 "\n" \
"paddw %%mm3, %%mm2\n" \
"paddw %%mm2, " #in0 "\n" \
"paddw " #in0 ", %%mm6\n"
__asm__ volatile (
"movl %4, %%ecx\n"
"pxor %%mm6, %%mm6\n"
"pcmpeqw %%mm7, %%mm7\n"
"psllw $15, %%mm7\n"
"packsswb %%mm7, %%mm7\n"
"movq (%0), %%mm0\n"
"movq (%1), %%mm2\n"
"movq 8(%0), %%mm1\n"
"movq 8(%1), %%mm3\n"
"add %3, %0\n"
"add %3, %1\n"
"psubb %%mm2, %%mm0\n"
"psubb %%mm3, %%mm1\n"
"pxor %%mm7, %%mm0\n"
"pxor %%mm7, %%mm1\n"
"jmp 2f\n"
"1:\n"
SUM(%%mm4, %%mm5, %%mm0, %%mm1)
"2:\n"
SUM(%%mm0, %%mm1, %%mm4, %%mm5)
"subl $2, %%ecx\n"
"jnz 1b\n"
"movq %%mm6, %%mm0\n"
"psrlq $32, %%mm6\n"
"paddw %%mm6, %%mm0\n"
"movq %%mm0, %%mm6\n"
"psrlq $16, %%mm0\n"
"paddw %%mm6, %%mm0\n"
"movd %%mm0, %2\n"
: "+r" (pix1), "+r" (pix2), "=r" (tmp)
: "r" (stride), "m" (h)
: "%ecx");
return tmp & 0x7FFF;
}
#undef SUM
DECLARE_ASM_CONST(8, uint64_t, round_tab)[3] = {
0x0000000000000000ULL,
0x0001000100010001ULL,
0x0002000200020002ULL,
};
static inline void sad8_1_mmx(uint8_t *blk1, uint8_t *blk2,
ptrdiff_t stride, int h)
{
x86_reg len = -stride * h;
__asm__ volatile (
".p2align 4 \n\t"
"1: \n\t"
"movq (%1, %%"FF_REG_a"), %%mm0 \n\t"
"movq (%2, %%"FF_REG_a"), %%mm2 \n\t"
"movq (%2, %%"FF_REG_a"), %%mm4 \n\t"
"add %3, %%"FF_REG_a" \n\t"
"psubusb %%mm0, %%mm2 \n\t"
"psubusb %%mm4, %%mm0 \n\t"
"movq (%1, %%"FF_REG_a"), %%mm1 \n\t"
"movq (%2, %%"FF_REG_a"), %%mm3 \n\t"
"movq (%2, %%"FF_REG_a"), %%mm5 \n\t"
"psubusb %%mm1, %%mm3 \n\t"
"psubusb %%mm5, %%mm1 \n\t"
"por %%mm2, %%mm0 \n\t"
"por %%mm1, %%mm3 \n\t"
"movq %%mm0, %%mm1 \n\t"
"movq %%mm3, %%mm2 \n\t"
"punpcklbw %%mm7, %%mm0 \n\t"
"punpckhbw %%mm7, %%mm1 \n\t"
"punpcklbw %%mm7, %%mm3 \n\t"
"punpckhbw %%mm7, %%mm2 \n\t"
"paddw %%mm1, %%mm0 \n\t"
"paddw %%mm3, %%mm2 \n\t"
"paddw %%mm2, %%mm0 \n\t"
"paddw %%mm0, %%mm6 \n\t"
"add %3, %%"FF_REG_a" \n\t"
" js 1b \n\t"
: "+a" (len)
: "r" (blk1 - len), "r" (blk2 - len), "r" (stride));
}
static inline void sad8_2_mmx(uint8_t *blk1a, uint8_t *blk1b, uint8_t *blk2,
ptrdiff_t stride, int h)
{
x86_reg len = -stride * h;
__asm__ volatile (
".p2align 4 \n\t"
"1: \n\t"
"movq (%1, %%"FF_REG_a"), %%mm0 \n\t"
"movq (%2, %%"FF_REG_a"), %%mm1 \n\t"
"movq (%1, %%"FF_REG_a"), %%mm2 \n\t"
"movq (%2, %%"FF_REG_a"), %%mm3 \n\t"
"punpcklbw %%mm7, %%mm0 \n\t"
"punpcklbw %%mm7, %%mm1 \n\t"
"punpckhbw %%mm7, %%mm2 \n\t"
"punpckhbw %%mm7, %%mm3 \n\t"
"paddw %%mm0, %%mm1 \n\t"
"paddw %%mm2, %%mm3 \n\t"
"movq (%3, %%"FF_REG_a"), %%mm4 \n\t"
"movq (%3, %%"FF_REG_a"), %%mm2 \n\t"
"paddw %%mm5, %%mm1 \n\t"
"paddw %%mm5, %%mm3 \n\t"
"psrlw $1, %%mm1 \n\t"
"psrlw $1, %%mm3 \n\t"
"packuswb %%mm3, %%mm1 \n\t"
"psubusb %%mm1, %%mm4 \n\t"
"psubusb %%mm2, %%mm1 \n\t"
"por %%mm4, %%mm1 \n\t"
"movq %%mm1, %%mm0 \n\t"
"punpcklbw %%mm7, %%mm0 \n\t"
"punpckhbw %%mm7, %%mm1 \n\t"
"paddw %%mm1, %%mm0 \n\t"
"paddw %%mm0, %%mm6 \n\t"
"add %4, %%"FF_REG_a" \n\t"
" js 1b \n\t"
: "+a" (len)
: "r" (blk1a - len), "r" (blk1b - len), "r" (blk2 - len),
"r" (stride));
}
static inline void sad8_4_mmx(uint8_t *blk1, uint8_t *blk2,
ptrdiff_t stride, int h)
{
x86_reg len = -stride * h;
__asm__ volatile (
"movq (%1, %%"FF_REG_a"), %%mm0\n\t"
"movq 1(%1, %%"FF_REG_a"), %%mm2\n\t"
"movq %%mm0, %%mm1 \n\t"
"movq %%mm2, %%mm3 \n\t"
"punpcklbw %%mm7, %%mm0 \n\t"
"punpckhbw %%mm7, %%mm1 \n\t"
"punpcklbw %%mm7, %%mm2 \n\t"
"punpckhbw %%mm7, %%mm3 \n\t"
"paddw %%mm2, %%mm0 \n\t"
"paddw %%mm3, %%mm1 \n\t"
".p2align 4 \n\t"
"1: \n\t"
"movq (%2, %%"FF_REG_a"), %%mm2\n\t"
"movq 1(%2, %%"FF_REG_a"), %%mm4\n\t"
"movq %%mm2, %%mm3 \n\t"
"movq %%mm4, %%mm5 \n\t"
"punpcklbw %%mm7, %%mm2 \n\t"
"punpckhbw %%mm7, %%mm3 \n\t"
"punpcklbw %%mm7, %%mm4 \n\t"
"punpckhbw %%mm7, %%mm5 \n\t"
"paddw %%mm4, %%mm2 \n\t"
"paddw %%mm5, %%mm3 \n\t"
"movq %5, %%mm5 \n\t"
"paddw %%mm2, %%mm0 \n\t"
"paddw %%mm3, %%mm1 \n\t"
"paddw %%mm5, %%mm0 \n\t"
"paddw %%mm5, %%mm1 \n\t"
"movq (%3, %%"FF_REG_a"), %%mm4 \n\t"
"movq (%3, %%"FF_REG_a"), %%mm5 \n\t"
"psrlw $2, %%mm0 \n\t"
"psrlw $2, %%mm1 \n\t"
"packuswb %%mm1, %%mm0 \n\t"
"psubusb %%mm0, %%mm4 \n\t"
"psubusb %%mm5, %%mm0 \n\t"
"por %%mm4, %%mm0 \n\t"
"movq %%mm0, %%mm4 \n\t"
"punpcklbw %%mm7, %%mm0 \n\t"
"punpckhbw %%mm7, %%mm4 \n\t"
"paddw %%mm0, %%mm6 \n\t"
"paddw %%mm4, %%mm6 \n\t"
"movq %%mm2, %%mm0 \n\t"
"movq %%mm3, %%mm1 \n\t"
"add %4, %%"FF_REG_a" \n\t"
" js 1b \n\t"
: "+a" (len)
: "r" (blk1 - len), "r" (blk1 - len + stride), "r" (blk2 - len),
"r" (stride), "m" (round_tab[2]));
}
static inline int sum_mmx(void)
{
int ret;
__asm__ volatile (
"movq %%mm6, %%mm0 \n\t"
"psrlq $32, %%mm6 \n\t"
"paddw %%mm0, %%mm6 \n\t"
"movq %%mm6, %%mm0 \n\t"
"psrlq $16, %%mm6 \n\t"
"paddw %%mm0, %%mm6 \n\t"
"movd %%mm6, %0 \n\t"
: "=r" (ret));
return ret & 0xFFFF;
}
static inline void sad8_x2a_mmx(uint8_t *blk1, uint8_t *blk2,
ptrdiff_t stride, int h)
{
sad8_2_mmx(blk1, blk1 + 1, blk2, stride, h);
}
static inline void sad8_y2a_mmx(uint8_t *blk1, uint8_t *blk2,
ptrdiff_t stride, int h)
{
sad8_2_mmx(blk1, blk1 + stride, blk2, stride, h);
}
#define PIX_SAD(suf) \
static int sad8_ ## suf(MpegEncContext *v, uint8_t *blk2, \
uint8_t *blk1, ptrdiff_t stride, int h) \
{ \
av_assert2(h == 8); \
__asm__ volatile ( \
"pxor %%mm7, %%mm7 \n\t" \
"pxor %%mm6, %%mm6 \n\t" \
:); \
\
sad8_1_ ## suf(blk1, blk2, stride, 8); \
\
return sum_ ## suf(); \
} \
\
static int sad8_x2_ ## suf(MpegEncContext *v, uint8_t *blk2, \
uint8_t *blk1, ptrdiff_t stride, int h) \
{ \
av_assert2(h == 8); \
__asm__ volatile ( \
"pxor %%mm7, %%mm7 \n\t" \
"pxor %%mm6, %%mm6 \n\t" \
"movq %0, %%mm5 \n\t" \
:: "m" (round_tab[1])); \
\
sad8_x2a_ ## suf(blk1, blk2, stride, 8); \
\
return sum_ ## suf(); \
} \
\
static int sad8_y2_ ## suf(MpegEncContext *v, uint8_t *blk2, \
uint8_t *blk1, ptrdiff_t stride, int h) \
{ \
av_assert2(h == 8); \
__asm__ volatile ( \
"pxor %%mm7, %%mm7 \n\t" \
"pxor %%mm6, %%mm6 \n\t" \
"movq %0, %%mm5 \n\t" \
:: "m" (round_tab[1])); \
\
sad8_y2a_ ## suf(blk1, blk2, stride, 8); \
\
return sum_ ## suf(); \
} \
\
static int sad8_xy2_ ## suf(MpegEncContext *v, uint8_t *blk2, \
uint8_t *blk1, ptrdiff_t stride, int h) \
{ \
av_assert2(h == 8); \
__asm__ volatile ( \
"pxor %%mm7, %%mm7 \n\t" \
"pxor %%mm6, %%mm6 \n\t" \
::); \
\
sad8_4_ ## suf(blk1, blk2, stride, 8); \
\
return sum_ ## suf(); \
} \
\
static int sad16_ ## suf(MpegEncContext *v, uint8_t *blk2, \
uint8_t *blk1, ptrdiff_t stride, int h) \
{ \
__asm__ volatile ( \
"pxor %%mm7, %%mm7 \n\t" \
"pxor %%mm6, %%mm6 \n\t" \
:); \
\
sad8_1_ ## suf(blk1, blk2, stride, h); \
sad8_1_ ## suf(blk1 + 8, blk2 + 8, stride, h); \
\
return sum_ ## suf(); \
} \
\
static int sad16_x2_ ## suf(MpegEncContext *v, uint8_t *blk2, \
uint8_t *blk1, ptrdiff_t stride, int h) \
{ \
__asm__ volatile ( \
"pxor %%mm7, %%mm7 \n\t" \
"pxor %%mm6, %%mm6 \n\t" \
"movq %0, %%mm5 \n\t" \
:: "m" (round_tab[1])); \
\
sad8_x2a_ ## suf(blk1, blk2, stride, h); \
sad8_x2a_ ## suf(blk1 + 8, blk2 + 8, stride, h); \
\
return sum_ ## suf(); \
} \
\
static int sad16_y2_ ## suf(MpegEncContext *v, uint8_t *blk2, \
uint8_t *blk1, ptrdiff_t stride, int h) \
{ \
__asm__ volatile ( \
"pxor %%mm7, %%mm7 \n\t" \
"pxor %%mm6, %%mm6 \n\t" \
"movq %0, %%mm5 \n\t" \
:: "m" (round_tab[1])); \
\
sad8_y2a_ ## suf(blk1, blk2, stride, h); \
sad8_y2a_ ## suf(blk1 + 8, blk2 + 8, stride, h); \
\
return sum_ ## suf(); \
} \
\
static int sad16_xy2_ ## suf(MpegEncContext *v, uint8_t *blk2, \
uint8_t *blk1, ptrdiff_t stride, int h) \
{ \
__asm__ volatile ( \
"pxor %%mm7, %%mm7 \n\t" \
"pxor %%mm6, %%mm6 \n\t" \
::); \
\
sad8_4_ ## suf(blk1, blk2, stride, h); \
sad8_4_ ## suf(blk1 + 8, blk2 + 8, stride, h); \
\
return sum_ ## suf(); \
} \
PIX_SAD(mmx)
#endif /* HAVE_INLINE_ASM */
av_cold void ff_me_cmp_init_x86(MECmpContext *c, AVCodecContext *avctx)
{
int cpu_flags = av_get_cpu_flags();
#if HAVE_INLINE_ASM
if (INLINE_MMX(cpu_flags)) {
c->pix_abs[0][0] = sad16_mmx;
c->pix_abs[0][1] = sad16_x2_mmx;
c->pix_abs[0][2] = sad16_y2_mmx;
c->pix_abs[0][3] = sad16_xy2_mmx;
c->pix_abs[1][0] = sad8_mmx;
c->pix_abs[1][1] = sad8_x2_mmx;
c->pix_abs[1][2] = sad8_y2_mmx;
c->pix_abs[1][3] = sad8_xy2_mmx;
c->sad[0] = sad16_mmx;
c->sad[1] = sad8_mmx;
c->vsad[4] = vsad_intra16_mmx;
if (!(avctx->flags & AV_CODEC_FLAG_BITEXACT)) {
c->vsad[0] = vsad16_mmx;
}
}
#endif /* HAVE_INLINE_ASM */
if (EXTERNAL_MMX(cpu_flags)) {
c->hadamard8_diff[0] = ff_hadamard8_diff16_mmx;
c->hadamard8_diff[1] = ff_hadamard8_diff_mmx;
c->sum_abs_dctelem = ff_sum_abs_dctelem_mmx;
c->sse[0] = ff_sse16_mmx;
c->sse[1] = ff_sse8_mmx;
#if HAVE_X86ASM
c->nsse[0] = nsse16_mmx;
c->nsse[1] = nsse8_mmx;
#endif
}
if (EXTERNAL_MMXEXT(cpu_flags)) {
c->hadamard8_diff[0] = ff_hadamard8_diff16_mmxext;
c->hadamard8_diff[1] = ff_hadamard8_diff_mmxext;
c->sum_abs_dctelem = ff_sum_abs_dctelem_mmxext;
c->sad[0] = ff_sad16_mmxext;
c->sad[1] = ff_sad8_mmxext;
c->pix_abs[0][0] = ff_sad16_mmxext;
c->pix_abs[0][1] = ff_sad16_x2_mmxext;
c->pix_abs[0][2] = ff_sad16_y2_mmxext;
c->pix_abs[1][0] = ff_sad8_mmxext;
c->pix_abs[1][1] = ff_sad8_x2_mmxext;
c->pix_abs[1][2] = ff_sad8_y2_mmxext;
c->vsad[4] = ff_vsad_intra16_mmxext;
c->vsad[5] = ff_vsad_intra8_mmxext;
if (!(avctx->flags & AV_CODEC_FLAG_BITEXACT)) {
c->pix_abs[0][3] = ff_sad16_approx_xy2_mmxext;
c->pix_abs[1][3] = ff_sad8_approx_xy2_mmxext;
c->vsad[0] = ff_vsad16_approx_mmxext;
c->vsad[1] = ff_vsad8_approx_mmxext;
}
}
if (EXTERNAL_SSE2(cpu_flags)) {
c->sse[0] = ff_sse16_sse2;
c->sum_abs_dctelem = ff_sum_abs_dctelem_sse2;
#if HAVE_ALIGNED_STACK
c->hadamard8_diff[0] = ff_hadamard8_diff16_sse2;
c->hadamard8_diff[1] = ff_hadamard8_diff_sse2;
#endif
if (!(cpu_flags & AV_CPU_FLAG_SSE2SLOW) && avctx->codec_id != AV_CODEC_ID_SNOW) {
c->sad[0] = ff_sad16_sse2;
c->pix_abs[0][0] = ff_sad16_sse2;
c->pix_abs[0][1] = ff_sad16_x2_sse2;
c->pix_abs[0][2] = ff_sad16_y2_sse2;
c->vsad[4] = ff_vsad_intra16_sse2;
if (!(avctx->flags & AV_CODEC_FLAG_BITEXACT)) {
c->pix_abs[0][3] = ff_sad16_approx_xy2_sse2;
c->vsad[0] = ff_vsad16_approx_sse2;
}
}
}
if (EXTERNAL_SSSE3(cpu_flags)) {
c->sum_abs_dctelem = ff_sum_abs_dctelem_ssse3;
#if HAVE_ALIGNED_STACK
c->hadamard8_diff[0] = ff_hadamard8_diff16_ssse3;
c->hadamard8_diff[1] = ff_hadamard8_diff_ssse3;
#endif
}
}

46
media/ffvpx/libavcodec/x86/moz.build
View file

@ -5,12 +5,58 @@
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
SOURCES += [
'aacpsdsp.asm',
'aacpsdsp_init.c',
'bswapdsp.asm',
'bswapdsp_init.c',
'constants.c',
'dct32.asm',
'dct_init.c',
'fdct.c',
'fdctdsp_init.c',
'fft.asm',
'fft_init.c',
'flacdsp.asm',
'flacdsp_init.c',
'fpel.asm',
'h264_chromamc.asm',
'h264_chromamc_10bit.asm',
'h264_deblock.asm',
'h264_deblock_10bit.asm',
'h264_idct.asm',
'h264_idct_10bit.asm',
'h264_intrapred.asm',
'h264_intrapred_10bit.asm',
'h264_intrapred_init.c',
'h264_qpel.c',
'h264_qpel_10bit.asm',
'h264_qpel_8bit.asm',
'h264_weight.asm',
'h264_weight_10bit.asm',
'h264chroma_init.c',
'h264dsp_init.c',
'hevc_add_res.asm',
'hevc_deblock.asm',
'hevc_idct.asm',
'hevc_mc.asm',
'hevc_sao.asm',
'hevc_sao_10bit.asm',
'hevcdsp_init.c',
'idctdsp.asm',
'idctdsp_init.c',
'imdct36.asm',
'mdct15.asm',
'mdct15_init.c',
'me_cmp.asm',
'me_cmp_init.c',
'mpegaudiodsp.c',
'pixblockdsp.asm',
'pixblockdsp_init.c',
'qpel.asm',
'sbrdsp.asm',
'sbrdsp_init.c',
'simple_idct.asm',
'simple_idct10.asm',
'videodsp.asm',
'videodsp_init.c',
'vp8dsp.asm',

289
media/ffvpx/libavcodec/x86/mpegaudiodsp.c Normal file
View file

@ -0,0 +1,289 @@
/*
* SIMD-optimized MP3 decoding functions
* Copyright (c) 2010 Vitor Sessak
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "libavutil/attributes.h"
#include "libavutil/cpu.h"
#include "libavutil/internal.h"
#include "libavutil/x86/asm.h"
#include "libavutil/x86/cpu.h"
#include "libavcodec/mpegaudiodsp.h"
#define DECL(CPU)\
static void imdct36_blocks_ ## CPU(float *out, float *buf, float *in, int count, int switch_point, int block_type);\
void ff_imdct36_float_ ## CPU(float *out, float *buf, float *in, float *win);
#if HAVE_X86ASM
#if ARCH_X86_32
DECL(sse)
#endif
DECL(sse2)
DECL(sse3)
DECL(ssse3)
DECL(avx)
#endif /* HAVE_X86ASM */
void ff_four_imdct36_float_sse(float *out, float *buf, float *in, float *win,
float *tmpbuf);
void ff_four_imdct36_float_avx(float *out, float *buf, float *in, float *win,
float *tmpbuf);
DECLARE_ALIGNED(16, static float, mdct_win_sse)[2][4][4*40];
#if HAVE_6REGS && HAVE_SSE_INLINE
#define MACS(rt, ra, rb) rt+=(ra)*(rb)
#define MLSS(rt, ra, rb) rt-=(ra)*(rb)
#define SUM8(op, sum, w, p) \
{ \
op(sum, (w)[0 * 64], (p)[0 * 64]); \
op(sum, (w)[1 * 64], (p)[1 * 64]); \
op(sum, (w)[2 * 64], (p)[2 * 64]); \
op(sum, (w)[3 * 64], (p)[3 * 64]); \
op(sum, (w)[4 * 64], (p)[4 * 64]); \
op(sum, (w)[5 * 64], (p)[5 * 64]); \
op(sum, (w)[6 * 64], (p)[6 * 64]); \
op(sum, (w)[7 * 64], (p)[7 * 64]); \
}
static void apply_window(const float *buf, const float *win1,
const float *win2, float *sum1, float *sum2, int len)
{
x86_reg count = - 4*len;
const float *win1a = win1+len;
const float *win2a = win2+len;
const float *bufa = buf+len;
float *sum1a = sum1+len;
float *sum2a = sum2+len;
#define MULT(a, b) \
"movaps " #a "(%1,%0), %%xmm1 \n\t" \
"movaps " #a "(%3,%0), %%xmm2 \n\t" \
"mulps %%xmm2, %%xmm1 \n\t" \
"subps %%xmm1, %%xmm0 \n\t" \
"mulps " #b "(%2,%0), %%xmm2 \n\t" \
"subps %%xmm2, %%xmm4 \n\t" \
__asm__ volatile(
"1: \n\t"
"xorps %%xmm0, %%xmm0 \n\t"
"xorps %%xmm4, %%xmm4 \n\t"
MULT( 0, 0)
MULT( 256, 64)
MULT( 512, 128)
MULT( 768, 192)
MULT(1024, 256)
MULT(1280, 320)
MULT(1536, 384)
MULT(1792, 448)
"movaps %%xmm0, (%4,%0) \n\t"
"movaps %%xmm4, (%5,%0) \n\t"
"add $16, %0 \n\t"
"jl 1b \n\t"
:"+&r"(count)
:"r"(win1a), "r"(win2a), "r"(bufa), "r"(sum1a), "r"(sum2a)
);
#undef MULT
}
static void apply_window_mp3(float *in, float *win, int *unused, float *out,
ptrdiff_t incr)
{
LOCAL_ALIGNED_16(float, suma, [17]);
LOCAL_ALIGNED_16(float, sumb, [17]);
LOCAL_ALIGNED_16(float, sumc, [17]);
LOCAL_ALIGNED_16(float, sumd, [17]);
float sum;
/* copy to avoid wrap */
__asm__ volatile(
"movaps 0(%0), %%xmm0 \n\t" \
"movaps 16(%0), %%xmm1 \n\t" \
"movaps 32(%0), %%xmm2 \n\t" \
"movaps 48(%0), %%xmm3 \n\t" \
"movaps %%xmm0, 0(%1) \n\t" \
"movaps %%xmm1, 16(%1) \n\t" \
"movaps %%xmm2, 32(%1) \n\t" \
"movaps %%xmm3, 48(%1) \n\t" \
"movaps 64(%0), %%xmm0 \n\t" \
"movaps 80(%0), %%xmm1 \n\t" \
"movaps 96(%0), %%xmm2 \n\t" \
"movaps 112(%0), %%xmm3 \n\t" \
"movaps %%xmm0, 64(%1) \n\t" \
"movaps %%xmm1, 80(%1) \n\t" \
"movaps %%xmm2, 96(%1) \n\t" \
"movaps %%xmm3, 112(%1) \n\t"
::"r"(in), "r"(in+512)
:"memory"
);
apply_window(in + 16, win , win + 512, suma, sumc, 16);
apply_window(in + 32, win + 48, win + 640, sumb, sumd, 16);
SUM8(MACS, suma[0], win + 32, in + 48);
sumc[ 0] = 0;
sumb[16] = 0;
sumd[16] = 0;
#define SUMS(suma, sumb, sumc, sumd, out1, out2) \
"movups " #sumd "(%4), %%xmm0 \n\t" \
"shufps $0x1b, %%xmm0, %%xmm0 \n\t" \
"subps " #suma "(%1), %%xmm0 \n\t" \
"movaps %%xmm0," #out1 "(%0) \n\t" \
\
"movups " #sumc "(%3), %%xmm0 \n\t" \
"shufps $0x1b, %%xmm0, %%xmm0 \n\t" \
"addps " #sumb "(%2), %%xmm0 \n\t" \
"movaps %%xmm0," #out2 "(%0) \n\t"
if (incr == 1) {
__asm__ volatile(
SUMS( 0, 48, 4, 52, 0, 112)
SUMS(16, 32, 20, 36, 16, 96)
SUMS(32, 16, 36, 20, 32, 80)
SUMS(48, 0, 52, 4, 48, 64)
:"+&r"(out)
:"r"(&suma[0]), "r"(&sumb[0]), "r"(&sumc[0]), "r"(&sumd[0])
:"memory"
);
out += 16*incr;
} else {
int j;
float *out2 = out + 32 * incr;
out[0 ] = -suma[ 0];
out += incr;
out2 -= incr;
for(j=1;j<16;j++) {
*out = -suma[ j] + sumd[16-j];
*out2 = sumb[16-j] + sumc[ j];
out += incr;
out2 -= incr;
}
}
sum = 0;
SUM8(MLSS, sum, win + 16 + 32, in + 32);
*out = sum;
}
#endif /* HAVE_6REGS && HAVE_SSE_INLINE */
#if HAVE_X86ASM
#define DECL_IMDCT_BLOCKS(CPU1, CPU2) \
static void imdct36_blocks_ ## CPU1(float *out, float *buf, float *in, \
int count, int switch_point, int block_type) \
{ \
int align_end = count - (count & 3); \
int j; \
for (j = 0; j < align_end; j+= 4) { \
LOCAL_ALIGNED_16(float, tmpbuf, [1024]); \
float *win = mdct_win_sse[switch_point && j < 4][block_type]; \
/* apply window & overlap with previous buffer */ \
\
/* select window */ \
ff_four_imdct36_float_ ## CPU2(out, buf, in, win, tmpbuf); \
in += 4*18; \
buf += 4*18; \
out += 4; \
} \
for (; j < count; j++) { \
/* apply window & overlap with previous buffer */ \
\
/* select window */ \
int win_idx = (switch_point && j < 2) ? 0 : block_type; \
float *win = ff_mdct_win_float[win_idx + (4 & -(j & 1))]; \
\
ff_imdct36_float_ ## CPU1(out, buf, in, win); \
\
in += 18; \
buf++; \
out++; \
} \
}
#if HAVE_SSE
#if ARCH_X86_32
DECL_IMDCT_BLOCKS(sse,sse)
#endif
DECL_IMDCT_BLOCKS(sse2,sse)
DECL_IMDCT_BLOCKS(sse3,sse)
DECL_IMDCT_BLOCKS(ssse3,sse)
#endif
#if HAVE_AVX_EXTERNAL
DECL_IMDCT_BLOCKS(avx,avx)
#endif
#endif /* HAVE_X86ASM */
av_cold void ff_mpadsp_init_x86(MPADSPContext *s)
{
av_unused int cpu_flags = av_get_cpu_flags();
int i, j;
for (j = 0; j < 4; j++) {
for (i = 0; i < 40; i ++) {
mdct_win_sse[0][j][4*i ] = ff_mdct_win_float[j ][i];
mdct_win_sse[0][j][4*i + 1] = ff_mdct_win_float[j + 4][i];
mdct_win_sse[0][j][4*i + 2] = ff_mdct_win_float[j ][i];
mdct_win_sse[0][j][4*i + 3] = ff_mdct_win_float[j + 4][i];
mdct_win_sse[1][j][4*i ] = ff_mdct_win_float[0 ][i];
mdct_win_sse[1][j][4*i + 1] = ff_mdct_win_float[4 ][i];
mdct_win_sse[1][j][4*i + 2] = ff_mdct_win_float[j ][i];
mdct_win_sse[1][j][4*i + 3] = ff_mdct_win_float[j + 4][i];
}
}
#if HAVE_6REGS && HAVE_SSE_INLINE
if (INLINE_SSE(cpu_flags)) {
s->apply_window_float = apply_window_mp3;
}
#endif /* HAVE_SSE_INLINE */
#if HAVE_X86ASM
#if HAVE_SSE
#if ARCH_X86_32
if (EXTERNAL_SSE(cpu_flags)) {
s->imdct36_blocks_float = imdct36_blocks_sse;
}
#endif
if (EXTERNAL_SSE2(cpu_flags)) {
s->imdct36_blocks_float = imdct36_blocks_sse2;
}
if (EXTERNAL_SSE3(cpu_flags)) {
s->imdct36_blocks_float = imdct36_blocks_sse3;
}
if (EXTERNAL_SSSE3(cpu_flags)) {
s->imdct36_blocks_float = imdct36_blocks_ssse3;
}
#endif
#if HAVE_AVX_EXTERNAL
if (EXTERNAL_AVX(cpu_flags)) {
s->imdct36_blocks_float = imdct36_blocks_avx;
}
#endif
#endif /* HAVE_X86ASM */
}

45
media/ffvpx/libavcodec/x86/opus_dsp_init.c Normal file
View file

@ -0,0 +1,45 @@
/*
* Opus encoder assembly optimizations
* Copyright (C) 2017 Ivan Kalvachev <ikalvachev@gmail.com>
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "config.h"
#include "libavutil/x86/cpu.h"
#include "libavcodec/opus_pvq.h"
extern float ff_pvq_search_approx_sse2(float *X, int *y, int K, int N);
extern float ff_pvq_search_approx_sse4(float *X, int *y, int K, int N);
extern float ff_pvq_search_exact_avx (float *X, int *y, int K, int N);
av_cold void ff_opus_dsp_init_x86(CeltPVQ *s)
{
int cpu_flags = av_get_cpu_flags();
#if CONFIG_OPUS_ENCODER
if (EXTERNAL_SSE2(cpu_flags))
s->pvq_search = ff_pvq_search_approx_sse2;
if (EXTERNAL_SSE4(cpu_flags))
s->pvq_search = ff_pvq_search_approx_sse4;
if (EXTERNAL_AVX_FAST(cpu_flags))
s->pvq_search = ff_pvq_search_exact_avx;
#endif
}

128
media/ffvpx/libavcodec/x86/pixblockdsp.asm Normal file
View file

@ -0,0 +1,128 @@
;*****************************************************************************
;* SIMD-optimized pixel operations
;*****************************************************************************
;* Copyright (c) 2000, 2001 Fabrice Bellard
;* Copyright (c) 2002-2004 Michael Niedermayer <michaelni@gmx.at>
;*
;* This file is part of FFmpeg.
;*
;* FFmpeg is free software; you can redistribute it and/or
;* modify it under the terms of the GNU Lesser General Public
;* License as published by the Free Software Foundation; either
;* version 2.1 of the License, or (at your option) any later version.
;*
;* FFmpeg is distributed in the hope that it will be useful,
;* but WITHOUT ANY WARRANTY; without even the implied warranty of
;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
;* Lesser General Public License for more details.
;*
;* You should have received a copy of the GNU Lesser General Public
;* License along with FFmpeg; if not, write to the Free Software
;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
;*****************************************************************************
%include "libavutil/x86/x86util.asm"
SECTION .text
INIT_MMX mmx
; void ff_get_pixels_mmx(int16_t *block, const uint8_t *pixels, ptrdiff_t stride)
cglobal get_pixels, 3,4
add r0, 128
mov r3, -128
pxor m7, m7
.loop:
mova m0, [r1]
mova m2, [r1+r2]
mova m1, m0
mova m3, m2
punpcklbw m0, m7
punpckhbw m1, m7
punpcklbw m2, m7
punpckhbw m3, m7
mova [r0+r3+ 0], m0
mova [r0+r3+ 8], m1
mova [r0+r3+16], m2
mova [r0+r3+24], m3
lea r1, [r1+r2*2]
add r3, 32
js .loop
REP_RET
INIT_XMM sse2
cglobal get_pixels, 3, 4, 5
lea r3, [r2*3]
pxor m4, m4
movh m0, [r1]
movh m1, [r1+r2]
movh m2, [r1+r2*2]
movh m3, [r1+r3]
lea r1, [r1+r2*4]
punpcklbw m0, m4
punpcklbw m1, m4
punpcklbw m2, m4
punpcklbw m3, m4
mova [r0], m0
mova [r0+0x10], m1
mova [r0+0x20], m2
mova [r0+0x30], m3
movh m0, [r1]
movh m1, [r1+r2*1]
movh m2, [r1+r2*2]
movh m3, [r1+r3]
punpcklbw m0, m4
punpcklbw m1, m4
punpcklbw m2, m4
punpcklbw m3, m4
mova [r0+0x40], m0
mova [r0+0x50], m1
mova [r0+0x60], m2
mova [r0+0x70], m3
RET
; void ff_diff_pixels_mmx(int16_t *block, const uint8_t *s1, const uint8_t *s2,
; ptrdiff_t stride);
%macro DIFF_PIXELS 0
cglobal diff_pixels, 4,5,5
pxor m4, m4
add r0, 128
mov r4, -128
.loop:
movq m0, [r1]
movq m2, [r2]
%if mmsize == 8
movq m1, m0
movq m3, m2
punpcklbw m0, m4
punpckhbw m1, m4
punpcklbw m2, m4
punpckhbw m3, m4
%else
movq m1, [r1+r3]
movq m3, [r2+r3]
punpcklbw m0, m4
punpcklbw m1, m4
punpcklbw m2, m4
punpcklbw m3, m4
%endif
psubw m0, m2
psubw m1, m3
mova [r0+r4+0], m0
mova [r0+r4+mmsize], m1
%if mmsize == 8
add r1, r3
add r2, r3
%else
lea r1, [r1+r3*2]
lea r2, [r2+r3*2]
%endif
add r4, 2 * mmsize
jne .loop
RET
%endmacro
INIT_MMX mmx
DIFF_PIXELS
INIT_XMM sse2
DIFF_PIXELS

52
media/ffvpx/libavcodec/x86/pixblockdsp_init.c Normal file
View file

@ -0,0 +1,52 @@
/*
* SIMD-optimized pixel operations
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "libavutil/attributes.h"
#include "libavutil/cpu.h"
#include "libavutil/x86/cpu.h"
#include "libavcodec/pixblockdsp.h"
void ff_get_pixels_mmx(int16_t *block, const uint8_t *pixels, ptrdiff_t stride);
void ff_get_pixels_sse2(int16_t *block, const uint8_t *pixels, ptrdiff_t stride);
void ff_diff_pixels_mmx(int16_t *block, const uint8_t *s1, const uint8_t *s2,
ptrdiff_t stride);
void ff_diff_pixels_sse2(int16_t *block, const uint8_t *s1, const uint8_t *s2,
ptrdiff_t stride);
av_cold void ff_pixblockdsp_init_x86(PixblockDSPContext *c,
AVCodecContext *avctx,
unsigned high_bit_depth)
{
int cpu_flags = av_get_cpu_flags();
if (EXTERNAL_MMX(cpu_flags)) {
if (!high_bit_depth)
c->get_pixels = ff_get_pixels_mmx;
c->diff_pixels_unaligned =
c->diff_pixels = ff_diff_pixels_mmx;
}
if (EXTERNAL_SSE2(cpu_flags)) {
if (!high_bit_depth)
c->get_pixels = ff_get_pixels_sse2;
c->diff_pixels_unaligned =
c->diff_pixels = ff_diff_pixels_sse2;
}
}

179
media/ffvpx/libavcodec/x86/qpel.asm Normal file
View file

@ -0,0 +1,179 @@
;******************************************************************************
;* SIMD-optimized quarterpel functions
;* Copyright (c) 2008 Loren Merritt
;* Copyright (c) 2003-2013 Michael Niedermayer
;* Copyright (c) 2013 Daniel Kang
;*
;* This file is part of FFmpeg.
;*
;* FFmpeg is free software; you can redistribute it and/or
;* modify it under the terms of the GNU Lesser General Public
;* License as published by the Free Software Foundation; either
;* version 2.1 of the License, or (at your option) any later version.
;*
;* FFmpeg is distributed in the hope that it will be useful,
;* but WITHOUT ANY WARRANTY; without even the implied warranty of
;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
;* Lesser General Public License for more details.
;*
;* You should have received a copy of the GNU Lesser General Public
;* License along with FFmpeg; if not, write to the Free Software
;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
;******************************************************************************
%include "libavutil/x86/x86util.asm"
SECTION .text
%macro op_avgh 3
movh %3, %2
pavgb %1, %3
movh %2, %1
%endmacro
%macro op_avg 2
pavgb %1, %2
mova %2, %1
%endmacro
%macro op_puth 2-3
movh %2, %1
%endmacro
%macro op_put 2
mova %2, %1
%endmacro
; void ff_put/avg_pixels4_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,
; int dstStride, int src1Stride, int h)
%macro PIXELS4_L2 1
%define OP op_%1h
cglobal %1_pixels4_l2, 6,6
movsxdifnidn r3, r3d
movsxdifnidn r4, r4d
test r5d, 1
je .loop
movd m0, [r1]
movd m1, [r2]
add r1, r4
add r2, 4
pavgb m0, m1
OP m0, [r0], m3
add r0, r3
dec r5d
.loop:
mova m0, [r1]
mova m1, [r1+r4]
lea r1, [r1+2*r4]
pavgb m0, [r2]
pavgb m1, [r2+4]
OP m0, [r0], m3
OP m1, [r0+r3], m3
lea r0, [r0+2*r3]
mova m0, [r1]
mova m1, [r1+r4]
lea r1, [r1+2*r4]
pavgb m0, [r2+8]
pavgb m1, [r2+12]
OP m0, [r0], m3
OP m1, [r0+r3], m3
lea r0, [r0+2*r3]
add r2, 16
sub r5d, 4
jne .loop
REP_RET
%endmacro
INIT_MMX mmxext
PIXELS4_L2 put
PIXELS4_L2 avg
; void ff_put/avg_pixels8_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,
; int dstStride, int src1Stride, int h)
%macro PIXELS8_L2 1
%define OP op_%1
cglobal %1_pixels8_l2, 6,6
movsxdifnidn r3, r3d
movsxdifnidn r4, r4d
test r5d, 1
je .loop
mova m0, [r1]
mova m1, [r2]
add r1, r4
add r2, 8
pavgb m0, m1
OP m0, [r0]
add r0, r3
dec r5d
.loop:
mova m0, [r1]
mova m1, [r1+r4]
lea r1, [r1+2*r4]
pavgb m0, [r2]
pavgb m1, [r2+8]
OP m0, [r0]
OP m1, [r0+r3]
lea r0, [r0+2*r3]
mova m0, [r1]
mova m1, [r1+r4]
lea r1, [r1+2*r4]
pavgb m0, [r2+16]
pavgb m1, [r2+24]
OP m0, [r0]
OP m1, [r0+r3]
lea r0, [r0+2*r3]
add r2, 32
sub r5d, 4
jne .loop
REP_RET
%endmacro
INIT_MMX mmxext
PIXELS8_L2 put
PIXELS8_L2 avg
; void ff_put/avg_pixels16_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,
; int dstStride, int src1Stride, int h)
%macro PIXELS16_L2 1
%define OP op_%1
cglobal %1_pixels16_l2, 6,6
movsxdifnidn r3, r3d
movsxdifnidn r4, r4d
test r5d, 1
je .loop
mova m0, [r1]
mova m1, [r1+8]
pavgb m0, [r2]
pavgb m1, [r2+8]
add r1, r4
add r2, 16
OP m0, [r0]
OP m1, [r0+8]
add r0, r3
dec r5d
.loop:
mova m0, [r1]
mova m1, [r1+8]
add r1, r4
pavgb m0, [r2]
pavgb m1, [r2+8]
OP m0, [r0]
OP m1, [r0+8]
add r0, r3
mova m0, [r1]
mova m1, [r1+8]
add r1, r4
pavgb m0, [r2+16]
pavgb m1, [r2+24]
OP m0, [r0]
OP m1, [r0+8]
add r0, r3
add r2, 32
sub r5d, 2
jne .loop
REP_RET
%endmacro
INIT_MMX mmxext
PIXELS16_L2 put
PIXELS16_L2 avg

548
media/ffvpx/libavcodec/x86/sbrdsp.asm Normal file
View file

@ -0,0 +1,548 @@
;******************************************************************************
;* AAC Spectral Band Replication decoding functions
;* Copyright (C) 2012 Christophe Gisquet <christophe.gisquet@gmail.com>
;*
;* This file is part of FFmpeg.
;*
;* FFmpeg is free software; you can redistribute it and/or
;* modify it under the terms of the GNU Lesser General Public
;* License as published by the Free Software Foundation; either
;* version 2.1 of the License, or (at your option) any later version.
;*
;* FFmpeg is distributed in the hope that it will be useful,
;* but WITHOUT ANY WARRANTY; without even the implied warranty of
;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
;* Lesser General Public License for more details.
;*
;* You should have received a copy of the GNU Lesser General Public
;* License along with FFmpeg; if not, write to the Free Software
;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
;******************************************************************************
%include "libavutil/x86/x86util.asm"
SECTION_RODATA
; mask equivalent for multiply by -1.0 1.0
ps_mask times 2 dd 1<<31, 0
ps_mask2 times 2 dd 0, 1<<31
ps_mask3 dd 0, 0, 0, 1<<31
ps_noise0 times 2 dd 1.0, 0.0,
ps_noise2 times 2 dd -1.0, 0.0
ps_noise13 dd 0.0, 1.0, 0.0, -1.0
dd 0.0, -1.0, 0.0, 1.0
dd 0.0, 1.0, 0.0, -1.0
cextern sbr_noise_table
cextern ps_neg
SECTION .text
INIT_XMM sse
cglobal sbr_sum_square, 2, 3, 6
mov r2d, r1d
xorps m0, m0
xorps m1, m1
sar r2, 3
jz .prepare
.loop:
movu m2, [r0 + 0]
movu m3, [r0 + 16]
movu m4, [r0 + 32]
movu m5, [r0 + 48]
mulps m2, m2
mulps m3, m3
mulps m4, m4
mulps m5, m5
addps m0, m2
addps m1, m3
addps m0, m4
addps m1, m5
add r0, 64
dec r2
jnz .loop
.prepare:
and r1, 7
sar r1, 1
jz .end
; len is a multiple of 2, thus there are at least 4 elements to process
.endloop:
movu m2, [r0]
add r0, 16
mulps m2, m2
dec r1
addps m0, m2
jnz .endloop
.end:
addps m0, m1
movhlps m2, m0
addps m0, m2
movss m1, m0
shufps m0, m0, 1
addss m0, m1
%if ARCH_X86_64 == 0
movss r0m, m0
fld dword r0m
%endif
RET
%define STEP 40*4*2
cglobal sbr_hf_g_filt, 5, 6, 5
lea r1, [r1 + 8*r4] ; offset by ixh elements into X_high
mov r5, r3
and r3, 0xFC
lea r2, [r2 + r3*4]
lea r0, [r0 + r3*8]
neg r3
jz .loop1
.loop4:
movlps m0, [r2 + 4*r3 + 0]
movlps m1, [r2 + 4*r3 + 8]
movlps m2, [r1 + 0*STEP]
movlps m3, [r1 + 2*STEP]
movhps m2, [r1 + 1*STEP]
movhps m3, [r1 + 3*STEP]
unpcklps m0, m0
unpcklps m1, m1
mulps m0, m2
mulps m1, m3
movu [r0 + 8*r3 + 0], m0
movu [r0 + 8*r3 + 16], m1
add r1, 4*STEP
add r3, 4
jnz .loop4
and r5, 3 ; number of single element loops
jz .end
.loop1: ; element 0 and 1 can be computed at the same time
movss m0, [r2]
movlps m2, [r1]
unpcklps m0, m0
mulps m2, m0
movlps [r0], m2
add r0, 8
add r2, 4
add r1, STEP
dec r5
jnz .loop1
.end:
RET
; void ff_sbr_hf_gen_sse(float (*X_high)[2], const float (*X_low)[2],
; const float alpha0[2], const float alpha1[2],
; float bw, int start, int end)
;
cglobal sbr_hf_gen, 4,4,8, X_high, X_low, alpha0, alpha1, BW, S, E
; load alpha factors
%define bw m0
%if ARCH_X86_64 == 0 || WIN64
movss bw, BWm
%endif
movlps m2, [alpha1q]
movlps m1, [alpha0q]
shufps bw, bw, 0
mulps m2, bw ; (a1[0] a1[1])*bw
mulps m1, bw ; (a0[0] a0[1])*bw = (a2 a3)
mulps m2, bw ; (a1[0] a1[1])*bw*bw = (a0 a1)
mova m3, m1
mova m4, m2
; Set pointers
%if ARCH_X86_64 == 0 || WIN64
; start and end 6th and 7th args on stack
mov r2d, Sm
mov r3d, Em
DEFINE_ARGS X_high, X_low, start, end
%else
; BW does not actually occupy a register, so shift by 1
DEFINE_ARGS X_high, X_low, alpha0, alpha1, start, end
movsxd startq, startd
movsxd endq, endd
%endif
sub startq, endq ; neg num of loops
lea X_highq, [X_highq + endq*2*4]
lea X_lowq, [X_lowq + endq*2*4 - 2*2*4]
shl startq, 3 ; offset from num loops
mova m0, [X_lowq + startq]
shufps m3, m3, q1111
shufps m4, m4, q1111
xorps m3, [ps_mask]
shufps m1, m1, q0000
shufps m2, m2, q0000
xorps m4, [ps_mask]
.loop2:
movu m7, [X_lowq + startq + 8] ; BbCc
mova m6, m0
mova m5, m7
shufps m0, m0, q2301 ; aAbB
shufps m7, m7, q2301 ; bBcC
mulps m0, m4
mulps m7, m3
mulps m6, m2
mulps m5, m1
addps m7, m0
mova m0, [X_lowq + startq + 16] ; CcDd
addps m7, m0
addps m6, m5
addps m7, m6
mova [X_highq + startq], m7
add startq, 16
jnz .loop2
RET
cglobal sbr_sum64x5, 1,2,4,z
lea r1q, [zq+ 256]
.loop:
mova m0, [zq+ 0]
mova m2, [zq+ 16]
mova m1, [zq+ 256]
mova m3, [zq+ 272]
addps m0, [zq+ 512]
addps m2, [zq+ 528]
addps m1, [zq+ 768]
addps m3, [zq+ 784]
addps m0, [zq+1024]
addps m2, [zq+1040]
addps m0, m1
addps m2, m3
mova [zq], m0
mova [zq+16], m2
add zq, 32
cmp zq, r1q
jne .loop
REP_RET
INIT_XMM sse
cglobal sbr_qmf_post_shuffle, 2,3,4,W,z
lea r2q, [zq + (64-4)*4]
mova m3, [ps_neg]
.loop:
mova m1, [zq]
xorps m0, m3, [r2q]
shufps m0, m0, m0, q0123
unpcklps m2, m0, m1
unpckhps m0, m0, m1
mova [Wq + 0], m2
mova [Wq + 16], m0
add Wq, 32
sub r2q, 16
add zq, 16
cmp zq, r2q
jl .loop
REP_RET
INIT_XMM sse
cglobal sbr_neg_odd_64, 1,2,4,z
lea r1q, [zq+256]
.loop:
mova m0, [zq+ 0]
mova m1, [zq+16]
mova m2, [zq+32]
mova m3, [zq+48]
xorps m0, [ps_mask2]
xorps m1, [ps_mask2]
xorps m2, [ps_mask2]
xorps m3, [ps_mask2]
mova [zq+ 0], m0
mova [zq+16], m1
mova [zq+32], m2
mova [zq+48], m3
add zq, 64
cmp zq, r1q
jne .loop
REP_RET
; void ff_sbr_qmf_deint_bfly_sse2(float *v, const float *src0, const float *src1)
%macro SBR_QMF_DEINT_BFLY 0
cglobal sbr_qmf_deint_bfly, 3,5,8, v,src0,src1,vrev,c
mov cq, 64*4-2*mmsize
lea vrevq, [vq + 64*4]
.loop:
mova m0, [src0q+cq]
mova m1, [src1q]
mova m4, [src0q+cq+mmsize]
mova m5, [src1q+mmsize]
%if cpuflag(sse2)
pshufd m2, m0, q0123
pshufd m3, m1, q0123
pshufd m6, m4, q0123
pshufd m7, m5, q0123
%else
shufps m2, m0, m0, q0123
shufps m3, m1, m1, q0123
shufps m6, m4, m4, q0123
shufps m7, m5, m5, q0123
%endif
addps m5, m2
subps m0, m7
addps m1, m6
subps m4, m3
mova [vrevq], m1
mova [vrevq+mmsize], m5
mova [vq+cq], m0
mova [vq+cq+mmsize], m4
add src1q, 2*mmsize
add vrevq, 2*mmsize
sub cq, 2*mmsize
jge .loop
REP_RET
%endmacro
INIT_XMM sse
SBR_QMF_DEINT_BFLY
INIT_XMM sse2
SBR_QMF_DEINT_BFLY
INIT_XMM sse2
cglobal sbr_qmf_pre_shuffle, 1,4,6,z
%define OFFSET (32*4-2*mmsize)
mov r3q, OFFSET
lea r1q, [zq + (32+1)*4]
lea r2q, [zq + 64*4]
mova m5, [ps_neg]
.loop:
movu m0, [r1q]
movu m2, [r1q + mmsize]
movu m1, [zq + r3q + 4 + mmsize]
movu m3, [zq + r3q + 4]
pxor m2, m5
pxor m0, m5
pshufd m2, m2, q0123
pshufd m0, m0, q0123
SBUTTERFLY dq, 2, 3, 4
SBUTTERFLY dq, 0, 1, 4
mova [r2q + 2*r3q + 0*mmsize], m2
mova [r2q + 2*r3q + 1*mmsize], m3
mova [r2q + 2*r3q + 2*mmsize], m0
mova [r2q + 2*r3q + 3*mmsize], m1
add r1q, 2*mmsize
sub r3q, 2*mmsize
jge .loop
movq m2, [zq]
movq [r2q], m2
REP_RET
%ifdef PIC
%define NREGS 1
%if UNIX64
%define NOISE_TABLE r6q ; r5q is m_max
%else
%define NOISE_TABLE r5q
%endif
%else
%define NREGS 0
%define NOISE_TABLE sbr_noise_table
%endif
%macro LOAD_NST 1
%ifdef PIC
lea NOISE_TABLE, [%1]
mova m0, [kxq + NOISE_TABLE]
%else
mova m0, [kxq + %1]
%endif
%endmacro
INIT_XMM sse2
; sbr_hf_apply_noise_0(float (*Y)[2], const float *s_m,
; const float *q_filt, int noise,
; int kx, int m_max)
cglobal sbr_hf_apply_noise_0, 5,5+NREGS+UNIX64,8, Y,s_m,q_filt,noise,kx,m_max
mova m0, [ps_noise0]
jmp apply_noise_main
; sbr_hf_apply_noise_1(float (*Y)[2], const float *s_m,
; const float *q_filt, int noise,
; int kx, int m_max)
cglobal sbr_hf_apply_noise_1, 5,5+NREGS+UNIX64,8, Y,s_m,q_filt,noise,kx,m_max
and kxq, 1
shl kxq, 4
LOAD_NST ps_noise13
jmp apply_noise_main
; sbr_hf_apply_noise_2(float (*Y)[2], const float *s_m,
; const float *q_filt, int noise,
; int kx, int m_max)
cglobal sbr_hf_apply_noise_2, 5,5+NREGS+UNIX64,8, Y,s_m,q_filt,noise,kx,m_max
mova m0, [ps_noise2]
jmp apply_noise_main
; sbr_hf_apply_noise_3(float (*Y)[2], const float *s_m,
; const float *q_filt, int noise,
; int kx, int m_max)
cglobal sbr_hf_apply_noise_3, 5,5+NREGS+UNIX64,8, Y,s_m,q_filt,noise,kx,m_max
and kxq, 1
shl kxq, 4
LOAD_NST ps_noise13+16
apply_noise_main:
%if ARCH_X86_64 == 0 || WIN64
mov kxd, m_maxm
DEFINE_ARGS Y, s_m, q_filt, noise, count
%else
DEFINE_ARGS Y, s_m, q_filt, noise, kx, count
%endif
movsxdifnidn noiseq, noised
dec noiseq
shl countd, 2
%ifdef PIC
lea NOISE_TABLE, [sbr_noise_table]
%endif
lea Yq, [Yq + 2*countq]
add s_mq, countq
add q_filtq, countq
shl noiseq, 3
pxor m5, m5
neg countq
.loop:
mova m1, [q_filtq + countq]
movu m3, [noiseq + NOISE_TABLE + 1*mmsize]
movu m4, [noiseq + NOISE_TABLE + 2*mmsize]
add noiseq, 2*mmsize
and noiseq, 0x1ff<<3
punpckhdq m2, m1, m1
punpckldq m1, m1
mulps m1, m3 ; m2 = q_filt[m] * ff_sbr_noise_table[noise]
mulps m2, m4 ; m2 = q_filt[m] * ff_sbr_noise_table[noise]
mova m3, [s_mq + countq]
; TODO: replace by a vpermd in AVX2
punpckhdq m4, m3, m3
punpckldq m3, m3
pcmpeqd m6, m3, m5 ; m6 == 0
pcmpeqd m7, m4, m5 ; m7 == 0
mulps m3, m0 ; s_m[m] * phi_sign
mulps m4, m0 ; s_m[m] * phi_sign
pand m1, m6
pand m2, m7
movu m6, [Yq + 2*countq]
movu m7, [Yq + 2*countq + mmsize]
addps m3, m1
addps m4, m2
addps m6, m3
addps m7, m4
movu [Yq + 2*countq], m6
movu [Yq + 2*countq + mmsize], m7
add countq, mmsize
jl .loop
RET
INIT_XMM sse
cglobal sbr_qmf_deint_neg, 2,4,4,v,src,vrev,c
%define COUNT 32*4
%define OFFSET 32*4
mov cq, -COUNT
lea vrevq, [vq + OFFSET + COUNT]
add vq, OFFSET-mmsize
add srcq, 2*COUNT
mova m3, [ps_neg]
.loop:
mova m0, [srcq + 2*cq + 0*mmsize]
mova m1, [srcq + 2*cq + 1*mmsize]
shufps m2, m0, m1, q2020
shufps m1, m0, q1313
xorps m2, m3
mova [vq], m1
mova [vrevq + cq], m2
sub vq, mmsize
add cq, mmsize
jl .loop
REP_RET
%macro SBR_AUTOCORRELATE 0
cglobal sbr_autocorrelate, 2,3,8,32, x, phi, cnt
mov cntq, 37*8
add xq, cntq
neg cntq
%if cpuflag(sse3)
%define MOVH movsd
movddup m5, [xq+cntq]
%else
%define MOVH movlps
movlps m5, [xq+cntq]
movlhps m5, m5
%endif
MOVH m7, [xq+cntq+8 ]
MOVH m1, [xq+cntq+16]
shufps m7, m7, q0110
shufps m1, m1, q0110
mulps m3, m5, m7 ; x[0][0] * x[1][0], x[0][1] * x[1][1], x[0][0] * x[1][1], x[0][1] * x[1][0]
mulps m4, m5, m5 ; x[0][0] * x[0][0], x[0][1] * x[0][1];
mulps m5, m1 ; real_sum2 = x[0][0] * x[2][0], x[0][1] * x[2][1]; imag_sum2 = x[0][0] * x[2][1], x[0][1] * x[2][0]
movaps [rsp ], m3
movaps [rsp+16], m4
add cntq, 8
MOVH m2, [xq+cntq+16]
movlhps m7, m7
shufps m2, m2, q0110
mulps m6, m7, m1 ; real_sum1 = x[1][0] * x[2][0], x[1][1] * x[2][1]; imag_sum1 += x[1][0] * x[2][1], x[1][1] * x[2][0]
mulps m4, m7, m2
mulps m7, m7 ; real_sum0 = x[1][0] * x[1][0], x[1][1] * x[1][1];
addps m5, m4 ; real_sum2 += x[1][0] * x[3][0], x[1][1] * x[3][1]; imag_sum2 += x[1][0] * x[3][1], x[1][1] * x[3][0]
align 16
.loop:
add cntq, 8
MOVH m0, [xq+cntq+16]
movlhps m1, m1
shufps m0, m0, q0110
mulps m3, m1, m2
mulps m4, m1, m0
mulps m1, m1
addps m6, m3 ; real_sum1 += x[i][0] * x[i + 1][0], x[i][1] * x[i + 1][1]; imag_sum1 += x[i][0] * x[i + 1][1], x[i][1] * x[i + 1][0];
addps m5, m4 ; real_sum2 += x[i][0] * x[i + 2][0], x[i][1] * x[i + 2][1]; imag_sum2 += x[i][0] * x[i + 2][1], x[i][1] * x[i + 2][0];
addps m7, m1 ; real_sum0 += x[i][0] * x[i][0], x[i][1] * x[i][1];
add cntq, 8
MOVH m1, [xq+cntq+16]
movlhps m2, m2
shufps m1, m1, q0110
mulps m3, m2, m0
mulps m4, m2, m1
mulps m2, m2
addps m6, m3 ; real_sum1 += x[i][0] * x[i + 1][0], x[i][1] * x[i + 1][1]; imag_sum1 += x[i][0] * x[i + 1][1], x[i][1] * x[i + 1][0];
addps m5, m4 ; real_sum2 += x[i][0] * x[i + 2][0], x[i][1] * x[i + 2][1]; imag_sum2 += x[i][0] * x[i + 2][1], x[i][1] * x[i + 2][0];
addps m7, m2 ; real_sum0 += x[i][0] * x[i][0], x[i][1] * x[i][1];
add cntq, 8
MOVH m2, [xq+cntq+16]
movlhps m0, m0
shufps m2, m2, q0110
mulps m3, m0, m1
mulps m4, m0, m2
mulps m0, m0
addps m6, m3 ; real_sum1 += x[i][0] * x[i + 1][0], x[i][1] * x[i + 1][1]; imag_sum1 += x[i][0] * x[i + 1][1], x[i][1] * x[i + 1][0];
addps m5, m4 ; real_sum2 += x[i][0] * x[i + 2][0], x[i][1] * x[i + 2][1]; imag_sum2 += x[i][0] * x[i + 2][1], x[i][1] * x[i + 2][0];
addps m7, m0 ; real_sum0 += x[i][0] * x[i][0], x[i][1] * x[i][1];
jl .loop
movlhps m1, m1
mulps m2, m1
mulps m1, m1
addps m2, m6 ; real_sum1 + x[38][0] * x[39][0], x[38][1] * x[39][1]; imag_sum1 + x[38][0] * x[39][1], x[38][1] * x[39][0];
addps m1, m7 ; real_sum0 + x[38][0] * x[38][0], x[38][1] * x[38][1];
addps m6, [rsp ] ; real_sum1 + x[ 0][0] * x[ 1][0], x[ 0][1] * x[ 1][1]; imag_sum1 + x[ 0][0] * x[ 1][1], x[ 0][1] * x[ 1][0];
addps m7, [rsp+16] ; real_sum0 + x[ 0][0] * x[ 0][0], x[ 0][1] * x[ 0][1];
xorps m2, [ps_mask3]
xorps m5, [ps_mask3]
xorps m6, [ps_mask3]
HADDPS m2, m5, m3
HADDPS m7, m6, m4
%if cpuflag(sse3)
movshdup m0, m1
%else
movss m0, m1
shufps m1, m1, q0001
%endif
addss m1, m0
movaps [phiq ], m2
movhps [phiq+0x18], m7
movss [phiq+0x28], m7
movss [phiq+0x10], m1
RET
%endmacro
INIT_XMM sse
SBR_AUTOCORRELATE
INIT_XMM sse3
SBR_AUTOCORRELATE

87
media/ffvpx/libavcodec/x86/sbrdsp_init.c Normal file
View file

@ -0,0 +1,87 @@
/*
* AAC Spectral Band Replication decoding functions
* Copyright (c) 2012 Christophe Gisquet <christophe.gisquet@gmail.com>
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "config.h"
#include "libavutil/attributes.h"
#include "libavutil/cpu.h"
#include "libavutil/x86/cpu.h"
#include "libavcodec/sbrdsp.h"
float ff_sbr_sum_square_sse(float (*x)[2], int n);
void ff_sbr_sum64x5_sse(float *z);
void ff_sbr_hf_g_filt_sse(float (*Y)[2], const float (*X_high)[40][2],
const float *g_filt, int m_max, intptr_t ixh);
void ff_sbr_hf_gen_sse(float (*X_high)[2], const float (*X_low)[2],
const float alpha0[2], const float alpha1[2],
float bw, int start, int end);
void ff_sbr_neg_odd_64_sse(float *z);
void ff_sbr_qmf_post_shuffle_sse(float W[32][2], const float *z);
void ff_sbr_qmf_deint_bfly_sse(float *v, const float *src0, const float *src1);
void ff_sbr_qmf_deint_bfly_sse2(float *v, const float *src0, const float *src1);
void ff_sbr_qmf_pre_shuffle_sse2(float *z);
void ff_sbr_hf_apply_noise_0_sse2(float (*Y)[2], const float *s_m,
const float *q_filt, int noise,
int kx, int m_max);
void ff_sbr_hf_apply_noise_1_sse2(float (*Y)[2], const float *s_m,
const float *q_filt, int noise,
int kx, int m_max);
void ff_sbr_hf_apply_noise_2_sse2(float (*Y)[2], const float *s_m,
const float *q_filt, int noise,
int kx, int m_max);
void ff_sbr_hf_apply_noise_3_sse2(float (*Y)[2], const float *s_m,
const float *q_filt, int noise,
int kx, int m_max);
void ff_sbr_qmf_deint_neg_sse(float *v, const float *src);
void ff_sbr_autocorrelate_sse (const float x[40][2], float phi[3][2][2]);
void ff_sbr_autocorrelate_sse3(const float x[40][2], float phi[3][2][2]);
av_cold void ff_sbrdsp_init_x86(SBRDSPContext *s)
{
int cpu_flags = av_get_cpu_flags();
if (EXTERNAL_SSE(cpu_flags)) {
s->neg_odd_64 = ff_sbr_neg_odd_64_sse;
s->sum_square = ff_sbr_sum_square_sse;
s->sum64x5 = ff_sbr_sum64x5_sse;
s->hf_g_filt = ff_sbr_hf_g_filt_sse;
s->hf_gen = ff_sbr_hf_gen_sse;
s->qmf_post_shuffle = ff_sbr_qmf_post_shuffle_sse;
s->qmf_deint_bfly = ff_sbr_qmf_deint_bfly_sse;
s->qmf_deint_neg = ff_sbr_qmf_deint_neg_sse;
s->autocorrelate = ff_sbr_autocorrelate_sse;
}
if (EXTERNAL_SSE2(cpu_flags)) {
s->qmf_deint_bfly = ff_sbr_qmf_deint_bfly_sse2;
s->qmf_pre_shuffle = ff_sbr_qmf_pre_shuffle_sse2;
s->hf_apply_noise[0] = ff_sbr_hf_apply_noise_0_sse2;
s->hf_apply_noise[1] = ff_sbr_hf_apply_noise_1_sse2;
s->hf_apply_noise[2] = ff_sbr_hf_apply_noise_2_sse2;
s->hf_apply_noise[3] = ff_sbr_hf_apply_noise_3_sse2;
}
if (EXTERNAL_SSE3(cpu_flags)) {
s->autocorrelate = ff_sbr_autocorrelate_sse3;
}
}

889
media/ffvpx/libavcodec/x86/simple_idct.asm Normal file
View file

@ -0,0 +1,889 @@
;
; Simple IDCT MMX
;
; Copyright (c) 2001, 2002 Michael Niedermayer <michaelni@gmx.at>
;
; Conversion from gcc syntax to x264asm syntax with minimal modifications
; by James Darnley <jdarnley@obe.tv>.
;
; This file is part of FFmpeg.
;
; FFmpeg is free software; you can redistribute it and/or
; modify it under the terms of the GNU Lesser General Public
; License as published by the Free Software Foundation; either
; version 2.1 of the License, or (at your option) any later version.
;
; FFmpeg is distributed in the hope that it will be useful,
; but WITHOUT ANY WARRANTY; without even the implied warranty of
; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
; Lesser General Public License for more details.
;
; You should have received a copy of the GNU Lesser General Public
; License along with FFmpeg; if not, write to the Free Software
; Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
;/
%include "libavutil/x86/x86util.asm"
SECTION_RODATA
cextern pb_80
wm1010: dw 0, 0xffff, 0, 0xffff
d40000: dd 4 << 16, 0
; 23170.475006
; 22725.260826
; 21406.727617
; 19265.545870
; 16384.000000
; 12872.826198
; 8866.956905
; 4520.335430
%define C0 23170 ; cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
%define C1 22725 ; cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
%define C2 21407 ; cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
%define C3 19266 ; cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
%define C4 16383 ; cos(i*M_PI/16)*sqrt(2)*(1<<14) - 0.5
%define C5 12873 ; cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
%define C6 8867 ; cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
%define C7 4520 ; cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
%define ROW_SHIFT 11
%define COL_SHIFT 20 ; 6
coeffs:
dw 1 << (ROW_SHIFT - 1), 0
dw 1 << (ROW_SHIFT - 1), 0
dw 1 << (ROW_SHIFT - 1), 1
dw 1 << (ROW_SHIFT - 1), 0
dw C4, C4, C4, C4
dw C4, -C4, C4, -C4
dw C2, C6, C2, C6
dw C6, -C2, C6, -C2
dw C1, C3, C1, C3
dw C5, C7, C5, C7
dw C3, -C7, C3, -C7
dw -C1, -C5, -C1, -C5
dw C5, -C1, C5, -C1
dw C7, C3, C7, C3
dw C7, -C5, C7, -C5
dw C3, -C1, C3, -C1
SECTION .text
%macro DC_COND_IDCT 7
movq mm0, [blockq + %1] ; R4 R0 r4 r0
movq mm1, [blockq + %2] ; R6 R2 r6 r2
movq mm2, [blockq + %3] ; R3 R1 r3 r1
movq mm3, [blockq + %4] ; R7 R5 r7 r5
movq mm4, [wm1010]
pand mm4, mm0
por mm4, mm1
por mm4, mm2
por mm4, mm3
packssdw mm4, mm4
movd t0d, mm4
or t0d, t0d
jz %%1
movq mm4, [coeffs + 16] ; C4 C4 C4 C4
pmaddwd mm4, mm0 ; C4R4+C4R0 C4r4+C4r0
movq mm5, [coeffs + 24] ; -C4 C4 -C4 C4
pmaddwd mm0, mm5 ; -C4R4+C4R0 -C4r4+C4r0
movq mm5, [coeffs + 32] ; C6 C2 C6 C2
pmaddwd mm5, mm1 ; C6R6+C2R2 C6r6+C2r2
movq mm6, [coeffs + 40] ; -C2 C6 -C2 C6
pmaddwd mm1, mm6 ; -C2R6+C6R2 -C2r6+C6r2
movq mm7, [coeffs + 48] ; C3 C1 C3 C1
pmaddwd mm7, mm2 ; C3R3+C1R1 C3r3+C1r1
paddd mm4, [coeffs + 8]
movq mm6, mm4 ; C4R4+C4R0 C4r4+C4r0
paddd mm4, mm5 ; A0 a0
psubd mm6, mm5 ; A3 a3
movq mm5, [coeffs + 56] ; C7 C5 C7 C5
pmaddwd mm5, mm3 ; C7R7+C5R5 C7r7+C5r5
paddd mm0, [coeffs + 8]
paddd mm1, mm0 ; A1 a1
paddd mm0, mm0
psubd mm0, mm1 ; A2 a2
pmaddwd mm2, [coeffs + 64] ; -C7R3+C3R1 -C7r3+C3r1
paddd mm7, mm5 ; B0 b0
movq mm5, [coeffs + 72] ; -C5 -C1 -C5 -C1
pmaddwd mm5, mm3 ; -C5R7-C1R5 -C5r7-C1r5
paddd mm7, mm4 ; A0+B0 a0+b0
paddd mm4, mm4 ; 2A0 2a0
psubd mm4, mm7 ; A0-B0 a0-b0
paddd mm5, mm2 ; B1 b1
psrad mm7, %7
psrad mm4, %7
movq mm2, mm1 ; A1 a1
paddd mm1, mm5 ; A1+B1 a1+b1
psubd mm2, mm5 ; A1-B1 a1-b1
psrad mm1, %7
psrad mm2, %7
packssdw mm7, mm1 ; A1+B1 a1+b1 A0+B0 a0+b0
packssdw mm2, mm4 ; A0-B0 a0-b0 A1-B1 a1-b1
movq [%5], mm7
movq mm1, [blockq + %3] ; R3 R1 r3 r1
movq mm4, [coeffs + 80] ; -C1 C5 -C1 C5
movq [24 + %5], mm2
pmaddwd mm4, mm1 ; -C1R3+C5R1 -C1r3+C5r1
movq mm7, [coeffs + 88] ; C3 C7 C3 C7
pmaddwd mm1, [coeffs + 96] ; -C5R3+C7R1 -C5r3+C7r1
pmaddwd mm7, mm3 ; C3R7+C7R5 C3r7+C7r5
movq mm2, mm0 ; A2 a2
pmaddwd mm3, [coeffs + 104] ; -C1R7+C3R5 -C1r7+C3r5
paddd mm4, mm7 ; B2 b2
paddd mm2, mm4 ; A2+B2 a2+b2
psubd mm0, mm4 ; a2-B2 a2-b2
psrad mm2, %7
psrad mm0, %7
movq mm4, mm6 ; A3 a3
paddd mm3, mm1 ; B3 b3
paddd mm6, mm3 ; A3+B3 a3+b3
psubd mm4, mm3 ; a3-B3 a3-b3
psrad mm6, %7
packssdw mm2, mm6 ; A3+B3 a3+b3 A2+B2 a2+b2
movq [8 + %5], mm2
psrad mm4, %7
packssdw mm4, mm0 ; A2-B2 a2-b2 A3-B3 a3-b3
movq [16 + %5], mm4
jmp %%2
%%1:
pslld mm0, 16
paddd mm0, [d40000]
psrad mm0, 13
packssdw mm0, mm0
movq [%5], mm0
movq [8 + %5], mm0
movq [16 + %5], mm0
movq [24 + %5], mm0
%%2:
%endmacro
%macro Z_COND_IDCT 8
movq mm0, [blockq + %1] ; R4 R0 r4 r0
movq mm1, [blockq + %2] ; R6 R2 r6 r2
movq mm2, [blockq + %3] ; R3 R1 r3 r1
movq mm3, [blockq + %4] ; R7 R5 r7 r5
movq mm4, mm0
por mm4, mm1
por mm4, mm2
por mm4, mm3
packssdw mm4, mm4
movd t0d, mm4
or t0d, t0d
jz %8
movq mm4, [coeffs + 16] ; C4 C4 C4 C4
pmaddwd mm4, mm0 ; C4R4+C4R0 C4r4+C4r0
movq mm5, [coeffs + 24] ; -C4 C4 -C4 C4
pmaddwd mm0, mm5 ; -C4R4+C4R0 -C4r4+C4r0
movq mm5, [coeffs + 32] ; C6 C2 C6 C2
pmaddwd mm5, mm1 ; C6R6+C2R2 C6r6+C2r2
movq mm6, [coeffs + 40] ; -C2 C6 -C2 C6
pmaddwd mm1, mm6 ; -C2R6+C6R2 -C2r6+C6r2
movq mm7, [coeffs + 48] ; C3 C1 C3 C1
pmaddwd mm7, mm2 ; C3R3+C1R1 C3r3+C1r1
paddd mm4, [coeffs]
movq mm6, mm4 ; C4R4+C4R0 C4r4+C4r0
paddd mm4, mm5 ; A0 a0
psubd mm6, mm5 ; A3 a3
movq mm5, [coeffs + 56] ; C7 C5 C7 C5
pmaddwd mm5, mm3 ; C7R7+C5R5 C7r7+C5r5
paddd mm0, [coeffs]
paddd mm1, mm0 ; A1 a1
paddd mm0, mm0
psubd mm0, mm1 ; A2 a2
pmaddwd mm2, [coeffs + 64] ; -C7R3+C3R1 -C7r3+C3r1
paddd mm7, mm5 ; B0 b0
movq mm5, [coeffs + 72] ; -C5 -C1 -C5 -C1
pmaddwd mm5, mm3 ; -C5R7-C1R5 -C5r7-C1r5
paddd mm7, mm4 ; A0+B0 a0+b0
paddd mm4, mm4 ; 2A0 2a0
psubd mm4, mm7 ; A0-B0 a0-b0
paddd mm5, mm2 ; B1 b1
psrad mm7, %7
psrad mm4, %7
movq mm2, mm1 ; A1 a1
paddd mm1, mm5 ; A1+B1 a1+b1
psubd mm2, mm5 ; A1-B1 a1-b1
psrad mm1, %7
psrad mm2, %7
packssdw mm7, mm1 ; A1+B1 a1+b1 A0+B0 a0+b0
packssdw mm2, mm4 ; A0-B0 a0-b0 A1-B1 a1-b1
movq [%5], mm7
movq mm1, [blockq + %3] ; R3 R1 r3 r1
movq mm4, [coeffs + 80] ; -C1 C5 -C1 C5
movq [24 + %5], mm2
pmaddwd mm4, mm1 ; -C1R3+C5R1 -C1r3+C5r1
movq mm7, [coeffs + 88] ; C3 C7 C3 C7
pmaddwd mm1, [coeffs + 96] ; -C5R3+C7R1 -C5r3+C7r1
pmaddwd mm7, mm3 ; C3R7+C7R5 C3r7+C7r5
movq mm2, mm0 ; A2 a2
pmaddwd mm3, [coeffs + 104] ; -C1R7+C3R5 -C1r7+C3r5
paddd mm4, mm7 ; B2 b2
paddd mm2, mm4 ; A2+B2 a2+b2
psubd mm0, mm4 ; a2-B2 a2-b2
psrad mm2, %7
psrad mm0, %7
movq mm4, mm6 ; A3 a3
paddd mm3, mm1 ; B3 b3
paddd mm6, mm3 ; A3+B3 a3+b3
psubd mm4, mm3 ; a3-B3 a3-b3
psrad mm6, %7
packssdw mm2, mm6 ; A3+B3 a3+b3 A2+B2 a2+b2
movq [8 + %5], mm2
psrad mm4, %7
packssdw mm4, mm0 ; A2-B2 a2-b2 A3-B3 a3-b3
movq [16 + %5], mm4
%endmacro
%macro IDCT1 6
movq mm0, %1 ; R4 R0 r4 r0
movq mm1, %2 ; R6 R2 r6 r2
movq mm2, %3 ; R3 R1 r3 r1
movq mm3, %4 ; R7 R5 r7 r5
movq mm4, [coeffs + 16] ; C4 C4 C4 C4
pmaddwd mm4, mm0 ; C4R4+C4R0 C4r4+C4r0
movq mm5, [coeffs + 24] ; -C4 C4 -C4 C4
pmaddwd mm0, mm5 ; -C4R4+C4R0 -C4r4+C4r0
movq mm5, [coeffs + 32] ; C6 C2 C6 C2
pmaddwd mm5, mm1 ; C6R6+C2R2 C6r6+C2r2
movq mm6, [coeffs + 40] ; -C2 C6 -C2 C6
pmaddwd mm1, mm6 ; -C2R6+C6R2 -C2r6+C6r2
movq mm6, mm4 ; C4R4+C4R0 C4r4+C4r0
movq mm7, [coeffs + 48] ; C3 C1 C3 C1
pmaddwd mm7, mm2 ; C3R3+C1R1 C3r3+C1r1
paddd mm4, mm5 ; A0 a0
psubd mm6, mm5 ; A3 a3
movq mm5, mm0 ; -C4R4+C4R0 -C4r4+C4r0
paddd mm0, mm1 ; A1 a1
psubd mm5, mm1 ; A2 a2
movq mm1, [coeffs + 56] ; C7 C5 C7 C5
pmaddwd mm1, mm3 ; C7R7+C5R5 C7r7+C5r5
pmaddwd mm2, [coeffs + 64] ; -C7R3+C3R1 -C7r3+C3r1
paddd mm7, mm1 ; B0 b0
movq mm1, [coeffs + 72] ; -C5 -C1 -C5 -C1
pmaddwd mm1, mm3 ; -C5R7-C1R5 -C5r7-C1r5
paddd mm7, mm4 ; A0+B0 a0+b0
paddd mm4, mm4 ; 2A0 2a0
psubd mm4, mm7 ; A0-B0 a0-b0
paddd mm1, mm2 ; B1 b1
psrad mm7, %6
psrad mm4, %6
movq mm2, mm0 ; A1 a1
paddd mm0, mm1 ; A1+B1 a1+b1
psubd mm2, mm1 ; A1-B1 a1-b1
psrad mm0, %6
psrad mm2, %6
packssdw mm7, mm7 ; A0+B0 a0+b0
movd [%5], mm7
packssdw mm0, mm0 ; A1+B1 a1+b1
movd [16 + %5], mm0
packssdw mm2, mm2 ; A1-B1 a1-b1
movd [96 + %5], mm2
packssdw mm4, mm4 ; A0-B0 a0-b0
movd [112 + %5], mm4
movq mm0, %3 ; R3 R1 r3 r1
movq mm4, [coeffs + 80] ; -C1 C5 -C1 C5
pmaddwd mm4, mm0 ; -C1R3+C5R1 -C1r3+C5r1
movq mm7, [coeffs + 88] ; C3 C7 C3 C7
pmaddwd mm0, [coeffs + 96] ; -C5R3+C7R1 -C5r3+C7r1
pmaddwd mm7, mm3 ; C3R7+C7R5 C3r7+C7r5
movq mm2, mm5 ; A2 a2
pmaddwd mm3, [coeffs + 104] ; -C1R7+C3R5 -C1r7+C3r5
paddd mm4, mm7 ; B2 b2
paddd mm2, mm4 ; A2+B2 a2+b2
psubd mm5, mm4 ; a2-B2 a2-b2
psrad mm2, %6
psrad mm5, %6
movq mm4, mm6 ; A3 a3
paddd mm3, mm0 ; B3 b3
paddd mm6, mm3 ; A3+B3 a3+b3
psubd mm4, mm3 ; a3-B3 a3-b3
psrad mm6, %6
psrad mm4, %6
packssdw mm2, mm2 ; A2+B2 a2+b2
packssdw mm6, mm6 ; A3+B3 a3+b3
movd [32 + %5], mm2
packssdw mm4, mm4 ; A3-B3 a3-b3
packssdw mm5, mm5 ; A2-B2 a2-b2
movd [48 + %5], mm6
movd [64 + %5], mm4
movd [80 + %5], mm5
%endmacro
%macro IDCT2 6
movq mm0, %1 ; R4 R0 r4 r0
movq mm1, %2 ; R6 R2 r6 r2
movq mm3, %4 ; R7 R5 r7 r5
movq mm4, [coeffs + 16] ; C4 C4 C4 C4
pmaddwd mm4, mm0 ; C4R4+C4R0 C4r4+C4r0
movq mm5, [coeffs + 24] ; -C4 C4 -C4 C4
pmaddwd mm0, mm5 ; -C4R4+C4R0 -C4r4+C4r0
movq mm5, [coeffs + 32] ; C6 C2 C6 C2
pmaddwd mm5, mm1 ; C6R6+C2R2 C6r6+C2r2
movq mm6, [coeffs + 40] ; -C2 C6 -C2 C6
pmaddwd mm1, mm6 ; -C2R6+C6R2 -C2r6+C6r2
movq mm6, mm4 ; C4R4+C4R0 C4r4+C4r0
paddd mm4, mm5 ; A0 a0
psubd mm6, mm5 ; A3 a3
movq mm5, mm0 ; -C4R4+C4R0 -C4r4+C4r0
paddd mm0, mm1 ; A1 a1
psubd mm5, mm1 ; A2 a2
movq mm1, [coeffs + 56] ; C7 C5 C7 C5
pmaddwd mm1, mm3 ; C7R7+C5R5 C7r7+C5r5
movq mm7, [coeffs + 72] ; -C5 -C1 -C5 -C1
pmaddwd mm7, mm3 ; -C5R7-C1R5 -C5r7-C1r5
paddd mm1, mm4 ; A0+B0 a0+b0
paddd mm4, mm4 ; 2A0 2a0
psubd mm4, mm1 ; A0-B0 a0-b0
psrad mm1, %6
psrad mm4, %6
movq mm2, mm0 ; A1 a1
paddd mm0, mm7 ; A1+B1 a1+b1
psubd mm2, mm7 ; A1-B1 a1-b1
psrad mm0, %6
psrad mm2, %6
packssdw mm1, mm1 ; A0+B0 a0+b0
movd [%5], mm1
packssdw mm0, mm0 ; A1+B1 a1+b1
movd [16 + %5], mm0
packssdw mm2, mm2 ; A1-B1 a1-b1
movd [96 + %5], mm2
packssdw mm4, mm4 ; A0-B0 a0-b0
movd [112 + %5], mm4
movq mm1, [coeffs + 88] ; C3 C7 C3 C7
pmaddwd mm1, mm3 ; C3R7+C7R5 C3r7+C7r5
movq mm2, mm5 ; A2 a2
pmaddwd mm3, [coeffs + 104] ; -C1R7+C3R5 -C1r7+C3r5
paddd mm2, mm1 ; A2+B2 a2+b2
psubd mm5, mm1 ; a2-B2 a2-b2
psrad mm2, %6
psrad mm5, %6
movq mm1, mm6 ; A3 a3
paddd mm6, mm3 ; A3+B3 a3+b3
psubd mm1, mm3 ; a3-B3 a3-b3
psrad mm6, %6
psrad mm1, %6
packssdw mm2, mm2 ; A2+B2 a2+b2
packssdw mm6, mm6 ; A3+B3 a3+b3
movd [32 + %5], mm2
packssdw mm1, mm1 ; A3-B3 a3-b3
packssdw mm5, mm5 ; A2-B2 a2-b2
movd [48 + %5], mm6
movd [64 + %5], mm1
movd [80 + %5], mm5
%endmacro
%macro IDCT3 6
movq mm0, %1 ; R4 R0 r4 r0
movq mm3, %4 ; R7 R5 r7 r5
movq mm4, [coeffs + 16] ; C4 C4 C4 C4
pmaddwd mm4, mm0 ; C4R4+C4R0 C4r4+C4r0
movq mm5, [coeffs + 24] ; -C4 C4 -C4 C4
pmaddwd mm0, mm5 ; -C4R4+C4R0 -C4r4+C4r0
movq mm6, mm4 ; C4R4+C4R0 C4r4+C4r0
movq mm5, mm0 ; -C4R4+C4R0 -C4r4+C4r0
movq mm1, [coeffs + 56] ; C7 C5 C7 C5
pmaddwd mm1, mm3 ; C7R7+C5R5 C7r7+C5r5
movq mm7, [coeffs + 72] ; -C5 -C1 -C5 -C1
pmaddwd mm7, mm3 ; -C5R7-C1R5 -C5r7-C1r5
paddd mm1, mm4 ; A0+B0 a0+b0
paddd mm4, mm4 ; 2A0 2a0
psubd mm4, mm1 ; A0-B0 a0-b0
psrad mm1, %6
psrad mm4, %6
movq mm2, mm0 ; A1 a1
paddd mm0, mm7 ; A1+B1 a1+b1
psubd mm2, mm7 ; A1-B1 a1-b1
psrad mm0, %6
psrad mm2, %6
packssdw mm1, mm1 ; A0+B0 a0+b0
movd [%5], mm1
packssdw mm0, mm0 ; A1+B1 a1+b1
movd [16 + %5], mm0
packssdw mm2, mm2 ; A1-B1 a1-b1
movd [96 + %5], mm2
packssdw mm4, mm4 ; A0-B0 a0-b0
movd [112 + %5], mm4
movq mm1, [coeffs + 88] ; C3 C7 C3 C7
pmaddwd mm1, mm3 ; C3R7+C7R5 C3r7+C7r5
movq mm2, mm5 ; A2 a2
pmaddwd mm3, [coeffs + 104] ; -C1R7+C3R5 -C1r7+C3r5
paddd mm2, mm1 ; A2+B2 a2+b2
psubd mm5, mm1 ; a2-B2 a2-b2
psrad mm2, %6
psrad mm5, %6
movq mm1, mm6 ; A3 a3
paddd mm6, mm3 ; A3+B3 a3+b3
psubd mm1, mm3 ; a3-B3 a3-b3
psrad mm6, %6
psrad mm1, %6
packssdw mm2, mm2 ; A2+B2 a2+b2
packssdw mm6, mm6 ; A3+B3 a3+b3
movd [32 + %5], mm2
packssdw mm1, mm1 ; A3-B3 a3-b3
packssdw mm5, mm5 ; A2-B2 a2-b2
movd [48 + %5], mm6
movd [64 + %5], mm1
movd [80 + %5], mm5
%endmacro
%macro IDCT4 6
movq mm0, %1 ; R4 R0 r4 r0
movq mm2, %3 ; R3 R1 r3 r1
movq mm3, %4 ; R7 R5 r7 r5
movq mm4, [coeffs + 16] ; C4 C4 C4 C4
pmaddwd mm4, mm0 ; C4R4+C4R0 C4r4+C4r0
movq mm5, [coeffs + 24] ; -C4 C4 -C4 C4
pmaddwd mm0, mm5 ; -C4R4+C4R0 -C4r4+C4r0
movq mm6, mm4 ; C4R4+C4R0 C4r4+C4r0
movq mm7, [coeffs + 48] ; C3 C1 C3 C1
pmaddwd mm7, mm2 ; C3R3+C1R1 C3r3+C1r1
movq mm5, mm0 ; -C4R4+C4R0 -C4r4+C4r0
movq mm1, [coeffs + 56] ; C7 C5 C7 C5
pmaddwd mm1, mm3 ; C7R7+C5R5 C7r7+C5r5
pmaddwd mm2, [coeffs + 64] ; -C7R3+C3R1 -C7r3+C3r1
paddd mm7, mm1 ; B0 b0
movq mm1, [coeffs + 72] ; -C5 -C1 -C5 -C1
pmaddwd mm1, mm3 ; -C5R7-C1R5 -C5r7-C1r5
paddd mm7, mm4 ; A0+B0 a0+b0
paddd mm4, mm4 ; 2A0 2a0
psubd mm4, mm7 ; A0-B0 a0-b0
paddd mm1, mm2 ; B1 b1
psrad mm7, %6
psrad mm4, %6
movq mm2, mm0 ; A1 a1
paddd mm0, mm1 ; A1+B1 a1+b1
psubd mm2, mm1 ; A1-B1 a1-b1
psrad mm0, %6
psrad mm2, %6
packssdw mm7, mm7 ; A0+B0 a0+b0
movd [%5], mm7
packssdw mm0, mm0 ; A1+B1 a1+b1
movd [16 + %5], mm0
packssdw mm2, mm2 ; A1-B1 a1-b1
movd [96 + %5], mm2
packssdw mm4, mm4 ; A0-B0 a0-b0
movd [112 + %5], mm4
movq mm0, %3 ; R3 R1 r3 r1
movq mm4, [coeffs + 80] ; -C1 C5 -C1 C5
pmaddwd mm4, mm0 ; -C1R3+C5R1 -C1r3+C5r1
movq mm7, [coeffs + 88] ; C3 C7 C3 C7
pmaddwd mm0, [coeffs + 96] ; -C5R3+C7R1 -C5r3+C7r1
pmaddwd mm7, mm3 ; C3R7+C7R5 C3r7+C7r5
movq mm2, mm5 ; A2 a2
pmaddwd mm3, [coeffs + 104] ; -C1R7+C3R5 -C1r7+C3r5
paddd mm4, mm7 ; B2 b2
paddd mm2, mm4 ; A2+B2 a2+b2
psubd mm5, mm4 ; a2-B2 a2-b2
psrad mm2, %6
psrad mm5, %6
movq mm4, mm6 ; A3 a3
paddd mm3, mm0 ; B3 b3
paddd mm6, mm3 ; A3+B3 a3+b3
psubd mm4, mm3 ; a3-B3 a3-b3
psrad mm6, %6
psrad mm4, %6
packssdw mm2, mm2 ; A2+B2 a2+b2
packssdw mm6, mm6 ; A3+B3 a3+b3
movd [32 + %5], mm2
packssdw mm4, mm4 ; A3-B3 a3-b3
packssdw mm5, mm5 ; A2-B2 a2-b2
movd [48 + %5], mm6
movd [64 + %5], mm4
movd [80 + %5], mm5
%endmacro
%macro IDCT5 6
movq mm0, %1 ; R4 R0 r4 r0
movq mm2, %3 ; R3 R1 r3 r1
movq mm4, [coeffs + 16] ; C4 C4 C4 C4
pmaddwd mm4, mm0 ; C4R4+C4R0 C4r4+C4r0
movq mm5, [coeffs + 24] ; -C4 C4 -C4 C4
pmaddwd mm0, mm5 ; -C4R4+C4R0 -C4r4+C4r0
movq mm6, mm4 ; C4R4+C4R0 C4r4+C4r0
movq mm7, [coeffs + 48] ; C3 C1 C3 C1
pmaddwd mm7, mm2 ; C3R3+C1R1 C3r3+C1r1
movq mm5, mm0 ; -C4R4+C4R0 -C4r4+C4r0
movq mm3, [coeffs + 64]
pmaddwd mm3, mm2 ; -C7R3+C3R1 -C7r3+C3r1
paddd mm7, mm4 ; A0+B0 a0+b0
paddd mm4, mm4 ; 2A0 2a0
psubd mm4, mm7 ; A0-B0 a0-b0
psrad mm7, %6
psrad mm4, %6
movq mm1, mm0 ; A1 a1
paddd mm0, mm3 ; A1+B1 a1+b1
psubd mm1, mm3 ; A1-B1 a1-b1
psrad mm0, %6
psrad mm1, %6
packssdw mm7, mm7 ; A0+B0 a0+b0
movd [%5], mm7
packssdw mm0, mm0 ; A1+B1 a1+b1
movd [16 + %5], mm0
packssdw mm1, mm1 ; A1-B1 a1-b1
movd [96 + %5], mm1
packssdw mm4, mm4 ; A0-B0 a0-b0
movd [112 + %5], mm4
movq mm4, [coeffs + 80] ; -C1 C5 -C1 C5
pmaddwd mm4, mm2 ; -C1R3+C5R1 -C1r3+C5r1
pmaddwd mm2, [coeffs + 96] ; -C5R3+C7R1 -C5r3+C7r1
movq mm1, mm5 ; A2 a2
paddd mm1, mm4 ; A2+B2 a2+b2
psubd mm5, mm4 ; a2-B2 a2-b2
psrad mm1, %6
psrad mm5, %6
movq mm4, mm6 ; A3 a3
paddd mm6, mm2 ; A3+B3 a3+b3
psubd mm4, mm2 ; a3-B3 a3-b3
psrad mm6, %6
psrad mm4, %6
packssdw mm1, mm1 ; A2+B2 a2+b2
packssdw mm6, mm6 ; A3+B3 a3+b3
movd [32 + %5], mm1
packssdw mm4, mm4 ; A3-B3 a3-b3
packssdw mm5, mm5 ; A2-B2 a2-b2
movd [48 + %5], mm6
movd [64 + %5], mm4
movd [80 + %5], mm5
%endmacro
%macro IDCT6 6
movq mm0, [%1] ; R4 R0 r4 r0
movq mm1, [%2] ; R6 R2 r6 r2
movq mm4, [coeffs + 16] ; C4 C4 C4 C4
pmaddwd mm4, mm0 ; C4R4+C4R0 C4r4+C4r0
movq mm5, [coeffs + 24] ; -C4 C4 -C4 C4
pmaddwd mm0, mm5 ; -C4R4+C4R0 -C4r4+C4r0
movq mm5, [coeffs + 32] ; C6 C2 C6 C2
pmaddwd mm5, mm1 ; C6R6+C2R2 C6r6+C2r2
movq mm6, [coeffs + 40] ; -C2 C6 -C2 C6
pmaddwd mm1, mm6 ; -C2R6+C6R2 -C2r6+C6r2
movq mm6, mm4 ; C4R4+C4R0 C4r4+C4r0
paddd mm4, mm5 ; A0 a0
psubd mm6, mm5 ; A3 a3
movq mm5, mm0 ; -C4R4+C4R0 -C4r4+C4r0
paddd mm0, mm1 ; A1 a1
psubd mm5, mm1 ; A2 a2
movq mm2, [8 + %1] ; R4 R0 r4 r0
movq mm3, [8 + %2] ; R6 R2 r6 r2
movq mm1, [coeffs + 16] ; C4 C4 C4 C4
pmaddwd mm1, mm2 ; C4R4+C4R0 C4r4+C4r0
movq mm7, [coeffs + 24] ; -C4 C4 -C4 C4
pmaddwd mm2, mm7 ; -C4R4+C4R0 -C4r4+C4r0
movq mm7, [coeffs + 32] ; C6 C2 C6 C2
pmaddwd mm7, mm3 ; C6R6+C2R2 C6r6+C2r2
pmaddwd mm3, [coeffs + 40] ; -C2R6+C6R2 -C2r6+C6r2
paddd mm7, mm1 ; A0 a0
paddd mm1, mm1 ; 2C0 2c0
psubd mm1, mm7 ; A3 a3
paddd mm3, mm2 ; A1 a1
paddd mm2, mm2 ; 2C1 2c1
psubd mm2, mm3 ; A2 a2
psrad mm4, %6
psrad mm7, %6
psrad mm3, %6
packssdw mm4, mm7 ; A0 a0
movq [%5], mm4
psrad mm0, %6
packssdw mm0, mm3 ; A1 a1
movq [16 + %5], mm0
movq [96 + %5], mm0
movq [112 + %5], mm4
psrad mm5, %6
psrad mm6, %6
psrad mm2, %6
packssdw mm5, mm2 ; A2-B2 a2-b2
movq [32 + %5], mm5
psrad mm1, %6
packssdw mm6, mm1 ; A3+B3 a3+b3
movq [48 + %5], mm6
movq [64 + %5], mm6
movq [80 + %5], mm5
%endmacro
%macro IDCT7 6
movq mm0, %1 ; R4 R0 r4 r0
movq mm1, %2 ; R6 R2 r6 r2
movq mm2, %3 ; R3 R1 r3 r1
movq mm4, [coeffs + 16] ; C4 C4 C4 C4
pmaddwd mm4, mm0 ; C4R4+C4R0 C4r4+C4r0
movq mm5, [coeffs + 24] ; -C4 C4 -C4 C4
pmaddwd mm0, mm5 ; -C4R4+C4R0 -C4r4+C4r0
movq mm5, [coeffs + 32] ; C6 C2 C6 C2
pmaddwd mm5, mm1 ; C6R6+C2R2 C6r6+C2r2
movq mm6, [coeffs + 40] ; -C2 C6 -C2 C6
pmaddwd mm1, mm6 ; -C2R6+C6R2 -C2r6+C6r2
movq mm6, mm4 ; C4R4+C4R0 C4r4+C4r0
movq mm7, [coeffs + 48] ; C3 C1 C3 C1
pmaddwd mm7, mm2 ; C3R3+C1R1 C3r3+C1r1
paddd mm4, mm5 ; A0 a0
psubd mm6, mm5 ; A3 a3
movq mm5, mm0 ; -C4R4+C4R0 -C4r4+C4r0
paddd mm0, mm1 ; A1 a1
psubd mm5, mm1 ; A2 a2
movq mm1, [coeffs + 64]
pmaddwd mm1, mm2 ; -C7R3+C3R1 -C7r3+C3r1
paddd mm7, mm4 ; A0+B0 a0+b0
paddd mm4, mm4 ; 2A0 2a0
psubd mm4, mm7 ; A0-B0 a0-b0
psrad mm7, %6
psrad mm4, %6
movq mm3, mm0 ; A1 a1
paddd mm0, mm1 ; A1+B1 a1+b1
psubd mm3, mm1 ; A1-B1 a1-b1
psrad mm0, %6
psrad mm3, %6
packssdw mm7, mm7 ; A0+B0 a0+b0
movd [%5], mm7
packssdw mm0, mm0 ; A1+B1 a1+b1
movd [16 + %5], mm0
packssdw mm3, mm3 ; A1-B1 a1-b1
movd [96 + %5], mm3
packssdw mm4, mm4 ; A0-B0 a0-b0
movd [112 + %5], mm4
movq mm4, [coeffs + 80] ; -C1 C5 -C1 C5
pmaddwd mm4, mm2 ; -C1R3+C5R1 -C1r3+C5r1
pmaddwd mm2, [coeffs + 96] ; -C5R3+C7R1 -C5r3+C7r1
movq mm3, mm5 ; A2 a2
paddd mm3, mm4 ; A2+B2 a2+b2
psubd mm5, mm4 ; a2-B2 a2-b2
psrad mm3, %6
psrad mm5, %6
movq mm4, mm6 ; A3 a3
paddd mm6, mm2 ; A3+B3 a3+b3
psubd mm4, mm2 ; a3-B3 a3-b3
psrad mm6, %6
packssdw mm3, mm3 ; A2+B2 a2+b2
movd [32 + %5], mm3
psrad mm4, %6
packssdw mm6, mm6 ; A3+B3 a3+b3
movd [48 + %5], mm6
packssdw mm4, mm4 ; A3-B3 a3-b3
packssdw mm5, mm5 ; A2-B2 a2-b2
movd [64 + %5], mm4
movd [80 + %5], mm5
%endmacro
%macro IDCT8 6
movq mm0, [%1] ; R4 R0 r4 r0
movq mm4, [coeffs + 16] ; C4 C4 C4 C4
pmaddwd mm4, mm0 ; C4R4+C4R0 C4r4+C4r0
movq mm5, [coeffs + 24] ; -C4 C4 -C4 C4
pmaddwd mm0, mm5 ; -C4R4+C4R0 -C4r4+C4r0
psrad mm4, %6
psrad mm0, %6
movq mm2, [8 + %1] ; R4 R0 r4 r0
movq mm1, [coeffs + 16] ; C4 C4 C4 C4
pmaddwd mm1, mm2 ; C4R4+C4R0 C4r4+C4r0
movq mm7, [coeffs + 24] ; -C4 C4 -C4 C4
pmaddwd mm2, mm7 ; -C4R4+C4R0 -C4r4+C4r0
movq mm7, [coeffs + 32] ; C6 C2 C6 C2
psrad mm1, %6
packssdw mm4, mm1 ; A0 a0
movq [%5], mm4
psrad mm2, %6
packssdw mm0, mm2 ; A1 a1
movq [16 + %5], mm0
movq [96 + %5], mm0
movq [112 + %5], mm4
movq [32 + %5], mm0
movq [48 + %5], mm4
movq [64 + %5], mm4
movq [80 + %5], mm0
%endmacro
%macro IDCT 0
DC_COND_IDCT 0, 8, 16, 24, rsp + 0, null, 11
Z_COND_IDCT 32, 40, 48, 56, rsp + 32, null, 11, %%4
Z_COND_IDCT 64, 72, 80, 88, rsp + 64, null, 11, %%2
Z_COND_IDCT 96, 104, 112, 120, rsp + 96, null, 11, %%1
IDCT1 [rsp + 0], [rsp + 64], [rsp + 32], [rsp + 96], blockq + 0, 20
IDCT1 [rsp + 8], [rsp + 72], [rsp + 40], [rsp + 104], blockq + 4, 20
IDCT1 [rsp + 16], [rsp + 80], [rsp + 48], [rsp + 112], blockq + 8, 20
IDCT1 [rsp + 24], [rsp + 88], [rsp + 56], [rsp + 120], blockq + 12, 20
jmp %%9
ALIGN 16
%%4:
Z_COND_IDCT 64, 72, 80, 88, rsp + 64, null, 11, %%6
Z_COND_IDCT 96, 104, 112, 120, rsp + 96, null, 11, %%5
IDCT2 [rsp + 0], [rsp + 64], [rsp + 32], [rsp + 96], blockq + 0, 20
IDCT2 [rsp + 8], [rsp + 72], [rsp + 40], [rsp + 104], blockq + 4, 20
IDCT2 [rsp + 16], [rsp + 80], [rsp + 48], [rsp + 112], blockq + 8, 20
IDCT2 [rsp + 24], [rsp + 88], [rsp + 56], [rsp + 120], blockq + 12, 20
jmp %%9
ALIGN 16
%%6:
Z_COND_IDCT 96, 104, 112, 120, rsp + 96, null, 11, %%7
IDCT3 [rsp + 0], [rsp + 64], [rsp + 32], [rsp + 96], blockq + 0, 20
IDCT3 [rsp + 8], [rsp + 72], [rsp + 40], [rsp + 104], blockq + 4, 20
IDCT3 [rsp + 16], [rsp + 80], [rsp + 48], [rsp + 112], blockq + 8, 20
IDCT3 [rsp + 24], [rsp + 88], [rsp + 56], [rsp + 120], blockq + 12, 20
jmp %%9
ALIGN 16
%%2:
Z_COND_IDCT 96, 104, 112, 120, rsp + 96, null, 11, %%3
IDCT4 [rsp + 0], [rsp + 64], [rsp + 32], [rsp + 96], blockq + 0, 20
IDCT4 [rsp + 8], [rsp + 72], [rsp + 40], [rsp + 104], blockq + 4, 20
IDCT4 [rsp + 16], [rsp + 80], [rsp + 48], [rsp + 112], blockq + 8, 20
IDCT4 [rsp + 24], [rsp + 88], [rsp + 56], [rsp + 120], blockq + 12, 20
jmp %%9
ALIGN 16
%%3:
IDCT5 [rsp + 0], [rsp + 64], [rsp + 32], [rsp + 96], blockq + 0, 20
IDCT5 [rsp + 8], [rsp + 72], [rsp + 40], [rsp + 104], blockq + 4, 20
IDCT5 [rsp + 16], [rsp + 80], [rsp + 48], [rsp + 112], blockq + 8, 20
IDCT5 [rsp + 24], [rsp + 88], [rsp + 56], [rsp + 120], blockq + 12, 20
jmp %%9
ALIGN 16
%%5:
IDCT6 rsp + 0, rsp + 64, rsp + 32, rsp + 96, blockq + 0, 20
IDCT6 rsp + 16, rsp + 80, rsp + 48, rsp + 112, blockq + 8, 20
jmp %%9
ALIGN 16
%%1:
IDCT7 [rsp + 0], [rsp + 64], [rsp + 32], [rsp + 96], blockq + 0, 20
IDCT7 [rsp + 8], [rsp + 72], [rsp + 40], [rsp + 104], blockq + 4, 20
IDCT7 [rsp + 16], [rsp + 80], [rsp + 48], [rsp + 112], blockq + 8, 20
IDCT7 [rsp + 24], [rsp + 88], [rsp + 56], [rsp + 120], blockq + 12, 20
jmp %%9
ALIGN 16
%%7:
IDCT8 rsp + 0, rsp + 64, rsp + 32, rsp + 96, blockq + 0, 20
IDCT8 rsp + 16, rsp + 80, rsp + 48, rsp + 112, blockq + 8, 20
%%9:
%endmacro
%macro PUT_PIXELS_CLAMPED_HALF 1
mova m0, [blockq+mmsize*0+%1]
mova m1, [blockq+mmsize*2+%1]
%if mmsize == 8
mova m2, [blockq+mmsize*4+%1]
mova m3, [blockq+mmsize*6+%1]
%endif
packuswb m0, [blockq+mmsize*1+%1]
packuswb m1, [blockq+mmsize*3+%1]
%if mmsize == 8
packuswb m2, [blockq+mmsize*5+%1]
packuswb m3, [blockq+mmsize*7+%1]
movq [pixelsq], m0
movq [lsizeq+pixelsq], m1
movq [2*lsizeq+pixelsq], m2
movq [lsize3q+pixelsq], m3
%else
movq [pixelsq], m0
movhps [lsizeq+pixelsq], m0
movq [2*lsizeq+pixelsq], m1
movhps [lsize3q+pixelsq], m1
%endif
%endmacro
%macro ADD_PIXELS_CLAMPED 1
mova m0, [blockq+mmsize*0+%1]
mova m1, [blockq+mmsize*1+%1]
%if mmsize == 8
mova m5, [blockq+mmsize*2+%1]
mova m6, [blockq+mmsize*3+%1]
%endif
movq m2, [pixelsq]
movq m3, [pixelsq+lsizeq]
%if mmsize == 8
mova m7, m2
punpcklbw m2, m4
punpckhbw m7, m4
paddsw m0, m2
paddsw m1, m7
mova m7, m3
punpcklbw m3, m4
punpckhbw m7, m4
paddsw m5, m3
paddsw m6, m7
%else
punpcklbw m2, m4
punpcklbw m3, m4
paddsw m0, m2
paddsw m1, m3
%endif
packuswb m0, m1
%if mmsize == 8
packuswb m5, m6
movq [pixelsq], m0
movq [pixelsq+lsizeq], m5
%else
movq [pixelsq], m0
movhps [pixelsq+lsizeq], m0
%endif
%endmacro
INIT_MMX mmx
cglobal simple_idct, 1, 2, 8, 128, block, t0
IDCT
RET
cglobal simple_idct_put, 3, 5, 8, 128, pixels, lsize, block, lsize3, t0
IDCT
lea lsize3q, [lsizeq*3]
PUT_PIXELS_CLAMPED_HALF 0
lea pixelsq, [pixelsq+lsizeq*4]
PUT_PIXELS_CLAMPED_HALF 64
RET
cglobal simple_idct_add, 3, 4, 8, 128, pixels, lsize, block, t0
IDCT
pxor m4, m4
ADD_PIXELS_CLAMPED 0
lea pixelsq, [pixelsq+lsizeq*2]
ADD_PIXELS_CLAMPED 32
lea pixelsq, [pixelsq+lsizeq*2]
ADD_PIXELS_CLAMPED 64
lea pixelsq, [pixelsq+lsizeq*2]
ADD_PIXELS_CLAMPED 96
RET
INIT_XMM sse2
cglobal simple_idct_put, 3, 5, 8, 128, pixels, lsize, block, lsize3, t0
IDCT
lea lsize3q, [lsizeq*3]
PUT_PIXELS_CLAMPED_HALF 0
lea pixelsq, [pixelsq+lsizeq*4]
PUT_PIXELS_CLAMPED_HALF 64
RET
cglobal simple_idct_add, 3, 4, 8, 128, pixels, lsize, block, t0
IDCT
pxor m4, m4
ADD_PIXELS_CLAMPED 0
lea pixelsq, [pixelsq+lsizeq*2]
ADD_PIXELS_CLAMPED 32
lea pixelsq, [pixelsq+lsizeq*2]
ADD_PIXELS_CLAMPED 64
lea pixelsq, [pixelsq+lsizeq*2]
ADD_PIXELS_CLAMPED 96
RET

53
media/ffvpx/libavcodec/x86/simple_idct.h Normal file
View file

@ -0,0 +1,53 @@
/*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#ifndef AVCODEC_X86_SIMPLE_IDCT_H
#define AVCODEC_X86_SIMPLE_IDCT_H
#include <stddef.h>
#include <stdint.h>
void ff_simple_idct_mmx(int16_t *block);
void ff_simple_idct_add_mmx(uint8_t *dest, ptrdiff_t line_size, int16_t *block);
void ff_simple_idct_put_mmx(uint8_t *dest, ptrdiff_t line_size, int16_t *block);
void ff_simple_idct_add_sse2(uint8_t *dest, ptrdiff_t line_size, int16_t *block);
void ff_simple_idct_put_sse2(uint8_t *dest, ptrdiff_t line_size, int16_t *block);
void ff_simple_idct8_sse2(int16_t *block);
void ff_simple_idct8_avx(int16_t *block);
void ff_simple_idct8_put_sse2(uint8_t *dest, ptrdiff_t line_size, int16_t *block);
void ff_simple_idct8_put_avx(uint8_t *dest, ptrdiff_t line_size, int16_t *block);
void ff_simple_idct8_add_sse2(uint8_t *dest, ptrdiff_t line_size, int16_t *block);
void ff_simple_idct8_add_avx(uint8_t *dest, ptrdiff_t line_size, int16_t *block);
void ff_simple_idct10_sse2(int16_t *block);
void ff_simple_idct10_avx(int16_t *block);
void ff_simple_idct10_put_sse2(uint8_t *dest, ptrdiff_t line_size, int16_t *block);
void ff_simple_idct10_put_avx(uint8_t *dest, ptrdiff_t line_size, int16_t *block);
void ff_simple_idct12_sse2(int16_t *block);
void ff_simple_idct12_avx(int16_t *block);
void ff_simple_idct12_put_sse2(uint8_t *dest, ptrdiff_t line_size, int16_t *block);
void ff_simple_idct12_put_avx(uint8_t *dest, ptrdiff_t line_size, int16_t *block);
#endif /* AVCODEC_X86_SIMPLE_IDCT_H */

205
media/ffvpx/libavcodec/x86/simple_idct10.asm Normal file
View file

@ -0,0 +1,205 @@
;******************************************************************************
;* x86-SIMD-optimized IDCT for prores
;* this is identical to "simple" IDCT written by Michael Niedermayer
;* except for the clip range
;*
;* Copyright (c) 2011 Ronald S. Bultje <rsbultje@gmail.com>
;* Copyright (c) 2015 Christophe Gisquet
;*
;* This file is part of FFmpeg.
;*
;* FFmpeg is free software; you can redistribute it and/or
;* modify it under the terms of the GNU Lesser General Public
;* License as published by the Free Software Foundation; either
;* version 2.1 of the License, or (at your option) any later version.
;*
;* FFmpeg is distributed in the hope that it will be useful,
;* but WITHOUT ANY WARRANTY; without even the implied warranty of
;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
;* Lesser General Public License for more details.
;*
;* You should have received a copy of the GNU Lesser General Public
;* License along with FFmpeg; if not, write to the Free Software
;* 51, Inc., Foundation Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
;******************************************************************************
%include "libavutil/x86/x86util.asm"
%if ARCH_X86_64
SECTION_RODATA
cextern pw_2
cextern pw_16
cextern pw_32
cextern pw_1023
cextern pw_4095
pd_round_11: times 4 dd 1<<(11-1)
pd_round_12: times 4 dd 1<<(12-1)
pd_round_15: times 4 dd 1<<(15-1)
pd_round_19: times 4 dd 1<<(19-1)
pd_round_20: times 4 dd 1<<(20-1)
%macro CONST_DEC 3
const %1
times 4 dw %2, %3
%endmacro
%define W1sh2 22725 ; W1 = 90901 = 22725<<2 + 1
%define W2sh2 21407 ; W2 = 85627 = 21407<<2 - 1
%define W3sh2 19265 ; W3 = 77062 = 19265<<2 + 2
%define W4sh2 16384 ; W4 = 65535 = 16384<<2 - 1
%define W3sh2_lo 19266
%define W4sh2_lo 16383
%define W5sh2 12873 ; W5 = 51491 = 12873<<2 - 1
%define W6sh2 8867 ; W6 = 35468 = 8867<<2
%define W7sh2 4520 ; W7 = 18081 = 4520<<2 + 1
CONST_DEC w4_plus_w2_hi, W4sh2, +W2sh2
CONST_DEC w4_min_w2_hi, W4sh2, -W2sh2
CONST_DEC w4_plus_w6_hi, W4sh2, +W6sh2
CONST_DEC w4_min_w6_hi, W4sh2, -W6sh2
CONST_DEC w1_plus_w3_hi, W1sh2, +W3sh2
CONST_DEC w3_min_w1_hi, W3sh2, -W1sh2
CONST_DEC w7_plus_w3_hi, W7sh2, +W3sh2
CONST_DEC w3_min_w7_hi, W3sh2, -W7sh2
CONST_DEC w1_plus_w5, W1sh2, +W5sh2
CONST_DEC w5_min_w1, W5sh2, -W1sh2
CONST_DEC w5_plus_w7, W5sh2, +W7sh2
CONST_DEC w7_min_w5, W7sh2, -W5sh2
CONST_DEC w4_plus_w2_lo, W4sh2_lo, +W2sh2
CONST_DEC w4_min_w2_lo, W4sh2_lo, -W2sh2
CONST_DEC w4_plus_w6_lo, W4sh2_lo, +W6sh2
CONST_DEC w4_min_w6_lo, W4sh2_lo, -W6sh2
CONST_DEC w1_plus_w3_lo, W1sh2, +W3sh2_lo
CONST_DEC w3_min_w1_lo, W3sh2_lo, -W1sh2
CONST_DEC w7_plus_w3_lo, W7sh2, +W3sh2_lo
CONST_DEC w3_min_w7_lo, W3sh2_lo, -W7sh2
%include "libavcodec/x86/simple_idct10_template.asm"
SECTION .text
%macro STORE_HI_LO 12
movq %1, %9
movq %3, %10
movq %5, %11
movq %7, %12
movhps %2, %9
movhps %4, %10
movhps %6, %11
movhps %8, %12
%endmacro
%macro LOAD_ZXBW_8 16
pmovzxbw %1, %9
pmovzxbw %2, %10
pmovzxbw %3, %11
pmovzxbw %4, %12
pmovzxbw %5, %13
pmovzxbw %6, %14
pmovzxbw %7, %15
pmovzxbw %8, %16
%endmacro
%macro LOAD_ZXBW_4 9
movh %1, %5
movh %2, %6
movh %3, %7
movh %4, %8
punpcklbw %1, %9
punpcklbw %2, %9
punpcklbw %3, %9
punpcklbw %4, %9
%endmacro
%define PASS4ROWS(base, stride, stride3) \
[base], [base + stride], [base + 2*stride], [base + stride3]
%macro idct_fn 0
define_constants _lo
cglobal simple_idct8, 1, 1, 16, 32, block
IDCT_FN "", 11, pw_32, 20, "store"
RET
cglobal simple_idct8_put, 3, 4, 16, 32, pixels, lsize, block
IDCT_FN "", 11, pw_32, 20
lea r3, [3*lsizeq]
lea r2, [pixelsq + r3]
packuswb m8, m0
packuswb m1, m2
packuswb m4, m11
packuswb m9, m10
STORE_HI_LO PASS8ROWS(pixelsq, r2, lsizeq, r3), m8, m1, m4, m9
RET
cglobal simple_idct8_add, 3, 4, 16, 32, pixels, lsize, block
IDCT_FN "", 11, pw_32, 20
lea r2, [3*lsizeq]
%if cpuflag(sse4)
lea r3, [pixelsq + r2]
LOAD_ZXBW_8 m3, m5, m6, m7, m12, m13, m14, m15, PASS8ROWS(pixelsq, r3, lsizeq, r2)
paddsw m8, m3
paddsw m0, m5
paddsw m1, m6
paddsw m2, m7
paddsw m4, m12
paddsw m11, m13
paddsw m9, m14
paddsw m10, m15
%else
pxor m12, m12
LOAD_ZXBW_4 m3, m5, m6, m7, PASS4ROWS(pixelsq, lsizeq, r2), m12
paddsw m8, m3
paddsw m0, m5
paddsw m1, m6
paddsw m2, m7
lea r3, [pixelsq + 4*lsizeq]
LOAD_ZXBW_4 m3, m5, m6, m7, PASS4ROWS(r3, lsizeq, r2), m12
paddsw m4, m3
paddsw m11, m5
paddsw m9, m6
paddsw m10, m7
lea r3, [pixelsq + r2]
%endif
packuswb m8, m0
packuswb m1, m2
packuswb m4, m11
packuswb m9, m10
STORE_HI_LO PASS8ROWS(pixelsq, r3, lsizeq, r2), m8, m1, m4, m9
RET
define_constants _hi
cglobal simple_idct10, 1, 1, 16, block
IDCT_FN "", 12, "", 19, "store"
RET
cglobal simple_idct10_put, 3, 3, 16, pixels, lsize, block
IDCT_FN "", 12, "", 19, "put", 0, pw_1023
RET
cglobal simple_idct12, 1, 1, 16, block
; coeffs are already 15bits, adding the offset would cause
; overflow in the input
IDCT_FN "", 15, pw_2, 16, "store"
RET
cglobal simple_idct12_put, 3, 3, 16, pixels, lsize, block
; range isn't known, so the C simple_idct range is used
; Also, using a bias on input overflows, so use the bias
; on output of the first butterfly instead
IDCT_FN "", 15, pw_2, 16, "put", 0, pw_4095
RET
%endmacro
INIT_XMM sse2
idct_fn
%if HAVE_AVX_EXTERNAL
INIT_XMM avx
idct_fn
%endif
%endif

369
media/ffvpx/libavcodec/x86/simple_idct10_template.asm Normal file
View file

@ -0,0 +1,369 @@
;******************************************************************************
;* x86-SIMD-optimized IDCT for prores
;* this is identical to "simple" IDCT written by Michael Niedermayer
;* except for the clip range
;*
;* Copyright (c) 2011 Ronald S. Bultje <rsbultje@gmail.com>
;*
;* This file is part of FFmpeg.
;*
;* FFmpeg is free software; you can redistribute it and/or
;* modify it under the terms of the GNU Lesser General Public
;* License as published by the Free Software Foundation; either
;* version 2.1 of the License, or (at your option) any later version.
;*
;* FFmpeg is distributed in the hope that it will be useful,
;* but WITHOUT ANY WARRANTY; without even the implied warranty of
;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
;* Lesser General Public License for more details.
;*
;* You should have received a copy of the GNU Lesser General Public
;* License along with FFmpeg; if not, write to the Free Software
;* 51, Inc., Foundation Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
;******************************************************************************
; add SECTION_RODATA and proper include before including this file!
%if ARCH_X86_64
%macro define_constants 1
%undef w4_plus_w2
%undef w4_min_w2
%undef w4_plus_w6
%undef w4_min_w6
%undef w1_plus_w3
%undef w3_min_w1
%undef w7_plus_w3
%undef w3_min_w7
%define w4_plus_w2 w4_plus_w2%1
%define w4_min_w2 w4_min_w2%1
%define w4_plus_w6 w4_plus_w6%1
%define w4_min_w6 w4_min_w6%1
%define w1_plus_w3 w1_plus_w3%1
%define w3_min_w1 w3_min_w1%1
%define w7_plus_w3 w7_plus_w3%1
%define w3_min_w7 w3_min_w7%1
%endmacro
; interleave data while maintaining source
; %1=type, %2=dstlo, %3=dsthi, %4=src, %5=interleave
%macro SBUTTERFLY3 5
punpckl%1 m%2, m%4, m%5
punpckh%1 m%3, m%4, m%5
%endmacro
; %1/%2=src1/dst1, %3/%4=dst2, %5/%6=src2, %7=shift
; action: %3/%4 = %1/%2 - %5/%6; %1/%2 += %5/%6
; %1/%2/%3/%4 >>= %7; dword -> word (in %1/%3)
%macro SUMSUB_SHPK 7
psubd %3, %1, %5 ; { a0 - b0 }[0-3]
psubd %4, %2, %6 ; { a0 - b0 }[4-7]
paddd %1, %5 ; { a0 + b0 }[0-3]
paddd %2, %6 ; { a0 + b0 }[4-7]
psrad %1, %7
psrad %2, %7
psrad %3, %7
psrad %4, %7
packssdw %1, %2 ; row[0]
packssdw %3, %4 ; row[7]
%endmacro
; %1 = initial bias ("" if nop)
; %2 = number of bits to shift at the end
; %3 = qmat (for prores)
%macro IDCT_1D 2-3
; a0 = (W4 * row[0]) + (1 << (15 - 1));
; a1 = a0;
; a2 = a0;
; a3 = a0;
; a0 += W2 * row[2];
; a1 += W6 * row[2];
; a2 -= W6 * row[2];
; a3 -= W2 * row[2];
%ifstr %1
mova m15, [pd_round_ %+ %2]
%else
paddw m10, [%1]
%endif
SBUTTERFLY3 wd, 0, 1, 10, 8 ; { row[0], row[2] }[0-3]/[4-7]
pmaddwd m2, m0, [w4_plus_w6]
pmaddwd m3, m1, [w4_plus_w6]
pmaddwd m4, m0, [w4_min_w6]
pmaddwd m5, m1, [w4_min_w6]
pmaddwd m6, m0, [w4_min_w2]
pmaddwd m7, m1, [w4_min_w2]
pmaddwd m0, [w4_plus_w2]
pmaddwd m1, [w4_plus_w2]
%ifstr %1
; Adding 1<<(%2-1) for >=15 bits values
paddd m2, m15
paddd m3, m15
paddd m4, m15
paddd m5, m15
paddd m6, m15
paddd m7, m15
paddd m0, m15
paddd m1, m15
%endif
; a0: -1*row[0]-1*row[2]
; a1: -1*row[0]
; a2: -1*row[0]
; a3: -1*row[0]+1*row[2]
; a0 += W4*row[4] + W6*row[6]; i.e. -1*row[4]
; a1 -= W4*row[4] + W2*row[6]; i.e. -1*row[4]-1*row[6]
; a2 -= W4*row[4] - W2*row[6]; i.e. -1*row[4]+1*row[6]
; a3 += W4*row[4] - W6*row[6]; i.e. -1*row[4]
SBUTTERFLY3 wd, 8, 9, 13, 12 ; { row[4], row[6] }[0-3]/[4-7]
pmaddwd m10, m8, [w4_plus_w6]
pmaddwd m11, m9, [w4_plus_w6]
paddd m0, m10 ; a0[0-3]
paddd m1, m11 ; a0[4-7]
pmaddwd m10, m8, [w4_min_w6]
pmaddwd m11, m9, [w4_min_w6]
paddd m6, m10 ; a3[0-3]
paddd m7, m11 ; a3[4-7]
pmaddwd m10, m8, [w4_min_w2]
pmaddwd m11, m9, [w4_min_w2]
pmaddwd m8, [w4_plus_w2]
pmaddwd m9, [w4_plus_w2]
psubd m4, m10 ; a2[0-3] intermediate
psubd m5, m11 ; a2[4-7] intermediate
psubd m2, m8 ; a1[0-3] intermediate
psubd m3, m9 ; a1[4-7] intermediate
; load/store
mova [blockq+ 0], m0
mova [blockq+ 32], m2
mova [blockq+ 64], m4
mova [blockq+ 96], m6
mova m10,[blockq+ 16] ; { row[1] }[0-7]
mova m8, [blockq+ 48] ; { row[3] }[0-7]
mova m13,[blockq+ 80] ; { row[5] }[0-7]
mova m14,[blockq+112] ; { row[7] }[0-7]
mova [blockq+ 16], m1
mova [blockq+ 48], m3
mova [blockq+ 80], m5
mova [blockq+112], m7
%if %0 == 3
pmullw m10,[%3+ 16]
pmullw m8, [%3+ 48]
pmullw m13,[%3+ 80]
pmullw m14,[%3+112]
%endif
; b0 = MUL(W1, row[1]);
; MAC(b0, W3, row[3]);
; b1 = MUL(W3, row[1]);
; MAC(b1, -W7, row[3]);
; b2 = MUL(W5, row[1]);
; MAC(b2, -W1, row[3]);
; b3 = MUL(W7, row[1]);
; MAC(b3, -W5, row[3]);
SBUTTERFLY3 wd, 0, 1, 10, 8 ; { row[1], row[3] }[0-3]/[4-7]
pmaddwd m2, m0, [w3_min_w7]
pmaddwd m3, m1, [w3_min_w7]
pmaddwd m4, m0, [w5_min_w1]
pmaddwd m5, m1, [w5_min_w1]
pmaddwd m6, m0, [w7_min_w5]
pmaddwd m7, m1, [w7_min_w5]
pmaddwd m0, [w1_plus_w3]
pmaddwd m1, [w1_plus_w3]
; b0: +1*row[1]+2*row[3]
; b1: +2*row[1]-1*row[3]
; b2: -1*row[1]-1*row[3]
; b3: +1*row[1]+1*row[3]
; MAC(b0, W5, row[5]);
; MAC(b0, W7, row[7]);
; MAC(b1, -W1, row[5]);
; MAC(b1, -W5, row[7]);
; MAC(b2, W7, row[5]);
; MAC(b2, W3, row[7]);
; MAC(b3, W3, row[5]);
; MAC(b3, -W1, row[7]);
SBUTTERFLY3 wd, 8, 9, 13, 14 ; { row[5], row[7] }[0-3]/[4-7]
; b0: -1*row[5]+1*row[7]
; b1: -1*row[5]+1*row[7]
; b2: +1*row[5]+2*row[7]
; b3: +2*row[5]-1*row[7]
pmaddwd m10, m8, [w1_plus_w5]
pmaddwd m11, m9, [w1_plus_w5]
pmaddwd m12, m8, [w5_plus_w7]
pmaddwd m13, m9, [w5_plus_w7]
psubd m2, m10 ; b1[0-3]
psubd m3, m11 ; b1[4-7]
paddd m0, m12 ; b0[0-3]
paddd m1, m13 ; b0[4-7]
pmaddwd m12, m8, [w7_plus_w3]
pmaddwd m13, m9, [w7_plus_w3]
pmaddwd m8, [w3_min_w1]
pmaddwd m9, [w3_min_w1]
paddd m4, m12 ; b2[0-3]
paddd m5, m13 ; b2[4-7]
paddd m6, m8 ; b3[0-3]
paddd m7, m9 ; b3[4-7]
; row[0] = (a0 + b0) >> 15;
; row[7] = (a0 - b0) >> 15;
; row[1] = (a1 + b1) >> 15;
; row[6] = (a1 - b1) >> 15;
; row[2] = (a2 + b2) >> 15;
; row[5] = (a2 - b2) >> 15;
; row[3] = (a3 + b3) >> 15;
; row[4] = (a3 - b3) >> 15;
mova m8, [blockq+ 0] ; a0[0-3]
mova m9, [blockq+16] ; a0[4-7]
SUMSUB_SHPK m8, m9, m10, m11, m0, m1, %2
mova m0, [blockq+32] ; a1[0-3]
mova m1, [blockq+48] ; a1[4-7]
SUMSUB_SHPK m0, m1, m9, m11, m2, m3, %2
mova m1, [blockq+64] ; a2[0-3]
mova m2, [blockq+80] ; a2[4-7]
SUMSUB_SHPK m1, m2, m11, m3, m4, m5, %2
mova m2, [blockq+96] ; a3[0-3]
mova m3, [blockq+112] ; a3[4-7]
SUMSUB_SHPK m2, m3, m4, m5, m6, m7, %2
%endmacro
; void ff_prores_idct_put_10_<opt>(uint8_t *pixels, ptrdiff_t stride,
; int16_t *block, const int16_t *qmat);
; %1 = row shift
; %2 = row bias macro
; %3 = column shift
; %4 = column bias macro
; %5 = final action (nothing, "store", "put", "add")
; %6 = min pixel value
; %7 = max pixel value
; %8 = qmat (for prores)
%macro IDCT_FN 4-8
; for (i = 0; i < 8; i++)
; idctRowCondDC(block + i*8);
mova m10,[blockq+ 0] ; { row[0] }[0-7]
mova m8, [blockq+32] ; { row[2] }[0-7]
mova m13,[blockq+64] ; { row[4] }[0-7]
mova m12,[blockq+96] ; { row[6] }[0-7]
%if %0 == 8
pmullw m10,[%8+ 0]
pmullw m8, [%8+32]
pmullw m13,[%8+64]
pmullw m12,[%8+96]
IDCT_1D %1, %2, %8
%elif %2 == 11
; This copies the DC-only shortcut. When there is only a DC coefficient the
; C shifts the value and splats it to all coeffs rather than multiplying and
; doing the full IDCT. This causes a difference on 8-bit because the
; coefficient is 16383 rather than 16384 (which you can get with shifting).
por m1, m8, m13
por m1, m12
por m1, [blockq+ 16] ; { row[1] }[0-7]
por m1, [blockq+ 48] ; { row[3] }[0-7]
por m1, [blockq+ 80] ; { row[5] }[0-7]
por m1, [blockq+112] ; { row[7] }[0-7]
pxor m2, m2
pcmpeqw m1, m2
psllw m2, m10, 3
pand m2, m1
pcmpeqb m3, m3
pxor m1, m3
mova [rsp], m1
mova [rsp+16], m2
IDCT_1D %1, %2
mova m5, [rsp]
mova m6, [rsp+16]
pand m8, m5
por m8, m6
pand m0, m5
por m0, m6
pand m1, m5
por m1, m6
pand m2, m5
por m2, m6
pand m4, m5
por m4, m6
pand m11, m5
por m11, m6
pand m9, m5
por m9, m6
pand m10, m5
por m10, m6
%else
IDCT_1D %1, %2
%endif
; transpose for second part of IDCT
TRANSPOSE8x8W 8, 0, 1, 2, 4, 11, 9, 10, 3
mova [blockq+ 16], m0
mova [blockq+ 48], m2
mova [blockq+ 80], m11
mova [blockq+112], m10
SWAP 8, 10
SWAP 1, 8
SWAP 4, 13
SWAP 9, 12
; for (i = 0; i < 8; i++)
; idctSparseColAdd(dest + i, line_size, block + i);
IDCT_1D %3, %4
; clip/store
%if %0 >= 5
%ifidn %5,"store"
; No clamping, means pure idct
mova [blockq+ 0], m8
mova [blockq+ 16], m0
mova [blockq+ 32], m1
mova [blockq+ 48], m2
mova [blockq+ 64], m4
mova [blockq+ 80], m11
mova [blockq+ 96], m9
mova [blockq+112], m10
%elifidn %5,"put"
%ifidn %6, 0
pxor m3, m3
%else
mova m3, [%6]
%endif ; ifidn %6, 0
mova m5, [%7]
pmaxsw m8, m3
pmaxsw m0, m3
pmaxsw m1, m3
pmaxsw m2, m3
pmaxsw m4, m3
pmaxsw m11, m3
pmaxsw m9, m3
pmaxsw m10, m3
pminsw m8, m5
pminsw m0, m5
pminsw m1, m5
pminsw m2, m5
pminsw m4, m5
pminsw m11, m5
pminsw m9, m5
pminsw m10, m5
lea r2, [r1*3]
mova [r0 ], m8
mova [r0+r1 ], m0
mova [r0+r1*2], m1
mova [r0+r2 ], m2
lea r0, [r0+r1*4]
mova [r0 ], m4
mova [r0+r1 ], m11
mova [r0+r1*2], m9
mova [r0+r2 ], m10
%endif ; %5 action
%endif; if %0 >= 5
%endmacro
%endif