wuggy/Dactyloidae
wuggy/Dactyloidae
1
0
Fork 0
mirror of https://repo.dactyloidae.xyz/Dactyloidae/UXP.git synced 2026-10-27 01:27:30 +09:00
Code Issues Projects Releases Packages Wiki Activity

aom: update libaom to 0ec86ac7ae1e32a7e70410fa4972a655ec3670a4

Browse source
This commit is contained in:
Roy Tam 2019-02-22 21:52:01 +08:00
commit eb361970c5
438 changed files with 52661 additions and 21905 deletions
Download patch file Download diff file Expand all files Collapse all files

21
third_party/aom/aom_dsp/x86/aom_asm_stubs.c vendored
View file

@ -21,13 +21,13 @@ filter8_1dfunction aom_filter_block1d8_v8_sse2;
filter8_1dfunction aom_filter_block1d8_h8_sse2;
filter8_1dfunction aom_filter_block1d4_v8_sse2;
filter8_1dfunction aom_filter_block1d4_h8_sse2;
filter8_1dfunction aom_filter_block1d16_v4_sse2;
filter8_1dfunction aom_filter_block1d16_h4_sse2;
#define aom_filter_block1d16_h4_sse2 aom_filter_block1d16_h8_sse2
#define aom_filter_block1d16_v4_sse2 aom_filter_block1d16_v8_sse2
#define aom_filter_block1d8_h4_sse2 aom_filter_block1d8_h8_sse2
#define aom_filter_block1d8_v4_sse2 aom_filter_block1d8_v8_sse2
#define aom_filter_block1d4_h4_sse2 aom_filter_block1d4_h8_sse2
#define aom_filter_block1d4_v4_sse2 aom_filter_block1d4_v8_sse2
filter8_1dfunction aom_filter_block1d8_h4_sse2;
filter8_1dfunction aom_filter_block1d8_v4_sse2;
filter8_1dfunction aom_filter_block1d4_h4_sse2;
filter8_1dfunction aom_filter_block1d4_v4_sse2;
filter8_1dfunction aom_filter_block1d16_v2_sse2;
filter8_1dfunction aom_filter_block1d16_h2_sse2;
@ -49,7 +49,6 @@ filter8_1dfunction aom_filter_block1d4_h2_sse2;
FUN_CONV_1D(horiz, x_step_q4, filter_x, h, src, , sse2);
FUN_CONV_1D(vert, y_step_q4, filter_y, v, src - src_stride * 3, , sse2);
#if ARCH_X86_64
highbd_filter8_1dfunction aom_highbd_filter_block1d16_v8_sse2;
highbd_filter8_1dfunction aom_highbd_filter_block1d16_h8_sse2;
highbd_filter8_1dfunction aom_highbd_filter_block1d8_v8_sse2;
@ -57,6 +56,13 @@ highbd_filter8_1dfunction aom_highbd_filter_block1d8_h8_sse2;
highbd_filter8_1dfunction aom_highbd_filter_block1d4_v8_sse2;
highbd_filter8_1dfunction aom_highbd_filter_block1d4_h8_sse2;
highbd_filter8_1dfunction aom_highbd_filter_block1d16_v4_sse2;
highbd_filter8_1dfunction aom_highbd_filter_block1d16_h4_sse2;
highbd_filter8_1dfunction aom_highbd_filter_block1d8_v4_sse2;
highbd_filter8_1dfunction aom_highbd_filter_block1d8_h4_sse2;
highbd_filter8_1dfunction aom_highbd_filter_block1d4_v4_sse2;
highbd_filter8_1dfunction aom_highbd_filter_block1d4_h4_sse2;
highbd_filter8_1dfunction aom_highbd_filter_block1d16_v2_sse2;
highbd_filter8_1dfunction aom_highbd_filter_block1d16_h2_sse2;
highbd_filter8_1dfunction aom_highbd_filter_block1d8_v2_sse2;
@ -85,5 +91,4 @@ highbd_filter8_1dfunction aom_highbd_filter_block1d4_h2_sse2;
HIGH_FUN_CONV_1D(horiz, x_step_q4, filter_x, h, src, , sse2);
HIGH_FUN_CONV_1D(vert, y_step_q4, filter_y, v, src - src_stride * 3, , sse2);
#endif // ARCH_X86_64
#endif // HAVE_SSE2

75
third_party/aom/aom_dsp/x86/aom_high_subpixel_bilinear_sse2.asm vendored
View file

@ -67,7 +67,6 @@
dec rcx
%endm
%if ARCH_X86_64
%macro HIGH_GET_PARAM 0
mov rdx, arg(5) ;filter ptr
mov rsi, arg(0) ;src_ptr
@ -86,14 +85,17 @@
mov rdx, 0x00010001
movsxd rcx, DWORD PTR arg(6) ;bps
movq xmm8, rdx
movq xmm3, rdx
movq xmm5, rcx
pshufd xmm8, xmm8, 0b
movdqa xmm1, xmm8
psllw xmm8, xmm5
psubw xmm8, xmm1 ;max value (for clamping)
pshufd xmm3, xmm3, 0b
movdqa xmm1, xmm3
psllw xmm3, xmm5
psubw xmm3, xmm1 ;max value (for clamping)
pxor xmm5, xmm5 ;min value (for clamping)
movdqa max, xmm3
movdqa min, xmm5
movsxd rax, DWORD PTR arg(1) ;pixels_per_line
movsxd rdx, DWORD PTR arg(3) ;out_pitch
movsxd rcx, DWORD PTR arg(4) ;output_height
@ -113,8 +115,8 @@
packssdw xmm0, xmm6 ;pack back to word
;clamp the values
pminsw xmm0, xmm8
pmaxsw xmm0, xmm5
pminsw xmm0, max
pmaxsw xmm0, min
%if %1
movdqu xmm1, [rdi]
@ -128,36 +130,36 @@
%endm
%macro HIGH_APPLY_FILTER_16 1
movdqa xmm9, xmm0
movdqa xmm5, xmm0
movdqa xmm6, xmm2
punpckhwd xmm9, xmm1
punpckhwd xmm5, xmm1
punpckhwd xmm6, xmm3
punpcklwd xmm0, xmm1
punpcklwd xmm2, xmm3
pmaddwd xmm9, xmm7
pmaddwd xmm5, xmm7
pmaddwd xmm6, xmm7
pmaddwd xmm0, xmm7
pmaddwd xmm2, xmm7
paddd xmm9, xmm4 ;rounding
paddd xmm5, xmm4 ;rounding
paddd xmm6, xmm4
paddd xmm0, xmm4
paddd xmm2, xmm4
psrad xmm9, 7 ;shift
psrad xmm5, 7 ;shift
psrad xmm6, 7
psrad xmm0, 7
psrad xmm2, 7
packssdw xmm0, xmm9 ;pack back to word
packssdw xmm0, xmm5 ;pack back to word
packssdw xmm2, xmm6 ;pack back to word
;clamp the values
pminsw xmm0, xmm8
pmaxsw xmm0, xmm5
pminsw xmm2, xmm8
pmaxsw xmm2, xmm5
pminsw xmm0, max
pmaxsw xmm0, min
pminsw xmm2, max
pmaxsw xmm2, min
%if %1
movdqu xmm1, [rdi]
@ -172,7 +174,6 @@
lea rdi, [rdi + 2*rdx]
dec rcx
%endm
%endif
SECTION .text
@ -200,7 +201,6 @@ sym(aom_highbd_filter_block1d4_v2_sse2):
pop rbp
ret
%if ARCH_X86_64
global sym(aom_highbd_filter_block1d8_v2_sse2) PRIVATE
sym(aom_highbd_filter_block1d8_v2_sse2):
push rbp
@ -211,6 +211,11 @@ sym(aom_highbd_filter_block1d8_v2_sse2):
push rdi
; end prolog
ALIGN_STACK 16, rax
sub rsp, 16 * 2
%define max [rsp + 16 * 0]
%define min [rsp + 16 * 1]
HIGH_GET_PARAM
.loop:
movdqu xmm0, [rsi] ;0
@ -219,6 +224,9 @@ sym(aom_highbd_filter_block1d8_v2_sse2):
HIGH_APPLY_FILTER_8 0
jnz .loop
add rsp, 16 * 2
pop rsp
; begin epilog
pop rdi
pop rsi
@ -237,6 +245,11 @@ sym(aom_highbd_filter_block1d16_v2_sse2):
push rdi
; end prolog
ALIGN_STACK 16, rax
sub rsp, 16 * 2
%define max [rsp + 16 * 0]
%define min [rsp + 16 * 1]
HIGH_GET_PARAM
.loop:
movdqu xmm0, [rsi] ;0
@ -247,6 +260,9 @@ sym(aom_highbd_filter_block1d16_v2_sse2):
HIGH_APPLY_FILTER_16 0
jnz .loop
add rsp, 16 * 2
pop rsp
; begin epilog
pop rdi
pop rsi
@ -254,7 +270,6 @@ sym(aom_highbd_filter_block1d16_v2_sse2):
UNSHADOW_ARGS
pop rbp
ret
%endif
global sym(aom_highbd_filter_block1d4_h2_sse2) PRIVATE
sym(aom_highbd_filter_block1d4_h2_sse2):
@ -281,7 +296,6 @@ sym(aom_highbd_filter_block1d4_h2_sse2):
pop rbp
ret
%if ARCH_X86_64
global sym(aom_highbd_filter_block1d8_h2_sse2) PRIVATE
sym(aom_highbd_filter_block1d8_h2_sse2):
push rbp
@ -292,6 +306,11 @@ sym(aom_highbd_filter_block1d8_h2_sse2):
push rdi
; end prolog
ALIGN_STACK 16, rax
sub rsp, 16 * 2
%define max [rsp + 16 * 0]
%define min [rsp + 16 * 1]
HIGH_GET_PARAM
.loop:
movdqu xmm0, [rsi] ;load src
@ -300,6 +319,9 @@ sym(aom_highbd_filter_block1d8_h2_sse2):
HIGH_APPLY_FILTER_8 0
jnz .loop
add rsp, 16 * 2
pop rsp
; begin epilog
pop rdi
pop rsi
@ -318,6 +340,11 @@ sym(aom_highbd_filter_block1d16_h2_sse2):
push rdi
; end prolog
ALIGN_STACK 16, rax
sub rsp, 16 * 2
%define max [rsp + 16 * 0]
%define min [rsp + 16 * 1]
HIGH_GET_PARAM
.loop:
movdqu xmm0, [rsi] ;load src
@ -328,6 +355,9 @@ sym(aom_highbd_filter_block1d16_h2_sse2):
HIGH_APPLY_FILTER_16 0
jnz .loop
add rsp, 16 * 2
pop rsp
; begin epilog
pop rdi
pop rsi
@ -335,4 +365,3 @@ sym(aom_highbd_filter_block1d16_h2_sse2):
UNSHADOW_ARGS
pop rbp
ret
%endif

569
third_party/aom/aom_dsp/x86/aom_subpixel_8t_intrin_sse2.c vendored Normal file
View file

@ -0,0 +1,569 @@
/*
* Copyright (c) 2018, Alliance for Open Media. All rights reserved
*
* This source code is subject to the terms of the BSD 2 Clause License and
* the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
* was not distributed with this source code in the LICENSE file, you can
* obtain it at www.aomedia.org/license/software. If the Alliance for Open
* Media Patent License 1.0 was not distributed with this source code in the
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
*/
#include <emmintrin.h> // SSE2
#include "config/aom_dsp_rtcd.h"
#include "aom_dsp/x86/convolve.h"
#include "aom_ports/mem.h"
void aom_filter_block1d16_h4_sse2(const uint8_t *src_ptr,
ptrdiff_t src_pixels_per_line,
uint8_t *output_ptr, ptrdiff_t output_pitch,
uint32_t output_height,
const int16_t *filter) {
__m128i filtersReg;
__m128i addFilterReg32;
__m128i secondFilters, thirdFilters;
__m128i srcRegFilt32b1_1, srcRegFilt32b1_2, srcRegFilt32b2_1,
srcRegFilt32b2_2;
__m128i srcReg32b1, srcReg32b2;
unsigned int i;
src_ptr -= 3;
addFilterReg32 = _mm_set1_epi16(32);
filtersReg = _mm_loadu_si128((const __m128i *)filter);
filtersReg = _mm_srai_epi16(filtersReg, 1);
// coeffs 0 1 0 1 2 3 2 3
const __m128i tmp_0 = _mm_unpacklo_epi32(filtersReg, filtersReg);
// coeffs 4 5 4 5 6 7 6 7
const __m128i tmp_1 = _mm_unpackhi_epi32(filtersReg, filtersReg);
secondFilters = _mm_unpackhi_epi64(tmp_0, tmp_0); // coeffs 2 3 2 3 2 3 2 3
thirdFilters = _mm_unpacklo_epi64(tmp_1, tmp_1); // coeffs 4 5 4 5 4 5 4 5
for (i = output_height; i > 0; i -= 1) {
srcReg32b1 = _mm_loadu_si128((const __m128i *)src_ptr);
__m128i ss_2 = _mm_srli_si128(srcReg32b1, 2);
__m128i ss_4 = _mm_srli_si128(srcReg32b1, 4);
__m128i ss_1_1 = _mm_unpacklo_epi8(ss_2, _mm_setzero_si128());
__m128i ss_2_1 = _mm_unpacklo_epi8(ss_4, _mm_setzero_si128());
__m128i d1 = _mm_madd_epi16(ss_1_1, secondFilters);
__m128i d2 = _mm_madd_epi16(ss_2_1, thirdFilters);
srcRegFilt32b1_1 = _mm_add_epi32(d1, d2);
__m128i ss_1 = _mm_srli_si128(srcReg32b1, 3);
__m128i ss_3 = _mm_srli_si128(srcReg32b1, 5);
__m128i ss_1_2 = _mm_unpacklo_epi8(ss_1, _mm_setzero_si128());
__m128i ss_2_2 = _mm_unpacklo_epi8(ss_3, _mm_setzero_si128());
d1 = _mm_madd_epi16(ss_1_2, secondFilters);
d2 = _mm_madd_epi16(ss_2_2, thirdFilters);
srcRegFilt32b1_2 = _mm_add_epi32(d1, d2);
__m128i res_lo = _mm_unpacklo_epi32(srcRegFilt32b1_1, srcRegFilt32b1_2);
__m128i res_hi = _mm_unpackhi_epi32(srcRegFilt32b1_1, srcRegFilt32b1_2);
srcRegFilt32b1_1 = _mm_packs_epi32(res_lo, res_hi);
// reading stride of the next 16 bytes
// (part of it was being read by earlier read)
srcReg32b2 = _mm_loadu_si128((const __m128i *)(src_ptr + 8));
ss_2 = _mm_srli_si128(srcReg32b2, 2);
ss_4 = _mm_srli_si128(srcReg32b2, 4);
ss_1_1 = _mm_unpacklo_epi8(ss_2, _mm_setzero_si128());
ss_2_1 = _mm_unpacklo_epi8(ss_4, _mm_setzero_si128());
d1 = _mm_madd_epi16(ss_1_1, secondFilters);
d2 = _mm_madd_epi16(ss_2_1, thirdFilters);
srcRegFilt32b2_1 = _mm_add_epi32(d1, d2);
ss_1 = _mm_srli_si128(srcReg32b2, 3);
ss_3 = _mm_srli_si128(srcReg32b2, 5);
ss_1_2 = _mm_unpacklo_epi8(ss_1, _mm_setzero_si128());
ss_2_2 = _mm_unpacklo_epi8(ss_3, _mm_setzero_si128());
d1 = _mm_madd_epi16(ss_1_2, secondFilters);
d2 = _mm_madd_epi16(ss_2_2, thirdFilters);
srcRegFilt32b2_2 = _mm_add_epi32(d1, d2);
res_lo = _mm_unpacklo_epi32(srcRegFilt32b2_1, srcRegFilt32b2_2);
res_hi = _mm_unpackhi_epi32(srcRegFilt32b2_1, srcRegFilt32b2_2);
srcRegFilt32b2_1 = _mm_packs_epi32(res_lo, res_hi);
// shift by 6 bit each 16 bit
srcRegFilt32b1_1 = _mm_adds_epi16(srcRegFilt32b1_1, addFilterReg32);
srcRegFilt32b2_1 = _mm_adds_epi16(srcRegFilt32b2_1, addFilterReg32);
srcRegFilt32b1_1 = _mm_srai_epi16(srcRegFilt32b1_1, 6);
srcRegFilt32b2_1 = _mm_srai_epi16(srcRegFilt32b2_1, 6);
// shrink to 8 bit each 16 bits, the first lane contain the first
// convolve result and the second lane contain the second convolve result
srcRegFilt32b1_1 = _mm_packus_epi16(srcRegFilt32b1_1, srcRegFilt32b2_1);
src_ptr += src_pixels_per_line;
_mm_store_si128((__m128i *)output_ptr, srcRegFilt32b1_1);
output_ptr += output_pitch;
}
}
void aom_filter_block1d16_v4_sse2(const uint8_t *src_ptr, ptrdiff_t src_pitch,
uint8_t *output_ptr, ptrdiff_t out_pitch,
uint32_t output_height,
const int16_t *filter) {
__m128i filtersReg;
__m128i srcReg2, srcReg3, srcReg4, srcReg5, srcReg6;
__m128i srcReg23_lo, srcReg23_hi, srcReg34_lo, srcReg34_hi;
__m128i srcReg45_lo, srcReg45_hi, srcReg56_lo, srcReg56_hi;
__m128i resReg23_lo, resReg34_lo, resReg45_lo, resReg56_lo;
__m128i resReg23_hi, resReg34_hi, resReg45_hi, resReg56_hi;
__m128i resReg23_45_lo, resReg34_56_lo, resReg23_45_hi, resReg34_56_hi;
__m128i resReg23_45, resReg34_56;
__m128i addFilterReg32, secondFilters, thirdFilters;
__m128i tmp_0, tmp_1;
unsigned int i;
ptrdiff_t src_stride, dst_stride;
addFilterReg32 = _mm_set1_epi16(32);
filtersReg = _mm_loadu_si128((const __m128i *)filter);
filtersReg = _mm_srai_epi16(filtersReg, 1);
// coeffs 0 1 0 1 2 3 2 3
const __m128i tmp0 = _mm_unpacklo_epi32(filtersReg, filtersReg);
// coeffs 4 5 4 5 6 7 6 7
const __m128i tmp1 = _mm_unpackhi_epi32(filtersReg, filtersReg);
secondFilters = _mm_unpackhi_epi64(tmp0, tmp0); // coeffs 2 3 2 3 2 3 2 3
thirdFilters = _mm_unpacklo_epi64(tmp1, tmp1); // coeffs 4 5 4 5 4 5 4 5
// multiply the size of the source and destination stride by two
src_stride = src_pitch << 1;
dst_stride = out_pitch << 1;
srcReg2 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 2));
srcReg3 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 3));
srcReg23_lo = _mm_unpacklo_epi8(srcReg2, srcReg3);
srcReg23_hi = _mm_unpackhi_epi8(srcReg2, srcReg3);
__m128i resReg23_lo_1 = _mm_unpacklo_epi8(srcReg23_lo, _mm_setzero_si128());
__m128i resReg23_lo_2 = _mm_unpackhi_epi8(srcReg23_lo, _mm_setzero_si128());
__m128i resReg23_hi_1 = _mm_unpacklo_epi8(srcReg23_hi, _mm_setzero_si128());
__m128i resReg23_hi_2 = _mm_unpackhi_epi8(srcReg23_hi, _mm_setzero_si128());
srcReg4 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 4));
srcReg34_lo = _mm_unpacklo_epi8(srcReg3, srcReg4);
srcReg34_hi = _mm_unpackhi_epi8(srcReg3, srcReg4);
__m128i resReg34_lo_1 = _mm_unpacklo_epi8(srcReg34_lo, _mm_setzero_si128());
__m128i resReg34_lo_2 = _mm_unpackhi_epi8(srcReg34_lo, _mm_setzero_si128());
__m128i resReg34_hi_1 = _mm_unpacklo_epi8(srcReg34_hi, _mm_setzero_si128());
__m128i resReg34_hi_2 = _mm_unpackhi_epi8(srcReg34_hi, _mm_setzero_si128());
for (i = output_height; i > 1; i -= 2) {
srcReg5 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 5));
srcReg45_lo = _mm_unpacklo_epi8(srcReg4, srcReg5);
srcReg45_hi = _mm_unpackhi_epi8(srcReg4, srcReg5);
srcReg6 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 6));
srcReg56_lo = _mm_unpacklo_epi8(srcReg5, srcReg6);
srcReg56_hi = _mm_unpackhi_epi8(srcReg5, srcReg6);
// multiply 2 adjacent elements with the filter and add the result
tmp_0 = _mm_madd_epi16(resReg23_lo_1, secondFilters);
tmp_1 = _mm_madd_epi16(resReg23_lo_2, secondFilters);
resReg23_lo = _mm_packs_epi32(tmp_0, tmp_1);
tmp_0 = _mm_madd_epi16(resReg34_lo_1, secondFilters);
tmp_1 = _mm_madd_epi16(resReg34_lo_2, secondFilters);
resReg34_lo = _mm_packs_epi32(tmp_0, tmp_1);
__m128i resReg45_lo_1 = _mm_unpacklo_epi8(srcReg45_lo, _mm_setzero_si128());
__m128i resReg45_lo_2 = _mm_unpackhi_epi8(srcReg45_lo, _mm_setzero_si128());
tmp_0 = _mm_madd_epi16(resReg45_lo_1, thirdFilters);
tmp_1 = _mm_madd_epi16(resReg45_lo_2, thirdFilters);
resReg45_lo = _mm_packs_epi32(tmp_0, tmp_1);
__m128i resReg56_lo_1 = _mm_unpacklo_epi8(srcReg56_lo, _mm_setzero_si128());
__m128i resReg56_lo_2 = _mm_unpackhi_epi8(srcReg56_lo, _mm_setzero_si128());
tmp_0 = _mm_madd_epi16(resReg56_lo_1, thirdFilters);
tmp_1 = _mm_madd_epi16(resReg56_lo_2, thirdFilters);
resReg56_lo = _mm_packs_epi32(tmp_0, tmp_1);
// add and saturate the results together
resReg23_45_lo = _mm_adds_epi16(resReg23_lo, resReg45_lo);
resReg34_56_lo = _mm_adds_epi16(resReg34_lo, resReg56_lo);
// multiply 2 adjacent elements with the filter and add the result
tmp_0 = _mm_madd_epi16(resReg23_hi_1, secondFilters);
tmp_1 = _mm_madd_epi16(resReg23_hi_2, secondFilters);
resReg23_hi = _mm_packs_epi32(tmp_0, tmp_1);
tmp_0 = _mm_madd_epi16(resReg34_hi_1, secondFilters);
tmp_1 = _mm_madd_epi16(resReg34_hi_2, secondFilters);
resReg34_hi = _mm_packs_epi32(tmp_0, tmp_1);
__m128i resReg45_hi_1 = _mm_unpacklo_epi8(srcReg45_hi, _mm_setzero_si128());
__m128i resReg45_hi_2 = _mm_unpackhi_epi8(srcReg45_hi, _mm_setzero_si128());
tmp_0 = _mm_madd_epi16(resReg45_hi_1, thirdFilters);
tmp_1 = _mm_madd_epi16(resReg45_hi_2, thirdFilters);
resReg45_hi = _mm_packs_epi32(tmp_0, tmp_1);
__m128i resReg56_hi_1 = _mm_unpacklo_epi8(srcReg56_hi, _mm_setzero_si128());
__m128i resReg56_hi_2 = _mm_unpackhi_epi8(srcReg56_hi, _mm_setzero_si128());
tmp_0 = _mm_madd_epi16(resReg56_hi_1, thirdFilters);
tmp_1 = _mm_madd_epi16(resReg56_hi_2, thirdFilters);
resReg56_hi = _mm_packs_epi32(tmp_0, tmp_1);
// add and saturate the results together
resReg23_45_hi = _mm_adds_epi16(resReg23_hi, resReg45_hi);
resReg34_56_hi = _mm_adds_epi16(resReg34_hi, resReg56_hi);
// shift by 6 bit each 16 bit
resReg23_45_lo = _mm_adds_epi16(resReg23_45_lo, addFilterReg32);
resReg34_56_lo = _mm_adds_epi16(resReg34_56_lo, addFilterReg32);
resReg23_45_hi = _mm_adds_epi16(resReg23_45_hi, addFilterReg32);
resReg34_56_hi = _mm_adds_epi16(resReg34_56_hi, addFilterReg32);
resReg23_45_lo = _mm_srai_epi16(resReg23_45_lo, 6);
resReg34_56_lo = _mm_srai_epi16(resReg34_56_lo, 6);
resReg23_45_hi = _mm_srai_epi16(resReg23_45_hi, 6);
resReg34_56_hi = _mm_srai_epi16(resReg34_56_hi, 6);
// shrink to 8 bit each 16 bits, the first lane contain the first
// convolve result and the second lane contain the second convolve
// result
resReg23_45 = _mm_packus_epi16(resReg23_45_lo, resReg23_45_hi);
resReg34_56 = _mm_packus_epi16(resReg34_56_lo, resReg34_56_hi);
src_ptr += src_stride;
_mm_store_si128((__m128i *)output_ptr, (resReg23_45));
_mm_store_si128((__m128i *)(output_ptr + out_pitch), (resReg34_56));
output_ptr += dst_stride;
// save part of the registers for next strides
resReg23_lo_1 = resReg45_lo_1;
resReg23_lo_2 = resReg45_lo_2;
resReg23_hi_1 = resReg45_hi_1;
resReg23_hi_2 = resReg45_hi_2;
resReg34_lo_1 = resReg56_lo_1;
resReg34_lo_2 = resReg56_lo_2;
resReg34_hi_1 = resReg56_hi_1;
resReg34_hi_2 = resReg56_hi_2;
srcReg4 = srcReg6;
}
}
void aom_filter_block1d8_h4_sse2(const uint8_t *src_ptr,
ptrdiff_t src_pixels_per_line,
uint8_t *output_ptr, ptrdiff_t output_pitch,
uint32_t output_height,
const int16_t *filter) {
__m128i filtersReg;
__m128i addFilterReg32;
__m128i secondFilters, thirdFilters;
__m128i srcRegFilt32b1_1, srcRegFilt32b1_2;
__m128i srcReg32b1;
unsigned int i;
src_ptr -= 3;
addFilterReg32 = _mm_set1_epi16(32);
filtersReg = _mm_loadu_si128((const __m128i *)filter);
filtersReg = _mm_srai_epi16(filtersReg, 1);
// coeffs 0 1 0 1 2 3 2 3
const __m128i tmp_0 = _mm_unpacklo_epi32(filtersReg, filtersReg);
// coeffs 4 5 4 5 6 7 6 7
const __m128i tmp_1 = _mm_unpackhi_epi32(filtersReg, filtersReg);
secondFilters = _mm_unpackhi_epi64(tmp_0, tmp_0); // coeffs 2 3 2 3 2 3 2 3
thirdFilters = _mm_unpacklo_epi64(tmp_1, tmp_1); // coeffs 4 5 4 5 4 5 4 5
for (i = output_height; i > 0; i -= 1) {
srcReg32b1 = _mm_loadu_si128((const __m128i *)src_ptr);
__m128i ss_2 = _mm_srli_si128(srcReg32b1, 2);
__m128i ss_4 = _mm_srli_si128(srcReg32b1, 4);
ss_2 = _mm_unpacklo_epi8(ss_2, _mm_setzero_si128());
ss_4 = _mm_unpacklo_epi8(ss_4, _mm_setzero_si128());
__m128i d1 = _mm_madd_epi16(ss_2, secondFilters);
__m128i d2 = _mm_madd_epi16(ss_4, thirdFilters);
srcRegFilt32b1_1 = _mm_add_epi32(d1, d2);
__m128i ss_3 = _mm_srli_si128(srcReg32b1, 3);
__m128i ss_5 = _mm_srli_si128(srcReg32b1, 5);
ss_3 = _mm_unpacklo_epi8(ss_3, _mm_setzero_si128());
ss_5 = _mm_unpacklo_epi8(ss_5, _mm_setzero_si128());
d1 = _mm_madd_epi16(ss_3, secondFilters);
d2 = _mm_madd_epi16(ss_5, thirdFilters);
srcRegFilt32b1_2 = _mm_add_epi32(d1, d2);
__m128i res_lo = _mm_unpacklo_epi32(srcRegFilt32b1_1, srcRegFilt32b1_2);
__m128i res_hi = _mm_unpackhi_epi32(srcRegFilt32b1_1, srcRegFilt32b1_2);
srcRegFilt32b1_1 = _mm_packs_epi32(res_lo, res_hi);
// shift by 6 bit each 16 bit
srcRegFilt32b1_1 = _mm_adds_epi16(srcRegFilt32b1_1, addFilterReg32);
srcRegFilt32b1_1 = _mm_srai_epi16(srcRegFilt32b1_1, 6);
// shrink to 8 bit each 16 bits, the first lane contain the first
// convolve result and the second lane contain the second convolve result
srcRegFilt32b1_1 = _mm_packus_epi16(srcRegFilt32b1_1, _mm_setzero_si128());
src_ptr += src_pixels_per_line;
_mm_storel_epi64((__m128i *)output_ptr, srcRegFilt32b1_1);
output_ptr += output_pitch;
}
}
void aom_filter_block1d8_v4_sse2(const uint8_t *src_ptr, ptrdiff_t src_pitch,
uint8_t *output_ptr, ptrdiff_t out_pitch,
uint32_t output_height,
const int16_t *filter) {
__m128i filtersReg;
__m128i srcReg2, srcReg3, srcReg4, srcReg5, srcReg6;
__m128i srcReg23_lo, srcReg34_lo;
__m128i srcReg45_lo, srcReg56_lo;
__m128i resReg23_lo, resReg34_lo, resReg45_lo, resReg56_lo;
__m128i resReg23_45_lo, resReg34_56_lo;
__m128i resReg23_45, resReg34_56;
__m128i addFilterReg32, secondFilters, thirdFilters;
__m128i tmp_0, tmp_1;
unsigned int i;
ptrdiff_t src_stride, dst_stride;
addFilterReg32 = _mm_set1_epi16(32);
filtersReg = _mm_loadu_si128((const __m128i *)filter);
filtersReg = _mm_srai_epi16(filtersReg, 1);
// coeffs 0 1 0 1 2 3 2 3
const __m128i tmp0 = _mm_unpacklo_epi32(filtersReg, filtersReg);
// coeffs 4 5 4 5 6 7 6 7
const __m128i tmp1 = _mm_unpackhi_epi32(filtersReg, filtersReg);
secondFilters = _mm_unpackhi_epi64(tmp0, tmp0); // coeffs 2 3 2 3 2 3 2 3
thirdFilters = _mm_unpacklo_epi64(tmp1, tmp1); // coeffs 4 5 4 5 4 5 4 5
// multiply the size of the source and destination stride by two
src_stride = src_pitch << 1;
dst_stride = out_pitch << 1;
srcReg2 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 2));
srcReg3 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 3));
srcReg23_lo = _mm_unpacklo_epi8(srcReg2, srcReg3);
__m128i resReg23_lo_1 = _mm_unpacklo_epi8(srcReg23_lo, _mm_setzero_si128());
__m128i resReg23_lo_2 = _mm_unpackhi_epi8(srcReg23_lo, _mm_setzero_si128());
srcReg4 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 4));
srcReg34_lo = _mm_unpacklo_epi8(srcReg3, srcReg4);
__m128i resReg34_lo_1 = _mm_unpacklo_epi8(srcReg34_lo, _mm_setzero_si128());
__m128i resReg34_lo_2 = _mm_unpackhi_epi8(srcReg34_lo, _mm_setzero_si128());
for (i = output_height; i > 1; i -= 2) {
srcReg5 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 5));
srcReg45_lo = _mm_unpacklo_epi8(srcReg4, srcReg5);
srcReg6 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 6));
srcReg56_lo = _mm_unpacklo_epi8(srcReg5, srcReg6);
// multiply 2 adjacent elements with the filter and add the result
tmp_0 = _mm_madd_epi16(resReg23_lo_1, secondFilters);
tmp_1 = _mm_madd_epi16(resReg23_lo_2, secondFilters);
resReg23_lo = _mm_packs_epi32(tmp_0, tmp_1);
tmp_0 = _mm_madd_epi16(resReg34_lo_1, secondFilters);
tmp_1 = _mm_madd_epi16(resReg34_lo_2, secondFilters);
resReg34_lo = _mm_packs_epi32(tmp_0, tmp_1);
__m128i resReg45_lo_1 = _mm_unpacklo_epi8(srcReg45_lo, _mm_setzero_si128());
__m128i resReg45_lo_2 = _mm_unpackhi_epi8(srcReg45_lo, _mm_setzero_si128());
tmp_0 = _mm_madd_epi16(resReg45_lo_1, thirdFilters);
tmp_1 = _mm_madd_epi16(resReg45_lo_2, thirdFilters);
resReg45_lo = _mm_packs_epi32(tmp_0, tmp_1);
__m128i resReg56_lo_1 = _mm_unpacklo_epi8(srcReg56_lo, _mm_setzero_si128());
__m128i resReg56_lo_2 = _mm_unpackhi_epi8(srcReg56_lo, _mm_setzero_si128());
tmp_0 = _mm_madd_epi16(resReg56_lo_1, thirdFilters);
tmp_1 = _mm_madd_epi16(resReg56_lo_2, thirdFilters);
resReg56_lo = _mm_packs_epi32(tmp_0, tmp_1);
// add and saturate the results together
resReg23_45_lo = _mm_adds_epi16(resReg23_lo, resReg45_lo);
resReg34_56_lo = _mm_adds_epi16(resReg34_lo, resReg56_lo);
// shift by 6 bit each 16 bit
resReg23_45_lo = _mm_adds_epi16(resReg23_45_lo, addFilterReg32);
resReg34_56_lo = _mm_adds_epi16(resReg34_56_lo, addFilterReg32);
resReg23_45_lo = _mm_srai_epi16(resReg23_45_lo, 6);
resReg34_56_lo = _mm_srai_epi16(resReg34_56_lo, 6);
// shrink to 8 bit each 16 bits, the first lane contain the first
// convolve result and the second lane contain the second convolve
// result
resReg23_45 = _mm_packus_epi16(resReg23_45_lo, _mm_setzero_si128());
resReg34_56 = _mm_packus_epi16(resReg34_56_lo, _mm_setzero_si128());
src_ptr += src_stride;
_mm_storel_epi64((__m128i *)output_ptr, (resReg23_45));
_mm_storel_epi64((__m128i *)(output_ptr + out_pitch), (resReg34_56));
output_ptr += dst_stride;
// save part of the registers for next strides
resReg23_lo_1 = resReg45_lo_1;
resReg23_lo_2 = resReg45_lo_2;
resReg34_lo_1 = resReg56_lo_1;
resReg34_lo_2 = resReg56_lo_2;
srcReg4 = srcReg6;
}
}
void aom_filter_block1d4_h4_sse2(const uint8_t *src_ptr,
ptrdiff_t src_pixels_per_line,
uint8_t *output_ptr, ptrdiff_t output_pitch,
uint32_t output_height,
const int16_t *filter) {
__m128i filtersReg;
__m128i addFilterReg32;
__m128i secondFilters, thirdFilters;
__m128i srcRegFilt32b1_1;
__m128i srcReg32b1;
unsigned int i;
src_ptr -= 3;
addFilterReg32 = _mm_set1_epi16(32);
filtersReg = _mm_loadu_si128((const __m128i *)filter);
filtersReg = _mm_srai_epi16(filtersReg, 1);
// coeffs 0 1 0 1 2 3 2 3
const __m128i tmp_0 = _mm_unpacklo_epi32(filtersReg, filtersReg);
// coeffs 4 5 4 5 6 7 6 7
const __m128i tmp_1 = _mm_unpackhi_epi32(filtersReg, filtersReg);
secondFilters = _mm_unpackhi_epi64(tmp_0, tmp_0); // coeffs 2 3 2 3 2 3 2 3
thirdFilters = _mm_unpacklo_epi64(tmp_1, tmp_1); // coeffs 4 5 4 5 4 5 4 5
for (i = output_height; i > 0; i -= 1) {
srcReg32b1 = _mm_loadu_si128((const __m128i *)src_ptr);
__m128i ss_2 = _mm_srli_si128(srcReg32b1, 2);
__m128i ss_3 = _mm_srli_si128(srcReg32b1, 3);
__m128i ss_4 = _mm_srli_si128(srcReg32b1, 4);
__m128i ss_5 = _mm_srli_si128(srcReg32b1, 5);
ss_2 = _mm_unpacklo_epi8(ss_2, _mm_setzero_si128());
ss_3 = _mm_unpacklo_epi8(ss_3, _mm_setzero_si128());
ss_4 = _mm_unpacklo_epi8(ss_4, _mm_setzero_si128());
ss_5 = _mm_unpacklo_epi8(ss_5, _mm_setzero_si128());
__m128i ss_1_1 = _mm_unpacklo_epi32(ss_2, ss_3);
__m128i ss_1_2 = _mm_unpacklo_epi32(ss_4, ss_5);
__m128i d1 = _mm_madd_epi16(ss_1_1, secondFilters);
__m128i d2 = _mm_madd_epi16(ss_1_2, thirdFilters);
srcRegFilt32b1_1 = _mm_add_epi32(d1, d2);
srcRegFilt32b1_1 = _mm_packs_epi32(srcRegFilt32b1_1, _mm_setzero_si128());
// shift by 6 bit each 16 bit
srcRegFilt32b1_1 = _mm_adds_epi16(srcRegFilt32b1_1, addFilterReg32);
srcRegFilt32b1_1 = _mm_srai_epi16(srcRegFilt32b1_1, 6);
// shrink to 8 bit each 16 bits, the first lane contain the first
// convolve result and the second lane contain the second convolve result
srcRegFilt32b1_1 = _mm_packus_epi16(srcRegFilt32b1_1, _mm_setzero_si128());
src_ptr += src_pixels_per_line;
*((uint32_t *)(output_ptr)) = _mm_cvtsi128_si32(srcRegFilt32b1_1);
output_ptr += output_pitch;
}
}
void aom_filter_block1d4_v4_sse2(const uint8_t *src_ptr, ptrdiff_t src_pitch,
uint8_t *output_ptr, ptrdiff_t out_pitch,
uint32_t output_height,
const int16_t *filter) {
__m128i filtersReg;
__m128i srcReg2, srcReg3, srcReg4, srcReg5, srcReg6;
__m128i srcReg23, srcReg34, srcReg45, srcReg56;
__m128i resReg23_34, resReg45_56;
__m128i resReg23_34_45_56;
__m128i addFilterReg32, secondFilters, thirdFilters;
__m128i tmp_0, tmp_1;
unsigned int i;
ptrdiff_t src_stride, dst_stride;
addFilterReg32 = _mm_set1_epi16(32);
filtersReg = _mm_loadu_si128((const __m128i *)filter);
filtersReg = _mm_srai_epi16(filtersReg, 1);
// coeffs 0 1 0 1 2 3 2 3
const __m128i tmp0 = _mm_unpacklo_epi32(filtersReg, filtersReg);
// coeffs 4 5 4 5 6 7 6 7
const __m128i tmp1 = _mm_unpackhi_epi32(filtersReg, filtersReg);
secondFilters = _mm_unpackhi_epi64(tmp0, tmp0); // coeffs 2 3 2 3 2 3 2 3
thirdFilters = _mm_unpacklo_epi64(tmp1, tmp1); // coeffs 4 5 4 5 4 5 4 5
// multiply the size of the source and destination stride by two
src_stride = src_pitch << 1;
dst_stride = out_pitch << 1;
srcReg2 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 2));
srcReg3 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 3));
srcReg23 = _mm_unpacklo_epi8(srcReg2, srcReg3);
__m128i resReg23 = _mm_unpacklo_epi8(srcReg23, _mm_setzero_si128());
srcReg4 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 4));
srcReg34 = _mm_unpacklo_epi8(srcReg3, srcReg4);
__m128i resReg34 = _mm_unpacklo_epi8(srcReg34, _mm_setzero_si128());
for (i = output_height; i > 1; i -= 2) {
srcReg5 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 5));
srcReg45 = _mm_unpacklo_epi8(srcReg4, srcReg5);
srcReg6 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 6));
srcReg56 = _mm_unpacklo_epi8(srcReg5, srcReg6);
// multiply 2 adjacent elements with the filter and add the result
tmp_0 = _mm_madd_epi16(resReg23, secondFilters);
tmp_1 = _mm_madd_epi16(resReg34, secondFilters);
resReg23_34 = _mm_packs_epi32(tmp_0, tmp_1);
__m128i resReg45 = _mm_unpacklo_epi8(srcReg45, _mm_setzero_si128());
__m128i resReg56 = _mm_unpacklo_epi8(srcReg56, _mm_setzero_si128());
tmp_0 = _mm_madd_epi16(resReg45, thirdFilters);
tmp_1 = _mm_madd_epi16(resReg56, thirdFilters);
resReg45_56 = _mm_packs_epi32(tmp_0, tmp_1);
// add and saturate the results together
resReg23_34_45_56 = _mm_adds_epi16(resReg23_34, resReg45_56);
// shift by 6 bit each 16 bit
resReg23_34_45_56 = _mm_adds_epi16(resReg23_34_45_56, addFilterReg32);
resReg23_34_45_56 = _mm_srai_epi16(resReg23_34_45_56, 6);
// shrink to 8 bit each 16 bits, the first lane contain the first
// convolve result and the second lane contain the second convolve
// result
resReg23_34_45_56 =
_mm_packus_epi16(resReg23_34_45_56, _mm_setzero_si128());
src_ptr += src_stride;
*((uint32_t *)(output_ptr)) = _mm_cvtsi128_si32(resReg23_34_45_56);
*((uint32_t *)(output_ptr + out_pitch)) =
_mm_cvtsi128_si32(_mm_srli_si128(resReg23_34_45_56, 4));
output_ptr += dst_stride;
// save part of the registers for next strides
resReg23 = resReg45;
resReg34 = resReg56;
srcReg4 = srcReg6;
}
}

469
third_party/aom/aom_dsp/x86/aom_subpixel_8t_intrin_ssse3.c vendored
View file

@ -45,11 +45,153 @@ DECLARE_ALIGNED(16, static const uint8_t, filt4_global[16]) = {
6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14
};
DECLARE_ALIGNED(32, static const uint8_t, filt_h4[]) = {
0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 0, 1, 1,
2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 2, 3, 3, 4, 4, 5,
5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 2, 3, 3, 4, 4, 5, 5, 6, 6,
7, 7, 8, 8, 9, 9, 10, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10,
10, 11, 11, 12, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11,
12, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 6, 7,
7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14
};
DECLARE_ALIGNED(32, static const uint8_t, filtd4[]) = {
2, 3, 4, 5, 3, 4, 5, 6, 4, 5, 6, 7, 5, 6, 7, 8,
2, 3, 4, 5, 3, 4, 5, 6, 4, 5, 6, 7, 5, 6, 7, 8,
};
// These are reused by the avx2 intrinsics.
filter8_1dfunction aom_filter_block1d8_v8_intrin_ssse3;
filter8_1dfunction aom_filter_block1d8_h8_intrin_ssse3;
filter8_1dfunction aom_filter_block1d4_h8_intrin_ssse3;
static void aom_filter_block1d4_h4_ssse3(
const uint8_t *src_ptr, ptrdiff_t src_pixels_per_line, uint8_t *output_ptr,
ptrdiff_t output_pitch, uint32_t output_height, const int16_t *filter) {
__m128i filtersReg;
__m128i addFilterReg32, filt1Reg, firstFilters, srcReg32b1, srcRegFilt32b1_1;
unsigned int i;
src_ptr -= 3;
addFilterReg32 = _mm_set1_epi16(32);
filtersReg = _mm_loadu_si128((const __m128i *)filter);
filtersReg = _mm_srai_epi16(filtersReg, 1);
// converting the 16 bit (short) to 8 bit (byte) and have the same data
// in both lanes of 128 bit register.
filtersReg = _mm_packs_epi16(filtersReg, filtersReg);
firstFilters = _mm_shuffle_epi8(filtersReg, _mm_set1_epi32(0x5040302u));
filt1Reg = _mm_load_si128((__m128i const *)(filtd4));
for (i = output_height; i > 0; i -= 1) {
// load the 2 strides of source
srcReg32b1 = _mm_loadu_si128((const __m128i *)src_ptr);
// filter the source buffer
srcRegFilt32b1_1 = _mm_shuffle_epi8(srcReg32b1, filt1Reg);
// multiply 4 adjacent elements with the filter and add the result
srcRegFilt32b1_1 = _mm_maddubs_epi16(srcRegFilt32b1_1, firstFilters);
srcRegFilt32b1_1 = _mm_hadds_epi16(srcRegFilt32b1_1, _mm_setzero_si128());
// shift by 6 bit each 16 bit
srcRegFilt32b1_1 = _mm_adds_epi16(srcRegFilt32b1_1, addFilterReg32);
srcRegFilt32b1_1 = _mm_srai_epi16(srcRegFilt32b1_1, 6);
// shrink to 8 bit each 16 bits, the first lane contain the first
// convolve result and the second lane contain the second convolve result
srcRegFilt32b1_1 = _mm_packus_epi16(srcRegFilt32b1_1, _mm_setzero_si128());
src_ptr += src_pixels_per_line;
*((uint32_t *)(output_ptr)) = _mm_cvtsi128_si32(srcRegFilt32b1_1);
output_ptr += output_pitch;
}
}
static void aom_filter_block1d4_v4_ssse3(
const uint8_t *src_ptr, ptrdiff_t src_pitch, uint8_t *output_ptr,
ptrdiff_t out_pitch, uint32_t output_height, const int16_t *filter) {
__m128i filtersReg;
__m128i addFilterReg32;
__m128i srcReg2, srcReg3, srcReg23, srcReg4, srcReg34, srcReg5, srcReg45,
srcReg6, srcReg56;
__m128i srcReg23_34_lo, srcReg45_56_lo;
__m128i srcReg2345_3456_lo, srcReg2345_3456_hi;
__m128i resReglo, resReghi;
__m128i firstFilters;
unsigned int i;
ptrdiff_t src_stride, dst_stride;
addFilterReg32 = _mm_set1_epi16(32);
filtersReg = _mm_loadu_si128((const __m128i *)filter);
// converting the 16 bit (short) to 8 bit (byte) and have the
// same data in both lanes of 128 bit register.
filtersReg = _mm_srai_epi16(filtersReg, 1);
filtersReg = _mm_packs_epi16(filtersReg, filtersReg);
firstFilters = _mm_shuffle_epi8(filtersReg, _mm_set1_epi32(0x5040302u));
// multiple the size of the source and destination stride by two
src_stride = src_pitch << 1;
dst_stride = out_pitch << 1;
srcReg2 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 2));
srcReg3 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 3));
srcReg23 = _mm_unpacklo_epi32(srcReg2, srcReg3);
srcReg4 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 4));
// have consecutive loads on the same 256 register
srcReg34 = _mm_unpacklo_epi32(srcReg3, srcReg4);
srcReg23_34_lo = _mm_unpacklo_epi8(srcReg23, srcReg34);
for (i = output_height; i > 1; i -= 2) {
srcReg5 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 5));
srcReg45 = _mm_unpacklo_epi32(srcReg4, srcReg5);
srcReg6 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 6));
srcReg56 = _mm_unpacklo_epi32(srcReg5, srcReg6);
// merge every two consecutive registers
srcReg45_56_lo = _mm_unpacklo_epi8(srcReg45, srcReg56);
srcReg2345_3456_lo = _mm_unpacklo_epi16(srcReg23_34_lo, srcReg45_56_lo);
srcReg2345_3456_hi = _mm_unpackhi_epi16(srcReg23_34_lo, srcReg45_56_lo);
// multiply 2 adjacent elements with the filter and add the result
resReglo = _mm_maddubs_epi16(srcReg2345_3456_lo, firstFilters);
resReghi = _mm_maddubs_epi16(srcReg2345_3456_hi, firstFilters);
resReglo = _mm_hadds_epi16(resReglo, _mm_setzero_si128());
resReghi = _mm_hadds_epi16(resReghi, _mm_setzero_si128());
// shift by 6 bit each 16 bit
resReglo = _mm_adds_epi16(resReglo, addFilterReg32);
resReghi = _mm_adds_epi16(resReghi, addFilterReg32);
resReglo = _mm_srai_epi16(resReglo, 6);
resReghi = _mm_srai_epi16(resReghi, 6);
// shrink to 8 bit each 16 bits, the first lane contain the first
// convolve result and the second lane contain the second convolve
// result
resReglo = _mm_packus_epi16(resReglo, resReglo);
resReghi = _mm_packus_epi16(resReghi, resReghi);
src_ptr += src_stride;
*((uint32_t *)(output_ptr)) = _mm_cvtsi128_si32(resReglo);
*((uint32_t *)(output_ptr + out_pitch)) = _mm_cvtsi128_si32(resReghi);
output_ptr += dst_stride;
// save part of the registers for next strides
srcReg23_34_lo = srcReg45_56_lo;
srcReg4 = srcReg6;
}
}
void aom_filter_block1d4_h8_intrin_ssse3(
const uint8_t *src_ptr, ptrdiff_t src_pixels_per_line, uint8_t *output_ptr,
ptrdiff_t output_pitch, uint32_t output_height, const int16_t *filter) {
@ -118,6 +260,145 @@ void aom_filter_block1d4_h8_intrin_ssse3(
}
}
static void aom_filter_block1d8_h4_ssse3(
const uint8_t *src_ptr, ptrdiff_t src_pixels_per_line, uint8_t *output_ptr,
ptrdiff_t output_pitch, uint32_t output_height, const int16_t *filter) {
__m128i filtersReg;
__m128i addFilterReg32, filt2Reg, filt3Reg;
__m128i secondFilters, thirdFilters;
__m128i srcRegFilt32b1_1, srcRegFilt32b2, srcRegFilt32b3;
__m128i srcReg32b1;
unsigned int i;
src_ptr -= 3;
addFilterReg32 = _mm_set1_epi16(32);
filtersReg = _mm_loadu_si128((const __m128i *)filter);
filtersReg = _mm_srai_epi16(filtersReg, 1);
// converting the 16 bit (short) to 8 bit (byte) and have the same data
// in both lanes of 128 bit register.
filtersReg = _mm_packs_epi16(filtersReg, filtersReg);
// duplicate only the second 16 bits (third and forth byte)
// across 256 bit register
secondFilters = _mm_shuffle_epi8(filtersReg, _mm_set1_epi16(0x302u));
// duplicate only the third 16 bits (fifth and sixth byte)
// across 256 bit register
thirdFilters = _mm_shuffle_epi8(filtersReg, _mm_set1_epi16(0x504u));
filt2Reg = _mm_load_si128((__m128i const *)(filt_h4 + 32));
filt3Reg = _mm_load_si128((__m128i const *)(filt_h4 + 32 * 2));
for (i = output_height; i > 0; i -= 1) {
srcReg32b1 = _mm_loadu_si128((const __m128i *)src_ptr);
// filter the source buffer
srcRegFilt32b3 = _mm_shuffle_epi8(srcReg32b1, filt2Reg);
srcRegFilt32b2 = _mm_shuffle_epi8(srcReg32b1, filt3Reg);
// multiply 2 adjacent elements with the filter and add the result
srcRegFilt32b3 = _mm_maddubs_epi16(srcRegFilt32b3, secondFilters);
srcRegFilt32b2 = _mm_maddubs_epi16(srcRegFilt32b2, thirdFilters);
srcRegFilt32b1_1 = _mm_adds_epi16(srcRegFilt32b3, srcRegFilt32b2);
// shift by 6 bit each 16 bit
srcRegFilt32b1_1 = _mm_adds_epi16(srcRegFilt32b1_1, addFilterReg32);
srcRegFilt32b1_1 = _mm_srai_epi16(srcRegFilt32b1_1, 6);
// shrink to 8 bit each 16 bits
srcRegFilt32b1_1 = _mm_packus_epi16(srcRegFilt32b1_1, _mm_setzero_si128());
src_ptr += src_pixels_per_line;
_mm_storel_epi64((__m128i *)output_ptr, srcRegFilt32b1_1);
output_ptr += output_pitch;
}
}
static void aom_filter_block1d8_v4_ssse3(
const uint8_t *src_ptr, ptrdiff_t src_pitch, uint8_t *output_ptr,
ptrdiff_t out_pitch, uint32_t output_height, const int16_t *filter) {
__m128i filtersReg;
__m128i srcReg2, srcReg3, srcReg4, srcReg5, srcReg6;
__m128i srcReg23, srcReg34, srcReg45, srcReg56;
__m128i resReg23, resReg34, resReg45, resReg56;
__m128i resReg23_45, resReg34_56;
__m128i addFilterReg32, secondFilters, thirdFilters;
unsigned int i;
ptrdiff_t src_stride, dst_stride;
addFilterReg32 = _mm_set1_epi16(32);
filtersReg = _mm_loadu_si128((const __m128i *)filter);
// converting the 16 bit (short) to 8 bit (byte) and have the
// same data in both lanes of 128 bit register.
filtersReg = _mm_srai_epi16(filtersReg, 1);
filtersReg = _mm_packs_epi16(filtersReg, filtersReg);
// duplicate only the second 16 bits (third and forth byte)
// across 128 bit register
secondFilters = _mm_shuffle_epi8(filtersReg, _mm_set1_epi16(0x302u));
// duplicate only the third 16 bits (fifth and sixth byte)
// across 128 bit register
thirdFilters = _mm_shuffle_epi8(filtersReg, _mm_set1_epi16(0x504u));
// multiple the size of the source and destination stride by two
src_stride = src_pitch << 1;
dst_stride = out_pitch << 1;
srcReg2 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 2));
srcReg3 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 3));
srcReg23 = _mm_unpacklo_epi8(srcReg2, srcReg3);
srcReg4 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 4));
// have consecutive loads on the same 256 register
srcReg34 = _mm_unpacklo_epi8(srcReg3, srcReg4);
for (i = output_height; i > 1; i -= 2) {
srcReg5 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 5));
srcReg45 = _mm_unpacklo_epi8(srcReg4, srcReg5);
srcReg6 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 6));
srcReg56 = _mm_unpacklo_epi8(srcReg5, srcReg6);
// multiply 2 adjacent elements with the filter and add the result
resReg23 = _mm_maddubs_epi16(srcReg23, secondFilters);
resReg34 = _mm_maddubs_epi16(srcReg34, secondFilters);
resReg45 = _mm_maddubs_epi16(srcReg45, thirdFilters);
resReg56 = _mm_maddubs_epi16(srcReg56, thirdFilters);
// add and saturate the results together
resReg23_45 = _mm_adds_epi16(resReg23, resReg45);
resReg34_56 = _mm_adds_epi16(resReg34, resReg56);
// shift by 6 bit each 16 bit
resReg23_45 = _mm_adds_epi16(resReg23_45, addFilterReg32);
resReg34_56 = _mm_adds_epi16(resReg34_56, addFilterReg32);
resReg23_45 = _mm_srai_epi16(resReg23_45, 6);
resReg34_56 = _mm_srai_epi16(resReg34_56, 6);
// shrink to 8 bit each 16 bits, the first lane contain the first
// convolve result and the second lane contain the second convolve
// result
resReg23_45 = _mm_packus_epi16(resReg23_45, _mm_setzero_si128());
resReg34_56 = _mm_packus_epi16(resReg34_56, _mm_setzero_si128());
src_ptr += src_stride;
_mm_storel_epi64((__m128i *)output_ptr, (resReg23_45));
_mm_storel_epi64((__m128i *)(output_ptr + out_pitch), (resReg34_56));
output_ptr += dst_stride;
// save part of the registers for next strides
srcReg23 = srcReg45;
srcReg34 = srcReg56;
srcReg4 = srcReg6;
}
}
void aom_filter_block1d8_h8_intrin_ssse3(
const uint8_t *src_ptr, ptrdiff_t src_pixels_per_line, uint8_t *output_ptr,
ptrdiff_t output_pitch, uint32_t output_height, const int16_t *filter) {
@ -280,6 +561,187 @@ void aom_filter_block1d8_v8_intrin_ssse3(
}
}
static void aom_filter_block1d16_h4_ssse3(
const uint8_t *src_ptr, ptrdiff_t src_pixels_per_line, uint8_t *output_ptr,
ptrdiff_t output_pitch, uint32_t output_height, const int16_t *filter) {
__m128i filtersReg;
__m128i addFilterReg32, filt2Reg, filt3Reg;
__m128i secondFilters, thirdFilters;
__m128i srcRegFilt32b1_1, srcRegFilt32b2_1, srcRegFilt32b2, srcRegFilt32b3;
__m128i srcReg32b1, srcReg32b2;
unsigned int i;
src_ptr -= 3;
addFilterReg32 = _mm_set1_epi16(32);
filtersReg = _mm_loadu_si128((const __m128i *)filter);
filtersReg = _mm_srai_epi16(filtersReg, 1);
// converting the 16 bit (short) to 8 bit (byte) and have the same data
// in both lanes of 128 bit register.
filtersReg = _mm_packs_epi16(filtersReg, filtersReg);
// duplicate only the second 16 bits (third and forth byte)
// across 256 bit register
secondFilters = _mm_shuffle_epi8(filtersReg, _mm_set1_epi16(0x302u));
// duplicate only the third 16 bits (fifth and sixth byte)
// across 256 bit register
thirdFilters = _mm_shuffle_epi8(filtersReg, _mm_set1_epi16(0x504u));
filt2Reg = _mm_load_si128((__m128i const *)(filt_h4 + 32));
filt3Reg = _mm_load_si128((__m128i const *)(filt_h4 + 32 * 2));
for (i = output_height; i > 0; i -= 1) {
srcReg32b1 = _mm_loadu_si128((const __m128i *)src_ptr);
// filter the source buffer
srcRegFilt32b3 = _mm_shuffle_epi8(srcReg32b1, filt2Reg);
srcRegFilt32b2 = _mm_shuffle_epi8(srcReg32b1, filt3Reg);
// multiply 2 adjacent elements with the filter and add the result
srcRegFilt32b3 = _mm_maddubs_epi16(srcRegFilt32b3, secondFilters);
srcRegFilt32b2 = _mm_maddubs_epi16(srcRegFilt32b2, thirdFilters);
srcRegFilt32b1_1 = _mm_adds_epi16(srcRegFilt32b3, srcRegFilt32b2);
// reading stride of the next 16 bytes
// (part of it was being read by earlier read)
srcReg32b2 = _mm_loadu_si128((const __m128i *)(src_ptr + 8));
// filter the source buffer
srcRegFilt32b3 = _mm_shuffle_epi8(srcReg32b2, filt2Reg);
srcRegFilt32b2 = _mm_shuffle_epi8(srcReg32b2, filt3Reg);
// multiply 2 adjacent elements with the filter and add the result
srcRegFilt32b3 = _mm_maddubs_epi16(srcRegFilt32b3, secondFilters);
srcRegFilt32b2 = _mm_maddubs_epi16(srcRegFilt32b2, thirdFilters);
// add and saturate the results together
srcRegFilt32b2_1 = _mm_adds_epi16(srcRegFilt32b3, srcRegFilt32b2);
// shift by 6 bit each 16 bit
srcRegFilt32b1_1 = _mm_adds_epi16(srcRegFilt32b1_1, addFilterReg32);
srcRegFilt32b2_1 = _mm_adds_epi16(srcRegFilt32b2_1, addFilterReg32);
srcRegFilt32b1_1 = _mm_srai_epi16(srcRegFilt32b1_1, 6);
srcRegFilt32b2_1 = _mm_srai_epi16(srcRegFilt32b2_1, 6);
// shrink to 8 bit each 16 bits, the first lane contain the first
// convolve result and the second lane contain the second convolve result
srcRegFilt32b1_1 = _mm_packus_epi16(srcRegFilt32b1_1, srcRegFilt32b2_1);
src_ptr += src_pixels_per_line;
_mm_store_si128((__m128i *)output_ptr, srcRegFilt32b1_1);
output_ptr += output_pitch;
}
}
static void aom_filter_block1d16_v4_ssse3(
const uint8_t *src_ptr, ptrdiff_t src_pitch, uint8_t *output_ptr,
ptrdiff_t out_pitch, uint32_t output_height, const int16_t *filter) {
__m128i filtersReg;
__m128i srcReg2, srcReg3, srcReg4, srcReg5, srcReg6;
__m128i srcReg23_lo, srcReg23_hi, srcReg34_lo, srcReg34_hi;
__m128i srcReg45_lo, srcReg45_hi, srcReg56_lo, srcReg56_hi;
__m128i resReg23_lo, resReg34_lo, resReg45_lo, resReg56_lo;
__m128i resReg23_hi, resReg34_hi, resReg45_hi, resReg56_hi;
__m128i resReg23_45_lo, resReg34_56_lo, resReg23_45_hi, resReg34_56_hi;
__m128i resReg23_45, resReg34_56;
__m128i addFilterReg32, secondFilters, thirdFilters;
unsigned int i;
ptrdiff_t src_stride, dst_stride;
addFilterReg32 = _mm_set1_epi16(32);
filtersReg = _mm_loadu_si128((const __m128i *)filter);
// converting the 16 bit (short) to 8 bit (byte) and have the
// same data in both lanes of 128 bit register.
filtersReg = _mm_srai_epi16(filtersReg, 1);
filtersReg = _mm_packs_epi16(filtersReg, filtersReg);
// duplicate only the second 16 bits (third and forth byte)
// across 128 bit register
secondFilters = _mm_shuffle_epi8(filtersReg, _mm_set1_epi16(0x302u));
// duplicate only the third 16 bits (fifth and sixth byte)
// across 128 bit register
thirdFilters = _mm_shuffle_epi8(filtersReg, _mm_set1_epi16(0x504u));
// multiple the size of the source and destination stride by two
src_stride = src_pitch << 1;
dst_stride = out_pitch << 1;
srcReg2 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 2));
srcReg3 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 3));
srcReg23_lo = _mm_unpacklo_epi8(srcReg2, srcReg3);
srcReg23_hi = _mm_unpackhi_epi8(srcReg2, srcReg3);
srcReg4 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 4));
// have consecutive loads on the same 256 register
srcReg34_lo = _mm_unpacklo_epi8(srcReg3, srcReg4);
srcReg34_hi = _mm_unpackhi_epi8(srcReg3, srcReg4);
for (i = output_height; i > 1; i -= 2) {
srcReg5 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 5));
srcReg45_lo = _mm_unpacklo_epi8(srcReg4, srcReg5);
srcReg45_hi = _mm_unpackhi_epi8(srcReg4, srcReg5);
srcReg6 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 6));
srcReg56_lo = _mm_unpacklo_epi8(srcReg5, srcReg6);
srcReg56_hi = _mm_unpackhi_epi8(srcReg5, srcReg6);
// multiply 2 adjacent elements with the filter and add the result
resReg23_lo = _mm_maddubs_epi16(srcReg23_lo, secondFilters);
resReg34_lo = _mm_maddubs_epi16(srcReg34_lo, secondFilters);
resReg45_lo = _mm_maddubs_epi16(srcReg45_lo, thirdFilters);
resReg56_lo = _mm_maddubs_epi16(srcReg56_lo, thirdFilters);
// add and saturate the results together
resReg23_45_lo = _mm_adds_epi16(resReg23_lo, resReg45_lo);
resReg34_56_lo = _mm_adds_epi16(resReg34_lo, resReg56_lo);
// multiply 2 adjacent elements with the filter and add the result
resReg23_hi = _mm_maddubs_epi16(srcReg23_hi, secondFilters);
resReg34_hi = _mm_maddubs_epi16(srcReg34_hi, secondFilters);
resReg45_hi = _mm_maddubs_epi16(srcReg45_hi, thirdFilters);
resReg56_hi = _mm_maddubs_epi16(srcReg56_hi, thirdFilters);
// add and saturate the results together
resReg23_45_hi = _mm_adds_epi16(resReg23_hi, resReg45_hi);
resReg34_56_hi = _mm_adds_epi16(resReg34_hi, resReg56_hi);
// shift by 6 bit each 16 bit
resReg23_45_lo = _mm_adds_epi16(resReg23_45_lo, addFilterReg32);
resReg34_56_lo = _mm_adds_epi16(resReg34_56_lo, addFilterReg32);
resReg23_45_hi = _mm_adds_epi16(resReg23_45_hi, addFilterReg32);
resReg34_56_hi = _mm_adds_epi16(resReg34_56_hi, addFilterReg32);
resReg23_45_lo = _mm_srai_epi16(resReg23_45_lo, 6);
resReg34_56_lo = _mm_srai_epi16(resReg34_56_lo, 6);
resReg23_45_hi = _mm_srai_epi16(resReg23_45_hi, 6);
resReg34_56_hi = _mm_srai_epi16(resReg34_56_hi, 6);
// shrink to 8 bit each 16 bits, the first lane contain the first
// convolve result and the second lane contain the second convolve
// result
resReg23_45 = _mm_packus_epi16(resReg23_45_lo, resReg23_45_hi);
resReg34_56 = _mm_packus_epi16(resReg34_56_lo, resReg34_56_hi);
src_ptr += src_stride;
_mm_store_si128((__m128i *)output_ptr, (resReg23_45));
_mm_store_si128((__m128i *)(output_ptr + out_pitch), (resReg34_56));
output_ptr += dst_stride;
// save part of the registers for next strides
srcReg23_lo = srcReg45_lo;
srcReg34_lo = srcReg56_lo;
srcReg23_hi = srcReg45_hi;
srcReg34_hi = srcReg56_hi;
srcReg4 = srcReg6;
}
}
filter8_1dfunction aom_filter_block1d16_v8_ssse3;
filter8_1dfunction aom_filter_block1d16_h8_ssse3;
filter8_1dfunction aom_filter_block1d8_v8_ssse3;
@ -287,13 +749,6 @@ filter8_1dfunction aom_filter_block1d8_h8_ssse3;
filter8_1dfunction aom_filter_block1d4_v8_ssse3;
filter8_1dfunction aom_filter_block1d4_h8_ssse3;
#define aom_filter_block1d16_h4_ssse3 aom_filter_block1d16_h8_ssse3
#define aom_filter_block1d16_v4_ssse3 aom_filter_block1d16_v8_ssse3
#define aom_filter_block1d8_h4_ssse3 aom_filter_block1d8_h8_ssse3
#define aom_filter_block1d8_v4_ssse3 aom_filter_block1d8_v8_ssse3
#define aom_filter_block1d4_h4_ssse3 aom_filter_block1d4_h8_ssse3
#define aom_filter_block1d4_v4_ssse3 aom_filter_block1d4_v8_ssse3
filter8_1dfunction aom_filter_block1d16_v2_ssse3;
filter8_1dfunction aom_filter_block1d16_h2_ssse3;
filter8_1dfunction aom_filter_block1d8_v2_ssse3;

249
third_party/aom/aom_dsp/x86/avg_intrin_avx2.c vendored Normal file
View file

@ -0,0 +1,249 @@
/*
* Copyright (c) 2016, Alliance for Open Media. All rights reserved
*
* This source code is subject to the terms of the BSD 2 Clause License and
* the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
* was not distributed with this source code in the LICENSE file, you can
* obtain it at www.aomedia.org/license/software. If the Alliance for Open
* Media Patent License 1.0 was not distributed with this source code in the
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
*/
#include <immintrin.h>
#include "config/aom_dsp_rtcd.h"
#include "aom/aom_integer.h"
#include "aom_dsp/x86/bitdepth_conversion_avx2.h"
#include "aom_ports/mem.h"
static void hadamard_col8x2_avx2(__m256i *in, int iter) {
__m256i a0 = in[0];
__m256i a1 = in[1];
__m256i a2 = in[2];
__m256i a3 = in[3];
__m256i a4 = in[4];
__m256i a5 = in[5];
__m256i a6 = in[6];
__m256i a7 = in[7];
__m256i b0 = _mm256_add_epi16(a0, a1);
__m256i b1 = _mm256_sub_epi16(a0, a1);
__m256i b2 = _mm256_add_epi16(a2, a3);
__m256i b3 = _mm256_sub_epi16(a2, a3);
__m256i b4 = _mm256_add_epi16(a4, a5);
__m256i b5 = _mm256_sub_epi16(a4, a5);
__m256i b6 = _mm256_add_epi16(a6, a7);
__m256i b7 = _mm256_sub_epi16(a6, a7);
a0 = _mm256_add_epi16(b0, b2);
a1 = _mm256_add_epi16(b1, b3);
a2 = _mm256_sub_epi16(b0, b2);
a3 = _mm256_sub_epi16(b1, b3);
a4 = _mm256_add_epi16(b4, b6);
a5 = _mm256_add_epi16(b5, b7);
a6 = _mm256_sub_epi16(b4, b6);
a7 = _mm256_sub_epi16(b5, b7);
if (iter == 0) {
b0 = _mm256_add_epi16(a0, a4);
b7 = _mm256_add_epi16(a1, a5);
b3 = _mm256_add_epi16(a2, a6);
b4 = _mm256_add_epi16(a3, a7);
b2 = _mm256_sub_epi16(a0, a4);
b6 = _mm256_sub_epi16(a1, a5);
b1 = _mm256_sub_epi16(a2, a6);
b5 = _mm256_sub_epi16(a3, a7);
a0 = _mm256_unpacklo_epi16(b0, b1);
a1 = _mm256_unpacklo_epi16(b2, b3);
a2 = _mm256_unpackhi_epi16(b0, b1);
a3 = _mm256_unpackhi_epi16(b2, b3);
a4 = _mm256_unpacklo_epi16(b4, b5);
a5 = _mm256_unpacklo_epi16(b6, b7);
a6 = _mm256_unpackhi_epi16(b4, b5);
a7 = _mm256_unpackhi_epi16(b6, b7);
b0 = _mm256_unpacklo_epi32(a0, a1);
b1 = _mm256_unpacklo_epi32(a4, a5);
b2 = _mm256_unpackhi_epi32(a0, a1);
b3 = _mm256_unpackhi_epi32(a4, a5);
b4 = _mm256_unpacklo_epi32(a2, a3);
b5 = _mm256_unpacklo_epi32(a6, a7);
b6 = _mm256_unpackhi_epi32(a2, a3);
b7 = _mm256_unpackhi_epi32(a6, a7);
in[0] = _mm256_unpacklo_epi64(b0, b1);
in[1] = _mm256_unpackhi_epi64(b0, b1);
in[2] = _mm256_unpacklo_epi64(b2, b3);
in[3] = _mm256_unpackhi_epi64(b2, b3);
in[4] = _mm256_unpacklo_epi64(b4, b5);
in[5] = _mm256_unpackhi_epi64(b4, b5);
in[6] = _mm256_unpacklo_epi64(b6, b7);
in[7] = _mm256_unpackhi_epi64(b6, b7);
} else {
in[0] = _mm256_add_epi16(a0, a4);
in[7] = _mm256_add_epi16(a1, a5);
in[3] = _mm256_add_epi16(a2, a6);
in[4] = _mm256_add_epi16(a3, a7);
in[2] = _mm256_sub_epi16(a0, a4);
in[6] = _mm256_sub_epi16(a1, a5);
in[1] = _mm256_sub_epi16(a2, a6);
in[5] = _mm256_sub_epi16(a3, a7);
}
}
static void hadamard_8x8x2_avx2(const int16_t *src_diff, ptrdiff_t src_stride,
int16_t *coeff) {
__m256i src[8];
src[0] = _mm256_loadu_si256((const __m256i *)src_diff);
src[1] = _mm256_loadu_si256((const __m256i *)(src_diff += src_stride));
src[2] = _mm256_loadu_si256((const __m256i *)(src_diff += src_stride));
src[3] = _mm256_loadu_si256((const __m256i *)(src_diff += src_stride));
src[4] = _mm256_loadu_si256((const __m256i *)(src_diff += src_stride));
src[5] = _mm256_loadu_si256((const __m256i *)(src_diff += src_stride));
src[6] = _mm256_loadu_si256((const __m256i *)(src_diff += src_stride));
src[7] = _mm256_loadu_si256((const __m256i *)(src_diff += src_stride));
hadamard_col8x2_avx2(src, 0);
hadamard_col8x2_avx2(src, 1);
_mm256_storeu_si256((__m256i *)coeff,
_mm256_permute2x128_si256(src[0], src[1], 0x20));
coeff += 16;
_mm256_storeu_si256((__m256i *)coeff,
_mm256_permute2x128_si256(src[2], src[3], 0x20));
coeff += 16;
_mm256_storeu_si256((__m256i *)coeff,
_mm256_permute2x128_si256(src[4], src[5], 0x20));
coeff += 16;
_mm256_storeu_si256((__m256i *)coeff,
_mm256_permute2x128_si256(src[6], src[7], 0x20));
coeff += 16;
_mm256_storeu_si256((__m256i *)coeff,
_mm256_permute2x128_si256(src[0], src[1], 0x31));
coeff += 16;
_mm256_storeu_si256((__m256i *)coeff,
_mm256_permute2x128_si256(src[2], src[3], 0x31));
coeff += 16;
_mm256_storeu_si256((__m256i *)coeff,
_mm256_permute2x128_si256(src[4], src[5], 0x31));
coeff += 16;
_mm256_storeu_si256((__m256i *)coeff,
_mm256_permute2x128_si256(src[6], src[7], 0x31));
}
static INLINE void hadamard_16x16_avx2(const int16_t *src_diff,
ptrdiff_t src_stride, tran_low_t *coeff,
int is_final) {
DECLARE_ALIGNED(32, int16_t, temp_coeff[16 * 16]);
int16_t *t_coeff = temp_coeff;
int16_t *coeff16 = (int16_t *)coeff;
int idx;
for (idx = 0; idx < 2; ++idx) {
const int16_t *src_ptr = src_diff + idx * 8 * src_stride;
hadamard_8x8x2_avx2(src_ptr, src_stride, t_coeff + (idx * 64 * 2));
}
for (idx = 0; idx < 64; idx += 16) {
const __m256i coeff0 = _mm256_loadu_si256((const __m256i *)t_coeff);
const __m256i coeff1 = _mm256_loadu_si256((const __m256i *)(t_coeff + 64));
const __m256i coeff2 = _mm256_loadu_si256((const __m256i *)(t_coeff + 128));
const __m256i coeff3 = _mm256_loadu_si256((const __m256i *)(t_coeff + 192));
__m256i b0 = _mm256_add_epi16(coeff0, coeff1);
__m256i b1 = _mm256_sub_epi16(coeff0, coeff1);
__m256i b2 = _mm256_add_epi16(coeff2, coeff3);
__m256i b3 = _mm256_sub_epi16(coeff2, coeff3);
b0 = _mm256_srai_epi16(b0, 1);
b1 = _mm256_srai_epi16(b1, 1);
b2 = _mm256_srai_epi16(b2, 1);
b3 = _mm256_srai_epi16(b3, 1);
if (is_final) {
store_tran_low(_mm256_add_epi16(b0, b2), coeff);
store_tran_low(_mm256_add_epi16(b1, b3), coeff + 64);
store_tran_low(_mm256_sub_epi16(b0, b2), coeff + 128);
store_tran_low(_mm256_sub_epi16(b1, b3), coeff + 192);
coeff += 16;
} else {
_mm256_storeu_si256((__m256i *)coeff16, _mm256_add_epi16(b0, b2));
_mm256_storeu_si256((__m256i *)(coeff16 + 64), _mm256_add_epi16(b1, b3));
_mm256_storeu_si256((__m256i *)(coeff16 + 128), _mm256_sub_epi16(b0, b2));
_mm256_storeu_si256((__m256i *)(coeff16 + 192), _mm256_sub_epi16(b1, b3));
coeff16 += 16;
}
t_coeff += 16;
}
}
void aom_hadamard_16x16_avx2(const int16_t *src_diff, ptrdiff_t src_stride,
tran_low_t *coeff) {
hadamard_16x16_avx2(src_diff, src_stride, coeff, 1);
}
void aom_hadamard_32x32_avx2(const int16_t *src_diff, ptrdiff_t src_stride,
tran_low_t *coeff) {
// For high bitdepths, it is unnecessary to store_tran_low
// (mult/unpack/store), then load_tran_low (load/pack) the same memory in the
// next stage. Output to an intermediate buffer first, then store_tran_low()
// in the final stage.
DECLARE_ALIGNED(32, int16_t, temp_coeff[32 * 32]);
int16_t *t_coeff = temp_coeff;
int idx;
for (idx = 0; idx < 4; ++idx) {
// src_diff: 9 bit, dynamic range [-255, 255]
const int16_t *src_ptr =
src_diff + (idx >> 1) * 16 * src_stride + (idx & 0x01) * 16;
hadamard_16x16_avx2(src_ptr, src_stride,
(tran_low_t *)(t_coeff + idx * 256), 0);
}
for (idx = 0; idx < 256; idx += 16) {
const __m256i coeff0 = _mm256_loadu_si256((const __m256i *)t_coeff);
const __m256i coeff1 = _mm256_loadu_si256((const __m256i *)(t_coeff + 256));
const __m256i coeff2 = _mm256_loadu_si256((const __m256i *)(t_coeff + 512));
const __m256i coeff3 = _mm256_loadu_si256((const __m256i *)(t_coeff + 768));
__m256i b0 = _mm256_add_epi16(coeff0, coeff1);
__m256i b1 = _mm256_sub_epi16(coeff0, coeff1);
__m256i b2 = _mm256_add_epi16(coeff2, coeff3);
__m256i b3 = _mm256_sub_epi16(coeff2, coeff3);
b0 = _mm256_srai_epi16(b0, 2);
b1 = _mm256_srai_epi16(b1, 2);
b2 = _mm256_srai_epi16(b2, 2);
b3 = _mm256_srai_epi16(b3, 2);
store_tran_low(_mm256_add_epi16(b0, b2), coeff);
store_tran_low(_mm256_add_epi16(b1, b3), coeff + 256);
store_tran_low(_mm256_sub_epi16(b0, b2), coeff + 512);
store_tran_low(_mm256_sub_epi16(b1, b3), coeff + 768);
coeff += 16;
t_coeff += 16;
}
}
int aom_satd_avx2(const tran_low_t *coeff, int length) {
const __m256i one = _mm256_set1_epi16(1);
__m256i accum = _mm256_setzero_si256();
int i;
for (i = 0; i < length; i += 16) {
const __m256i src_line = load_tran_low(coeff);
const __m256i abs = _mm256_abs_epi16(src_line);
const __m256i sum = _mm256_madd_epi16(abs, one);
accum = _mm256_add_epi32(accum, sum);
coeff += 16;
}
{ // 32 bit horizontal add
const __m256i a = _mm256_srli_si256(accum, 8);
const __m256i b = _mm256_add_epi32(accum, a);
const __m256i c = _mm256_srli_epi64(b, 32);
const __m256i d = _mm256_add_epi32(b, c);
const __m128i accum_128 = _mm_add_epi32(_mm256_castsi256_si128(d),
_mm256_extractf128_si256(d, 1));
return _mm_cvtsi128_si32(accum_128);
}
}

285
third_party/aom/aom_dsp/x86/avg_intrin_sse2.c vendored Normal file
View file

@ -0,0 +1,285 @@
/*
* Copyright (c) 2016, Alliance for Open Media. All rights reserved
*
* This source code is subject to the terms of the BSD 2 Clause License and
* the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
* was not distributed with this source code in the LICENSE file, you can
* obtain it at www.aomedia.org/license/software. If the Alliance for Open
* Media Patent License 1.0 was not distributed with this source code in the
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
*/
#include <immintrin.h>
#include "config/aom_dsp_rtcd.h"
#include "aom/aom_integer.h"
#include "aom_dsp/x86/bitdepth_conversion_sse2.h"
#include "aom_ports/mem.h"
static void hadamard_col8_sse2(__m128i *in, int iter) {
__m128i a0 = in[0];
__m128i a1 = in[1];
__m128i a2 = in[2];
__m128i a3 = in[3];
__m128i a4 = in[4];
__m128i a5 = in[5];
__m128i a6 = in[6];
__m128i a7 = in[7];
__m128i b0 = _mm_add_epi16(a0, a1);
__m128i b1 = _mm_sub_epi16(a0, a1);
__m128i b2 = _mm_add_epi16(a2, a3);
__m128i b3 = _mm_sub_epi16(a2, a3);
__m128i b4 = _mm_add_epi16(a4, a5);
__m128i b5 = _mm_sub_epi16(a4, a5);
__m128i b6 = _mm_add_epi16(a6, a7);
__m128i b7 = _mm_sub_epi16(a6, a7);
a0 = _mm_add_epi16(b0, b2);
a1 = _mm_add_epi16(b1, b3);
a2 = _mm_sub_epi16(b0, b2);
a3 = _mm_sub_epi16(b1, b3);
a4 = _mm_add_epi16(b4, b6);
a5 = _mm_add_epi16(b5, b7);
a6 = _mm_sub_epi16(b4, b6);
a7 = _mm_sub_epi16(b5, b7);
if (iter == 0) {
b0 = _mm_add_epi16(a0, a4);
b7 = _mm_add_epi16(a1, a5);
b3 = _mm_add_epi16(a2, a6);
b4 = _mm_add_epi16(a3, a7);
b2 = _mm_sub_epi16(a0, a4);
b6 = _mm_sub_epi16(a1, a5);
b1 = _mm_sub_epi16(a2, a6);
b5 = _mm_sub_epi16(a3, a7);
a0 = _mm_unpacklo_epi16(b0, b1);
a1 = _mm_unpacklo_epi16(b2, b3);
a2 = _mm_unpackhi_epi16(b0, b1);
a3 = _mm_unpackhi_epi16(b2, b3);
a4 = _mm_unpacklo_epi16(b4, b5);
a5 = _mm_unpacklo_epi16(b6, b7);
a6 = _mm_unpackhi_epi16(b4, b5);
a7 = _mm_unpackhi_epi16(b6, b7);
b0 = _mm_unpacklo_epi32(a0, a1);
b1 = _mm_unpacklo_epi32(a4, a5);
b2 = _mm_unpackhi_epi32(a0, a1);
b3 = _mm_unpackhi_epi32(a4, a5);
b4 = _mm_unpacklo_epi32(a2, a3);
b5 = _mm_unpacklo_epi32(a6, a7);
b6 = _mm_unpackhi_epi32(a2, a3);
b7 = _mm_unpackhi_epi32(a6, a7);
in[0] = _mm_unpacklo_epi64(b0, b1);
in[1] = _mm_unpackhi_epi64(b0, b1);
in[2] = _mm_unpacklo_epi64(b2, b3);
in[3] = _mm_unpackhi_epi64(b2, b3);
in[4] = _mm_unpacklo_epi64(b4, b5);
in[5] = _mm_unpackhi_epi64(b4, b5);
in[6] = _mm_unpacklo_epi64(b6, b7);
in[7] = _mm_unpackhi_epi64(b6, b7);
} else {
in[0] = _mm_add_epi16(a0, a4);
in[7] = _mm_add_epi16(a1, a5);
in[3] = _mm_add_epi16(a2, a6);
in[4] = _mm_add_epi16(a3, a7);
in[2] = _mm_sub_epi16(a0, a4);
in[6] = _mm_sub_epi16(a1, a5);
in[1] = _mm_sub_epi16(a2, a6);
in[5] = _mm_sub_epi16(a3, a7);
}
}
static INLINE void hadamard_8x8_sse2(const int16_t *src_diff,
ptrdiff_t src_stride, tran_low_t *coeff,
int is_final) {
__m128i src[8];
src[0] = _mm_load_si128((const __m128i *)src_diff);
src[1] = _mm_load_si128((const __m128i *)(src_diff += src_stride));
src[2] = _mm_load_si128((const __m128i *)(src_diff += src_stride));
src[3] = _mm_load_si128((const __m128i *)(src_diff += src_stride));
src[4] = _mm_load_si128((const __m128i *)(src_diff += src_stride));
src[5] = _mm_load_si128((const __m128i *)(src_diff += src_stride));
src[6] = _mm_load_si128((const __m128i *)(src_diff += src_stride));
src[7] = _mm_load_si128((const __m128i *)(src_diff += src_stride));
hadamard_col8_sse2(src, 0);
hadamard_col8_sse2(src, 1);
if (is_final) {
store_tran_low(src[0], coeff);
coeff += 8;
store_tran_low(src[1], coeff);
coeff += 8;
store_tran_low(src[2], coeff);
coeff += 8;
store_tran_low(src[3], coeff);
coeff += 8;
store_tran_low(src[4], coeff);
coeff += 8;
store_tran_low(src[5], coeff);
coeff += 8;
store_tran_low(src[6], coeff);
coeff += 8;
store_tran_low(src[7], coeff);
} else {
int16_t *coeff16 = (int16_t *)coeff;
_mm_store_si128((__m128i *)coeff16, src[0]);
coeff16 += 8;
_mm_store_si128((__m128i *)coeff16, src[1]);
coeff16 += 8;
_mm_store_si128((__m128i *)coeff16, src[2]);
coeff16 += 8;
_mm_store_si128((__m128i *)coeff16, src[3]);
coeff16 += 8;
_mm_store_si128((__m128i *)coeff16, src[4]);
coeff16 += 8;
_mm_store_si128((__m128i *)coeff16, src[5]);
coeff16 += 8;
_mm_store_si128((__m128i *)coeff16, src[6]);
coeff16 += 8;
_mm_store_si128((__m128i *)coeff16, src[7]);
}
}
void aom_hadamard_8x8_sse2(const int16_t *src_diff, ptrdiff_t src_stride,
tran_low_t *coeff) {
hadamard_8x8_sse2(src_diff, src_stride, coeff, 1);
}
static INLINE void hadamard_16x16_sse2(const int16_t *src_diff,
ptrdiff_t src_stride, tran_low_t *coeff,
int is_final) {
// For high bitdepths, it is unnecessary to store_tran_low
// (mult/unpack/store), then load_tran_low (load/pack) the same memory in the
// next stage. Output to an intermediate buffer first, then store_tran_low()
// in the final stage.
DECLARE_ALIGNED(32, int16_t, temp_coeff[16 * 16]);
int16_t *t_coeff = temp_coeff;
int16_t *coeff16 = (int16_t *)coeff;
int idx;
for (idx = 0; idx < 4; ++idx) {
const int16_t *src_ptr =
src_diff + (idx >> 1) * 8 * src_stride + (idx & 0x01) * 8;
hadamard_8x8_sse2(src_ptr, src_stride, (tran_low_t *)(t_coeff + idx * 64),
0);
}
for (idx = 0; idx < 64; idx += 8) {
__m128i coeff0 = _mm_load_si128((const __m128i *)t_coeff);
__m128i coeff1 = _mm_load_si128((const __m128i *)(t_coeff + 64));
__m128i coeff2 = _mm_load_si128((const __m128i *)(t_coeff + 128));
__m128i coeff3 = _mm_load_si128((const __m128i *)(t_coeff + 192));
__m128i b0 = _mm_add_epi16(coeff0, coeff1);
__m128i b1 = _mm_sub_epi16(coeff0, coeff1);
__m128i b2 = _mm_add_epi16(coeff2, coeff3);
__m128i b3 = _mm_sub_epi16(coeff2, coeff3);
b0 = _mm_srai_epi16(b0, 1);
b1 = _mm_srai_epi16(b1, 1);
b2 = _mm_srai_epi16(b2, 1);
b3 = _mm_srai_epi16(b3, 1);
coeff0 = _mm_add_epi16(b0, b2);
coeff1 = _mm_add_epi16(b1, b3);
coeff2 = _mm_sub_epi16(b0, b2);
coeff3 = _mm_sub_epi16(b1, b3);
if (is_final) {
store_tran_low(coeff0, coeff);
store_tran_low(coeff1, coeff + 64);
store_tran_low(coeff2, coeff + 128);
store_tran_low(coeff3, coeff + 192);
coeff += 8;
} else {
_mm_store_si128((__m128i *)coeff16, coeff0);
_mm_store_si128((__m128i *)(coeff16 + 64), coeff1);
_mm_store_si128((__m128i *)(coeff16 + 128), coeff2);
_mm_store_si128((__m128i *)(coeff16 + 192), coeff3);
coeff16 += 8;
}
t_coeff += 8;
}
}
void aom_hadamard_16x16_sse2(const int16_t *src_diff, ptrdiff_t src_stride,
tran_low_t *coeff) {
hadamard_16x16_sse2(src_diff, src_stride, coeff, 1);
}
void aom_hadamard_32x32_sse2(const int16_t *src_diff, ptrdiff_t src_stride,
tran_low_t *coeff) {
// For high bitdepths, it is unnecessary to store_tran_low
// (mult/unpack/store), then load_tran_low (load/pack) the same memory in the
// next stage. Output to an intermediate buffer first, then store_tran_low()
// in the final stage.
DECLARE_ALIGNED(32, int16_t, temp_coeff[32 * 32]);
int16_t *t_coeff = temp_coeff;
int idx;
for (idx = 0; idx < 4; ++idx) {
const int16_t *src_ptr =
src_diff + (idx >> 1) * 16 * src_stride + (idx & 0x01) * 16;
hadamard_16x16_sse2(src_ptr, src_stride,
(tran_low_t *)(t_coeff + idx * 256), 0);
}
for (idx = 0; idx < 256; idx += 8) {
__m128i coeff0 = _mm_load_si128((const __m128i *)t_coeff);
__m128i coeff1 = _mm_load_si128((const __m128i *)(t_coeff + 256));
__m128i coeff2 = _mm_load_si128((const __m128i *)(t_coeff + 512));
__m128i coeff3 = _mm_load_si128((const __m128i *)(t_coeff + 768));
__m128i b0 = _mm_add_epi16(coeff0, coeff1);
__m128i b1 = _mm_sub_epi16(coeff0, coeff1);
__m128i b2 = _mm_add_epi16(coeff2, coeff3);
__m128i b3 = _mm_sub_epi16(coeff2, coeff3);
b0 = _mm_srai_epi16(b0, 2);
b1 = _mm_srai_epi16(b1, 2);
b2 = _mm_srai_epi16(b2, 2);
b3 = _mm_srai_epi16(b3, 2);
coeff0 = _mm_add_epi16(b0, b2);
coeff1 = _mm_add_epi16(b1, b3);
store_tran_low(coeff0, coeff);
store_tran_low(coeff1, coeff + 256);
coeff2 = _mm_sub_epi16(b0, b2);
coeff3 = _mm_sub_epi16(b1, b3);
store_tran_low(coeff2, coeff + 512);
store_tran_low(coeff3, coeff + 768);
coeff += 8;
t_coeff += 8;
}
}
int aom_satd_sse2(const tran_low_t *coeff, int length) {
int i;
const __m128i zero = _mm_setzero_si128();
__m128i accum = zero;
for (i = 0; i < length; i += 8) {
const __m128i src_line = load_tran_low(coeff);
const __m128i inv = _mm_sub_epi16(zero, src_line);
const __m128i abs = _mm_max_epi16(src_line, inv); // abs(src_line)
const __m128i abs_lo = _mm_unpacklo_epi16(abs, zero);
const __m128i abs_hi = _mm_unpackhi_epi16(abs, zero);
const __m128i sum = _mm_add_epi32(abs_lo, abs_hi);
accum = _mm_add_epi32(accum, sum);
coeff += 8;
}
{ // cascading summation of accum
__m128i hi = _mm_srli_si128(accum, 8);
accum = _mm_add_epi32(accum, hi);
hi = _mm_srli_epi64(accum, 32);
accum = _mm_add_epi32(accum, hi);
}
return _mm_cvtsi128_si32(accum);
}

32
third_party/aom/aom_dsp/x86/bitdepth_conversion_avx2.h vendored Normal file
View file

@ -0,0 +1,32 @@
/*
* Copyright (c) 2016, Alliance for Open Media. All rights reserved
*
* This source code is subject to the terms of the BSD 2 Clause License and
* the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
* was not distributed with this source code in the LICENSE file, you can
* obtain it at www.aomedia.org/license/software. If the Alliance for Open
* Media Patent License 1.0 was not distributed with this source code in the
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
*/
#include <immintrin.h>
#include "config/aom_config.h"
#include "aom/aom_integer.h"
#include "aom_dsp/aom_dsp_common.h"
static INLINE __m256i load_tran_low(const tran_low_t *a) {
const __m256i a_low = _mm256_loadu_si256((const __m256i *)a);
const __m256i a_high = _mm256_loadu_si256((const __m256i *)(a + 8));
return _mm256_packs_epi32(a_low, a_high);
}
static INLINE void store_tran_low(__m256i a, tran_low_t *b) {
const __m256i one = _mm256_set1_epi16(1);
const __m256i a_hi = _mm256_mulhi_epi16(a, one);
const __m256i a_lo = _mm256_mullo_epi16(a, one);
const __m256i a_1 = _mm256_unpacklo_epi16(a_lo, a_hi);
const __m256i a_2 = _mm256_unpackhi_epi16(a_lo, a_hi);
_mm256_storeu_si256((__m256i *)b, a_1);
_mm256_storeu_si256((__m256i *)(b + 8), a_2);
}

35
third_party/aom/aom_dsp/x86/bitdepth_conversion_sse2.h vendored Normal file
View file

@ -0,0 +1,35 @@
/*
* Copyright (c) 2016, Alliance for Open Media. All rights reserved
*
* This source code is subject to the terms of the BSD 2 Clause License and
* the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
* was not distributed with this source code in the LICENSE file, you can
* obtain it at www.aomedia.org/license/software. If the Alliance for Open
* Media Patent License 1.0 was not distributed with this source code in the
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
*/
#include <xmmintrin.h>
#include "config/aom_config.h"
#include "aom/aom_integer.h"
#include "aom_dsp/aom_dsp_common.h"
// Load 8 16 bit values. If the source is 32 bits then pack down with
// saturation.
static INLINE __m128i load_tran_low(const tran_low_t *a) {
const __m128i a_low = _mm_load_si128((const __m128i *)a);
return _mm_packs_epi32(a_low, *(const __m128i *)(a + 4));
}
// Store 8 16 bit values. If the destination is 32 bits then sign extend the
// values by multiplying by 1.
static INLINE void store_tran_low(__m128i a, tran_low_t *b) {
const __m128i one = _mm_set1_epi16(1);
const __m128i a_hi = _mm_mulhi_epi16(a, one);
const __m128i a_lo = _mm_mullo_epi16(a, one);
const __m128i a_1 = _mm_unpacklo_epi16(a_lo, a_hi);
const __m128i a_2 = _mm_unpackhi_epi16(a_lo, a_hi);
_mm_store_si128((__m128i *)(b), a_1);
_mm_store_si128((__m128i *)(b + 4), a_2);
}

472
third_party/aom/aom_dsp/x86/blend_a64_mask_avx2.c vendored
View file

@ -898,3 +898,475 @@ void aom_blend_a64_mask_avx2(uint8_t *dst, uint32_t dst_stride,
}
}
}
//////////////////////////////////////////////////////////////////////////////
// aom_highbd_blend_a64_d16_mask_avx2()
//////////////////////////////////////////////////////////////////////////////
static INLINE void highbd_blend_a64_d16_mask_w4_avx2(
uint16_t *dst, int dst_stride, const CONV_BUF_TYPE *src0, int src0_stride,
const CONV_BUF_TYPE *src1, int src1_stride, const __m256i *mask0,
const __m256i *round_offset, int shift, const __m256i *clip_low,
const __m256i *clip_high, const __m256i *mask_max) {
// Load 4x u16 pixels from each of 4 rows from each source
const __m256i s0 = _mm256_set_epi64x(*(uint64_t *)(src0 + 3 * src0_stride),
*(uint64_t *)(src0 + 2 * src0_stride),
*(uint64_t *)(src0 + 1 * src0_stride),
*(uint64_t *)(src0 + 0 * src0_stride));
const __m256i s1 = _mm256_set_epi64x(*(uint64_t *)(src1 + 3 * src1_stride),
*(uint64_t *)(src1 + 2 * src1_stride),
*(uint64_t *)(src1 + 1 * src1_stride),
*(uint64_t *)(src1 + 0 * src1_stride));
// Generate the inverse mask
const __m256i mask1 = _mm256_sub_epi16(*mask_max, *mask0);
// Multiply each mask by the respective source
const __m256i mul0_highs = _mm256_mulhi_epu16(*mask0, s0);
const __m256i mul0_lows = _mm256_mullo_epi16(*mask0, s0);
const __m256i mul0h = _mm256_unpackhi_epi16(mul0_lows, mul0_highs);
const __m256i mul0l = _mm256_unpacklo_epi16(mul0_lows, mul0_highs);
// Note that AVX2 unpack orders 64-bit words as [3 1] [2 0] to keep within
// lanes Later, packs does the same again which cancels this out with no need
// for a permute. The intermediate values being reordered makes no difference
const __m256i mul1_highs = _mm256_mulhi_epu16(mask1, s1);
const __m256i mul1_lows = _mm256_mullo_epi16(mask1, s1);
const __m256i mul1h = _mm256_unpackhi_epi16(mul1_lows, mul1_highs);
const __m256i mul1l = _mm256_unpacklo_epi16(mul1_lows, mul1_highs);
const __m256i sumh = _mm256_add_epi32(mul0h, mul1h);
const __m256i suml = _mm256_add_epi32(mul0l, mul1l);
const __m256i roundh =
_mm256_srai_epi32(_mm256_sub_epi32(sumh, *round_offset), shift);
const __m256i roundl =
_mm256_srai_epi32(_mm256_sub_epi32(suml, *round_offset), shift);
const __m256i pack = _mm256_packs_epi32(roundl, roundh);
const __m256i clip =
_mm256_min_epi16(_mm256_max_epi16(pack, *clip_low), *clip_high);
// _mm256_extract_epi64 doesn't exist on x86, so do it the old-fashioned way:
const __m128i cliph = _mm256_extracti128_si256(clip, 1);
xx_storel_64(dst + 3 * dst_stride, _mm_srli_si128(cliph, 8));
xx_storel_64(dst + 2 * dst_stride, cliph);
const __m128i clipl = _mm256_castsi256_si128(clip);
xx_storel_64(dst + 1 * dst_stride, _mm_srli_si128(clipl, 8));
xx_storel_64(dst + 0 * dst_stride, clipl);
}
static INLINE void highbd_blend_a64_d16_mask_subw0_subh0_w4_avx2(
uint16_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0,
uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride,
const uint8_t *mask, uint32_t mask_stride, int h,
const __m256i *round_offset, int shift, const __m256i *clip_low,
const __m256i *clip_high, const __m256i *mask_max) {
do {
// Load 8x u8 pixels from each of 4 rows of the mask, pad each to u16
const __m128i mask08 = _mm_set_epi32(*(uint32_t *)(mask + 3 * mask_stride),
*(uint32_t *)(mask + 2 * mask_stride),
*(uint32_t *)(mask + 1 * mask_stride),
*(uint32_t *)(mask + 0 * mask_stride));
const __m256i mask0 = _mm256_cvtepu8_epi16(mask08);
highbd_blend_a64_d16_mask_w4_avx2(dst, dst_stride, src0, src0_stride, src1,
src1_stride, &mask0, round_offset, shift,
clip_low, clip_high, mask_max);
dst += dst_stride * 4;
src0 += src0_stride * 4;
src1 += src1_stride * 4;
mask += mask_stride * 4;
} while (h -= 4);
}
static INLINE void highbd_blend_a64_d16_mask_subw1_subh1_w4_avx2(
uint16_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0,
uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride,
const uint8_t *mask, uint32_t mask_stride, int h,
const __m256i *round_offset, int shift, const __m256i *clip_low,
const __m256i *clip_high, const __m256i *mask_max) {
const __m256i one_b = _mm256_set1_epi8(1);
const __m256i two_w = _mm256_set1_epi16(2);
do {
// Load 8 pixels from each of 8 rows of mask,
// (saturating) add together rows then use madd to add adjacent pixels
// Finally, divide each value by 4 (with rounding)
const __m256i m0246 =
_mm256_set_epi64x(*(uint64_t *)(mask + 6 * mask_stride),
*(uint64_t *)(mask + 4 * mask_stride),
*(uint64_t *)(mask + 2 * mask_stride),
*(uint64_t *)(mask + 0 * mask_stride));
const __m256i m1357 =
_mm256_set_epi64x(*(uint64_t *)(mask + 7 * mask_stride),
*(uint64_t *)(mask + 5 * mask_stride),
*(uint64_t *)(mask + 3 * mask_stride),
*(uint64_t *)(mask + 1 * mask_stride));
const __m256i addrows = _mm256_adds_epu8(m0246, m1357);
const __m256i adjacent = _mm256_maddubs_epi16(addrows, one_b);
const __m256i mask0 =
_mm256_srli_epi16(_mm256_add_epi16(adjacent, two_w), 2);
highbd_blend_a64_d16_mask_w4_avx2(dst, dst_stride, src0, src0_stride, src1,
src1_stride, &mask0, round_offset, shift,
clip_low, clip_high, mask_max);
dst += dst_stride * 4;
src0 += src0_stride * 4;
src1 += src1_stride * 4;
mask += mask_stride * 8;
} while (h -= 4);
}
static INLINE void highbd_blend_a64_d16_mask_w8_avx2(
uint16_t *dst, int dst_stride, const CONV_BUF_TYPE *src0, int src0_stride,
const CONV_BUF_TYPE *src1, int src1_stride, const __m256i *mask0a,
const __m256i *mask0b, const __m256i *round_offset, int shift,
const __m256i *clip_low, const __m256i *clip_high,
const __m256i *mask_max) {
// Load 8x u16 pixels from each of 4 rows from each source
const __m256i s0a =
yy_loadu2_128(src0 + 0 * src0_stride, src0 + 1 * src0_stride);
const __m256i s0b =
yy_loadu2_128(src0 + 2 * src0_stride, src0 + 3 * src0_stride);
const __m256i s1a =
yy_loadu2_128(src1 + 0 * src1_stride, src1 + 1 * src1_stride);
const __m256i s1b =
yy_loadu2_128(src1 + 2 * src1_stride, src1 + 3 * src1_stride);
// Generate inverse masks
const __m256i mask1a = _mm256_sub_epi16(*mask_max, *mask0a);
const __m256i mask1b = _mm256_sub_epi16(*mask_max, *mask0b);
// Multiply sources by respective masks
const __m256i mul0a_highs = _mm256_mulhi_epu16(*mask0a, s0a);
const __m256i mul0a_lows = _mm256_mullo_epi16(*mask0a, s0a);
const __m256i mul0ah = _mm256_unpackhi_epi16(mul0a_lows, mul0a_highs);
const __m256i mul0al = _mm256_unpacklo_epi16(mul0a_lows, mul0a_highs);
// Note that AVX2 unpack orders 64-bit words as [3 1] [2 0] to keep within
// lanes Later, packs does the same again which cancels this out with no need
// for a permute. The intermediate values being reordered makes no difference
const __m256i mul1a_highs = _mm256_mulhi_epu16(mask1a, s1a);
const __m256i mul1a_lows = _mm256_mullo_epi16(mask1a, s1a);
const __m256i mul1ah = _mm256_unpackhi_epi16(mul1a_lows, mul1a_highs);
const __m256i mul1al = _mm256_unpacklo_epi16(mul1a_lows, mul1a_highs);
const __m256i sumah = _mm256_add_epi32(mul0ah, mul1ah);
const __m256i sumal = _mm256_add_epi32(mul0al, mul1al);
const __m256i mul0b_highs = _mm256_mulhi_epu16(*mask0b, s0b);
const __m256i mul0b_lows = _mm256_mullo_epi16(*mask0b, s0b);
const __m256i mul0bh = _mm256_unpackhi_epi16(mul0b_lows, mul0b_highs);
const __m256i mul0bl = _mm256_unpacklo_epi16(mul0b_lows, mul0b_highs);
const __m256i mul1b_highs = _mm256_mulhi_epu16(mask1b, s1b);
const __m256i mul1b_lows = _mm256_mullo_epi16(mask1b, s1b);
const __m256i mul1bh = _mm256_unpackhi_epi16(mul1b_lows, mul1b_highs);
const __m256i mul1bl = _mm256_unpacklo_epi16(mul1b_lows, mul1b_highs);
const __m256i sumbh = _mm256_add_epi32(mul0bh, mul1bh);
const __m256i sumbl = _mm256_add_epi32(mul0bl, mul1bl);
// Divide down each result, with rounding
const __m256i roundah =
_mm256_srai_epi32(_mm256_sub_epi32(sumah, *round_offset), shift);
const __m256i roundal =
_mm256_srai_epi32(_mm256_sub_epi32(sumal, *round_offset), shift);
const __m256i roundbh =
_mm256_srai_epi32(_mm256_sub_epi32(sumbh, *round_offset), shift);
const __m256i roundbl =
_mm256_srai_epi32(_mm256_sub_epi32(sumbl, *round_offset), shift);
// Pack each i32 down to an i16 with saturation, then clip to valid range
const __m256i packa = _mm256_packs_epi32(roundal, roundah);
const __m256i clipa =
_mm256_min_epi16(_mm256_max_epi16(packa, *clip_low), *clip_high);
const __m256i packb = _mm256_packs_epi32(roundbl, roundbh);
const __m256i clipb =
_mm256_min_epi16(_mm256_max_epi16(packb, *clip_low), *clip_high);
// Store 8x u16 pixels to each of 4 rows in the destination
yy_storeu2_128(dst + 0 * dst_stride, dst + 1 * dst_stride, clipa);
yy_storeu2_128(dst + 2 * dst_stride, dst + 3 * dst_stride, clipb);
}
static INLINE void highbd_blend_a64_d16_mask_subw0_subh0_w8_avx2(
uint16_t *dst, int dst_stride, const CONV_BUF_TYPE *src0, int src0_stride,
const CONV_BUF_TYPE *src1, int src1_stride, const uint8_t *mask,
int mask_stride, int h, const __m256i *round_offset, int shift,
const __m256i *clip_low, const __m256i *clip_high,
const __m256i *mask_max) {
do {
// Load 8x u8 pixels from each of 4 rows in the mask
const __m128i mask0a8 =
_mm_set_epi64x(*(uint64_t *)mask, *(uint64_t *)(mask + mask_stride));
const __m128i mask0b8 =
_mm_set_epi64x(*(uint64_t *)(mask + 2 * mask_stride),
*(uint64_t *)(mask + 3 * mask_stride));
const __m256i mask0a = _mm256_cvtepu8_epi16(mask0a8);
const __m256i mask0b = _mm256_cvtepu8_epi16(mask0b8);
highbd_blend_a64_d16_mask_w8_avx2(
dst, dst_stride, src0, src0_stride, src1, src1_stride, &mask0a, &mask0b,
round_offset, shift, clip_low, clip_high, mask_max);
dst += dst_stride * 4;
src0 += src0_stride * 4;
src1 += src1_stride * 4;
mask += mask_stride * 4;
} while (h -= 4);
}
static INLINE void highbd_blend_a64_d16_mask_subw1_subh1_w8_avx2(
uint16_t *dst, int dst_stride, const CONV_BUF_TYPE *src0, int src0_stride,
const CONV_BUF_TYPE *src1, int src1_stride, const uint8_t *mask,
int mask_stride, int h, const __m256i *round_offset, int shift,
const __m256i *clip_low, const __m256i *clip_high,
const __m256i *mask_max) {
const __m256i one_b = _mm256_set1_epi8(1);
const __m256i two_w = _mm256_set1_epi16(2);
do {
// Load 16x u8 pixels from each of 8 rows in the mask,
// (saturating) add together rows then use madd to add adjacent pixels
// Finally, divide each value by 4 (with rounding)
const __m256i m02 =
yy_loadu2_128(mask + 0 * mask_stride, mask + 2 * mask_stride);
const __m256i m13 =
yy_loadu2_128(mask + 1 * mask_stride, mask + 3 * mask_stride);
const __m256i m0123 =
_mm256_maddubs_epi16(_mm256_adds_epu8(m02, m13), one_b);
const __m256i mask_0a =
_mm256_srli_epi16(_mm256_add_epi16(m0123, two_w), 2);
const __m256i m46 =
yy_loadu2_128(mask + 4 * mask_stride, mask + 6 * mask_stride);
const __m256i m57 =
yy_loadu2_128(mask + 5 * mask_stride, mask + 7 * mask_stride);
const __m256i m4567 =
_mm256_maddubs_epi16(_mm256_adds_epu8(m46, m57), one_b);
const __m256i mask_0b =
_mm256_srli_epi16(_mm256_add_epi16(m4567, two_w), 2);
highbd_blend_a64_d16_mask_w8_avx2(
dst, dst_stride, src0, src0_stride, src1, src1_stride, &mask_0a,
&mask_0b, round_offset, shift, clip_low, clip_high, mask_max);
dst += dst_stride * 4;
src0 += src0_stride * 4;
src1 += src1_stride * 4;
mask += mask_stride * 8;
} while (h -= 4);
}
static INLINE void highbd_blend_a64_d16_mask_w16_avx2(
uint16_t *dst, int dst_stride, const CONV_BUF_TYPE *src0, int src0_stride,
const CONV_BUF_TYPE *src1, int src1_stride, const __m256i *mask0a,
const __m256i *mask0b, const __m256i *round_offset, int shift,
const __m256i *clip_low, const __m256i *clip_high,
const __m256i *mask_max) {
// Load 16x pixels from each of 2 rows from each source
const __m256i s0a = yy_loadu_256(src0);
const __m256i s0b = yy_loadu_256(src0 + src0_stride);
const __m256i s1a = yy_loadu_256(src1);
const __m256i s1b = yy_loadu_256(src1 + src1_stride);
// Calculate inverse masks
const __m256i mask1a = _mm256_sub_epi16(*mask_max, *mask0a);
const __m256i mask1b = _mm256_sub_epi16(*mask_max, *mask0b);
// Multiply each source by appropriate mask
const __m256i mul0a_highs = _mm256_mulhi_epu16(*mask0a, s0a);
const __m256i mul0a_lows = _mm256_mullo_epi16(*mask0a, s0a);
const __m256i mul0ah = _mm256_unpackhi_epi16(mul0a_lows, mul0a_highs);
const __m256i mul0al = _mm256_unpacklo_epi16(mul0a_lows, mul0a_highs);
// Note that AVX2 unpack orders 64-bit words as [3 1] [2 0] to keep within
// lanes Later, packs does the same again which cancels this out with no need
// for a permute. The intermediate values being reordered makes no difference
const __m256i mul1a_highs = _mm256_mulhi_epu16(mask1a, s1a);
const __m256i mul1a_lows = _mm256_mullo_epi16(mask1a, s1a);
const __m256i mul1ah = _mm256_unpackhi_epi16(mul1a_lows, mul1a_highs);
const __m256i mul1al = _mm256_unpacklo_epi16(mul1a_lows, mul1a_highs);
const __m256i mulah = _mm256_add_epi32(mul0ah, mul1ah);
const __m256i mulal = _mm256_add_epi32(mul0al, mul1al);
const __m256i mul0b_highs = _mm256_mulhi_epu16(*mask0b, s0b);
const __m256i mul0b_lows = _mm256_mullo_epi16(*mask0b, s0b);
const __m256i mul0bh = _mm256_unpackhi_epi16(mul0b_lows, mul0b_highs);
const __m256i mul0bl = _mm256_unpacklo_epi16(mul0b_lows, mul0b_highs);
const __m256i mul1b_highs = _mm256_mulhi_epu16(mask1b, s1b);
const __m256i mul1b_lows = _mm256_mullo_epi16(mask1b, s1b);
const __m256i mul1bh = _mm256_unpackhi_epi16(mul1b_lows, mul1b_highs);
const __m256i mul1bl = _mm256_unpacklo_epi16(mul1b_lows, mul1b_highs);
const __m256i mulbh = _mm256_add_epi32(mul0bh, mul1bh);
const __m256i mulbl = _mm256_add_epi32(mul0bl, mul1bl);
const __m256i resah =
_mm256_srai_epi32(_mm256_sub_epi32(mulah, *round_offset), shift);
const __m256i resal =
_mm256_srai_epi32(_mm256_sub_epi32(mulal, *round_offset), shift);
const __m256i resbh =
_mm256_srai_epi32(_mm256_sub_epi32(mulbh, *round_offset), shift);
const __m256i resbl =
_mm256_srai_epi32(_mm256_sub_epi32(mulbl, *round_offset), shift);
// Signed saturating pack from i32 to i16:
const __m256i packa = _mm256_packs_epi32(resal, resah);
const __m256i packb = _mm256_packs_epi32(resbl, resbh);
// Clip the values to the valid range
const __m256i clipa =
_mm256_min_epi16(_mm256_max_epi16(packa, *clip_low), *clip_high);
const __m256i clipb =
_mm256_min_epi16(_mm256_max_epi16(packb, *clip_low), *clip_high);
// Store 16 pixels
yy_storeu_256(dst, clipa);
yy_storeu_256(dst + dst_stride, clipb);
}
static INLINE void highbd_blend_a64_d16_mask_subw0_subh0_w16_avx2(
uint16_t *dst, int dst_stride, const CONV_BUF_TYPE *src0, int src0_stride,
const CONV_BUF_TYPE *src1, int src1_stride, const uint8_t *mask,
int mask_stride, int h, int w, const __m256i *round_offset, int shift,
const __m256i *clip_low, const __m256i *clip_high,
const __m256i *mask_max) {
for (int i = 0; i < h; i += 2) {
for (int j = 0; j < w; j += 16) {
// Load 16x u8 alpha-mask values from each of two rows and pad to u16
const __m128i masks_a8 = xx_loadu_128(mask + j);
const __m128i masks_b8 = xx_loadu_128(mask + mask_stride + j);
const __m256i mask0a = _mm256_cvtepu8_epi16(masks_a8);
const __m256i mask0b = _mm256_cvtepu8_epi16(masks_b8);
highbd_blend_a64_d16_mask_w16_avx2(
dst + j, dst_stride, src0 + j, src0_stride, src1 + j, src1_stride,
&mask0a, &mask0b, round_offset, shift, clip_low, clip_high, mask_max);
}
dst += dst_stride * 2;
src0 += src0_stride * 2;
src1 += src1_stride * 2;
mask += mask_stride * 2;
}
}
static INLINE void highbd_blend_a64_d16_mask_subw1_subh1_w16_avx2(
uint16_t *dst, int dst_stride, const CONV_BUF_TYPE *src0, int src0_stride,
const CONV_BUF_TYPE *src1, int src1_stride, const uint8_t *mask,
int mask_stride, int h, int w, const __m256i *round_offset, int shift,
const __m256i *clip_low, const __m256i *clip_high,
const __m256i *mask_max) {
const __m256i one_b = _mm256_set1_epi8(1);
const __m256i two_w = _mm256_set1_epi16(2);
for (int i = 0; i < h; i += 2) {
for (int j = 0; j < w; j += 16) {
// Load 32x u8 alpha-mask values from each of four rows
// (saturating) add pairs of rows, then use madd to add adjacent values
// Finally, divide down each result with rounding
const __m256i m0 = yy_loadu_256(mask + 0 * mask_stride + 2 * j);
const __m256i m1 = yy_loadu_256(mask + 1 * mask_stride + 2 * j);
const __m256i m2 = yy_loadu_256(mask + 2 * mask_stride + 2 * j);
const __m256i m3 = yy_loadu_256(mask + 3 * mask_stride + 2 * j);
const __m256i m01_8 = _mm256_adds_epu8(m0, m1);
const __m256i m23_8 = _mm256_adds_epu8(m2, m3);
const __m256i m01 = _mm256_maddubs_epi16(m01_8, one_b);
const __m256i m23 = _mm256_maddubs_epi16(m23_8, one_b);
const __m256i mask0a = _mm256_srli_epi16(_mm256_add_epi16(m01, two_w), 2);
const __m256i mask0b = _mm256_srli_epi16(_mm256_add_epi16(m23, two_w), 2);
highbd_blend_a64_d16_mask_w16_avx2(
dst + j, dst_stride, src0 + j, src0_stride, src1 + j, src1_stride,
&mask0a, &mask0b, round_offset, shift, clip_low, clip_high, mask_max);
}
dst += dst_stride * 2;
src0 += src0_stride * 2;
src1 += src1_stride * 2;
mask += mask_stride * 4;
}
}
void aom_highbd_blend_a64_d16_mask_avx2(
uint8_t *dst8, uint32_t dst_stride, const CONV_BUF_TYPE *src0,
uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride,
const uint8_t *mask, uint32_t mask_stride, int w, int h, int subw, int subh,
ConvolveParams *conv_params, const int bd) {
uint16_t *dst = CONVERT_TO_SHORTPTR(dst8);
const int round_bits =
2 * FILTER_BITS - conv_params->round_0 - conv_params->round_1;
const int32_t round_offset =
((1 << (round_bits + bd)) + (1 << (round_bits + bd - 1)) -
(1 << (round_bits - 1)))
<< AOM_BLEND_A64_ROUND_BITS;
const __m256i v_round_offset = _mm256_set1_epi32(round_offset);
const int shift = round_bits + AOM_BLEND_A64_ROUND_BITS;
const __m256i clip_low = _mm256_set1_epi16(0);
const __m256i clip_high = _mm256_set1_epi16((1 << bd) - 1);
const __m256i mask_max = _mm256_set1_epi16(AOM_BLEND_A64_MAX_ALPHA);
assert(IMPLIES((void *)src0 == dst, src0_stride == dst_stride));
assert(IMPLIES((void *)src1 == dst, src1_stride == dst_stride));
assert(h >= 4);
assert(w >= 4);
assert(IS_POWER_OF_TWO(h));
assert(IS_POWER_OF_TWO(w));
if (subw == 0 && subh == 0) {
switch (w) {
case 4:
highbd_blend_a64_d16_mask_subw0_subh0_w4_avx2(
dst, dst_stride, src0, src0_stride, src1, src1_stride, mask,
mask_stride, h, &v_round_offset, shift, &clip_low, &clip_high,
&mask_max);
break;
case 8:
highbd_blend_a64_d16_mask_subw0_subh0_w8_avx2(
dst, dst_stride, src0, src0_stride, src1, src1_stride, mask,
mask_stride, h, &v_round_offset, shift, &clip_low, &clip_high,
&mask_max);
break;
default: // >= 16
highbd_blend_a64_d16_mask_subw0_subh0_w16_avx2(
dst, dst_stride, src0, src0_stride, src1, src1_stride, mask,
mask_stride, h, w, &v_round_offset, shift, &clip_low, &clip_high,
&mask_max);
break;
}
} else if (subw == 1 && subh == 1) {
switch (w) {
case 4:
highbd_blend_a64_d16_mask_subw1_subh1_w4_avx2(
dst, dst_stride, src0, src0_stride, src1, src1_stride, mask,
mask_stride, h, &v_round_offset, shift, &clip_low, &clip_high,
&mask_max);
break;
case 8:
highbd_blend_a64_d16_mask_subw1_subh1_w8_avx2(
dst, dst_stride, src0, src0_stride, src1, src1_stride, mask,
mask_stride, h, &v_round_offset, shift, &clip_low, &clip_high,
&mask_max);
break;
default: // >= 16
highbd_blend_a64_d16_mask_subw1_subh1_w16_avx2(
dst, dst_stride, src0, src0_stride, src1, src1_stride, mask,
mask_stride, h, w, &v_round_offset, shift, &clip_low, &clip_high,
&mask_max);
break;
}
} else {
// Sub-sampling in only one axis doesn't seem to happen very much, so fall
// back to the vanilla C implementation instead of having all the optimised
// code for these.
aom_highbd_blend_a64_d16_mask_c(dst8, dst_stride, src0, src0_stride, src1,
src1_stride, mask, mask_stride, w, h, subw,
subh, conv_params, bd);
}
}

449
third_party/aom/aom_dsp/x86/blend_a64_mask_sse4.c vendored
View file

@ -1107,3 +1107,452 @@ void aom_lowbd_blend_a64_d16_mask_sse4_1(
}
}
}
//////////////////////////////////////////////////////////////////////////////
// aom_highbd_blend_a64_d16_mask_sse4_1()
//////////////////////////////////////////////////////////////////////////////
static INLINE void highbd_blend_a64_d16_mask_w4_sse4_1(
uint16_t *dst, int dst_stride, const CONV_BUF_TYPE *src0, int src0_stride,
const CONV_BUF_TYPE *src1, int src1_stride, const __m128i *mask0a,
const __m128i *mask0b, const __m128i *round_offset, int shift,
const __m128i *clip_low, const __m128i *clip_high,
const __m128i *mask_max) {
// Load 4 pixels from each of 4 rows from each source
const __m128i s0a =
_mm_set_epi64x(*(uint64_t *)src0, *(uint64_t *)(src0 + src0_stride));
const __m128i s0b = _mm_set_epi64x(*(uint64_t *)(src0 + 2 * src0_stride),
*(uint64_t *)(src0 + 3 * src0_stride));
const __m128i s1a =
_mm_set_epi64x(*(uint64_t *)(src1), *(uint64_t *)(src1 + src1_stride));
const __m128i s1b = _mm_set_epi64x(*(uint64_t *)(src1 + 2 * src1_stride),
*(uint64_t *)(src1 + 3 * src1_stride));
// Generate the inverse masks
const __m128i mask1a = _mm_sub_epi16(*mask_max, *mask0a);
const __m128i mask1b = _mm_sub_epi16(*mask_max, *mask0b);
// Multiply each mask by the respective source
const __m128i mul0a_highs = _mm_mulhi_epu16(*mask0a, s0a);
const __m128i mul0a_lows = _mm_mullo_epi16(*mask0a, s0a);
const __m128i mul0ah = _mm_unpackhi_epi16(mul0a_lows, mul0a_highs);
const __m128i mul0al = _mm_unpacklo_epi16(mul0a_lows, mul0a_highs);
const __m128i mul1a_highs = _mm_mulhi_epu16(mask1a, s1a);
const __m128i mul1a_lows = _mm_mullo_epi16(mask1a, s1a);
const __m128i mul1ah = _mm_unpackhi_epi16(mul1a_lows, mul1a_highs);
const __m128i mul1al = _mm_unpacklo_epi16(mul1a_lows, mul1a_highs);
const __m128i mul0b_highs = _mm_mulhi_epu16(*mask0b, s0b);
const __m128i mul0b_lows = _mm_mullo_epi16(*mask0b, s0b);
const __m128i mul0bh = _mm_unpackhi_epi16(mul0b_lows, mul0b_highs);
const __m128i mul0bl = _mm_unpacklo_epi16(mul0b_lows, mul0b_highs);
const __m128i mul1b_highs = _mm_mulhi_epu16(mask1b, s1b);
const __m128i mul1b_lows = _mm_mullo_epi16(mask1b, s1b);
const __m128i mul1bh = _mm_unpackhi_epi16(mul1b_lows, mul1b_highs);
const __m128i mul1bl = _mm_unpacklo_epi16(mul1b_lows, mul1b_highs);
const __m128i sumah = _mm_add_epi32(mul0ah, mul1ah);
const __m128i sumal = _mm_add_epi32(mul0al, mul1al);
const __m128i sumbh = _mm_add_epi32(mul0bh, mul1bh);
const __m128i sumbl = _mm_add_epi32(mul0bl, mul1bl);
const __m128i roundah =
_mm_srai_epi32(_mm_sub_epi32(sumah, *round_offset), shift);
const __m128i roundbh =
_mm_srai_epi32(_mm_sub_epi32(sumbh, *round_offset), shift);
const __m128i roundal =
_mm_srai_epi32(_mm_sub_epi32(sumal, *round_offset), shift);
const __m128i roundbl =
_mm_srai_epi32(_mm_sub_epi32(sumbl, *round_offset), shift);
const __m128i packa = _mm_packs_epi32(roundal, roundah);
const __m128i packb = _mm_packs_epi32(roundbl, roundbh);
const __m128i clipa =
_mm_min_epi16(_mm_max_epi16(packa, *clip_low), *clip_high);
const __m128i clipb =
_mm_min_epi16(_mm_max_epi16(packb, *clip_low), *clip_high);
xx_storel_64(dst, _mm_srli_si128(clipa, 8));
xx_storel_64(dst + dst_stride, clipa);
xx_storel_64(dst + 2 * dst_stride, _mm_srli_si128(clipb, 8));
xx_storel_64(dst + 3 * dst_stride, clipb);
}
static INLINE void highbd_blend_a64_d16_mask_subw0_subh0_w4_sse4_1(
uint16_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0,
uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride,
const uint8_t *mask, uint32_t mask_stride, int h,
const __m128i *round_offset, int shift, const __m128i *clip_low,
const __m128i *clip_high, const __m128i *mask_max) {
do {
const __m128i mask0a8 = _mm_set_epi32(0, 0, *(uint32_t *)mask,
*(uint32_t *)(mask + mask_stride));
const __m128i mask0b8 =
_mm_set_epi32(0, 0, *(uint32_t *)(mask + 2 * mask_stride),
*(uint32_t *)(mask + 3 * mask_stride));
const __m128i mask0a = _mm_cvtepu8_epi16(mask0a8);
const __m128i mask0b = _mm_cvtepu8_epi16(mask0b8);
highbd_blend_a64_d16_mask_w4_sse4_1(
dst, dst_stride, src0, src0_stride, src1, src1_stride, &mask0a, &mask0b,
round_offset, shift, clip_low, clip_high, mask_max);
dst += dst_stride * 4;
src0 += src0_stride * 4;
src1 += src1_stride * 4;
mask += mask_stride * 4;
} while (h -= 4);
}
static INLINE void highbd_blend_a64_d16_mask_subw1_subh1_w4_sse4_1(
uint16_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0,
uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride,
const uint8_t *mask, uint32_t mask_stride, int h,
const __m128i *round_offset, int shift, const __m128i *clip_low,
const __m128i *clip_high, const __m128i *mask_max) {
const __m128i one_b = _mm_set1_epi8(1);
const __m128i two_w = _mm_set1_epi16(2);
do {
// Load 8 pixels from each of 8 rows of mask,
// (saturating) add together rows then use madd to add adjacent pixels
// Finally, divide each value by 4 (with rounding)
const __m128i m02 = _mm_set_epi64x(*(uint64_t *)(mask),
*(uint64_t *)(mask + 2 * mask_stride));
const __m128i m13 = _mm_set_epi64x(*(uint64_t *)(mask + mask_stride),
*(uint64_t *)(mask + 3 * mask_stride));
const __m128i m0123 = _mm_maddubs_epi16(_mm_adds_epu8(m02, m13), one_b);
const __m128i mask_0a = _mm_srli_epi16(_mm_add_epi16(m0123, two_w), 2);
const __m128i m46 = _mm_set_epi64x(*(uint64_t *)(mask + 4 * mask_stride),
*(uint64_t *)(mask + 6 * mask_stride));
const __m128i m57 = _mm_set_epi64x(*(uint64_t *)(mask + 5 * mask_stride),
*(uint64_t *)(mask + 7 * mask_stride));
const __m128i m4567 = _mm_maddubs_epi16(_mm_adds_epu8(m46, m57), one_b);
const __m128i mask_0b = _mm_srli_epi16(_mm_add_epi16(m4567, two_w), 2);
highbd_blend_a64_d16_mask_w4_sse4_1(
dst, dst_stride, src0, src0_stride, src1, src1_stride, &mask_0a,
&mask_0b, round_offset, shift, clip_low, clip_high, mask_max);
dst += dst_stride * 4;
src0 += src0_stride * 4;
src1 += src1_stride * 4;
mask += mask_stride * 8;
} while (h -= 4);
}
static INLINE void highbd_blend_a64_d16_mask_w8_sse4_1(
uint16_t *dst, int dst_stride, const CONV_BUF_TYPE *src0, int src0_stride,
const CONV_BUF_TYPE *src1, int src1_stride, const __m128i *mask0a,
const __m128i *mask0b, const __m128i *round_offset, int shift,
const __m128i *clip_low, const __m128i *clip_high,
const __m128i *max_mask) {
// Load 8x pixels from each of 2 rows from each source
const __m128i s0a = xx_loadu_128(src0);
const __m128i s0b = xx_loadu_128(src0 + src0_stride);
const __m128i s1a = xx_loadu_128(src1);
const __m128i s1b = xx_loadu_128(src1 + src1_stride);
// Generate inverse masks
const __m128i mask1a = _mm_sub_epi16(*max_mask, *mask0a);
const __m128i mask1b = _mm_sub_epi16(*max_mask, *mask0b);
// Multiply sources by respective masks
const __m128i mul0a_highs = _mm_mulhi_epu16(*mask0a, s0a);
const __m128i mul0a_lows = _mm_mullo_epi16(*mask0a, s0a);
const __m128i mul0ah = _mm_unpackhi_epi16(mul0a_lows, mul0a_highs);
const __m128i mul0al = _mm_unpacklo_epi16(mul0a_lows, mul0a_highs);
const __m128i mul1a_highs = _mm_mulhi_epu16(mask1a, s1a);
const __m128i mul1a_lows = _mm_mullo_epi16(mask1a, s1a);
const __m128i mul1ah = _mm_unpackhi_epi16(mul1a_lows, mul1a_highs);
const __m128i mul1al = _mm_unpacklo_epi16(mul1a_lows, mul1a_highs);
const __m128i sumah = _mm_add_epi32(mul0ah, mul1ah);
const __m128i sumal = _mm_add_epi32(mul0al, mul1al);
const __m128i mul0b_highs = _mm_mulhi_epu16(*mask0b, s0b);
const __m128i mul0b_lows = _mm_mullo_epi16(*mask0b, s0b);
const __m128i mul0bh = _mm_unpackhi_epi16(mul0b_lows, mul0b_highs);
const __m128i mul0bl = _mm_unpacklo_epi16(mul0b_lows, mul0b_highs);
const __m128i mul1b_highs = _mm_mulhi_epu16(mask1b, s1b);
const __m128i mul1b_lows = _mm_mullo_epi16(mask1b, s1b);
const __m128i mul1bh = _mm_unpackhi_epi16(mul1b_lows, mul1b_highs);
const __m128i mul1bl = _mm_unpacklo_epi16(mul1b_lows, mul1b_highs);
const __m128i sumbh = _mm_add_epi32(mul0bh, mul1bh);
const __m128i sumbl = _mm_add_epi32(mul0bl, mul1bl);
const __m128i roundah =
_mm_srai_epi32(_mm_sub_epi32(sumah, *round_offset), shift);
const __m128i roundal =
_mm_srai_epi32(_mm_sub_epi32(sumal, *round_offset), shift);
const __m128i roundbh =
_mm_srai_epi32(_mm_sub_epi32(sumbh, *round_offset), shift);
const __m128i roundbl =
_mm_srai_epi32(_mm_sub_epi32(sumbl, *round_offset), shift);
const __m128i packa = _mm_packs_epi32(roundal, roundah);
const __m128i clipa =
_mm_min_epi16(_mm_max_epi16(packa, *clip_low), *clip_high);
const __m128i packb = _mm_packs_epi32(roundbl, roundbh);
const __m128i clipb =
_mm_min_epi16(_mm_max_epi16(packb, *clip_low), *clip_high);
xx_storeu_128(dst, clipa);
xx_storeu_128(dst + dst_stride, clipb);
}
static INLINE void highbd_blend_a64_d16_mask_subw0_subh0_w8_sse4_1(
uint16_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0,
uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride,
const uint8_t *mask, uint32_t mask_stride, int h,
const __m128i *round_offset, int shift, const __m128i *clip_low,
const __m128i *clip_high, const __m128i *max_mask) {
do {
const __m128i mask0a = _mm_cvtepu8_epi16(xx_loadl_64(mask));
const __m128i mask0b = _mm_cvtepu8_epi16(xx_loadl_64(mask + mask_stride));
highbd_blend_a64_d16_mask_w8_sse4_1(
dst, dst_stride, src0, src0_stride, src1, src1_stride, &mask0a, &mask0b,
round_offset, shift, clip_low, clip_high, max_mask);
dst += dst_stride * 2;
src0 += src0_stride * 2;
src1 += src1_stride * 2;
mask += mask_stride * 2;
} while (h -= 2);
}
static INLINE void highbd_blend_a64_d16_mask_subw1_subh1_w8_sse4_1(
uint16_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0,
uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride,
const uint8_t *mask, uint32_t mask_stride, int h,
const __m128i *round_offset, int shift, const __m128i *clip_low,
const __m128i *clip_high, const __m128i *max_mask) {
const __m128i one_b = _mm_set1_epi8(1);
const __m128i two_w = _mm_set1_epi16(2);
do {
const __m128i mask_thisrowa = xx_loadu_128(mask);
const __m128i mask_nextrowa = xx_loadu_128(mask + mask_stride);
const __m128i mask_thisrowb = xx_loadu_128(mask + 2 * mask_stride);
const __m128i mask_nextrowb = xx_loadu_128(mask + 3 * mask_stride);
const __m128i mask_bothrowsa = _mm_adds_epu8(mask_thisrowa, mask_nextrowa);
const __m128i mask_bothrowsb = _mm_adds_epu8(mask_thisrowb, mask_nextrowb);
const __m128i mask_16a = _mm_maddubs_epi16(mask_bothrowsa, one_b);
const __m128i mask_16b = _mm_maddubs_epi16(mask_bothrowsb, one_b);
const __m128i mask_sa = _mm_srli_epi16(_mm_add_epi16(mask_16a, two_w), 2);
const __m128i mask_sb = _mm_srli_epi16(_mm_add_epi16(mask_16b, two_w), 2);
highbd_blend_a64_d16_mask_w8_sse4_1(
dst, dst_stride, src0, src0_stride, src1, src1_stride, &mask_sa,
&mask_sb, round_offset, shift, clip_low, clip_high, max_mask);
dst += dst_stride * 2;
src0 += src0_stride * 2;
src1 += src1_stride * 2;
mask += mask_stride * 4;
} while (h -= 2);
}
static INLINE void highbd_blend_a64_d16_mask_w16_sse4_1(
uint16_t *dst, const CONV_BUF_TYPE *src0, const CONV_BUF_TYPE *src1,
const __m128i *round_offset, int shift, const __m128i *mask0l,
const __m128i *mask0h, const __m128i *clip_low, const __m128i *clip_high,
const __m128i *mask_max) {
// Load 16x u16 pixels for this row from each src
const __m128i s0l = xx_loadu_128(src0);
const __m128i s0h = xx_loadu_128(src0 + 8);
const __m128i s1l = xx_loadu_128(src1);
const __m128i s1h = xx_loadu_128(src1 + 8);
// Calculate inverse masks
const __m128i mask1h = _mm_sub_epi16(*mask_max, *mask0h);
const __m128i mask1l = _mm_sub_epi16(*mask_max, *mask0l);
const __m128i mul0_highs = _mm_mulhi_epu16(*mask0h, s0h);
const __m128i mul0_lows = _mm_mullo_epi16(*mask0h, s0h);
const __m128i mul0h = _mm_unpackhi_epi16(mul0_lows, mul0_highs);
const __m128i mul0l = _mm_unpacklo_epi16(mul0_lows, mul0_highs);
const __m128i mul1_highs = _mm_mulhi_epu16(mask1h, s1h);
const __m128i mul1_lows = _mm_mullo_epi16(mask1h, s1h);
const __m128i mul1h = _mm_unpackhi_epi16(mul1_lows, mul1_highs);
const __m128i mul1l = _mm_unpacklo_epi16(mul1_lows, mul1_highs);
const __m128i mulhh = _mm_add_epi32(mul0h, mul1h);
const __m128i mulhl = _mm_add_epi32(mul0l, mul1l);
const __m128i mul2_highs = _mm_mulhi_epu16(*mask0l, s0l);
const __m128i mul2_lows = _mm_mullo_epi16(*mask0l, s0l);
const __m128i mul2h = _mm_unpackhi_epi16(mul2_lows, mul2_highs);
const __m128i mul2l = _mm_unpacklo_epi16(mul2_lows, mul2_highs);
const __m128i mul3_highs = _mm_mulhi_epu16(mask1l, s1l);
const __m128i mul3_lows = _mm_mullo_epi16(mask1l, s1l);
const __m128i mul3h = _mm_unpackhi_epi16(mul3_lows, mul3_highs);
const __m128i mul3l = _mm_unpacklo_epi16(mul3_lows, mul3_highs);
const __m128i mullh = _mm_add_epi32(mul2h, mul3h);
const __m128i mulll = _mm_add_epi32(mul2l, mul3l);
const __m128i reshh =
_mm_srai_epi32(_mm_sub_epi32(mulhh, *round_offset), shift);
const __m128i reshl =
_mm_srai_epi32(_mm_sub_epi32(mulhl, *round_offset), shift);
const __m128i reslh =
_mm_srai_epi32(_mm_sub_epi32(mullh, *round_offset), shift);
const __m128i resll =
_mm_srai_epi32(_mm_sub_epi32(mulll, *round_offset), shift);
// Signed saturating pack from i32 to i16:
const __m128i packh = _mm_packs_epi32(reshl, reshh);
const __m128i packl = _mm_packs_epi32(resll, reslh);
// Clip the values to the valid range
const __m128i cliph =
_mm_min_epi16(_mm_max_epi16(packh, *clip_low), *clip_high);
const __m128i clipl =
_mm_min_epi16(_mm_max_epi16(packl, *clip_low), *clip_high);
// Store 16 pixels
xx_storeu_128(dst, clipl);
xx_storeu_128(dst + 8, cliph);
}
static INLINE void highbd_blend_a64_d16_mask_subw0_subh0_w16_sse4_1(
uint16_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0,
uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride,
const uint8_t *mask, uint32_t mask_stride, int h, int w,
const __m128i *round_offset, int shift, const __m128i *clip_low,
const __m128i *clip_high, const __m128i *mask_max) {
for (int i = 0; i < h; i++) {
for (int j = 0; j < w; j += 16) {
// Load 16x u8 alpha-mask values and pad to u16
const __m128i masks_u8 = xx_loadu_128(mask + j);
const __m128i mask0l = _mm_cvtepu8_epi16(masks_u8);
const __m128i mask0h = _mm_cvtepu8_epi16(_mm_srli_si128(masks_u8, 8));
highbd_blend_a64_d16_mask_w16_sse4_1(
dst + j, src0 + j, src1 + j, round_offset, shift, &mask0l, &mask0h,
clip_low, clip_high, mask_max);
}
dst += dst_stride;
src0 += src0_stride;
src1 += src1_stride;
mask += mask_stride;
}
}
static INLINE void highbd_blend_a64_d16_mask_subw1_subh1_w16_sse4_1(
uint16_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0,
uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride,
const uint8_t *mask, uint32_t mask_stride, int h, int w,
const __m128i *round_offset, int shift, const __m128i *clip_low,
const __m128i *clip_high, const __m128i *mask_max) {
const __m128i one_b = _mm_set1_epi8(1);
const __m128i two_w = _mm_set1_epi16(2);
for (int i = 0; i < h; i++) {
for (int j = 0; j < w; j += 16) {
const __m128i m_i00 = xx_loadu_128(mask + 2 * j);
const __m128i m_i01 = xx_loadu_128(mask + 2 * j + 16);
const __m128i m_i10 = xx_loadu_128(mask + mask_stride + 2 * j);
const __m128i m_i11 = xx_loadu_128(mask + mask_stride + 2 * j + 16);
const __m128i m0_ac = _mm_adds_epu8(m_i00, m_i10);
const __m128i m1_ac = _mm_adds_epu8(m_i01, m_i11);
const __m128i m0_acbd = _mm_maddubs_epi16(m0_ac, one_b);
const __m128i m1_acbd = _mm_maddubs_epi16(m1_ac, one_b);
const __m128i mask_l = _mm_srli_epi16(_mm_add_epi16(m0_acbd, two_w), 2);
const __m128i mask_h = _mm_srli_epi16(_mm_add_epi16(m1_acbd, two_w), 2);
highbd_blend_a64_d16_mask_w16_sse4_1(
dst + j, src0 + j, src1 + j, round_offset, shift, &mask_l, &mask_h,
clip_low, clip_high, mask_max);
}
dst += dst_stride;
src0 += src0_stride;
src1 += src1_stride;
mask += mask_stride * 2;
}
}
void aom_highbd_blend_a64_d16_mask_sse4_1(
uint8_t *dst8, uint32_t dst_stride, const CONV_BUF_TYPE *src0,
uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride,
const uint8_t *mask, uint32_t mask_stride, int w, int h, int subw, int subh,
ConvolveParams *conv_params, const int bd) {
uint16_t *dst = CONVERT_TO_SHORTPTR(dst8);
const int round_bits =
2 * FILTER_BITS - conv_params->round_0 - conv_params->round_1;
const int32_t round_offset =
((1 << (round_bits + bd)) + (1 << (round_bits + bd - 1)) -
(1 << (round_bits - 1)))
<< AOM_BLEND_A64_ROUND_BITS;
const __m128i v_round_offset = _mm_set1_epi32(round_offset);
const int shift = round_bits + AOM_BLEND_A64_ROUND_BITS;
const __m128i clip_low = _mm_set1_epi16(0);
const __m128i clip_high = _mm_set1_epi16((1 << bd) - 1);
const __m128i mask_max = _mm_set1_epi16(AOM_BLEND_A64_MAX_ALPHA);
assert(IMPLIES((void *)src0 == dst, src0_stride == dst_stride));
assert(IMPLIES((void *)src1 == dst, src1_stride == dst_stride));
assert(h >= 4);
assert(w >= 4);
assert(IS_POWER_OF_TWO(h));
assert(IS_POWER_OF_TWO(w));
if (subw == 0 && subh == 0) {
switch (w) {
case 4:
highbd_blend_a64_d16_mask_subw0_subh0_w4_sse4_1(
dst, dst_stride, src0, src0_stride, src1, src1_stride, mask,
mask_stride, h, &v_round_offset, shift, &clip_low, &clip_high,
&mask_max);
break;
case 8:
highbd_blend_a64_d16_mask_subw0_subh0_w8_sse4_1(
dst, dst_stride, src0, src0_stride, src1, src1_stride, mask,
mask_stride, h, &v_round_offset, shift, &clip_low, &clip_high,
&mask_max);
break;
default: // >=16
highbd_blend_a64_d16_mask_subw0_subh0_w16_sse4_1(
dst, dst_stride, src0, src0_stride, src1, src1_stride, mask,
mask_stride, h, w, &v_round_offset, shift, &clip_low, &clip_high,
&mask_max);
break;
}
} else if (subw == 1 && subh == 1) {
switch (w) {
case 4:
highbd_blend_a64_d16_mask_subw1_subh1_w4_sse4_1(
dst, dst_stride, src0, src0_stride, src1, src1_stride, mask,
mask_stride, h, &v_round_offset, shift, &clip_low, &clip_high,
&mask_max);
break;
case 8:
highbd_blend_a64_d16_mask_subw1_subh1_w8_sse4_1(
dst, dst_stride, src0, src0_stride, src1, src1_stride, mask,
mask_stride, h, &v_round_offset, shift, &clip_low, &clip_high,
&mask_max);
break;
default: // >=16
highbd_blend_a64_d16_mask_subw1_subh1_w16_sse4_1(
dst, dst_stride, src0, src0_stride, src1, src1_stride, mask,
mask_stride, h, w, &v_round_offset, shift, &clip_low, &clip_high,
&mask_max);
break;
}
} else {
// Sub-sampling in only one axis doesn't seem to happen very much, so fall
// back to the vanilla C implementation instead of having all the optimised
// code for these.
aom_highbd_blend_a64_d16_mask_c(dst8, dst_stride, src0, src0_stride, src1,
src1_stride, mask, mask_stride, w, h, subw,
subh, conv_params, bd);
}
}

25
third_party/aom/aom_dsp/x86/convolve.h vendored
View file

@ -122,7 +122,30 @@ typedef void highbd_filter8_1dfunction(const uint16_t *src_ptr,
uint16_t *src = CONVERT_TO_SHORTPTR(src8); \
uint16_t *dst = CONVERT_TO_SHORTPTR(dst8); \
if (step_q4 == 16 && filter[3] != 128) { \
if (filter[0] | filter[1] | filter[2]) { \
if (((filter[0] | filter[1] | filter[6] | filter[7]) == 0) && \
(filter[2] | filter[5])) { \
while (w >= 16) { \
aom_highbd_filter_block1d16_##dir##4_##avg##opt( \
src_start, src_stride, dst, dst_stride, h, filter, bd); \
src += 16; \
dst += 16; \
w -= 16; \
} \
while (w >= 8) { \
aom_highbd_filter_block1d8_##dir##4_##avg##opt( \
src_start, src_stride, dst, dst_stride, h, filter, bd); \
src += 8; \
dst += 8; \
w -= 8; \
} \
while (w >= 4) { \
aom_highbd_filter_block1d4_##dir##4_##avg##opt( \
src_start, src_stride, dst, dst_stride, h, filter, bd); \
src += 4; \
dst += 4; \
w -= 4; \
} \
} else if (filter[0] | filter[1] | filter[2]) { \
while (w >= 16) { \
aom_highbd_filter_block1d16_##dir##8_##avg##opt( \
src_start, src_stride, dst, dst_stride, h, filter, bd); \

42
third_party/aom/aom_dsp/x86/convolve_avx2.h vendored
View file

@ -34,6 +34,31 @@ DECLARE_ALIGNED(32, static const uint8_t, filt4_d4_global_avx2[]) = {
2, 3, 4, 5, 3, 4, 5, 6, 4, 5, 6, 7, 5, 6, 7, 8,
};
DECLARE_ALIGNED(32, static const uint8_t, filt_center_global_avx2[32]) = {
3, 255, 4, 255, 5, 255, 6, 255, 7, 255, 8, 255, 9, 255, 10, 255,
3, 255, 4, 255, 5, 255, 6, 255, 7, 255, 8, 255, 9, 255, 10, 255
};
DECLARE_ALIGNED(32, static const uint8_t, filt1_global_avx2[32]) = {
0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8,
0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8
};
DECLARE_ALIGNED(32, static const uint8_t, filt2_global_avx2[32]) = {
2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10,
2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10
};
DECLARE_ALIGNED(32, static const uint8_t, filt3_global_avx2[32]) = {
4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12,
4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12
};
DECLARE_ALIGNED(32, static const uint8_t, filt4_global_avx2[32]) = {
6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14,
6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14
};
static INLINE void prepare_coeffs_lowbd(
const InterpFilterParams *const filter_params, const int subpel_q4,
__m256i *const coeffs /* [4] */) {
@ -108,6 +133,15 @@ static INLINE __m256i convolve(const __m256i *const s,
return res;
}
static INLINE __m256i convolve_4tap(const __m256i *const s,
const __m256i *const coeffs) {
const __m256i res_1 = _mm256_madd_epi16(s[0], coeffs[0]);
const __m256i res_2 = _mm256_madd_epi16(s[1], coeffs[1]);
const __m256i res = _mm256_add_epi32(res_1, res_2);
return res;
}
static INLINE __m256i convolve_lowbd_x(const __m256i data,
const __m256i *const coeffs,
const __m256i *const filt) {
@ -138,9 +172,9 @@ static INLINE void add_store_aligned_256(CONV_BUF_TYPE *const dst,
static INLINE __m256i comp_avg(const __m256i *const data_ref_0,
const __m256i *const res_unsigned,
const __m256i *const wt,
const int use_jnt_comp_avg) {
const int use_dist_wtd_comp_avg) {
__m256i res;
if (use_jnt_comp_avg) {
if (use_dist_wtd_comp_avg) {
const __m256i data_lo = _mm256_unpacklo_epi16(*data_ref_0, *res_unsigned);
const __m256i data_hi = _mm256_unpackhi_epi16(*data_ref_0, *res_unsigned);
@ -172,9 +206,9 @@ static INLINE __m256i highbd_comp_avg(const __m256i *const data_ref_0,
const __m256i *const res_unsigned,
const __m256i *const wt0,
const __m256i *const wt1,
const int use_jnt_comp_avg) {
const int use_dist_wtd_comp_avg) {
__m256i res;
if (use_jnt_comp_avg) {
if (use_dist_wtd_comp_avg) {
const __m256i wt0_res = _mm256_mullo_epi32(*data_ref_0, *wt0);
const __m256i wt1_res = _mm256_mullo_epi32(*res_unsigned, *wt1);
const __m256i wt_res = _mm256_add_epi32(wt0_res, wt1_res);

4
third_party/aom/aom_dsp/x86/convolve_sse2.h vendored
View file

@ -78,9 +78,9 @@ static INLINE __m128i convolve_hi_y(const __m128i *const s,
static INLINE __m128i comp_avg(const __m128i *const data_ref_0,
const __m128i *const res_unsigned,
const __m128i *const wt,
const int use_jnt_comp_avg) {
const int use_dist_wtd_avg) {
__m128i res;
if (use_jnt_comp_avg) {
if (use_dist_wtd_avg) {
const __m128i data_lo = _mm_unpacklo_epi16(*data_ref_0, *res_unsigned);
const __m128i data_hi = _mm_unpackhi_epi16(*data_ref_0, *res_unsigned);

4
third_party/aom/aom_dsp/x86/convolve_sse4_1.h vendored
View file

@ -35,9 +35,9 @@ static INLINE __m128i highbd_comp_avg_sse4_1(const __m128i *const data_ref_0,
const __m128i *const res_unsigned,
const __m128i *const wt0,
const __m128i *const wt1,
const int use_jnt_comp_avg) {
const int use_dist_wtd_avg) {
__m128i res;
if (use_jnt_comp_avg) {
if (use_dist_wtd_avg) {
const __m128i wt0_res = _mm_mullo_epi32(*data_ref_0, *wt0);
const __m128i wt1_res = _mm_mullo_epi32(*res_unsigned, *wt1);

1
third_party/aom/aom_dsp/x86/fft_avx2.c vendored
View file

@ -11,6 +11,7 @@
#include <immintrin.h>
#include "config/aom_dsp_rtcd.h"
#include "aom_dsp/aom_dsp_common.h"
#include "aom_dsp/fft_common.h"

1
third_party/aom/aom_dsp/x86/fft_sse2.c vendored
View file

@ -11,6 +11,7 @@ s * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
#include <xmmintrin.h>
#include "config/aom_dsp_rtcd.h"
#include "aom_dsp/aom_dsp_common.h"
#include "aom_dsp/fft_common.h"

40
third_party/aom/aom_dsp/x86/fwd_txfm_sse2.c vendored
View file

@ -17,46 +17,6 @@
#include "aom_dsp/aom_dsp_common.h"
#include "aom_dsp/x86/fwd_txfm_sse2.h"
void aom_fdct8x8_1_sse2(const int16_t *input, tran_low_t *output, int stride) {
__m128i in0 = _mm_load_si128((const __m128i *)(input + 0 * stride));
__m128i in1 = _mm_load_si128((const __m128i *)(input + 1 * stride));
__m128i in2 = _mm_load_si128((const __m128i *)(input + 2 * stride));
__m128i in3 = _mm_load_si128((const __m128i *)(input + 3 * stride));
__m128i u0, u1, sum;
u0 = _mm_add_epi16(in0, in1);
u1 = _mm_add_epi16(in2, in3);
in0 = _mm_load_si128((const __m128i *)(input + 4 * stride));
in1 = _mm_load_si128((const __m128i *)(input + 5 * stride));
in2 = _mm_load_si128((const __m128i *)(input + 6 * stride));
in3 = _mm_load_si128((const __m128i *)(input + 7 * stride));
sum = _mm_add_epi16(u0, u1);
in0 = _mm_add_epi16(in0, in1);
in2 = _mm_add_epi16(in2, in3);
sum = _mm_add_epi16(sum, in0);
u0 = _mm_setzero_si128();
sum = _mm_add_epi16(sum, in2);
in0 = _mm_unpacklo_epi16(u0, sum);
in1 = _mm_unpackhi_epi16(u0, sum);
in0 = _mm_srai_epi32(in0, 16);
in1 = _mm_srai_epi32(in1, 16);
sum = _mm_add_epi32(in0, in1);
in0 = _mm_unpacklo_epi32(sum, u0);
in1 = _mm_unpackhi_epi32(sum, u0);
sum = _mm_add_epi32(in0, in1);
in0 = _mm_srli_si128(sum, 8);
in1 = _mm_add_epi32(sum, in0);
output[0] = (tran_low_t)_mm_cvtsi128_si32(in1);
}
#define DCT_HIGH_BIT_DEPTH 0
#define FDCT8x8_2D aom_fdct8x8_sse2
#include "aom_dsp/x86/fwd_txfm_impl_sse2.h"

325
third_party/aom/aom_dsp/x86/highbd_convolve_avx2.c vendored
View file

@ -20,6 +20,14 @@
// -----------------------------------------------------------------------------
// Copy and average
static const uint8_t ip_shuffle_f2f3[32] = { 0, 1, 2, 3, 2, 3, 4, 5, 4, 5, 6,
7, 6, 7, 8, 9, 0, 1, 2, 3, 2, 3,
4, 5, 4, 5, 6, 7, 6, 7, 8, 9 };
static const uint8_t ip_shuffle_f4f5[32] = { 4, 5, 6, 7, 6, 7, 8, 9,
8, 9, 10, 11, 10, 11, 12, 13,
4, 5, 6, 7, 6, 7, 8, 9,
8, 9, 10, 11, 10, 11, 12, 13 };
void aom_highbd_convolve_copy_avx2(const uint8_t *src8, ptrdiff_t src_stride,
uint8_t *dst8, ptrdiff_t dst_stride,
const int16_t *filter_x, int filter_x_stride,
@ -444,6 +452,17 @@ static INLINE void pack_filters(const int16_t *filter, __m256i *f /*f[4]*/) {
f[3] = _mm256_shuffle_epi8(hh, p3);
}
static INLINE void pack_filters_4tap(const int16_t *filter,
__m256i *f /*f[4]*/) {
const __m128i h = _mm_loadu_si128((const __m128i *)filter);
const __m256i coeff = _mm256_broadcastsi128_si256(h);
// coeffs 2 3 2 3 2 3 2 3
f[0] = _mm256_shuffle_epi32(coeff, 0x55);
// coeffs 4 5 4 5 4 5 4 5
f[1] = _mm256_shuffle_epi32(coeff, 0xaa);
}
static INLINE void filter_8x1_pixels(const __m256i *sig /*sig[4]*/,
const __m256i *fil /*fil[4]*/,
__m256i *y) {
@ -544,6 +563,176 @@ static void aom_highbd_filter_block1d16_h8_avx2(
} while (height > 0);
}
static void aom_highbd_filter_block1d4_h4_avx2(
const uint16_t *src_ptr, ptrdiff_t src_pitch, uint16_t *dst_ptr,
ptrdiff_t dst_pitch, uint32_t height, const int16_t *filter, int bd) {
const __m256i rounding = _mm256_set1_epi32(1 << (CONV8_ROUNDING_BITS - 1));
__m256i ff[2], s[2];
uint32_t i;
const __m256i clip_pixel =
_mm256_set1_epi16(bd == 10 ? 1023 : (bd == 12 ? 4095 : 255));
const __m256i zero = _mm256_setzero_si256();
static const uint8_t shuffle_mask[32] = { 0, 1, 2, 3, 2, 3, 4, 5, 4, 5, 6,
7, 6, 7, 8, 9, 0, 1, 2, 3, 2, 3,
4, 5, 4, 5, 6, 7, 6, 7, 8, 9 };
__m256i mask = _mm256_loadu_si256((__m256i *)shuffle_mask);
__m256i ip_mask_f2f3 = _mm256_loadu_si256((__m256i *)ip_shuffle_f2f3);
__m256i ip_mask_f4f5 = _mm256_loadu_si256((__m256i *)ip_shuffle_f4f5);
pack_filters_4tap(filter, ff);
src_ptr -= 3;
for (i = 0; i <= (height - 2); i += 2) {
__m256i row0 = _mm256_castsi128_si256(
_mm_loadu_si128((__m128i *)&src_ptr[i * src_pitch + 2]));
__m256i row1 = _mm256_castsi128_si256(
_mm_loadu_si128((__m128i *)&src_ptr[(i + 1) * src_pitch + 2]));
s[0] = _mm256_inserti128_si256(row0, _mm256_castsi256_si128(row1), 1);
s[1] = _mm256_alignr_epi8(s[0], s[0], 4);
s[0] = _mm256_shuffle_epi8(s[0], mask);
s[1] = _mm256_shuffle_epi8(s[1], mask);
__m256i res = convolve_4tap(s, ff);
res =
_mm256_srai_epi32(_mm256_add_epi32(res, rounding), CONV8_ROUNDING_BITS);
res = _mm256_packs_epi32(res, res);
res = _mm256_min_epi16(res, clip_pixel);
res = _mm256_max_epi16(res, zero);
_mm_storel_epi64((__m128i *)&dst_ptr[i * dst_pitch],
_mm256_castsi256_si128(res));
_mm_storel_epi64((__m128i *)&dst_ptr[(i + 1) * dst_pitch],
_mm256_extracti128_si256(res, 1));
}
if (height % 2 != 0) {
i = height - 1;
const __m256i row0_0 = _mm256_castsi128_si256(
_mm_loadu_si128((__m128i *)&src_ptr[i * src_pitch + 2]));
const __m256i row0_1 = _mm256_castsi128_si256(
_mm_loadu_si128((__m128i *)&src_ptr[i * src_pitch + 6]));
const __m256i r0 =
_mm256_inserti128_si256(row0_0, _mm256_castsi256_si128(row0_1), 1);
s[0] = _mm256_shuffle_epi8(r0, ip_mask_f2f3);
s[1] = _mm256_shuffle_epi8(r0, ip_mask_f4f5);
__m256i res = convolve_4tap(s, ff);
res =
_mm256_srai_epi32(_mm256_add_epi32(res, rounding), CONV8_ROUNDING_BITS);
res = _mm256_packs_epi32(res, res);
res = _mm256_min_epi16(res, clip_pixel);
res = _mm256_max_epi16(res, zero);
_mm_storel_epi64((__m128i *)&dst_ptr[i * dst_pitch],
_mm256_castsi256_si128(res));
}
}
static void aom_highbd_filter_block1d8_h4_avx2(
const uint16_t *src_ptr, ptrdiff_t src_pitch, uint16_t *dst_ptr,
ptrdiff_t dst_pitch, uint32_t height, const int16_t *filter, int bd) {
const __m256i rounding = _mm256_set1_epi32(1 << (CONV8_ROUNDING_BITS - 1));
__m256i ff[2], s[2];
uint32_t i = 0;
const __m256i clip_pixel =
_mm256_set1_epi16(bd == 10 ? 1023 : (bd == 12 ? 4095 : 255));
const __m256i zero = _mm256_setzero_si256();
static const uint8_t shuffle_mask[32] = { 0, 1, 8, 9, 2, 3, 10, 11,
4, 5, 12, 13, 6, 7, 14, 15,
0, 1, 8, 9, 2, 3, 10, 11,
4, 5, 12, 13, 6, 7, 14, 15 };
__m256i mask = _mm256_loadu_si256((__m256i *)shuffle_mask);
__m256i ip_mask_f2f3 = _mm256_loadu_si256((__m256i *)ip_shuffle_f2f3);
__m256i ip_mask_f4f5 = _mm256_loadu_si256((__m256i *)ip_shuffle_f4f5);
pack_filters_4tap(filter, ff);
src_ptr -= 3;
/* Horizontal filter */
for (i = 0; i <= (height - 2); i += 2) {
const __m256i row0 =
_mm256_loadu_si256((__m256i *)&src_ptr[i * src_pitch + 2]);
__m256i row1 =
_mm256_loadu_si256((__m256i *)&src_ptr[(i + 1) * src_pitch + 2]);
const __m256i r0 =
_mm256_inserti128_si256(row0, _mm256_castsi256_si128(row1), 1);
const __m256i r1 = _mm256_permute2x128_si256(row0, row1, 0x31);
// even pixels
s[0] = r0;
s[1] = _mm256_alignr_epi8(r1, r0, 4);
__m256i res_even = convolve_4tap(s, ff);
res_even = _mm256_srai_epi32(_mm256_add_epi32(res_even, rounding),
CONV8_ROUNDING_BITS);
// odd pixels
s[0] = _mm256_alignr_epi8(r1, r0, 2);
s[1] = _mm256_alignr_epi8(r1, r0, 6);
__m256i res_odd = convolve_4tap(s, ff);
res_odd = _mm256_srai_epi32(_mm256_add_epi32(res_odd, rounding),
CONV8_ROUNDING_BITS);
__m256i res = _mm256_packs_epi32(res_even, res_odd);
res = _mm256_shuffle_epi8(res, mask);
res = _mm256_min_epi16(res, clip_pixel);
res = _mm256_max_epi16(res, zero);
_mm_storeu_si128((__m128i *)&dst_ptr[i * dst_pitch],
_mm256_castsi256_si128(res));
_mm_storeu_si128((__m128i *)&dst_ptr[i * dst_pitch + dst_pitch],
_mm256_extracti128_si256(res, 1));
}
if (height % 2 != 0) {
i = height - 1;
const __m256i row0_0 =
_mm256_loadu_si256((__m256i *)&src_ptr[i * src_pitch + 2]);
const __m256i row0_1 =
_mm256_loadu_si256((__m256i *)&src_ptr[i * src_pitch + 6]);
const __m256i r0 =
_mm256_inserti128_si256(row0_0, _mm256_castsi256_si128(row0_1), 1);
s[0] = _mm256_shuffle_epi8(r0, ip_mask_f2f3);
s[1] = _mm256_shuffle_epi8(r0, ip_mask_f4f5);
__m256i res = convolve_4tap(s, ff);
res =
_mm256_srai_epi32(_mm256_add_epi32(res, rounding), CONV8_ROUNDING_BITS);
res = _mm256_packs_epi32(res, res);
res = _mm256_min_epi16(res, clip_pixel);
res = _mm256_max_epi16(res, zero);
_mm_storel_epi64((__m128i *)&dst_ptr[i * dst_pitch],
_mm256_castsi256_si128(res));
_mm_storel_epi64((__m128i *)&dst_ptr[i * dst_pitch + 4],
_mm256_extracti128_si256(res, 1));
}
}
static void aom_highbd_filter_block1d16_h4_avx2(
const uint16_t *src_ptr, ptrdiff_t src_pitch, uint16_t *dst_ptr,
ptrdiff_t dst_pitch, uint32_t height, const int16_t *filter, int bd) {
aom_highbd_filter_block1d8_h4_avx2(src_ptr, src_pitch, dst_ptr, dst_pitch,
height, filter, bd);
aom_highbd_filter_block1d8_h4_avx2(src_ptr + 8, src_pitch, dst_ptr + 8,
dst_pitch, height, filter, bd);
}
// -----------------------------------------------------------------------------
// 2-tap horizontal filtering
@ -875,6 +1064,142 @@ static void aom_highbd_filter_block1d16_v8_avx2(
} while (height > 0);
}
static void aom_highbd_filter_block1d4_v4_avx2(
const uint16_t *src_ptr, ptrdiff_t src_pitch, uint16_t *dst_ptr,
ptrdiff_t dst_pitch, uint32_t height, const int16_t *filter, int bd) {
const int bits = FILTER_BITS;
const __m128i round_shift_bits = _mm_cvtsi32_si128(bits);
const __m256i round_const_bits = _mm256_set1_epi32((1 << bits) >> 1);
const __m256i clip_pixel =
_mm256_set1_epi32(bd == 10 ? 1023 : (bd == 12 ? 4095 : 255));
const __m256i zero = _mm256_setzero_si256();
uint32_t i;
__m256i s[2], ff[2];
pack_filters_4tap(filter, ff);
const uint16_t *data = src_ptr;
/* Vertical filter */
{
__m128i s2 = _mm_loadl_epi64((__m128i *)(data + 2 * src_pitch));
__m128i s3 = _mm_loadl_epi64((__m128i *)(data + 3 * src_pitch));
__m256i s23 = _mm256_inserti128_si256(_mm256_castsi128_si256(s2), s3, 1);
__m128i s4 = _mm_loadl_epi64((__m128i *)(data + 4 * src_pitch));
__m256i s34 = _mm256_inserti128_si256(_mm256_castsi128_si256(s3), s4, 1);
s[0] = _mm256_unpacklo_epi16(s23, s34);
for (i = 0; i < height; i += 2) {
data = &src_ptr[i * src_pitch];
__m128i s5 = _mm_loadl_epi64((__m128i *)(data + 5 * src_pitch));
__m128i s6 = _mm_loadl_epi64((__m128i *)(data + 6 * src_pitch));
__m256i s45 = _mm256_inserti128_si256(_mm256_castsi128_si256(s4), s5, 1);
__m256i s56 = _mm256_inserti128_si256(_mm256_castsi128_si256(s5), s6, 1);
s[1] = _mm256_unpacklo_epi16(s45, s56);
const __m256i res_a = convolve_4tap(s, ff);
__m256i res_a_round = _mm256_sra_epi32(
_mm256_add_epi32(res_a, round_const_bits), round_shift_bits);
__m256i res_16bit = _mm256_min_epi32(res_a_round, clip_pixel);
res_16bit = _mm256_max_epi32(res_16bit, zero);
res_16bit = _mm256_packs_epi32(res_16bit, res_16bit);
_mm_storel_epi64((__m128i *)&dst_ptr[i * dst_pitch],
_mm256_castsi256_si128(res_16bit));
_mm_storel_epi64((__m128i *)&dst_ptr[i * dst_pitch + dst_pitch],
_mm256_extracti128_si256(res_16bit, 1));
s[0] = s[1];
s4 = s6;
}
}
}
static void aom_highbd_filter_block1d8_v4_avx2(
const uint16_t *src_ptr, ptrdiff_t src_pitch, uint16_t *dst_ptr,
ptrdiff_t dst_pitch, uint32_t height, const int16_t *filter, int bd) {
const int bits = FILTER_BITS;
const __m128i round_shift_bits = _mm_cvtsi32_si128(bits);
const __m256i round_const_bits = _mm256_set1_epi32((1 << bits) >> 1);
const __m256i clip_pixel =
_mm256_set1_epi16(bd == 10 ? 1023 : (bd == 12 ? 4095 : 255));
const __m256i zero = _mm256_setzero_si256();
__m256i s[4], ff[2];
uint32_t i;
pack_filters_4tap(filter, ff);
const uint16_t *data = src_ptr;
/* Vertical filter */
{
__m128i s2 = _mm_loadu_si128((__m128i *)(data + 2 * src_pitch));
__m128i s3 = _mm_loadu_si128((__m128i *)(data + 3 * src_pitch));
__m256i s23 = _mm256_inserti128_si256(_mm256_castsi128_si256(s2), s3, 1);
__m128i s4 = _mm_loadu_si128((__m128i *)(data + 4 * src_pitch));
__m256i s34 = _mm256_inserti128_si256(_mm256_castsi128_si256(s3), s4, 1);
s[0] = _mm256_unpacklo_epi16(s23, s34);
s[2] = _mm256_unpackhi_epi16(s23, s34);
for (i = 0; i < height; i += 2) {
data = &src_ptr[i * src_pitch];
__m128i s5 = _mm_loadu_si128((__m128i *)(data + 5 * src_pitch));
__m128i s6 = _mm_loadu_si128((__m128i *)(data + 6 * src_pitch));
__m256i s45 = _mm256_inserti128_si256(_mm256_castsi128_si256(s4), s5, 1);
__m256i s56 = _mm256_inserti128_si256(_mm256_castsi128_si256(s5), s6, 1);
s[1] = _mm256_unpacklo_epi16(s45, s56);
s[3] = _mm256_unpackhi_epi16(s45, s56);
const __m256i res_a = convolve_4tap(s, ff);
__m256i res_a_round = _mm256_sra_epi32(
_mm256_add_epi32(res_a, round_const_bits), round_shift_bits);
const __m256i res_b = convolve_4tap(s + 2, ff);
__m256i res_b_round = _mm256_sra_epi32(
_mm256_add_epi32(res_b, round_const_bits), round_shift_bits);
__m256i res_16bit = _mm256_packs_epi32(res_a_round, res_b_round);
res_16bit = _mm256_min_epi16(res_16bit, clip_pixel);
res_16bit = _mm256_max_epi16(res_16bit, zero);
_mm_storeu_si128((__m128i *)&dst_ptr[i * dst_pitch],
_mm256_castsi256_si128(res_16bit));
_mm_storeu_si128((__m128i *)&dst_ptr[i * dst_pitch + dst_pitch],
_mm256_extracti128_si256(res_16bit, 1));
s[0] = s[1];
s[2] = s[3];
s4 = s6;
}
}
}
static void aom_highbd_filter_block1d16_v4_avx2(
const uint16_t *src_ptr, ptrdiff_t src_pitch, uint16_t *dst_ptr,
ptrdiff_t dst_pitch, uint32_t height, const int16_t *filter, int bd) {
aom_highbd_filter_block1d8_v4_avx2(src_ptr, src_pitch, dst_ptr, dst_pitch,
height, filter, bd);
aom_highbd_filter_block1d8_v4_avx2(src_ptr + 8, src_pitch, dst_ptr + 8,
dst_pitch, height, filter, bd);
}
// -----------------------------------------------------------------------------
// 2-tap vertical filtering

351
third_party/aom/aom_dsp/x86/highbd_convolve_sse2.c vendored Normal file
View file

@ -0,0 +1,351 @@
/*
* Copyright (c) 2018, Alliance for Open Media. All rights reserved
*
* This source code is subject to the terms of the BSD 2 Clause License and
* the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
* was not distributed with this source code in the LICENSE file, you can
* obtain it at www.aomedia.org/license/software. If the Alliance for Open
* Media Patent License 1.0 was not distributed with this source code in the
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
*/
#include <emmintrin.h>
#include "config/aom_dsp_rtcd.h"
#include "aom_dsp/x86/convolve.h"
// -----------------------------------------------------------------------------
void aom_highbd_filter_block1d4_v4_sse2(const uint16_t *src_ptr,
ptrdiff_t src_pitch, uint16_t *dst_ptr,
ptrdiff_t dst_pitch, uint32_t height,
const int16_t *filter, int bd) {
__m128i filtersReg;
__m128i srcReg2, srcReg3, srcReg4, srcReg5, srcReg6;
__m128i srcReg23_lo, srcReg34_lo;
__m128i srcReg45_lo, srcReg56_lo;
__m128i resReg23_lo, resReg34_lo, resReg45_lo, resReg56_lo;
__m128i resReg23_45_lo, resReg34_56_lo;
__m128i resReg23_45, resReg34_56;
__m128i addFilterReg64, secondFilters, thirdFilters;
unsigned int i;
ptrdiff_t src_stride, dst_stride;
const __m128i max = _mm_set1_epi16((1 << bd) - 1);
addFilterReg64 = _mm_set1_epi32(64);
filtersReg = _mm_loadu_si128((const __m128i *)filter);
// coeffs 0 1 0 1 2 3 2 3
const __m128i tmp0 = _mm_unpacklo_epi32(filtersReg, filtersReg);
// coeffs 4 5 4 5 6 7 6 7
const __m128i tmp1 = _mm_unpackhi_epi32(filtersReg, filtersReg);
secondFilters = _mm_unpackhi_epi64(tmp0, tmp0); // coeffs 2 3 2 3 2 3 2 3
thirdFilters = _mm_unpacklo_epi64(tmp1, tmp1); // coeffs 4 5 4 5 4 5 4 5
// multiply the size of the source and destination stride by two
src_stride = src_pitch << 1;
dst_stride = dst_pitch << 1;
srcReg2 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 2));
srcReg3 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 3));
srcReg23_lo = _mm_unpacklo_epi16(srcReg2, srcReg3);
srcReg4 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 4));
srcReg34_lo = _mm_unpacklo_epi16(srcReg3, srcReg4);
for (i = height; i > 1; i -= 2) {
srcReg5 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 5));
srcReg45_lo = _mm_unpacklo_epi16(srcReg4, srcReg5);
srcReg6 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 6));
srcReg56_lo = _mm_unpacklo_epi16(srcReg5, srcReg6);
// multiply 2 adjacent elements with the filter and add the result
resReg23_lo = _mm_madd_epi16(srcReg23_lo, secondFilters);
resReg34_lo = _mm_madd_epi16(srcReg34_lo, secondFilters);
resReg45_lo = _mm_madd_epi16(srcReg45_lo, thirdFilters);
resReg56_lo = _mm_madd_epi16(srcReg56_lo, thirdFilters);
resReg23_45_lo = _mm_add_epi32(resReg23_lo, resReg45_lo);
resReg34_56_lo = _mm_add_epi32(resReg34_lo, resReg56_lo);
// shift by 7 bit each 32 bit
resReg23_45_lo = _mm_add_epi32(resReg23_45_lo, addFilterReg64);
resReg34_56_lo = _mm_add_epi32(resReg34_56_lo, addFilterReg64);
resReg23_45_lo = _mm_srai_epi32(resReg23_45_lo, 7);
resReg34_56_lo = _mm_srai_epi32(resReg34_56_lo, 7);
// shrink to 16 bit each 32 bits, the first lane contain the first
// convolve result and the second lane contain the second convolve
// result
resReg23_45 = _mm_packs_epi32(resReg23_45_lo, _mm_setzero_si128());
resReg34_56 = _mm_packs_epi32(resReg34_56_lo, _mm_setzero_si128());
resReg23_45 = _mm_max_epi16(resReg23_45, _mm_setzero_si128());
resReg23_45 = _mm_min_epi16(resReg23_45, max);
resReg34_56 = _mm_max_epi16(resReg34_56, _mm_setzero_si128());
resReg34_56 = _mm_min_epi16(resReg34_56, max);
src_ptr += src_stride;
_mm_storel_epi64((__m128i *)dst_ptr, (resReg23_45));
_mm_storel_epi64((__m128i *)(dst_ptr + dst_pitch), (resReg34_56));
dst_ptr += dst_stride;
// save part of the registers for next strides
srcReg23_lo = srcReg45_lo;
srcReg34_lo = srcReg56_lo;
srcReg4 = srcReg6;
}
}
void aom_highbd_filter_block1d4_h4_sse2(const uint16_t *src_ptr,
ptrdiff_t src_pitch, uint16_t *dst_ptr,
ptrdiff_t dst_pitch, uint32_t height,
const int16_t *filter, int bd) {
__m128i filtersReg;
__m128i addFilterReg64;
__m128i secondFilters, thirdFilters;
__m128i srcRegFilt32b1_1;
__m128i srcReg32b1;
unsigned int i;
src_ptr -= 3;
addFilterReg64 = _mm_set1_epi32(64);
filtersReg = _mm_loadu_si128((const __m128i *)filter);
const __m128i max = _mm_set1_epi16((1 << bd) - 1);
// coeffs 0 1 0 1 2 3 2 3
const __m128i tmp_0 = _mm_unpacklo_epi32(filtersReg, filtersReg);
// coeffs 4 5 4 5 6 7 6 7
const __m128i tmp_1 = _mm_unpackhi_epi32(filtersReg, filtersReg);
secondFilters = _mm_unpackhi_epi64(tmp_0, tmp_0); // coeffs 2 3 2 3 2 3 2 3
thirdFilters = _mm_unpacklo_epi64(tmp_1, tmp_1); // coeffs 4 5 4 5 4 5 4 5
for (i = height; i > 0; i -= 1) {
srcReg32b1 = _mm_loadu_si128((const __m128i *)(src_ptr + 2));
__m128i ss_3_1 = _mm_srli_si128(srcReg32b1, 2);
__m128i ss_4_1 = _mm_srli_si128(srcReg32b1, 4);
__m128i ss_5_1 = _mm_srli_si128(srcReg32b1, 6);
__m128i ss_23 = _mm_unpacklo_epi32(srcReg32b1, ss_3_1);
__m128i ss_45 = _mm_unpacklo_epi32(ss_4_1, ss_5_1);
ss_23 = _mm_madd_epi16(ss_23, secondFilters);
ss_45 = _mm_madd_epi16(ss_45, thirdFilters);
srcRegFilt32b1_1 = _mm_add_epi32(ss_23, ss_45);
// shift by 7 bit each 32 bit
srcRegFilt32b1_1 = _mm_add_epi32(srcRegFilt32b1_1, addFilterReg64);
srcRegFilt32b1_1 = _mm_srai_epi32(srcRegFilt32b1_1, 7);
srcRegFilt32b1_1 = _mm_packs_epi32(srcRegFilt32b1_1, _mm_setzero_si128());
srcRegFilt32b1_1 = _mm_max_epi16(srcRegFilt32b1_1, _mm_setzero_si128());
srcRegFilt32b1_1 = _mm_min_epi16(srcRegFilt32b1_1, max);
src_ptr += src_pitch;
_mm_storel_epi64((__m128i *)dst_ptr, srcRegFilt32b1_1);
dst_ptr += dst_pitch;
}
}
void aom_highbd_filter_block1d8_v4_sse2(const uint16_t *src_ptr,
ptrdiff_t src_pitch, uint16_t *dst_ptr,
ptrdiff_t dst_pitch, uint32_t height,
const int16_t *filter, int bd) {
__m128i filtersReg;
__m128i srcReg2, srcReg3, srcReg4, srcReg5, srcReg6;
__m128i srcReg23_lo, srcReg23_hi, srcReg34_lo, srcReg34_hi;
__m128i srcReg45_lo, srcReg45_hi, srcReg56_lo, srcReg56_hi;
__m128i resReg23_lo, resReg34_lo, resReg45_lo, resReg56_lo;
__m128i resReg23_hi, resReg34_hi, resReg45_hi, resReg56_hi;
__m128i resReg23_45_lo, resReg34_56_lo, resReg23_45_hi, resReg34_56_hi;
__m128i resReg23_45, resReg34_56;
__m128i addFilterReg64, secondFilters, thirdFilters;
unsigned int i;
ptrdiff_t src_stride, dst_stride;
const __m128i max = _mm_set1_epi16((1 << bd) - 1);
addFilterReg64 = _mm_set1_epi32(64);
filtersReg = _mm_loadu_si128((const __m128i *)filter);
// coeffs 0 1 0 1 2 3 2 3
const __m128i tmp0 = _mm_unpacklo_epi32(filtersReg, filtersReg);
// coeffs 4 5 4 5 6 7 6 7
const __m128i tmp1 = _mm_unpackhi_epi32(filtersReg, filtersReg);
secondFilters = _mm_unpackhi_epi64(tmp0, tmp0); // coeffs 2 3 2 3 2 3 2 3
thirdFilters = _mm_unpacklo_epi64(tmp1, tmp1); // coeffs 4 5 4 5 4 5 4 5
// multiple the size of the source and destination stride by two
src_stride = src_pitch << 1;
dst_stride = dst_pitch << 1;
srcReg2 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 2));
srcReg3 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 3));
srcReg23_lo = _mm_unpacklo_epi16(srcReg2, srcReg3);
srcReg23_hi = _mm_unpackhi_epi16(srcReg2, srcReg3);
srcReg4 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 4));
srcReg34_lo = _mm_unpacklo_epi16(srcReg3, srcReg4);
srcReg34_hi = _mm_unpackhi_epi16(srcReg3, srcReg4);
for (i = height; i > 1; i -= 2) {
srcReg5 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 5));
srcReg45_lo = _mm_unpacklo_epi16(srcReg4, srcReg5);
srcReg45_hi = _mm_unpackhi_epi16(srcReg4, srcReg5);
srcReg6 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 6));
srcReg56_lo = _mm_unpacklo_epi16(srcReg5, srcReg6);
srcReg56_hi = _mm_unpackhi_epi16(srcReg5, srcReg6);
// multiply 2 adjacent elements with the filter and add the result
resReg23_lo = _mm_madd_epi16(srcReg23_lo, secondFilters);
resReg34_lo = _mm_madd_epi16(srcReg34_lo, secondFilters);
resReg45_lo = _mm_madd_epi16(srcReg45_lo, thirdFilters);
resReg56_lo = _mm_madd_epi16(srcReg56_lo, thirdFilters);
resReg23_45_lo = _mm_add_epi32(resReg23_lo, resReg45_lo);
resReg34_56_lo = _mm_add_epi32(resReg34_lo, resReg56_lo);
// multiply 2 adjacent elements with the filter and add the result
resReg23_hi = _mm_madd_epi16(srcReg23_hi, secondFilters);
resReg34_hi = _mm_madd_epi16(srcReg34_hi, secondFilters);
resReg45_hi = _mm_madd_epi16(srcReg45_hi, thirdFilters);
resReg56_hi = _mm_madd_epi16(srcReg56_hi, thirdFilters);
resReg23_45_hi = _mm_add_epi32(resReg23_hi, resReg45_hi);
resReg34_56_hi = _mm_add_epi32(resReg34_hi, resReg56_hi);
// shift by 7 bit each 32 bit
resReg23_45_lo = _mm_add_epi32(resReg23_45_lo, addFilterReg64);
resReg34_56_lo = _mm_add_epi32(resReg34_56_lo, addFilterReg64);
resReg23_45_hi = _mm_add_epi32(resReg23_45_hi, addFilterReg64);
resReg34_56_hi = _mm_add_epi32(resReg34_56_hi, addFilterReg64);
resReg23_45_lo = _mm_srai_epi32(resReg23_45_lo, 7);
resReg34_56_lo = _mm_srai_epi32(resReg34_56_lo, 7);
resReg23_45_hi = _mm_srai_epi32(resReg23_45_hi, 7);
resReg34_56_hi = _mm_srai_epi32(resReg34_56_hi, 7);
// shrink to 16 bit each 32 bits, the first lane contain the first
// convolve result and the second lane contain the second convolve
// result
resReg23_45 = _mm_packs_epi32(resReg23_45_lo, resReg23_45_hi);
resReg34_56 = _mm_packs_epi32(resReg34_56_lo, resReg34_56_hi);
resReg23_45 = _mm_max_epi16(resReg23_45, _mm_setzero_si128());
resReg23_45 = _mm_min_epi16(resReg23_45, max);
resReg34_56 = _mm_max_epi16(resReg34_56, _mm_setzero_si128());
resReg34_56 = _mm_min_epi16(resReg34_56, max);
src_ptr += src_stride;
_mm_store_si128((__m128i *)dst_ptr, (resReg23_45));
_mm_store_si128((__m128i *)(dst_ptr + dst_pitch), (resReg34_56));
dst_ptr += dst_stride;
// save part of the registers for next strides
srcReg23_lo = srcReg45_lo;
srcReg23_hi = srcReg45_hi;
srcReg34_lo = srcReg56_lo;
srcReg34_hi = srcReg56_hi;
srcReg4 = srcReg6;
}
}
void aom_highbd_filter_block1d8_h4_sse2(const uint16_t *src_ptr,
ptrdiff_t src_pitch, uint16_t *dst_ptr,
ptrdiff_t dst_pitch, uint32_t height,
const int16_t *filter, int bd) {
__m128i filtersReg;
__m128i addFilterReg64;
__m128i secondFilters, thirdFilters;
__m128i srcRegFilt32b1_1, srcRegFilt32b1_2;
__m128i srcReg32b1, srcReg32b2;
unsigned int i;
src_ptr -= 3;
addFilterReg64 = _mm_set1_epi32(64);
filtersReg = _mm_loadu_si128((const __m128i *)filter);
const __m128i max = _mm_set1_epi16((1 << bd) - 1);
// coeffs 0 1 0 1 2 3 2 3
const __m128i tmp_0 = _mm_unpacklo_epi32(filtersReg, filtersReg);
// coeffs 4 5 4 5 6 7 6 7
const __m128i tmp_1 = _mm_unpackhi_epi32(filtersReg, filtersReg);
secondFilters = _mm_unpackhi_epi64(tmp_0, tmp_0); // coeffs 2 3 2 3 2 3 2 3
thirdFilters = _mm_unpacklo_epi64(tmp_1, tmp_1); // coeffs 4 5 4 5 4 5 4 5
for (i = height; i > 0; i -= 1) {
srcReg32b1 = _mm_loadu_si128((const __m128i *)(src_ptr + 2));
srcReg32b2 = _mm_loadu_si128((const __m128i *)(src_ptr + 6));
__m128i ss_4_1 = _mm_srli_si128(srcReg32b1, 4);
__m128i ss_4_2 = _mm_srli_si128(srcReg32b2, 4);
__m128i ss_4 = _mm_unpacklo_epi64(ss_4_1, ss_4_2);
__m128i d1 = _mm_madd_epi16(srcReg32b1, secondFilters);
__m128i d2 = _mm_madd_epi16(ss_4, thirdFilters);
srcRegFilt32b1_1 = _mm_add_epi32(d1, d2);
__m128i ss_3_1 = _mm_srli_si128(srcReg32b1, 2);
__m128i ss_5_1 = _mm_srli_si128(srcReg32b1, 6);
__m128i ss_3_2 = _mm_srli_si128(srcReg32b2, 2);
__m128i ss_5_2 = _mm_srli_si128(srcReg32b2, 6);
__m128i ss_3 = _mm_unpacklo_epi64(ss_3_1, ss_3_2);
__m128i ss_5 = _mm_unpacklo_epi64(ss_5_1, ss_5_2);
d1 = _mm_madd_epi16(ss_3, secondFilters);
d2 = _mm_madd_epi16(ss_5, thirdFilters);
srcRegFilt32b1_2 = _mm_add_epi32(d1, d2);
__m128i res_lo_1 = _mm_unpacklo_epi32(srcRegFilt32b1_1, srcRegFilt32b1_2);
__m128i res_hi_1 = _mm_unpackhi_epi32(srcRegFilt32b1_1, srcRegFilt32b1_2);
// shift by 7 bit each 32 bit
res_lo_1 = _mm_add_epi32(res_lo_1, addFilterReg64);
res_hi_1 = _mm_add_epi32(res_hi_1, addFilterReg64);
res_lo_1 = _mm_srai_epi32(res_lo_1, 7);
res_hi_1 = _mm_srai_epi32(res_hi_1, 7);
srcRegFilt32b1_1 = _mm_packs_epi32(res_lo_1, res_hi_1);
srcRegFilt32b1_1 = _mm_max_epi16(srcRegFilt32b1_1, _mm_setzero_si128());
srcRegFilt32b1_1 = _mm_min_epi16(srcRegFilt32b1_1, max);
src_ptr += src_pitch;
_mm_store_si128((__m128i *)dst_ptr, srcRegFilt32b1_1);
dst_ptr += dst_pitch;
}
}
void aom_highbd_filter_block1d16_v4_sse2(const uint16_t *src_ptr,
ptrdiff_t src_pitch, uint16_t *dst_ptr,
ptrdiff_t dst_pitch, uint32_t height,
const int16_t *filter, int bd) {
aom_highbd_filter_block1d8_v4_sse2(src_ptr, src_pitch, dst_ptr, dst_pitch,
height, filter, bd);
aom_highbd_filter_block1d8_v4_sse2((src_ptr + 8), src_pitch, (dst_ptr + 8),
dst_pitch, height, filter, bd);
}
void aom_highbd_filter_block1d16_h4_sse2(const uint16_t *src_ptr,
ptrdiff_t src_pitch, uint16_t *dst_ptr,
ptrdiff_t dst_pitch, uint32_t height,
const int16_t *filter, int bd) {
aom_highbd_filter_block1d8_h4_sse2(src_ptr, src_pitch, dst_ptr, dst_pitch,
height, filter, bd);
aom_highbd_filter_block1d8_h4_sse2((src_ptr + 8), src_pitch, (dst_ptr + 8),
dst_pitch, height, filter, bd);
}

259
third_party/aom/aom_dsp/x86/highbd_intrapred_asm_sse2.asm vendored Normal file
View file

@ -0,0 +1,259 @@
;
; Copyright (c) 2016, Alliance for Open Media. All rights reserved
;
; This source code is subject to the terms of the BSD 2 Clause License and
; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
; was not distributed with this source code in the LICENSE file, you can
; obtain it at www.aomedia.org/license/software. If the Alliance for Open
; Media Patent License 1.0 was not distributed with this source code in the
; PATENTS file, you can obtain it at www.aomedia.org/license/patent.
;
;
%include "third_party/x86inc/x86inc.asm"
SECTION_RODATA
pw_4: times 8 dw 4
pw_8: times 8 dw 8
pw_16: times 4 dd 16
pw_32: times 4 dd 32
SECTION .text
INIT_XMM sse2
cglobal highbd_dc_predictor_4x4, 4, 5, 4, dst, stride, above, left, goffset
GET_GOT goffsetq
movq m0, [aboveq]
movq m2, [leftq]
paddw m0, m2
pshuflw m1, m0, 0xe
paddw m0, m1
pshuflw m1, m0, 0x1
paddw m0, m1
paddw m0, [GLOBAL(pw_4)]
psraw m0, 3
pshuflw m0, m0, 0x0
movq [dstq ], m0
movq [dstq+strideq*2], m0
lea dstq, [dstq+strideq*4]
movq [dstq ], m0
movq [dstq+strideq*2], m0
RESTORE_GOT
RET
INIT_XMM sse2
cglobal highbd_dc_predictor_8x8, 4, 5, 4, dst, stride, above, left, goffset
GET_GOT goffsetq
pxor m1, m1
mova m0, [aboveq]
mova m2, [leftq]
DEFINE_ARGS dst, stride, stride3, one
mov oned, 0x00010001
lea stride3q, [strideq*3]
movd m3, oned
pshufd m3, m3, 0x0
paddw m0, m2
pmaddwd m0, m3
packssdw m0, m1
pmaddwd m0, m3
packssdw m0, m1
pmaddwd m0, m3
paddw m0, [GLOBAL(pw_8)]
psrlw m0, 4
pshuflw m0, m0, 0x0
punpcklqdq m0, m0
mova [dstq ], m0
mova [dstq+strideq*2 ], m0
mova [dstq+strideq*4 ], m0
mova [dstq+stride3q*2], m0
lea dstq, [dstq+strideq*8]
mova [dstq ], m0
mova [dstq+strideq*2 ], m0
mova [dstq+strideq*4 ], m0
mova [dstq+stride3q*2], m0
RESTORE_GOT
RET
INIT_XMM sse2
cglobal highbd_dc_predictor_16x16, 4, 5, 5, dst, stride, above, left, goffset
GET_GOT goffsetq
pxor m1, m1
mova m0, [aboveq]
mova m3, [aboveq+16]
mova m2, [leftq]
mova m4, [leftq+16]
DEFINE_ARGS dst, stride, stride3, lines4
lea stride3q, [strideq*3]
mov lines4d, 4
paddw m0, m2
paddw m0, m3
paddw m0, m4
movhlps m2, m0
paddw m0, m2
punpcklwd m0, m1
movhlps m2, m0
paddd m0, m2
punpckldq m0, m1
movhlps m2, m0
paddd m0, m2
paddd m0, [GLOBAL(pw_16)]
psrad m0, 5
pshuflw m0, m0, 0x0
punpcklqdq m0, m0
.loop:
mova [dstq ], m0
mova [dstq +16], m0
mova [dstq+strideq*2 ], m0
mova [dstq+strideq*2 +16], m0
mova [dstq+strideq*4 ], m0
mova [dstq+strideq*4 +16], m0
mova [dstq+stride3q*2 ], m0
mova [dstq+stride3q*2+16], m0
lea dstq, [dstq+strideq*8]
dec lines4d
jnz .loop
RESTORE_GOT
REP_RET
INIT_XMM sse2
cglobal highbd_dc_predictor_32x32, 4, 5, 7, dst, stride, above, left, goffset
GET_GOT goffsetq
mova m0, [aboveq]
mova m2, [aboveq+16]
mova m3, [aboveq+32]
mova m4, [aboveq+48]
paddw m0, m2
paddw m3, m4
mova m2, [leftq]
mova m4, [leftq+16]
mova m5, [leftq+32]
mova m6, [leftq+48]
paddw m2, m4
paddw m5, m6
paddw m0, m3
paddw m2, m5
pxor m1, m1
paddw m0, m2
DEFINE_ARGS dst, stride, stride3, lines4
lea stride3q, [strideq*3]
mov lines4d, 8
movhlps m2, m0
paddw m0, m2
punpcklwd m0, m1
movhlps m2, m0
paddd m0, m2
punpckldq m0, m1
movhlps m2, m0
paddd m0, m2
paddd m0, [GLOBAL(pw_32)]
psrad m0, 6
pshuflw m0, m0, 0x0
punpcklqdq m0, m0
.loop:
mova [dstq ], m0
mova [dstq +16 ], m0
mova [dstq +32 ], m0
mova [dstq +48 ], m0
mova [dstq+strideq*2 ], m0
mova [dstq+strideq*2+16 ], m0
mova [dstq+strideq*2+32 ], m0
mova [dstq+strideq*2+48 ], m0
mova [dstq+strideq*4 ], m0
mova [dstq+strideq*4+16 ], m0
mova [dstq+strideq*4+32 ], m0
mova [dstq+strideq*4+48 ], m0
mova [dstq+stride3q*2 ], m0
mova [dstq+stride3q*2 +16], m0
mova [dstq+stride3q*2 +32], m0
mova [dstq+stride3q*2 +48], m0
lea dstq, [dstq+strideq*8]
dec lines4d
jnz .loop
RESTORE_GOT
REP_RET
INIT_XMM sse2
cglobal highbd_v_predictor_4x4, 3, 3, 1, dst, stride, above
movq m0, [aboveq]
movq [dstq ], m0
movq [dstq+strideq*2], m0
lea dstq, [dstq+strideq*4]
movq [dstq ], m0
movq [dstq+strideq*2], m0
RET
INIT_XMM sse2
cglobal highbd_v_predictor_8x8, 3, 3, 1, dst, stride, above
mova m0, [aboveq]
DEFINE_ARGS dst, stride, stride3
lea stride3q, [strideq*3]
mova [dstq ], m0
mova [dstq+strideq*2 ], m0
mova [dstq+strideq*4 ], m0
mova [dstq+stride3q*2], m0
lea dstq, [dstq+strideq*8]
mova [dstq ], m0
mova [dstq+strideq*2 ], m0
mova [dstq+strideq*4 ], m0
mova [dstq+stride3q*2], m0
RET
INIT_XMM sse2
cglobal highbd_v_predictor_16x16, 3, 4, 2, dst, stride, above
mova m0, [aboveq]
mova m1, [aboveq+16]
DEFINE_ARGS dst, stride, stride3, nlines4
lea stride3q, [strideq*3]
mov nlines4d, 4
.loop:
mova [dstq ], m0
mova [dstq +16], m1
mova [dstq+strideq*2 ], m0
mova [dstq+strideq*2 +16], m1
mova [dstq+strideq*4 ], m0
mova [dstq+strideq*4 +16], m1
mova [dstq+stride3q*2 ], m0
mova [dstq+stride3q*2+16], m1
lea dstq, [dstq+strideq*8]
dec nlines4d
jnz .loop
REP_RET
INIT_XMM sse2
cglobal highbd_v_predictor_32x32, 3, 4, 4, dst, stride, above
mova m0, [aboveq]
mova m1, [aboveq+16]
mova m2, [aboveq+32]
mova m3, [aboveq+48]
DEFINE_ARGS dst, stride, stride3, nlines4
lea stride3q, [strideq*3]
mov nlines4d, 8
.loop:
mova [dstq ], m0
mova [dstq +16], m1
mova [dstq +32], m2
mova [dstq +48], m3
mova [dstq+strideq*2 ], m0
mova [dstq+strideq*2 +16], m1
mova [dstq+strideq*2 +32], m2
mova [dstq+strideq*2 +48], m3
mova [dstq+strideq*4 ], m0
mova [dstq+strideq*4 +16], m1
mova [dstq+strideq*4 +32], m2
mova [dstq+strideq*4 +48], m3
mova [dstq+stride3q*2 ], m0
mova [dstq+stride3q*2 +16], m1
mova [dstq+stride3q*2 +32], m2
mova [dstq+stride3q*2 +48], m3
lea dstq, [dstq+strideq*8]
dec nlines4d
jnz .loop
REP_RET

4
third_party/aom/aom_dsp/x86/highbd_loopfilter_sse2.c vendored
View file

@ -727,8 +727,8 @@ void aom_highbd_lpf_horizontal_14_dual_sse2(
_limit1, _thresh1, bd);
for (i = 0; i < 6; i++) {
_mm_store_si128((__m128i *)(s - (i + 1) * pitch), p[i]);
_mm_store_si128((__m128i *)(s + i * pitch), q[i]);
_mm_storeu_si128((__m128i *)(s - (i + 1) * pitch), p[i]);
_mm_storeu_si128((__m128i *)(s + i * pitch), q[i]);
}
}

52
third_party/aom/aom_dsp/x86/highbd_subpel_variance_impl_sse2.asm vendored
View file

@ -114,45 +114,33 @@ SECTION .text
cglobal highbd_sub_pixel_avg_variance%1xh, 7, 7, 13, src, src_stride, \
x_offset, y_offset, \
dst, dst_stride, \
sec, sec_stride, height, sse, \
g_bilin_filter, g_pw_8
sec, sec_stride, height, sse
%define block_height dword heightm
%define sec_str sec_stridemp
; Store bilin_filter and pw_8 location in stack
%if GET_GOT_DEFINED == 1
GET_GOT eax
add esp, 4 ; restore esp
%endif
lea ecx, [GLOBAL(bilin_filter_m)]
mov g_bilin_filterm, ecx
lea ecx, [GLOBAL(pw_8)]
mov g_pw_8m, ecx
LOAD_IF_USED 0, 1 ; load eax, ecx back
%else
cglobal highbd_sub_pixel_variance%1xh, 7, 7, 13, src, src_stride, \
x_offset, y_offset, \
dst, dst_stride, height, sse, \
g_bilin_filter, g_pw_8
dst, dst_stride, height, sse
%define block_height heightd
; Store bilin_filter and pw_8 location in stack
%if GET_GOT_DEFINED == 1
GET_GOT eax
add esp, 4 ; restore esp
%endif
lea ecx, [GLOBAL(bilin_filter_m)]
mov g_bilin_filterm, ecx
lea ecx, [GLOBAL(pw_8)]
mov g_pw_8m, ecx
LOAD_IF_USED 0, 1 ; load eax, ecx back
%endif
; reuse argument stack space
%define g_bilin_filterm x_offsetm
%define g_pw_8m y_offsetm
; Store bilin_filter and pw_8 location in stack
%if GET_GOT_DEFINED == 1
GET_GOT eax
add esp, 4 ; restore esp
%endif
lea ecx, [GLOBAL(bilin_filter_m)]
mov g_bilin_filterm, ecx
lea ecx, [GLOBAL(pw_8)]
mov g_pw_8m, ecx
LOAD_IF_USED 0, 1 ; load eax, ecx back
%else
%if %2 == 1 ; avg
cglobal highbd_sub_pixel_avg_variance%1xh, 7, 7, 13, src, src_stride, \

48
third_party/aom/aom_dsp/x86/highbd_subtract_sse2.c vendored
View file

@ -29,15 +29,15 @@ static void subtract_4x4(int16_t *diff, ptrdiff_t diff_stride,
__m128i x0, x1, x2, x3;
int64_t *store_diff = (int64_t *)(diff + 0 * diff_stride);
u0 = _mm_loadu_si128((__m128i const *)(src + 0 * src_stride));
u1 = _mm_loadu_si128((__m128i const *)(src + 1 * src_stride));
u2 = _mm_loadu_si128((__m128i const *)(src + 2 * src_stride));
u3 = _mm_loadu_si128((__m128i const *)(src + 3 * src_stride));
u0 = _mm_loadl_epi64((__m128i const *)(src + 0 * src_stride));
u1 = _mm_loadl_epi64((__m128i const *)(src + 1 * src_stride));
u2 = _mm_loadl_epi64((__m128i const *)(src + 2 * src_stride));
u3 = _mm_loadl_epi64((__m128i const *)(src + 3 * src_stride));
v0 = _mm_loadu_si128((__m128i const *)(pred + 0 * pred_stride));
v1 = _mm_loadu_si128((__m128i const *)(pred + 1 * pred_stride));
v2 = _mm_loadu_si128((__m128i const *)(pred + 2 * pred_stride));
v3 = _mm_loadu_si128((__m128i const *)(pred + 3 * pred_stride));
v0 = _mm_loadl_epi64((__m128i const *)(pred + 0 * pred_stride));
v1 = _mm_loadl_epi64((__m128i const *)(pred + 1 * pred_stride));
v2 = _mm_loadl_epi64((__m128i const *)(pred + 2 * pred_stride));
v3 = _mm_loadl_epi64((__m128i const *)(pred + 3 * pred_stride));
x0 = _mm_sub_epi16(u0, v0);
x1 = _mm_sub_epi16(u1, v1);
@ -61,23 +61,23 @@ static void subtract_4x8(int16_t *diff, ptrdiff_t diff_stride,
__m128i x0, x1, x2, x3, x4, x5, x6, x7;
int64_t *store_diff = (int64_t *)(diff + 0 * diff_stride);
u0 = _mm_loadu_si128((__m128i const *)(src + 0 * src_stride));
u1 = _mm_loadu_si128((__m128i const *)(src + 1 * src_stride));
u2 = _mm_loadu_si128((__m128i const *)(src + 2 * src_stride));
u3 = _mm_loadu_si128((__m128i const *)(src + 3 * src_stride));
u4 = _mm_loadu_si128((__m128i const *)(src + 4 * src_stride));
u5 = _mm_loadu_si128((__m128i const *)(src + 5 * src_stride));
u6 = _mm_loadu_si128((__m128i const *)(src + 6 * src_stride));
u7 = _mm_loadu_si128((__m128i const *)(src + 7 * src_stride));
u0 = _mm_loadl_epi64((__m128i const *)(src + 0 * src_stride));
u1 = _mm_loadl_epi64((__m128i const *)(src + 1 * src_stride));
u2 = _mm_loadl_epi64((__m128i const *)(src + 2 * src_stride));
u3 = _mm_loadl_epi64((__m128i const *)(src + 3 * src_stride));
u4 = _mm_loadl_epi64((__m128i const *)(src + 4 * src_stride));
u5 = _mm_loadl_epi64((__m128i const *)(src + 5 * src_stride));
u6 = _mm_loadl_epi64((__m128i const *)(src + 6 * src_stride));
u7 = _mm_loadl_epi64((__m128i const *)(src + 7 * src_stride));
v0 = _mm_loadu_si128((__m128i const *)(pred + 0 * pred_stride));
v1 = _mm_loadu_si128((__m128i const *)(pred + 1 * pred_stride));
v2 = _mm_loadu_si128((__m128i const *)(pred + 2 * pred_stride));
v3 = _mm_loadu_si128((__m128i const *)(pred + 3 * pred_stride));
v4 = _mm_loadu_si128((__m128i const *)(pred + 4 * pred_stride));
v5 = _mm_loadu_si128((__m128i const *)(pred + 5 * pred_stride));
v6 = _mm_loadu_si128((__m128i const *)(pred + 6 * pred_stride));
v7 = _mm_loadu_si128((__m128i const *)(pred + 7 * pred_stride));
v0 = _mm_loadl_epi64((__m128i const *)(pred + 0 * pred_stride));
v1 = _mm_loadl_epi64((__m128i const *)(pred + 1 * pred_stride));
v2 = _mm_loadl_epi64((__m128i const *)(pred + 2 * pred_stride));
v3 = _mm_loadl_epi64((__m128i const *)(pred + 3 * pred_stride));
v4 = _mm_loadl_epi64((__m128i const *)(pred + 4 * pred_stride));
v5 = _mm_loadl_epi64((__m128i const *)(pred + 5 * pred_stride));
v6 = _mm_loadl_epi64((__m128i const *)(pred + 6 * pred_stride));
v7 = _mm_loadl_epi64((__m128i const *)(pred + 7 * pred_stride));
x0 = _mm_sub_epi16(u0, v0);
x1 = _mm_sub_epi16(u1, v1);

59
third_party/aom/aom_dsp/x86/highbd_variance_sse2.c vendored
View file

@ -603,7 +603,7 @@ void aom_highbd_upsampled_pred_sse2(MACROBLOCKD *xd,
const int ref_num = 0;
const int is_intrabc = is_intrabc_block(mi);
const struct scale_factors *const sf =
is_intrabc ? &cm->sf_identity : &xd->block_refs[ref_num]->sf;
is_intrabc ? &cm->sf_identity : xd->block_ref_scale_factors[ref_num];
const int is_scaled = av1_is_scaled(sf);
if (is_scaled) {
@ -677,11 +677,8 @@ void aom_highbd_upsampled_pred_sse2(MACROBLOCKD *xd,
}
}
const InterpFilterParams *filter =
(subpel_search == 1)
? av1_get_4tap_interp_filter_params(EIGHTTAP_REGULAR)
: av1_get_interp_filter_params_with_block_size(EIGHTTAP_REGULAR, 8);
const InterpFilterParams *filter = av1_get_filter(subpel_search);
int filter_taps = (subpel_search <= USE_4_TAPS) ? 4 : SUBPEL_TAPS;
if (!subpel_x_q3 && !subpel_y_q3) {
uint16_t *ref = CONVERT_TO_SHORTPTR(ref8);
uint16_t *comp_pred = CONVERT_TO_SHORTPTR(comp_pred8);
@ -729,17 +726,20 @@ void aom_highbd_upsampled_pred_sse2(MACROBLOCKD *xd,
av1_get_interp_filter_subpel_kernel(filter, subpel_x_q3 << 1);
const int16_t *const kernel_y =
av1_get_interp_filter_subpel_kernel(filter, subpel_y_q3 << 1);
const uint8_t *ref_start = ref8 - ref_stride * ((filter_taps >> 1) - 1);
uint16_t *temp_start_horiz = (subpel_search <= USE_4_TAPS)
? temp + (filter_taps >> 1) * MAX_SB_SIZE
: temp;
uint16_t *temp_start_vert = temp + MAX_SB_SIZE * ((filter->taps >> 1) - 1);
const int intermediate_height =
(((height - 1) * 8 + subpel_y_q3) >> 3) + filter->taps;
(((height - 1) * 8 + subpel_y_q3) >> 3) + filter_taps;
assert(intermediate_height <= (MAX_SB_SIZE * 2 + 16) + 16);
aom_highbd_convolve8_horiz(ref8 - ref_stride * ((filter->taps >> 1) - 1),
ref_stride, CONVERT_TO_BYTEPTR(temp),
MAX_SB_SIZE, kernel_x, 16, NULL, -1, width,
intermediate_height, bd);
aom_highbd_convolve8_vert(
CONVERT_TO_BYTEPTR(temp + MAX_SB_SIZE * ((filter->taps >> 1) - 1)),
MAX_SB_SIZE, comp_pred8, width, NULL, -1, kernel_y, 16, width, height,
bd);
aom_highbd_convolve8_horiz(
ref_start, ref_stride, CONVERT_TO_BYTEPTR(temp_start_horiz),
MAX_SB_SIZE, kernel_x, 16, NULL, -1, width, intermediate_height, bd);
aom_highbd_convolve8_vert(CONVERT_TO_BYTEPTR(temp_start_vert), MAX_SB_SIZE,
comp_pred8, width, NULL, -1, kernel_y, 16, width,
height, bd);
}
}
@ -765,11 +765,11 @@ void aom_highbd_comp_avg_upsampled_pred_sse2(
}
}
static INLINE void highbd_compute_jnt_comp_avg(__m128i *p0, __m128i *p1,
const __m128i *w0,
const __m128i *w1,
const __m128i *r,
void *const result) {
static INLINE void highbd_compute_dist_wtd_comp_avg(__m128i *p0, __m128i *p1,
const __m128i *w0,
const __m128i *w1,
const __m128i *r,
void *const result) {
assert(DIST_PRECISION_BITS <= 4);
__m128i mult0 = _mm_mullo_epi16(*p0, *w0);
__m128i mult1 = _mm_mullo_epi16(*p1, *w1);
@ -780,11 +780,10 @@ static INLINE void highbd_compute_jnt_comp_avg(__m128i *p0, __m128i *p1,
xx_storeu_128(result, shift);
}
void aom_highbd_jnt_comp_avg_pred_sse2(uint8_t *comp_pred8,
const uint8_t *pred8, int width,
int height, const uint8_t *ref8,
int ref_stride,
const JNT_COMP_PARAMS *jcp_param) {
void aom_highbd_dist_wtd_comp_avg_pred_sse2(
uint8_t *comp_pred8, const uint8_t *pred8, int width, int height,
const uint8_t *ref8, int ref_stride,
const DIST_WTD_COMP_PARAMS *jcp_param) {
int i;
const uint16_t wt0 = (uint16_t)jcp_param->fwd_offset;
const uint16_t wt1 = (uint16_t)jcp_param->bck_offset;
@ -806,7 +805,7 @@ void aom_highbd_jnt_comp_avg_pred_sse2(uint8_t *comp_pred8,
__m128i p0 = xx_loadu_128(ref);
__m128i p1 = xx_loadu_128(pred);
highbd_compute_jnt_comp_avg(&p0, &p1, &w0, &w1, &r, comp_pred);
highbd_compute_dist_wtd_comp_avg(&p0, &p1, &w0, &w1, &r, comp_pred);
comp_pred += 8;
pred += 8;
@ -823,7 +822,7 @@ void aom_highbd_jnt_comp_avg_pred_sse2(uint8_t *comp_pred8,
__m128i p0 = _mm_unpacklo_epi64(p0_0, p0_1);
__m128i p1 = xx_loadu_128(pred);
highbd_compute_jnt_comp_avg(&p0, &p1, &w0, &w1, &r, comp_pred);
highbd_compute_dist_wtd_comp_avg(&p0, &p1, &w0, &w1, &r, comp_pred);
comp_pred += 8;
pred += 8;
@ -832,11 +831,11 @@ void aom_highbd_jnt_comp_avg_pred_sse2(uint8_t *comp_pred8,
}
}
void aom_highbd_jnt_comp_avg_upsampled_pred_sse2(
void aom_highbd_dist_wtd_comp_avg_upsampled_pred_sse2(
MACROBLOCKD *xd, const struct AV1Common *const cm, int mi_row, int mi_col,
const MV *const mv, uint8_t *comp_pred8, const uint8_t *pred8, int width,
int height, int subpel_x_q3, int subpel_y_q3, const uint8_t *ref8,
int ref_stride, int bd, const JNT_COMP_PARAMS *jcp_param,
int ref_stride, int bd, const DIST_WTD_COMP_PARAMS *jcp_param,
int subpel_search) {
uint16_t *pred = CONVERT_TO_SHORTPTR(pred8);
int n;
@ -860,7 +859,7 @@ void aom_highbd_jnt_comp_avg_upsampled_pred_sse2(
__m128i p0 = xx_loadu_128(comp_pred16);
__m128i p1 = xx_loadu_128(pred);
highbd_compute_jnt_comp_avg(&p0, &p1, &w0, &w1, &r, comp_pred16);
highbd_compute_dist_wtd_comp_avg(&p0, &p1, &w0, &w1, &r, comp_pred16);
comp_pred16 += 8;
pred += 8;

608
third_party/aom/aom_dsp/x86/intrapred_asm_sse2.asm vendored Normal file
View file

@ -0,0 +1,608 @@
;
; Copyright (c) 2016, Alliance for Open Media. All rights reserved
;
; This source code is subject to the terms of the BSD 2 Clause License and
; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
; was not distributed with this source code in the LICENSE file, you can
; obtain it at www.aomedia.org/license/software. If the Alliance for Open
; Media Patent License 1.0 was not distributed with this source code in the
; PATENTS file, you can obtain it at www.aomedia.org/license/patent.
;
;
%include "third_party/x86inc/x86inc.asm"
SECTION_RODATA
pb_1: times 16 db 1
pw_4: times 8 dw 4
pw_8: times 8 dw 8
pw_16: times 8 dw 16
pw_32: times 8 dw 32
dc_128: times 16 db 128
pw2_4: times 8 dw 2
pw2_8: times 8 dw 4
pw2_16: times 8 dw 8
pw2_32: times 8 dw 16
SECTION .text
INIT_XMM sse2
cglobal dc_predictor_4x4, 4, 5, 3, dst, stride, above, left, goffset
GET_GOT goffsetq
movd m2, [leftq]
movd m0, [aboveq]
pxor m1, m1
punpckldq m0, m2
psadbw m0, m1
paddw m0, [GLOBAL(pw_4)]
psraw m0, 3
pshuflw m0, m0, 0x0
packuswb m0, m0
movd [dstq ], m0
movd [dstq+strideq], m0
lea dstq, [dstq+strideq*2]
movd [dstq ], m0
movd [dstq+strideq], m0
RESTORE_GOT
RET
INIT_XMM sse2
cglobal dc_left_predictor_4x4, 2, 5, 2, dst, stride, above, left, goffset
movifnidn leftq, leftmp
GET_GOT goffsetq
pxor m1, m1
movd m0, [leftq]
psadbw m0, m1
paddw m0, [GLOBAL(pw2_4)]
psraw m0, 2
pshuflw m0, m0, 0x0
packuswb m0, m0
movd [dstq ], m0
movd [dstq+strideq], m0
lea dstq, [dstq+strideq*2]
movd [dstq ], m0
movd [dstq+strideq], m0
RESTORE_GOT
RET
INIT_XMM sse2
cglobal dc_top_predictor_4x4, 3, 5, 2, dst, stride, above, left, goffset
GET_GOT goffsetq
pxor m1, m1
movd m0, [aboveq]
psadbw m0, m1
paddw m0, [GLOBAL(pw2_4)]
psraw m0, 2
pshuflw m0, m0, 0x0
packuswb m0, m0
movd [dstq ], m0
movd [dstq+strideq], m0
lea dstq, [dstq+strideq*2]
movd [dstq ], m0
movd [dstq+strideq], m0
RESTORE_GOT
RET
INIT_XMM sse2
cglobal dc_predictor_8x8, 4, 5, 3, dst, stride, above, left, goffset
GET_GOT goffsetq
pxor m1, m1
movq m0, [aboveq]
movq m2, [leftq]
DEFINE_ARGS dst, stride, stride3
lea stride3q, [strideq*3]
psadbw m0, m1
psadbw m2, m1
paddw m0, m2
paddw m0, [GLOBAL(pw_8)]
psraw m0, 4
punpcklbw m0, m0
pshuflw m0, m0, 0x0
movq [dstq ], m0
movq [dstq+strideq ], m0
movq [dstq+strideq*2], m0
movq [dstq+stride3q ], m0
lea dstq, [dstq+strideq*4]
movq [dstq ], m0
movq [dstq+strideq ], m0
movq [dstq+strideq*2], m0
movq [dstq+stride3q ], m0
RESTORE_GOT
RET
INIT_XMM sse2
cglobal dc_top_predictor_8x8, 3, 5, 2, dst, stride, above, left, goffset
GET_GOT goffsetq
pxor m1, m1
movq m0, [aboveq]
DEFINE_ARGS dst, stride, stride3
lea stride3q, [strideq*3]
psadbw m0, m1
paddw m0, [GLOBAL(pw2_8)]
psraw m0, 3
punpcklbw m0, m0
pshuflw m0, m0, 0x0
movq [dstq ], m0
movq [dstq+strideq ], m0
movq [dstq+strideq*2], m0
movq [dstq+stride3q ], m0
lea dstq, [dstq+strideq*4]
movq [dstq ], m0
movq [dstq+strideq ], m0
movq [dstq+strideq*2], m0
movq [dstq+stride3q ], m0
RESTORE_GOT
RET
INIT_XMM sse2
cglobal dc_left_predictor_8x8, 2, 5, 2, dst, stride, above, left, goffset
movifnidn leftq, leftmp
GET_GOT goffsetq
pxor m1, m1
movq m0, [leftq]
DEFINE_ARGS dst, stride, stride3
lea stride3q, [strideq*3]
psadbw m0, m1
paddw m0, [GLOBAL(pw2_8)]
psraw m0, 3
punpcklbw m0, m0
pshuflw m0, m0, 0x0
movq [dstq ], m0
movq [dstq+strideq ], m0
movq [dstq+strideq*2], m0
movq [dstq+stride3q ], m0
lea dstq, [dstq+strideq*4]
movq [dstq ], m0
movq [dstq+strideq ], m0
movq [dstq+strideq*2], m0
movq [dstq+stride3q ], m0
RESTORE_GOT
RET
INIT_XMM sse2
cglobal dc_128_predictor_4x4, 2, 5, 1, dst, stride, above, left, goffset
GET_GOT goffsetq
DEFINE_ARGS dst, stride, stride3
lea stride3q, [strideq*3]
movd m0, [GLOBAL(dc_128)]
movd [dstq ], m0
movd [dstq+strideq ], m0
movd [dstq+strideq*2], m0
movd [dstq+stride3q ], m0
RESTORE_GOT
RET
INIT_XMM sse2
cglobal dc_128_predictor_8x8, 2, 5, 1, dst, stride, above, left, goffset
GET_GOT goffsetq
DEFINE_ARGS dst, stride, stride3
lea stride3q, [strideq*3]
movq m0, [GLOBAL(dc_128)]
movq [dstq ], m0
movq [dstq+strideq ], m0
movq [dstq+strideq*2], m0
movq [dstq+stride3q ], m0
lea dstq, [dstq+strideq*4]
movq [dstq ], m0
movq [dstq+strideq ], m0
movq [dstq+strideq*2], m0
movq [dstq+stride3q ], m0
RESTORE_GOT
RET
INIT_XMM sse2
cglobal dc_predictor_16x16, 4, 5, 3, dst, stride, above, left, goffset
GET_GOT goffsetq
pxor m1, m1
mova m0, [aboveq]
mova m2, [leftq]
DEFINE_ARGS dst, stride, stride3, lines4
lea stride3q, [strideq*3]
mov lines4d, 4
psadbw m0, m1
psadbw m2, m1
paddw m0, m2
movhlps m2, m0
paddw m0, m2
paddw m0, [GLOBAL(pw_16)]
psraw m0, 5
pshuflw m0, m0, 0x0
punpcklqdq m0, m0
packuswb m0, m0
.loop:
mova [dstq ], m0
mova [dstq+strideq ], m0
mova [dstq+strideq*2], m0
mova [dstq+stride3q ], m0
lea dstq, [dstq+strideq*4]
dec lines4d
jnz .loop
RESTORE_GOT
REP_RET
INIT_XMM sse2
cglobal dc_top_predictor_16x16, 4, 5, 3, dst, stride, above, left, goffset
GET_GOT goffsetq
pxor m1, m1
mova m0, [aboveq]
DEFINE_ARGS dst, stride, stride3, lines4
lea stride3q, [strideq*3]
mov lines4d, 4
psadbw m0, m1
movhlps m2, m0
paddw m0, m2
paddw m0, [GLOBAL(pw2_16)]
psraw m0, 4
pshuflw m0, m0, 0x0
punpcklqdq m0, m0
packuswb m0, m0
.loop:
mova [dstq ], m0
mova [dstq+strideq ], m0
mova [dstq+strideq*2], m0
mova [dstq+stride3q ], m0
lea dstq, [dstq+strideq*4]
dec lines4d
jnz .loop
RESTORE_GOT
REP_RET
INIT_XMM sse2
cglobal dc_left_predictor_16x16, 4, 5, 3, dst, stride, above, left, goffset
GET_GOT goffsetq
pxor m1, m1
mova m0, [leftq]
DEFINE_ARGS dst, stride, stride3, lines4
lea stride3q, [strideq*3]
mov lines4d, 4
psadbw m0, m1
movhlps m2, m0
paddw m0, m2
paddw m0, [GLOBAL(pw2_16)]
psraw m0, 4
pshuflw m0, m0, 0x0
punpcklqdq m0, m0
packuswb m0, m0
.loop:
mova [dstq ], m0
mova [dstq+strideq ], m0
mova [dstq+strideq*2], m0
mova [dstq+stride3q ], m0
lea dstq, [dstq+strideq*4]
dec lines4d
jnz .loop
RESTORE_GOT
REP_RET
INIT_XMM sse2
cglobal dc_128_predictor_16x16, 4, 5, 3, dst, stride, above, left, goffset
GET_GOT goffsetq
DEFINE_ARGS dst, stride, stride3, lines4
lea stride3q, [strideq*3]
mov lines4d, 4
mova m0, [GLOBAL(dc_128)]
.loop:
mova [dstq ], m0
mova [dstq+strideq ], m0
mova [dstq+strideq*2], m0
mova [dstq+stride3q ], m0
lea dstq, [dstq+strideq*4]
dec lines4d
jnz .loop
RESTORE_GOT
RET
INIT_XMM sse2
cglobal dc_predictor_32x32, 4, 5, 5, dst, stride, above, left, goffset
GET_GOT goffsetq
pxor m1, m1
mova m0, [aboveq]
mova m2, [aboveq+16]
mova m3, [leftq]
mova m4, [leftq+16]
DEFINE_ARGS dst, stride, stride3, lines4
lea stride3q, [strideq*3]
mov lines4d, 8
psadbw m0, m1
psadbw m2, m1
psadbw m3, m1
psadbw m4, m1
paddw m0, m2
paddw m0, m3
paddw m0, m4
movhlps m2, m0
paddw m0, m2
paddw m0, [GLOBAL(pw_32)]
psraw m0, 6
pshuflw m0, m0, 0x0
punpcklqdq m0, m0
packuswb m0, m0
.loop:
mova [dstq ], m0
mova [dstq +16], m0
mova [dstq+strideq ], m0
mova [dstq+strideq +16], m0
mova [dstq+strideq*2 ], m0
mova [dstq+strideq*2+16], m0
mova [dstq+stride3q ], m0
mova [dstq+stride3q +16], m0
lea dstq, [dstq+strideq*4]
dec lines4d
jnz .loop
RESTORE_GOT
REP_RET
INIT_XMM sse2
cglobal dc_top_predictor_32x32, 4, 5, 5, dst, stride, above, left, goffset
GET_GOT goffsetq
pxor m1, m1
mova m0, [aboveq]
mova m2, [aboveq+16]
DEFINE_ARGS dst, stride, stride3, lines4
lea stride3q, [strideq*3]
mov lines4d, 8
psadbw m0, m1
psadbw m2, m1
paddw m0, m2
movhlps m2, m0
paddw m0, m2
paddw m0, [GLOBAL(pw2_32)]
psraw m0, 5
pshuflw m0, m0, 0x0
punpcklqdq m0, m0
packuswb m0, m0
.loop:
mova [dstq ], m0
mova [dstq +16], m0
mova [dstq+strideq ], m0
mova [dstq+strideq +16], m0
mova [dstq+strideq*2 ], m0
mova [dstq+strideq*2+16], m0
mova [dstq+stride3q ], m0
mova [dstq+stride3q +16], m0
lea dstq, [dstq+strideq*4]
dec lines4d
jnz .loop
RESTORE_GOT
REP_RET
INIT_XMM sse2
cglobal dc_left_predictor_32x32, 4, 5, 5, dst, stride, above, left, goffset
GET_GOT goffsetq
pxor m1, m1
mova m0, [leftq]
mova m2, [leftq+16]
DEFINE_ARGS dst, stride, stride3, lines4
lea stride3q, [strideq*3]
mov lines4d, 8
psadbw m0, m1
psadbw m2, m1
paddw m0, m2
movhlps m2, m0
paddw m0, m2
paddw m0, [GLOBAL(pw2_32)]
psraw m0, 5
pshuflw m0, m0, 0x0
punpcklqdq m0, m0
packuswb m0, m0
.loop:
mova [dstq ], m0
mova [dstq +16], m0
mova [dstq+strideq ], m0
mova [dstq+strideq +16], m0
mova [dstq+strideq*2 ], m0
mova [dstq+strideq*2+16], m0
mova [dstq+stride3q ], m0
mova [dstq+stride3q +16], m0
lea dstq, [dstq+strideq*4]
dec lines4d
jnz .loop
RESTORE_GOT
REP_RET
INIT_XMM sse2
cglobal dc_128_predictor_32x32, 4, 5, 3, dst, stride, above, left, goffset
GET_GOT goffsetq
DEFINE_ARGS dst, stride, stride3, lines4
lea stride3q, [strideq*3]
mov lines4d, 8
mova m0, [GLOBAL(dc_128)]
.loop:
mova [dstq ], m0
mova [dstq +16], m0
mova [dstq+strideq ], m0
mova [dstq+strideq +16], m0
mova [dstq+strideq*2 ], m0
mova [dstq+strideq*2+16], m0
mova [dstq+stride3q ], m0
mova [dstq+stride3q +16], m0
lea dstq, [dstq+strideq*4]
dec lines4d
jnz .loop
RESTORE_GOT
RET
INIT_XMM sse2
cglobal v_predictor_4x4, 3, 3, 1, dst, stride, above
movd m0, [aboveq]
movd [dstq ], m0
movd [dstq+strideq], m0
lea dstq, [dstq+strideq*2]
movd [dstq ], m0
movd [dstq+strideq], m0
RET
INIT_XMM sse2
cglobal v_predictor_8x8, 3, 3, 1, dst, stride, above
movq m0, [aboveq]
DEFINE_ARGS dst, stride, stride3
lea stride3q, [strideq*3]
movq [dstq ], m0
movq [dstq+strideq ], m0
movq [dstq+strideq*2], m0
movq [dstq+stride3q ], m0
lea dstq, [dstq+strideq*4]
movq [dstq ], m0
movq [dstq+strideq ], m0
movq [dstq+strideq*2], m0
movq [dstq+stride3q ], m0
RET
INIT_XMM sse2
cglobal v_predictor_16x16, 3, 4, 1, dst, stride, above
mova m0, [aboveq]
DEFINE_ARGS dst, stride, stride3, nlines4
lea stride3q, [strideq*3]
mov nlines4d, 4
.loop:
mova [dstq ], m0
mova [dstq+strideq ], m0
mova [dstq+strideq*2], m0
mova [dstq+stride3q ], m0
lea dstq, [dstq+strideq*4]
dec nlines4d
jnz .loop
REP_RET
INIT_XMM sse2
cglobal v_predictor_32x32, 3, 4, 2, dst, stride, above
mova m0, [aboveq]
mova m1, [aboveq+16]
DEFINE_ARGS dst, stride, stride3, nlines4
lea stride3q, [strideq*3]
mov nlines4d, 8
.loop:
mova [dstq ], m0
mova [dstq +16], m1
mova [dstq+strideq ], m0
mova [dstq+strideq +16], m1
mova [dstq+strideq*2 ], m0
mova [dstq+strideq*2+16], m1
mova [dstq+stride3q ], m0
mova [dstq+stride3q +16], m1
lea dstq, [dstq+strideq*4]
dec nlines4d
jnz .loop
REP_RET
INIT_XMM sse2
cglobal h_predictor_4x4, 2, 4, 4, dst, stride, line, left
movifnidn leftq, leftmp
movd m0, [leftq]
punpcklbw m0, m0
punpcklbw m0, m0
pshufd m1, m0, 0x1
movd [dstq ], m0
movd [dstq+strideq], m1
pshufd m2, m0, 0x2
lea dstq, [dstq+strideq*2]
pshufd m3, m0, 0x3
movd [dstq ], m2
movd [dstq+strideq], m3
RET
INIT_XMM sse2
cglobal h_predictor_8x8, 2, 5, 3, dst, stride, line, left
movifnidn leftq, leftmp
mov lineq, -2
DEFINE_ARGS dst, stride, line, left, stride3
lea stride3q, [strideq*3]
movq m0, [leftq ]
punpcklbw m0, m0 ; l1 l1 l2 l2 ... l8 l8
.loop:
pshuflw m1, m0, 0x0 ; l1 l1 l1 l1 l1 l1 l1 l1
pshuflw m2, m0, 0x55 ; l2 l2 l2 l2 l2 l2 l2 l2
movq [dstq ], m1
movq [dstq+strideq], m2
pshuflw m1, m0, 0xaa
pshuflw m2, m0, 0xff
movq [dstq+strideq*2], m1
movq [dstq+stride3q ], m2
pshufd m0, m0, 0xe ; [63:0] l5 l5 l6 l6 l7 l7 l8 l8
inc lineq
lea dstq, [dstq+strideq*4]
jnz .loop
REP_RET
INIT_XMM sse2
cglobal h_predictor_16x16, 2, 5, 3, dst, stride, line, left
movifnidn leftq, leftmp
mov lineq, -4
DEFINE_ARGS dst, stride, line, left, stride3
lea stride3q, [strideq*3]
.loop:
movd m0, [leftq]
punpcklbw m0, m0
punpcklbw m0, m0 ; l1 to l4 each repeated 4 times
pshufd m1, m0, 0x0 ; l1 repeated 16 times
pshufd m2, m0, 0x55 ; l2 repeated 16 times
mova [dstq ], m1
mova [dstq+strideq ], m2
pshufd m1, m0, 0xaa
pshufd m2, m0, 0xff
mova [dstq+strideq*2], m1
mova [dstq+stride3q ], m2
inc lineq
lea leftq, [leftq+4 ]
lea dstq, [dstq+strideq*4]
jnz .loop
REP_RET
INIT_XMM sse2
cglobal h_predictor_32x32, 2, 5, 3, dst, stride, line, left
movifnidn leftq, leftmp
mov lineq, -8
DEFINE_ARGS dst, stride, line, left, stride3
lea stride3q, [strideq*3]
.loop:
movd m0, [leftq]
punpcklbw m0, m0
punpcklbw m0, m0 ; l1 to l4 each repeated 4 times
pshufd m1, m0, 0x0 ; l1 repeated 16 times
pshufd m2, m0, 0x55 ; l2 repeated 16 times
mova [dstq ], m1
mova [dstq+16 ], m1
mova [dstq+strideq ], m2
mova [dstq+strideq+16 ], m2
pshufd m1, m0, 0xaa
pshufd m2, m0, 0xff
mova [dstq+strideq*2 ], m1
mova [dstq+strideq*2+16], m1
mova [dstq+stride3q ], m2
mova [dstq+stride3q+16 ], m2
inc lineq
lea leftq, [leftq+4 ]
lea dstq, [dstq+strideq*4]
jnz .loop
REP_RET

3573
third_party/aom/aom_dsp/x86/intrapred_avx2.c vendored
View file

File diff suppressed because it is too large Load diff

64
third_party/aom/aom_dsp/x86/jnt_sad_ssse3.c vendored
View file

@ -192,47 +192,47 @@ unsigned int aom_sad128xh_sse2(const uint8_t *a, int a_stride, const uint8_t *b,
return res;
}
#define jnt_sadMxN_sse2(m, n) \
unsigned int aom_jnt_sad##m##x##n##_avg_ssse3( \
#define dist_wtd_sadMxN_sse2(m, n) \
unsigned int aom_dist_wtd_sad##m##x##n##_avg_ssse3( \
const uint8_t *src, int src_stride, const uint8_t *ref, int ref_stride, \
const uint8_t *second_pred, const JNT_COMP_PARAMS *jcp_param) { \
const uint8_t *second_pred, const DIST_WTD_COMP_PARAMS *jcp_param) { \
uint8_t comp_pred[m * n]; \
aom_jnt_comp_avg_pred(comp_pred, second_pred, m, n, ref, ref_stride, \
jcp_param); \
aom_dist_wtd_comp_avg_pred(comp_pred, second_pred, m, n, ref, ref_stride, \
jcp_param); \
return aom_sad##m##xh_sse2(src, src_stride, comp_pred, m, m, n); \
}
#define jnt_sadMxN_avx2(m, n) \
unsigned int aom_jnt_sad##m##x##n##_avg_avx2( \
#define dist_wtd_sadMxN_avx2(m, n) \
unsigned int aom_dist_wtd_sad##m##x##n##_avg_avx2( \
const uint8_t *src, int src_stride, const uint8_t *ref, int ref_stride, \
const uint8_t *second_pred, const JNT_COMP_PARAMS *jcp_param) { \
const uint8_t *second_pred, const DIST_WTD_COMP_PARAMS *jcp_param) { \
uint8_t comp_pred[m * n]; \
aom_jnt_comp_avg_pred(comp_pred, second_pred, m, n, ref, ref_stride, \
jcp_param); \
aom_dist_wtd_comp_avg_pred(comp_pred, second_pred, m, n, ref, ref_stride, \
jcp_param); \
return aom_sad##m##xh_avx2(src, src_stride, comp_pred, m, m, n); \
}
/* clang-format off */
jnt_sadMxN_sse2(128, 128)
jnt_sadMxN_sse2(128, 64)
jnt_sadMxN_sse2(64, 128)
jnt_sadMxN_sse2(64, 64)
jnt_sadMxN_sse2(64, 32)
jnt_sadMxN_sse2(32, 64)
jnt_sadMxN_sse2(32, 32)
jnt_sadMxN_sse2(32, 16)
jnt_sadMxN_sse2(16, 32)
jnt_sadMxN_sse2(16, 16)
jnt_sadMxN_sse2(16, 8)
jnt_sadMxN_sse2(8, 16)
jnt_sadMxN_sse2(8, 8)
jnt_sadMxN_sse2(8, 4)
jnt_sadMxN_sse2(4, 8)
jnt_sadMxN_sse2(4, 4)
jnt_sadMxN_sse2(4, 16)
jnt_sadMxN_sse2(16, 4)
jnt_sadMxN_sse2(8, 32)
jnt_sadMxN_sse2(32, 8)
jnt_sadMxN_sse2(16, 64)
jnt_sadMxN_sse2(64, 16)
dist_wtd_sadMxN_sse2(128, 128)
dist_wtd_sadMxN_sse2(128, 64)
dist_wtd_sadMxN_sse2(64, 128)
dist_wtd_sadMxN_sse2(64, 64)
dist_wtd_sadMxN_sse2(64, 32)
dist_wtd_sadMxN_sse2(32, 64)
dist_wtd_sadMxN_sse2(32, 32)
dist_wtd_sadMxN_sse2(32, 16)
dist_wtd_sadMxN_sse2(16, 32)
dist_wtd_sadMxN_sse2(16, 16)
dist_wtd_sadMxN_sse2(16, 8)
dist_wtd_sadMxN_sse2(8, 16)
dist_wtd_sadMxN_sse2(8, 8)
dist_wtd_sadMxN_sse2(8, 4)
dist_wtd_sadMxN_sse2(4, 8)
dist_wtd_sadMxN_sse2(4, 4)
dist_wtd_sadMxN_sse2(4, 16)
dist_wtd_sadMxN_sse2(16, 4)
dist_wtd_sadMxN_sse2(8, 32)
dist_wtd_sadMxN_sse2(32, 8)
dist_wtd_sadMxN_sse2(16, 64)
dist_wtd_sadMxN_sse2(64, 16)
/* clang-format on */

102
third_party/aom/aom_dsp/x86/jnt_variance_ssse3.c vendored
View file

@ -29,7 +29,7 @@ void aom_var_filter_block2d_bil_second_pass_ssse3(
unsigned int pixel_step, unsigned int output_height,
unsigned int output_width, const uint8_t *filter);
static INLINE void compute_jnt_comp_avg(__m128i *p0, __m128i *p1,
static INLINE void compute_dist_wtd_avg(__m128i *p0, __m128i *p1,
const __m128i *w, const __m128i *r,
void *const result) {
__m128i p_lo = _mm_unpacklo_epi8(*p0, *p1);
@ -45,10 +45,10 @@ static INLINE void compute_jnt_comp_avg(__m128i *p0, __m128i *p1,
xx_storeu_128(result, _mm_packus_epi16(shift_lo, shift_hi));
}
void aom_jnt_comp_avg_pred_ssse3(uint8_t *comp_pred, const uint8_t *pred,
int width, int height, const uint8_t *ref,
int ref_stride,
const JNT_COMP_PARAMS *jcp_param) {
void aom_dist_wtd_comp_avg_pred_ssse3(uint8_t *comp_pred, const uint8_t *pred,
int width, int height, const uint8_t *ref,
int ref_stride,
const DIST_WTD_COMP_PARAMS *jcp_param) {
int i;
const uint8_t w0 = (uint8_t)jcp_param->fwd_offset;
const uint8_t w1 = (uint8_t)jcp_param->bck_offset;
@ -67,7 +67,7 @@ void aom_jnt_comp_avg_pred_ssse3(uint8_t *comp_pred, const uint8_t *pred,
__m128i p0 = xx_loadu_128(ref);
__m128i p1 = xx_loadu_128(pred);
compute_jnt_comp_avg(&p0, &p1, &w, &r, comp_pred);
compute_dist_wtd_avg(&p0, &p1, &w, &r, comp_pred);
comp_pred += 16;
pred += 16;
@ -85,7 +85,7 @@ void aom_jnt_comp_avg_pred_ssse3(uint8_t *comp_pred, const uint8_t *pred,
__m128i p0 = _mm_unpacklo_epi64(p0_0, p0_1);
__m128i p1 = xx_loadu_128(pred);
compute_jnt_comp_avg(&p0, &p1, &w, &r, comp_pred);
compute_dist_wtd_avg(&p0, &p1, &w, &r, comp_pred);
comp_pred += 16;
pred += 16;
@ -107,7 +107,7 @@ void aom_jnt_comp_avg_pred_ssse3(uint8_t *comp_pred, const uint8_t *pred,
row3[0], row3[1], row3[2], row3[3]);
__m128i p1 = xx_loadu_128(pred);
compute_jnt_comp_avg(&p0, &p1, &w, &r, comp_pred);
compute_dist_wtd_avg(&p0, &p1, &w, &r, comp_pred);
comp_pred += 16;
pred += 16;
@ -116,11 +116,11 @@ void aom_jnt_comp_avg_pred_ssse3(uint8_t *comp_pred, const uint8_t *pred,
}
}
void aom_jnt_comp_avg_upsampled_pred_ssse3(
void aom_dist_wtd_comp_avg_upsampled_pred_ssse3(
MACROBLOCKD *xd, const struct AV1Common *const cm, int mi_row, int mi_col,
const MV *const mv, uint8_t *comp_pred, const uint8_t *pred, int width,
int height, int subpel_x_q3, int subpel_y_q3, const uint8_t *ref,
int ref_stride, const JNT_COMP_PARAMS *jcp_param, int subpel_search) {
int ref_stride, const DIST_WTD_COMP_PARAMS *jcp_param, int subpel_search) {
int n;
int i;
aom_upsampled_pred(xd, cm, mi_row, mi_col, mv, comp_pred, width, height,
@ -141,52 +141,52 @@ void aom_jnt_comp_avg_upsampled_pred_ssse3(
__m128i p0 = xx_loadu_128(comp_pred);
__m128i p1 = xx_loadu_128(pred);
compute_jnt_comp_avg(&p0, &p1, &w, &r, comp_pred);
compute_dist_wtd_avg(&p0, &p1, &w, &r, comp_pred);
comp_pred += 16;
pred += 16;
}
}
#define JNT_SUBPIX_AVG_VAR(W, H) \
uint32_t aom_jnt_sub_pixel_avg_variance##W##x##H##_ssse3( \
const uint8_t *a, int a_stride, int xoffset, int yoffset, \
const uint8_t *b, int b_stride, uint32_t *sse, \
const uint8_t *second_pred, const JNT_COMP_PARAMS *jcp_param) { \
uint16_t fdata3[(H + 1) * W]; \
uint8_t temp2[H * W]; \
DECLARE_ALIGNED(16, uint8_t, temp3[H * W]); \
\
aom_var_filter_block2d_bil_first_pass_ssse3( \
a, fdata3, a_stride, 1, H + 1, W, bilinear_filters_2t[xoffset]); \
aom_var_filter_block2d_bil_second_pass_ssse3( \
fdata3, temp2, W, W, H, W, bilinear_filters_2t[yoffset]); \
\
aom_jnt_comp_avg_pred_ssse3(temp3, second_pred, W, H, temp2, W, \
jcp_param); \
\
return aom_variance##W##x##H(temp3, W, b, b_stride, sse); \
#define DIST_WTD_SUBPIX_AVG_VAR(W, H) \
uint32_t aom_dist_wtd_sub_pixel_avg_variance##W##x##H##_ssse3( \
const uint8_t *a, int a_stride, int xoffset, int yoffset, \
const uint8_t *b, int b_stride, uint32_t *sse, \
const uint8_t *second_pred, const DIST_WTD_COMP_PARAMS *jcp_param) { \
uint16_t fdata3[(H + 1) * W]; \
uint8_t temp2[H * W]; \
DECLARE_ALIGNED(16, uint8_t, temp3[H * W]); \
\
aom_var_filter_block2d_bil_first_pass_ssse3( \
a, fdata3, a_stride, 1, H + 1, W, bilinear_filters_2t[xoffset]); \
aom_var_filter_block2d_bil_second_pass_ssse3( \
fdata3, temp2, W, W, H, W, bilinear_filters_2t[yoffset]); \
\
aom_dist_wtd_comp_avg_pred_ssse3(temp3, second_pred, W, H, temp2, W, \
jcp_param); \
\
return aom_variance##W##x##H(temp3, W, b, b_stride, sse); \
}
JNT_SUBPIX_AVG_VAR(128, 128)
JNT_SUBPIX_AVG_VAR(128, 64)
JNT_SUBPIX_AVG_VAR(64, 128)
JNT_SUBPIX_AVG_VAR(64, 64)
JNT_SUBPIX_AVG_VAR(64, 32)
JNT_SUBPIX_AVG_VAR(32, 64)
JNT_SUBPIX_AVG_VAR(32, 32)
JNT_SUBPIX_AVG_VAR(32, 16)
JNT_SUBPIX_AVG_VAR(16, 32)
JNT_SUBPIX_AVG_VAR(16, 16)
JNT_SUBPIX_AVG_VAR(16, 8)
JNT_SUBPIX_AVG_VAR(8, 16)
JNT_SUBPIX_AVG_VAR(8, 8)
JNT_SUBPIX_AVG_VAR(8, 4)
JNT_SUBPIX_AVG_VAR(4, 8)
JNT_SUBPIX_AVG_VAR(4, 4)
JNT_SUBPIX_AVG_VAR(4, 16)
JNT_SUBPIX_AVG_VAR(16, 4)
JNT_SUBPIX_AVG_VAR(8, 32)
JNT_SUBPIX_AVG_VAR(32, 8)
JNT_SUBPIX_AVG_VAR(16, 64)
JNT_SUBPIX_AVG_VAR(64, 16)
DIST_WTD_SUBPIX_AVG_VAR(128, 128)
DIST_WTD_SUBPIX_AVG_VAR(128, 64)
DIST_WTD_SUBPIX_AVG_VAR(64, 128)
DIST_WTD_SUBPIX_AVG_VAR(64, 64)
DIST_WTD_SUBPIX_AVG_VAR(64, 32)
DIST_WTD_SUBPIX_AVG_VAR(32, 64)
DIST_WTD_SUBPIX_AVG_VAR(32, 32)
DIST_WTD_SUBPIX_AVG_VAR(32, 16)
DIST_WTD_SUBPIX_AVG_VAR(16, 32)
DIST_WTD_SUBPIX_AVG_VAR(16, 16)
DIST_WTD_SUBPIX_AVG_VAR(16, 8)
DIST_WTD_SUBPIX_AVG_VAR(8, 16)
DIST_WTD_SUBPIX_AVG_VAR(8, 8)
DIST_WTD_SUBPIX_AVG_VAR(8, 4)
DIST_WTD_SUBPIX_AVG_VAR(4, 8)
DIST_WTD_SUBPIX_AVG_VAR(4, 4)
DIST_WTD_SUBPIX_AVG_VAR(4, 16)
DIST_WTD_SUBPIX_AVG_VAR(16, 4)
DIST_WTD_SUBPIX_AVG_VAR(8, 32)
DIST_WTD_SUBPIX_AVG_VAR(32, 8)
DIST_WTD_SUBPIX_AVG_VAR(16, 64)
DIST_WTD_SUBPIX_AVG_VAR(64, 16)

453
third_party/aom/aom_dsp/x86/loopfilter_sse2.c vendored
View file

@ -16,347 +16,12 @@
#include "aom_dsp/x86/synonyms.h"
#include "aom_ports/mem.h"
#include "aom_ports/emmintrin_compat.h"
#include "aom_dsp/x86/lpf_common_sse2.h"
static INLINE __m128i abs_diff(__m128i a, __m128i b) {
return _mm_or_si128(_mm_subs_epu8(a, b), _mm_subs_epu8(b, a));
}
static INLINE void transpose4x8_8x4_low_sse2(__m128i *x0, __m128i *x1,
__m128i *x2, __m128i *x3,
__m128i *d0, __m128i *d1,
__m128i *d2, __m128i *d3) {
// input
// x0 00 01 02 03 04 05 06 07 xx xx xx xx xx xx xx xx
// x1 10 11 12 13 14 15 16 17 xx xx xx xx xx xx xx xx
// x2 20 21 22 23 24 25 26 27 xx xx xx xx xx xx xx xx
// x3 30 31 32 33 34 35 36 37 xx xx xx xx xx xx xx xx
// output
// 00 10 20 30 xx xx xx xx xx xx xx xx xx xx xx xx
// 01 11 21 31 xx xx xx xx xx xx xx xx xx xx xx xx
// 02 12 22 32 xx xx xx xx xx xx xx xx xx xx xx xx
// 03 13 23 33 xx xx xx xx xx xx xx xx xx xx xx xx
__m128i w0, w1;
w0 = _mm_unpacklo_epi8(
*x0, *x1); // 00 10 01 11 02 12 03 13 04 14 05 15 06 16 07 17
w1 = _mm_unpacklo_epi8(
*x2, *x3); // 20 30 21 31 22 32 23 33 24 34 25 35 26 36 27 37
*d0 = _mm_unpacklo_epi16(
w0, w1); // 00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33
*d1 = _mm_srli_si128(*d0,
4); // 01 11 21 31 xx xx xx xx xx xx xx xx xx xx xx xx
*d2 = _mm_srli_si128(*d0,
8); // 02 12 22 32 xx xx xx xx xx xx xx xx xx xx xx xx
*d3 = _mm_srli_si128(*d0,
12); // 03 13 23 33 xx xx xx xx xx xx xx xx xx xx xx xx
}
static INLINE void transpose4x8_8x4_sse2(__m128i *x0, __m128i *x1, __m128i *x2,
__m128i *x3, __m128i *d0, __m128i *d1,
__m128i *d2, __m128i *d3, __m128i *d4,
__m128i *d5, __m128i *d6,
__m128i *d7) {
// input
// x0 00 01 02 03 04 05 06 07 xx xx xx xx xx xx xx xx
// x1 10 11 12 13 14 15 16 17 xx xx xx xx xx xx xx xx
// x2 20 21 22 23 24 25 26 27 xx xx xx xx xx xx xx xx
// x3 30 31 32 33 34 35 36 37 xx xx xx xx xx xx xx xx
// output
// 00 10 20 30 xx xx xx xx xx xx xx xx xx xx xx xx
// 01 11 21 31 xx xx xx xx xx xx xx xx xx xx xx xx
// 02 12 22 32 xx xx xx xx xx xx xx xx xx xx xx xx
// 03 13 23 33 xx xx xx xx xx xx xx xx xx xx xx xx
// 04 14 24 34 xx xx xx xx xx xx xx xx xx xx xx xx
// 05 15 25 35 xx xx xx xx xx xx xx xx xx xx xx xx
// 06 16 26 36 xx xx xx xx xx xx xx xx xx xx xx xx
// 07 17 27 37 xx xx xx xx xx xx xx xx xx xx xx xx
__m128i w0, w1, ww0, ww1;
w0 = _mm_unpacklo_epi8(
*x0, *x1); // 00 10 01 11 02 12 03 13 04 14 05 15 06 16 07 17
w1 = _mm_unpacklo_epi8(
*x2, *x3); // 20 30 21 31 22 32 23 33 24 34 25 35 26 36 27 37
ww0 = _mm_unpacklo_epi16(
w0, w1); // 00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33
ww1 = _mm_unpackhi_epi16(
w0, w1); // 04 14 24 34 05 15 25 35 06 16 26 36 07 17 27 37
*d0 = ww0; // 00 10 20 30 xx xx xx xx xx xx xx xx xx xx xx xx
*d1 = _mm_srli_si128(ww0,
4); // 01 11 21 31 xx xx xx xx xx xx xx xx xx xx xx xx
*d2 = _mm_srli_si128(ww0,
8); // 02 12 22 32 xx xx xx xx xx xx xx xx xx xx xx xx
*d3 = _mm_srli_si128(ww0,
12); // 03 13 23 33 xx xx xx xx xx xx xx xx xx xx xx xx
*d4 = ww1; // 04 14 24 34 xx xx xx xx xx xx xx xx xx xx xx xx
*d5 = _mm_srli_si128(ww1,
4); // 05 15 25 35 xx xx xx xx xx xx xx xx xx xx xx xx
*d6 = _mm_srli_si128(ww1,
8); // 06 16 26 36 xx xx xx xx xx xx xx xx xx xx xx xx
*d7 = _mm_srli_si128(ww1,
12); // 07 17 27 37 xx xx xx xx xx xx xx xx xx xx xx xx
}
static INLINE void transpose8x8_low_sse2(__m128i *x0, __m128i *x1, __m128i *x2,
__m128i *x3, __m128i *x4, __m128i *x5,
__m128i *x6, __m128i *x7, __m128i *d0,
__m128i *d1, __m128i *d2,
__m128i *d3) {
// input
// x0 00 01 02 03 04 05 06 07
// x1 10 11 12 13 14 15 16 17
// x2 20 21 22 23 24 25 26 27
// x3 30 31 32 33 34 35 36 37
// x4 40 41 42 43 44 45 46 47
// x5 50 51 52 53 54 55 56 57
// x6 60 61 62 63 64 65 66 67
// x7 70 71 72 73 74 75 76 77
// output
// d0 00 10 20 30 40 50 60 70 xx xx xx xx xx xx xx
// d1 01 11 21 31 41 51 61 71 xx xx xx xx xx xx xx xx
// d2 02 12 22 32 42 52 62 72 xx xx xx xx xx xx xx xx
// d3 03 13 23 33 43 53 63 73 xx xx xx xx xx xx xx xx
__m128i w0, w1, w2, w3, w4, w5;
w0 = _mm_unpacklo_epi8(
*x0, *x1); // 00 10 01 11 02 12 03 13 04 14 05 15 06 16 07 17
w1 = _mm_unpacklo_epi8(
*x2, *x3); // 20 30 21 31 22 32 23 33 24 34 25 35 26 36 27 37
w2 = _mm_unpacklo_epi8(
*x4, *x5); // 40 50 41 51 42 52 43 53 44 54 45 55 46 56 47 57
w3 = _mm_unpacklo_epi8(
*x6, *x7); // 60 70 61 71 62 72 63 73 64 74 65 75 66 76 67 77
w4 = _mm_unpacklo_epi16(
w0, w1); // 00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33
w5 = _mm_unpacklo_epi16(
w2, w3); // 40 50 60 70 41 51 61 71 42 52 62 72 43 53 63 73
*d0 = _mm_unpacklo_epi32(
w4, w5); // 00 10 20 30 40 50 60 70 01 11 21 31 41 51 61 71
*d1 = _mm_srli_si128(*d0, 8);
*d2 = _mm_unpackhi_epi32(
w4, w5); // 02 12 22 32 42 52 62 72 03 13 23 33 43 53 63 73
*d3 = _mm_srli_si128(*d2, 8);
}
static INLINE void transpose8x8_sse2(__m128i *x0, __m128i *x1, __m128i *x2,
__m128i *x3, __m128i *x4, __m128i *x5,
__m128i *x6, __m128i *x7, __m128i *d0d1,
__m128i *d2d3, __m128i *d4d5,
__m128i *d6d7) {
__m128i w0, w1, w2, w3, w4, w5, w6, w7;
// x0 00 01 02 03 04 05 06 07
// x1 10 11 12 13 14 15 16 17
w0 = _mm_unpacklo_epi8(
*x0, *x1); // 00 10 01 11 02 12 03 13 04 14 05 15 06 16 07 17
// x2 20 21 22 23 24 25 26 27
// x3 30 31 32 33 34 35 36 37
w1 = _mm_unpacklo_epi8(
*x2, *x3); // 20 30 21 31 22 32 23 33 24 34 25 35 26 36 27 37
// x4 40 41 42 43 44 45 46 47
// x5 50 51 52 53 54 55 56 57
w2 = _mm_unpacklo_epi8(
*x4, *x5); // 40 50 41 51 42 52 43 53 44 54 45 55 46 56 47 57
// x6 60 61 62 63 64 65 66 67
// x7 70 71 72 73 74 75 76 77
w3 = _mm_unpacklo_epi8(
*x6, *x7); // 60 70 61 71 62 72 63 73 64 74 65 75 66 76 67 77
w4 = _mm_unpacklo_epi16(
w0, w1); // 00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33
w5 = _mm_unpacklo_epi16(
w2, w3); // 40 50 60 70 41 51 61 71 42 52 62 72 43 53 63 73
*d0d1 = _mm_unpacklo_epi32(
w4, w5); // 00 10 20 30 40 50 60 70 01 11 21 31 41 51 61 71
*d2d3 = _mm_unpackhi_epi32(
w4, w5); // 02 12 22 32 42 52 62 72 03 13 23 33 43 53 63 73
w6 = _mm_unpackhi_epi16(
w0, w1); // 04 14 24 34 05 15 25 35 06 16 26 36 07 17 27 37
w7 = _mm_unpackhi_epi16(
w2, w3); // 44 54 64 74 45 55 65 75 46 56 66 76 47 57 67 77
*d4d5 = _mm_unpacklo_epi32(
w6, w7); // 04 14 24 34 44 54 64 74 05 15 25 35 45 55 65 75
*d6d7 = _mm_unpackhi_epi32(
w6, w7); // 06 16 26 36 46 56 66 76 07 17 27 37 47 57 67 77
}
static INLINE void transpose16x8_8x16_sse2(
__m128i *x0, __m128i *x1, __m128i *x2, __m128i *x3, __m128i *x4,
__m128i *x5, __m128i *x6, __m128i *x7, __m128i *x8, __m128i *x9,
__m128i *x10, __m128i *x11, __m128i *x12, __m128i *x13, __m128i *x14,
__m128i *x15, __m128i *d0, __m128i *d1, __m128i *d2, __m128i *d3,
__m128i *d4, __m128i *d5, __m128i *d6, __m128i *d7) {
__m128i w0, w1, w2, w3, w4, w5, w6, w7, w8, w9;
__m128i w10, w11, w12, w13, w14, w15;
w0 = _mm_unpacklo_epi8(*x0, *x1);
w1 = _mm_unpacklo_epi8(*x2, *x3);
w2 = _mm_unpacklo_epi8(*x4, *x5);
w3 = _mm_unpacklo_epi8(*x6, *x7);
w8 = _mm_unpacklo_epi8(*x8, *x9);
w9 = _mm_unpacklo_epi8(*x10, *x11);
w10 = _mm_unpacklo_epi8(*x12, *x13);
w11 = _mm_unpacklo_epi8(*x14, *x15);
w4 = _mm_unpacklo_epi16(w0, w1);
w5 = _mm_unpacklo_epi16(w2, w3);
w12 = _mm_unpacklo_epi16(w8, w9);
w13 = _mm_unpacklo_epi16(w10, w11);
w6 = _mm_unpacklo_epi32(w4, w5);
w7 = _mm_unpackhi_epi32(w4, w5);
w14 = _mm_unpacklo_epi32(w12, w13);
w15 = _mm_unpackhi_epi32(w12, w13);
// Store first 4-line result
*d0 = _mm_unpacklo_epi64(w6, w14);
*d1 = _mm_unpackhi_epi64(w6, w14);
*d2 = _mm_unpacklo_epi64(w7, w15);
*d3 = _mm_unpackhi_epi64(w7, w15);
w4 = _mm_unpackhi_epi16(w0, w1);
w5 = _mm_unpackhi_epi16(w2, w3);
w12 = _mm_unpackhi_epi16(w8, w9);
w13 = _mm_unpackhi_epi16(w10, w11);
w6 = _mm_unpacklo_epi32(w4, w5);
w7 = _mm_unpackhi_epi32(w4, w5);
w14 = _mm_unpacklo_epi32(w12, w13);
w15 = _mm_unpackhi_epi32(w12, w13);
// Store second 4-line result
*d4 = _mm_unpacklo_epi64(w6, w14);
*d5 = _mm_unpackhi_epi64(w6, w14);
*d6 = _mm_unpacklo_epi64(w7, w15);
*d7 = _mm_unpackhi_epi64(w7, w15);
}
// this function treats its input as 2 parallel 8x4 matrices, transposes each of
// them independently while flipping the second matrix horizontaly Used for 14
// taps filter pq pairs inverse
static INLINE void transpose_pq_14_inv_sse2(__m128i *x0, __m128i *x1,
__m128i *x2, __m128i *x3,
__m128i *x4, __m128i *x5,
__m128i *x6, __m128i *x7,
__m128i *pq0, __m128i *pq1,
__m128i *pq2, __m128i *pq3) {
__m128i w10, w11, w12, w13;
__m128i w0, w1, w2, w3, w4, w5;
__m128i d0, d1, d2, d3;
w0 = _mm_unpacklo_epi8(
*x0, *x1); // p 00 10 01 11 02 12 03 13 04 14 05 15 06 16 07 17
w1 = _mm_unpacklo_epi8(
*x2, *x3); // p 20 30 21 31 22 32 23 33 24 34 25 35 26 36 27 37
w2 = _mm_unpacklo_epi8(
*x4, *x5); // p 40 50 41 51 42 52 43 53 44 54 45 55 46 56 47 57
w3 = _mm_unpacklo_epi8(
*x6, *x7); // p 60 70 61 71 62 72 63 73 64 74 65 75 66 76 67 77
w4 = _mm_unpacklo_epi16(
w0, w1); // 00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33
w5 = _mm_unpacklo_epi16(
w2, w3); // 40 50 60 70 41 51 61 71 42 52 62 72 43 53 63 73
d0 = _mm_unpacklo_epi32(
w4, w5); // 00 10 20 30 40 50 60 70 01 11 21 31 41 51 61 71
d2 = _mm_unpackhi_epi32(
w4, w5); // 02 12 22 32 42 52 62 72 03 13 23 33 43 53 63 73
w10 = _mm_unpacklo_epi8(
*x7, *x6); // q xx xx xx xx xx xx xx xx 00 10 01 11 02 12 03 13
w11 = _mm_unpacklo_epi8(
*x5, *x4); // q xx xx xx xx xx xx xx xx 20 30 21 31 22 32 23 33
w12 = _mm_unpacklo_epi8(
*x3, *x2); // q xx xx xx xx xx xx xx xx 40 50 41 51 42 52 43 53
w13 = _mm_unpacklo_epi8(
*x1, *x0); // q xx xx xx xx xx xx xx xx 60 70 61 71 62 72 63 73
w4 = _mm_unpackhi_epi16(
w10, w11); // 00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33
w5 = _mm_unpackhi_epi16(
w12, w13); // 40 50 60 70 41 51 61 71 42 52 62 72 43 53 63 73
d1 = _mm_unpacklo_epi32(
w4, w5); // 00 10 20 30 40 50 60 70 01 11 21 31 41 51 61 71
d3 = _mm_unpackhi_epi32(
w4, w5); // 02 12 22 32 42 52 62 72 03 13 23 33 43 53 63 73
*pq0 = _mm_unpacklo_epi64(d0, d1); // pq
*pq1 = _mm_unpackhi_epi64(d0, d1); // pq
*pq2 = _mm_unpacklo_epi64(d2, d3); // pq
*pq3 = _mm_unpackhi_epi64(d2, d3); // pq
}
static INLINE void transpose8x16_16x8_sse2(
__m128i *x0, __m128i *x1, __m128i *x2, __m128i *x3, __m128i *x4,
__m128i *x5, __m128i *x6, __m128i *x7, __m128i *d0d1, __m128i *d2d3,
__m128i *d4d5, __m128i *d6d7, __m128i *d8d9, __m128i *d10d11,
__m128i *d12d13, __m128i *d14d15) {
__m128i w0, w1, w2, w3, w4, w5, w6, w7, w8, w9;
__m128i w10, w11, w12, w13, w14, w15;
w0 = _mm_unpacklo_epi8(*x0, *x1);
w1 = _mm_unpacklo_epi8(*x2, *x3);
w2 = _mm_unpacklo_epi8(*x4, *x5);
w3 = _mm_unpacklo_epi8(*x6, *x7);
w8 = _mm_unpackhi_epi8(*x0, *x1);
w9 = _mm_unpackhi_epi8(*x2, *x3);
w10 = _mm_unpackhi_epi8(*x4, *x5);
w11 = _mm_unpackhi_epi8(*x6, *x7);
w4 = _mm_unpacklo_epi16(w0, w1);
w5 = _mm_unpacklo_epi16(w2, w3);
w12 = _mm_unpacklo_epi16(w8, w9);
w13 = _mm_unpacklo_epi16(w10, w11);
w6 = _mm_unpacklo_epi32(w4, w5);
w7 = _mm_unpackhi_epi32(w4, w5);
w14 = _mm_unpacklo_epi32(w12, w13);
w15 = _mm_unpackhi_epi32(w12, w13);
// Store first 4-line result
*d0d1 = _mm_unpacklo_epi64(w6, w14);
*d2d3 = _mm_unpackhi_epi64(w6, w14);
*d4d5 = _mm_unpacklo_epi64(w7, w15);
*d6d7 = _mm_unpackhi_epi64(w7, w15);
w4 = _mm_unpackhi_epi16(w0, w1);
w5 = _mm_unpackhi_epi16(w2, w3);
w12 = _mm_unpackhi_epi16(w8, w9);
w13 = _mm_unpackhi_epi16(w10, w11);
w6 = _mm_unpacklo_epi32(w4, w5);
w7 = _mm_unpackhi_epi32(w4, w5);
w14 = _mm_unpacklo_epi32(w12, w13);
w15 = _mm_unpackhi_epi32(w12, w13);
// Store second 4-line result
*d8d9 = _mm_unpacklo_epi64(w6, w14);
*d10d11 = _mm_unpackhi_epi64(w6, w14);
*d12d13 = _mm_unpacklo_epi64(w7, w15);
*d14d15 = _mm_unpackhi_epi64(w7, w15);
}
// this function treats its input as 2 parallel 8x4 matrices, transposes each of
// them to 4x8 independently while flipping the second matrix horizontaly. Used
// for 14 taps pq pairs creation
@ -416,6 +81,63 @@ static INLINE void transpose_pq_14_sse2(__m128i *x0, __m128i *x1, __m128i *x2,
ww2); // 07 17 27 37 08 18 28 38 xx xx xx xx xx xx xx xx
}
// this function treats its input as 2 parallel 8x4 matrices, transposes each of
// them independently while flipping the second matrix horizontaly Used for 14
// taps filter pq pairs inverse
static INLINE void transpose_pq_14_inv_sse2(__m128i *x0, __m128i *x1,
__m128i *x2, __m128i *x3,
__m128i *x4, __m128i *x5,
__m128i *x6, __m128i *x7,
__m128i *pq0, __m128i *pq1,
__m128i *pq2, __m128i *pq3) {
__m128i w10, w11, w12, w13;
__m128i w0, w1, w2, w3, w4, w5;
__m128i d0, d1, d2, d3;
w0 = _mm_unpacklo_epi8(
*x0, *x1); // p 00 10 01 11 02 12 03 13 04 14 05 15 06 16 07 17
w1 = _mm_unpacklo_epi8(
*x2, *x3); // p 20 30 21 31 22 32 23 33 24 34 25 35 26 36 27 37
w2 = _mm_unpacklo_epi8(
*x4, *x5); // p 40 50 41 51 42 52 43 53 44 54 45 55 46 56 47 57
w3 = _mm_unpacklo_epi8(
*x6, *x7); // p 60 70 61 71 62 72 63 73 64 74 65 75 66 76 67 77
w4 = _mm_unpacklo_epi16(
w0, w1); // 00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33
w5 = _mm_unpacklo_epi16(
w2, w3); // 40 50 60 70 41 51 61 71 42 52 62 72 43 53 63 73
d0 = _mm_unpacklo_epi32(
w4, w5); // 00 10 20 30 40 50 60 70 01 11 21 31 41 51 61 71
d2 = _mm_unpackhi_epi32(
w4, w5); // 02 12 22 32 42 52 62 72 03 13 23 33 43 53 63 73
w10 = _mm_unpacklo_epi8(
*x7, *x6); // q xx xx xx xx xx xx xx xx 00 10 01 11 02 12 03 13
w11 = _mm_unpacklo_epi8(
*x5, *x4); // q xx xx xx xx xx xx xx xx 20 30 21 31 22 32 23 33
w12 = _mm_unpacklo_epi8(
*x3, *x2); // q xx xx xx xx xx xx xx xx 40 50 41 51 42 52 43 53
w13 = _mm_unpacklo_epi8(
*x1, *x0); // q xx xx xx xx xx xx xx xx 60 70 61 71 62 72 63 73
w4 = _mm_unpackhi_epi16(
w10, w11); // 00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33
w5 = _mm_unpackhi_epi16(
w12, w13); // 40 50 60 70 41 51 61 71 42 52 62 72 43 53 63 73
d1 = _mm_unpacklo_epi32(
w4, w5); // 00 10 20 30 40 50 60 70 01 11 21 31 41 51 61 71
d3 = _mm_unpackhi_epi32(
w4, w5); // 02 12 22 32 42 52 62 72 03 13 23 33 43 53 63 73
*pq0 = _mm_unpacklo_epi64(d0, d1); // pq
*pq1 = _mm_unpackhi_epi64(d0, d1); // pq
*pq2 = _mm_unpacklo_epi64(d2, d3); // pq
*pq3 = _mm_unpackhi_epi64(d2, d3); // pq
}
static AOM_FORCE_INLINE void filter4_sse2(__m128i *p1p0, __m128i *q1q0,
__m128i *hev, __m128i *mask,
__m128i *qs1qs0, __m128i *ps1ps0) {
@ -616,10 +338,10 @@ void aom_lpf_horizontal_4_sse2(uint8_t *s, int p /* pitch */,
__m128i qs1qs0, ps1ps0;
__m128i p1, p0, q0, q1;
p1 = _mm_cvtsi32_si128(*(int *)(s - 2 * p));
p0 = _mm_cvtsi32_si128(*(int *)(s - 1 * p));
q0 = _mm_cvtsi32_si128(*(int *)(s + 0 * p));
q1 = _mm_cvtsi32_si128(*(int *)(s + 1 * p));
p1 = xx_loadl_32(s - 2 * p);
p0 = xx_loadl_32(s - 1 * p);
q0 = xx_loadl_32(s - 0 * p);
q1 = xx_loadl_32(s + 1 * p);
lpf_internal_4_sse2(&p1, &p0, &q0, &q1, &limit, &thresh, &qs1qs0, &ps1ps0);
@ -1241,23 +963,16 @@ void aom_lpf_horizontal_14_sse2(unsigned char *s, int p,
__m128i limit = _mm_load_si128((const __m128i *)_limit);
__m128i thresh = _mm_load_si128((const __m128i *)_thresh);
q4p4 = _mm_unpacklo_epi32(_mm_cvtsi32_si128(*(int *)(s - 5 * p)),
_mm_cvtsi32_si128(*(int *)(s + 4 * p)));
q3p3 = _mm_unpacklo_epi32(_mm_cvtsi32_si128(*(int *)(s - 4 * p)),
_mm_cvtsi32_si128(*(int *)(s + 3 * p)));
q2p2 = _mm_unpacklo_epi32(_mm_cvtsi32_si128(*(int *)(s - 3 * p)),
_mm_cvtsi32_si128(*(int *)(s + 2 * p)));
q1p1 = _mm_unpacklo_epi32(_mm_cvtsi32_si128(*(int *)(s - 2 * p)),
_mm_cvtsi32_si128(*(int *)(s + 1 * p)));
q4p4 = _mm_unpacklo_epi32(xx_loadl_32(s - 5 * p), xx_loadl_32(s + 4 * p));
q3p3 = _mm_unpacklo_epi32(xx_loadl_32(s - 4 * p), xx_loadl_32(s + 3 * p));
q2p2 = _mm_unpacklo_epi32(xx_loadl_32(s - 3 * p), xx_loadl_32(s + 2 * p));
q1p1 = _mm_unpacklo_epi32(xx_loadl_32(s - 2 * p), xx_loadl_32(s + 1 * p));
q0p0 = _mm_unpacklo_epi32(_mm_cvtsi32_si128(*(int *)(s - 1 * p)),
_mm_cvtsi32_si128(*(int *)(s - 0 * p)));
q0p0 = _mm_unpacklo_epi32(xx_loadl_32(s - 1 * p), xx_loadl_32(s - 0 * p));
q5p5 = _mm_unpacklo_epi32(_mm_cvtsi32_si128(*(int *)(s - 6 * p)),
_mm_cvtsi32_si128(*(int *)(s + 5 * p)));
q5p5 = _mm_unpacklo_epi32(xx_loadl_32(s - 6 * p), xx_loadl_32(s + 5 * p));
q6p6 = _mm_unpacklo_epi32(_mm_cvtsi32_si128(*(int *)(s - 7 * p)),
_mm_cvtsi32_si128(*(int *)(s + 6 * p)));
q6p6 = _mm_unpacklo_epi32(xx_loadl_32(s - 7 * p), xx_loadl_32(s + 6 * p));
lpf_internal_14_sse2(&q6p6, &q5p5, &q4p4, &q3p3, &q2p2, &q1p1, &q0p0, &blimit,
&limit, &thresh);
@ -1543,12 +1258,12 @@ void aom_lpf_horizontal_6_sse2(unsigned char *s, int p,
__m128i limit = _mm_load_si128((__m128i *)_limit);
__m128i thresh = _mm_load_si128((__m128i *)_thresh);
p2 = _mm_cvtsi32_si128(*(int *)(s - 3 * p));
p1 = _mm_cvtsi32_si128(*(int *)(s - 2 * p));
p0 = _mm_cvtsi32_si128(*(int *)(s - 1 * p));
q0 = _mm_cvtsi32_si128(*(int *)(s - 0 * p));
q1 = _mm_cvtsi32_si128(*(int *)(s + 1 * p));
q2 = _mm_cvtsi32_si128(*(int *)(s + 2 * p));
p2 = xx_loadl_32(s - 3 * p);
p1 = xx_loadl_32(s - 2 * p);
p0 = xx_loadl_32(s - 1 * p);
q0 = xx_loadl_32(s - 0 * p);
q1 = xx_loadl_32(s + 1 * p);
q2 = xx_loadl_32(s + 2 * p);
lpf_internal_6_sse2(&p2, &q2, &p1, &q1, &p0, &q0, &q1q0, &p1p0, &blimit,
&limit, &thresh);
@ -1895,20 +1610,20 @@ void aom_lpf_horizontal_8_sse2(unsigned char *s, int p,
const unsigned char *_blimit,
const unsigned char *_limit,
const unsigned char *_thresh) {
__m128i p2, p1, p0, q0, q1, q2, p3, q3;
__m128i p3, p2, p1, p0, q0, q1, q2, q3;
__m128i q1q0, p1p0;
__m128i blimit = _mm_load_si128((const __m128i *)_blimit);
__m128i limit = _mm_load_si128((const __m128i *)_limit);
__m128i thresh = _mm_load_si128((const __m128i *)_thresh);
p3 = _mm_cvtsi32_si128(*(int *)(s - 4 * p));
p2 = _mm_cvtsi32_si128(*(int *)(s - 3 * p));
p1 = _mm_cvtsi32_si128(*(int *)(s - 2 * p));
p0 = _mm_cvtsi32_si128(*(int *)(s - 1 * p));
q0 = _mm_cvtsi32_si128(*(int *)(s - 0 * p));
q1 = _mm_cvtsi32_si128(*(int *)(s + 1 * p));
q2 = _mm_cvtsi32_si128(*(int *)(s + 2 * p));
q3 = _mm_cvtsi32_si128(*(int *)(s + 3 * p));
p3 = xx_loadl_32(s - 4 * p);
p2 = xx_loadl_32(s - 3 * p);
p1 = xx_loadl_32(s - 2 * p);
p0 = xx_loadl_32(s - 1 * p);
q0 = xx_loadl_32(s - 0 * p);
q1 = xx_loadl_32(s + 1 * p);
q2 = xx_loadl_32(s + 2 * p);
q3 = xx_loadl_32(s + 3 * p);
lpf_internal_8_sse2(&p3, &q3, &p2, &q2, &p1, &q1, &p0, &q0, &q1q0, &p1p0,
&blimit, &limit, &thresh);

280
third_party/aom/aom_dsp/x86/lpf_common_sse2.h vendored
View file

@ -212,4 +212,284 @@ static INLINE void highbd_transpose8x16_sse2(
d4 + 1, d5 + 1, d6 + 1, d7 + 1);
}
// Low bit depth functions
static INLINE void transpose4x8_8x4_low_sse2(__m128i *x0, __m128i *x1,
__m128i *x2, __m128i *x3,
__m128i *d0, __m128i *d1,
__m128i *d2, __m128i *d3) {
// input
// x0 00 01 02 03 04 05 06 07 xx xx xx xx xx xx xx xx
// x1 10 11 12 13 14 15 16 17 xx xx xx xx xx xx xx xx
// x2 20 21 22 23 24 25 26 27 xx xx xx xx xx xx xx xx
// x3 30 31 32 33 34 35 36 37 xx xx xx xx xx xx xx xx
// output
// 00 10 20 30 xx xx xx xx xx xx xx xx xx xx xx xx
// 01 11 21 31 xx xx xx xx xx xx xx xx xx xx xx xx
// 02 12 22 32 xx xx xx xx xx xx xx xx xx xx xx xx
// 03 13 23 33 xx xx xx xx xx xx xx xx xx xx xx xx
__m128i w0, w1;
w0 = _mm_unpacklo_epi8(
*x0, *x1); // 00 10 01 11 02 12 03 13 04 14 05 15 06 16 07 17
w1 = _mm_unpacklo_epi8(
*x2, *x3); // 20 30 21 31 22 32 23 33 24 34 25 35 26 36 27 37
*d0 = _mm_unpacklo_epi16(
w0, w1); // 00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33
*d1 = _mm_srli_si128(*d0,
4); // 01 11 21 31 xx xx xx xx xx xx xx xx xx xx xx xx
*d2 = _mm_srli_si128(*d0,
8); // 02 12 22 32 xx xx xx xx xx xx xx xx xx xx xx xx
*d3 = _mm_srli_si128(*d0,
12); // 03 13 23 33 xx xx xx xx xx xx xx xx xx xx xx xx
}
static INLINE void transpose4x8_8x4_sse2(__m128i *x0, __m128i *x1, __m128i *x2,
__m128i *x3, __m128i *d0, __m128i *d1,
__m128i *d2, __m128i *d3, __m128i *d4,
__m128i *d5, __m128i *d6,
__m128i *d7) {
// input
// x0 00 01 02 03 04 05 06 07 xx xx xx xx xx xx xx xx
// x1 10 11 12 13 14 15 16 17 xx xx xx xx xx xx xx xx
// x2 20 21 22 23 24 25 26 27 xx xx xx xx xx xx xx xx
// x3 30 31 32 33 34 35 36 37 xx xx xx xx xx xx xx xx
// output
// 00 10 20 30 xx xx xx xx xx xx xx xx xx xx xx xx
// 01 11 21 31 xx xx xx xx xx xx xx xx xx xx xx xx
// 02 12 22 32 xx xx xx xx xx xx xx xx xx xx xx xx
// 03 13 23 33 xx xx xx xx xx xx xx xx xx xx xx xx
// 04 14 24 34 xx xx xx xx xx xx xx xx xx xx xx xx
// 05 15 25 35 xx xx xx xx xx xx xx xx xx xx xx xx
// 06 16 26 36 xx xx xx xx xx xx xx xx xx xx xx xx
// 07 17 27 37 xx xx xx xx xx xx xx xx xx xx xx xx
__m128i w0, w1, ww0, ww1;
w0 = _mm_unpacklo_epi8(
*x0, *x1); // 00 10 01 11 02 12 03 13 04 14 05 15 06 16 07 17
w1 = _mm_unpacklo_epi8(
*x2, *x3); // 20 30 21 31 22 32 23 33 24 34 25 35 26 36 27 37
ww0 = _mm_unpacklo_epi16(
w0, w1); // 00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33
ww1 = _mm_unpackhi_epi16(
w0, w1); // 04 14 24 34 05 15 25 35 06 16 26 36 07 17 27 37
*d0 = ww0; // 00 10 20 30 xx xx xx xx xx xx xx xx xx xx xx xx
*d1 = _mm_srli_si128(ww0,
4); // 01 11 21 31 xx xx xx xx xx xx xx xx xx xx xx xx
*d2 = _mm_srli_si128(ww0,
8); // 02 12 22 32 xx xx xx xx xx xx xx xx xx xx xx xx
*d3 = _mm_srli_si128(ww0,
12); // 03 13 23 33 xx xx xx xx xx xx xx xx xx xx xx xx
*d4 = ww1; // 04 14 24 34 xx xx xx xx xx xx xx xx xx xx xx xx
*d5 = _mm_srli_si128(ww1,
4); // 05 15 25 35 xx xx xx xx xx xx xx xx xx xx xx xx
*d6 = _mm_srli_si128(ww1,
8); // 06 16 26 36 xx xx xx xx xx xx xx xx xx xx xx xx
*d7 = _mm_srli_si128(ww1,
12); // 07 17 27 37 xx xx xx xx xx xx xx xx xx xx xx xx
}
static INLINE void transpose8x8_low_sse2(__m128i *x0, __m128i *x1, __m128i *x2,
__m128i *x3, __m128i *x4, __m128i *x5,
__m128i *x6, __m128i *x7, __m128i *d0,
__m128i *d1, __m128i *d2,
__m128i *d3) {
// input
// x0 00 01 02 03 04 05 06 07
// x1 10 11 12 13 14 15 16 17
// x2 20 21 22 23 24 25 26 27
// x3 30 31 32 33 34 35 36 37
// x4 40 41 42 43 44 45 46 47
// x5 50 51 52 53 54 55 56 57
// x6 60 61 62 63 64 65 66 67
// x7 70 71 72 73 74 75 76 77
// output
// d0 00 10 20 30 40 50 60 70 xx xx xx xx xx xx xx
// d1 01 11 21 31 41 51 61 71 xx xx xx xx xx xx xx xx
// d2 02 12 22 32 42 52 62 72 xx xx xx xx xx xx xx xx
// d3 03 13 23 33 43 53 63 73 xx xx xx xx xx xx xx xx
__m128i w0, w1, w2, w3, w4, w5;
w0 = _mm_unpacklo_epi8(
*x0, *x1); // 00 10 01 11 02 12 03 13 04 14 05 15 06 16 07 17
w1 = _mm_unpacklo_epi8(
*x2, *x3); // 20 30 21 31 22 32 23 33 24 34 25 35 26 36 27 37
w2 = _mm_unpacklo_epi8(
*x4, *x5); // 40 50 41 51 42 52 43 53 44 54 45 55 46 56 47 57
w3 = _mm_unpacklo_epi8(
*x6, *x7); // 60 70 61 71 62 72 63 73 64 74 65 75 66 76 67 77
w4 = _mm_unpacklo_epi16(
w0, w1); // 00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33
w5 = _mm_unpacklo_epi16(
w2, w3); // 40 50 60 70 41 51 61 71 42 52 62 72 43 53 63 73
*d0 = _mm_unpacklo_epi32(
w4, w5); // 00 10 20 30 40 50 60 70 01 11 21 31 41 51 61 71
*d1 = _mm_srli_si128(*d0, 8);
*d2 = _mm_unpackhi_epi32(
w4, w5); // 02 12 22 32 42 52 62 72 03 13 23 33 43 53 63 73
*d3 = _mm_srli_si128(*d2, 8);
}
static INLINE void transpose8x8_sse2(__m128i *x0, __m128i *x1, __m128i *x2,
__m128i *x3, __m128i *x4, __m128i *x5,
__m128i *x6, __m128i *x7, __m128i *d0d1,
__m128i *d2d3, __m128i *d4d5,
__m128i *d6d7) {
__m128i w0, w1, w2, w3, w4, w5, w6, w7;
// x0 00 01 02 03 04 05 06 07
// x1 10 11 12 13 14 15 16 17
w0 = _mm_unpacklo_epi8(
*x0, *x1); // 00 10 01 11 02 12 03 13 04 14 05 15 06 16 07 17
// x2 20 21 22 23 24 25 26 27
// x3 30 31 32 33 34 35 36 37
w1 = _mm_unpacklo_epi8(
*x2, *x3); // 20 30 21 31 22 32 23 33 24 34 25 35 26 36 27 37
// x4 40 41 42 43 44 45 46 47
// x5 50 51 52 53 54 55 56 57
w2 = _mm_unpacklo_epi8(
*x4, *x5); // 40 50 41 51 42 52 43 53 44 54 45 55 46 56 47 57
// x6 60 61 62 63 64 65 66 67
// x7 70 71 72 73 74 75 76 77
w3 = _mm_unpacklo_epi8(
*x6, *x7); // 60 70 61 71 62 72 63 73 64 74 65 75 66 76 67 77
w4 = _mm_unpacklo_epi16(
w0, w1); // 00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33
w5 = _mm_unpacklo_epi16(
w2, w3); // 40 50 60 70 41 51 61 71 42 52 62 72 43 53 63 73
*d0d1 = _mm_unpacklo_epi32(
w4, w5); // 00 10 20 30 40 50 60 70 01 11 21 31 41 51 61 71
*d2d3 = _mm_unpackhi_epi32(
w4, w5); // 02 12 22 32 42 52 62 72 03 13 23 33 43 53 63 73
w6 = _mm_unpackhi_epi16(
w0, w1); // 04 14 24 34 05 15 25 35 06 16 26 36 07 17 27 37
w7 = _mm_unpackhi_epi16(
w2, w3); // 44 54 64 74 45 55 65 75 46 56 66 76 47 57 67 77
*d4d5 = _mm_unpacklo_epi32(
w6, w7); // 04 14 24 34 44 54 64 74 05 15 25 35 45 55 65 75
*d6d7 = _mm_unpackhi_epi32(
w6, w7); // 06 16 26 36 46 56 66 76 07 17 27 37 47 57 67 77
}
static INLINE void transpose16x8_8x16_sse2(
__m128i *x0, __m128i *x1, __m128i *x2, __m128i *x3, __m128i *x4,
__m128i *x5, __m128i *x6, __m128i *x7, __m128i *x8, __m128i *x9,
__m128i *x10, __m128i *x11, __m128i *x12, __m128i *x13, __m128i *x14,
__m128i *x15, __m128i *d0, __m128i *d1, __m128i *d2, __m128i *d3,
__m128i *d4, __m128i *d5, __m128i *d6, __m128i *d7) {
__m128i w0, w1, w2, w3, w4, w5, w6, w7, w8, w9;
__m128i w10, w11, w12, w13, w14, w15;
w0 = _mm_unpacklo_epi8(*x0, *x1);
w1 = _mm_unpacklo_epi8(*x2, *x3);
w2 = _mm_unpacklo_epi8(*x4, *x5);
w3 = _mm_unpacklo_epi8(*x6, *x7);
w8 = _mm_unpacklo_epi8(*x8, *x9);
w9 = _mm_unpacklo_epi8(*x10, *x11);
w10 = _mm_unpacklo_epi8(*x12, *x13);
w11 = _mm_unpacklo_epi8(*x14, *x15);
w4 = _mm_unpacklo_epi16(w0, w1);
w5 = _mm_unpacklo_epi16(w2, w3);
w12 = _mm_unpacklo_epi16(w8, w9);
w13 = _mm_unpacklo_epi16(w10, w11);
w6 = _mm_unpacklo_epi32(w4, w5);
w7 = _mm_unpackhi_epi32(w4, w5);
w14 = _mm_unpacklo_epi32(w12, w13);
w15 = _mm_unpackhi_epi32(w12, w13);
// Store first 4-line result
*d0 = _mm_unpacklo_epi64(w6, w14);
*d1 = _mm_unpackhi_epi64(w6, w14);
*d2 = _mm_unpacklo_epi64(w7, w15);
*d3 = _mm_unpackhi_epi64(w7, w15);
w4 = _mm_unpackhi_epi16(w0, w1);
w5 = _mm_unpackhi_epi16(w2, w3);
w12 = _mm_unpackhi_epi16(w8, w9);
w13 = _mm_unpackhi_epi16(w10, w11);
w6 = _mm_unpacklo_epi32(w4, w5);
w7 = _mm_unpackhi_epi32(w4, w5);
w14 = _mm_unpacklo_epi32(w12, w13);
w15 = _mm_unpackhi_epi32(w12, w13);
// Store second 4-line result
*d4 = _mm_unpacklo_epi64(w6, w14);
*d5 = _mm_unpackhi_epi64(w6, w14);
*d6 = _mm_unpacklo_epi64(w7, w15);
*d7 = _mm_unpackhi_epi64(w7, w15);
}
static INLINE void transpose8x16_16x8_sse2(
__m128i *x0, __m128i *x1, __m128i *x2, __m128i *x3, __m128i *x4,
__m128i *x5, __m128i *x6, __m128i *x7, __m128i *d0d1, __m128i *d2d3,
__m128i *d4d5, __m128i *d6d7, __m128i *d8d9, __m128i *d10d11,
__m128i *d12d13, __m128i *d14d15) {
__m128i w0, w1, w2, w3, w4, w5, w6, w7, w8, w9;
__m128i w10, w11, w12, w13, w14, w15;
w0 = _mm_unpacklo_epi8(*x0, *x1);
w1 = _mm_unpacklo_epi8(*x2, *x3);
w2 = _mm_unpacklo_epi8(*x4, *x5);
w3 = _mm_unpacklo_epi8(*x6, *x7);
w8 = _mm_unpackhi_epi8(*x0, *x1);
w9 = _mm_unpackhi_epi8(*x2, *x3);
w10 = _mm_unpackhi_epi8(*x4, *x5);
w11 = _mm_unpackhi_epi8(*x6, *x7);
w4 = _mm_unpacklo_epi16(w0, w1);
w5 = _mm_unpacklo_epi16(w2, w3);
w12 = _mm_unpacklo_epi16(w8, w9);
w13 = _mm_unpacklo_epi16(w10, w11);
w6 = _mm_unpacklo_epi32(w4, w5);
w7 = _mm_unpackhi_epi32(w4, w5);
w14 = _mm_unpacklo_epi32(w12, w13);
w15 = _mm_unpackhi_epi32(w12, w13);
// Store first 4-line result
*d0d1 = _mm_unpacklo_epi64(w6, w14);
*d2d3 = _mm_unpackhi_epi64(w6, w14);
*d4d5 = _mm_unpacklo_epi64(w7, w15);
*d6d7 = _mm_unpackhi_epi64(w7, w15);
w4 = _mm_unpackhi_epi16(w0, w1);
w5 = _mm_unpackhi_epi16(w2, w3);
w12 = _mm_unpackhi_epi16(w8, w9);
w13 = _mm_unpackhi_epi16(w10, w11);
w6 = _mm_unpacklo_epi32(w4, w5);
w7 = _mm_unpackhi_epi32(w4, w5);
w14 = _mm_unpacklo_epi32(w12, w13);
w15 = _mm_unpackhi_epi32(w12, w13);
// Store second 4-line result
*d8d9 = _mm_unpacklo_epi64(w6, w14);
*d10d11 = _mm_unpackhi_epi64(w6, w14);
*d12d13 = _mm_unpacklo_epi64(w7, w15);
*d14d15 = _mm_unpackhi_epi64(w7, w15);
}
#endif // AOM_AOM_DSP_X86_LPF_COMMON_SSE2_H_

69
third_party/aom/aom_dsp/x86/masked_variance_intrin_ssse3.c vendored
View file

@ -218,15 +218,15 @@ static void bilinear_filter(const uint8_t *src, int src_stride, int xoffset,
}
}
static INLINE __m128i filter_block_2rows(const __m128i a0, const __m128i b0,
const __m128i a1, const __m128i b1,
const __m128i filter) {
__m128i v0 = _mm_unpacklo_epi8(a0, b0);
v0 = _mm_maddubs_epi16(v0, filter);
static INLINE __m128i filter_block_2rows(const __m128i *a0, const __m128i *b0,
const __m128i *a1, const __m128i *b1,
const __m128i *filter) {
__m128i v0 = _mm_unpacklo_epi8(*a0, *b0);
v0 = _mm_maddubs_epi16(v0, *filter);
v0 = xx_roundn_epu16(v0, FILTER_BITS);
__m128i v1 = _mm_unpacklo_epi8(a1, b1);
v1 = _mm_maddubs_epi16(v1, filter);
__m128i v1 = _mm_unpacklo_epi8(*a1, *b1);
v1 = _mm_maddubs_epi16(v1, *filter);
v1 = xx_roundn_epu16(v1, FILTER_BITS);
return _mm_packus_epi16(v0, v1);
@ -262,7 +262,7 @@ static void bilinear_filter8xh(const uint8_t *src, int src_stride, int xoffset,
const __m128i z0 = _mm_srli_si128(x0, 1);
const __m128i x1 = _mm_loadu_si128((__m128i *)&src[src_stride]);
const __m128i z1 = _mm_srli_si128(x1, 1);
const __m128i res = filter_block_2rows(x0, z0, x1, z1, hfilter_vec);
const __m128i res = filter_block_2rows(&x0, &z0, &x1, &z1, &hfilter_vec);
_mm_storeu_si128((__m128i *)b, res);
src += src_stride * 2;
@ -296,7 +296,7 @@ static void bilinear_filter8xh(const uint8_t *src, int src_stride, int xoffset,
const __m128i x = _mm_loadl_epi64((__m128i *)dst);
const __m128i y = _mm_loadl_epi64((__m128i *)&dst[8]);
const __m128i z = _mm_loadl_epi64((__m128i *)&dst[16]);
const __m128i res = filter_block_2rows(x, y, y, z, vfilter_vec);
const __m128i res = filter_block_2rows(&x, &y, &y, &z, &vfilter_vec);
_mm_storeu_si128((__m128i *)dst, res);
dst += 16;
@ -343,7 +343,7 @@ static void bilinear_filter4xh(const uint8_t *src, int src_stride, int xoffset,
const __m128i b0 = _mm_unpacklo_epi32(z0, z1);
const __m128i a1 = _mm_unpacklo_epi32(x2, x3);
const __m128i b1 = _mm_unpacklo_epi32(z2, z3);
const __m128i res = filter_block_2rows(a0, b0, a1, b1, hfilter_vec);
const __m128i res = filter_block_2rows(&a0, &b0, &a1, &b1, &hfilter_vec);
_mm_storeu_si128((__m128i *)b, res);
src += src_stride * 4;
@ -384,7 +384,7 @@ static void bilinear_filter4xh(const uint8_t *src, int src_stride, int xoffset,
const __m128i b0 = _mm_unpacklo_epi32(b, c);
const __m128i a1 = _mm_unpacklo_epi32(c, d);
const __m128i b1 = _mm_unpacklo_epi32(d, e);
const __m128i res = filter_block_2rows(a0, b0, a1, b1, vfilter_vec);
const __m128i res = filter_block_2rows(&a0, &b0, &a1, &b1, &vfilter_vec);
_mm_storeu_si128((__m128i *)dst, res);
dst += 16;
@ -392,29 +392,29 @@ static void bilinear_filter4xh(const uint8_t *src, int src_stride, int xoffset,
}
}
static INLINE void accumulate_block(const __m128i src, const __m128i a,
const __m128i b, const __m128i m,
static INLINE void accumulate_block(const __m128i *src, const __m128i *a,
const __m128i *b, const __m128i *m,
__m128i *sum, __m128i *sum_sq) {
const __m128i zero = _mm_setzero_si128();
const __m128i one = _mm_set1_epi16(1);
const __m128i mask_max = _mm_set1_epi8((1 << AOM_BLEND_A64_ROUND_BITS));
const __m128i m_inv = _mm_sub_epi8(mask_max, m);
const __m128i m_inv = _mm_sub_epi8(mask_max, *m);
// Calculate 16 predicted pixels.
// Note that the maximum value of any entry of 'pred_l' or 'pred_r'
// is 64 * 255, so we have plenty of space to add rounding constants.
const __m128i data_l = _mm_unpacklo_epi8(a, b);
const __m128i mask_l = _mm_unpacklo_epi8(m, m_inv);
const __m128i data_l = _mm_unpacklo_epi8(*a, *b);
const __m128i mask_l = _mm_unpacklo_epi8(*m, m_inv);
__m128i pred_l = _mm_maddubs_epi16(data_l, mask_l);
pred_l = xx_roundn_epu16(pred_l, AOM_BLEND_A64_ROUND_BITS);
const __m128i data_r = _mm_unpackhi_epi8(a, b);
const __m128i mask_r = _mm_unpackhi_epi8(m, m_inv);
const __m128i data_r = _mm_unpackhi_epi8(*a, *b);
const __m128i mask_r = _mm_unpackhi_epi8(*m, m_inv);
__m128i pred_r = _mm_maddubs_epi16(data_r, mask_r);
pred_r = xx_roundn_epu16(pred_r, AOM_BLEND_A64_ROUND_BITS);
const __m128i src_l = _mm_unpacklo_epi8(src, zero);
const __m128i src_r = _mm_unpackhi_epi8(src, zero);
const __m128i src_l = _mm_unpacklo_epi8(*src, zero);
const __m128i src_r = _mm_unpackhi_epi8(*src, zero);
const __m128i diff_l = _mm_sub_epi16(pred_l, src_l);
const __m128i diff_r = _mm_sub_epi16(pred_r, src_r);
@ -440,7 +440,7 @@ static void masked_variance(const uint8_t *src_ptr, int src_stride,
const __m128i a = _mm_loadu_si128((const __m128i *)&a_ptr[x]);
const __m128i b = _mm_loadu_si128((const __m128i *)&b_ptr[x]);
const __m128i m = _mm_loadu_si128((const __m128i *)&m_ptr[x]);
accumulate_block(src, a, b, m, &sum, &sum_sq);
accumulate_block(&src, &a, &b, &m, &sum, &sum_sq);
}
src_ptr += src_stride;
@ -471,7 +471,7 @@ static void masked_variance8xh(const uint8_t *src_ptr, int src_stride,
const __m128i m =
_mm_unpacklo_epi64(_mm_loadl_epi64((const __m128i *)m_ptr),
_mm_loadl_epi64((const __m128i *)&m_ptr[m_stride]));
accumulate_block(src, a, b, m, &sum, &sum_sq);
accumulate_block(&src, &a, &b, &m, &sum, &sum_sq);
src_ptr += src_stride * 2;
a_ptr += 16;
@ -503,7 +503,7 @@ static void masked_variance4xh(const uint8_t *src_ptr, int src_stride,
const __m128i m = _mm_setr_epi32(
*(uint32_t *)m_ptr, *(uint32_t *)&m_ptr[m_stride],
*(uint32_t *)&m_ptr[m_stride * 2], *(uint32_t *)&m_ptr[m_stride * 3]);
accumulate_block(src, a, b, m, &sum, &sum_sq);
accumulate_block(&src, &a, &b, &m, &sum, &sum_sq);
src_ptr += src_stride * 4;
a_ptr += 16;
@ -797,17 +797,17 @@ static void highbd_bilinear_filter(const uint16_t *src, int src_stride,
}
}
static INLINE __m128i highbd_filter_block_2rows(const __m128i a0,
const __m128i b0,
const __m128i a1,
const __m128i b1,
const __m128i filter) {
__m128i v0 = _mm_unpacklo_epi16(a0, b0);
v0 = _mm_madd_epi16(v0, filter);
static INLINE __m128i highbd_filter_block_2rows(const __m128i *a0,
const __m128i *b0,
const __m128i *a1,
const __m128i *b1,
const __m128i *filter) {
__m128i v0 = _mm_unpacklo_epi16(*a0, *b0);
v0 = _mm_madd_epi16(v0, *filter);
v0 = xx_roundn_epu32(v0, FILTER_BITS);
__m128i v1 = _mm_unpacklo_epi16(a1, b1);
v1 = _mm_madd_epi16(v1, filter);
__m128i v1 = _mm_unpacklo_epi16(*a1, *b1);
v1 = _mm_madd_epi16(v1, *filter);
v1 = xx_roundn_epu32(v1, FILTER_BITS);
return _mm_packs_epi32(v0, v1);
@ -845,7 +845,7 @@ static void highbd_bilinear_filter4xh(const uint16_t *src, int src_stride,
const __m128i x1 = _mm_loadu_si128((__m128i *)&src[src_stride]);
const __m128i z1 = _mm_srli_si128(x1, 2);
const __m128i res =
highbd_filter_block_2rows(x0, z0, x1, z1, hfilter_vec);
highbd_filter_block_2rows(&x0, &z0, &x1, &z1, &hfilter_vec);
_mm_storeu_si128((__m128i *)b, res);
src += src_stride * 2;
@ -879,7 +879,8 @@ static void highbd_bilinear_filter4xh(const uint16_t *src, int src_stride,
const __m128i x = _mm_loadl_epi64((__m128i *)dst);
const __m128i y = _mm_loadl_epi64((__m128i *)&dst[4]);
const __m128i z = _mm_loadl_epi64((__m128i *)&dst[8]);
const __m128i res = highbd_filter_block_2rows(x, y, y, z, vfilter_vec);
const __m128i res =
highbd_filter_block_2rows(&x, &y, &y, &z, &vfilter_vec);
_mm_storeu_si128((__m128i *)dst, res);
dst += 8;

43
third_party/aom/aom_dsp/x86/quantize_avx_x86_64.asm vendored
View file

@ -126,7 +126,7 @@ cglobal quantize_%1, 0, %2, 15, coeff, ncoeff, zbin, round, quant, \
punpckhqdq m3, m3
pmullw m13, m3 ; dqc[i] = qc[i] * q
; Store 16bit numbers as 32bit numbers in array pointed to by qcoeff
; Store 16bit numbers as 32bit numbers in array pointed to by dqcoeff
pcmpgtw m6, m5, m8
punpckhwd m6, m8, m6
pmovsxwd m11, m8
@ -198,10 +198,7 @@ DEFINE_ARGS coeff, ncoeff, zbin, round, quant, shift, \
mova m4, [r2] ; m4 = shift
mov r4, dqcoeffmp
mov r5, iscanmp
%ifidn %1, b_32x32
psllw m4, 1
%endif
pxor m5, m5 ; m5 = dedicated zero
pxor m5, m5 ; m5 = dedicated zero
DEFINE_ARGS coeff, ncoeff, d1, qcoeff, dqcoeff, iscan, d2, d3, d4, eob
@ -255,9 +252,26 @@ DEFINE_ARGS coeff, ncoeff, zbin, round, quant, shift, \
pmulhw m13, m11, m2 ; m13 = m11*q>>16
paddw m8, m6 ; m8 += m6
paddw m13, m11 ; m13 += m11
%ifidn %1, b_32x32
pmullw m5, m8, m4 ; store the lower 16 bits of m8*qsh
%endif
pmulhw m8, m4 ; m8 = m8*qsh>>16
%ifidn %1, b_32x32
psllw m8, 1
psrlw m5, 15
por m8, m5
%endif
punpckhqdq m4, m4
%ifidn %1, b_32x32
pmullw m5, m13, m4 ; store the lower 16 bits of m13*qsh
%endif
pmulhw m13, m4 ; m13 = m13*qsh>>16
%ifidn %1, b_32x32
psllw m13, 1
psrlw m5, 15
por m13, m5
pxor m5, m5 ; reset m5 to zero register
%endif
psignw m8, m9 ; m8 = reinsert sign
psignw m13, m10 ; m13 = reinsert sign
pand m8, m7
@ -289,7 +303,7 @@ DEFINE_ARGS coeff, ncoeff, zbin, round, quant, shift, \
psignw m13, m10
%endif
; store 16bit numbers as 32bit numbers in array pointed to by qcoeff
; store 16bit numbers as 32bit numbers in array pointed to by dqcoeff
pcmpgtw m6, m5, m8
punpckhwd m6, m8, m6
pmovsxwd m11, m8
@ -359,8 +373,23 @@ DEFINE_ARGS coeff, ncoeff, zbin, round, quant, shift, \
pmulhw m13, m11, m2 ; m13 = m11*q>>16
paddw m14, m6 ; m14 += m6
paddw m13, m11 ; m13 += m11
%ifidn %1, b_32x32
pmullw m5, m14, m4 ; store the lower 16 bits of m14*qsh
%endif
pmulhw m14, m4 ; m14 = m14*qsh>>16
%ifidn %1, b_32x32
psllw m14, 1
psrlw m5, 15
por m14, m5
pmullw m5, m13, m4 ; store the lower 16 bits of m13*qsh
%endif
pmulhw m13, m4 ; m13 = m13*qsh>>16
%ifidn %1, b_32x32
psllw m13, 1
psrlw m5, 15
por m13, m5
pxor m5, m5 ; reset m5 to zero register
%endif
psignw m14, m9 ; m14 = reinsert sign
psignw m13, m10 ; m13 = reinsert sign
pand m14, m7
@ -391,7 +420,7 @@ DEFINE_ARGS coeff, ncoeff, zbin, round, quant, shift, \
psignw m13, m10
%endif
; store 16bit numbers as 32bit numbers in array pointed to by qcoeff
; store 16bit numbers as 32bit numbers in array pointed to by dqcoeff
pcmpgtw m6, m5, m14
punpckhwd m6, m14, m6
pmovsxwd m11, m14

22
third_party/aom/aom_dsp/x86/quantize_sse2.c vendored
View file

@ -18,28 +18,6 @@
#include "aom/aom_integer.h"
#include "aom_dsp/x86/quantize_x86.h"
static INLINE __m128i load_coefficients(const tran_low_t *coeff_ptr) {
assert(sizeof(tran_low_t) == 4);
return _mm_setr_epi16((int16_t)coeff_ptr[0], (int16_t)coeff_ptr[1],
(int16_t)coeff_ptr[2], (int16_t)coeff_ptr[3],
(int16_t)coeff_ptr[4], (int16_t)coeff_ptr[5],
(int16_t)coeff_ptr[6], (int16_t)coeff_ptr[7]);
}
static INLINE void store_coefficients(__m128i coeff_vals,
tran_low_t *coeff_ptr) {
assert(sizeof(tran_low_t) == 4);
__m128i one = _mm_set1_epi16(1);
__m128i coeff_vals_hi = _mm_mulhi_epi16(coeff_vals, one);
__m128i coeff_vals_lo = _mm_mullo_epi16(coeff_vals, one);
__m128i coeff_vals_1 = _mm_unpacklo_epi16(coeff_vals_lo, coeff_vals_hi);
__m128i coeff_vals_2 = _mm_unpackhi_epi16(coeff_vals_lo, coeff_vals_hi);
_mm_store_si128((__m128i *)(coeff_ptr), coeff_vals_1);
_mm_store_si128((__m128i *)(coeff_ptr + 4), coeff_vals_2);
}
void aom_quantize_b_sse2(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
const int16_t *zbin_ptr, const int16_t *round_ptr,
const int16_t *quant_ptr,

192
third_party/aom/aom_dsp/x86/quantize_ssse3.c vendored Normal file
View file

@ -0,0 +1,192 @@
/*
* Copyright (c) 2019, Alliance for Open Media. All rights reserved
*
* This source code is subject to the terms of the BSD 2 Clause License and
* the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
* was not distributed with this source code in the LICENSE file, you can
* obtain it at www.aomedia.org/license/software. If the Alliance for Open
* Media Patent License 1.0 was not distributed with this source code in the
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
*/
#include <assert.h>
#include <tmmintrin.h>
#include <emmintrin.h>
#include <xmmintrin.h>
#include "config/aom_dsp_rtcd.h"
#include "aom/aom_integer.h"
#include "aom_dsp/x86/quantize_x86.h"
static INLINE void calculate_qcoeff_64x64(__m128i *coeff, const __m128i round,
const __m128i quant,
const __m128i *shift) {
__m128i tmp, qcoeff, tmp1;
qcoeff = _mm_adds_epi16(*coeff, round);
tmp = _mm_mulhi_epi16(qcoeff, quant);
qcoeff = _mm_add_epi16(tmp, qcoeff);
tmp = _mm_mullo_epi16(qcoeff, *shift);
tmp = _mm_srli_epi16(tmp, 14);
tmp1 = _mm_mulhi_epi16(qcoeff, *shift);
tmp1 = _mm_slli_epi16(tmp1, 2);
*coeff = _mm_or_si128(tmp, tmp1);
}
static INLINE void calculate_dqcoeff_and_store_64x64(const __m128i qcoeff,
const __m128i dequant,
const __m128i zero,
tran_low_t *dqcoeff) {
// Un-sign to bias rounding like C.
const __m128i coeff = _mm_abs_epi16(qcoeff);
const __m128i sign_0 = _mm_unpacklo_epi16(zero, qcoeff);
const __m128i sign_1 = _mm_unpackhi_epi16(zero, qcoeff);
const __m128i low = _mm_mullo_epi16(coeff, dequant);
const __m128i high = _mm_mulhi_epi16(coeff, dequant);
__m128i dqcoeff32_0 = _mm_unpacklo_epi16(low, high);
__m128i dqcoeff32_1 = _mm_unpackhi_epi16(low, high);
// "Divide" by 4.
dqcoeff32_0 = _mm_srli_epi32(dqcoeff32_0, 2);
dqcoeff32_1 = _mm_srli_epi32(dqcoeff32_1, 2);
dqcoeff32_0 = _mm_sign_epi32(dqcoeff32_0, sign_0);
dqcoeff32_1 = _mm_sign_epi32(dqcoeff32_1, sign_1);
_mm_store_si128((__m128i *)(dqcoeff), dqcoeff32_0);
_mm_store_si128((__m128i *)(dqcoeff + 4), dqcoeff32_1);
}
void aom_quantize_b_64x64_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
const int16_t *zbin_ptr,
const int16_t *round_ptr,
const int16_t *quant_ptr,
const int16_t *quant_shift_ptr,
tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr,
const int16_t *dequant_ptr, uint16_t *eob_ptr,
const int16_t *scan, const int16_t *iscan) {
const __m128i zero = _mm_setzero_si128();
const __m128i one = _mm_set1_epi16(1);
const __m128i two = _mm_set1_epi16(2);
int index;
__m128i zbin, round, quant, dequant, shift;
__m128i coeff0, coeff1, qcoeff0, qcoeff1;
__m128i cmp_mask0, cmp_mask1, all_zero;
__m128i eob = zero, eob0;
(void)scan;
(void)n_coeffs;
// Setup global values.
zbin = _mm_load_si128((const __m128i *)zbin_ptr);
round = _mm_load_si128((const __m128i *)round_ptr);
quant = _mm_load_si128((const __m128i *)quant_ptr);
dequant = _mm_load_si128((const __m128i *)dequant_ptr);
shift = _mm_load_si128((const __m128i *)quant_shift_ptr);
// Shift with rounding.
zbin = _mm_add_epi16(zbin, two);
round = _mm_add_epi16(round, two);
zbin = _mm_srli_epi16(zbin, 2);
round = _mm_srli_epi16(round, 2);
zbin = _mm_sub_epi16(zbin, one);
// Do DC and first 15 AC.
coeff0 = load_coefficients(coeff_ptr);
coeff1 = load_coefficients(coeff_ptr + 8);
qcoeff0 = _mm_abs_epi16(coeff0);
qcoeff1 = _mm_abs_epi16(coeff1);
cmp_mask0 = _mm_cmpgt_epi16(qcoeff0, zbin);
zbin = _mm_unpackhi_epi64(zbin, zbin);
cmp_mask1 = _mm_cmpgt_epi16(qcoeff1, zbin);
all_zero = _mm_or_si128(cmp_mask0, cmp_mask1);
if (_mm_movemask_epi8(all_zero) == 0) {
_mm_store_si128((__m128i *)(qcoeff_ptr), zero);
_mm_store_si128((__m128i *)(qcoeff_ptr + 4), zero);
_mm_store_si128((__m128i *)(qcoeff_ptr + 8), zero);
_mm_store_si128((__m128i *)(qcoeff_ptr + 12), zero);
_mm_store_si128((__m128i *)(dqcoeff_ptr), zero);
_mm_store_si128((__m128i *)(dqcoeff_ptr + 4), zero);
_mm_store_si128((__m128i *)(dqcoeff_ptr + 8), zero);
_mm_store_si128((__m128i *)(dqcoeff_ptr + 12), zero);
round = _mm_unpackhi_epi64(round, round);
quant = _mm_unpackhi_epi64(quant, quant);
shift = _mm_unpackhi_epi64(shift, shift);
dequant = _mm_unpackhi_epi64(dequant, dequant);
} else {
calculate_qcoeff_64x64(&qcoeff0, round, quant, &shift);
round = _mm_unpackhi_epi64(round, round);
quant = _mm_unpackhi_epi64(quant, quant);
shift = _mm_unpackhi_epi64(shift, shift);
calculate_qcoeff_64x64(&qcoeff1, round, quant, &shift);
// Reinsert signs.
qcoeff0 = _mm_sign_epi16(qcoeff0, coeff0);
qcoeff1 = _mm_sign_epi16(qcoeff1, coeff1);
// Mask out zbin threshold coeffs.
qcoeff0 = _mm_and_si128(qcoeff0, cmp_mask0);
qcoeff1 = _mm_and_si128(qcoeff1, cmp_mask1);
store_coefficients(qcoeff0, qcoeff_ptr);
store_coefficients(qcoeff1, qcoeff_ptr + 8);
calculate_dqcoeff_and_store_64x64(qcoeff0, dequant, zero, dqcoeff_ptr);
dequant = _mm_unpackhi_epi64(dequant, dequant);
calculate_dqcoeff_and_store_64x64(qcoeff1, dequant, zero, dqcoeff_ptr + 8);
eob =
scan_for_eob(&qcoeff0, &qcoeff1, cmp_mask0, cmp_mask1, iscan, 0, zero);
}
// AC only loop.
for (index = 16; index < 1024; index += 16) {
coeff0 = load_coefficients(coeff_ptr + index);
coeff1 = load_coefficients(coeff_ptr + index + 8);
qcoeff0 = _mm_abs_epi16(coeff0);
qcoeff1 = _mm_abs_epi16(coeff1);
cmp_mask0 = _mm_cmpgt_epi16(qcoeff0, zbin);
cmp_mask1 = _mm_cmpgt_epi16(qcoeff1, zbin);
all_zero = _mm_or_si128(cmp_mask0, cmp_mask1);
if (_mm_movemask_epi8(all_zero) == 0) {
_mm_store_si128((__m128i *)(qcoeff_ptr + index), zero);
_mm_store_si128((__m128i *)(qcoeff_ptr + index + 4), zero);
_mm_store_si128((__m128i *)(qcoeff_ptr + index + 8), zero);
_mm_store_si128((__m128i *)(qcoeff_ptr + index + 12), zero);
_mm_store_si128((__m128i *)(dqcoeff_ptr + index), zero);
_mm_store_si128((__m128i *)(dqcoeff_ptr + index + 4), zero);
_mm_store_si128((__m128i *)(dqcoeff_ptr + index + 8), zero);
_mm_store_si128((__m128i *)(dqcoeff_ptr + index + 12), zero);
continue;
}
calculate_qcoeff_64x64(&qcoeff0, round, quant, &shift);
calculate_qcoeff_64x64(&qcoeff1, round, quant, &shift);
qcoeff0 = _mm_sign_epi16(qcoeff0, coeff0);
qcoeff1 = _mm_sign_epi16(qcoeff1, coeff1);
qcoeff0 = _mm_and_si128(qcoeff0, cmp_mask0);
qcoeff1 = _mm_and_si128(qcoeff1, cmp_mask1);
store_coefficients(qcoeff0, qcoeff_ptr + index);
store_coefficients(qcoeff1, qcoeff_ptr + index + 8);
calculate_dqcoeff_and_store_64x64(qcoeff0, dequant, zero,
dqcoeff_ptr + index);
calculate_dqcoeff_and_store_64x64(qcoeff1, dequant, zero,
dqcoeff_ptr + 8 + index);
eob0 = scan_for_eob(&qcoeff0, &qcoeff1, cmp_mask0, cmp_mask1, iscan, index,
zero);
eob = _mm_max_epi16(eob, eob0);
}
*eob_ptr = accumulate_eob(eob);
}

40
third_party/aom/aom_dsp/x86/quantize_ssse3_x86_64.asm vendored
View file

@ -48,9 +48,6 @@ cglobal quantize_%1, 0, %2, 15, coeff, ncoeff, zbin, round, quant, \
mov r3, qcoeffmp
mov r4, dqcoeffmp
mov r5, iscanmp
%ifidn %1, b_32x32
psllw m4, 1
%endif
pxor m5, m5 ; m5 = dedicated zero
DEFINE_ARGS coeff, ncoeff, d1, qcoeff, dqcoeff, iscan, d2, d3, d4, eob
lea coeffq, [ coeffq+ncoeffq*4]
@ -78,9 +75,26 @@ cglobal quantize_%1, 0, %2, 15, coeff, ncoeff, zbin, round, quant, \
pmulhw m13, m11, m2 ; m13 = m11*q>>16
paddw m8, m6 ; m8 += m6
paddw m13, m11 ; m13 += m11
%ifidn %1, b_32x32
pmullw m5, m8, m4 ; store the lower 16 bits of m8*qsh
%endif
pmulhw m8, m4 ; m8 = m8*qsh>>16
%ifidn %1, b_32x32
psllw m8, 1
psrlw m5, 15
por m8, m5
%endif
punpckhqdq m4, m4
%ifidn %1, b_32x32
pmullw m5, m13, m4 ; store the lower 16 bits of m13*qsh
%endif
pmulhw m13, m4 ; m13 = m13*qsh>>16
%ifidn %1, b_32x32
psllw m13, 1
psrlw m5, 15
por m13, m5
pxor m5, m5 ; reset m5 to zero register
%endif
psignw m8, m9 ; m8 = reinsert sign
psignw m13, m10 ; m13 = reinsert sign
pand m8, m7
@ -117,7 +131,7 @@ cglobal quantize_%1, 0, %2, 15, coeff, ncoeff, zbin, round, quant, \
psignw m8, m9
psignw m13, m10
%endif
; store 16bit numbers as 32bit numbers in array pointed to by qcoeff
; store 16bit numbers as 32bit numbers in array pointed to by dqcoeff
mova m11, m8
mova m6, m8
pcmpgtw m5, m8
@ -169,12 +183,28 @@ cglobal quantize_%1, 0, %2, 15, coeff, ncoeff, zbin, round, quant, \
pmulhw m13, m11, m2 ; m13 = m11*q>>16
paddw m14, m6 ; m14 += m6
paddw m13, m11 ; m13 += m11
%ifidn %1, b_32x32
pmullw m5, m14, m4 ; store the lower 16 bits of m14*qsh
%endif
pmulhw m14, m4 ; m14 = m14*qsh>>16
%ifidn %1, b_32x32
psllw m14, 1
psrlw m5, 15
por m14, m5
pmullw m5, m13, m4 ; store the lower 16 bits of m13*qsh
%endif
pmulhw m13, m4 ; m13 = m13*qsh>>16
%ifidn %1, b_32x32
psllw m13, 1
psrlw m5, 15
por m13, m5
pxor m5, m5 ; reset m5 to zero register
%endif
psignw m14, m9 ; m14 = reinsert sign
psignw m13, m10 ; m13 = reinsert sign
pand m14, m7
pand m13, m12
; store 16bit numbers as 32bit numbers in array pointed to by qcoeff
pxor m11, m11
mova m11, m14
@ -207,7 +237,7 @@ cglobal quantize_%1, 0, %2, 15, coeff, ncoeff, zbin, round, quant, \
psignw m13, m10
%endif
; store 16bit numbers as 32bit numbers in array pointed to by qcoeff
; store 16bit numbers as 32bit numbers in array pointed to by dqcoeff
mova m11, m14
mova m6, m14
pcmpgtw m5, m14

20
third_party/aom/aom_dsp/x86/quantize_x86.h vendored
View file

@ -75,3 +75,23 @@ static INLINE int16_t accumulate_eob(__m128i eob) {
eob = _mm_max_epi16(eob, eob_shuffled);
return _mm_extract_epi16(eob, 1);
}
static INLINE __m128i load_coefficients(const tran_low_t *coeff_ptr) {
assert(sizeof(tran_low_t) == 4);
const __m128i coeff1 = _mm_load_si128((__m128i *)(coeff_ptr));
const __m128i coeff2 = _mm_load_si128((__m128i *)(coeff_ptr + 4));
return _mm_packs_epi32(coeff1, coeff2);
}
static INLINE void store_coefficients(__m128i coeff_vals,
tran_low_t *coeff_ptr) {
assert(sizeof(tran_low_t) == 4);
__m128i one = _mm_set1_epi16(1);
__m128i coeff_vals_hi = _mm_mulhi_epi16(coeff_vals, one);
__m128i coeff_vals_lo = _mm_mullo_epi16(coeff_vals, one);
__m128i coeff_vals_1 = _mm_unpacklo_epi16(coeff_vals_lo, coeff_vals_hi);
__m128i coeff_vals_2 = _mm_unpackhi_epi16(coeff_vals_lo, coeff_vals_hi);
_mm_store_si128((__m128i *)(coeff_ptr), coeff_vals_1);
_mm_store_si128((__m128i *)(coeff_ptr + 4), coeff_vals_2);
}

308
third_party/aom/aom_dsp/x86/sse_avx2.c vendored
View file

@ -21,12 +21,11 @@ static INLINE void sse_w32_avx2(__m256i *sum, const uint8_t *a,
const uint8_t *b) {
const __m256i v_a0 = yy_loadu_256(a);
const __m256i v_b0 = yy_loadu_256(b);
const __m256i v_a00_w = _mm256_cvtepu8_epi16(_mm256_castsi256_si128(v_a0));
const __m256i v_a01_w =
_mm256_cvtepu8_epi16(_mm256_extracti128_si256(v_a0, 1));
const __m256i v_b00_w = _mm256_cvtepu8_epi16(_mm256_castsi256_si128(v_b0));
const __m256i v_b01_w =
_mm256_cvtepu8_epi16(_mm256_extracti128_si256(v_b0, 1));
const __m256i zero = _mm256_setzero_si256();
const __m256i v_a00_w = _mm256_unpacklo_epi8(v_a0, zero);
const __m256i v_a01_w = _mm256_unpackhi_epi8(v_a0, zero);
const __m256i v_b00_w = _mm256_unpacklo_epi8(v_b0, zero);
const __m256i v_b01_w = _mm256_unpackhi_epi8(v_b0, zero);
const __m256i v_d00_w = _mm256_sub_epi16(v_a00_w, v_b00_w);
const __m256i v_d01_w = _mm256_sub_epi16(v_a01_w, v_b01_w);
*sum = _mm256_add_epi32(*sum, _mm256_madd_epi16(v_d00_w, v_d00_w));
@ -35,43 +34,76 @@ static INLINE void sse_w32_avx2(__m256i *sum, const uint8_t *a,
static INLINE int64_t summary_all_avx2(const __m256i *sum_all) {
int64_t sum;
const __m256i sum0_4x64 =
_mm256_cvtepu32_epi64(_mm256_castsi256_si128(*sum_all));
const __m256i sum1_4x64 =
_mm256_cvtepu32_epi64(_mm256_extracti128_si256(*sum_all, 1));
__m256i zero = _mm256_setzero_si256();
const __m256i sum0_4x64 = _mm256_unpacklo_epi32(*sum_all, zero);
const __m256i sum1_4x64 = _mm256_unpackhi_epi32(*sum_all, zero);
const __m256i sum_4x64 = _mm256_add_epi64(sum0_4x64, sum1_4x64);
const __m128i sum_2x64 = _mm_add_epi64(_mm256_castsi256_si128(sum_4x64),
_mm256_extracti128_si256(sum_4x64, 1));
const __m128i sum_1x64 = _mm_add_epi64(sum_2x64, _mm_srli_si128(sum_2x64, 8));
xx_storel_64(&sum, sum_1x64);
return sum;
}
static INLINE void summary_32_avx2(const __m256i *sum32, __m256i *sum) {
const __m256i sum0_4x64 =
_mm256_cvtepu32_epi64(_mm256_castsi256_si128(*sum32));
const __m256i sum1_4x64 =
_mm256_cvtepu32_epi64(_mm256_extracti128_si256(*sum32, 1));
const __m256i sum_4x64 = _mm256_add_epi64(sum0_4x64, sum1_4x64);
*sum = _mm256_add_epi64(*sum, sum_4x64);
}
static INLINE int64_t summary_4x64_avx2(const __m256i sum_4x64) {
int64_t sum;
const __m128i sum_2x64 = _mm_add_epi64(_mm256_castsi256_si128(sum_4x64),
_mm256_extracti128_si256(sum_4x64, 1));
const __m128i sum_1x64 = _mm_add_epi64(sum_2x64, _mm_srli_si128(sum_2x64, 8));
xx_storel_64(&sum, sum_1x64);
return sum;
}
static INLINE void sse_w4x4_avx2(const uint8_t *a, int a_stride,
const uint8_t *b, int b_stride, __m256i *sum) {
const __m128i v_a0 = xx_loadl_32(a);
const __m128i v_a1 = xx_loadl_32(a + a_stride);
const __m128i v_a2 = xx_loadl_32(a + a_stride * 2);
const __m128i v_a3 = xx_loadl_32(a + a_stride * 3);
const __m128i v_b0 = xx_loadl_32(b);
const __m128i v_b1 = xx_loadl_32(b + b_stride);
const __m128i v_b2 = xx_loadl_32(b + b_stride * 2);
const __m128i v_b3 = xx_loadl_32(b + b_stride * 3);
const __m128i v_a0123 = _mm_unpacklo_epi64(_mm_unpacklo_epi32(v_a0, v_a1),
_mm_unpacklo_epi32(v_a2, v_a3));
const __m128i v_b0123 = _mm_unpacklo_epi64(_mm_unpacklo_epi32(v_b0, v_b1),
_mm_unpacklo_epi32(v_b2, v_b3));
const __m256i v_a_w = _mm256_cvtepu8_epi16(v_a0123);
const __m256i v_b_w = _mm256_cvtepu8_epi16(v_b0123);
const __m256i v_d_w = _mm256_sub_epi16(v_a_w, v_b_w);
*sum = _mm256_add_epi32(*sum, _mm256_madd_epi16(v_d_w, v_d_w));
}
static INLINE void sse_w8x2_avx2(const uint8_t *a, int a_stride,
const uint8_t *b, int b_stride, __m256i *sum) {
const __m128i v_a0 = xx_loadl_64(a);
const __m128i v_a1 = xx_loadl_64(a + a_stride);
const __m128i v_b0 = xx_loadl_64(b);
const __m128i v_b1 = xx_loadl_64(b + b_stride);
const __m256i v_a_w = _mm256_cvtepu8_epi16(_mm_unpacklo_epi64(v_a0, v_a1));
const __m256i v_b_w = _mm256_cvtepu8_epi16(_mm_unpacklo_epi64(v_b0, v_b1));
const __m256i v_d_w = _mm256_sub_epi16(v_a_w, v_b_w);
*sum = _mm256_add_epi32(*sum, _mm256_madd_epi16(v_d_w, v_d_w));
}
int64_t aom_sse_avx2(const uint8_t *a, int a_stride, const uint8_t *b,
int b_stride, int width, int height) {
int32_t y = 0;
int64_t sse = 0;
__m256i sum = _mm256_setzero_si256();
__m256i zero = _mm256_setzero_si256();
switch (width) {
case 4:
do {
const __m128i v_a0 = xx_loadl_32(a);
const __m128i v_a1 = xx_loadl_32(a + a_stride);
const __m128i v_a2 = xx_loadl_32(a + a_stride * 2);
const __m128i v_a3 = xx_loadl_32(a + a_stride * 3);
const __m128i v_b0 = xx_loadl_32(b);
const __m128i v_b1 = xx_loadl_32(b + b_stride);
const __m128i v_b2 = xx_loadl_32(b + b_stride * 2);
const __m128i v_b3 = xx_loadl_32(b + b_stride * 3);
const __m128i v_a0123 = _mm_unpacklo_epi64(
_mm_unpacklo_epi32(v_a0, v_a1), _mm_unpacklo_epi32(v_a2, v_a3));
const __m128i v_b0123 = _mm_unpacklo_epi64(
_mm_unpacklo_epi32(v_b0, v_b1), _mm_unpacklo_epi32(v_b2, v_b3));
const __m256i v_a_w = _mm256_cvtepu8_epi16(v_a0123);
const __m256i v_b_w = _mm256_cvtepu8_epi16(v_b0123);
const __m256i v_d_w = _mm256_sub_epi16(v_a_w, v_b_w);
sum = _mm256_add_epi32(sum, _mm256_madd_epi16(v_d_w, v_d_w));
sse_w4x4_avx2(a, a_stride, b, b_stride, &sum);
a += a_stride << 2;
b += b_stride << 2;
y += 4;
@ -80,16 +112,7 @@ int64_t aom_sse_avx2(const uint8_t *a, int a_stride, const uint8_t *b,
break;
case 8:
do {
const __m128i v_a0 = xx_loadl_64(a);
const __m128i v_a1 = xx_loadl_64(a + a_stride);
const __m128i v_b0 = xx_loadl_64(b);
const __m128i v_b1 = xx_loadl_64(b + b_stride);
const __m256i v_a_w =
_mm256_cvtepu8_epi16(_mm_unpacklo_epi64(v_a0, v_a1));
const __m256i v_b_w =
_mm256_cvtepu8_epi16(_mm_unpacklo_epi64(v_b0, v_b1));
const __m256i v_d_w = _mm256_sub_epi16(v_a_w, v_b_w);
sum = _mm256_add_epi32(sum, _mm256_madd_epi16(v_d_w, v_d_w));
sse_w8x2_avx2(a, a_stride, b, b_stride, &sum);
a += a_stride << 1;
b += b_stride << 1;
y += 2;
@ -99,14 +122,26 @@ int64_t aom_sse_avx2(const uint8_t *a, int a_stride, const uint8_t *b,
case 16:
do {
const __m128i v_a0 = xx_loadu_128(a);
const __m128i v_a1 = xx_loadu_128(a + a_stride);
const __m128i v_b0 = xx_loadu_128(b);
const __m256i v_a_w = _mm256_cvtepu8_epi16(v_a0);
const __m256i v_b_w = _mm256_cvtepu8_epi16(v_b0);
const __m256i v_d_w = _mm256_sub_epi16(v_a_w, v_b_w);
sum = _mm256_add_epi32(sum, _mm256_madd_epi16(v_d_w, v_d_w));
a += a_stride;
b += b_stride;
y += 1;
const __m128i v_b1 = xx_loadu_128(b + b_stride);
const __m256i v_a =
_mm256_insertf128_si256(_mm256_castsi128_si256(v_a0), v_a1, 0x01);
const __m256i v_b =
_mm256_insertf128_si256(_mm256_castsi128_si256(v_b0), v_b1, 0x01);
const __m256i v_al = _mm256_unpacklo_epi8(v_a, zero);
const __m256i v_au = _mm256_unpackhi_epi8(v_a, zero);
const __m256i v_bl = _mm256_unpacklo_epi8(v_b, zero);
const __m256i v_bu = _mm256_unpackhi_epi8(v_b, zero);
const __m256i v_asub = _mm256_sub_epi16(v_al, v_bl);
const __m256i v_bsub = _mm256_sub_epi16(v_au, v_bu);
const __m256i temp =
_mm256_add_epi32(_mm256_madd_epi16(v_asub, v_asub),
_mm256_madd_epi16(v_bsub, v_bsub));
sum = _mm256_add_epi32(sum, temp);
a += a_stride << 1;
b += b_stride << 1;
y += 2;
} while (y < height);
sse = summary_all_avx2(&sum);
break;
@ -141,7 +176,36 @@ int64_t aom_sse_avx2(const uint8_t *a, int a_stride, const uint8_t *b,
} while (y < height);
sse = summary_all_avx2(&sum);
break;
default: break;
default:
if ((width & 0x07) == 0) {
do {
int i = 0;
do {
sse_w8x2_avx2(a + i, a_stride, b + i, b_stride, &sum);
i += 8;
} while (i < width);
a += a_stride << 1;
b += b_stride << 1;
y += 2;
} while (y < height);
} else {
do {
int i = 0;
do {
sse_w8x2_avx2(a + i, a_stride, b + i, b_stride, &sum);
const uint8_t *a2 = a + i + (a_stride << 1);
const uint8_t *b2 = b + i + (b_stride << 1);
sse_w8x2_avx2(a2, a_stride, b2, b_stride, &sum);
i += 8;
} while (i + 4 < width);
sse_w4x4_avx2(a + i, a_stride, b + i, b_stride, &sum);
a += a_stride << 2;
b += b_stride << 2;
y += 4;
} while (y < height);
}
sse = summary_all_avx2(&sum);
break;
}
return sse;
@ -155,6 +219,33 @@ static INLINE void highbd_sse_w16_avx2(__m256i *sum, const uint16_t *a,
*sum = _mm256_add_epi32(*sum, _mm256_madd_epi16(v_d_w, v_d_w));
}
static INLINE void highbd_sse_w4x4_avx2(__m256i *sum, const uint16_t *a,
int a_stride, const uint16_t *b,
int b_stride) {
const __m128i v_a0 = xx_loadl_64(a);
const __m128i v_a1 = xx_loadl_64(a + a_stride);
const __m128i v_a2 = xx_loadl_64(a + a_stride * 2);
const __m128i v_a3 = xx_loadl_64(a + a_stride * 3);
const __m128i v_b0 = xx_loadl_64(b);
const __m128i v_b1 = xx_loadl_64(b + b_stride);
const __m128i v_b2 = xx_loadl_64(b + b_stride * 2);
const __m128i v_b3 = xx_loadl_64(b + b_stride * 3);
const __m256i v_a_w = yy_set_m128i(_mm_unpacklo_epi64(v_a0, v_a1),
_mm_unpacklo_epi64(v_a2, v_a3));
const __m256i v_b_w = yy_set_m128i(_mm_unpacklo_epi64(v_b0, v_b1),
_mm_unpacklo_epi64(v_b2, v_b3));
const __m256i v_d_w = _mm256_sub_epi16(v_a_w, v_b_w);
*sum = _mm256_add_epi32(*sum, _mm256_madd_epi16(v_d_w, v_d_w));
}
static INLINE void highbd_sse_w8x2_avx2(__m256i *sum, const uint16_t *a,
int a_stride, const uint16_t *b,
int b_stride) {
const __m256i v_a_w = yy_loadu2_128(a + a_stride, a);
const __m256i v_b_w = yy_loadu2_128(b + b_stride, b);
const __m256i v_d_w = _mm256_sub_epi16(v_a_w, v_b_w);
*sum = _mm256_add_epi32(*sum, _mm256_madd_epi16(v_d_w, v_d_w));
}
int64_t aom_highbd_sse_avx2(const uint8_t *a8, int a_stride, const uint8_t *b8,
int b_stride, int width, int height) {
int32_t y = 0;
@ -165,20 +256,7 @@ int64_t aom_highbd_sse_avx2(const uint8_t *a8, int a_stride, const uint8_t *b8,
switch (width) {
case 4:
do {
const __m128i v_a0 = xx_loadl_64(a);
const __m128i v_a1 = xx_loadl_64(a + a_stride);
const __m128i v_a2 = xx_loadl_64(a + a_stride * 2);
const __m128i v_a3 = xx_loadl_64(a + a_stride * 3);
const __m128i v_b0 = xx_loadl_64(b);
const __m128i v_b1 = xx_loadl_64(b + b_stride);
const __m128i v_b2 = xx_loadl_64(b + b_stride * 2);
const __m128i v_b3 = xx_loadl_64(b + b_stride * 3);
const __m256i v_a_w = yy_set_m128i(_mm_unpacklo_epi64(v_a0, v_a1),
_mm_unpacklo_epi64(v_a2, v_a3));
const __m256i v_b_w = yy_set_m128i(_mm_unpacklo_epi64(v_b0, v_b1),
_mm_unpacklo_epi64(v_b2, v_b3));
const __m256i v_d_w = _mm256_sub_epi16(v_a_w, v_b_w);
sum = _mm256_add_epi32(sum, _mm256_madd_epi16(v_d_w, v_d_w));
highbd_sse_w4x4_avx2(&sum, a, a_stride, b, b_stride);
a += a_stride << 2;
b += b_stride << 2;
y += 4;
@ -187,10 +265,7 @@ int64_t aom_highbd_sse_avx2(const uint8_t *a8, int a_stride, const uint8_t *b8,
break;
case 8:
do {
const __m256i v_a_w = yy_loadu2_128(a + a_stride, a);
const __m256i v_b_w = yy_loadu2_128(b + b_stride, b);
const __m256i v_d_w = _mm256_sub_epi16(v_a_w, v_b_w);
sum = _mm256_add_epi32(sum, _mm256_madd_epi16(v_d_w, v_d_w));
highbd_sse_w8x2_avx2(&sum, a, a_stride, b, b_stride);
a += a_stride << 1;
b += b_stride << 1;
y += 2;
@ -208,43 +283,98 @@ int64_t aom_highbd_sse_avx2(const uint8_t *a8, int a_stride, const uint8_t *b8,
break;
case 32:
do {
highbd_sse_w16_avx2(&sum, a, b);
highbd_sse_w16_avx2(&sum, a + 16, b + 16);
a += a_stride;
b += b_stride;
y += 1;
int l = 0;
__m256i sum32 = _mm256_setzero_si256();
do {
highbd_sse_w16_avx2(&sum32, a, b);
highbd_sse_w16_avx2(&sum32, a + 16, b + 16);
a += a_stride;
b += b_stride;
l += 1;
} while (l < 64 && l < (height - y));
summary_32_avx2(&sum32, &sum);
y += 64;
} while (y < height);
sse = summary_all_avx2(&sum);
sse = summary_4x64_avx2(sum);
break;
case 64:
do {
highbd_sse_w16_avx2(&sum, a, b);
highbd_sse_w16_avx2(&sum, a + 16 * 1, b + 16 * 1);
highbd_sse_w16_avx2(&sum, a + 16 * 2, b + 16 * 2);
highbd_sse_w16_avx2(&sum, a + 16 * 3, b + 16 * 3);
a += a_stride;
b += b_stride;
y += 1;
int l = 0;
__m256i sum32 = _mm256_setzero_si256();
do {
highbd_sse_w16_avx2(&sum32, a, b);
highbd_sse_w16_avx2(&sum32, a + 16 * 1, b + 16 * 1);
highbd_sse_w16_avx2(&sum32, a + 16 * 2, b + 16 * 2);
highbd_sse_w16_avx2(&sum32, a + 16 * 3, b + 16 * 3);
a += a_stride;
b += b_stride;
l += 1;
} while (l < 32 && l < (height - y));
summary_32_avx2(&sum32, &sum);
y += 32;
} while (y < height);
sse = summary_all_avx2(&sum);
sse = summary_4x64_avx2(sum);
break;
case 128:
do {
highbd_sse_w16_avx2(&sum, a, b);
highbd_sse_w16_avx2(&sum, a + 16 * 1, b + 16 * 1);
highbd_sse_w16_avx2(&sum, a + 16 * 2, b + 16 * 2);
highbd_sse_w16_avx2(&sum, a + 16 * 3, b + 16 * 3);
highbd_sse_w16_avx2(&sum, a + 16 * 4, b + 16 * 4);
highbd_sse_w16_avx2(&sum, a + 16 * 5, b + 16 * 5);
highbd_sse_w16_avx2(&sum, a + 16 * 6, b + 16 * 6);
highbd_sse_w16_avx2(&sum, a + 16 * 7, b + 16 * 7);
a += a_stride;
b += b_stride;
y += 1;
int l = 0;
__m256i sum32 = _mm256_setzero_si256();
do {
highbd_sse_w16_avx2(&sum32, a, b);
highbd_sse_w16_avx2(&sum32, a + 16 * 1, b + 16 * 1);
highbd_sse_w16_avx2(&sum32, a + 16 * 2, b + 16 * 2);
highbd_sse_w16_avx2(&sum32, a + 16 * 3, b + 16 * 3);
highbd_sse_w16_avx2(&sum32, a + 16 * 4, b + 16 * 4);
highbd_sse_w16_avx2(&sum32, a + 16 * 5, b + 16 * 5);
highbd_sse_w16_avx2(&sum32, a + 16 * 6, b + 16 * 6);
highbd_sse_w16_avx2(&sum32, a + 16 * 7, b + 16 * 7);
a += a_stride;
b += b_stride;
l += 1;
} while (l < 16 && l < (height - y));
summary_32_avx2(&sum32, &sum);
y += 16;
} while (y < height);
sse = summary_all_avx2(&sum);
sse = summary_4x64_avx2(sum);
break;
default:
if (width & 0x7) {
do {
int i = 0;
__m256i sum32 = _mm256_setzero_si256();
do {
highbd_sse_w8x2_avx2(&sum32, a + i, a_stride, b + i, b_stride);
const uint16_t *a2 = a + i + (a_stride << 1);
const uint16_t *b2 = b + i + (b_stride << 1);
highbd_sse_w8x2_avx2(&sum32, a2, a_stride, b2, b_stride);
i += 8;
} while (i + 4 < width);
highbd_sse_w4x4_avx2(&sum32, a + i, a_stride, b + i, b_stride);
summary_32_avx2(&sum32, &sum);
a += a_stride << 2;
b += b_stride << 2;
y += 4;
} while (y < height);
} else {
do {
int l = 0;
__m256i sum32 = _mm256_setzero_si256();
do {
int i = 0;
do {
highbd_sse_w8x2_avx2(&sum32, a + i, a_stride, b + i, b_stride);
i += 8;
} while (i < width);
a += a_stride << 1;
b += b_stride << 1;
l += 2;
} while (l < 8 && l < (height - y));
summary_32_avx2(&sum32, &sum);
y += 8;
} while (y < height);
}
sse = summary_4x64_avx2(sum);
break;
default: break;
}
return sse;
}

249
third_party/aom/aom_dsp/x86/sse_sse4.c vendored
View file

@ -28,6 +28,13 @@ static INLINE int64_t summary_all_sse4(const __m128i *sum_all) {
return sum;
}
static INLINE void summary_32_sse4(const __m128i *sum32, __m128i *sum64) {
const __m128i sum0 = _mm_cvtepu32_epi64(*sum32);
const __m128i sum1 = _mm_cvtepu32_epi64(_mm_srli_si128(*sum32, 8));
*sum64 = _mm_add_epi64(sum0, *sum64);
*sum64 = _mm_add_epi64(sum1, *sum64);
}
static INLINE void sse_w16_sse4_1(__m128i *sum, const uint8_t *a,
const uint8_t *b) {
const __m128i v_a0 = xx_loadu_128(a);
@ -42,6 +49,28 @@ static INLINE void sse_w16_sse4_1(__m128i *sum, const uint8_t *a,
*sum = _mm_add_epi32(*sum, _mm_madd_epi16(v_d01_w, v_d01_w));
}
static INLINE void aom_sse4x2_sse4_1(const uint8_t *a, int a_stride,
const uint8_t *b, int b_stride,
__m128i *sum) {
const __m128i v_a0 = xx_loadl_32(a);
const __m128i v_a1 = xx_loadl_32(a + a_stride);
const __m128i v_b0 = xx_loadl_32(b);
const __m128i v_b1 = xx_loadl_32(b + b_stride);
const __m128i v_a_w = _mm_cvtepu8_epi16(_mm_unpacklo_epi32(v_a0, v_a1));
const __m128i v_b_w = _mm_cvtepu8_epi16(_mm_unpacklo_epi32(v_b0, v_b1));
const __m128i v_d_w = _mm_sub_epi16(v_a_w, v_b_w);
*sum = _mm_add_epi32(*sum, _mm_madd_epi16(v_d_w, v_d_w));
}
static INLINE void aom_sse8_sse4_1(const uint8_t *a, const uint8_t *b,
__m128i *sum) {
const __m128i v_a0 = xx_loadl_64(a);
const __m128i v_b0 = xx_loadl_64(b);
const __m128i v_a_w = _mm_cvtepu8_epi16(v_a0);
const __m128i v_b_w = _mm_cvtepu8_epi16(v_b0);
const __m128i v_d_w = _mm_sub_epi16(v_a_w, v_b_w);
*sum = _mm_add_epi32(*sum, _mm_madd_epi16(v_d_w, v_d_w));
}
int64_t aom_sse_sse4_1(const uint8_t *a, int a_stride, const uint8_t *b,
int b_stride, int width, int height) {
int y = 0;
@ -50,14 +79,7 @@ int64_t aom_sse_sse4_1(const uint8_t *a, int a_stride, const uint8_t *b,
switch (width) {
case 4:
do {
const __m128i v_a0 = xx_loadl_32(a);
const __m128i v_a1 = xx_loadl_32(a + a_stride);
const __m128i v_b0 = xx_loadl_32(b);
const __m128i v_b1 = xx_loadl_32(b + b_stride);
const __m128i v_a_w = _mm_cvtepu8_epi16(_mm_unpacklo_epi32(v_a0, v_a1));
const __m128i v_b_w = _mm_cvtepu8_epi16(_mm_unpacklo_epi32(v_b0, v_b1));
const __m128i v_d_w = _mm_sub_epi16(v_a_w, v_b_w);
sum = _mm_add_epi32(sum, _mm_madd_epi16(v_d_w, v_d_w));
aom_sse4x2_sse4_1(a, a_stride, b, b_stride, &sum);
a += a_stride << 1;
b += b_stride << 1;
y += 2;
@ -66,12 +88,7 @@ int64_t aom_sse_sse4_1(const uint8_t *a, int a_stride, const uint8_t *b,
break;
case 8:
do {
const __m128i v_a0 = xx_loadl_64(a);
const __m128i v_b0 = xx_loadl_64(b);
const __m128i v_a_w = _mm_cvtepu8_epi16(v_a0);
const __m128i v_b_w = _mm_cvtepu8_epi16(v_b0);
const __m128i v_d_w = _mm_sub_epi16(v_a_w, v_b_w);
sum = _mm_add_epi32(sum, _mm_madd_epi16(v_d_w, v_d_w));
aom_sse8_sse4_1(a, b, &sum);
a += a_stride;
b += b_stride;
y += 1;
@ -125,12 +142,52 @@ int64_t aom_sse_sse4_1(const uint8_t *a, int a_stride, const uint8_t *b,
} while (y < height);
sse = summary_all_sse4(&sum);
break;
default: break;
default:
if (width & 0x07) {
do {
int i = 0;
do {
aom_sse8_sse4_1(a + i, b + i, &sum);
aom_sse8_sse4_1(a + i + a_stride, b + i + b_stride, &sum);
i += 8;
} while (i + 4 < width);
aom_sse4x2_sse4_1(a + i, a_stride, b + i, b_stride, &sum);
a += (a_stride << 1);
b += (b_stride << 1);
y += 2;
} while (y < height);
} else {
do {
int i = 0;
do {
aom_sse8_sse4_1(a + i, b + i, &sum);
i += 8;
} while (i < width);
a += a_stride;
b += b_stride;
y += 1;
} while (y < height);
}
sse = summary_all_sse4(&sum);
break;
}
return sse;
}
static INLINE void highbd_sse_w4x2_sse4_1(__m128i *sum, const uint16_t *a,
int a_stride, const uint16_t *b,
int b_stride) {
const __m128i v_a0 = xx_loadl_64(a);
const __m128i v_a1 = xx_loadl_64(a + a_stride);
const __m128i v_b0 = xx_loadl_64(b);
const __m128i v_b1 = xx_loadl_64(b + b_stride);
const __m128i v_a_w = _mm_unpacklo_epi64(v_a0, v_a1);
const __m128i v_b_w = _mm_unpacklo_epi64(v_b0, v_b1);
const __m128i v_d_w = _mm_sub_epi16(v_a_w, v_b_w);
*sum = _mm_add_epi32(*sum, _mm_madd_epi16(v_d_w, v_d_w));
}
static INLINE void highbd_sse_w8_sse4_1(__m128i *sum, const uint16_t *a,
const uint16_t *b) {
const __m128i v_a_w = xx_loadu_128(a);
@ -150,14 +207,7 @@ int64_t aom_highbd_sse_sse4_1(const uint8_t *a8, int a_stride,
switch (width) {
case 4:
do {
const __m128i v_a0 = xx_loadl_64(a);
const __m128i v_a1 = xx_loadl_64(a + a_stride);
const __m128i v_b0 = xx_loadl_64(b);
const __m128i v_b1 = xx_loadl_64(b + b_stride);
const __m128i v_a_w = _mm_unpacklo_epi64(v_a0, v_a1);
const __m128i v_b_w = _mm_unpacklo_epi64(v_b0, v_b1);
const __m128i v_d_w = _mm_sub_epi16(v_a_w, v_b_w);
sum = _mm_add_epi32(sum, _mm_madd_epi16(v_d_w, v_d_w));
highbd_sse_w4x2_sse4_1(&sum, a, a_stride, b, b_stride);
a += a_stride << 1;
b += b_stride << 1;
y += 2;
@ -175,67 +225,126 @@ int64_t aom_highbd_sse_sse4_1(const uint8_t *a8, int a_stride,
break;
case 16:
do {
highbd_sse_w8_sse4_1(&sum, a, b);
highbd_sse_w8_sse4_1(&sum, a + 8, b + 8);
a += a_stride;
b += b_stride;
y += 1;
int l = 0;
__m128i sum32 = _mm_setzero_si128();
do {
highbd_sse_w8_sse4_1(&sum32, a, b);
highbd_sse_w8_sse4_1(&sum32, a + 8, b + 8);
a += a_stride;
b += b_stride;
l += 1;
} while (l < 64 && l < (height - y));
summary_32_sse4(&sum32, &sum);
y += 64;
} while (y < height);
sse = summary_all_sse4(&sum);
xx_storel_64(&sse, _mm_add_epi64(sum, _mm_srli_si128(sum, 8)));
break;
case 32:
do {
highbd_sse_w8_sse4_1(&sum, a, b);
highbd_sse_w8_sse4_1(&sum, a + 8 * 1, b + 8 * 1);
highbd_sse_w8_sse4_1(&sum, a + 8 * 2, b + 8 * 2);
highbd_sse_w8_sse4_1(&sum, a + 8 * 3, b + 8 * 3);
a += a_stride;
b += b_stride;
y += 1;
int l = 0;
__m128i sum32 = _mm_setzero_si128();
do {
highbd_sse_w8_sse4_1(&sum32, a, b);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 1, b + 8 * 1);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 2, b + 8 * 2);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 3, b + 8 * 3);
a += a_stride;
b += b_stride;
l += 1;
} while (l < 32 && l < (height - y));
summary_32_sse4(&sum32, &sum);
y += 32;
} while (y < height);
sse = summary_all_sse4(&sum);
xx_storel_64(&sse, _mm_add_epi64(sum, _mm_srli_si128(sum, 8)));
break;
case 64:
do {
highbd_sse_w8_sse4_1(&sum, a, b);
highbd_sse_w8_sse4_1(&sum, a + 8 * 1, b + 8 * 1);
highbd_sse_w8_sse4_1(&sum, a + 8 * 2, b + 8 * 2);
highbd_sse_w8_sse4_1(&sum, a + 8 * 3, b + 8 * 3);
highbd_sse_w8_sse4_1(&sum, a + 8 * 4, b + 8 * 4);
highbd_sse_w8_sse4_1(&sum, a + 8 * 5, b + 8 * 5);
highbd_sse_w8_sse4_1(&sum, a + 8 * 6, b + 8 * 6);
highbd_sse_w8_sse4_1(&sum, a + 8 * 7, b + 8 * 7);
a += a_stride;
b += b_stride;
y += 1;
int l = 0;
__m128i sum32 = _mm_setzero_si128();
do {
highbd_sse_w8_sse4_1(&sum32, a, b);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 1, b + 8 * 1);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 2, b + 8 * 2);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 3, b + 8 * 3);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 4, b + 8 * 4);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 5, b + 8 * 5);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 6, b + 8 * 6);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 7, b + 8 * 7);
a += a_stride;
b += b_stride;
l += 1;
} while (l < 16 && l < (height - y));
summary_32_sse4(&sum32, &sum);
y += 16;
} while (y < height);
sse = summary_all_sse4(&sum);
xx_storel_64(&sse, _mm_add_epi64(sum, _mm_srli_si128(sum, 8)));
break;
case 128:
do {
highbd_sse_w8_sse4_1(&sum, a, b);
highbd_sse_w8_sse4_1(&sum, a + 8 * 1, b + 8 * 1);
highbd_sse_w8_sse4_1(&sum, a + 8 * 2, b + 8 * 2);
highbd_sse_w8_sse4_1(&sum, a + 8 * 3, b + 8 * 3);
highbd_sse_w8_sse4_1(&sum, a + 8 * 4, b + 8 * 4);
highbd_sse_w8_sse4_1(&sum, a + 8 * 5, b + 8 * 5);
highbd_sse_w8_sse4_1(&sum, a + 8 * 6, b + 8 * 6);
highbd_sse_w8_sse4_1(&sum, a + 8 * 7, b + 8 * 7);
highbd_sse_w8_sse4_1(&sum, a + 8 * 8, b + 8 * 8);
highbd_sse_w8_sse4_1(&sum, a + 8 * 9, b + 8 * 9);
highbd_sse_w8_sse4_1(&sum, a + 8 * 10, b + 8 * 10);
highbd_sse_w8_sse4_1(&sum, a + 8 * 11, b + 8 * 11);
highbd_sse_w8_sse4_1(&sum, a + 8 * 12, b + 8 * 12);
highbd_sse_w8_sse4_1(&sum, a + 8 * 13, b + 8 * 13);
highbd_sse_w8_sse4_1(&sum, a + 8 * 14, b + 8 * 14);
highbd_sse_w8_sse4_1(&sum, a + 8 * 15, b + 8 * 15);
a += a_stride;
b += b_stride;
y += 1;
int l = 0;
__m128i sum32 = _mm_setzero_si128();
do {
highbd_sse_w8_sse4_1(&sum32, a, b);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 1, b + 8 * 1);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 2, b + 8 * 2);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 3, b + 8 * 3);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 4, b + 8 * 4);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 5, b + 8 * 5);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 6, b + 8 * 6);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 7, b + 8 * 7);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 8, b + 8 * 8);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 9, b + 8 * 9);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 10, b + 8 * 10);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 11, b + 8 * 11);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 12, b + 8 * 12);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 13, b + 8 * 13);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 14, b + 8 * 14);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 15, b + 8 * 15);
a += a_stride;
b += b_stride;
l += 1;
} while (l < 8 && l < (height - y));
summary_32_sse4(&sum32, &sum);
y += 8;
} while (y < height);
sse = summary_all_sse4(&sum);
xx_storel_64(&sse, _mm_add_epi64(sum, _mm_srli_si128(sum, 8)));
break;
default:
if (width & 0x7) {
do {
__m128i sum32 = _mm_setzero_si128();
int i = 0;
do {
highbd_sse_w8_sse4_1(&sum32, a + i, b + i);
highbd_sse_w8_sse4_1(&sum32, a + i + a_stride, b + i + b_stride);
i += 8;
} while (i + 4 < width);
highbd_sse_w4x2_sse4_1(&sum32, a + i, a_stride, b + i, b_stride);
a += (a_stride << 1);
b += (b_stride << 1);
y += 2;
summary_32_sse4(&sum32, &sum);
} while (y < height);
} else {
do {
int l = 0;
__m128i sum32 = _mm_setzero_si128();
do {
int i = 0;
do {
highbd_sse_w8_sse4_1(&sum32, a + i, b + i);
i += 8;
} while (i < width);
a += a_stride;
b += b_stride;
l += 1;
} while (l < 8 && l < (height - y));
summary_32_sse4(&sum32, &sum);
y += 8;
} while (y < height);
}
xx_storel_64(&sse, _mm_add_epi64(sum, _mm_srli_si128(sum, 8)));
break;
default: break;
}
return sse;
}

222
third_party/aom/aom_dsp/x86/ssim_sse2_x86_64.asm vendored Normal file
View file

@ -0,0 +1,222 @@
;
; Copyright (c) 2016, Alliance for Open Media. All rights reserved
;
; This source code is subject to the terms of the BSD 2 Clause License and
; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
; was not distributed with this source code in the LICENSE file, you can
; obtain it at www.aomedia.org/license/software. If the Alliance for Open
; Media Patent License 1.0 was not distributed with this source code in the
; PATENTS file, you can obtain it at www.aomedia.org/license/patent.
;
;
%include "aom_ports/x86_abi_support.asm"
; tabulate_ssim - sums sum_s,sum_r,sum_sq_s,sum_sq_r, sum_sxr
%macro TABULATE_SSIM 0
paddusw xmm15, xmm3 ; sum_s
paddusw xmm14, xmm4 ; sum_r
movdqa xmm1, xmm3
pmaddwd xmm1, xmm1
paddd xmm13, xmm1 ; sum_sq_s
movdqa xmm2, xmm4
pmaddwd xmm2, xmm2
paddd xmm12, xmm2 ; sum_sq_r
pmaddwd xmm3, xmm4
paddd xmm11, xmm3 ; sum_sxr
%endmacro
; Sum across the register %1 starting with q words
%macro SUM_ACROSS_Q 1
movdqa xmm2,%1
punpckldq %1,xmm0
punpckhdq xmm2,xmm0
paddq %1,xmm2
movdqa xmm2,%1
punpcklqdq %1,xmm0
punpckhqdq xmm2,xmm0
paddq %1,xmm2
%endmacro
; Sum across the register %1 starting with q words
%macro SUM_ACROSS_W 1
movdqa xmm1, %1
punpcklwd %1,xmm0
punpckhwd xmm1,xmm0
paddd %1, xmm1
SUM_ACROSS_Q %1
%endmacro
SECTION .text
;void ssim_parms_sse2(
; unsigned char *s,
; int sp,
; unsigned char *r,
; int rp
; uint32_t *sum_s,
; uint32_t *sum_r,
; uint32_t *sum_sq_s,
; uint32_t *sum_sq_r,
; uint32_t *sum_sxr);
;
; TODO: Use parm passing through structure, probably don't need the pxors
; ( calling app will initialize to 0 ) could easily fit everything in sse2
; without too much hastle, and can probably do better estimates with psadw
; or pavgb At this point this is just meant to be first pass for calculating
; all the parms needed for 16x16 ssim so we can play with dssim as distortion
; in mode selection code.
global sym(aom_ssim_parms_16x16_sse2) PRIVATE
sym(aom_ssim_parms_16x16_sse2):
push rbp
mov rbp, rsp
SHADOW_ARGS_TO_STACK 9
SAVE_XMM 15
push rsi
push rdi
; end prolog
mov rsi, arg(0) ;s
mov rcx, arg(1) ;sp
mov rdi, arg(2) ;r
mov rax, arg(3) ;rp
pxor xmm0, xmm0
pxor xmm15,xmm15 ;sum_s
pxor xmm14,xmm14 ;sum_r
pxor xmm13,xmm13 ;sum_sq_s
pxor xmm12,xmm12 ;sum_sq_r
pxor xmm11,xmm11 ;sum_sxr
mov rdx, 16 ;row counter
.NextRow:
;grab source and reference pixels
movdqu xmm5, [rsi]
movdqu xmm6, [rdi]
movdqa xmm3, xmm5
movdqa xmm4, xmm6
punpckhbw xmm3, xmm0 ; high_s
punpckhbw xmm4, xmm0 ; high_r
TABULATE_SSIM
movdqa xmm3, xmm5
movdqa xmm4, xmm6
punpcklbw xmm3, xmm0 ; low_s
punpcklbw xmm4, xmm0 ; low_r
TABULATE_SSIM
add rsi, rcx ; next s row
add rdi, rax ; next r row
dec rdx ; counter
jnz .NextRow
SUM_ACROSS_W xmm15
SUM_ACROSS_W xmm14
SUM_ACROSS_Q xmm13
SUM_ACROSS_Q xmm12
SUM_ACROSS_Q xmm11
mov rdi,arg(4)
movd [rdi], xmm15;
mov rdi,arg(5)
movd [rdi], xmm14;
mov rdi,arg(6)
movd [rdi], xmm13;
mov rdi,arg(7)
movd [rdi], xmm12;
mov rdi,arg(8)
movd [rdi], xmm11;
; begin epilog
pop rdi
pop rsi
RESTORE_XMM
UNSHADOW_ARGS
pop rbp
ret
;void ssim_parms_sse2(
; unsigned char *s,
; int sp,
; unsigned char *r,
; int rp
; uint32_t *sum_s,
; uint32_t *sum_r,
; uint32_t *sum_sq_s,
; uint32_t *sum_sq_r,
; uint32_t *sum_sxr);
;
; TODO: Use parm passing through structure, probably don't need the pxors
; ( calling app will initialize to 0 ) could easily fit everything in sse2
; without too much hastle, and can probably do better estimates with psadw
; or pavgb At this point this is just meant to be first pass for calculating
; all the parms needed for 16x16 ssim so we can play with dssim as distortion
; in mode selection code.
global sym(aom_ssim_parms_8x8_sse2) PRIVATE
sym(aom_ssim_parms_8x8_sse2):
push rbp
mov rbp, rsp
SHADOW_ARGS_TO_STACK 9
SAVE_XMM 15
push rsi
push rdi
; end prolog
mov rsi, arg(0) ;s
mov rcx, arg(1) ;sp
mov rdi, arg(2) ;r
mov rax, arg(3) ;rp
pxor xmm0, xmm0
pxor xmm15,xmm15 ;sum_s
pxor xmm14,xmm14 ;sum_r
pxor xmm13,xmm13 ;sum_sq_s
pxor xmm12,xmm12 ;sum_sq_r
pxor xmm11,xmm11 ;sum_sxr
mov rdx, 8 ;row counter
.NextRow:
;grab source and reference pixels
movq xmm3, [rsi]
movq xmm4, [rdi]
punpcklbw xmm3, xmm0 ; low_s
punpcklbw xmm4, xmm0 ; low_r
TABULATE_SSIM
add rsi, rcx ; next s row
add rdi, rax ; next r row
dec rdx ; counter
jnz .NextRow
SUM_ACROSS_W xmm15
SUM_ACROSS_W xmm14
SUM_ACROSS_Q xmm13
SUM_ACROSS_Q xmm12
SUM_ACROSS_Q xmm11
mov rdi,arg(4)
movd [rdi], xmm15;
mov rdi,arg(5)
movd [rdi], xmm14;
mov rdi,arg(6)
movd [rdi], xmm13;
mov rdi,arg(7)
movd [rdi], xmm12;
mov rdi,arg(8)
movd [rdi], xmm11;
; begin epilog
pop rdi
pop rsi
RESTORE_XMM
UNSHADOW_ARGS
pop rbp
ret

51
third_party/aom/aom_dsp/x86/subpel_variance_sse2.asm vendored
View file

@ -135,44 +135,33 @@ SECTION .text
%if %2 == 1 ; avg
cglobal sub_pixel_avg_variance%1xh, 7, 7, 13, src, src_stride, \
x_offset, y_offset, dst, dst_stride, \
sec, sec_stride, height, sse, \
g_bilin_filter, g_pw_8
sec, sec_stride, height, sse
%define block_height dword heightm
%define sec_str sec_stridemp
;Store bilin_filter and pw_8 location in stack
%if GET_GOT_DEFINED == 1
GET_GOT eax
add esp, 4 ; restore esp
%endif
lea ecx, [GLOBAL(bilin_filter_m)]
mov g_bilin_filterm, ecx
lea ecx, [GLOBAL(pw_8)]
mov g_pw_8m, ecx
LOAD_IF_USED 0, 1 ; load eax, ecx back
%else
cglobal sub_pixel_variance%1xh, 7, 7, 13, src, src_stride, \
x_offset, y_offset, dst, dst_stride, \
height, sse, g_bilin_filter, g_pw_8
height, sse
%define block_height heightd
;Store bilin_filter and pw_8 location in stack
%if GET_GOT_DEFINED == 1
GET_GOT eax
add esp, 4 ; restore esp
%endif
lea ecx, [GLOBAL(bilin_filter_m)]
mov g_bilin_filterm, ecx
lea ecx, [GLOBAL(pw_8)]
mov g_pw_8m, ecx
LOAD_IF_USED 0, 1 ; load eax, ecx back
%endif
; reuse argument stack space
%define g_bilin_filterm x_offsetm
%define g_pw_8m y_offsetm
;Store bilin_filter and pw_8 location in stack
%if GET_GOT_DEFINED == 1
GET_GOT eax
add esp, 4 ; restore esp
%endif
lea ecx, [GLOBAL(bilin_filter_m)]
mov g_bilin_filterm, ecx
lea ecx, [GLOBAL(pw_8)]
mov g_pw_8m, ecx
LOAD_IF_USED 0, 1 ; load eax, ecx back
%else
%if %2 == 1 ; avg
cglobal sub_pixel_avg_variance%1xh, 7, 7, 13, src, src_stride, \

8
third_party/aom/aom_dsp/x86/synonyms.h vendored
View file

@ -13,6 +13,7 @@
#define AOM_AOM_DSP_X86_SYNONYMS_H_
#include <immintrin.h>
#include <string.h>
#include "config/aom_config.h"
@ -28,7 +29,9 @@
// Loads and stores to do away with the tedium of casting the address
// to the right type.
static INLINE __m128i xx_loadl_32(const void *a) {
return _mm_cvtsi32_si128(*(const uint32_t *)a);
int val;
memcpy(&val, a, sizeof(val));
return _mm_cvtsi32_si128(val);
}
static INLINE __m128i xx_loadl_64(const void *a) {
@ -44,7 +47,8 @@ static INLINE __m128i xx_loadu_128(const void *a) {
}
static INLINE void xx_storel_32(void *const a, const __m128i v) {
*(uint32_t *)a = _mm_cvtsi128_si32(v);
const int val = _mm_cvtsi128_si32(v);
memcpy(a, &val, sizeof(val));
}
static INLINE void xx_storel_64(void *const a, const __m128i v) {

5
third_party/aom/aom_dsp/x86/synonyms_avx2.h vendored
View file

@ -67,6 +67,11 @@ static INLINE __m256i yy_loadu2_128(const void *hi, const void *lo) {
return yy_set_m128i(mhi, mlo);
}
static INLINE void yy_storeu2_128(void *hi, void *lo, const __m256i a) {
_mm_storeu_si128((__m128i *)hi, _mm256_extracti128_si256(a, 1));
_mm_storeu_si128((__m128i *)lo, _mm256_castsi256_si128(a));
}
static INLINE __m256i yy_roundn_epu16(__m256i v_val_w, int bits) {
const __m256i v_s_w = _mm256_srli_epi16(v_val_w, bits - 1);
return _mm256_avg_epu16(v_s_w, _mm256_setzero_si256());

50
third_party/aom/aom_dsp/x86/txfm_common_avx2.h vendored
View file

@ -20,9 +20,6 @@
extern "C" {
#endif
typedef void (*transform_1d_avx2)(const __m256i *input, __m256i *output,
int8_t cos_bit);
static INLINE __m256i pair_set_w16_epi16(int16_t a, int16_t b) {
return _mm256_set1_epi32(
(int32_t)(((uint16_t)(a)) | (((uint32_t)(b)) << 16)));
@ -192,6 +189,53 @@ static INLINE void round_shift_16bit_w16_avx2(__m256i *in, int size, int bit) {
}
}
static INLINE __m256i av1_round_shift_32_avx2(__m256i vec, int bit) {
__m256i tmp, round;
round = _mm256_set1_epi32(1 << (bit - 1));
tmp = _mm256_add_epi32(vec, round);
return _mm256_srai_epi32(tmp, bit);
}
static INLINE void av1_round_shift_array_32_avx2(__m256i *input,
__m256i *output,
const int size,
const int bit) {
if (bit > 0) {
int i;
for (i = 0; i < size; i++) {
output[i] = av1_round_shift_32_avx2(input[i], bit);
}
} else {
int i;
for (i = 0; i < size; i++) {
output[i] = _mm256_slli_epi32(input[i], -bit);
}
}
}
static INLINE void av1_round_shift_rect_array_32_avx2(__m256i *input,
__m256i *output,
const int size,
const int bit,
const int val) {
const __m256i sqrt2 = _mm256_set1_epi32(val);
if (bit > 0) {
int i;
for (i = 0; i < size; i++) {
const __m256i r0 = av1_round_shift_32_avx2(input[i], bit);
const __m256i r1 = _mm256_mullo_epi32(sqrt2, r0);
output[i] = av1_round_shift_32_avx2(r1, NewSqrt2Bits);
}
} else {
int i;
for (i = 0; i < size; i++) {
const __m256i r0 = _mm256_slli_epi32(input[i], -bit);
const __m256i r1 = _mm256_mullo_epi32(sqrt2, r0);
output[i] = av1_round_shift_32_avx2(r1, NewSqrt2Bits);
}
}
}
#ifdef __cplusplus
}
#endif

24
third_party/aom/aom_dsp/x86/variance_sse2.c vendored
View file

@ -494,7 +494,7 @@ void aom_upsampled_pred_sse2(MACROBLOCKD *xd, const struct AV1Common *const cm,
const int ref_num = 0;
const int is_intrabc = is_intrabc_block(mi);
const struct scale_factors *const sf =
is_intrabc ? &cm->sf_identity : &xd->block_refs[ref_num]->sf;
is_intrabc ? &cm->sf_identity : xd->block_ref_scale_factors[ref_num];
const int is_scaled = av1_is_scaled(sf);
if (is_scaled) {
@ -571,11 +571,10 @@ void aom_upsampled_pred_sse2(MACROBLOCKD *xd, const struct AV1Common *const cm,
}
}
const InterpFilterParams *filter =
(subpel_search == 1)
? av1_get_4tap_interp_filter_params(EIGHTTAP_REGULAR)
: av1_get_interp_filter_params_with_block_size(EIGHTTAP_REGULAR, 8);
int filter_taps = (subpel_search == 1) ? 4 : SUBPEL_TAPS;
const InterpFilterParams *filter = av1_get_filter(subpel_search);
// (TODO:yunqing) 2-tap case uses 4-tap functions since there is no SIMD for
// 2-tap yet.
int filter_taps = (subpel_search <= USE_4_TAPS) ? 4 : SUBPEL_TAPS;
if (!subpel_x_q3 && !subpel_y_q3) {
if (width >= 16) {
@ -638,20 +637,13 @@ void aom_upsampled_pred_sse2(MACROBLOCKD *xd, const struct AV1Common *const cm,
const int16_t *const kernel_y =
av1_get_interp_filter_subpel_kernel(filter, subpel_y_q3 << 1);
const uint8_t *ref_start = ref - ref_stride * ((filter_taps >> 1) - 1);
uint8_t *temp_start_horiz =
(subpel_search == 1) ? temp + (filter_taps >> 1) * MAX_SB_SIZE : temp;
uint8_t *temp_start_horiz = (subpel_search <= USE_4_TAPS)
? temp + (filter_taps >> 1) * MAX_SB_SIZE
: temp;
uint8_t *temp_start_vert = temp + MAX_SB_SIZE * ((filter->taps >> 1) - 1);
int intermediate_height =
(((height - 1) * 8 + subpel_y_q3) >> 3) + filter_taps;
assert(intermediate_height <= (MAX_SB_SIZE * 2 + 16) + 16);
// TODO(Deepa): Remove the memset below when we have
// 4 tap simd for sse2 and ssse3.
if (subpel_search == 1) {
memset(temp_start_vert - 3 * MAX_SB_SIZE, 0, width);
memset(temp_start_vert - 2 * MAX_SB_SIZE, 0, width);
memset(temp_start_vert + (height + 2) * MAX_SB_SIZE, 0, width);
memset(temp_start_vert + (height + 3) * MAX_SB_SIZE, 0, width);
}
aom_convolve8_horiz(ref_start, ref_stride, temp_start_horiz, MAX_SB_SIZE,
kernel_x, 16, NULL, -1, width, intermediate_height);
aom_convolve8_vert(temp_start_vert, MAX_SB_SIZE, comp_pred, width, NULL, -1,