mirror of
https://repo.dactyloidae.xyz/Dactyloidae/UXP.git
synced 2026-10-06 07:17:30 +09:00
Update libaom to rev b25610052a1398032320008d69b51d2da94f5928
This commit is contained in:
trav90
Roy Tam
parent
8493557d20
commit
3d109dd4fc
240 changed files with 16977 additions and 6139 deletions
8
third_party/aom/aom_dsp/aom_dsp.cmake
vendored
8
third_party/aom/aom_dsp/aom_dsp.cmake
vendored
|
|
@ -83,6 +83,7 @@ list(APPEND AOM_DSP_COMMON_INTRIN_SSE4_1
|
|||
list(APPEND AOM_DSP_COMMON_INTRIN_AVX2
|
||||
"${AOM_ROOT}/aom_dsp/x86/aom_subpixel_8t_intrin_avx2.c"
|
||||
"${AOM_ROOT}/aom_dsp/x86/common_avx2.h"
|
||||
"${AOM_ROOT}/aom_dsp/x86/txfm_common_avx2.h"
|
||||
"${AOM_ROOT}/aom_dsp/x86/convolve_avx2.h"
|
||||
"${AOM_ROOT}/aom_dsp/x86/fft_avx2.c"
|
||||
"${AOM_ROOT}/aom_dsp/x86/highbd_convolve_avx2.c"
|
||||
|
|
@ -190,13 +191,16 @@ if(CONFIG_AV1_ENCODER)
|
|||
"${AOM_ROOT}/aom_dsp/x86/ssim_opt_x86_64.asm")
|
||||
|
||||
list(APPEND AOM_DSP_ENCODER_INTRIN_AVX2
|
||||
"${AOM_ROOT}/aom_dsp/x86/masked_sad_intrin_avx2.c"
|
||||
"${AOM_ROOT}/aom_dsp/x86/subtract_avx2.c"
|
||||
"${AOM_ROOT}/aom_dsp/x86/highbd_quantize_intrin_avx2.c"
|
||||
"${AOM_ROOT}/aom_dsp/x86/sad4d_avx2.c"
|
||||
"${AOM_ROOT}/aom_dsp/x86/sad_avx2.c"
|
||||
"${AOM_ROOT}/aom_dsp/x86/sad_highbd_avx2.c"
|
||||
"${AOM_ROOT}/aom_dsp/x86/sad_impl_avx2.c"
|
||||
"${AOM_ROOT}/aom_dsp/x86/variance_avx2.c"
|
||||
"${AOM_ROOT}/aom_dsp/x86/variance_impl_avx2.c")
|
||||
"${AOM_ROOT}/aom_dsp/x86/variance_impl_avx2.c"
|
||||
"${AOM_ROOT}/aom_dsp/x86/obmc_sad_avx2.c")
|
||||
|
||||
list(APPEND AOM_DSP_ENCODER_ASM_SSSE3_X86_64
|
||||
"${AOM_ROOT}/aom_dsp/x86/quantize_ssse3_x86_64.asm")
|
||||
|
|
@ -205,9 +209,11 @@ if(CONFIG_AV1_ENCODER)
|
|||
"${AOM_ROOT}/aom_dsp/x86/quantize_avx_x86_64.asm")
|
||||
|
||||
list(APPEND AOM_DSP_ENCODER_INTRIN_SSSE3
|
||||
"${AOM_ROOT}/aom_dsp/x86/masked_sad_intrin_ssse3.h"
|
||||
"${AOM_ROOT}/aom_dsp/x86/masked_sad_intrin_ssse3.c"
|
||||
"${AOM_ROOT}/aom_dsp/x86/masked_variance_intrin_ssse3.h"
|
||||
"${AOM_ROOT}/aom_dsp/x86/masked_variance_intrin_ssse3.c"
|
||||
"${AOM_ROOT}/aom_dsp/x86/variance_impl_ssse3.c"
|
||||
"${AOM_ROOT}/aom_dsp/x86/jnt_variance_ssse3.c"
|
||||
"${AOM_ROOT}/aom_dsp/x86/jnt_sad_ssse3.c")
|
||||
|
||||
|
|
|
|||
2
third_party/aom/aom_dsp/aom_dsp_rtcd.c
vendored
2
third_party/aom/aom_dsp/aom_dsp_rtcd.c
vendored
|
|
@ -15,4 +15,4 @@
|
|||
|
||||
#include "aom_ports/aom_once.h"
|
||||
|
||||
void aom_dsp_rtcd() { once(setup_rtcd_internal); }
|
||||
void aom_dsp_rtcd() { aom_once(setup_rtcd_internal); }
|
||||
|
|
|
|||
23
third_party/aom/aom_dsp/aom_dsp_rtcd_defs.pl
vendored
23
third_party/aom/aom_dsp/aom_dsp_rtcd_defs.pl
vendored
|
|
@ -377,7 +377,7 @@ add_proto qw/void aom_lpf_vertical_14_dual/, "uint8_t *s, int pitch, const uint8
|
|||
specialize qw/aom_lpf_vertical_14_dual sse2/;
|
||||
|
||||
add_proto qw/void aom_lpf_vertical_6/, "uint8_t *s, int pitch, const uint8_t *blimit, const uint8_t *limit, const uint8_t *thresh";
|
||||
specialize qw/aom_lpf_vertical_6 sse2/;
|
||||
specialize qw/aom_lpf_vertical_6 sse2 neon/;
|
||||
|
||||
add_proto qw/void aom_lpf_vertical_8/, "uint8_t *s, int pitch, const uint8_t *blimit, const uint8_t *limit, const uint8_t *thresh";
|
||||
specialize qw/aom_lpf_vertical_8 sse2 neon/;
|
||||
|
|
@ -386,13 +386,13 @@ add_proto qw/void aom_lpf_vertical_8_dual/, "uint8_t *s, int pitch, const uint8_
|
|||
specialize qw/aom_lpf_vertical_8_dual sse2/;
|
||||
|
||||
add_proto qw/void aom_lpf_vertical_4/, "uint8_t *s, int pitch, const uint8_t *blimit, const uint8_t *limit, const uint8_t *thresh";
|
||||
specialize qw/aom_lpf_vertical_4 sse2/;
|
||||
specialize qw/aom_lpf_vertical_4 sse2 neon/;
|
||||
|
||||
add_proto qw/void aom_lpf_vertical_4_dual/, "uint8_t *s, int pitch, const uint8_t *blimit0, const uint8_t *limit0, const uint8_t *thresh0, const uint8_t *blimit1, const uint8_t *limit1, const uint8_t *thresh1";
|
||||
specialize qw/aom_lpf_vertical_4_dual sse2/;
|
||||
|
||||
add_proto qw/void aom_lpf_horizontal_14/, "uint8_t *s, int pitch, const uint8_t *blimit, const uint8_t *limit, const uint8_t *thresh";
|
||||
specialize qw/aom_lpf_horizontal_14 sse2/;
|
||||
specialize qw/aom_lpf_horizontal_14 sse2 neon/;
|
||||
|
||||
add_proto qw/void aom_lpf_horizontal_14_dual/, "uint8_t *s, int pitch, const uint8_t *blimit0, const uint8_t *limit0, const uint8_t *thresh0, const uint8_t *blimit1, const uint8_t *limit1, const uint8_t *thresh1";
|
||||
specialize qw/aom_lpf_horizontal_14_dual sse2/;
|
||||
|
|
@ -410,7 +410,7 @@ add_proto qw/void aom_lpf_horizontal_8_dual/, "uint8_t *s, int pitch, const uint
|
|||
specialize qw/aom_lpf_horizontal_8_dual sse2/;
|
||||
|
||||
add_proto qw/void aom_lpf_horizontal_4/, "uint8_t *s, int pitch, const uint8_t *blimit, const uint8_t *limit, const uint8_t *thresh";
|
||||
specialize qw/aom_lpf_horizontal_4 sse2/;
|
||||
specialize qw/aom_lpf_horizontal_4 sse2 neon/;
|
||||
|
||||
add_proto qw/void aom_lpf_horizontal_4_dual/, "uint8_t *s, int pitch, const uint8_t *blimit0, const uint8_t *limit0, const uint8_t *thresh0, const uint8_t *blimit1, const uint8_t *limit1, const uint8_t *thresh1";
|
||||
specialize qw/aom_lpf_horizontal_4_dual sse2/;
|
||||
|
|
@ -564,7 +564,7 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
|
|||
# Block subtraction
|
||||
#
|
||||
add_proto qw/void aom_subtract_block/, "int rows, int cols, int16_t *diff_ptr, ptrdiff_t diff_stride, const uint8_t *src_ptr, ptrdiff_t src_stride, const uint8_t *pred_ptr, ptrdiff_t pred_stride";
|
||||
specialize qw/aom_subtract_block neon msa sse2/;
|
||||
specialize qw/aom_subtract_block neon msa sse2 avx2/;
|
||||
|
||||
add_proto qw/void aom_highbd_subtract_block/, "int rows, int cols, int16_t *diff_ptr, ptrdiff_t diff_stride, const uint8_t *src_ptr, ptrdiff_t src_stride, const uint8_t *pred_ptr, ptrdiff_t pred_stride, int bd";
|
||||
specialize qw/aom_highbd_subtract_block sse2/;
|
||||
|
|
@ -732,14 +732,14 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
|
|||
foreach (@block_sizes) {
|
||||
($w, $h) = @$_;
|
||||
add_proto qw/unsigned int/, "aom_masked_sad${w}x${h}", "const uint8_t *src, int src_stride, const uint8_t *ref, int ref_stride, const uint8_t *second_pred, const uint8_t *msk, int msk_stride, int invert_mask";
|
||||
specialize "aom_masked_sad${w}x${h}", qw/ssse3/;
|
||||
specialize "aom_masked_sad${w}x${h}", qw/ssse3 avx2/;
|
||||
}
|
||||
|
||||
|
||||
foreach (@block_sizes) {
|
||||
($w, $h) = @$_;
|
||||
add_proto qw/unsigned int/, "aom_highbd_masked_sad${w}x${h}", "const uint8_t *src8, int src_stride, const uint8_t *ref8, int ref_stride, const uint8_t *second_pred8, const uint8_t *msk, int msk_stride, int invert_mask";
|
||||
specialize "aom_highbd_masked_sad${w}x${h}", qw/ssse3/;
|
||||
specialize "aom_highbd_masked_sad${w}x${h}", qw/ssse3 avx2/;
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -750,7 +750,7 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
|
|||
($w, $h) = @$_;
|
||||
add_proto qw/unsigned int/, "aom_obmc_sad${w}x${h}", "const uint8_t *pre, int pre_stride, const int32_t *wsrc, const int32_t *mask";
|
||||
if (! (($w == 128 && $h == 32) || ($w == 32 && $h == 128))) {
|
||||
specialize "aom_obmc_sad${w}x${h}", qw/sse4_1/;
|
||||
specialize "aom_obmc_sad${w}x${h}", qw/sse4_1 avx2/;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -759,7 +759,7 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
|
|||
($w, $h) = @$_;
|
||||
add_proto qw/unsigned int/, "aom_highbd_obmc_sad${w}x${h}", "const uint8_t *pre, int pre_stride, const int32_t *wsrc, const int32_t *mask";
|
||||
if (! (($w == 128 && $h == 32) || ($w == 32 && $h == 128))) {
|
||||
specialize "aom_highbd_obmc_sad${w}x${h}", qw/sse4_1/;
|
||||
specialize "aom_highbd_obmc_sad${w}x${h}", qw/sse4_1 avx2/;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1102,6 +1102,7 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
|
|||
add_proto qw/unsigned int/, "aom_obmc_variance${w}x${h}", "const uint8_t *pre, int pre_stride, const int32_t *wsrc, const int32_t *mask, unsigned int *sse";
|
||||
add_proto qw/unsigned int/, "aom_obmc_sub_pixel_variance${w}x${h}", "const uint8_t *pre, int pre_stride, int xoffset, int yoffset, const int32_t *wsrc, const int32_t *mask, unsigned int *sse";
|
||||
specialize "aom_obmc_variance${w}x${h}", q/sse4_1/;
|
||||
specialize "aom_obmc_sub_pixel_variance${w}x${h}", q/sse4_1/;
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -1539,9 +1540,7 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
|
|||
specialize qw/aom_comp_mask_pred ssse3 avx2/;
|
||||
|
||||
add_proto qw/void aom_highbd_comp_mask_pred/, "uint16_t *comp_pred, const uint8_t *pred8, int width, int height, const uint8_t *ref8, int ref_stride, const uint8_t *mask, int mask_stride, int invert_mask";
|
||||
add_proto qw/void aom_highbd_comp_mask_upsampled_pred/, "MACROBLOCKD *xd, const struct AV1Common *const cm, int mi_row, int mi_col, const MV *const mv, uint16_t *comp_pred, const uint8_t *pred8, int width,
|
||||
int height, int subpel_x_q3, int subpel_y_q3, const uint8_t *ref8, int ref_stride, const uint8_t *mask, int mask_stride, int invert_mask, int bd";
|
||||
|
||||
specialize qw/aom_highbd_comp_mask_pred avx2/;
|
||||
|
||||
} # CONFIG_AV1_ENCODER
|
||||
|
||||
|
|
|
|||
60
third_party/aom/aom_dsp/arm/intrapred_neon.c
vendored
60
third_party/aom/aom_dsp/arm/intrapred_neon.c
vendored
|
|
@ -528,3 +528,63 @@ void aom_h_predictor_32x32_neon(uint8_t *dst, ptrdiff_t stride,
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
static INLINE void highbd_dc_predictor(uint16_t *dst, ptrdiff_t stride, int bw,
|
||||
const uint16_t *above,
|
||||
const uint16_t *left) {
|
||||
assert(bw >= 4);
|
||||
assert(IS_POWER_OF_TWO(bw));
|
||||
int expected_dc, sum = 0;
|
||||
const int count = bw * 2;
|
||||
uint32x4_t sum_q = vdupq_n_u32(0);
|
||||
uint32x2_t sum_d;
|
||||
uint16_t *dst_1;
|
||||
if (bw >= 8) {
|
||||
for (int i = 0; i < bw; i += 8) {
|
||||
sum_q = vpadalq_u16(sum_q, vld1q_u16(above));
|
||||
sum_q = vpadalq_u16(sum_q, vld1q_u16(left));
|
||||
above += 8;
|
||||
left += 8;
|
||||
}
|
||||
sum_d = vadd_u32(vget_low_u32(sum_q), vget_high_u32(sum_q));
|
||||
sum = vget_lane_s32(vreinterpret_s32_u64(vpaddl_u32(sum_d)), 0);
|
||||
expected_dc = (sum + (count >> 1)) / count;
|
||||
const uint16x8_t dc = vdupq_n_u16((uint16_t)expected_dc);
|
||||
for (int r = 0; r < bw; r++) {
|
||||
dst_1 = dst;
|
||||
for (int i = 0; i < bw; i += 8) {
|
||||
vst1q_u16(dst_1, dc);
|
||||
dst_1 += 8;
|
||||
}
|
||||
dst += stride;
|
||||
}
|
||||
} else { // 4x4
|
||||
sum_q = vaddl_u16(vld1_u16(above), vld1_u16(left));
|
||||
sum_d = vadd_u32(vget_low_u32(sum_q), vget_high_u32(sum_q));
|
||||
sum = vget_lane_s32(vreinterpret_s32_u64(vpaddl_u32(sum_d)), 0);
|
||||
expected_dc = (sum + (count >> 1)) / count;
|
||||
const uint16x4_t dc = vdup_n_u16((uint16_t)expected_dc);
|
||||
for (int r = 0; r < bw; r++) {
|
||||
vst1_u16(dst, dc);
|
||||
dst += stride;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#define intra_pred_highbd_sized_neon(type, width) \
|
||||
void aom_highbd_##type##_predictor_##width##x##width##_neon( \
|
||||
uint16_t *dst, ptrdiff_t stride, const uint16_t *above, \
|
||||
const uint16_t *left, int bd) { \
|
||||
(void)bd; \
|
||||
highbd_##type##_predictor(dst, stride, width, above, left); \
|
||||
}
|
||||
|
||||
#define intra_pred_square(type) \
|
||||
intra_pred_highbd_sized_neon(type, 4); \
|
||||
intra_pred_highbd_sized_neon(type, 8); \
|
||||
intra_pred_highbd_sized_neon(type, 16); \
|
||||
intra_pred_highbd_sized_neon(type, 32); \
|
||||
intra_pred_highbd_sized_neon(type, 64);
|
||||
|
||||
intra_pred_square(dc);
|
||||
#undef intra_pred_square
|
||||
|
|
|
|||
228
third_party/aom/aom_dsp/arm/loopfilter_neon.c
vendored
228
third_party/aom/aom_dsp/arm/loopfilter_neon.c
vendored
|
|
@ -52,6 +52,36 @@ static INLINE uint8x8_t lpf_mask(uint8x8_t p3q3, uint8x8_t p2q2, uint8x8_t p1q1,
|
|||
return mask_8x8;
|
||||
}
|
||||
|
||||
static INLINE uint8x8_t lpf_mask2(uint8x8_t p1q1, uint8x8_t p0q0,
|
||||
const uint8_t blimit, const uint8_t limit) {
|
||||
uint32x2x2_t p0q0_p1q1;
|
||||
uint16x8_t temp_16x8;
|
||||
uint16x4_t temp0_16x4, temp1_16x4;
|
||||
const uint16x4_t blimit_16x4 = vdup_n_u16(blimit);
|
||||
const uint8x8_t limit_8x8 = vdup_n_u8(limit);
|
||||
uint8x8_t mask_8x8, temp_8x8;
|
||||
|
||||
mask_8x8 = vabd_u8(p1q1, p0q0);
|
||||
mask_8x8 = vcle_u8(mask_8x8, limit_8x8);
|
||||
|
||||
temp_8x8 = vreinterpret_u8_u32(vrev64_u32(vreinterpret_u32_u8(mask_8x8)));
|
||||
mask_8x8 = vand_u8(mask_8x8, temp_8x8);
|
||||
|
||||
p0q0_p1q1 = vtrn_u32(vreinterpret_u32_u8(p0q0), vreinterpret_u32_u8(p1q1));
|
||||
temp_8x8 = vabd_u8(vreinterpret_u8_u32(p0q0_p1q1.val[0]),
|
||||
vreinterpret_u8_u32(p0q0_p1q1.val[1]));
|
||||
temp_16x8 = vmovl_u8(temp_8x8);
|
||||
temp0_16x4 = vshl_n_u16(vget_low_u16(temp_16x8), 1);
|
||||
temp1_16x4 = vshr_n_u16(vget_high_u16(temp_16x8), 1);
|
||||
temp0_16x4 = vadd_u16(temp0_16x4, temp1_16x4);
|
||||
temp0_16x4 = vcle_u16(temp0_16x4, blimit_16x4);
|
||||
temp_8x8 = vmovn_u16(vcombine_u16(temp0_16x4, temp0_16x4));
|
||||
|
||||
mask_8x8 = vand_u8(mask_8x8, temp_8x8);
|
||||
|
||||
return mask_8x8;
|
||||
}
|
||||
|
||||
static INLINE uint8x8_t lpf_flat_mask4(uint8x8_t p3q3, uint8x8_t p2q2,
|
||||
uint8x8_t p1q1, uint8x8_t p0q0) {
|
||||
const uint8x8_t thresh_8x8 = vdup_n_u8(1); // for bd==8 threshold is always 1
|
||||
|
|
@ -523,6 +553,68 @@ static void lpf_6_neon(uint8x8_t *p2q2, uint8x8_t *p1q1, uint8x8_t *p0q0,
|
|||
}
|
||||
}
|
||||
|
||||
static void lpf_4_neon(uint8x8_t *p1q1, uint8x8_t *p0q0, const uint8_t blimit,
|
||||
const uint8_t limit, const uint8_t thresh) {
|
||||
int32x2x2_t ps0_qs0, ps1_qs1;
|
||||
int16x8_t filter_s16;
|
||||
const uint8x8_t thresh_f4 = vdup_n_u8(thresh);
|
||||
uint8x8_t mask_8x8, temp0_8x8, temp1_8x8;
|
||||
int8x8_t ps0_s8, ps1_s8, qs0_s8, qs1_s8, temp_s8;
|
||||
int8x8_t op0, oq0, op1, oq1;
|
||||
int8x8_t pq_s0, pq_s1;
|
||||
int8x8_t filter_s8, filter1_s8, filter2_s8;
|
||||
int8x8_t hev_8x8;
|
||||
const int8x8_t sign_mask = vdup_n_s8(0x80);
|
||||
const int8x8_t val_4 = vdup_n_s8(4);
|
||||
const int8x8_t val_3 = vdup_n_s8(3);
|
||||
|
||||
// Calculate filter mask
|
||||
mask_8x8 = lpf_mask2(*p1q1, *p0q0, blimit, limit);
|
||||
|
||||
pq_s0 = veor_s8(vreinterpret_s8_u8(*p0q0), sign_mask);
|
||||
pq_s1 = veor_s8(vreinterpret_s8_u8(*p1q1), sign_mask);
|
||||
|
||||
ps0_qs0 = vtrn_s32(vreinterpret_s32_s8(pq_s0), vreinterpret_s32_s8(pq_s0));
|
||||
ps1_qs1 = vtrn_s32(vreinterpret_s32_s8(pq_s1), vreinterpret_s32_s8(pq_s1));
|
||||
ps0_s8 = vreinterpret_s8_s32(ps0_qs0.val[0]);
|
||||
qs0_s8 = vreinterpret_s8_s32(ps0_qs0.val[1]);
|
||||
ps1_s8 = vreinterpret_s8_s32(ps1_qs1.val[0]);
|
||||
qs1_s8 = vreinterpret_s8_s32(ps1_qs1.val[1]);
|
||||
|
||||
// hev_mask
|
||||
temp0_8x8 = vcgt_u8(vabd_u8(*p0q0, *p1q1), thresh_f4);
|
||||
temp1_8x8 = vreinterpret_u8_u32(vrev64_u32(vreinterpret_u32_u8(temp0_8x8)));
|
||||
hev_8x8 = vreinterpret_s8_u8(vorr_u8(temp0_8x8, temp1_8x8));
|
||||
|
||||
// add outer taps if we have high edge variance
|
||||
filter_s8 = vqsub_s8(ps1_s8, qs1_s8);
|
||||
filter_s8 = vand_s8(filter_s8, hev_8x8);
|
||||
|
||||
// inner taps
|
||||
temp_s8 = vqsub_s8(qs0_s8, ps0_s8);
|
||||
filter_s16 = vmovl_s8(filter_s8);
|
||||
filter_s16 = vmlal_s8(filter_s16, temp_s8, val_3);
|
||||
filter_s8 = vqmovn_s16(filter_s16);
|
||||
filter_s8 = vand_s8(filter_s8, vreinterpret_s8_u8(mask_8x8));
|
||||
|
||||
filter1_s8 = vqadd_s8(filter_s8, val_4);
|
||||
filter2_s8 = vqadd_s8(filter_s8, val_3);
|
||||
filter1_s8 = vshr_n_s8(filter1_s8, 3);
|
||||
filter2_s8 = vshr_n_s8(filter2_s8, 3);
|
||||
|
||||
oq0 = veor_s8(vqsub_s8(qs0_s8, filter1_s8), sign_mask);
|
||||
op0 = veor_s8(vqadd_s8(ps0_s8, filter2_s8), sign_mask);
|
||||
|
||||
filter_s8 = vrshr_n_s8(filter1_s8, 1);
|
||||
filter_s8 = vbic_s8(filter_s8, hev_8x8);
|
||||
|
||||
oq1 = veor_s8(vqsub_s8(qs1_s8, filter_s8), sign_mask);
|
||||
op1 = veor_s8(vqadd_s8(ps1_s8, filter_s8), sign_mask);
|
||||
|
||||
*p0q0 = vreinterpret_u8_s8(vext_s8(op0, oq0, 4));
|
||||
*p1q1 = vreinterpret_u8_s8(vext_s8(op1, oq1, 4));
|
||||
}
|
||||
|
||||
void aom_lpf_vertical_14_neon(uint8_t *src, int stride, const uint8_t *blimit,
|
||||
const uint8_t *limit, const uint8_t *thresh) {
|
||||
uint8x16_t row0, row1, row2, row3;
|
||||
|
|
@ -646,6 +738,125 @@ void aom_lpf_vertical_8_neon(uint8_t *src, int stride, const uint8_t *blimit,
|
|||
store_u8_8x4(src - 4, stride, p3q0, p2q1, p1q2, p0q3);
|
||||
}
|
||||
|
||||
void aom_lpf_vertical_6_neon(uint8_t *src, int stride, const uint8_t *blimit,
|
||||
const uint8_t *limit, const uint8_t *thresh) {
|
||||
uint32x2x2_t p2q2_p1q1, pxqy_p0q0;
|
||||
uint32x2_t pq_rev;
|
||||
uint8x8_t pxq0, p2q1, p1q2, p0qy;
|
||||
uint8x8_t p0q0, p1q1, p2q2, pxqy;
|
||||
|
||||
// row0: px p2 p1 p0 | q0 q1 q2 qy
|
||||
// row1: px p2 p1 p0 | q0 q1 q2 qy
|
||||
// row2: px p2 p1 p0 | q0 q1 q2 qy
|
||||
// row3: px p2 p1 p0 | q0 q1 q2 qy
|
||||
load_u8_8x4(src - 4, stride, &pxq0, &p2q1, &p1q2, &p0qy);
|
||||
|
||||
transpose_u8_8x4(&pxq0, &p2q1, &p1q2, &p0qy);
|
||||
|
||||
pq_rev = vrev64_u32(vreinterpret_u32_u8(p0qy));
|
||||
pxqy_p0q0 = vtrn_u32(vreinterpret_u32_u8(pxq0), pq_rev);
|
||||
|
||||
pq_rev = vrev64_u32(vreinterpret_u32_u8(p1q2));
|
||||
p2q2_p1q1 = vtrn_u32(vreinterpret_u32_u8(p2q1), pq_rev);
|
||||
|
||||
p0q0 = vreinterpret_u8_u32(vrev64_u32(pxqy_p0q0.val[1]));
|
||||
p1q1 = vreinterpret_u8_u32(vrev64_u32(p2q2_p1q1.val[1]));
|
||||
p2q2 = vreinterpret_u8_u32(p2q2_p1q1.val[0]);
|
||||
pxqy = vreinterpret_u8_u32(pxqy_p0q0.val[0]);
|
||||
|
||||
lpf_6_neon(&p2q2, &p1q1, &p0q0, *blimit, *limit, *thresh);
|
||||
|
||||
pq_rev = vrev64_u32(vreinterpret_u32_u8(p0q0));
|
||||
pxqy_p0q0 = vtrn_u32(vreinterpret_u32_u8(pxqy), pq_rev);
|
||||
|
||||
pq_rev = vrev64_u32(vreinterpret_u32_u8(p1q1));
|
||||
p2q2_p1q1 = vtrn_u32(vreinterpret_u32_u8(p2q2), pq_rev);
|
||||
|
||||
p0qy = vreinterpret_u8_u32(vrev64_u32(pxqy_p0q0.val[1]));
|
||||
p1q2 = vreinterpret_u8_u32(vrev64_u32(p2q2_p1q1.val[1]));
|
||||
p2q1 = vreinterpret_u8_u32(p2q2_p1q1.val[0]);
|
||||
pxq0 = vreinterpret_u8_u32(pxqy_p0q0.val[0]);
|
||||
transpose_u8_8x4(&pxq0, &p2q1, &p1q2, &p0qy);
|
||||
|
||||
store_u8_8x4(src - 4, stride, pxq0, p2q1, p1q2, p0qy);
|
||||
}
|
||||
|
||||
void aom_lpf_vertical_4_neon(uint8_t *src, int stride, const uint8_t *blimit,
|
||||
const uint8_t *limit, const uint8_t *thresh) {
|
||||
uint32x2x2_t p1q0_p0q1, p1q1_p0q0, p1p0_q1q0;
|
||||
uint32x2_t pq_rev;
|
||||
uint8x8_t UNINITIALIZED_IS_SAFE(p1p0), q0q1, p0q0, p1q1;
|
||||
|
||||
// row0: p1 p0 | q0 q1
|
||||
// row1: p1 p0 | q0 q1
|
||||
// row2: p1 p0 | q0 q1
|
||||
// row3: p1 p0 | q0 q1
|
||||
load_u8_4x1(src - 2, &p1p0, 0);
|
||||
load_u8_4x1((src - 2) + 1 * stride, &p1p0, 1);
|
||||
load_u8_4x1((src - 2) + 2 * stride, &q0q1, 0);
|
||||
load_u8_4x1((src - 2) + 3 * stride, &q0q1, 1);
|
||||
|
||||
transpose_u8_4x4(&p1p0, &q0q1);
|
||||
|
||||
p1q0_p0q1 = vtrn_u32(vreinterpret_u32_u8(p1p0), vreinterpret_u32_u8(q0q1));
|
||||
|
||||
pq_rev = vrev64_u32(p1q0_p0q1.val[1]);
|
||||
p1q1_p0q0 = vtrn_u32(p1q0_p0q1.val[0], pq_rev);
|
||||
|
||||
p1q1 = vreinterpret_u8_u32(p1q1_p0q0.val[0]);
|
||||
p0q0 = vreinterpret_u8_u32(p1q1_p0q0.val[1]);
|
||||
|
||||
lpf_4_neon(&p1q1, &p0q0, *blimit, *limit, *thresh);
|
||||
|
||||
p1p0_q1q0 = vtrn_u32(vreinterpret_u32_u8(p1q1), vreinterpret_u32_u8(p0q0));
|
||||
|
||||
p1p0 = vreinterpret_u8_u32(p1p0_q1q0.val[0]);
|
||||
q0q1 = vreinterpret_u8_u32(vrev64_u32(p1p0_q1q0.val[1]));
|
||||
|
||||
transpose_u8_4x4(&p1p0, &q0q1);
|
||||
|
||||
store_u8_4x1(src - 2, p1p0, 0);
|
||||
store_u8_4x1((src - 2) + 1 * stride, q0q1, 0);
|
||||
store_u8_4x1((src - 2) + 2 * stride, p1p0, 1);
|
||||
store_u8_4x1((src - 2) + 3 * stride, q0q1, 1);
|
||||
}
|
||||
|
||||
void aom_lpf_horizontal_14_neon(uint8_t *src, int stride, const uint8_t *blimit,
|
||||
const uint8_t *limit, const uint8_t *thresh) {
|
||||
uint8x8_t p0q0, p1q1, p2q2, p3q3, p4q4, p5q5, UNINITIALIZED_IS_SAFE(p6q6);
|
||||
|
||||
load_u8_4x1(src - 7 * stride, &p6q6, 0);
|
||||
load_u8_4x1(src - 6 * stride, &p5q5, 0);
|
||||
load_u8_4x1(src - 5 * stride, &p4q4, 0);
|
||||
load_u8_4x1(src - 4 * stride, &p3q3, 0);
|
||||
load_u8_4x1(src - 3 * stride, &p2q2, 0);
|
||||
load_u8_4x1(src - 2 * stride, &p1q1, 0);
|
||||
load_u8_4x1(src - 1 * stride, &p0q0, 0);
|
||||
load_u8_4x1(src + 0 * stride, &p0q0, 1);
|
||||
load_u8_4x1(src + 1 * stride, &p1q1, 1);
|
||||
load_u8_4x1(src + 2 * stride, &p2q2, 1);
|
||||
load_u8_4x1(src + 3 * stride, &p3q3, 1);
|
||||
load_u8_4x1(src + 4 * stride, &p4q4, 1);
|
||||
load_u8_4x1(src + 5 * stride, &p5q5, 1);
|
||||
load_u8_4x1(src + 6 * stride, &p6q6, 1);
|
||||
|
||||
lpf_14_neon(&p6q6, &p5q5, &p4q4, &p3q3, &p2q2, &p1q1, &p0q0, *blimit, *limit,
|
||||
*thresh);
|
||||
|
||||
store_u8_4x1(src - 6 * stride, p5q5, 0);
|
||||
store_u8_4x1(src - 5 * stride, p4q4, 0);
|
||||
store_u8_4x1(src - 4 * stride, p3q3, 0);
|
||||
store_u8_4x1(src - 3 * stride, p2q2, 0);
|
||||
store_u8_4x1(src - 2 * stride, p1q1, 0);
|
||||
store_u8_4x1(src - 1 * stride, p0q0, 0);
|
||||
store_u8_4x1(src + 0 * stride, p0q0, 1);
|
||||
store_u8_4x1(src + 1 * stride, p1q1, 1);
|
||||
store_u8_4x1(src + 2 * stride, p2q2, 1);
|
||||
store_u8_4x1(src + 3 * stride, p3q3, 1);
|
||||
store_u8_4x1(src + 4 * stride, p4q4, 1);
|
||||
store_u8_4x1(src + 5 * stride, p5q5, 1);
|
||||
}
|
||||
|
||||
void aom_lpf_horizontal_8_neon(uint8_t *src, int stride, const uint8_t *blimit,
|
||||
const uint8_t *limit, const uint8_t *thresh) {
|
||||
uint8x8_t p0q0, p1q1, p2q2, p3q3;
|
||||
|
|
@ -698,3 +909,20 @@ void aom_lpf_horizontal_6_neon(uint8_t *src, int stride, const uint8_t *blimit,
|
|||
vst1_lane_u32((uint32_t *)(src + 1 * stride), vreinterpret_u32_u8(p1q1), 1);
|
||||
vst1_lane_u32((uint32_t *)(src + 2 * stride), vreinterpret_u32_u8(p2q2), 1);
|
||||
}
|
||||
|
||||
void aom_lpf_horizontal_4_neon(uint8_t *src, int stride, const uint8_t *blimit,
|
||||
const uint8_t *limit, const uint8_t *thresh) {
|
||||
uint8x8_t p0q0, UNINITIALIZED_IS_SAFE(p1q1);
|
||||
|
||||
load_u8_4x1(src - 2 * stride, &p1q1, 0);
|
||||
load_u8_4x1(src - 1 * stride, &p0q0, 0);
|
||||
load_u8_4x1(src + 0 * stride, &p0q0, 1);
|
||||
load_u8_4x1(src + 1 * stride, &p1q1, 1);
|
||||
|
||||
lpf_4_neon(&p1q1, &p0q0, *blimit, *limit, *thresh);
|
||||
|
||||
store_u8_4x1(src - 2 * stride, p1q1, 0);
|
||||
store_u8_4x1(src - 1 * stride, p0q0, 0);
|
||||
store_u8_4x1(src + 0 * stride, p0q0, 1);
|
||||
store_u8_4x1(src + 1 * stride, p1q1, 1);
|
||||
}
|
||||
|
|
|
|||
7
third_party/aom/aom_dsp/bitreader_buffer.c
vendored
7
third_party/aom/aom_dsp/bitreader_buffer.c
vendored
|
|
@ -8,11 +8,14 @@
|
|||
* Media Patent License 1.0 was not distributed with this source code in the
|
||||
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
|
||||
*/
|
||||
|
||||
#include <assert.h>
|
||||
|
||||
#include "config/aom_config.h"
|
||||
|
||||
#include "aom_dsp/bitreader_buffer.h"
|
||||
|
||||
size_t aom_rb_bytes_read(struct aom_read_bit_buffer *rb) {
|
||||
size_t aom_rb_bytes_read(const struct aom_read_bit_buffer *rb) {
|
||||
return (rb->bit_offset + 7) >> 3;
|
||||
}
|
||||
|
||||
|
|
@ -31,6 +34,7 @@ int aom_rb_read_bit(struct aom_read_bit_buffer *rb) {
|
|||
}
|
||||
|
||||
int aom_rb_read_literal(struct aom_read_bit_buffer *rb, int bits) {
|
||||
assert(bits <= 31);
|
||||
int value = 0, bit;
|
||||
for (bit = bits - 1; bit >= 0; bit--) value |= aom_rb_read_bit(rb) << bit;
|
||||
return value;
|
||||
|
|
@ -38,6 +42,7 @@ int aom_rb_read_literal(struct aom_read_bit_buffer *rb, int bits) {
|
|||
|
||||
uint32_t aom_rb_read_unsigned_literal(struct aom_read_bit_buffer *rb,
|
||||
int bits) {
|
||||
assert(bits <= 32);
|
||||
uint32_t value = 0;
|
||||
int bit;
|
||||
for (bit = bits - 1; bit >= 0; bit--)
|
||||
|
|
|
|||
2
third_party/aom/aom_dsp/bitreader_buffer.h
vendored
2
third_party/aom/aom_dsp/bitreader_buffer.h
vendored
|
|
@ -31,7 +31,7 @@ struct aom_read_bit_buffer {
|
|||
aom_rb_error_handler error_handler;
|
||||
};
|
||||
|
||||
size_t aom_rb_bytes_read(struct aom_read_bit_buffer *rb);
|
||||
size_t aom_rb_bytes_read(const struct aom_read_bit_buffer *rb);
|
||||
|
||||
int aom_rb_read_bit(struct aom_read_bit_buffer *rb);
|
||||
|
||||
|
|
|
|||
3
third_party/aom/aom_dsp/bitwriter_buffer.c
vendored
3
third_party/aom/aom_dsp/bitwriter_buffer.c
vendored
|
|
@ -9,6 +9,7 @@
|
|||
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
|
||||
*/
|
||||
|
||||
#include <assert.h>
|
||||
#include <limits.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
|
|
@ -49,12 +50,14 @@ void aom_wb_overwrite_bit(struct aom_write_bit_buffer *wb, int bit) {
|
|||
}
|
||||
|
||||
void aom_wb_write_literal(struct aom_write_bit_buffer *wb, int data, int bits) {
|
||||
assert(bits <= 31);
|
||||
int bit;
|
||||
for (bit = bits - 1; bit >= 0; bit--) aom_wb_write_bit(wb, (data >> bit) & 1);
|
||||
}
|
||||
|
||||
void aom_wb_write_unsigned_literal(struct aom_write_bit_buffer *wb,
|
||||
uint32_t data, int bits) {
|
||||
assert(bits <= 32);
|
||||
int bit;
|
||||
for (bit = bits - 1; bit >= 0; bit--) aom_wb_write_bit(wb, (data >> bit) & 1);
|
||||
}
|
||||
|
|
|
|||
40
third_party/aom/aom_dsp/grain_synthesis.c
vendored
40
third_party/aom/aom_dsp/grain_synthesis.c
vendored
|
|
@ -17,6 +17,7 @@
|
|||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
#include <stdlib.h>
|
||||
#include <assert.h>
|
||||
#include "aom_dsp/grain_synthesis.h"
|
||||
#include "aom_mem/aom_mem.h"
|
||||
|
||||
|
|
@ -237,7 +238,7 @@ static int grain_max;
|
|||
|
||||
static uint16_t random_register = 0; // random number generator register
|
||||
|
||||
static void init_arrays(aom_film_grain_t *params, int luma_stride,
|
||||
static void init_arrays(const aom_film_grain_t *params, int luma_stride,
|
||||
int chroma_stride, int ***pred_pos_luma_p,
|
||||
int ***pred_pos_chroma_p, int **luma_grain_block,
|
||||
int **cb_grain_block, int **cr_grain_block,
|
||||
|
|
@ -331,7 +332,7 @@ static void init_arrays(aom_film_grain_t *params, int luma_stride,
|
|||
(int *)aom_malloc(sizeof(**cr_grain_block) * chroma_grain_samples);
|
||||
}
|
||||
|
||||
static void dealloc_arrays(aom_film_grain_t *params, int ***pred_pos_luma,
|
||||
static void dealloc_arrays(const aom_film_grain_t *params, int ***pred_pos_luma,
|
||||
int ***pred_pos_chroma, int **luma_grain_block,
|
||||
int **cb_grain_block, int **cr_grain_block,
|
||||
int **y_line_buf, int **cb_line_buf,
|
||||
|
|
@ -396,10 +397,14 @@ static void init_random_generator(int luma_line, uint16_t seed) {
|
|||
}
|
||||
|
||||
static void generate_luma_grain_block(
|
||||
aom_film_grain_t *params, int **pred_pos_luma, int *luma_grain_block,
|
||||
const aom_film_grain_t *params, int **pred_pos_luma, int *luma_grain_block,
|
||||
int luma_block_size_y, int luma_block_size_x, int luma_grain_stride,
|
||||
int left_pad, int top_pad, int right_pad, int bottom_pad) {
|
||||
if (params->num_y_points == 0) return;
|
||||
if (params->num_y_points == 0) {
|
||||
memset(luma_grain_block, 0,
|
||||
sizeof(*luma_grain_block) * luma_block_size_y * luma_grain_stride);
|
||||
return;
|
||||
}
|
||||
|
||||
int bit_depth = params->bit_depth;
|
||||
int gauss_sec_shift = 12 - bit_depth + params->grain_scale_shift;
|
||||
|
|
@ -431,7 +436,7 @@ static void generate_luma_grain_block(
|
|||
}
|
||||
|
||||
static void generate_chroma_grain_blocks(
|
||||
aom_film_grain_t *params,
|
||||
const aom_film_grain_t *params,
|
||||
// int** pred_pos_luma,
|
||||
int **pred_pos_chroma, int *luma_grain_block, int *cb_grain_block,
|
||||
int *cr_grain_block, int luma_grain_stride, int chroma_block_size_y,
|
||||
|
|
@ -443,7 +448,7 @@ static void generate_chroma_grain_blocks(
|
|||
int num_pos_chroma = 2 * params->ar_coeff_lag * (params->ar_coeff_lag + 1);
|
||||
if (params->num_y_points > 0) ++num_pos_chroma;
|
||||
int rounding_offset = (1 << (params->ar_coeff_shift - 1));
|
||||
int chroma_grain_samples = chroma_block_size_y * chroma_block_size_x;
|
||||
int chroma_grain_block_size = chroma_block_size_y * chroma_grain_stride;
|
||||
|
||||
if (params->num_cb_points || params->chroma_scaling_from_luma) {
|
||||
init_random_generator(7 << 5, params->random_seed);
|
||||
|
|
@ -455,7 +460,8 @@ static void generate_chroma_grain_blocks(
|
|||
((1 << gauss_sec_shift) >> 1)) >>
|
||||
gauss_sec_shift;
|
||||
} else {
|
||||
memset(cr_grain_block, 0, sizeof(*cr_grain_block) * chroma_grain_samples);
|
||||
memset(cb_grain_block, 0,
|
||||
sizeof(*cb_grain_block) * chroma_grain_block_size);
|
||||
}
|
||||
|
||||
if (params->num_cr_points || params->chroma_scaling_from_luma) {
|
||||
|
|
@ -468,7 +474,8 @@ static void generate_chroma_grain_blocks(
|
|||
((1 << gauss_sec_shift) >> 1)) >>
|
||||
gauss_sec_shift;
|
||||
} else {
|
||||
memset(cb_grain_block, 0, sizeof(*cb_grain_block) * chroma_grain_samples);
|
||||
memset(cr_grain_block, 0,
|
||||
sizeof(*cr_grain_block) * chroma_grain_block_size);
|
||||
}
|
||||
|
||||
for (int i = top_pad; i < chroma_block_size_y - bottom_pad; i++)
|
||||
|
|
@ -522,7 +529,7 @@ static void generate_chroma_grain_blocks(
|
|||
}
|
||||
}
|
||||
|
||||
static void init_scaling_function(int scaling_points[][2], int num_points,
|
||||
static void init_scaling_function(const int scaling_points[][2], int num_points,
|
||||
int scaling_lut[]) {
|
||||
if (num_points == 0) return;
|
||||
|
||||
|
|
@ -559,7 +566,7 @@ static int scale_LUT(int *scaling_lut, int index, int bit_depth) {
|
|||
(bit_depth - 8));
|
||||
}
|
||||
|
||||
static void add_noise_to_block(aom_film_grain_t *params, uint8_t *luma,
|
||||
static void add_noise_to_block(const aom_film_grain_t *params, uint8_t *luma,
|
||||
uint8_t *cb, uint8_t *cr, int luma_stride,
|
||||
int chroma_stride, int *luma_grain,
|
||||
int *cb_grain, int *cr_grain,
|
||||
|
|
@ -675,7 +682,7 @@ static void add_noise_to_block(aom_film_grain_t *params, uint8_t *luma,
|
|||
}
|
||||
|
||||
static void add_noise_to_block_hbd(
|
||||
aom_film_grain_t *params, uint16_t *luma, uint16_t *cb, uint16_t *cr,
|
||||
const aom_film_grain_t *params, uint16_t *luma, uint16_t *cb, uint16_t *cr,
|
||||
int luma_stride, int chroma_stride, int *luma_grain, int *cb_grain,
|
||||
int *cr_grain, int luma_grain_stride, int chroma_grain_stride,
|
||||
int half_luma_height, int half_luma_width, int bit_depth,
|
||||
|
|
@ -903,7 +910,7 @@ static void hor_boundary_overlap(int *top_block, int top_stride,
|
|||
}
|
||||
}
|
||||
|
||||
void av1_add_film_grain(aom_film_grain_t *params, aom_image_t *src,
|
||||
void av1_add_film_grain(const aom_film_grain_t *params, const aom_image_t *src,
|
||||
aom_image_t *dst) {
|
||||
uint8_t *luma, *cb, *cr;
|
||||
int height, width, luma_stride, chroma_stride;
|
||||
|
|
@ -950,6 +957,11 @@ void av1_add_film_grain(aom_film_grain_t *params, aom_image_t *src,
|
|||
exit(1);
|
||||
}
|
||||
|
||||
assert(params->bit_depth == src->bit_depth);
|
||||
|
||||
dst->fmt = src->fmt;
|
||||
dst->bit_depth = src->bit_depth;
|
||||
|
||||
dst->r_w = src->r_w;
|
||||
dst->r_h = src->r_h;
|
||||
dst->d_w = src->d_w;
|
||||
|
|
@ -999,15 +1011,13 @@ void av1_add_film_grain(aom_film_grain_t *params, aom_image_t *src,
|
|||
luma_stride = dst->stride[AOM_PLANE_Y] >> use_high_bit_depth;
|
||||
chroma_stride = dst->stride[AOM_PLANE_U] >> use_high_bit_depth;
|
||||
|
||||
params->bit_depth = dst->bit_depth;
|
||||
|
||||
av1_add_film_grain_run(params, luma, cb, cr, height, width, luma_stride,
|
||||
chroma_stride, use_high_bit_depth, chroma_subsamp_y,
|
||||
chroma_subsamp_x, mc_identity);
|
||||
return;
|
||||
}
|
||||
|
||||
void av1_add_film_grain_run(aom_film_grain_t *params, uint8_t *luma,
|
||||
void av1_add_film_grain_run(const aom_film_grain_t *params, uint8_t *luma,
|
||||
uint8_t *cb, uint8_t *cr, int height, int width,
|
||||
int luma_stride, int chroma_stride,
|
||||
int use_high_bit_depth, int chroma_subsamp_y,
|
||||
|
|
|
|||
10
third_party/aom/aom_dsp/grain_synthesis.h
vendored
10
third_party/aom/aom_dsp/grain_synthesis.h
vendored
|
|
@ -72,7 +72,7 @@ typedef struct {
|
|||
|
||||
int clip_to_restricted_range;
|
||||
|
||||
int bit_depth; // video bit depth
|
||||
unsigned int bit_depth; // video bit depth
|
||||
|
||||
int chroma_scaling_from_luma;
|
||||
|
||||
|
|
@ -94,7 +94,7 @@ typedef struct {
|
|||
* \param[in] luma_stride luma plane stride
|
||||
* \param[in] chroma_stride chroma plane stride
|
||||
*/
|
||||
void av1_add_film_grain_run(aom_film_grain_t *grain_params, uint8_t *luma,
|
||||
void av1_add_film_grain_run(const aom_film_grain_t *grain_params, uint8_t *luma,
|
||||
uint8_t *cb, uint8_t *cr, int height, int width,
|
||||
int luma_stride, int chroma_stride,
|
||||
int use_high_bit_depth, int chroma_subsamp_y,
|
||||
|
|
@ -106,10 +106,10 @@ void av1_add_film_grain_run(aom_film_grain_t *grain_params, uint8_t *luma,
|
|||
*
|
||||
* \param[in] grain_params Grain parameters
|
||||
* \param[in] src Source image
|
||||
* \param[in] dst Resulting image with grain
|
||||
* \param[out] dst Resulting image with grain
|
||||
*/
|
||||
void av1_add_film_grain(aom_film_grain_t *grain_params, aom_image_t *src,
|
||||
aom_image_t *dst);
|
||||
void av1_add_film_grain(const aom_film_grain_t *grain_params,
|
||||
const aom_image_t *src, aom_image_t *dst);
|
||||
|
||||
#ifdef __cplusplus
|
||||
} // extern "C"
|
||||
|
|
|
|||
186
third_party/aom/aom_dsp/noise_model.c
vendored
186
third_party/aom/aom_dsp/noise_model.c
vendored
|
|
@ -1458,3 +1458,189 @@ int aom_wiener_denoise_2d(const uint8_t *const data[3], uint8_t *denoised[3],
|
|||
}
|
||||
return init_success;
|
||||
}
|
||||
|
||||
struct aom_denoise_and_model_t {
|
||||
int block_size;
|
||||
int bit_depth;
|
||||
float noise_level;
|
||||
|
||||
// Size of current denoised buffer and flat_block buffer
|
||||
int width;
|
||||
int height;
|
||||
int y_stride;
|
||||
int uv_stride;
|
||||
int num_blocks_w;
|
||||
int num_blocks_h;
|
||||
|
||||
// Buffers for image and noise_psd allocated on the fly
|
||||
float *noise_psd[3];
|
||||
uint8_t *denoised[3];
|
||||
uint8_t *flat_blocks;
|
||||
|
||||
aom_flat_block_finder_t flat_block_finder;
|
||||
aom_noise_model_t noise_model;
|
||||
};
|
||||
|
||||
struct aom_denoise_and_model_t *aom_denoise_and_model_alloc(int bit_depth,
|
||||
int block_size,
|
||||
float noise_level) {
|
||||
struct aom_denoise_and_model_t *ctx =
|
||||
(struct aom_denoise_and_model_t *)aom_malloc(
|
||||
sizeof(struct aom_denoise_and_model_t));
|
||||
if (!ctx) {
|
||||
fprintf(stderr, "Unable to allocate denoise_and_model struct\n");
|
||||
return NULL;
|
||||
}
|
||||
memset(ctx, 0, sizeof(*ctx));
|
||||
|
||||
ctx->block_size = block_size;
|
||||
ctx->noise_level = noise_level;
|
||||
ctx->bit_depth = bit_depth;
|
||||
|
||||
ctx->noise_psd[0] =
|
||||
aom_malloc(sizeof(*ctx->noise_psd[0]) * block_size * block_size);
|
||||
ctx->noise_psd[1] =
|
||||
aom_malloc(sizeof(*ctx->noise_psd[1]) * block_size * block_size);
|
||||
ctx->noise_psd[2] =
|
||||
aom_malloc(sizeof(*ctx->noise_psd[2]) * block_size * block_size);
|
||||
if (!ctx->noise_psd[0] || !ctx->noise_psd[1] || !ctx->noise_psd[2]) {
|
||||
fprintf(stderr, "Unable to allocate noise PSD buffers\n");
|
||||
aom_denoise_and_model_free(ctx);
|
||||
return NULL;
|
||||
}
|
||||
return ctx;
|
||||
}
|
||||
|
||||
void aom_denoise_and_model_free(struct aom_denoise_and_model_t *ctx) {
|
||||
aom_free(ctx->flat_blocks);
|
||||
for (int i = 0; i < 3; ++i) {
|
||||
aom_free(ctx->denoised[i]);
|
||||
aom_free(ctx->noise_psd[i]);
|
||||
}
|
||||
aom_noise_model_free(&ctx->noise_model);
|
||||
aom_flat_block_finder_free(&ctx->flat_block_finder);
|
||||
aom_free(ctx);
|
||||
}
|
||||
|
||||
static int denoise_and_model_realloc_if_necessary(
|
||||
struct aom_denoise_and_model_t *ctx, YV12_BUFFER_CONFIG *sd) {
|
||||
if (ctx->width == sd->y_width && ctx->height == sd->y_height &&
|
||||
ctx->y_stride == sd->y_stride && ctx->uv_stride == sd->uv_stride)
|
||||
return 1;
|
||||
const int use_highbd = (sd->flags & YV12_FLAG_HIGHBITDEPTH) != 0;
|
||||
const int block_size = ctx->block_size;
|
||||
|
||||
ctx->width = sd->y_width;
|
||||
ctx->height = sd->y_height;
|
||||
ctx->y_stride = sd->y_stride;
|
||||
ctx->uv_stride = sd->uv_stride;
|
||||
|
||||
for (int i = 0; i < 3; ++i) {
|
||||
aom_free(ctx->denoised[i]);
|
||||
ctx->denoised[i] = NULL;
|
||||
}
|
||||
aom_free(ctx->flat_blocks);
|
||||
ctx->flat_blocks = NULL;
|
||||
|
||||
ctx->denoised[0] = aom_malloc((sd->y_stride * sd->y_height) << use_highbd);
|
||||
ctx->denoised[1] = aom_malloc((sd->uv_stride * sd->uv_height) << use_highbd);
|
||||
ctx->denoised[2] = aom_malloc((sd->uv_stride * sd->uv_height) << use_highbd);
|
||||
if (!ctx->denoised[0] || !ctx->denoised[1] || !ctx->denoised[2]) {
|
||||
fprintf(stderr, "Unable to allocate denoise buffers\n");
|
||||
return 0;
|
||||
}
|
||||
ctx->num_blocks_w = (sd->y_width + ctx->block_size - 1) / ctx->block_size;
|
||||
ctx->num_blocks_h = (sd->y_height + ctx->block_size - 1) / ctx->block_size;
|
||||
ctx->flat_blocks = aom_malloc(ctx->num_blocks_w * ctx->num_blocks_h);
|
||||
|
||||
aom_flat_block_finder_free(&ctx->flat_block_finder);
|
||||
if (!aom_flat_block_finder_init(&ctx->flat_block_finder, ctx->block_size,
|
||||
ctx->bit_depth, use_highbd)) {
|
||||
fprintf(stderr, "Unable to init flat block finder\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
const aom_noise_model_params_t params = { AOM_NOISE_SHAPE_SQUARE, 3,
|
||||
ctx->bit_depth, use_highbd };
|
||||
aom_noise_model_free(&ctx->noise_model);
|
||||
if (!aom_noise_model_init(&ctx->noise_model, params)) {
|
||||
fprintf(stderr, "Unable to init noise model\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Simply use a flat PSD (although we could use the flat blocks to estimate
|
||||
// PSD) those to estimate an actual noise PSD)
|
||||
const float y_noise_level =
|
||||
aom_noise_psd_get_default_value(ctx->block_size, ctx->noise_level);
|
||||
const float uv_noise_level = aom_noise_psd_get_default_value(
|
||||
ctx->block_size >> sd->subsampling_x, ctx->noise_level);
|
||||
for (int i = 0; i < block_size * block_size; ++i) {
|
||||
ctx->noise_psd[0][i] = y_noise_level;
|
||||
ctx->noise_psd[1][i] = ctx->noise_psd[2][i] = uv_noise_level;
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
int aom_denoise_and_model_run(struct aom_denoise_and_model_t *ctx,
|
||||
YV12_BUFFER_CONFIG *sd,
|
||||
aom_film_grain_t *film_grain) {
|
||||
const int block_size = ctx->block_size;
|
||||
const int use_highbd = (sd->flags & YV12_FLAG_HIGHBITDEPTH) != 0;
|
||||
uint8_t *raw_data[3] = {
|
||||
use_highbd ? (uint8_t *)CONVERT_TO_SHORTPTR(sd->y_buffer) : sd->y_buffer,
|
||||
use_highbd ? (uint8_t *)CONVERT_TO_SHORTPTR(sd->u_buffer) : sd->u_buffer,
|
||||
use_highbd ? (uint8_t *)CONVERT_TO_SHORTPTR(sd->v_buffer) : sd->v_buffer,
|
||||
};
|
||||
const uint8_t *const data[3] = { raw_data[0], raw_data[1], raw_data[2] };
|
||||
int strides[3] = { sd->y_stride, sd->uv_stride, sd->uv_stride };
|
||||
int chroma_sub_log2[2] = { sd->subsampling_x, sd->subsampling_y };
|
||||
|
||||
if (!denoise_and_model_realloc_if_necessary(ctx, sd)) {
|
||||
fprintf(stderr, "Unable to realloc buffers\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
aom_flat_block_finder_run(&ctx->flat_block_finder, data[0], sd->y_width,
|
||||
sd->y_height, strides[0], ctx->flat_blocks);
|
||||
|
||||
if (!aom_wiener_denoise_2d(data, ctx->denoised, sd->y_width, sd->y_height,
|
||||
strides, chroma_sub_log2, ctx->noise_psd,
|
||||
block_size, ctx->bit_depth, use_highbd)) {
|
||||
fprintf(stderr, "Unable to denoise image\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
const aom_noise_status_t status = aom_noise_model_update(
|
||||
&ctx->noise_model, data, (const uint8_t *const *)ctx->denoised,
|
||||
sd->y_width, sd->y_height, strides, chroma_sub_log2, ctx->flat_blocks,
|
||||
block_size);
|
||||
int have_noise_estimate = 0;
|
||||
if (status == AOM_NOISE_STATUS_OK) {
|
||||
have_noise_estimate = 1;
|
||||
} else if (status == AOM_NOISE_STATUS_DIFFERENT_NOISE_TYPE) {
|
||||
aom_noise_model_save_latest(&ctx->noise_model);
|
||||
have_noise_estimate = 1;
|
||||
} else {
|
||||
// Unable to update noise model; proceed if we have a previous estimate.
|
||||
have_noise_estimate =
|
||||
(ctx->noise_model.combined_state[0].strength_solver.num_equations > 0);
|
||||
}
|
||||
|
||||
film_grain->apply_grain = 0;
|
||||
if (have_noise_estimate) {
|
||||
if (!aom_noise_model_get_grain_parameters(&ctx->noise_model, film_grain)) {
|
||||
fprintf(stderr, "Unable to get grain parameters.\n");
|
||||
return 0;
|
||||
}
|
||||
if (!film_grain->random_seed) {
|
||||
film_grain->random_seed = 1071;
|
||||
}
|
||||
memcpy(raw_data[0], ctx->denoised[0],
|
||||
(strides[0] * sd->y_height) << use_highbd);
|
||||
memcpy(raw_data[1], ctx->denoised[1],
|
||||
(strides[1] * sd->uv_height) << use_highbd);
|
||||
memcpy(raw_data[2], ctx->denoised[2],
|
||||
(strides[2] * sd->uv_height) << use_highbd);
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
|
|
|||
37
third_party/aom/aom_dsp/noise_model.h
vendored
37
third_party/aom/aom_dsp/noise_model.h
vendored
|
|
@ -18,6 +18,7 @@ extern "C" {
|
|||
|
||||
#include <stdint.h>
|
||||
#include "aom_dsp/grain_synthesis.h"
|
||||
#include "aom_scale/yv12config.h"
|
||||
|
||||
/*!\brief Wrapper of data required to represent linear system of eqns and soln.
|
||||
*/
|
||||
|
|
@ -280,6 +281,42 @@ int aom_wiener_denoise_2d(const uint8_t *const data[3], uint8_t *denoised[3],
|
|||
int w, int h, int stride[3], int chroma_sub_log2[2],
|
||||
float *noise_psd[3], int block_size, int bit_depth,
|
||||
int use_highbd);
|
||||
|
||||
struct aom_denoise_and_model_t;
|
||||
|
||||
/*!\brief Denoise the buffer and model the residual noise.
|
||||
*
|
||||
* This is meant to be called sequentially on input frames. The input buffer
|
||||
* is denoised and the residual noise is modelled. The current noise estimate
|
||||
* is populated in film_grain. Returns true on success. The grain.apply_grain
|
||||
* parameter will be true when the input buffer was successfully denoised and
|
||||
* grain was modelled. Returns false on error.
|
||||
*
|
||||
* \param[in] ctx Struct allocated with aom_denoise_and_model_alloc
|
||||
* that holds some buffers for denoising and the current
|
||||
* noise estimate.
|
||||
* \param[in/out] buf The raw input buffer to be denoised.
|
||||
* \param[out] grain Output film grain parameters
|
||||
*/
|
||||
int aom_denoise_and_model_run(struct aom_denoise_and_model_t *ctx,
|
||||
YV12_BUFFER_CONFIG *buf, aom_film_grain_t *grain);
|
||||
|
||||
/*!\brief Allocates a context that can be used for denoising and noise modeling.
|
||||
*
|
||||
* \param[in] bit_depth Bit depth of buffers this will be run on.
|
||||
* \param[in] block_size Block size for noise modeling and flat block
|
||||
* estimation
|
||||
* \param[in] noise_level The noise_level (2.5 for moderate noise, and 5 for
|
||||
* higher levels of noise)
|
||||
*/
|
||||
struct aom_denoise_and_model_t *aom_denoise_and_model_alloc(int bit_depth,
|
||||
int block_size,
|
||||
float noise_level);
|
||||
|
||||
/*!\brief Frees the denoise context allocated with aom_denoise_and_model_alloc
|
||||
*/
|
||||
void aom_denoise_and_model_free(struct aom_denoise_and_model_t *denoise_model);
|
||||
|
||||
#ifdef __cplusplus
|
||||
} // extern "C"
|
||||
#endif // __cplusplus
|
||||
|
|
|
|||
|
|
@ -289,6 +289,15 @@ SIMD_INLINE v256 v256_shr_u32(v256 a, unsigned int c) {
|
|||
SIMD_INLINE v256 v256_shr_s32(v256 a, unsigned int c) {
|
||||
return c_v256_shr_s32(a, c);
|
||||
}
|
||||
SIMD_INLINE v256 v256_shl_64(v256 a, unsigned int c) {
|
||||
return c_v256_shl_64(a, c);
|
||||
}
|
||||
SIMD_INLINE v256 v256_shr_u64(v256 a, unsigned int c) {
|
||||
return c_v256_shr_u64(a, c);
|
||||
}
|
||||
SIMD_INLINE v256 v256_shr_s64(v256 a, unsigned int c) {
|
||||
return c_v256_shr_s64(a, c);
|
||||
}
|
||||
|
||||
SIMD_INLINE v256 v256_shr_n_byte(v256 a, unsigned int n) {
|
||||
return c_v256_shr_n_byte(a, n);
|
||||
|
|
|
|||
39
third_party/aom/aom_dsp/variance.c
vendored
39
third_party/aom/aom_dsp/variance.c
vendored
|
|
@ -386,7 +386,7 @@ void aom_upsampled_pred_c(MACROBLOCKD *xd, const AV1_COMMON *const cm,
|
|||
}
|
||||
}
|
||||
|
||||
const InterpFilterParams filter =
|
||||
const InterpFilterParams *filter =
|
||||
av1_get_interp_filter_params_with_block_size(EIGHTTAP_REGULAR, 8);
|
||||
|
||||
if (!subpel_x_q3 && !subpel_y_q3) {
|
||||
|
|
@ -413,12 +413,12 @@ void aom_upsampled_pred_c(MACROBLOCKD *xd, const AV1_COMMON *const cm,
|
|||
const int16_t *const kernel_y =
|
||||
av1_get_interp_filter_subpel_kernel(filter, subpel_y_q3 << 1);
|
||||
const int intermediate_height =
|
||||
(((height - 1) * 8 + subpel_y_q3) >> 3) + filter.taps;
|
||||
(((height - 1) * 8 + subpel_y_q3) >> 3) + filter->taps;
|
||||
assert(intermediate_height <= (MAX_SB_SIZE * 2 + 16) + 16);
|
||||
aom_convolve8_horiz(ref - ref_stride * ((filter.taps >> 1) - 1), ref_stride,
|
||||
temp, MAX_SB_SIZE, kernel_x, 16, NULL, -1, width,
|
||||
intermediate_height);
|
||||
aom_convolve8_vert(temp + MAX_SB_SIZE * ((filter.taps >> 1) - 1),
|
||||
aom_convolve8_horiz(ref - ref_stride * ((filter->taps >> 1) - 1),
|
||||
ref_stride, temp, MAX_SB_SIZE, kernel_x, 16, NULL, -1,
|
||||
width, intermediate_height);
|
||||
aom_convolve8_vert(temp + MAX_SB_SIZE * ((filter->taps >> 1) - 1),
|
||||
MAX_SB_SIZE, comp_pred, width, NULL, -1, kernel_y, 16,
|
||||
width, height);
|
||||
}
|
||||
|
|
@ -974,7 +974,7 @@ void aom_highbd_upsampled_pred_c(MACROBLOCKD *xd,
|
|||
}
|
||||
}
|
||||
|
||||
const InterpFilterParams filter =
|
||||
const InterpFilterParams *filter =
|
||||
av1_get_interp_filter_params_with_block_size(EIGHTTAP_REGULAR, 8);
|
||||
|
||||
if (!subpel_x_q3 && !subpel_y_q3) {
|
||||
|
|
@ -1004,14 +1004,14 @@ void aom_highbd_upsampled_pred_c(MACROBLOCKD *xd,
|
|||
const int16_t *const kernel_y =
|
||||
av1_get_interp_filter_subpel_kernel(filter, subpel_y_q3 << 1);
|
||||
const int intermediate_height =
|
||||
(((height - 1) * 8 + subpel_y_q3) >> 3) + filter.taps;
|
||||
(((height - 1) * 8 + subpel_y_q3) >> 3) + filter->taps;
|
||||
assert(intermediate_height <= (MAX_SB_SIZE * 2 + 16) + 16);
|
||||
aom_highbd_convolve8_horiz(ref8 - ref_stride * ((filter.taps >> 1) - 1),
|
||||
aom_highbd_convolve8_horiz(ref8 - ref_stride * ((filter->taps >> 1) - 1),
|
||||
ref_stride, CONVERT_TO_BYTEPTR(temp),
|
||||
MAX_SB_SIZE, kernel_x, 16, NULL, -1, width,
|
||||
intermediate_height, bd);
|
||||
aom_highbd_convolve8_vert(
|
||||
CONVERT_TO_BYTEPTR(temp + MAX_SB_SIZE * ((filter.taps >> 1) - 1)),
|
||||
CONVERT_TO_BYTEPTR(temp + MAX_SB_SIZE * ((filter->taps >> 1) - 1)),
|
||||
MAX_SB_SIZE, CONVERT_TO_BYTEPTR(comp_pred), width, NULL, -1, kernel_y,
|
||||
16, width, height, bd);
|
||||
}
|
||||
|
|
@ -1185,29 +1185,18 @@ void aom_highbd_comp_mask_pred_c(uint16_t *comp_pred, const uint8_t *pred8,
|
|||
}
|
||||
}
|
||||
|
||||
void aom_highbd_comp_mask_upsampled_pred_c(
|
||||
void aom_highbd_comp_mask_upsampled_pred(
|
||||
MACROBLOCKD *xd, const struct AV1Common *const cm, int mi_row, int mi_col,
|
||||
const MV *const mv, uint16_t *comp_pred, const uint8_t *pred8, int width,
|
||||
int height, int subpel_x_q3, int subpel_y_q3, const uint8_t *ref8,
|
||||
int ref_stride, const uint8_t *mask, int mask_stride, int invert_mask,
|
||||
int bd) {
|
||||
int i, j;
|
||||
|
||||
uint16_t *pred = CONVERT_TO_SHORTPTR(pred8);
|
||||
aom_highbd_upsampled_pred(xd, cm, mi_row, mi_col, mv, comp_pred, width,
|
||||
height, subpel_x_q3, subpel_y_q3, ref8, ref_stride,
|
||||
bd);
|
||||
for (i = 0; i < height; ++i) {
|
||||
for (j = 0; j < width; ++j) {
|
||||
if (!invert_mask)
|
||||
comp_pred[j] = AOM_BLEND_A64(mask[j], comp_pred[j], pred[j]);
|
||||
else
|
||||
comp_pred[j] = AOM_BLEND_A64(mask[j], pred[j], comp_pred[j]);
|
||||
}
|
||||
comp_pred += width;
|
||||
pred += width;
|
||||
mask += mask_stride;
|
||||
}
|
||||
aom_highbd_comp_mask_pred(comp_pred, pred8, width, height,
|
||||
CONVERT_TO_BYTEPTR(comp_pred), width, mask,
|
||||
mask_stride, invert_mask);
|
||||
}
|
||||
|
||||
#define HIGHBD_MASK_SUBPIX_VAR(W, H) \
|
||||
|
|
|
|||
7
third_party/aom/aom_dsp/variance.h
vendored
7
third_party/aom/aom_dsp/variance.h
vendored
|
|
@ -76,6 +76,13 @@ void aom_comp_mask_upsampled_pred(
|
|||
int height, int subpel_x_q3, int subpel_y_q3, const uint8_t *ref,
|
||||
int ref_stride, const uint8_t *mask, int mask_stride, int invert_mask);
|
||||
|
||||
void aom_highbd_comp_mask_upsampled_pred(
|
||||
MACROBLOCKD *xd, const struct AV1Common *const cm, int mi_row, int mi_col,
|
||||
const MV *const mv, uint16_t *comp_pred, const uint8_t *pred8, int width,
|
||||
int height, int subpel_x_q3, int subpel_y_q3, const uint8_t *ref8,
|
||||
int ref_stride, const uint8_t *mask, int mask_stride, int invert_mask,
|
||||
int bd);
|
||||
|
||||
typedef unsigned int (*aom_obmc_sad_fn_t)(const uint8_t *pred, int pred_stride,
|
||||
const int32_t *wsrc,
|
||||
const int32_t *msk);
|
||||
|
|
|
|||
|
|
@ -41,25 +41,163 @@
|
|||
#define MM256_BROADCASTSI128_SI256(x) _mm256_broadcastsi128_si256(x)
|
||||
#endif // __clang__
|
||||
|
||||
static void aom_filter_block1d16_h8_avx2(
|
||||
static INLINE void xx_storeu2_epi32(const uint8_t *output_ptr,
|
||||
const ptrdiff_t stride, const __m256i *a) {
|
||||
*((uint32_t *)(output_ptr)) = _mm_cvtsi128_si32(_mm256_castsi256_si128(*a));
|
||||
*((uint32_t *)(output_ptr + stride)) =
|
||||
_mm_cvtsi128_si32(_mm256_extracti128_si256(*a, 1));
|
||||
}
|
||||
|
||||
static INLINE __m256i xx_loadu2_epi64(const void *hi, const void *lo) {
|
||||
__m256i a = _mm256_castsi128_si256(_mm_loadl_epi64((const __m128i *)(lo)));
|
||||
a = _mm256_inserti128_si256(a, _mm_loadl_epi64((const __m128i *)(hi)), 1);
|
||||
return a;
|
||||
}
|
||||
|
||||
static INLINE void xx_storeu2_epi64(const uint8_t *output_ptr,
|
||||
const ptrdiff_t stride, const __m256i *a) {
|
||||
_mm_storel_epi64((__m128i *)output_ptr, _mm256_castsi256_si128(*a));
|
||||
_mm_storel_epi64((__m128i *)(output_ptr + stride),
|
||||
_mm256_extractf128_si256(*a, 1));
|
||||
}
|
||||
|
||||
static INLINE __m256i xx_loadu2_mi128(const void *hi, const void *lo) {
|
||||
__m256i a = _mm256_castsi128_si256(_mm_loadu_si128((const __m128i *)(lo)));
|
||||
a = _mm256_inserti128_si256(a, _mm_loadu_si128((const __m128i *)(hi)), 1);
|
||||
return a;
|
||||
}
|
||||
|
||||
static INLINE void xx_store2_mi128(const uint8_t *output_ptr,
|
||||
const ptrdiff_t stride, const __m256i *a) {
|
||||
_mm_store_si128((__m128i *)output_ptr, _mm256_castsi256_si128(*a));
|
||||
_mm_store_si128((__m128i *)(output_ptr + stride),
|
||||
_mm256_extractf128_si256(*a, 1));
|
||||
}
|
||||
|
||||
static void aom_filter_block1d4_h8_avx2(
|
||||
const uint8_t *src_ptr, ptrdiff_t src_pixels_per_line, uint8_t *output_ptr,
|
||||
ptrdiff_t output_pitch, uint32_t output_height, const int16_t *filter) {
|
||||
__m128i filtersReg;
|
||||
__m256i addFilterReg64, filt1Reg, filt2Reg, filt3Reg, filt4Reg;
|
||||
__m256i firstFilters, secondFilters, thirdFilters, forthFilters;
|
||||
__m256i srcRegFilt32b1_1, srcRegFilt32b2_1, srcRegFilt32b2, srcRegFilt32b3;
|
||||
__m256i srcReg32b1, srcReg32b2, filtersReg32;
|
||||
__m256i addFilterReg32, filt1Reg, filt2Reg;
|
||||
__m256i firstFilters, secondFilters;
|
||||
__m256i srcRegFilt32b1_1, srcRegFilt32b2;
|
||||
__m256i srcReg32b1;
|
||||
unsigned int i;
|
||||
ptrdiff_t src_stride, dst_stride;
|
||||
|
||||
// create a register with 0,64,0,64,0,64,0,64,0,64,0,64,0,64,0,64
|
||||
addFilterReg64 = _mm256_set1_epi32((int)0x0400040u);
|
||||
src_ptr -= 3;
|
||||
addFilterReg32 = _mm256_set1_epi16(32);
|
||||
filtersReg = _mm_loadu_si128((const __m128i *)filter);
|
||||
filtersReg = _mm_srai_epi16(filtersReg, 1);
|
||||
// converting the 16 bit (short) to 8 bit (byte) and have the same data
|
||||
// in both lanes of 128 bit register.
|
||||
filtersReg = _mm_packs_epi16(filtersReg, filtersReg);
|
||||
// have the same data in both lanes of a 256 bit register
|
||||
filtersReg32 = MM256_BROADCASTSI128_SI256(filtersReg);
|
||||
const __m256i filtersReg32 = MM256_BROADCASTSI128_SI256(filtersReg);
|
||||
|
||||
// duplicate only the first 32 bits
|
||||
firstFilters = _mm256_shuffle_epi32(filtersReg32, 0);
|
||||
// duplicate only the second 32 bits
|
||||
secondFilters = _mm256_shuffle_epi32(filtersReg32, 0x55);
|
||||
|
||||
filt1Reg = _mm256_load_si256((__m256i const *)filt_d4_global_avx2);
|
||||
filt2Reg = _mm256_load_si256((__m256i const *)(filt_d4_global_avx2 + 32));
|
||||
|
||||
// multiple the size of the source and destination stride by two
|
||||
src_stride = src_pixels_per_line << 1;
|
||||
dst_stride = output_pitch << 1;
|
||||
for (i = output_height; i > 1; i -= 2) {
|
||||
// load the 2 strides of source
|
||||
srcReg32b1 = xx_loadu2_mi128(src_ptr + src_pixels_per_line, src_ptr);
|
||||
|
||||
// filter the source buffer
|
||||
srcRegFilt32b1_1 = _mm256_shuffle_epi8(srcReg32b1, filt1Reg);
|
||||
|
||||
// multiply 4 adjacent elements with the filter and add the result
|
||||
srcRegFilt32b1_1 = _mm256_maddubs_epi16(srcRegFilt32b1_1, firstFilters);
|
||||
|
||||
// filter the source buffer
|
||||
srcRegFilt32b2 = _mm256_shuffle_epi8(srcReg32b1, filt2Reg);
|
||||
|
||||
// multiply 4 adjacent elements with the filter and add the result
|
||||
srcRegFilt32b2 = _mm256_maddubs_epi16(srcRegFilt32b2, secondFilters);
|
||||
|
||||
srcRegFilt32b1_1 = _mm256_adds_epi16(srcRegFilt32b1_1, srcRegFilt32b2);
|
||||
|
||||
srcRegFilt32b1_1 =
|
||||
_mm256_hadds_epi16(srcRegFilt32b1_1, _mm256_setzero_si256());
|
||||
|
||||
// shift by 6 bit each 16 bit
|
||||
srcRegFilt32b1_1 = _mm256_adds_epi16(srcRegFilt32b1_1, addFilterReg32);
|
||||
srcRegFilt32b1_1 = _mm256_srai_epi16(srcRegFilt32b1_1, 6);
|
||||
|
||||
// shrink to 8 bit each 16 bits, the first lane contain the first
|
||||
// convolve result and the second lane contain the second convolve result
|
||||
srcRegFilt32b1_1 =
|
||||
_mm256_packus_epi16(srcRegFilt32b1_1, _mm256_setzero_si256());
|
||||
|
||||
src_ptr += src_stride;
|
||||
|
||||
xx_storeu2_epi32(output_ptr, output_pitch, &srcRegFilt32b1_1);
|
||||
output_ptr += dst_stride;
|
||||
}
|
||||
|
||||
// if the number of strides is odd.
|
||||
// process only 4 bytes
|
||||
if (i > 0) {
|
||||
__m128i srcReg1, srcRegFilt1_1;
|
||||
__m128i srcRegFilt2;
|
||||
|
||||
srcReg1 = _mm_loadu_si128((const __m128i *)(src_ptr));
|
||||
|
||||
// filter the source buffer
|
||||
srcRegFilt1_1 = _mm_shuffle_epi8(srcReg1, _mm256_castsi256_si128(filt1Reg));
|
||||
|
||||
// multiply 4 adjacent elements with the filter and add the result
|
||||
srcRegFilt1_1 =
|
||||
_mm_maddubs_epi16(srcRegFilt1_1, _mm256_castsi256_si128(firstFilters));
|
||||
|
||||
// filter the source buffer
|
||||
srcRegFilt2 = _mm_shuffle_epi8(srcReg1, _mm256_castsi256_si128(filt2Reg));
|
||||
|
||||
// multiply 4 adjacent elements with the filter and add the result
|
||||
srcRegFilt2 =
|
||||
_mm_maddubs_epi16(srcRegFilt2, _mm256_castsi256_si128(secondFilters));
|
||||
|
||||
srcRegFilt1_1 = _mm_adds_epi16(srcRegFilt1_1, srcRegFilt2);
|
||||
srcRegFilt1_1 = _mm_hadds_epi16(srcRegFilt1_1, _mm_setzero_si128());
|
||||
// shift by 6 bit each 16 bit
|
||||
srcRegFilt1_1 =
|
||||
_mm_adds_epi16(srcRegFilt1_1, _mm256_castsi256_si128(addFilterReg32));
|
||||
srcRegFilt1_1 = _mm_srai_epi16(srcRegFilt1_1, 6);
|
||||
|
||||
// shrink to 8 bit each 16 bits, the first lane contain the first
|
||||
// convolve result and the second lane contain the second convolve result
|
||||
srcRegFilt1_1 = _mm_packus_epi16(srcRegFilt1_1, _mm_setzero_si128());
|
||||
|
||||
// save 4 bytes
|
||||
*((uint32_t *)(output_ptr)) = _mm_cvtsi128_si32(srcRegFilt1_1);
|
||||
}
|
||||
}
|
||||
|
||||
static void aom_filter_block1d8_h8_avx2(
|
||||
const uint8_t *src_ptr, ptrdiff_t src_pixels_per_line, uint8_t *output_ptr,
|
||||
ptrdiff_t output_pitch, uint32_t output_height, const int16_t *filter) {
|
||||
__m128i filtersReg;
|
||||
__m256i addFilterReg32, filt1Reg, filt2Reg, filt3Reg, filt4Reg;
|
||||
__m256i firstFilters, secondFilters, thirdFilters, forthFilters;
|
||||
__m256i srcRegFilt32b1_1, srcRegFilt32b2, srcRegFilt32b3;
|
||||
__m256i srcReg32b1;
|
||||
unsigned int i;
|
||||
ptrdiff_t src_stride, dst_stride;
|
||||
src_ptr -= 3;
|
||||
addFilterReg32 = _mm256_set1_epi16(32);
|
||||
filtersReg = _mm_loadu_si128((const __m128i *)filter);
|
||||
filtersReg = _mm_srai_epi16(filtersReg, 1);
|
||||
// converting the 16 bit (short) to 8 bit (byte) and have the same data
|
||||
// in both lanes of 128 bit register.
|
||||
filtersReg = _mm_packs_epi16(filtersReg, filtersReg);
|
||||
// have the same data in both lanes of a 256 bit register
|
||||
const __m256i filtersReg32 = MM256_BROADCASTSI128_SI256(filtersReg);
|
||||
|
||||
// duplicate only the first 16 bits (first and second byte)
|
||||
// across 256 bit register
|
||||
|
|
@ -74,22 +212,17 @@ static void aom_filter_block1d16_h8_avx2(
|
|||
// across 256 bit register
|
||||
forthFilters = _mm256_shuffle_epi8(filtersReg32, _mm256_set1_epi16(0x706u));
|
||||
|
||||
filt1Reg = _mm256_load_si256((__m256i const *)filt1_global_avx2);
|
||||
filt2Reg = _mm256_load_si256((__m256i const *)filt2_global_avx2);
|
||||
filt3Reg = _mm256_load_si256((__m256i const *)filt3_global_avx2);
|
||||
filt4Reg = _mm256_load_si256((__m256i const *)filt4_global_avx2);
|
||||
filt1Reg = _mm256_load_si256((__m256i const *)filt_global_avx2);
|
||||
filt2Reg = _mm256_load_si256((__m256i const *)(filt_global_avx2 + 32));
|
||||
filt3Reg = _mm256_load_si256((__m256i const *)(filt_global_avx2 + 32 * 2));
|
||||
filt4Reg = _mm256_load_si256((__m256i const *)(filt_global_avx2 + 32 * 3));
|
||||
|
||||
// multiple the size of the source and destination stride by two
|
||||
src_stride = src_pixels_per_line << 1;
|
||||
dst_stride = output_pitch << 1;
|
||||
for (i = output_height; i > 1; i -= 2) {
|
||||
// load the 2 strides of source
|
||||
srcReg32b1 =
|
||||
_mm256_castsi128_si256(_mm_loadu_si128((const __m128i *)(src_ptr - 3)));
|
||||
srcReg32b1 = _mm256_inserti128_si256(
|
||||
srcReg32b1,
|
||||
_mm_loadu_si128((const __m128i *)(src_ptr + src_pixels_per_line - 3)),
|
||||
1);
|
||||
srcReg32b1 = xx_loadu2_mi128(src_ptr + src_pixels_per_line, src_ptr);
|
||||
|
||||
// filter the source buffer
|
||||
srcRegFilt32b1_1 = _mm256_shuffle_epi8(srcReg32b1, filt1Reg);
|
||||
|
|
@ -110,80 +243,31 @@ static void aom_filter_block1d16_h8_avx2(
|
|||
srcRegFilt32b3 = _mm256_maddubs_epi16(srcRegFilt32b3, secondFilters);
|
||||
srcRegFilt32b2 = _mm256_maddubs_epi16(srcRegFilt32b2, thirdFilters);
|
||||
|
||||
// add and saturate the results together
|
||||
srcRegFilt32b1_1 = _mm256_adds_epi16(
|
||||
srcRegFilt32b1_1, _mm256_min_epi16(srcRegFilt32b3, srcRegFilt32b2));
|
||||
__m256i sum23 = _mm256_adds_epi16(srcRegFilt32b3, srcRegFilt32b2);
|
||||
srcRegFilt32b1_1 = _mm256_adds_epi16(srcRegFilt32b1_1, sum23);
|
||||
|
||||
// reading 2 strides of the next 16 bytes
|
||||
// (part of it was being read by earlier read)
|
||||
srcReg32b2 =
|
||||
_mm256_castsi128_si256(_mm_loadu_si128((const __m128i *)(src_ptr + 5)));
|
||||
srcReg32b2 = _mm256_inserti128_si256(
|
||||
srcReg32b2,
|
||||
_mm_loadu_si128((const __m128i *)(src_ptr + src_pixels_per_line + 5)),
|
||||
1);
|
||||
|
||||
// add and saturate the results together
|
||||
srcRegFilt32b1_1 = _mm256_adds_epi16(
|
||||
srcRegFilt32b1_1, _mm256_max_epi16(srcRegFilt32b3, srcRegFilt32b2));
|
||||
|
||||
// filter the source buffer
|
||||
srcRegFilt32b2_1 = _mm256_shuffle_epi8(srcReg32b2, filt1Reg);
|
||||
srcRegFilt32b2 = _mm256_shuffle_epi8(srcReg32b2, filt4Reg);
|
||||
|
||||
// multiply 2 adjacent elements with the filter and add the result
|
||||
srcRegFilt32b2_1 = _mm256_maddubs_epi16(srcRegFilt32b2_1, firstFilters);
|
||||
srcRegFilt32b2 = _mm256_maddubs_epi16(srcRegFilt32b2, forthFilters);
|
||||
|
||||
// add and saturate the results together
|
||||
srcRegFilt32b2_1 = _mm256_adds_epi16(srcRegFilt32b2_1, srcRegFilt32b2);
|
||||
|
||||
// filter the source buffer
|
||||
srcRegFilt32b3 = _mm256_shuffle_epi8(srcReg32b2, filt2Reg);
|
||||
srcRegFilt32b2 = _mm256_shuffle_epi8(srcReg32b2, filt3Reg);
|
||||
|
||||
// multiply 2 adjacent elements with the filter and add the result
|
||||
srcRegFilt32b3 = _mm256_maddubs_epi16(srcRegFilt32b3, secondFilters);
|
||||
srcRegFilt32b2 = _mm256_maddubs_epi16(srcRegFilt32b2, thirdFilters);
|
||||
|
||||
// add and saturate the results together
|
||||
srcRegFilt32b2_1 = _mm256_adds_epi16(
|
||||
srcRegFilt32b2_1, _mm256_min_epi16(srcRegFilt32b3, srcRegFilt32b2));
|
||||
srcRegFilt32b2_1 = _mm256_adds_epi16(
|
||||
srcRegFilt32b2_1, _mm256_max_epi16(srcRegFilt32b3, srcRegFilt32b2));
|
||||
|
||||
srcRegFilt32b1_1 = _mm256_adds_epi16(srcRegFilt32b1_1, addFilterReg64);
|
||||
|
||||
srcRegFilt32b2_1 = _mm256_adds_epi16(srcRegFilt32b2_1, addFilterReg64);
|
||||
|
||||
// shift by 7 bit each 16 bit
|
||||
srcRegFilt32b1_1 = _mm256_srai_epi16(srcRegFilt32b1_1, 7);
|
||||
srcRegFilt32b2_1 = _mm256_srai_epi16(srcRegFilt32b2_1, 7);
|
||||
// shift by 6 bit each 16 bit
|
||||
srcRegFilt32b1_1 = _mm256_adds_epi16(srcRegFilt32b1_1, addFilterReg32);
|
||||
srcRegFilt32b1_1 = _mm256_srai_epi16(srcRegFilt32b1_1, 6);
|
||||
|
||||
// shrink to 8 bit each 16 bits, the first lane contain the first
|
||||
// convolve result and the second lane contain the second convolve
|
||||
// result
|
||||
srcRegFilt32b1_1 = _mm256_packus_epi16(srcRegFilt32b1_1, srcRegFilt32b2_1);
|
||||
// convolve result and the second lane contain the second convolve result
|
||||
srcRegFilt32b1_1 =
|
||||
_mm256_packus_epi16(srcRegFilt32b1_1, _mm256_setzero_si256());
|
||||
|
||||
src_ptr += src_stride;
|
||||
|
||||
// save 16 bytes
|
||||
_mm_store_si128((__m128i *)output_ptr,
|
||||
_mm256_castsi256_si128(srcRegFilt32b1_1));
|
||||
|
||||
// save the next 16 bits
|
||||
_mm_store_si128((__m128i *)(output_ptr + output_pitch),
|
||||
_mm256_extractf128_si256(srcRegFilt32b1_1, 1));
|
||||
xx_storeu2_epi64(output_ptr, output_pitch, &srcRegFilt32b1_1);
|
||||
output_ptr += dst_stride;
|
||||
}
|
||||
|
||||
// if the number of strides is odd.
|
||||
// process only 16 bytes
|
||||
// process only 8 bytes
|
||||
if (i > 0) {
|
||||
__m128i srcReg1, srcReg2, srcRegFilt1_1, srcRegFilt2_1;
|
||||
__m128i srcReg1, srcRegFilt1_1;
|
||||
__m128i srcRegFilt2, srcRegFilt3;
|
||||
|
||||
srcReg1 = _mm_loadu_si128((const __m128i *)(src_ptr - 3));
|
||||
srcReg1 = _mm_loadu_si128((const __m128i *)(src_ptr));
|
||||
|
||||
// filter the source buffer
|
||||
srcRegFilt1_1 = _mm_shuffle_epi8(srcReg1, _mm256_castsi256_si128(filt1Reg));
|
||||
|
|
@ -210,15 +294,172 @@ static void aom_filter_block1d16_h8_avx2(
|
|||
|
||||
// add and saturate the results together
|
||||
srcRegFilt1_1 =
|
||||
_mm_adds_epi16(srcRegFilt1_1, _mm_min_epi16(srcRegFilt3, srcRegFilt2));
|
||||
_mm_adds_epi16(srcRegFilt1_1, _mm_adds_epi16(srcRegFilt3, srcRegFilt2));
|
||||
|
||||
// reading the next 16 bytes
|
||||
// shift by 6 bit each 16 bit
|
||||
srcRegFilt1_1 =
|
||||
_mm_adds_epi16(srcRegFilt1_1, _mm256_castsi256_si128(addFilterReg32));
|
||||
srcRegFilt1_1 = _mm_srai_epi16(srcRegFilt1_1, 6);
|
||||
|
||||
// shrink to 8 bit each 16 bits, the first lane contain the first
|
||||
// convolve result and the second lane contain the second convolve
|
||||
// result
|
||||
srcRegFilt1_1 = _mm_packus_epi16(srcRegFilt1_1, _mm_setzero_si128());
|
||||
|
||||
// save 8 bytes
|
||||
_mm_storel_epi64((__m128i *)output_ptr, srcRegFilt1_1);
|
||||
}
|
||||
}
|
||||
|
||||
static void aom_filter_block1d16_h8_avx2(
|
||||
const uint8_t *src_ptr, ptrdiff_t src_pixels_per_line, uint8_t *output_ptr,
|
||||
ptrdiff_t output_pitch, uint32_t output_height, const int16_t *filter) {
|
||||
__m128i filtersReg;
|
||||
__m256i addFilterReg32, filt1Reg, filt2Reg, filt3Reg, filt4Reg;
|
||||
__m256i firstFilters, secondFilters, thirdFilters, forthFilters;
|
||||
__m256i srcRegFilt32b1_1, srcRegFilt32b2_1, srcRegFilt32b2, srcRegFilt32b3;
|
||||
__m256i srcReg32b1, srcReg32b2, filtersReg32;
|
||||
unsigned int i;
|
||||
ptrdiff_t src_stride, dst_stride;
|
||||
src_ptr -= 3;
|
||||
addFilterReg32 = _mm256_set1_epi16(32);
|
||||
filtersReg = _mm_loadu_si128((const __m128i *)filter);
|
||||
filtersReg = _mm_srai_epi16(filtersReg, 1);
|
||||
// converting the 16 bit (short) to 8 bit (byte) and have the same data
|
||||
// in both lanes of 128 bit register.
|
||||
filtersReg = _mm_packs_epi16(filtersReg, filtersReg);
|
||||
// have the same data in both lanes of a 256 bit register
|
||||
filtersReg32 = MM256_BROADCASTSI128_SI256(filtersReg);
|
||||
|
||||
// duplicate only the first 16 bits (first and second byte)
|
||||
// across 256 bit register
|
||||
firstFilters = _mm256_shuffle_epi8(filtersReg32, _mm256_set1_epi16(0x100u));
|
||||
// duplicate only the second 16 bits (third and forth byte)
|
||||
// across 256 bit register
|
||||
secondFilters = _mm256_shuffle_epi8(filtersReg32, _mm256_set1_epi16(0x302u));
|
||||
// duplicate only the third 16 bits (fifth and sixth byte)
|
||||
// across 256 bit register
|
||||
thirdFilters = _mm256_shuffle_epi8(filtersReg32, _mm256_set1_epi16(0x504u));
|
||||
// duplicate only the forth 16 bits (seventh and eighth byte)
|
||||
// across 256 bit register
|
||||
forthFilters = _mm256_shuffle_epi8(filtersReg32, _mm256_set1_epi16(0x706u));
|
||||
|
||||
filt1Reg = _mm256_load_si256((__m256i const *)filt_global_avx2);
|
||||
filt2Reg = _mm256_load_si256((__m256i const *)(filt_global_avx2 + 32));
|
||||
filt3Reg = _mm256_load_si256((__m256i const *)(filt_global_avx2 + 32 * 2));
|
||||
filt4Reg = _mm256_load_si256((__m256i const *)(filt_global_avx2 + 32 * 3));
|
||||
|
||||
// multiple the size of the source and destination stride by two
|
||||
src_stride = src_pixels_per_line << 1;
|
||||
dst_stride = output_pitch << 1;
|
||||
for (i = output_height; i > 1; i -= 2) {
|
||||
// load the 2 strides of source
|
||||
srcReg32b1 = xx_loadu2_mi128(src_ptr + src_pixels_per_line, src_ptr);
|
||||
|
||||
// filter the source buffer
|
||||
srcRegFilt32b1_1 = _mm256_shuffle_epi8(srcReg32b1, filt1Reg);
|
||||
srcRegFilt32b2 = _mm256_shuffle_epi8(srcReg32b1, filt4Reg);
|
||||
|
||||
// multiply 2 adjacent elements with the filter and add the result
|
||||
srcRegFilt32b1_1 = _mm256_maddubs_epi16(srcRegFilt32b1_1, firstFilters);
|
||||
srcRegFilt32b2 = _mm256_maddubs_epi16(srcRegFilt32b2, forthFilters);
|
||||
|
||||
// add and saturate the results together
|
||||
srcRegFilt32b1_1 = _mm256_adds_epi16(srcRegFilt32b1_1, srcRegFilt32b2);
|
||||
|
||||
// filter the source buffer
|
||||
srcRegFilt32b3 = _mm256_shuffle_epi8(srcReg32b1, filt2Reg);
|
||||
srcRegFilt32b2 = _mm256_shuffle_epi8(srcReg32b1, filt3Reg);
|
||||
|
||||
// multiply 2 adjacent elements with the filter and add the result
|
||||
srcRegFilt32b3 = _mm256_maddubs_epi16(srcRegFilt32b3, secondFilters);
|
||||
srcRegFilt32b2 = _mm256_maddubs_epi16(srcRegFilt32b2, thirdFilters);
|
||||
|
||||
__m256i sum23 = _mm256_adds_epi16(srcRegFilt32b3, srcRegFilt32b2);
|
||||
srcRegFilt32b1_1 = _mm256_adds_epi16(srcRegFilt32b1_1, sum23);
|
||||
|
||||
// reading 2 strides of the next 16 bytes
|
||||
// (part of it was being read by earlier read)
|
||||
srcReg2 = _mm_loadu_si128((const __m128i *)(src_ptr + 5));
|
||||
srcReg32b2 =
|
||||
xx_loadu2_mi128(src_ptr + src_pixels_per_line + 8, src_ptr + 8);
|
||||
|
||||
// filter the source buffer
|
||||
srcRegFilt32b2_1 = _mm256_shuffle_epi8(srcReg32b2, filt1Reg);
|
||||
srcRegFilt32b2 = _mm256_shuffle_epi8(srcReg32b2, filt4Reg);
|
||||
|
||||
// multiply 2 adjacent elements with the filter and add the result
|
||||
srcRegFilt32b2_1 = _mm256_maddubs_epi16(srcRegFilt32b2_1, firstFilters);
|
||||
srcRegFilt32b2 = _mm256_maddubs_epi16(srcRegFilt32b2, forthFilters);
|
||||
|
||||
// add and saturate the results together
|
||||
srcRegFilt32b2_1 = _mm256_adds_epi16(srcRegFilt32b2_1, srcRegFilt32b2);
|
||||
|
||||
// filter the source buffer
|
||||
srcRegFilt32b3 = _mm256_shuffle_epi8(srcReg32b2, filt2Reg);
|
||||
srcRegFilt32b2 = _mm256_shuffle_epi8(srcReg32b2, filt3Reg);
|
||||
|
||||
// multiply 2 adjacent elements with the filter and add the result
|
||||
srcRegFilt32b3 = _mm256_maddubs_epi16(srcRegFilt32b3, secondFilters);
|
||||
srcRegFilt32b2 = _mm256_maddubs_epi16(srcRegFilt32b2, thirdFilters);
|
||||
|
||||
// add and saturate the results together
|
||||
srcRegFilt32b2_1 = _mm256_adds_epi16(
|
||||
srcRegFilt32b2_1, _mm256_adds_epi16(srcRegFilt32b3, srcRegFilt32b2));
|
||||
|
||||
// shift by 6 bit each 16 bit
|
||||
srcRegFilt32b1_1 = _mm256_adds_epi16(srcRegFilt32b1_1, addFilterReg32);
|
||||
srcRegFilt32b2_1 = _mm256_adds_epi16(srcRegFilt32b2_1, addFilterReg32);
|
||||
srcRegFilt32b1_1 = _mm256_srai_epi16(srcRegFilt32b1_1, 6);
|
||||
srcRegFilt32b2_1 = _mm256_srai_epi16(srcRegFilt32b2_1, 6);
|
||||
|
||||
// shrink to 8 bit each 16 bits, the first lane contain the first
|
||||
// convolve result and the second lane contain the second convolve result
|
||||
srcRegFilt32b1_1 = _mm256_packus_epi16(srcRegFilt32b1_1, srcRegFilt32b2_1);
|
||||
|
||||
src_ptr += src_stride;
|
||||
|
||||
xx_store2_mi128(output_ptr, output_pitch, &srcRegFilt32b1_1);
|
||||
output_ptr += dst_stride;
|
||||
}
|
||||
|
||||
// if the number of strides is odd.
|
||||
// process only 16 bytes
|
||||
if (i > 0) {
|
||||
__m128i srcReg1, srcReg2, srcRegFilt1_1, srcRegFilt2_1;
|
||||
__m128i srcRegFilt2, srcRegFilt3;
|
||||
|
||||
srcReg1 = _mm_loadu_si128((const __m128i *)(src_ptr));
|
||||
|
||||
// filter the source buffer
|
||||
srcRegFilt1_1 = _mm_shuffle_epi8(srcReg1, _mm256_castsi256_si128(filt1Reg));
|
||||
srcRegFilt2 = _mm_shuffle_epi8(srcReg1, _mm256_castsi256_si128(filt4Reg));
|
||||
|
||||
// multiply 2 adjacent elements with the filter and add the result
|
||||
srcRegFilt1_1 =
|
||||
_mm_maddubs_epi16(srcRegFilt1_1, _mm256_castsi256_si128(firstFilters));
|
||||
srcRegFilt2 =
|
||||
_mm_maddubs_epi16(srcRegFilt2, _mm256_castsi256_si128(forthFilters));
|
||||
|
||||
// add and saturate the results together
|
||||
srcRegFilt1_1 = _mm_adds_epi16(srcRegFilt1_1, srcRegFilt2);
|
||||
|
||||
// filter the source buffer
|
||||
srcRegFilt3 = _mm_shuffle_epi8(srcReg1, _mm256_castsi256_si128(filt2Reg));
|
||||
srcRegFilt2 = _mm_shuffle_epi8(srcReg1, _mm256_castsi256_si128(filt3Reg));
|
||||
|
||||
// multiply 2 adjacent elements with the filter and add the result
|
||||
srcRegFilt3 =
|
||||
_mm_maddubs_epi16(srcRegFilt3, _mm256_castsi256_si128(secondFilters));
|
||||
srcRegFilt2 =
|
||||
_mm_maddubs_epi16(srcRegFilt2, _mm256_castsi256_si128(thirdFilters));
|
||||
|
||||
// add and saturate the results together
|
||||
srcRegFilt1_1 =
|
||||
_mm_adds_epi16(srcRegFilt1_1, _mm_max_epi16(srcRegFilt3, srcRegFilt2));
|
||||
_mm_adds_epi16(srcRegFilt1_1, _mm_adds_epi16(srcRegFilt3, srcRegFilt2));
|
||||
|
||||
// reading the next 16 bytes
|
||||
// (part of it was being read by earlier read)
|
||||
srcReg2 = _mm_loadu_si128((const __m128i *)(src_ptr + 8));
|
||||
|
||||
// filter the source buffer
|
||||
srcRegFilt2_1 = _mm_shuffle_epi8(srcReg2, _mm256_castsi256_si128(filt1Reg));
|
||||
|
|
@ -245,19 +486,16 @@ static void aom_filter_block1d16_h8_avx2(
|
|||
|
||||
// add and saturate the results together
|
||||
srcRegFilt2_1 =
|
||||
_mm_adds_epi16(srcRegFilt2_1, _mm_min_epi16(srcRegFilt3, srcRegFilt2));
|
||||
srcRegFilt2_1 =
|
||||
_mm_adds_epi16(srcRegFilt2_1, _mm_max_epi16(srcRegFilt3, srcRegFilt2));
|
||||
_mm_adds_epi16(srcRegFilt2_1, _mm_adds_epi16(srcRegFilt3, srcRegFilt2));
|
||||
|
||||
// shift by 6 bit each 16 bit
|
||||
srcRegFilt1_1 =
|
||||
_mm_adds_epi16(srcRegFilt1_1, _mm256_castsi256_si128(addFilterReg64));
|
||||
_mm_adds_epi16(srcRegFilt1_1, _mm256_castsi256_si128(addFilterReg32));
|
||||
srcRegFilt1_1 = _mm_srai_epi16(srcRegFilt1_1, 6);
|
||||
|
||||
srcRegFilt2_1 =
|
||||
_mm_adds_epi16(srcRegFilt2_1, _mm256_castsi256_si128(addFilterReg64));
|
||||
|
||||
// shift by 7 bit each 16 bit
|
||||
srcRegFilt1_1 = _mm_srai_epi16(srcRegFilt1_1, 7);
|
||||
srcRegFilt2_1 = _mm_srai_epi16(srcRegFilt2_1, 7);
|
||||
_mm_adds_epi16(srcRegFilt2_1, _mm256_castsi256_si128(addFilterReg32));
|
||||
srcRegFilt2_1 = _mm_srai_epi16(srcRegFilt2_1, 6);
|
||||
|
||||
// shrink to 8 bit each 16 bits, the first lane contain the first
|
||||
// convolve result and the second lane contain the second convolve
|
||||
|
|
@ -269,11 +507,11 @@ static void aom_filter_block1d16_h8_avx2(
|
|||
}
|
||||
}
|
||||
|
||||
static void aom_filter_block1d16_v8_avx2(
|
||||
static void aom_filter_block1d8_v8_avx2(
|
||||
const uint8_t *src_ptr, ptrdiff_t src_pitch, uint8_t *output_ptr,
|
||||
ptrdiff_t out_pitch, uint32_t output_height, const int16_t *filter) {
|
||||
__m128i filtersReg;
|
||||
__m256i addFilterReg64;
|
||||
__m256i addFilterReg32;
|
||||
__m256i srcReg32b1, srcReg32b2, srcReg32b3, srcReg32b4, srcReg32b5;
|
||||
__m256i srcReg32b6, srcReg32b7, srcReg32b8, srcReg32b9, srcReg32b10;
|
||||
__m256i srcReg32b11, srcReg32b12, filtersReg32;
|
||||
|
|
@ -281,11 +519,11 @@ static void aom_filter_block1d16_v8_avx2(
|
|||
unsigned int i;
|
||||
ptrdiff_t src_stride, dst_stride;
|
||||
|
||||
// create a register with 0,64,0,64,0,64,0,64,0,64,0,64,0,64,0,64
|
||||
addFilterReg64 = _mm256_set1_epi32((int)0x0400040u);
|
||||
addFilterReg32 = _mm256_set1_epi16(32);
|
||||
filtersReg = _mm_loadu_si128((const __m128i *)filter);
|
||||
// converting the 16 bit (short) to 8 bit (byte) and have the
|
||||
// same data in both lanes of 128 bit register.
|
||||
filtersReg = _mm_srai_epi16(filtersReg, 1);
|
||||
filtersReg = _mm_packs_epi16(filtersReg, filtersReg);
|
||||
// have the same data in both lanes of a 256 bit register
|
||||
filtersReg32 = MM256_BROADCASTSI128_SI256(filtersReg);
|
||||
|
|
@ -308,49 +546,178 @@ static void aom_filter_block1d16_v8_avx2(
|
|||
dst_stride = out_pitch << 1;
|
||||
|
||||
// load 16 bytes 7 times in stride of src_pitch
|
||||
srcReg32b1 =
|
||||
_mm256_castsi128_si256(_mm_loadu_si128((const __m128i *)(src_ptr)));
|
||||
srcReg32b2 = _mm256_castsi128_si256(
|
||||
_mm_loadu_si128((const __m128i *)(src_ptr + src_pitch)));
|
||||
srcReg32b3 = _mm256_castsi128_si256(
|
||||
_mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 2)));
|
||||
srcReg32b4 = _mm256_castsi128_si256(
|
||||
_mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 3)));
|
||||
srcReg32b5 = _mm256_castsi128_si256(
|
||||
_mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 4)));
|
||||
srcReg32b6 = _mm256_castsi128_si256(
|
||||
_mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 5)));
|
||||
srcReg32b1 = xx_loadu2_epi64(src_ptr + src_pitch, src_ptr);
|
||||
srcReg32b3 =
|
||||
xx_loadu2_epi64(src_ptr + src_pitch * 3, src_ptr + src_pitch * 2);
|
||||
srcReg32b5 =
|
||||
xx_loadu2_epi64(src_ptr + src_pitch * 5, src_ptr + src_pitch * 4);
|
||||
srcReg32b7 = _mm256_castsi128_si256(
|
||||
_mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 6)));
|
||||
|
||||
// have each consecutive loads on the same 256 register
|
||||
srcReg32b2 = _mm256_permute2x128_si256(srcReg32b1, srcReg32b3, 0x21);
|
||||
srcReg32b4 = _mm256_permute2x128_si256(srcReg32b3, srcReg32b5, 0x21);
|
||||
srcReg32b6 = _mm256_permute2x128_si256(srcReg32b5, srcReg32b7, 0x21);
|
||||
// merge every two consecutive registers except the last one
|
||||
srcReg32b10 = _mm256_unpacklo_epi8(srcReg32b1, srcReg32b2);
|
||||
srcReg32b11 = _mm256_unpacklo_epi8(srcReg32b3, srcReg32b4);
|
||||
srcReg32b2 = _mm256_unpacklo_epi8(srcReg32b5, srcReg32b6);
|
||||
|
||||
for (i = output_height; i > 1; i -= 2) {
|
||||
// load the last 2 loads of 16 bytes and have every two
|
||||
// consecutive loads in the same 256 bit register
|
||||
srcReg32b8 = _mm256_castsi128_si256(
|
||||
_mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 7)));
|
||||
srcReg32b7 = _mm256_inserti128_si256(srcReg32b7,
|
||||
_mm256_castsi256_si128(srcReg32b8), 1);
|
||||
srcReg32b9 = _mm256_castsi128_si256(
|
||||
_mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 8)));
|
||||
srcReg32b8 = _mm256_inserti128_si256(srcReg32b8,
|
||||
_mm256_castsi256_si128(srcReg32b9), 1);
|
||||
|
||||
// merge every two consecutive registers
|
||||
// save
|
||||
srcReg32b4 = _mm256_unpacklo_epi8(srcReg32b7, srcReg32b8);
|
||||
|
||||
// multiply 2 adjacent elements with the filter and add the result
|
||||
srcReg32b10 = _mm256_maddubs_epi16(srcReg32b10, firstFilters);
|
||||
srcReg32b6 = _mm256_maddubs_epi16(srcReg32b4, forthFilters);
|
||||
|
||||
// add and saturate the results together
|
||||
srcReg32b10 = _mm256_adds_epi16(srcReg32b10, srcReg32b6);
|
||||
|
||||
// multiply 2 adjacent elements with the filter and add the result
|
||||
srcReg32b8 = _mm256_maddubs_epi16(srcReg32b11, secondFilters);
|
||||
srcReg32b12 = _mm256_maddubs_epi16(srcReg32b2, thirdFilters);
|
||||
|
||||
// add and saturate the results together
|
||||
srcReg32b10 = _mm256_adds_epi16(srcReg32b10,
|
||||
_mm256_adds_epi16(srcReg32b8, srcReg32b12));
|
||||
|
||||
// shift by 6 bit each 16 bit
|
||||
srcReg32b10 = _mm256_adds_epi16(srcReg32b10, addFilterReg32);
|
||||
srcReg32b10 = _mm256_srai_epi16(srcReg32b10, 6);
|
||||
|
||||
// shrink to 8 bit each 16 bits, the first lane contain the first
|
||||
// convolve result and the second lane contain the second convolve
|
||||
// result
|
||||
srcReg32b1 = _mm256_packus_epi16(srcReg32b10, _mm256_setzero_si256());
|
||||
|
||||
src_ptr += src_stride;
|
||||
|
||||
xx_storeu2_epi64(output_ptr, out_pitch, &srcReg32b1);
|
||||
|
||||
output_ptr += dst_stride;
|
||||
|
||||
// save part of the registers for next strides
|
||||
srcReg32b10 = srcReg32b11;
|
||||
srcReg32b11 = srcReg32b2;
|
||||
srcReg32b2 = srcReg32b4;
|
||||
srcReg32b7 = srcReg32b9;
|
||||
}
|
||||
if (i > 0) {
|
||||
__m128i srcRegFilt1, srcRegFilt4, srcRegFilt6, srcRegFilt8;
|
||||
// load the last 16 bytes
|
||||
srcRegFilt8 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 7));
|
||||
|
||||
// merge the last 2 results together
|
||||
srcRegFilt4 =
|
||||
_mm_unpacklo_epi8(_mm256_castsi256_si128(srcReg32b7), srcRegFilt8);
|
||||
|
||||
// multiply 2 adjacent elements with the filter and add the result
|
||||
srcRegFilt1 = _mm_maddubs_epi16(_mm256_castsi256_si128(srcReg32b10),
|
||||
_mm256_castsi256_si128(firstFilters));
|
||||
srcRegFilt4 =
|
||||
_mm_maddubs_epi16(srcRegFilt4, _mm256_castsi256_si128(forthFilters));
|
||||
|
||||
// add and saturate the results together
|
||||
srcRegFilt1 = _mm_adds_epi16(srcRegFilt1, srcRegFilt4);
|
||||
|
||||
// multiply 2 adjacent elements with the filter and add the result
|
||||
srcRegFilt4 = _mm_maddubs_epi16(_mm256_castsi256_si128(srcReg32b11),
|
||||
_mm256_castsi256_si128(secondFilters));
|
||||
|
||||
// multiply 2 adjacent elements with the filter and add the result
|
||||
srcRegFilt6 = _mm_maddubs_epi16(_mm256_castsi256_si128(srcReg32b2),
|
||||
_mm256_castsi256_si128(thirdFilters));
|
||||
|
||||
// add and saturate the results together
|
||||
srcRegFilt1 =
|
||||
_mm_adds_epi16(srcRegFilt1, _mm_adds_epi16(srcRegFilt4, srcRegFilt6));
|
||||
|
||||
// shift by 6 bit each 16 bit
|
||||
srcRegFilt1 =
|
||||
_mm_adds_epi16(srcRegFilt1, _mm256_castsi256_si128(addFilterReg32));
|
||||
srcRegFilt1 = _mm_srai_epi16(srcRegFilt1, 6);
|
||||
|
||||
// shrink to 8 bit each 16 bits, the first lane contain the first
|
||||
// convolve result and the second lane contain the second convolve result
|
||||
srcRegFilt1 = _mm_packus_epi16(srcRegFilt1, _mm_setzero_si128());
|
||||
|
||||
// save 8 bytes
|
||||
_mm_storel_epi64((__m128i *)output_ptr, srcRegFilt1);
|
||||
}
|
||||
}
|
||||
|
||||
static void aom_filter_block1d16_v8_avx2(
|
||||
const uint8_t *src_ptr, ptrdiff_t src_pitch, uint8_t *output_ptr,
|
||||
ptrdiff_t out_pitch, uint32_t output_height, const int16_t *filter) {
|
||||
__m128i filtersReg;
|
||||
__m256i addFilterReg32;
|
||||
__m256i srcReg32b1, srcReg32b2, srcReg32b3, srcReg32b4, srcReg32b5;
|
||||
__m256i srcReg32b6, srcReg32b7, srcReg32b8, srcReg32b9, srcReg32b10;
|
||||
__m256i srcReg32b11, srcReg32b12, filtersReg32;
|
||||
__m256i firstFilters, secondFilters, thirdFilters, forthFilters;
|
||||
unsigned int i;
|
||||
ptrdiff_t src_stride, dst_stride;
|
||||
|
||||
addFilterReg32 = _mm256_set1_epi16(32);
|
||||
filtersReg = _mm_loadu_si128((const __m128i *)filter);
|
||||
// converting the 16 bit (short) to 8 bit (byte) and have the
|
||||
// same data in both lanes of 128 bit register.
|
||||
filtersReg = _mm_srai_epi16(filtersReg, 1);
|
||||
filtersReg = _mm_packs_epi16(filtersReg, filtersReg);
|
||||
// have the same data in both lanes of a 256 bit register
|
||||
filtersReg32 = MM256_BROADCASTSI128_SI256(filtersReg);
|
||||
|
||||
// duplicate only the first 16 bits (first and second byte)
|
||||
// across 256 bit register
|
||||
firstFilters = _mm256_shuffle_epi8(filtersReg32, _mm256_set1_epi16(0x100u));
|
||||
// duplicate only the second 16 bits (third and forth byte)
|
||||
// across 256 bit register
|
||||
secondFilters = _mm256_shuffle_epi8(filtersReg32, _mm256_set1_epi16(0x302u));
|
||||
// duplicate only the third 16 bits (fifth and sixth byte)
|
||||
// across 256 bit register
|
||||
thirdFilters = _mm256_shuffle_epi8(filtersReg32, _mm256_set1_epi16(0x504u));
|
||||
// duplicate only the forth 16 bits (seventh and eighth byte)
|
||||
// across 256 bit register
|
||||
forthFilters = _mm256_shuffle_epi8(filtersReg32, _mm256_set1_epi16(0x706u));
|
||||
|
||||
// multiple the size of the source and destination stride by two
|
||||
src_stride = src_pitch << 1;
|
||||
dst_stride = out_pitch << 1;
|
||||
|
||||
// load 16 bytes 7 times in stride of src_pitch
|
||||
srcReg32b1 = xx_loadu2_mi128(src_ptr + src_pitch, src_ptr);
|
||||
srcReg32b3 =
|
||||
xx_loadu2_mi128(src_ptr + src_pitch * 3, src_ptr + src_pitch * 2);
|
||||
srcReg32b5 =
|
||||
xx_loadu2_mi128(src_ptr + src_pitch * 5, src_ptr + src_pitch * 4);
|
||||
srcReg32b7 = _mm256_castsi128_si256(
|
||||
_mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 6)));
|
||||
|
||||
// have each consecutive loads on the same 256 register
|
||||
srcReg32b1 = _mm256_inserti128_si256(srcReg32b1,
|
||||
_mm256_castsi256_si128(srcReg32b2), 1);
|
||||
srcReg32b2 = _mm256_inserti128_si256(srcReg32b2,
|
||||
_mm256_castsi256_si128(srcReg32b3), 1);
|
||||
srcReg32b3 = _mm256_inserti128_si256(srcReg32b3,
|
||||
_mm256_castsi256_si128(srcReg32b4), 1);
|
||||
srcReg32b4 = _mm256_inserti128_si256(srcReg32b4,
|
||||
_mm256_castsi256_si128(srcReg32b5), 1);
|
||||
srcReg32b5 = _mm256_inserti128_si256(srcReg32b5,
|
||||
_mm256_castsi256_si128(srcReg32b6), 1);
|
||||
srcReg32b6 = _mm256_inserti128_si256(srcReg32b6,
|
||||
_mm256_castsi256_si128(srcReg32b7), 1);
|
||||
|
||||
srcReg32b2 = _mm256_permute2x128_si256(srcReg32b1, srcReg32b3, 0x21);
|
||||
srcReg32b4 = _mm256_permute2x128_si256(srcReg32b3, srcReg32b5, 0x21);
|
||||
srcReg32b6 = _mm256_permute2x128_si256(srcReg32b5, srcReg32b7, 0x21);
|
||||
// merge every two consecutive registers except the last one
|
||||
srcReg32b10 = _mm256_unpacklo_epi8(srcReg32b1, srcReg32b2);
|
||||
srcReg32b1 = _mm256_unpackhi_epi8(srcReg32b1, srcReg32b2);
|
||||
|
||||
// save
|
||||
srcReg32b11 = _mm256_unpacklo_epi8(srcReg32b3, srcReg32b4);
|
||||
|
||||
// save
|
||||
srcReg32b3 = _mm256_unpackhi_epi8(srcReg32b3, srcReg32b4);
|
||||
|
||||
// save
|
||||
srcReg32b2 = _mm256_unpacklo_epi8(srcReg32b5, srcReg32b6);
|
||||
|
||||
// save
|
||||
srcReg32b5 = _mm256_unpackhi_epi8(srcReg32b5, srcReg32b6);
|
||||
|
||||
for (i = output_height; i > 1; i -= 2) {
|
||||
|
|
@ -383,9 +750,7 @@ static void aom_filter_block1d16_v8_avx2(
|
|||
|
||||
// add and saturate the results together
|
||||
srcReg32b10 = _mm256_adds_epi16(srcReg32b10,
|
||||
_mm256_min_epi16(srcReg32b8, srcReg32b12));
|
||||
srcReg32b10 = _mm256_adds_epi16(srcReg32b10,
|
||||
_mm256_max_epi16(srcReg32b8, srcReg32b12));
|
||||
_mm256_adds_epi16(srcReg32b8, srcReg32b12));
|
||||
|
||||
// multiply 2 adjacent elements with the filter and add the result
|
||||
srcReg32b1 = _mm256_maddubs_epi16(srcReg32b1, firstFilters);
|
||||
|
|
@ -399,16 +764,13 @@ static void aom_filter_block1d16_v8_avx2(
|
|||
|
||||
// add and saturate the results together
|
||||
srcReg32b1 = _mm256_adds_epi16(srcReg32b1,
|
||||
_mm256_min_epi16(srcReg32b8, srcReg32b12));
|
||||
srcReg32b1 = _mm256_adds_epi16(srcReg32b1,
|
||||
_mm256_max_epi16(srcReg32b8, srcReg32b12));
|
||||
_mm256_adds_epi16(srcReg32b8, srcReg32b12));
|
||||
|
||||
srcReg32b10 = _mm256_adds_epi16(srcReg32b10, addFilterReg64);
|
||||
srcReg32b1 = _mm256_adds_epi16(srcReg32b1, addFilterReg64);
|
||||
|
||||
// shift by 7 bit each 16 bit
|
||||
srcReg32b10 = _mm256_srai_epi16(srcReg32b10, 7);
|
||||
srcReg32b1 = _mm256_srai_epi16(srcReg32b1, 7);
|
||||
// shift by 6 bit each 16 bit
|
||||
srcReg32b10 = _mm256_adds_epi16(srcReg32b10, addFilterReg32);
|
||||
srcReg32b1 = _mm256_adds_epi16(srcReg32b1, addFilterReg32);
|
||||
srcReg32b10 = _mm256_srai_epi16(srcReg32b10, 6);
|
||||
srcReg32b1 = _mm256_srai_epi16(srcReg32b1, 6);
|
||||
|
||||
// shrink to 8 bit each 16 bits, the first lane contain the first
|
||||
// convolve result and the second lane contain the second convolve
|
||||
|
|
@ -417,12 +779,7 @@ static void aom_filter_block1d16_v8_avx2(
|
|||
|
||||
src_ptr += src_stride;
|
||||
|
||||
// save 16 bytes
|
||||
_mm_store_si128((__m128i *)output_ptr, _mm256_castsi256_si128(srcReg32b1));
|
||||
|
||||
// save the next 16 bits
|
||||
_mm_store_si128((__m128i *)(output_ptr + out_pitch),
|
||||
_mm256_extractf128_si256(srcReg32b1, 1));
|
||||
xx_store2_mi128(output_ptr, out_pitch, &srcReg32b1);
|
||||
|
||||
output_ptr += dst_stride;
|
||||
|
||||
|
|
@ -475,24 +832,17 @@ static void aom_filter_block1d16_v8_avx2(
|
|||
|
||||
// add and saturate the results together
|
||||
srcRegFilt1 =
|
||||
_mm_adds_epi16(srcRegFilt1, _mm_min_epi16(srcRegFilt4, srcRegFilt6));
|
||||
_mm_adds_epi16(srcRegFilt1, _mm_adds_epi16(srcRegFilt4, srcRegFilt6));
|
||||
srcRegFilt3 =
|
||||
_mm_adds_epi16(srcRegFilt3, _mm_min_epi16(srcRegFilt5, srcRegFilt7));
|
||||
_mm_adds_epi16(srcRegFilt3, _mm_adds_epi16(srcRegFilt5, srcRegFilt7));
|
||||
|
||||
// add and saturate the results together
|
||||
// shift by 6 bit each 16 bit
|
||||
srcRegFilt1 =
|
||||
_mm_adds_epi16(srcRegFilt1, _mm_max_epi16(srcRegFilt4, srcRegFilt6));
|
||||
_mm_adds_epi16(srcRegFilt1, _mm256_castsi256_si128(addFilterReg32));
|
||||
srcRegFilt3 =
|
||||
_mm_adds_epi16(srcRegFilt3, _mm_max_epi16(srcRegFilt5, srcRegFilt7));
|
||||
|
||||
srcRegFilt1 =
|
||||
_mm_adds_epi16(srcRegFilt1, _mm256_castsi256_si128(addFilterReg64));
|
||||
srcRegFilt3 =
|
||||
_mm_adds_epi16(srcRegFilt3, _mm256_castsi256_si128(addFilterReg64));
|
||||
|
||||
// shift by 7 bit each 16 bit
|
||||
srcRegFilt1 = _mm_srai_epi16(srcRegFilt1, 7);
|
||||
srcRegFilt3 = _mm_srai_epi16(srcRegFilt3, 7);
|
||||
_mm_adds_epi16(srcRegFilt3, _mm256_castsi256_si128(addFilterReg32));
|
||||
srcRegFilt1 = _mm_srai_epi16(srcRegFilt1, 6);
|
||||
srcRegFilt3 = _mm_srai_epi16(srcRegFilt3, 6);
|
||||
|
||||
// shrink to 8 bit each 16 bits, the first lane contain the first
|
||||
// convolve result and the second lane contain the second convolve
|
||||
|
|
@ -506,21 +856,6 @@ static void aom_filter_block1d16_v8_avx2(
|
|||
|
||||
#if HAVE_AVX2 && HAVE_SSSE3
|
||||
filter8_1dfunction aom_filter_block1d4_v8_ssse3;
|
||||
#if ARCH_X86_64
|
||||
filter8_1dfunction aom_filter_block1d8_v8_intrin_ssse3;
|
||||
filter8_1dfunction aom_filter_block1d8_h8_intrin_ssse3;
|
||||
filter8_1dfunction aom_filter_block1d4_h8_intrin_ssse3;
|
||||
#define aom_filter_block1d8_v8_avx2 aom_filter_block1d8_v8_intrin_ssse3
|
||||
#define aom_filter_block1d8_h8_avx2 aom_filter_block1d8_h8_intrin_ssse3
|
||||
#define aom_filter_block1d4_h8_avx2 aom_filter_block1d4_h8_intrin_ssse3
|
||||
#else // ARCH_X86
|
||||
filter8_1dfunction aom_filter_block1d8_v8_ssse3;
|
||||
filter8_1dfunction aom_filter_block1d8_h8_ssse3;
|
||||
filter8_1dfunction aom_filter_block1d4_h8_ssse3;
|
||||
#define aom_filter_block1d8_v8_avx2 aom_filter_block1d8_v8_ssse3
|
||||
#define aom_filter_block1d8_h8_avx2 aom_filter_block1d8_h8_ssse3
|
||||
#define aom_filter_block1d4_h8_avx2 aom_filter_block1d4_h8_ssse3
|
||||
#endif // ARCH_X86_64
|
||||
filter8_1dfunction aom_filter_block1d16_v2_ssse3;
|
||||
filter8_1dfunction aom_filter_block1d16_h2_ssse3;
|
||||
filter8_1dfunction aom_filter_block1d8_v2_ssse3;
|
||||
|
|
|
|||
32
third_party/aom/aom_dsp/x86/convolve_avx2.h
vendored
32
third_party/aom/aom_dsp/x86/convolve_avx2.h
vendored
|
|
@ -13,31 +13,27 @@
|
|||
#define AOM_DSP_X86_CONVOLVE_AVX2_H_
|
||||
|
||||
// filters for 16
|
||||
DECLARE_ALIGNED(32, static const uint8_t, filt1_global_avx2[32]) = {
|
||||
0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8,
|
||||
0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8
|
||||
DECLARE_ALIGNED(32, static const uint8_t, filt_global_avx2[]) = {
|
||||
0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 0, 1, 1,
|
||||
2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 2, 3, 3, 4, 4, 5,
|
||||
5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 2, 3, 3, 4, 4, 5, 5, 6, 6,
|
||||
7, 7, 8, 8, 9, 9, 10, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10,
|
||||
10, 11, 11, 12, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11,
|
||||
12, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 6, 7,
|
||||
7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14
|
||||
};
|
||||
|
||||
DECLARE_ALIGNED(32, static const uint8_t, filt2_global_avx2[32]) = {
|
||||
2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10,
|
||||
2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10
|
||||
};
|
||||
|
||||
DECLARE_ALIGNED(32, static const uint8_t, filt3_global_avx2[32]) = {
|
||||
4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12,
|
||||
4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12
|
||||
};
|
||||
|
||||
DECLARE_ALIGNED(32, static const uint8_t, filt4_global_avx2[32]) = {
|
||||
6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14,
|
||||
6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14
|
||||
DECLARE_ALIGNED(32, static const uint8_t, filt_d4_global_avx2[]) = {
|
||||
0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6, 0, 1, 2, 3, 1, 2,
|
||||
3, 4, 2, 3, 4, 5, 3, 4, 5, 6, 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9,
|
||||
7, 8, 9, 10, 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10,
|
||||
};
|
||||
|
||||
static INLINE void prepare_coeffs_lowbd(
|
||||
const InterpFilterParams *const filter_params, const int subpel_q4,
|
||||
__m256i *const coeffs /* [4] */) {
|
||||
const int16_t *const filter = av1_get_interp_filter_subpel_kernel(
|
||||
*filter_params, subpel_q4 & SUBPEL_MASK);
|
||||
filter_params, subpel_q4 & SUBPEL_MASK);
|
||||
const __m128i coeffs_8 = _mm_loadu_si128((__m128i *)filter);
|
||||
const __m256i filter_coeffs = _mm256_broadcastsi128_si256(coeffs_8);
|
||||
|
||||
|
|
@ -65,7 +61,7 @@ static INLINE void prepare_coeffs(const InterpFilterParams *const filter_params,
|
|||
const int subpel_q4,
|
||||
__m256i *const coeffs /* [4] */) {
|
||||
const int16_t *filter = av1_get_interp_filter_subpel_kernel(
|
||||
*filter_params, subpel_q4 & SUBPEL_MASK);
|
||||
filter_params, subpel_q4 & SUBPEL_MASK);
|
||||
|
||||
const __m128i coeff_8 = _mm_loadu_si128((__m128i *)filter);
|
||||
const __m256i coeff = _mm256_broadcastsi128_si256(coeff_8);
|
||||
|
|
|
|||
2
third_party/aom/aom_dsp/x86/convolve_sse2.h
vendored
2
third_party/aom/aom_dsp/x86/convolve_sse2.h
vendored
|
|
@ -19,7 +19,7 @@ static INLINE void prepare_coeffs(const InterpFilterParams *const filter_params,
|
|||
const int subpel_q4,
|
||||
__m128i *const coeffs /* [4] */) {
|
||||
const int16_t *filter = av1_get_interp_filter_subpel_kernel(
|
||||
*filter_params, subpel_q4 & SUBPEL_MASK);
|
||||
filter_params, subpel_q4 & SUBPEL_MASK);
|
||||
const __m128i coeff = _mm_loadu_si128((__m128i *)filter);
|
||||
|
||||
// coeffs 0 1 0 1 0 1 0 1
|
||||
|
|
|
|||
|
|
@ -105,8 +105,8 @@ void aom_highbd_convolve_copy_avx2(const uint8_t *src8, ptrdiff_t src_stride,
|
|||
|
||||
void av1_highbd_convolve_y_sr_avx2(const uint16_t *src, int src_stride,
|
||||
uint16_t *dst, int dst_stride, int w, int h,
|
||||
InterpFilterParams *filter_params_x,
|
||||
InterpFilterParams *filter_params_y,
|
||||
const InterpFilterParams *filter_params_x,
|
||||
const InterpFilterParams *filter_params_y,
|
||||
const int subpel_x_q4, const int subpel_y_q4,
|
||||
ConvolveParams *conv_params, int bd) {
|
||||
int i, j;
|
||||
|
|
@ -254,8 +254,8 @@ void av1_highbd_convolve_y_sr_avx2(const uint16_t *src, int src_stride,
|
|||
|
||||
void av1_highbd_convolve_x_sr_avx2(const uint16_t *src, int src_stride,
|
||||
uint16_t *dst, int dst_stride, int w, int h,
|
||||
InterpFilterParams *filter_params_x,
|
||||
InterpFilterParams *filter_params_y,
|
||||
const InterpFilterParams *filter_params_x,
|
||||
const InterpFilterParams *filter_params_y,
|
||||
const int subpel_x_q4, const int subpel_y_q4,
|
||||
ConvolveParams *conv_params, int bd) {
|
||||
int i, j;
|
||||
|
|
|
|||
|
|
@ -18,8 +18,8 @@
|
|||
|
||||
void av1_highbd_convolve_y_sr_ssse3(const uint16_t *src, int src_stride,
|
||||
uint16_t *dst, int dst_stride, int w, int h,
|
||||
InterpFilterParams *filter_params_x,
|
||||
InterpFilterParams *filter_params_y,
|
||||
const InterpFilterParams *filter_params_x,
|
||||
const InterpFilterParams *filter_params_y,
|
||||
const int subpel_x_q4,
|
||||
const int subpel_y_q4,
|
||||
ConvolveParams *conv_params, int bd) {
|
||||
|
|
@ -166,8 +166,8 @@ void av1_highbd_convolve_y_sr_ssse3(const uint16_t *src, int src_stride,
|
|||
|
||||
void av1_highbd_convolve_x_sr_ssse3(const uint16_t *src, int src_stride,
|
||||
uint16_t *dst, int dst_stride, int w, int h,
|
||||
InterpFilterParams *filter_params_x,
|
||||
InterpFilterParams *filter_params_y,
|
||||
const InterpFilterParams *filter_params_x,
|
||||
const InterpFilterParams *filter_params_y,
|
||||
const int subpel_x_q4,
|
||||
const int subpel_y_q4,
|
||||
ConvolveParams *conv_params, int bd) {
|
||||
|
|
|
|||
|
|
@ -676,7 +676,7 @@ void aom_highbd_upsampled_pred_sse2(MACROBLOCKD *xd,
|
|||
}
|
||||
}
|
||||
|
||||
const InterpFilterParams filter =
|
||||
const InterpFilterParams *filter =
|
||||
av1_get_interp_filter_params_with_block_size(EIGHTTAP_REGULAR, 8);
|
||||
|
||||
if (!subpel_x_q3 && !subpel_y_q3) {
|
||||
|
|
@ -726,14 +726,14 @@ void aom_highbd_upsampled_pred_sse2(MACROBLOCKD *xd,
|
|||
const int16_t *const kernel_y =
|
||||
av1_get_interp_filter_subpel_kernel(filter, subpel_y_q3 << 1);
|
||||
const int intermediate_height =
|
||||
(((height - 1) * 8 + subpel_y_q3) >> 3) + filter.taps;
|
||||
(((height - 1) * 8 + subpel_y_q3) >> 3) + filter->taps;
|
||||
assert(intermediate_height <= (MAX_SB_SIZE * 2 + 16) + 16);
|
||||
aom_highbd_convolve8_horiz(ref8 - ref_stride * ((filter.taps >> 1) - 1),
|
||||
aom_highbd_convolve8_horiz(ref8 - ref_stride * ((filter->taps >> 1) - 1),
|
||||
ref_stride, CONVERT_TO_BYTEPTR(temp),
|
||||
MAX_SB_SIZE, kernel_x, 16, NULL, -1, width,
|
||||
intermediate_height, bd);
|
||||
aom_highbd_convolve8_vert(
|
||||
CONVERT_TO_BYTEPTR(temp + MAX_SB_SIZE * ((filter.taps >> 1) - 1)),
|
||||
CONVERT_TO_BYTEPTR(temp + MAX_SB_SIZE * ((filter->taps >> 1) - 1)),
|
||||
MAX_SB_SIZE, CONVERT_TO_BYTEPTR(comp_pred), width, NULL, -1, kernel_y,
|
||||
16, width, height, bd);
|
||||
}
|
||||
|
|
|
|||
110
third_party/aom/aom_dsp/x86/jnt_variance_ssse3.c
vendored
110
third_party/aom/aom_dsp/x86/jnt_variance_ssse3.c
vendored
|
|
@ -22,118 +22,12 @@
|
|||
void aom_var_filter_block2d_bil_first_pass_ssse3(
|
||||
const uint8_t *a, uint16_t *b, unsigned int src_pixels_per_line,
|
||||
unsigned int pixel_step, unsigned int output_height,
|
||||
unsigned int output_width, const uint8_t *filter) {
|
||||
// Note: filter[0], filter[1] could be {128, 0}, where 128 will overflow
|
||||
// in computation using _mm_maddubs_epi16.
|
||||
// Change {128, 0} to {64, 0} and reduce FILTER_BITS by 1 to avoid overflow.
|
||||
const int16_t round = (1 << (FILTER_BITS - 1)) >> 1;
|
||||
const __m128i r = _mm_set1_epi16(round);
|
||||
const uint8_t f0 = filter[0] >> 1;
|
||||
const uint8_t f1 = filter[1] >> 1;
|
||||
const __m128i filters = _mm_setr_epi8(f0, f1, f0, f1, f0, f1, f0, f1, f0, f1,
|
||||
f0, f1, f0, f1, f0, f1);
|
||||
const __m128i shuffle_mask =
|
||||
_mm_setr_epi8(0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8);
|
||||
unsigned int i, j;
|
||||
(void)pixel_step;
|
||||
|
||||
if (output_width >= 8) {
|
||||
for (i = 0; i < output_height; ++i) {
|
||||
for (j = 0; j < output_width; j += 8) {
|
||||
// load source
|
||||
__m128i source_low = xx_loadl_64(a);
|
||||
__m128i source_hi = _mm_setzero_si128();
|
||||
|
||||
// avoid load undefined memory
|
||||
if (a + 8 != NULL) source_hi = xx_loadl_64(a + 8);
|
||||
__m128i source = _mm_unpacklo_epi64(source_low, source_hi);
|
||||
|
||||
// shuffle to:
|
||||
// { a[0], a[1], a[1], a[2], a[2], a[3], a[3], a[4],
|
||||
// a[4], a[5], a[5], a[6], a[6], a[7], a[7], a[8] }
|
||||
__m128i source_shuffle = _mm_shuffle_epi8(source, shuffle_mask);
|
||||
|
||||
// b[i] = a[i] * filter[0] + a[i + 1] * filter[1]
|
||||
__m128i res = _mm_maddubs_epi16(source_shuffle, filters);
|
||||
|
||||
// round
|
||||
res = _mm_srai_epi16(_mm_add_epi16(res, r), FILTER_BITS - 1);
|
||||
|
||||
xx_storeu_128(b, res);
|
||||
|
||||
a += 8;
|
||||
b += 8;
|
||||
}
|
||||
|
||||
a += src_pixels_per_line - output_width;
|
||||
}
|
||||
} else {
|
||||
for (i = 0; i < output_height; ++i) {
|
||||
// load source, only first 5 values are meaningful:
|
||||
// { a[0], a[1], a[2], a[3], a[4], xxxx }
|
||||
__m128i source = xx_loadl_64(a);
|
||||
|
||||
// shuffle, up to the first 8 are useful
|
||||
// { a[0], a[1], a[1], a[2], a[2], a[3], a[3], a[4],
|
||||
// a[4], a[5], a[5], a[6], a[6], a[7], a[7], a[8] }
|
||||
__m128i source_shuffle = _mm_shuffle_epi8(source, shuffle_mask);
|
||||
|
||||
__m128i res = _mm_maddubs_epi16(source_shuffle, filters);
|
||||
res = _mm_srai_epi16(_mm_add_epi16(res, r), FILTER_BITS - 1);
|
||||
|
||||
xx_storel_64(b, res);
|
||||
|
||||
a += src_pixels_per_line;
|
||||
b += output_width;
|
||||
}
|
||||
}
|
||||
}
|
||||
unsigned int output_width, const uint8_t *filter);
|
||||
|
||||
void aom_var_filter_block2d_bil_second_pass_ssse3(
|
||||
const uint16_t *a, uint8_t *b, unsigned int src_pixels_per_line,
|
||||
unsigned int pixel_step, unsigned int output_height,
|
||||
unsigned int output_width, const uint8_t *filter) {
|
||||
const int16_t round = (1 << FILTER_BITS) >> 1;
|
||||
const __m128i r = _mm_set1_epi32(round);
|
||||
const __m128i filters =
|
||||
_mm_setr_epi16(filter[0], filter[1], filter[0], filter[1], filter[0],
|
||||
filter[1], filter[0], filter[1]);
|
||||
const __m128i shuffle_mask =
|
||||
_mm_setr_epi8(0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15);
|
||||
const __m128i mask =
|
||||
_mm_setr_epi8(0, 4, 8, 12, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1);
|
||||
unsigned int i, j;
|
||||
|
||||
for (i = 0; i < output_height; ++i) {
|
||||
for (j = 0; j < output_width; j += 4) {
|
||||
// load source as:
|
||||
// { a[0], a[1], a[2], a[3], a[w], a[w+1], a[w+2], a[w+3] }
|
||||
__m128i source1 = xx_loadl_64(a);
|
||||
__m128i source2 = xx_loadl_64(a + pixel_step);
|
||||
__m128i source = _mm_unpacklo_epi64(source1, source2);
|
||||
|
||||
// shuffle source to:
|
||||
// { a[0], a[w], a[1], a[w+1], a[2], a[w+2], a[3], a[w+3] }
|
||||
__m128i source_shuffle = _mm_shuffle_epi8(source, shuffle_mask);
|
||||
|
||||
// b[i] = a[i] * filter[0] + a[w + i] * filter[1]
|
||||
__m128i res = _mm_madd_epi16(source_shuffle, filters);
|
||||
|
||||
// round
|
||||
res = _mm_srai_epi32(_mm_add_epi32(res, r), FILTER_BITS);
|
||||
|
||||
// shuffle to get each lower 8 bit of every 32 bit
|
||||
res = _mm_shuffle_epi8(res, mask);
|
||||
|
||||
xx_storel_32(b, res);
|
||||
|
||||
a += 4;
|
||||
b += 4;
|
||||
}
|
||||
|
||||
a += src_pixels_per_line - output_width;
|
||||
}
|
||||
}
|
||||
unsigned int output_width, const uint8_t *filter);
|
||||
|
||||
static INLINE void compute_jnt_comp_avg(__m128i *p0, __m128i *p1,
|
||||
const __m128i *w, const __m128i *r,
|
||||
|
|
|
|||
390
third_party/aom/aom_dsp/x86/masked_sad_intrin_avx2.c
vendored
Normal file
390
third_party/aom/aom_dsp/x86/masked_sad_intrin_avx2.c
vendored
Normal file
|
|
@ -0,0 +1,390 @@
|
|||
/*
|
||||
* Copyright (c) 2018, Alliance for Open Media. All rights reserved
|
||||
*
|
||||
* This source code is subject to the terms of the BSD 2 Clause License and
|
||||
* the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
|
||||
* was not distributed with this source code in the LICENSE file, you can
|
||||
* obtain it at www.aomedia.org/license/software. If the Alliance for Open
|
||||
* Media Patent License 1.0 was not distributed with this source code in the
|
||||
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
|
||||
*/
|
||||
|
||||
#include <stdio.h>
|
||||
#include <tmmintrin.h>
|
||||
|
||||
#include "config/aom_config.h"
|
||||
#include "config/aom_dsp_rtcd.h"
|
||||
|
||||
#include "aom_dsp/blend.h"
|
||||
#include "aom/aom_integer.h"
|
||||
#include "aom_dsp/x86/synonyms.h"
|
||||
#include "aom_dsp/x86//masked_sad_intrin_ssse3.h"
|
||||
|
||||
static INLINE unsigned int masked_sad32xh_avx2(
|
||||
const uint8_t *src_ptr, int src_stride, const uint8_t *a_ptr, int a_stride,
|
||||
const uint8_t *b_ptr, int b_stride, const uint8_t *m_ptr, int m_stride,
|
||||
int width, int height) {
|
||||
int x, y;
|
||||
__m256i res = _mm256_setzero_si256();
|
||||
const __m256i mask_max = _mm256_set1_epi8((1 << AOM_BLEND_A64_ROUND_BITS));
|
||||
const __m256i round_scale =
|
||||
_mm256_set1_epi16(1 << (15 - AOM_BLEND_A64_ROUND_BITS));
|
||||
for (y = 0; y < height; y++) {
|
||||
for (x = 0; x < width; x += 32) {
|
||||
const __m256i src = _mm256_lddqu_si256((const __m256i *)&src_ptr[x]);
|
||||
const __m256i a = _mm256_lddqu_si256((const __m256i *)&a_ptr[x]);
|
||||
const __m256i b = _mm256_lddqu_si256((const __m256i *)&b_ptr[x]);
|
||||
const __m256i m = _mm256_lddqu_si256((const __m256i *)&m_ptr[x]);
|
||||
const __m256i m_inv = _mm256_sub_epi8(mask_max, m);
|
||||
|
||||
// Calculate 16 predicted pixels.
|
||||
// Note that the maximum value of any entry of 'pred_l' or 'pred_r'
|
||||
// is 64 * 255, so we have plenty of space to add rounding constants.
|
||||
const __m256i data_l = _mm256_unpacklo_epi8(a, b);
|
||||
const __m256i mask_l = _mm256_unpacklo_epi8(m, m_inv);
|
||||
__m256i pred_l = _mm256_maddubs_epi16(data_l, mask_l);
|
||||
pred_l = _mm256_mulhrs_epi16(pred_l, round_scale);
|
||||
|
||||
const __m256i data_r = _mm256_unpackhi_epi8(a, b);
|
||||
const __m256i mask_r = _mm256_unpackhi_epi8(m, m_inv);
|
||||
__m256i pred_r = _mm256_maddubs_epi16(data_r, mask_r);
|
||||
pred_r = _mm256_mulhrs_epi16(pred_r, round_scale);
|
||||
|
||||
const __m256i pred = _mm256_packus_epi16(pred_l, pred_r);
|
||||
res = _mm256_add_epi32(res, _mm256_sad_epu8(pred, src));
|
||||
}
|
||||
|
||||
src_ptr += src_stride;
|
||||
a_ptr += a_stride;
|
||||
b_ptr += b_stride;
|
||||
m_ptr += m_stride;
|
||||
}
|
||||
// At this point, we have two 32-bit partial SADs in lanes 0 and 2 of 'res'.
|
||||
res = _mm256_shuffle_epi32(res, 0xd8);
|
||||
res = _mm256_permute4x64_epi64(res, 0xd8);
|
||||
res = _mm256_hadd_epi32(res, res);
|
||||
res = _mm256_hadd_epi32(res, res);
|
||||
int32_t sad = _mm256_extract_epi32(res, 0);
|
||||
return (sad + 31) >> 6;
|
||||
}
|
||||
|
||||
static INLINE __m256i xx_loadu2_m128i(const void *hi, const void *lo) {
|
||||
__m128i a0 = _mm_lddqu_si128((const __m128i *)(lo));
|
||||
__m128i a1 = _mm_lddqu_si128((const __m128i *)(hi));
|
||||
__m256i a = _mm256_castsi128_si256(a0);
|
||||
return _mm256_inserti128_si256(a, a1, 1);
|
||||
}
|
||||
|
||||
static INLINE unsigned int masked_sad16xh_avx2(
|
||||
const uint8_t *src_ptr, int src_stride, const uint8_t *a_ptr, int a_stride,
|
||||
const uint8_t *b_ptr, int b_stride, const uint8_t *m_ptr, int m_stride,
|
||||
int height) {
|
||||
int y;
|
||||
__m256i res = _mm256_setzero_si256();
|
||||
const __m256i mask_max = _mm256_set1_epi8((1 << AOM_BLEND_A64_ROUND_BITS));
|
||||
const __m256i round_scale =
|
||||
_mm256_set1_epi16(1 << (15 - AOM_BLEND_A64_ROUND_BITS));
|
||||
for (y = 0; y < height; y += 2) {
|
||||
const __m256i src = xx_loadu2_m128i(src_ptr + src_stride, src_ptr);
|
||||
const __m256i a = xx_loadu2_m128i(a_ptr + a_stride, a_ptr);
|
||||
const __m256i b = xx_loadu2_m128i(b_ptr + b_stride, b_ptr);
|
||||
const __m256i m = xx_loadu2_m128i(m_ptr + m_stride, m_ptr);
|
||||
const __m256i m_inv = _mm256_sub_epi8(mask_max, m);
|
||||
|
||||
// Calculate 16 predicted pixels.
|
||||
// Note that the maximum value of any entry of 'pred_l' or 'pred_r'
|
||||
// is 64 * 255, so we have plenty of space to add rounding constants.
|
||||
const __m256i data_l = _mm256_unpacklo_epi8(a, b);
|
||||
const __m256i mask_l = _mm256_unpacklo_epi8(m, m_inv);
|
||||
__m256i pred_l = _mm256_maddubs_epi16(data_l, mask_l);
|
||||
pred_l = _mm256_mulhrs_epi16(pred_l, round_scale);
|
||||
|
||||
const __m256i data_r = _mm256_unpackhi_epi8(a, b);
|
||||
const __m256i mask_r = _mm256_unpackhi_epi8(m, m_inv);
|
||||
__m256i pred_r = _mm256_maddubs_epi16(data_r, mask_r);
|
||||
pred_r = _mm256_mulhrs_epi16(pred_r, round_scale);
|
||||
|
||||
const __m256i pred = _mm256_packus_epi16(pred_l, pred_r);
|
||||
res = _mm256_add_epi32(res, _mm256_sad_epu8(pred, src));
|
||||
|
||||
src_ptr += src_stride << 1;
|
||||
a_ptr += a_stride << 1;
|
||||
b_ptr += b_stride << 1;
|
||||
m_ptr += m_stride << 1;
|
||||
}
|
||||
// At this point, we have two 32-bit partial SADs in lanes 0 and 2 of 'res'.
|
||||
res = _mm256_shuffle_epi32(res, 0xd8);
|
||||
res = _mm256_permute4x64_epi64(res, 0xd8);
|
||||
res = _mm256_hadd_epi32(res, res);
|
||||
res = _mm256_hadd_epi32(res, res);
|
||||
int32_t sad = _mm256_extract_epi32(res, 0);
|
||||
return (sad + 31) >> 6;
|
||||
}
|
||||
|
||||
static INLINE unsigned int aom_masked_sad_avx2(
|
||||
const uint8_t *src, int src_stride, const uint8_t *ref, int ref_stride,
|
||||
const uint8_t *second_pred, const uint8_t *msk, int msk_stride,
|
||||
int invert_mask, int m, int n) {
|
||||
unsigned int sad;
|
||||
if (!invert_mask) {
|
||||
switch (m) {
|
||||
case 4:
|
||||
sad = aom_masked_sad4xh_ssse3(src, src_stride, ref, ref_stride,
|
||||
second_pred, m, msk, msk_stride, n);
|
||||
break;
|
||||
case 8:
|
||||
sad = aom_masked_sad8xh_ssse3(src, src_stride, ref, ref_stride,
|
||||
second_pred, m, msk, msk_stride, n);
|
||||
break;
|
||||
case 16:
|
||||
sad = masked_sad16xh_avx2(src, src_stride, ref, ref_stride, second_pred,
|
||||
m, msk, msk_stride, n);
|
||||
break;
|
||||
default:
|
||||
sad = masked_sad32xh_avx2(src, src_stride, ref, ref_stride, second_pred,
|
||||
m, msk, msk_stride, m, n);
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
switch (m) {
|
||||
case 4:
|
||||
sad = aom_masked_sad4xh_ssse3(src, src_stride, second_pred, m, ref,
|
||||
ref_stride, msk, msk_stride, n);
|
||||
break;
|
||||
case 8:
|
||||
sad = aom_masked_sad8xh_ssse3(src, src_stride, second_pred, m, ref,
|
||||
ref_stride, msk, msk_stride, n);
|
||||
break;
|
||||
case 16:
|
||||
sad = masked_sad16xh_avx2(src, src_stride, second_pred, m, ref,
|
||||
ref_stride, msk, msk_stride, n);
|
||||
break;
|
||||
default:
|
||||
sad = masked_sad32xh_avx2(src, src_stride, second_pred, m, ref,
|
||||
ref_stride, msk, msk_stride, m, n);
|
||||
break;
|
||||
}
|
||||
}
|
||||
return sad;
|
||||
}
|
||||
|
||||
#define MASKSADMXN_AVX2(m, n) \
|
||||
unsigned int aom_masked_sad##m##x##n##_avx2( \
|
||||
const uint8_t *src, int src_stride, const uint8_t *ref, int ref_stride, \
|
||||
const uint8_t *second_pred, const uint8_t *msk, int msk_stride, \
|
||||
int invert_mask) { \
|
||||
return aom_masked_sad_avx2(src, src_stride, ref, ref_stride, second_pred, \
|
||||
msk, msk_stride, invert_mask, m, n); \
|
||||
}
|
||||
|
||||
MASKSADMXN_AVX2(4, 4)
|
||||
MASKSADMXN_AVX2(4, 8)
|
||||
MASKSADMXN_AVX2(8, 4)
|
||||
MASKSADMXN_AVX2(8, 8)
|
||||
MASKSADMXN_AVX2(8, 16)
|
||||
MASKSADMXN_AVX2(16, 8)
|
||||
MASKSADMXN_AVX2(16, 16)
|
||||
MASKSADMXN_AVX2(16, 32)
|
||||
MASKSADMXN_AVX2(32, 16)
|
||||
MASKSADMXN_AVX2(32, 32)
|
||||
MASKSADMXN_AVX2(32, 64)
|
||||
MASKSADMXN_AVX2(64, 32)
|
||||
MASKSADMXN_AVX2(64, 64)
|
||||
MASKSADMXN_AVX2(64, 128)
|
||||
MASKSADMXN_AVX2(128, 64)
|
||||
MASKSADMXN_AVX2(128, 128)
|
||||
MASKSADMXN_AVX2(4, 16)
|
||||
MASKSADMXN_AVX2(16, 4)
|
||||
MASKSADMXN_AVX2(8, 32)
|
||||
MASKSADMXN_AVX2(32, 8)
|
||||
MASKSADMXN_AVX2(16, 64)
|
||||
MASKSADMXN_AVX2(64, 16)
|
||||
|
||||
static INLINE unsigned int highbd_masked_sad8xh_avx2(
|
||||
const uint8_t *src8, int src_stride, const uint8_t *a8, int a_stride,
|
||||
const uint8_t *b8, int b_stride, const uint8_t *m_ptr, int m_stride,
|
||||
int height) {
|
||||
const uint16_t *src_ptr = CONVERT_TO_SHORTPTR(src8);
|
||||
const uint16_t *a_ptr = CONVERT_TO_SHORTPTR(a8);
|
||||
const uint16_t *b_ptr = CONVERT_TO_SHORTPTR(b8);
|
||||
int y;
|
||||
__m256i res = _mm256_setzero_si256();
|
||||
const __m256i mask_max = _mm256_set1_epi16((1 << AOM_BLEND_A64_ROUND_BITS));
|
||||
const __m256i round_const =
|
||||
_mm256_set1_epi32((1 << AOM_BLEND_A64_ROUND_BITS) >> 1);
|
||||
const __m256i one = _mm256_set1_epi16(1);
|
||||
|
||||
for (y = 0; y < height; y += 2) {
|
||||
const __m256i src = xx_loadu2_m128i(src_ptr + src_stride, src_ptr);
|
||||
const __m256i a = xx_loadu2_m128i(a_ptr + a_stride, a_ptr);
|
||||
const __m256i b = xx_loadu2_m128i(b_ptr + b_stride, b_ptr);
|
||||
// Zero-extend mask to 16 bits
|
||||
const __m256i m = _mm256_cvtepu8_epi16(_mm_unpacklo_epi64(
|
||||
_mm_loadl_epi64((const __m128i *)(m_ptr)),
|
||||
_mm_loadl_epi64((const __m128i *)(m_ptr + m_stride))));
|
||||
const __m256i m_inv = _mm256_sub_epi16(mask_max, m);
|
||||
|
||||
const __m256i data_l = _mm256_unpacklo_epi16(a, b);
|
||||
const __m256i mask_l = _mm256_unpacklo_epi16(m, m_inv);
|
||||
__m256i pred_l = _mm256_madd_epi16(data_l, mask_l);
|
||||
pred_l = _mm256_srai_epi32(_mm256_add_epi32(pred_l, round_const),
|
||||
AOM_BLEND_A64_ROUND_BITS);
|
||||
|
||||
const __m256i data_r = _mm256_unpackhi_epi16(a, b);
|
||||
const __m256i mask_r = _mm256_unpackhi_epi16(m, m_inv);
|
||||
__m256i pred_r = _mm256_madd_epi16(data_r, mask_r);
|
||||
pred_r = _mm256_srai_epi32(_mm256_add_epi32(pred_r, round_const),
|
||||
AOM_BLEND_A64_ROUND_BITS);
|
||||
|
||||
// Note: the maximum value in pred_l/r is (2^bd)-1 < 2^15,
|
||||
// so it is safe to do signed saturation here.
|
||||
const __m256i pred = _mm256_packs_epi32(pred_l, pred_r);
|
||||
// There is no 16-bit SAD instruction, so we have to synthesize
|
||||
// an 8-element SAD. We do this by storing 4 32-bit partial SADs,
|
||||
// and accumulating them at the end
|
||||
const __m256i diff = _mm256_abs_epi16(_mm256_sub_epi16(pred, src));
|
||||
res = _mm256_add_epi32(res, _mm256_madd_epi16(diff, one));
|
||||
|
||||
src_ptr += src_stride << 1;
|
||||
a_ptr += a_stride << 1;
|
||||
b_ptr += b_stride << 1;
|
||||
m_ptr += m_stride << 1;
|
||||
}
|
||||
// At this point, we have four 32-bit partial SADs stored in 'res'.
|
||||
res = _mm256_hadd_epi32(res, res);
|
||||
res = _mm256_hadd_epi32(res, res);
|
||||
int sad = _mm256_extract_epi32(res, 0) + _mm256_extract_epi32(res, 4);
|
||||
return (sad + 31) >> 6;
|
||||
}
|
||||
|
||||
static INLINE unsigned int highbd_masked_sad16xh_avx2(
|
||||
const uint8_t *src8, int src_stride, const uint8_t *a8, int a_stride,
|
||||
const uint8_t *b8, int b_stride, const uint8_t *m_ptr, int m_stride,
|
||||
int width, int height) {
|
||||
const uint16_t *src_ptr = CONVERT_TO_SHORTPTR(src8);
|
||||
const uint16_t *a_ptr = CONVERT_TO_SHORTPTR(a8);
|
||||
const uint16_t *b_ptr = CONVERT_TO_SHORTPTR(b8);
|
||||
int x, y;
|
||||
__m256i res = _mm256_setzero_si256();
|
||||
const __m256i mask_max = _mm256_set1_epi16((1 << AOM_BLEND_A64_ROUND_BITS));
|
||||
const __m256i round_const =
|
||||
_mm256_set1_epi32((1 << AOM_BLEND_A64_ROUND_BITS) >> 1);
|
||||
const __m256i one = _mm256_set1_epi16(1);
|
||||
|
||||
for (y = 0; y < height; y++) {
|
||||
for (x = 0; x < width; x += 16) {
|
||||
const __m256i src = _mm256_lddqu_si256((const __m256i *)&src_ptr[x]);
|
||||
const __m256i a = _mm256_lddqu_si256((const __m256i *)&a_ptr[x]);
|
||||
const __m256i b = _mm256_lddqu_si256((const __m256i *)&b_ptr[x]);
|
||||
// Zero-extend mask to 16 bits
|
||||
const __m256i m =
|
||||
_mm256_cvtepu8_epi16(_mm_lddqu_si128((const __m128i *)&m_ptr[x]));
|
||||
const __m256i m_inv = _mm256_sub_epi16(mask_max, m);
|
||||
|
||||
const __m256i data_l = _mm256_unpacklo_epi16(a, b);
|
||||
const __m256i mask_l = _mm256_unpacklo_epi16(m, m_inv);
|
||||
__m256i pred_l = _mm256_madd_epi16(data_l, mask_l);
|
||||
pred_l = _mm256_srai_epi32(_mm256_add_epi32(pred_l, round_const),
|
||||
AOM_BLEND_A64_ROUND_BITS);
|
||||
|
||||
const __m256i data_r = _mm256_unpackhi_epi16(a, b);
|
||||
const __m256i mask_r = _mm256_unpackhi_epi16(m, m_inv);
|
||||
__m256i pred_r = _mm256_madd_epi16(data_r, mask_r);
|
||||
pred_r = _mm256_srai_epi32(_mm256_add_epi32(pred_r, round_const),
|
||||
AOM_BLEND_A64_ROUND_BITS);
|
||||
|
||||
// Note: the maximum value in pred_l/r is (2^bd)-1 < 2^15,
|
||||
// so it is safe to do signed saturation here.
|
||||
const __m256i pred = _mm256_packs_epi32(pred_l, pred_r);
|
||||
// There is no 16-bit SAD instruction, so we have to synthesize
|
||||
// an 8-element SAD. We do this by storing 4 32-bit partial SADs,
|
||||
// and accumulating them at the end
|
||||
const __m256i diff = _mm256_abs_epi16(_mm256_sub_epi16(pred, src));
|
||||
res = _mm256_add_epi32(res, _mm256_madd_epi16(diff, one));
|
||||
}
|
||||
|
||||
src_ptr += src_stride;
|
||||
a_ptr += a_stride;
|
||||
b_ptr += b_stride;
|
||||
m_ptr += m_stride;
|
||||
}
|
||||
// At this point, we have four 32-bit partial SADs stored in 'res'.
|
||||
res = _mm256_hadd_epi32(res, res);
|
||||
res = _mm256_hadd_epi32(res, res);
|
||||
int sad = _mm256_extract_epi32(res, 0) + _mm256_extract_epi32(res, 4);
|
||||
return (sad + 31) >> 6;
|
||||
}
|
||||
|
||||
static INLINE unsigned int aom_highbd_masked_sad_avx2(
|
||||
const uint8_t *src, int src_stride, const uint8_t *ref, int ref_stride,
|
||||
const uint8_t *second_pred, const uint8_t *msk, int msk_stride,
|
||||
int invert_mask, int m, int n) {
|
||||
unsigned int sad;
|
||||
if (!invert_mask) {
|
||||
switch (m) {
|
||||
case 4:
|
||||
sad =
|
||||
aom_highbd_masked_sad4xh_ssse3(src, src_stride, ref, ref_stride,
|
||||
second_pred, m, msk, msk_stride, n);
|
||||
break;
|
||||
case 8:
|
||||
sad = highbd_masked_sad8xh_avx2(src, src_stride, ref, ref_stride,
|
||||
second_pred, m, msk, msk_stride, n);
|
||||
break;
|
||||
default:
|
||||
sad = highbd_masked_sad16xh_avx2(src, src_stride, ref, ref_stride,
|
||||
second_pred, m, msk, msk_stride, m, n);
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
switch (m) {
|
||||
case 4:
|
||||
sad =
|
||||
aom_highbd_masked_sad4xh_ssse3(src, src_stride, second_pred, m, ref,
|
||||
ref_stride, msk, msk_stride, n);
|
||||
break;
|
||||
case 8:
|
||||
sad = highbd_masked_sad8xh_avx2(src, src_stride, second_pred, m, ref,
|
||||
ref_stride, msk, msk_stride, n);
|
||||
break;
|
||||
default:
|
||||
sad = highbd_masked_sad16xh_avx2(src, src_stride, second_pred, m, ref,
|
||||
ref_stride, msk, msk_stride, m, n);
|
||||
break;
|
||||
}
|
||||
}
|
||||
return sad;
|
||||
}
|
||||
|
||||
#define HIGHBD_MASKSADMXN_AVX2(m, n) \
|
||||
unsigned int aom_highbd_masked_sad##m##x##n##_avx2( \
|
||||
const uint8_t *src8, int src_stride, const uint8_t *ref8, \
|
||||
int ref_stride, const uint8_t *second_pred8, const uint8_t *msk, \
|
||||
int msk_stride, int invert_mask) { \
|
||||
return aom_highbd_masked_sad_avx2(src8, src_stride, ref8, ref_stride, \
|
||||
second_pred8, msk, msk_stride, \
|
||||
invert_mask, m, n); \
|
||||
}
|
||||
|
||||
HIGHBD_MASKSADMXN_AVX2(4, 4);
|
||||
HIGHBD_MASKSADMXN_AVX2(4, 8);
|
||||
HIGHBD_MASKSADMXN_AVX2(8, 4);
|
||||
HIGHBD_MASKSADMXN_AVX2(8, 8);
|
||||
HIGHBD_MASKSADMXN_AVX2(8, 16);
|
||||
HIGHBD_MASKSADMXN_AVX2(16, 8);
|
||||
HIGHBD_MASKSADMXN_AVX2(16, 16);
|
||||
HIGHBD_MASKSADMXN_AVX2(16, 32);
|
||||
HIGHBD_MASKSADMXN_AVX2(32, 16);
|
||||
HIGHBD_MASKSADMXN_AVX2(32, 32);
|
||||
HIGHBD_MASKSADMXN_AVX2(32, 64);
|
||||
HIGHBD_MASKSADMXN_AVX2(64, 32);
|
||||
HIGHBD_MASKSADMXN_AVX2(64, 64);
|
||||
HIGHBD_MASKSADMXN_AVX2(64, 128);
|
||||
HIGHBD_MASKSADMXN_AVX2(128, 64);
|
||||
HIGHBD_MASKSADMXN_AVX2(128, 128);
|
||||
HIGHBD_MASKSADMXN_AVX2(4, 16);
|
||||
HIGHBD_MASKSADMXN_AVX2(16, 4);
|
||||
HIGHBD_MASKSADMXN_AVX2(8, 32);
|
||||
HIGHBD_MASKSADMXN_AVX2(32, 8);
|
||||
HIGHBD_MASKSADMXN_AVX2(16, 64);
|
||||
HIGHBD_MASKSADMXN_AVX2(64, 16);
|
||||
|
|
@ -19,6 +19,8 @@
|
|||
#include "aom/aom_integer.h"
|
||||
#include "aom_dsp/x86/synonyms.h"
|
||||
|
||||
#include "aom_dsp/x86//masked_sad_intrin_ssse3.h"
|
||||
|
||||
// For width a multiple of 16
|
||||
static INLINE unsigned int masked_sad_ssse3(const uint8_t *src_ptr,
|
||||
int src_stride,
|
||||
|
|
@ -27,16 +29,6 @@ static INLINE unsigned int masked_sad_ssse3(const uint8_t *src_ptr,
|
|||
const uint8_t *m_ptr, int m_stride,
|
||||
int width, int height);
|
||||
|
||||
static INLINE unsigned int masked_sad8xh_ssse3(
|
||||
const uint8_t *src_ptr, int src_stride, const uint8_t *a_ptr, int a_stride,
|
||||
const uint8_t *b_ptr, int b_stride, const uint8_t *m_ptr, int m_stride,
|
||||
int height);
|
||||
|
||||
static INLINE unsigned int masked_sad4xh_ssse3(
|
||||
const uint8_t *src_ptr, int src_stride, const uint8_t *a_ptr, int a_stride,
|
||||
const uint8_t *b_ptr, int b_stride, const uint8_t *m_ptr, int m_stride,
|
||||
int height);
|
||||
|
||||
#define MASKSADMXN_SSSE3(m, n) \
|
||||
unsigned int aom_masked_sad##m##x##n##_ssse3( \
|
||||
const uint8_t *src, int src_stride, const uint8_t *ref, int ref_stride, \
|
||||
|
|
@ -56,11 +48,11 @@ static INLINE unsigned int masked_sad4xh_ssse3(
|
|||
const uint8_t *second_pred, const uint8_t *msk, int msk_stride, \
|
||||
int invert_mask) { \
|
||||
if (!invert_mask) \
|
||||
return masked_sad8xh_ssse3(src, src_stride, ref, ref_stride, \
|
||||
second_pred, 8, msk, msk_stride, n); \
|
||||
return aom_masked_sad8xh_ssse3(src, src_stride, ref, ref_stride, \
|
||||
second_pred, 8, msk, msk_stride, n); \
|
||||
else \
|
||||
return masked_sad8xh_ssse3(src, src_stride, second_pred, 8, ref, \
|
||||
ref_stride, msk, msk_stride, n); \
|
||||
return aom_masked_sad8xh_ssse3(src, src_stride, second_pred, 8, ref, \
|
||||
ref_stride, msk, msk_stride, n); \
|
||||
}
|
||||
|
||||
#define MASKSAD4XN_SSSE3(n) \
|
||||
|
|
@ -69,11 +61,11 @@ static INLINE unsigned int masked_sad4xh_ssse3(
|
|||
const uint8_t *second_pred, const uint8_t *msk, int msk_stride, \
|
||||
int invert_mask) { \
|
||||
if (!invert_mask) \
|
||||
return masked_sad4xh_ssse3(src, src_stride, ref, ref_stride, \
|
||||
second_pred, 4, msk, msk_stride, n); \
|
||||
return aom_masked_sad4xh_ssse3(src, src_stride, ref, ref_stride, \
|
||||
second_pred, 4, msk, msk_stride, n); \
|
||||
else \
|
||||
return masked_sad4xh_ssse3(src, src_stride, second_pred, 4, ref, \
|
||||
ref_stride, msk, msk_stride, n); \
|
||||
return aom_masked_sad4xh_ssse3(src, src_stride, second_pred, 4, ref, \
|
||||
ref_stride, msk, msk_stride, n); \
|
||||
}
|
||||
|
||||
MASKSADMXN_SSSE3(128, 128)
|
||||
|
|
@ -145,10 +137,11 @@ static INLINE unsigned int masked_sad_ssse3(const uint8_t *src_ptr,
|
|||
return (sad + 31) >> 6;
|
||||
}
|
||||
|
||||
static INLINE unsigned int masked_sad8xh_ssse3(
|
||||
const uint8_t *src_ptr, int src_stride, const uint8_t *a_ptr, int a_stride,
|
||||
const uint8_t *b_ptr, int b_stride, const uint8_t *m_ptr, int m_stride,
|
||||
int height) {
|
||||
unsigned int aom_masked_sad8xh_ssse3(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *a_ptr, int a_stride,
|
||||
const uint8_t *b_ptr, int b_stride,
|
||||
const uint8_t *m_ptr, int m_stride,
|
||||
int height) {
|
||||
int y;
|
||||
__m128i res = _mm_setzero_si128();
|
||||
const __m128i mask_max = _mm_set1_epi8((1 << AOM_BLEND_A64_ROUND_BITS));
|
||||
|
|
@ -189,10 +182,11 @@ static INLINE unsigned int masked_sad8xh_ssse3(
|
|||
return (sad + 31) >> 6;
|
||||
}
|
||||
|
||||
static INLINE unsigned int masked_sad4xh_ssse3(
|
||||
const uint8_t *src_ptr, int src_stride, const uint8_t *a_ptr, int a_stride,
|
||||
const uint8_t *b_ptr, int b_stride, const uint8_t *m_ptr, int m_stride,
|
||||
int height) {
|
||||
unsigned int aom_masked_sad4xh_ssse3(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *a_ptr, int a_stride,
|
||||
const uint8_t *b_ptr, int b_stride,
|
||||
const uint8_t *m_ptr, int m_stride,
|
||||
int height) {
|
||||
int y;
|
||||
__m128i res = _mm_setzero_si128();
|
||||
const __m128i mask_max = _mm_set1_epi8((1 << AOM_BLEND_A64_ROUND_BITS));
|
||||
|
|
@ -238,11 +232,6 @@ static INLINE unsigned int highbd_masked_sad_ssse3(
|
|||
const uint8_t *b8, int b_stride, const uint8_t *m_ptr, int m_stride,
|
||||
int width, int height);
|
||||
|
||||
static INLINE unsigned int highbd_masked_sad4xh_ssse3(
|
||||
const uint8_t *src8, int src_stride, const uint8_t *a8, int a_stride,
|
||||
const uint8_t *b8, int b_stride, const uint8_t *m_ptr, int m_stride,
|
||||
int height);
|
||||
|
||||
#define HIGHBD_MASKSADMXN_SSSE3(m, n) \
|
||||
unsigned int aom_highbd_masked_sad##m##x##n##_ssse3( \
|
||||
const uint8_t *src8, int src_stride, const uint8_t *ref8, \
|
||||
|
|
@ -262,11 +251,13 @@ static INLINE unsigned int highbd_masked_sad4xh_ssse3(
|
|||
int ref_stride, const uint8_t *second_pred8, const uint8_t *msk, \
|
||||
int msk_stride, int invert_mask) { \
|
||||
if (!invert_mask) \
|
||||
return highbd_masked_sad4xh_ssse3(src8, src_stride, ref8, ref_stride, \
|
||||
second_pred8, 4, msk, msk_stride, n); \
|
||||
return aom_highbd_masked_sad4xh_ssse3(src8, src_stride, ref8, \
|
||||
ref_stride, second_pred8, 4, msk, \
|
||||
msk_stride, n); \
|
||||
else \
|
||||
return highbd_masked_sad4xh_ssse3(src8, src_stride, second_pred8, 4, \
|
||||
ref8, ref_stride, msk, msk_stride, n); \
|
||||
return aom_highbd_masked_sad4xh_ssse3(src8, src_stride, second_pred8, 4, \
|
||||
ref8, ref_stride, msk, msk_stride, \
|
||||
n); \
|
||||
}
|
||||
|
||||
HIGHBD_MASKSADMXN_SSSE3(128, 128)
|
||||
|
|
@ -350,10 +341,11 @@ static INLINE unsigned int highbd_masked_sad_ssse3(
|
|||
return (sad + 31) >> 6;
|
||||
}
|
||||
|
||||
static INLINE unsigned int highbd_masked_sad4xh_ssse3(
|
||||
const uint8_t *src8, int src_stride, const uint8_t *a8, int a_stride,
|
||||
const uint8_t *b8, int b_stride, const uint8_t *m_ptr, int m_stride,
|
||||
int height) {
|
||||
unsigned int aom_highbd_masked_sad4xh_ssse3(const uint8_t *src8, int src_stride,
|
||||
const uint8_t *a8, int a_stride,
|
||||
const uint8_t *b8, int b_stride,
|
||||
const uint8_t *m_ptr, int m_stride,
|
||||
int height) {
|
||||
const uint16_t *src_ptr = CONVERT_TO_SHORTPTR(src8);
|
||||
const uint16_t *a_ptr = CONVERT_TO_SHORTPTR(a8);
|
||||
const uint16_t *b_ptr = CONVERT_TO_SHORTPTR(b8);
|
||||
|
|
|
|||
33
third_party/aom/aom_dsp/x86/masked_sad_intrin_ssse3.h
vendored
Normal file
33
third_party/aom/aom_dsp/x86/masked_sad_intrin_ssse3.h
vendored
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
/*
|
||||
* Copyright (c) 2018, Alliance for Open Media. All rights reserved
|
||||
*
|
||||
* This source code is subject to the terms of the BSD 2 Clause License and
|
||||
* the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
|
||||
* was not distributed with this source code in the LICENSE file, you can
|
||||
* obtain it at www.aomedia.org/license/software. If the Alliance for Open
|
||||
* Media Patent License 1.0 was not distributed with this source code in the
|
||||
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
|
||||
*/
|
||||
|
||||
#ifndef _AOM_DSP_X86_MASKED_SAD_INTRIN_SSSE3_H
|
||||
#define _AOM_DSP_X86_MASKED_SAD_INTRIN_SSSE3_H
|
||||
|
||||
unsigned int aom_masked_sad8xh_ssse3(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *a_ptr, int a_stride,
|
||||
const uint8_t *b_ptr, int b_stride,
|
||||
const uint8_t *m_ptr, int m_stride,
|
||||
int height);
|
||||
|
||||
unsigned int aom_masked_sad4xh_ssse3(const uint8_t *src_ptr, int src_stride,
|
||||
const uint8_t *a_ptr, int a_stride,
|
||||
const uint8_t *b_ptr, int b_stride,
|
||||
const uint8_t *m_ptr, int m_stride,
|
||||
int height);
|
||||
|
||||
unsigned int aom_highbd_masked_sad4xh_ssse3(const uint8_t *src8, int src_stride,
|
||||
const uint8_t *a8, int a_stride,
|
||||
const uint8_t *b8, int b_stride,
|
||||
const uint8_t *m_ptr, int m_stride,
|
||||
int height);
|
||||
|
||||
#endif
|
||||
270
third_party/aom/aom_dsp/x86/obmc_sad_avx2.c
vendored
Normal file
270
third_party/aom/aom_dsp/x86/obmc_sad_avx2.c
vendored
Normal file
|
|
@ -0,0 +1,270 @@
|
|||
/*
|
||||
* Copyright (c) 2018, Alliance for Open Media. All rights reserved
|
||||
*
|
||||
* This source code is subject to the terms of the BSD 2 Clause License and
|
||||
* the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
|
||||
* was not distributed with this source code in the LICENSE file, you can
|
||||
* obtain it at www.aomedia.org/license/software. If the Alliance for Open
|
||||
* Media Patent License 1.0 was not distributed with this source code in the
|
||||
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
|
||||
*/
|
||||
|
||||
#include <assert.h>
|
||||
#include <immintrin.h>
|
||||
|
||||
#include "config/aom_config.h"
|
||||
|
||||
#include "aom_ports/mem.h"
|
||||
#include "aom/aom_integer.h"
|
||||
|
||||
#include "aom_dsp/aom_dsp_common.h"
|
||||
#include "aom_dsp/x86/obmc_intrinsic_ssse3.h"
|
||||
#include "aom_dsp/x86/synonyms.h"
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// 8 bit
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
static INLINE unsigned int obmc_sad_w4_avx2(const uint8_t *pre,
|
||||
const int pre_stride,
|
||||
const int32_t *wsrc,
|
||||
const int32_t *mask,
|
||||
const int height) {
|
||||
int n = 0;
|
||||
__m256i v_sad_d = _mm256_setzero_si256();
|
||||
const __m256i v_bias_d = _mm256_set1_epi32((1 << 12) >> 1);
|
||||
|
||||
do {
|
||||
const __m128i v_p_b_0 = xx_loadl_32(pre);
|
||||
const __m128i v_p_b_1 = xx_loadl_32(pre + pre_stride);
|
||||
const __m128i v_p_b = _mm_unpacklo_epi32(v_p_b_0, v_p_b_1);
|
||||
const __m256i v_m_d = _mm256_lddqu_si256((__m256i *)(mask + n));
|
||||
const __m256i v_w_d = _mm256_lddqu_si256((__m256i *)(wsrc + n));
|
||||
|
||||
const __m256i v_p_d = _mm256_cvtepu8_epi32(v_p_b);
|
||||
|
||||
// Values in both pre and mask fit in 15 bits, and are packed at 32 bit
|
||||
// boundaries. We use pmaddwd, as it has lower latency on Haswell
|
||||
// than pmulld but produces the same result with these inputs.
|
||||
const __m256i v_pm_d = _mm256_madd_epi16(v_p_d, v_m_d);
|
||||
|
||||
const __m256i v_diff_d = _mm256_sub_epi32(v_w_d, v_pm_d);
|
||||
const __m256i v_absdiff_d = _mm256_abs_epi32(v_diff_d);
|
||||
|
||||
// Rounded absolute difference
|
||||
const __m256i v_tmp_d = _mm256_add_epi32(v_absdiff_d, v_bias_d);
|
||||
const __m256i v_rad_d = _mm256_srli_epi32(v_tmp_d, 12);
|
||||
|
||||
v_sad_d = _mm256_add_epi32(v_sad_d, v_rad_d);
|
||||
|
||||
n += 8;
|
||||
pre += pre_stride << 1;
|
||||
} while (n < 8 * (height >> 1));
|
||||
|
||||
__m128i v_sad_d_0 = _mm256_castsi256_si128(v_sad_d);
|
||||
__m128i v_sad_d_1 = _mm256_extracti128_si256(v_sad_d, 1);
|
||||
v_sad_d_0 = _mm_add_epi32(v_sad_d_0, v_sad_d_1);
|
||||
return xx_hsum_epi32_si32(v_sad_d_0);
|
||||
}
|
||||
|
||||
static INLINE unsigned int obmc_sad_w8n_avx2(
|
||||
const uint8_t *pre, const int pre_stride, const int32_t *wsrc,
|
||||
const int32_t *mask, const int width, const int height) {
|
||||
const int pre_step = pre_stride - width;
|
||||
int n = 0;
|
||||
__m256i v_sad_d = _mm256_setzero_si256();
|
||||
const __m256i v_bias_d = _mm256_set1_epi32((1 << 12) >> 1);
|
||||
assert(width >= 8);
|
||||
assert(IS_POWER_OF_TWO(width));
|
||||
|
||||
do {
|
||||
const __m128i v_p0_b = xx_loadl_64(pre + n);
|
||||
const __m256i v_m0_d = _mm256_lddqu_si256((__m256i *)(mask + n));
|
||||
const __m256i v_w0_d = _mm256_lddqu_si256((__m256i *)(wsrc + n));
|
||||
|
||||
const __m256i v_p0_d = _mm256_cvtepu8_epi32(v_p0_b);
|
||||
|
||||
// Values in both pre and mask fit in 15 bits, and are packed at 32 bit
|
||||
// boundaries. We use pmaddwd, as it has lower latency on Haswell
|
||||
// than pmulld but produces the same result with these inputs.
|
||||
const __m256i v_pm0_d = _mm256_madd_epi16(v_p0_d, v_m0_d);
|
||||
|
||||
const __m256i v_diff0_d = _mm256_sub_epi32(v_w0_d, v_pm0_d);
|
||||
const __m256i v_absdiff0_d = _mm256_abs_epi32(v_diff0_d);
|
||||
|
||||
// Rounded absolute difference
|
||||
const __m256i v_tmp_d = _mm256_add_epi32(v_absdiff0_d, v_bias_d);
|
||||
const __m256i v_rad0_d = _mm256_srli_epi32(v_tmp_d, 12);
|
||||
|
||||
v_sad_d = _mm256_add_epi32(v_sad_d, v_rad0_d);
|
||||
|
||||
n += 8;
|
||||
|
||||
if ((n & (width - 1)) == 0) pre += pre_step;
|
||||
} while (n < width * height);
|
||||
|
||||
__m128i v_sad_d_0 = _mm256_castsi256_si128(v_sad_d);
|
||||
__m128i v_sad_d_1 = _mm256_extracti128_si256(v_sad_d, 1);
|
||||
v_sad_d_0 = _mm_add_epi32(v_sad_d_0, v_sad_d_1);
|
||||
return xx_hsum_epi32_si32(v_sad_d_0);
|
||||
}
|
||||
|
||||
#define OBMCSADWXH(w, h) \
|
||||
unsigned int aom_obmc_sad##w##x##h##_avx2( \
|
||||
const uint8_t *pre, int pre_stride, const int32_t *wsrc, \
|
||||
const int32_t *msk) { \
|
||||
if (w == 4) { \
|
||||
return obmc_sad_w4_avx2(pre, pre_stride, wsrc, msk, h); \
|
||||
} else { \
|
||||
return obmc_sad_w8n_avx2(pre, pre_stride, wsrc, msk, w, h); \
|
||||
} \
|
||||
}
|
||||
|
||||
OBMCSADWXH(128, 128)
|
||||
OBMCSADWXH(128, 64)
|
||||
OBMCSADWXH(64, 128)
|
||||
OBMCSADWXH(64, 64)
|
||||
OBMCSADWXH(64, 32)
|
||||
OBMCSADWXH(32, 64)
|
||||
OBMCSADWXH(32, 32)
|
||||
OBMCSADWXH(32, 16)
|
||||
OBMCSADWXH(16, 32)
|
||||
OBMCSADWXH(16, 16)
|
||||
OBMCSADWXH(16, 8)
|
||||
OBMCSADWXH(8, 16)
|
||||
OBMCSADWXH(8, 8)
|
||||
OBMCSADWXH(8, 4)
|
||||
OBMCSADWXH(4, 8)
|
||||
OBMCSADWXH(4, 4)
|
||||
OBMCSADWXH(4, 16)
|
||||
OBMCSADWXH(16, 4)
|
||||
OBMCSADWXH(8, 32)
|
||||
OBMCSADWXH(32, 8)
|
||||
OBMCSADWXH(16, 64)
|
||||
OBMCSADWXH(64, 16)
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// High bit-depth
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
static INLINE unsigned int hbd_obmc_sad_w4_avx2(const uint8_t *pre8,
|
||||
const int pre_stride,
|
||||
const int32_t *wsrc,
|
||||
const int32_t *mask,
|
||||
const int height) {
|
||||
const uint16_t *pre = CONVERT_TO_SHORTPTR(pre8);
|
||||
int n = 0;
|
||||
__m256i v_sad_d = _mm256_setzero_si256();
|
||||
const __m256i v_bias_d = _mm256_set1_epi32((1 << 12) >> 1);
|
||||
do {
|
||||
const __m128i v_p_w_0 = xx_loadl_64(pre);
|
||||
const __m128i v_p_w_1 = xx_loadl_64(pre + pre_stride);
|
||||
const __m128i v_p_w = _mm_unpacklo_epi64(v_p_w_0, v_p_w_1);
|
||||
const __m256i v_m_d = _mm256_lddqu_si256((__m256i *)(mask + n));
|
||||
const __m256i v_w_d = _mm256_lddqu_si256((__m256i *)(wsrc + n));
|
||||
|
||||
const __m256i v_p_d = _mm256_cvtepu16_epi32(v_p_w);
|
||||
|
||||
// Values in both pre and mask fit in 15 bits, and are packed at 32 bit
|
||||
// boundaries. We use pmaddwd, as it has lower latency on Haswell
|
||||
// than pmulld but produces the same result with these inputs.
|
||||
const __m256i v_pm_d = _mm256_madd_epi16(v_p_d, v_m_d);
|
||||
|
||||
const __m256i v_diff_d = _mm256_sub_epi32(v_w_d, v_pm_d);
|
||||
const __m256i v_absdiff_d = _mm256_abs_epi32(v_diff_d);
|
||||
|
||||
// Rounded absolute difference
|
||||
|
||||
const __m256i v_tmp_d = _mm256_add_epi32(v_absdiff_d, v_bias_d);
|
||||
const __m256i v_rad_d = _mm256_srli_epi32(v_tmp_d, 12);
|
||||
|
||||
v_sad_d = _mm256_add_epi32(v_sad_d, v_rad_d);
|
||||
|
||||
n += 8;
|
||||
|
||||
pre += pre_stride << 1;
|
||||
} while (n < 8 * (height >> 1));
|
||||
|
||||
__m128i v_sad_d_0 = _mm256_castsi256_si128(v_sad_d);
|
||||
__m128i v_sad_d_1 = _mm256_extracti128_si256(v_sad_d, 1);
|
||||
v_sad_d_0 = _mm_add_epi32(v_sad_d_0, v_sad_d_1);
|
||||
return xx_hsum_epi32_si32(v_sad_d_0);
|
||||
}
|
||||
|
||||
static INLINE unsigned int hbd_obmc_sad_w8n_avx2(
|
||||
const uint8_t *pre8, const int pre_stride, const int32_t *wsrc,
|
||||
const int32_t *mask, const int width, const int height) {
|
||||
const uint16_t *pre = CONVERT_TO_SHORTPTR(pre8);
|
||||
const int pre_step = pre_stride - width;
|
||||
int n = 0;
|
||||
__m256i v_sad_d = _mm256_setzero_si256();
|
||||
const __m256i v_bias_d = _mm256_set1_epi32((1 << 12) >> 1);
|
||||
|
||||
assert(width >= 8);
|
||||
assert(IS_POWER_OF_TWO(width));
|
||||
|
||||
do {
|
||||
const __m128i v_p0_w = _mm_lddqu_si128((__m128i *)(pre + n));
|
||||
const __m256i v_m0_d = _mm256_lddqu_si256((__m256i *)(mask + n));
|
||||
const __m256i v_w0_d = _mm256_lddqu_si256((__m256i *)(wsrc + n));
|
||||
|
||||
const __m256i v_p0_d = _mm256_cvtepu16_epi32(v_p0_w);
|
||||
|
||||
// Values in both pre and mask fit in 15 bits, and are packed at 32 bit
|
||||
// boundaries. We use pmaddwd, as it has lower latency on Haswell
|
||||
// than pmulld but produces the same result with these inputs.
|
||||
const __m256i v_pm0_d = _mm256_madd_epi16(v_p0_d, v_m0_d);
|
||||
|
||||
const __m256i v_diff0_d = _mm256_sub_epi32(v_w0_d, v_pm0_d);
|
||||
const __m256i v_absdiff0_d = _mm256_abs_epi32(v_diff0_d);
|
||||
|
||||
// Rounded absolute difference
|
||||
const __m256i v_tmp_d = _mm256_add_epi32(v_absdiff0_d, v_bias_d);
|
||||
const __m256i v_rad0_d = _mm256_srli_epi32(v_tmp_d, 12);
|
||||
|
||||
v_sad_d = _mm256_add_epi32(v_sad_d, v_rad0_d);
|
||||
|
||||
n += 8;
|
||||
|
||||
if (n % width == 0) pre += pre_step;
|
||||
} while (n < width * height);
|
||||
|
||||
__m128i v_sad_d_0 = _mm256_castsi256_si128(v_sad_d);
|
||||
__m128i v_sad_d_1 = _mm256_extracti128_si256(v_sad_d, 1);
|
||||
v_sad_d_0 = _mm_add_epi32(v_sad_d_0, v_sad_d_1);
|
||||
return xx_hsum_epi32_si32(v_sad_d_0);
|
||||
}
|
||||
|
||||
#define HBD_OBMCSADWXH(w, h) \
|
||||
unsigned int aom_highbd_obmc_sad##w##x##h##_avx2( \
|
||||
const uint8_t *pre, int pre_stride, const int32_t *wsrc, \
|
||||
const int32_t *mask) { \
|
||||
if (w == 4) { \
|
||||
return hbd_obmc_sad_w4_avx2(pre, pre_stride, wsrc, mask, h); \
|
||||
} else { \
|
||||
return hbd_obmc_sad_w8n_avx2(pre, pre_stride, wsrc, mask, w, h); \
|
||||
} \
|
||||
}
|
||||
|
||||
HBD_OBMCSADWXH(128, 128)
|
||||
HBD_OBMCSADWXH(128, 64)
|
||||
HBD_OBMCSADWXH(64, 128)
|
||||
HBD_OBMCSADWXH(64, 64)
|
||||
HBD_OBMCSADWXH(64, 32)
|
||||
HBD_OBMCSADWXH(32, 64)
|
||||
HBD_OBMCSADWXH(32, 32)
|
||||
HBD_OBMCSADWXH(32, 16)
|
||||
HBD_OBMCSADWXH(16, 32)
|
||||
HBD_OBMCSADWXH(16, 16)
|
||||
HBD_OBMCSADWXH(16, 8)
|
||||
HBD_OBMCSADWXH(8, 16)
|
||||
HBD_OBMCSADWXH(8, 8)
|
||||
HBD_OBMCSADWXH(8, 4)
|
||||
HBD_OBMCSADWXH(4, 8)
|
||||
HBD_OBMCSADWXH(4, 4)
|
||||
HBD_OBMCSADWXH(4, 16)
|
||||
HBD_OBMCSADWXH(16, 4)
|
||||
HBD_OBMCSADWXH(8, 32)
|
||||
HBD_OBMCSADWXH(32, 8)
|
||||
HBD_OBMCSADWXH(16, 64)
|
||||
HBD_OBMCSADWXH(64, 16)
|
||||
50
third_party/aom/aom_dsp/x86/obmc_variance_sse4.c
vendored
50
third_party/aom/aom_dsp/x86/obmc_variance_sse4.c
vendored
|
|
@ -26,6 +26,16 @@
|
|||
// 8 bit
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
void aom_var_filter_block2d_bil_first_pass_ssse3(
|
||||
const uint8_t *a, uint16_t *b, unsigned int src_pixels_per_line,
|
||||
unsigned int pixel_step, unsigned int output_height,
|
||||
unsigned int output_width, const uint8_t *filter);
|
||||
|
||||
void aom_var_filter_block2d_bil_second_pass_ssse3(
|
||||
const uint16_t *a, uint8_t *b, unsigned int src_pixels_per_line,
|
||||
unsigned int pixel_step, unsigned int output_height,
|
||||
unsigned int output_width, const uint8_t *filter);
|
||||
|
||||
static INLINE void obmc_variance_w4(const uint8_t *pre, const int pre_stride,
|
||||
const int32_t *wsrc, const int32_t *mask,
|
||||
unsigned int *const sse, int *const sum,
|
||||
|
|
@ -152,6 +162,46 @@ OBMCVARWXH(32, 8)
|
|||
OBMCVARWXH(16, 64)
|
||||
OBMCVARWXH(64, 16)
|
||||
|
||||
#include "config/aom_dsp_rtcd.h"
|
||||
|
||||
#define OBMC_SUBPIX_VAR(W, H) \
|
||||
uint32_t aom_obmc_sub_pixel_variance##W##x##H##_sse4_1( \
|
||||
const uint8_t *pre, int pre_stride, int xoffset, int yoffset, \
|
||||
const int32_t *wsrc, const int32_t *mask, unsigned int *sse) { \
|
||||
uint16_t fdata3[(H + 1) * W]; \
|
||||
uint8_t temp2[H * W]; \
|
||||
\
|
||||
aom_var_filter_block2d_bil_first_pass_ssse3( \
|
||||
pre, fdata3, pre_stride, 1, H + 1, W, bilinear_filters_2t[xoffset]); \
|
||||
aom_var_filter_block2d_bil_second_pass_ssse3( \
|
||||
fdata3, temp2, W, W, H, W, bilinear_filters_2t[yoffset]); \
|
||||
\
|
||||
return aom_obmc_variance##W##x##H##_sse4_1(temp2, W, wsrc, mask, sse); \
|
||||
}
|
||||
|
||||
OBMC_SUBPIX_VAR(128, 128)
|
||||
OBMC_SUBPIX_VAR(128, 64)
|
||||
OBMC_SUBPIX_VAR(64, 128)
|
||||
OBMC_SUBPIX_VAR(64, 64)
|
||||
OBMC_SUBPIX_VAR(64, 32)
|
||||
OBMC_SUBPIX_VAR(32, 64)
|
||||
OBMC_SUBPIX_VAR(32, 32)
|
||||
OBMC_SUBPIX_VAR(32, 16)
|
||||
OBMC_SUBPIX_VAR(16, 32)
|
||||
OBMC_SUBPIX_VAR(16, 16)
|
||||
OBMC_SUBPIX_VAR(16, 8)
|
||||
OBMC_SUBPIX_VAR(8, 16)
|
||||
OBMC_SUBPIX_VAR(8, 8)
|
||||
OBMC_SUBPIX_VAR(8, 4)
|
||||
OBMC_SUBPIX_VAR(4, 8)
|
||||
OBMC_SUBPIX_VAR(4, 4)
|
||||
OBMC_SUBPIX_VAR(4, 16)
|
||||
OBMC_SUBPIX_VAR(16, 4)
|
||||
OBMC_SUBPIX_VAR(8, 32)
|
||||
OBMC_SUBPIX_VAR(32, 8)
|
||||
OBMC_SUBPIX_VAR(16, 64)
|
||||
OBMC_SUBPIX_VAR(64, 16)
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// High bit-depth
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
|
|
|||
108
third_party/aom/aom_dsp/x86/subtract_avx2.c
vendored
Normal file
108
third_party/aom/aom_dsp/x86/subtract_avx2.c
vendored
Normal file
|
|
@ -0,0 +1,108 @@
|
|||
/*
|
||||
* Copyright (c) 2018, Alliance for Open Media. All rights reserved
|
||||
*
|
||||
* This source code is subject to the terms of the BSD 2 Clause License and
|
||||
* the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
|
||||
* was not distributed with this source code in the LICENSE file, you can
|
||||
* obtain it at www.aomedia.org/license/software. If the Alliance for Open
|
||||
* Media Patent License 1.0 was not distributed with this source code in the
|
||||
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
|
||||
*/
|
||||
#include <immintrin.h>
|
||||
|
||||
#include "config/aom_dsp_rtcd.h"
|
||||
|
||||
static INLINE void subtract32_avx2(int16_t *diff_ptr, const uint8_t *src_ptr,
|
||||
const uint8_t *pred_ptr) {
|
||||
__m256i s = _mm256_lddqu_si256((__m256i *)(src_ptr));
|
||||
__m256i p = _mm256_lddqu_si256((__m256i *)(pred_ptr));
|
||||
__m256i s_0 = _mm256_cvtepu8_epi16(_mm256_castsi256_si128(s));
|
||||
__m256i s_1 = _mm256_cvtepu8_epi16(_mm256_extracti128_si256(s, 1));
|
||||
__m256i p_0 = _mm256_cvtepu8_epi16(_mm256_castsi256_si128(p));
|
||||
__m256i p_1 = _mm256_cvtepu8_epi16(_mm256_extracti128_si256(p, 1));
|
||||
const __m256i d_0 = _mm256_sub_epi16(s_0, p_0);
|
||||
const __m256i d_1 = _mm256_sub_epi16(s_1, p_1);
|
||||
_mm256_store_si256((__m256i *)(diff_ptr), d_0);
|
||||
_mm256_store_si256((__m256i *)(diff_ptr + 16), d_1);
|
||||
}
|
||||
|
||||
static INLINE void aom_subtract_block_16xn_avx2(
|
||||
int rows, int16_t *diff_ptr, ptrdiff_t diff_stride, const uint8_t *src_ptr,
|
||||
ptrdiff_t src_stride, const uint8_t *pred_ptr, ptrdiff_t pred_stride) {
|
||||
for (int32_t j = 0; j < rows; ++j) {
|
||||
__m128i s = _mm_lddqu_si128((__m128i *)(src_ptr));
|
||||
__m128i p = _mm_lddqu_si128((__m128i *)(pred_ptr));
|
||||
__m256i s_0 = _mm256_cvtepu8_epi16(s);
|
||||
__m256i p_0 = _mm256_cvtepu8_epi16(p);
|
||||
const __m256i d_0 = _mm256_sub_epi16(s_0, p_0);
|
||||
_mm256_store_si256((__m256i *)(diff_ptr), d_0);
|
||||
src_ptr += src_stride;
|
||||
pred_ptr += pred_stride;
|
||||
diff_ptr += diff_stride;
|
||||
}
|
||||
}
|
||||
|
||||
static INLINE void aom_subtract_block_32xn_avx2(
|
||||
int rows, int16_t *diff_ptr, ptrdiff_t diff_stride, const uint8_t *src_ptr,
|
||||
ptrdiff_t src_stride, const uint8_t *pred_ptr, ptrdiff_t pred_stride) {
|
||||
for (int32_t j = 0; j < rows; ++j) {
|
||||
subtract32_avx2(diff_ptr, src_ptr, pred_ptr);
|
||||
src_ptr += src_stride;
|
||||
pred_ptr += pred_stride;
|
||||
diff_ptr += diff_stride;
|
||||
}
|
||||
}
|
||||
|
||||
static INLINE void aom_subtract_block_64xn_avx2(
|
||||
int rows, int16_t *diff_ptr, ptrdiff_t diff_stride, const uint8_t *src_ptr,
|
||||
ptrdiff_t src_stride, const uint8_t *pred_ptr, ptrdiff_t pred_stride) {
|
||||
for (int32_t j = 0; j < rows; ++j) {
|
||||
subtract32_avx2(diff_ptr, src_ptr, pred_ptr);
|
||||
subtract32_avx2(diff_ptr + 32, src_ptr + 32, pred_ptr + 32);
|
||||
src_ptr += src_stride;
|
||||
pred_ptr += pred_stride;
|
||||
diff_ptr += diff_stride;
|
||||
}
|
||||
}
|
||||
|
||||
static INLINE void aom_subtract_block_128xn_avx2(
|
||||
int rows, int16_t *diff_ptr, ptrdiff_t diff_stride, const uint8_t *src_ptr,
|
||||
ptrdiff_t src_stride, const uint8_t *pred_ptr, ptrdiff_t pred_stride) {
|
||||
for (int32_t j = 0; j < rows; ++j) {
|
||||
subtract32_avx2(diff_ptr, src_ptr, pred_ptr);
|
||||
subtract32_avx2(diff_ptr + 32, src_ptr + 32, pred_ptr + 32);
|
||||
subtract32_avx2(diff_ptr + 64, src_ptr + 64, pred_ptr + 64);
|
||||
subtract32_avx2(diff_ptr + 96, src_ptr + 96, pred_ptr + 96);
|
||||
src_ptr += src_stride;
|
||||
pred_ptr += pred_stride;
|
||||
diff_ptr += diff_stride;
|
||||
}
|
||||
}
|
||||
|
||||
void aom_subtract_block_avx2(int rows, int cols, int16_t *diff_ptr,
|
||||
ptrdiff_t diff_stride, const uint8_t *src_ptr,
|
||||
ptrdiff_t src_stride, const uint8_t *pred_ptr,
|
||||
ptrdiff_t pred_stride) {
|
||||
switch (cols) {
|
||||
case 16:
|
||||
aom_subtract_block_16xn_avx2(rows, diff_ptr, diff_stride, src_ptr,
|
||||
src_stride, pred_ptr, pred_stride);
|
||||
break;
|
||||
case 32:
|
||||
aom_subtract_block_32xn_avx2(rows, diff_ptr, diff_stride, src_ptr,
|
||||
src_stride, pred_ptr, pred_stride);
|
||||
break;
|
||||
case 64:
|
||||
aom_subtract_block_64xn_avx2(rows, diff_ptr, diff_stride, src_ptr,
|
||||
src_stride, pred_ptr, pred_stride);
|
||||
break;
|
||||
case 128:
|
||||
aom_subtract_block_128xn_avx2(rows, diff_ptr, diff_stride, src_ptr,
|
||||
src_stride, pred_ptr, pred_stride);
|
||||
break;
|
||||
default:
|
||||
aom_subtract_block_sse2(rows, cols, diff_ptr, diff_stride, src_ptr,
|
||||
src_stride, pred_ptr, pred_stride);
|
||||
break;
|
||||
}
|
||||
}
|
||||
199
third_party/aom/aom_dsp/x86/txfm_common_avx2.h
vendored
Normal file
199
third_party/aom/aom_dsp/x86/txfm_common_avx2.h
vendored
Normal file
|
|
@ -0,0 +1,199 @@
|
|||
/*
|
||||
* Copyright (c) 2018, Alliance for Open Media. All rights reserved
|
||||
*
|
||||
* This source code is subject to the terms of the BSD 2 Clause License and
|
||||
* the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
|
||||
* was not distributed with this source code in the LICENSE file, you can
|
||||
* obtain it at www.aomedia.org/license/software. If the Alliance for Open
|
||||
* Media Patent License 1.0 was not distributed with this source code in the
|
||||
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
|
||||
*/
|
||||
|
||||
#ifndef AOM_DSP_X86_TXFM_COMMON_AVX2_H_
|
||||
#define AOM_DSP_X86_TXFM_COMMON_AVX2_H_
|
||||
|
||||
#include <emmintrin.h>
|
||||
#include "aom/aom_integer.h"
|
||||
#include "aom_dsp/x86/synonyms.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
typedef void (*transform_1d_avx2)(const __m256i *input, __m256i *output,
|
||||
int8_t cos_bit);
|
||||
|
||||
static INLINE __m256i pair_set_w16_epi16(int16_t a, int16_t b) {
|
||||
return _mm256_set1_epi32(
|
||||
(int32_t)(((uint16_t)(a)) | (((uint32_t)(b)) << 16)));
|
||||
}
|
||||
|
||||
static INLINE void btf_16_w16_avx2(const __m256i w0, const __m256i w1,
|
||||
__m256i *in0, __m256i *in1, const __m256i _r,
|
||||
const int32_t cos_bit) {
|
||||
__m256i t0 = _mm256_unpacklo_epi16(*in0, *in1);
|
||||
__m256i t1 = _mm256_unpackhi_epi16(*in0, *in1);
|
||||
__m256i u0 = _mm256_madd_epi16(t0, w0);
|
||||
__m256i u1 = _mm256_madd_epi16(t1, w0);
|
||||
__m256i v0 = _mm256_madd_epi16(t0, w1);
|
||||
__m256i v1 = _mm256_madd_epi16(t1, w1);
|
||||
|
||||
__m256i a0 = _mm256_add_epi32(u0, _r);
|
||||
__m256i a1 = _mm256_add_epi32(u1, _r);
|
||||
__m256i b0 = _mm256_add_epi32(v0, _r);
|
||||
__m256i b1 = _mm256_add_epi32(v1, _r);
|
||||
|
||||
__m256i c0 = _mm256_srai_epi32(a0, cos_bit);
|
||||
__m256i c1 = _mm256_srai_epi32(a1, cos_bit);
|
||||
__m256i d0 = _mm256_srai_epi32(b0, cos_bit);
|
||||
__m256i d1 = _mm256_srai_epi32(b1, cos_bit);
|
||||
|
||||
*in0 = _mm256_packs_epi32(c0, c1);
|
||||
*in1 = _mm256_packs_epi32(d0, d1);
|
||||
}
|
||||
|
||||
static INLINE void btf_16_adds_subs_avx2(__m256i *in0, __m256i *in1) {
|
||||
const __m256i _in0 = *in0;
|
||||
const __m256i _in1 = *in1;
|
||||
*in0 = _mm256_adds_epi16(_in0, _in1);
|
||||
*in1 = _mm256_subs_epi16(_in0, _in1);
|
||||
}
|
||||
|
||||
static INLINE void btf_32_add_sub_avx2(__m256i *in0, __m256i *in1) {
|
||||
const __m256i _in0 = *in0;
|
||||
const __m256i _in1 = *in1;
|
||||
*in0 = _mm256_add_epi32(_in0, _in1);
|
||||
*in1 = _mm256_sub_epi32(_in0, _in1);
|
||||
}
|
||||
|
||||
static INLINE void btf_16_adds_subs_out_avx2(__m256i *out0, __m256i *out1,
|
||||
__m256i in0, __m256i in1) {
|
||||
const __m256i _in0 = in0;
|
||||
const __m256i _in1 = in1;
|
||||
*out0 = _mm256_adds_epi16(_in0, _in1);
|
||||
*out1 = _mm256_subs_epi16(_in0, _in1);
|
||||
}
|
||||
|
||||
static INLINE void btf_32_add_sub_out_avx2(__m256i *out0, __m256i *out1,
|
||||
__m256i in0, __m256i in1) {
|
||||
const __m256i _in0 = in0;
|
||||
const __m256i _in1 = in1;
|
||||
*out0 = _mm256_add_epi32(_in0, _in1);
|
||||
*out1 = _mm256_sub_epi32(_in0, _in1);
|
||||
}
|
||||
|
||||
static INLINE __m256i load_16bit_to_16bit_avx2(const int16_t *a) {
|
||||
return _mm256_load_si256((const __m256i *)a);
|
||||
}
|
||||
|
||||
static INLINE void load_buffer_16bit_to_16bit_avx2(const int16_t *in,
|
||||
int stride, __m256i *out,
|
||||
int out_size) {
|
||||
for (int i = 0; i < out_size; ++i) {
|
||||
out[i] = load_16bit_to_16bit_avx2(in + i * stride);
|
||||
}
|
||||
}
|
||||
|
||||
static INLINE void load_buffer_16bit_to_16bit_flip_avx2(const int16_t *in,
|
||||
int stride,
|
||||
__m256i *out,
|
||||
int out_size) {
|
||||
for (int i = 0; i < out_size; ++i) {
|
||||
out[out_size - i - 1] = load_16bit_to_16bit_avx2(in + i * stride);
|
||||
}
|
||||
}
|
||||
|
||||
static INLINE __m256i load_32bit_to_16bit_w16_avx2(const int32_t *a) {
|
||||
const __m256i a_low = _mm256_lddqu_si256((const __m256i *)a);
|
||||
const __m256i b = _mm256_packs_epi32(a_low, *(const __m256i *)(a + 8));
|
||||
return _mm256_permute4x64_epi64(b, 0xD8);
|
||||
}
|
||||
|
||||
static INLINE void load_buffer_32bit_to_16bit_w16_avx2(const int32_t *in,
|
||||
int stride, __m256i *out,
|
||||
int out_size) {
|
||||
for (int i = 0; i < out_size; ++i) {
|
||||
out[i] = load_32bit_to_16bit_w16_avx2(in + i * stride);
|
||||
}
|
||||
}
|
||||
|
||||
static INLINE void transpose_16bit_16x16_avx2(const __m256i *const in,
|
||||
__m256i *const out) {
|
||||
// Unpack 16 bit elements. Goes from:
|
||||
// in[0]: 00 01 02 03 08 09 0a 0b 04 05 06 07 0c 0d 0e 0f
|
||||
// in[1]: 10 11 12 13 18 19 1a 1b 14 15 16 17 1c 1d 1e 1f
|
||||
// in[2]: 20 21 22 23 28 29 2a 2b 24 25 26 27 2c 2d 2e 2f
|
||||
// in[3]: 30 31 32 33 38 39 3a 3b 34 35 36 37 3c 3d 3e 3f
|
||||
// in[4]: 40 41 42 43 48 49 4a 4b 44 45 46 47 4c 4d 4e 4f
|
||||
// in[5]: 50 51 52 53 58 59 5a 5b 54 55 56 57 5c 5d 5e 5f
|
||||
// in[6]: 60 61 62 63 68 69 6a 6b 64 65 66 67 6c 6d 6e 6f
|
||||
// in[7]: 70 71 72 73 78 79 7a 7b 74 75 76 77 7c 7d 7e 7f
|
||||
// in[8]: 80 81 82 83 88 89 8a 8b 84 85 86 87 8c 8d 8e 8f
|
||||
// to:
|
||||
// a0: 00 10 01 11 02 12 03 13 04 14 05 15 06 16 07 17
|
||||
// a1: 20 30 21 31 22 32 23 33 24 34 25 35 26 36 27 37
|
||||
// a2: 40 50 41 51 42 52 43 53 44 54 45 55 46 56 47 57
|
||||
// a3: 60 70 61 71 62 72 63 73 64 74 65 75 66 76 67 77
|
||||
// ...
|
||||
__m256i a[16];
|
||||
for (int i = 0; i < 16; i += 2) {
|
||||
a[i / 2 + 0] = _mm256_unpacklo_epi16(in[i], in[i + 1]);
|
||||
a[i / 2 + 8] = _mm256_unpackhi_epi16(in[i], in[i + 1]);
|
||||
}
|
||||
__m256i b[16];
|
||||
for (int i = 0; i < 16; i += 2) {
|
||||
b[i / 2 + 0] = _mm256_unpacklo_epi32(a[i], a[i + 1]);
|
||||
b[i / 2 + 8] = _mm256_unpackhi_epi32(a[i], a[i + 1]);
|
||||
}
|
||||
__m256i c[16];
|
||||
for (int i = 0; i < 16; i += 2) {
|
||||
c[i / 2 + 0] = _mm256_unpacklo_epi64(b[i], b[i + 1]);
|
||||
c[i / 2 + 8] = _mm256_unpackhi_epi64(b[i], b[i + 1]);
|
||||
}
|
||||
out[0 + 0] = _mm256_permute2x128_si256(c[0], c[1], 0x20);
|
||||
out[1 + 0] = _mm256_permute2x128_si256(c[8], c[9], 0x20);
|
||||
out[2 + 0] = _mm256_permute2x128_si256(c[4], c[5], 0x20);
|
||||
out[3 + 0] = _mm256_permute2x128_si256(c[12], c[13], 0x20);
|
||||
|
||||
out[0 + 8] = _mm256_permute2x128_si256(c[0], c[1], 0x31);
|
||||
out[1 + 8] = _mm256_permute2x128_si256(c[8], c[9], 0x31);
|
||||
out[2 + 8] = _mm256_permute2x128_si256(c[4], c[5], 0x31);
|
||||
out[3 + 8] = _mm256_permute2x128_si256(c[12], c[13], 0x31);
|
||||
|
||||
out[4 + 0] = _mm256_permute2x128_si256(c[0 + 2], c[1 + 2], 0x20);
|
||||
out[5 + 0] = _mm256_permute2x128_si256(c[8 + 2], c[9 + 2], 0x20);
|
||||
out[6 + 0] = _mm256_permute2x128_si256(c[4 + 2], c[5 + 2], 0x20);
|
||||
out[7 + 0] = _mm256_permute2x128_si256(c[12 + 2], c[13 + 2], 0x20);
|
||||
|
||||
out[4 + 8] = _mm256_permute2x128_si256(c[0 + 2], c[1 + 2], 0x31);
|
||||
out[5 + 8] = _mm256_permute2x128_si256(c[8 + 2], c[9 + 2], 0x31);
|
||||
out[6 + 8] = _mm256_permute2x128_si256(c[4 + 2], c[5 + 2], 0x31);
|
||||
out[7 + 8] = _mm256_permute2x128_si256(c[12 + 2], c[13 + 2], 0x31);
|
||||
}
|
||||
|
||||
static INLINE void flip_buf_avx2(__m256i *in, __m256i *out, int size) {
|
||||
for (int i = 0; i < size; ++i) {
|
||||
out[size - i - 1] = in[i];
|
||||
}
|
||||
}
|
||||
|
||||
static INLINE void round_shift_16bit_w16_avx2(__m256i *in, int size, int bit) {
|
||||
if (bit < 0) {
|
||||
bit = -bit;
|
||||
__m256i round = _mm256_set1_epi16(1 << (bit - 1));
|
||||
for (int i = 0; i < size; ++i) {
|
||||
in[i] = _mm256_adds_epi16(in[i], round);
|
||||
in[i] = _mm256_srai_epi16(in[i], bit);
|
||||
}
|
||||
} else if (bit > 0) {
|
||||
for (int i = 0; i < size; ++i) {
|
||||
in[i] = _mm256_slli_epi16(in[i], bit);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif // AOM_DSP_X86_TXFM_COMMON_AVX2_H_
|
||||
113
third_party/aom/aom_dsp/x86/variance_avx2.c
vendored
113
third_party/aom/aom_dsp/x86/variance_avx2.c
vendored
|
|
@ -324,6 +324,12 @@ static INLINE __m256i mm256_loadu2(const uint8_t *p0, const uint8_t *p1) {
|
|||
return _mm256_insertf128_si256(d, _mm_loadu_si128((const __m128i *)p0), 1);
|
||||
}
|
||||
|
||||
static INLINE __m256i mm256_loadu2_16(const uint16_t *p0, const uint16_t *p1) {
|
||||
const __m256i d =
|
||||
_mm256_castsi128_si256(_mm_loadu_si128((const __m128i *)p1));
|
||||
return _mm256_insertf128_si256(d, _mm_loadu_si128((const __m128i *)p0), 1);
|
||||
}
|
||||
|
||||
static INLINE void comp_mask_pred_line_avx2(const __m256i s0, const __m256i s1,
|
||||
const __m256i a,
|
||||
uint8_t *comp_pred) {
|
||||
|
|
@ -401,3 +407,110 @@ void aom_comp_mask_pred_avx2(uint8_t *comp_pred, const uint8_t *pred, int width,
|
|||
} while (i < height);
|
||||
}
|
||||
}
|
||||
|
||||
static INLINE __m256i highbd_comp_mask_pred_line_avx2(const __m256i s0,
|
||||
const __m256i s1,
|
||||
const __m256i a) {
|
||||
const __m256i alpha_max = _mm256_set1_epi16((1 << AOM_BLEND_A64_ROUND_BITS));
|
||||
const __m256i round_const =
|
||||
_mm256_set1_epi32((1 << AOM_BLEND_A64_ROUND_BITS) >> 1);
|
||||
const __m256i a_inv = _mm256_sub_epi16(alpha_max, a);
|
||||
|
||||
const __m256i s_lo = _mm256_unpacklo_epi16(s0, s1);
|
||||
const __m256i a_lo = _mm256_unpacklo_epi16(a, a_inv);
|
||||
const __m256i pred_lo = _mm256_madd_epi16(s_lo, a_lo);
|
||||
const __m256i pred_l = _mm256_srai_epi32(
|
||||
_mm256_add_epi32(pred_lo, round_const), AOM_BLEND_A64_ROUND_BITS);
|
||||
|
||||
const __m256i s_hi = _mm256_unpackhi_epi16(s0, s1);
|
||||
const __m256i a_hi = _mm256_unpackhi_epi16(a, a_inv);
|
||||
const __m256i pred_hi = _mm256_madd_epi16(s_hi, a_hi);
|
||||
const __m256i pred_h = _mm256_srai_epi32(
|
||||
_mm256_add_epi32(pred_hi, round_const), AOM_BLEND_A64_ROUND_BITS);
|
||||
|
||||
const __m256i comp = _mm256_packs_epi32(pred_l, pred_h);
|
||||
|
||||
return comp;
|
||||
}
|
||||
|
||||
void aom_highbd_comp_mask_pred_avx2(uint16_t *comp_pred, const uint8_t *pred8,
|
||||
int width, int height, const uint8_t *ref8,
|
||||
int ref_stride, const uint8_t *mask,
|
||||
int mask_stride, int invert_mask) {
|
||||
int i = 0;
|
||||
uint16_t *pred = CONVERT_TO_SHORTPTR(pred8);
|
||||
uint16_t *ref = CONVERT_TO_SHORTPTR(ref8);
|
||||
const uint16_t *src0 = invert_mask ? pred : ref;
|
||||
const uint16_t *src1 = invert_mask ? ref : pred;
|
||||
const int stride0 = invert_mask ? width : ref_stride;
|
||||
const int stride1 = invert_mask ? ref_stride : width;
|
||||
const __m256i zero = _mm256_setzero_si256();
|
||||
|
||||
if (width == 8) {
|
||||
do {
|
||||
const __m256i s0 = mm256_loadu2_16(src0 + stride0, src0);
|
||||
const __m256i s1 = mm256_loadu2_16(src1 + stride1, src1);
|
||||
|
||||
const __m128i m_l = _mm_loadl_epi64((const __m128i *)mask);
|
||||
const __m128i m_h = _mm_loadl_epi64((const __m128i *)(mask + 8));
|
||||
|
||||
__m256i m = _mm256_castsi128_si256(m_l);
|
||||
m = _mm256_insertf128_si256(m, m_h, 1);
|
||||
const __m256i m_16 = _mm256_unpacklo_epi8(m, zero);
|
||||
|
||||
const __m256i comp = highbd_comp_mask_pred_line_avx2(s0, s1, m_16);
|
||||
|
||||
_mm_storeu_si128((__m128i *)(comp_pred), _mm256_castsi256_si128(comp));
|
||||
|
||||
_mm_storeu_si128((__m128i *)(comp_pred + width),
|
||||
_mm256_extractf128_si256(comp, 1));
|
||||
|
||||
src0 += (stride0 << 1);
|
||||
src1 += (stride1 << 1);
|
||||
mask += (mask_stride << 1);
|
||||
comp_pred += (width << 1);
|
||||
i += 2;
|
||||
} while (i < height);
|
||||
} else if (width == 16) {
|
||||
do {
|
||||
const __m256i s0 = _mm256_loadu_si256((const __m256i *)(src0));
|
||||
const __m256i s1 = _mm256_loadu_si256((const __m256i *)(src1));
|
||||
const __m256i m_16 =
|
||||
_mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i *)mask));
|
||||
|
||||
const __m256i comp = highbd_comp_mask_pred_line_avx2(s0, s1, m_16);
|
||||
|
||||
_mm256_storeu_si256((__m256i *)comp_pred, comp);
|
||||
|
||||
src0 += stride0;
|
||||
src1 += stride1;
|
||||
mask += mask_stride;
|
||||
comp_pred += width;
|
||||
i += 1;
|
||||
} while (i < height);
|
||||
} else if (width == 32) {
|
||||
do {
|
||||
const __m256i s0 = _mm256_loadu_si256((const __m256i *)src0);
|
||||
const __m256i s2 = _mm256_loadu_si256((const __m256i *)(src0 + 16));
|
||||
const __m256i s1 = _mm256_loadu_si256((const __m256i *)src1);
|
||||
const __m256i s3 = _mm256_loadu_si256((const __m256i *)(src1 + 16));
|
||||
|
||||
const __m256i m01_16 =
|
||||
_mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i *)mask));
|
||||
const __m256i m23_16 =
|
||||
_mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i *)(mask + 16)));
|
||||
|
||||
const __m256i comp = highbd_comp_mask_pred_line_avx2(s0, s1, m01_16);
|
||||
const __m256i comp1 = highbd_comp_mask_pred_line_avx2(s2, s3, m23_16);
|
||||
|
||||
_mm256_storeu_si256((__m256i *)comp_pred, comp);
|
||||
_mm256_storeu_si256((__m256i *)(comp_pred + 16), comp1);
|
||||
|
||||
src0 += stride0;
|
||||
src1 += stride1;
|
||||
mask += mask_stride;
|
||||
comp_pred += width;
|
||||
i += 1;
|
||||
} while (i < height);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
129
third_party/aom/aom_dsp/x86/variance_impl_ssse3.c
vendored
Normal file
129
third_party/aom/aom_dsp/x86/variance_impl_ssse3.c
vendored
Normal file
|
|
@ -0,0 +1,129 @@
|
|||
/*
|
||||
* Copyright (c) 2018, Alliance for Open Media. All rights reserved
|
||||
*
|
||||
* This source code is subject to the terms of the BSD 2 Clause License and
|
||||
* the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
|
||||
* was not distributed with this source code in the LICENSE file, you can
|
||||
* obtain it at www.aomedia.org/license/software. If the Alliance for Open
|
||||
* Media Patent License 1.0 was not distributed with this source code in the
|
||||
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
|
||||
*/
|
||||
|
||||
#include <tmmintrin.h>
|
||||
|
||||
#include "config/aom_config.h"
|
||||
#include "config/aom_dsp_rtcd.h"
|
||||
|
||||
#include "aom_dsp/x86/synonyms.h"
|
||||
|
||||
void aom_var_filter_block2d_bil_first_pass_ssse3(
|
||||
const uint8_t *a, uint16_t *b, unsigned int src_pixels_per_line,
|
||||
unsigned int pixel_step, unsigned int output_height,
|
||||
unsigned int output_width, const uint8_t *filter) {
|
||||
// Note: filter[0], filter[1] could be {128, 0}, where 128 will overflow
|
||||
// in computation using _mm_maddubs_epi16.
|
||||
// Change {128, 0} to {64, 0} and reduce FILTER_BITS by 1 to avoid overflow.
|
||||
const int16_t round = (1 << (FILTER_BITS - 1)) >> 1;
|
||||
const __m128i r = _mm_set1_epi16(round);
|
||||
const uint8_t f0 = filter[0] >> 1;
|
||||
const uint8_t f1 = filter[1] >> 1;
|
||||
const __m128i filters = _mm_setr_epi8(f0, f1, f0, f1, f0, f1, f0, f1, f0, f1,
|
||||
f0, f1, f0, f1, f0, f1);
|
||||
unsigned int i, j;
|
||||
(void)pixel_step;
|
||||
|
||||
if (output_width >= 8) {
|
||||
for (i = 0; i < output_height; ++i) {
|
||||
for (j = 0; j < output_width; j += 8) {
|
||||
// load source
|
||||
__m128i source_low = xx_loadl_64(a);
|
||||
__m128i source_hi = xx_loadl_64(a + 1);
|
||||
|
||||
// unpack to:
|
||||
// { a[0], a[1], a[1], a[2], a[2], a[3], a[3], a[4],
|
||||
// a[4], a[5], a[5], a[6], a[6], a[7], a[7], a[8] }
|
||||
__m128i source = _mm_unpacklo_epi8(source_low, source_hi);
|
||||
|
||||
// b[i] = a[i] * filter[0] + a[i + 1] * filter[1]
|
||||
__m128i res = _mm_maddubs_epi16(source, filters);
|
||||
|
||||
// round
|
||||
res = _mm_srai_epi16(_mm_add_epi16(res, r), FILTER_BITS - 1);
|
||||
|
||||
xx_storeu_128(b, res);
|
||||
|
||||
a += 8;
|
||||
b += 8;
|
||||
}
|
||||
|
||||
a += src_pixels_per_line - output_width;
|
||||
}
|
||||
} else {
|
||||
const __m128i shuffle_mask =
|
||||
_mm_setr_epi8(0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8);
|
||||
for (i = 0; i < output_height; ++i) {
|
||||
// load source, only first 5 values are meaningful:
|
||||
// { a[0], a[1], a[2], a[3], a[4], xxxx }
|
||||
__m128i source = xx_loadl_64(a);
|
||||
|
||||
// shuffle, up to the first 8 are useful
|
||||
// { a[0], a[1], a[1], a[2], a[2], a[3], a[3], a[4],
|
||||
// a[4], a[5], a[5], a[6], a[6], a[7], a[7], a[8] }
|
||||
__m128i source_shuffle = _mm_shuffle_epi8(source, shuffle_mask);
|
||||
|
||||
__m128i res = _mm_maddubs_epi16(source_shuffle, filters);
|
||||
res = _mm_srai_epi16(_mm_add_epi16(res, r), FILTER_BITS - 1);
|
||||
|
||||
xx_storel_64(b, res);
|
||||
|
||||
a += src_pixels_per_line;
|
||||
b += output_width;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void aom_var_filter_block2d_bil_second_pass_ssse3(
|
||||
const uint16_t *a, uint8_t *b, unsigned int src_pixels_per_line,
|
||||
unsigned int pixel_step, unsigned int output_height,
|
||||
unsigned int output_width, const uint8_t *filter) {
|
||||
const int16_t round = (1 << FILTER_BITS) >> 1;
|
||||
const __m128i r = _mm_set1_epi32(round);
|
||||
const __m128i filters =
|
||||
_mm_setr_epi16(filter[0], filter[1], filter[0], filter[1], filter[0],
|
||||
filter[1], filter[0], filter[1]);
|
||||
const __m128i shuffle_mask =
|
||||
_mm_setr_epi8(0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15);
|
||||
const __m128i mask =
|
||||
_mm_setr_epi8(0, 4, 8, 12, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1);
|
||||
unsigned int i, j;
|
||||
|
||||
for (i = 0; i < output_height; ++i) {
|
||||
for (j = 0; j < output_width; j += 4) {
|
||||
// load source as:
|
||||
// { a[0], a[1], a[2], a[3], a[w], a[w+1], a[w+2], a[w+3] }
|
||||
__m128i source1 = xx_loadl_64(a);
|
||||
__m128i source2 = xx_loadl_64(a + pixel_step);
|
||||
__m128i source = _mm_unpacklo_epi64(source1, source2);
|
||||
|
||||
// shuffle source to:
|
||||
// { a[0], a[w], a[1], a[w+1], a[2], a[w+2], a[3], a[w+3] }
|
||||
__m128i source_shuffle = _mm_shuffle_epi8(source, shuffle_mask);
|
||||
|
||||
// b[i] = a[i] * filter[0] + a[w + i] * filter[1]
|
||||
__m128i res = _mm_madd_epi16(source_shuffle, filters);
|
||||
|
||||
// round
|
||||
res = _mm_srai_epi32(_mm_add_epi32(res, r), FILTER_BITS);
|
||||
|
||||
// shuffle to get each lower 8 bit of every 32 bit
|
||||
res = _mm_shuffle_epi8(res, mask);
|
||||
|
||||
xx_storel_32(b, res);
|
||||
|
||||
a += 4;
|
||||
b += 4;
|
||||
}
|
||||
|
||||
a += src_pixels_per_line - output_width;
|
||||
}
|
||||
}
|
||||
12
third_party/aom/aom_dsp/x86/variance_sse2.c
vendored
12
third_party/aom/aom_dsp/x86/variance_sse2.c
vendored
|
|
@ -569,7 +569,7 @@ void aom_upsampled_pred_sse2(MACROBLOCKD *xd, const struct AV1Common *const cm,
|
|||
}
|
||||
}
|
||||
|
||||
const InterpFilterParams filter =
|
||||
const InterpFilterParams *filter =
|
||||
av1_get_interp_filter_params_with_block_size(EIGHTTAP_REGULAR, 8);
|
||||
|
||||
if (!subpel_x_q3 && !subpel_y_q3) {
|
||||
|
|
@ -633,12 +633,12 @@ void aom_upsampled_pred_sse2(MACROBLOCKD *xd, const struct AV1Common *const cm,
|
|||
const int16_t *const kernel_y =
|
||||
av1_get_interp_filter_subpel_kernel(filter, subpel_y_q3 << 1);
|
||||
const int intermediate_height =
|
||||
(((height - 1) * 8 + subpel_y_q3) >> 3) + filter.taps;
|
||||
(((height - 1) * 8 + subpel_y_q3) >> 3) + filter->taps;
|
||||
assert(intermediate_height <= (MAX_SB_SIZE * 2 + 16) + 16);
|
||||
aom_convolve8_horiz(ref - ref_stride * ((filter.taps >> 1) - 1), ref_stride,
|
||||
temp, MAX_SB_SIZE, kernel_x, 16, NULL, -1, width,
|
||||
intermediate_height);
|
||||
aom_convolve8_vert(temp + MAX_SB_SIZE * ((filter.taps >> 1) - 1),
|
||||
aom_convolve8_horiz(ref - ref_stride * ((filter->taps >> 1) - 1),
|
||||
ref_stride, temp, MAX_SB_SIZE, kernel_x, 16, NULL, -1,
|
||||
width, intermediate_height);
|
||||
aom_convolve8_vert(temp + MAX_SB_SIZE * ((filter->taps >> 1) - 1),
|
||||
MAX_SB_SIZE, comp_pred, width, NULL, -1, kernel_y, 16,
|
||||
width, height);
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue