wuggy/Dactyloidae
wuggy/Dactyloidae
1
0
Fork 0
mirror of https://repo.dactyloidae.xyz/Dactyloidae/UXP.git synced 2026-10-18 13:17:29 +09:00
Code Issues Projects Releases Packages Wiki Activity

aom: update libaom to 0ec86ac7ae1e32a7e70410fa4972a655ec3670a4

Browse source
This commit is contained in:
Roy Tam 2019-02-22 21:52:01 +08:00
commit eb361970c5
438 changed files with 52661 additions and 21905 deletions
Download patch file Download diff file Expand all files Collapse all files

2
third_party/aom/aom_dsp/add_noise.c vendored
View file

@ -40,7 +40,7 @@ void aom_plane_add_noise_c(uint8_t *start, char *noise, char blackclamp[16],
}
static double gaussian(double sigma, double mu, double x) {
return 1 / (sigma * sqrt(2.0 * 3.14159265)) *
return 1 / (sigma * sqrt(2.0 * PI)) *
(exp(-(x - mu) * (x - mu) / (2 * sigma * sigma)));
}

30
third_party/aom/aom_dsp/aom_dsp.cmake vendored
View file

@ -35,6 +35,7 @@ list(APPEND AOM_DSP_COMMON_SOURCES
"${AOM_ROOT}/aom_dsp/intrapred_common.h"
"${AOM_ROOT}/aom_dsp/loopfilter.c"
"${AOM_ROOT}/aom_dsp/prob.h"
"${AOM_ROOT}/aom_dsp/recenter.h"
"${AOM_ROOT}/aom_dsp/simd/v128_intrinsics.h"
"${AOM_ROOT}/aom_dsp/simd/v128_intrinsics_c.h"
"${AOM_ROOT}/aom_dsp/simd/v256_intrinsics.h"
@ -43,7 +44,8 @@ list(APPEND AOM_DSP_COMMON_SOURCES
"${AOM_ROOT}/aom_dsp/simd/v64_intrinsics_c.h"
"${AOM_ROOT}/aom_dsp/subtract.c"
"${AOM_ROOT}/aom_dsp/txfm_common.h"
"${AOM_ROOT}/aom_dsp/x86/convolve_common_intrin.h")
"${AOM_ROOT}/aom_dsp/x86/convolve_common_intrin.h"
"${AOM_ROOT}/aom_dsp/avg.c")
list(APPEND AOM_DSP_COMMON_ASM_SSE2
"${AOM_ROOT}/aom_dsp/x86/aom_convolve_copy_sse2.asm"
@ -51,15 +53,17 @@ list(APPEND AOM_DSP_COMMON_ASM_SSE2
"${AOM_ROOT}/aom_dsp/x86/aom_high_subpixel_bilinear_sse2.asm"
"${AOM_ROOT}/aom_dsp/x86/aom_subpixel_8t_sse2.asm"
"${AOM_ROOT}/aom_dsp/x86/aom_subpixel_bilinear_sse2.asm"
"${AOM_ROOT}/aom_dsp/x86/highbd_intrapred_sse2.asm"
"${AOM_ROOT}/aom_dsp/x86/intrapred_sse2.asm"
"${AOM_ROOT}/aom_dsp/x86/highbd_intrapred_asm_sse2.asm"
"${AOM_ROOT}/aom_dsp/x86/intrapred_asm_sse2.asm"
"${AOM_ROOT}/aom_dsp/x86/inv_wht_sse2.asm")
list(APPEND AOM_DSP_COMMON_INTRIN_SSE2
"${AOM_ROOT}/aom_dsp/x86/aom_subpixel_8t_intrin_sse2.c"
"${AOM_ROOT}/aom_dsp/x86/aom_asm_stubs.c"
"${AOM_ROOT}/aom_dsp/x86/convolve.h"
"${AOM_ROOT}/aom_dsp/x86/convolve_sse2.h"
"${AOM_ROOT}/aom_dsp/x86/fft_sse2.c"
"${AOM_ROOT}/aom_dsp/x86/highbd_convolve_sse2.c"
"${AOM_ROOT}/aom_dsp/x86/highbd_intrapred_sse2.c"
"${AOM_ROOT}/aom_dsp/x86/highbd_loopfilter_sse2.c"
"${AOM_ROOT}/aom_dsp/x86/intrapred_sse2.c"
@ -68,7 +72,9 @@ list(APPEND AOM_DSP_COMMON_INTRIN_SSE2
"${AOM_ROOT}/aom_dsp/x86/mem_sse2.h"
"${AOM_ROOT}/aom_dsp/x86/transpose_sse2.h"
"${AOM_ROOT}/aom_dsp/x86/txfm_common_sse2.h"
"${AOM_ROOT}/aom_dsp/x86/sum_squares_sse2.h")
"${AOM_ROOT}/aom_dsp/x86/sum_squares_sse2.h"
"${AOM_ROOT}/aom_dsp/x86/avg_intrin_sse2.c"
"${AOM_ROOT}/aom_dsp/x86/bitdepth_conversion_sse2.h")
list(APPEND AOM_DSP_COMMON_ASM_SSSE3
"${AOM_ROOT}/aom_dsp/x86/aom_subpixel_8t_ssse3.asm"
@ -94,7 +100,9 @@ list(APPEND AOM_DSP_COMMON_INTRIN_AVX2
"${AOM_ROOT}/aom_dsp/x86/highbd_convolve_avx2.c"
"${AOM_ROOT}/aom_dsp/x86/highbd_loopfilter_avx2.c"
"${AOM_ROOT}/aom_dsp/x86/intrapred_avx2.c"
"${AOM_ROOT}/aom_dsp/x86/blend_a64_mask_avx2.c")
"${AOM_ROOT}/aom_dsp/x86/blend_a64_mask_avx2.c"
"${AOM_ROOT}/aom_dsp/x86/avg_intrin_avx2.c"
"${AOM_ROOT}/aom_dsp/x86/bitdepth_conversion_avx2.h")
list(APPEND AOM_DSP_COMMON_INTRIN_NEON
"${AOM_ROOT}/aom_dsp/arm/fwd_txfm_neon.c"
@ -175,6 +183,9 @@ if(CONFIG_AV1_ENCODER)
"${AOM_ROOT}/aom_dsp/x86/subpel_variance_sse2.asm"
"${AOM_ROOT}/aom_dsp/x86/subtract_sse2.asm")
list(APPEND AOM_DSP_ENCODER_ASM_SSE2_X86_64
"${AOM_ROOT}/aom_dsp/x86/ssim_sse2_x86_64.asm")
list(APPEND AOM_DSP_ENCODER_INTRIN_SSE2
"${AOM_ROOT}/aom_dsp/x86/fwd_txfm_impl_sse2.h"
"${AOM_ROOT}/aom_dsp/x86/fwd_txfm_sse2.c"
@ -189,7 +200,7 @@ if(CONFIG_AV1_ENCODER)
list(APPEND AOM_DSP_ENCODER_ASM_SSSE3_X86_64
"${AOM_ROOT}/aom_dsp/x86/fwd_txfm_ssse3_x86_64.asm"
"${AOM_ROOT}/aom_dsp/x86/ssim_opt_x86_64.asm")
"${AOM_ROOT}/aom_dsp/x86/quantize_ssse3_x86_64.asm")
list(APPEND AOM_DSP_ENCODER_INTRIN_AVX2
"${AOM_ROOT}/aom_dsp/x86/masked_sad_intrin_avx2.c"
@ -207,9 +218,6 @@ if(CONFIG_AV1_ENCODER)
"${AOM_ROOT}/aom_dsp/x86/obmc_variance_avx2.c"
"${AOM_ROOT}/aom_dsp/x86/sum_squares_avx2.c")
list(APPEND AOM_DSP_ENCODER_ASM_SSSE3_X86_64
"${AOM_ROOT}/aom_dsp/x86/quantize_ssse3_x86_64.asm")
list(APPEND AOM_DSP_ENCODER_AVX_ASM_X86_64
"${AOM_ROOT}/aom_dsp/x86/quantize_avx_x86_64.asm")
@ -218,6 +226,7 @@ if(CONFIG_AV1_ENCODER)
"${AOM_ROOT}/aom_dsp/x86/masked_sad_intrin_ssse3.c"
"${AOM_ROOT}/aom_dsp/x86/masked_variance_intrin_ssse3.h"
"${AOM_ROOT}/aom_dsp/x86/masked_variance_intrin_ssse3.c"
"${AOM_ROOT}/aom_dsp/x86/quantize_ssse3.c"
"${AOM_ROOT}/aom_dsp/x86/variance_impl_ssse3.c"
"${AOM_ROOT}/aom_dsp/x86/jnt_variance_ssse3.c"
"${AOM_ROOT}/aom_dsp/x86/jnt_sad_ssse3.c")
@ -278,6 +287,9 @@ function(setup_aom_dsp_targets)
"AOM_DSP_COMMON_INTRIN_SSE2" "aom")
if(CONFIG_AV1_ENCODER)
if("${AOM_TARGET_CPU}" STREQUAL "x86_64")
list(APPEND AOM_DSP_ENCODER_ASM_SSE2 ${AOM_DSP_ENCODER_ASM_SSE2_X86_64})
endif()
add_asm_library("aom_dsp_encoder_sse2" "AOM_DSP_ENCODER_ASM_SSE2" "aom")
add_intrinsics_object_library("-msse2" "sse2" "aom_dsp_encoder"
"AOM_DSP_ENCODER_INTRIN_SSE2" "aom")

144
third_party/aom/aom_dsp/aom_dsp_rtcd_defs.pl vendored
View file

@ -362,10 +362,10 @@ add_proto qw/void aom_highbd_convolve_copy/, "const uint8_t *src, ptrdiff_t src_
specialize qw/aom_highbd_convolve_copy sse2 avx2/;
add_proto qw/void aom_highbd_convolve8_horiz/, "const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, const int16_t *filter_y, int y_step_q4, int w, int h, int bps";
specialize qw/aom_highbd_convolve8_horiz avx2/, "$sse2_x86_64";
specialize qw/aom_highbd_convolve8_horiz sse2 avx2/;
add_proto qw/void aom_highbd_convolve8_vert/, "const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, const int16_t *filter_y, int y_step_q4, int w, int h, int bps";
specialize qw/aom_highbd_convolve8_vert avx2/, "$sse2_x86_64";
specialize qw/aom_highbd_convolve8_vert sse2 avx2/;
#
# Loopfilter
@ -466,10 +466,6 @@ specialize qw/aom_highbd_lpf_horizontal_4 sse2/;
add_proto qw/void aom_highbd_lpf_horizontal_4_dual/, "uint16_t *s, int pitch, const uint8_t *blimit0, const uint8_t *limit0, const uint8_t *thresh0, const uint8_t *blimit1, const uint8_t *limit1, const uint8_t *thresh1, int bd";
specialize qw/aom_highbd_lpf_horizontal_4_dual sse2 avx2/;
# Helper functions.
add_proto qw/void av1_round_shift_array/, "int32_t *arr, int size, int bit";
specialize "av1_round_shift_array", qw/sse4_1 neon/;
#
# Encoder functions.
#
@ -526,6 +522,7 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
specialize qw/aom_quantize_b_32x32/, "$ssse3_x86_64", "$avx_x86_64";
add_proto qw/void aom_quantize_b_64x64/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan";
specialize qw/aom_quantize_b_64x64 ssse3/;
} # CONFIG_AV1_ENCODER
if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
@ -544,7 +541,6 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
#
add_proto qw/void aom_lowbd_blend_a64_d16_mask/, "uint8_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0, uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride, const uint8_t *mask, uint32_t mask_stride, int w, int h, int subx, int suby, ConvolveParams *conv_params";
specialize qw/aom_lowbd_blend_a64_d16_mask sse4_1 avx2 neon/;
add_proto qw/void aom_highbd_blend_a64_d16_mask/, "uint8_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0, uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride, const uint8_t *mask, uint32_t mask_stride, int w, int h, int subx, int suby, ConvolveParams *conv_params, const int bd";
add_proto qw/void aom_blend_a64_mask/, "uint8_t *dst, uint32_t dst_stride, const uint8_t *src0, uint32_t src0_stride, const uint8_t *src1, uint32_t src1_stride, const uint8_t *mask, uint32_t mask_stride, int w, int h, int subx, int suby";
add_proto qw/void aom_blend_a64_hmask/, "uint8_t *dst, uint32_t dst_stride, const uint8_t *src0, uint32_t src0_stride, const uint8_t *src1, uint32_t src1_stride, const uint8_t *mask, int w, int h";
add_proto qw/void aom_blend_a64_vmask/, "uint8_t *dst, uint32_t dst_stride, const uint8_t *src0, uint32_t src0_stride, const uint8_t *src1, uint32_t src1_stride, const uint8_t *mask, int w, int h";
@ -555,9 +551,11 @@ specialize "aom_blend_a64_vmask", qw/sse4_1 neon/;
add_proto qw/void aom_highbd_blend_a64_mask/, "uint8_t *dst, uint32_t dst_stride, const uint8_t *src0, uint32_t src0_stride, const uint8_t *src1, uint32_t src1_stride, const uint8_t *mask, uint32_t mask_stride, int w, int h, int subx, int suby, int bd";
add_proto qw/void aom_highbd_blend_a64_hmask/, "uint8_t *dst, uint32_t dst_stride, const uint8_t *src0, uint32_t src0_stride, const uint8_t *src1, uint32_t src1_stride, const uint8_t *mask, int w, int h, int bd";
add_proto qw/void aom_highbd_blend_a64_vmask/, "uint8_t *dst, uint32_t dst_stride, const uint8_t *src0, uint32_t src0_stride, const uint8_t *src1, uint32_t src1_stride, const uint8_t *mask, int w, int h, int bd";
add_proto qw/void aom_highbd_blend_a64_d16_mask/, "uint8_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0, uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride, const uint8_t *mask, uint32_t mask_stride, int w, int h, int subx, int suby, ConvolveParams *conv_params, const int bd";
specialize "aom_highbd_blend_a64_mask", qw/sse4_1/;
specialize "aom_highbd_blend_a64_hmask", qw/sse4_1/;
specialize "aom_highbd_blend_a64_vmask", qw/sse4_1/;
specialize "aom_highbd_blend_a64_d16_mask", qw/sse4_1 avx2/;
if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
#
@ -595,7 +593,7 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
($w, $h) = @$_;
add_proto qw/unsigned int/, "aom_sad${w}x${h}", "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride";
add_proto qw/unsigned int/, "aom_sad${w}x${h}_avg", "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred";
add_proto qw/unsigned int/, "aom_jnt_sad${w}x${h}_avg", "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred, const JNT_COMP_PARAMS *jcp_param";
add_proto qw/unsigned int/, "aom_dist_wtd_sad${w}x${h}_avg", "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred, const DIST_WTD_COMP_PARAMS *jcp_param";
}
specialize qw/aom_sad128x128 avx2 sse2/;
@ -646,29 +644,29 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
specialize qw/aom_sad16x64_avg sse2/;
specialize qw/aom_sad64x16_avg sse2/;
specialize qw/aom_jnt_sad128x128_avg ssse3/;
specialize qw/aom_jnt_sad128x64_avg ssse3/;
specialize qw/aom_jnt_sad64x128_avg ssse3/;
specialize qw/aom_jnt_sad64x64_avg ssse3/;
specialize qw/aom_jnt_sad64x32_avg ssse3/;
specialize qw/aom_jnt_sad32x64_avg ssse3/;
specialize qw/aom_jnt_sad32x32_avg ssse3/;
specialize qw/aom_jnt_sad32x16_avg ssse3/;
specialize qw/aom_jnt_sad16x32_avg ssse3/;
specialize qw/aom_jnt_sad16x16_avg ssse3/;
specialize qw/aom_jnt_sad16x8_avg ssse3/;
specialize qw/aom_jnt_sad8x16_avg ssse3/;
specialize qw/aom_jnt_sad8x8_avg ssse3/;
specialize qw/aom_jnt_sad8x4_avg ssse3/;
specialize qw/aom_jnt_sad4x8_avg ssse3/;
specialize qw/aom_jnt_sad4x4_avg ssse3/;
specialize qw/aom_dist_wtd_sad128x128_avg ssse3/;
specialize qw/aom_dist_wtd_sad128x64_avg ssse3/;
specialize qw/aom_dist_wtd_sad64x128_avg ssse3/;
specialize qw/aom_dist_wtd_sad64x64_avg ssse3/;
specialize qw/aom_dist_wtd_sad64x32_avg ssse3/;
specialize qw/aom_dist_wtd_sad32x64_avg ssse3/;
specialize qw/aom_dist_wtd_sad32x32_avg ssse3/;
specialize qw/aom_dist_wtd_sad32x16_avg ssse3/;
specialize qw/aom_dist_wtd_sad16x32_avg ssse3/;
specialize qw/aom_dist_wtd_sad16x16_avg ssse3/;
specialize qw/aom_dist_wtd_sad16x8_avg ssse3/;
specialize qw/aom_dist_wtd_sad8x16_avg ssse3/;
specialize qw/aom_dist_wtd_sad8x8_avg ssse3/;
specialize qw/aom_dist_wtd_sad8x4_avg ssse3/;
specialize qw/aom_dist_wtd_sad4x8_avg ssse3/;
specialize qw/aom_dist_wtd_sad4x4_avg ssse3/;
specialize qw/aom_jnt_sad4x16_avg ssse3/;
specialize qw/aom_jnt_sad16x4_avg ssse3/;
specialize qw/aom_jnt_sad8x32_avg ssse3/;
specialize qw/aom_jnt_sad32x8_avg ssse3/;
specialize qw/aom_jnt_sad16x64_avg ssse3/;
specialize qw/aom_jnt_sad64x16_avg ssse3/;
specialize qw/aom_dist_wtd_sad4x16_avg ssse3/;
specialize qw/aom_dist_wtd_sad16x4_avg ssse3/;
specialize qw/aom_dist_wtd_sad8x32_avg ssse3/;
specialize qw/aom_dist_wtd_sad32x8_avg ssse3/;
specialize qw/aom_dist_wtd_sad16x64_avg ssse3/;
specialize qw/aom_dist_wtd_sad64x16_avg ssse3/;
add_proto qw/unsigned int/, "aom_sad4xh", "const uint8_t *a, int a_stride, const uint8_t *b, int b_stride, int width, int height";
add_proto qw/unsigned int/, "aom_sad8xh", "const uint8_t *a, int a_stride, const uint8_t *b, int b_stride, int width, int height";
@ -693,7 +691,7 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
specialize "aom_highbd_sad${w}x${h}", qw/sse2/;
specialize "aom_highbd_sad${w}x${h}_avg", qw/sse2/;
}
add_proto qw/unsigned int/, "aom_highbd_jnt_sad${w}x${h}_avg", "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred, const JNT_COMP_PARAMS* jcp_param";
add_proto qw/unsigned int/, "aom_highbd_dist_wtd_sad${w}x${h}_avg", "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred, const DIST_WTD_COMP_PARAMS* jcp_param";
}
specialize qw/aom_highbd_sad128x128 avx2/;
specialize qw/aom_highbd_sad128x64 avx2/;
@ -837,6 +835,21 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
specialize qw/aom_highbd_sad16x64x4d sse2/;
specialize qw/aom_highbd_sad64x16x4d sse2/;
#
# hamadard transform and satd for implmenting temporal dependency model
#
add_proto qw/void aom_hadamard_8x8/, "const int16_t *src_diff, ptrdiff_t src_stride, tran_low_t *coeff";
specialize qw/aom_hadamard_8x8 sse2/;
add_proto qw/void aom_hadamard_16x16/, "const int16_t *src_diff, ptrdiff_t src_stride, tran_low_t *coeff";
specialize qw/aom_hadamard_16x16 avx2 sse2/;
add_proto qw/void aom_hadamard_32x32/, "const int16_t *src_diff, ptrdiff_t src_stride, tran_low_t *coeff";
specialize qw/aom_hadamard_32x32 avx2 sse2/;
add_proto qw/int aom_satd/, "const tran_low_t *coeff, int length";
specialize qw/aom_satd avx2 sse2/;
#
# Structured Similarity (SSIM)
#
@ -903,11 +916,11 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
int ref_stride, int subpel_search";
specialize qw/aom_comp_avg_upsampled_pred sse2/;
add_proto qw/void aom_jnt_comp_avg_upsampled_pred/, "MACROBLOCKD *xd, const struct AV1Common *const cm, int mi_row, int mi_col,
add_proto qw/void aom_dist_wtd_comp_avg_upsampled_pred/, "MACROBLOCKD *xd, const struct AV1Common *const cm, int mi_row, int mi_col,
const MV *const mv, uint8_t *comp_pred, const uint8_t *pred, int width,
int height, int subpel_x_q3, int subpel_y_q3, const uint8_t *ref,
int ref_stride, const JNT_COMP_PARAMS *jcp_param, int subpel_search";
specialize qw/aom_jnt_comp_avg_upsampled_pred ssse3/;
int ref_stride, const DIST_WTD_COMP_PARAMS *jcp_param, int subpel_search";
specialize qw/aom_dist_wtd_comp_avg_upsampled_pred ssse3/;
add_proto qw/void aom_comp_mask_upsampled_pred/, "MACROBLOCKD *xd, const struct AV1Common *const cm, int mi_row, int mi_col,
const MV *const mv, uint8_t *comp_pred, const uint8_t *pred, int width,
@ -916,7 +929,6 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
int subpel_search";
specialize qw/aom_comp_mask_upsampled_pred sse2/;
add_proto qw/void aom_highbd_upsampled_pred/, "MACROBLOCKD *xd, const struct AV1Common *const cm, int mi_row, int mi_col,
const MV *const mv, uint8_t *comp_pred8, int width, int height, int subpel_x_q3,
int subpel_y_q3, const uint8_t *ref8, int ref_stride, int bd, int subpel_search";
@ -927,11 +939,11 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
int height, int subpel_x_q3, int subpel_y_q3, const uint8_t *ref8, int ref_stride, int bd, int subpel_search";
specialize qw/aom_highbd_comp_avg_upsampled_pred sse2/;
add_proto qw/void aom_highbd_jnt_comp_avg_upsampled_pred/, "MACROBLOCKD *xd, const struct AV1Common *const cm, int mi_row, int mi_col,
add_proto qw/void aom_highbd_dist_wtd_comp_avg_upsampled_pred/, "MACROBLOCKD *xd, const struct AV1Common *const cm, int mi_row, int mi_col,
const MV *const mv, uint8_t *comp_pred8, const uint8_t *pred8, int width,
int height, int subpel_x_q3, int subpel_y_q3, const uint8_t *ref8,
int ref_stride, int bd, const JNT_COMP_PARAMS *jcp_param, int subpel_search";
specialize qw/aom_highbd_jnt_comp_avg_upsampled_pred sse2/;
int ref_stride, int bd, const DIST_WTD_COMP_PARAMS *jcp_param, int subpel_search";
specialize qw/aom_highbd_dist_wtd_comp_avg_upsampled_pred sse2/;
#
@ -957,7 +969,7 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
add_proto qw/unsigned int/, "aom_variance${w}x${h}", "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
add_proto qw/uint32_t/, "aom_sub_pixel_variance${w}x${h}", "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse";
add_proto qw/uint32_t/, "aom_sub_pixel_avg_variance${w}x${h}", "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred";
add_proto qw/uint32_t/, "aom_jnt_sub_pixel_avg_variance${w}x${h}", "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred, const JNT_COMP_PARAMS *jcp_param";
add_proto qw/uint32_t/, "aom_dist_wtd_sub_pixel_avg_variance${w}x${h}", "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred, const DIST_WTD_COMP_PARAMS *jcp_param";
}
specialize qw/aom_variance128x128 sse2 avx2 /;
specialize qw/aom_variance128x64 sse2 avx2 /;
@ -1029,30 +1041,30 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
specialize qw/aom_sub_pixel_avg_variance16x64 sse2 ssse3/;
specialize qw/aom_sub_pixel_avg_variance64x16 sse2 ssse3/;
specialize qw/aom_jnt_sub_pixel_avg_variance64x64 ssse3/;
specialize qw/aom_jnt_sub_pixel_avg_variance64x32 ssse3/;
specialize qw/aom_jnt_sub_pixel_avg_variance32x64 ssse3/;
specialize qw/aom_jnt_sub_pixel_avg_variance32x32 ssse3/;
specialize qw/aom_jnt_sub_pixel_avg_variance32x16 ssse3/;
specialize qw/aom_jnt_sub_pixel_avg_variance16x32 ssse3/;
specialize qw/aom_jnt_sub_pixel_avg_variance16x16 ssse3/;
specialize qw/aom_jnt_sub_pixel_avg_variance16x8 ssse3/;
specialize qw/aom_jnt_sub_pixel_avg_variance8x16 ssse3/;
specialize qw/aom_jnt_sub_pixel_avg_variance8x8 ssse3/;
specialize qw/aom_jnt_sub_pixel_avg_variance8x4 ssse3/;
specialize qw/aom_jnt_sub_pixel_avg_variance4x8 ssse3/;
specialize qw/aom_jnt_sub_pixel_avg_variance4x4 ssse3/;
specialize qw/aom_dist_wtd_sub_pixel_avg_variance64x64 ssse3/;
specialize qw/aom_dist_wtd_sub_pixel_avg_variance64x32 ssse3/;
specialize qw/aom_dist_wtd_sub_pixel_avg_variance32x64 ssse3/;
specialize qw/aom_dist_wtd_sub_pixel_avg_variance32x32 ssse3/;
specialize qw/aom_dist_wtd_sub_pixel_avg_variance32x16 ssse3/;
specialize qw/aom_dist_wtd_sub_pixel_avg_variance16x32 ssse3/;
specialize qw/aom_dist_wtd_sub_pixel_avg_variance16x16 ssse3/;
specialize qw/aom_dist_wtd_sub_pixel_avg_variance16x8 ssse3/;
specialize qw/aom_dist_wtd_sub_pixel_avg_variance8x16 ssse3/;
specialize qw/aom_dist_wtd_sub_pixel_avg_variance8x8 ssse3/;
specialize qw/aom_dist_wtd_sub_pixel_avg_variance8x4 ssse3/;
specialize qw/aom_dist_wtd_sub_pixel_avg_variance4x8 ssse3/;
specialize qw/aom_dist_wtd_sub_pixel_avg_variance4x4 ssse3/;
specialize qw/aom_jnt_sub_pixel_avg_variance4x16 ssse3/;
specialize qw/aom_jnt_sub_pixel_avg_variance16x4 ssse3/;
specialize qw/aom_jnt_sub_pixel_avg_variance8x32 ssse3/;
specialize qw/aom_jnt_sub_pixel_avg_variance32x8 ssse3/;
specialize qw/aom_jnt_sub_pixel_avg_variance16x64 ssse3/;
specialize qw/aom_jnt_sub_pixel_avg_variance64x16 ssse3/;
specialize qw/aom_dist_wtd_sub_pixel_avg_variance4x16 ssse3/;
specialize qw/aom_dist_wtd_sub_pixel_avg_variance16x4 ssse3/;
specialize qw/aom_dist_wtd_sub_pixel_avg_variance8x32 ssse3/;
specialize qw/aom_dist_wtd_sub_pixel_avg_variance32x8 ssse3/;
specialize qw/aom_dist_wtd_sub_pixel_avg_variance16x64 ssse3/;
specialize qw/aom_dist_wtd_sub_pixel_avg_variance64x16 ssse3/;
specialize qw/aom_jnt_sub_pixel_avg_variance128x128 ssse3/;
specialize qw/aom_jnt_sub_pixel_avg_variance128x64 ssse3/;
specialize qw/aom_jnt_sub_pixel_avg_variance64x128 ssse3/;
specialize qw/aom_dist_wtd_sub_pixel_avg_variance128x128 ssse3/;
specialize qw/aom_dist_wtd_sub_pixel_avg_variance128x64 ssse3/;
specialize qw/aom_dist_wtd_sub_pixel_avg_variance64x128 ssse3/;
foreach $bd (8, 10, 12) {
@ -1084,7 +1096,7 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
specialize "aom_highbd_${bd}_sub_pixel_avg_variance${w}x${h}", "sse4_1";
}
add_proto qw/uint32_t/, "aom_highbd_${bd}_jnt_sub_pixel_avg_variance${w}x${h}", "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred, const JNT_COMP_PARAMS* jcp_param";
add_proto qw/uint32_t/, "aom_highbd_${bd}_dist_wtd_sub_pixel_avg_variance${w}x${h}", "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred, const DIST_WTD_COMP_PARAMS* jcp_param";
}
}
@ -1173,8 +1185,8 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
#
add_proto qw/void aom_comp_avg_pred/, "uint8_t *comp_pred, const uint8_t *pred, int width, int height, const uint8_t *ref, int ref_stride";
add_proto qw/void aom_jnt_comp_avg_pred/, "uint8_t *comp_pred, const uint8_t *pred, int width, int height, const uint8_t *ref, int ref_stride, const JNT_COMP_PARAMS *jcp_param";
specialize qw/aom_jnt_comp_avg_pred ssse3/;
add_proto qw/void aom_dist_wtd_comp_avg_pred/, "uint8_t *comp_pred, const uint8_t *pred, int width, int height, const uint8_t *ref, int ref_stride, const DIST_WTD_COMP_PARAMS *jcp_param";
specialize qw/aom_dist_wtd_comp_avg_pred ssse3/;
add_proto qw/unsigned int aom_highbd_12_variance128x128/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
specialize qw/aom_highbd_12_variance128x128 sse2/;
@ -1340,8 +1352,8 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
add_proto qw/void aom_highbd_comp_avg_pred/, "uint8_t *comp_pred8, const uint8_t *pred8, int width, int height, const uint8_t *ref8, int ref_stride";
add_proto qw/void aom_highbd_jnt_comp_avg_pred/, "uint8_t *comp_pred8, const uint8_t *pred8, int width, int height, const uint8_t *ref8, int ref_stride, const JNT_COMP_PARAMS *jcp_param";
specialize qw/aom_highbd_jnt_comp_avg_pred sse2/;
add_proto qw/void aom_highbd_dist_wtd_comp_avg_pred/, "uint8_t *comp_pred8, const uint8_t *pred8, int width, int height, const uint8_t *ref8, int ref_stride, const DIST_WTD_COMP_PARAMS *jcp_param";
specialize qw/aom_highbd_dist_wtd_comp_avg_pred sse2/;
#
# Subpixel Variance

17
third_party/aom/aom_dsp/arm/loopfilter_neon.c vendored
View file

@ -785,16 +785,15 @@ void aom_lpf_vertical_4_neon(uint8_t *src, int stride, const uint8_t *blimit,
const uint8_t *limit, const uint8_t *thresh) {
uint32x2x2_t p1q0_p0q1, p1q1_p0q0, p1p0_q1q0;
uint32x2_t pq_rev;
uint8x8_t UNINITIALIZED_IS_SAFE(p1p0), q0q1, p0q0, p1q1;
uint8x8_t UNINITIALIZED_IS_SAFE(p1p0), UNINITIALIZED_IS_SAFE(q0q1);
uint8x8_t p0q0, p1q1;
// row0: p1 p0 | q0 q1
// row1: p1 p0 | q0 q1
// row2: p1 p0 | q0 q1
// row3: p1 p0 | q0 q1
load_u8_4x1(src - 2, &p1p0, 0);
load_u8_4x1((src - 2) + 1 * stride, &p1p0, 1);
load_u8_4x1((src - 2) + 2 * stride, &q0q1, 0);
load_u8_4x1((src - 2) + 3 * stride, &q0q1, 1);
load_unaligned_u8_4x4(src - 2, stride, (uint32x2_t *)&p1p0,
(uint32x2_t *)&q0q1);
transpose_u8_4x4(&p1p0, &q0q1);
@ -815,10 +814,10 @@ void aom_lpf_vertical_4_neon(uint8_t *src, int stride, const uint8_t *blimit,
transpose_u8_4x4(&p1p0, &q0q1);
store_u8_4x1(src - 2, p1p0, 0);
store_u8_4x1((src - 2) + 1 * stride, q0q1, 0);
store_u8_4x1((src - 2) + 2 * stride, p1p0, 1);
store_u8_4x1((src - 2) + 3 * stride, q0q1, 1);
store_unaligned_u8_4x1(src - 2, p1p0, 0);
store_unaligned_u8_4x1((src - 2) + 1 * stride, q0q1, 0);
store_unaligned_u8_4x1((src - 2) + 2 * stride, p1p0, 1);
store_unaligned_u8_4x1((src - 2) + 3 * stride, q0q1, 1);
}
void aom_lpf_horizontal_14_neon(uint8_t *src, int stride, const uint8_t *blimit,

148
third_party/aom/aom_dsp/avg.c vendored Normal file
View file

@ -0,0 +1,148 @@
/*
* Copyright (c) 2016, Alliance for Open Media. All rights reserved
*
* This source code is subject to the terms of the BSD 2 Clause License and
* the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
* was not distributed with this source code in the LICENSE file, you can
* obtain it at www.aomedia.org/license/software. If the Alliance for Open
* Media Patent License 1.0 was not distributed with this source code in the
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
*/
#include <stdlib.h>
#include "config/aom_dsp_rtcd.h"
#include "aom_ports/mem.h"
// src_diff: first pass, 9 bit, dynamic range [-255, 255]
// second pass, 12 bit, dynamic range [-2040, 2040]
static void hadamard_col8(const int16_t *src_diff, ptrdiff_t src_stride,
int16_t *coeff) {
int16_t b0 = src_diff[0 * src_stride] + src_diff[1 * src_stride];
int16_t b1 = src_diff[0 * src_stride] - src_diff[1 * src_stride];
int16_t b2 = src_diff[2 * src_stride] + src_diff[3 * src_stride];
int16_t b3 = src_diff[2 * src_stride] - src_diff[3 * src_stride];
int16_t b4 = src_diff[4 * src_stride] + src_diff[5 * src_stride];
int16_t b5 = src_diff[4 * src_stride] - src_diff[5 * src_stride];
int16_t b6 = src_diff[6 * src_stride] + src_diff[7 * src_stride];
int16_t b7 = src_diff[6 * src_stride] - src_diff[7 * src_stride];
int16_t c0 = b0 + b2;
int16_t c1 = b1 + b3;
int16_t c2 = b0 - b2;
int16_t c3 = b1 - b3;
int16_t c4 = b4 + b6;
int16_t c5 = b5 + b7;
int16_t c6 = b4 - b6;
int16_t c7 = b5 - b7;
coeff[0] = c0 + c4;
coeff[7] = c1 + c5;
coeff[3] = c2 + c6;
coeff[4] = c3 + c7;
coeff[2] = c0 - c4;
coeff[6] = c1 - c5;
coeff[1] = c2 - c6;
coeff[5] = c3 - c7;
}
// The order of the output coeff of the hadamard is not important. For
// optimization purposes the final transpose may be skipped.
void aom_hadamard_8x8_c(const int16_t *src_diff, ptrdiff_t src_stride,
tran_low_t *coeff) {
int idx;
int16_t buffer[64];
int16_t buffer2[64];
int16_t *tmp_buf = &buffer[0];
for (idx = 0; idx < 8; ++idx) {
hadamard_col8(src_diff, src_stride, tmp_buf); // src_diff: 9 bit
// dynamic range [-255, 255]
tmp_buf += 8;
++src_diff;
}
tmp_buf = &buffer[0];
for (idx = 0; idx < 8; ++idx) {
hadamard_col8(tmp_buf, 8, buffer2 + 8 * idx); // tmp_buf: 12 bit
// dynamic range [-2040, 2040]
// buffer2: 15 bit
// dynamic range [-16320, 16320]
++tmp_buf;
}
for (idx = 0; idx < 64; ++idx) coeff[idx] = (tran_low_t)buffer2[idx];
}
// In place 16x16 2D Hadamard transform
void aom_hadamard_16x16_c(const int16_t *src_diff, ptrdiff_t src_stride,
tran_low_t *coeff) {
int idx;
for (idx = 0; idx < 4; ++idx) {
// src_diff: 9 bit, dynamic range [-255, 255]
const int16_t *src_ptr =
src_diff + (idx >> 1) * 8 * src_stride + (idx & 0x01) * 8;
aom_hadamard_8x8_c(src_ptr, src_stride, coeff + idx * 64);
}
// coeff: 15 bit, dynamic range [-16320, 16320]
for (idx = 0; idx < 64; ++idx) {
tran_low_t a0 = coeff[0];
tran_low_t a1 = coeff[64];
tran_low_t a2 = coeff[128];
tran_low_t a3 = coeff[192];
tran_low_t b0 = (a0 + a1) >> 1; // (a0 + a1): 16 bit, [-32640, 32640]
tran_low_t b1 = (a0 - a1) >> 1; // b0-b3: 15 bit, dynamic range
tran_low_t b2 = (a2 + a3) >> 1; // [-16320, 16320]
tran_low_t b3 = (a2 - a3) >> 1;
coeff[0] = b0 + b2; // 16 bit, [-32640, 32640]
coeff[64] = b1 + b3;
coeff[128] = b0 - b2;
coeff[192] = b1 - b3;
++coeff;
}
}
void aom_hadamard_32x32_c(const int16_t *src_diff, ptrdiff_t src_stride,
tran_low_t *coeff) {
int idx;
for (idx = 0; idx < 4; ++idx) {
// src_diff: 9 bit, dynamic range [-255, 255]
const int16_t *src_ptr =
src_diff + (idx >> 1) * 16 * src_stride + (idx & 0x01) * 16;
aom_hadamard_16x16_c(src_ptr, src_stride, coeff + idx * 256);
}
// coeff: 15 bit, dynamic range [-16320, 16320]
for (idx = 0; idx < 256; ++idx) {
tran_low_t a0 = coeff[0];
tran_low_t a1 = coeff[256];
tran_low_t a2 = coeff[512];
tran_low_t a3 = coeff[768];
tran_low_t b0 = (a0 + a1) >> 2; // (a0 + a1): 16 bit, [-32640, 32640]
tran_low_t b1 = (a0 - a1) >> 2; // b0-b3: 15 bit, dynamic range
tran_low_t b2 = (a2 + a3) >> 2; // [-16320, 16320]
tran_low_t b3 = (a2 - a3) >> 2;
coeff[0] = b0 + b2; // 16 bit, [-32640, 32640]
coeff[256] = b1 + b3;
coeff[512] = b0 - b2;
coeff[768] = b1 - b3;
++coeff;
}
}
// coeff: 16 bits, dynamic range [-32640, 32640].
// length: value range {16, 64, 256, 1024}.
int aom_satd_c(const tran_low_t *coeff, int length) {
int i;
int satd = 0;
for (i = 0; i < length; ++i) satd += abs(coeff[i]);
// satd: 26 bits, dynamic range [-32640 * 1024, 32640 * 1024]
return satd;
}

69
third_party/aom/aom_dsp/binary_codes_reader.c vendored
View file

@ -10,29 +10,9 @@
*/
#include "aom_dsp/binary_codes_reader.h"
#include "aom_dsp/recenter.h"
#include "av1/common/common.h"
// Inverse recenters a non-negative literal v around a reference r
static uint16_t inv_recenter_nonneg(uint16_t r, uint16_t v) {
if (v > (r << 1))
return v;
else if ((v & 1) == 0)
return (v >> 1) + r;
else
return r - ((v + 1) >> 1);
}
// Inverse recenters a non-negative literal v in [0, n-1] around a
// reference r also in [0, n-1]
static uint16_t inv_recenter_finite_nonneg(uint16_t n, uint16_t r, uint16_t v) {
if ((r << 1) <= n) {
return inv_recenter_nonneg(r, v);
} else {
return n - 1 - inv_recenter_nonneg(n - 1 - r, v);
}
}
uint16_t aom_read_primitive_quniform_(aom_reader *r,
uint16_t n ACCT_STR_PARAM) {
if (n <= 1) return 0;
@ -42,15 +22,6 @@ uint16_t aom_read_primitive_quniform_(aom_reader *r,
return v < m ? v : (v << 1) - m + aom_read_bit(r, ACCT_STR_NAME);
}
static uint16_t aom_rb_read_primitive_quniform(struct aom_read_bit_buffer *rb,
uint16_t n) {
if (n <= 1) return 0;
const int l = get_msb(n) + 1;
const int m = (1 << l) - n;
const int v = aom_rb_read_literal(rb, l - 1);
return v < m ? v : (v << 1) - m + aom_rb_read_bit(rb);
}
// Decode finite subexponential code that for a symbol v in [0, n-1] with
// parameter k
uint16_t aom_read_primitive_subexpfin_(aom_reader *r, uint16_t n,
@ -78,46 +49,8 @@ uint16_t aom_read_primitive_subexpfin_(aom_reader *r, uint16_t n,
return 0;
}
static uint16_t aom_rb_read_primitive_subexpfin(struct aom_read_bit_buffer *rb,
uint16_t n, uint16_t k) {
int i = 0;
int mk = 0;
while (1) {
int b = (i ? k + i - 1 : k);
int a = (1 << b);
if (n <= mk + 3 * a) {
return aom_rb_read_primitive_quniform(rb, n - mk) + mk;
}
if (!aom_rb_read_bit(rb)) {
return aom_rb_read_literal(rb, b) + mk;
}
i = i + 1;
mk += a;
}
assert(0);
return 0;
}
uint16_t aom_read_primitive_refsubexpfin_(aom_reader *r, uint16_t n, uint16_t k,
uint16_t ref ACCT_STR_PARAM) {
return inv_recenter_finite_nonneg(
n, ref, aom_read_primitive_subexpfin(r, n, k, ACCT_STR_NAME));
}
static uint16_t aom_rb_read_primitive_refsubexpfin(
struct aom_read_bit_buffer *rb, uint16_t n, uint16_t k, uint16_t ref) {
return inv_recenter_finite_nonneg(n, ref,
aom_rb_read_primitive_subexpfin(rb, n, k));
}
int16_t aom_rb_read_signed_primitive_refsubexpfin(
struct aom_read_bit_buffer *rb, uint16_t n, uint16_t k, int16_t ref) {
ref += n - 1;
const uint16_t scaled_n = (n << 1) - 1;
return aom_rb_read_primitive_refsubexpfin(rb, scaled_n, k, ref) - n + 1;
}

3
third_party/aom/aom_dsp/binary_codes_reader.h vendored
View file

@ -37,9 +37,6 @@ uint16_t aom_read_primitive_subexpfin_(aom_reader *r, uint16_t n,
uint16_t aom_read_primitive_refsubexpfin_(aom_reader *r, uint16_t n, uint16_t k,
uint16_t ref ACCT_STR_PARAM);
int16_t aom_rb_read_signed_primitive_refsubexpfin(
struct aom_read_bit_buffer *rb, uint16_t n, uint16_t k, int16_t ref);
#ifdef __cplusplus
} // extern "C"
#endif

76
third_party/aom/aom_dsp/binary_codes_writer.c vendored
View file

@ -11,29 +11,10 @@
#include "aom_dsp/bitwriter.h"
#include "aom_dsp/binary_codes_writer.h"
#include "aom_dsp/recenter.h"
#include "aom_ports/bitops.h"
#include "av1/common/common.h"
// Recenters a non-negative literal v around a reference r
static uint16_t recenter_nonneg(uint16_t r, uint16_t v) {
if (v > (r << 1))
return v;
else if (v >= r)
return ((v - r) << 1);
else
return ((r - v) << 1) - 1;
}
// Recenters a non-negative literal v in [0, n-1] around a
// reference r also in [0, n-1]
static uint16_t recenter_finite_nonneg(uint16_t n, uint16_t r, uint16_t v) {
if ((r << 1) <= n) {
return recenter_nonneg(r, v);
} else {
return recenter_nonneg(n - 1 - r, n - 1 - v);
}
}
// Codes a symbol v in [-2^mag_bits, 2^mag_bits].
// mag_bits is number of bits for magnitude. The alphabet is of size
// 2 * 2^mag_bits + 1, symmetric around 0, where one bit is used to
@ -69,19 +50,6 @@ void aom_write_primitive_quniform(aom_writer *w, uint16_t n, uint16_t v) {
}
}
static void aom_wb_write_primitive_quniform(struct aom_write_bit_buffer *wb,
uint16_t n, uint16_t v) {
if (n <= 1) return;
const int l = get_msb(n) + 1;
const int m = (1 << l) - n;
if (v < m) {
aom_wb_write_literal(wb, v, l - 1);
} else {
aom_wb_write_literal(wb, m + ((v - m) >> 1), l - 1);
aom_wb_write_bit(wb, (v - m) & 1);
}
}
int aom_count_primitive_quniform(uint16_t n, uint16_t v) {
if (n <= 1) return 0;
const int l = get_msb(n) + 1;
@ -114,31 +82,6 @@ void aom_write_primitive_subexpfin(aom_writer *w, uint16_t n, uint16_t k,
}
}
static void aom_wb_write_primitive_subexpfin(struct aom_write_bit_buffer *wb,
uint16_t n, uint16_t k,
uint16_t v) {
int i = 0;
int mk = 0;
while (1) {
int b = (i ? k + i - 1 : k);
int a = (1 << b);
if (n <= mk + 3 * a) {
aom_wb_write_primitive_quniform(wb, n - mk, v - mk);
break;
} else {
int t = (v >= mk + a);
aom_wb_write_bit(wb, t);
if (t) {
i = i + 1;
mk += a;
} else {
aom_wb_write_literal(wb, v - mk, b);
break;
}
}
}
}
int aom_count_primitive_subexpfin(uint16_t n, uint16_t k, uint16_t v) {
int count = 0;
int i = 0;
@ -172,12 +115,6 @@ void aom_write_primitive_refsubexpfin(aom_writer *w, uint16_t n, uint16_t k,
aom_write_primitive_subexpfin(w, n, k, recenter_finite_nonneg(n, ref, v));
}
static void aom_wb_write_primitive_refsubexpfin(struct aom_write_bit_buffer *wb,
uint16_t n, uint16_t k,
uint16_t ref, uint16_t v) {
aom_wb_write_primitive_subexpfin(wb, n, k, recenter_finite_nonneg(n, ref, v));
}
void aom_write_signed_primitive_refsubexpfin(aom_writer *w, uint16_t n,
uint16_t k, int16_t ref,
int16_t v) {
@ -187,15 +124,6 @@ void aom_write_signed_primitive_refsubexpfin(aom_writer *w, uint16_t n,
aom_write_primitive_refsubexpfin(w, scaled_n, k, ref, v);
}
void aom_wb_write_signed_primitive_refsubexpfin(struct aom_write_bit_buffer *wb,
uint16_t n, uint16_t k,
int16_t ref, int16_t v) {
ref += n - 1;
v += n - 1;
const uint16_t scaled_n = (n << 1) - 1;
aom_wb_write_primitive_refsubexpfin(wb, scaled_n, k, ref, v);
}
int aom_count_primitive_refsubexpfin(uint16_t n, uint16_t k, uint16_t ref,
uint16_t v) {
return aom_count_primitive_subexpfin(n, k, recenter_finite_nonneg(n, ref, v));

5
third_party/aom/aom_dsp/binary_codes_writer.h vendored
View file

@ -49,10 +49,6 @@ void aom_write_signed_primitive_refsubexpfin(aom_writer *w, uint16_t n,
uint16_t k, int16_t ref,
int16_t v);
void aom_wb_write_signed_primitive_refsubexpfin(struct aom_write_bit_buffer *wb,
uint16_t n, uint16_t k,
int16_t ref, int16_t v);
// Functions that counts bits for the above primitives
int aom_count_primitive_symmetric(int16_t v, unsigned int mag_bits);
int aom_count_primitive_quniform(uint16_t n, uint16_t v);
@ -61,6 +57,7 @@ int aom_count_primitive_refsubexpfin(uint16_t n, uint16_t k, uint16_t ref,
uint16_t v);
int aom_count_signed_primitive_refsubexpfin(uint16_t n, uint16_t k, int16_t ref,
int16_t v);
#ifdef __cplusplus
} // extern "C"
#endif

4
third_party/aom/aom_dsp/bitreader.h vendored
View file

@ -65,10 +65,6 @@ static INLINE const uint8_t *aom_reader_find_end(aom_reader *r) {
return aom_daala_reader_find_end(r);
}
static INLINE int aom_reader_has_error(aom_reader *r) {
return aom_daala_reader_has_error(r);
}
// Returns true if the bit reader has tried to decode more data from the buffer
// than was actually provided.
static INLINE int aom_reader_has_overflowed(const aom_reader *r) {

49
third_party/aom/aom_dsp/bitreader_buffer.c vendored
View file

@ -14,6 +14,8 @@
#include "config/aom_config.h"
#include "aom_dsp/bitreader_buffer.h"
#include "aom_dsp/recenter.h"
#include "aom_ports/bitops.h"
size_t aom_rb_bytes_read(const struct aom_read_bit_buffer *rb) {
return (rb->bit_offset + 7) >> 3;
@ -65,3 +67,50 @@ uint32_t aom_rb_read_uvlc(struct aom_read_bit_buffer *rb) {
const uint32_t value = aom_rb_read_literal(rb, leading_zeros);
return base + value;
}
static uint16_t aom_rb_read_primitive_quniform(struct aom_read_bit_buffer *rb,
uint16_t n) {
if (n <= 1) return 0;
const int l = get_msb(n) + 1;
const int m = (1 << l) - n;
const int v = aom_rb_read_literal(rb, l - 1);
return v < m ? v : (v << 1) - m + aom_rb_read_bit(rb);
}
static uint16_t aom_rb_read_primitive_subexpfin(struct aom_read_bit_buffer *rb,
uint16_t n, uint16_t k) {
int i = 0;
int mk = 0;
while (1) {
int b = (i ? k + i - 1 : k);
int a = (1 << b);
if (n <= mk + 3 * a) {
return aom_rb_read_primitive_quniform(rb, n - mk) + mk;
}
if (!aom_rb_read_bit(rb)) {
return aom_rb_read_literal(rb, b) + mk;
}
i = i + 1;
mk += a;
}
assert(0);
return 0;
}
static uint16_t aom_rb_read_primitive_refsubexpfin(
struct aom_read_bit_buffer *rb, uint16_t n, uint16_t k, uint16_t ref) {
return inv_recenter_finite_nonneg(n, ref,
aom_rb_read_primitive_subexpfin(rb, n, k));
}
int16_t aom_rb_read_signed_primitive_refsubexpfin(
struct aom_read_bit_buffer *rb, uint16_t n, uint16_t k, int16_t ref) {
ref += n - 1;
const uint16_t scaled_n = (n << 1) - 1;
return aom_rb_read_primitive_refsubexpfin(rb, scaled_n, k, ref) - n + 1;
}

3
third_party/aom/aom_dsp/bitreader_buffer.h vendored
View file

@ -43,6 +43,9 @@ int aom_rb_read_inv_signed_literal(struct aom_read_bit_buffer *rb, int bits);
uint32_t aom_rb_read_uvlc(struct aom_read_bit_buffer *rb);
int16_t aom_rb_read_signed_primitive_refsubexpfin(
struct aom_read_bit_buffer *rb, uint16_t n, uint16_t k, int16_t ref);
#ifdef __cplusplus
} // extern "C"
#endif

55
third_party/aom/aom_dsp/bitwriter_buffer.c vendored
View file

@ -16,6 +16,8 @@
#include "config/aom_config.h"
#include "aom_dsp/bitwriter_buffer.h"
#include "aom_dsp/recenter.h"
#include "aom_ports/bitops.h"
int aom_wb_is_byte_aligned(const struct aom_write_bit_buffer *wb) {
return (wb->bit_offset % CHAR_BIT == 0);
@ -85,3 +87,56 @@ void aom_wb_write_uvlc(struct aom_write_bit_buffer *wb, uint32_t v) {
aom_wb_write_literal(wb, 0, leading_zeroes >> 1);
aom_wb_write_unsigned_literal(wb, v, (leading_zeroes + 1) >> 1);
}
static void aom_wb_write_primitive_quniform(struct aom_write_bit_buffer *wb,
uint16_t n, uint16_t v) {
if (n <= 1) return;
const int l = get_msb(n) + 1;
const int m = (1 << l) - n;
if (v < m) {
aom_wb_write_literal(wb, v, l - 1);
} else {
aom_wb_write_literal(wb, m + ((v - m) >> 1), l - 1);
aom_wb_write_bit(wb, (v - m) & 1);
}
}
static void aom_wb_write_primitive_subexpfin(struct aom_write_bit_buffer *wb,
uint16_t n, uint16_t k,
uint16_t v) {
int i = 0;
int mk = 0;
while (1) {
int b = (i ? k + i - 1 : k);
int a = (1 << b);
if (n <= mk + 3 * a) {
aom_wb_write_primitive_quniform(wb, n - mk, v - mk);
break;
} else {
int t = (v >= mk + a);
aom_wb_write_bit(wb, t);
if (t) {
i = i + 1;
mk += a;
} else {
aom_wb_write_literal(wb, v - mk, b);
break;
}
}
}
}
static void aom_wb_write_primitive_refsubexpfin(struct aom_write_bit_buffer *wb,
uint16_t n, uint16_t k,
uint16_t ref, uint16_t v) {
aom_wb_write_primitive_subexpfin(wb, n, k, recenter_finite_nonneg(n, ref, v));
}
void aom_wb_write_signed_primitive_refsubexpfin(struct aom_write_bit_buffer *wb,
uint16_t n, uint16_t k,
int16_t ref, int16_t v) {
ref += n - 1;
v += n - 1;
const uint16_t scaled_n = (n << 1) - 1;
aom_wb_write_primitive_refsubexpfin(wb, scaled_n, k, ref, v);
}

4
third_party/aom/aom_dsp/bitwriter_buffer.h vendored
View file

@ -44,6 +44,10 @@ void aom_wb_write_inv_signed_literal(struct aom_write_bit_buffer *wb, int data,
void aom_wb_write_uvlc(struct aom_write_bit_buffer *wb, uint32_t v);
void aom_wb_write_signed_primitive_refsubexpfin(struct aom_write_bit_buffer *wb,
uint16_t n, uint16_t k,
int16_t ref, int16_t v);
#ifdef __cplusplus
} // extern "C"
#endif

4
third_party/aom/aom_dsp/blend_a64_mask.c vendored
View file

@ -22,7 +22,7 @@
// as described for AOM_BLEND_A64 in aom_dsp/blend.h. src0 or src1 can
// be the same as dst, or dst can be different from both sources.
// NOTE(david.barker): The input and output of aom_blend_a64_d32_mask_c() are
// NOTE(david.barker): The input and output of aom_blend_a64_d16_mask_c() are
// in a higher intermediate precision, and will later be rounded down to pixel
// precision.
// Thus, in order to avoid double-rounding, we want to use normal right shifts
@ -30,7 +30,7 @@
// This works because of the identity:
// ROUND_POWER_OF_TWO(x >> y, z) == ROUND_POWER_OF_TWO(x, y+z)
//
// In contrast, the output of the non-d32 functions will not be further rounded,
// In contrast, the output of the non-d16 functions will not be further rounded,
// so we *should* use ROUND_POWER_OF_TWO there.
void aom_lowbd_blend_a64_d16_mask_c(

4
third_party/aom/aom_dsp/daalaboolreader.h vendored
View file

@ -101,10 +101,6 @@ static INLINE int aom_daala_read(daala_reader *r, int prob) {
return bit;
}
static INLINE int aom_daala_reader_has_error(daala_reader *r) {
return r->ec.error;
}
static INLINE int daala_read_symbol(daala_reader *r, const aom_cdf_prob *cdf,
int nsymbs) {
int symb;

1
third_party/aom/aom_dsp/entcode.h vendored
View file

@ -24,6 +24,7 @@
on a larger type, you can speed up the decoder by using it here.*/
typedef uint32_t od_ec_window;
/*The size in bits of od_ec_window.*/
#define OD_EC_WINDOW_SIZE ((int)sizeof(od_ec_window) * CHAR_BIT)
/*The resolution of fractional-precision bit usage measurements, i.e.,

24
third_party/aom/aom_dsp/entdec.c vendored
View file

@ -87,11 +87,25 @@ static void od_ec_dec_refill(od_ec_dec *dec) {
end = dec->end;
s = OD_EC_WINDOW_SIZE - 9 - (cnt + 15);
for (; s >= 0 && bptr < end; s -= 8, bptr++) {
/*Each time a byte is inserted into the window (dif), bptr advances and cnt
is incremented by 8, so the total number of consumed bits (the return
value of od_ec_dec_tell) does not change.*/
assert(s <= OD_EC_WINDOW_SIZE - 8);
dif ^= (od_ec_window)bptr[0] << s;
cnt += 8;
}
if (bptr >= end) {
/*We've reached the end of the buffer. It is perfectly valid for us to need
to fill the window with additional bits past the end of the buffer (and
this happens in normal operation). These bits should all just be taken
as zero. But we cannot increment bptr past 'end' (this is undefined
behavior), so we start to increment dec->tell_offs. We also don't want
to keep testing bptr against 'end', so we set cnt to OD_EC_LOTS_OF_BITS
and adjust dec->tell_offs so that the total number of unconsumed bits in
the window (dec->cnt - dec->tell_offs) does not change. This effectively
puts lots of zero bits into the window, and means we won't try to refill
it from the buffer for a very long time (at which point we'll put lots
of zero bits into the window again).*/
dec->tell_offs += OD_EC_LOTS_OF_BITS - cnt;
cnt = OD_EC_LOTS_OF_BITS;
}
@ -112,8 +126,9 @@ static int od_ec_dec_normalize(od_ec_dec *dec, od_ec_window dif, unsigned rng,
int ret) {
int d;
assert(rng <= 65535U);
// The number of leading zeros in the 16-bit binary representation of rng.
/*The number of leading zeros in the 16-bit binary representation of rng.*/
d = 16 - OD_ILOG_NZ(rng);
/*d bits in dec->dif are consumed.*/
dec->cnt -= d;
/*This is equivalent to shifting in 1's instead of 0's.*/
dec->dif = ((dif + 1) << d) - 1;
@ -124,7 +139,7 @@ static int od_ec_dec_normalize(od_ec_dec *dec, od_ec_window dif, unsigned rng,
/*Initializes the decoder.
buf: The input buffer to use.
Return: 0 on success, or a negative value on error.*/
storage: The size in bytes of the input buffer.*/
void od_ec_dec_init(od_ec_dec *dec, const unsigned char *buf,
uint32_t storage) {
dec->buf = buf;
@ -134,7 +149,6 @@ void od_ec_dec_init(od_ec_dec *dec, const unsigned char *buf,
dec->dif = ((od_ec_window)1 << (OD_EC_WINDOW_SIZE - 1)) - 1;
dec->rng = 0x8000;
dec->cnt = -15;
dec->error = 0;
od_ec_dec_refill(dec);
}
@ -215,6 +229,10 @@ int od_ec_decode_cdf_q15(od_ec_dec *dec, const uint16_t *icdf, int nsyms) {
This will always be slightly larger than the exact value (e.g., all
rounding error is in the positive direction).*/
int od_ec_dec_tell(const od_ec_dec *dec) {
/*There is a window of bits stored in dec->dif. The difference
(dec->bptr - dec->buf) tells us how many bytes have been read into this
window. The difference (dec->cnt - dec->tell_offs) tells us how many of
the bits in that window remain unconsumed.*/
return (int)((dec->bptr - dec->buf) * 8 - dec->cnt + dec->tell_offs);
}

4
third_party/aom/aom_dsp/entdec.h vendored
View file

@ -34,7 +34,7 @@ struct od_ec_dec {
const unsigned char *buf;
/*An offset used to keep track of tell after reaching the end of the stream.
This is constant throughout most of the decoding process, but becomes
important once we hit the end of the buffer and stop incrementing pointers
important once we hit the end of the buffer and stop incrementing bptr
(and instead pretend cnt has lots of bits).*/
int32_t tell_offs;
/*The end of the current input buffer.*/
@ -53,8 +53,6 @@ struct od_ec_dec {
uint16_t rng;
/*The number of bits of data in the current value.*/
int16_t cnt;
/*Nonzero if an error occurred.*/
int error;
};
/*See entdec.c for further documentation.*/

2
third_party/aom/aom_dsp/entenc.c vendored
View file

@ -60,7 +60,7 @@ static void od_ec_enc_normalize(od_ec_enc *enc, od_ec_window low,
int s;
c = enc->cnt;
assert(rng <= 65535U);
// The number of leading zeros in the 16-bit binary representation of rng.
/*The number of leading zeros in the 16-bit binary representation of rng.*/
d = 16 - OD_ILOG_NZ(rng);
s = c + d;
/*TODO: Right now we flush every time we have at least one byte available.

70
third_party/aom/aom_dsp/grain_synthesis.h vendored
View file

@ -20,6 +20,8 @@
extern "C" {
#endif
#include <string.h>
#include "aom_dsp/aom_dsp_common.h"
#include "aom/aom_image.h"
@ -28,6 +30,9 @@ extern "C" {
* This structure contains input parameters for film grain synthesis
*/
typedef struct {
// This structure is compared element-by-element in the function
// av1_check_grain_params_equiv: this function must be updated if any changes
// are made to this structure.
int apply_grain;
int update_parameters;
@ -79,8 +84,73 @@ typedef struct {
int grain_scale_shift;
uint16_t random_seed;
// This structure is compared element-by-element in the function
// av1_check_grain_params_equiv: this function must be updated if any changes
// are made to this structure.
} aom_film_grain_t;
/*!\brief Check if two film grain parameters structs are equivalent
*
* Check if two film grain parameters are equal, except for the
* update_parameters and random_seed elements which are ignored.
*
* \param[in] pa The first set of parameters to compare
* \param[in] pb The second set of parameters to compare
* \return Returns 1 if the params are equivalent, 0 otherwise
*/
static INLINE int av1_check_grain_params_equiv(
const aom_film_grain_t *const pa, const aom_film_grain_t *const pb) {
if (pa->apply_grain != pb->apply_grain) return 0;
// Don't compare update_parameters
if (pa->num_y_points != pb->num_y_points) return 0;
if (memcmp(pa->scaling_points_y, pb->scaling_points_y,
pa->num_y_points * 2 * sizeof(*pa->scaling_points_y)) != 0)
return 0;
if (pa->num_cb_points != pb->num_cb_points) return 0;
if (memcmp(pa->scaling_points_cb, pb->scaling_points_cb,
pa->num_cb_points * 2 * sizeof(*pa->scaling_points_cb)) != 0)
return 0;
if (pa->num_cr_points != pb->num_cr_points) return 0;
if (memcmp(pa->scaling_points_cr, pb->scaling_points_cr,
pa->num_cr_points * 2 * sizeof(*pa->scaling_points_cr)) != 0)
return 0;
if (pa->scaling_shift != pb->scaling_shift) return 0;
if (pa->ar_coeff_lag != pb->ar_coeff_lag) return 0;
const int num_pos = 2 * pa->ar_coeff_lag * (pa->ar_coeff_lag + 1);
if (memcmp(pa->ar_coeffs_y, pb->ar_coeffs_y,
num_pos * sizeof(*pa->ar_coeffs_y)) != 0)
return 0;
if (memcmp(pa->ar_coeffs_cb, pb->ar_coeffs_cb,
num_pos * sizeof(*pa->ar_coeffs_cb)) != 0)
return 0;
if (memcmp(pa->ar_coeffs_cr, pb->ar_coeffs_cr,
num_pos * sizeof(*pa->ar_coeffs_cr)) != 0)
return 0;
if (pa->ar_coeff_shift != pb->ar_coeff_shift) return 0;
if (pa->cb_mult != pb->cb_mult) return 0;
if (pa->cb_luma_mult != pb->cb_luma_mult) return 0;
if (pa->cb_offset != pb->cb_offset) return 0;
if (pa->cr_mult != pb->cr_mult) return 0;
if (pa->cr_luma_mult != pb->cr_luma_mult) return 0;
if (pa->cr_offset != pb->cr_offset) return 0;
if (pa->overlap_flag != pb->overlap_flag) return 0;
if (pa->clip_to_restricted_range != pb->clip_to_restricted_range) return 0;
if (pa->bit_depth != pb->bit_depth) return 0;
if (pa->chroma_scaling_from_luma != pb->chroma_scaling_from_luma) return 0;
if (pa->grain_scale_shift != pb->grain_scale_shift) return 0;
return 1;
}
/*!\brief Add film grain
*
* Add film grain to an image

3
third_party/aom/aom_dsp/grain_table.c vendored
View file

@ -134,7 +134,8 @@ static void grain_table_entry_read(FILE *file,
}
}
void grain_table_entry_write(FILE *file, aom_film_grain_table_entry_t *entry) {
static void grain_table_entry_write(FILE *file,
aom_film_grain_table_entry_t *entry) {
const aom_film_grain_t *pars = &entry->params;
fprintf(file, "E %" PRId64 " %" PRId64 " %d %d %d\n", entry->start_time,
entry->end_time, pars->apply_grain, pars->random_seed,

32
third_party/aom/aom_dsp/intrapred.c vendored
View file

@ -239,23 +239,23 @@ static INLINE int divide_using_multiply_shift(int num, int shift1,
return interm * multiplier >> shift2;
}
// The constants (multiplier and shifts) for a given block size are obtained
// as follows:
// - Let sum_w_h = block width + block height.
// - Shift 'sum_w_h' right until we reach an odd number. Let the number of
// shifts for that block size be called 'shift1' (see the parameter in
// dc_predictor_rect() function), and let the odd number be 'd'. [d has only 2
// possible values: d = 3 for a 1:2 rect block and d = 5 for a 1:4 rect
// block].
// - Find multipliers for (i) dividing by 3, and (ii) dividing by 5,
// using the "Algorithm 1" in:
// http://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=1467632
// by ensuring that m + n = 16 (in that algorithm). This ensures that our 2nd
// shift will be 16, regardless of the block size.
// The constants (multiplier and shifts) for a given block size are obtained
// as follows:
// - Let sum_w_h = block width + block height.
// - Shift 'sum_w_h' right until we reach an odd number. Let the number of
// shifts for that block size be called 'shift1' (see the parameter in
// dc_predictor_rect() function), and let the odd number be 'd'. [d has only 2
// possible values: d = 3 for a 1:2 rect block and d = 5 for a 1:4 rect
// block].
// - Find multipliers for (i) dividing by 3, and (ii) dividing by 5,
// using the "Algorithm 1" in:
// http://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=1467632
// by ensuring that m + n = 16 (in that algorithm). This ensures that our 2nd
// shift will be 16, regardless of the block size.
// Note: For low bitdepth, assembly code may be optimized by using smaller
// constants for smaller block sizes, where the range of the 'sum' is
// restricted to fewer bits.
// Note: For low bitdepth, assembly code may be optimized by using smaller
// constants for smaller block sizes, where the range of the 'sum' is
// restricted to fewer bits.
#define DC_MULTIPLIER_1X2 0x5556
#define DC_MULTIPLIER_1X4 0x3334

8
third_party/aom/aom_dsp/noise_model.h vendored
View file

@ -158,10 +158,10 @@ int aom_flat_block_finder_run(const aom_flat_block_finder_t *block_finder,
int stride, uint8_t *flat_blocks);
// The noise shape indicates the allowed coefficients in the AR model.
typedef enum {
enum {
AOM_NOISE_SHAPE_DIAMOND = 0,
AOM_NOISE_SHAPE_SQUARE = 1
} aom_noise_shape;
} UENUM1BYTE(aom_noise_shape);
// The parameters of the noise model include the shape type, lag, the
// bit depth of the input images provided, and whether the input images
@ -202,13 +202,13 @@ typedef struct {
} aom_noise_model_t;
/*!\brief Result of a noise model update. */
typedef enum {
enum {
AOM_NOISE_STATUS_OK = 0,
AOM_NOISE_STATUS_INVALID_ARGUMENT,
AOM_NOISE_STATUS_INSUFFICIENT_FLAT_BLOCKS,
AOM_NOISE_STATUS_DIFFERENT_NOISE_TYPE,
AOM_NOISE_STATUS_INTERNAL_ERROR,
} aom_noise_status_t;
} UENUM1BYTE(aom_noise_status_t);
/*!\brief Initializes a noise model with the given parameters.
*

2
third_party/aom/aom_dsp/prob.h vendored
View file

@ -641,7 +641,7 @@ static INLINE uint8_t get_prob(unsigned int num, unsigned int den) {
}
}
static INLINE void update_cdf(aom_cdf_prob *cdf, int val, int nsymbs) {
static INLINE void update_cdf(aom_cdf_prob *cdf, int8_t val, int nsymbs) {
int rate;
int i, tmp;

194
third_party/aom/aom_dsp/quantize.c vendored
View file

@ -12,6 +12,68 @@
#include "aom_dsp/quantize.h"
#include "aom_mem/aom_mem.h"
void quantize_b_adaptive_helper_c(
const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr,
const int16_t *round_ptr, const int16_t *quant_ptr,
const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr,
tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr,
const int16_t *scan, const int16_t *iscan, const qm_val_t *qm_ptr,
const qm_val_t *iqm_ptr, const int log_scale) {
const int zbins[2] = { ROUND_POWER_OF_TWO(zbin_ptr[0], log_scale),
ROUND_POWER_OF_TWO(zbin_ptr[1], log_scale) };
const int nzbins[2] = { zbins[0] * -1, zbins[1] * -1 };
int i, non_zero_count = (int)n_coeffs, eob = -1;
(void)iscan;
memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr));
memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr));
// Pre-scan pass
for (i = (int)n_coeffs - 1; i >= 0; i--) {
const int rc = scan[i];
const qm_val_t wt = qm_ptr != NULL ? qm_ptr[rc] : (1 << AOM_QM_BITS);
const int coeff = coeff_ptr[rc] * wt;
int prescan_add = ROUND_POWER_OF_TWO(dequant_ptr[rc != 0] * 325, 7);
if (coeff < (zbins[rc != 0] * (1 << AOM_QM_BITS) + prescan_add) &&
coeff > (nzbins[rc != 0] * (1 << AOM_QM_BITS) - prescan_add))
non_zero_count--;
else
break;
}
// Quantization pass: All coefficients with index >= zero_flag are
// skippable. Note: zero_flag can be zero.
for (i = 0; i < non_zero_count; i++) {
const int rc = scan[i];
const int coeff = coeff_ptr[rc];
const int coeff_sign = (coeff >> 31);
const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign;
int tmp32;
const qm_val_t wt = qm_ptr != NULL ? qm_ptr[rc] : (1 << AOM_QM_BITS);
if (abs_coeff * wt >= (zbins[rc != 0] << AOM_QM_BITS)) {
int64_t tmp =
clamp(abs_coeff + ROUND_POWER_OF_TWO(round_ptr[rc != 0], log_scale),
INT16_MIN, INT16_MAX);
tmp *= wt;
tmp32 = (int)(((((tmp * quant_ptr[rc != 0]) >> 16) + tmp) *
quant_shift_ptr[rc != 0]) >>
(16 - log_scale + AOM_QM_BITS)); // quantization
qcoeff_ptr[rc] = (tmp32 ^ coeff_sign) - coeff_sign;
const int iwt = iqm_ptr != NULL ? iqm_ptr[rc] : (1 << AOM_QM_BITS);
const int dequant =
(dequant_ptr[rc != 0] * iwt + (1 << (AOM_QM_BITS - 1))) >>
AOM_QM_BITS;
const tran_low_t abs_dqcoeff = (tmp32 * dequant) >> log_scale;
dqcoeff_ptr[rc] = (tran_low_t)((abs_dqcoeff ^ coeff_sign) - coeff_sign);
if (tmp32) eob = i;
}
}
*eob_ptr = eob + 1;
}
void quantize_b_helper_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
const int16_t *zbin_ptr, const int16_t *round_ptr,
const int16_t *quant_ptr,
@ -74,6 +136,64 @@ void quantize_b_helper_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
*eob_ptr = eob + 1;
}
void highbd_quantize_b_adaptive_helper_c(
const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr,
const int16_t *round_ptr, const int16_t *quant_ptr,
const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr,
tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr,
const int16_t *scan, const int16_t *iscan, const qm_val_t *qm_ptr,
const qm_val_t *iqm_ptr, const int log_scale) {
int i, eob = -1;
const int zbins[2] = { ROUND_POWER_OF_TWO(zbin_ptr[0], log_scale),
ROUND_POWER_OF_TWO(zbin_ptr[1], log_scale) };
const int nzbins[2] = { zbins[0] * -1, zbins[1] * -1 };
int dequant;
int idx_arr[4096];
(void)iscan;
int idx = 0;
memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr));
memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr));
// Pre-scan pass
for (i = 0; i < n_coeffs; i++) {
const int rc = scan[i];
const qm_val_t wt = qm_ptr != NULL ? qm_ptr[rc] : (1 << AOM_QM_BITS);
const int coeff = coeff_ptr[rc] * wt;
// If the coefficient is out of the base ZBIN range, keep it for
// quantization.
int prescan_add = ROUND_POWER_OF_TWO(dequant_ptr[rc != 0] * 325, 7);
if (coeff >= (zbins[rc != 0] * (1 << AOM_QM_BITS) + prescan_add) &&
coeff <= (nzbins[rc != 0] * (1 << AOM_QM_BITS) - prescan_add))
idx_arr[idx++] = i;
}
// Quantization pass: only process the coefficients selected in
// pre-scan pass. Note: idx can be zero.
for (i = 0; i < idx; i++) {
const int rc = scan[idx_arr[i]];
const int coeff = coeff_ptr[rc];
const int coeff_sign = (coeff >> 31);
const qm_val_t wt = qm_ptr != NULL ? qm_ptr[rc] : (1 << AOM_QM_BITS);
const qm_val_t iwt = iqm_ptr != NULL ? iqm_ptr[rc] : (1 << AOM_QM_BITS);
const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign;
const int64_t tmp1 =
abs_coeff + ROUND_POWER_OF_TWO(round_ptr[rc != 0], log_scale);
const int64_t tmpw = tmp1 * wt;
const int64_t tmp2 = ((tmpw * quant_ptr[rc != 0]) >> 16) + tmpw;
const int abs_qcoeff = (int)((tmp2 * quant_shift_ptr[rc != 0]) >>
(16 - log_scale + AOM_QM_BITS));
qcoeff_ptr[rc] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign);
dequant =
(dequant_ptr[rc != 0] * iwt + (1 << (AOM_QM_BITS - 1))) >> AOM_QM_BITS;
const tran_low_t abs_dqcoeff = (abs_qcoeff * dequant) >> log_scale;
dqcoeff_ptr[rc] = (tran_low_t)((abs_dqcoeff ^ coeff_sign) - coeff_sign);
if (abs_qcoeff) eob = idx_arr[i];
}
*eob_ptr = eob + 1;
}
void highbd_quantize_b_helper_c(
const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr,
const int16_t *round_ptr, const int16_t *quant_ptr,
@ -133,6 +253,80 @@ void highbd_quantize_b_helper_c(
/* These functions should only be called when quantisation matrices
are not used. */
void aom_quantize_b_adaptive_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
const int16_t *zbin_ptr,
const int16_t *round_ptr,
const int16_t *quant_ptr,
const int16_t *quant_shift_ptr,
tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr,
const int16_t *dequant_ptr, uint16_t *eob_ptr,
const int16_t *scan, const int16_t *iscan) {
quantize_b_adaptive_helper_c(coeff_ptr, n_coeffs, zbin_ptr, round_ptr,
quant_ptr, quant_shift_ptr, qcoeff_ptr,
dqcoeff_ptr, dequant_ptr, eob_ptr, scan, iscan,
NULL, NULL, 0);
}
void aom_quantize_b_32x32_adaptive_c(
const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr,
const int16_t *round_ptr, const int16_t *quant_ptr,
const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr,
tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr,
const int16_t *scan, const int16_t *iscan) {
quantize_b_adaptive_helper_c(coeff_ptr, n_coeffs, zbin_ptr, round_ptr,
quant_ptr, quant_shift_ptr, qcoeff_ptr,
dqcoeff_ptr, dequant_ptr, eob_ptr, scan, iscan,
NULL, NULL, 1);
}
void aom_quantize_b_64x64_adaptive_c(
const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr,
const int16_t *round_ptr, const int16_t *quant_ptr,
const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr,
tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr,
const int16_t *scan, const int16_t *iscan) {
quantize_b_adaptive_helper_c(coeff_ptr, n_coeffs, zbin_ptr, round_ptr,
quant_ptr, quant_shift_ptr, qcoeff_ptr,
dqcoeff_ptr, dequant_ptr, eob_ptr, scan, iscan,
NULL, NULL, 2);
}
void aom_highbd_quantize_b_adaptive_c(
const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr,
const int16_t *round_ptr, const int16_t *quant_ptr,
const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr,
tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr,
const int16_t *scan, const int16_t *iscan) {
highbd_quantize_b_adaptive_helper_c(coeff_ptr, n_coeffs, zbin_ptr, round_ptr,
quant_ptr, quant_shift_ptr, qcoeff_ptr,
dqcoeff_ptr, dequant_ptr, eob_ptr, scan,
iscan, NULL, NULL, 0);
}
void aom_highbd_quantize_b_32x32_adaptive_c(
const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr,
const int16_t *round_ptr, const int16_t *quant_ptr,
const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr,
tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr,
const int16_t *scan, const int16_t *iscan) {
highbd_quantize_b_adaptive_helper_c(coeff_ptr, n_coeffs, zbin_ptr, round_ptr,
quant_ptr, quant_shift_ptr, qcoeff_ptr,
dqcoeff_ptr, dequant_ptr, eob_ptr, scan,
iscan, NULL, NULL, 1);
}
void aom_highbd_quantize_b_64x64_adaptive_c(
const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr,
const int16_t *round_ptr, const int16_t *quant_ptr,
const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr,
tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr,
const int16_t *scan, const int16_t *iscan) {
highbd_quantize_b_adaptive_helper_c(coeff_ptr, n_coeffs, zbin_ptr, round_ptr,
quant_ptr, quant_shift_ptr, qcoeff_ptr,
dqcoeff_ptr, dequant_ptr, eob_ptr, scan,
iscan, NULL, NULL, 2);
}
void aom_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
const int16_t *zbin_ptr, const int16_t *round_ptr,
const int16_t *quant_ptr, const int16_t *quant_shift_ptr,

60
third_party/aom/aom_dsp/quantize.h vendored
View file

@ -20,6 +20,66 @@
extern "C" {
#endif
void quantize_b_adaptive_helper_c(
const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr,
const int16_t *round_ptr, const int16_t *quant_ptr,
const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr,
tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr,
const int16_t *scan, const int16_t *iscan, const qm_val_t *qm_ptr,
const qm_val_t *iqm_ptr, const int log_scale);
void aom_quantize_b_adaptive_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
const int16_t *zbin_ptr,
const int16_t *round_ptr,
const int16_t *quant_ptr,
const int16_t *quant_shift_ptr,
tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr,
const int16_t *dequant_ptr, uint16_t *eob_ptr,
const int16_t *scan, const int16_t *iscan);
void aom_quantize_b_32x32_adaptive_c(
const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr,
const int16_t *round_ptr, const int16_t *quant_ptr,
const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr,
tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr,
const int16_t *scan, const int16_t *iscan);
void aom_quantize_b_64x64_adaptive_c(
const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr,
const int16_t *round_ptr, const int16_t *quant_ptr,
const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr,
tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr,
const int16_t *scan, const int16_t *iscan);
void highbd_quantize_b_adaptive_helper_c(
const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr,
const int16_t *round_ptr, const int16_t *quant_ptr,
const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr,
tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr,
const int16_t *scan, const int16_t *iscan, const qm_val_t *qm_ptr,
const qm_val_t *iqm_ptr, const int log_scale);
void aom_highbd_quantize_b_adaptive_c(
const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr,
const int16_t *round_ptr, const int16_t *quant_ptr,
const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr,
tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr,
const int16_t *scan, const int16_t *iscan);
void aom_highbd_quantize_b_32x32_adaptive_c(
const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr,
const int16_t *round_ptr, const int16_t *quant_ptr,
const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr,
tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr,
const int16_t *scan, const int16_t *iscan);
void aom_highbd_quantize_b_64x64_adaptive_c(
const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr,
const int16_t *round_ptr, const int16_t *quant_ptr,
const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr,
tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr,
const int16_t *scan, const int16_t *iscan);
void quantize_b_helper_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
const int16_t *zbin_ptr, const int16_t *round_ptr,
const int16_t *quant_ptr,

61
third_party/aom/aom_dsp/recenter.h vendored Normal file
View file

@ -0,0 +1,61 @@
/*
* Copyright (c) 2018, Alliance for Open Media. All rights reserved
*
* This source code is subject to the terms of the BSD 2 Clause License and
* the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
* was not distributed with this source code in the LICENSE file, you can
* obtain it at www.aomedia.org/license/software. If the Alliance for Open
* Media Patent License 1.0 was not distributed with this source code in the
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
*/
#ifndef AOM_AOM_DSP_RECENTER_H_
#define AOM_AOM_DSP_RECENTER_H_
#include "config/aom_config.h"
#include "aom/aom_integer.h"
// Inverse recenters a non-negative literal v around a reference r
static INLINE uint16_t inv_recenter_nonneg(uint16_t r, uint16_t v) {
if (v > (r << 1))
return v;
else if ((v & 1) == 0)
return (v >> 1) + r;
else
return r - ((v + 1) >> 1);
}
// Inverse recenters a non-negative literal v in [0, n-1] around a
// reference r also in [0, n-1]
static INLINE uint16_t inv_recenter_finite_nonneg(uint16_t n, uint16_t r,
uint16_t v) {
if ((r << 1) <= n) {
return inv_recenter_nonneg(r, v);
} else {
return n - 1 - inv_recenter_nonneg(n - 1 - r, v);
}
}
// Recenters a non-negative literal v around a reference r
static INLINE uint16_t recenter_nonneg(uint16_t r, uint16_t v) {
if (v > (r << 1))
return v;
else if (v >= r)
return ((v - r) << 1);
else
return ((r - v) << 1) - 1;
}
// Recenters a non-negative literal v in [0, n-1] around a
// reference r also in [0, n-1]
static INLINE uint16_t recenter_finite_nonneg(uint16_t n, uint16_t r,
uint16_t v) {
if ((r << 1) <= n) {
return recenter_nonneg(r, v);
} else {
return recenter_nonneg(n - 1 - r, n - 1 - v);
}
}
#endif // AOM_AOM_DSP_RECENTER_H_

219
third_party/aom/aom_dsp/sad.c vendored
View file

@ -25,7 +25,9 @@ static INLINE unsigned int sad(const uint8_t *a, int a_stride, const uint8_t *b,
unsigned int sad = 0;
for (y = 0; y < height; y++) {
for (x = 0; x < width; x++) sad += abs(a[x] - b[x]);
for (x = 0; x < width; x++) {
sad += abs(a[x] - b[x]);
}
a += a_stride;
b += b_stride;
@ -52,12 +54,12 @@ static INLINE unsigned int sad(const uint8_t *a, int a_stride, const uint8_t *b,
aom_comp_avg_pred(comp_pred, second_pred, m, n, ref, ref_stride); \
return sad(src, src_stride, comp_pred, m, m, n); \
} \
unsigned int aom_jnt_sad##m##x##n##_avg_c( \
unsigned int aom_dist_wtd_sad##m##x##n##_avg_c( \
const uint8_t *src, int src_stride, const uint8_t *ref, int ref_stride, \
const uint8_t *second_pred, const JNT_COMP_PARAMS *jcp_param) { \
const uint8_t *second_pred, const DIST_WTD_COMP_PARAMS *jcp_param) { \
uint8_t comp_pred[m * n]; \
aom_jnt_comp_avg_pred_c(comp_pred, second_pred, m, n, ref, ref_stride, \
jcp_param); \
aom_dist_wtd_comp_avg_pred_c(comp_pred, second_pred, m, n, ref, \
ref_stride, jcp_param); \
return sad(src, src_stride, comp_pred, m, m, n); \
}
@ -67,75 +69,75 @@ static INLINE unsigned int sad(const uint8_t *a, int a_stride, const uint8_t *b,
const uint8_t *const ref_array[], \
int ref_stride, uint32_t *sad_array) { \
int i; \
for (i = 0; i < 4; ++i) \
for (i = 0; i < 4; ++i) { \
sad_array[i] = \
aom_sad##m##x##n##_c(src, src_stride, ref_array[i], ref_stride); \
} \
}
/* clang-format off */
// 128x128
sadMxN(128, 128)
sadMxNx4D(128, 128)
sadMxN(128, 128);
sadMxNx4D(128, 128);
// 128x64
sadMxN(128, 64)
sadMxNx4D(128, 64)
sadMxN(128, 64);
sadMxNx4D(128, 64);
// 64x128
sadMxN(64, 128)
sadMxNx4D(64, 128)
sadMxN(64, 128);
sadMxNx4D(64, 128);
// 64x64
sadMxN(64, 64)
sadMxNx4D(64, 64)
sadMxN(64, 64);
sadMxNx4D(64, 64);
// 64x32
sadMxN(64, 32)
sadMxNx4D(64, 32)
sadMxN(64, 32);
sadMxNx4D(64, 32);
// 32x64
sadMxN(32, 64)
sadMxNx4D(32, 64)
sadMxN(32, 64);
sadMxNx4D(32, 64);
// 32x32
sadMxN(32, 32)
sadMxNx4D(32, 32)
sadMxN(32, 32);
sadMxNx4D(32, 32);
// 32x16
sadMxN(32, 16)
sadMxNx4D(32, 16)
sadMxN(32, 16);
sadMxNx4D(32, 16);
// 16x32
sadMxN(16, 32)
sadMxNx4D(16, 32)
sadMxN(16, 32);
sadMxNx4D(16, 32);
// 16x16
sadMxN(16, 16)
sadMxNx4D(16, 16)
sadMxN(16, 16);
sadMxNx4D(16, 16);
// 16x8
sadMxN(16, 8)
sadMxNx4D(16, 8)
sadMxN(16, 8);
sadMxNx4D(16, 8);
// 8x16
sadMxN(8, 16)
sadMxNx4D(8, 16)
sadMxN(8, 16);
sadMxNx4D(8, 16);
// 8x8
sadMxN(8, 8)
sadMxNx4D(8, 8)
sadMxN(8, 8);
sadMxNx4D(8, 8);
// 8x4
sadMxN(8, 4)
sadMxNx4D(8, 4)
sadMxN(8, 4);
sadMxNx4D(8, 4);
// 4x8
sadMxN(4, 8)
sadMxNx4D(4, 8)
sadMxN(4, 8);
sadMxNx4D(4, 8);
// 4x4
sadMxN(4, 4)
sadMxNx4D(4, 4)
sadMxN(4, 4);
sadMxNx4D(4, 4);
sadMxh(128);
sadMxh(64);
@ -144,30 +146,30 @@ sadMxh(16);
sadMxh(8);
sadMxh(4);
sadMxN(4, 16)
sadMxNx4D(4, 16)
sadMxN(16, 4)
sadMxNx4D(16, 4)
sadMxN(8, 32)
sadMxNx4D(8, 32)
sadMxN(32, 8)
sadMxNx4D(32, 8)
sadMxN(16, 64)
sadMxNx4D(16, 64)
sadMxN(64, 16)
sadMxNx4D(64, 16)
sadMxN(4, 16);
sadMxNx4D(4, 16);
sadMxN(16, 4);
sadMxNx4D(16, 4);
sadMxN(8, 32);
sadMxNx4D(8, 32);
sadMxN(32, 8);
sadMxNx4D(32, 8);
sadMxN(16, 64);
sadMxNx4D(16, 64);
sadMxN(64, 16);
sadMxNx4D(64, 16);
/* clang-format on */
static INLINE
unsigned int highbd_sad(const uint8_t *a8, int a_stride, const uint8_t *b8,
int b_stride, int width, int height) {
static INLINE unsigned int highbd_sad(const uint8_t *a8, int a_stride,
const uint8_t *b8, int b_stride,
int width, int height) {
int y, x;
unsigned int sad = 0;
const uint16_t *a = CONVERT_TO_SHORTPTR(a8);
const uint16_t *b = CONVERT_TO_SHORTPTR(b8);
for (y = 0; y < height; y++) {
for (x = 0; x < width; x++) sad += abs(a[x] - b[x]);
for (x = 0; x < width; x++) {
sad += abs(a[x] - b[x]);
}
a += a_stride;
b += b_stride;
@ -182,7 +184,9 @@ static INLINE unsigned int highbd_sadb(const uint8_t *a8, int a_stride,
unsigned int sad = 0;
const uint16_t *a = CONVERT_TO_SHORTPTR(a8);
for (y = 0; y < height; y++) {
for (x = 0; x < width; x++) sad += abs(a[x] - b[x]);
for (x = 0; x < width; x++) {
sad += abs(a[x] - b[x]);
}
a += a_stride;
b += b_stride;
@ -204,12 +208,13 @@ static INLINE unsigned int highbd_sadb(const uint8_t *a8, int a_stride,
ref, ref_stride); \
return highbd_sadb(src, src_stride, comp_pred, m, m, n); \
} \
unsigned int aom_highbd_jnt_sad##m##x##n##_avg_c( \
unsigned int aom_highbd_dist_wtd_sad##m##x##n##_avg_c( \
const uint8_t *src, int src_stride, const uint8_t *ref, int ref_stride, \
const uint8_t *second_pred, const JNT_COMP_PARAMS *jcp_param) { \
const uint8_t *second_pred, const DIST_WTD_COMP_PARAMS *jcp_param) { \
uint16_t comp_pred[m * n]; \
aom_highbd_jnt_comp_avg_pred(CONVERT_TO_BYTEPTR(comp_pred), second_pred, \
m, n, ref, ref_stride, jcp_param); \
aom_highbd_dist_wtd_comp_avg_pred(CONVERT_TO_BYTEPTR(comp_pred), \
second_pred, m, n, ref, ref_stride, \
jcp_param); \
return highbd_sadb(src, src_stride, comp_pred, m, m, n); \
}
@ -224,81 +229,79 @@ static INLINE unsigned int highbd_sadb(const uint8_t *a8, int a_stride,
} \
}
/* clang-format off */
// 128x128
highbd_sadMxN(128, 128)
highbd_sadMxNx4D(128, 128)
highbd_sadMxN(128, 128);
highbd_sadMxNx4D(128, 128);
// 128x64
highbd_sadMxN(128, 64)
highbd_sadMxNx4D(128, 64)
highbd_sadMxN(128, 64);
highbd_sadMxNx4D(128, 64);
// 64x128
highbd_sadMxN(64, 128)
highbd_sadMxNx4D(64, 128)
highbd_sadMxN(64, 128);
highbd_sadMxNx4D(64, 128);
// 64x64
highbd_sadMxN(64, 64)
highbd_sadMxNx4D(64, 64)
highbd_sadMxN(64, 64);
highbd_sadMxNx4D(64, 64);
// 64x32
highbd_sadMxN(64, 32)
highbd_sadMxNx4D(64, 32)
highbd_sadMxN(64, 32);
highbd_sadMxNx4D(64, 32);
// 32x64
highbd_sadMxN(32, 64)
highbd_sadMxNx4D(32, 64)
highbd_sadMxN(32, 64);
highbd_sadMxNx4D(32, 64);
// 32x32
highbd_sadMxN(32, 32)
highbd_sadMxNx4D(32, 32)
highbd_sadMxN(32, 32);
highbd_sadMxNx4D(32, 32);
// 32x16
highbd_sadMxN(32, 16)
highbd_sadMxNx4D(32, 16)
highbd_sadMxN(32, 16);
highbd_sadMxNx4D(32, 16);
// 16x32
highbd_sadMxN(16, 32)
highbd_sadMxNx4D(16, 32)
highbd_sadMxN(16, 32);
highbd_sadMxNx4D(16, 32);
// 16x16
highbd_sadMxN(16, 16)
highbd_sadMxNx4D(16, 16)
highbd_sadMxN(16, 16);
highbd_sadMxNx4D(16, 16);
// 16x8
highbd_sadMxN(16, 8)
highbd_sadMxNx4D(16, 8)
highbd_sadMxN(16, 8);
highbd_sadMxNx4D(16, 8);
// 8x16
highbd_sadMxN(8, 16)
highbd_sadMxNx4D(8, 16)
highbd_sadMxN(8, 16);
highbd_sadMxNx4D(8, 16);
// 8x8
highbd_sadMxN(8, 8)
highbd_sadMxNx4D(8, 8)
highbd_sadMxN(8, 8);
highbd_sadMxNx4D(8, 8);
// 8x4
highbd_sadMxN(8, 4)
highbd_sadMxNx4D(8, 4)
highbd_sadMxN(8, 4);
highbd_sadMxNx4D(8, 4);
// 4x8
highbd_sadMxN(4, 8)
highbd_sadMxNx4D(4, 8)
highbd_sadMxN(4, 8);
highbd_sadMxNx4D(4, 8);
// 4x4
highbd_sadMxN(4, 4)
highbd_sadMxNx4D(4, 4)
highbd_sadMxN(4, 4);
highbd_sadMxNx4D(4, 4);
highbd_sadMxN(4, 16)
highbd_sadMxNx4D(4, 16)
highbd_sadMxN(16, 4)
highbd_sadMxNx4D(16, 4)
highbd_sadMxN(8, 32)
highbd_sadMxNx4D(8, 32)
highbd_sadMxN(32, 8)
highbd_sadMxNx4D(32, 8)
highbd_sadMxN(16, 64)
highbd_sadMxNx4D(16, 64)
highbd_sadMxN(64, 16)
highbd_sadMxNx4D(64, 16)
/* clang-format on */
highbd_sadMxN(4, 16);
highbd_sadMxNx4D(4, 16);
highbd_sadMxN(16, 4);
highbd_sadMxNx4D(16, 4);
highbd_sadMxN(8, 32);
highbd_sadMxNx4D(8, 32);
highbd_sadMxN(32, 8);
highbd_sadMxNx4D(32, 8);
highbd_sadMxN(16, 64);
highbd_sadMxNx4D(16, 64);
highbd_sadMxN(64, 16);
highbd_sadMxNx4D(64, 16);

2
third_party/aom/aom_dsp/txfm_common.h vendored
View file

@ -39,7 +39,7 @@ typedef struct txfm_param {
// Constants:
// for (int i = 1; i< 32; ++i)
// printf("static const int cospi_%d_64 = %.0f;\n", i,
// round(16384 * cos(i*M_PI/64)));
// round(16384 * cos(i*PI/64)));
// Note: sin(k*Pi/64) = cos((32-k)*Pi/64)
static const tran_high_t cospi_1_64 = 16364;
static const tran_high_t cospi_2_64 = 16305;

370
third_party/aom/aom_dsp/variance.c vendored
View file

@ -164,40 +164,40 @@ void aom_var_filter_block2d_bil_second_pass_c(const uint16_t *a, uint8_t *b,
return aom_variance##W##x##H##_c(temp2, W, b, b_stride, sse); \
}
#define SUBPIX_AVG_VAR(W, H) \
uint32_t aom_sub_pixel_avg_variance##W##x##H##_c( \
const uint8_t *a, int a_stride, int xoffset, int yoffset, \
const uint8_t *b, int b_stride, uint32_t *sse, \
const uint8_t *second_pred) { \
uint16_t fdata3[(H + 1) * W]; \
uint8_t temp2[H * W]; \
DECLARE_ALIGNED(16, uint8_t, temp3[H * W]); \
\
aom_var_filter_block2d_bil_first_pass_c(a, fdata3, a_stride, 1, H + 1, W, \
bilinear_filters_2t[xoffset]); \
aom_var_filter_block2d_bil_second_pass_c(fdata3, temp2, W, W, H, W, \
bilinear_filters_2t[yoffset]); \
\
aom_comp_avg_pred(temp3, second_pred, W, H, temp2, W); \
\
return aom_variance##W##x##H##_c(temp3, W, b, b_stride, sse); \
} \
uint32_t aom_jnt_sub_pixel_avg_variance##W##x##H##_c( \
const uint8_t *a, int a_stride, int xoffset, int yoffset, \
const uint8_t *b, int b_stride, uint32_t *sse, \
const uint8_t *second_pred, const JNT_COMP_PARAMS *jcp_param) { \
uint16_t fdata3[(H + 1) * W]; \
uint8_t temp2[H * W]; \
DECLARE_ALIGNED(16, uint8_t, temp3[H * W]); \
\
aom_var_filter_block2d_bil_first_pass_c(a, fdata3, a_stride, 1, H + 1, W, \
bilinear_filters_2t[xoffset]); \
aom_var_filter_block2d_bil_second_pass_c(fdata3, temp2, W, W, H, W, \
bilinear_filters_2t[yoffset]); \
\
aom_jnt_comp_avg_pred(temp3, second_pred, W, H, temp2, W, jcp_param); \
\
return aom_variance##W##x##H(temp3, W, b, b_stride, sse); \
#define SUBPIX_AVG_VAR(W, H) \
uint32_t aom_sub_pixel_avg_variance##W##x##H##_c( \
const uint8_t *a, int a_stride, int xoffset, int yoffset, \
const uint8_t *b, int b_stride, uint32_t *sse, \
const uint8_t *second_pred) { \
uint16_t fdata3[(H + 1) * W]; \
uint8_t temp2[H * W]; \
DECLARE_ALIGNED(16, uint8_t, temp3[H * W]); \
\
aom_var_filter_block2d_bil_first_pass_c(a, fdata3, a_stride, 1, H + 1, W, \
bilinear_filters_2t[xoffset]); \
aom_var_filter_block2d_bil_second_pass_c(fdata3, temp2, W, W, H, W, \
bilinear_filters_2t[yoffset]); \
\
aom_comp_avg_pred(temp3, second_pred, W, H, temp2, W); \
\
return aom_variance##W##x##H##_c(temp3, W, b, b_stride, sse); \
} \
uint32_t aom_dist_wtd_sub_pixel_avg_variance##W##x##H##_c( \
const uint8_t *a, int a_stride, int xoffset, int yoffset, \
const uint8_t *b, int b_stride, uint32_t *sse, \
const uint8_t *second_pred, const DIST_WTD_COMP_PARAMS *jcp_param) { \
uint16_t fdata3[(H + 1) * W]; \
uint8_t temp2[H * W]; \
DECLARE_ALIGNED(16, uint8_t, temp3[H * W]); \
\
aom_var_filter_block2d_bil_first_pass_c(a, fdata3, a_stride, 1, H + 1, W, \
bilinear_filters_2t[xoffset]); \
aom_var_filter_block2d_bil_second_pass_c(fdata3, temp2, W, W, H, W, \
bilinear_filters_2t[yoffset]); \
\
aom_dist_wtd_comp_avg_pred(temp3, second_pred, W, H, temp2, W, jcp_param); \
\
return aom_variance##W##x##H(temp3, W, b, b_stride, sse); \
}
/* Identical to the variance call except it takes an additional parameter, sum,
@ -291,7 +291,7 @@ void aom_upsampled_pred_c(MACROBLOCKD *xd, const AV1_COMMON *const cm,
const int ref_num = 0;
const int is_intrabc = is_intrabc_block(mi);
const struct scale_factors *const sf =
is_intrabc ? &cm->sf_identity : &xd->block_refs[ref_num]->sf;
is_intrabc ? &cm->sf_identity : xd->block_ref_scale_factors[ref_num];
const int is_scaled = av1_is_scaled(sf);
if (is_scaled) {
@ -368,10 +368,7 @@ void aom_upsampled_pred_c(MACROBLOCKD *xd, const AV1_COMMON *const cm,
}
}
const InterpFilterParams *filter =
(subpel_search == 1)
? av1_get_4tap_interp_filter_params(EIGHTTAP_REGULAR)
: av1_get_interp_filter_params_with_block_size(EIGHTTAP_REGULAR, 8);
const InterpFilterParams *filter = av1_get_filter(subpel_search);
if (!subpel_x_q3 && !subpel_y_q3) {
for (int i = 0; i < height; i++) {
@ -427,9 +424,10 @@ void aom_comp_avg_upsampled_pred_c(MACROBLOCKD *xd, const AV1_COMMON *const cm,
}
}
void aom_jnt_comp_avg_pred_c(uint8_t *comp_pred, const uint8_t *pred, int width,
int height, const uint8_t *ref, int ref_stride,
const JNT_COMP_PARAMS *jcp_param) {
void aom_dist_wtd_comp_avg_pred_c(uint8_t *comp_pred, const uint8_t *pred,
int width, int height, const uint8_t *ref,
int ref_stride,
const DIST_WTD_COMP_PARAMS *jcp_param) {
int i, j;
const int fwd_offset = jcp_param->fwd_offset;
const int bck_offset = jcp_param->bck_offset;
@ -446,11 +444,11 @@ void aom_jnt_comp_avg_pred_c(uint8_t *comp_pred, const uint8_t *pred, int width,
}
}
void aom_jnt_comp_avg_upsampled_pred_c(
void aom_dist_wtd_comp_avg_upsampled_pred_c(
MACROBLOCKD *xd, const AV1_COMMON *const cm, int mi_row, int mi_col,
const MV *const mv, uint8_t *comp_pred, const uint8_t *pred, int width,
int height, int subpel_x_q3, int subpel_y_q3, const uint8_t *ref,
int ref_stride, const JNT_COMP_PARAMS *jcp_param, int subpel_search) {
int ref_stride, const DIST_WTD_COMP_PARAMS *jcp_param, int subpel_search) {
int i, j;
const int fwd_offset = jcp_param->fwd_offset;
const int bck_offset = jcp_param->bck_offset;
@ -691,125 +689,128 @@ void aom_highbd_var_filter_block2d_bil_second_pass(
dst, dst_stride, sse); \
}
#define HIGHBD_SUBPIX_AVG_VAR(W, H) \
uint32_t aom_highbd_8_sub_pixel_avg_variance##W##x##H##_c( \
const uint8_t *src, int src_stride, int xoffset, int yoffset, \
const uint8_t *dst, int dst_stride, uint32_t *sse, \
const uint8_t *second_pred) { \
uint16_t fdata3[(H + 1) * W]; \
uint16_t temp2[H * W]; \
DECLARE_ALIGNED(16, uint16_t, temp3[H * W]); \
\
aom_highbd_var_filter_block2d_bil_first_pass( \
src, fdata3, src_stride, 1, H + 1, W, bilinear_filters_2t[xoffset]); \
aom_highbd_var_filter_block2d_bil_second_pass( \
fdata3, temp2, W, W, H, W, bilinear_filters_2t[yoffset]); \
\
aom_highbd_comp_avg_pred_c(CONVERT_TO_BYTEPTR(temp3), second_pred, W, H, \
CONVERT_TO_BYTEPTR(temp2), W); \
\
return aom_highbd_8_variance##W##x##H##_c(CONVERT_TO_BYTEPTR(temp3), W, \
dst, dst_stride, sse); \
} \
\
uint32_t aom_highbd_10_sub_pixel_avg_variance##W##x##H##_c( \
const uint8_t *src, int src_stride, int xoffset, int yoffset, \
const uint8_t *dst, int dst_stride, uint32_t *sse, \
const uint8_t *second_pred) { \
uint16_t fdata3[(H + 1) * W]; \
uint16_t temp2[H * W]; \
DECLARE_ALIGNED(16, uint16_t, temp3[H * W]); \
\
aom_highbd_var_filter_block2d_bil_first_pass( \
src, fdata3, src_stride, 1, H + 1, W, bilinear_filters_2t[xoffset]); \
aom_highbd_var_filter_block2d_bil_second_pass( \
fdata3, temp2, W, W, H, W, bilinear_filters_2t[yoffset]); \
\
aom_highbd_comp_avg_pred_c(CONVERT_TO_BYTEPTR(temp3), second_pred, W, H, \
CONVERT_TO_BYTEPTR(temp2), W); \
\
return aom_highbd_10_variance##W##x##H##_c(CONVERT_TO_BYTEPTR(temp3), W, \
dst, dst_stride, sse); \
} \
\
uint32_t aom_highbd_12_sub_pixel_avg_variance##W##x##H##_c( \
const uint8_t *src, int src_stride, int xoffset, int yoffset, \
const uint8_t *dst, int dst_stride, uint32_t *sse, \
const uint8_t *second_pred) { \
uint16_t fdata3[(H + 1) * W]; \
uint16_t temp2[H * W]; \
DECLARE_ALIGNED(16, uint16_t, temp3[H * W]); \
\
aom_highbd_var_filter_block2d_bil_first_pass( \
src, fdata3, src_stride, 1, H + 1, W, bilinear_filters_2t[xoffset]); \
aom_highbd_var_filter_block2d_bil_second_pass( \
fdata3, temp2, W, W, H, W, bilinear_filters_2t[yoffset]); \
\
aom_highbd_comp_avg_pred_c(CONVERT_TO_BYTEPTR(temp3), second_pred, W, H, \
CONVERT_TO_BYTEPTR(temp2), W); \
\
return aom_highbd_12_variance##W##x##H##_c(CONVERT_TO_BYTEPTR(temp3), W, \
dst, dst_stride, sse); \
} \
\
uint32_t aom_highbd_8_jnt_sub_pixel_avg_variance##W##x##H##_c( \
const uint8_t *src, int src_stride, int xoffset, int yoffset, \
const uint8_t *dst, int dst_stride, uint32_t *sse, \
const uint8_t *second_pred, const JNT_COMP_PARAMS *jcp_param) { \
uint16_t fdata3[(H + 1) * W]; \
uint16_t temp2[H * W]; \
DECLARE_ALIGNED(16, uint16_t, temp3[H * W]); \
\
aom_highbd_var_filter_block2d_bil_first_pass( \
src, fdata3, src_stride, 1, H + 1, W, bilinear_filters_2t[xoffset]); \
aom_highbd_var_filter_block2d_bil_second_pass( \
fdata3, temp2, W, W, H, W, bilinear_filters_2t[yoffset]); \
\
aom_highbd_jnt_comp_avg_pred(CONVERT_TO_BYTEPTR(temp3), second_pred, W, H, \
CONVERT_TO_BYTEPTR(temp2), W, jcp_param); \
\
return aom_highbd_8_variance##W##x##H(CONVERT_TO_BYTEPTR(temp3), W, dst, \
dst_stride, sse); \
} \
\
uint32_t aom_highbd_10_jnt_sub_pixel_avg_variance##W##x##H##_c( \
const uint8_t *src, int src_stride, int xoffset, int yoffset, \
const uint8_t *dst, int dst_stride, uint32_t *sse, \
const uint8_t *second_pred, const JNT_COMP_PARAMS *jcp_param) { \
uint16_t fdata3[(H + 1) * W]; \
uint16_t temp2[H * W]; \
DECLARE_ALIGNED(16, uint16_t, temp3[H * W]); \
\
aom_highbd_var_filter_block2d_bil_first_pass( \
src, fdata3, src_stride, 1, H + 1, W, bilinear_filters_2t[xoffset]); \
aom_highbd_var_filter_block2d_bil_second_pass( \
fdata3, temp2, W, W, H, W, bilinear_filters_2t[yoffset]); \
\
aom_highbd_jnt_comp_avg_pred(CONVERT_TO_BYTEPTR(temp3), second_pred, W, H, \
CONVERT_TO_BYTEPTR(temp2), W, jcp_param); \
\
return aom_highbd_10_variance##W##x##H(CONVERT_TO_BYTEPTR(temp3), W, dst, \
dst_stride, sse); \
} \
\
uint32_t aom_highbd_12_jnt_sub_pixel_avg_variance##W##x##H##_c( \
const uint8_t *src, int src_stride, int xoffset, int yoffset, \
const uint8_t *dst, int dst_stride, uint32_t *sse, \
const uint8_t *second_pred, const JNT_COMP_PARAMS *jcp_param) { \
uint16_t fdata3[(H + 1) * W]; \
uint16_t temp2[H * W]; \
DECLARE_ALIGNED(16, uint16_t, temp3[H * W]); \
\
aom_highbd_var_filter_block2d_bil_first_pass( \
src, fdata3, src_stride, 1, H + 1, W, bilinear_filters_2t[xoffset]); \
aom_highbd_var_filter_block2d_bil_second_pass( \
fdata3, temp2, W, W, H, W, bilinear_filters_2t[yoffset]); \
\
aom_highbd_jnt_comp_avg_pred(CONVERT_TO_BYTEPTR(temp3), second_pred, W, H, \
CONVERT_TO_BYTEPTR(temp2), W, jcp_param); \
\
return aom_highbd_12_variance##W##x##H(CONVERT_TO_BYTEPTR(temp3), W, dst, \
dst_stride, sse); \
#define HIGHBD_SUBPIX_AVG_VAR(W, H) \
uint32_t aom_highbd_8_sub_pixel_avg_variance##W##x##H##_c( \
const uint8_t *src, int src_stride, int xoffset, int yoffset, \
const uint8_t *dst, int dst_stride, uint32_t *sse, \
const uint8_t *second_pred) { \
uint16_t fdata3[(H + 1) * W]; \
uint16_t temp2[H * W]; \
DECLARE_ALIGNED(16, uint16_t, temp3[H * W]); \
\
aom_highbd_var_filter_block2d_bil_first_pass( \
src, fdata3, src_stride, 1, H + 1, W, bilinear_filters_2t[xoffset]); \
aom_highbd_var_filter_block2d_bil_second_pass( \
fdata3, temp2, W, W, H, W, bilinear_filters_2t[yoffset]); \
\
aom_highbd_comp_avg_pred_c(CONVERT_TO_BYTEPTR(temp3), second_pred, W, H, \
CONVERT_TO_BYTEPTR(temp2), W); \
\
return aom_highbd_8_variance##W##x##H##_c(CONVERT_TO_BYTEPTR(temp3), W, \
dst, dst_stride, sse); \
} \
\
uint32_t aom_highbd_10_sub_pixel_avg_variance##W##x##H##_c( \
const uint8_t *src, int src_stride, int xoffset, int yoffset, \
const uint8_t *dst, int dst_stride, uint32_t *sse, \
const uint8_t *second_pred) { \
uint16_t fdata3[(H + 1) * W]; \
uint16_t temp2[H * W]; \
DECLARE_ALIGNED(16, uint16_t, temp3[H * W]); \
\
aom_highbd_var_filter_block2d_bil_first_pass( \
src, fdata3, src_stride, 1, H + 1, W, bilinear_filters_2t[xoffset]); \
aom_highbd_var_filter_block2d_bil_second_pass( \
fdata3, temp2, W, W, H, W, bilinear_filters_2t[yoffset]); \
\
aom_highbd_comp_avg_pred_c(CONVERT_TO_BYTEPTR(temp3), second_pred, W, H, \
CONVERT_TO_BYTEPTR(temp2), W); \
\
return aom_highbd_10_variance##W##x##H##_c(CONVERT_TO_BYTEPTR(temp3), W, \
dst, dst_stride, sse); \
} \
\
uint32_t aom_highbd_12_sub_pixel_avg_variance##W##x##H##_c( \
const uint8_t *src, int src_stride, int xoffset, int yoffset, \
const uint8_t *dst, int dst_stride, uint32_t *sse, \
const uint8_t *second_pred) { \
uint16_t fdata3[(H + 1) * W]; \
uint16_t temp2[H * W]; \
DECLARE_ALIGNED(16, uint16_t, temp3[H * W]); \
\
aom_highbd_var_filter_block2d_bil_first_pass( \
src, fdata3, src_stride, 1, H + 1, W, bilinear_filters_2t[xoffset]); \
aom_highbd_var_filter_block2d_bil_second_pass( \
fdata3, temp2, W, W, H, W, bilinear_filters_2t[yoffset]); \
\
aom_highbd_comp_avg_pred_c(CONVERT_TO_BYTEPTR(temp3), second_pred, W, H, \
CONVERT_TO_BYTEPTR(temp2), W); \
\
return aom_highbd_12_variance##W##x##H##_c(CONVERT_TO_BYTEPTR(temp3), W, \
dst, dst_stride, sse); \
} \
\
uint32_t aom_highbd_8_dist_wtd_sub_pixel_avg_variance##W##x##H##_c( \
const uint8_t *src, int src_stride, int xoffset, int yoffset, \
const uint8_t *dst, int dst_stride, uint32_t *sse, \
const uint8_t *second_pred, const DIST_WTD_COMP_PARAMS *jcp_param) { \
uint16_t fdata3[(H + 1) * W]; \
uint16_t temp2[H * W]; \
DECLARE_ALIGNED(16, uint16_t, temp3[H * W]); \
\
aom_highbd_var_filter_block2d_bil_first_pass( \
src, fdata3, src_stride, 1, H + 1, W, bilinear_filters_2t[xoffset]); \
aom_highbd_var_filter_block2d_bil_second_pass( \
fdata3, temp2, W, W, H, W, bilinear_filters_2t[yoffset]); \
\
aom_highbd_dist_wtd_comp_avg_pred(CONVERT_TO_BYTEPTR(temp3), second_pred, \
W, H, CONVERT_TO_BYTEPTR(temp2), W, \
jcp_param); \
\
return aom_highbd_8_variance##W##x##H(CONVERT_TO_BYTEPTR(temp3), W, dst, \
dst_stride, sse); \
} \
\
uint32_t aom_highbd_10_dist_wtd_sub_pixel_avg_variance##W##x##H##_c( \
const uint8_t *src, int src_stride, int xoffset, int yoffset, \
const uint8_t *dst, int dst_stride, uint32_t *sse, \
const uint8_t *second_pred, const DIST_WTD_COMP_PARAMS *jcp_param) { \
uint16_t fdata3[(H + 1) * W]; \
uint16_t temp2[H * W]; \
DECLARE_ALIGNED(16, uint16_t, temp3[H * W]); \
\
aom_highbd_var_filter_block2d_bil_first_pass( \
src, fdata3, src_stride, 1, H + 1, W, bilinear_filters_2t[xoffset]); \
aom_highbd_var_filter_block2d_bil_second_pass( \
fdata3, temp2, W, W, H, W, bilinear_filters_2t[yoffset]); \
\
aom_highbd_dist_wtd_comp_avg_pred(CONVERT_TO_BYTEPTR(temp3), second_pred, \
W, H, CONVERT_TO_BYTEPTR(temp2), W, \
jcp_param); \
\
return aom_highbd_10_variance##W##x##H(CONVERT_TO_BYTEPTR(temp3), W, dst, \
dst_stride, sse); \
} \
\
uint32_t aom_highbd_12_dist_wtd_sub_pixel_avg_variance##W##x##H##_c( \
const uint8_t *src, int src_stride, int xoffset, int yoffset, \
const uint8_t *dst, int dst_stride, uint32_t *sse, \
const uint8_t *second_pred, const DIST_WTD_COMP_PARAMS *jcp_param) { \
uint16_t fdata3[(H + 1) * W]; \
uint16_t temp2[H * W]; \
DECLARE_ALIGNED(16, uint16_t, temp3[H * W]); \
\
aom_highbd_var_filter_block2d_bil_first_pass( \
src, fdata3, src_stride, 1, H + 1, W, bilinear_filters_2t[xoffset]); \
aom_highbd_var_filter_block2d_bil_second_pass( \
fdata3, temp2, W, W, H, W, bilinear_filters_2t[yoffset]); \
\
aom_highbd_dist_wtd_comp_avg_pred(CONVERT_TO_BYTEPTR(temp3), second_pred, \
W, H, CONVERT_TO_BYTEPTR(temp2), W, \
jcp_param); \
\
return aom_highbd_12_variance##W##x##H(CONVERT_TO_BYTEPTR(temp3), W, dst, \
dst_stride, sse); \
}
/* All three forms of the variance are available in the same sizes. */
@ -883,7 +884,7 @@ void aom_highbd_upsampled_pred_c(MACROBLOCKD *xd,
const int ref_num = 0;
const int is_intrabc = is_intrabc_block(mi);
const struct scale_factors *const sf =
is_intrabc ? &cm->sf_identity : &xd->block_refs[ref_num]->sf;
is_intrabc ? &cm->sf_identity : xd->block_ref_scale_factors[ref_num];
const int is_scaled = av1_is_scaled(sf);
if (is_scaled) {
@ -959,10 +960,7 @@ void aom_highbd_upsampled_pred_c(MACROBLOCKD *xd,
}
}
const InterpFilterParams *filter =
(subpel_search == 1)
? av1_get_4tap_interp_filter_params(EIGHTTAP_REGULAR)
: av1_get_interp_filter_params_with_block_size(EIGHTTAP_REGULAR, 8);
const InterpFilterParams *filter = av1_get_filter(subpel_search);
if (!subpel_x_q3 && !subpel_y_q3) {
const uint16_t *ref = CONVERT_TO_SHORTPTR(ref8);
@ -975,13 +973,13 @@ void aom_highbd_upsampled_pred_c(MACROBLOCKD *xd,
} else if (!subpel_y_q3) {
const int16_t *const kernel =
av1_get_interp_filter_subpel_kernel(filter, subpel_x_q3 << 1);
aom_highbd_convolve8_horiz(ref8, ref_stride, comp_pred8, width, kernel, 16,
NULL, -1, width, height, bd);
aom_highbd_convolve8_horiz_c(ref8, ref_stride, comp_pred8, width, kernel,
16, NULL, -1, width, height, bd);
} else if (!subpel_x_q3) {
const int16_t *const kernel =
av1_get_interp_filter_subpel_kernel(filter, subpel_y_q3 << 1);
aom_highbd_convolve8_vert(ref8, ref_stride, comp_pred8, width, NULL, -1,
kernel, 16, width, height, bd);
aom_highbd_convolve8_vert_c(ref8, ref_stride, comp_pred8, width, NULL, -1,
kernel, 16, width, height, bd);
} else {
DECLARE_ALIGNED(16, uint16_t,
temp[((MAX_SB_SIZE + 16) + 16) * MAX_SB_SIZE]);
@ -992,11 +990,11 @@ void aom_highbd_upsampled_pred_c(MACROBLOCKD *xd,
const int intermediate_height =
(((height - 1) * 8 + subpel_y_q3) >> 3) + filter->taps;
assert(intermediate_height <= (MAX_SB_SIZE * 2 + 16) + 16);
aom_highbd_convolve8_horiz(ref8 - ref_stride * ((filter->taps >> 1) - 1),
ref_stride, CONVERT_TO_BYTEPTR(temp),
MAX_SB_SIZE, kernel_x, 16, NULL, -1, width,
intermediate_height, bd);
aom_highbd_convolve8_vert(
aom_highbd_convolve8_horiz_c(ref8 - ref_stride * ((filter->taps >> 1) - 1),
ref_stride, CONVERT_TO_BYTEPTR(temp),
MAX_SB_SIZE, kernel_x, 16, NULL, -1, width,
intermediate_height, bd);
aom_highbd_convolve8_vert_c(
CONVERT_TO_BYTEPTR(temp + MAX_SB_SIZE * ((filter->taps >> 1) - 1)),
MAX_SB_SIZE, comp_pred8, width, NULL, -1, kernel_y, 16, width, height,
bd);
@ -1024,10 +1022,10 @@ void aom_highbd_comp_avg_upsampled_pred_c(
}
}
void aom_highbd_jnt_comp_avg_pred_c(uint8_t *comp_pred8, const uint8_t *pred8,
int width, int height, const uint8_t *ref8,
int ref_stride,
const JNT_COMP_PARAMS *jcp_param) {
void aom_highbd_dist_wtd_comp_avg_pred_c(
uint8_t *comp_pred8, const uint8_t *pred8, int width, int height,
const uint8_t *ref8, int ref_stride,
const DIST_WTD_COMP_PARAMS *jcp_param) {
int i, j;
const int fwd_offset = jcp_param->fwd_offset;
const int bck_offset = jcp_param->bck_offset;
@ -1047,20 +1045,20 @@ void aom_highbd_jnt_comp_avg_pred_c(uint8_t *comp_pred8, const uint8_t *pred8,
}
}
void aom_highbd_jnt_comp_avg_upsampled_pred_c(
void aom_highbd_dist_wtd_comp_avg_upsampled_pred_c(
MACROBLOCKD *xd, const struct AV1Common *const cm, int mi_row, int mi_col,
const MV *const mv, uint8_t *comp_pred8, const uint8_t *pred8, int width,
int height, int subpel_x_q3, int subpel_y_q3, const uint8_t *ref8,
int ref_stride, int bd, const JNT_COMP_PARAMS *jcp_param,
int ref_stride, int bd, const DIST_WTD_COMP_PARAMS *jcp_param,
int subpel_search) {
int i, j;
const int fwd_offset = jcp_param->fwd_offset;
const int bck_offset = jcp_param->bck_offset;
const uint16_t *pred = CONVERT_TO_SHORTPTR(pred8);
uint16_t *comp_pred = CONVERT_TO_SHORTPTR(comp_pred8);
aom_highbd_upsampled_pred(xd, cm, mi_row, mi_col, mv, comp_pred8, width,
height, subpel_x_q3, subpel_y_q3, ref8, ref_stride,
bd, subpel_search);
aom_highbd_upsampled_pred_c(xd, cm, mi_row, mi_col, mv, comp_pred8, width,
height, subpel_x_q3, subpel_y_q3, ref8,
ref_stride, bd, subpel_search);
for (i = 0; i < height; i++) {
for (j = 0; j < width; j++) {

15
third_party/aom/aom_dsp/variance.h vendored
View file

@ -50,15 +50,14 @@ typedef unsigned int (*aom_subp_avg_variance_fn_t)(
const uint8_t *a, int a_stride, int xoffset, int yoffset, const uint8_t *b,
int b_stride, unsigned int *sse, const uint8_t *second_pred);
typedef unsigned int (*aom_jnt_sad_avg_fn_t)(const uint8_t *a, int a_stride,
const uint8_t *b, int b_stride,
const uint8_t *second_pred,
const JNT_COMP_PARAMS *jcp_param);
typedef unsigned int (*aom_dist_wtd_sad_avg_fn_t)(
const uint8_t *a, int a_stride, const uint8_t *b, int b_stride,
const uint8_t *second_pred, const DIST_WTD_COMP_PARAMS *jcp_param);
typedef unsigned int (*aom_jnt_subp_avg_variance_fn_t)(
typedef unsigned int (*aom_dist_wtd_subp_avg_variance_fn_t)(
const uint8_t *a, int a_stride, int xoffset, int yoffset, const uint8_t *b,
int b_stride, unsigned int *sse, const uint8_t *second_pred,
const JNT_COMP_PARAMS *jcp_param);
const DIST_WTD_COMP_PARAMS *jcp_param);
typedef unsigned int (*aom_masked_sad_fn_t)(const uint8_t *src, int src_stride,
const uint8_t *ref, int ref_stride,
@ -101,8 +100,8 @@ typedef struct aom_variance_vtable {
aom_obmc_sad_fn_t osdf;
aom_obmc_variance_fn_t ovf;
aom_obmc_subpixvariance_fn_t osvf;
aom_jnt_sad_avg_fn_t jsdaf;
aom_jnt_subp_avg_variance_fn_t jsvaf;
aom_dist_wtd_sad_avg_fn_t jsdaf;
aom_dist_wtd_subp_avg_variance_fn_t jsvaf;
} aom_variance_fn_ptr_t;
void aom_highbd_var_filter_block2d_bil_first_pass(

21
third_party/aom/aom_dsp/x86/aom_asm_stubs.c vendored
View file

@ -21,13 +21,13 @@ filter8_1dfunction aom_filter_block1d8_v8_sse2;
filter8_1dfunction aom_filter_block1d8_h8_sse2;
filter8_1dfunction aom_filter_block1d4_v8_sse2;
filter8_1dfunction aom_filter_block1d4_h8_sse2;
filter8_1dfunction aom_filter_block1d16_v4_sse2;
filter8_1dfunction aom_filter_block1d16_h4_sse2;
#define aom_filter_block1d16_h4_sse2 aom_filter_block1d16_h8_sse2
#define aom_filter_block1d16_v4_sse2 aom_filter_block1d16_v8_sse2
#define aom_filter_block1d8_h4_sse2 aom_filter_block1d8_h8_sse2
#define aom_filter_block1d8_v4_sse2 aom_filter_block1d8_v8_sse2
#define aom_filter_block1d4_h4_sse2 aom_filter_block1d4_h8_sse2
#define aom_filter_block1d4_v4_sse2 aom_filter_block1d4_v8_sse2
filter8_1dfunction aom_filter_block1d8_h4_sse2;
filter8_1dfunction aom_filter_block1d8_v4_sse2;
filter8_1dfunction aom_filter_block1d4_h4_sse2;
filter8_1dfunction aom_filter_block1d4_v4_sse2;
filter8_1dfunction aom_filter_block1d16_v2_sse2;
filter8_1dfunction aom_filter_block1d16_h2_sse2;
@ -49,7 +49,6 @@ filter8_1dfunction aom_filter_block1d4_h2_sse2;
FUN_CONV_1D(horiz, x_step_q4, filter_x, h, src, , sse2);
FUN_CONV_1D(vert, y_step_q4, filter_y, v, src - src_stride * 3, , sse2);
#if ARCH_X86_64
highbd_filter8_1dfunction aom_highbd_filter_block1d16_v8_sse2;
highbd_filter8_1dfunction aom_highbd_filter_block1d16_h8_sse2;
highbd_filter8_1dfunction aom_highbd_filter_block1d8_v8_sse2;
@ -57,6 +56,13 @@ highbd_filter8_1dfunction aom_highbd_filter_block1d8_h8_sse2;
highbd_filter8_1dfunction aom_highbd_filter_block1d4_v8_sse2;
highbd_filter8_1dfunction aom_highbd_filter_block1d4_h8_sse2;
highbd_filter8_1dfunction aom_highbd_filter_block1d16_v4_sse2;
highbd_filter8_1dfunction aom_highbd_filter_block1d16_h4_sse2;
highbd_filter8_1dfunction aom_highbd_filter_block1d8_v4_sse2;
highbd_filter8_1dfunction aom_highbd_filter_block1d8_h4_sse2;
highbd_filter8_1dfunction aom_highbd_filter_block1d4_v4_sse2;
highbd_filter8_1dfunction aom_highbd_filter_block1d4_h4_sse2;
highbd_filter8_1dfunction aom_highbd_filter_block1d16_v2_sse2;
highbd_filter8_1dfunction aom_highbd_filter_block1d16_h2_sse2;
highbd_filter8_1dfunction aom_highbd_filter_block1d8_v2_sse2;
@ -85,5 +91,4 @@ highbd_filter8_1dfunction aom_highbd_filter_block1d4_h2_sse2;
HIGH_FUN_CONV_1D(horiz, x_step_q4, filter_x, h, src, , sse2);
HIGH_FUN_CONV_1D(vert, y_step_q4, filter_y, v, src - src_stride * 3, , sse2);
#endif // ARCH_X86_64
#endif // HAVE_SSE2

75
third_party/aom/aom_dsp/x86/aom_high_subpixel_bilinear_sse2.asm vendored
View file

@ -67,7 +67,6 @@
dec rcx
%endm
%if ARCH_X86_64
%macro HIGH_GET_PARAM 0
mov rdx, arg(5) ;filter ptr
mov rsi, arg(0) ;src_ptr
@ -86,14 +85,17 @@
mov rdx, 0x00010001
movsxd rcx, DWORD PTR arg(6) ;bps
movq xmm8, rdx
movq xmm3, rdx
movq xmm5, rcx
pshufd xmm8, xmm8, 0b
movdqa xmm1, xmm8
psllw xmm8, xmm5
psubw xmm8, xmm1 ;max value (for clamping)
pshufd xmm3, xmm3, 0b
movdqa xmm1, xmm3
psllw xmm3, xmm5
psubw xmm3, xmm1 ;max value (for clamping)
pxor xmm5, xmm5 ;min value (for clamping)
movdqa max, xmm3
movdqa min, xmm5
movsxd rax, DWORD PTR arg(1) ;pixels_per_line
movsxd rdx, DWORD PTR arg(3) ;out_pitch
movsxd rcx, DWORD PTR arg(4) ;output_height
@ -113,8 +115,8 @@
packssdw xmm0, xmm6 ;pack back to word
;clamp the values
pminsw xmm0, xmm8
pmaxsw xmm0, xmm5
pminsw xmm0, max
pmaxsw xmm0, min
%if %1
movdqu xmm1, [rdi]
@ -128,36 +130,36 @@
%endm
%macro HIGH_APPLY_FILTER_16 1
movdqa xmm9, xmm0
movdqa xmm5, xmm0
movdqa xmm6, xmm2
punpckhwd xmm9, xmm1
punpckhwd xmm5, xmm1
punpckhwd xmm6, xmm3
punpcklwd xmm0, xmm1
punpcklwd xmm2, xmm3
pmaddwd xmm9, xmm7
pmaddwd xmm5, xmm7
pmaddwd xmm6, xmm7
pmaddwd xmm0, xmm7
pmaddwd xmm2, xmm7
paddd xmm9, xmm4 ;rounding
paddd xmm5, xmm4 ;rounding
paddd xmm6, xmm4
paddd xmm0, xmm4
paddd xmm2, xmm4
psrad xmm9, 7 ;shift
psrad xmm5, 7 ;shift
psrad xmm6, 7
psrad xmm0, 7
psrad xmm2, 7
packssdw xmm0, xmm9 ;pack back to word
packssdw xmm0, xmm5 ;pack back to word
packssdw xmm2, xmm6 ;pack back to word
;clamp the values
pminsw xmm0, xmm8
pmaxsw xmm0, xmm5
pminsw xmm2, xmm8
pmaxsw xmm2, xmm5
pminsw xmm0, max
pmaxsw xmm0, min
pminsw xmm2, max
pmaxsw xmm2, min
%if %1
movdqu xmm1, [rdi]
@ -172,7 +174,6 @@
lea rdi, [rdi + 2*rdx]
dec rcx
%endm
%endif
SECTION .text
@ -200,7 +201,6 @@ sym(aom_highbd_filter_block1d4_v2_sse2):
pop rbp
ret
%if ARCH_X86_64
global sym(aom_highbd_filter_block1d8_v2_sse2) PRIVATE
sym(aom_highbd_filter_block1d8_v2_sse2):
push rbp
@ -211,6 +211,11 @@ sym(aom_highbd_filter_block1d8_v2_sse2):
push rdi
; end prolog
ALIGN_STACK 16, rax
sub rsp, 16 * 2
%define max [rsp + 16 * 0]
%define min [rsp + 16 * 1]
HIGH_GET_PARAM
.loop:
movdqu xmm0, [rsi] ;0
@ -219,6 +224,9 @@ sym(aom_highbd_filter_block1d8_v2_sse2):
HIGH_APPLY_FILTER_8 0
jnz .loop
add rsp, 16 * 2
pop rsp
; begin epilog
pop rdi
pop rsi
@ -237,6 +245,11 @@ sym(aom_highbd_filter_block1d16_v2_sse2):
push rdi
; end prolog
ALIGN_STACK 16, rax
sub rsp, 16 * 2
%define max [rsp + 16 * 0]
%define min [rsp + 16 * 1]
HIGH_GET_PARAM
.loop:
movdqu xmm0, [rsi] ;0
@ -247,6 +260,9 @@ sym(aom_highbd_filter_block1d16_v2_sse2):
HIGH_APPLY_FILTER_16 0
jnz .loop
add rsp, 16 * 2
pop rsp
; begin epilog
pop rdi
pop rsi
@ -254,7 +270,6 @@ sym(aom_highbd_filter_block1d16_v2_sse2):
UNSHADOW_ARGS
pop rbp
ret
%endif
global sym(aom_highbd_filter_block1d4_h2_sse2) PRIVATE
sym(aom_highbd_filter_block1d4_h2_sse2):
@ -281,7 +296,6 @@ sym(aom_highbd_filter_block1d4_h2_sse2):
pop rbp
ret
%if ARCH_X86_64
global sym(aom_highbd_filter_block1d8_h2_sse2) PRIVATE
sym(aom_highbd_filter_block1d8_h2_sse2):
push rbp
@ -292,6 +306,11 @@ sym(aom_highbd_filter_block1d8_h2_sse2):
push rdi
; end prolog
ALIGN_STACK 16, rax
sub rsp, 16 * 2
%define max [rsp + 16 * 0]
%define min [rsp + 16 * 1]
HIGH_GET_PARAM
.loop:
movdqu xmm0, [rsi] ;load src
@ -300,6 +319,9 @@ sym(aom_highbd_filter_block1d8_h2_sse2):
HIGH_APPLY_FILTER_8 0
jnz .loop
add rsp, 16 * 2
pop rsp
; begin epilog
pop rdi
pop rsi
@ -318,6 +340,11 @@ sym(aom_highbd_filter_block1d16_h2_sse2):
push rdi
; end prolog
ALIGN_STACK 16, rax
sub rsp, 16 * 2
%define max [rsp + 16 * 0]
%define min [rsp + 16 * 1]
HIGH_GET_PARAM
.loop:
movdqu xmm0, [rsi] ;load src
@ -328,6 +355,9 @@ sym(aom_highbd_filter_block1d16_h2_sse2):
HIGH_APPLY_FILTER_16 0
jnz .loop
add rsp, 16 * 2
pop rsp
; begin epilog
pop rdi
pop rsi
@ -335,4 +365,3 @@ sym(aom_highbd_filter_block1d16_h2_sse2):
UNSHADOW_ARGS
pop rbp
ret
%endif

569
third_party/aom/aom_dsp/x86/aom_subpixel_8t_intrin_sse2.c vendored Normal file
View file

@ -0,0 +1,569 @@
/*
* Copyright (c) 2018, Alliance for Open Media. All rights reserved
*
* This source code is subject to the terms of the BSD 2 Clause License and
* the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
* was not distributed with this source code in the LICENSE file, you can
* obtain it at www.aomedia.org/license/software. If the Alliance for Open
* Media Patent License 1.0 was not distributed with this source code in the
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
*/
#include <emmintrin.h> // SSE2
#include "config/aom_dsp_rtcd.h"
#include "aom_dsp/x86/convolve.h"
#include "aom_ports/mem.h"
void aom_filter_block1d16_h4_sse2(const uint8_t *src_ptr,
ptrdiff_t src_pixels_per_line,
uint8_t *output_ptr, ptrdiff_t output_pitch,
uint32_t output_height,
const int16_t *filter) {
__m128i filtersReg;
__m128i addFilterReg32;
__m128i secondFilters, thirdFilters;
__m128i srcRegFilt32b1_1, srcRegFilt32b1_2, srcRegFilt32b2_1,
srcRegFilt32b2_2;
__m128i srcReg32b1, srcReg32b2;
unsigned int i;
src_ptr -= 3;
addFilterReg32 = _mm_set1_epi16(32);
filtersReg = _mm_loadu_si128((const __m128i *)filter);
filtersReg = _mm_srai_epi16(filtersReg, 1);
// coeffs 0 1 0 1 2 3 2 3
const __m128i tmp_0 = _mm_unpacklo_epi32(filtersReg, filtersReg);
// coeffs 4 5 4 5 6 7 6 7
const __m128i tmp_1 = _mm_unpackhi_epi32(filtersReg, filtersReg);
secondFilters = _mm_unpackhi_epi64(tmp_0, tmp_0); // coeffs 2 3 2 3 2 3 2 3
thirdFilters = _mm_unpacklo_epi64(tmp_1, tmp_1); // coeffs 4 5 4 5 4 5 4 5
for (i = output_height; i > 0; i -= 1) {
srcReg32b1 = _mm_loadu_si128((const __m128i *)src_ptr);
__m128i ss_2 = _mm_srli_si128(srcReg32b1, 2);
__m128i ss_4 = _mm_srli_si128(srcReg32b1, 4);
__m128i ss_1_1 = _mm_unpacklo_epi8(ss_2, _mm_setzero_si128());
__m128i ss_2_1 = _mm_unpacklo_epi8(ss_4, _mm_setzero_si128());
__m128i d1 = _mm_madd_epi16(ss_1_1, secondFilters);
__m128i d2 = _mm_madd_epi16(ss_2_1, thirdFilters);
srcRegFilt32b1_1 = _mm_add_epi32(d1, d2);
__m128i ss_1 = _mm_srli_si128(srcReg32b1, 3);
__m128i ss_3 = _mm_srli_si128(srcReg32b1, 5);
__m128i ss_1_2 = _mm_unpacklo_epi8(ss_1, _mm_setzero_si128());
__m128i ss_2_2 = _mm_unpacklo_epi8(ss_3, _mm_setzero_si128());
d1 = _mm_madd_epi16(ss_1_2, secondFilters);
d2 = _mm_madd_epi16(ss_2_2, thirdFilters);
srcRegFilt32b1_2 = _mm_add_epi32(d1, d2);
__m128i res_lo = _mm_unpacklo_epi32(srcRegFilt32b1_1, srcRegFilt32b1_2);
__m128i res_hi = _mm_unpackhi_epi32(srcRegFilt32b1_1, srcRegFilt32b1_2);
srcRegFilt32b1_1 = _mm_packs_epi32(res_lo, res_hi);
// reading stride of the next 16 bytes
// (part of it was being read by earlier read)
srcReg32b2 = _mm_loadu_si128((const __m128i *)(src_ptr + 8));
ss_2 = _mm_srli_si128(srcReg32b2, 2);
ss_4 = _mm_srli_si128(srcReg32b2, 4);
ss_1_1 = _mm_unpacklo_epi8(ss_2, _mm_setzero_si128());
ss_2_1 = _mm_unpacklo_epi8(ss_4, _mm_setzero_si128());
d1 = _mm_madd_epi16(ss_1_1, secondFilters);
d2 = _mm_madd_epi16(ss_2_1, thirdFilters);
srcRegFilt32b2_1 = _mm_add_epi32(d1, d2);
ss_1 = _mm_srli_si128(srcReg32b2, 3);
ss_3 = _mm_srli_si128(srcReg32b2, 5);
ss_1_2 = _mm_unpacklo_epi8(ss_1, _mm_setzero_si128());
ss_2_2 = _mm_unpacklo_epi8(ss_3, _mm_setzero_si128());
d1 = _mm_madd_epi16(ss_1_2, secondFilters);
d2 = _mm_madd_epi16(ss_2_2, thirdFilters);
srcRegFilt32b2_2 = _mm_add_epi32(d1, d2);
res_lo = _mm_unpacklo_epi32(srcRegFilt32b2_1, srcRegFilt32b2_2);
res_hi = _mm_unpackhi_epi32(srcRegFilt32b2_1, srcRegFilt32b2_2);
srcRegFilt32b2_1 = _mm_packs_epi32(res_lo, res_hi);
// shift by 6 bit each 16 bit
srcRegFilt32b1_1 = _mm_adds_epi16(srcRegFilt32b1_1, addFilterReg32);
srcRegFilt32b2_1 = _mm_adds_epi16(srcRegFilt32b2_1, addFilterReg32);
srcRegFilt32b1_1 = _mm_srai_epi16(srcRegFilt32b1_1, 6);
srcRegFilt32b2_1 = _mm_srai_epi16(srcRegFilt32b2_1, 6);
// shrink to 8 bit each 16 bits, the first lane contain the first
// convolve result and the second lane contain the second convolve result
srcRegFilt32b1_1 = _mm_packus_epi16(srcRegFilt32b1_1, srcRegFilt32b2_1);
src_ptr += src_pixels_per_line;
_mm_store_si128((__m128i *)output_ptr, srcRegFilt32b1_1);
output_ptr += output_pitch;
}
}
void aom_filter_block1d16_v4_sse2(const uint8_t *src_ptr, ptrdiff_t src_pitch,
uint8_t *output_ptr, ptrdiff_t out_pitch,
uint32_t output_height,
const int16_t *filter) {
__m128i filtersReg;
__m128i srcReg2, srcReg3, srcReg4, srcReg5, srcReg6;
__m128i srcReg23_lo, srcReg23_hi, srcReg34_lo, srcReg34_hi;
__m128i srcReg45_lo, srcReg45_hi, srcReg56_lo, srcReg56_hi;
__m128i resReg23_lo, resReg34_lo, resReg45_lo, resReg56_lo;
__m128i resReg23_hi, resReg34_hi, resReg45_hi, resReg56_hi;
__m128i resReg23_45_lo, resReg34_56_lo, resReg23_45_hi, resReg34_56_hi;
__m128i resReg23_45, resReg34_56;
__m128i addFilterReg32, secondFilters, thirdFilters;
__m128i tmp_0, tmp_1;
unsigned int i;
ptrdiff_t src_stride, dst_stride;
addFilterReg32 = _mm_set1_epi16(32);
filtersReg = _mm_loadu_si128((const __m128i *)filter);
filtersReg = _mm_srai_epi16(filtersReg, 1);
// coeffs 0 1 0 1 2 3 2 3
const __m128i tmp0 = _mm_unpacklo_epi32(filtersReg, filtersReg);
// coeffs 4 5 4 5 6 7 6 7
const __m128i tmp1 = _mm_unpackhi_epi32(filtersReg, filtersReg);
secondFilters = _mm_unpackhi_epi64(tmp0, tmp0); // coeffs 2 3 2 3 2 3 2 3
thirdFilters = _mm_unpacklo_epi64(tmp1, tmp1); // coeffs 4 5 4 5 4 5 4 5
// multiply the size of the source and destination stride by two
src_stride = src_pitch << 1;
dst_stride = out_pitch << 1;
srcReg2 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 2));
srcReg3 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 3));
srcReg23_lo = _mm_unpacklo_epi8(srcReg2, srcReg3);
srcReg23_hi = _mm_unpackhi_epi8(srcReg2, srcReg3);
__m128i resReg23_lo_1 = _mm_unpacklo_epi8(srcReg23_lo, _mm_setzero_si128());
__m128i resReg23_lo_2 = _mm_unpackhi_epi8(srcReg23_lo, _mm_setzero_si128());
__m128i resReg23_hi_1 = _mm_unpacklo_epi8(srcReg23_hi, _mm_setzero_si128());
__m128i resReg23_hi_2 = _mm_unpackhi_epi8(srcReg23_hi, _mm_setzero_si128());
srcReg4 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 4));
srcReg34_lo = _mm_unpacklo_epi8(srcReg3, srcReg4);
srcReg34_hi = _mm_unpackhi_epi8(srcReg3, srcReg4);
__m128i resReg34_lo_1 = _mm_unpacklo_epi8(srcReg34_lo, _mm_setzero_si128());
__m128i resReg34_lo_2 = _mm_unpackhi_epi8(srcReg34_lo, _mm_setzero_si128());
__m128i resReg34_hi_1 = _mm_unpacklo_epi8(srcReg34_hi, _mm_setzero_si128());
__m128i resReg34_hi_2 = _mm_unpackhi_epi8(srcReg34_hi, _mm_setzero_si128());
for (i = output_height; i > 1; i -= 2) {
srcReg5 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 5));
srcReg45_lo = _mm_unpacklo_epi8(srcReg4, srcReg5);
srcReg45_hi = _mm_unpackhi_epi8(srcReg4, srcReg5);
srcReg6 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 6));
srcReg56_lo = _mm_unpacklo_epi8(srcReg5, srcReg6);
srcReg56_hi = _mm_unpackhi_epi8(srcReg5, srcReg6);
// multiply 2 adjacent elements with the filter and add the result
tmp_0 = _mm_madd_epi16(resReg23_lo_1, secondFilters);
tmp_1 = _mm_madd_epi16(resReg23_lo_2, secondFilters);
resReg23_lo = _mm_packs_epi32(tmp_0, tmp_1);
tmp_0 = _mm_madd_epi16(resReg34_lo_1, secondFilters);
tmp_1 = _mm_madd_epi16(resReg34_lo_2, secondFilters);
resReg34_lo = _mm_packs_epi32(tmp_0, tmp_1);
__m128i resReg45_lo_1 = _mm_unpacklo_epi8(srcReg45_lo, _mm_setzero_si128());
__m128i resReg45_lo_2 = _mm_unpackhi_epi8(srcReg45_lo, _mm_setzero_si128());
tmp_0 = _mm_madd_epi16(resReg45_lo_1, thirdFilters);
tmp_1 = _mm_madd_epi16(resReg45_lo_2, thirdFilters);
resReg45_lo = _mm_packs_epi32(tmp_0, tmp_1);
__m128i resReg56_lo_1 = _mm_unpacklo_epi8(srcReg56_lo, _mm_setzero_si128());
__m128i resReg56_lo_2 = _mm_unpackhi_epi8(srcReg56_lo, _mm_setzero_si128());
tmp_0 = _mm_madd_epi16(resReg56_lo_1, thirdFilters);
tmp_1 = _mm_madd_epi16(resReg56_lo_2, thirdFilters);
resReg56_lo = _mm_packs_epi32(tmp_0, tmp_1);
// add and saturate the results together
resReg23_45_lo = _mm_adds_epi16(resReg23_lo, resReg45_lo);
resReg34_56_lo = _mm_adds_epi16(resReg34_lo, resReg56_lo);
// multiply 2 adjacent elements with the filter and add the result
tmp_0 = _mm_madd_epi16(resReg23_hi_1, secondFilters);
tmp_1 = _mm_madd_epi16(resReg23_hi_2, secondFilters);
resReg23_hi = _mm_packs_epi32(tmp_0, tmp_1);
tmp_0 = _mm_madd_epi16(resReg34_hi_1, secondFilters);
tmp_1 = _mm_madd_epi16(resReg34_hi_2, secondFilters);
resReg34_hi = _mm_packs_epi32(tmp_0, tmp_1);
__m128i resReg45_hi_1 = _mm_unpacklo_epi8(srcReg45_hi, _mm_setzero_si128());
__m128i resReg45_hi_2 = _mm_unpackhi_epi8(srcReg45_hi, _mm_setzero_si128());
tmp_0 = _mm_madd_epi16(resReg45_hi_1, thirdFilters);
tmp_1 = _mm_madd_epi16(resReg45_hi_2, thirdFilters);
resReg45_hi = _mm_packs_epi32(tmp_0, tmp_1);
__m128i resReg56_hi_1 = _mm_unpacklo_epi8(srcReg56_hi, _mm_setzero_si128());
__m128i resReg56_hi_2 = _mm_unpackhi_epi8(srcReg56_hi, _mm_setzero_si128());
tmp_0 = _mm_madd_epi16(resReg56_hi_1, thirdFilters);
tmp_1 = _mm_madd_epi16(resReg56_hi_2, thirdFilters);
resReg56_hi = _mm_packs_epi32(tmp_0, tmp_1);
// add and saturate the results together
resReg23_45_hi = _mm_adds_epi16(resReg23_hi, resReg45_hi);
resReg34_56_hi = _mm_adds_epi16(resReg34_hi, resReg56_hi);
// shift by 6 bit each 16 bit
resReg23_45_lo = _mm_adds_epi16(resReg23_45_lo, addFilterReg32);
resReg34_56_lo = _mm_adds_epi16(resReg34_56_lo, addFilterReg32);
resReg23_45_hi = _mm_adds_epi16(resReg23_45_hi, addFilterReg32);
resReg34_56_hi = _mm_adds_epi16(resReg34_56_hi, addFilterReg32);
resReg23_45_lo = _mm_srai_epi16(resReg23_45_lo, 6);
resReg34_56_lo = _mm_srai_epi16(resReg34_56_lo, 6);
resReg23_45_hi = _mm_srai_epi16(resReg23_45_hi, 6);
resReg34_56_hi = _mm_srai_epi16(resReg34_56_hi, 6);
// shrink to 8 bit each 16 bits, the first lane contain the first
// convolve result and the second lane contain the second convolve
// result
resReg23_45 = _mm_packus_epi16(resReg23_45_lo, resReg23_45_hi);
resReg34_56 = _mm_packus_epi16(resReg34_56_lo, resReg34_56_hi);
src_ptr += src_stride;
_mm_store_si128((__m128i *)output_ptr, (resReg23_45));
_mm_store_si128((__m128i *)(output_ptr + out_pitch), (resReg34_56));
output_ptr += dst_stride;
// save part of the registers for next strides
resReg23_lo_1 = resReg45_lo_1;
resReg23_lo_2 = resReg45_lo_2;
resReg23_hi_1 = resReg45_hi_1;
resReg23_hi_2 = resReg45_hi_2;
resReg34_lo_1 = resReg56_lo_1;
resReg34_lo_2 = resReg56_lo_2;
resReg34_hi_1 = resReg56_hi_1;
resReg34_hi_2 = resReg56_hi_2;
srcReg4 = srcReg6;
}
}
void aom_filter_block1d8_h4_sse2(const uint8_t *src_ptr,
ptrdiff_t src_pixels_per_line,
uint8_t *output_ptr, ptrdiff_t output_pitch,
uint32_t output_height,
const int16_t *filter) {
__m128i filtersReg;
__m128i addFilterReg32;
__m128i secondFilters, thirdFilters;
__m128i srcRegFilt32b1_1, srcRegFilt32b1_2;
__m128i srcReg32b1;
unsigned int i;
src_ptr -= 3;
addFilterReg32 = _mm_set1_epi16(32);
filtersReg = _mm_loadu_si128((const __m128i *)filter);
filtersReg = _mm_srai_epi16(filtersReg, 1);
// coeffs 0 1 0 1 2 3 2 3
const __m128i tmp_0 = _mm_unpacklo_epi32(filtersReg, filtersReg);
// coeffs 4 5 4 5 6 7 6 7
const __m128i tmp_1 = _mm_unpackhi_epi32(filtersReg, filtersReg);
secondFilters = _mm_unpackhi_epi64(tmp_0, tmp_0); // coeffs 2 3 2 3 2 3 2 3
thirdFilters = _mm_unpacklo_epi64(tmp_1, tmp_1); // coeffs 4 5 4 5 4 5 4 5
for (i = output_height; i > 0; i -= 1) {
srcReg32b1 = _mm_loadu_si128((const __m128i *)src_ptr);
__m128i ss_2 = _mm_srli_si128(srcReg32b1, 2);
__m128i ss_4 = _mm_srli_si128(srcReg32b1, 4);
ss_2 = _mm_unpacklo_epi8(ss_2, _mm_setzero_si128());
ss_4 = _mm_unpacklo_epi8(ss_4, _mm_setzero_si128());
__m128i d1 = _mm_madd_epi16(ss_2, secondFilters);
__m128i d2 = _mm_madd_epi16(ss_4, thirdFilters);
srcRegFilt32b1_1 = _mm_add_epi32(d1, d2);
__m128i ss_3 = _mm_srli_si128(srcReg32b1, 3);
__m128i ss_5 = _mm_srli_si128(srcReg32b1, 5);
ss_3 = _mm_unpacklo_epi8(ss_3, _mm_setzero_si128());
ss_5 = _mm_unpacklo_epi8(ss_5, _mm_setzero_si128());
d1 = _mm_madd_epi16(ss_3, secondFilters);
d2 = _mm_madd_epi16(ss_5, thirdFilters);
srcRegFilt32b1_2 = _mm_add_epi32(d1, d2);
__m128i res_lo = _mm_unpacklo_epi32(srcRegFilt32b1_1, srcRegFilt32b1_2);
__m128i res_hi = _mm_unpackhi_epi32(srcRegFilt32b1_1, srcRegFilt32b1_2);
srcRegFilt32b1_1 = _mm_packs_epi32(res_lo, res_hi);
// shift by 6 bit each 16 bit
srcRegFilt32b1_1 = _mm_adds_epi16(srcRegFilt32b1_1, addFilterReg32);
srcRegFilt32b1_1 = _mm_srai_epi16(srcRegFilt32b1_1, 6);
// shrink to 8 bit each 16 bits, the first lane contain the first
// convolve result and the second lane contain the second convolve result
srcRegFilt32b1_1 = _mm_packus_epi16(srcRegFilt32b1_1, _mm_setzero_si128());
src_ptr += src_pixels_per_line;
_mm_storel_epi64((__m128i *)output_ptr, srcRegFilt32b1_1);
output_ptr += output_pitch;
}
}
void aom_filter_block1d8_v4_sse2(const uint8_t *src_ptr, ptrdiff_t src_pitch,
uint8_t *output_ptr, ptrdiff_t out_pitch,
uint32_t output_height,
const int16_t *filter) {
__m128i filtersReg;
__m128i srcReg2, srcReg3, srcReg4, srcReg5, srcReg6;
__m128i srcReg23_lo, srcReg34_lo;
__m128i srcReg45_lo, srcReg56_lo;
__m128i resReg23_lo, resReg34_lo, resReg45_lo, resReg56_lo;
__m128i resReg23_45_lo, resReg34_56_lo;
__m128i resReg23_45, resReg34_56;
__m128i addFilterReg32, secondFilters, thirdFilters;
__m128i tmp_0, tmp_1;
unsigned int i;
ptrdiff_t src_stride, dst_stride;
addFilterReg32 = _mm_set1_epi16(32);
filtersReg = _mm_loadu_si128((const __m128i *)filter);
filtersReg = _mm_srai_epi16(filtersReg, 1);
// coeffs 0 1 0 1 2 3 2 3
const __m128i tmp0 = _mm_unpacklo_epi32(filtersReg, filtersReg);
// coeffs 4 5 4 5 6 7 6 7
const __m128i tmp1 = _mm_unpackhi_epi32(filtersReg, filtersReg);
secondFilters = _mm_unpackhi_epi64(tmp0, tmp0); // coeffs 2 3 2 3 2 3 2 3
thirdFilters = _mm_unpacklo_epi64(tmp1, tmp1); // coeffs 4 5 4 5 4 5 4 5
// multiply the size of the source and destination stride by two
src_stride = src_pitch << 1;
dst_stride = out_pitch << 1;
srcReg2 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 2));
srcReg3 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 3));
srcReg23_lo = _mm_unpacklo_epi8(srcReg2, srcReg3);
__m128i resReg23_lo_1 = _mm_unpacklo_epi8(srcReg23_lo, _mm_setzero_si128());
__m128i resReg23_lo_2 = _mm_unpackhi_epi8(srcReg23_lo, _mm_setzero_si128());
srcReg4 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 4));
srcReg34_lo = _mm_unpacklo_epi8(srcReg3, srcReg4);
__m128i resReg34_lo_1 = _mm_unpacklo_epi8(srcReg34_lo, _mm_setzero_si128());
__m128i resReg34_lo_2 = _mm_unpackhi_epi8(srcReg34_lo, _mm_setzero_si128());
for (i = output_height; i > 1; i -= 2) {
srcReg5 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 5));
srcReg45_lo = _mm_unpacklo_epi8(srcReg4, srcReg5);
srcReg6 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 6));
srcReg56_lo = _mm_unpacklo_epi8(srcReg5, srcReg6);
// multiply 2 adjacent elements with the filter and add the result
tmp_0 = _mm_madd_epi16(resReg23_lo_1, secondFilters);
tmp_1 = _mm_madd_epi16(resReg23_lo_2, secondFilters);
resReg23_lo = _mm_packs_epi32(tmp_0, tmp_1);
tmp_0 = _mm_madd_epi16(resReg34_lo_1, secondFilters);
tmp_1 = _mm_madd_epi16(resReg34_lo_2, secondFilters);
resReg34_lo = _mm_packs_epi32(tmp_0, tmp_1);
__m128i resReg45_lo_1 = _mm_unpacklo_epi8(srcReg45_lo, _mm_setzero_si128());
__m128i resReg45_lo_2 = _mm_unpackhi_epi8(srcReg45_lo, _mm_setzero_si128());
tmp_0 = _mm_madd_epi16(resReg45_lo_1, thirdFilters);
tmp_1 = _mm_madd_epi16(resReg45_lo_2, thirdFilters);
resReg45_lo = _mm_packs_epi32(tmp_0, tmp_1);
__m128i resReg56_lo_1 = _mm_unpacklo_epi8(srcReg56_lo, _mm_setzero_si128());
__m128i resReg56_lo_2 = _mm_unpackhi_epi8(srcReg56_lo, _mm_setzero_si128());
tmp_0 = _mm_madd_epi16(resReg56_lo_1, thirdFilters);
tmp_1 = _mm_madd_epi16(resReg56_lo_2, thirdFilters);
resReg56_lo = _mm_packs_epi32(tmp_0, tmp_1);
// add and saturate the results together
resReg23_45_lo = _mm_adds_epi16(resReg23_lo, resReg45_lo);
resReg34_56_lo = _mm_adds_epi16(resReg34_lo, resReg56_lo);
// shift by 6 bit each 16 bit
resReg23_45_lo = _mm_adds_epi16(resReg23_45_lo, addFilterReg32);
resReg34_56_lo = _mm_adds_epi16(resReg34_56_lo, addFilterReg32);
resReg23_45_lo = _mm_srai_epi16(resReg23_45_lo, 6);
resReg34_56_lo = _mm_srai_epi16(resReg34_56_lo, 6);
// shrink to 8 bit each 16 bits, the first lane contain the first
// convolve result and the second lane contain the second convolve
// result
resReg23_45 = _mm_packus_epi16(resReg23_45_lo, _mm_setzero_si128());
resReg34_56 = _mm_packus_epi16(resReg34_56_lo, _mm_setzero_si128());
src_ptr += src_stride;
_mm_storel_epi64((__m128i *)output_ptr, (resReg23_45));
_mm_storel_epi64((__m128i *)(output_ptr + out_pitch), (resReg34_56));
output_ptr += dst_stride;
// save part of the registers for next strides
resReg23_lo_1 = resReg45_lo_1;
resReg23_lo_2 = resReg45_lo_2;
resReg34_lo_1 = resReg56_lo_1;
resReg34_lo_2 = resReg56_lo_2;
srcReg4 = srcReg6;
}
}
void aom_filter_block1d4_h4_sse2(const uint8_t *src_ptr,
ptrdiff_t src_pixels_per_line,
uint8_t *output_ptr, ptrdiff_t output_pitch,
uint32_t output_height,
const int16_t *filter) {
__m128i filtersReg;
__m128i addFilterReg32;
__m128i secondFilters, thirdFilters;
__m128i srcRegFilt32b1_1;
__m128i srcReg32b1;
unsigned int i;
src_ptr -= 3;
addFilterReg32 = _mm_set1_epi16(32);
filtersReg = _mm_loadu_si128((const __m128i *)filter);
filtersReg = _mm_srai_epi16(filtersReg, 1);
// coeffs 0 1 0 1 2 3 2 3
const __m128i tmp_0 = _mm_unpacklo_epi32(filtersReg, filtersReg);
// coeffs 4 5 4 5 6 7 6 7
const __m128i tmp_1 = _mm_unpackhi_epi32(filtersReg, filtersReg);
secondFilters = _mm_unpackhi_epi64(tmp_0, tmp_0); // coeffs 2 3 2 3 2 3 2 3
thirdFilters = _mm_unpacklo_epi64(tmp_1, tmp_1); // coeffs 4 5 4 5 4 5 4 5
for (i = output_height; i > 0; i -= 1) {
srcReg32b1 = _mm_loadu_si128((const __m128i *)src_ptr);
__m128i ss_2 = _mm_srli_si128(srcReg32b1, 2);
__m128i ss_3 = _mm_srli_si128(srcReg32b1, 3);
__m128i ss_4 = _mm_srli_si128(srcReg32b1, 4);
__m128i ss_5 = _mm_srli_si128(srcReg32b1, 5);
ss_2 = _mm_unpacklo_epi8(ss_2, _mm_setzero_si128());
ss_3 = _mm_unpacklo_epi8(ss_3, _mm_setzero_si128());
ss_4 = _mm_unpacklo_epi8(ss_4, _mm_setzero_si128());
ss_5 = _mm_unpacklo_epi8(ss_5, _mm_setzero_si128());
__m128i ss_1_1 = _mm_unpacklo_epi32(ss_2, ss_3);
__m128i ss_1_2 = _mm_unpacklo_epi32(ss_4, ss_5);
__m128i d1 = _mm_madd_epi16(ss_1_1, secondFilters);
__m128i d2 = _mm_madd_epi16(ss_1_2, thirdFilters);
srcRegFilt32b1_1 = _mm_add_epi32(d1, d2);
srcRegFilt32b1_1 = _mm_packs_epi32(srcRegFilt32b1_1, _mm_setzero_si128());
// shift by 6 bit each 16 bit
srcRegFilt32b1_1 = _mm_adds_epi16(srcRegFilt32b1_1, addFilterReg32);
srcRegFilt32b1_1 = _mm_srai_epi16(srcRegFilt32b1_1, 6);
// shrink to 8 bit each 16 bits, the first lane contain the first
// convolve result and the second lane contain the second convolve result
srcRegFilt32b1_1 = _mm_packus_epi16(srcRegFilt32b1_1, _mm_setzero_si128());
src_ptr += src_pixels_per_line;
*((uint32_t *)(output_ptr)) = _mm_cvtsi128_si32(srcRegFilt32b1_1);
output_ptr += output_pitch;
}
}
void aom_filter_block1d4_v4_sse2(const uint8_t *src_ptr, ptrdiff_t src_pitch,
uint8_t *output_ptr, ptrdiff_t out_pitch,
uint32_t output_height,
const int16_t *filter) {
__m128i filtersReg;
__m128i srcReg2, srcReg3, srcReg4, srcReg5, srcReg6;
__m128i srcReg23, srcReg34, srcReg45, srcReg56;
__m128i resReg23_34, resReg45_56;
__m128i resReg23_34_45_56;
__m128i addFilterReg32, secondFilters, thirdFilters;
__m128i tmp_0, tmp_1;
unsigned int i;
ptrdiff_t src_stride, dst_stride;
addFilterReg32 = _mm_set1_epi16(32);
filtersReg = _mm_loadu_si128((const __m128i *)filter);
filtersReg = _mm_srai_epi16(filtersReg, 1);
// coeffs 0 1 0 1 2 3 2 3
const __m128i tmp0 = _mm_unpacklo_epi32(filtersReg, filtersReg);
// coeffs 4 5 4 5 6 7 6 7
const __m128i tmp1 = _mm_unpackhi_epi32(filtersReg, filtersReg);
secondFilters = _mm_unpackhi_epi64(tmp0, tmp0); // coeffs 2 3 2 3 2 3 2 3
thirdFilters = _mm_unpacklo_epi64(tmp1, tmp1); // coeffs 4 5 4 5 4 5 4 5
// multiply the size of the source and destination stride by two
src_stride = src_pitch << 1;
dst_stride = out_pitch << 1;
srcReg2 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 2));
srcReg3 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 3));
srcReg23 = _mm_unpacklo_epi8(srcReg2, srcReg3);
__m128i resReg23 = _mm_unpacklo_epi8(srcReg23, _mm_setzero_si128());
srcReg4 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 4));
srcReg34 = _mm_unpacklo_epi8(srcReg3, srcReg4);
__m128i resReg34 = _mm_unpacklo_epi8(srcReg34, _mm_setzero_si128());
for (i = output_height; i > 1; i -= 2) {
srcReg5 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 5));
srcReg45 = _mm_unpacklo_epi8(srcReg4, srcReg5);
srcReg6 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 6));
srcReg56 = _mm_unpacklo_epi8(srcReg5, srcReg6);
// multiply 2 adjacent elements with the filter and add the result
tmp_0 = _mm_madd_epi16(resReg23, secondFilters);
tmp_1 = _mm_madd_epi16(resReg34, secondFilters);
resReg23_34 = _mm_packs_epi32(tmp_0, tmp_1);
__m128i resReg45 = _mm_unpacklo_epi8(srcReg45, _mm_setzero_si128());
__m128i resReg56 = _mm_unpacklo_epi8(srcReg56, _mm_setzero_si128());
tmp_0 = _mm_madd_epi16(resReg45, thirdFilters);
tmp_1 = _mm_madd_epi16(resReg56, thirdFilters);
resReg45_56 = _mm_packs_epi32(tmp_0, tmp_1);
// add and saturate the results together
resReg23_34_45_56 = _mm_adds_epi16(resReg23_34, resReg45_56);
// shift by 6 bit each 16 bit
resReg23_34_45_56 = _mm_adds_epi16(resReg23_34_45_56, addFilterReg32);
resReg23_34_45_56 = _mm_srai_epi16(resReg23_34_45_56, 6);
// shrink to 8 bit each 16 bits, the first lane contain the first
// convolve result and the second lane contain the second convolve
// result
resReg23_34_45_56 =
_mm_packus_epi16(resReg23_34_45_56, _mm_setzero_si128());
src_ptr += src_stride;
*((uint32_t *)(output_ptr)) = _mm_cvtsi128_si32(resReg23_34_45_56);
*((uint32_t *)(output_ptr + out_pitch)) =
_mm_cvtsi128_si32(_mm_srli_si128(resReg23_34_45_56, 4));
output_ptr += dst_stride;
// save part of the registers for next strides
resReg23 = resReg45;
resReg34 = resReg56;
srcReg4 = srcReg6;
}
}

469
third_party/aom/aom_dsp/x86/aom_subpixel_8t_intrin_ssse3.c vendored
View file

@ -45,11 +45,153 @@ DECLARE_ALIGNED(16, static const uint8_t, filt4_global[16]) = {
6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14
};
DECLARE_ALIGNED(32, static const uint8_t, filt_h4[]) = {
0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 0, 1, 1,
2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 2, 3, 3, 4, 4, 5,
5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 2, 3, 3, 4, 4, 5, 5, 6, 6,
7, 7, 8, 8, 9, 9, 10, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10,
10, 11, 11, 12, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11,
12, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 6, 7,
7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14
};
DECLARE_ALIGNED(32, static const uint8_t, filtd4[]) = {
2, 3, 4, 5, 3, 4, 5, 6, 4, 5, 6, 7, 5, 6, 7, 8,
2, 3, 4, 5, 3, 4, 5, 6, 4, 5, 6, 7, 5, 6, 7, 8,
};
// These are reused by the avx2 intrinsics.
filter8_1dfunction aom_filter_block1d8_v8_intrin_ssse3;
filter8_1dfunction aom_filter_block1d8_h8_intrin_ssse3;
filter8_1dfunction aom_filter_block1d4_h8_intrin_ssse3;
static void aom_filter_block1d4_h4_ssse3(
const uint8_t *src_ptr, ptrdiff_t src_pixels_per_line, uint8_t *output_ptr,
ptrdiff_t output_pitch, uint32_t output_height, const int16_t *filter) {
__m128i filtersReg;
__m128i addFilterReg32, filt1Reg, firstFilters, srcReg32b1, srcRegFilt32b1_1;
unsigned int i;
src_ptr -= 3;
addFilterReg32 = _mm_set1_epi16(32);
filtersReg = _mm_loadu_si128((const __m128i *)filter);
filtersReg = _mm_srai_epi16(filtersReg, 1);
// converting the 16 bit (short) to 8 bit (byte) and have the same data
// in both lanes of 128 bit register.
filtersReg = _mm_packs_epi16(filtersReg, filtersReg);
firstFilters = _mm_shuffle_epi8(filtersReg, _mm_set1_epi32(0x5040302u));
filt1Reg = _mm_load_si128((__m128i const *)(filtd4));
for (i = output_height; i > 0; i -= 1) {
// load the 2 strides of source
srcReg32b1 = _mm_loadu_si128((const __m128i *)src_ptr);
// filter the source buffer
srcRegFilt32b1_1 = _mm_shuffle_epi8(srcReg32b1, filt1Reg);
// multiply 4 adjacent elements with the filter and add the result
srcRegFilt32b1_1 = _mm_maddubs_epi16(srcRegFilt32b1_1, firstFilters);
srcRegFilt32b1_1 = _mm_hadds_epi16(srcRegFilt32b1_1, _mm_setzero_si128());
// shift by 6 bit each 16 bit
srcRegFilt32b1_1 = _mm_adds_epi16(srcRegFilt32b1_1, addFilterReg32);
srcRegFilt32b1_1 = _mm_srai_epi16(srcRegFilt32b1_1, 6);
// shrink to 8 bit each 16 bits, the first lane contain the first
// convolve result and the second lane contain the second convolve result
srcRegFilt32b1_1 = _mm_packus_epi16(srcRegFilt32b1_1, _mm_setzero_si128());
src_ptr += src_pixels_per_line;
*((uint32_t *)(output_ptr)) = _mm_cvtsi128_si32(srcRegFilt32b1_1);
output_ptr += output_pitch;
}
}
static void aom_filter_block1d4_v4_ssse3(
const uint8_t *src_ptr, ptrdiff_t src_pitch, uint8_t *output_ptr,
ptrdiff_t out_pitch, uint32_t output_height, const int16_t *filter) {
__m128i filtersReg;
__m128i addFilterReg32;
__m128i srcReg2, srcReg3, srcReg23, srcReg4, srcReg34, srcReg5, srcReg45,
srcReg6, srcReg56;
__m128i srcReg23_34_lo, srcReg45_56_lo;
__m128i srcReg2345_3456_lo, srcReg2345_3456_hi;
__m128i resReglo, resReghi;
__m128i firstFilters;
unsigned int i;
ptrdiff_t src_stride, dst_stride;
addFilterReg32 = _mm_set1_epi16(32);
filtersReg = _mm_loadu_si128((const __m128i *)filter);
// converting the 16 bit (short) to 8 bit (byte) and have the
// same data in both lanes of 128 bit register.
filtersReg = _mm_srai_epi16(filtersReg, 1);
filtersReg = _mm_packs_epi16(filtersReg, filtersReg);
firstFilters = _mm_shuffle_epi8(filtersReg, _mm_set1_epi32(0x5040302u));
// multiple the size of the source and destination stride by two
src_stride = src_pitch << 1;
dst_stride = out_pitch << 1;
srcReg2 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 2));
srcReg3 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 3));
srcReg23 = _mm_unpacklo_epi32(srcReg2, srcReg3);
srcReg4 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 4));
// have consecutive loads on the same 256 register
srcReg34 = _mm_unpacklo_epi32(srcReg3, srcReg4);
srcReg23_34_lo = _mm_unpacklo_epi8(srcReg23, srcReg34);
for (i = output_height; i > 1; i -= 2) {
srcReg5 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 5));
srcReg45 = _mm_unpacklo_epi32(srcReg4, srcReg5);
srcReg6 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 6));
srcReg56 = _mm_unpacklo_epi32(srcReg5, srcReg6);
// merge every two consecutive registers
srcReg45_56_lo = _mm_unpacklo_epi8(srcReg45, srcReg56);
srcReg2345_3456_lo = _mm_unpacklo_epi16(srcReg23_34_lo, srcReg45_56_lo);
srcReg2345_3456_hi = _mm_unpackhi_epi16(srcReg23_34_lo, srcReg45_56_lo);
// multiply 2 adjacent elements with the filter and add the result
resReglo = _mm_maddubs_epi16(srcReg2345_3456_lo, firstFilters);
resReghi = _mm_maddubs_epi16(srcReg2345_3456_hi, firstFilters);
resReglo = _mm_hadds_epi16(resReglo, _mm_setzero_si128());
resReghi = _mm_hadds_epi16(resReghi, _mm_setzero_si128());
// shift by 6 bit each 16 bit
resReglo = _mm_adds_epi16(resReglo, addFilterReg32);
resReghi = _mm_adds_epi16(resReghi, addFilterReg32);
resReglo = _mm_srai_epi16(resReglo, 6);
resReghi = _mm_srai_epi16(resReghi, 6);
// shrink to 8 bit each 16 bits, the first lane contain the first
// convolve result and the second lane contain the second convolve
// result
resReglo = _mm_packus_epi16(resReglo, resReglo);
resReghi = _mm_packus_epi16(resReghi, resReghi);
src_ptr += src_stride;
*((uint32_t *)(output_ptr)) = _mm_cvtsi128_si32(resReglo);
*((uint32_t *)(output_ptr + out_pitch)) = _mm_cvtsi128_si32(resReghi);
output_ptr += dst_stride;
// save part of the registers for next strides
srcReg23_34_lo = srcReg45_56_lo;
srcReg4 = srcReg6;
}
}
void aom_filter_block1d4_h8_intrin_ssse3(
const uint8_t *src_ptr, ptrdiff_t src_pixels_per_line, uint8_t *output_ptr,
ptrdiff_t output_pitch, uint32_t output_height, const int16_t *filter) {
@ -118,6 +260,145 @@ void aom_filter_block1d4_h8_intrin_ssse3(
}
}
static void aom_filter_block1d8_h4_ssse3(
const uint8_t *src_ptr, ptrdiff_t src_pixels_per_line, uint8_t *output_ptr,
ptrdiff_t output_pitch, uint32_t output_height, const int16_t *filter) {
__m128i filtersReg;
__m128i addFilterReg32, filt2Reg, filt3Reg;
__m128i secondFilters, thirdFilters;
__m128i srcRegFilt32b1_1, srcRegFilt32b2, srcRegFilt32b3;
__m128i srcReg32b1;
unsigned int i;
src_ptr -= 3;
addFilterReg32 = _mm_set1_epi16(32);
filtersReg = _mm_loadu_si128((const __m128i *)filter);
filtersReg = _mm_srai_epi16(filtersReg, 1);
// converting the 16 bit (short) to 8 bit (byte) and have the same data
// in both lanes of 128 bit register.
filtersReg = _mm_packs_epi16(filtersReg, filtersReg);
// duplicate only the second 16 bits (third and forth byte)
// across 256 bit register
secondFilters = _mm_shuffle_epi8(filtersReg, _mm_set1_epi16(0x302u));
// duplicate only the third 16 bits (fifth and sixth byte)
// across 256 bit register
thirdFilters = _mm_shuffle_epi8(filtersReg, _mm_set1_epi16(0x504u));
filt2Reg = _mm_load_si128((__m128i const *)(filt_h4 + 32));
filt3Reg = _mm_load_si128((__m128i const *)(filt_h4 + 32 * 2));
for (i = output_height; i > 0; i -= 1) {
srcReg32b1 = _mm_loadu_si128((const __m128i *)src_ptr);
// filter the source buffer
srcRegFilt32b3 = _mm_shuffle_epi8(srcReg32b1, filt2Reg);
srcRegFilt32b2 = _mm_shuffle_epi8(srcReg32b1, filt3Reg);
// multiply 2 adjacent elements with the filter and add the result
srcRegFilt32b3 = _mm_maddubs_epi16(srcRegFilt32b3, secondFilters);
srcRegFilt32b2 = _mm_maddubs_epi16(srcRegFilt32b2, thirdFilters);
srcRegFilt32b1_1 = _mm_adds_epi16(srcRegFilt32b3, srcRegFilt32b2);
// shift by 6 bit each 16 bit
srcRegFilt32b1_1 = _mm_adds_epi16(srcRegFilt32b1_1, addFilterReg32);
srcRegFilt32b1_1 = _mm_srai_epi16(srcRegFilt32b1_1, 6);
// shrink to 8 bit each 16 bits
srcRegFilt32b1_1 = _mm_packus_epi16(srcRegFilt32b1_1, _mm_setzero_si128());
src_ptr += src_pixels_per_line;
_mm_storel_epi64((__m128i *)output_ptr, srcRegFilt32b1_1);
output_ptr += output_pitch;
}
}
static void aom_filter_block1d8_v4_ssse3(
const uint8_t *src_ptr, ptrdiff_t src_pitch, uint8_t *output_ptr,
ptrdiff_t out_pitch, uint32_t output_height, const int16_t *filter) {
__m128i filtersReg;
__m128i srcReg2, srcReg3, srcReg4, srcReg5, srcReg6;
__m128i srcReg23, srcReg34, srcReg45, srcReg56;
__m128i resReg23, resReg34, resReg45, resReg56;
__m128i resReg23_45, resReg34_56;
__m128i addFilterReg32, secondFilters, thirdFilters;
unsigned int i;
ptrdiff_t src_stride, dst_stride;
addFilterReg32 = _mm_set1_epi16(32);
filtersReg = _mm_loadu_si128((const __m128i *)filter);
// converting the 16 bit (short) to 8 bit (byte) and have the
// same data in both lanes of 128 bit register.
filtersReg = _mm_srai_epi16(filtersReg, 1);
filtersReg = _mm_packs_epi16(filtersReg, filtersReg);
// duplicate only the second 16 bits (third and forth byte)
// across 128 bit register
secondFilters = _mm_shuffle_epi8(filtersReg, _mm_set1_epi16(0x302u));
// duplicate only the third 16 bits (fifth and sixth byte)
// across 128 bit register
thirdFilters = _mm_shuffle_epi8(filtersReg, _mm_set1_epi16(0x504u));
// multiple the size of the source and destination stride by two
src_stride = src_pitch << 1;
dst_stride = out_pitch << 1;
srcReg2 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 2));
srcReg3 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 3));
srcReg23 = _mm_unpacklo_epi8(srcReg2, srcReg3);
srcReg4 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 4));
// have consecutive loads on the same 256 register
srcReg34 = _mm_unpacklo_epi8(srcReg3, srcReg4);
for (i = output_height; i > 1; i -= 2) {
srcReg5 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 5));
srcReg45 = _mm_unpacklo_epi8(srcReg4, srcReg5);
srcReg6 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 6));
srcReg56 = _mm_unpacklo_epi8(srcReg5, srcReg6);
// multiply 2 adjacent elements with the filter and add the result
resReg23 = _mm_maddubs_epi16(srcReg23, secondFilters);
resReg34 = _mm_maddubs_epi16(srcReg34, secondFilters);
resReg45 = _mm_maddubs_epi16(srcReg45, thirdFilters);
resReg56 = _mm_maddubs_epi16(srcReg56, thirdFilters);
// add and saturate the results together
resReg23_45 = _mm_adds_epi16(resReg23, resReg45);
resReg34_56 = _mm_adds_epi16(resReg34, resReg56);
// shift by 6 bit each 16 bit
resReg23_45 = _mm_adds_epi16(resReg23_45, addFilterReg32);
resReg34_56 = _mm_adds_epi16(resReg34_56, addFilterReg32);
resReg23_45 = _mm_srai_epi16(resReg23_45, 6);
resReg34_56 = _mm_srai_epi16(resReg34_56, 6);
// shrink to 8 bit each 16 bits, the first lane contain the first
// convolve result and the second lane contain the second convolve
// result
resReg23_45 = _mm_packus_epi16(resReg23_45, _mm_setzero_si128());
resReg34_56 = _mm_packus_epi16(resReg34_56, _mm_setzero_si128());
src_ptr += src_stride;
_mm_storel_epi64((__m128i *)output_ptr, (resReg23_45));
_mm_storel_epi64((__m128i *)(output_ptr + out_pitch), (resReg34_56));
output_ptr += dst_stride;
// save part of the registers for next strides
srcReg23 = srcReg45;
srcReg34 = srcReg56;
srcReg4 = srcReg6;
}
}
void aom_filter_block1d8_h8_intrin_ssse3(
const uint8_t *src_ptr, ptrdiff_t src_pixels_per_line, uint8_t *output_ptr,
ptrdiff_t output_pitch, uint32_t output_height, const int16_t *filter) {
@ -280,6 +561,187 @@ void aom_filter_block1d8_v8_intrin_ssse3(
}
}
static void aom_filter_block1d16_h4_ssse3(
const uint8_t *src_ptr, ptrdiff_t src_pixels_per_line, uint8_t *output_ptr,
ptrdiff_t output_pitch, uint32_t output_height, const int16_t *filter) {
__m128i filtersReg;
__m128i addFilterReg32, filt2Reg, filt3Reg;
__m128i secondFilters, thirdFilters;
__m128i srcRegFilt32b1_1, srcRegFilt32b2_1, srcRegFilt32b2, srcRegFilt32b3;
__m128i srcReg32b1, srcReg32b2;
unsigned int i;
src_ptr -= 3;
addFilterReg32 = _mm_set1_epi16(32);
filtersReg = _mm_loadu_si128((const __m128i *)filter);
filtersReg = _mm_srai_epi16(filtersReg, 1);
// converting the 16 bit (short) to 8 bit (byte) and have the same data
// in both lanes of 128 bit register.
filtersReg = _mm_packs_epi16(filtersReg, filtersReg);
// duplicate only the second 16 bits (third and forth byte)
// across 256 bit register
secondFilters = _mm_shuffle_epi8(filtersReg, _mm_set1_epi16(0x302u));
// duplicate only the third 16 bits (fifth and sixth byte)
// across 256 bit register
thirdFilters = _mm_shuffle_epi8(filtersReg, _mm_set1_epi16(0x504u));
filt2Reg = _mm_load_si128((__m128i const *)(filt_h4 + 32));
filt3Reg = _mm_load_si128((__m128i const *)(filt_h4 + 32 * 2));
for (i = output_height; i > 0; i -= 1) {
srcReg32b1 = _mm_loadu_si128((const __m128i *)src_ptr);
// filter the source buffer
srcRegFilt32b3 = _mm_shuffle_epi8(srcReg32b1, filt2Reg);
srcRegFilt32b2 = _mm_shuffle_epi8(srcReg32b1, filt3Reg);
// multiply 2 adjacent elements with the filter and add the result
srcRegFilt32b3 = _mm_maddubs_epi16(srcRegFilt32b3, secondFilters);
srcRegFilt32b2 = _mm_maddubs_epi16(srcRegFilt32b2, thirdFilters);
srcRegFilt32b1_1 = _mm_adds_epi16(srcRegFilt32b3, srcRegFilt32b2);
// reading stride of the next 16 bytes
// (part of it was being read by earlier read)
srcReg32b2 = _mm_loadu_si128((const __m128i *)(src_ptr + 8));
// filter the source buffer
srcRegFilt32b3 = _mm_shuffle_epi8(srcReg32b2, filt2Reg);
srcRegFilt32b2 = _mm_shuffle_epi8(srcReg32b2, filt3Reg);
// multiply 2 adjacent elements with the filter and add the result
srcRegFilt32b3 = _mm_maddubs_epi16(srcRegFilt32b3, secondFilters);
srcRegFilt32b2 = _mm_maddubs_epi16(srcRegFilt32b2, thirdFilters);
// add and saturate the results together
srcRegFilt32b2_1 = _mm_adds_epi16(srcRegFilt32b3, srcRegFilt32b2);
// shift by 6 bit each 16 bit
srcRegFilt32b1_1 = _mm_adds_epi16(srcRegFilt32b1_1, addFilterReg32);
srcRegFilt32b2_1 = _mm_adds_epi16(srcRegFilt32b2_1, addFilterReg32);
srcRegFilt32b1_1 = _mm_srai_epi16(srcRegFilt32b1_1, 6);
srcRegFilt32b2_1 = _mm_srai_epi16(srcRegFilt32b2_1, 6);
// shrink to 8 bit each 16 bits, the first lane contain the first
// convolve result and the second lane contain the second convolve result
srcRegFilt32b1_1 = _mm_packus_epi16(srcRegFilt32b1_1, srcRegFilt32b2_1);
src_ptr += src_pixels_per_line;
_mm_store_si128((__m128i *)output_ptr, srcRegFilt32b1_1);
output_ptr += output_pitch;
}
}
static void aom_filter_block1d16_v4_ssse3(
const uint8_t *src_ptr, ptrdiff_t src_pitch, uint8_t *output_ptr,
ptrdiff_t out_pitch, uint32_t output_height, const int16_t *filter) {
__m128i filtersReg;
__m128i srcReg2, srcReg3, srcReg4, srcReg5, srcReg6;
__m128i srcReg23_lo, srcReg23_hi, srcReg34_lo, srcReg34_hi;
__m128i srcReg45_lo, srcReg45_hi, srcReg56_lo, srcReg56_hi;
__m128i resReg23_lo, resReg34_lo, resReg45_lo, resReg56_lo;
__m128i resReg23_hi, resReg34_hi, resReg45_hi, resReg56_hi;
__m128i resReg23_45_lo, resReg34_56_lo, resReg23_45_hi, resReg34_56_hi;
__m128i resReg23_45, resReg34_56;
__m128i addFilterReg32, secondFilters, thirdFilters;
unsigned int i;
ptrdiff_t src_stride, dst_stride;
addFilterReg32 = _mm_set1_epi16(32);
filtersReg = _mm_loadu_si128((const __m128i *)filter);
// converting the 16 bit (short) to 8 bit (byte) and have the
// same data in both lanes of 128 bit register.
filtersReg = _mm_srai_epi16(filtersReg, 1);
filtersReg = _mm_packs_epi16(filtersReg, filtersReg);
// duplicate only the second 16 bits (third and forth byte)
// across 128 bit register
secondFilters = _mm_shuffle_epi8(filtersReg, _mm_set1_epi16(0x302u));
// duplicate only the third 16 bits (fifth and sixth byte)
// across 128 bit register
thirdFilters = _mm_shuffle_epi8(filtersReg, _mm_set1_epi16(0x504u));
// multiple the size of the source and destination stride by two
src_stride = src_pitch << 1;
dst_stride = out_pitch << 1;
srcReg2 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 2));
srcReg3 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 3));
srcReg23_lo = _mm_unpacklo_epi8(srcReg2, srcReg3);
srcReg23_hi = _mm_unpackhi_epi8(srcReg2, srcReg3);
srcReg4 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 4));
// have consecutive loads on the same 256 register
srcReg34_lo = _mm_unpacklo_epi8(srcReg3, srcReg4);
srcReg34_hi = _mm_unpackhi_epi8(srcReg3, srcReg4);
for (i = output_height; i > 1; i -= 2) {
srcReg5 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 5));
srcReg45_lo = _mm_unpacklo_epi8(srcReg4, srcReg5);
srcReg45_hi = _mm_unpackhi_epi8(srcReg4, srcReg5);
srcReg6 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 6));
srcReg56_lo = _mm_unpacklo_epi8(srcReg5, srcReg6);
srcReg56_hi = _mm_unpackhi_epi8(srcReg5, srcReg6);
// multiply 2 adjacent elements with the filter and add the result
resReg23_lo = _mm_maddubs_epi16(srcReg23_lo, secondFilters);
resReg34_lo = _mm_maddubs_epi16(srcReg34_lo, secondFilters);
resReg45_lo = _mm_maddubs_epi16(srcReg45_lo, thirdFilters);
resReg56_lo = _mm_maddubs_epi16(srcReg56_lo, thirdFilters);
// add and saturate the results together
resReg23_45_lo = _mm_adds_epi16(resReg23_lo, resReg45_lo);
resReg34_56_lo = _mm_adds_epi16(resReg34_lo, resReg56_lo);
// multiply 2 adjacent elements with the filter and add the result
resReg23_hi = _mm_maddubs_epi16(srcReg23_hi, secondFilters);
resReg34_hi = _mm_maddubs_epi16(srcReg34_hi, secondFilters);
resReg45_hi = _mm_maddubs_epi16(srcReg45_hi, thirdFilters);
resReg56_hi = _mm_maddubs_epi16(srcReg56_hi, thirdFilters);
// add and saturate the results together
resReg23_45_hi = _mm_adds_epi16(resReg23_hi, resReg45_hi);
resReg34_56_hi = _mm_adds_epi16(resReg34_hi, resReg56_hi);
// shift by 6 bit each 16 bit
resReg23_45_lo = _mm_adds_epi16(resReg23_45_lo, addFilterReg32);
resReg34_56_lo = _mm_adds_epi16(resReg34_56_lo, addFilterReg32);
resReg23_45_hi = _mm_adds_epi16(resReg23_45_hi, addFilterReg32);
resReg34_56_hi = _mm_adds_epi16(resReg34_56_hi, addFilterReg32);
resReg23_45_lo = _mm_srai_epi16(resReg23_45_lo, 6);
resReg34_56_lo = _mm_srai_epi16(resReg34_56_lo, 6);
resReg23_45_hi = _mm_srai_epi16(resReg23_45_hi, 6);
resReg34_56_hi = _mm_srai_epi16(resReg34_56_hi, 6);
// shrink to 8 bit each 16 bits, the first lane contain the first
// convolve result and the second lane contain the second convolve
// result
resReg23_45 = _mm_packus_epi16(resReg23_45_lo, resReg23_45_hi);
resReg34_56 = _mm_packus_epi16(resReg34_56_lo, resReg34_56_hi);
src_ptr += src_stride;
_mm_store_si128((__m128i *)output_ptr, (resReg23_45));
_mm_store_si128((__m128i *)(output_ptr + out_pitch), (resReg34_56));
output_ptr += dst_stride;
// save part of the registers for next strides
srcReg23_lo = srcReg45_lo;
srcReg34_lo = srcReg56_lo;
srcReg23_hi = srcReg45_hi;
srcReg34_hi = srcReg56_hi;
srcReg4 = srcReg6;
}
}
filter8_1dfunction aom_filter_block1d16_v8_ssse3;
filter8_1dfunction aom_filter_block1d16_h8_ssse3;
filter8_1dfunction aom_filter_block1d8_v8_ssse3;
@ -287,13 +749,6 @@ filter8_1dfunction aom_filter_block1d8_h8_ssse3;
filter8_1dfunction aom_filter_block1d4_v8_ssse3;
filter8_1dfunction aom_filter_block1d4_h8_ssse3;
#define aom_filter_block1d16_h4_ssse3 aom_filter_block1d16_h8_ssse3
#define aom_filter_block1d16_v4_ssse3 aom_filter_block1d16_v8_ssse3
#define aom_filter_block1d8_h4_ssse3 aom_filter_block1d8_h8_ssse3
#define aom_filter_block1d8_v4_ssse3 aom_filter_block1d8_v8_ssse3
#define aom_filter_block1d4_h4_ssse3 aom_filter_block1d4_h8_ssse3
#define aom_filter_block1d4_v4_ssse3 aom_filter_block1d4_v8_ssse3
filter8_1dfunction aom_filter_block1d16_v2_ssse3;
filter8_1dfunction aom_filter_block1d16_h2_ssse3;
filter8_1dfunction aom_filter_block1d8_v2_ssse3;

249
third_party/aom/aom_dsp/x86/avg_intrin_avx2.c vendored Normal file
View file

@ -0,0 +1,249 @@
/*
* Copyright (c) 2016, Alliance for Open Media. All rights reserved
*
* This source code is subject to the terms of the BSD 2 Clause License and
* the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
* was not distributed with this source code in the LICENSE file, you can
* obtain it at www.aomedia.org/license/software. If the Alliance for Open
* Media Patent License 1.0 was not distributed with this source code in the
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
*/
#include <immintrin.h>
#include "config/aom_dsp_rtcd.h"
#include "aom/aom_integer.h"
#include "aom_dsp/x86/bitdepth_conversion_avx2.h"
#include "aom_ports/mem.h"
static void hadamard_col8x2_avx2(__m256i *in, int iter) {
__m256i a0 = in[0];
__m256i a1 = in[1];
__m256i a2 = in[2];
__m256i a3 = in[3];
__m256i a4 = in[4];
__m256i a5 = in[5];
__m256i a6 = in[6];
__m256i a7 = in[7];
__m256i b0 = _mm256_add_epi16(a0, a1);
__m256i b1 = _mm256_sub_epi16(a0, a1);
__m256i b2 = _mm256_add_epi16(a2, a3);
__m256i b3 = _mm256_sub_epi16(a2, a3);
__m256i b4 = _mm256_add_epi16(a4, a5);
__m256i b5 = _mm256_sub_epi16(a4, a5);
__m256i b6 = _mm256_add_epi16(a6, a7);
__m256i b7 = _mm256_sub_epi16(a6, a7);
a0 = _mm256_add_epi16(b0, b2);
a1 = _mm256_add_epi16(b1, b3);
a2 = _mm256_sub_epi16(b0, b2);
a3 = _mm256_sub_epi16(b1, b3);
a4 = _mm256_add_epi16(b4, b6);
a5 = _mm256_add_epi16(b5, b7);
a6 = _mm256_sub_epi16(b4, b6);
a7 = _mm256_sub_epi16(b5, b7);
if (iter == 0) {
b0 = _mm256_add_epi16(a0, a4);
b7 = _mm256_add_epi16(a1, a5);
b3 = _mm256_add_epi16(a2, a6);
b4 = _mm256_add_epi16(a3, a7);
b2 = _mm256_sub_epi16(a0, a4);
b6 = _mm256_sub_epi16(a1, a5);
b1 = _mm256_sub_epi16(a2, a6);
b5 = _mm256_sub_epi16(a3, a7);
a0 = _mm256_unpacklo_epi16(b0, b1);
a1 = _mm256_unpacklo_epi16(b2, b3);
a2 = _mm256_unpackhi_epi16(b0, b1);
a3 = _mm256_unpackhi_epi16(b2, b3);
a4 = _mm256_unpacklo_epi16(b4, b5);
a5 = _mm256_unpacklo_epi16(b6, b7);
a6 = _mm256_unpackhi_epi16(b4, b5);
a7 = _mm256_unpackhi_epi16(b6, b7);
b0 = _mm256_unpacklo_epi32(a0, a1);
b1 = _mm256_unpacklo_epi32(a4, a5);
b2 = _mm256_unpackhi_epi32(a0, a1);
b3 = _mm256_unpackhi_epi32(a4, a5);
b4 = _mm256_unpacklo_epi32(a2, a3);
b5 = _mm256_unpacklo_epi32(a6, a7);
b6 = _mm256_unpackhi_epi32(a2, a3);
b7 = _mm256_unpackhi_epi32(a6, a7);
in[0] = _mm256_unpacklo_epi64(b0, b1);
in[1] = _mm256_unpackhi_epi64(b0, b1);
in[2] = _mm256_unpacklo_epi64(b2, b3);
in[3] = _mm256_unpackhi_epi64(b2, b3);
in[4] = _mm256_unpacklo_epi64(b4, b5);
in[5] = _mm256_unpackhi_epi64(b4, b5);
in[6] = _mm256_unpacklo_epi64(b6, b7);
in[7] = _mm256_unpackhi_epi64(b6, b7);
} else {
in[0] = _mm256_add_epi16(a0, a4);
in[7] = _mm256_add_epi16(a1, a5);
in[3] = _mm256_add_epi16(a2, a6);
in[4] = _mm256_add_epi16(a3, a7);
in[2] = _mm256_sub_epi16(a0, a4);
in[6] = _mm256_sub_epi16(a1, a5);
in[1] = _mm256_sub_epi16(a2, a6);
in[5] = _mm256_sub_epi16(a3, a7);
}
}
static void hadamard_8x8x2_avx2(const int16_t *src_diff, ptrdiff_t src_stride,
int16_t *coeff) {
__m256i src[8];
src[0] = _mm256_loadu_si256((const __m256i *)src_diff);
src[1] = _mm256_loadu_si256((const __m256i *)(src_diff += src_stride));
src[2] = _mm256_loadu_si256((const __m256i *)(src_diff += src_stride));
src[3] = _mm256_loadu_si256((const __m256i *)(src_diff += src_stride));
src[4] = _mm256_loadu_si256((const __m256i *)(src_diff += src_stride));
src[5] = _mm256_loadu_si256((const __m256i *)(src_diff += src_stride));
src[6] = _mm256_loadu_si256((const __m256i *)(src_diff += src_stride));
src[7] = _mm256_loadu_si256((const __m256i *)(src_diff += src_stride));
hadamard_col8x2_avx2(src, 0);
hadamard_col8x2_avx2(src, 1);
_mm256_storeu_si256((__m256i *)coeff,
_mm256_permute2x128_si256(src[0], src[1], 0x20));
coeff += 16;
_mm256_storeu_si256((__m256i *)coeff,
_mm256_permute2x128_si256(src[2], src[3], 0x20));
coeff += 16;
_mm256_storeu_si256((__m256i *)coeff,
_mm256_permute2x128_si256(src[4], src[5], 0x20));
coeff += 16;
_mm256_storeu_si256((__m256i *)coeff,
_mm256_permute2x128_si256(src[6], src[7], 0x20));
coeff += 16;
_mm256_storeu_si256((__m256i *)coeff,
_mm256_permute2x128_si256(src[0], src[1], 0x31));
coeff += 16;
_mm256_storeu_si256((__m256i *)coeff,
_mm256_permute2x128_si256(src[2], src[3], 0x31));
coeff += 16;
_mm256_storeu_si256((__m256i *)coeff,
_mm256_permute2x128_si256(src[4], src[5], 0x31));
coeff += 16;
_mm256_storeu_si256((__m256i *)coeff,
_mm256_permute2x128_si256(src[6], src[7], 0x31));
}
static INLINE void hadamard_16x16_avx2(const int16_t *src_diff,
ptrdiff_t src_stride, tran_low_t *coeff,
int is_final) {
DECLARE_ALIGNED(32, int16_t, temp_coeff[16 * 16]);
int16_t *t_coeff = temp_coeff;
int16_t *coeff16 = (int16_t *)coeff;
int idx;
for (idx = 0; idx < 2; ++idx) {
const int16_t *src_ptr = src_diff + idx * 8 * src_stride;
hadamard_8x8x2_avx2(src_ptr, src_stride, t_coeff + (idx * 64 * 2));
}
for (idx = 0; idx < 64; idx += 16) {
const __m256i coeff0 = _mm256_loadu_si256((const __m256i *)t_coeff);
const __m256i coeff1 = _mm256_loadu_si256((const __m256i *)(t_coeff + 64));
const __m256i coeff2 = _mm256_loadu_si256((const __m256i *)(t_coeff + 128));
const __m256i coeff3 = _mm256_loadu_si256((const __m256i *)(t_coeff + 192));
__m256i b0 = _mm256_add_epi16(coeff0, coeff1);
__m256i b1 = _mm256_sub_epi16(coeff0, coeff1);
__m256i b2 = _mm256_add_epi16(coeff2, coeff3);
__m256i b3 = _mm256_sub_epi16(coeff2, coeff3);
b0 = _mm256_srai_epi16(b0, 1);
b1 = _mm256_srai_epi16(b1, 1);
b2 = _mm256_srai_epi16(b2, 1);
b3 = _mm256_srai_epi16(b3, 1);
if (is_final) {
store_tran_low(_mm256_add_epi16(b0, b2), coeff);
store_tran_low(_mm256_add_epi16(b1, b3), coeff + 64);
store_tran_low(_mm256_sub_epi16(b0, b2), coeff + 128);
store_tran_low(_mm256_sub_epi16(b1, b3), coeff + 192);
coeff += 16;
} else {
_mm256_storeu_si256((__m256i *)coeff16, _mm256_add_epi16(b0, b2));
_mm256_storeu_si256((__m256i *)(coeff16 + 64), _mm256_add_epi16(b1, b3));
_mm256_storeu_si256((__m256i *)(coeff16 + 128), _mm256_sub_epi16(b0, b2));
_mm256_storeu_si256((__m256i *)(coeff16 + 192), _mm256_sub_epi16(b1, b3));
coeff16 += 16;
}
t_coeff += 16;
}
}
void aom_hadamard_16x16_avx2(const int16_t *src_diff, ptrdiff_t src_stride,
tran_low_t *coeff) {
hadamard_16x16_avx2(src_diff, src_stride, coeff, 1);
}
void aom_hadamard_32x32_avx2(const int16_t *src_diff, ptrdiff_t src_stride,
tran_low_t *coeff) {
// For high bitdepths, it is unnecessary to store_tran_low
// (mult/unpack/store), then load_tran_low (load/pack) the same memory in the
// next stage. Output to an intermediate buffer first, then store_tran_low()
// in the final stage.
DECLARE_ALIGNED(32, int16_t, temp_coeff[32 * 32]);
int16_t *t_coeff = temp_coeff;
int idx;
for (idx = 0; idx < 4; ++idx) {
// src_diff: 9 bit, dynamic range [-255, 255]
const int16_t *src_ptr =
src_diff + (idx >> 1) * 16 * src_stride + (idx & 0x01) * 16;
hadamard_16x16_avx2(src_ptr, src_stride,
(tran_low_t *)(t_coeff + idx * 256), 0);
}
for (idx = 0; idx < 256; idx += 16) {
const __m256i coeff0 = _mm256_loadu_si256((const __m256i *)t_coeff);
const __m256i coeff1 = _mm256_loadu_si256((const __m256i *)(t_coeff + 256));
const __m256i coeff2 = _mm256_loadu_si256((const __m256i *)(t_coeff + 512));
const __m256i coeff3 = _mm256_loadu_si256((const __m256i *)(t_coeff + 768));
__m256i b0 = _mm256_add_epi16(coeff0, coeff1);
__m256i b1 = _mm256_sub_epi16(coeff0, coeff1);
__m256i b2 = _mm256_add_epi16(coeff2, coeff3);
__m256i b3 = _mm256_sub_epi16(coeff2, coeff3);
b0 = _mm256_srai_epi16(b0, 2);
b1 = _mm256_srai_epi16(b1, 2);
b2 = _mm256_srai_epi16(b2, 2);
b3 = _mm256_srai_epi16(b3, 2);
store_tran_low(_mm256_add_epi16(b0, b2), coeff);
store_tran_low(_mm256_add_epi16(b1, b3), coeff + 256);
store_tran_low(_mm256_sub_epi16(b0, b2), coeff + 512);
store_tran_low(_mm256_sub_epi16(b1, b3), coeff + 768);
coeff += 16;
t_coeff += 16;
}
}
int aom_satd_avx2(const tran_low_t *coeff, int length) {
const __m256i one = _mm256_set1_epi16(1);
__m256i accum = _mm256_setzero_si256();
int i;
for (i = 0; i < length; i += 16) {
const __m256i src_line = load_tran_low(coeff);
const __m256i abs = _mm256_abs_epi16(src_line);
const __m256i sum = _mm256_madd_epi16(abs, one);
accum = _mm256_add_epi32(accum, sum);
coeff += 16;
}
{ // 32 bit horizontal add
const __m256i a = _mm256_srli_si256(accum, 8);
const __m256i b = _mm256_add_epi32(accum, a);
const __m256i c = _mm256_srli_epi64(b, 32);
const __m256i d = _mm256_add_epi32(b, c);
const __m128i accum_128 = _mm_add_epi32(_mm256_castsi256_si128(d),
_mm256_extractf128_si256(d, 1));
return _mm_cvtsi128_si32(accum_128);
}
}

285
third_party/aom/aom_dsp/x86/avg_intrin_sse2.c vendored Normal file
View file

@ -0,0 +1,285 @@
/*
* Copyright (c) 2016, Alliance for Open Media. All rights reserved
*
* This source code is subject to the terms of the BSD 2 Clause License and
* the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
* was not distributed with this source code in the LICENSE file, you can
* obtain it at www.aomedia.org/license/software. If the Alliance for Open
* Media Patent License 1.0 was not distributed with this source code in the
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
*/
#include <immintrin.h>
#include "config/aom_dsp_rtcd.h"
#include "aom/aom_integer.h"
#include "aom_dsp/x86/bitdepth_conversion_sse2.h"
#include "aom_ports/mem.h"
static void hadamard_col8_sse2(__m128i *in, int iter) {
__m128i a0 = in[0];
__m128i a1 = in[1];
__m128i a2 = in[2];
__m128i a3 = in[3];
__m128i a4 = in[4];
__m128i a5 = in[5];
__m128i a6 = in[6];
__m128i a7 = in[7];
__m128i b0 = _mm_add_epi16(a0, a1);
__m128i b1 = _mm_sub_epi16(a0, a1);
__m128i b2 = _mm_add_epi16(a2, a3);
__m128i b3 = _mm_sub_epi16(a2, a3);
__m128i b4 = _mm_add_epi16(a4, a5);
__m128i b5 = _mm_sub_epi16(a4, a5);
__m128i b6 = _mm_add_epi16(a6, a7);
__m128i b7 = _mm_sub_epi16(a6, a7);
a0 = _mm_add_epi16(b0, b2);
a1 = _mm_add_epi16(b1, b3);
a2 = _mm_sub_epi16(b0, b2);
a3 = _mm_sub_epi16(b1, b3);
a4 = _mm_add_epi16(b4, b6);
a5 = _mm_add_epi16(b5, b7);
a6 = _mm_sub_epi16(b4, b6);
a7 = _mm_sub_epi16(b5, b7);
if (iter == 0) {
b0 = _mm_add_epi16(a0, a4);
b7 = _mm_add_epi16(a1, a5);
b3 = _mm_add_epi16(a2, a6);
b4 = _mm_add_epi16(a3, a7);
b2 = _mm_sub_epi16(a0, a4);
b6 = _mm_sub_epi16(a1, a5);
b1 = _mm_sub_epi16(a2, a6);
b5 = _mm_sub_epi16(a3, a7);
a0 = _mm_unpacklo_epi16(b0, b1);
a1 = _mm_unpacklo_epi16(b2, b3);
a2 = _mm_unpackhi_epi16(b0, b1);
a3 = _mm_unpackhi_epi16(b2, b3);
a4 = _mm_unpacklo_epi16(b4, b5);
a5 = _mm_unpacklo_epi16(b6, b7);
a6 = _mm_unpackhi_epi16(b4, b5);
a7 = _mm_unpackhi_epi16(b6, b7);
b0 = _mm_unpacklo_epi32(a0, a1);
b1 = _mm_unpacklo_epi32(a4, a5);
b2 = _mm_unpackhi_epi32(a0, a1);
b3 = _mm_unpackhi_epi32(a4, a5);
b4 = _mm_unpacklo_epi32(a2, a3);
b5 = _mm_unpacklo_epi32(a6, a7);
b6 = _mm_unpackhi_epi32(a2, a3);
b7 = _mm_unpackhi_epi32(a6, a7);
in[0] = _mm_unpacklo_epi64(b0, b1);
in[1] = _mm_unpackhi_epi64(b0, b1);
in[2] = _mm_unpacklo_epi64(b2, b3);
in[3] = _mm_unpackhi_epi64(b2, b3);
in[4] = _mm_unpacklo_epi64(b4, b5);
in[5] = _mm_unpackhi_epi64(b4, b5);
in[6] = _mm_unpacklo_epi64(b6, b7);
in[7] = _mm_unpackhi_epi64(b6, b7);
} else {
in[0] = _mm_add_epi16(a0, a4);
in[7] = _mm_add_epi16(a1, a5);
in[3] = _mm_add_epi16(a2, a6);
in[4] = _mm_add_epi16(a3, a7);
in[2] = _mm_sub_epi16(a0, a4);
in[6] = _mm_sub_epi16(a1, a5);
in[1] = _mm_sub_epi16(a2, a6);
in[5] = _mm_sub_epi16(a3, a7);
}
}
static INLINE void hadamard_8x8_sse2(const int16_t *src_diff,
ptrdiff_t src_stride, tran_low_t *coeff,
int is_final) {
__m128i src[8];
src[0] = _mm_load_si128((const __m128i *)src_diff);
src[1] = _mm_load_si128((const __m128i *)(src_diff += src_stride));
src[2] = _mm_load_si128((const __m128i *)(src_diff += src_stride));
src[3] = _mm_load_si128((const __m128i *)(src_diff += src_stride));
src[4] = _mm_load_si128((const __m128i *)(src_diff += src_stride));
src[5] = _mm_load_si128((const __m128i *)(src_diff += src_stride));
src[6] = _mm_load_si128((const __m128i *)(src_diff += src_stride));
src[7] = _mm_load_si128((const __m128i *)(src_diff += src_stride));
hadamard_col8_sse2(src, 0);
hadamard_col8_sse2(src, 1);
if (is_final) {
store_tran_low(src[0], coeff);
coeff += 8;
store_tran_low(src[1], coeff);
coeff += 8;
store_tran_low(src[2], coeff);
coeff += 8;
store_tran_low(src[3], coeff);
coeff += 8;
store_tran_low(src[4], coeff);
coeff += 8;
store_tran_low(src[5], coeff);
coeff += 8;
store_tran_low(src[6], coeff);
coeff += 8;
store_tran_low(src[7], coeff);
} else {
int16_t *coeff16 = (int16_t *)coeff;
_mm_store_si128((__m128i *)coeff16, src[0]);
coeff16 += 8;
_mm_store_si128((__m128i *)coeff16, src[1]);
coeff16 += 8;
_mm_store_si128((__m128i *)coeff16, src[2]);
coeff16 += 8;
_mm_store_si128((__m128i *)coeff16, src[3]);
coeff16 += 8;
_mm_store_si128((__m128i *)coeff16, src[4]);
coeff16 += 8;
_mm_store_si128((__m128i *)coeff16, src[5]);
coeff16 += 8;
_mm_store_si128((__m128i *)coeff16, src[6]);
coeff16 += 8;
_mm_store_si128((__m128i *)coeff16, src[7]);
}
}
void aom_hadamard_8x8_sse2(const int16_t *src_diff, ptrdiff_t src_stride,
tran_low_t *coeff) {
hadamard_8x8_sse2(src_diff, src_stride, coeff, 1);
}
static INLINE void hadamard_16x16_sse2(const int16_t *src_diff,
ptrdiff_t src_stride, tran_low_t *coeff,
int is_final) {
// For high bitdepths, it is unnecessary to store_tran_low
// (mult/unpack/store), then load_tran_low (load/pack) the same memory in the
// next stage. Output to an intermediate buffer first, then store_tran_low()
// in the final stage.
DECLARE_ALIGNED(32, int16_t, temp_coeff[16 * 16]);
int16_t *t_coeff = temp_coeff;
int16_t *coeff16 = (int16_t *)coeff;
int idx;
for (idx = 0; idx < 4; ++idx) {
const int16_t *src_ptr =
src_diff + (idx >> 1) * 8 * src_stride + (idx & 0x01) * 8;
hadamard_8x8_sse2(src_ptr, src_stride, (tran_low_t *)(t_coeff + idx * 64),
0);
}
for (idx = 0; idx < 64; idx += 8) {
__m128i coeff0 = _mm_load_si128((const __m128i *)t_coeff);
__m128i coeff1 = _mm_load_si128((const __m128i *)(t_coeff + 64));
__m128i coeff2 = _mm_load_si128((const __m128i *)(t_coeff + 128));
__m128i coeff3 = _mm_load_si128((const __m128i *)(t_coeff + 192));
__m128i b0 = _mm_add_epi16(coeff0, coeff1);
__m128i b1 = _mm_sub_epi16(coeff0, coeff1);
__m128i b2 = _mm_add_epi16(coeff2, coeff3);
__m128i b3 = _mm_sub_epi16(coeff2, coeff3);
b0 = _mm_srai_epi16(b0, 1);
b1 = _mm_srai_epi16(b1, 1);
b2 = _mm_srai_epi16(b2, 1);
b3 = _mm_srai_epi16(b3, 1);
coeff0 = _mm_add_epi16(b0, b2);
coeff1 = _mm_add_epi16(b1, b3);
coeff2 = _mm_sub_epi16(b0, b2);
coeff3 = _mm_sub_epi16(b1, b3);
if (is_final) {
store_tran_low(coeff0, coeff);
store_tran_low(coeff1, coeff + 64);
store_tran_low(coeff2, coeff + 128);
store_tran_low(coeff3, coeff + 192);
coeff += 8;
} else {
_mm_store_si128((__m128i *)coeff16, coeff0);
_mm_store_si128((__m128i *)(coeff16 + 64), coeff1);
_mm_store_si128((__m128i *)(coeff16 + 128), coeff2);
_mm_store_si128((__m128i *)(coeff16 + 192), coeff3);
coeff16 += 8;
}
t_coeff += 8;
}
}
void aom_hadamard_16x16_sse2(const int16_t *src_diff, ptrdiff_t src_stride,
tran_low_t *coeff) {
hadamard_16x16_sse2(src_diff, src_stride, coeff, 1);
}
void aom_hadamard_32x32_sse2(const int16_t *src_diff, ptrdiff_t src_stride,
tran_low_t *coeff) {
// For high bitdepths, it is unnecessary to store_tran_low
// (mult/unpack/store), then load_tran_low (load/pack) the same memory in the
// next stage. Output to an intermediate buffer first, then store_tran_low()
// in the final stage.
DECLARE_ALIGNED(32, int16_t, temp_coeff[32 * 32]);
int16_t *t_coeff = temp_coeff;
int idx;
for (idx = 0; idx < 4; ++idx) {
const int16_t *src_ptr =
src_diff + (idx >> 1) * 16 * src_stride + (idx & 0x01) * 16;
hadamard_16x16_sse2(src_ptr, src_stride,
(tran_low_t *)(t_coeff + idx * 256), 0);
}
for (idx = 0; idx < 256; idx += 8) {
__m128i coeff0 = _mm_load_si128((const __m128i *)t_coeff);
__m128i coeff1 = _mm_load_si128((const __m128i *)(t_coeff + 256));
__m128i coeff2 = _mm_load_si128((const __m128i *)(t_coeff + 512));
__m128i coeff3 = _mm_load_si128((const __m128i *)(t_coeff + 768));
__m128i b0 = _mm_add_epi16(coeff0, coeff1);
__m128i b1 = _mm_sub_epi16(coeff0, coeff1);
__m128i b2 = _mm_add_epi16(coeff2, coeff3);
__m128i b3 = _mm_sub_epi16(coeff2, coeff3);
b0 = _mm_srai_epi16(b0, 2);
b1 = _mm_srai_epi16(b1, 2);
b2 = _mm_srai_epi16(b2, 2);
b3 = _mm_srai_epi16(b3, 2);
coeff0 = _mm_add_epi16(b0, b2);
coeff1 = _mm_add_epi16(b1, b3);
store_tran_low(coeff0, coeff);
store_tran_low(coeff1, coeff + 256);
coeff2 = _mm_sub_epi16(b0, b2);
coeff3 = _mm_sub_epi16(b1, b3);
store_tran_low(coeff2, coeff + 512);
store_tran_low(coeff3, coeff + 768);
coeff += 8;
t_coeff += 8;
}
}
int aom_satd_sse2(const tran_low_t *coeff, int length) {
int i;
const __m128i zero = _mm_setzero_si128();
__m128i accum = zero;
for (i = 0; i < length; i += 8) {
const __m128i src_line = load_tran_low(coeff);
const __m128i inv = _mm_sub_epi16(zero, src_line);
const __m128i abs = _mm_max_epi16(src_line, inv); // abs(src_line)
const __m128i abs_lo = _mm_unpacklo_epi16(abs, zero);
const __m128i abs_hi = _mm_unpackhi_epi16(abs, zero);
const __m128i sum = _mm_add_epi32(abs_lo, abs_hi);
accum = _mm_add_epi32(accum, sum);
coeff += 8;
}
{ // cascading summation of accum
__m128i hi = _mm_srli_si128(accum, 8);
accum = _mm_add_epi32(accum, hi);
hi = _mm_srli_epi64(accum, 32);
accum = _mm_add_epi32(accum, hi);
}
return _mm_cvtsi128_si32(accum);
}

32
third_party/aom/aom_dsp/x86/bitdepth_conversion_avx2.h vendored Normal file
View file

@ -0,0 +1,32 @@
/*
* Copyright (c) 2016, Alliance for Open Media. All rights reserved
*
* This source code is subject to the terms of the BSD 2 Clause License and
* the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
* was not distributed with this source code in the LICENSE file, you can
* obtain it at www.aomedia.org/license/software. If the Alliance for Open
* Media Patent License 1.0 was not distributed with this source code in the
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
*/
#include <immintrin.h>
#include "config/aom_config.h"
#include "aom/aom_integer.h"
#include "aom_dsp/aom_dsp_common.h"
static INLINE __m256i load_tran_low(const tran_low_t *a) {
const __m256i a_low = _mm256_loadu_si256((const __m256i *)a);
const __m256i a_high = _mm256_loadu_si256((const __m256i *)(a + 8));
return _mm256_packs_epi32(a_low, a_high);
}
static INLINE void store_tran_low(__m256i a, tran_low_t *b) {
const __m256i one = _mm256_set1_epi16(1);
const __m256i a_hi = _mm256_mulhi_epi16(a, one);
const __m256i a_lo = _mm256_mullo_epi16(a, one);
const __m256i a_1 = _mm256_unpacklo_epi16(a_lo, a_hi);
const __m256i a_2 = _mm256_unpackhi_epi16(a_lo, a_hi);
_mm256_storeu_si256((__m256i *)b, a_1);
_mm256_storeu_si256((__m256i *)(b + 8), a_2);
}

35
third_party/aom/aom_dsp/x86/bitdepth_conversion_sse2.h vendored Normal file
View file

@ -0,0 +1,35 @@
/*
* Copyright (c) 2016, Alliance for Open Media. All rights reserved
*
* This source code is subject to the terms of the BSD 2 Clause License and
* the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
* was not distributed with this source code in the LICENSE file, you can
* obtain it at www.aomedia.org/license/software. If the Alliance for Open
* Media Patent License 1.0 was not distributed with this source code in the
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
*/
#include <xmmintrin.h>
#include "config/aom_config.h"
#include "aom/aom_integer.h"
#include "aom_dsp/aom_dsp_common.h"
// Load 8 16 bit values. If the source is 32 bits then pack down with
// saturation.
static INLINE __m128i load_tran_low(const tran_low_t *a) {
const __m128i a_low = _mm_load_si128((const __m128i *)a);
return _mm_packs_epi32(a_low, *(const __m128i *)(a + 4));
}
// Store 8 16 bit values. If the destination is 32 bits then sign extend the
// values by multiplying by 1.
static INLINE void store_tran_low(__m128i a, tran_low_t *b) {
const __m128i one = _mm_set1_epi16(1);
const __m128i a_hi = _mm_mulhi_epi16(a, one);
const __m128i a_lo = _mm_mullo_epi16(a, one);
const __m128i a_1 = _mm_unpacklo_epi16(a_lo, a_hi);
const __m128i a_2 = _mm_unpackhi_epi16(a_lo, a_hi);
_mm_store_si128((__m128i *)(b), a_1);
_mm_store_si128((__m128i *)(b + 4), a_2);
}

472
third_party/aom/aom_dsp/x86/blend_a64_mask_avx2.c vendored
View file

@ -898,3 +898,475 @@ void aom_blend_a64_mask_avx2(uint8_t *dst, uint32_t dst_stride,
}
}
}
//////////////////////////////////////////////////////////////////////////////
// aom_highbd_blend_a64_d16_mask_avx2()
//////////////////////////////////////////////////////////////////////////////
static INLINE void highbd_blend_a64_d16_mask_w4_avx2(
uint16_t *dst, int dst_stride, const CONV_BUF_TYPE *src0, int src0_stride,
const CONV_BUF_TYPE *src1, int src1_stride, const __m256i *mask0,
const __m256i *round_offset, int shift, const __m256i *clip_low,
const __m256i *clip_high, const __m256i *mask_max) {
// Load 4x u16 pixels from each of 4 rows from each source
const __m256i s0 = _mm256_set_epi64x(*(uint64_t *)(src0 + 3 * src0_stride),
*(uint64_t *)(src0 + 2 * src0_stride),
*(uint64_t *)(src0 + 1 * src0_stride),
*(uint64_t *)(src0 + 0 * src0_stride));
const __m256i s1 = _mm256_set_epi64x(*(uint64_t *)(src1 + 3 * src1_stride),
*(uint64_t *)(src1 + 2 * src1_stride),
*(uint64_t *)(src1 + 1 * src1_stride),
*(uint64_t *)(src1 + 0 * src1_stride));
// Generate the inverse mask
const __m256i mask1 = _mm256_sub_epi16(*mask_max, *mask0);
// Multiply each mask by the respective source
const __m256i mul0_highs = _mm256_mulhi_epu16(*mask0, s0);
const __m256i mul0_lows = _mm256_mullo_epi16(*mask0, s0);
const __m256i mul0h = _mm256_unpackhi_epi16(mul0_lows, mul0_highs);
const __m256i mul0l = _mm256_unpacklo_epi16(mul0_lows, mul0_highs);
// Note that AVX2 unpack orders 64-bit words as [3 1] [2 0] to keep within
// lanes Later, packs does the same again which cancels this out with no need
// for a permute. The intermediate values being reordered makes no difference
const __m256i mul1_highs = _mm256_mulhi_epu16(mask1, s1);
const __m256i mul1_lows = _mm256_mullo_epi16(mask1, s1);
const __m256i mul1h = _mm256_unpackhi_epi16(mul1_lows, mul1_highs);
const __m256i mul1l = _mm256_unpacklo_epi16(mul1_lows, mul1_highs);
const __m256i sumh = _mm256_add_epi32(mul0h, mul1h);
const __m256i suml = _mm256_add_epi32(mul0l, mul1l);
const __m256i roundh =
_mm256_srai_epi32(_mm256_sub_epi32(sumh, *round_offset), shift);
const __m256i roundl =
_mm256_srai_epi32(_mm256_sub_epi32(suml, *round_offset), shift);
const __m256i pack = _mm256_packs_epi32(roundl, roundh);
const __m256i clip =
_mm256_min_epi16(_mm256_max_epi16(pack, *clip_low), *clip_high);
// _mm256_extract_epi64 doesn't exist on x86, so do it the old-fashioned way:
const __m128i cliph = _mm256_extracti128_si256(clip, 1);
xx_storel_64(dst + 3 * dst_stride, _mm_srli_si128(cliph, 8));
xx_storel_64(dst + 2 * dst_stride, cliph);
const __m128i clipl = _mm256_castsi256_si128(clip);
xx_storel_64(dst + 1 * dst_stride, _mm_srli_si128(clipl, 8));
xx_storel_64(dst + 0 * dst_stride, clipl);
}
static INLINE void highbd_blend_a64_d16_mask_subw0_subh0_w4_avx2(
uint16_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0,
uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride,
const uint8_t *mask, uint32_t mask_stride, int h,
const __m256i *round_offset, int shift, const __m256i *clip_low,
const __m256i *clip_high, const __m256i *mask_max) {
do {
// Load 8x u8 pixels from each of 4 rows of the mask, pad each to u16
const __m128i mask08 = _mm_set_epi32(*(uint32_t *)(mask + 3 * mask_stride),
*(uint32_t *)(mask + 2 * mask_stride),
*(uint32_t *)(mask + 1 * mask_stride),
*(uint32_t *)(mask + 0 * mask_stride));
const __m256i mask0 = _mm256_cvtepu8_epi16(mask08);
highbd_blend_a64_d16_mask_w4_avx2(dst, dst_stride, src0, src0_stride, src1,
src1_stride, &mask0, round_offset, shift,
clip_low, clip_high, mask_max);
dst += dst_stride * 4;
src0 += src0_stride * 4;
src1 += src1_stride * 4;
mask += mask_stride * 4;
} while (h -= 4);
}
static INLINE void highbd_blend_a64_d16_mask_subw1_subh1_w4_avx2(
uint16_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0,
uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride,
const uint8_t *mask, uint32_t mask_stride, int h,
const __m256i *round_offset, int shift, const __m256i *clip_low,
const __m256i *clip_high, const __m256i *mask_max) {
const __m256i one_b = _mm256_set1_epi8(1);
const __m256i two_w = _mm256_set1_epi16(2);
do {
// Load 8 pixels from each of 8 rows of mask,
// (saturating) add together rows then use madd to add adjacent pixels
// Finally, divide each value by 4 (with rounding)
const __m256i m0246 =
_mm256_set_epi64x(*(uint64_t *)(mask + 6 * mask_stride),
*(uint64_t *)(mask + 4 * mask_stride),
*(uint64_t *)(mask + 2 * mask_stride),
*(uint64_t *)(mask + 0 * mask_stride));
const __m256i m1357 =
_mm256_set_epi64x(*(uint64_t *)(mask + 7 * mask_stride),
*(uint64_t *)(mask + 5 * mask_stride),
*(uint64_t *)(mask + 3 * mask_stride),
*(uint64_t *)(mask + 1 * mask_stride));
const __m256i addrows = _mm256_adds_epu8(m0246, m1357);
const __m256i adjacent = _mm256_maddubs_epi16(addrows, one_b);
const __m256i mask0 =
_mm256_srli_epi16(_mm256_add_epi16(adjacent, two_w), 2);
highbd_blend_a64_d16_mask_w4_avx2(dst, dst_stride, src0, src0_stride, src1,
src1_stride, &mask0, round_offset, shift,
clip_low, clip_high, mask_max);
dst += dst_stride * 4;
src0 += src0_stride * 4;
src1 += src1_stride * 4;
mask += mask_stride * 8;
} while (h -= 4);
}
static INLINE void highbd_blend_a64_d16_mask_w8_avx2(
uint16_t *dst, int dst_stride, const CONV_BUF_TYPE *src0, int src0_stride,
const CONV_BUF_TYPE *src1, int src1_stride, const __m256i *mask0a,
const __m256i *mask0b, const __m256i *round_offset, int shift,
const __m256i *clip_low, const __m256i *clip_high,
const __m256i *mask_max) {
// Load 8x u16 pixels from each of 4 rows from each source
const __m256i s0a =
yy_loadu2_128(src0 + 0 * src0_stride, src0 + 1 * src0_stride);
const __m256i s0b =
yy_loadu2_128(src0 + 2 * src0_stride, src0 + 3 * src0_stride);
const __m256i s1a =
yy_loadu2_128(src1 + 0 * src1_stride, src1 + 1 * src1_stride);
const __m256i s1b =
yy_loadu2_128(src1 + 2 * src1_stride, src1 + 3 * src1_stride);
// Generate inverse masks
const __m256i mask1a = _mm256_sub_epi16(*mask_max, *mask0a);
const __m256i mask1b = _mm256_sub_epi16(*mask_max, *mask0b);
// Multiply sources by respective masks
const __m256i mul0a_highs = _mm256_mulhi_epu16(*mask0a, s0a);
const __m256i mul0a_lows = _mm256_mullo_epi16(*mask0a, s0a);
const __m256i mul0ah = _mm256_unpackhi_epi16(mul0a_lows, mul0a_highs);
const __m256i mul0al = _mm256_unpacklo_epi16(mul0a_lows, mul0a_highs);
// Note that AVX2 unpack orders 64-bit words as [3 1] [2 0] to keep within
// lanes Later, packs does the same again which cancels this out with no need
// for a permute. The intermediate values being reordered makes no difference
const __m256i mul1a_highs = _mm256_mulhi_epu16(mask1a, s1a);
const __m256i mul1a_lows = _mm256_mullo_epi16(mask1a, s1a);
const __m256i mul1ah = _mm256_unpackhi_epi16(mul1a_lows, mul1a_highs);
const __m256i mul1al = _mm256_unpacklo_epi16(mul1a_lows, mul1a_highs);
const __m256i sumah = _mm256_add_epi32(mul0ah, mul1ah);
const __m256i sumal = _mm256_add_epi32(mul0al, mul1al);
const __m256i mul0b_highs = _mm256_mulhi_epu16(*mask0b, s0b);
const __m256i mul0b_lows = _mm256_mullo_epi16(*mask0b, s0b);
const __m256i mul0bh = _mm256_unpackhi_epi16(mul0b_lows, mul0b_highs);
const __m256i mul0bl = _mm256_unpacklo_epi16(mul0b_lows, mul0b_highs);
const __m256i mul1b_highs = _mm256_mulhi_epu16(mask1b, s1b);
const __m256i mul1b_lows = _mm256_mullo_epi16(mask1b, s1b);
const __m256i mul1bh = _mm256_unpackhi_epi16(mul1b_lows, mul1b_highs);
const __m256i mul1bl = _mm256_unpacklo_epi16(mul1b_lows, mul1b_highs);
const __m256i sumbh = _mm256_add_epi32(mul0bh, mul1bh);
const __m256i sumbl = _mm256_add_epi32(mul0bl, mul1bl);
// Divide down each result, with rounding
const __m256i roundah =
_mm256_srai_epi32(_mm256_sub_epi32(sumah, *round_offset), shift);
const __m256i roundal =
_mm256_srai_epi32(_mm256_sub_epi32(sumal, *round_offset), shift);
const __m256i roundbh =
_mm256_srai_epi32(_mm256_sub_epi32(sumbh, *round_offset), shift);
const __m256i roundbl =
_mm256_srai_epi32(_mm256_sub_epi32(sumbl, *round_offset), shift);
// Pack each i32 down to an i16 with saturation, then clip to valid range
const __m256i packa = _mm256_packs_epi32(roundal, roundah);
const __m256i clipa =
_mm256_min_epi16(_mm256_max_epi16(packa, *clip_low), *clip_high);
const __m256i packb = _mm256_packs_epi32(roundbl, roundbh);
const __m256i clipb =
_mm256_min_epi16(_mm256_max_epi16(packb, *clip_low), *clip_high);
// Store 8x u16 pixels to each of 4 rows in the destination
yy_storeu2_128(dst + 0 * dst_stride, dst + 1 * dst_stride, clipa);
yy_storeu2_128(dst + 2 * dst_stride, dst + 3 * dst_stride, clipb);
}
static INLINE void highbd_blend_a64_d16_mask_subw0_subh0_w8_avx2(
uint16_t *dst, int dst_stride, const CONV_BUF_TYPE *src0, int src0_stride,
const CONV_BUF_TYPE *src1, int src1_stride, const uint8_t *mask,
int mask_stride, int h, const __m256i *round_offset, int shift,
const __m256i *clip_low, const __m256i *clip_high,
const __m256i *mask_max) {
do {
// Load 8x u8 pixels from each of 4 rows in the mask
const __m128i mask0a8 =
_mm_set_epi64x(*(uint64_t *)mask, *(uint64_t *)(mask + mask_stride));
const __m128i mask0b8 =
_mm_set_epi64x(*(uint64_t *)(mask + 2 * mask_stride),
*(uint64_t *)(mask + 3 * mask_stride));
const __m256i mask0a = _mm256_cvtepu8_epi16(mask0a8);
const __m256i mask0b = _mm256_cvtepu8_epi16(mask0b8);
highbd_blend_a64_d16_mask_w8_avx2(
dst, dst_stride, src0, src0_stride, src1, src1_stride, &mask0a, &mask0b,
round_offset, shift, clip_low, clip_high, mask_max);
dst += dst_stride * 4;
src0 += src0_stride * 4;
src1 += src1_stride * 4;
mask += mask_stride * 4;
} while (h -= 4);
}
static INLINE void highbd_blend_a64_d16_mask_subw1_subh1_w8_avx2(
uint16_t *dst, int dst_stride, const CONV_BUF_TYPE *src0, int src0_stride,
const CONV_BUF_TYPE *src1, int src1_stride, const uint8_t *mask,
int mask_stride, int h, const __m256i *round_offset, int shift,
const __m256i *clip_low, const __m256i *clip_high,
const __m256i *mask_max) {
const __m256i one_b = _mm256_set1_epi8(1);
const __m256i two_w = _mm256_set1_epi16(2);
do {
// Load 16x u8 pixels from each of 8 rows in the mask,
// (saturating) add together rows then use madd to add adjacent pixels
// Finally, divide each value by 4 (with rounding)
const __m256i m02 =
yy_loadu2_128(mask + 0 * mask_stride, mask + 2 * mask_stride);
const __m256i m13 =
yy_loadu2_128(mask + 1 * mask_stride, mask + 3 * mask_stride);
const __m256i m0123 =
_mm256_maddubs_epi16(_mm256_adds_epu8(m02, m13), one_b);
const __m256i mask_0a =
_mm256_srli_epi16(_mm256_add_epi16(m0123, two_w), 2);
const __m256i m46 =
yy_loadu2_128(mask + 4 * mask_stride, mask + 6 * mask_stride);
const __m256i m57 =
yy_loadu2_128(mask + 5 * mask_stride, mask + 7 * mask_stride);
const __m256i m4567 =
_mm256_maddubs_epi16(_mm256_adds_epu8(m46, m57), one_b);
const __m256i mask_0b =
_mm256_srli_epi16(_mm256_add_epi16(m4567, two_w), 2);
highbd_blend_a64_d16_mask_w8_avx2(
dst, dst_stride, src0, src0_stride, src1, src1_stride, &mask_0a,
&mask_0b, round_offset, shift, clip_low, clip_high, mask_max);
dst += dst_stride * 4;
src0 += src0_stride * 4;
src1 += src1_stride * 4;
mask += mask_stride * 8;
} while (h -= 4);
}
static INLINE void highbd_blend_a64_d16_mask_w16_avx2(
uint16_t *dst, int dst_stride, const CONV_BUF_TYPE *src0, int src0_stride,
const CONV_BUF_TYPE *src1, int src1_stride, const __m256i *mask0a,
const __m256i *mask0b, const __m256i *round_offset, int shift,
const __m256i *clip_low, const __m256i *clip_high,
const __m256i *mask_max) {
// Load 16x pixels from each of 2 rows from each source
const __m256i s0a = yy_loadu_256(src0);
const __m256i s0b = yy_loadu_256(src0 + src0_stride);
const __m256i s1a = yy_loadu_256(src1);
const __m256i s1b = yy_loadu_256(src1 + src1_stride);
// Calculate inverse masks
const __m256i mask1a = _mm256_sub_epi16(*mask_max, *mask0a);
const __m256i mask1b = _mm256_sub_epi16(*mask_max, *mask0b);
// Multiply each source by appropriate mask
const __m256i mul0a_highs = _mm256_mulhi_epu16(*mask0a, s0a);
const __m256i mul0a_lows = _mm256_mullo_epi16(*mask0a, s0a);
const __m256i mul0ah = _mm256_unpackhi_epi16(mul0a_lows, mul0a_highs);
const __m256i mul0al = _mm256_unpacklo_epi16(mul0a_lows, mul0a_highs);
// Note that AVX2 unpack orders 64-bit words as [3 1] [2 0] to keep within
// lanes Later, packs does the same again which cancels this out with no need
// for a permute. The intermediate values being reordered makes no difference
const __m256i mul1a_highs = _mm256_mulhi_epu16(mask1a, s1a);
const __m256i mul1a_lows = _mm256_mullo_epi16(mask1a, s1a);
const __m256i mul1ah = _mm256_unpackhi_epi16(mul1a_lows, mul1a_highs);
const __m256i mul1al = _mm256_unpacklo_epi16(mul1a_lows, mul1a_highs);
const __m256i mulah = _mm256_add_epi32(mul0ah, mul1ah);
const __m256i mulal = _mm256_add_epi32(mul0al, mul1al);
const __m256i mul0b_highs = _mm256_mulhi_epu16(*mask0b, s0b);
const __m256i mul0b_lows = _mm256_mullo_epi16(*mask0b, s0b);
const __m256i mul0bh = _mm256_unpackhi_epi16(mul0b_lows, mul0b_highs);
const __m256i mul0bl = _mm256_unpacklo_epi16(mul0b_lows, mul0b_highs);
const __m256i mul1b_highs = _mm256_mulhi_epu16(mask1b, s1b);
const __m256i mul1b_lows = _mm256_mullo_epi16(mask1b, s1b);
const __m256i mul1bh = _mm256_unpackhi_epi16(mul1b_lows, mul1b_highs);
const __m256i mul1bl = _mm256_unpacklo_epi16(mul1b_lows, mul1b_highs);
const __m256i mulbh = _mm256_add_epi32(mul0bh, mul1bh);
const __m256i mulbl = _mm256_add_epi32(mul0bl, mul1bl);
const __m256i resah =
_mm256_srai_epi32(_mm256_sub_epi32(mulah, *round_offset), shift);
const __m256i resal =
_mm256_srai_epi32(_mm256_sub_epi32(mulal, *round_offset), shift);
const __m256i resbh =
_mm256_srai_epi32(_mm256_sub_epi32(mulbh, *round_offset), shift);
const __m256i resbl =
_mm256_srai_epi32(_mm256_sub_epi32(mulbl, *round_offset), shift);
// Signed saturating pack from i32 to i16:
const __m256i packa = _mm256_packs_epi32(resal, resah);
const __m256i packb = _mm256_packs_epi32(resbl, resbh);
// Clip the values to the valid range
const __m256i clipa =
_mm256_min_epi16(_mm256_max_epi16(packa, *clip_low), *clip_high);
const __m256i clipb =
_mm256_min_epi16(_mm256_max_epi16(packb, *clip_low), *clip_high);
// Store 16 pixels
yy_storeu_256(dst, clipa);
yy_storeu_256(dst + dst_stride, clipb);
}
static INLINE void highbd_blend_a64_d16_mask_subw0_subh0_w16_avx2(
uint16_t *dst, int dst_stride, const CONV_BUF_TYPE *src0, int src0_stride,
const CONV_BUF_TYPE *src1, int src1_stride, const uint8_t *mask,
int mask_stride, int h, int w, const __m256i *round_offset, int shift,
const __m256i *clip_low, const __m256i *clip_high,
const __m256i *mask_max) {
for (int i = 0; i < h; i += 2) {
for (int j = 0; j < w; j += 16) {
// Load 16x u8 alpha-mask values from each of two rows and pad to u16
const __m128i masks_a8 = xx_loadu_128(mask + j);
const __m128i masks_b8 = xx_loadu_128(mask + mask_stride + j);
const __m256i mask0a = _mm256_cvtepu8_epi16(masks_a8);
const __m256i mask0b = _mm256_cvtepu8_epi16(masks_b8);
highbd_blend_a64_d16_mask_w16_avx2(
dst + j, dst_stride, src0 + j, src0_stride, src1 + j, src1_stride,
&mask0a, &mask0b, round_offset, shift, clip_low, clip_high, mask_max);
}
dst += dst_stride * 2;
src0 += src0_stride * 2;
src1 += src1_stride * 2;
mask += mask_stride * 2;
}
}
static INLINE void highbd_blend_a64_d16_mask_subw1_subh1_w16_avx2(
uint16_t *dst, int dst_stride, const CONV_BUF_TYPE *src0, int src0_stride,
const CONV_BUF_TYPE *src1, int src1_stride, const uint8_t *mask,
int mask_stride, int h, int w, const __m256i *round_offset, int shift,
const __m256i *clip_low, const __m256i *clip_high,
const __m256i *mask_max) {
const __m256i one_b = _mm256_set1_epi8(1);
const __m256i two_w = _mm256_set1_epi16(2);
for (int i = 0; i < h; i += 2) {
for (int j = 0; j < w; j += 16) {
// Load 32x u8 alpha-mask values from each of four rows
// (saturating) add pairs of rows, then use madd to add adjacent values
// Finally, divide down each result with rounding
const __m256i m0 = yy_loadu_256(mask + 0 * mask_stride + 2 * j);
const __m256i m1 = yy_loadu_256(mask + 1 * mask_stride + 2 * j);
const __m256i m2 = yy_loadu_256(mask + 2 * mask_stride + 2 * j);
const __m256i m3 = yy_loadu_256(mask + 3 * mask_stride + 2 * j);
const __m256i m01_8 = _mm256_adds_epu8(m0, m1);
const __m256i m23_8 = _mm256_adds_epu8(m2, m3);
const __m256i m01 = _mm256_maddubs_epi16(m01_8, one_b);
const __m256i m23 = _mm256_maddubs_epi16(m23_8, one_b);
const __m256i mask0a = _mm256_srli_epi16(_mm256_add_epi16(m01, two_w), 2);
const __m256i mask0b = _mm256_srli_epi16(_mm256_add_epi16(m23, two_w), 2);
highbd_blend_a64_d16_mask_w16_avx2(
dst + j, dst_stride, src0 + j, src0_stride, src1 + j, src1_stride,
&mask0a, &mask0b, round_offset, shift, clip_low, clip_high, mask_max);
}
dst += dst_stride * 2;
src0 += src0_stride * 2;
src1 += src1_stride * 2;
mask += mask_stride * 4;
}
}
void aom_highbd_blend_a64_d16_mask_avx2(
uint8_t *dst8, uint32_t dst_stride, const CONV_BUF_TYPE *src0,
uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride,
const uint8_t *mask, uint32_t mask_stride, int w, int h, int subw, int subh,
ConvolveParams *conv_params, const int bd) {
uint16_t *dst = CONVERT_TO_SHORTPTR(dst8);
const int round_bits =
2 * FILTER_BITS - conv_params->round_0 - conv_params->round_1;
const int32_t round_offset =
((1 << (round_bits + bd)) + (1 << (round_bits + bd - 1)) -
(1 << (round_bits - 1)))
<< AOM_BLEND_A64_ROUND_BITS;
const __m256i v_round_offset = _mm256_set1_epi32(round_offset);
const int shift = round_bits + AOM_BLEND_A64_ROUND_BITS;
const __m256i clip_low = _mm256_set1_epi16(0);
const __m256i clip_high = _mm256_set1_epi16((1 << bd) - 1);
const __m256i mask_max = _mm256_set1_epi16(AOM_BLEND_A64_MAX_ALPHA);
assert(IMPLIES((void *)src0 == dst, src0_stride == dst_stride));
assert(IMPLIES((void *)src1 == dst, src1_stride == dst_stride));
assert(h >= 4);
assert(w >= 4);
assert(IS_POWER_OF_TWO(h));
assert(IS_POWER_OF_TWO(w));
if (subw == 0 && subh == 0) {
switch (w) {
case 4:
highbd_blend_a64_d16_mask_subw0_subh0_w4_avx2(
dst, dst_stride, src0, src0_stride, src1, src1_stride, mask,
mask_stride, h, &v_round_offset, shift, &clip_low, &clip_high,
&mask_max);
break;
case 8:
highbd_blend_a64_d16_mask_subw0_subh0_w8_avx2(
dst, dst_stride, src0, src0_stride, src1, src1_stride, mask,
mask_stride, h, &v_round_offset, shift, &clip_low, &clip_high,
&mask_max);
break;
default: // >= 16
highbd_blend_a64_d16_mask_subw0_subh0_w16_avx2(
dst, dst_stride, src0, src0_stride, src1, src1_stride, mask,
mask_stride, h, w, &v_round_offset, shift, &clip_low, &clip_high,
&mask_max);
break;
}
} else if (subw == 1 && subh == 1) {
switch (w) {
case 4:
highbd_blend_a64_d16_mask_subw1_subh1_w4_avx2(
dst, dst_stride, src0, src0_stride, src1, src1_stride, mask,
mask_stride, h, &v_round_offset, shift, &clip_low, &clip_high,
&mask_max);
break;
case 8:
highbd_blend_a64_d16_mask_subw1_subh1_w8_avx2(
dst, dst_stride, src0, src0_stride, src1, src1_stride, mask,
mask_stride, h, &v_round_offset, shift, &clip_low, &clip_high,
&mask_max);
break;
default: // >= 16
highbd_blend_a64_d16_mask_subw1_subh1_w16_avx2(
dst, dst_stride, src0, src0_stride, src1, src1_stride, mask,
mask_stride, h, w, &v_round_offset, shift, &clip_low, &clip_high,
&mask_max);
break;
}
} else {
// Sub-sampling in only one axis doesn't seem to happen very much, so fall
// back to the vanilla C implementation instead of having all the optimised
// code for these.
aom_highbd_blend_a64_d16_mask_c(dst8, dst_stride, src0, src0_stride, src1,
src1_stride, mask, mask_stride, w, h, subw,
subh, conv_params, bd);
}
}

449
third_party/aom/aom_dsp/x86/blend_a64_mask_sse4.c vendored
View file

@ -1107,3 +1107,452 @@ void aom_lowbd_blend_a64_d16_mask_sse4_1(
}
}
}
//////////////////////////////////////////////////////////////////////////////
// aom_highbd_blend_a64_d16_mask_sse4_1()
//////////////////////////////////////////////////////////////////////////////
static INLINE void highbd_blend_a64_d16_mask_w4_sse4_1(
uint16_t *dst, int dst_stride, const CONV_BUF_TYPE *src0, int src0_stride,
const CONV_BUF_TYPE *src1, int src1_stride, const __m128i *mask0a,
const __m128i *mask0b, const __m128i *round_offset, int shift,
const __m128i *clip_low, const __m128i *clip_high,
const __m128i *mask_max) {
// Load 4 pixels from each of 4 rows from each source
const __m128i s0a =
_mm_set_epi64x(*(uint64_t *)src0, *(uint64_t *)(src0 + src0_stride));
const __m128i s0b = _mm_set_epi64x(*(uint64_t *)(src0 + 2 * src0_stride),
*(uint64_t *)(src0 + 3 * src0_stride));
const __m128i s1a =
_mm_set_epi64x(*(uint64_t *)(src1), *(uint64_t *)(src1 + src1_stride));
const __m128i s1b = _mm_set_epi64x(*(uint64_t *)(src1 + 2 * src1_stride),
*(uint64_t *)(src1 + 3 * src1_stride));
// Generate the inverse masks
const __m128i mask1a = _mm_sub_epi16(*mask_max, *mask0a);
const __m128i mask1b = _mm_sub_epi16(*mask_max, *mask0b);
// Multiply each mask by the respective source
const __m128i mul0a_highs = _mm_mulhi_epu16(*mask0a, s0a);
const __m128i mul0a_lows = _mm_mullo_epi16(*mask0a, s0a);
const __m128i mul0ah = _mm_unpackhi_epi16(mul0a_lows, mul0a_highs);
const __m128i mul0al = _mm_unpacklo_epi16(mul0a_lows, mul0a_highs);
const __m128i mul1a_highs = _mm_mulhi_epu16(mask1a, s1a);
const __m128i mul1a_lows = _mm_mullo_epi16(mask1a, s1a);
const __m128i mul1ah = _mm_unpackhi_epi16(mul1a_lows, mul1a_highs);
const __m128i mul1al = _mm_unpacklo_epi16(mul1a_lows, mul1a_highs);
const __m128i mul0b_highs = _mm_mulhi_epu16(*mask0b, s0b);
const __m128i mul0b_lows = _mm_mullo_epi16(*mask0b, s0b);
const __m128i mul0bh = _mm_unpackhi_epi16(mul0b_lows, mul0b_highs);
const __m128i mul0bl = _mm_unpacklo_epi16(mul0b_lows, mul0b_highs);
const __m128i mul1b_highs = _mm_mulhi_epu16(mask1b, s1b);
const __m128i mul1b_lows = _mm_mullo_epi16(mask1b, s1b);
const __m128i mul1bh = _mm_unpackhi_epi16(mul1b_lows, mul1b_highs);
const __m128i mul1bl = _mm_unpacklo_epi16(mul1b_lows, mul1b_highs);
const __m128i sumah = _mm_add_epi32(mul0ah, mul1ah);
const __m128i sumal = _mm_add_epi32(mul0al, mul1al);
const __m128i sumbh = _mm_add_epi32(mul0bh, mul1bh);
const __m128i sumbl = _mm_add_epi32(mul0bl, mul1bl);
const __m128i roundah =
_mm_srai_epi32(_mm_sub_epi32(sumah, *round_offset), shift);
const __m128i roundbh =
_mm_srai_epi32(_mm_sub_epi32(sumbh, *round_offset), shift);
const __m128i roundal =
_mm_srai_epi32(_mm_sub_epi32(sumal, *round_offset), shift);
const __m128i roundbl =
_mm_srai_epi32(_mm_sub_epi32(sumbl, *round_offset), shift);
const __m128i packa = _mm_packs_epi32(roundal, roundah);
const __m128i packb = _mm_packs_epi32(roundbl, roundbh);
const __m128i clipa =
_mm_min_epi16(_mm_max_epi16(packa, *clip_low), *clip_high);
const __m128i clipb =
_mm_min_epi16(_mm_max_epi16(packb, *clip_low), *clip_high);
xx_storel_64(dst, _mm_srli_si128(clipa, 8));
xx_storel_64(dst + dst_stride, clipa);
xx_storel_64(dst + 2 * dst_stride, _mm_srli_si128(clipb, 8));
xx_storel_64(dst + 3 * dst_stride, clipb);
}
static INLINE void highbd_blend_a64_d16_mask_subw0_subh0_w4_sse4_1(
uint16_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0,
uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride,
const uint8_t *mask, uint32_t mask_stride, int h,
const __m128i *round_offset, int shift, const __m128i *clip_low,
const __m128i *clip_high, const __m128i *mask_max) {
do {
const __m128i mask0a8 = _mm_set_epi32(0, 0, *(uint32_t *)mask,
*(uint32_t *)(mask + mask_stride));
const __m128i mask0b8 =
_mm_set_epi32(0, 0, *(uint32_t *)(mask + 2 * mask_stride),
*(uint32_t *)(mask + 3 * mask_stride));
const __m128i mask0a = _mm_cvtepu8_epi16(mask0a8);
const __m128i mask0b = _mm_cvtepu8_epi16(mask0b8);
highbd_blend_a64_d16_mask_w4_sse4_1(
dst, dst_stride, src0, src0_stride, src1, src1_stride, &mask0a, &mask0b,
round_offset, shift, clip_low, clip_high, mask_max);
dst += dst_stride * 4;
src0 += src0_stride * 4;
src1 += src1_stride * 4;
mask += mask_stride * 4;
} while (h -= 4);
}
static INLINE void highbd_blend_a64_d16_mask_subw1_subh1_w4_sse4_1(
uint16_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0,
uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride,
const uint8_t *mask, uint32_t mask_stride, int h,
const __m128i *round_offset, int shift, const __m128i *clip_low,
const __m128i *clip_high, const __m128i *mask_max) {
const __m128i one_b = _mm_set1_epi8(1);
const __m128i two_w = _mm_set1_epi16(2);
do {
// Load 8 pixels from each of 8 rows of mask,
// (saturating) add together rows then use madd to add adjacent pixels
// Finally, divide each value by 4 (with rounding)
const __m128i m02 = _mm_set_epi64x(*(uint64_t *)(mask),
*(uint64_t *)(mask + 2 * mask_stride));
const __m128i m13 = _mm_set_epi64x(*(uint64_t *)(mask + mask_stride),
*(uint64_t *)(mask + 3 * mask_stride));
const __m128i m0123 = _mm_maddubs_epi16(_mm_adds_epu8(m02, m13), one_b);
const __m128i mask_0a = _mm_srli_epi16(_mm_add_epi16(m0123, two_w), 2);
const __m128i m46 = _mm_set_epi64x(*(uint64_t *)(mask + 4 * mask_stride),
*(uint64_t *)(mask + 6 * mask_stride));
const __m128i m57 = _mm_set_epi64x(*(uint64_t *)(mask + 5 * mask_stride),
*(uint64_t *)(mask + 7 * mask_stride));
const __m128i m4567 = _mm_maddubs_epi16(_mm_adds_epu8(m46, m57), one_b);
const __m128i mask_0b = _mm_srli_epi16(_mm_add_epi16(m4567, two_w), 2);
highbd_blend_a64_d16_mask_w4_sse4_1(
dst, dst_stride, src0, src0_stride, src1, src1_stride, &mask_0a,
&mask_0b, round_offset, shift, clip_low, clip_high, mask_max);
dst += dst_stride * 4;
src0 += src0_stride * 4;
src1 += src1_stride * 4;
mask += mask_stride * 8;
} while (h -= 4);
}
static INLINE void highbd_blend_a64_d16_mask_w8_sse4_1(
uint16_t *dst, int dst_stride, const CONV_BUF_TYPE *src0, int src0_stride,
const CONV_BUF_TYPE *src1, int src1_stride, const __m128i *mask0a,
const __m128i *mask0b, const __m128i *round_offset, int shift,
const __m128i *clip_low, const __m128i *clip_high,
const __m128i *max_mask) {
// Load 8x pixels from each of 2 rows from each source
const __m128i s0a = xx_loadu_128(src0);
const __m128i s0b = xx_loadu_128(src0 + src0_stride);
const __m128i s1a = xx_loadu_128(src1);
const __m128i s1b = xx_loadu_128(src1 + src1_stride);
// Generate inverse masks
const __m128i mask1a = _mm_sub_epi16(*max_mask, *mask0a);
const __m128i mask1b = _mm_sub_epi16(*max_mask, *mask0b);
// Multiply sources by respective masks
const __m128i mul0a_highs = _mm_mulhi_epu16(*mask0a, s0a);
const __m128i mul0a_lows = _mm_mullo_epi16(*mask0a, s0a);
const __m128i mul0ah = _mm_unpackhi_epi16(mul0a_lows, mul0a_highs);
const __m128i mul0al = _mm_unpacklo_epi16(mul0a_lows, mul0a_highs);
const __m128i mul1a_highs = _mm_mulhi_epu16(mask1a, s1a);
const __m128i mul1a_lows = _mm_mullo_epi16(mask1a, s1a);
const __m128i mul1ah = _mm_unpackhi_epi16(mul1a_lows, mul1a_highs);
const __m128i mul1al = _mm_unpacklo_epi16(mul1a_lows, mul1a_highs);
const __m128i sumah = _mm_add_epi32(mul0ah, mul1ah);
const __m128i sumal = _mm_add_epi32(mul0al, mul1al);
const __m128i mul0b_highs = _mm_mulhi_epu16(*mask0b, s0b);
const __m128i mul0b_lows = _mm_mullo_epi16(*mask0b, s0b);
const __m128i mul0bh = _mm_unpackhi_epi16(mul0b_lows, mul0b_highs);
const __m128i mul0bl = _mm_unpacklo_epi16(mul0b_lows, mul0b_highs);
const __m128i mul1b_highs = _mm_mulhi_epu16(mask1b, s1b);
const __m128i mul1b_lows = _mm_mullo_epi16(mask1b, s1b);
const __m128i mul1bh = _mm_unpackhi_epi16(mul1b_lows, mul1b_highs);
const __m128i mul1bl = _mm_unpacklo_epi16(mul1b_lows, mul1b_highs);
const __m128i sumbh = _mm_add_epi32(mul0bh, mul1bh);
const __m128i sumbl = _mm_add_epi32(mul0bl, mul1bl);
const __m128i roundah =
_mm_srai_epi32(_mm_sub_epi32(sumah, *round_offset), shift);
const __m128i roundal =
_mm_srai_epi32(_mm_sub_epi32(sumal, *round_offset), shift);
const __m128i roundbh =
_mm_srai_epi32(_mm_sub_epi32(sumbh, *round_offset), shift);
const __m128i roundbl =
_mm_srai_epi32(_mm_sub_epi32(sumbl, *round_offset), shift);
const __m128i packa = _mm_packs_epi32(roundal, roundah);
const __m128i clipa =
_mm_min_epi16(_mm_max_epi16(packa, *clip_low), *clip_high);
const __m128i packb = _mm_packs_epi32(roundbl, roundbh);
const __m128i clipb =
_mm_min_epi16(_mm_max_epi16(packb, *clip_low), *clip_high);
xx_storeu_128(dst, clipa);
xx_storeu_128(dst + dst_stride, clipb);
}
static INLINE void highbd_blend_a64_d16_mask_subw0_subh0_w8_sse4_1(
uint16_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0,
uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride,
const uint8_t *mask, uint32_t mask_stride, int h,
const __m128i *round_offset, int shift, const __m128i *clip_low,
const __m128i *clip_high, const __m128i *max_mask) {
do {
const __m128i mask0a = _mm_cvtepu8_epi16(xx_loadl_64(mask));
const __m128i mask0b = _mm_cvtepu8_epi16(xx_loadl_64(mask + mask_stride));
highbd_blend_a64_d16_mask_w8_sse4_1(
dst, dst_stride, src0, src0_stride, src1, src1_stride, &mask0a, &mask0b,
round_offset, shift, clip_low, clip_high, max_mask);
dst += dst_stride * 2;
src0 += src0_stride * 2;
src1 += src1_stride * 2;
mask += mask_stride * 2;
} while (h -= 2);
}
static INLINE void highbd_blend_a64_d16_mask_subw1_subh1_w8_sse4_1(
uint16_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0,
uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride,
const uint8_t *mask, uint32_t mask_stride, int h,
const __m128i *round_offset, int shift, const __m128i *clip_low,
const __m128i *clip_high, const __m128i *max_mask) {
const __m128i one_b = _mm_set1_epi8(1);
const __m128i two_w = _mm_set1_epi16(2);
do {
const __m128i mask_thisrowa = xx_loadu_128(mask);
const __m128i mask_nextrowa = xx_loadu_128(mask + mask_stride);
const __m128i mask_thisrowb = xx_loadu_128(mask + 2 * mask_stride);
const __m128i mask_nextrowb = xx_loadu_128(mask + 3 * mask_stride);
const __m128i mask_bothrowsa = _mm_adds_epu8(mask_thisrowa, mask_nextrowa);
const __m128i mask_bothrowsb = _mm_adds_epu8(mask_thisrowb, mask_nextrowb);
const __m128i mask_16a = _mm_maddubs_epi16(mask_bothrowsa, one_b);
const __m128i mask_16b = _mm_maddubs_epi16(mask_bothrowsb, one_b);
const __m128i mask_sa = _mm_srli_epi16(_mm_add_epi16(mask_16a, two_w), 2);
const __m128i mask_sb = _mm_srli_epi16(_mm_add_epi16(mask_16b, two_w), 2);
highbd_blend_a64_d16_mask_w8_sse4_1(
dst, dst_stride, src0, src0_stride, src1, src1_stride, &mask_sa,
&mask_sb, round_offset, shift, clip_low, clip_high, max_mask);
dst += dst_stride * 2;
src0 += src0_stride * 2;
src1 += src1_stride * 2;
mask += mask_stride * 4;
} while (h -= 2);
}
static INLINE void highbd_blend_a64_d16_mask_w16_sse4_1(
uint16_t *dst, const CONV_BUF_TYPE *src0, const CONV_BUF_TYPE *src1,
const __m128i *round_offset, int shift, const __m128i *mask0l,
const __m128i *mask0h, const __m128i *clip_low, const __m128i *clip_high,
const __m128i *mask_max) {
// Load 16x u16 pixels for this row from each src
const __m128i s0l = xx_loadu_128(src0);
const __m128i s0h = xx_loadu_128(src0 + 8);
const __m128i s1l = xx_loadu_128(src1);
const __m128i s1h = xx_loadu_128(src1 + 8);
// Calculate inverse masks
const __m128i mask1h = _mm_sub_epi16(*mask_max, *mask0h);
const __m128i mask1l = _mm_sub_epi16(*mask_max, *mask0l);
const __m128i mul0_highs = _mm_mulhi_epu16(*mask0h, s0h);
const __m128i mul0_lows = _mm_mullo_epi16(*mask0h, s0h);
const __m128i mul0h = _mm_unpackhi_epi16(mul0_lows, mul0_highs);
const __m128i mul0l = _mm_unpacklo_epi16(mul0_lows, mul0_highs);
const __m128i mul1_highs = _mm_mulhi_epu16(mask1h, s1h);
const __m128i mul1_lows = _mm_mullo_epi16(mask1h, s1h);
const __m128i mul1h = _mm_unpackhi_epi16(mul1_lows, mul1_highs);
const __m128i mul1l = _mm_unpacklo_epi16(mul1_lows, mul1_highs);
const __m128i mulhh = _mm_add_epi32(mul0h, mul1h);
const __m128i mulhl = _mm_add_epi32(mul0l, mul1l);
const __m128i mul2_highs = _mm_mulhi_epu16(*mask0l, s0l);
const __m128i mul2_lows = _mm_mullo_epi16(*mask0l, s0l);
const __m128i mul2h = _mm_unpackhi_epi16(mul2_lows, mul2_highs);
const __m128i mul2l = _mm_unpacklo_epi16(mul2_lows, mul2_highs);
const __m128i mul3_highs = _mm_mulhi_epu16(mask1l, s1l);
const __m128i mul3_lows = _mm_mullo_epi16(mask1l, s1l);
const __m128i mul3h = _mm_unpackhi_epi16(mul3_lows, mul3_highs);
const __m128i mul3l = _mm_unpacklo_epi16(mul3_lows, mul3_highs);
const __m128i mullh = _mm_add_epi32(mul2h, mul3h);
const __m128i mulll = _mm_add_epi32(mul2l, mul3l);
const __m128i reshh =
_mm_srai_epi32(_mm_sub_epi32(mulhh, *round_offset), shift);
const __m128i reshl =
_mm_srai_epi32(_mm_sub_epi32(mulhl, *round_offset), shift);
const __m128i reslh =
_mm_srai_epi32(_mm_sub_epi32(mullh, *round_offset), shift);
const __m128i resll =
_mm_srai_epi32(_mm_sub_epi32(mulll, *round_offset), shift);
// Signed saturating pack from i32 to i16:
const __m128i packh = _mm_packs_epi32(reshl, reshh);
const __m128i packl = _mm_packs_epi32(resll, reslh);
// Clip the values to the valid range
const __m128i cliph =
_mm_min_epi16(_mm_max_epi16(packh, *clip_low), *clip_high);
const __m128i clipl =
_mm_min_epi16(_mm_max_epi16(packl, *clip_low), *clip_high);
// Store 16 pixels
xx_storeu_128(dst, clipl);
xx_storeu_128(dst + 8, cliph);
}
static INLINE void highbd_blend_a64_d16_mask_subw0_subh0_w16_sse4_1(
uint16_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0,
uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride,
const uint8_t *mask, uint32_t mask_stride, int h, int w,
const __m128i *round_offset, int shift, const __m128i *clip_low,
const __m128i *clip_high, const __m128i *mask_max) {
for (int i = 0; i < h; i++) {
for (int j = 0; j < w; j += 16) {
// Load 16x u8 alpha-mask values and pad to u16
const __m128i masks_u8 = xx_loadu_128(mask + j);
const __m128i mask0l = _mm_cvtepu8_epi16(masks_u8);
const __m128i mask0h = _mm_cvtepu8_epi16(_mm_srli_si128(masks_u8, 8));
highbd_blend_a64_d16_mask_w16_sse4_1(
dst + j, src0 + j, src1 + j, round_offset, shift, &mask0l, &mask0h,
clip_low, clip_high, mask_max);
}
dst += dst_stride;
src0 += src0_stride;
src1 += src1_stride;
mask += mask_stride;
}
}
static INLINE void highbd_blend_a64_d16_mask_subw1_subh1_w16_sse4_1(
uint16_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0,
uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride,
const uint8_t *mask, uint32_t mask_stride, int h, int w,
const __m128i *round_offset, int shift, const __m128i *clip_low,
const __m128i *clip_high, const __m128i *mask_max) {
const __m128i one_b = _mm_set1_epi8(1);
const __m128i two_w = _mm_set1_epi16(2);
for (int i = 0; i < h; i++) {
for (int j = 0; j < w; j += 16) {
const __m128i m_i00 = xx_loadu_128(mask + 2 * j);
const __m128i m_i01 = xx_loadu_128(mask + 2 * j + 16);
const __m128i m_i10 = xx_loadu_128(mask + mask_stride + 2 * j);
const __m128i m_i11 = xx_loadu_128(mask + mask_stride + 2 * j + 16);
const __m128i m0_ac = _mm_adds_epu8(m_i00, m_i10);
const __m128i m1_ac = _mm_adds_epu8(m_i01, m_i11);
const __m128i m0_acbd = _mm_maddubs_epi16(m0_ac, one_b);
const __m128i m1_acbd = _mm_maddubs_epi16(m1_ac, one_b);
const __m128i mask_l = _mm_srli_epi16(_mm_add_epi16(m0_acbd, two_w), 2);
const __m128i mask_h = _mm_srli_epi16(_mm_add_epi16(m1_acbd, two_w), 2);
highbd_blend_a64_d16_mask_w16_sse4_1(
dst + j, src0 + j, src1 + j, round_offset, shift, &mask_l, &mask_h,
clip_low, clip_high, mask_max);
}
dst += dst_stride;
src0 += src0_stride;
src1 += src1_stride;
mask += mask_stride * 2;
}
}
void aom_highbd_blend_a64_d16_mask_sse4_1(
uint8_t *dst8, uint32_t dst_stride, const CONV_BUF_TYPE *src0,
uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride,
const uint8_t *mask, uint32_t mask_stride, int w, int h, int subw, int subh,
ConvolveParams *conv_params, const int bd) {
uint16_t *dst = CONVERT_TO_SHORTPTR(dst8);
const int round_bits =
2 * FILTER_BITS - conv_params->round_0 - conv_params->round_1;
const int32_t round_offset =
((1 << (round_bits + bd)) + (1 << (round_bits + bd - 1)) -
(1 << (round_bits - 1)))
<< AOM_BLEND_A64_ROUND_BITS;
const __m128i v_round_offset = _mm_set1_epi32(round_offset);
const int shift = round_bits + AOM_BLEND_A64_ROUND_BITS;
const __m128i clip_low = _mm_set1_epi16(0);
const __m128i clip_high = _mm_set1_epi16((1 << bd) - 1);
const __m128i mask_max = _mm_set1_epi16(AOM_BLEND_A64_MAX_ALPHA);
assert(IMPLIES((void *)src0 == dst, src0_stride == dst_stride));
assert(IMPLIES((void *)src1 == dst, src1_stride == dst_stride));
assert(h >= 4);
assert(w >= 4);
assert(IS_POWER_OF_TWO(h));
assert(IS_POWER_OF_TWO(w));
if (subw == 0 && subh == 0) {
switch (w) {
case 4:
highbd_blend_a64_d16_mask_subw0_subh0_w4_sse4_1(
dst, dst_stride, src0, src0_stride, src1, src1_stride, mask,
mask_stride, h, &v_round_offset, shift, &clip_low, &clip_high,
&mask_max);
break;
case 8:
highbd_blend_a64_d16_mask_subw0_subh0_w8_sse4_1(
dst, dst_stride, src0, src0_stride, src1, src1_stride, mask,
mask_stride, h, &v_round_offset, shift, &clip_low, &clip_high,
&mask_max);
break;
default: // >=16
highbd_blend_a64_d16_mask_subw0_subh0_w16_sse4_1(
dst, dst_stride, src0, src0_stride, src1, src1_stride, mask,
mask_stride, h, w, &v_round_offset, shift, &clip_low, &clip_high,
&mask_max);
break;
}
} else if (subw == 1 && subh == 1) {
switch (w) {
case 4:
highbd_blend_a64_d16_mask_subw1_subh1_w4_sse4_1(
dst, dst_stride, src0, src0_stride, src1, src1_stride, mask,
mask_stride, h, &v_round_offset, shift, &clip_low, &clip_high,
&mask_max);
break;
case 8:
highbd_blend_a64_d16_mask_subw1_subh1_w8_sse4_1(
dst, dst_stride, src0, src0_stride, src1, src1_stride, mask,
mask_stride, h, &v_round_offset, shift, &clip_low, &clip_high,
&mask_max);
break;
default: // >=16
highbd_blend_a64_d16_mask_subw1_subh1_w16_sse4_1(
dst, dst_stride, src0, src0_stride, src1, src1_stride, mask,
mask_stride, h, w, &v_round_offset, shift, &clip_low, &clip_high,
&mask_max);
break;
}
} else {
// Sub-sampling in only one axis doesn't seem to happen very much, so fall
// back to the vanilla C implementation instead of having all the optimised
// code for these.
aom_highbd_blend_a64_d16_mask_c(dst8, dst_stride, src0, src0_stride, src1,
src1_stride, mask, mask_stride, w, h, subw,
subh, conv_params, bd);
}
}

25
third_party/aom/aom_dsp/x86/convolve.h vendored
View file

@ -122,7 +122,30 @@ typedef void highbd_filter8_1dfunction(const uint16_t *src_ptr,
uint16_t *src = CONVERT_TO_SHORTPTR(src8); \
uint16_t *dst = CONVERT_TO_SHORTPTR(dst8); \
if (step_q4 == 16 && filter[3] != 128) { \
if (filter[0] | filter[1] | filter[2]) { \
if (((filter[0] | filter[1] | filter[6] | filter[7]) == 0) && \
(filter[2] | filter[5])) { \
while (w >= 16) { \
aom_highbd_filter_block1d16_##dir##4_##avg##opt( \
src_start, src_stride, dst, dst_stride, h, filter, bd); \
src += 16; \
dst += 16; \
w -= 16; \
} \
while (w >= 8) { \
aom_highbd_filter_block1d8_##dir##4_##avg##opt( \
src_start, src_stride, dst, dst_stride, h, filter, bd); \
src += 8; \
dst += 8; \
w -= 8; \
} \
while (w >= 4) { \
aom_highbd_filter_block1d4_##dir##4_##avg##opt( \
src_start, src_stride, dst, dst_stride, h, filter, bd); \
src += 4; \
dst += 4; \
w -= 4; \
} \
} else if (filter[0] | filter[1] | filter[2]) { \
while (w >= 16) { \
aom_highbd_filter_block1d16_##dir##8_##avg##opt( \
src_start, src_stride, dst, dst_stride, h, filter, bd); \

42
third_party/aom/aom_dsp/x86/convolve_avx2.h vendored
View file

@ -34,6 +34,31 @@ DECLARE_ALIGNED(32, static const uint8_t, filt4_d4_global_avx2[]) = {
2, 3, 4, 5, 3, 4, 5, 6, 4, 5, 6, 7, 5, 6, 7, 8,
};
DECLARE_ALIGNED(32, static const uint8_t, filt_center_global_avx2[32]) = {
3, 255, 4, 255, 5, 255, 6, 255, 7, 255, 8, 255, 9, 255, 10, 255,
3, 255, 4, 255, 5, 255, 6, 255, 7, 255, 8, 255, 9, 255, 10, 255
};
DECLARE_ALIGNED(32, static const uint8_t, filt1_global_avx2[32]) = {
0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8,
0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8
};
DECLARE_ALIGNED(32, static const uint8_t, filt2_global_avx2[32]) = {
2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10,
2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10
};
DECLARE_ALIGNED(32, static const uint8_t, filt3_global_avx2[32]) = {
4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12,
4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12
};
DECLARE_ALIGNED(32, static const uint8_t, filt4_global_avx2[32]) = {
6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14,
6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14
};
static INLINE void prepare_coeffs_lowbd(
const InterpFilterParams *const filter_params, const int subpel_q4,
__m256i *const coeffs /* [4] */) {
@ -108,6 +133,15 @@ static INLINE __m256i convolve(const __m256i *const s,
return res;
}
static INLINE __m256i convolve_4tap(const __m256i *const s,
const __m256i *const coeffs) {
const __m256i res_1 = _mm256_madd_epi16(s[0], coeffs[0]);
const __m256i res_2 = _mm256_madd_epi16(s[1], coeffs[1]);
const __m256i res = _mm256_add_epi32(res_1, res_2);
return res;
}
static INLINE __m256i convolve_lowbd_x(const __m256i data,
const __m256i *const coeffs,
const __m256i *const filt) {
@ -138,9 +172,9 @@ static INLINE void add_store_aligned_256(CONV_BUF_TYPE *const dst,
static INLINE __m256i comp_avg(const __m256i *const data_ref_0,
const __m256i *const res_unsigned,
const __m256i *const wt,
const int use_jnt_comp_avg) {
const int use_dist_wtd_comp_avg) {
__m256i res;
if (use_jnt_comp_avg) {
if (use_dist_wtd_comp_avg) {
const __m256i data_lo = _mm256_unpacklo_epi16(*data_ref_0, *res_unsigned);
const __m256i data_hi = _mm256_unpackhi_epi16(*data_ref_0, *res_unsigned);
@ -172,9 +206,9 @@ static INLINE __m256i highbd_comp_avg(const __m256i *const data_ref_0,
const __m256i *const res_unsigned,
const __m256i *const wt0,
const __m256i *const wt1,
const int use_jnt_comp_avg) {
const int use_dist_wtd_comp_avg) {
__m256i res;
if (use_jnt_comp_avg) {
if (use_dist_wtd_comp_avg) {
const __m256i wt0_res = _mm256_mullo_epi32(*data_ref_0, *wt0);
const __m256i wt1_res = _mm256_mullo_epi32(*res_unsigned, *wt1);
const __m256i wt_res = _mm256_add_epi32(wt0_res, wt1_res);

4
third_party/aom/aom_dsp/x86/convolve_sse2.h vendored
View file

@ -78,9 +78,9 @@ static INLINE __m128i convolve_hi_y(const __m128i *const s,
static INLINE __m128i comp_avg(const __m128i *const data_ref_0,
const __m128i *const res_unsigned,
const __m128i *const wt,
const int use_jnt_comp_avg) {
const int use_dist_wtd_avg) {
__m128i res;
if (use_jnt_comp_avg) {
if (use_dist_wtd_avg) {
const __m128i data_lo = _mm_unpacklo_epi16(*data_ref_0, *res_unsigned);
const __m128i data_hi = _mm_unpackhi_epi16(*data_ref_0, *res_unsigned);

4
third_party/aom/aom_dsp/x86/convolve_sse4_1.h vendored
View file

@ -35,9 +35,9 @@ static INLINE __m128i highbd_comp_avg_sse4_1(const __m128i *const data_ref_0,
const __m128i *const res_unsigned,
const __m128i *const wt0,
const __m128i *const wt1,
const int use_jnt_comp_avg) {
const int use_dist_wtd_avg) {
__m128i res;
if (use_jnt_comp_avg) {
if (use_dist_wtd_avg) {
const __m128i wt0_res = _mm_mullo_epi32(*data_ref_0, *wt0);
const __m128i wt1_res = _mm_mullo_epi32(*res_unsigned, *wt1);

1
third_party/aom/aom_dsp/x86/fft_avx2.c vendored
View file

@ -11,6 +11,7 @@
#include <immintrin.h>
#include "config/aom_dsp_rtcd.h"
#include "aom_dsp/aom_dsp_common.h"
#include "aom_dsp/fft_common.h"

1
third_party/aom/aom_dsp/x86/fft_sse2.c vendored
View file

@ -11,6 +11,7 @@ s * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
#include <xmmintrin.h>
#include "config/aom_dsp_rtcd.h"
#include "aom_dsp/aom_dsp_common.h"
#include "aom_dsp/fft_common.h"

40
third_party/aom/aom_dsp/x86/fwd_txfm_sse2.c vendored
View file

@ -17,46 +17,6 @@
#include "aom_dsp/aom_dsp_common.h"
#include "aom_dsp/x86/fwd_txfm_sse2.h"
void aom_fdct8x8_1_sse2(const int16_t *input, tran_low_t *output, int stride) {
__m128i in0 = _mm_load_si128((const __m128i *)(input + 0 * stride));
__m128i in1 = _mm_load_si128((const __m128i *)(input + 1 * stride));
__m128i in2 = _mm_load_si128((const __m128i *)(input + 2 * stride));
__m128i in3 = _mm_load_si128((const __m128i *)(input + 3 * stride));
__m128i u0, u1, sum;
u0 = _mm_add_epi16(in0, in1);
u1 = _mm_add_epi16(in2, in3);
in0 = _mm_load_si128((const __m128i *)(input + 4 * stride));
in1 = _mm_load_si128((const __m128i *)(input + 5 * stride));
in2 = _mm_load_si128((const __m128i *)(input + 6 * stride));
in3 = _mm_load_si128((const __m128i *)(input + 7 * stride));
sum = _mm_add_epi16(u0, u1);
in0 = _mm_add_epi16(in0, in1);
in2 = _mm_add_epi16(in2, in3);
sum = _mm_add_epi16(sum, in0);
u0 = _mm_setzero_si128();
sum = _mm_add_epi16(sum, in2);
in0 = _mm_unpacklo_epi16(u0, sum);
in1 = _mm_unpackhi_epi16(u0, sum);
in0 = _mm_srai_epi32(in0, 16);
in1 = _mm_srai_epi32(in1, 16);
sum = _mm_add_epi32(in0, in1);
in0 = _mm_unpacklo_epi32(sum, u0);
in1 = _mm_unpackhi_epi32(sum, u0);
sum = _mm_add_epi32(in0, in1);
in0 = _mm_srli_si128(sum, 8);
in1 = _mm_add_epi32(sum, in0);
output[0] = (tran_low_t)_mm_cvtsi128_si32(in1);
}
#define DCT_HIGH_BIT_DEPTH 0
#define FDCT8x8_2D aom_fdct8x8_sse2
#include "aom_dsp/x86/fwd_txfm_impl_sse2.h"

325
third_party/aom/aom_dsp/x86/highbd_convolve_avx2.c vendored
View file

@ -20,6 +20,14 @@
// -----------------------------------------------------------------------------
// Copy and average
static const uint8_t ip_shuffle_f2f3[32] = { 0, 1, 2, 3, 2, 3, 4, 5, 4, 5, 6,
7, 6, 7, 8, 9, 0, 1, 2, 3, 2, 3,
4, 5, 4, 5, 6, 7, 6, 7, 8, 9 };
static const uint8_t ip_shuffle_f4f5[32] = { 4, 5, 6, 7, 6, 7, 8, 9,
8, 9, 10, 11, 10, 11, 12, 13,
4, 5, 6, 7, 6, 7, 8, 9,
8, 9, 10, 11, 10, 11, 12, 13 };
void aom_highbd_convolve_copy_avx2(const uint8_t *src8, ptrdiff_t src_stride,
uint8_t *dst8, ptrdiff_t dst_stride,
const int16_t *filter_x, int filter_x_stride,
@ -444,6 +452,17 @@ static INLINE void pack_filters(const int16_t *filter, __m256i *f /*f[4]*/) {
f[3] = _mm256_shuffle_epi8(hh, p3);
}
static INLINE void pack_filters_4tap(const int16_t *filter,
__m256i *f /*f[4]*/) {
const __m128i h = _mm_loadu_si128((const __m128i *)filter);
const __m256i coeff = _mm256_broadcastsi128_si256(h);
// coeffs 2 3 2 3 2 3 2 3
f[0] = _mm256_shuffle_epi32(coeff, 0x55);
// coeffs 4 5 4 5 4 5 4 5
f[1] = _mm256_shuffle_epi32(coeff, 0xaa);
}
static INLINE void filter_8x1_pixels(const __m256i *sig /*sig[4]*/,
const __m256i *fil /*fil[4]*/,
__m256i *y) {
@ -544,6 +563,176 @@ static void aom_highbd_filter_block1d16_h8_avx2(
} while (height > 0);
}
static void aom_highbd_filter_block1d4_h4_avx2(
const uint16_t *src_ptr, ptrdiff_t src_pitch, uint16_t *dst_ptr,
ptrdiff_t dst_pitch, uint32_t height, const int16_t *filter, int bd) {
const __m256i rounding = _mm256_set1_epi32(1 << (CONV8_ROUNDING_BITS - 1));
__m256i ff[2], s[2];
uint32_t i;
const __m256i clip_pixel =
_mm256_set1_epi16(bd == 10 ? 1023 : (bd == 12 ? 4095 : 255));
const __m256i zero = _mm256_setzero_si256();
static const uint8_t shuffle_mask[32] = { 0, 1, 2, 3, 2, 3, 4, 5, 4, 5, 6,
7, 6, 7, 8, 9, 0, 1, 2, 3, 2, 3,
4, 5, 4, 5, 6, 7, 6, 7, 8, 9 };
__m256i mask = _mm256_loadu_si256((__m256i *)shuffle_mask);
__m256i ip_mask_f2f3 = _mm256_loadu_si256((__m256i *)ip_shuffle_f2f3);
__m256i ip_mask_f4f5 = _mm256_loadu_si256((__m256i *)ip_shuffle_f4f5);
pack_filters_4tap(filter, ff);
src_ptr -= 3;
for (i = 0; i <= (height - 2); i += 2) {
__m256i row0 = _mm256_castsi128_si256(
_mm_loadu_si128((__m128i *)&src_ptr[i * src_pitch + 2]));
__m256i row1 = _mm256_castsi128_si256(
_mm_loadu_si128((__m128i *)&src_ptr[(i + 1) * src_pitch + 2]));
s[0] = _mm256_inserti128_si256(row0, _mm256_castsi256_si128(row1), 1);
s[1] = _mm256_alignr_epi8(s[0], s[0], 4);
s[0] = _mm256_shuffle_epi8(s[0], mask);
s[1] = _mm256_shuffle_epi8(s[1], mask);
__m256i res = convolve_4tap(s, ff);
res =
_mm256_srai_epi32(_mm256_add_epi32(res, rounding), CONV8_ROUNDING_BITS);
res = _mm256_packs_epi32(res, res);
res = _mm256_min_epi16(res, clip_pixel);
res = _mm256_max_epi16(res, zero);
_mm_storel_epi64((__m128i *)&dst_ptr[i * dst_pitch],
_mm256_castsi256_si128(res));
_mm_storel_epi64((__m128i *)&dst_ptr[(i + 1) * dst_pitch],
_mm256_extracti128_si256(res, 1));
}
if (height % 2 != 0) {
i = height - 1;
const __m256i row0_0 = _mm256_castsi128_si256(
_mm_loadu_si128((__m128i *)&src_ptr[i * src_pitch + 2]));
const __m256i row0_1 = _mm256_castsi128_si256(
_mm_loadu_si128((__m128i *)&src_ptr[i * src_pitch + 6]));
const __m256i r0 =
_mm256_inserti128_si256(row0_0, _mm256_castsi256_si128(row0_1), 1);
s[0] = _mm256_shuffle_epi8(r0, ip_mask_f2f3);
s[1] = _mm256_shuffle_epi8(r0, ip_mask_f4f5);
__m256i res = convolve_4tap(s, ff);
res =
_mm256_srai_epi32(_mm256_add_epi32(res, rounding), CONV8_ROUNDING_BITS);
res = _mm256_packs_epi32(res, res);
res = _mm256_min_epi16(res, clip_pixel);
res = _mm256_max_epi16(res, zero);
_mm_storel_epi64((__m128i *)&dst_ptr[i * dst_pitch],
_mm256_castsi256_si128(res));
}
}
static void aom_highbd_filter_block1d8_h4_avx2(
const uint16_t *src_ptr, ptrdiff_t src_pitch, uint16_t *dst_ptr,
ptrdiff_t dst_pitch, uint32_t height, const int16_t *filter, int bd) {
const __m256i rounding = _mm256_set1_epi32(1 << (CONV8_ROUNDING_BITS - 1));
__m256i ff[2], s[2];
uint32_t i = 0;
const __m256i clip_pixel =
_mm256_set1_epi16(bd == 10 ? 1023 : (bd == 12 ? 4095 : 255));
const __m256i zero = _mm256_setzero_si256();
static const uint8_t shuffle_mask[32] = { 0, 1, 8, 9, 2, 3, 10, 11,
4, 5, 12, 13, 6, 7, 14, 15,
0, 1, 8, 9, 2, 3, 10, 11,
4, 5, 12, 13, 6, 7, 14, 15 };
__m256i mask = _mm256_loadu_si256((__m256i *)shuffle_mask);
__m256i ip_mask_f2f3 = _mm256_loadu_si256((__m256i *)ip_shuffle_f2f3);
__m256i ip_mask_f4f5 = _mm256_loadu_si256((__m256i *)ip_shuffle_f4f5);
pack_filters_4tap(filter, ff);
src_ptr -= 3;
/* Horizontal filter */
for (i = 0; i <= (height - 2); i += 2) {
const __m256i row0 =
_mm256_loadu_si256((__m256i *)&src_ptr[i * src_pitch + 2]);
__m256i row1 =
_mm256_loadu_si256((__m256i *)&src_ptr[(i + 1) * src_pitch + 2]);
const __m256i r0 =
_mm256_inserti128_si256(row0, _mm256_castsi256_si128(row1), 1);
const __m256i r1 = _mm256_permute2x128_si256(row0, row1, 0x31);
// even pixels
s[0] = r0;
s[1] = _mm256_alignr_epi8(r1, r0, 4);
__m256i res_even = convolve_4tap(s, ff);
res_even = _mm256_srai_epi32(_mm256_add_epi32(res_even, rounding),
CONV8_ROUNDING_BITS);
// odd pixels
s[0] = _mm256_alignr_epi8(r1, r0, 2);
s[1] = _mm256_alignr_epi8(r1, r0, 6);
__m256i res_odd = convolve_4tap(s, ff);
res_odd = _mm256_srai_epi32(_mm256_add_epi32(res_odd, rounding),
CONV8_ROUNDING_BITS);
__m256i res = _mm256_packs_epi32(res_even, res_odd);
res = _mm256_shuffle_epi8(res, mask);
res = _mm256_min_epi16(res, clip_pixel);
res = _mm256_max_epi16(res, zero);
_mm_storeu_si128((__m128i *)&dst_ptr[i * dst_pitch],
_mm256_castsi256_si128(res));
_mm_storeu_si128((__m128i *)&dst_ptr[i * dst_pitch + dst_pitch],
_mm256_extracti128_si256(res, 1));
}
if (height % 2 != 0) {
i = height - 1;
const __m256i row0_0 =
_mm256_loadu_si256((__m256i *)&src_ptr[i * src_pitch + 2]);
const __m256i row0_1 =
_mm256_loadu_si256((__m256i *)&src_ptr[i * src_pitch + 6]);
const __m256i r0 =
_mm256_inserti128_si256(row0_0, _mm256_castsi256_si128(row0_1), 1);
s[0] = _mm256_shuffle_epi8(r0, ip_mask_f2f3);
s[1] = _mm256_shuffle_epi8(r0, ip_mask_f4f5);
__m256i res = convolve_4tap(s, ff);
res =
_mm256_srai_epi32(_mm256_add_epi32(res, rounding), CONV8_ROUNDING_BITS);
res = _mm256_packs_epi32(res, res);
res = _mm256_min_epi16(res, clip_pixel);
res = _mm256_max_epi16(res, zero);
_mm_storel_epi64((__m128i *)&dst_ptr[i * dst_pitch],
_mm256_castsi256_si128(res));
_mm_storel_epi64((__m128i *)&dst_ptr[i * dst_pitch + 4],
_mm256_extracti128_si256(res, 1));
}
}
static void aom_highbd_filter_block1d16_h4_avx2(
const uint16_t *src_ptr, ptrdiff_t src_pitch, uint16_t *dst_ptr,
ptrdiff_t dst_pitch, uint32_t height, const int16_t *filter, int bd) {
aom_highbd_filter_block1d8_h4_avx2(src_ptr, src_pitch, dst_ptr, dst_pitch,
height, filter, bd);
aom_highbd_filter_block1d8_h4_avx2(src_ptr + 8, src_pitch, dst_ptr + 8,
dst_pitch, height, filter, bd);
}
// -----------------------------------------------------------------------------
// 2-tap horizontal filtering
@ -875,6 +1064,142 @@ static void aom_highbd_filter_block1d16_v8_avx2(
} while (height > 0);
}
static void aom_highbd_filter_block1d4_v4_avx2(
const uint16_t *src_ptr, ptrdiff_t src_pitch, uint16_t *dst_ptr,
ptrdiff_t dst_pitch, uint32_t height, const int16_t *filter, int bd) {
const int bits = FILTER_BITS;
const __m128i round_shift_bits = _mm_cvtsi32_si128(bits);
const __m256i round_const_bits = _mm256_set1_epi32((1 << bits) >> 1);
const __m256i clip_pixel =
_mm256_set1_epi32(bd == 10 ? 1023 : (bd == 12 ? 4095 : 255));
const __m256i zero = _mm256_setzero_si256();
uint32_t i;
__m256i s[2], ff[2];
pack_filters_4tap(filter, ff);
const uint16_t *data = src_ptr;
/* Vertical filter */
{
__m128i s2 = _mm_loadl_epi64((__m128i *)(data + 2 * src_pitch));
__m128i s3 = _mm_loadl_epi64((__m128i *)(data + 3 * src_pitch));
__m256i s23 = _mm256_inserti128_si256(_mm256_castsi128_si256(s2), s3, 1);
__m128i s4 = _mm_loadl_epi64((__m128i *)(data + 4 * src_pitch));
__m256i s34 = _mm256_inserti128_si256(_mm256_castsi128_si256(s3), s4, 1);
s[0] = _mm256_unpacklo_epi16(s23, s34);
for (i = 0; i < height; i += 2) {
data = &src_ptr[i * src_pitch];
__m128i s5 = _mm_loadl_epi64((__m128i *)(data + 5 * src_pitch));
__m128i s6 = _mm_loadl_epi64((__m128i *)(data + 6 * src_pitch));
__m256i s45 = _mm256_inserti128_si256(_mm256_castsi128_si256(s4), s5, 1);
__m256i s56 = _mm256_inserti128_si256(_mm256_castsi128_si256(s5), s6, 1);
s[1] = _mm256_unpacklo_epi16(s45, s56);
const __m256i res_a = convolve_4tap(s, ff);
__m256i res_a_round = _mm256_sra_epi32(
_mm256_add_epi32(res_a, round_const_bits), round_shift_bits);
__m256i res_16bit = _mm256_min_epi32(res_a_round, clip_pixel);
res_16bit = _mm256_max_epi32(res_16bit, zero);
res_16bit = _mm256_packs_epi32(res_16bit, res_16bit);
_mm_storel_epi64((__m128i *)&dst_ptr[i * dst_pitch],
_mm256_castsi256_si128(res_16bit));
_mm_storel_epi64((__m128i *)&dst_ptr[i * dst_pitch + dst_pitch],
_mm256_extracti128_si256(res_16bit, 1));
s[0] = s[1];
s4 = s6;
}
}
}
static void aom_highbd_filter_block1d8_v4_avx2(
const uint16_t *src_ptr, ptrdiff_t src_pitch, uint16_t *dst_ptr,
ptrdiff_t dst_pitch, uint32_t height, const int16_t *filter, int bd) {
const int bits = FILTER_BITS;
const __m128i round_shift_bits = _mm_cvtsi32_si128(bits);
const __m256i round_const_bits = _mm256_set1_epi32((1 << bits) >> 1);
const __m256i clip_pixel =
_mm256_set1_epi16(bd == 10 ? 1023 : (bd == 12 ? 4095 : 255));
const __m256i zero = _mm256_setzero_si256();
__m256i s[4], ff[2];
uint32_t i;
pack_filters_4tap(filter, ff);
const uint16_t *data = src_ptr;
/* Vertical filter */
{
__m128i s2 = _mm_loadu_si128((__m128i *)(data + 2 * src_pitch));
__m128i s3 = _mm_loadu_si128((__m128i *)(data + 3 * src_pitch));
__m256i s23 = _mm256_inserti128_si256(_mm256_castsi128_si256(s2), s3, 1);
__m128i s4 = _mm_loadu_si128((__m128i *)(data + 4 * src_pitch));
__m256i s34 = _mm256_inserti128_si256(_mm256_castsi128_si256(s3), s4, 1);
s[0] = _mm256_unpacklo_epi16(s23, s34);
s[2] = _mm256_unpackhi_epi16(s23, s34);
for (i = 0; i < height; i += 2) {
data = &src_ptr[i * src_pitch];
__m128i s5 = _mm_loadu_si128((__m128i *)(data + 5 * src_pitch));
__m128i s6 = _mm_loadu_si128((__m128i *)(data + 6 * src_pitch));
__m256i s45 = _mm256_inserti128_si256(_mm256_castsi128_si256(s4), s5, 1);
__m256i s56 = _mm256_inserti128_si256(_mm256_castsi128_si256(s5), s6, 1);
s[1] = _mm256_unpacklo_epi16(s45, s56);
s[3] = _mm256_unpackhi_epi16(s45, s56);
const __m256i res_a = convolve_4tap(s, ff);
__m256i res_a_round = _mm256_sra_epi32(
_mm256_add_epi32(res_a, round_const_bits), round_shift_bits);
const __m256i res_b = convolve_4tap(s + 2, ff);
__m256i res_b_round = _mm256_sra_epi32(
_mm256_add_epi32(res_b, round_const_bits), round_shift_bits);
__m256i res_16bit = _mm256_packs_epi32(res_a_round, res_b_round);
res_16bit = _mm256_min_epi16(res_16bit, clip_pixel);
res_16bit = _mm256_max_epi16(res_16bit, zero);
_mm_storeu_si128((__m128i *)&dst_ptr[i * dst_pitch],
_mm256_castsi256_si128(res_16bit));
_mm_storeu_si128((__m128i *)&dst_ptr[i * dst_pitch + dst_pitch],
_mm256_extracti128_si256(res_16bit, 1));
s[0] = s[1];
s[2] = s[3];
s4 = s6;
}
}
}
static void aom_highbd_filter_block1d16_v4_avx2(
const uint16_t *src_ptr, ptrdiff_t src_pitch, uint16_t *dst_ptr,
ptrdiff_t dst_pitch, uint32_t height, const int16_t *filter, int bd) {
aom_highbd_filter_block1d8_v4_avx2(src_ptr, src_pitch, dst_ptr, dst_pitch,
height, filter, bd);
aom_highbd_filter_block1d8_v4_avx2(src_ptr + 8, src_pitch, dst_ptr + 8,
dst_pitch, height, filter, bd);
}
// -----------------------------------------------------------------------------
// 2-tap vertical filtering

351
third_party/aom/aom_dsp/x86/highbd_convolve_sse2.c vendored Normal file
View file

@ -0,0 +1,351 @@
/*
* Copyright (c) 2018, Alliance for Open Media. All rights reserved
*
* This source code is subject to the terms of the BSD 2 Clause License and
* the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
* was not distributed with this source code in the LICENSE file, you can
* obtain it at www.aomedia.org/license/software. If the Alliance for Open
* Media Patent License 1.0 was not distributed with this source code in the
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
*/
#include <emmintrin.h>
#include "config/aom_dsp_rtcd.h"
#include "aom_dsp/x86/convolve.h"
// -----------------------------------------------------------------------------
void aom_highbd_filter_block1d4_v4_sse2(const uint16_t *src_ptr,
ptrdiff_t src_pitch, uint16_t *dst_ptr,
ptrdiff_t dst_pitch, uint32_t height,
const int16_t *filter, int bd) {
__m128i filtersReg;
__m128i srcReg2, srcReg3, srcReg4, srcReg5, srcReg6;
__m128i srcReg23_lo, srcReg34_lo;
__m128i srcReg45_lo, srcReg56_lo;
__m128i resReg23_lo, resReg34_lo, resReg45_lo, resReg56_lo;
__m128i resReg23_45_lo, resReg34_56_lo;
__m128i resReg23_45, resReg34_56;
__m128i addFilterReg64, secondFilters, thirdFilters;
unsigned int i;
ptrdiff_t src_stride, dst_stride;
const __m128i max = _mm_set1_epi16((1 << bd) - 1);
addFilterReg64 = _mm_set1_epi32(64);
filtersReg = _mm_loadu_si128((const __m128i *)filter);
// coeffs 0 1 0 1 2 3 2 3
const __m128i tmp0 = _mm_unpacklo_epi32(filtersReg, filtersReg);
// coeffs 4 5 4 5 6 7 6 7
const __m128i tmp1 = _mm_unpackhi_epi32(filtersReg, filtersReg);
secondFilters = _mm_unpackhi_epi64(tmp0, tmp0); // coeffs 2 3 2 3 2 3 2 3
thirdFilters = _mm_unpacklo_epi64(tmp1, tmp1); // coeffs 4 5 4 5 4 5 4 5
// multiply the size of the source and destination stride by two
src_stride = src_pitch << 1;
dst_stride = dst_pitch << 1;
srcReg2 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 2));
srcReg3 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 3));
srcReg23_lo = _mm_unpacklo_epi16(srcReg2, srcReg3);
srcReg4 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 4));
srcReg34_lo = _mm_unpacklo_epi16(srcReg3, srcReg4);
for (i = height; i > 1; i -= 2) {
srcReg5 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 5));
srcReg45_lo = _mm_unpacklo_epi16(srcReg4, srcReg5);
srcReg6 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 6));
srcReg56_lo = _mm_unpacklo_epi16(srcReg5, srcReg6);
// multiply 2 adjacent elements with the filter and add the result
resReg23_lo = _mm_madd_epi16(srcReg23_lo, secondFilters);
resReg34_lo = _mm_madd_epi16(srcReg34_lo, secondFilters);
resReg45_lo = _mm_madd_epi16(srcReg45_lo, thirdFilters);
resReg56_lo = _mm_madd_epi16(srcReg56_lo, thirdFilters);
resReg23_45_lo = _mm_add_epi32(resReg23_lo, resReg45_lo);
resReg34_56_lo = _mm_add_epi32(resReg34_lo, resReg56_lo);
// shift by 7 bit each 32 bit
resReg23_45_lo = _mm_add_epi32(resReg23_45_lo, addFilterReg64);
resReg34_56_lo = _mm_add_epi32(resReg34_56_lo, addFilterReg64);
resReg23_45_lo = _mm_srai_epi32(resReg23_45_lo, 7);
resReg34_56_lo = _mm_srai_epi32(resReg34_56_lo, 7);
// shrink to 16 bit each 32 bits, the first lane contain the first
// convolve result and the second lane contain the second convolve
// result
resReg23_45 = _mm_packs_epi32(resReg23_45_lo, _mm_setzero_si128());
resReg34_56 = _mm_packs_epi32(resReg34_56_lo, _mm_setzero_si128());
resReg23_45 = _mm_max_epi16(resReg23_45, _mm_setzero_si128());
resReg23_45 = _mm_min_epi16(resReg23_45, max);
resReg34_56 = _mm_max_epi16(resReg34_56, _mm_setzero_si128());
resReg34_56 = _mm_min_epi16(resReg34_56, max);
src_ptr += src_stride;
_mm_storel_epi64((__m128i *)dst_ptr, (resReg23_45));
_mm_storel_epi64((__m128i *)(dst_ptr + dst_pitch), (resReg34_56));
dst_ptr += dst_stride;
// save part of the registers for next strides
srcReg23_lo = srcReg45_lo;
srcReg34_lo = srcReg56_lo;
srcReg4 = srcReg6;
}
}
void aom_highbd_filter_block1d4_h4_sse2(const uint16_t *src_ptr,
ptrdiff_t src_pitch, uint16_t *dst_ptr,
ptrdiff_t dst_pitch, uint32_t height,
const int16_t *filter, int bd) {
__m128i filtersReg;
__m128i addFilterReg64;
__m128i secondFilters, thirdFilters;
__m128i srcRegFilt32b1_1;
__m128i srcReg32b1;
unsigned int i;
src_ptr -= 3;
addFilterReg64 = _mm_set1_epi32(64);
filtersReg = _mm_loadu_si128((const __m128i *)filter);
const __m128i max = _mm_set1_epi16((1 << bd) - 1);
// coeffs 0 1 0 1 2 3 2 3
const __m128i tmp_0 = _mm_unpacklo_epi32(filtersReg, filtersReg);
// coeffs 4 5 4 5 6 7 6 7
const __m128i tmp_1 = _mm_unpackhi_epi32(filtersReg, filtersReg);
secondFilters = _mm_unpackhi_epi64(tmp_0, tmp_0); // coeffs 2 3 2 3 2 3 2 3
thirdFilters = _mm_unpacklo_epi64(tmp_1, tmp_1); // coeffs 4 5 4 5 4 5 4 5
for (i = height; i > 0; i -= 1) {
srcReg32b1 = _mm_loadu_si128((const __m128i *)(src_ptr + 2));
__m128i ss_3_1 = _mm_srli_si128(srcReg32b1, 2);
__m128i ss_4_1 = _mm_srli_si128(srcReg32b1, 4);
__m128i ss_5_1 = _mm_srli_si128(srcReg32b1, 6);
__m128i ss_23 = _mm_unpacklo_epi32(srcReg32b1, ss_3_1);
__m128i ss_45 = _mm_unpacklo_epi32(ss_4_1, ss_5_1);
ss_23 = _mm_madd_epi16(ss_23, secondFilters);
ss_45 = _mm_madd_epi16(ss_45, thirdFilters);
srcRegFilt32b1_1 = _mm_add_epi32(ss_23, ss_45);
// shift by 7 bit each 32 bit
srcRegFilt32b1_1 = _mm_add_epi32(srcRegFilt32b1_1, addFilterReg64);
srcRegFilt32b1_1 = _mm_srai_epi32(srcRegFilt32b1_1, 7);
srcRegFilt32b1_1 = _mm_packs_epi32(srcRegFilt32b1_1, _mm_setzero_si128());
srcRegFilt32b1_1 = _mm_max_epi16(srcRegFilt32b1_1, _mm_setzero_si128());
srcRegFilt32b1_1 = _mm_min_epi16(srcRegFilt32b1_1, max);
src_ptr += src_pitch;
_mm_storel_epi64((__m128i *)dst_ptr, srcRegFilt32b1_1);
dst_ptr += dst_pitch;
}
}
void aom_highbd_filter_block1d8_v4_sse2(const uint16_t *src_ptr,
ptrdiff_t src_pitch, uint16_t *dst_ptr,
ptrdiff_t dst_pitch, uint32_t height,
const int16_t *filter, int bd) {
__m128i filtersReg;
__m128i srcReg2, srcReg3, srcReg4, srcReg5, srcReg6;
__m128i srcReg23_lo, srcReg23_hi, srcReg34_lo, srcReg34_hi;
__m128i srcReg45_lo, srcReg45_hi, srcReg56_lo, srcReg56_hi;
__m128i resReg23_lo, resReg34_lo, resReg45_lo, resReg56_lo;
__m128i resReg23_hi, resReg34_hi, resReg45_hi, resReg56_hi;
__m128i resReg23_45_lo, resReg34_56_lo, resReg23_45_hi, resReg34_56_hi;
__m128i resReg23_45, resReg34_56;
__m128i addFilterReg64, secondFilters, thirdFilters;
unsigned int i;
ptrdiff_t src_stride, dst_stride;
const __m128i max = _mm_set1_epi16((1 << bd) - 1);
addFilterReg64 = _mm_set1_epi32(64);
filtersReg = _mm_loadu_si128((const __m128i *)filter);
// coeffs 0 1 0 1 2 3 2 3
const __m128i tmp0 = _mm_unpacklo_epi32(filtersReg, filtersReg);
// coeffs 4 5 4 5 6 7 6 7
const __m128i tmp1 = _mm_unpackhi_epi32(filtersReg, filtersReg);
secondFilters = _mm_unpackhi_epi64(tmp0, tmp0); // coeffs 2 3 2 3 2 3 2 3
thirdFilters = _mm_unpacklo_epi64(tmp1, tmp1); // coeffs 4 5 4 5 4 5 4 5
// multiple the size of the source and destination stride by two
src_stride = src_pitch << 1;
dst_stride = dst_pitch << 1;
srcReg2 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 2));
srcReg3 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 3));
srcReg23_lo = _mm_unpacklo_epi16(srcReg2, srcReg3);
srcReg23_hi = _mm_unpackhi_epi16(srcReg2, srcReg3);
srcReg4 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 4));
srcReg34_lo = _mm_unpacklo_epi16(srcReg3, srcReg4);
srcReg34_hi = _mm_unpackhi_epi16(srcReg3, srcReg4);
for (i = height; i > 1; i -= 2) {
srcReg5 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 5));
srcReg45_lo = _mm_unpacklo_epi16(srcReg4, srcReg5);
srcReg45_hi = _mm_unpackhi_epi16(srcReg4, srcReg5);
srcReg6 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 6));
srcReg56_lo = _mm_unpacklo_epi16(srcReg5, srcReg6);
srcReg56_hi = _mm_unpackhi_epi16(srcReg5, srcReg6);
// multiply 2 adjacent elements with the filter and add the result
resReg23_lo = _mm_madd_epi16(srcReg23_lo, secondFilters);
resReg34_lo = _mm_madd_epi16(srcReg34_lo, secondFilters);
resReg45_lo = _mm_madd_epi16(srcReg45_lo, thirdFilters);
resReg56_lo = _mm_madd_epi16(srcReg56_lo, thirdFilters);
resReg23_45_lo = _mm_add_epi32(resReg23_lo, resReg45_lo);
resReg34_56_lo = _mm_add_epi32(resReg34_lo, resReg56_lo);
// multiply 2 adjacent elements with the filter and add the result
resReg23_hi = _mm_madd_epi16(srcReg23_hi, secondFilters);
resReg34_hi = _mm_madd_epi16(srcReg34_hi, secondFilters);
resReg45_hi = _mm_madd_epi16(srcReg45_hi, thirdFilters);
resReg56_hi = _mm_madd_epi16(srcReg56_hi, thirdFilters);
resReg23_45_hi = _mm_add_epi32(resReg23_hi, resReg45_hi);
resReg34_56_hi = _mm_add_epi32(resReg34_hi, resReg56_hi);
// shift by 7 bit each 32 bit
resReg23_45_lo = _mm_add_epi32(resReg23_45_lo, addFilterReg64);
resReg34_56_lo = _mm_add_epi32(resReg34_56_lo, addFilterReg64);
resReg23_45_hi = _mm_add_epi32(resReg23_45_hi, addFilterReg64);
resReg34_56_hi = _mm_add_epi32(resReg34_56_hi, addFilterReg64);
resReg23_45_lo = _mm_srai_epi32(resReg23_45_lo, 7);
resReg34_56_lo = _mm_srai_epi32(resReg34_56_lo, 7);
resReg23_45_hi = _mm_srai_epi32(resReg23_45_hi, 7);
resReg34_56_hi = _mm_srai_epi32(resReg34_56_hi, 7);
// shrink to 16 bit each 32 bits, the first lane contain the first
// convolve result and the second lane contain the second convolve
// result
resReg23_45 = _mm_packs_epi32(resReg23_45_lo, resReg23_45_hi);
resReg34_56 = _mm_packs_epi32(resReg34_56_lo, resReg34_56_hi);
resReg23_45 = _mm_max_epi16(resReg23_45, _mm_setzero_si128());
resReg23_45 = _mm_min_epi16(resReg23_45, max);
resReg34_56 = _mm_max_epi16(resReg34_56, _mm_setzero_si128());
resReg34_56 = _mm_min_epi16(resReg34_56, max);
src_ptr += src_stride;
_mm_store_si128((__m128i *)dst_ptr, (resReg23_45));
_mm_store_si128((__m128i *)(dst_ptr + dst_pitch), (resReg34_56));
dst_ptr += dst_stride;
// save part of the registers for next strides
srcReg23_lo = srcReg45_lo;
srcReg23_hi = srcReg45_hi;
srcReg34_lo = srcReg56_lo;
srcReg34_hi = srcReg56_hi;
srcReg4 = srcReg6;
}
}
void aom_highbd_filter_block1d8_h4_sse2(const uint16_t *src_ptr,
ptrdiff_t src_pitch, uint16_t *dst_ptr,
ptrdiff_t dst_pitch, uint32_t height,
const int16_t *filter, int bd) {
__m128i filtersReg;
__m128i addFilterReg64;
__m128i secondFilters, thirdFilters;
__m128i srcRegFilt32b1_1, srcRegFilt32b1_2;
__m128i srcReg32b1, srcReg32b2;
unsigned int i;
src_ptr -= 3;
addFilterReg64 = _mm_set1_epi32(64);
filtersReg = _mm_loadu_si128((const __m128i *)filter);
const __m128i max = _mm_set1_epi16((1 << bd) - 1);
// coeffs 0 1 0 1 2 3 2 3
const __m128i tmp_0 = _mm_unpacklo_epi32(filtersReg, filtersReg);
// coeffs 4 5 4 5 6 7 6 7
const __m128i tmp_1 = _mm_unpackhi_epi32(filtersReg, filtersReg);
secondFilters = _mm_unpackhi_epi64(tmp_0, tmp_0); // coeffs 2 3 2 3 2 3 2 3
thirdFilters = _mm_unpacklo_epi64(tmp_1, tmp_1); // coeffs 4 5 4 5 4 5 4 5
for (i = height; i > 0; i -= 1) {
srcReg32b1 = _mm_loadu_si128((const __m128i *)(src_ptr + 2));
srcReg32b2 = _mm_loadu_si128((const __m128i *)(src_ptr + 6));
__m128i ss_4_1 = _mm_srli_si128(srcReg32b1, 4);
__m128i ss_4_2 = _mm_srli_si128(srcReg32b2, 4);
__m128i ss_4 = _mm_unpacklo_epi64(ss_4_1, ss_4_2);
__m128i d1 = _mm_madd_epi16(srcReg32b1, secondFilters);
__m128i d2 = _mm_madd_epi16(ss_4, thirdFilters);
srcRegFilt32b1_1 = _mm_add_epi32(d1, d2);
__m128i ss_3_1 = _mm_srli_si128(srcReg32b1, 2);
__m128i ss_5_1 = _mm_srli_si128(srcReg32b1, 6);
__m128i ss_3_2 = _mm_srli_si128(srcReg32b2, 2);
__m128i ss_5_2 = _mm_srli_si128(srcReg32b2, 6);
__m128i ss_3 = _mm_unpacklo_epi64(ss_3_1, ss_3_2);
__m128i ss_5 = _mm_unpacklo_epi64(ss_5_1, ss_5_2);
d1 = _mm_madd_epi16(ss_3, secondFilters);
d2 = _mm_madd_epi16(ss_5, thirdFilters);
srcRegFilt32b1_2 = _mm_add_epi32(d1, d2);
__m128i res_lo_1 = _mm_unpacklo_epi32(srcRegFilt32b1_1, srcRegFilt32b1_2);
__m128i res_hi_1 = _mm_unpackhi_epi32(srcRegFilt32b1_1, srcRegFilt32b1_2);
// shift by 7 bit each 32 bit
res_lo_1 = _mm_add_epi32(res_lo_1, addFilterReg64);
res_hi_1 = _mm_add_epi32(res_hi_1, addFilterReg64);
res_lo_1 = _mm_srai_epi32(res_lo_1, 7);
res_hi_1 = _mm_srai_epi32(res_hi_1, 7);
srcRegFilt32b1_1 = _mm_packs_epi32(res_lo_1, res_hi_1);
srcRegFilt32b1_1 = _mm_max_epi16(srcRegFilt32b1_1, _mm_setzero_si128());
srcRegFilt32b1_1 = _mm_min_epi16(srcRegFilt32b1_1, max);
src_ptr += src_pitch;
_mm_store_si128((__m128i *)dst_ptr, srcRegFilt32b1_1);
dst_ptr += dst_pitch;
}
}
void aom_highbd_filter_block1d16_v4_sse2(const uint16_t *src_ptr,
ptrdiff_t src_pitch, uint16_t *dst_ptr,
ptrdiff_t dst_pitch, uint32_t height,
const int16_t *filter, int bd) {
aom_highbd_filter_block1d8_v4_sse2(src_ptr, src_pitch, dst_ptr, dst_pitch,
height, filter, bd);
aom_highbd_filter_block1d8_v4_sse2((src_ptr + 8), src_pitch, (dst_ptr + 8),
dst_pitch, height, filter, bd);
}
void aom_highbd_filter_block1d16_h4_sse2(const uint16_t *src_ptr,
ptrdiff_t src_pitch, uint16_t *dst_ptr,
ptrdiff_t dst_pitch, uint32_t height,
const int16_t *filter, int bd) {
aom_highbd_filter_block1d8_h4_sse2(src_ptr, src_pitch, dst_ptr, dst_pitch,
height, filter, bd);
aom_highbd_filter_block1d8_h4_sse2((src_ptr + 8), src_pitch, (dst_ptr + 8),
dst_pitch, height, filter, bd);
}

259
third_party/aom/aom_dsp/x86/highbd_intrapred_asm_sse2.asm vendored Normal file
View file

@ -0,0 +1,259 @@
;
; Copyright (c) 2016, Alliance for Open Media. All rights reserved
;
; This source code is subject to the terms of the BSD 2 Clause License and
; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
; was not distributed with this source code in the LICENSE file, you can
; obtain it at www.aomedia.org/license/software. If the Alliance for Open
; Media Patent License 1.0 was not distributed with this source code in the
; PATENTS file, you can obtain it at www.aomedia.org/license/patent.
;
;
%include "third_party/x86inc/x86inc.asm"
SECTION_RODATA
pw_4: times 8 dw 4
pw_8: times 8 dw 8
pw_16: times 4 dd 16
pw_32: times 4 dd 32
SECTION .text
INIT_XMM sse2
cglobal highbd_dc_predictor_4x4, 4, 5, 4, dst, stride, above, left, goffset
GET_GOT goffsetq
movq m0, [aboveq]
movq m2, [leftq]
paddw m0, m2
pshuflw m1, m0, 0xe
paddw m0, m1
pshuflw m1, m0, 0x1
paddw m0, m1
paddw m0, [GLOBAL(pw_4)]
psraw m0, 3
pshuflw m0, m0, 0x0
movq [dstq ], m0
movq [dstq+strideq*2], m0
lea dstq, [dstq+strideq*4]
movq [dstq ], m0
movq [dstq+strideq*2], m0
RESTORE_GOT
RET
INIT_XMM sse2
cglobal highbd_dc_predictor_8x8, 4, 5, 4, dst, stride, above, left, goffset
GET_GOT goffsetq
pxor m1, m1
mova m0, [aboveq]
mova m2, [leftq]
DEFINE_ARGS dst, stride, stride3, one
mov oned, 0x00010001
lea stride3q, [strideq*3]
movd m3, oned
pshufd m3, m3, 0x0
paddw m0, m2
pmaddwd m0, m3
packssdw m0, m1
pmaddwd m0, m3
packssdw m0, m1
pmaddwd m0, m3
paddw m0, [GLOBAL(pw_8)]
psrlw m0, 4
pshuflw m0, m0, 0x0
punpcklqdq m0, m0
mova [dstq ], m0
mova [dstq+strideq*2 ], m0
mova [dstq+strideq*4 ], m0
mova [dstq+stride3q*2], m0
lea dstq, [dstq+strideq*8]
mova [dstq ], m0
mova [dstq+strideq*2 ], m0
mova [dstq+strideq*4 ], m0
mova [dstq+stride3q*2], m0
RESTORE_GOT
RET
INIT_XMM sse2
cglobal highbd_dc_predictor_16x16, 4, 5, 5, dst, stride, above, left, goffset
GET_GOT goffsetq
pxor m1, m1
mova m0, [aboveq]
mova m3, [aboveq+16]
mova m2, [leftq]
mova m4, [leftq+16]
DEFINE_ARGS dst, stride, stride3, lines4
lea stride3q, [strideq*3]
mov lines4d, 4
paddw m0, m2
paddw m0, m3
paddw m0, m4
movhlps m2, m0
paddw m0, m2
punpcklwd m0, m1
movhlps m2, m0
paddd m0, m2
punpckldq m0, m1
movhlps m2, m0
paddd m0, m2
paddd m0, [GLOBAL(pw_16)]
psrad m0, 5
pshuflw m0, m0, 0x0
punpcklqdq m0, m0
.loop:
mova [dstq ], m0
mova [dstq +16], m0
mova [dstq+strideq*2 ], m0
mova [dstq+strideq*2 +16], m0
mova [dstq+strideq*4 ], m0
mova [dstq+strideq*4 +16], m0
mova [dstq+stride3q*2 ], m0
mova [dstq+stride3q*2+16], m0
lea dstq, [dstq+strideq*8]
dec lines4d
jnz .loop
RESTORE_GOT
REP_RET
INIT_XMM sse2
cglobal highbd_dc_predictor_32x32, 4, 5, 7, dst, stride, above, left, goffset
GET_GOT goffsetq
mova m0, [aboveq]
mova m2, [aboveq+16]
mova m3, [aboveq+32]
mova m4, [aboveq+48]
paddw m0, m2
paddw m3, m4
mova m2, [leftq]
mova m4, [leftq+16]
mova m5, [leftq+32]
mova m6, [leftq+48]
paddw m2, m4
paddw m5, m6
paddw m0, m3
paddw m2, m5
pxor m1, m1
paddw m0, m2
DEFINE_ARGS dst, stride, stride3, lines4
lea stride3q, [strideq*3]
mov lines4d, 8
movhlps m2, m0
paddw m0, m2
punpcklwd m0, m1
movhlps m2, m0
paddd m0, m2
punpckldq m0, m1
movhlps m2, m0
paddd m0, m2
paddd m0, [GLOBAL(pw_32)]
psrad m0, 6
pshuflw m0, m0, 0x0
punpcklqdq m0, m0
.loop:
mova [dstq ], m0
mova [dstq +16 ], m0
mova [dstq +32 ], m0
mova [dstq +48 ], m0
mova [dstq+strideq*2 ], m0
mova [dstq+strideq*2+16 ], m0
mova [dstq+strideq*2+32 ], m0
mova [dstq+strideq*2+48 ], m0
mova [dstq+strideq*4 ], m0
mova [dstq+strideq*4+16 ], m0
mova [dstq+strideq*4+32 ], m0
mova [dstq+strideq*4+48 ], m0
mova [dstq+stride3q*2 ], m0
mova [dstq+stride3q*2 +16], m0
mova [dstq+stride3q*2 +32], m0
mova [dstq+stride3q*2 +48], m0
lea dstq, [dstq+strideq*8]
dec lines4d
jnz .loop
RESTORE_GOT
REP_RET
INIT_XMM sse2
cglobal highbd_v_predictor_4x4, 3, 3, 1, dst, stride, above
movq m0, [aboveq]
movq [dstq ], m0
movq [dstq+strideq*2], m0
lea dstq, [dstq+strideq*4]
movq [dstq ], m0
movq [dstq+strideq*2], m0
RET
INIT_XMM sse2
cglobal highbd_v_predictor_8x8, 3, 3, 1, dst, stride, above
mova m0, [aboveq]
DEFINE_ARGS dst, stride, stride3
lea stride3q, [strideq*3]
mova [dstq ], m0
mova [dstq+strideq*2 ], m0
mova [dstq+strideq*4 ], m0
mova [dstq+stride3q*2], m0
lea dstq, [dstq+strideq*8]
mova [dstq ], m0
mova [dstq+strideq*2 ], m0
mova [dstq+strideq*4 ], m0
mova [dstq+stride3q*2], m0
RET
INIT_XMM sse2
cglobal highbd_v_predictor_16x16, 3, 4, 2, dst, stride, above
mova m0, [aboveq]
mova m1, [aboveq+16]
DEFINE_ARGS dst, stride, stride3, nlines4
lea stride3q, [strideq*3]
mov nlines4d, 4
.loop:
mova [dstq ], m0
mova [dstq +16], m1
mova [dstq+strideq*2 ], m0
mova [dstq+strideq*2 +16], m1
mova [dstq+strideq*4 ], m0
mova [dstq+strideq*4 +16], m1
mova [dstq+stride3q*2 ], m0
mova [dstq+stride3q*2+16], m1
lea dstq, [dstq+strideq*8]
dec nlines4d
jnz .loop
REP_RET
INIT_XMM sse2
cglobal highbd_v_predictor_32x32, 3, 4, 4, dst, stride, above
mova m0, [aboveq]
mova m1, [aboveq+16]
mova m2, [aboveq+32]
mova m3, [aboveq+48]
DEFINE_ARGS dst, stride, stride3, nlines4
lea stride3q, [strideq*3]
mov nlines4d, 8
.loop:
mova [dstq ], m0
mova [dstq +16], m1
mova [dstq +32], m2
mova [dstq +48], m3
mova [dstq+strideq*2 ], m0
mova [dstq+strideq*2 +16], m1
mova [dstq+strideq*2 +32], m2
mova [dstq+strideq*2 +48], m3
mova [dstq+strideq*4 ], m0
mova [dstq+strideq*4 +16], m1
mova [dstq+strideq*4 +32], m2
mova [dstq+strideq*4 +48], m3
mova [dstq+stride3q*2 ], m0
mova [dstq+stride3q*2 +16], m1
mova [dstq+stride3q*2 +32], m2
mova [dstq+stride3q*2 +48], m3
lea dstq, [dstq+strideq*8]
dec nlines4d
jnz .loop
REP_RET

4
third_party/aom/aom_dsp/x86/highbd_loopfilter_sse2.c vendored
View file

@ -727,8 +727,8 @@ void aom_highbd_lpf_horizontal_14_dual_sse2(
_limit1, _thresh1, bd);
for (i = 0; i < 6; i++) {
_mm_store_si128((__m128i *)(s - (i + 1) * pitch), p[i]);
_mm_store_si128((__m128i *)(s + i * pitch), q[i]);
_mm_storeu_si128((__m128i *)(s - (i + 1) * pitch), p[i]);
_mm_storeu_si128((__m128i *)(s + i * pitch), q[i]);
}
}

52
third_party/aom/aom_dsp/x86/highbd_subpel_variance_impl_sse2.asm vendored
View file

@ -114,45 +114,33 @@ SECTION .text
cglobal highbd_sub_pixel_avg_variance%1xh, 7, 7, 13, src, src_stride, \
x_offset, y_offset, \
dst, dst_stride, \
sec, sec_stride, height, sse, \
g_bilin_filter, g_pw_8
sec, sec_stride, height, sse
%define block_height dword heightm
%define sec_str sec_stridemp
; Store bilin_filter and pw_8 location in stack
%if GET_GOT_DEFINED == 1
GET_GOT eax
add esp, 4 ; restore esp
%endif
lea ecx, [GLOBAL(bilin_filter_m)]
mov g_bilin_filterm, ecx
lea ecx, [GLOBAL(pw_8)]
mov g_pw_8m, ecx
LOAD_IF_USED 0, 1 ; load eax, ecx back
%else
cglobal highbd_sub_pixel_variance%1xh, 7, 7, 13, src, src_stride, \
x_offset, y_offset, \
dst, dst_stride, height, sse, \
g_bilin_filter, g_pw_8
dst, dst_stride, height, sse
%define block_height heightd
; Store bilin_filter and pw_8 location in stack
%if GET_GOT_DEFINED == 1
GET_GOT eax
add esp, 4 ; restore esp
%endif
lea ecx, [GLOBAL(bilin_filter_m)]
mov g_bilin_filterm, ecx
lea ecx, [GLOBAL(pw_8)]
mov g_pw_8m, ecx
LOAD_IF_USED 0, 1 ; load eax, ecx back
%endif
; reuse argument stack space
%define g_bilin_filterm x_offsetm
%define g_pw_8m y_offsetm
; Store bilin_filter and pw_8 location in stack
%if GET_GOT_DEFINED == 1
GET_GOT eax
add esp, 4 ; restore esp
%endif
lea ecx, [GLOBAL(bilin_filter_m)]
mov g_bilin_filterm, ecx
lea ecx, [GLOBAL(pw_8)]
mov g_pw_8m, ecx
LOAD_IF_USED 0, 1 ; load eax, ecx back
%else
%if %2 == 1 ; avg
cglobal highbd_sub_pixel_avg_variance%1xh, 7, 7, 13, src, src_stride, \

48
third_party/aom/aom_dsp/x86/highbd_subtract_sse2.c vendored
View file

@ -29,15 +29,15 @@ static void subtract_4x4(int16_t *diff, ptrdiff_t diff_stride,
__m128i x0, x1, x2, x3;
int64_t *store_diff = (int64_t *)(diff + 0 * diff_stride);
u0 = _mm_loadu_si128((__m128i const *)(src + 0 * src_stride));
u1 = _mm_loadu_si128((__m128i const *)(src + 1 * src_stride));
u2 = _mm_loadu_si128((__m128i const *)(src + 2 * src_stride));
u3 = _mm_loadu_si128((__m128i const *)(src + 3 * src_stride));
u0 = _mm_loadl_epi64((__m128i const *)(src + 0 * src_stride));
u1 = _mm_loadl_epi64((__m128i const *)(src + 1 * src_stride));
u2 = _mm_loadl_epi64((__m128i const *)(src + 2 * src_stride));
u3 = _mm_loadl_epi64((__m128i const *)(src + 3 * src_stride));
v0 = _mm_loadu_si128((__m128i const *)(pred + 0 * pred_stride));
v1 = _mm_loadu_si128((__m128i const *)(pred + 1 * pred_stride));
v2 = _mm_loadu_si128((__m128i const *)(pred + 2 * pred_stride));
v3 = _mm_loadu_si128((__m128i const *)(pred + 3 * pred_stride));
v0 = _mm_loadl_epi64((__m128i const *)(pred + 0 * pred_stride));
v1 = _mm_loadl_epi64((__m128i const *)(pred + 1 * pred_stride));
v2 = _mm_loadl_epi64((__m128i const *)(pred + 2 * pred_stride));
v3 = _mm_loadl_epi64((__m128i const *)(pred + 3 * pred_stride));
x0 = _mm_sub_epi16(u0, v0);
x1 = _mm_sub_epi16(u1, v1);
@ -61,23 +61,23 @@ static void subtract_4x8(int16_t *diff, ptrdiff_t diff_stride,
__m128i x0, x1, x2, x3, x4, x5, x6, x7;
int64_t *store_diff = (int64_t *)(diff + 0 * diff_stride);
u0 = _mm_loadu_si128((__m128i const *)(src + 0 * src_stride));
u1 = _mm_loadu_si128((__m128i const *)(src + 1 * src_stride));
u2 = _mm_loadu_si128((__m128i const *)(src + 2 * src_stride));
u3 = _mm_loadu_si128((__m128i const *)(src + 3 * src_stride));
u4 = _mm_loadu_si128((__m128i const *)(src + 4 * src_stride));
u5 = _mm_loadu_si128((__m128i const *)(src + 5 * src_stride));
u6 = _mm_loadu_si128((__m128i const *)(src + 6 * src_stride));
u7 = _mm_loadu_si128((__m128i const *)(src + 7 * src_stride));
u0 = _mm_loadl_epi64((__m128i const *)(src + 0 * src_stride));
u1 = _mm_loadl_epi64((__m128i const *)(src + 1 * src_stride));
u2 = _mm_loadl_epi64((__m128i const *)(src + 2 * src_stride));
u3 = _mm_loadl_epi64((__m128i const *)(src + 3 * src_stride));
u4 = _mm_loadl_epi64((__m128i const *)(src + 4 * src_stride));
u5 = _mm_loadl_epi64((__m128i const *)(src + 5 * src_stride));
u6 = _mm_loadl_epi64((__m128i const *)(src + 6 * src_stride));
u7 = _mm_loadl_epi64((__m128i const *)(src + 7 * src_stride));
v0 = _mm_loadu_si128((__m128i const *)(pred + 0 * pred_stride));
v1 = _mm_loadu_si128((__m128i const *)(pred + 1 * pred_stride));
v2 = _mm_loadu_si128((__m128i const *)(pred + 2 * pred_stride));
v3 = _mm_loadu_si128((__m128i const *)(pred + 3 * pred_stride));
v4 = _mm_loadu_si128((__m128i const *)(pred + 4 * pred_stride));
v5 = _mm_loadu_si128((__m128i const *)(pred + 5 * pred_stride));
v6 = _mm_loadu_si128((__m128i const *)(pred + 6 * pred_stride));
v7 = _mm_loadu_si128((__m128i const *)(pred + 7 * pred_stride));
v0 = _mm_loadl_epi64((__m128i const *)(pred + 0 * pred_stride));
v1 = _mm_loadl_epi64((__m128i const *)(pred + 1 * pred_stride));
v2 = _mm_loadl_epi64((__m128i const *)(pred + 2 * pred_stride));
v3 = _mm_loadl_epi64((__m128i const *)(pred + 3 * pred_stride));
v4 = _mm_loadl_epi64((__m128i const *)(pred + 4 * pred_stride));
v5 = _mm_loadl_epi64((__m128i const *)(pred + 5 * pred_stride));
v6 = _mm_loadl_epi64((__m128i const *)(pred + 6 * pred_stride));
v7 = _mm_loadl_epi64((__m128i const *)(pred + 7 * pred_stride));
x0 = _mm_sub_epi16(u0, v0);
x1 = _mm_sub_epi16(u1, v1);

59
third_party/aom/aom_dsp/x86/highbd_variance_sse2.c vendored
View file

@ -603,7 +603,7 @@ void aom_highbd_upsampled_pred_sse2(MACROBLOCKD *xd,
const int ref_num = 0;
const int is_intrabc = is_intrabc_block(mi);
const struct scale_factors *const sf =
is_intrabc ? &cm->sf_identity : &xd->block_refs[ref_num]->sf;
is_intrabc ? &cm->sf_identity : xd->block_ref_scale_factors[ref_num];
const int is_scaled = av1_is_scaled(sf);
if (is_scaled) {
@ -677,11 +677,8 @@ void aom_highbd_upsampled_pred_sse2(MACROBLOCKD *xd,
}
}
const InterpFilterParams *filter =
(subpel_search == 1)
? av1_get_4tap_interp_filter_params(EIGHTTAP_REGULAR)
: av1_get_interp_filter_params_with_block_size(EIGHTTAP_REGULAR, 8);
const InterpFilterParams *filter = av1_get_filter(subpel_search);
int filter_taps = (subpel_search <= USE_4_TAPS) ? 4 : SUBPEL_TAPS;
if (!subpel_x_q3 && !subpel_y_q3) {
uint16_t *ref = CONVERT_TO_SHORTPTR(ref8);
uint16_t *comp_pred = CONVERT_TO_SHORTPTR(comp_pred8);
@ -729,17 +726,20 @@ void aom_highbd_upsampled_pred_sse2(MACROBLOCKD *xd,
av1_get_interp_filter_subpel_kernel(filter, subpel_x_q3 << 1);
const int16_t *const kernel_y =
av1_get_interp_filter_subpel_kernel(filter, subpel_y_q3 << 1);
const uint8_t *ref_start = ref8 - ref_stride * ((filter_taps >> 1) - 1);
uint16_t *temp_start_horiz = (subpel_search <= USE_4_TAPS)
? temp + (filter_taps >> 1) * MAX_SB_SIZE
: temp;
uint16_t *temp_start_vert = temp + MAX_SB_SIZE * ((filter->taps >> 1) - 1);
const int intermediate_height =
(((height - 1) * 8 + subpel_y_q3) >> 3) + filter->taps;
(((height - 1) * 8 + subpel_y_q3) >> 3) + filter_taps;
assert(intermediate_height <= (MAX_SB_SIZE * 2 + 16) + 16);
aom_highbd_convolve8_horiz(ref8 - ref_stride * ((filter->taps >> 1) - 1),
ref_stride, CONVERT_TO_BYTEPTR(temp),
MAX_SB_SIZE, kernel_x, 16, NULL, -1, width,
intermediate_height, bd);
aom_highbd_convolve8_vert(
CONVERT_TO_BYTEPTR(temp + MAX_SB_SIZE * ((filter->taps >> 1) - 1)),
MAX_SB_SIZE, comp_pred8, width, NULL, -1, kernel_y, 16, width, height,
bd);
aom_highbd_convolve8_horiz(
ref_start, ref_stride, CONVERT_TO_BYTEPTR(temp_start_horiz),
MAX_SB_SIZE, kernel_x, 16, NULL, -1, width, intermediate_height, bd);
aom_highbd_convolve8_vert(CONVERT_TO_BYTEPTR(temp_start_vert), MAX_SB_SIZE,
comp_pred8, width, NULL, -1, kernel_y, 16, width,
height, bd);
}
}
@ -765,11 +765,11 @@ void aom_highbd_comp_avg_upsampled_pred_sse2(
}
}
static INLINE void highbd_compute_jnt_comp_avg(__m128i *p0, __m128i *p1,
const __m128i *w0,
const __m128i *w1,
const __m128i *r,
void *const result) {
static INLINE void highbd_compute_dist_wtd_comp_avg(__m128i *p0, __m128i *p1,
const __m128i *w0,
const __m128i *w1,
const __m128i *r,
void *const result) {
assert(DIST_PRECISION_BITS <= 4);
__m128i mult0 = _mm_mullo_epi16(*p0, *w0);
__m128i mult1 = _mm_mullo_epi16(*p1, *w1);
@ -780,11 +780,10 @@ static INLINE void highbd_compute_jnt_comp_avg(__m128i *p0, __m128i *p1,
xx_storeu_128(result, shift);
}
void aom_highbd_jnt_comp_avg_pred_sse2(uint8_t *comp_pred8,
const uint8_t *pred8, int width,
int height, const uint8_t *ref8,
int ref_stride,
const JNT_COMP_PARAMS *jcp_param) {
void aom_highbd_dist_wtd_comp_avg_pred_sse2(
uint8_t *comp_pred8, const uint8_t *pred8, int width, int height,
const uint8_t *ref8, int ref_stride,
const DIST_WTD_COMP_PARAMS *jcp_param) {
int i;
const uint16_t wt0 = (uint16_t)jcp_param->fwd_offset;
const uint16_t wt1 = (uint16_t)jcp_param->bck_offset;
@ -806,7 +805,7 @@ void aom_highbd_jnt_comp_avg_pred_sse2(uint8_t *comp_pred8,
__m128i p0 = xx_loadu_128(ref);
__m128i p1 = xx_loadu_128(pred);
highbd_compute_jnt_comp_avg(&p0, &p1, &w0, &w1, &r, comp_pred);
highbd_compute_dist_wtd_comp_avg(&p0, &p1, &w0, &w1, &r, comp_pred);
comp_pred += 8;
pred += 8;
@ -823,7 +822,7 @@ void aom_highbd_jnt_comp_avg_pred_sse2(uint8_t *comp_pred8,
__m128i p0 = _mm_unpacklo_epi64(p0_0, p0_1);
__m128i p1 = xx_loadu_128(pred);
highbd_compute_jnt_comp_avg(&p0, &p1, &w0, &w1, &r, comp_pred);
highbd_compute_dist_wtd_comp_avg(&p0, &p1, &w0, &w1, &r, comp_pred);
comp_pred += 8;
pred += 8;
@ -832,11 +831,11 @@ void aom_highbd_jnt_comp_avg_pred_sse2(uint8_t *comp_pred8,
}
}
void aom_highbd_jnt_comp_avg_upsampled_pred_sse2(
void aom_highbd_dist_wtd_comp_avg_upsampled_pred_sse2(
MACROBLOCKD *xd, const struct AV1Common *const cm, int mi_row, int mi_col,
const MV *const mv, uint8_t *comp_pred8, const uint8_t *pred8, int width,
int height, int subpel_x_q3, int subpel_y_q3, const uint8_t *ref8,
int ref_stride, int bd, const JNT_COMP_PARAMS *jcp_param,
int ref_stride, int bd, const DIST_WTD_COMP_PARAMS *jcp_param,
int subpel_search) {
uint16_t *pred = CONVERT_TO_SHORTPTR(pred8);
int n;
@ -860,7 +859,7 @@ void aom_highbd_jnt_comp_avg_upsampled_pred_sse2(
__m128i p0 = xx_loadu_128(comp_pred16);
__m128i p1 = xx_loadu_128(pred);
highbd_compute_jnt_comp_avg(&p0, &p1, &w0, &w1, &r, comp_pred16);
highbd_compute_dist_wtd_comp_avg(&p0, &p1, &w0, &w1, &r, comp_pred16);
comp_pred16 += 8;
pred += 8;

608
third_party/aom/aom_dsp/x86/intrapred_asm_sse2.asm vendored Normal file
View file

@ -0,0 +1,608 @@
;
; Copyright (c) 2016, Alliance for Open Media. All rights reserved
;
; This source code is subject to the terms of the BSD 2 Clause License and
; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
; was not distributed with this source code in the LICENSE file, you can
; obtain it at www.aomedia.org/license/software. If the Alliance for Open
; Media Patent License 1.0 was not distributed with this source code in the
; PATENTS file, you can obtain it at www.aomedia.org/license/patent.
;
;
%include "third_party/x86inc/x86inc.asm"
SECTION_RODATA
pb_1: times 16 db 1
pw_4: times 8 dw 4
pw_8: times 8 dw 8
pw_16: times 8 dw 16
pw_32: times 8 dw 32
dc_128: times 16 db 128
pw2_4: times 8 dw 2
pw2_8: times 8 dw 4
pw2_16: times 8 dw 8
pw2_32: times 8 dw 16
SECTION .text
INIT_XMM sse2
cglobal dc_predictor_4x4, 4, 5, 3, dst, stride, above, left, goffset
GET_GOT goffsetq
movd m2, [leftq]
movd m0, [aboveq]
pxor m1, m1
punpckldq m0, m2
psadbw m0, m1
paddw m0, [GLOBAL(pw_4)]
psraw m0, 3
pshuflw m0, m0, 0x0
packuswb m0, m0
movd [dstq ], m0
movd [dstq+strideq], m0
lea dstq, [dstq+strideq*2]
movd [dstq ], m0
movd [dstq+strideq], m0
RESTORE_GOT
RET
INIT_XMM sse2
cglobal dc_left_predictor_4x4, 2, 5, 2, dst, stride, above, left, goffset
movifnidn leftq, leftmp
GET_GOT goffsetq
pxor m1, m1
movd m0, [leftq]
psadbw m0, m1
paddw m0, [GLOBAL(pw2_4)]
psraw m0, 2
pshuflw m0, m0, 0x0
packuswb m0, m0
movd [dstq ], m0
movd [dstq+strideq], m0
lea dstq, [dstq+strideq*2]
movd [dstq ], m0
movd [dstq+strideq], m0
RESTORE_GOT
RET
INIT_XMM sse2
cglobal dc_top_predictor_4x4, 3, 5, 2, dst, stride, above, left, goffset
GET_GOT goffsetq
pxor m1, m1
movd m0, [aboveq]
psadbw m0, m1
paddw m0, [GLOBAL(pw2_4)]
psraw m0, 2
pshuflw m0, m0, 0x0
packuswb m0, m0
movd [dstq ], m0
movd [dstq+strideq], m0
lea dstq, [dstq+strideq*2]
movd [dstq ], m0
movd [dstq+strideq], m0
RESTORE_GOT
RET
INIT_XMM sse2
cglobal dc_predictor_8x8, 4, 5, 3, dst, stride, above, left, goffset
GET_GOT goffsetq
pxor m1, m1
movq m0, [aboveq]
movq m2, [leftq]
DEFINE_ARGS dst, stride, stride3
lea stride3q, [strideq*3]
psadbw m0, m1
psadbw m2, m1
paddw m0, m2
paddw m0, [GLOBAL(pw_8)]
psraw m0, 4
punpcklbw m0, m0
pshuflw m0, m0, 0x0
movq [dstq ], m0
movq [dstq+strideq ], m0
movq [dstq+strideq*2], m0
movq [dstq+stride3q ], m0
lea dstq, [dstq+strideq*4]
movq [dstq ], m0
movq [dstq+strideq ], m0
movq [dstq+strideq*2], m0
movq [dstq+stride3q ], m0
RESTORE_GOT
RET
INIT_XMM sse2
cglobal dc_top_predictor_8x8, 3, 5, 2, dst, stride, above, left, goffset
GET_GOT goffsetq
pxor m1, m1
movq m0, [aboveq]
DEFINE_ARGS dst, stride, stride3
lea stride3q, [strideq*3]
psadbw m0, m1
paddw m0, [GLOBAL(pw2_8)]
psraw m0, 3
punpcklbw m0, m0
pshuflw m0, m0, 0x0
movq [dstq ], m0
movq [dstq+strideq ], m0
movq [dstq+strideq*2], m0
movq [dstq+stride3q ], m0
lea dstq, [dstq+strideq*4]
movq [dstq ], m0
movq [dstq+strideq ], m0
movq [dstq+strideq*2], m0
movq [dstq+stride3q ], m0
RESTORE_GOT
RET
INIT_XMM sse2
cglobal dc_left_predictor_8x8, 2, 5, 2, dst, stride, above, left, goffset
movifnidn leftq, leftmp
GET_GOT goffsetq
pxor m1, m1
movq m0, [leftq]
DEFINE_ARGS dst, stride, stride3
lea stride3q, [strideq*3]
psadbw m0, m1
paddw m0, [GLOBAL(pw2_8)]
psraw m0, 3
punpcklbw m0, m0
pshuflw m0, m0, 0x0
movq [dstq ], m0
movq [dstq+strideq ], m0
movq [dstq+strideq*2], m0
movq [dstq+stride3q ], m0
lea dstq, [dstq+strideq*4]
movq [dstq ], m0
movq [dstq+strideq ], m0
movq [dstq+strideq*2], m0
movq [dstq+stride3q ], m0
RESTORE_GOT
RET
INIT_XMM sse2
cglobal dc_128_predictor_4x4, 2, 5, 1, dst, stride, above, left, goffset
GET_GOT goffsetq
DEFINE_ARGS dst, stride, stride3
lea stride3q, [strideq*3]
movd m0, [GLOBAL(dc_128)]
movd [dstq ], m0
movd [dstq+strideq ], m0
movd [dstq+strideq*2], m0
movd [dstq+stride3q ], m0
RESTORE_GOT
RET
INIT_XMM sse2
cglobal dc_128_predictor_8x8, 2, 5, 1, dst, stride, above, left, goffset
GET_GOT goffsetq
DEFINE_ARGS dst, stride, stride3
lea stride3q, [strideq*3]
movq m0, [GLOBAL(dc_128)]
movq [dstq ], m0
movq [dstq+strideq ], m0
movq [dstq+strideq*2], m0
movq [dstq+stride3q ], m0
lea dstq, [dstq+strideq*4]
movq [dstq ], m0
movq [dstq+strideq ], m0
movq [dstq+strideq*2], m0
movq [dstq+stride3q ], m0
RESTORE_GOT
RET
INIT_XMM sse2
cglobal dc_predictor_16x16, 4, 5, 3, dst, stride, above, left, goffset
GET_GOT goffsetq
pxor m1, m1
mova m0, [aboveq]
mova m2, [leftq]
DEFINE_ARGS dst, stride, stride3, lines4
lea stride3q, [strideq*3]
mov lines4d, 4
psadbw m0, m1
psadbw m2, m1
paddw m0, m2
movhlps m2, m0
paddw m0, m2
paddw m0, [GLOBAL(pw_16)]
psraw m0, 5
pshuflw m0, m0, 0x0
punpcklqdq m0, m0
packuswb m0, m0
.loop:
mova [dstq ], m0
mova [dstq+strideq ], m0
mova [dstq+strideq*2], m0
mova [dstq+stride3q ], m0
lea dstq, [dstq+strideq*4]
dec lines4d
jnz .loop
RESTORE_GOT
REP_RET
INIT_XMM sse2
cglobal dc_top_predictor_16x16, 4, 5, 3, dst, stride, above, left, goffset
GET_GOT goffsetq
pxor m1, m1
mova m0, [aboveq]
DEFINE_ARGS dst, stride, stride3, lines4
lea stride3q, [strideq*3]
mov lines4d, 4
psadbw m0, m1
movhlps m2, m0
paddw m0, m2
paddw m0, [GLOBAL(pw2_16)]
psraw m0, 4
pshuflw m0, m0, 0x0
punpcklqdq m0, m0
packuswb m0, m0
.loop:
mova [dstq ], m0
mova [dstq+strideq ], m0
mova [dstq+strideq*2], m0
mova [dstq+stride3q ], m0
lea dstq, [dstq+strideq*4]
dec lines4d
jnz .loop
RESTORE_GOT
REP_RET
INIT_XMM sse2
cglobal dc_left_predictor_16x16, 4, 5, 3, dst, stride, above, left, goffset
GET_GOT goffsetq
pxor m1, m1
mova m0, [leftq]
DEFINE_ARGS dst, stride, stride3, lines4
lea stride3q, [strideq*3]
mov lines4d, 4
psadbw m0, m1
movhlps m2, m0
paddw m0, m2
paddw m0, [GLOBAL(pw2_16)]
psraw m0, 4
pshuflw m0, m0, 0x0
punpcklqdq m0, m0
packuswb m0, m0
.loop:
mova [dstq ], m0
mova [dstq+strideq ], m0
mova [dstq+strideq*2], m0
mova [dstq+stride3q ], m0
lea dstq, [dstq+strideq*4]
dec lines4d
jnz .loop
RESTORE_GOT
REP_RET
INIT_XMM sse2
cglobal dc_128_predictor_16x16, 4, 5, 3, dst, stride, above, left, goffset
GET_GOT goffsetq
DEFINE_ARGS dst, stride, stride3, lines4
lea stride3q, [strideq*3]
mov lines4d, 4
mova m0, [GLOBAL(dc_128)]
.loop:
mova [dstq ], m0
mova [dstq+strideq ], m0
mova [dstq+strideq*2], m0
mova [dstq+stride3q ], m0
lea dstq, [dstq+strideq*4]
dec lines4d
jnz .loop
RESTORE_GOT
RET
INIT_XMM sse2
cglobal dc_predictor_32x32, 4, 5, 5, dst, stride, above, left, goffset
GET_GOT goffsetq
pxor m1, m1
mova m0, [aboveq]
mova m2, [aboveq+16]
mova m3, [leftq]
mova m4, [leftq+16]
DEFINE_ARGS dst, stride, stride3, lines4
lea stride3q, [strideq*3]
mov lines4d, 8
psadbw m0, m1
psadbw m2, m1
psadbw m3, m1
psadbw m4, m1
paddw m0, m2
paddw m0, m3
paddw m0, m4
movhlps m2, m0
paddw m0, m2
paddw m0, [GLOBAL(pw_32)]
psraw m0, 6
pshuflw m0, m0, 0x0
punpcklqdq m0, m0
packuswb m0, m0
.loop:
mova [dstq ], m0
mova [dstq +16], m0
mova [dstq+strideq ], m0
mova [dstq+strideq +16], m0
mova [dstq+strideq*2 ], m0
mova [dstq+strideq*2+16], m0
mova [dstq+stride3q ], m0
mova [dstq+stride3q +16], m0
lea dstq, [dstq+strideq*4]
dec lines4d
jnz .loop
RESTORE_GOT
REP_RET
INIT_XMM sse2
cglobal dc_top_predictor_32x32, 4, 5, 5, dst, stride, above, left, goffset
GET_GOT goffsetq
pxor m1, m1
mova m0, [aboveq]
mova m2, [aboveq+16]
DEFINE_ARGS dst, stride, stride3, lines4
lea stride3q, [strideq*3]
mov lines4d, 8
psadbw m0, m1
psadbw m2, m1
paddw m0, m2
movhlps m2, m0
paddw m0, m2
paddw m0, [GLOBAL(pw2_32)]
psraw m0, 5
pshuflw m0, m0, 0x0
punpcklqdq m0, m0
packuswb m0, m0
.loop:
mova [dstq ], m0
mova [dstq +16], m0
mova [dstq+strideq ], m0
mova [dstq+strideq +16], m0
mova [dstq+strideq*2 ], m0
mova [dstq+strideq*2+16], m0
mova [dstq+stride3q ], m0
mova [dstq+stride3q +16], m0
lea dstq, [dstq+strideq*4]
dec lines4d
jnz .loop
RESTORE_GOT
REP_RET
INIT_XMM sse2
cglobal dc_left_predictor_32x32, 4, 5, 5, dst, stride, above, left, goffset
GET_GOT goffsetq
pxor m1, m1
mova m0, [leftq]
mova m2, [leftq+16]
DEFINE_ARGS dst, stride, stride3, lines4
lea stride3q, [strideq*3]
mov lines4d, 8
psadbw m0, m1
psadbw m2, m1
paddw m0, m2
movhlps m2, m0
paddw m0, m2
paddw m0, [GLOBAL(pw2_32)]
psraw m0, 5
pshuflw m0, m0, 0x0
punpcklqdq m0, m0
packuswb m0, m0
.loop:
mova [dstq ], m0
mova [dstq +16], m0
mova [dstq+strideq ], m0
mova [dstq+strideq +16], m0
mova [dstq+strideq*2 ], m0
mova [dstq+strideq*2+16], m0
mova [dstq+stride3q ], m0
mova [dstq+stride3q +16], m0
lea dstq, [dstq+strideq*4]
dec lines4d
jnz .loop
RESTORE_GOT
REP_RET
INIT_XMM sse2
cglobal dc_128_predictor_32x32, 4, 5, 3, dst, stride, above, left, goffset
GET_GOT goffsetq
DEFINE_ARGS dst, stride, stride3, lines4
lea stride3q, [strideq*3]
mov lines4d, 8
mova m0, [GLOBAL(dc_128)]
.loop:
mova [dstq ], m0
mova [dstq +16], m0
mova [dstq+strideq ], m0
mova [dstq+strideq +16], m0
mova [dstq+strideq*2 ], m0
mova [dstq+strideq*2+16], m0
mova [dstq+stride3q ], m0
mova [dstq+stride3q +16], m0
lea dstq, [dstq+strideq*4]
dec lines4d
jnz .loop
RESTORE_GOT
RET
INIT_XMM sse2
cglobal v_predictor_4x4, 3, 3, 1, dst, stride, above
movd m0, [aboveq]
movd [dstq ], m0
movd [dstq+strideq], m0
lea dstq, [dstq+strideq*2]
movd [dstq ], m0
movd [dstq+strideq], m0
RET
INIT_XMM sse2
cglobal v_predictor_8x8, 3, 3, 1, dst, stride, above
movq m0, [aboveq]
DEFINE_ARGS dst, stride, stride3
lea stride3q, [strideq*3]
movq [dstq ], m0
movq [dstq+strideq ], m0
movq [dstq+strideq*2], m0
movq [dstq+stride3q ], m0
lea dstq, [dstq+strideq*4]
movq [dstq ], m0
movq [dstq+strideq ], m0
movq [dstq+strideq*2], m0
movq [dstq+stride3q ], m0
RET
INIT_XMM sse2
cglobal v_predictor_16x16, 3, 4, 1, dst, stride, above
mova m0, [aboveq]
DEFINE_ARGS dst, stride, stride3, nlines4
lea stride3q, [strideq*3]
mov nlines4d, 4
.loop:
mova [dstq ], m0
mova [dstq+strideq ], m0
mova [dstq+strideq*2], m0
mova [dstq+stride3q ], m0
lea dstq, [dstq+strideq*4]
dec nlines4d
jnz .loop
REP_RET
INIT_XMM sse2
cglobal v_predictor_32x32, 3, 4, 2, dst, stride, above
mova m0, [aboveq]
mova m1, [aboveq+16]
DEFINE_ARGS dst, stride, stride3, nlines4
lea stride3q, [strideq*3]
mov nlines4d, 8
.loop:
mova [dstq ], m0
mova [dstq +16], m1
mova [dstq+strideq ], m0
mova [dstq+strideq +16], m1
mova [dstq+strideq*2 ], m0
mova [dstq+strideq*2+16], m1
mova [dstq+stride3q ], m0
mova [dstq+stride3q +16], m1
lea dstq, [dstq+strideq*4]
dec nlines4d
jnz .loop
REP_RET
INIT_XMM sse2
cglobal h_predictor_4x4, 2, 4, 4, dst, stride, line, left
movifnidn leftq, leftmp
movd m0, [leftq]
punpcklbw m0, m0
punpcklbw m0, m0
pshufd m1, m0, 0x1
movd [dstq ], m0
movd [dstq+strideq], m1
pshufd m2, m0, 0x2
lea dstq, [dstq+strideq*2]
pshufd m3, m0, 0x3
movd [dstq ], m2
movd [dstq+strideq], m3
RET
INIT_XMM sse2
cglobal h_predictor_8x8, 2, 5, 3, dst, stride, line, left
movifnidn leftq, leftmp
mov lineq, -2
DEFINE_ARGS dst, stride, line, left, stride3
lea stride3q, [strideq*3]
movq m0, [leftq ]
punpcklbw m0, m0 ; l1 l1 l2 l2 ... l8 l8
.loop:
pshuflw m1, m0, 0x0 ; l1 l1 l1 l1 l1 l1 l1 l1
pshuflw m2, m0, 0x55 ; l2 l2 l2 l2 l2 l2 l2 l2
movq [dstq ], m1
movq [dstq+strideq], m2
pshuflw m1, m0, 0xaa
pshuflw m2, m0, 0xff
movq [dstq+strideq*2], m1
movq [dstq+stride3q ], m2
pshufd m0, m0, 0xe ; [63:0] l5 l5 l6 l6 l7 l7 l8 l8
inc lineq
lea dstq, [dstq+strideq*4]
jnz .loop
REP_RET
INIT_XMM sse2
cglobal h_predictor_16x16, 2, 5, 3, dst, stride, line, left
movifnidn leftq, leftmp
mov lineq, -4
DEFINE_ARGS dst, stride, line, left, stride3
lea stride3q, [strideq*3]
.loop:
movd m0, [leftq]
punpcklbw m0, m0
punpcklbw m0, m0 ; l1 to l4 each repeated 4 times
pshufd m1, m0, 0x0 ; l1 repeated 16 times
pshufd m2, m0, 0x55 ; l2 repeated 16 times
mova [dstq ], m1
mova [dstq+strideq ], m2
pshufd m1, m0, 0xaa
pshufd m2, m0, 0xff
mova [dstq+strideq*2], m1
mova [dstq+stride3q ], m2
inc lineq
lea leftq, [leftq+4 ]
lea dstq, [dstq+strideq*4]
jnz .loop
REP_RET
INIT_XMM sse2
cglobal h_predictor_32x32, 2, 5, 3, dst, stride, line, left
movifnidn leftq, leftmp
mov lineq, -8
DEFINE_ARGS dst, stride, line, left, stride3
lea stride3q, [strideq*3]
.loop:
movd m0, [leftq]
punpcklbw m0, m0
punpcklbw m0, m0 ; l1 to l4 each repeated 4 times
pshufd m1, m0, 0x0 ; l1 repeated 16 times
pshufd m2, m0, 0x55 ; l2 repeated 16 times
mova [dstq ], m1
mova [dstq+16 ], m1
mova [dstq+strideq ], m2
mova [dstq+strideq+16 ], m2
pshufd m1, m0, 0xaa
pshufd m2, m0, 0xff
mova [dstq+strideq*2 ], m1
mova [dstq+strideq*2+16], m1
mova [dstq+stride3q ], m2
mova [dstq+stride3q+16 ], m2
inc lineq
lea leftq, [leftq+4 ]
lea dstq, [dstq+strideq*4]
jnz .loop
REP_RET

3573
third_party/aom/aom_dsp/x86/intrapred_avx2.c vendored
View file

File diff suppressed because it is too large Load diff

64
third_party/aom/aom_dsp/x86/jnt_sad_ssse3.c vendored
View file

@ -192,47 +192,47 @@ unsigned int aom_sad128xh_sse2(const uint8_t *a, int a_stride, const uint8_t *b,
return res;
}
#define jnt_sadMxN_sse2(m, n) \
unsigned int aom_jnt_sad##m##x##n##_avg_ssse3( \
#define dist_wtd_sadMxN_sse2(m, n) \
unsigned int aom_dist_wtd_sad##m##x##n##_avg_ssse3( \
const uint8_t *src, int src_stride, const uint8_t *ref, int ref_stride, \
const uint8_t *second_pred, const JNT_COMP_PARAMS *jcp_param) { \
const uint8_t *second_pred, const DIST_WTD_COMP_PARAMS *jcp_param) { \
uint8_t comp_pred[m * n]; \
aom_jnt_comp_avg_pred(comp_pred, second_pred, m, n, ref, ref_stride, \
jcp_param); \
aom_dist_wtd_comp_avg_pred(comp_pred, second_pred, m, n, ref, ref_stride, \
jcp_param); \
return aom_sad##m##xh_sse2(src, src_stride, comp_pred, m, m, n); \
}
#define jnt_sadMxN_avx2(m, n) \
unsigned int aom_jnt_sad##m##x##n##_avg_avx2( \
#define dist_wtd_sadMxN_avx2(m, n) \
unsigned int aom_dist_wtd_sad##m##x##n##_avg_avx2( \
const uint8_t *src, int src_stride, const uint8_t *ref, int ref_stride, \
const uint8_t *second_pred, const JNT_COMP_PARAMS *jcp_param) { \
const uint8_t *second_pred, const DIST_WTD_COMP_PARAMS *jcp_param) { \
uint8_t comp_pred[m * n]; \
aom_jnt_comp_avg_pred(comp_pred, second_pred, m, n, ref, ref_stride, \
jcp_param); \
aom_dist_wtd_comp_avg_pred(comp_pred, second_pred, m, n, ref, ref_stride, \
jcp_param); \
return aom_sad##m##xh_avx2(src, src_stride, comp_pred, m, m, n); \
}
/* clang-format off */
jnt_sadMxN_sse2(128, 128)
jnt_sadMxN_sse2(128, 64)
jnt_sadMxN_sse2(64, 128)
jnt_sadMxN_sse2(64, 64)
jnt_sadMxN_sse2(64, 32)
jnt_sadMxN_sse2(32, 64)
jnt_sadMxN_sse2(32, 32)
jnt_sadMxN_sse2(32, 16)
jnt_sadMxN_sse2(16, 32)
jnt_sadMxN_sse2(16, 16)
jnt_sadMxN_sse2(16, 8)
jnt_sadMxN_sse2(8, 16)
jnt_sadMxN_sse2(8, 8)
jnt_sadMxN_sse2(8, 4)
jnt_sadMxN_sse2(4, 8)
jnt_sadMxN_sse2(4, 4)
jnt_sadMxN_sse2(4, 16)
jnt_sadMxN_sse2(16, 4)
jnt_sadMxN_sse2(8, 32)
jnt_sadMxN_sse2(32, 8)
jnt_sadMxN_sse2(16, 64)
jnt_sadMxN_sse2(64, 16)
dist_wtd_sadMxN_sse2(128, 128)
dist_wtd_sadMxN_sse2(128, 64)
dist_wtd_sadMxN_sse2(64, 128)
dist_wtd_sadMxN_sse2(64, 64)
dist_wtd_sadMxN_sse2(64, 32)
dist_wtd_sadMxN_sse2(32, 64)
dist_wtd_sadMxN_sse2(32, 32)
dist_wtd_sadMxN_sse2(32, 16)
dist_wtd_sadMxN_sse2(16, 32)
dist_wtd_sadMxN_sse2(16, 16)
dist_wtd_sadMxN_sse2(16, 8)
dist_wtd_sadMxN_sse2(8, 16)
dist_wtd_sadMxN_sse2(8, 8)
dist_wtd_sadMxN_sse2(8, 4)
dist_wtd_sadMxN_sse2(4, 8)
dist_wtd_sadMxN_sse2(4, 4)
dist_wtd_sadMxN_sse2(4, 16)
dist_wtd_sadMxN_sse2(16, 4)
dist_wtd_sadMxN_sse2(8, 32)
dist_wtd_sadMxN_sse2(32, 8)
dist_wtd_sadMxN_sse2(16, 64)
dist_wtd_sadMxN_sse2(64, 16)
/* clang-format on */

102
third_party/aom/aom_dsp/x86/jnt_variance_ssse3.c vendored
View file

@ -29,7 +29,7 @@ void aom_var_filter_block2d_bil_second_pass_ssse3(
unsigned int pixel_step, unsigned int output_height,
unsigned int output_width, const uint8_t *filter);
static INLINE void compute_jnt_comp_avg(__m128i *p0, __m128i *p1,
static INLINE void compute_dist_wtd_avg(__m128i *p0, __m128i *p1,
const __m128i *w, const __m128i *r,
void *const result) {
__m128i p_lo = _mm_unpacklo_epi8(*p0, *p1);
@ -45,10 +45,10 @@ static INLINE void compute_jnt_comp_avg(__m128i *p0, __m128i *p1,
xx_storeu_128(result, _mm_packus_epi16(shift_lo, shift_hi));
}
void aom_jnt_comp_avg_pred_ssse3(uint8_t *comp_pred, const uint8_t *pred,
int width, int height, const uint8_t *ref,
int ref_stride,
const JNT_COMP_PARAMS *jcp_param) {
void aom_dist_wtd_comp_avg_pred_ssse3(uint8_t *comp_pred, const uint8_t *pred,
int width, int height, const uint8_t *ref,
int ref_stride,
const DIST_WTD_COMP_PARAMS *jcp_param) {
int i;
const uint8_t w0 = (uint8_t)jcp_param->fwd_offset;
const uint8_t w1 = (uint8_t)jcp_param->bck_offset;
@ -67,7 +67,7 @@ void aom_jnt_comp_avg_pred_ssse3(uint8_t *comp_pred, const uint8_t *pred,
__m128i p0 = xx_loadu_128(ref);
__m128i p1 = xx_loadu_128(pred);
compute_jnt_comp_avg(&p0, &p1, &w, &r, comp_pred);
compute_dist_wtd_avg(&p0, &p1, &w, &r, comp_pred);
comp_pred += 16;
pred += 16;
@ -85,7 +85,7 @@ void aom_jnt_comp_avg_pred_ssse3(uint8_t *comp_pred, const uint8_t *pred,
__m128i p0 = _mm_unpacklo_epi64(p0_0, p0_1);
__m128i p1 = xx_loadu_128(pred);
compute_jnt_comp_avg(&p0, &p1, &w, &r, comp_pred);
compute_dist_wtd_avg(&p0, &p1, &w, &r, comp_pred);
comp_pred += 16;
pred += 16;
@ -107,7 +107,7 @@ void aom_jnt_comp_avg_pred_ssse3(uint8_t *comp_pred, const uint8_t *pred,
row3[0], row3[1], row3[2], row3[3]);
__m128i p1 = xx_loadu_128(pred);
compute_jnt_comp_avg(&p0, &p1, &w, &r, comp_pred);
compute_dist_wtd_avg(&p0, &p1, &w, &r, comp_pred);
comp_pred += 16;
pred += 16;
@ -116,11 +116,11 @@ void aom_jnt_comp_avg_pred_ssse3(uint8_t *comp_pred, const uint8_t *pred,
}
}
void aom_jnt_comp_avg_upsampled_pred_ssse3(
void aom_dist_wtd_comp_avg_upsampled_pred_ssse3(
MACROBLOCKD *xd, const struct AV1Common *const cm, int mi_row, int mi_col,
const MV *const mv, uint8_t *comp_pred, const uint8_t *pred, int width,
int height, int subpel_x_q3, int subpel_y_q3, const uint8_t *ref,
int ref_stride, const JNT_COMP_PARAMS *jcp_param, int subpel_search) {
int ref_stride, const DIST_WTD_COMP_PARAMS *jcp_param, int subpel_search) {
int n;
int i;
aom_upsampled_pred(xd, cm, mi_row, mi_col, mv, comp_pred, width, height,
@ -141,52 +141,52 @@ void aom_jnt_comp_avg_upsampled_pred_ssse3(
__m128i p0 = xx_loadu_128(comp_pred);
__m128i p1 = xx_loadu_128(pred);
compute_jnt_comp_avg(&p0, &p1, &w, &r, comp_pred);
compute_dist_wtd_avg(&p0, &p1, &w, &r, comp_pred);
comp_pred += 16;
pred += 16;
}
}
#define JNT_SUBPIX_AVG_VAR(W, H) \
uint32_t aom_jnt_sub_pixel_avg_variance##W##x##H##_ssse3( \
const uint8_t *a, int a_stride, int xoffset, int yoffset, \
const uint8_t *b, int b_stride, uint32_t *sse, \
const uint8_t *second_pred, const JNT_COMP_PARAMS *jcp_param) { \
uint16_t fdata3[(H + 1) * W]; \
uint8_t temp2[H * W]; \
DECLARE_ALIGNED(16, uint8_t, temp3[H * W]); \
\
aom_var_filter_block2d_bil_first_pass_ssse3( \
a, fdata3, a_stride, 1, H + 1, W, bilinear_filters_2t[xoffset]); \
aom_var_filter_block2d_bil_second_pass_ssse3( \
fdata3, temp2, W, W, H, W, bilinear_filters_2t[yoffset]); \
\
aom_jnt_comp_avg_pred_ssse3(temp3, second_pred, W, H, temp2, W, \
jcp_param); \
\
return aom_variance##W##x##H(temp3, W, b, b_stride, sse); \
#define DIST_WTD_SUBPIX_AVG_VAR(W, H) \
uint32_t aom_dist_wtd_sub_pixel_avg_variance##W##x##H##_ssse3( \
const uint8_t *a, int a_stride, int xoffset, int yoffset, \
const uint8_t *b, int b_stride, uint32_t *sse, \
const uint8_t *second_pred, const DIST_WTD_COMP_PARAMS *jcp_param) { \
uint16_t fdata3[(H + 1) * W]; \
uint8_t temp2[H * W]; \
DECLARE_ALIGNED(16, uint8_t, temp3[H * W]); \
\
aom_var_filter_block2d_bil_first_pass_ssse3( \
a, fdata3, a_stride, 1, H + 1, W, bilinear_filters_2t[xoffset]); \
aom_var_filter_block2d_bil_second_pass_ssse3( \
fdata3, temp2, W, W, H, W, bilinear_filters_2t[yoffset]); \
\
aom_dist_wtd_comp_avg_pred_ssse3(temp3, second_pred, W, H, temp2, W, \
jcp_param); \
\
return aom_variance##W##x##H(temp3, W, b, b_stride, sse); \
}
JNT_SUBPIX_AVG_VAR(128, 128)
JNT_SUBPIX_AVG_VAR(128, 64)
JNT_SUBPIX_AVG_VAR(64, 128)
JNT_SUBPIX_AVG_VAR(64, 64)
JNT_SUBPIX_AVG_VAR(64, 32)
JNT_SUBPIX_AVG_VAR(32, 64)
JNT_SUBPIX_AVG_VAR(32, 32)
JNT_SUBPIX_AVG_VAR(32, 16)
JNT_SUBPIX_AVG_VAR(16, 32)
JNT_SUBPIX_AVG_VAR(16, 16)
JNT_SUBPIX_AVG_VAR(16, 8)
JNT_SUBPIX_AVG_VAR(8, 16)
JNT_SUBPIX_AVG_VAR(8, 8)
JNT_SUBPIX_AVG_VAR(8, 4)
JNT_SUBPIX_AVG_VAR(4, 8)
JNT_SUBPIX_AVG_VAR(4, 4)
JNT_SUBPIX_AVG_VAR(4, 16)
JNT_SUBPIX_AVG_VAR(16, 4)
JNT_SUBPIX_AVG_VAR(8, 32)
JNT_SUBPIX_AVG_VAR(32, 8)
JNT_SUBPIX_AVG_VAR(16, 64)
JNT_SUBPIX_AVG_VAR(64, 16)
DIST_WTD_SUBPIX_AVG_VAR(128, 128)
DIST_WTD_SUBPIX_AVG_VAR(128, 64)
DIST_WTD_SUBPIX_AVG_VAR(64, 128)
DIST_WTD_SUBPIX_AVG_VAR(64, 64)
DIST_WTD_SUBPIX_AVG_VAR(64, 32)
DIST_WTD_SUBPIX_AVG_VAR(32, 64)
DIST_WTD_SUBPIX_AVG_VAR(32, 32)
DIST_WTD_SUBPIX_AVG_VAR(32, 16)
DIST_WTD_SUBPIX_AVG_VAR(16, 32)
DIST_WTD_SUBPIX_AVG_VAR(16, 16)
DIST_WTD_SUBPIX_AVG_VAR(16, 8)
DIST_WTD_SUBPIX_AVG_VAR(8, 16)
DIST_WTD_SUBPIX_AVG_VAR(8, 8)
DIST_WTD_SUBPIX_AVG_VAR(8, 4)
DIST_WTD_SUBPIX_AVG_VAR(4, 8)
DIST_WTD_SUBPIX_AVG_VAR(4, 4)
DIST_WTD_SUBPIX_AVG_VAR(4, 16)
DIST_WTD_SUBPIX_AVG_VAR(16, 4)
DIST_WTD_SUBPIX_AVG_VAR(8, 32)
DIST_WTD_SUBPIX_AVG_VAR(32, 8)
DIST_WTD_SUBPIX_AVG_VAR(16, 64)
DIST_WTD_SUBPIX_AVG_VAR(64, 16)

453
third_party/aom/aom_dsp/x86/loopfilter_sse2.c vendored
View file

@ -16,347 +16,12 @@
#include "aom_dsp/x86/synonyms.h"
#include "aom_ports/mem.h"
#include "aom_ports/emmintrin_compat.h"
#include "aom_dsp/x86/lpf_common_sse2.h"
static INLINE __m128i abs_diff(__m128i a, __m128i b) {
return _mm_or_si128(_mm_subs_epu8(a, b), _mm_subs_epu8(b, a));
}
static INLINE void transpose4x8_8x4_low_sse2(__m128i *x0, __m128i *x1,
__m128i *x2, __m128i *x3,
__m128i *d0, __m128i *d1,
__m128i *d2, __m128i *d3) {
// input
// x0 00 01 02 03 04 05 06 07 xx xx xx xx xx xx xx xx
// x1 10 11 12 13 14 15 16 17 xx xx xx xx xx xx xx xx
// x2 20 21 22 23 24 25 26 27 xx xx xx xx xx xx xx xx
// x3 30 31 32 33 34 35 36 37 xx xx xx xx xx xx xx xx
// output
// 00 10 20 30 xx xx xx xx xx xx xx xx xx xx xx xx
// 01 11 21 31 xx xx xx xx xx xx xx xx xx xx xx xx
// 02 12 22 32 xx xx xx xx xx xx xx xx xx xx xx xx
// 03 13 23 33 xx xx xx xx xx xx xx xx xx xx xx xx
__m128i w0, w1;
w0 = _mm_unpacklo_epi8(
*x0, *x1); // 00 10 01 11 02 12 03 13 04 14 05 15 06 16 07 17
w1 = _mm_unpacklo_epi8(
*x2, *x3); // 20 30 21 31 22 32 23 33 24 34 25 35 26 36 27 37
*d0 = _mm_unpacklo_epi16(
w0, w1); // 00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33
*d1 = _mm_srli_si128(*d0,
4); // 01 11 21 31 xx xx xx xx xx xx xx xx xx xx xx xx
*d2 = _mm_srli_si128(*d0,
8); // 02 12 22 32 xx xx xx xx xx xx xx xx xx xx xx xx
*d3 = _mm_srli_si128(*d0,
12); // 03 13 23 33 xx xx xx xx xx xx xx xx xx xx xx xx
}
static INLINE void transpose4x8_8x4_sse2(__m128i *x0, __m128i *x1, __m128i *x2,
__m128i *x3, __m128i *d0, __m128i *d1,
__m128i *d2, __m128i *d3, __m128i *d4,
__m128i *d5, __m128i *d6,
__m128i *d7) {
// input
// x0 00 01 02 03 04 05 06 07 xx xx xx xx xx xx xx xx
// x1 10 11 12 13 14 15 16 17 xx xx xx xx xx xx xx xx
// x2 20 21 22 23 24 25 26 27 xx xx xx xx xx xx xx xx
// x3 30 31 32 33 34 35 36 37 xx xx xx xx xx xx xx xx
// output
// 00 10 20 30 xx xx xx xx xx xx xx xx xx xx xx xx
// 01 11 21 31 xx xx xx xx xx xx xx xx xx xx xx xx
// 02 12 22 32 xx xx xx xx xx xx xx xx xx xx xx xx
// 03 13 23 33 xx xx xx xx xx xx xx xx xx xx xx xx
// 04 14 24 34 xx xx xx xx xx xx xx xx xx xx xx xx
// 05 15 25 35 xx xx xx xx xx xx xx xx xx xx xx xx
// 06 16 26 36 xx xx xx xx xx xx xx xx xx xx xx xx
// 07 17 27 37 xx xx xx xx xx xx xx xx xx xx xx xx
__m128i w0, w1, ww0, ww1;
w0 = _mm_unpacklo_epi8(
*x0, *x1); // 00 10 01 11 02 12 03 13 04 14 05 15 06 16 07 17
w1 = _mm_unpacklo_epi8(
*x2, *x3); // 20 30 21 31 22 32 23 33 24 34 25 35 26 36 27 37
ww0 = _mm_unpacklo_epi16(
w0, w1); // 00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33
ww1 = _mm_unpackhi_epi16(
w0, w1); // 04 14 24 34 05 15 25 35 06 16 26 36 07 17 27 37
*d0 = ww0; // 00 10 20 30 xx xx xx xx xx xx xx xx xx xx xx xx
*d1 = _mm_srli_si128(ww0,
4); // 01 11 21 31 xx xx xx xx xx xx xx xx xx xx xx xx
*d2 = _mm_srli_si128(ww0,
8); // 02 12 22 32 xx xx xx xx xx xx xx xx xx xx xx xx
*d3 = _mm_srli_si128(ww0,
12); // 03 13 23 33 xx xx xx xx xx xx xx xx xx xx xx xx
*d4 = ww1; // 04 14 24 34 xx xx xx xx xx xx xx xx xx xx xx xx
*d5 = _mm_srli_si128(ww1,
4); // 05 15 25 35 xx xx xx xx xx xx xx xx xx xx xx xx
*d6 = _mm_srli_si128(ww1,
8); // 06 16 26 36 xx xx xx xx xx xx xx xx xx xx xx xx
*d7 = _mm_srli_si128(ww1,
12); // 07 17 27 37 xx xx xx xx xx xx xx xx xx xx xx xx
}
static INLINE void transpose8x8_low_sse2(__m128i *x0, __m128i *x1, __m128i *x2,
__m128i *x3, __m128i *x4, __m128i *x5,
__m128i *x6, __m128i *x7, __m128i *d0,
__m128i *d1, __m128i *d2,
__m128i *d3) {
// input
// x0 00 01 02 03 04 05 06 07
// x1 10 11 12 13 14 15 16 17
// x2 20 21 22 23 24 25 26 27
// x3 30 31 32 33 34 35 36 37
// x4 40 41 42 43 44 45 46 47
// x5 50 51 52 53 54 55 56 57
// x6 60 61 62 63 64 65 66 67
// x7 70 71 72 73 74 75 76 77
// output
// d0 00 10 20 30 40 50 60 70 xx xx xx xx xx xx xx
// d1 01 11 21 31 41 51 61 71 xx xx xx xx xx xx xx xx
// d2 02 12 22 32 42 52 62 72 xx xx xx xx xx xx xx xx
// d3 03 13 23 33 43 53 63 73 xx xx xx xx xx xx xx xx
__m128i w0, w1, w2, w3, w4, w5;
w0 = _mm_unpacklo_epi8(
*x0, *x1); // 00 10 01 11 02 12 03 13 04 14 05 15 06 16 07 17
w1 = _mm_unpacklo_epi8(
*x2, *x3); // 20 30 21 31 22 32 23 33 24 34 25 35 26 36 27 37
w2 = _mm_unpacklo_epi8(
*x4, *x5); // 40 50 41 51 42 52 43 53 44 54 45 55 46 56 47 57
w3 = _mm_unpacklo_epi8(
*x6, *x7); // 60 70 61 71 62 72 63 73 64 74 65 75 66 76 67 77
w4 = _mm_unpacklo_epi16(
w0, w1); // 00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33
w5 = _mm_unpacklo_epi16(
w2, w3); // 40 50 60 70 41 51 61 71 42 52 62 72 43 53 63 73
*d0 = _mm_unpacklo_epi32(
w4, w5); // 00 10 20 30 40 50 60 70 01 11 21 31 41 51 61 71
*d1 = _mm_srli_si128(*d0, 8);
*d2 = _mm_unpackhi_epi32(
w4, w5); // 02 12 22 32 42 52 62 72 03 13 23 33 43 53 63 73
*d3 = _mm_srli_si128(*d2, 8);
}
static INLINE void transpose8x8_sse2(__m128i *x0, __m128i *x1, __m128i *x2,
__m128i *x3, __m128i *x4, __m128i *x5,
__m128i *x6, __m128i *x7, __m128i *d0d1,
__m128i *d2d3, __m128i *d4d5,
__m128i *d6d7) {
__m128i w0, w1, w2, w3, w4, w5, w6, w7;
// x0 00 01 02 03 04 05 06 07
// x1 10 11 12 13 14 15 16 17
w0 = _mm_unpacklo_epi8(
*x0, *x1); // 00 10 01 11 02 12 03 13 04 14 05 15 06 16 07 17
// x2 20 21 22 23 24 25 26 27
// x3 30 31 32 33 34 35 36 37
w1 = _mm_unpacklo_epi8(
*x2, *x3); // 20 30 21 31 22 32 23 33 24 34 25 35 26 36 27 37
// x4 40 41 42 43 44 45 46 47
// x5 50 51 52 53 54 55 56 57
w2 = _mm_unpacklo_epi8(
*x4, *x5); // 40 50 41 51 42 52 43 53 44 54 45 55 46 56 47 57
// x6 60 61 62 63 64 65 66 67
// x7 70 71 72 73 74 75 76 77
w3 = _mm_unpacklo_epi8(
*x6, *x7); // 60 70 61 71 62 72 63 73 64 74 65 75 66 76 67 77
w4 = _mm_unpacklo_epi16(
w0, w1); // 00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33
w5 = _mm_unpacklo_epi16(
w2, w3); // 40 50 60 70 41 51 61 71 42 52 62 72 43 53 63 73
*d0d1 = _mm_unpacklo_epi32(
w4, w5); // 00 10 20 30 40 50 60 70 01 11 21 31 41 51 61 71
*d2d3 = _mm_unpackhi_epi32(
w4, w5); // 02 12 22 32 42 52 62 72 03 13 23 33 43 53 63 73
w6 = _mm_unpackhi_epi16(
w0, w1); // 04 14 24 34 05 15 25 35 06 16 26 36 07 17 27 37
w7 = _mm_unpackhi_epi16(
w2, w3); // 44 54 64 74 45 55 65 75 46 56 66 76 47 57 67 77
*d4d5 = _mm_unpacklo_epi32(
w6, w7); // 04 14 24 34 44 54 64 74 05 15 25 35 45 55 65 75
*d6d7 = _mm_unpackhi_epi32(
w6, w7); // 06 16 26 36 46 56 66 76 07 17 27 37 47 57 67 77
}
static INLINE void transpose16x8_8x16_sse2(
__m128i *x0, __m128i *x1, __m128i *x2, __m128i *x3, __m128i *x4,
__m128i *x5, __m128i *x6, __m128i *x7, __m128i *x8, __m128i *x9,
__m128i *x10, __m128i *x11, __m128i *x12, __m128i *x13, __m128i *x14,
__m128i *x15, __m128i *d0, __m128i *d1, __m128i *d2, __m128i *d3,
__m128i *d4, __m128i *d5, __m128i *d6, __m128i *d7) {
__m128i w0, w1, w2, w3, w4, w5, w6, w7, w8, w9;
__m128i w10, w11, w12, w13, w14, w15;
w0 = _mm_unpacklo_epi8(*x0, *x1);
w1 = _mm_unpacklo_epi8(*x2, *x3);
w2 = _mm_unpacklo_epi8(*x4, *x5);
w3 = _mm_unpacklo_epi8(*x6, *x7);
w8 = _mm_unpacklo_epi8(*x8, *x9);
w9 = _mm_unpacklo_epi8(*x10, *x11);
w10 = _mm_unpacklo_epi8(*x12, *x13);
w11 = _mm_unpacklo_epi8(*x14, *x15);
w4 = _mm_unpacklo_epi16(w0, w1);
w5 = _mm_unpacklo_epi16(w2, w3);
w12 = _mm_unpacklo_epi16(w8, w9);
w13 = _mm_unpacklo_epi16(w10, w11);
w6 = _mm_unpacklo_epi32(w4, w5);
w7 = _mm_unpackhi_epi32(w4, w5);
w14 = _mm_unpacklo_epi32(w12, w13);
w15 = _mm_unpackhi_epi32(w12, w13);
// Store first 4-line result
*d0 = _mm_unpacklo_epi64(w6, w14);
*d1 = _mm_unpackhi_epi64(w6, w14);
*d2 = _mm_unpacklo_epi64(w7, w15);
*d3 = _mm_unpackhi_epi64(w7, w15);
w4 = _mm_unpackhi_epi16(w0, w1);
w5 = _mm_unpackhi_epi16(w2, w3);
w12 = _mm_unpackhi_epi16(w8, w9);
w13 = _mm_unpackhi_epi16(w10, w11);
w6 = _mm_unpacklo_epi32(w4, w5);
w7 = _mm_unpackhi_epi32(w4, w5);
w14 = _mm_unpacklo_epi32(w12, w13);
w15 = _mm_unpackhi_epi32(w12, w13);
// Store second 4-line result
*d4 = _mm_unpacklo_epi64(w6, w14);
*d5 = _mm_unpackhi_epi64(w6, w14);
*d6 = _mm_unpacklo_epi64(w7, w15);
*d7 = _mm_unpackhi_epi64(w7, w15);
}
// this function treats its input as 2 parallel 8x4 matrices, transposes each of
// them independently while flipping the second matrix horizontaly Used for 14
// taps filter pq pairs inverse
static INLINE void transpose_pq_14_inv_sse2(__m128i *x0, __m128i *x1,
__m128i *x2, __m128i *x3,
__m128i *x4, __m128i *x5,
__m128i *x6, __m128i *x7,
__m128i *pq0, __m128i *pq1,
__m128i *pq2, __m128i *pq3) {
__m128i w10, w11, w12, w13;
__m128i w0, w1, w2, w3, w4, w5;
__m128i d0, d1, d2, d3;
w0 = _mm_unpacklo_epi8(
*x0, *x1); // p 00 10 01 11 02 12 03 13 04 14 05 15 06 16 07 17
w1 = _mm_unpacklo_epi8(
*x2, *x3); // p 20 30 21 31 22 32 23 33 24 34 25 35 26 36 27 37
w2 = _mm_unpacklo_epi8(
*x4, *x5); // p 40 50 41 51 42 52 43 53 44 54 45 55 46 56 47 57
w3 = _mm_unpacklo_epi8(
*x6, *x7); // p 60 70 61 71 62 72 63 73 64 74 65 75 66 76 67 77
w4 = _mm_unpacklo_epi16(
w0, w1); // 00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33
w5 = _mm_unpacklo_epi16(
w2, w3); // 40 50 60 70 41 51 61 71 42 52 62 72 43 53 63 73
d0 = _mm_unpacklo_epi32(
w4, w5); // 00 10 20 30 40 50 60 70 01 11 21 31 41 51 61 71
d2 = _mm_unpackhi_epi32(
w4, w5); // 02 12 22 32 42 52 62 72 03 13 23 33 43 53 63 73
w10 = _mm_unpacklo_epi8(
*x7, *x6); // q xx xx xx xx xx xx xx xx 00 10 01 11 02 12 03 13
w11 = _mm_unpacklo_epi8(
*x5, *x4); // q xx xx xx xx xx xx xx xx 20 30 21 31 22 32 23 33
w12 = _mm_unpacklo_epi8(
*x3, *x2); // q xx xx xx xx xx xx xx xx 40 50 41 51 42 52 43 53
w13 = _mm_unpacklo_epi8(
*x1, *x0); // q xx xx xx xx xx xx xx xx 60 70 61 71 62 72 63 73
w4 = _mm_unpackhi_epi16(
w10, w11); // 00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33
w5 = _mm_unpackhi_epi16(
w12, w13); // 40 50 60 70 41 51 61 71 42 52 62 72 43 53 63 73
d1 = _mm_unpacklo_epi32(
w4, w5); // 00 10 20 30 40 50 60 70 01 11 21 31 41 51 61 71
d3 = _mm_unpackhi_epi32(
w4, w5); // 02 12 22 32 42 52 62 72 03 13 23 33 43 53 63 73
*pq0 = _mm_unpacklo_epi64(d0, d1); // pq
*pq1 = _mm_unpackhi_epi64(d0, d1); // pq
*pq2 = _mm_unpacklo_epi64(d2, d3); // pq
*pq3 = _mm_unpackhi_epi64(d2, d3); // pq
}
static INLINE void transpose8x16_16x8_sse2(
__m128i *x0, __m128i *x1, __m128i *x2, __m128i *x3, __m128i *x4,
__m128i *x5, __m128i *x6, __m128i *x7, __m128i *d0d1, __m128i *d2d3,
__m128i *d4d5, __m128i *d6d7, __m128i *d8d9, __m128i *d10d11,
__m128i *d12d13, __m128i *d14d15) {
__m128i w0, w1, w2, w3, w4, w5, w6, w7, w8, w9;
__m128i w10, w11, w12, w13, w14, w15;
w0 = _mm_unpacklo_epi8(*x0, *x1);
w1 = _mm_unpacklo_epi8(*x2, *x3);
w2 = _mm_unpacklo_epi8(*x4, *x5);
w3 = _mm_unpacklo_epi8(*x6, *x7);
w8 = _mm_unpackhi_epi8(*x0, *x1);
w9 = _mm_unpackhi_epi8(*x2, *x3);
w10 = _mm_unpackhi_epi8(*x4, *x5);
w11 = _mm_unpackhi_epi8(*x6, *x7);
w4 = _mm_unpacklo_epi16(w0, w1);
w5 = _mm_unpacklo_epi16(w2, w3);
w12 = _mm_unpacklo_epi16(w8, w9);
w13 = _mm_unpacklo_epi16(w10, w11);
w6 = _mm_unpacklo_epi32(w4, w5);
w7 = _mm_unpackhi_epi32(w4, w5);
w14 = _mm_unpacklo_epi32(w12, w13);
w15 = _mm_unpackhi_epi32(w12, w13);
// Store first 4-line result
*d0d1 = _mm_unpacklo_epi64(w6, w14);
*d2d3 = _mm_unpackhi_epi64(w6, w14);
*d4d5 = _mm_unpacklo_epi64(w7, w15);
*d6d7 = _mm_unpackhi_epi64(w7, w15);
w4 = _mm_unpackhi_epi16(w0, w1);
w5 = _mm_unpackhi_epi16(w2, w3);
w12 = _mm_unpackhi_epi16(w8, w9);
w13 = _mm_unpackhi_epi16(w10, w11);
w6 = _mm_unpacklo_epi32(w4, w5);
w7 = _mm_unpackhi_epi32(w4, w5);
w14 = _mm_unpacklo_epi32(w12, w13);
w15 = _mm_unpackhi_epi32(w12, w13);
// Store second 4-line result
*d8d9 = _mm_unpacklo_epi64(w6, w14);
*d10d11 = _mm_unpackhi_epi64(w6, w14);
*d12d13 = _mm_unpacklo_epi64(w7, w15);
*d14d15 = _mm_unpackhi_epi64(w7, w15);
}
// this function treats its input as 2 parallel 8x4 matrices, transposes each of
// them to 4x8 independently while flipping the second matrix horizontaly. Used
// for 14 taps pq pairs creation
@ -416,6 +81,63 @@ static INLINE void transpose_pq_14_sse2(__m128i *x0, __m128i *x1, __m128i *x2,
ww2); // 07 17 27 37 08 18 28 38 xx xx xx xx xx xx xx xx
}
// this function treats its input as 2 parallel 8x4 matrices, transposes each of
// them independently while flipping the second matrix horizontaly Used for 14
// taps filter pq pairs inverse
static INLINE void transpose_pq_14_inv_sse2(__m128i *x0, __m128i *x1,
__m128i *x2, __m128i *x3,
__m128i *x4, __m128i *x5,
__m128i *x6, __m128i *x7,
__m128i *pq0, __m128i *pq1,
__m128i *pq2, __m128i *pq3) {
__m128i w10, w11, w12, w13;
__m128i w0, w1, w2, w3, w4, w5;
__m128i d0, d1, d2, d3;
w0 = _mm_unpacklo_epi8(
*x0, *x1); // p 00 10 01 11 02 12 03 13 04 14 05 15 06 16 07 17
w1 = _mm_unpacklo_epi8(
*x2, *x3); // p 20 30 21 31 22 32 23 33 24 34 25 35 26 36 27 37
w2 = _mm_unpacklo_epi8(
*x4, *x5); // p 40 50 41 51 42 52 43 53 44 54 45 55 46 56 47 57
w3 = _mm_unpacklo_epi8(
*x6, *x7); // p 60 70 61 71 62 72 63 73 64 74 65 75 66 76 67 77
w4 = _mm_unpacklo_epi16(
w0, w1); // 00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33
w5 = _mm_unpacklo_epi16(
w2, w3); // 40 50 60 70 41 51 61 71 42 52 62 72 43 53 63 73
d0 = _mm_unpacklo_epi32(
w4, w5); // 00 10 20 30 40 50 60 70 01 11 21 31 41 51 61 71
d2 = _mm_unpackhi_epi32(
w4, w5); // 02 12 22 32 42 52 62 72 03 13 23 33 43 53 63 73
w10 = _mm_unpacklo_epi8(
*x7, *x6); // q xx xx xx xx xx xx xx xx 00 10 01 11 02 12 03 13
w11 = _mm_unpacklo_epi8(
*x5, *x4); // q xx xx xx xx xx xx xx xx 20 30 21 31 22 32 23 33
w12 = _mm_unpacklo_epi8(
*x3, *x2); // q xx xx xx xx xx xx xx xx 40 50 41 51 42 52 43 53
w13 = _mm_unpacklo_epi8(
*x1, *x0); // q xx xx xx xx xx xx xx xx 60 70 61 71 62 72 63 73
w4 = _mm_unpackhi_epi16(
w10, w11); // 00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33
w5 = _mm_unpackhi_epi16(
w12, w13); // 40 50 60 70 41 51 61 71 42 52 62 72 43 53 63 73
d1 = _mm_unpacklo_epi32(
w4, w5); // 00 10 20 30 40 50 60 70 01 11 21 31 41 51 61 71
d3 = _mm_unpackhi_epi32(
w4, w5); // 02 12 22 32 42 52 62 72 03 13 23 33 43 53 63 73
*pq0 = _mm_unpacklo_epi64(d0, d1); // pq
*pq1 = _mm_unpackhi_epi64(d0, d1); // pq
*pq2 = _mm_unpacklo_epi64(d2, d3); // pq
*pq3 = _mm_unpackhi_epi64(d2, d3); // pq
}
static AOM_FORCE_INLINE void filter4_sse2(__m128i *p1p0, __m128i *q1q0,
__m128i *hev, __m128i *mask,
__m128i *qs1qs0, __m128i *ps1ps0) {
@ -616,10 +338,10 @@ void aom_lpf_horizontal_4_sse2(uint8_t *s, int p /* pitch */,
__m128i qs1qs0, ps1ps0;
__m128i p1, p0, q0, q1;
p1 = _mm_cvtsi32_si128(*(int *)(s - 2 * p));
p0 = _mm_cvtsi32_si128(*(int *)(s - 1 * p));
q0 = _mm_cvtsi32_si128(*(int *)(s + 0 * p));
q1 = _mm_cvtsi32_si128(*(int *)(s + 1 * p));
p1 = xx_loadl_32(s - 2 * p);
p0 = xx_loadl_32(s - 1 * p);
q0 = xx_loadl_32(s - 0 * p);
q1 = xx_loadl_32(s + 1 * p);
lpf_internal_4_sse2(&p1, &p0, &q0, &q1, &limit, &thresh, &qs1qs0, &ps1ps0);
@ -1241,23 +963,16 @@ void aom_lpf_horizontal_14_sse2(unsigned char *s, int p,
__m128i limit = _mm_load_si128((const __m128i *)_limit);
__m128i thresh = _mm_load_si128((const __m128i *)_thresh);
q4p4 = _mm_unpacklo_epi32(_mm_cvtsi32_si128(*(int *)(s - 5 * p)),
_mm_cvtsi32_si128(*(int *)(s + 4 * p)));
q3p3 = _mm_unpacklo_epi32(_mm_cvtsi32_si128(*(int *)(s - 4 * p)),
_mm_cvtsi32_si128(*(int *)(s + 3 * p)));
q2p2 = _mm_unpacklo_epi32(_mm_cvtsi32_si128(*(int *)(s - 3 * p)),
_mm_cvtsi32_si128(*(int *)(s + 2 * p)));
q1p1 = _mm_unpacklo_epi32(_mm_cvtsi32_si128(*(int *)(s - 2 * p)),
_mm_cvtsi32_si128(*(int *)(s + 1 * p)));
q4p4 = _mm_unpacklo_epi32(xx_loadl_32(s - 5 * p), xx_loadl_32(s + 4 * p));
q3p3 = _mm_unpacklo_epi32(xx_loadl_32(s - 4 * p), xx_loadl_32(s + 3 * p));
q2p2 = _mm_unpacklo_epi32(xx_loadl_32(s - 3 * p), xx_loadl_32(s + 2 * p));
q1p1 = _mm_unpacklo_epi32(xx_loadl_32(s - 2 * p), xx_loadl_32(s + 1 * p));
q0p0 = _mm_unpacklo_epi32(_mm_cvtsi32_si128(*(int *)(s - 1 * p)),
_mm_cvtsi32_si128(*(int *)(s - 0 * p)));
q0p0 = _mm_unpacklo_epi32(xx_loadl_32(s - 1 * p), xx_loadl_32(s - 0 * p));
q5p5 = _mm_unpacklo_epi32(_mm_cvtsi32_si128(*(int *)(s - 6 * p)),
_mm_cvtsi32_si128(*(int *)(s + 5 * p)));
q5p5 = _mm_unpacklo_epi32(xx_loadl_32(s - 6 * p), xx_loadl_32(s + 5 * p));
q6p6 = _mm_unpacklo_epi32(_mm_cvtsi32_si128(*(int *)(s - 7 * p)),
_mm_cvtsi32_si128(*(int *)(s + 6 * p)));
q6p6 = _mm_unpacklo_epi32(xx_loadl_32(s - 7 * p), xx_loadl_32(s + 6 * p));
lpf_internal_14_sse2(&q6p6, &q5p5, &q4p4, &q3p3, &q2p2, &q1p1, &q0p0, &blimit,
&limit, &thresh);
@ -1543,12 +1258,12 @@ void aom_lpf_horizontal_6_sse2(unsigned char *s, int p,
__m128i limit = _mm_load_si128((__m128i *)_limit);
__m128i thresh = _mm_load_si128((__m128i *)_thresh);
p2 = _mm_cvtsi32_si128(*(int *)(s - 3 * p));
p1 = _mm_cvtsi32_si128(*(int *)(s - 2 * p));
p0 = _mm_cvtsi32_si128(*(int *)(s - 1 * p));
q0 = _mm_cvtsi32_si128(*(int *)(s - 0 * p));
q1 = _mm_cvtsi32_si128(*(int *)(s + 1 * p));
q2 = _mm_cvtsi32_si128(*(int *)(s + 2 * p));
p2 = xx_loadl_32(s - 3 * p);
p1 = xx_loadl_32(s - 2 * p);
p0 = xx_loadl_32(s - 1 * p);
q0 = xx_loadl_32(s - 0 * p);
q1 = xx_loadl_32(s + 1 * p);
q2 = xx_loadl_32(s + 2 * p);
lpf_internal_6_sse2(&p2, &q2, &p1, &q1, &p0, &q0, &q1q0, &p1p0, &blimit,
&limit, &thresh);
@ -1895,20 +1610,20 @@ void aom_lpf_horizontal_8_sse2(unsigned char *s, int p,
const unsigned char *_blimit,
const unsigned char *_limit,
const unsigned char *_thresh) {
__m128i p2, p1, p0, q0, q1, q2, p3, q3;
__m128i p3, p2, p1, p0, q0, q1, q2, q3;
__m128i q1q0, p1p0;
__m128i blimit = _mm_load_si128((const __m128i *)_blimit);
__m128i limit = _mm_load_si128((const __m128i *)_limit);
__m128i thresh = _mm_load_si128((const __m128i *)_thresh);
p3 = _mm_cvtsi32_si128(*(int *)(s - 4 * p));
p2 = _mm_cvtsi32_si128(*(int *)(s - 3 * p));
p1 = _mm_cvtsi32_si128(*(int *)(s - 2 * p));
p0 = _mm_cvtsi32_si128(*(int *)(s - 1 * p));
q0 = _mm_cvtsi32_si128(*(int *)(s - 0 * p));
q1 = _mm_cvtsi32_si128(*(int *)(s + 1 * p));
q2 = _mm_cvtsi32_si128(*(int *)(s + 2 * p));
q3 = _mm_cvtsi32_si128(*(int *)(s + 3 * p));
p3 = xx_loadl_32(s - 4 * p);
p2 = xx_loadl_32(s - 3 * p);
p1 = xx_loadl_32(s - 2 * p);
p0 = xx_loadl_32(s - 1 * p);
q0 = xx_loadl_32(s - 0 * p);
q1 = xx_loadl_32(s + 1 * p);
q2 = xx_loadl_32(s + 2 * p);
q3 = xx_loadl_32(s + 3 * p);
lpf_internal_8_sse2(&p3, &q3, &p2, &q2, &p1, &q1, &p0, &q0, &q1q0, &p1p0,
&blimit, &limit, &thresh);

280
third_party/aom/aom_dsp/x86/lpf_common_sse2.h vendored
View file

@ -212,4 +212,284 @@ static INLINE void highbd_transpose8x16_sse2(
d4 + 1, d5 + 1, d6 + 1, d7 + 1);
}
// Low bit depth functions
static INLINE void transpose4x8_8x4_low_sse2(__m128i *x0, __m128i *x1,
__m128i *x2, __m128i *x3,
__m128i *d0, __m128i *d1,
__m128i *d2, __m128i *d3) {
// input
// x0 00 01 02 03 04 05 06 07 xx xx xx xx xx xx xx xx
// x1 10 11 12 13 14 15 16 17 xx xx xx xx xx xx xx xx
// x2 20 21 22 23 24 25 26 27 xx xx xx xx xx xx xx xx
// x3 30 31 32 33 34 35 36 37 xx xx xx xx xx xx xx xx
// output
// 00 10 20 30 xx xx xx xx xx xx xx xx xx xx xx xx
// 01 11 21 31 xx xx xx xx xx xx xx xx xx xx xx xx
// 02 12 22 32 xx xx xx xx xx xx xx xx xx xx xx xx
// 03 13 23 33 xx xx xx xx xx xx xx xx xx xx xx xx
__m128i w0, w1;
w0 = _mm_unpacklo_epi8(
*x0, *x1); // 00 10 01 11 02 12 03 13 04 14 05 15 06 16 07 17
w1 = _mm_unpacklo_epi8(
*x2, *x3); // 20 30 21 31 22 32 23 33 24 34 25 35 26 36 27 37
*d0 = _mm_unpacklo_epi16(
w0, w1); // 00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33
*d1 = _mm_srli_si128(*d0,
4); // 01 11 21 31 xx xx xx xx xx xx xx xx xx xx xx xx
*d2 = _mm_srli_si128(*d0,
8); // 02 12 22 32 xx xx xx xx xx xx xx xx xx xx xx xx
*d3 = _mm_srli_si128(*d0,
12); // 03 13 23 33 xx xx xx xx xx xx xx xx xx xx xx xx
}
static INLINE void transpose4x8_8x4_sse2(__m128i *x0, __m128i *x1, __m128i *x2,
__m128i *x3, __m128i *d0, __m128i *d1,
__m128i *d2, __m128i *d3, __m128i *d4,
__m128i *d5, __m128i *d6,
__m128i *d7) {
// input
// x0 00 01 02 03 04 05 06 07 xx xx xx xx xx xx xx xx
// x1 10 11 12 13 14 15 16 17 xx xx xx xx xx xx xx xx
// x2 20 21 22 23 24 25 26 27 xx xx xx xx xx xx xx xx
// x3 30 31 32 33 34 35 36 37 xx xx xx xx xx xx xx xx
// output
// 00 10 20 30 xx xx xx xx xx xx xx xx xx xx xx xx
// 01 11 21 31 xx xx xx xx xx xx xx xx xx xx xx xx
// 02 12 22 32 xx xx xx xx xx xx xx xx xx xx xx xx
// 03 13 23 33 xx xx xx xx xx xx xx xx xx xx xx xx
// 04 14 24 34 xx xx xx xx xx xx xx xx xx xx xx xx
// 05 15 25 35 xx xx xx xx xx xx xx xx xx xx xx xx
// 06 16 26 36 xx xx xx xx xx xx xx xx xx xx xx xx
// 07 17 27 37 xx xx xx xx xx xx xx xx xx xx xx xx
__m128i w0, w1, ww0, ww1;
w0 = _mm_unpacklo_epi8(
*x0, *x1); // 00 10 01 11 02 12 03 13 04 14 05 15 06 16 07 17
w1 = _mm_unpacklo_epi8(
*x2, *x3); // 20 30 21 31 22 32 23 33 24 34 25 35 26 36 27 37
ww0 = _mm_unpacklo_epi16(
w0, w1); // 00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33
ww1 = _mm_unpackhi_epi16(
w0, w1); // 04 14 24 34 05 15 25 35 06 16 26 36 07 17 27 37
*d0 = ww0; // 00 10 20 30 xx xx xx xx xx xx xx xx xx xx xx xx
*d1 = _mm_srli_si128(ww0,
4); // 01 11 21 31 xx xx xx xx xx xx xx xx xx xx xx xx
*d2 = _mm_srli_si128(ww0,
8); // 02 12 22 32 xx xx xx xx xx xx xx xx xx xx xx xx
*d3 = _mm_srli_si128(ww0,
12); // 03 13 23 33 xx xx xx xx xx xx xx xx xx xx xx xx
*d4 = ww1; // 04 14 24 34 xx xx xx xx xx xx xx xx xx xx xx xx
*d5 = _mm_srli_si128(ww1,
4); // 05 15 25 35 xx xx xx xx xx xx xx xx xx xx xx xx
*d6 = _mm_srli_si128(ww1,
8); // 06 16 26 36 xx xx xx xx xx xx xx xx xx xx xx xx
*d7 = _mm_srli_si128(ww1,
12); // 07 17 27 37 xx xx xx xx xx xx xx xx xx xx xx xx
}
static INLINE void transpose8x8_low_sse2(__m128i *x0, __m128i *x1, __m128i *x2,
__m128i *x3, __m128i *x4, __m128i *x5,
__m128i *x6, __m128i *x7, __m128i *d0,
__m128i *d1, __m128i *d2,
__m128i *d3) {
// input
// x0 00 01 02 03 04 05 06 07
// x1 10 11 12 13 14 15 16 17
// x2 20 21 22 23 24 25 26 27
// x3 30 31 32 33 34 35 36 37
// x4 40 41 42 43 44 45 46 47
// x5 50 51 52 53 54 55 56 57
// x6 60 61 62 63 64 65 66 67
// x7 70 71 72 73 74 75 76 77
// output
// d0 00 10 20 30 40 50 60 70 xx xx xx xx xx xx xx
// d1 01 11 21 31 41 51 61 71 xx xx xx xx xx xx xx xx
// d2 02 12 22 32 42 52 62 72 xx xx xx xx xx xx xx xx
// d3 03 13 23 33 43 53 63 73 xx xx xx xx xx xx xx xx
__m128i w0, w1, w2, w3, w4, w5;
w0 = _mm_unpacklo_epi8(
*x0, *x1); // 00 10 01 11 02 12 03 13 04 14 05 15 06 16 07 17
w1 = _mm_unpacklo_epi8(
*x2, *x3); // 20 30 21 31 22 32 23 33 24 34 25 35 26 36 27 37
w2 = _mm_unpacklo_epi8(
*x4, *x5); // 40 50 41 51 42 52 43 53 44 54 45 55 46 56 47 57
w3 = _mm_unpacklo_epi8(
*x6, *x7); // 60 70 61 71 62 72 63 73 64 74 65 75 66 76 67 77
w4 = _mm_unpacklo_epi16(
w0, w1); // 00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33
w5 = _mm_unpacklo_epi16(
w2, w3); // 40 50 60 70 41 51 61 71 42 52 62 72 43 53 63 73
*d0 = _mm_unpacklo_epi32(
w4, w5); // 00 10 20 30 40 50 60 70 01 11 21 31 41 51 61 71
*d1 = _mm_srli_si128(*d0, 8);
*d2 = _mm_unpackhi_epi32(
w4, w5); // 02 12 22 32 42 52 62 72 03 13 23 33 43 53 63 73
*d3 = _mm_srli_si128(*d2, 8);
}
static INLINE void transpose8x8_sse2(__m128i *x0, __m128i *x1, __m128i *x2,
__m128i *x3, __m128i *x4, __m128i *x5,
__m128i *x6, __m128i *x7, __m128i *d0d1,
__m128i *d2d3, __m128i *d4d5,
__m128i *d6d7) {
__m128i w0, w1, w2, w3, w4, w5, w6, w7;
// x0 00 01 02 03 04 05 06 07
// x1 10 11 12 13 14 15 16 17
w0 = _mm_unpacklo_epi8(
*x0, *x1); // 00 10 01 11 02 12 03 13 04 14 05 15 06 16 07 17
// x2 20 21 22 23 24 25 26 27
// x3 30 31 32 33 34 35 36 37
w1 = _mm_unpacklo_epi8(
*x2, *x3); // 20 30 21 31 22 32 23 33 24 34 25 35 26 36 27 37
// x4 40 41 42 43 44 45 46 47
// x5 50 51 52 53 54 55 56 57
w2 = _mm_unpacklo_epi8(
*x4, *x5); // 40 50 41 51 42 52 43 53 44 54 45 55 46 56 47 57
// x6 60 61 62 63 64 65 66 67
// x7 70 71 72 73 74 75 76 77
w3 = _mm_unpacklo_epi8(
*x6, *x7); // 60 70 61 71 62 72 63 73 64 74 65 75 66 76 67 77
w4 = _mm_unpacklo_epi16(
w0, w1); // 00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33
w5 = _mm_unpacklo_epi16(
w2, w3); // 40 50 60 70 41 51 61 71 42 52 62 72 43 53 63 73
*d0d1 = _mm_unpacklo_epi32(
w4, w5); // 00 10 20 30 40 50 60 70 01 11 21 31 41 51 61 71
*d2d3 = _mm_unpackhi_epi32(
w4, w5); // 02 12 22 32 42 52 62 72 03 13 23 33 43 53 63 73
w6 = _mm_unpackhi_epi16(
w0, w1); // 04 14 24 34 05 15 25 35 06 16 26 36 07 17 27 37
w7 = _mm_unpackhi_epi16(
w2, w3); // 44 54 64 74 45 55 65 75 46 56 66 76 47 57 67 77
*d4d5 = _mm_unpacklo_epi32(
w6, w7); // 04 14 24 34 44 54 64 74 05 15 25 35 45 55 65 75
*d6d7 = _mm_unpackhi_epi32(
w6, w7); // 06 16 26 36 46 56 66 76 07 17 27 37 47 57 67 77
}
static INLINE void transpose16x8_8x16_sse2(
__m128i *x0, __m128i *x1, __m128i *x2, __m128i *x3, __m128i *x4,
__m128i *x5, __m128i *x6, __m128i *x7, __m128i *x8, __m128i *x9,
__m128i *x10, __m128i *x11, __m128i *x12, __m128i *x13, __m128i *x14,
__m128i *x15, __m128i *d0, __m128i *d1, __m128i *d2, __m128i *d3,
__m128i *d4, __m128i *d5, __m128i *d6, __m128i *d7) {
__m128i w0, w1, w2, w3, w4, w5, w6, w7, w8, w9;
__m128i w10, w11, w12, w13, w14, w15;
w0 = _mm_unpacklo_epi8(*x0, *x1);
w1 = _mm_unpacklo_epi8(*x2, *x3);
w2 = _mm_unpacklo_epi8(*x4, *x5);
w3 = _mm_unpacklo_epi8(*x6, *x7);
w8 = _mm_unpacklo_epi8(*x8, *x9);
w9 = _mm_unpacklo_epi8(*x10, *x11);
w10 = _mm_unpacklo_epi8(*x12, *x13);
w11 = _mm_unpacklo_epi8(*x14, *x15);
w4 = _mm_unpacklo_epi16(w0, w1);
w5 = _mm_unpacklo_epi16(w2, w3);
w12 = _mm_unpacklo_epi16(w8, w9);
w13 = _mm_unpacklo_epi16(w10, w11);
w6 = _mm_unpacklo_epi32(w4, w5);
w7 = _mm_unpackhi_epi32(w4, w5);
w14 = _mm_unpacklo_epi32(w12, w13);
w15 = _mm_unpackhi_epi32(w12, w13);
// Store first 4-line result
*d0 = _mm_unpacklo_epi64(w6, w14);
*d1 = _mm_unpackhi_epi64(w6, w14);
*d2 = _mm_unpacklo_epi64(w7, w15);
*d3 = _mm_unpackhi_epi64(w7, w15);
w4 = _mm_unpackhi_epi16(w0, w1);
w5 = _mm_unpackhi_epi16(w2, w3);
w12 = _mm_unpackhi_epi16(w8, w9);
w13 = _mm_unpackhi_epi16(w10, w11);
w6 = _mm_unpacklo_epi32(w4, w5);
w7 = _mm_unpackhi_epi32(w4, w5);
w14 = _mm_unpacklo_epi32(w12, w13);
w15 = _mm_unpackhi_epi32(w12, w13);
// Store second 4-line result
*d4 = _mm_unpacklo_epi64(w6, w14);
*d5 = _mm_unpackhi_epi64(w6, w14);
*d6 = _mm_unpacklo_epi64(w7, w15);
*d7 = _mm_unpackhi_epi64(w7, w15);
}
static INLINE void transpose8x16_16x8_sse2(
__m128i *x0, __m128i *x1, __m128i *x2, __m128i *x3, __m128i *x4,
__m128i *x5, __m128i *x6, __m128i *x7, __m128i *d0d1, __m128i *d2d3,
__m128i *d4d5, __m128i *d6d7, __m128i *d8d9, __m128i *d10d11,
__m128i *d12d13, __m128i *d14d15) {
__m128i w0, w1, w2, w3, w4, w5, w6, w7, w8, w9;
__m128i w10, w11, w12, w13, w14, w15;
w0 = _mm_unpacklo_epi8(*x0, *x1);
w1 = _mm_unpacklo_epi8(*x2, *x3);
w2 = _mm_unpacklo_epi8(*x4, *x5);
w3 = _mm_unpacklo_epi8(*x6, *x7);
w8 = _mm_unpackhi_epi8(*x0, *x1);
w9 = _mm_unpackhi_epi8(*x2, *x3);
w10 = _mm_unpackhi_epi8(*x4, *x5);
w11 = _mm_unpackhi_epi8(*x6, *x7);
w4 = _mm_unpacklo_epi16(w0, w1);
w5 = _mm_unpacklo_epi16(w2, w3);
w12 = _mm_unpacklo_epi16(w8, w9);
w13 = _mm_unpacklo_epi16(w10, w11);
w6 = _mm_unpacklo_epi32(w4, w5);
w7 = _mm_unpackhi_epi32(w4, w5);
w14 = _mm_unpacklo_epi32(w12, w13);
w15 = _mm_unpackhi_epi32(w12, w13);
// Store first 4-line result
*d0d1 = _mm_unpacklo_epi64(w6, w14);
*d2d3 = _mm_unpackhi_epi64(w6, w14);
*d4d5 = _mm_unpacklo_epi64(w7, w15);
*d6d7 = _mm_unpackhi_epi64(w7, w15);
w4 = _mm_unpackhi_epi16(w0, w1);
w5 = _mm_unpackhi_epi16(w2, w3);
w12 = _mm_unpackhi_epi16(w8, w9);
w13 = _mm_unpackhi_epi16(w10, w11);
w6 = _mm_unpacklo_epi32(w4, w5);
w7 = _mm_unpackhi_epi32(w4, w5);
w14 = _mm_unpacklo_epi32(w12, w13);
w15 = _mm_unpackhi_epi32(w12, w13);
// Store second 4-line result
*d8d9 = _mm_unpacklo_epi64(w6, w14);
*d10d11 = _mm_unpackhi_epi64(w6, w14);
*d12d13 = _mm_unpacklo_epi64(w7, w15);
*d14d15 = _mm_unpackhi_epi64(w7, w15);
}
#endif // AOM_AOM_DSP_X86_LPF_COMMON_SSE2_H_

69
third_party/aom/aom_dsp/x86/masked_variance_intrin_ssse3.c vendored
View file

@ -218,15 +218,15 @@ static void bilinear_filter(const uint8_t *src, int src_stride, int xoffset,
}
}
static INLINE __m128i filter_block_2rows(const __m128i a0, const __m128i b0,
const __m128i a1, const __m128i b1,
const __m128i filter) {
__m128i v0 = _mm_unpacklo_epi8(a0, b0);
v0 = _mm_maddubs_epi16(v0, filter);
static INLINE __m128i filter_block_2rows(const __m128i *a0, const __m128i *b0,
const __m128i *a1, const __m128i *b1,
const __m128i *filter) {
__m128i v0 = _mm_unpacklo_epi8(*a0, *b0);
v0 = _mm_maddubs_epi16(v0, *filter);
v0 = xx_roundn_epu16(v0, FILTER_BITS);
__m128i v1 = _mm_unpacklo_epi8(a1, b1);
v1 = _mm_maddubs_epi16(v1, filter);
__m128i v1 = _mm_unpacklo_epi8(*a1, *b1);
v1 = _mm_maddubs_epi16(v1, *filter);
v1 = xx_roundn_epu16(v1, FILTER_BITS);
return _mm_packus_epi16(v0, v1);
@ -262,7 +262,7 @@ static void bilinear_filter8xh(const uint8_t *src, int src_stride, int xoffset,
const __m128i z0 = _mm_srli_si128(x0, 1);
const __m128i x1 = _mm_loadu_si128((__m128i *)&src[src_stride]);
const __m128i z1 = _mm_srli_si128(x1, 1);
const __m128i res = filter_block_2rows(x0, z0, x1, z1, hfilter_vec);
const __m128i res = filter_block_2rows(&x0, &z0, &x1, &z1, &hfilter_vec);
_mm_storeu_si128((__m128i *)b, res);
src += src_stride * 2;
@ -296,7 +296,7 @@ static void bilinear_filter8xh(const uint8_t *src, int src_stride, int xoffset,
const __m128i x = _mm_loadl_epi64((__m128i *)dst);
const __m128i y = _mm_loadl_epi64((__m128i *)&dst[8]);
const __m128i z = _mm_loadl_epi64((__m128i *)&dst[16]);
const __m128i res = filter_block_2rows(x, y, y, z, vfilter_vec);
const __m128i res = filter_block_2rows(&x, &y, &y, &z, &vfilter_vec);
_mm_storeu_si128((__m128i *)dst, res);
dst += 16;
@ -343,7 +343,7 @@ static void bilinear_filter4xh(const uint8_t *src, int src_stride, int xoffset,
const __m128i b0 = _mm_unpacklo_epi32(z0, z1);
const __m128i a1 = _mm_unpacklo_epi32(x2, x3);
const __m128i b1 = _mm_unpacklo_epi32(z2, z3);
const __m128i res = filter_block_2rows(a0, b0, a1, b1, hfilter_vec);
const __m128i res = filter_block_2rows(&a0, &b0, &a1, &b1, &hfilter_vec);
_mm_storeu_si128((__m128i *)b, res);
src += src_stride * 4;
@ -384,7 +384,7 @@ static void bilinear_filter4xh(const uint8_t *src, int src_stride, int xoffset,
const __m128i b0 = _mm_unpacklo_epi32(b, c);
const __m128i a1 = _mm_unpacklo_epi32(c, d);
const __m128i b1 = _mm_unpacklo_epi32(d, e);
const __m128i res = filter_block_2rows(a0, b0, a1, b1, vfilter_vec);
const __m128i res = filter_block_2rows(&a0, &b0, &a1, &b1, &vfilter_vec);
_mm_storeu_si128((__m128i *)dst, res);
dst += 16;
@ -392,29 +392,29 @@ static void bilinear_filter4xh(const uint8_t *src, int src_stride, int xoffset,
}
}
static INLINE void accumulate_block(const __m128i src, const __m128i a,
const __m128i b, const __m128i m,
static INLINE void accumulate_block(const __m128i *src, const __m128i *a,
const __m128i *b, const __m128i *m,
__m128i *sum, __m128i *sum_sq) {
const __m128i zero = _mm_setzero_si128();
const __m128i one = _mm_set1_epi16(1);
const __m128i mask_max = _mm_set1_epi8((1 << AOM_BLEND_A64_ROUND_BITS));
const __m128i m_inv = _mm_sub_epi8(mask_max, m);
const __m128i m_inv = _mm_sub_epi8(mask_max, *m);
// Calculate 16 predicted pixels.
// Note that the maximum value of any entry of 'pred_l' or 'pred_r'
// is 64 * 255, so we have plenty of space to add rounding constants.
const __m128i data_l = _mm_unpacklo_epi8(a, b);
const __m128i mask_l = _mm_unpacklo_epi8(m, m_inv);
const __m128i data_l = _mm_unpacklo_epi8(*a, *b);
const __m128i mask_l = _mm_unpacklo_epi8(*m, m_inv);
__m128i pred_l = _mm_maddubs_epi16(data_l, mask_l);
pred_l = xx_roundn_epu16(pred_l, AOM_BLEND_A64_ROUND_BITS);
const __m128i data_r = _mm_unpackhi_epi8(a, b);
const __m128i mask_r = _mm_unpackhi_epi8(m, m_inv);
const __m128i data_r = _mm_unpackhi_epi8(*a, *b);
const __m128i mask_r = _mm_unpackhi_epi8(*m, m_inv);
__m128i pred_r = _mm_maddubs_epi16(data_r, mask_r);
pred_r = xx_roundn_epu16(pred_r, AOM_BLEND_A64_ROUND_BITS);
const __m128i src_l = _mm_unpacklo_epi8(src, zero);
const __m128i src_r = _mm_unpackhi_epi8(src, zero);
const __m128i src_l = _mm_unpacklo_epi8(*src, zero);
const __m128i src_r = _mm_unpackhi_epi8(*src, zero);
const __m128i diff_l = _mm_sub_epi16(pred_l, src_l);
const __m128i diff_r = _mm_sub_epi16(pred_r, src_r);
@ -440,7 +440,7 @@ static void masked_variance(const uint8_t *src_ptr, int src_stride,
const __m128i a = _mm_loadu_si128((const __m128i *)&a_ptr[x]);
const __m128i b = _mm_loadu_si128((const __m128i *)&b_ptr[x]);
const __m128i m = _mm_loadu_si128((const __m128i *)&m_ptr[x]);
accumulate_block(src, a, b, m, &sum, &sum_sq);
accumulate_block(&src, &a, &b, &m, &sum, &sum_sq);
}
src_ptr += src_stride;
@ -471,7 +471,7 @@ static void masked_variance8xh(const uint8_t *src_ptr, int src_stride,
const __m128i m =
_mm_unpacklo_epi64(_mm_loadl_epi64((const __m128i *)m_ptr),
_mm_loadl_epi64((const __m128i *)&m_ptr[m_stride]));
accumulate_block(src, a, b, m, &sum, &sum_sq);
accumulate_block(&src, &a, &b, &m, &sum, &sum_sq);
src_ptr += src_stride * 2;
a_ptr += 16;
@ -503,7 +503,7 @@ static void masked_variance4xh(const uint8_t *src_ptr, int src_stride,
const __m128i m = _mm_setr_epi32(
*(uint32_t *)m_ptr, *(uint32_t *)&m_ptr[m_stride],
*(uint32_t *)&m_ptr[m_stride * 2], *(uint32_t *)&m_ptr[m_stride * 3]);
accumulate_block(src, a, b, m, &sum, &sum_sq);
accumulate_block(&src, &a, &b, &m, &sum, &sum_sq);
src_ptr += src_stride * 4;
a_ptr += 16;
@ -797,17 +797,17 @@ static void highbd_bilinear_filter(const uint16_t *src, int src_stride,
}
}
static INLINE __m128i highbd_filter_block_2rows(const __m128i a0,
const __m128i b0,
const __m128i a1,
const __m128i b1,
const __m128i filter) {
__m128i v0 = _mm_unpacklo_epi16(a0, b0);
v0 = _mm_madd_epi16(v0, filter);
static INLINE __m128i highbd_filter_block_2rows(const __m128i *a0,
const __m128i *b0,
const __m128i *a1,
const __m128i *b1,
const __m128i *filter) {
__m128i v0 = _mm_unpacklo_epi16(*a0, *b0);
v0 = _mm_madd_epi16(v0, *filter);
v0 = xx_roundn_epu32(v0, FILTER_BITS);
__m128i v1 = _mm_unpacklo_epi16(a1, b1);
v1 = _mm_madd_epi16(v1, filter);
__m128i v1 = _mm_unpacklo_epi16(*a1, *b1);
v1 = _mm_madd_epi16(v1, *filter);
v1 = xx_roundn_epu32(v1, FILTER_BITS);
return _mm_packs_epi32(v0, v1);
@ -845,7 +845,7 @@ static void highbd_bilinear_filter4xh(const uint16_t *src, int src_stride,
const __m128i x1 = _mm_loadu_si128((__m128i *)&src[src_stride]);
const __m128i z1 = _mm_srli_si128(x1, 2);
const __m128i res =
highbd_filter_block_2rows(x0, z0, x1, z1, hfilter_vec);
highbd_filter_block_2rows(&x0, &z0, &x1, &z1, &hfilter_vec);
_mm_storeu_si128((__m128i *)b, res);
src += src_stride * 2;
@ -879,7 +879,8 @@ static void highbd_bilinear_filter4xh(const uint16_t *src, int src_stride,
const __m128i x = _mm_loadl_epi64((__m128i *)dst);
const __m128i y = _mm_loadl_epi64((__m128i *)&dst[4]);
const __m128i z = _mm_loadl_epi64((__m128i *)&dst[8]);
const __m128i res = highbd_filter_block_2rows(x, y, y, z, vfilter_vec);
const __m128i res =
highbd_filter_block_2rows(&x, &y, &y, &z, &vfilter_vec);
_mm_storeu_si128((__m128i *)dst, res);
dst += 8;

43
third_party/aom/aom_dsp/x86/quantize_avx_x86_64.asm vendored
View file

@ -126,7 +126,7 @@ cglobal quantize_%1, 0, %2, 15, coeff, ncoeff, zbin, round, quant, \
punpckhqdq m3, m3
pmullw m13, m3 ; dqc[i] = qc[i] * q
; Store 16bit numbers as 32bit numbers in array pointed to by qcoeff
; Store 16bit numbers as 32bit numbers in array pointed to by dqcoeff
pcmpgtw m6, m5, m8
punpckhwd m6, m8, m6
pmovsxwd m11, m8
@ -198,10 +198,7 @@ DEFINE_ARGS coeff, ncoeff, zbin, round, quant, shift, \
mova m4, [r2] ; m4 = shift
mov r4, dqcoeffmp
mov r5, iscanmp
%ifidn %1, b_32x32
psllw m4, 1
%endif
pxor m5, m5 ; m5 = dedicated zero
pxor m5, m5 ; m5 = dedicated zero
DEFINE_ARGS coeff, ncoeff, d1, qcoeff, dqcoeff, iscan, d2, d3, d4, eob
@ -255,9 +252,26 @@ DEFINE_ARGS coeff, ncoeff, zbin, round, quant, shift, \
pmulhw m13, m11, m2 ; m13 = m11*q>>16
paddw m8, m6 ; m8 += m6
paddw m13, m11 ; m13 += m11
%ifidn %1, b_32x32
pmullw m5, m8, m4 ; store the lower 16 bits of m8*qsh
%endif
pmulhw m8, m4 ; m8 = m8*qsh>>16
%ifidn %1, b_32x32
psllw m8, 1
psrlw m5, 15
por m8, m5
%endif
punpckhqdq m4, m4
%ifidn %1, b_32x32
pmullw m5, m13, m4 ; store the lower 16 bits of m13*qsh
%endif
pmulhw m13, m4 ; m13 = m13*qsh>>16
%ifidn %1, b_32x32
psllw m13, 1
psrlw m5, 15
por m13, m5
pxor m5, m5 ; reset m5 to zero register
%endif
psignw m8, m9 ; m8 = reinsert sign
psignw m13, m10 ; m13 = reinsert sign
pand m8, m7
@ -289,7 +303,7 @@ DEFINE_ARGS coeff, ncoeff, zbin, round, quant, shift, \
psignw m13, m10
%endif
; store 16bit numbers as 32bit numbers in array pointed to by qcoeff
; store 16bit numbers as 32bit numbers in array pointed to by dqcoeff
pcmpgtw m6, m5, m8
punpckhwd m6, m8, m6
pmovsxwd m11, m8
@ -359,8 +373,23 @@ DEFINE_ARGS coeff, ncoeff, zbin, round, quant, shift, \
pmulhw m13, m11, m2 ; m13 = m11*q>>16
paddw m14, m6 ; m14 += m6
paddw m13, m11 ; m13 += m11
%ifidn %1, b_32x32
pmullw m5, m14, m4 ; store the lower 16 bits of m14*qsh
%endif
pmulhw m14, m4 ; m14 = m14*qsh>>16
%ifidn %1, b_32x32
psllw m14, 1
psrlw m5, 15
por m14, m5
pmullw m5, m13, m4 ; store the lower 16 bits of m13*qsh
%endif
pmulhw m13, m4 ; m13 = m13*qsh>>16
%ifidn %1, b_32x32
psllw m13, 1
psrlw m5, 15
por m13, m5
pxor m5, m5 ; reset m5 to zero register
%endif
psignw m14, m9 ; m14 = reinsert sign
psignw m13, m10 ; m13 = reinsert sign
pand m14, m7
@ -391,7 +420,7 @@ DEFINE_ARGS coeff, ncoeff, zbin, round, quant, shift, \
psignw m13, m10
%endif
; store 16bit numbers as 32bit numbers in array pointed to by qcoeff
; store 16bit numbers as 32bit numbers in array pointed to by dqcoeff
pcmpgtw m6, m5, m14
punpckhwd m6, m14, m6
pmovsxwd m11, m14

22
third_party/aom/aom_dsp/x86/quantize_sse2.c vendored
View file

@ -18,28 +18,6 @@
#include "aom/aom_integer.h"
#include "aom_dsp/x86/quantize_x86.h"
static INLINE __m128i load_coefficients(const tran_low_t *coeff_ptr) {
assert(sizeof(tran_low_t) == 4);
return _mm_setr_epi16((int16_t)coeff_ptr[0], (int16_t)coeff_ptr[1],
(int16_t)coeff_ptr[2], (int16_t)coeff_ptr[3],
(int16_t)coeff_ptr[4], (int16_t)coeff_ptr[5],
(int16_t)coeff_ptr[6], (int16_t)coeff_ptr[7]);
}
static INLINE void store_coefficients(__m128i coeff_vals,
tran_low_t *coeff_ptr) {
assert(sizeof(tran_low_t) == 4);
__m128i one = _mm_set1_epi16(1);
__m128i coeff_vals_hi = _mm_mulhi_epi16(coeff_vals, one);
__m128i coeff_vals_lo = _mm_mullo_epi16(coeff_vals, one);
__m128i coeff_vals_1 = _mm_unpacklo_epi16(coeff_vals_lo, coeff_vals_hi);
__m128i coeff_vals_2 = _mm_unpackhi_epi16(coeff_vals_lo, coeff_vals_hi);
_mm_store_si128((__m128i *)(coeff_ptr), coeff_vals_1);
_mm_store_si128((__m128i *)(coeff_ptr + 4), coeff_vals_2);
}
void aom_quantize_b_sse2(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
const int16_t *zbin_ptr, const int16_t *round_ptr,
const int16_t *quant_ptr,

192
third_party/aom/aom_dsp/x86/quantize_ssse3.c vendored Normal file
View file

@ -0,0 +1,192 @@
/*
* Copyright (c) 2019, Alliance for Open Media. All rights reserved
*
* This source code is subject to the terms of the BSD 2 Clause License and
* the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
* was not distributed with this source code in the LICENSE file, you can
* obtain it at www.aomedia.org/license/software. If the Alliance for Open
* Media Patent License 1.0 was not distributed with this source code in the
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
*/
#include <assert.h>
#include <tmmintrin.h>
#include <emmintrin.h>
#include <xmmintrin.h>
#include "config/aom_dsp_rtcd.h"
#include "aom/aom_integer.h"
#include "aom_dsp/x86/quantize_x86.h"
static INLINE void calculate_qcoeff_64x64(__m128i *coeff, const __m128i round,
const __m128i quant,
const __m128i *shift) {
__m128i tmp, qcoeff, tmp1;
qcoeff = _mm_adds_epi16(*coeff, round);
tmp = _mm_mulhi_epi16(qcoeff, quant);
qcoeff = _mm_add_epi16(tmp, qcoeff);
tmp = _mm_mullo_epi16(qcoeff, *shift);
tmp = _mm_srli_epi16(tmp, 14);
tmp1 = _mm_mulhi_epi16(qcoeff, *shift);
tmp1 = _mm_slli_epi16(tmp1, 2);
*coeff = _mm_or_si128(tmp, tmp1);
}
static INLINE void calculate_dqcoeff_and_store_64x64(const __m128i qcoeff,
const __m128i dequant,
const __m128i zero,
tran_low_t *dqcoeff) {
// Un-sign to bias rounding like C.
const __m128i coeff = _mm_abs_epi16(qcoeff);
const __m128i sign_0 = _mm_unpacklo_epi16(zero, qcoeff);
const __m128i sign_1 = _mm_unpackhi_epi16(zero, qcoeff);
const __m128i low = _mm_mullo_epi16(coeff, dequant);
const __m128i high = _mm_mulhi_epi16(coeff, dequant);
__m128i dqcoeff32_0 = _mm_unpacklo_epi16(low, high);
__m128i dqcoeff32_1 = _mm_unpackhi_epi16(low, high);
// "Divide" by 4.
dqcoeff32_0 = _mm_srli_epi32(dqcoeff32_0, 2);
dqcoeff32_1 = _mm_srli_epi32(dqcoeff32_1, 2);
dqcoeff32_0 = _mm_sign_epi32(dqcoeff32_0, sign_0);
dqcoeff32_1 = _mm_sign_epi32(dqcoeff32_1, sign_1);
_mm_store_si128((__m128i *)(dqcoeff), dqcoeff32_0);
_mm_store_si128((__m128i *)(dqcoeff + 4), dqcoeff32_1);
}
void aom_quantize_b_64x64_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
const int16_t *zbin_ptr,
const int16_t *round_ptr,
const int16_t *quant_ptr,
const int16_t *quant_shift_ptr,
tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr,
const int16_t *dequant_ptr, uint16_t *eob_ptr,
const int16_t *scan, const int16_t *iscan) {
const __m128i zero = _mm_setzero_si128();
const __m128i one = _mm_set1_epi16(1);
const __m128i two = _mm_set1_epi16(2);
int index;
__m128i zbin, round, quant, dequant, shift;
__m128i coeff0, coeff1, qcoeff0, qcoeff1;
__m128i cmp_mask0, cmp_mask1, all_zero;
__m128i eob = zero, eob0;
(void)scan;
(void)n_coeffs;
// Setup global values.
zbin = _mm_load_si128((const __m128i *)zbin_ptr);
round = _mm_load_si128((const __m128i *)round_ptr);
quant = _mm_load_si128((const __m128i *)quant_ptr);
dequant = _mm_load_si128((const __m128i *)dequant_ptr);
shift = _mm_load_si128((const __m128i *)quant_shift_ptr);
// Shift with rounding.
zbin = _mm_add_epi16(zbin, two);
round = _mm_add_epi16(round, two);
zbin = _mm_srli_epi16(zbin, 2);
round = _mm_srli_epi16(round, 2);
zbin = _mm_sub_epi16(zbin, one);
// Do DC and first 15 AC.
coeff0 = load_coefficients(coeff_ptr);
coeff1 = load_coefficients(coeff_ptr + 8);
qcoeff0 = _mm_abs_epi16(coeff0);
qcoeff1 = _mm_abs_epi16(coeff1);
cmp_mask0 = _mm_cmpgt_epi16(qcoeff0, zbin);
zbin = _mm_unpackhi_epi64(zbin, zbin);
cmp_mask1 = _mm_cmpgt_epi16(qcoeff1, zbin);
all_zero = _mm_or_si128(cmp_mask0, cmp_mask1);
if (_mm_movemask_epi8(all_zero) == 0) {
_mm_store_si128((__m128i *)(qcoeff_ptr), zero);
_mm_store_si128((__m128i *)(qcoeff_ptr + 4), zero);
_mm_store_si128((__m128i *)(qcoeff_ptr + 8), zero);
_mm_store_si128((__m128i *)(qcoeff_ptr + 12), zero);
_mm_store_si128((__m128i *)(dqcoeff_ptr), zero);
_mm_store_si128((__m128i *)(dqcoeff_ptr + 4), zero);
_mm_store_si128((__m128i *)(dqcoeff_ptr + 8), zero);
_mm_store_si128((__m128i *)(dqcoeff_ptr + 12), zero);
round = _mm_unpackhi_epi64(round, round);
quant = _mm_unpackhi_epi64(quant, quant);
shift = _mm_unpackhi_epi64(shift, shift);
dequant = _mm_unpackhi_epi64(dequant, dequant);
} else {
calculate_qcoeff_64x64(&qcoeff0, round, quant, &shift);
round = _mm_unpackhi_epi64(round, round);
quant = _mm_unpackhi_epi64(quant, quant);
shift = _mm_unpackhi_epi64(shift, shift);
calculate_qcoeff_64x64(&qcoeff1, round, quant, &shift);
// Reinsert signs.
qcoeff0 = _mm_sign_epi16(qcoeff0, coeff0);
qcoeff1 = _mm_sign_epi16(qcoeff1, coeff1);
// Mask out zbin threshold coeffs.
qcoeff0 = _mm_and_si128(qcoeff0, cmp_mask0);
qcoeff1 = _mm_and_si128(qcoeff1, cmp_mask1);
store_coefficients(qcoeff0, qcoeff_ptr);
store_coefficients(qcoeff1, qcoeff_ptr + 8);
calculate_dqcoeff_and_store_64x64(qcoeff0, dequant, zero, dqcoeff_ptr);
dequant = _mm_unpackhi_epi64(dequant, dequant);
calculate_dqcoeff_and_store_64x64(qcoeff1, dequant, zero, dqcoeff_ptr + 8);
eob =
scan_for_eob(&qcoeff0, &qcoeff1, cmp_mask0, cmp_mask1, iscan, 0, zero);
}
// AC only loop.
for (index = 16; index < 1024; index += 16) {
coeff0 = load_coefficients(coeff_ptr + index);
coeff1 = load_coefficients(coeff_ptr + index + 8);
qcoeff0 = _mm_abs_epi16(coeff0);
qcoeff1 = _mm_abs_epi16(coeff1);
cmp_mask0 = _mm_cmpgt_epi16(qcoeff0, zbin);
cmp_mask1 = _mm_cmpgt_epi16(qcoeff1, zbin);
all_zero = _mm_or_si128(cmp_mask0, cmp_mask1);
if (_mm_movemask_epi8(all_zero) == 0) {
_mm_store_si128((__m128i *)(qcoeff_ptr + index), zero);
_mm_store_si128((__m128i *)(qcoeff_ptr + index + 4), zero);
_mm_store_si128((__m128i *)(qcoeff_ptr + index + 8), zero);
_mm_store_si128((__m128i *)(qcoeff_ptr + index + 12), zero);
_mm_store_si128((__m128i *)(dqcoeff_ptr + index), zero);
_mm_store_si128((__m128i *)(dqcoeff_ptr + index + 4), zero);
_mm_store_si128((__m128i *)(dqcoeff_ptr + index + 8), zero);
_mm_store_si128((__m128i *)(dqcoeff_ptr + index + 12), zero);
continue;
}
calculate_qcoeff_64x64(&qcoeff0, round, quant, &shift);
calculate_qcoeff_64x64(&qcoeff1, round, quant, &shift);
qcoeff0 = _mm_sign_epi16(qcoeff0, coeff0);
qcoeff1 = _mm_sign_epi16(qcoeff1, coeff1);
qcoeff0 = _mm_and_si128(qcoeff0, cmp_mask0);
qcoeff1 = _mm_and_si128(qcoeff1, cmp_mask1);
store_coefficients(qcoeff0, qcoeff_ptr + index);
store_coefficients(qcoeff1, qcoeff_ptr + index + 8);
calculate_dqcoeff_and_store_64x64(qcoeff0, dequant, zero,
dqcoeff_ptr + index);
calculate_dqcoeff_and_store_64x64(qcoeff1, dequant, zero,
dqcoeff_ptr + 8 + index);
eob0 = scan_for_eob(&qcoeff0, &qcoeff1, cmp_mask0, cmp_mask1, iscan, index,
zero);
eob = _mm_max_epi16(eob, eob0);
}
*eob_ptr = accumulate_eob(eob);
}

40
third_party/aom/aom_dsp/x86/quantize_ssse3_x86_64.asm vendored
View file

@ -48,9 +48,6 @@ cglobal quantize_%1, 0, %2, 15, coeff, ncoeff, zbin, round, quant, \
mov r3, qcoeffmp
mov r4, dqcoeffmp
mov r5, iscanmp
%ifidn %1, b_32x32
psllw m4, 1
%endif
pxor m5, m5 ; m5 = dedicated zero
DEFINE_ARGS coeff, ncoeff, d1, qcoeff, dqcoeff, iscan, d2, d3, d4, eob
lea coeffq, [ coeffq+ncoeffq*4]
@ -78,9 +75,26 @@ cglobal quantize_%1, 0, %2, 15, coeff, ncoeff, zbin, round, quant, \
pmulhw m13, m11, m2 ; m13 = m11*q>>16
paddw m8, m6 ; m8 += m6
paddw m13, m11 ; m13 += m11
%ifidn %1, b_32x32
pmullw m5, m8, m4 ; store the lower 16 bits of m8*qsh
%endif
pmulhw m8, m4 ; m8 = m8*qsh>>16
%ifidn %1, b_32x32
psllw m8, 1
psrlw m5, 15
por m8, m5
%endif
punpckhqdq m4, m4
%ifidn %1, b_32x32
pmullw m5, m13, m4 ; store the lower 16 bits of m13*qsh
%endif
pmulhw m13, m4 ; m13 = m13*qsh>>16
%ifidn %1, b_32x32
psllw m13, 1
psrlw m5, 15
por m13, m5
pxor m5, m5 ; reset m5 to zero register
%endif
psignw m8, m9 ; m8 = reinsert sign
psignw m13, m10 ; m13 = reinsert sign
pand m8, m7
@ -117,7 +131,7 @@ cglobal quantize_%1, 0, %2, 15, coeff, ncoeff, zbin, round, quant, \
psignw m8, m9
psignw m13, m10
%endif
; store 16bit numbers as 32bit numbers in array pointed to by qcoeff
; store 16bit numbers as 32bit numbers in array pointed to by dqcoeff
mova m11, m8
mova m6, m8
pcmpgtw m5, m8
@ -169,12 +183,28 @@ cglobal quantize_%1, 0, %2, 15, coeff, ncoeff, zbin, round, quant, \
pmulhw m13, m11, m2 ; m13 = m11*q>>16
paddw m14, m6 ; m14 += m6
paddw m13, m11 ; m13 += m11
%ifidn %1, b_32x32
pmullw m5, m14, m4 ; store the lower 16 bits of m14*qsh
%endif
pmulhw m14, m4 ; m14 = m14*qsh>>16
%ifidn %1, b_32x32
psllw m14, 1
psrlw m5, 15
por m14, m5
pmullw m5, m13, m4 ; store the lower 16 bits of m13*qsh
%endif
pmulhw m13, m4 ; m13 = m13*qsh>>16
%ifidn %1, b_32x32
psllw m13, 1
psrlw m5, 15
por m13, m5
pxor m5, m5 ; reset m5 to zero register
%endif
psignw m14, m9 ; m14 = reinsert sign
psignw m13, m10 ; m13 = reinsert sign
pand m14, m7
pand m13, m12
; store 16bit numbers as 32bit numbers in array pointed to by qcoeff
pxor m11, m11
mova m11, m14
@ -207,7 +237,7 @@ cglobal quantize_%1, 0, %2, 15, coeff, ncoeff, zbin, round, quant, \
psignw m13, m10
%endif
; store 16bit numbers as 32bit numbers in array pointed to by qcoeff
; store 16bit numbers as 32bit numbers in array pointed to by dqcoeff
mova m11, m14
mova m6, m14
pcmpgtw m5, m14

20
third_party/aom/aom_dsp/x86/quantize_x86.h vendored
View file

@ -75,3 +75,23 @@ static INLINE int16_t accumulate_eob(__m128i eob) {
eob = _mm_max_epi16(eob, eob_shuffled);
return _mm_extract_epi16(eob, 1);
}
static INLINE __m128i load_coefficients(const tran_low_t *coeff_ptr) {
assert(sizeof(tran_low_t) == 4);
const __m128i coeff1 = _mm_load_si128((__m128i *)(coeff_ptr));
const __m128i coeff2 = _mm_load_si128((__m128i *)(coeff_ptr + 4));
return _mm_packs_epi32(coeff1, coeff2);
}
static INLINE void store_coefficients(__m128i coeff_vals,
tran_low_t *coeff_ptr) {
assert(sizeof(tran_low_t) == 4);
__m128i one = _mm_set1_epi16(1);
__m128i coeff_vals_hi = _mm_mulhi_epi16(coeff_vals, one);
__m128i coeff_vals_lo = _mm_mullo_epi16(coeff_vals, one);
__m128i coeff_vals_1 = _mm_unpacklo_epi16(coeff_vals_lo, coeff_vals_hi);
__m128i coeff_vals_2 = _mm_unpackhi_epi16(coeff_vals_lo, coeff_vals_hi);
_mm_store_si128((__m128i *)(coeff_ptr), coeff_vals_1);
_mm_store_si128((__m128i *)(coeff_ptr + 4), coeff_vals_2);
}

308
third_party/aom/aom_dsp/x86/sse_avx2.c vendored
View file

@ -21,12 +21,11 @@ static INLINE void sse_w32_avx2(__m256i *sum, const uint8_t *a,
const uint8_t *b) {
const __m256i v_a0 = yy_loadu_256(a);
const __m256i v_b0 = yy_loadu_256(b);
const __m256i v_a00_w = _mm256_cvtepu8_epi16(_mm256_castsi256_si128(v_a0));
const __m256i v_a01_w =
_mm256_cvtepu8_epi16(_mm256_extracti128_si256(v_a0, 1));
const __m256i v_b00_w = _mm256_cvtepu8_epi16(_mm256_castsi256_si128(v_b0));
const __m256i v_b01_w =
_mm256_cvtepu8_epi16(_mm256_extracti128_si256(v_b0, 1));
const __m256i zero = _mm256_setzero_si256();
const __m256i v_a00_w = _mm256_unpacklo_epi8(v_a0, zero);
const __m256i v_a01_w = _mm256_unpackhi_epi8(v_a0, zero);
const __m256i v_b00_w = _mm256_unpacklo_epi8(v_b0, zero);
const __m256i v_b01_w = _mm256_unpackhi_epi8(v_b0, zero);
const __m256i v_d00_w = _mm256_sub_epi16(v_a00_w, v_b00_w);
const __m256i v_d01_w = _mm256_sub_epi16(v_a01_w, v_b01_w);
*sum = _mm256_add_epi32(*sum, _mm256_madd_epi16(v_d00_w, v_d00_w));
@ -35,43 +34,76 @@ static INLINE void sse_w32_avx2(__m256i *sum, const uint8_t *a,
static INLINE int64_t summary_all_avx2(const __m256i *sum_all) {
int64_t sum;
const __m256i sum0_4x64 =
_mm256_cvtepu32_epi64(_mm256_castsi256_si128(*sum_all));
const __m256i sum1_4x64 =
_mm256_cvtepu32_epi64(_mm256_extracti128_si256(*sum_all, 1));
__m256i zero = _mm256_setzero_si256();
const __m256i sum0_4x64 = _mm256_unpacklo_epi32(*sum_all, zero);
const __m256i sum1_4x64 = _mm256_unpackhi_epi32(*sum_all, zero);
const __m256i sum_4x64 = _mm256_add_epi64(sum0_4x64, sum1_4x64);
const __m128i sum_2x64 = _mm_add_epi64(_mm256_castsi256_si128(sum_4x64),
_mm256_extracti128_si256(sum_4x64, 1));
const __m128i sum_1x64 = _mm_add_epi64(sum_2x64, _mm_srli_si128(sum_2x64, 8));
xx_storel_64(&sum, sum_1x64);
return sum;
}
static INLINE void summary_32_avx2(const __m256i *sum32, __m256i *sum) {
const __m256i sum0_4x64 =
_mm256_cvtepu32_epi64(_mm256_castsi256_si128(*sum32));
const __m256i sum1_4x64 =
_mm256_cvtepu32_epi64(_mm256_extracti128_si256(*sum32, 1));
const __m256i sum_4x64 = _mm256_add_epi64(sum0_4x64, sum1_4x64);
*sum = _mm256_add_epi64(*sum, sum_4x64);
}
static INLINE int64_t summary_4x64_avx2(const __m256i sum_4x64) {
int64_t sum;
const __m128i sum_2x64 = _mm_add_epi64(_mm256_castsi256_si128(sum_4x64),
_mm256_extracti128_si256(sum_4x64, 1));
const __m128i sum_1x64 = _mm_add_epi64(sum_2x64, _mm_srli_si128(sum_2x64, 8));
xx_storel_64(&sum, sum_1x64);
return sum;
}
static INLINE void sse_w4x4_avx2(const uint8_t *a, int a_stride,
const uint8_t *b, int b_stride, __m256i *sum) {
const __m128i v_a0 = xx_loadl_32(a);
const __m128i v_a1 = xx_loadl_32(a + a_stride);
const __m128i v_a2 = xx_loadl_32(a + a_stride * 2);
const __m128i v_a3 = xx_loadl_32(a + a_stride * 3);
const __m128i v_b0 = xx_loadl_32(b);
const __m128i v_b1 = xx_loadl_32(b + b_stride);
const __m128i v_b2 = xx_loadl_32(b + b_stride * 2);
const __m128i v_b3 = xx_loadl_32(b + b_stride * 3);
const __m128i v_a0123 = _mm_unpacklo_epi64(_mm_unpacklo_epi32(v_a0, v_a1),
_mm_unpacklo_epi32(v_a2, v_a3));
const __m128i v_b0123 = _mm_unpacklo_epi64(_mm_unpacklo_epi32(v_b0, v_b1),
_mm_unpacklo_epi32(v_b2, v_b3));
const __m256i v_a_w = _mm256_cvtepu8_epi16(v_a0123);
const __m256i v_b_w = _mm256_cvtepu8_epi16(v_b0123);
const __m256i v_d_w = _mm256_sub_epi16(v_a_w, v_b_w);
*sum = _mm256_add_epi32(*sum, _mm256_madd_epi16(v_d_w, v_d_w));
}
static INLINE void sse_w8x2_avx2(const uint8_t *a, int a_stride,
const uint8_t *b, int b_stride, __m256i *sum) {
const __m128i v_a0 = xx_loadl_64(a);
const __m128i v_a1 = xx_loadl_64(a + a_stride);
const __m128i v_b0 = xx_loadl_64(b);
const __m128i v_b1 = xx_loadl_64(b + b_stride);
const __m256i v_a_w = _mm256_cvtepu8_epi16(_mm_unpacklo_epi64(v_a0, v_a1));
const __m256i v_b_w = _mm256_cvtepu8_epi16(_mm_unpacklo_epi64(v_b0, v_b1));
const __m256i v_d_w = _mm256_sub_epi16(v_a_w, v_b_w);
*sum = _mm256_add_epi32(*sum, _mm256_madd_epi16(v_d_w, v_d_w));
}
int64_t aom_sse_avx2(const uint8_t *a, int a_stride, const uint8_t *b,
int b_stride, int width, int height) {
int32_t y = 0;
int64_t sse = 0;
__m256i sum = _mm256_setzero_si256();
__m256i zero = _mm256_setzero_si256();
switch (width) {
case 4:
do {
const __m128i v_a0 = xx_loadl_32(a);
const __m128i v_a1 = xx_loadl_32(a + a_stride);
const __m128i v_a2 = xx_loadl_32(a + a_stride * 2);
const __m128i v_a3 = xx_loadl_32(a + a_stride * 3);
const __m128i v_b0 = xx_loadl_32(b);
const __m128i v_b1 = xx_loadl_32(b + b_stride);
const __m128i v_b2 = xx_loadl_32(b + b_stride * 2);
const __m128i v_b3 = xx_loadl_32(b + b_stride * 3);
const __m128i v_a0123 = _mm_unpacklo_epi64(
_mm_unpacklo_epi32(v_a0, v_a1), _mm_unpacklo_epi32(v_a2, v_a3));
const __m128i v_b0123 = _mm_unpacklo_epi64(
_mm_unpacklo_epi32(v_b0, v_b1), _mm_unpacklo_epi32(v_b2, v_b3));
const __m256i v_a_w = _mm256_cvtepu8_epi16(v_a0123);
const __m256i v_b_w = _mm256_cvtepu8_epi16(v_b0123);
const __m256i v_d_w = _mm256_sub_epi16(v_a_w, v_b_w);
sum = _mm256_add_epi32(sum, _mm256_madd_epi16(v_d_w, v_d_w));
sse_w4x4_avx2(a, a_stride, b, b_stride, &sum);
a += a_stride << 2;
b += b_stride << 2;
y += 4;
@ -80,16 +112,7 @@ int64_t aom_sse_avx2(const uint8_t *a, int a_stride, const uint8_t *b,
break;
case 8:
do {
const __m128i v_a0 = xx_loadl_64(a);
const __m128i v_a1 = xx_loadl_64(a + a_stride);
const __m128i v_b0 = xx_loadl_64(b);
const __m128i v_b1 = xx_loadl_64(b + b_stride);
const __m256i v_a_w =
_mm256_cvtepu8_epi16(_mm_unpacklo_epi64(v_a0, v_a1));
const __m256i v_b_w =
_mm256_cvtepu8_epi16(_mm_unpacklo_epi64(v_b0, v_b1));
const __m256i v_d_w = _mm256_sub_epi16(v_a_w, v_b_w);
sum = _mm256_add_epi32(sum, _mm256_madd_epi16(v_d_w, v_d_w));
sse_w8x2_avx2(a, a_stride, b, b_stride, &sum);
a += a_stride << 1;
b += b_stride << 1;
y += 2;
@ -99,14 +122,26 @@ int64_t aom_sse_avx2(const uint8_t *a, int a_stride, const uint8_t *b,
case 16:
do {
const __m128i v_a0 = xx_loadu_128(a);
const __m128i v_a1 = xx_loadu_128(a + a_stride);
const __m128i v_b0 = xx_loadu_128(b);
const __m256i v_a_w = _mm256_cvtepu8_epi16(v_a0);
const __m256i v_b_w = _mm256_cvtepu8_epi16(v_b0);
const __m256i v_d_w = _mm256_sub_epi16(v_a_w, v_b_w);
sum = _mm256_add_epi32(sum, _mm256_madd_epi16(v_d_w, v_d_w));
a += a_stride;
b += b_stride;
y += 1;
const __m128i v_b1 = xx_loadu_128(b + b_stride);
const __m256i v_a =
_mm256_insertf128_si256(_mm256_castsi128_si256(v_a0), v_a1, 0x01);
const __m256i v_b =
_mm256_insertf128_si256(_mm256_castsi128_si256(v_b0), v_b1, 0x01);
const __m256i v_al = _mm256_unpacklo_epi8(v_a, zero);
const __m256i v_au = _mm256_unpackhi_epi8(v_a, zero);
const __m256i v_bl = _mm256_unpacklo_epi8(v_b, zero);
const __m256i v_bu = _mm256_unpackhi_epi8(v_b, zero);
const __m256i v_asub = _mm256_sub_epi16(v_al, v_bl);
const __m256i v_bsub = _mm256_sub_epi16(v_au, v_bu);
const __m256i temp =
_mm256_add_epi32(_mm256_madd_epi16(v_asub, v_asub),
_mm256_madd_epi16(v_bsub, v_bsub));
sum = _mm256_add_epi32(sum, temp);
a += a_stride << 1;
b += b_stride << 1;
y += 2;
} while (y < height);
sse = summary_all_avx2(&sum);
break;
@ -141,7 +176,36 @@ int64_t aom_sse_avx2(const uint8_t *a, int a_stride, const uint8_t *b,
} while (y < height);
sse = summary_all_avx2(&sum);
break;
default: break;
default:
if ((width & 0x07) == 0) {
do {
int i = 0;
do {
sse_w8x2_avx2(a + i, a_stride, b + i, b_stride, &sum);
i += 8;
} while (i < width);
a += a_stride << 1;
b += b_stride << 1;
y += 2;
} while (y < height);
} else {
do {
int i = 0;
do {
sse_w8x2_avx2(a + i, a_stride, b + i, b_stride, &sum);
const uint8_t *a2 = a + i + (a_stride << 1);
const uint8_t *b2 = b + i + (b_stride << 1);
sse_w8x2_avx2(a2, a_stride, b2, b_stride, &sum);
i += 8;
} while (i + 4 < width);
sse_w4x4_avx2(a + i, a_stride, b + i, b_stride, &sum);
a += a_stride << 2;
b += b_stride << 2;
y += 4;
} while (y < height);
}
sse = summary_all_avx2(&sum);
break;
}
return sse;
@ -155,6 +219,33 @@ static INLINE void highbd_sse_w16_avx2(__m256i *sum, const uint16_t *a,
*sum = _mm256_add_epi32(*sum, _mm256_madd_epi16(v_d_w, v_d_w));
}
static INLINE void highbd_sse_w4x4_avx2(__m256i *sum, const uint16_t *a,
int a_stride, const uint16_t *b,
int b_stride) {
const __m128i v_a0 = xx_loadl_64(a);
const __m128i v_a1 = xx_loadl_64(a + a_stride);
const __m128i v_a2 = xx_loadl_64(a + a_stride * 2);
const __m128i v_a3 = xx_loadl_64(a + a_stride * 3);
const __m128i v_b0 = xx_loadl_64(b);
const __m128i v_b1 = xx_loadl_64(b + b_stride);
const __m128i v_b2 = xx_loadl_64(b + b_stride * 2);
const __m128i v_b3 = xx_loadl_64(b + b_stride * 3);
const __m256i v_a_w = yy_set_m128i(_mm_unpacklo_epi64(v_a0, v_a1),
_mm_unpacklo_epi64(v_a2, v_a3));
const __m256i v_b_w = yy_set_m128i(_mm_unpacklo_epi64(v_b0, v_b1),
_mm_unpacklo_epi64(v_b2, v_b3));
const __m256i v_d_w = _mm256_sub_epi16(v_a_w, v_b_w);
*sum = _mm256_add_epi32(*sum, _mm256_madd_epi16(v_d_w, v_d_w));
}
static INLINE void highbd_sse_w8x2_avx2(__m256i *sum, const uint16_t *a,
int a_stride, const uint16_t *b,
int b_stride) {
const __m256i v_a_w = yy_loadu2_128(a + a_stride, a);
const __m256i v_b_w = yy_loadu2_128(b + b_stride, b);
const __m256i v_d_w = _mm256_sub_epi16(v_a_w, v_b_w);
*sum = _mm256_add_epi32(*sum, _mm256_madd_epi16(v_d_w, v_d_w));
}
int64_t aom_highbd_sse_avx2(const uint8_t *a8, int a_stride, const uint8_t *b8,
int b_stride, int width, int height) {
int32_t y = 0;
@ -165,20 +256,7 @@ int64_t aom_highbd_sse_avx2(const uint8_t *a8, int a_stride, const uint8_t *b8,
switch (width) {
case 4:
do {
const __m128i v_a0 = xx_loadl_64(a);
const __m128i v_a1 = xx_loadl_64(a + a_stride);
const __m128i v_a2 = xx_loadl_64(a + a_stride * 2);
const __m128i v_a3 = xx_loadl_64(a + a_stride * 3);
const __m128i v_b0 = xx_loadl_64(b);
const __m128i v_b1 = xx_loadl_64(b + b_stride);
const __m128i v_b2 = xx_loadl_64(b + b_stride * 2);
const __m128i v_b3 = xx_loadl_64(b + b_stride * 3);
const __m256i v_a_w = yy_set_m128i(_mm_unpacklo_epi64(v_a0, v_a1),
_mm_unpacklo_epi64(v_a2, v_a3));
const __m256i v_b_w = yy_set_m128i(_mm_unpacklo_epi64(v_b0, v_b1),
_mm_unpacklo_epi64(v_b2, v_b3));
const __m256i v_d_w = _mm256_sub_epi16(v_a_w, v_b_w);
sum = _mm256_add_epi32(sum, _mm256_madd_epi16(v_d_w, v_d_w));
highbd_sse_w4x4_avx2(&sum, a, a_stride, b, b_stride);
a += a_stride << 2;
b += b_stride << 2;
y += 4;
@ -187,10 +265,7 @@ int64_t aom_highbd_sse_avx2(const uint8_t *a8, int a_stride, const uint8_t *b8,
break;
case 8:
do {
const __m256i v_a_w = yy_loadu2_128(a + a_stride, a);
const __m256i v_b_w = yy_loadu2_128(b + b_stride, b);
const __m256i v_d_w = _mm256_sub_epi16(v_a_w, v_b_w);
sum = _mm256_add_epi32(sum, _mm256_madd_epi16(v_d_w, v_d_w));
highbd_sse_w8x2_avx2(&sum, a, a_stride, b, b_stride);
a += a_stride << 1;
b += b_stride << 1;
y += 2;
@ -208,43 +283,98 @@ int64_t aom_highbd_sse_avx2(const uint8_t *a8, int a_stride, const uint8_t *b8,
break;
case 32:
do {
highbd_sse_w16_avx2(&sum, a, b);
highbd_sse_w16_avx2(&sum, a + 16, b + 16);
a += a_stride;
b += b_stride;
y += 1;
int l = 0;
__m256i sum32 = _mm256_setzero_si256();
do {
highbd_sse_w16_avx2(&sum32, a, b);
highbd_sse_w16_avx2(&sum32, a + 16, b + 16);
a += a_stride;
b += b_stride;
l += 1;
} while (l < 64 && l < (height - y));
summary_32_avx2(&sum32, &sum);
y += 64;
} while (y < height);
sse = summary_all_avx2(&sum);
sse = summary_4x64_avx2(sum);
break;
case 64:
do {
highbd_sse_w16_avx2(&sum, a, b);
highbd_sse_w16_avx2(&sum, a + 16 * 1, b + 16 * 1);
highbd_sse_w16_avx2(&sum, a + 16 * 2, b + 16 * 2);
highbd_sse_w16_avx2(&sum, a + 16 * 3, b + 16 * 3);
a += a_stride;
b += b_stride;
y += 1;
int l = 0;
__m256i sum32 = _mm256_setzero_si256();
do {
highbd_sse_w16_avx2(&sum32, a, b);
highbd_sse_w16_avx2(&sum32, a + 16 * 1, b + 16 * 1);
highbd_sse_w16_avx2(&sum32, a + 16 * 2, b + 16 * 2);
highbd_sse_w16_avx2(&sum32, a + 16 * 3, b + 16 * 3);
a += a_stride;
b += b_stride;
l += 1;
} while (l < 32 && l < (height - y));
summary_32_avx2(&sum32, &sum);
y += 32;
} while (y < height);
sse = summary_all_avx2(&sum);
sse = summary_4x64_avx2(sum);
break;
case 128:
do {
highbd_sse_w16_avx2(&sum, a, b);
highbd_sse_w16_avx2(&sum, a + 16 * 1, b + 16 * 1);
highbd_sse_w16_avx2(&sum, a + 16 * 2, b + 16 * 2);
highbd_sse_w16_avx2(&sum, a + 16 * 3, b + 16 * 3);
highbd_sse_w16_avx2(&sum, a + 16 * 4, b + 16 * 4);
highbd_sse_w16_avx2(&sum, a + 16 * 5, b + 16 * 5);
highbd_sse_w16_avx2(&sum, a + 16 * 6, b + 16 * 6);
highbd_sse_w16_avx2(&sum, a + 16 * 7, b + 16 * 7);
a += a_stride;
b += b_stride;
y += 1;
int l = 0;
__m256i sum32 = _mm256_setzero_si256();
do {
highbd_sse_w16_avx2(&sum32, a, b);
highbd_sse_w16_avx2(&sum32, a + 16 * 1, b + 16 * 1);
highbd_sse_w16_avx2(&sum32, a + 16 * 2, b + 16 * 2);
highbd_sse_w16_avx2(&sum32, a + 16 * 3, b + 16 * 3);
highbd_sse_w16_avx2(&sum32, a + 16 * 4, b + 16 * 4);
highbd_sse_w16_avx2(&sum32, a + 16 * 5, b + 16 * 5);
highbd_sse_w16_avx2(&sum32, a + 16 * 6, b + 16 * 6);
highbd_sse_w16_avx2(&sum32, a + 16 * 7, b + 16 * 7);
a += a_stride;
b += b_stride;
l += 1;
} while (l < 16 && l < (height - y));
summary_32_avx2(&sum32, &sum);
y += 16;
} while (y < height);
sse = summary_all_avx2(&sum);
sse = summary_4x64_avx2(sum);
break;
default:
if (width & 0x7) {
do {
int i = 0;
__m256i sum32 = _mm256_setzero_si256();
do {
highbd_sse_w8x2_avx2(&sum32, a + i, a_stride, b + i, b_stride);
const uint16_t *a2 = a + i + (a_stride << 1);
const uint16_t *b2 = b + i + (b_stride << 1);
highbd_sse_w8x2_avx2(&sum32, a2, a_stride, b2, b_stride);
i += 8;
} while (i + 4 < width);
highbd_sse_w4x4_avx2(&sum32, a + i, a_stride, b + i, b_stride);
summary_32_avx2(&sum32, &sum);
a += a_stride << 2;
b += b_stride << 2;
y += 4;
} while (y < height);
} else {
do {
int l = 0;
__m256i sum32 = _mm256_setzero_si256();
do {
int i = 0;
do {
highbd_sse_w8x2_avx2(&sum32, a + i, a_stride, b + i, b_stride);
i += 8;
} while (i < width);
a += a_stride << 1;
b += b_stride << 1;
l += 2;
} while (l < 8 && l < (height - y));
summary_32_avx2(&sum32, &sum);
y += 8;
} while (y < height);
}
sse = summary_4x64_avx2(sum);
break;
default: break;
}
return sse;
}

249
third_party/aom/aom_dsp/x86/sse_sse4.c vendored
View file

@ -28,6 +28,13 @@ static INLINE int64_t summary_all_sse4(const __m128i *sum_all) {
return sum;
}
static INLINE void summary_32_sse4(const __m128i *sum32, __m128i *sum64) {
const __m128i sum0 = _mm_cvtepu32_epi64(*sum32);
const __m128i sum1 = _mm_cvtepu32_epi64(_mm_srli_si128(*sum32, 8));
*sum64 = _mm_add_epi64(sum0, *sum64);
*sum64 = _mm_add_epi64(sum1, *sum64);
}
static INLINE void sse_w16_sse4_1(__m128i *sum, const uint8_t *a,
const uint8_t *b) {
const __m128i v_a0 = xx_loadu_128(a);
@ -42,6 +49,28 @@ static INLINE void sse_w16_sse4_1(__m128i *sum, const uint8_t *a,
*sum = _mm_add_epi32(*sum, _mm_madd_epi16(v_d01_w, v_d01_w));
}
static INLINE void aom_sse4x2_sse4_1(const uint8_t *a, int a_stride,
const uint8_t *b, int b_stride,
__m128i *sum) {
const __m128i v_a0 = xx_loadl_32(a);
const __m128i v_a1 = xx_loadl_32(a + a_stride);
const __m128i v_b0 = xx_loadl_32(b);
const __m128i v_b1 = xx_loadl_32(b + b_stride);
const __m128i v_a_w = _mm_cvtepu8_epi16(_mm_unpacklo_epi32(v_a0, v_a1));
const __m128i v_b_w = _mm_cvtepu8_epi16(_mm_unpacklo_epi32(v_b0, v_b1));
const __m128i v_d_w = _mm_sub_epi16(v_a_w, v_b_w);
*sum = _mm_add_epi32(*sum, _mm_madd_epi16(v_d_w, v_d_w));
}
static INLINE void aom_sse8_sse4_1(const uint8_t *a, const uint8_t *b,
__m128i *sum) {
const __m128i v_a0 = xx_loadl_64(a);
const __m128i v_b0 = xx_loadl_64(b);
const __m128i v_a_w = _mm_cvtepu8_epi16(v_a0);
const __m128i v_b_w = _mm_cvtepu8_epi16(v_b0);
const __m128i v_d_w = _mm_sub_epi16(v_a_w, v_b_w);
*sum = _mm_add_epi32(*sum, _mm_madd_epi16(v_d_w, v_d_w));
}
int64_t aom_sse_sse4_1(const uint8_t *a, int a_stride, const uint8_t *b,
int b_stride, int width, int height) {
int y = 0;
@ -50,14 +79,7 @@ int64_t aom_sse_sse4_1(const uint8_t *a, int a_stride, const uint8_t *b,
switch (width) {
case 4:
do {
const __m128i v_a0 = xx_loadl_32(a);
const __m128i v_a1 = xx_loadl_32(a + a_stride);
const __m128i v_b0 = xx_loadl_32(b);
const __m128i v_b1 = xx_loadl_32(b + b_stride);
const __m128i v_a_w = _mm_cvtepu8_epi16(_mm_unpacklo_epi32(v_a0, v_a1));
const __m128i v_b_w = _mm_cvtepu8_epi16(_mm_unpacklo_epi32(v_b0, v_b1));
const __m128i v_d_w = _mm_sub_epi16(v_a_w, v_b_w);
sum = _mm_add_epi32(sum, _mm_madd_epi16(v_d_w, v_d_w));
aom_sse4x2_sse4_1(a, a_stride, b, b_stride, &sum);
a += a_stride << 1;
b += b_stride << 1;
y += 2;
@ -66,12 +88,7 @@ int64_t aom_sse_sse4_1(const uint8_t *a, int a_stride, const uint8_t *b,
break;
case 8:
do {
const __m128i v_a0 = xx_loadl_64(a);
const __m128i v_b0 = xx_loadl_64(b);
const __m128i v_a_w = _mm_cvtepu8_epi16(v_a0);
const __m128i v_b_w = _mm_cvtepu8_epi16(v_b0);
const __m128i v_d_w = _mm_sub_epi16(v_a_w, v_b_w);
sum = _mm_add_epi32(sum, _mm_madd_epi16(v_d_w, v_d_w));
aom_sse8_sse4_1(a, b, &sum);
a += a_stride;
b += b_stride;
y += 1;
@ -125,12 +142,52 @@ int64_t aom_sse_sse4_1(const uint8_t *a, int a_stride, const uint8_t *b,
} while (y < height);
sse = summary_all_sse4(&sum);
break;
default: break;
default:
if (width & 0x07) {
do {
int i = 0;
do {
aom_sse8_sse4_1(a + i, b + i, &sum);
aom_sse8_sse4_1(a + i + a_stride, b + i + b_stride, &sum);
i += 8;
} while (i + 4 < width);
aom_sse4x2_sse4_1(a + i, a_stride, b + i, b_stride, &sum);
a += (a_stride << 1);
b += (b_stride << 1);
y += 2;
} while (y < height);
} else {
do {
int i = 0;
do {
aom_sse8_sse4_1(a + i, b + i, &sum);
i += 8;
} while (i < width);
a += a_stride;
b += b_stride;
y += 1;
} while (y < height);
}
sse = summary_all_sse4(&sum);
break;
}
return sse;
}
static INLINE void highbd_sse_w4x2_sse4_1(__m128i *sum, const uint16_t *a,
int a_stride, const uint16_t *b,
int b_stride) {
const __m128i v_a0 = xx_loadl_64(a);
const __m128i v_a1 = xx_loadl_64(a + a_stride);
const __m128i v_b0 = xx_loadl_64(b);
const __m128i v_b1 = xx_loadl_64(b + b_stride);
const __m128i v_a_w = _mm_unpacklo_epi64(v_a0, v_a1);
const __m128i v_b_w = _mm_unpacklo_epi64(v_b0, v_b1);
const __m128i v_d_w = _mm_sub_epi16(v_a_w, v_b_w);
*sum = _mm_add_epi32(*sum, _mm_madd_epi16(v_d_w, v_d_w));
}
static INLINE void highbd_sse_w8_sse4_1(__m128i *sum, const uint16_t *a,
const uint16_t *b) {
const __m128i v_a_w = xx_loadu_128(a);
@ -150,14 +207,7 @@ int64_t aom_highbd_sse_sse4_1(const uint8_t *a8, int a_stride,
switch (width) {
case 4:
do {
const __m128i v_a0 = xx_loadl_64(a);
const __m128i v_a1 = xx_loadl_64(a + a_stride);
const __m128i v_b0 = xx_loadl_64(b);
const __m128i v_b1 = xx_loadl_64(b + b_stride);
const __m128i v_a_w = _mm_unpacklo_epi64(v_a0, v_a1);
const __m128i v_b_w = _mm_unpacklo_epi64(v_b0, v_b1);
const __m128i v_d_w = _mm_sub_epi16(v_a_w, v_b_w);
sum = _mm_add_epi32(sum, _mm_madd_epi16(v_d_w, v_d_w));
highbd_sse_w4x2_sse4_1(&sum, a, a_stride, b, b_stride);
a += a_stride << 1;
b += b_stride << 1;
y += 2;
@ -175,67 +225,126 @@ int64_t aom_highbd_sse_sse4_1(const uint8_t *a8, int a_stride,
break;
case 16:
do {
highbd_sse_w8_sse4_1(&sum, a, b);
highbd_sse_w8_sse4_1(&sum, a + 8, b + 8);
a += a_stride;
b += b_stride;
y += 1;
int l = 0;
__m128i sum32 = _mm_setzero_si128();
do {
highbd_sse_w8_sse4_1(&sum32, a, b);
highbd_sse_w8_sse4_1(&sum32, a + 8, b + 8);
a += a_stride;
b += b_stride;
l += 1;
} while (l < 64 && l < (height - y));
summary_32_sse4(&sum32, &sum);
y += 64;
} while (y < height);
sse = summary_all_sse4(&sum);
xx_storel_64(&sse, _mm_add_epi64(sum, _mm_srli_si128(sum, 8)));
break;
case 32:
do {
highbd_sse_w8_sse4_1(&sum, a, b);
highbd_sse_w8_sse4_1(&sum, a + 8 * 1, b + 8 * 1);
highbd_sse_w8_sse4_1(&sum, a + 8 * 2, b + 8 * 2);
highbd_sse_w8_sse4_1(&sum, a + 8 * 3, b + 8 * 3);
a += a_stride;
b += b_stride;
y += 1;
int l = 0;
__m128i sum32 = _mm_setzero_si128();
do {
highbd_sse_w8_sse4_1(&sum32, a, b);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 1, b + 8 * 1);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 2, b + 8 * 2);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 3, b + 8 * 3);
a += a_stride;
b += b_stride;
l += 1;
} while (l < 32 && l < (height - y));
summary_32_sse4(&sum32, &sum);
y += 32;
} while (y < height);
sse = summary_all_sse4(&sum);
xx_storel_64(&sse, _mm_add_epi64(sum, _mm_srli_si128(sum, 8)));
break;
case 64:
do {
highbd_sse_w8_sse4_1(&sum, a, b);
highbd_sse_w8_sse4_1(&sum, a + 8 * 1, b + 8 * 1);
highbd_sse_w8_sse4_1(&sum, a + 8 * 2, b + 8 * 2);
highbd_sse_w8_sse4_1(&sum, a + 8 * 3, b + 8 * 3);
highbd_sse_w8_sse4_1(&sum, a + 8 * 4, b + 8 * 4);
highbd_sse_w8_sse4_1(&sum, a + 8 * 5, b + 8 * 5);
highbd_sse_w8_sse4_1(&sum, a + 8 * 6, b + 8 * 6);
highbd_sse_w8_sse4_1(&sum, a + 8 * 7, b + 8 * 7);
a += a_stride;
b += b_stride;
y += 1;
int l = 0;
__m128i sum32 = _mm_setzero_si128();
do {
highbd_sse_w8_sse4_1(&sum32, a, b);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 1, b + 8 * 1);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 2, b + 8 * 2);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 3, b + 8 * 3);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 4, b + 8 * 4);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 5, b + 8 * 5);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 6, b + 8 * 6);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 7, b + 8 * 7);
a += a_stride;
b += b_stride;
l += 1;
} while (l < 16 && l < (height - y));
summary_32_sse4(&sum32, &sum);
y += 16;
} while (y < height);
sse = summary_all_sse4(&sum);
xx_storel_64(&sse, _mm_add_epi64(sum, _mm_srli_si128(sum, 8)));
break;
case 128:
do {
highbd_sse_w8_sse4_1(&sum, a, b);
highbd_sse_w8_sse4_1(&sum, a + 8 * 1, b + 8 * 1);
highbd_sse_w8_sse4_1(&sum, a + 8 * 2, b + 8 * 2);
highbd_sse_w8_sse4_1(&sum, a + 8 * 3, b + 8 * 3);
highbd_sse_w8_sse4_1(&sum, a + 8 * 4, b + 8 * 4);
highbd_sse_w8_sse4_1(&sum, a + 8 * 5, b + 8 * 5);
highbd_sse_w8_sse4_1(&sum, a + 8 * 6, b + 8 * 6);
highbd_sse_w8_sse4_1(&sum, a + 8 * 7, b + 8 * 7);
highbd_sse_w8_sse4_1(&sum, a + 8 * 8, b + 8 * 8);
highbd_sse_w8_sse4_1(&sum, a + 8 * 9, b + 8 * 9);
highbd_sse_w8_sse4_1(&sum, a + 8 * 10, b + 8 * 10);
highbd_sse_w8_sse4_1(&sum, a + 8 * 11, b + 8 * 11);
highbd_sse_w8_sse4_1(&sum, a + 8 * 12, b + 8 * 12);
highbd_sse_w8_sse4_1(&sum, a + 8 * 13, b + 8 * 13);
highbd_sse_w8_sse4_1(&sum, a + 8 * 14, b + 8 * 14);
highbd_sse_w8_sse4_1(&sum, a + 8 * 15, b + 8 * 15);
a += a_stride;
b += b_stride;
y += 1;
int l = 0;
__m128i sum32 = _mm_setzero_si128();
do {
highbd_sse_w8_sse4_1(&sum32, a, b);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 1, b + 8 * 1);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 2, b + 8 * 2);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 3, b + 8 * 3);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 4, b + 8 * 4);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 5, b + 8 * 5);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 6, b + 8 * 6);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 7, b + 8 * 7);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 8, b + 8 * 8);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 9, b + 8 * 9);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 10, b + 8 * 10);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 11, b + 8 * 11);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 12, b + 8 * 12);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 13, b + 8 * 13);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 14, b + 8 * 14);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 15, b + 8 * 15);
a += a_stride;
b += b_stride;
l += 1;
} while (l < 8 && l < (height - y));
summary_32_sse4(&sum32, &sum);
y += 8;
} while (y < height);
sse = summary_all_sse4(&sum);
xx_storel_64(&sse, _mm_add_epi64(sum, _mm_srli_si128(sum, 8)));
break;
default:
if (width & 0x7) {
do {
__m128i sum32 = _mm_setzero_si128();
int i = 0;
do {
highbd_sse_w8_sse4_1(&sum32, a + i, b + i);
highbd_sse_w8_sse4_1(&sum32, a + i + a_stride, b + i + b_stride);
i += 8;
} while (i + 4 < width);
highbd_sse_w4x2_sse4_1(&sum32, a + i, a_stride, b + i, b_stride);
a += (a_stride << 1);
b += (b_stride << 1);
y += 2;
summary_32_sse4(&sum32, &sum);
} while (y < height);
} else {
do {
int l = 0;
__m128i sum32 = _mm_setzero_si128();
do {
int i = 0;
do {
highbd_sse_w8_sse4_1(&sum32, a + i, b + i);
i += 8;
} while (i < width);
a += a_stride;
b += b_stride;
l += 1;
} while (l < 8 && l < (height - y));
summary_32_sse4(&sum32, &sum);
y += 8;
} while (y < height);
}
xx_storel_64(&sse, _mm_add_epi64(sum, _mm_srli_si128(sum, 8)));
break;
default: break;
}
return sse;
}

222
third_party/aom/aom_dsp/x86/ssim_sse2_x86_64.asm vendored Normal file
View file

@ -0,0 +1,222 @@
;
; Copyright (c) 2016, Alliance for Open Media. All rights reserved
;
; This source code is subject to the terms of the BSD 2 Clause License and
; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
; was not distributed with this source code in the LICENSE file, you can
; obtain it at www.aomedia.org/license/software. If the Alliance for Open
; Media Patent License 1.0 was not distributed with this source code in the
; PATENTS file, you can obtain it at www.aomedia.org/license/patent.
;
;
%include "aom_ports/x86_abi_support.asm"
; tabulate_ssim - sums sum_s,sum_r,sum_sq_s,sum_sq_r, sum_sxr
%macro TABULATE_SSIM 0
paddusw xmm15, xmm3 ; sum_s
paddusw xmm14, xmm4 ; sum_r
movdqa xmm1, xmm3
pmaddwd xmm1, xmm1
paddd xmm13, xmm1 ; sum_sq_s
movdqa xmm2, xmm4
pmaddwd xmm2, xmm2
paddd xmm12, xmm2 ; sum_sq_r
pmaddwd xmm3, xmm4
paddd xmm11, xmm3 ; sum_sxr
%endmacro
; Sum across the register %1 starting with q words
%macro SUM_ACROSS_Q 1
movdqa xmm2,%1
punpckldq %1,xmm0
punpckhdq xmm2,xmm0
paddq %1,xmm2
movdqa xmm2,%1
punpcklqdq %1,xmm0
punpckhqdq xmm2,xmm0
paddq %1,xmm2
%endmacro
; Sum across the register %1 starting with q words
%macro SUM_ACROSS_W 1
movdqa xmm1, %1
punpcklwd %1,xmm0
punpckhwd xmm1,xmm0
paddd %1, xmm1
SUM_ACROSS_Q %1
%endmacro
SECTION .text
;void ssim_parms_sse2(
; unsigned char *s,
; int sp,
; unsigned char *r,
; int rp
; uint32_t *sum_s,
; uint32_t *sum_r,
; uint32_t *sum_sq_s,
; uint32_t *sum_sq_r,
; uint32_t *sum_sxr);
;
; TODO: Use parm passing through structure, probably don't need the pxors
; ( calling app will initialize to 0 ) could easily fit everything in sse2
; without too much hastle, and can probably do better estimates with psadw
; or pavgb At this point this is just meant to be first pass for calculating
; all the parms needed for 16x16 ssim so we can play with dssim as distortion
; in mode selection code.
global sym(aom_ssim_parms_16x16_sse2) PRIVATE
sym(aom_ssim_parms_16x16_sse2):
push rbp
mov rbp, rsp
SHADOW_ARGS_TO_STACK 9
SAVE_XMM 15
push rsi
push rdi
; end prolog
mov rsi, arg(0) ;s
mov rcx, arg(1) ;sp
mov rdi, arg(2) ;r
mov rax, arg(3) ;rp
pxor xmm0, xmm0
pxor xmm15,xmm15 ;sum_s
pxor xmm14,xmm14 ;sum_r
pxor xmm13,xmm13 ;sum_sq_s
pxor xmm12,xmm12 ;sum_sq_r
pxor xmm11,xmm11 ;sum_sxr
mov rdx, 16 ;row counter
.NextRow:
;grab source and reference pixels
movdqu xmm5, [rsi]
movdqu xmm6, [rdi]
movdqa xmm3, xmm5
movdqa xmm4, xmm6
punpckhbw xmm3, xmm0 ; high_s
punpckhbw xmm4, xmm0 ; high_r
TABULATE_SSIM
movdqa xmm3, xmm5
movdqa xmm4, xmm6
punpcklbw xmm3, xmm0 ; low_s
punpcklbw xmm4, xmm0 ; low_r
TABULATE_SSIM
add rsi, rcx ; next s row
add rdi, rax ; next r row
dec rdx ; counter
jnz .NextRow
SUM_ACROSS_W xmm15
SUM_ACROSS_W xmm14
SUM_ACROSS_Q xmm13
SUM_ACROSS_Q xmm12
SUM_ACROSS_Q xmm11
mov rdi,arg(4)
movd [rdi], xmm15;
mov rdi,arg(5)
movd [rdi], xmm14;
mov rdi,arg(6)
movd [rdi], xmm13;
mov rdi,arg(7)
movd [rdi], xmm12;
mov rdi,arg(8)
movd [rdi], xmm11;
; begin epilog
pop rdi
pop rsi
RESTORE_XMM
UNSHADOW_ARGS
pop rbp
ret
;void ssim_parms_sse2(
; unsigned char *s,
; int sp,
; unsigned char *r,
; int rp
; uint32_t *sum_s,
; uint32_t *sum_r,
; uint32_t *sum_sq_s,
; uint32_t *sum_sq_r,
; uint32_t *sum_sxr);
;
; TODO: Use parm passing through structure, probably don't need the pxors
; ( calling app will initialize to 0 ) could easily fit everything in sse2
; without too much hastle, and can probably do better estimates with psadw
; or pavgb At this point this is just meant to be first pass for calculating
; all the parms needed for 16x16 ssim so we can play with dssim as distortion
; in mode selection code.
global sym(aom_ssim_parms_8x8_sse2) PRIVATE
sym(aom_ssim_parms_8x8_sse2):
push rbp
mov rbp, rsp
SHADOW_ARGS_TO_STACK 9
SAVE_XMM 15
push rsi
push rdi
; end prolog
mov rsi, arg(0) ;s
mov rcx, arg(1) ;sp
mov rdi, arg(2) ;r
mov rax, arg(3) ;rp
pxor xmm0, xmm0
pxor xmm15,xmm15 ;sum_s
pxor xmm14,xmm14 ;sum_r
pxor xmm13,xmm13 ;sum_sq_s
pxor xmm12,xmm12 ;sum_sq_r
pxor xmm11,xmm11 ;sum_sxr
mov rdx, 8 ;row counter
.NextRow:
;grab source and reference pixels
movq xmm3, [rsi]
movq xmm4, [rdi]
punpcklbw xmm3, xmm0 ; low_s
punpcklbw xmm4, xmm0 ; low_r
TABULATE_SSIM
add rsi, rcx ; next s row
add rdi, rax ; next r row
dec rdx ; counter
jnz .NextRow
SUM_ACROSS_W xmm15
SUM_ACROSS_W xmm14
SUM_ACROSS_Q xmm13
SUM_ACROSS_Q xmm12
SUM_ACROSS_Q xmm11
mov rdi,arg(4)
movd [rdi], xmm15;
mov rdi,arg(5)
movd [rdi], xmm14;
mov rdi,arg(6)
movd [rdi], xmm13;
mov rdi,arg(7)
movd [rdi], xmm12;
mov rdi,arg(8)
movd [rdi], xmm11;
; begin epilog
pop rdi
pop rsi
RESTORE_XMM
UNSHADOW_ARGS
pop rbp
ret

51
third_party/aom/aom_dsp/x86/subpel_variance_sse2.asm vendored
View file

@ -135,44 +135,33 @@ SECTION .text
%if %2 == 1 ; avg
cglobal sub_pixel_avg_variance%1xh, 7, 7, 13, src, src_stride, \
x_offset, y_offset, dst, dst_stride, \
sec, sec_stride, height, sse, \
g_bilin_filter, g_pw_8
sec, sec_stride, height, sse
%define block_height dword heightm
%define sec_str sec_stridemp
;Store bilin_filter and pw_8 location in stack
%if GET_GOT_DEFINED == 1
GET_GOT eax
add esp, 4 ; restore esp
%endif
lea ecx, [GLOBAL(bilin_filter_m)]
mov g_bilin_filterm, ecx
lea ecx, [GLOBAL(pw_8)]
mov g_pw_8m, ecx
LOAD_IF_USED 0, 1 ; load eax, ecx back
%else
cglobal sub_pixel_variance%1xh, 7, 7, 13, src, src_stride, \
x_offset, y_offset, dst, dst_stride, \
height, sse, g_bilin_filter, g_pw_8
height, sse
%define block_height heightd
;Store bilin_filter and pw_8 location in stack
%if GET_GOT_DEFINED == 1
GET_GOT eax
add esp, 4 ; restore esp
%endif
lea ecx, [GLOBAL(bilin_filter_m)]
mov g_bilin_filterm, ecx
lea ecx, [GLOBAL(pw_8)]
mov g_pw_8m, ecx
LOAD_IF_USED 0, 1 ; load eax, ecx back
%endif
; reuse argument stack space
%define g_bilin_filterm x_offsetm
%define g_pw_8m y_offsetm
;Store bilin_filter and pw_8 location in stack
%if GET_GOT_DEFINED == 1
GET_GOT eax
add esp, 4 ; restore esp
%endif
lea ecx, [GLOBAL(bilin_filter_m)]
mov g_bilin_filterm, ecx
lea ecx, [GLOBAL(pw_8)]
mov g_pw_8m, ecx
LOAD_IF_USED 0, 1 ; load eax, ecx back
%else
%if %2 == 1 ; avg
cglobal sub_pixel_avg_variance%1xh, 7, 7, 13, src, src_stride, \

8
third_party/aom/aom_dsp/x86/synonyms.h vendored
View file

@ -13,6 +13,7 @@
#define AOM_AOM_DSP_X86_SYNONYMS_H_
#include <immintrin.h>
#include <string.h>
#include "config/aom_config.h"
@ -28,7 +29,9 @@
// Loads and stores to do away with the tedium of casting the address
// to the right type.
static INLINE __m128i xx_loadl_32(const void *a) {
return _mm_cvtsi32_si128(*(const uint32_t *)a);
int val;
memcpy(&val, a, sizeof(val));
return _mm_cvtsi32_si128(val);
}
static INLINE __m128i xx_loadl_64(const void *a) {
@ -44,7 +47,8 @@ static INLINE __m128i xx_loadu_128(const void *a) {
}
static INLINE void xx_storel_32(void *const a, const __m128i v) {
*(uint32_t *)a = _mm_cvtsi128_si32(v);
const int val = _mm_cvtsi128_si32(v);
memcpy(a, &val, sizeof(val));
}
static INLINE void xx_storel_64(void *const a, const __m128i v) {

5
third_party/aom/aom_dsp/x86/synonyms_avx2.h vendored
View file

@ -67,6 +67,11 @@ static INLINE __m256i yy_loadu2_128(const void *hi, const void *lo) {
return yy_set_m128i(mhi, mlo);
}
static INLINE void yy_storeu2_128(void *hi, void *lo, const __m256i a) {
_mm_storeu_si128((__m128i *)hi, _mm256_extracti128_si256(a, 1));
_mm_storeu_si128((__m128i *)lo, _mm256_castsi256_si128(a));
}
static INLINE __m256i yy_roundn_epu16(__m256i v_val_w, int bits) {
const __m256i v_s_w = _mm256_srli_epi16(v_val_w, bits - 1);
return _mm256_avg_epu16(v_s_w, _mm256_setzero_si256());

50
third_party/aom/aom_dsp/x86/txfm_common_avx2.h vendored
View file

@ -20,9 +20,6 @@
extern "C" {
#endif
typedef void (*transform_1d_avx2)(const __m256i *input, __m256i *output,
int8_t cos_bit);
static INLINE __m256i pair_set_w16_epi16(int16_t a, int16_t b) {
return _mm256_set1_epi32(
(int32_t)(((uint16_t)(a)) | (((uint32_t)(b)) << 16)));
@ -192,6 +189,53 @@ static INLINE void round_shift_16bit_w16_avx2(__m256i *in, int size, int bit) {
}
}
static INLINE __m256i av1_round_shift_32_avx2(__m256i vec, int bit) {
__m256i tmp, round;
round = _mm256_set1_epi32(1 << (bit - 1));
tmp = _mm256_add_epi32(vec, round);
return _mm256_srai_epi32(tmp, bit);
}
static INLINE void av1_round_shift_array_32_avx2(__m256i *input,
__m256i *output,
const int size,
const int bit) {
if (bit > 0) {
int i;
for (i = 0; i < size; i++) {
output[i] = av1_round_shift_32_avx2(input[i], bit);
}
} else {
int i;
for (i = 0; i < size; i++) {
output[i] = _mm256_slli_epi32(input[i], -bit);
}
}
}
static INLINE void av1_round_shift_rect_array_32_avx2(__m256i *input,
__m256i *output,
const int size,
const int bit,
const int val) {
const __m256i sqrt2 = _mm256_set1_epi32(val);
if (bit > 0) {
int i;
for (i = 0; i < size; i++) {
const __m256i r0 = av1_round_shift_32_avx2(input[i], bit);
const __m256i r1 = _mm256_mullo_epi32(sqrt2, r0);
output[i] = av1_round_shift_32_avx2(r1, NewSqrt2Bits);
}
} else {
int i;
for (i = 0; i < size; i++) {
const __m256i r0 = _mm256_slli_epi32(input[i], -bit);
const __m256i r1 = _mm256_mullo_epi32(sqrt2, r0);
output[i] = av1_round_shift_32_avx2(r1, NewSqrt2Bits);
}
}
}
#ifdef __cplusplus
}
#endif

24
third_party/aom/aom_dsp/x86/variance_sse2.c vendored
View file

@ -494,7 +494,7 @@ void aom_upsampled_pred_sse2(MACROBLOCKD *xd, const struct AV1Common *const cm,
const int ref_num = 0;
const int is_intrabc = is_intrabc_block(mi);
const struct scale_factors *const sf =
is_intrabc ? &cm->sf_identity : &xd->block_refs[ref_num]->sf;
is_intrabc ? &cm->sf_identity : xd->block_ref_scale_factors[ref_num];
const int is_scaled = av1_is_scaled(sf);
if (is_scaled) {
@ -571,11 +571,10 @@ void aom_upsampled_pred_sse2(MACROBLOCKD *xd, const struct AV1Common *const cm,
}
}
const InterpFilterParams *filter =
(subpel_search == 1)
? av1_get_4tap_interp_filter_params(EIGHTTAP_REGULAR)
: av1_get_interp_filter_params_with_block_size(EIGHTTAP_REGULAR, 8);
int filter_taps = (subpel_search == 1) ? 4 : SUBPEL_TAPS;
const InterpFilterParams *filter = av1_get_filter(subpel_search);
// (TODO:yunqing) 2-tap case uses 4-tap functions since there is no SIMD for
// 2-tap yet.
int filter_taps = (subpel_search <= USE_4_TAPS) ? 4 : SUBPEL_TAPS;
if (!subpel_x_q3 && !subpel_y_q3) {
if (width >= 16) {
@ -638,20 +637,13 @@ void aom_upsampled_pred_sse2(MACROBLOCKD *xd, const struct AV1Common *const cm,
const int16_t *const kernel_y =
av1_get_interp_filter_subpel_kernel(filter, subpel_y_q3 << 1);
const uint8_t *ref_start = ref - ref_stride * ((filter_taps >> 1) - 1);
uint8_t *temp_start_horiz =
(subpel_search == 1) ? temp + (filter_taps >> 1) * MAX_SB_SIZE : temp;
uint8_t *temp_start_horiz = (subpel_search <= USE_4_TAPS)
? temp + (filter_taps >> 1) * MAX_SB_SIZE
: temp;
uint8_t *temp_start_vert = temp + MAX_SB_SIZE * ((filter->taps >> 1) - 1);
int intermediate_height =
(((height - 1) * 8 + subpel_y_q3) >> 3) + filter_taps;
assert(intermediate_height <= (MAX_SB_SIZE * 2 + 16) + 16);
// TODO(Deepa): Remove the memset below when we have
// 4 tap simd for sse2 and ssse3.
if (subpel_search == 1) {
memset(temp_start_vert - 3 * MAX_SB_SIZE, 0, width);
memset(temp_start_vert - 2 * MAX_SB_SIZE, 0, width);
memset(temp_start_vert + (height + 2) * MAX_SB_SIZE, 0, width);
memset(temp_start_vert + (height + 3) * MAX_SB_SIZE, 0, width);
}
aom_convolve8_horiz(ref_start, ref_stride, temp_start_horiz, MAX_SB_SIZE,
kernel_x, 16, NULL, -1, width, intermediate_height);
aom_convolve8_vert(temp_start_vert, MAX_SB_SIZE, comp_pred, width, NULL, -1,