diff --git a/media/libaom/README_MCP b/media/libaom/README_MCP index 5ae9f96c42..bab18e4b5a 100644 --- a/media/libaom/README_MCP +++ b/media/libaom/README_MCP @@ -10,4 +10,4 @@ The upstream aom git repository is: https://aomedia.googlesource.com/aom -The git commit ID used was 1e227d41f0616de9548a673a83a21ef990b62591. +The git commit ID used was 0ec86ac7ae1e32a7e70410fa4972a655ec3670a4. diff --git a/media/libaom/config/generic/config/aom_config.asm b/media/libaom/config/generic/config/aom_config.asm index 897eeb4fd1..a236ca9dd2 100644 --- a/media/libaom/config/generic/config/aom_config.asm +++ b/media/libaom/config/generic/config/aom_config.asm @@ -1,5 +1,5 @@ ; -; Copyright (c) 2018, Alliance for Open Media. All rights reserved +; Copyright (c) 2019, Alliance for Open Media. All rights reserved ; ; This source code is subject to the terms of the BSD 2 Clause License and ; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License @@ -22,18 +22,17 @@ CONFIG_AV1_ENCODER equ 0 CONFIG_BIG_ENDIAN equ 0 CONFIG_BITSTREAM_DEBUG equ 0 CONFIG_COEFFICIENT_RANGE_CHECKING equ 0 -CONFIG_COLLECT_INTER_MODE_RD_STATS equ 0 +CONFIG_COLLECT_PARTITION_STATS equ 0 CONFIG_COLLECT_RD_STATS equ 0 CONFIG_DEBUG equ 0 CONFIG_DENOISE equ 1 +CONFIG_DISABLE_FULL_PIXEL_SPLIT_8X8 equ 1 CONFIG_DIST_8X8 equ 0 CONFIG_ENTROPY_STATS equ 0 CONFIG_FILEOPTIONS equ 1 -CONFIG_FIX_GF_LENGTH equ 1 CONFIG_FP_MB_STATS equ 0 CONFIG_GCC equ 1 CONFIG_GCOV equ 0 -CONFIG_GLOBAL_MOTION_SEARCH equ 1 CONFIG_GPROF equ 0 CONFIG_INSPECTION equ 0 CONFIG_INTERNAL_STATS equ 0 @@ -44,15 +43,16 @@ CONFIG_MAX_DECODE_PROFILE equ 2 CONFIG_MISMATCH_DEBUG equ 0 CONFIG_MULTITHREAD equ 1 CONFIG_NORMAL_TILE_MODE equ 0 +CONFIG_ONE_PASS_SVM equ 0 CONFIG_OS_SUPPORT equ 1 CONFIG_PIC equ 0 CONFIG_RD_DEBUG equ 0 -CONFIG_REDUCED_ENCODER_BORDER equ 0 CONFIG_RUNTIME_CPU_DETECT equ 1 CONFIG_SHARED equ 0 CONFIG_SHARP_SETTINGS equ 0 CONFIG_SIZE_LIMIT equ 0 CONFIG_SPATIAL_RESAMPLING equ 1 +CONFIG_SPEED_STATS equ 0 CONFIG_STATIC equ 1 CONFIG_WEBM_IO equ 0 DECODE_HEIGHT_LIMIT equ 0 diff --git a/media/libaom/config/generic/config/aom_config.h b/media/libaom/config/generic/config/aom_config.h index 54fb243e11..3b3798a451 100644 --- a/media/libaom/config/generic/config/aom_config.h +++ b/media/libaom/config/generic/config/aom_config.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, Alliance for Open Media. All rights reserved + * Copyright (c) 2019, Alliance for Open Media. All rights reserved * * This source code is subject to the terms of the BSD 2 Clause License and * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License @@ -24,18 +24,17 @@ #define CONFIG_BIG_ENDIAN 0 #define CONFIG_BITSTREAM_DEBUG 0 #define CONFIG_COEFFICIENT_RANGE_CHECKING 0 -#define CONFIG_COLLECT_INTER_MODE_RD_STATS 0 +#define CONFIG_COLLECT_PARTITION_STATS 0 #define CONFIG_COLLECT_RD_STATS 0 #define CONFIG_DEBUG 0 #define CONFIG_DENOISE 1 +#define CONFIG_DISABLE_FULL_PIXEL_SPLIT_8X8 1 #define CONFIG_DIST_8X8 0 #define CONFIG_ENTROPY_STATS 0 #define CONFIG_FILEOPTIONS 1 -#define CONFIG_FIX_GF_LENGTH 1 #define CONFIG_FP_MB_STATS 0 #define CONFIG_GCC 1 #define CONFIG_GCOV 0 -#define CONFIG_GLOBAL_MOTION_SEARCH 1 #define CONFIG_GPROF 0 #define CONFIG_INSPECTION 0 #define CONFIG_INTERNAL_STATS 0 @@ -46,15 +45,16 @@ #define CONFIG_MISMATCH_DEBUG 0 #define CONFIG_MULTITHREAD 1 #define CONFIG_NORMAL_TILE_MODE 0 +#define CONFIG_ONE_PASS_SVM 0 #define CONFIG_OS_SUPPORT 1 #define CONFIG_PIC 0 #define CONFIG_RD_DEBUG 0 -#define CONFIG_REDUCED_ENCODER_BORDER 0 #define CONFIG_RUNTIME_CPU_DETECT 1 #define CONFIG_SHARED 0 #define CONFIG_SHARP_SETTINGS 0 #define CONFIG_SIZE_LIMIT 0 #define CONFIG_SPATIAL_RESAMPLING 1 +#define CONFIG_SPEED_STATS 0 #define CONFIG_STATIC 1 #define CONFIG_WEBM_IO 0 #define DECODE_HEIGHT_LIMIT 0 @@ -76,7 +76,5 @@ #define HAVE_SSSE3 0 #define HAVE_VSX 0 #define HAVE_WXWIDGETS 0 -#define INCLUDE_INSTALL_DIR INSTALLDIR/include #define INLINE inline -#define LIB_INSTALL_DIR INSTALLDIR/lib #endif /* AOM_CONFIG_H_ */ diff --git a/media/libaom/config/generic/config/aom_dsp_rtcd.h b/media/libaom/config/generic/config/aom_dsp_rtcd.h index c3b61b8a8b..62222ee19e 100644 --- a/media/libaom/config/generic/config/aom_dsp_rtcd.h +++ b/media/libaom/config/generic/config/aom_dsp_rtcd.h @@ -1360,9 +1360,6 @@ void aom_v_predictor_8x4_c(uint8_t *dst, ptrdiff_t y_stride, const uint8_t *abov void aom_v_predictor_8x8_c(uint8_t *dst, ptrdiff_t y_stride, const uint8_t *above, const uint8_t *left); #define aom_v_predictor_8x8 aom_v_predictor_8x8_c -void av1_round_shift_array_c(int32_t *arr, int size, int bit); -#define av1_round_shift_array av1_round_shift_array_c - void aom_dsp_rtcd(void); #include "config/aom_config.h" diff --git a/media/libaom/config/generic/config/aom_scale_rtcd.h b/media/libaom/config/generic/config/aom_scale_rtcd.h index 1ee9bef3d7..8b92a2d57f 100644 --- a/media/libaom/config/generic/config/aom_scale_rtcd.h +++ b/media/libaom/config/generic/config/aom_scale_rtcd.h @@ -59,13 +59,22 @@ void aom_yv12_copy_y_c(const struct yv12_buffer_config *src_ybc, struct yv12_buf void aom_yv12_extend_frame_borders_c(struct yv12_buffer_config *ybf, const int num_planes); #define aom_yv12_extend_frame_borders aom_yv12_extend_frame_borders_c -void aom_yv12_partial_copy_u_c(const struct yv12_buffer_config *src_bc, struct yv12_buffer_config *dst_bc, int hstart, int hend, int vstart, int vend); +void aom_yv12_partial_coloc_copy_u_c(const struct yv12_buffer_config *src_bc, struct yv12_buffer_config *dst_bc, int hstart, int hend, int vstart, int vend); +#define aom_yv12_partial_coloc_copy_u aom_yv12_partial_coloc_copy_u_c + +void aom_yv12_partial_coloc_copy_v_c(const struct yv12_buffer_config *src_bc, struct yv12_buffer_config *dst_bc, int hstart, int hend, int vstart, int vend); +#define aom_yv12_partial_coloc_copy_v aom_yv12_partial_coloc_copy_v_c + +void aom_yv12_partial_coloc_copy_y_c(const struct yv12_buffer_config *src_ybc, struct yv12_buffer_config *dst_ybc, int hstart, int hend, int vstart, int vend); +#define aom_yv12_partial_coloc_copy_y aom_yv12_partial_coloc_copy_y_c + +void aom_yv12_partial_copy_u_c(const struct yv12_buffer_config *src_bc, int hstart1, int hend1, int vstart1, int vend1, struct yv12_buffer_config *dst_bc, int hstart2, int vstart2); #define aom_yv12_partial_copy_u aom_yv12_partial_copy_u_c -void aom_yv12_partial_copy_v_c(const struct yv12_buffer_config *src_bc, struct yv12_buffer_config *dst_bc, int hstart, int hend, int vstart, int vend); +void aom_yv12_partial_copy_v_c(const struct yv12_buffer_config *src_bc, int hstart1, int hend1, int vstart1, int vend1, struct yv12_buffer_config *dst_bc, int hstart2, int vstart2); #define aom_yv12_partial_copy_v aom_yv12_partial_copy_v_c -void aom_yv12_partial_copy_y_c(const struct yv12_buffer_config *src_ybc, struct yv12_buffer_config *dst_ybc, int hstart, int hend, int vstart, int vend); +void aom_yv12_partial_copy_y_c(const struct yv12_buffer_config *src_ybc, int hstart1, int hend1, int vstart1, int vend1, struct yv12_buffer_config *dst_ybc, int hstart2, int vstart2); #define aom_yv12_partial_copy_y aom_yv12_partial_copy_y_c void aom_scale_rtcd(void); diff --git a/media/libaom/config/generic/config/av1_rtcd.h b/media/libaom/config/generic/config/av1_rtcd.h index e2fe11bbdc..ab42789f64 100644 --- a/media/libaom/config/generic/config/av1_rtcd.h +++ b/media/libaom/config/generic/config/av1_rtcd.h @@ -31,6 +31,8 @@ struct txfm_param; struct aom_variance_vtable; struct search_site_config; struct yv12_buffer_config; +struct NN_CONFIG; +typedef struct NN_CONFIG NN_CONFIG; /* Function pointers return by CfL functions */ typedef void (*cfl_subsample_lbd_fn)(const uint8_t *input, int input_stride, @@ -81,6 +83,18 @@ void av1_convolve_x_sr_c(const uint8_t *src, int src_stride, uint8_t *dst, int d void av1_convolve_y_sr_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); #define av1_convolve_y_sr av1_convolve_y_sr_c +void av1_dist_wtd_convolve_2d_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +#define av1_dist_wtd_convolve_2d av1_dist_wtd_convolve_2d_c + +void av1_dist_wtd_convolve_2d_copy_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +#define av1_dist_wtd_convolve_2d_copy av1_dist_wtd_convolve_2d_copy_c + +void av1_dist_wtd_convolve_x_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +#define av1_dist_wtd_convolve_x av1_dist_wtd_convolve_x_c + +void av1_dist_wtd_convolve_y_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +#define av1_dist_wtd_convolve_y av1_dist_wtd_convolve_y_c + void av1_dr_prediction_z1_c(uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_above, int dx, int dy); #define av1_dr_prediction_z1 av1_dr_prediction_z1_c @@ -132,6 +146,18 @@ void av1_highbd_convolve_x_sr_c(const uint16_t *src, int src_stride, uint16_t *d void av1_highbd_convolve_y_sr_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); #define av1_highbd_convolve_y_sr av1_highbd_convolve_y_sr_c +void av1_highbd_dist_wtd_convolve_2d_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +#define av1_highbd_dist_wtd_convolve_2d av1_highbd_dist_wtd_convolve_2d_c + +void av1_highbd_dist_wtd_convolve_2d_copy_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +#define av1_highbd_dist_wtd_convolve_2d_copy av1_highbd_dist_wtd_convolve_2d_copy_c + +void av1_highbd_dist_wtd_convolve_x_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +#define av1_highbd_dist_wtd_convolve_x av1_highbd_dist_wtd_convolve_x_c + +void av1_highbd_dist_wtd_convolve_y_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +#define av1_highbd_dist_wtd_convolve_y av1_highbd_dist_wtd_convolve_y_c + void av1_highbd_dr_prediction_z1_c(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_above, int dx, int dy, int bd); #define av1_highbd_dr_prediction_z1 av1_highbd_dr_prediction_z1_c @@ -144,20 +170,20 @@ void av1_highbd_dr_prediction_z3_c(uint16_t *dst, ptrdiff_t stride, int bw, int void av1_highbd_inv_txfm_add_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); #define av1_highbd_inv_txfm_add av1_highbd_inv_txfm_add_c -void av1_highbd_inv_txfm_add_16x16_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -#define av1_highbd_inv_txfm_add_16x16 av1_highbd_inv_txfm_add_16x16_c +void av1_highbd_inv_txfm_add_16x4_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +#define av1_highbd_inv_txfm_add_16x4 av1_highbd_inv_txfm_add_16x4_c -void av1_highbd_inv_txfm_add_16x8_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -#define av1_highbd_inv_txfm_add_16x8 av1_highbd_inv_txfm_add_16x8_c - -void av1_highbd_inv_txfm_add_32x32_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -#define av1_highbd_inv_txfm_add_32x32 av1_highbd_inv_txfm_add_32x32_c +void av1_highbd_inv_txfm_add_4x16_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +#define av1_highbd_inv_txfm_add_4x16 av1_highbd_inv_txfm_add_4x16_c void av1_highbd_inv_txfm_add_4x4_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); #define av1_highbd_inv_txfm_add_4x4 av1_highbd_inv_txfm_add_4x4_c -void av1_highbd_inv_txfm_add_8x16_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -#define av1_highbd_inv_txfm_add_8x16 av1_highbd_inv_txfm_add_8x16_c +void av1_highbd_inv_txfm_add_4x8_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +#define av1_highbd_inv_txfm_add_4x8 av1_highbd_inv_txfm_add_4x8_c + +void av1_highbd_inv_txfm_add_8x4_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +#define av1_highbd_inv_txfm_add_8x4 av1_highbd_inv_txfm_add_8x4_c void av1_highbd_inv_txfm_add_8x8_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); #define av1_highbd_inv_txfm_add_8x8 av1_highbd_inv_txfm_add_8x8_c @@ -168,18 +194,6 @@ void av1_highbd_iwht4x4_16_add_c(const tran_low_t *input, uint8_t *dest, int des void av1_highbd_iwht4x4_1_add_c(const tran_low_t *input, uint8_t *dest, int dest_stride, int bd); #define av1_highbd_iwht4x4_1_add av1_highbd_iwht4x4_1_add_c -void av1_highbd_jnt_convolve_2d_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -#define av1_highbd_jnt_convolve_2d av1_highbd_jnt_convolve_2d_c - -void av1_highbd_jnt_convolve_2d_copy_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -#define av1_highbd_jnt_convolve_2d_copy av1_highbd_jnt_convolve_2d_copy_c - -void av1_highbd_jnt_convolve_x_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -#define av1_highbd_jnt_convolve_x av1_highbd_jnt_convolve_x_c - -void av1_highbd_jnt_convolve_y_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -#define av1_highbd_jnt_convolve_y av1_highbd_jnt_convolve_y_c - void av1_highbd_warp_affine_c(const int32_t *mat, const uint16_t *ref, int width, int height, int stride, uint16_t *pred, int p_col, int p_row, int p_width, int p_height, int p_stride, int subsampling_x, int subsampling_y, int bd, ConvolveParams *conv_params, int16_t alpha, int16_t beta, int16_t gamma, int16_t delta); #define av1_highbd_warp_affine av1_highbd_warp_affine_c @@ -246,17 +260,8 @@ void av1_inv_txfm2d_add_8x8_c(const int32_t *input, uint16_t *output, int stride void av1_inv_txfm_add_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); #define av1_inv_txfm_add av1_inv_txfm_add_c -void av1_jnt_convolve_2d_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -#define av1_jnt_convolve_2d av1_jnt_convolve_2d_c - -void av1_jnt_convolve_2d_copy_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -#define av1_jnt_convolve_2d_copy av1_jnt_convolve_2d_copy_c - -void av1_jnt_convolve_x_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -#define av1_jnt_convolve_x av1_jnt_convolve_x_c - -void av1_jnt_convolve_y_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -#define av1_jnt_convolve_y av1_jnt_convolve_y_c +void av1_round_shift_array_c(int32_t *arr, int size, int bit); +#define av1_round_shift_array av1_round_shift_array_c int av1_selfguided_restoration_c(const uint8_t *dgd8, int width, int height, int dgd_stride, int32_t *flt0, int32_t *flt1, int flt_stride, @@ -275,7 +280,7 @@ void av1_warp_affine_c(const int32_t *mat, const uint8_t *ref, int width, int he void av1_wiener_convolve_add_src_c(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, const int16_t *filter_y, int y_step_q4, int w, int h, const ConvolveParams *conv_params); #define av1_wiener_convolve_add_src av1_wiener_convolve_add_src_c -void cdef_filter_block_c(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int max, int coeff_shift); +void cdef_filter_block_c(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int coeff_shift); #define cdef_filter_block cdef_filter_block_c int cdef_find_dir_c(const uint16_t *img, int stride, int32_t *var, int coeff_shift); diff --git a/media/libaom/config/linux/arm/config/aom_config.asm b/media/libaom/config/linux/arm/config/aom_config.asm index f5db6f6ca6..c680ea1ba7 100644 --- a/media/libaom/config/linux/arm/config/aom_config.asm +++ b/media/libaom/config/linux/arm/config/aom_config.asm @@ -1,5 +1,5 @@ ; -; Copyright (c) 2018, Alliance for Open Media. All rights reserved +; Copyright (c) 2019, Alliance for Open Media. All rights reserved ; ; This source code is subject to the terms of the BSD 2 Clause License and ; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License @@ -22,18 +22,17 @@ .equ CONFIG_BIG_ENDIAN, 0 .equ CONFIG_BITSTREAM_DEBUG, 0 .equ CONFIG_COEFFICIENT_RANGE_CHECKING, 0 -.equ CONFIG_COLLECT_INTER_MODE_RD_STATS, 0 +.equ CONFIG_COLLECT_PARTITION_STATS, 0 .equ CONFIG_COLLECT_RD_STATS, 0 .equ CONFIG_DEBUG, 0 .equ CONFIG_DENOISE, 1 +.equ CONFIG_DISABLE_FULL_PIXEL_SPLIT_8X8, 1 .equ CONFIG_DIST_8X8, 0 .equ CONFIG_ENTROPY_STATS, 0 .equ CONFIG_FILEOPTIONS, 1 -.equ CONFIG_FIX_GF_LENGTH, 1 .equ CONFIG_FP_MB_STATS, 0 .equ CONFIG_GCC, 1 .equ CONFIG_GCOV, 0 -.equ CONFIG_GLOBAL_MOTION_SEARCH, 1 .equ CONFIG_GPROF, 0 .equ CONFIG_INSPECTION, 0 .equ CONFIG_INTERNAL_STATS, 0 @@ -44,15 +43,16 @@ .equ CONFIG_MISMATCH_DEBUG, 0 .equ CONFIG_MULTITHREAD, 1 .equ CONFIG_NORMAL_TILE_MODE, 0 +.equ CONFIG_ONE_PASS_SVM, 0 .equ CONFIG_OS_SUPPORT, 1 .equ CONFIG_PIC, 1 .equ CONFIG_RD_DEBUG, 0 -.equ CONFIG_REDUCED_ENCODER_BORDER, 0 .equ CONFIG_RUNTIME_CPU_DETECT, 1 .equ CONFIG_SHARED, 0 .equ CONFIG_SHARP_SETTINGS, 0 .equ CONFIG_SIZE_LIMIT, 0 .equ CONFIG_SPATIAL_RESAMPLING, 1 +.equ CONFIG_SPEED_STATS, 0 .equ CONFIG_STATIC, 1 .equ CONFIG_WEBM_IO, 0 .equ DECODE_HEIGHT_LIMIT, 0 diff --git a/media/libaom/config/linux/arm/config/aom_config.h b/media/libaom/config/linux/arm/config/aom_config.h index 850eebfbd4..ffc5188369 100644 --- a/media/libaom/config/linux/arm/config/aom_config.h +++ b/media/libaom/config/linux/arm/config/aom_config.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, Alliance for Open Media. All rights reserved + * Copyright (c) 2019, Alliance for Open Media. All rights reserved * * This source code is subject to the terms of the BSD 2 Clause License and * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License @@ -24,18 +24,17 @@ #define CONFIG_BIG_ENDIAN 0 #define CONFIG_BITSTREAM_DEBUG 0 #define CONFIG_COEFFICIENT_RANGE_CHECKING 0 -#define CONFIG_COLLECT_INTER_MODE_RD_STATS 0 +#define CONFIG_COLLECT_PARTITION_STATS 0 #define CONFIG_COLLECT_RD_STATS 0 #define CONFIG_DEBUG 0 #define CONFIG_DENOISE 1 +#define CONFIG_DISABLE_FULL_PIXEL_SPLIT_8X8 1 #define CONFIG_DIST_8X8 0 #define CONFIG_ENTROPY_STATS 0 #define CONFIG_FILEOPTIONS 1 -#define CONFIG_FIX_GF_LENGTH 1 #define CONFIG_FP_MB_STATS 0 #define CONFIG_GCC 1 #define CONFIG_GCOV 0 -#define CONFIG_GLOBAL_MOTION_SEARCH 1 #define CONFIG_GPROF 0 #define CONFIG_INSPECTION 0 #define CONFIG_INTERNAL_STATS 0 @@ -46,15 +45,16 @@ #define CONFIG_MISMATCH_DEBUG 0 #define CONFIG_MULTITHREAD 1 #define CONFIG_NORMAL_TILE_MODE 0 +#define CONFIG_ONE_PASS_SVM 0 #define CONFIG_OS_SUPPORT 1 #define CONFIG_PIC 1 #define CONFIG_RD_DEBUG 0 -#define CONFIG_REDUCED_ENCODER_BORDER 0 #define CONFIG_RUNTIME_CPU_DETECT 1 #define CONFIG_SHARED 0 #define CONFIG_SHARP_SETTINGS 0 #define CONFIG_SIZE_LIMIT 0 #define CONFIG_SPATIAL_RESAMPLING 1 +#define CONFIG_SPEED_STATS 0 #define CONFIG_STATIC 1 #define CONFIG_WEBM_IO 0 #define DECODE_HEIGHT_LIMIT 0 @@ -76,7 +76,5 @@ #define HAVE_SSSE3 0 #define HAVE_VSX 0 #define HAVE_WXWIDGETS 0 -#define INCLUDE_INSTALL_DIR INSTALLDIR/include #define INLINE inline -#define LIB_INSTALL_DIR INSTALLDIR/lib #endif /* AOM_CONFIG_H_ */ diff --git a/media/libaom/config/linux/arm/config/aom_dsp_rtcd.h b/media/libaom/config/linux/arm/config/aom_dsp_rtcd.h index d80f8972b1..22fa7a1bbb 100644 --- a/media/libaom/config/linux/arm/config/aom_dsp_rtcd.h +++ b/media/libaom/config/linux/arm/config/aom_dsp_rtcd.h @@ -1400,10 +1400,6 @@ void aom_v_predictor_8x8_c(uint8_t *dst, ptrdiff_t y_stride, const uint8_t *abov void aom_v_predictor_8x8_neon(uint8_t *dst, ptrdiff_t y_stride, const uint8_t *above, const uint8_t *left); RTCD_EXTERN void (*aom_v_predictor_8x8)(uint8_t *dst, ptrdiff_t y_stride, const uint8_t *above, const uint8_t *left); -void av1_round_shift_array_c(int32_t *arr, int size, int bit); -void av1_round_shift_array_neon(int32_t *arr, int size, int bit); -RTCD_EXTERN void (*av1_round_shift_array)(int32_t *arr, int size, int bit); - void aom_dsp_rtcd(void); #include "config/aom_config.h" @@ -1496,8 +1492,6 @@ static void setup_rtcd_internal(void) if (flags & HAS_NEON) aom_v_predictor_4x4 = aom_v_predictor_4x4_neon; aom_v_predictor_8x8 = aom_v_predictor_8x8_c; if (flags & HAS_NEON) aom_v_predictor_8x8 = aom_v_predictor_8x8_neon; - av1_round_shift_array = av1_round_shift_array_c; - if (flags & HAS_NEON) av1_round_shift_array = av1_round_shift_array_neon; } #endif diff --git a/media/libaom/config/linux/arm/config/aom_scale_rtcd.h b/media/libaom/config/linux/arm/config/aom_scale_rtcd.h index 165d730936..7260bd372f 100644 --- a/media/libaom/config/linux/arm/config/aom_scale_rtcd.h +++ b/media/libaom/config/linux/arm/config/aom_scale_rtcd.h @@ -59,13 +59,22 @@ void aom_yv12_copy_y_c(const struct yv12_buffer_config *src_ybc, struct yv12_buf void aom_yv12_extend_frame_borders_c(struct yv12_buffer_config *ybf, const int num_planes); #define aom_yv12_extend_frame_borders aom_yv12_extend_frame_borders_c -void aom_yv12_partial_copy_u_c(const struct yv12_buffer_config *src_bc, struct yv12_buffer_config *dst_bc, int hstart, int hend, int vstart, int vend); +void aom_yv12_partial_coloc_copy_u_c(const struct yv12_buffer_config *src_bc, struct yv12_buffer_config *dst_bc, int hstart, int hend, int vstart, int vend); +#define aom_yv12_partial_coloc_copy_u aom_yv12_partial_coloc_copy_u_c + +void aom_yv12_partial_coloc_copy_v_c(const struct yv12_buffer_config *src_bc, struct yv12_buffer_config *dst_bc, int hstart, int hend, int vstart, int vend); +#define aom_yv12_partial_coloc_copy_v aom_yv12_partial_coloc_copy_v_c + +void aom_yv12_partial_coloc_copy_y_c(const struct yv12_buffer_config *src_ybc, struct yv12_buffer_config *dst_ybc, int hstart, int hend, int vstart, int vend); +#define aom_yv12_partial_coloc_copy_y aom_yv12_partial_coloc_copy_y_c + +void aom_yv12_partial_copy_u_c(const struct yv12_buffer_config *src_bc, int hstart1, int hend1, int vstart1, int vend1, struct yv12_buffer_config *dst_bc, int hstart2, int vstart2); #define aom_yv12_partial_copy_u aom_yv12_partial_copy_u_c -void aom_yv12_partial_copy_v_c(const struct yv12_buffer_config *src_bc, struct yv12_buffer_config *dst_bc, int hstart, int hend, int vstart, int vend); +void aom_yv12_partial_copy_v_c(const struct yv12_buffer_config *src_bc, int hstart1, int hend1, int vstart1, int vend1, struct yv12_buffer_config *dst_bc, int hstart2, int vstart2); #define aom_yv12_partial_copy_v aom_yv12_partial_copy_v_c -void aom_yv12_partial_copy_y_c(const struct yv12_buffer_config *src_ybc, struct yv12_buffer_config *dst_ybc, int hstart, int hend, int vstart, int vend); +void aom_yv12_partial_copy_y_c(const struct yv12_buffer_config *src_ybc, int hstart1, int hend1, int vstart1, int vend1, struct yv12_buffer_config *dst_ybc, int hstart2, int vstart2); #define aom_yv12_partial_copy_y aom_yv12_partial_copy_y_c void aom_scale_rtcd(void); diff --git a/media/libaom/config/linux/arm/config/av1_rtcd.h b/media/libaom/config/linux/arm/config/av1_rtcd.h index b54b4514dd..d51b6816cf 100644 --- a/media/libaom/config/linux/arm/config/av1_rtcd.h +++ b/media/libaom/config/linux/arm/config/av1_rtcd.h @@ -31,6 +31,8 @@ struct txfm_param; struct aom_variance_vtable; struct search_site_config; struct yv12_buffer_config; +struct NN_CONFIG; +typedef struct NN_CONFIG NN_CONFIG; /* Function pointers return by CfL functions */ typedef void (*cfl_subsample_lbd_fn)(const uint8_t *input, int input_stride, @@ -87,6 +89,22 @@ void av1_convolve_y_sr_c(const uint8_t *src, int src_stride, uint8_t *dst, int d void av1_convolve_y_sr_neon(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); RTCD_EXTERN void (*av1_convolve_y_sr)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_2d_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_2d_neon(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +RTCD_EXTERN void (*av1_dist_wtd_convolve_2d)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); + +void av1_dist_wtd_convolve_2d_copy_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_2d_copy_neon(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +RTCD_EXTERN void (*av1_dist_wtd_convolve_2d_copy)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); + +void av1_dist_wtd_convolve_x_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_x_neon(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +RTCD_EXTERN void (*av1_dist_wtd_convolve_x)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); + +void av1_dist_wtd_convolve_y_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_y_neon(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +RTCD_EXTERN void (*av1_dist_wtd_convolve_y)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); + void av1_dr_prediction_z1_c(uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_above, int dx, int dy); #define av1_dr_prediction_z1 av1_dr_prediction_z1_c @@ -138,6 +156,18 @@ void av1_highbd_convolve_x_sr_c(const uint16_t *src, int src_stride, uint16_t *d void av1_highbd_convolve_y_sr_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); #define av1_highbd_convolve_y_sr av1_highbd_convolve_y_sr_c +void av1_highbd_dist_wtd_convolve_2d_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +#define av1_highbd_dist_wtd_convolve_2d av1_highbd_dist_wtd_convolve_2d_c + +void av1_highbd_dist_wtd_convolve_2d_copy_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +#define av1_highbd_dist_wtd_convolve_2d_copy av1_highbd_dist_wtd_convolve_2d_copy_c + +void av1_highbd_dist_wtd_convolve_x_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +#define av1_highbd_dist_wtd_convolve_x av1_highbd_dist_wtd_convolve_x_c + +void av1_highbd_dist_wtd_convolve_y_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +#define av1_highbd_dist_wtd_convolve_y av1_highbd_dist_wtd_convolve_y_c + void av1_highbd_dr_prediction_z1_c(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_above, int dx, int dy, int bd); #define av1_highbd_dr_prediction_z1 av1_highbd_dr_prediction_z1_c @@ -150,20 +180,20 @@ void av1_highbd_dr_prediction_z3_c(uint16_t *dst, ptrdiff_t stride, int bw, int void av1_highbd_inv_txfm_add_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); #define av1_highbd_inv_txfm_add av1_highbd_inv_txfm_add_c -void av1_highbd_inv_txfm_add_16x16_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -#define av1_highbd_inv_txfm_add_16x16 av1_highbd_inv_txfm_add_16x16_c +void av1_highbd_inv_txfm_add_16x4_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +#define av1_highbd_inv_txfm_add_16x4 av1_highbd_inv_txfm_add_16x4_c -void av1_highbd_inv_txfm_add_16x8_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -#define av1_highbd_inv_txfm_add_16x8 av1_highbd_inv_txfm_add_16x8_c - -void av1_highbd_inv_txfm_add_32x32_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -#define av1_highbd_inv_txfm_add_32x32 av1_highbd_inv_txfm_add_32x32_c +void av1_highbd_inv_txfm_add_4x16_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +#define av1_highbd_inv_txfm_add_4x16 av1_highbd_inv_txfm_add_4x16_c void av1_highbd_inv_txfm_add_4x4_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); #define av1_highbd_inv_txfm_add_4x4 av1_highbd_inv_txfm_add_4x4_c -void av1_highbd_inv_txfm_add_8x16_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -#define av1_highbd_inv_txfm_add_8x16 av1_highbd_inv_txfm_add_8x16_c +void av1_highbd_inv_txfm_add_4x8_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +#define av1_highbd_inv_txfm_add_4x8 av1_highbd_inv_txfm_add_4x8_c + +void av1_highbd_inv_txfm_add_8x4_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +#define av1_highbd_inv_txfm_add_8x4 av1_highbd_inv_txfm_add_8x4_c void av1_highbd_inv_txfm_add_8x8_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); #define av1_highbd_inv_txfm_add_8x8 av1_highbd_inv_txfm_add_8x8_c @@ -174,18 +204,6 @@ void av1_highbd_iwht4x4_16_add_c(const tran_low_t *input, uint8_t *dest, int des void av1_highbd_iwht4x4_1_add_c(const tran_low_t *input, uint8_t *dest, int dest_stride, int bd); #define av1_highbd_iwht4x4_1_add av1_highbd_iwht4x4_1_add_c -void av1_highbd_jnt_convolve_2d_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -#define av1_highbd_jnt_convolve_2d av1_highbd_jnt_convolve_2d_c - -void av1_highbd_jnt_convolve_2d_copy_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -#define av1_highbd_jnt_convolve_2d_copy av1_highbd_jnt_convolve_2d_copy_c - -void av1_highbd_jnt_convolve_x_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -#define av1_highbd_jnt_convolve_x av1_highbd_jnt_convolve_x_c - -void av1_highbd_jnt_convolve_y_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -#define av1_highbd_jnt_convolve_y av1_highbd_jnt_convolve_y_c - void av1_highbd_warp_affine_c(const int32_t *mat, const uint16_t *ref, int width, int height, int stride, uint16_t *pred, int p_col, int p_row, int p_width, int p_height, int p_stride, int subsampling_x, int subsampling_y, int bd, ConvolveParams *conv_params, int16_t alpha, int16_t beta, int16_t gamma, int16_t delta); #define av1_highbd_warp_affine av1_highbd_warp_affine_c @@ -253,21 +271,9 @@ void av1_inv_txfm_add_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, con void av1_inv_txfm_add_neon(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); RTCD_EXTERN void (*av1_inv_txfm_add)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -void av1_jnt_convolve_2d_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -void av1_jnt_convolve_2d_neon(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -RTCD_EXTERN void (*av1_jnt_convolve_2d)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); - -void av1_jnt_convolve_2d_copy_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -void av1_jnt_convolve_2d_copy_neon(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -RTCD_EXTERN void (*av1_jnt_convolve_2d_copy)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); - -void av1_jnt_convolve_x_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -void av1_jnt_convolve_x_neon(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -RTCD_EXTERN void (*av1_jnt_convolve_x)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); - -void av1_jnt_convolve_y_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -void av1_jnt_convolve_y_neon(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -RTCD_EXTERN void (*av1_jnt_convolve_y)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_round_shift_array_c(int32_t *arr, int size, int bit); +void av1_round_shift_array_neon(int32_t *arr, int size, int bit); +RTCD_EXTERN void (*av1_round_shift_array)(int32_t *arr, int size, int bit); int av1_selfguided_restoration_c(const uint8_t *dgd8, int width, int height, int dgd_stride, int32_t *flt0, int32_t *flt1, int flt_stride, @@ -293,9 +299,9 @@ void av1_wiener_convolve_add_src_c(const uint8_t *src, ptrdiff_t src_stride, uin void av1_wiener_convolve_add_src_neon(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, const int16_t *filter_y, int y_step_q4, int w, int h, const ConvolveParams *conv_params); RTCD_EXTERN void (*av1_wiener_convolve_add_src)(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, const int16_t *filter_y, int y_step_q4, int w, int h, const ConvolveParams *conv_params); -void cdef_filter_block_c(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int max, int coeff_shift); -void cdef_filter_block_neon(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int max, int coeff_shift); -RTCD_EXTERN void (*cdef_filter_block)(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int max, int coeff_shift); +void cdef_filter_block_c(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int coeff_shift); +void cdef_filter_block_neon(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int coeff_shift); +RTCD_EXTERN void (*cdef_filter_block)(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int coeff_shift); int cdef_find_dir_c(const uint16_t *img, int stride, int32_t *var, int coeff_shift); int cdef_find_dir_neon(const uint16_t *img, int stride, int32_t *var, int coeff_shift); @@ -369,16 +375,18 @@ static void setup_rtcd_internal(void) if (flags & HAS_NEON) av1_convolve_x_sr = av1_convolve_x_sr_neon; av1_convolve_y_sr = av1_convolve_y_sr_c; if (flags & HAS_NEON) av1_convolve_y_sr = av1_convolve_y_sr_neon; + av1_dist_wtd_convolve_2d = av1_dist_wtd_convolve_2d_c; + if (flags & HAS_NEON) av1_dist_wtd_convolve_2d = av1_dist_wtd_convolve_2d_neon; + av1_dist_wtd_convolve_2d_copy = av1_dist_wtd_convolve_2d_copy_c; + if (flags & HAS_NEON) av1_dist_wtd_convolve_2d_copy = av1_dist_wtd_convolve_2d_copy_neon; + av1_dist_wtd_convolve_x = av1_dist_wtd_convolve_x_c; + if (flags & HAS_NEON) av1_dist_wtd_convolve_x = av1_dist_wtd_convolve_x_neon; + av1_dist_wtd_convolve_y = av1_dist_wtd_convolve_y_c; + if (flags & HAS_NEON) av1_dist_wtd_convolve_y = av1_dist_wtd_convolve_y_neon; av1_inv_txfm_add = av1_inv_txfm_add_c; if (flags & HAS_NEON) av1_inv_txfm_add = av1_inv_txfm_add_neon; - av1_jnt_convolve_2d = av1_jnt_convolve_2d_c; - if (flags & HAS_NEON) av1_jnt_convolve_2d = av1_jnt_convolve_2d_neon; - av1_jnt_convolve_2d_copy = av1_jnt_convolve_2d_copy_c; - if (flags & HAS_NEON) av1_jnt_convolve_2d_copy = av1_jnt_convolve_2d_copy_neon; - av1_jnt_convolve_x = av1_jnt_convolve_x_c; - if (flags & HAS_NEON) av1_jnt_convolve_x = av1_jnt_convolve_x_neon; - av1_jnt_convolve_y = av1_jnt_convolve_y_c; - if (flags & HAS_NEON) av1_jnt_convolve_y = av1_jnt_convolve_y_neon; + av1_round_shift_array = av1_round_shift_array_c; + if (flags & HAS_NEON) av1_round_shift_array = av1_round_shift_array_neon; av1_selfguided_restoration = av1_selfguided_restoration_c; if (flags & HAS_NEON) av1_selfguided_restoration = av1_selfguided_restoration_neon; av1_warp_affine = av1_warp_affine_c; diff --git a/media/libaom/config/linux/ia32/config/aom_config.asm b/media/libaom/config/linux/ia32/config/aom_config.asm index 128f85ec27..d037bc4231 100644 --- a/media/libaom/config/linux/ia32/config/aom_config.asm +++ b/media/libaom/config/linux/ia32/config/aom_config.asm @@ -1,5 +1,5 @@ ; -; Copyright (c) 2018, Alliance for Open Media. All rights reserved +; Copyright (c) 2019, Alliance for Open Media. All rights reserved ; ; This source code is subject to the terms of the BSD 2 Clause License and ; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License @@ -22,18 +22,17 @@ CONFIG_AV1_ENCODER equ 0 CONFIG_BIG_ENDIAN equ 0 CONFIG_BITSTREAM_DEBUG equ 0 CONFIG_COEFFICIENT_RANGE_CHECKING equ 0 -CONFIG_COLLECT_INTER_MODE_RD_STATS equ 0 +CONFIG_COLLECT_PARTITION_STATS equ 0 CONFIG_COLLECT_RD_STATS equ 0 CONFIG_DEBUG equ 0 CONFIG_DENOISE equ 1 +CONFIG_DISABLE_FULL_PIXEL_SPLIT_8X8 equ 1 CONFIG_DIST_8X8 equ 0 CONFIG_ENTROPY_STATS equ 0 CONFIG_FILEOPTIONS equ 1 -CONFIG_FIX_GF_LENGTH equ 1 CONFIG_FP_MB_STATS equ 0 CONFIG_GCC equ 1 CONFIG_GCOV equ 0 -CONFIG_GLOBAL_MOTION_SEARCH equ 1 CONFIG_GPROF equ 0 CONFIG_INSPECTION equ 0 CONFIG_INTERNAL_STATS equ 0 @@ -44,15 +43,16 @@ CONFIG_MAX_DECODE_PROFILE equ 2 CONFIG_MISMATCH_DEBUG equ 0 CONFIG_MULTITHREAD equ 1 CONFIG_NORMAL_TILE_MODE equ 0 +CONFIG_ONE_PASS_SVM equ 0 CONFIG_OS_SUPPORT equ 1 CONFIG_PIC equ 1 CONFIG_RD_DEBUG equ 0 -CONFIG_REDUCED_ENCODER_BORDER equ 0 CONFIG_RUNTIME_CPU_DETECT equ 1 CONFIG_SHARED equ 0 CONFIG_SHARP_SETTINGS equ 0 CONFIG_SIZE_LIMIT equ 0 CONFIG_SPATIAL_RESAMPLING equ 1 +CONFIG_SPEED_STATS equ 0 CONFIG_STATIC equ 1 CONFIG_WEBM_IO equ 0 DECODE_HEIGHT_LIMIT equ 0 diff --git a/media/libaom/config/linux/ia32/config/aom_config.h b/media/libaom/config/linux/ia32/config/aom_config.h index 02f6f72434..67201081b2 100644 --- a/media/libaom/config/linux/ia32/config/aom_config.h +++ b/media/libaom/config/linux/ia32/config/aom_config.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, Alliance for Open Media. All rights reserved + * Copyright (c) 2019, Alliance for Open Media. All rights reserved * * This source code is subject to the terms of the BSD 2 Clause License and * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License @@ -24,18 +24,17 @@ #define CONFIG_BIG_ENDIAN 0 #define CONFIG_BITSTREAM_DEBUG 0 #define CONFIG_COEFFICIENT_RANGE_CHECKING 0 -#define CONFIG_COLLECT_INTER_MODE_RD_STATS 0 +#define CONFIG_COLLECT_PARTITION_STATS 0 #define CONFIG_COLLECT_RD_STATS 0 #define CONFIG_DEBUG 0 #define CONFIG_DENOISE 1 +#define CONFIG_DISABLE_FULL_PIXEL_SPLIT_8X8 1 #define CONFIG_DIST_8X8 0 #define CONFIG_ENTROPY_STATS 0 #define CONFIG_FILEOPTIONS 1 -#define CONFIG_FIX_GF_LENGTH 1 #define CONFIG_FP_MB_STATS 0 #define CONFIG_GCC 1 #define CONFIG_GCOV 0 -#define CONFIG_GLOBAL_MOTION_SEARCH 1 #define CONFIG_GPROF 0 #define CONFIG_INSPECTION 0 #define CONFIG_INTERNAL_STATS 0 @@ -46,15 +45,16 @@ #define CONFIG_MISMATCH_DEBUG 0 #define CONFIG_MULTITHREAD 1 #define CONFIG_NORMAL_TILE_MODE 0 +#define CONFIG_ONE_PASS_SVM 0 #define CONFIG_OS_SUPPORT 1 #define CONFIG_PIC 1 #define CONFIG_RD_DEBUG 0 -#define CONFIG_REDUCED_ENCODER_BORDER 0 #define CONFIG_RUNTIME_CPU_DETECT 1 #define CONFIG_SHARED 0 #define CONFIG_SHARP_SETTINGS 0 #define CONFIG_SIZE_LIMIT 0 #define CONFIG_SPATIAL_RESAMPLING 1 +#define CONFIG_SPEED_STATS 0 #define CONFIG_STATIC 1 #define CONFIG_WEBM_IO 0 #define DECODE_HEIGHT_LIMIT 0 @@ -76,7 +76,5 @@ #define HAVE_SSSE3 1 #define HAVE_VSX 0 #define HAVE_WXWIDGETS 0 -#define INCLUDE_INSTALL_DIR INSTALLDIR/include #define INLINE inline -#define LIB_INSTALL_DIR INSTALLDIR/lib #endif /* AOM_CONFIG_H_ */ diff --git a/media/libaom/config/linux/ia32/config/aom_dsp_rtcd.h b/media/libaom/config/linux/ia32/config/aom_dsp_rtcd.h index 5b3762f109..a60a7aea6e 100644 --- a/media/libaom/config/linux/ia32/config/aom_dsp_rtcd.h +++ b/media/libaom/config/linux/ia32/config/aom_dsp_rtcd.h @@ -472,7 +472,9 @@ void aom_h_predictor_8x8_sse2(uint8_t *dst, ptrdiff_t y_stride, const uint8_t *a RTCD_EXTERN void (*aom_h_predictor_8x8)(uint8_t *dst, ptrdiff_t y_stride, const uint8_t *above, const uint8_t *left); void aom_highbd_blend_a64_d16_mask_c(uint8_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0, uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride, const uint8_t *mask, uint32_t mask_stride, int w, int h, int subx, int suby, ConvolveParams *conv_params, const int bd); -#define aom_highbd_blend_a64_d16_mask aom_highbd_blend_a64_d16_mask_c +void aom_highbd_blend_a64_d16_mask_sse4_1(uint8_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0, uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride, const uint8_t *mask, uint32_t mask_stride, int w, int h, int subx, int suby, ConvolveParams *conv_params, const int bd); +void aom_highbd_blend_a64_d16_mask_avx2(uint8_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0, uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride, const uint8_t *mask, uint32_t mask_stride, int w, int h, int subx, int suby, ConvolveParams *conv_params, const int bd); +RTCD_EXTERN void (*aom_highbd_blend_a64_d16_mask)(uint8_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0, uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride, const uint8_t *mask, uint32_t mask_stride, int w, int h, int subx, int suby, ConvolveParams *conv_params, const int bd); void aom_highbd_blend_a64_hmask_c(uint8_t *dst, uint32_t dst_stride, const uint8_t *src0, uint32_t src0_stride, const uint8_t *src1, uint32_t src1_stride, const uint8_t *mask, int w, int h, int bd); void aom_highbd_blend_a64_hmask_sse4_1(uint8_t *dst, uint32_t dst_stride, const uint8_t *src0, uint32_t src0_stride, const uint8_t *src1, uint32_t src1_stride, const uint8_t *mask, int w, int h, int bd); @@ -487,10 +489,12 @@ void aom_highbd_blend_a64_vmask_sse4_1(uint8_t *dst, uint32_t dst_stride, const RTCD_EXTERN void (*aom_highbd_blend_a64_vmask)(uint8_t *dst, uint32_t dst_stride, const uint8_t *src0, uint32_t src0_stride, const uint8_t *src1, uint32_t src1_stride, const uint8_t *mask, int w, int h, int bd); void aom_highbd_convolve8_horiz_c(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, const int16_t *filter_y, int y_step_q4, int w, int h, int bps); +void aom_highbd_convolve8_horiz_sse2(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, const int16_t *filter_y, int y_step_q4, int w, int h, int bps); void aom_highbd_convolve8_horiz_avx2(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, const int16_t *filter_y, int y_step_q4, int w, int h, int bps); RTCD_EXTERN void (*aom_highbd_convolve8_horiz)(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, const int16_t *filter_y, int y_step_q4, int w, int h, int bps); void aom_highbd_convolve8_vert_c(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, const int16_t *filter_y, int y_step_q4, int w, int h, int bps); +void aom_highbd_convolve8_vert_sse2(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, const int16_t *filter_y, int y_step_q4, int w, int h, int bps); void aom_highbd_convolve8_vert_avx2(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, const int16_t *filter_y, int y_step_q4, int w, int h, int bps); RTCD_EXTERN void (*aom_highbd_convolve8_vert)(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, const int16_t *filter_y, int y_step_q4, int w, int h, int bps); @@ -1709,10 +1713,6 @@ void aom_v_predictor_8x8_c(uint8_t *dst, ptrdiff_t y_stride, const uint8_t *abov void aom_v_predictor_8x8_sse2(uint8_t *dst, ptrdiff_t y_stride, const uint8_t *above, const uint8_t *left); RTCD_EXTERN void (*aom_v_predictor_8x8)(uint8_t *dst, ptrdiff_t y_stride, const uint8_t *above, const uint8_t *left); -void av1_round_shift_array_c(int32_t *arr, int size, int bit); -void av1_round_shift_array_sse4_1(int32_t *arr, int size, int bit); -RTCD_EXTERN void (*av1_round_shift_array)(int32_t *arr, int size, int bit); - void aom_dsp_rtcd(void); #ifdef RTCD_C @@ -1955,6 +1955,9 @@ static void setup_rtcd_internal(void) if (flags & HAS_SSE2) aom_h_predictor_8x4 = aom_h_predictor_8x4_sse2; aom_h_predictor_8x8 = aom_h_predictor_8x8_c; if (flags & HAS_SSE2) aom_h_predictor_8x8 = aom_h_predictor_8x8_sse2; + aom_highbd_blend_a64_d16_mask = aom_highbd_blend_a64_d16_mask_c; + if (flags & HAS_SSE4_1) aom_highbd_blend_a64_d16_mask = aom_highbd_blend_a64_d16_mask_sse4_1; + if (flags & HAS_AVX2) aom_highbd_blend_a64_d16_mask = aom_highbd_blend_a64_d16_mask_avx2; aom_highbd_blend_a64_hmask = aom_highbd_blend_a64_hmask_c; if (flags & HAS_SSE4_1) aom_highbd_blend_a64_hmask = aom_highbd_blend_a64_hmask_sse4_1; aom_highbd_blend_a64_mask = aom_highbd_blend_a64_mask_c; @@ -1962,8 +1965,10 @@ static void setup_rtcd_internal(void) aom_highbd_blend_a64_vmask = aom_highbd_blend_a64_vmask_c; if (flags & HAS_SSE4_1) aom_highbd_blend_a64_vmask = aom_highbd_blend_a64_vmask_sse4_1; aom_highbd_convolve8_horiz = aom_highbd_convolve8_horiz_c; + if (flags & HAS_SSE2) aom_highbd_convolve8_horiz = aom_highbd_convolve8_horiz_sse2; if (flags & HAS_AVX2) aom_highbd_convolve8_horiz = aom_highbd_convolve8_horiz_avx2; aom_highbd_convolve8_vert = aom_highbd_convolve8_vert_c; + if (flags & HAS_SSE2) aom_highbd_convolve8_vert = aom_highbd_convolve8_vert_sse2; if (flags & HAS_AVX2) aom_highbd_convolve8_vert = aom_highbd_convolve8_vert_avx2; aom_highbd_convolve_copy = aom_highbd_convolve_copy_c; if (flags & HAS_SSE2) aom_highbd_convolve_copy = aom_highbd_convolve_copy_sse2; @@ -2367,8 +2372,6 @@ static void setup_rtcd_internal(void) if (flags & HAS_SSE2) aom_v_predictor_8x4 = aom_v_predictor_8x4_sse2; aom_v_predictor_8x8 = aom_v_predictor_8x8_c; if (flags & HAS_SSE2) aom_v_predictor_8x8 = aom_v_predictor_8x8_sse2; - av1_round_shift_array = av1_round_shift_array_c; - if (flags & HAS_SSE4_1) av1_round_shift_array = av1_round_shift_array_sse4_1; } #endif diff --git a/media/libaom/config/linux/ia32/config/aom_scale_rtcd.h b/media/libaom/config/linux/ia32/config/aom_scale_rtcd.h index 6b56795cd4..b6e814905b 100644 --- a/media/libaom/config/linux/ia32/config/aom_scale_rtcd.h +++ b/media/libaom/config/linux/ia32/config/aom_scale_rtcd.h @@ -59,13 +59,22 @@ void aom_yv12_copy_y_c(const struct yv12_buffer_config *src_ybc, struct yv12_buf void aom_yv12_extend_frame_borders_c(struct yv12_buffer_config *ybf, const int num_planes); #define aom_yv12_extend_frame_borders aom_yv12_extend_frame_borders_c -void aom_yv12_partial_copy_u_c(const struct yv12_buffer_config *src_bc, struct yv12_buffer_config *dst_bc, int hstart, int hend, int vstart, int vend); +void aom_yv12_partial_coloc_copy_u_c(const struct yv12_buffer_config *src_bc, struct yv12_buffer_config *dst_bc, int hstart, int hend, int vstart, int vend); +#define aom_yv12_partial_coloc_copy_u aom_yv12_partial_coloc_copy_u_c + +void aom_yv12_partial_coloc_copy_v_c(const struct yv12_buffer_config *src_bc, struct yv12_buffer_config *dst_bc, int hstart, int hend, int vstart, int vend); +#define aom_yv12_partial_coloc_copy_v aom_yv12_partial_coloc_copy_v_c + +void aom_yv12_partial_coloc_copy_y_c(const struct yv12_buffer_config *src_ybc, struct yv12_buffer_config *dst_ybc, int hstart, int hend, int vstart, int vend); +#define aom_yv12_partial_coloc_copy_y aom_yv12_partial_coloc_copy_y_c + +void aom_yv12_partial_copy_u_c(const struct yv12_buffer_config *src_bc, int hstart1, int hend1, int vstart1, int vend1, struct yv12_buffer_config *dst_bc, int hstart2, int vstart2); #define aom_yv12_partial_copy_u aom_yv12_partial_copy_u_c -void aom_yv12_partial_copy_v_c(const struct yv12_buffer_config *src_bc, struct yv12_buffer_config *dst_bc, int hstart, int hend, int vstart, int vend); +void aom_yv12_partial_copy_v_c(const struct yv12_buffer_config *src_bc, int hstart1, int hend1, int vstart1, int vend1, struct yv12_buffer_config *dst_bc, int hstart2, int vstart2); #define aom_yv12_partial_copy_v aom_yv12_partial_copy_v_c -void aom_yv12_partial_copy_y_c(const struct yv12_buffer_config *src_ybc, struct yv12_buffer_config *dst_ybc, int hstart, int hend, int vstart, int vend); +void aom_yv12_partial_copy_y_c(const struct yv12_buffer_config *src_ybc, int hstart1, int hend1, int vstart1, int vend1, struct yv12_buffer_config *dst_ybc, int hstart2, int vstart2); #define aom_yv12_partial_copy_y aom_yv12_partial_copy_y_c void aom_scale_rtcd(void); diff --git a/media/libaom/config/linux/ia32/config/av1_rtcd.h b/media/libaom/config/linux/ia32/config/av1_rtcd.h index a2a6f14ff5..8e69b643ef 100644 --- a/media/libaom/config/linux/ia32/config/av1_rtcd.h +++ b/media/libaom/config/linux/ia32/config/av1_rtcd.h @@ -31,6 +31,8 @@ struct txfm_param; struct aom_variance_vtable; struct search_site_config; struct yv12_buffer_config; +struct NN_CONFIG; +typedef struct NN_CONFIG NN_CONFIG; /* Function pointers return by CfL functions */ typedef void (*cfl_subsample_lbd_fn)(const uint8_t *input, int input_stride, @@ -99,14 +101,38 @@ void av1_convolve_y_sr_sse2(const uint8_t *src, int src_stride, uint8_t *dst, in void av1_convolve_y_sr_avx2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); RTCD_EXTERN void (*av1_convolve_y_sr)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_2d_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_2d_sse2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_2d_ssse3(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_2d_avx2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +RTCD_EXTERN void (*av1_dist_wtd_convolve_2d)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); + +void av1_dist_wtd_convolve_2d_copy_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_2d_copy_sse2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_2d_copy_avx2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +RTCD_EXTERN void (*av1_dist_wtd_convolve_2d_copy)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); + +void av1_dist_wtd_convolve_x_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_x_sse2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_x_avx2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +RTCD_EXTERN void (*av1_dist_wtd_convolve_x)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); + +void av1_dist_wtd_convolve_y_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_y_sse2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_y_avx2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +RTCD_EXTERN void (*av1_dist_wtd_convolve_y)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); + void av1_dr_prediction_z1_c(uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_above, int dx, int dy); -#define av1_dr_prediction_z1 av1_dr_prediction_z1_c +void av1_dr_prediction_z1_avx2(uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_above, int dx, int dy); +RTCD_EXTERN void (*av1_dr_prediction_z1)(uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_above, int dx, int dy); void av1_dr_prediction_z2_c(uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_above, int upsample_left, int dx, int dy); -#define av1_dr_prediction_z2 av1_dr_prediction_z2_c +void av1_dr_prediction_z2_avx2(uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_above, int upsample_left, int dx, int dy); +RTCD_EXTERN void (*av1_dr_prediction_z2)(uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_above, int upsample_left, int dx, int dy); void av1_dr_prediction_z3_c(uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_left, int dx, int dy); -#define av1_dr_prediction_z3 av1_dr_prediction_z3_c +void av1_dr_prediction_z3_avx2(uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_left, int dx, int dy); +RTCD_EXTERN void (*av1_dr_prediction_z3)(uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_left, int dx, int dy); void av1_filter_intra_edge_c(uint8_t *p, int sz, int strength); void av1_filter_intra_edge_sse4_1(uint8_t *p, int sz, int strength); @@ -163,40 +189,61 @@ void av1_highbd_convolve_y_sr_ssse3(const uint16_t *src, int src_stride, uint16_ void av1_highbd_convolve_y_sr_avx2(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); RTCD_EXTERN void (*av1_highbd_convolve_y_sr)(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +void av1_highbd_dist_wtd_convolve_2d_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +void av1_highbd_dist_wtd_convolve_2d_sse4_1(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +void av1_highbd_dist_wtd_convolve_2d_avx2(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +RTCD_EXTERN void (*av1_highbd_dist_wtd_convolve_2d)(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); + +void av1_highbd_dist_wtd_convolve_2d_copy_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +void av1_highbd_dist_wtd_convolve_2d_copy_sse4_1(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +void av1_highbd_dist_wtd_convolve_2d_copy_avx2(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +RTCD_EXTERN void (*av1_highbd_dist_wtd_convolve_2d_copy)(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); + +void av1_highbd_dist_wtd_convolve_x_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +void av1_highbd_dist_wtd_convolve_x_sse4_1(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +void av1_highbd_dist_wtd_convolve_x_avx2(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +RTCD_EXTERN void (*av1_highbd_dist_wtd_convolve_x)(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); + +void av1_highbd_dist_wtd_convolve_y_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +void av1_highbd_dist_wtd_convolve_y_sse4_1(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +void av1_highbd_dist_wtd_convolve_y_avx2(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +RTCD_EXTERN void (*av1_highbd_dist_wtd_convolve_y)(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); + void av1_highbd_dr_prediction_z1_c(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_above, int dx, int dy, int bd); -#define av1_highbd_dr_prediction_z1 av1_highbd_dr_prediction_z1_c +void av1_highbd_dr_prediction_z1_avx2(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_above, int dx, int dy, int bd); +RTCD_EXTERN void (*av1_highbd_dr_prediction_z1)(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_above, int dx, int dy, int bd); void av1_highbd_dr_prediction_z2_c(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_above, int upsample_left, int dx, int dy, int bd); #define av1_highbd_dr_prediction_z2 av1_highbd_dr_prediction_z2_c void av1_highbd_dr_prediction_z3_c(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_left, int dx, int dy, int bd); -#define av1_highbd_dr_prediction_z3 av1_highbd_dr_prediction_z3_c +void av1_highbd_dr_prediction_z3_avx2(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_left, int dx, int dy, int bd); +RTCD_EXTERN void (*av1_highbd_dr_prediction_z3)(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_left, int dx, int dy, int bd); void av1_highbd_inv_txfm_add_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); void av1_highbd_inv_txfm_add_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); void av1_highbd_inv_txfm_add_avx2(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); RTCD_EXTERN void (*av1_highbd_inv_txfm_add)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -void av1_highbd_inv_txfm_add_16x16_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -void av1_highbd_inv_txfm_add_16x16_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -RTCD_EXTERN void (*av1_highbd_inv_txfm_add_16x16)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +void av1_highbd_inv_txfm_add_16x4_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +void av1_highbd_inv_txfm_add_16x4_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +RTCD_EXTERN void (*av1_highbd_inv_txfm_add_16x4)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -void av1_highbd_inv_txfm_add_16x8_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -void av1_highbd_inv_txfm_add_16x8_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -RTCD_EXTERN void (*av1_highbd_inv_txfm_add_16x8)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); - -void av1_highbd_inv_txfm_add_32x32_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -void av1_highbd_inv_txfm_add_32x32_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -void av1_highbd_inv_txfm_add_32x32_avx2(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -RTCD_EXTERN void (*av1_highbd_inv_txfm_add_32x32)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +void av1_highbd_inv_txfm_add_4x16_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +void av1_highbd_inv_txfm_add_4x16_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +RTCD_EXTERN void (*av1_highbd_inv_txfm_add_4x16)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); void av1_highbd_inv_txfm_add_4x4_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); void av1_highbd_inv_txfm_add_4x4_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); RTCD_EXTERN void (*av1_highbd_inv_txfm_add_4x4)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -void av1_highbd_inv_txfm_add_8x16_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -void av1_highbd_inv_txfm_add_8x16_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -RTCD_EXTERN void (*av1_highbd_inv_txfm_add_8x16)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +void av1_highbd_inv_txfm_add_4x8_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +void av1_highbd_inv_txfm_add_4x8_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +RTCD_EXTERN void (*av1_highbd_inv_txfm_add_4x8)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); + +void av1_highbd_inv_txfm_add_8x4_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +void av1_highbd_inv_txfm_add_8x4_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +RTCD_EXTERN void (*av1_highbd_inv_txfm_add_8x4)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); void av1_highbd_inv_txfm_add_8x8_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); void av1_highbd_inv_txfm_add_8x8_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); @@ -208,26 +255,6 @@ void av1_highbd_iwht4x4_16_add_c(const tran_low_t *input, uint8_t *dest, int des void av1_highbd_iwht4x4_1_add_c(const tran_low_t *input, uint8_t *dest, int dest_stride, int bd); #define av1_highbd_iwht4x4_1_add av1_highbd_iwht4x4_1_add_c -void av1_highbd_jnt_convolve_2d_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -void av1_highbd_jnt_convolve_2d_sse4_1(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -void av1_highbd_jnt_convolve_2d_avx2(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -RTCD_EXTERN void (*av1_highbd_jnt_convolve_2d)(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); - -void av1_highbd_jnt_convolve_2d_copy_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -void av1_highbd_jnt_convolve_2d_copy_sse4_1(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -void av1_highbd_jnt_convolve_2d_copy_avx2(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -RTCD_EXTERN void (*av1_highbd_jnt_convolve_2d_copy)(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); - -void av1_highbd_jnt_convolve_x_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -void av1_highbd_jnt_convolve_x_sse4_1(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -void av1_highbd_jnt_convolve_x_avx2(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -RTCD_EXTERN void (*av1_highbd_jnt_convolve_x)(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); - -void av1_highbd_jnt_convolve_y_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -void av1_highbd_jnt_convolve_y_sse4_1(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -void av1_highbd_jnt_convolve_y_avx2(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -RTCD_EXTERN void (*av1_highbd_jnt_convolve_y)(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); - void av1_highbd_warp_affine_c(const int32_t *mat, const uint16_t *ref, int width, int height, int stride, uint16_t *pred, int p_col, int p_row, int p_width, int p_height, int p_stride, int subsampling_x, int subsampling_y, int bd, ConvolveParams *conv_params, int16_t alpha, int16_t beta, int16_t gamma, int16_t delta); void av1_highbd_warp_affine_sse4_1(const int32_t *mat, const uint16_t *ref, int width, int height, int stride, uint16_t *pred, int p_col, int p_row, int p_width, int p_height, int p_stride, int subsampling_x, int subsampling_y, int bd, ConvolveParams *conv_params, int16_t alpha, int16_t beta, int16_t gamma, int16_t delta); RTCD_EXTERN void (*av1_highbd_warp_affine)(const int32_t *mat, const uint16_t *ref, int width, int height, int stride, uint16_t *pred, int p_col, int p_row, int p_width, int p_height, int p_stride, int subsampling_x, int subsampling_y, int bd, ConvolveParams *conv_params, int16_t alpha, int16_t beta, int16_t gamma, int16_t delta); @@ -301,25 +328,9 @@ void av1_inv_txfm_add_ssse3(const tran_low_t *dqcoeff, uint8_t *dst, int stride, void av1_inv_txfm_add_avx2(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); RTCD_EXTERN void (*av1_inv_txfm_add)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -void av1_jnt_convolve_2d_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -void av1_jnt_convolve_2d_ssse3(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -void av1_jnt_convolve_2d_avx2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -RTCD_EXTERN void (*av1_jnt_convolve_2d)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); - -void av1_jnt_convolve_2d_copy_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -void av1_jnt_convolve_2d_copy_sse2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -void av1_jnt_convolve_2d_copy_avx2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -RTCD_EXTERN void (*av1_jnt_convolve_2d_copy)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); - -void av1_jnt_convolve_x_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -void av1_jnt_convolve_x_sse2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -void av1_jnt_convolve_x_avx2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -RTCD_EXTERN void (*av1_jnt_convolve_x)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); - -void av1_jnt_convolve_y_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -void av1_jnt_convolve_y_sse2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -void av1_jnt_convolve_y_avx2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -RTCD_EXTERN void (*av1_jnt_convolve_y)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_round_shift_array_c(int32_t *arr, int size, int bit); +void av1_round_shift_array_sse4_1(int32_t *arr, int size, int bit); +RTCD_EXTERN void (*av1_round_shift_array)(int32_t *arr, int size, int bit); int av1_selfguided_restoration_c(const uint8_t *dgd8, int width, int height, int dgd_stride, int32_t *flt0, int32_t *flt1, int flt_stride, @@ -351,12 +362,12 @@ void av1_wiener_convolve_add_src_sse2(const uint8_t *src, ptrdiff_t src_stride, void av1_wiener_convolve_add_src_avx2(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, const int16_t *filter_y, int y_step_q4, int w, int h, const ConvolveParams *conv_params); RTCD_EXTERN void (*av1_wiener_convolve_add_src)(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, const int16_t *filter_y, int y_step_q4, int w, int h, const ConvolveParams *conv_params); -void cdef_filter_block_c(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int max, int coeff_shift); -void cdef_filter_block_sse2(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int max, int coeff_shift); -void cdef_filter_block_ssse3(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int max, int coeff_shift); -void cdef_filter_block_sse4_1(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int max, int coeff_shift); -void cdef_filter_block_avx2(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int max, int coeff_shift); -RTCD_EXTERN void (*cdef_filter_block)(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int max, int coeff_shift); +void cdef_filter_block_c(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int coeff_shift); +void cdef_filter_block_sse2(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int coeff_shift); +void cdef_filter_block_ssse3(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int coeff_shift); +void cdef_filter_block_sse4_1(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int coeff_shift); +void cdef_filter_block_avx2(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int coeff_shift); +RTCD_EXTERN void (*cdef_filter_block)(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int coeff_shift); int cdef_find_dir_c(const uint16_t *img, int stride, int32_t *var, int coeff_shift); int cdef_find_dir_sse2(const uint16_t *img, int stride, int32_t *var, int coeff_shift); @@ -462,6 +473,25 @@ static void setup_rtcd_internal(void) av1_convolve_y_sr = av1_convolve_y_sr_c; if (flags & HAS_SSE2) av1_convolve_y_sr = av1_convolve_y_sr_sse2; if (flags & HAS_AVX2) av1_convolve_y_sr = av1_convolve_y_sr_avx2; + av1_dist_wtd_convolve_2d = av1_dist_wtd_convolve_2d_c; + if (flags & HAS_SSE2) av1_dist_wtd_convolve_2d = av1_dist_wtd_convolve_2d_sse2; + if (flags & HAS_SSSE3) av1_dist_wtd_convolve_2d = av1_dist_wtd_convolve_2d_ssse3; + if (flags & HAS_AVX2) av1_dist_wtd_convolve_2d = av1_dist_wtd_convolve_2d_avx2; + av1_dist_wtd_convolve_2d_copy = av1_dist_wtd_convolve_2d_copy_c; + if (flags & HAS_SSE2) av1_dist_wtd_convolve_2d_copy = av1_dist_wtd_convolve_2d_copy_sse2; + if (flags & HAS_AVX2) av1_dist_wtd_convolve_2d_copy = av1_dist_wtd_convolve_2d_copy_avx2; + av1_dist_wtd_convolve_x = av1_dist_wtd_convolve_x_c; + if (flags & HAS_SSE2) av1_dist_wtd_convolve_x = av1_dist_wtd_convolve_x_sse2; + if (flags & HAS_AVX2) av1_dist_wtd_convolve_x = av1_dist_wtd_convolve_x_avx2; + av1_dist_wtd_convolve_y = av1_dist_wtd_convolve_y_c; + if (flags & HAS_SSE2) av1_dist_wtd_convolve_y = av1_dist_wtd_convolve_y_sse2; + if (flags & HAS_AVX2) av1_dist_wtd_convolve_y = av1_dist_wtd_convolve_y_avx2; + av1_dr_prediction_z1 = av1_dr_prediction_z1_c; + if (flags & HAS_AVX2) av1_dr_prediction_z1 = av1_dr_prediction_z1_avx2; + av1_dr_prediction_z2 = av1_dr_prediction_z2_c; + if (flags & HAS_AVX2) av1_dr_prediction_z2 = av1_dr_prediction_z2_avx2; + av1_dr_prediction_z3 = av1_dr_prediction_z3_c; + if (flags & HAS_AVX2) av1_dr_prediction_z3 = av1_dr_prediction_z3_avx2; av1_filter_intra_edge = av1_filter_intra_edge_c; if (flags & HAS_SSE4_1) av1_filter_intra_edge = av1_filter_intra_edge_sse4_1; av1_filter_intra_edge_high = av1_filter_intra_edge_high_c; @@ -484,34 +514,37 @@ static void setup_rtcd_internal(void) av1_highbd_convolve_y_sr = av1_highbd_convolve_y_sr_c; if (flags & HAS_SSSE3) av1_highbd_convolve_y_sr = av1_highbd_convolve_y_sr_ssse3; if (flags & HAS_AVX2) av1_highbd_convolve_y_sr = av1_highbd_convolve_y_sr_avx2; + av1_highbd_dist_wtd_convolve_2d = av1_highbd_dist_wtd_convolve_2d_c; + if (flags & HAS_SSE4_1) av1_highbd_dist_wtd_convolve_2d = av1_highbd_dist_wtd_convolve_2d_sse4_1; + if (flags & HAS_AVX2) av1_highbd_dist_wtd_convolve_2d = av1_highbd_dist_wtd_convolve_2d_avx2; + av1_highbd_dist_wtd_convolve_2d_copy = av1_highbd_dist_wtd_convolve_2d_copy_c; + if (flags & HAS_SSE4_1) av1_highbd_dist_wtd_convolve_2d_copy = av1_highbd_dist_wtd_convolve_2d_copy_sse4_1; + if (flags & HAS_AVX2) av1_highbd_dist_wtd_convolve_2d_copy = av1_highbd_dist_wtd_convolve_2d_copy_avx2; + av1_highbd_dist_wtd_convolve_x = av1_highbd_dist_wtd_convolve_x_c; + if (flags & HAS_SSE4_1) av1_highbd_dist_wtd_convolve_x = av1_highbd_dist_wtd_convolve_x_sse4_1; + if (flags & HAS_AVX2) av1_highbd_dist_wtd_convolve_x = av1_highbd_dist_wtd_convolve_x_avx2; + av1_highbd_dist_wtd_convolve_y = av1_highbd_dist_wtd_convolve_y_c; + if (flags & HAS_SSE4_1) av1_highbd_dist_wtd_convolve_y = av1_highbd_dist_wtd_convolve_y_sse4_1; + if (flags & HAS_AVX2) av1_highbd_dist_wtd_convolve_y = av1_highbd_dist_wtd_convolve_y_avx2; + av1_highbd_dr_prediction_z1 = av1_highbd_dr_prediction_z1_c; + if (flags & HAS_AVX2) av1_highbd_dr_prediction_z1 = av1_highbd_dr_prediction_z1_avx2; + av1_highbd_dr_prediction_z3 = av1_highbd_dr_prediction_z3_c; + if (flags & HAS_AVX2) av1_highbd_dr_prediction_z3 = av1_highbd_dr_prediction_z3_avx2; av1_highbd_inv_txfm_add = av1_highbd_inv_txfm_add_c; if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add = av1_highbd_inv_txfm_add_sse4_1; if (flags & HAS_AVX2) av1_highbd_inv_txfm_add = av1_highbd_inv_txfm_add_avx2; - av1_highbd_inv_txfm_add_16x16 = av1_highbd_inv_txfm_add_16x16_c; - if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add_16x16 = av1_highbd_inv_txfm_add_16x16_sse4_1; - av1_highbd_inv_txfm_add_16x8 = av1_highbd_inv_txfm_add_16x8_c; - if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add_16x8 = av1_highbd_inv_txfm_add_16x8_sse4_1; - av1_highbd_inv_txfm_add_32x32 = av1_highbd_inv_txfm_add_32x32_c; - if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add_32x32 = av1_highbd_inv_txfm_add_32x32_sse4_1; - if (flags & HAS_AVX2) av1_highbd_inv_txfm_add_32x32 = av1_highbd_inv_txfm_add_32x32_avx2; + av1_highbd_inv_txfm_add_16x4 = av1_highbd_inv_txfm_add_16x4_c; + if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add_16x4 = av1_highbd_inv_txfm_add_16x4_sse4_1; + av1_highbd_inv_txfm_add_4x16 = av1_highbd_inv_txfm_add_4x16_c; + if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add_4x16 = av1_highbd_inv_txfm_add_4x16_sse4_1; av1_highbd_inv_txfm_add_4x4 = av1_highbd_inv_txfm_add_4x4_c; if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add_4x4 = av1_highbd_inv_txfm_add_4x4_sse4_1; - av1_highbd_inv_txfm_add_8x16 = av1_highbd_inv_txfm_add_8x16_c; - if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add_8x16 = av1_highbd_inv_txfm_add_8x16_sse4_1; + av1_highbd_inv_txfm_add_4x8 = av1_highbd_inv_txfm_add_4x8_c; + if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add_4x8 = av1_highbd_inv_txfm_add_4x8_sse4_1; + av1_highbd_inv_txfm_add_8x4 = av1_highbd_inv_txfm_add_8x4_c; + if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add_8x4 = av1_highbd_inv_txfm_add_8x4_sse4_1; av1_highbd_inv_txfm_add_8x8 = av1_highbd_inv_txfm_add_8x8_c; if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add_8x8 = av1_highbd_inv_txfm_add_8x8_sse4_1; - av1_highbd_jnt_convolve_2d = av1_highbd_jnt_convolve_2d_c; - if (flags & HAS_SSE4_1) av1_highbd_jnt_convolve_2d = av1_highbd_jnt_convolve_2d_sse4_1; - if (flags & HAS_AVX2) av1_highbd_jnt_convolve_2d = av1_highbd_jnt_convolve_2d_avx2; - av1_highbd_jnt_convolve_2d_copy = av1_highbd_jnt_convolve_2d_copy_c; - if (flags & HAS_SSE4_1) av1_highbd_jnt_convolve_2d_copy = av1_highbd_jnt_convolve_2d_copy_sse4_1; - if (flags & HAS_AVX2) av1_highbd_jnt_convolve_2d_copy = av1_highbd_jnt_convolve_2d_copy_avx2; - av1_highbd_jnt_convolve_x = av1_highbd_jnt_convolve_x_c; - if (flags & HAS_SSE4_1) av1_highbd_jnt_convolve_x = av1_highbd_jnt_convolve_x_sse4_1; - if (flags & HAS_AVX2) av1_highbd_jnt_convolve_x = av1_highbd_jnt_convolve_x_avx2; - av1_highbd_jnt_convolve_y = av1_highbd_jnt_convolve_y_c; - if (flags & HAS_SSE4_1) av1_highbd_jnt_convolve_y = av1_highbd_jnt_convolve_y_sse4_1; - if (flags & HAS_AVX2) av1_highbd_jnt_convolve_y = av1_highbd_jnt_convolve_y_avx2; av1_highbd_warp_affine = av1_highbd_warp_affine_c; if (flags & HAS_SSE4_1) av1_highbd_warp_affine = av1_highbd_warp_affine_sse4_1; av1_highbd_wiener_convolve_add_src = av1_highbd_wiener_convolve_add_src_c; @@ -524,18 +557,8 @@ static void setup_rtcd_internal(void) av1_inv_txfm_add = av1_inv_txfm_add_c; if (flags & HAS_SSSE3) av1_inv_txfm_add = av1_inv_txfm_add_ssse3; if (flags & HAS_AVX2) av1_inv_txfm_add = av1_inv_txfm_add_avx2; - av1_jnt_convolve_2d = av1_jnt_convolve_2d_c; - if (flags & HAS_SSSE3) av1_jnt_convolve_2d = av1_jnt_convolve_2d_ssse3; - if (flags & HAS_AVX2) av1_jnt_convolve_2d = av1_jnt_convolve_2d_avx2; - av1_jnt_convolve_2d_copy = av1_jnt_convolve_2d_copy_c; - if (flags & HAS_SSE2) av1_jnt_convolve_2d_copy = av1_jnt_convolve_2d_copy_sse2; - if (flags & HAS_AVX2) av1_jnt_convolve_2d_copy = av1_jnt_convolve_2d_copy_avx2; - av1_jnt_convolve_x = av1_jnt_convolve_x_c; - if (flags & HAS_SSE2) av1_jnt_convolve_x = av1_jnt_convolve_x_sse2; - if (flags & HAS_AVX2) av1_jnt_convolve_x = av1_jnt_convolve_x_avx2; - av1_jnt_convolve_y = av1_jnt_convolve_y_c; - if (flags & HAS_SSE2) av1_jnt_convolve_y = av1_jnt_convolve_y_sse2; - if (flags & HAS_AVX2) av1_jnt_convolve_y = av1_jnt_convolve_y_avx2; + av1_round_shift_array = av1_round_shift_array_c; + if (flags & HAS_SSE4_1) av1_round_shift_array = av1_round_shift_array_sse4_1; av1_selfguided_restoration = av1_selfguided_restoration_c; if (flags & HAS_SSE4_1) av1_selfguided_restoration = av1_selfguided_restoration_sse4_1; if (flags & HAS_AVX2) av1_selfguided_restoration = av1_selfguided_restoration_avx2; diff --git a/media/libaom/config/linux/x64/config/aom_config.asm b/media/libaom/config/linux/x64/config/aom_config.asm index 4ab2dedb42..294d0294b6 100644 --- a/media/libaom/config/linux/x64/config/aom_config.asm +++ b/media/libaom/config/linux/x64/config/aom_config.asm @@ -1,5 +1,5 @@ ; -; Copyright (c) 2018, Alliance for Open Media. All rights reserved +; Copyright (c) 2019, Alliance for Open Media. All rights reserved ; ; This source code is subject to the terms of the BSD 2 Clause License and ; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License @@ -22,18 +22,17 @@ CONFIG_AV1_ENCODER equ 0 CONFIG_BIG_ENDIAN equ 0 CONFIG_BITSTREAM_DEBUG equ 0 CONFIG_COEFFICIENT_RANGE_CHECKING equ 0 -CONFIG_COLLECT_INTER_MODE_RD_STATS equ 0 +CONFIG_COLLECT_PARTITION_STATS equ 0 CONFIG_COLLECT_RD_STATS equ 0 CONFIG_DEBUG equ 0 CONFIG_DENOISE equ 1 +CONFIG_DISABLE_FULL_PIXEL_SPLIT_8X8 equ 1 CONFIG_DIST_8X8 equ 0 CONFIG_ENTROPY_STATS equ 0 CONFIG_FILEOPTIONS equ 1 -CONFIG_FIX_GF_LENGTH equ 1 CONFIG_FP_MB_STATS equ 0 CONFIG_GCC equ 1 CONFIG_GCOV equ 0 -CONFIG_GLOBAL_MOTION_SEARCH equ 1 CONFIG_GPROF equ 0 CONFIG_INSPECTION equ 0 CONFIG_INTERNAL_STATS equ 0 @@ -44,15 +43,16 @@ CONFIG_MAX_DECODE_PROFILE equ 2 CONFIG_MISMATCH_DEBUG equ 0 CONFIG_MULTITHREAD equ 1 CONFIG_NORMAL_TILE_MODE equ 0 +CONFIG_ONE_PASS_SVM equ 0 CONFIG_OS_SUPPORT equ 1 CONFIG_PIC equ 0 CONFIG_RD_DEBUG equ 0 -CONFIG_REDUCED_ENCODER_BORDER equ 0 CONFIG_RUNTIME_CPU_DETECT equ 1 CONFIG_SHARED equ 0 CONFIG_SHARP_SETTINGS equ 0 CONFIG_SIZE_LIMIT equ 0 CONFIG_SPATIAL_RESAMPLING equ 1 +CONFIG_SPEED_STATS equ 0 CONFIG_STATIC equ 1 CONFIG_WEBM_IO equ 0 DECODE_HEIGHT_LIMIT equ 0 diff --git a/media/libaom/config/linux/x64/config/aom_config.h b/media/libaom/config/linux/x64/config/aom_config.h index 9a0be7cd11..ffff5abe6a 100644 --- a/media/libaom/config/linux/x64/config/aom_config.h +++ b/media/libaom/config/linux/x64/config/aom_config.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, Alliance for Open Media. All rights reserved + * Copyright (c) 2019, Alliance for Open Media. All rights reserved * * This source code is subject to the terms of the BSD 2 Clause License and * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License @@ -24,18 +24,17 @@ #define CONFIG_BIG_ENDIAN 0 #define CONFIG_BITSTREAM_DEBUG 0 #define CONFIG_COEFFICIENT_RANGE_CHECKING 0 -#define CONFIG_COLLECT_INTER_MODE_RD_STATS 0 +#define CONFIG_COLLECT_PARTITION_STATS 0 #define CONFIG_COLLECT_RD_STATS 0 #define CONFIG_DEBUG 0 #define CONFIG_DENOISE 1 +#define CONFIG_DISABLE_FULL_PIXEL_SPLIT_8X8 1 #define CONFIG_DIST_8X8 0 #define CONFIG_ENTROPY_STATS 0 #define CONFIG_FILEOPTIONS 1 -#define CONFIG_FIX_GF_LENGTH 1 #define CONFIG_FP_MB_STATS 0 #define CONFIG_GCC 1 #define CONFIG_GCOV 0 -#define CONFIG_GLOBAL_MOTION_SEARCH 1 #define CONFIG_GPROF 0 #define CONFIG_INSPECTION 0 #define CONFIG_INTERNAL_STATS 0 @@ -46,15 +45,16 @@ #define CONFIG_MISMATCH_DEBUG 0 #define CONFIG_MULTITHREAD 1 #define CONFIG_NORMAL_TILE_MODE 0 +#define CONFIG_ONE_PASS_SVM 0 #define CONFIG_OS_SUPPORT 1 #define CONFIG_PIC 0 #define CONFIG_RD_DEBUG 0 -#define CONFIG_REDUCED_ENCODER_BORDER 0 #define CONFIG_RUNTIME_CPU_DETECT 1 #define CONFIG_SHARED 0 #define CONFIG_SHARP_SETTINGS 0 #define CONFIG_SIZE_LIMIT 0 #define CONFIG_SPATIAL_RESAMPLING 1 +#define CONFIG_SPEED_STATS 0 #define CONFIG_STATIC 1 #define CONFIG_WEBM_IO 0 #define DECODE_HEIGHT_LIMIT 0 @@ -76,7 +76,5 @@ #define HAVE_SSSE3 1 #define HAVE_VSX 0 #define HAVE_WXWIDGETS 0 -#define INCLUDE_INSTALL_DIR INSTALLDIR/include #define INLINE inline -#define LIB_INSTALL_DIR INSTALLDIR/lib #endif /* AOM_CONFIG_H_ */ diff --git a/media/libaom/config/linux/x64/config/aom_dsp_rtcd.h b/media/libaom/config/linux/x64/config/aom_dsp_rtcd.h index 2856d4ede7..5826faff95 100644 --- a/media/libaom/config/linux/x64/config/aom_dsp_rtcd.h +++ b/media/libaom/config/linux/x64/config/aom_dsp_rtcd.h @@ -472,7 +472,9 @@ void aom_h_predictor_8x8_sse2(uint8_t *dst, ptrdiff_t y_stride, const uint8_t *a #define aom_h_predictor_8x8 aom_h_predictor_8x8_sse2 void aom_highbd_blend_a64_d16_mask_c(uint8_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0, uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride, const uint8_t *mask, uint32_t mask_stride, int w, int h, int subx, int suby, ConvolveParams *conv_params, const int bd); -#define aom_highbd_blend_a64_d16_mask aom_highbd_blend_a64_d16_mask_c +void aom_highbd_blend_a64_d16_mask_sse4_1(uint8_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0, uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride, const uint8_t *mask, uint32_t mask_stride, int w, int h, int subx, int suby, ConvolveParams *conv_params, const int bd); +void aom_highbd_blend_a64_d16_mask_avx2(uint8_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0, uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride, const uint8_t *mask, uint32_t mask_stride, int w, int h, int subx, int suby, ConvolveParams *conv_params, const int bd); +RTCD_EXTERN void (*aom_highbd_blend_a64_d16_mask)(uint8_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0, uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride, const uint8_t *mask, uint32_t mask_stride, int w, int h, int subx, int suby, ConvolveParams *conv_params, const int bd); void aom_highbd_blend_a64_hmask_c(uint8_t *dst, uint32_t dst_stride, const uint8_t *src0, uint32_t src0_stride, const uint8_t *src1, uint32_t src1_stride, const uint8_t *mask, int w, int h, int bd); void aom_highbd_blend_a64_hmask_sse4_1(uint8_t *dst, uint32_t dst_stride, const uint8_t *src0, uint32_t src0_stride, const uint8_t *src1, uint32_t src1_stride, const uint8_t *mask, int w, int h, int bd); @@ -1711,10 +1713,6 @@ void aom_v_predictor_8x8_c(uint8_t *dst, ptrdiff_t y_stride, const uint8_t *abov void aom_v_predictor_8x8_sse2(uint8_t *dst, ptrdiff_t y_stride, const uint8_t *above, const uint8_t *left); #define aom_v_predictor_8x8 aom_v_predictor_8x8_sse2 -void av1_round_shift_array_c(int32_t *arr, int size, int bit); -void av1_round_shift_array_sse4_1(int32_t *arr, int size, int bit); -RTCD_EXTERN void (*av1_round_shift_array)(int32_t *arr, int size, int bit); - void aom_dsp_rtcd(void); #ifdef RTCD_C @@ -1788,6 +1786,9 @@ static void setup_rtcd_internal(void) if (flags & HAS_AVX2) aom_dc_top_predictor_64x64 = aom_dc_top_predictor_64x64_avx2; aom_h_predictor_32x32 = aom_h_predictor_32x32_sse2; if (flags & HAS_AVX2) aom_h_predictor_32x32 = aom_h_predictor_32x32_avx2; + aom_highbd_blend_a64_d16_mask = aom_highbd_blend_a64_d16_mask_c; + if (flags & HAS_SSE4_1) aom_highbd_blend_a64_d16_mask = aom_highbd_blend_a64_d16_mask_sse4_1; + if (flags & HAS_AVX2) aom_highbd_blend_a64_d16_mask = aom_highbd_blend_a64_d16_mask_avx2; aom_highbd_blend_a64_hmask = aom_highbd_blend_a64_hmask_c; if (flags & HAS_SSE4_1) aom_highbd_blend_a64_hmask = aom_highbd_blend_a64_hmask_sse4_1; aom_highbd_blend_a64_mask = aom_highbd_blend_a64_mask_c; @@ -1989,8 +1990,6 @@ static void setup_rtcd_internal(void) if (flags & HAS_AVX2) aom_v_predictor_64x32 = aom_v_predictor_64x32_avx2; aom_v_predictor_64x64 = aom_v_predictor_64x64_sse2; if (flags & HAS_AVX2) aom_v_predictor_64x64 = aom_v_predictor_64x64_avx2; - av1_round_shift_array = av1_round_shift_array_c; - if (flags & HAS_SSE4_1) av1_round_shift_array = av1_round_shift_array_sse4_1; } #endif diff --git a/media/libaom/config/linux/x64/config/aom_scale_rtcd.h b/media/libaom/config/linux/x64/config/aom_scale_rtcd.h index 6b56795cd4..b6e814905b 100644 --- a/media/libaom/config/linux/x64/config/aom_scale_rtcd.h +++ b/media/libaom/config/linux/x64/config/aom_scale_rtcd.h @@ -59,13 +59,22 @@ void aom_yv12_copy_y_c(const struct yv12_buffer_config *src_ybc, struct yv12_buf void aom_yv12_extend_frame_borders_c(struct yv12_buffer_config *ybf, const int num_planes); #define aom_yv12_extend_frame_borders aom_yv12_extend_frame_borders_c -void aom_yv12_partial_copy_u_c(const struct yv12_buffer_config *src_bc, struct yv12_buffer_config *dst_bc, int hstart, int hend, int vstart, int vend); +void aom_yv12_partial_coloc_copy_u_c(const struct yv12_buffer_config *src_bc, struct yv12_buffer_config *dst_bc, int hstart, int hend, int vstart, int vend); +#define aom_yv12_partial_coloc_copy_u aom_yv12_partial_coloc_copy_u_c + +void aom_yv12_partial_coloc_copy_v_c(const struct yv12_buffer_config *src_bc, struct yv12_buffer_config *dst_bc, int hstart, int hend, int vstart, int vend); +#define aom_yv12_partial_coloc_copy_v aom_yv12_partial_coloc_copy_v_c + +void aom_yv12_partial_coloc_copy_y_c(const struct yv12_buffer_config *src_ybc, struct yv12_buffer_config *dst_ybc, int hstart, int hend, int vstart, int vend); +#define aom_yv12_partial_coloc_copy_y aom_yv12_partial_coloc_copy_y_c + +void aom_yv12_partial_copy_u_c(const struct yv12_buffer_config *src_bc, int hstart1, int hend1, int vstart1, int vend1, struct yv12_buffer_config *dst_bc, int hstart2, int vstart2); #define aom_yv12_partial_copy_u aom_yv12_partial_copy_u_c -void aom_yv12_partial_copy_v_c(const struct yv12_buffer_config *src_bc, struct yv12_buffer_config *dst_bc, int hstart, int hend, int vstart, int vend); +void aom_yv12_partial_copy_v_c(const struct yv12_buffer_config *src_bc, int hstart1, int hend1, int vstart1, int vend1, struct yv12_buffer_config *dst_bc, int hstart2, int vstart2); #define aom_yv12_partial_copy_v aom_yv12_partial_copy_v_c -void aom_yv12_partial_copy_y_c(const struct yv12_buffer_config *src_ybc, struct yv12_buffer_config *dst_ybc, int hstart, int hend, int vstart, int vend); +void aom_yv12_partial_copy_y_c(const struct yv12_buffer_config *src_ybc, int hstart1, int hend1, int vstart1, int vend1, struct yv12_buffer_config *dst_ybc, int hstart2, int vstart2); #define aom_yv12_partial_copy_y aom_yv12_partial_copy_y_c void aom_scale_rtcd(void); diff --git a/media/libaom/config/linux/x64/config/av1_rtcd.h b/media/libaom/config/linux/x64/config/av1_rtcd.h index d273182086..80bacb8dd5 100644 --- a/media/libaom/config/linux/x64/config/av1_rtcd.h +++ b/media/libaom/config/linux/x64/config/av1_rtcd.h @@ -31,6 +31,8 @@ struct txfm_param; struct aom_variance_vtable; struct search_site_config; struct yv12_buffer_config; +struct NN_CONFIG; +typedef struct NN_CONFIG NN_CONFIG; /* Function pointers return by CfL functions */ typedef void (*cfl_subsample_lbd_fn)(const uint8_t *input, int input_stride, @@ -99,14 +101,38 @@ void av1_convolve_y_sr_sse2(const uint8_t *src, int src_stride, uint8_t *dst, in void av1_convolve_y_sr_avx2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); RTCD_EXTERN void (*av1_convolve_y_sr)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_2d_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_2d_sse2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_2d_ssse3(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_2d_avx2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +RTCD_EXTERN void (*av1_dist_wtd_convolve_2d)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); + +void av1_dist_wtd_convolve_2d_copy_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_2d_copy_sse2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_2d_copy_avx2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +RTCD_EXTERN void (*av1_dist_wtd_convolve_2d_copy)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); + +void av1_dist_wtd_convolve_x_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_x_sse2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_x_avx2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +RTCD_EXTERN void (*av1_dist_wtd_convolve_x)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); + +void av1_dist_wtd_convolve_y_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_y_sse2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_y_avx2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +RTCD_EXTERN void (*av1_dist_wtd_convolve_y)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); + void av1_dr_prediction_z1_c(uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_above, int dx, int dy); -#define av1_dr_prediction_z1 av1_dr_prediction_z1_c +void av1_dr_prediction_z1_avx2(uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_above, int dx, int dy); +RTCD_EXTERN void (*av1_dr_prediction_z1)(uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_above, int dx, int dy); void av1_dr_prediction_z2_c(uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_above, int upsample_left, int dx, int dy); -#define av1_dr_prediction_z2 av1_dr_prediction_z2_c +void av1_dr_prediction_z2_avx2(uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_above, int upsample_left, int dx, int dy); +RTCD_EXTERN void (*av1_dr_prediction_z2)(uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_above, int upsample_left, int dx, int dy); void av1_dr_prediction_z3_c(uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_left, int dx, int dy); -#define av1_dr_prediction_z3 av1_dr_prediction_z3_c +void av1_dr_prediction_z3_avx2(uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_left, int dx, int dy); +RTCD_EXTERN void (*av1_dr_prediction_z3)(uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_left, int dx, int dy); void av1_filter_intra_edge_c(uint8_t *p, int sz, int strength); void av1_filter_intra_edge_sse4_1(uint8_t *p, int sz, int strength); @@ -166,40 +192,61 @@ void av1_highbd_convolve_y_sr_ssse3(const uint16_t *src, int src_stride, uint16_ void av1_highbd_convolve_y_sr_avx2(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); RTCD_EXTERN void (*av1_highbd_convolve_y_sr)(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +void av1_highbd_dist_wtd_convolve_2d_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +void av1_highbd_dist_wtd_convolve_2d_sse4_1(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +void av1_highbd_dist_wtd_convolve_2d_avx2(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +RTCD_EXTERN void (*av1_highbd_dist_wtd_convolve_2d)(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); + +void av1_highbd_dist_wtd_convolve_2d_copy_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +void av1_highbd_dist_wtd_convolve_2d_copy_sse4_1(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +void av1_highbd_dist_wtd_convolve_2d_copy_avx2(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +RTCD_EXTERN void (*av1_highbd_dist_wtd_convolve_2d_copy)(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); + +void av1_highbd_dist_wtd_convolve_x_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +void av1_highbd_dist_wtd_convolve_x_sse4_1(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +void av1_highbd_dist_wtd_convolve_x_avx2(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +RTCD_EXTERN void (*av1_highbd_dist_wtd_convolve_x)(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); + +void av1_highbd_dist_wtd_convolve_y_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +void av1_highbd_dist_wtd_convolve_y_sse4_1(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +void av1_highbd_dist_wtd_convolve_y_avx2(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +RTCD_EXTERN void (*av1_highbd_dist_wtd_convolve_y)(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); + void av1_highbd_dr_prediction_z1_c(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_above, int dx, int dy, int bd); -#define av1_highbd_dr_prediction_z1 av1_highbd_dr_prediction_z1_c +void av1_highbd_dr_prediction_z1_avx2(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_above, int dx, int dy, int bd); +RTCD_EXTERN void (*av1_highbd_dr_prediction_z1)(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_above, int dx, int dy, int bd); void av1_highbd_dr_prediction_z2_c(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_above, int upsample_left, int dx, int dy, int bd); #define av1_highbd_dr_prediction_z2 av1_highbd_dr_prediction_z2_c void av1_highbd_dr_prediction_z3_c(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_left, int dx, int dy, int bd); -#define av1_highbd_dr_prediction_z3 av1_highbd_dr_prediction_z3_c +void av1_highbd_dr_prediction_z3_avx2(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_left, int dx, int dy, int bd); +RTCD_EXTERN void (*av1_highbd_dr_prediction_z3)(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_left, int dx, int dy, int bd); void av1_highbd_inv_txfm_add_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); void av1_highbd_inv_txfm_add_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); void av1_highbd_inv_txfm_add_avx2(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); RTCD_EXTERN void (*av1_highbd_inv_txfm_add)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -void av1_highbd_inv_txfm_add_16x16_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -void av1_highbd_inv_txfm_add_16x16_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -RTCD_EXTERN void (*av1_highbd_inv_txfm_add_16x16)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +void av1_highbd_inv_txfm_add_16x4_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +void av1_highbd_inv_txfm_add_16x4_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +RTCD_EXTERN void (*av1_highbd_inv_txfm_add_16x4)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -void av1_highbd_inv_txfm_add_16x8_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -void av1_highbd_inv_txfm_add_16x8_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -RTCD_EXTERN void (*av1_highbd_inv_txfm_add_16x8)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); - -void av1_highbd_inv_txfm_add_32x32_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -void av1_highbd_inv_txfm_add_32x32_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -void av1_highbd_inv_txfm_add_32x32_avx2(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -RTCD_EXTERN void (*av1_highbd_inv_txfm_add_32x32)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +void av1_highbd_inv_txfm_add_4x16_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +void av1_highbd_inv_txfm_add_4x16_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +RTCD_EXTERN void (*av1_highbd_inv_txfm_add_4x16)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); void av1_highbd_inv_txfm_add_4x4_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); void av1_highbd_inv_txfm_add_4x4_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); RTCD_EXTERN void (*av1_highbd_inv_txfm_add_4x4)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -void av1_highbd_inv_txfm_add_8x16_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -void av1_highbd_inv_txfm_add_8x16_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -RTCD_EXTERN void (*av1_highbd_inv_txfm_add_8x16)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +void av1_highbd_inv_txfm_add_4x8_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +void av1_highbd_inv_txfm_add_4x8_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +RTCD_EXTERN void (*av1_highbd_inv_txfm_add_4x8)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); + +void av1_highbd_inv_txfm_add_8x4_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +void av1_highbd_inv_txfm_add_8x4_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +RTCD_EXTERN void (*av1_highbd_inv_txfm_add_8x4)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); void av1_highbd_inv_txfm_add_8x8_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); void av1_highbd_inv_txfm_add_8x8_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); @@ -211,26 +258,6 @@ void av1_highbd_iwht4x4_16_add_c(const tran_low_t *input, uint8_t *dest, int des void av1_highbd_iwht4x4_1_add_c(const tran_low_t *input, uint8_t *dest, int dest_stride, int bd); #define av1_highbd_iwht4x4_1_add av1_highbd_iwht4x4_1_add_c -void av1_highbd_jnt_convolve_2d_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -void av1_highbd_jnt_convolve_2d_sse4_1(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -void av1_highbd_jnt_convolve_2d_avx2(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -RTCD_EXTERN void (*av1_highbd_jnt_convolve_2d)(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); - -void av1_highbd_jnt_convolve_2d_copy_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -void av1_highbd_jnt_convolve_2d_copy_sse4_1(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -void av1_highbd_jnt_convolve_2d_copy_avx2(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -RTCD_EXTERN void (*av1_highbd_jnt_convolve_2d_copy)(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); - -void av1_highbd_jnt_convolve_x_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -void av1_highbd_jnt_convolve_x_sse4_1(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -void av1_highbd_jnt_convolve_x_avx2(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -RTCD_EXTERN void (*av1_highbd_jnt_convolve_x)(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); - -void av1_highbd_jnt_convolve_y_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -void av1_highbd_jnt_convolve_y_sse4_1(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -void av1_highbd_jnt_convolve_y_avx2(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -RTCD_EXTERN void (*av1_highbd_jnt_convolve_y)(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); - void av1_highbd_warp_affine_c(const int32_t *mat, const uint16_t *ref, int width, int height, int stride, uint16_t *pred, int p_col, int p_row, int p_width, int p_height, int p_stride, int subsampling_x, int subsampling_y, int bd, ConvolveParams *conv_params, int16_t alpha, int16_t beta, int16_t gamma, int16_t delta); void av1_highbd_warp_affine_sse4_1(const int32_t *mat, const uint16_t *ref, int width, int height, int stride, uint16_t *pred, int p_col, int p_row, int p_width, int p_height, int p_stride, int subsampling_x, int subsampling_y, int bd, ConvolveParams *conv_params, int16_t alpha, int16_t beta, int16_t gamma, int16_t delta); RTCD_EXTERN void (*av1_highbd_warp_affine)(const int32_t *mat, const uint16_t *ref, int width, int height, int stride, uint16_t *pred, int p_col, int p_row, int p_width, int p_height, int p_stride, int subsampling_x, int subsampling_y, int bd, ConvolveParams *conv_params, int16_t alpha, int16_t beta, int16_t gamma, int16_t delta); @@ -304,25 +331,9 @@ void av1_inv_txfm_add_ssse3(const tran_low_t *dqcoeff, uint8_t *dst, int stride, void av1_inv_txfm_add_avx2(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); RTCD_EXTERN void (*av1_inv_txfm_add)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -void av1_jnt_convolve_2d_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -void av1_jnt_convolve_2d_ssse3(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -void av1_jnt_convolve_2d_avx2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -RTCD_EXTERN void (*av1_jnt_convolve_2d)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); - -void av1_jnt_convolve_2d_copy_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -void av1_jnt_convolve_2d_copy_sse2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -void av1_jnt_convolve_2d_copy_avx2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -RTCD_EXTERN void (*av1_jnt_convolve_2d_copy)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); - -void av1_jnt_convolve_x_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -void av1_jnt_convolve_x_sse2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -void av1_jnt_convolve_x_avx2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -RTCD_EXTERN void (*av1_jnt_convolve_x)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); - -void av1_jnt_convolve_y_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -void av1_jnt_convolve_y_sse2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -void av1_jnt_convolve_y_avx2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -RTCD_EXTERN void (*av1_jnt_convolve_y)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_round_shift_array_c(int32_t *arr, int size, int bit); +void av1_round_shift_array_sse4_1(int32_t *arr, int size, int bit); +RTCD_EXTERN void (*av1_round_shift_array)(int32_t *arr, int size, int bit); int av1_selfguided_restoration_c(const uint8_t *dgd8, int width, int height, int dgd_stride, int32_t *flt0, int32_t *flt1, int flt_stride, @@ -354,12 +365,12 @@ void av1_wiener_convolve_add_src_sse2(const uint8_t *src, ptrdiff_t src_stride, void av1_wiener_convolve_add_src_avx2(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, const int16_t *filter_y, int y_step_q4, int w, int h, const ConvolveParams *conv_params); RTCD_EXTERN void (*av1_wiener_convolve_add_src)(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, const int16_t *filter_y, int y_step_q4, int w, int h, const ConvolveParams *conv_params); -void cdef_filter_block_c(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int max, int coeff_shift); -void cdef_filter_block_sse2(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int max, int coeff_shift); -void cdef_filter_block_ssse3(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int max, int coeff_shift); -void cdef_filter_block_sse4_1(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int max, int coeff_shift); -void cdef_filter_block_avx2(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int max, int coeff_shift); -RTCD_EXTERN void (*cdef_filter_block)(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int max, int coeff_shift); +void cdef_filter_block_c(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int coeff_shift); +void cdef_filter_block_sse2(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int coeff_shift); +void cdef_filter_block_ssse3(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int coeff_shift); +void cdef_filter_block_sse4_1(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int coeff_shift); +void cdef_filter_block_avx2(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int coeff_shift); +RTCD_EXTERN void (*cdef_filter_block)(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int coeff_shift); int cdef_find_dir_c(const uint16_t *img, int stride, int32_t *var, int coeff_shift); int cdef_find_dir_sse2(const uint16_t *img, int stride, int32_t *var, int coeff_shift); @@ -461,6 +472,21 @@ static void setup_rtcd_internal(void) if (flags & HAS_AVX2) av1_convolve_x_sr = av1_convolve_x_sr_avx2; av1_convolve_y_sr = av1_convolve_y_sr_sse2; if (flags & HAS_AVX2) av1_convolve_y_sr = av1_convolve_y_sr_avx2; + av1_dist_wtd_convolve_2d = av1_dist_wtd_convolve_2d_sse2; + if (flags & HAS_SSSE3) av1_dist_wtd_convolve_2d = av1_dist_wtd_convolve_2d_ssse3; + if (flags & HAS_AVX2) av1_dist_wtd_convolve_2d = av1_dist_wtd_convolve_2d_avx2; + av1_dist_wtd_convolve_2d_copy = av1_dist_wtd_convolve_2d_copy_sse2; + if (flags & HAS_AVX2) av1_dist_wtd_convolve_2d_copy = av1_dist_wtd_convolve_2d_copy_avx2; + av1_dist_wtd_convolve_x = av1_dist_wtd_convolve_x_sse2; + if (flags & HAS_AVX2) av1_dist_wtd_convolve_x = av1_dist_wtd_convolve_x_avx2; + av1_dist_wtd_convolve_y = av1_dist_wtd_convolve_y_sse2; + if (flags & HAS_AVX2) av1_dist_wtd_convolve_y = av1_dist_wtd_convolve_y_avx2; + av1_dr_prediction_z1 = av1_dr_prediction_z1_c; + if (flags & HAS_AVX2) av1_dr_prediction_z1 = av1_dr_prediction_z1_avx2; + av1_dr_prediction_z2 = av1_dr_prediction_z2_c; + if (flags & HAS_AVX2) av1_dr_prediction_z2 = av1_dr_prediction_z2_avx2; + av1_dr_prediction_z3 = av1_dr_prediction_z3_c; + if (flags & HAS_AVX2) av1_dr_prediction_z3 = av1_dr_prediction_z3_avx2; av1_filter_intra_edge = av1_filter_intra_edge_c; if (flags & HAS_SSE4_1) av1_filter_intra_edge = av1_filter_intra_edge_sse4_1; av1_filter_intra_edge_high = av1_filter_intra_edge_high_c; @@ -482,34 +508,37 @@ static void setup_rtcd_internal(void) av1_highbd_convolve_y_sr = av1_highbd_convolve_y_sr_c; if (flags & HAS_SSSE3) av1_highbd_convolve_y_sr = av1_highbd_convolve_y_sr_ssse3; if (flags & HAS_AVX2) av1_highbd_convolve_y_sr = av1_highbd_convolve_y_sr_avx2; + av1_highbd_dist_wtd_convolve_2d = av1_highbd_dist_wtd_convolve_2d_c; + if (flags & HAS_SSE4_1) av1_highbd_dist_wtd_convolve_2d = av1_highbd_dist_wtd_convolve_2d_sse4_1; + if (flags & HAS_AVX2) av1_highbd_dist_wtd_convolve_2d = av1_highbd_dist_wtd_convolve_2d_avx2; + av1_highbd_dist_wtd_convolve_2d_copy = av1_highbd_dist_wtd_convolve_2d_copy_c; + if (flags & HAS_SSE4_1) av1_highbd_dist_wtd_convolve_2d_copy = av1_highbd_dist_wtd_convolve_2d_copy_sse4_1; + if (flags & HAS_AVX2) av1_highbd_dist_wtd_convolve_2d_copy = av1_highbd_dist_wtd_convolve_2d_copy_avx2; + av1_highbd_dist_wtd_convolve_x = av1_highbd_dist_wtd_convolve_x_c; + if (flags & HAS_SSE4_1) av1_highbd_dist_wtd_convolve_x = av1_highbd_dist_wtd_convolve_x_sse4_1; + if (flags & HAS_AVX2) av1_highbd_dist_wtd_convolve_x = av1_highbd_dist_wtd_convolve_x_avx2; + av1_highbd_dist_wtd_convolve_y = av1_highbd_dist_wtd_convolve_y_c; + if (flags & HAS_SSE4_1) av1_highbd_dist_wtd_convolve_y = av1_highbd_dist_wtd_convolve_y_sse4_1; + if (flags & HAS_AVX2) av1_highbd_dist_wtd_convolve_y = av1_highbd_dist_wtd_convolve_y_avx2; + av1_highbd_dr_prediction_z1 = av1_highbd_dr_prediction_z1_c; + if (flags & HAS_AVX2) av1_highbd_dr_prediction_z1 = av1_highbd_dr_prediction_z1_avx2; + av1_highbd_dr_prediction_z3 = av1_highbd_dr_prediction_z3_c; + if (flags & HAS_AVX2) av1_highbd_dr_prediction_z3 = av1_highbd_dr_prediction_z3_avx2; av1_highbd_inv_txfm_add = av1_highbd_inv_txfm_add_c; if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add = av1_highbd_inv_txfm_add_sse4_1; if (flags & HAS_AVX2) av1_highbd_inv_txfm_add = av1_highbd_inv_txfm_add_avx2; - av1_highbd_inv_txfm_add_16x16 = av1_highbd_inv_txfm_add_16x16_c; - if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add_16x16 = av1_highbd_inv_txfm_add_16x16_sse4_1; - av1_highbd_inv_txfm_add_16x8 = av1_highbd_inv_txfm_add_16x8_c; - if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add_16x8 = av1_highbd_inv_txfm_add_16x8_sse4_1; - av1_highbd_inv_txfm_add_32x32 = av1_highbd_inv_txfm_add_32x32_c; - if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add_32x32 = av1_highbd_inv_txfm_add_32x32_sse4_1; - if (flags & HAS_AVX2) av1_highbd_inv_txfm_add_32x32 = av1_highbd_inv_txfm_add_32x32_avx2; + av1_highbd_inv_txfm_add_16x4 = av1_highbd_inv_txfm_add_16x4_c; + if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add_16x4 = av1_highbd_inv_txfm_add_16x4_sse4_1; + av1_highbd_inv_txfm_add_4x16 = av1_highbd_inv_txfm_add_4x16_c; + if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add_4x16 = av1_highbd_inv_txfm_add_4x16_sse4_1; av1_highbd_inv_txfm_add_4x4 = av1_highbd_inv_txfm_add_4x4_c; if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add_4x4 = av1_highbd_inv_txfm_add_4x4_sse4_1; - av1_highbd_inv_txfm_add_8x16 = av1_highbd_inv_txfm_add_8x16_c; - if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add_8x16 = av1_highbd_inv_txfm_add_8x16_sse4_1; + av1_highbd_inv_txfm_add_4x8 = av1_highbd_inv_txfm_add_4x8_c; + if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add_4x8 = av1_highbd_inv_txfm_add_4x8_sse4_1; + av1_highbd_inv_txfm_add_8x4 = av1_highbd_inv_txfm_add_8x4_c; + if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add_8x4 = av1_highbd_inv_txfm_add_8x4_sse4_1; av1_highbd_inv_txfm_add_8x8 = av1_highbd_inv_txfm_add_8x8_c; if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add_8x8 = av1_highbd_inv_txfm_add_8x8_sse4_1; - av1_highbd_jnt_convolve_2d = av1_highbd_jnt_convolve_2d_c; - if (flags & HAS_SSE4_1) av1_highbd_jnt_convolve_2d = av1_highbd_jnt_convolve_2d_sse4_1; - if (flags & HAS_AVX2) av1_highbd_jnt_convolve_2d = av1_highbd_jnt_convolve_2d_avx2; - av1_highbd_jnt_convolve_2d_copy = av1_highbd_jnt_convolve_2d_copy_c; - if (flags & HAS_SSE4_1) av1_highbd_jnt_convolve_2d_copy = av1_highbd_jnt_convolve_2d_copy_sse4_1; - if (flags & HAS_AVX2) av1_highbd_jnt_convolve_2d_copy = av1_highbd_jnt_convolve_2d_copy_avx2; - av1_highbd_jnt_convolve_x = av1_highbd_jnt_convolve_x_c; - if (flags & HAS_SSE4_1) av1_highbd_jnt_convolve_x = av1_highbd_jnt_convolve_x_sse4_1; - if (flags & HAS_AVX2) av1_highbd_jnt_convolve_x = av1_highbd_jnt_convolve_x_avx2; - av1_highbd_jnt_convolve_y = av1_highbd_jnt_convolve_y_c; - if (flags & HAS_SSE4_1) av1_highbd_jnt_convolve_y = av1_highbd_jnt_convolve_y_sse4_1; - if (flags & HAS_AVX2) av1_highbd_jnt_convolve_y = av1_highbd_jnt_convolve_y_avx2; av1_highbd_warp_affine = av1_highbd_warp_affine_c; if (flags & HAS_SSE4_1) av1_highbd_warp_affine = av1_highbd_warp_affine_sse4_1; av1_highbd_wiener_convolve_add_src = av1_highbd_wiener_convolve_add_src_c; @@ -522,15 +551,8 @@ static void setup_rtcd_internal(void) av1_inv_txfm_add = av1_inv_txfm_add_c; if (flags & HAS_SSSE3) av1_inv_txfm_add = av1_inv_txfm_add_ssse3; if (flags & HAS_AVX2) av1_inv_txfm_add = av1_inv_txfm_add_avx2; - av1_jnt_convolve_2d = av1_jnt_convolve_2d_c; - if (flags & HAS_SSSE3) av1_jnt_convolve_2d = av1_jnt_convolve_2d_ssse3; - if (flags & HAS_AVX2) av1_jnt_convolve_2d = av1_jnt_convolve_2d_avx2; - av1_jnt_convolve_2d_copy = av1_jnt_convolve_2d_copy_sse2; - if (flags & HAS_AVX2) av1_jnt_convolve_2d_copy = av1_jnt_convolve_2d_copy_avx2; - av1_jnt_convolve_x = av1_jnt_convolve_x_sse2; - if (flags & HAS_AVX2) av1_jnt_convolve_x = av1_jnt_convolve_x_avx2; - av1_jnt_convolve_y = av1_jnt_convolve_y_sse2; - if (flags & HAS_AVX2) av1_jnt_convolve_y = av1_jnt_convolve_y_avx2; + av1_round_shift_array = av1_round_shift_array_c; + if (flags & HAS_SSE4_1) av1_round_shift_array = av1_round_shift_array_sse4_1; av1_selfguided_restoration = av1_selfguided_restoration_c; if (flags & HAS_SSE4_1) av1_selfguided_restoration = av1_selfguided_restoration_sse4_1; if (flags & HAS_AVX2) av1_selfguided_restoration = av1_selfguided_restoration_avx2; diff --git a/media/libaom/config/mac/x64/config/aom_config.asm b/media/libaom/config/mac/x64/config/aom_config.asm index 4ab2dedb42..294d0294b6 100644 --- a/media/libaom/config/mac/x64/config/aom_config.asm +++ b/media/libaom/config/mac/x64/config/aom_config.asm @@ -1,5 +1,5 @@ ; -; Copyright (c) 2018, Alliance for Open Media. All rights reserved +; Copyright (c) 2019, Alliance for Open Media. All rights reserved ; ; This source code is subject to the terms of the BSD 2 Clause License and ; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License @@ -22,18 +22,17 @@ CONFIG_AV1_ENCODER equ 0 CONFIG_BIG_ENDIAN equ 0 CONFIG_BITSTREAM_DEBUG equ 0 CONFIG_COEFFICIENT_RANGE_CHECKING equ 0 -CONFIG_COLLECT_INTER_MODE_RD_STATS equ 0 +CONFIG_COLLECT_PARTITION_STATS equ 0 CONFIG_COLLECT_RD_STATS equ 0 CONFIG_DEBUG equ 0 CONFIG_DENOISE equ 1 +CONFIG_DISABLE_FULL_PIXEL_SPLIT_8X8 equ 1 CONFIG_DIST_8X8 equ 0 CONFIG_ENTROPY_STATS equ 0 CONFIG_FILEOPTIONS equ 1 -CONFIG_FIX_GF_LENGTH equ 1 CONFIG_FP_MB_STATS equ 0 CONFIG_GCC equ 1 CONFIG_GCOV equ 0 -CONFIG_GLOBAL_MOTION_SEARCH equ 1 CONFIG_GPROF equ 0 CONFIG_INSPECTION equ 0 CONFIG_INTERNAL_STATS equ 0 @@ -44,15 +43,16 @@ CONFIG_MAX_DECODE_PROFILE equ 2 CONFIG_MISMATCH_DEBUG equ 0 CONFIG_MULTITHREAD equ 1 CONFIG_NORMAL_TILE_MODE equ 0 +CONFIG_ONE_PASS_SVM equ 0 CONFIG_OS_SUPPORT equ 1 CONFIG_PIC equ 0 CONFIG_RD_DEBUG equ 0 -CONFIG_REDUCED_ENCODER_BORDER equ 0 CONFIG_RUNTIME_CPU_DETECT equ 1 CONFIG_SHARED equ 0 CONFIG_SHARP_SETTINGS equ 0 CONFIG_SIZE_LIMIT equ 0 CONFIG_SPATIAL_RESAMPLING equ 1 +CONFIG_SPEED_STATS equ 0 CONFIG_STATIC equ 1 CONFIG_WEBM_IO equ 0 DECODE_HEIGHT_LIMIT equ 0 diff --git a/media/libaom/config/mac/x64/config/aom_config.h b/media/libaom/config/mac/x64/config/aom_config.h index 9a0be7cd11..ffff5abe6a 100644 --- a/media/libaom/config/mac/x64/config/aom_config.h +++ b/media/libaom/config/mac/x64/config/aom_config.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, Alliance for Open Media. All rights reserved + * Copyright (c) 2019, Alliance for Open Media. All rights reserved * * This source code is subject to the terms of the BSD 2 Clause License and * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License @@ -24,18 +24,17 @@ #define CONFIG_BIG_ENDIAN 0 #define CONFIG_BITSTREAM_DEBUG 0 #define CONFIG_COEFFICIENT_RANGE_CHECKING 0 -#define CONFIG_COLLECT_INTER_MODE_RD_STATS 0 +#define CONFIG_COLLECT_PARTITION_STATS 0 #define CONFIG_COLLECT_RD_STATS 0 #define CONFIG_DEBUG 0 #define CONFIG_DENOISE 1 +#define CONFIG_DISABLE_FULL_PIXEL_SPLIT_8X8 1 #define CONFIG_DIST_8X8 0 #define CONFIG_ENTROPY_STATS 0 #define CONFIG_FILEOPTIONS 1 -#define CONFIG_FIX_GF_LENGTH 1 #define CONFIG_FP_MB_STATS 0 #define CONFIG_GCC 1 #define CONFIG_GCOV 0 -#define CONFIG_GLOBAL_MOTION_SEARCH 1 #define CONFIG_GPROF 0 #define CONFIG_INSPECTION 0 #define CONFIG_INTERNAL_STATS 0 @@ -46,15 +45,16 @@ #define CONFIG_MISMATCH_DEBUG 0 #define CONFIG_MULTITHREAD 1 #define CONFIG_NORMAL_TILE_MODE 0 +#define CONFIG_ONE_PASS_SVM 0 #define CONFIG_OS_SUPPORT 1 #define CONFIG_PIC 0 #define CONFIG_RD_DEBUG 0 -#define CONFIG_REDUCED_ENCODER_BORDER 0 #define CONFIG_RUNTIME_CPU_DETECT 1 #define CONFIG_SHARED 0 #define CONFIG_SHARP_SETTINGS 0 #define CONFIG_SIZE_LIMIT 0 #define CONFIG_SPATIAL_RESAMPLING 1 +#define CONFIG_SPEED_STATS 0 #define CONFIG_STATIC 1 #define CONFIG_WEBM_IO 0 #define DECODE_HEIGHT_LIMIT 0 @@ -76,7 +76,5 @@ #define HAVE_SSSE3 1 #define HAVE_VSX 0 #define HAVE_WXWIDGETS 0 -#define INCLUDE_INSTALL_DIR INSTALLDIR/include #define INLINE inline -#define LIB_INSTALL_DIR INSTALLDIR/lib #endif /* AOM_CONFIG_H_ */ diff --git a/media/libaom/config/mac/x64/config/aom_dsp_rtcd.h b/media/libaom/config/mac/x64/config/aom_dsp_rtcd.h index 2856d4ede7..5826faff95 100644 --- a/media/libaom/config/mac/x64/config/aom_dsp_rtcd.h +++ b/media/libaom/config/mac/x64/config/aom_dsp_rtcd.h @@ -472,7 +472,9 @@ void aom_h_predictor_8x8_sse2(uint8_t *dst, ptrdiff_t y_stride, const uint8_t *a #define aom_h_predictor_8x8 aom_h_predictor_8x8_sse2 void aom_highbd_blend_a64_d16_mask_c(uint8_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0, uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride, const uint8_t *mask, uint32_t mask_stride, int w, int h, int subx, int suby, ConvolveParams *conv_params, const int bd); -#define aom_highbd_blend_a64_d16_mask aom_highbd_blend_a64_d16_mask_c +void aom_highbd_blend_a64_d16_mask_sse4_1(uint8_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0, uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride, const uint8_t *mask, uint32_t mask_stride, int w, int h, int subx, int suby, ConvolveParams *conv_params, const int bd); +void aom_highbd_blend_a64_d16_mask_avx2(uint8_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0, uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride, const uint8_t *mask, uint32_t mask_stride, int w, int h, int subx, int suby, ConvolveParams *conv_params, const int bd); +RTCD_EXTERN void (*aom_highbd_blend_a64_d16_mask)(uint8_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0, uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride, const uint8_t *mask, uint32_t mask_stride, int w, int h, int subx, int suby, ConvolveParams *conv_params, const int bd); void aom_highbd_blend_a64_hmask_c(uint8_t *dst, uint32_t dst_stride, const uint8_t *src0, uint32_t src0_stride, const uint8_t *src1, uint32_t src1_stride, const uint8_t *mask, int w, int h, int bd); void aom_highbd_blend_a64_hmask_sse4_1(uint8_t *dst, uint32_t dst_stride, const uint8_t *src0, uint32_t src0_stride, const uint8_t *src1, uint32_t src1_stride, const uint8_t *mask, int w, int h, int bd); @@ -1711,10 +1713,6 @@ void aom_v_predictor_8x8_c(uint8_t *dst, ptrdiff_t y_stride, const uint8_t *abov void aom_v_predictor_8x8_sse2(uint8_t *dst, ptrdiff_t y_stride, const uint8_t *above, const uint8_t *left); #define aom_v_predictor_8x8 aom_v_predictor_8x8_sse2 -void av1_round_shift_array_c(int32_t *arr, int size, int bit); -void av1_round_shift_array_sse4_1(int32_t *arr, int size, int bit); -RTCD_EXTERN void (*av1_round_shift_array)(int32_t *arr, int size, int bit); - void aom_dsp_rtcd(void); #ifdef RTCD_C @@ -1788,6 +1786,9 @@ static void setup_rtcd_internal(void) if (flags & HAS_AVX2) aom_dc_top_predictor_64x64 = aom_dc_top_predictor_64x64_avx2; aom_h_predictor_32x32 = aom_h_predictor_32x32_sse2; if (flags & HAS_AVX2) aom_h_predictor_32x32 = aom_h_predictor_32x32_avx2; + aom_highbd_blend_a64_d16_mask = aom_highbd_blend_a64_d16_mask_c; + if (flags & HAS_SSE4_1) aom_highbd_blend_a64_d16_mask = aom_highbd_blend_a64_d16_mask_sse4_1; + if (flags & HAS_AVX2) aom_highbd_blend_a64_d16_mask = aom_highbd_blend_a64_d16_mask_avx2; aom_highbd_blend_a64_hmask = aom_highbd_blend_a64_hmask_c; if (flags & HAS_SSE4_1) aom_highbd_blend_a64_hmask = aom_highbd_blend_a64_hmask_sse4_1; aom_highbd_blend_a64_mask = aom_highbd_blend_a64_mask_c; @@ -1989,8 +1990,6 @@ static void setup_rtcd_internal(void) if (flags & HAS_AVX2) aom_v_predictor_64x32 = aom_v_predictor_64x32_avx2; aom_v_predictor_64x64 = aom_v_predictor_64x64_sse2; if (flags & HAS_AVX2) aom_v_predictor_64x64 = aom_v_predictor_64x64_avx2; - av1_round_shift_array = av1_round_shift_array_c; - if (flags & HAS_SSE4_1) av1_round_shift_array = av1_round_shift_array_sse4_1; } #endif diff --git a/media/libaom/config/mac/x64/config/aom_scale_rtcd.h b/media/libaom/config/mac/x64/config/aom_scale_rtcd.h index 6b56795cd4..b6e814905b 100644 --- a/media/libaom/config/mac/x64/config/aom_scale_rtcd.h +++ b/media/libaom/config/mac/x64/config/aom_scale_rtcd.h @@ -59,13 +59,22 @@ void aom_yv12_copy_y_c(const struct yv12_buffer_config *src_ybc, struct yv12_buf void aom_yv12_extend_frame_borders_c(struct yv12_buffer_config *ybf, const int num_planes); #define aom_yv12_extend_frame_borders aom_yv12_extend_frame_borders_c -void aom_yv12_partial_copy_u_c(const struct yv12_buffer_config *src_bc, struct yv12_buffer_config *dst_bc, int hstart, int hend, int vstart, int vend); +void aom_yv12_partial_coloc_copy_u_c(const struct yv12_buffer_config *src_bc, struct yv12_buffer_config *dst_bc, int hstart, int hend, int vstart, int vend); +#define aom_yv12_partial_coloc_copy_u aom_yv12_partial_coloc_copy_u_c + +void aom_yv12_partial_coloc_copy_v_c(const struct yv12_buffer_config *src_bc, struct yv12_buffer_config *dst_bc, int hstart, int hend, int vstart, int vend); +#define aom_yv12_partial_coloc_copy_v aom_yv12_partial_coloc_copy_v_c + +void aom_yv12_partial_coloc_copy_y_c(const struct yv12_buffer_config *src_ybc, struct yv12_buffer_config *dst_ybc, int hstart, int hend, int vstart, int vend); +#define aom_yv12_partial_coloc_copy_y aom_yv12_partial_coloc_copy_y_c + +void aom_yv12_partial_copy_u_c(const struct yv12_buffer_config *src_bc, int hstart1, int hend1, int vstart1, int vend1, struct yv12_buffer_config *dst_bc, int hstart2, int vstart2); #define aom_yv12_partial_copy_u aom_yv12_partial_copy_u_c -void aom_yv12_partial_copy_v_c(const struct yv12_buffer_config *src_bc, struct yv12_buffer_config *dst_bc, int hstart, int hend, int vstart, int vend); +void aom_yv12_partial_copy_v_c(const struct yv12_buffer_config *src_bc, int hstart1, int hend1, int vstart1, int vend1, struct yv12_buffer_config *dst_bc, int hstart2, int vstart2); #define aom_yv12_partial_copy_v aom_yv12_partial_copy_v_c -void aom_yv12_partial_copy_y_c(const struct yv12_buffer_config *src_ybc, struct yv12_buffer_config *dst_ybc, int hstart, int hend, int vstart, int vend); +void aom_yv12_partial_copy_y_c(const struct yv12_buffer_config *src_ybc, int hstart1, int hend1, int vstart1, int vend1, struct yv12_buffer_config *dst_ybc, int hstart2, int vstart2); #define aom_yv12_partial_copy_y aom_yv12_partial_copy_y_c void aom_scale_rtcd(void); diff --git a/media/libaom/config/mac/x64/config/av1_rtcd.h b/media/libaom/config/mac/x64/config/av1_rtcd.h index d273182086..80bacb8dd5 100644 --- a/media/libaom/config/mac/x64/config/av1_rtcd.h +++ b/media/libaom/config/mac/x64/config/av1_rtcd.h @@ -31,6 +31,8 @@ struct txfm_param; struct aom_variance_vtable; struct search_site_config; struct yv12_buffer_config; +struct NN_CONFIG; +typedef struct NN_CONFIG NN_CONFIG; /* Function pointers return by CfL functions */ typedef void (*cfl_subsample_lbd_fn)(const uint8_t *input, int input_stride, @@ -99,14 +101,38 @@ void av1_convolve_y_sr_sse2(const uint8_t *src, int src_stride, uint8_t *dst, in void av1_convolve_y_sr_avx2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); RTCD_EXTERN void (*av1_convolve_y_sr)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_2d_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_2d_sse2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_2d_ssse3(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_2d_avx2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +RTCD_EXTERN void (*av1_dist_wtd_convolve_2d)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); + +void av1_dist_wtd_convolve_2d_copy_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_2d_copy_sse2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_2d_copy_avx2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +RTCD_EXTERN void (*av1_dist_wtd_convolve_2d_copy)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); + +void av1_dist_wtd_convolve_x_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_x_sse2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_x_avx2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +RTCD_EXTERN void (*av1_dist_wtd_convolve_x)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); + +void av1_dist_wtd_convolve_y_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_y_sse2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_y_avx2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +RTCD_EXTERN void (*av1_dist_wtd_convolve_y)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); + void av1_dr_prediction_z1_c(uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_above, int dx, int dy); -#define av1_dr_prediction_z1 av1_dr_prediction_z1_c +void av1_dr_prediction_z1_avx2(uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_above, int dx, int dy); +RTCD_EXTERN void (*av1_dr_prediction_z1)(uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_above, int dx, int dy); void av1_dr_prediction_z2_c(uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_above, int upsample_left, int dx, int dy); -#define av1_dr_prediction_z2 av1_dr_prediction_z2_c +void av1_dr_prediction_z2_avx2(uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_above, int upsample_left, int dx, int dy); +RTCD_EXTERN void (*av1_dr_prediction_z2)(uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_above, int upsample_left, int dx, int dy); void av1_dr_prediction_z3_c(uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_left, int dx, int dy); -#define av1_dr_prediction_z3 av1_dr_prediction_z3_c +void av1_dr_prediction_z3_avx2(uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_left, int dx, int dy); +RTCD_EXTERN void (*av1_dr_prediction_z3)(uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_left, int dx, int dy); void av1_filter_intra_edge_c(uint8_t *p, int sz, int strength); void av1_filter_intra_edge_sse4_1(uint8_t *p, int sz, int strength); @@ -166,40 +192,61 @@ void av1_highbd_convolve_y_sr_ssse3(const uint16_t *src, int src_stride, uint16_ void av1_highbd_convolve_y_sr_avx2(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); RTCD_EXTERN void (*av1_highbd_convolve_y_sr)(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +void av1_highbd_dist_wtd_convolve_2d_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +void av1_highbd_dist_wtd_convolve_2d_sse4_1(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +void av1_highbd_dist_wtd_convolve_2d_avx2(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +RTCD_EXTERN void (*av1_highbd_dist_wtd_convolve_2d)(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); + +void av1_highbd_dist_wtd_convolve_2d_copy_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +void av1_highbd_dist_wtd_convolve_2d_copy_sse4_1(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +void av1_highbd_dist_wtd_convolve_2d_copy_avx2(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +RTCD_EXTERN void (*av1_highbd_dist_wtd_convolve_2d_copy)(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); + +void av1_highbd_dist_wtd_convolve_x_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +void av1_highbd_dist_wtd_convolve_x_sse4_1(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +void av1_highbd_dist_wtd_convolve_x_avx2(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +RTCD_EXTERN void (*av1_highbd_dist_wtd_convolve_x)(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); + +void av1_highbd_dist_wtd_convolve_y_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +void av1_highbd_dist_wtd_convolve_y_sse4_1(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +void av1_highbd_dist_wtd_convolve_y_avx2(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +RTCD_EXTERN void (*av1_highbd_dist_wtd_convolve_y)(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); + void av1_highbd_dr_prediction_z1_c(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_above, int dx, int dy, int bd); -#define av1_highbd_dr_prediction_z1 av1_highbd_dr_prediction_z1_c +void av1_highbd_dr_prediction_z1_avx2(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_above, int dx, int dy, int bd); +RTCD_EXTERN void (*av1_highbd_dr_prediction_z1)(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_above, int dx, int dy, int bd); void av1_highbd_dr_prediction_z2_c(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_above, int upsample_left, int dx, int dy, int bd); #define av1_highbd_dr_prediction_z2 av1_highbd_dr_prediction_z2_c void av1_highbd_dr_prediction_z3_c(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_left, int dx, int dy, int bd); -#define av1_highbd_dr_prediction_z3 av1_highbd_dr_prediction_z3_c +void av1_highbd_dr_prediction_z3_avx2(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_left, int dx, int dy, int bd); +RTCD_EXTERN void (*av1_highbd_dr_prediction_z3)(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_left, int dx, int dy, int bd); void av1_highbd_inv_txfm_add_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); void av1_highbd_inv_txfm_add_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); void av1_highbd_inv_txfm_add_avx2(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); RTCD_EXTERN void (*av1_highbd_inv_txfm_add)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -void av1_highbd_inv_txfm_add_16x16_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -void av1_highbd_inv_txfm_add_16x16_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -RTCD_EXTERN void (*av1_highbd_inv_txfm_add_16x16)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +void av1_highbd_inv_txfm_add_16x4_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +void av1_highbd_inv_txfm_add_16x4_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +RTCD_EXTERN void (*av1_highbd_inv_txfm_add_16x4)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -void av1_highbd_inv_txfm_add_16x8_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -void av1_highbd_inv_txfm_add_16x8_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -RTCD_EXTERN void (*av1_highbd_inv_txfm_add_16x8)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); - -void av1_highbd_inv_txfm_add_32x32_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -void av1_highbd_inv_txfm_add_32x32_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -void av1_highbd_inv_txfm_add_32x32_avx2(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -RTCD_EXTERN void (*av1_highbd_inv_txfm_add_32x32)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +void av1_highbd_inv_txfm_add_4x16_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +void av1_highbd_inv_txfm_add_4x16_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +RTCD_EXTERN void (*av1_highbd_inv_txfm_add_4x16)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); void av1_highbd_inv_txfm_add_4x4_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); void av1_highbd_inv_txfm_add_4x4_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); RTCD_EXTERN void (*av1_highbd_inv_txfm_add_4x4)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -void av1_highbd_inv_txfm_add_8x16_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -void av1_highbd_inv_txfm_add_8x16_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -RTCD_EXTERN void (*av1_highbd_inv_txfm_add_8x16)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +void av1_highbd_inv_txfm_add_4x8_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +void av1_highbd_inv_txfm_add_4x8_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +RTCD_EXTERN void (*av1_highbd_inv_txfm_add_4x8)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); + +void av1_highbd_inv_txfm_add_8x4_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +void av1_highbd_inv_txfm_add_8x4_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +RTCD_EXTERN void (*av1_highbd_inv_txfm_add_8x4)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); void av1_highbd_inv_txfm_add_8x8_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); void av1_highbd_inv_txfm_add_8x8_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); @@ -211,26 +258,6 @@ void av1_highbd_iwht4x4_16_add_c(const tran_low_t *input, uint8_t *dest, int des void av1_highbd_iwht4x4_1_add_c(const tran_low_t *input, uint8_t *dest, int dest_stride, int bd); #define av1_highbd_iwht4x4_1_add av1_highbd_iwht4x4_1_add_c -void av1_highbd_jnt_convolve_2d_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -void av1_highbd_jnt_convolve_2d_sse4_1(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -void av1_highbd_jnt_convolve_2d_avx2(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -RTCD_EXTERN void (*av1_highbd_jnt_convolve_2d)(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); - -void av1_highbd_jnt_convolve_2d_copy_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -void av1_highbd_jnt_convolve_2d_copy_sse4_1(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -void av1_highbd_jnt_convolve_2d_copy_avx2(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -RTCD_EXTERN void (*av1_highbd_jnt_convolve_2d_copy)(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); - -void av1_highbd_jnt_convolve_x_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -void av1_highbd_jnt_convolve_x_sse4_1(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -void av1_highbd_jnt_convolve_x_avx2(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -RTCD_EXTERN void (*av1_highbd_jnt_convolve_x)(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); - -void av1_highbd_jnt_convolve_y_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -void av1_highbd_jnt_convolve_y_sse4_1(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -void av1_highbd_jnt_convolve_y_avx2(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -RTCD_EXTERN void (*av1_highbd_jnt_convolve_y)(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); - void av1_highbd_warp_affine_c(const int32_t *mat, const uint16_t *ref, int width, int height, int stride, uint16_t *pred, int p_col, int p_row, int p_width, int p_height, int p_stride, int subsampling_x, int subsampling_y, int bd, ConvolveParams *conv_params, int16_t alpha, int16_t beta, int16_t gamma, int16_t delta); void av1_highbd_warp_affine_sse4_1(const int32_t *mat, const uint16_t *ref, int width, int height, int stride, uint16_t *pred, int p_col, int p_row, int p_width, int p_height, int p_stride, int subsampling_x, int subsampling_y, int bd, ConvolveParams *conv_params, int16_t alpha, int16_t beta, int16_t gamma, int16_t delta); RTCD_EXTERN void (*av1_highbd_warp_affine)(const int32_t *mat, const uint16_t *ref, int width, int height, int stride, uint16_t *pred, int p_col, int p_row, int p_width, int p_height, int p_stride, int subsampling_x, int subsampling_y, int bd, ConvolveParams *conv_params, int16_t alpha, int16_t beta, int16_t gamma, int16_t delta); @@ -304,25 +331,9 @@ void av1_inv_txfm_add_ssse3(const tran_low_t *dqcoeff, uint8_t *dst, int stride, void av1_inv_txfm_add_avx2(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); RTCD_EXTERN void (*av1_inv_txfm_add)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -void av1_jnt_convolve_2d_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -void av1_jnt_convolve_2d_ssse3(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -void av1_jnt_convolve_2d_avx2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -RTCD_EXTERN void (*av1_jnt_convolve_2d)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); - -void av1_jnt_convolve_2d_copy_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -void av1_jnt_convolve_2d_copy_sse2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -void av1_jnt_convolve_2d_copy_avx2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -RTCD_EXTERN void (*av1_jnt_convolve_2d_copy)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); - -void av1_jnt_convolve_x_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -void av1_jnt_convolve_x_sse2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -void av1_jnt_convolve_x_avx2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -RTCD_EXTERN void (*av1_jnt_convolve_x)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); - -void av1_jnt_convolve_y_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -void av1_jnt_convolve_y_sse2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -void av1_jnt_convolve_y_avx2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -RTCD_EXTERN void (*av1_jnt_convolve_y)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_round_shift_array_c(int32_t *arr, int size, int bit); +void av1_round_shift_array_sse4_1(int32_t *arr, int size, int bit); +RTCD_EXTERN void (*av1_round_shift_array)(int32_t *arr, int size, int bit); int av1_selfguided_restoration_c(const uint8_t *dgd8, int width, int height, int dgd_stride, int32_t *flt0, int32_t *flt1, int flt_stride, @@ -354,12 +365,12 @@ void av1_wiener_convolve_add_src_sse2(const uint8_t *src, ptrdiff_t src_stride, void av1_wiener_convolve_add_src_avx2(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, const int16_t *filter_y, int y_step_q4, int w, int h, const ConvolveParams *conv_params); RTCD_EXTERN void (*av1_wiener_convolve_add_src)(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, const int16_t *filter_y, int y_step_q4, int w, int h, const ConvolveParams *conv_params); -void cdef_filter_block_c(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int max, int coeff_shift); -void cdef_filter_block_sse2(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int max, int coeff_shift); -void cdef_filter_block_ssse3(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int max, int coeff_shift); -void cdef_filter_block_sse4_1(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int max, int coeff_shift); -void cdef_filter_block_avx2(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int max, int coeff_shift); -RTCD_EXTERN void (*cdef_filter_block)(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int max, int coeff_shift); +void cdef_filter_block_c(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int coeff_shift); +void cdef_filter_block_sse2(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int coeff_shift); +void cdef_filter_block_ssse3(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int coeff_shift); +void cdef_filter_block_sse4_1(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int coeff_shift); +void cdef_filter_block_avx2(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int coeff_shift); +RTCD_EXTERN void (*cdef_filter_block)(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int coeff_shift); int cdef_find_dir_c(const uint16_t *img, int stride, int32_t *var, int coeff_shift); int cdef_find_dir_sse2(const uint16_t *img, int stride, int32_t *var, int coeff_shift); @@ -461,6 +472,21 @@ static void setup_rtcd_internal(void) if (flags & HAS_AVX2) av1_convolve_x_sr = av1_convolve_x_sr_avx2; av1_convolve_y_sr = av1_convolve_y_sr_sse2; if (flags & HAS_AVX2) av1_convolve_y_sr = av1_convolve_y_sr_avx2; + av1_dist_wtd_convolve_2d = av1_dist_wtd_convolve_2d_sse2; + if (flags & HAS_SSSE3) av1_dist_wtd_convolve_2d = av1_dist_wtd_convolve_2d_ssse3; + if (flags & HAS_AVX2) av1_dist_wtd_convolve_2d = av1_dist_wtd_convolve_2d_avx2; + av1_dist_wtd_convolve_2d_copy = av1_dist_wtd_convolve_2d_copy_sse2; + if (flags & HAS_AVX2) av1_dist_wtd_convolve_2d_copy = av1_dist_wtd_convolve_2d_copy_avx2; + av1_dist_wtd_convolve_x = av1_dist_wtd_convolve_x_sse2; + if (flags & HAS_AVX2) av1_dist_wtd_convolve_x = av1_dist_wtd_convolve_x_avx2; + av1_dist_wtd_convolve_y = av1_dist_wtd_convolve_y_sse2; + if (flags & HAS_AVX2) av1_dist_wtd_convolve_y = av1_dist_wtd_convolve_y_avx2; + av1_dr_prediction_z1 = av1_dr_prediction_z1_c; + if (flags & HAS_AVX2) av1_dr_prediction_z1 = av1_dr_prediction_z1_avx2; + av1_dr_prediction_z2 = av1_dr_prediction_z2_c; + if (flags & HAS_AVX2) av1_dr_prediction_z2 = av1_dr_prediction_z2_avx2; + av1_dr_prediction_z3 = av1_dr_prediction_z3_c; + if (flags & HAS_AVX2) av1_dr_prediction_z3 = av1_dr_prediction_z3_avx2; av1_filter_intra_edge = av1_filter_intra_edge_c; if (flags & HAS_SSE4_1) av1_filter_intra_edge = av1_filter_intra_edge_sse4_1; av1_filter_intra_edge_high = av1_filter_intra_edge_high_c; @@ -482,34 +508,37 @@ static void setup_rtcd_internal(void) av1_highbd_convolve_y_sr = av1_highbd_convolve_y_sr_c; if (flags & HAS_SSSE3) av1_highbd_convolve_y_sr = av1_highbd_convolve_y_sr_ssse3; if (flags & HAS_AVX2) av1_highbd_convolve_y_sr = av1_highbd_convolve_y_sr_avx2; + av1_highbd_dist_wtd_convolve_2d = av1_highbd_dist_wtd_convolve_2d_c; + if (flags & HAS_SSE4_1) av1_highbd_dist_wtd_convolve_2d = av1_highbd_dist_wtd_convolve_2d_sse4_1; + if (flags & HAS_AVX2) av1_highbd_dist_wtd_convolve_2d = av1_highbd_dist_wtd_convolve_2d_avx2; + av1_highbd_dist_wtd_convolve_2d_copy = av1_highbd_dist_wtd_convolve_2d_copy_c; + if (flags & HAS_SSE4_1) av1_highbd_dist_wtd_convolve_2d_copy = av1_highbd_dist_wtd_convolve_2d_copy_sse4_1; + if (flags & HAS_AVX2) av1_highbd_dist_wtd_convolve_2d_copy = av1_highbd_dist_wtd_convolve_2d_copy_avx2; + av1_highbd_dist_wtd_convolve_x = av1_highbd_dist_wtd_convolve_x_c; + if (flags & HAS_SSE4_1) av1_highbd_dist_wtd_convolve_x = av1_highbd_dist_wtd_convolve_x_sse4_1; + if (flags & HAS_AVX2) av1_highbd_dist_wtd_convolve_x = av1_highbd_dist_wtd_convolve_x_avx2; + av1_highbd_dist_wtd_convolve_y = av1_highbd_dist_wtd_convolve_y_c; + if (flags & HAS_SSE4_1) av1_highbd_dist_wtd_convolve_y = av1_highbd_dist_wtd_convolve_y_sse4_1; + if (flags & HAS_AVX2) av1_highbd_dist_wtd_convolve_y = av1_highbd_dist_wtd_convolve_y_avx2; + av1_highbd_dr_prediction_z1 = av1_highbd_dr_prediction_z1_c; + if (flags & HAS_AVX2) av1_highbd_dr_prediction_z1 = av1_highbd_dr_prediction_z1_avx2; + av1_highbd_dr_prediction_z3 = av1_highbd_dr_prediction_z3_c; + if (flags & HAS_AVX2) av1_highbd_dr_prediction_z3 = av1_highbd_dr_prediction_z3_avx2; av1_highbd_inv_txfm_add = av1_highbd_inv_txfm_add_c; if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add = av1_highbd_inv_txfm_add_sse4_1; if (flags & HAS_AVX2) av1_highbd_inv_txfm_add = av1_highbd_inv_txfm_add_avx2; - av1_highbd_inv_txfm_add_16x16 = av1_highbd_inv_txfm_add_16x16_c; - if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add_16x16 = av1_highbd_inv_txfm_add_16x16_sse4_1; - av1_highbd_inv_txfm_add_16x8 = av1_highbd_inv_txfm_add_16x8_c; - if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add_16x8 = av1_highbd_inv_txfm_add_16x8_sse4_1; - av1_highbd_inv_txfm_add_32x32 = av1_highbd_inv_txfm_add_32x32_c; - if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add_32x32 = av1_highbd_inv_txfm_add_32x32_sse4_1; - if (flags & HAS_AVX2) av1_highbd_inv_txfm_add_32x32 = av1_highbd_inv_txfm_add_32x32_avx2; + av1_highbd_inv_txfm_add_16x4 = av1_highbd_inv_txfm_add_16x4_c; + if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add_16x4 = av1_highbd_inv_txfm_add_16x4_sse4_1; + av1_highbd_inv_txfm_add_4x16 = av1_highbd_inv_txfm_add_4x16_c; + if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add_4x16 = av1_highbd_inv_txfm_add_4x16_sse4_1; av1_highbd_inv_txfm_add_4x4 = av1_highbd_inv_txfm_add_4x4_c; if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add_4x4 = av1_highbd_inv_txfm_add_4x4_sse4_1; - av1_highbd_inv_txfm_add_8x16 = av1_highbd_inv_txfm_add_8x16_c; - if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add_8x16 = av1_highbd_inv_txfm_add_8x16_sse4_1; + av1_highbd_inv_txfm_add_4x8 = av1_highbd_inv_txfm_add_4x8_c; + if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add_4x8 = av1_highbd_inv_txfm_add_4x8_sse4_1; + av1_highbd_inv_txfm_add_8x4 = av1_highbd_inv_txfm_add_8x4_c; + if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add_8x4 = av1_highbd_inv_txfm_add_8x4_sse4_1; av1_highbd_inv_txfm_add_8x8 = av1_highbd_inv_txfm_add_8x8_c; if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add_8x8 = av1_highbd_inv_txfm_add_8x8_sse4_1; - av1_highbd_jnt_convolve_2d = av1_highbd_jnt_convolve_2d_c; - if (flags & HAS_SSE4_1) av1_highbd_jnt_convolve_2d = av1_highbd_jnt_convolve_2d_sse4_1; - if (flags & HAS_AVX2) av1_highbd_jnt_convolve_2d = av1_highbd_jnt_convolve_2d_avx2; - av1_highbd_jnt_convolve_2d_copy = av1_highbd_jnt_convolve_2d_copy_c; - if (flags & HAS_SSE4_1) av1_highbd_jnt_convolve_2d_copy = av1_highbd_jnt_convolve_2d_copy_sse4_1; - if (flags & HAS_AVX2) av1_highbd_jnt_convolve_2d_copy = av1_highbd_jnt_convolve_2d_copy_avx2; - av1_highbd_jnt_convolve_x = av1_highbd_jnt_convolve_x_c; - if (flags & HAS_SSE4_1) av1_highbd_jnt_convolve_x = av1_highbd_jnt_convolve_x_sse4_1; - if (flags & HAS_AVX2) av1_highbd_jnt_convolve_x = av1_highbd_jnt_convolve_x_avx2; - av1_highbd_jnt_convolve_y = av1_highbd_jnt_convolve_y_c; - if (flags & HAS_SSE4_1) av1_highbd_jnt_convolve_y = av1_highbd_jnt_convolve_y_sse4_1; - if (flags & HAS_AVX2) av1_highbd_jnt_convolve_y = av1_highbd_jnt_convolve_y_avx2; av1_highbd_warp_affine = av1_highbd_warp_affine_c; if (flags & HAS_SSE4_1) av1_highbd_warp_affine = av1_highbd_warp_affine_sse4_1; av1_highbd_wiener_convolve_add_src = av1_highbd_wiener_convolve_add_src_c; @@ -522,15 +551,8 @@ static void setup_rtcd_internal(void) av1_inv_txfm_add = av1_inv_txfm_add_c; if (flags & HAS_SSSE3) av1_inv_txfm_add = av1_inv_txfm_add_ssse3; if (flags & HAS_AVX2) av1_inv_txfm_add = av1_inv_txfm_add_avx2; - av1_jnt_convolve_2d = av1_jnt_convolve_2d_c; - if (flags & HAS_SSSE3) av1_jnt_convolve_2d = av1_jnt_convolve_2d_ssse3; - if (flags & HAS_AVX2) av1_jnt_convolve_2d = av1_jnt_convolve_2d_avx2; - av1_jnt_convolve_2d_copy = av1_jnt_convolve_2d_copy_sse2; - if (flags & HAS_AVX2) av1_jnt_convolve_2d_copy = av1_jnt_convolve_2d_copy_avx2; - av1_jnt_convolve_x = av1_jnt_convolve_x_sse2; - if (flags & HAS_AVX2) av1_jnt_convolve_x = av1_jnt_convolve_x_avx2; - av1_jnt_convolve_y = av1_jnt_convolve_y_sse2; - if (flags & HAS_AVX2) av1_jnt_convolve_y = av1_jnt_convolve_y_avx2; + av1_round_shift_array = av1_round_shift_array_c; + if (flags & HAS_SSE4_1) av1_round_shift_array = av1_round_shift_array_sse4_1; av1_selfguided_restoration = av1_selfguided_restoration_c; if (flags & HAS_SSE4_1) av1_selfguided_restoration = av1_selfguided_restoration_sse4_1; if (flags & HAS_AVX2) av1_selfguided_restoration = av1_selfguided_restoration_avx2; diff --git a/media/libaom/config/win/ia32/config/aom_config.asm b/media/libaom/config/win/ia32/config/aom_config.asm index 6ae776c7c4..b3a8c2d596 100644 --- a/media/libaom/config/win/ia32/config/aom_config.asm +++ b/media/libaom/config/win/ia32/config/aom_config.asm @@ -1,5 +1,5 @@ ; -; Copyright (c) 2018, Alliance for Open Media. All rights reserved +; Copyright (c) 2019, Alliance for Open Media. All rights reserved ; ; This source code is subject to the terms of the BSD 2 Clause License and ; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License @@ -22,18 +22,17 @@ CONFIG_AV1_ENCODER equ 0 CONFIG_BIG_ENDIAN equ 0 CONFIG_BITSTREAM_DEBUG equ 0 CONFIG_COEFFICIENT_RANGE_CHECKING equ 0 -CONFIG_COLLECT_INTER_MODE_RD_STATS equ 0 +CONFIG_COLLECT_PARTITION_STATS equ 0 CONFIG_COLLECT_RD_STATS equ 0 CONFIG_DEBUG equ 0 CONFIG_DENOISE equ 1 +CONFIG_DISABLE_FULL_PIXEL_SPLIT_8X8 equ 1 CONFIG_DIST_8X8 equ 0 CONFIG_ENTROPY_STATS equ 0 CONFIG_FILEOPTIONS equ 1 -CONFIG_FIX_GF_LENGTH equ 1 CONFIG_FP_MB_STATS equ 0 CONFIG_GCC equ 1 CONFIG_GCOV equ 0 -CONFIG_GLOBAL_MOTION_SEARCH equ 1 CONFIG_GPROF equ 0 CONFIG_INSPECTION equ 0 CONFIG_INTERNAL_STATS equ 0 @@ -44,15 +43,16 @@ CONFIG_MAX_DECODE_PROFILE equ 2 CONFIG_MISMATCH_DEBUG equ 0 CONFIG_MULTITHREAD equ 1 CONFIG_NORMAL_TILE_MODE equ 0 +CONFIG_ONE_PASS_SVM equ 0 CONFIG_OS_SUPPORT equ 1 CONFIG_PIC equ 0 CONFIG_RD_DEBUG equ 0 -CONFIG_REDUCED_ENCODER_BORDER equ 0 CONFIG_RUNTIME_CPU_DETECT equ 1 CONFIG_SHARED equ 0 CONFIG_SHARP_SETTINGS equ 0 CONFIG_SIZE_LIMIT equ 0 CONFIG_SPATIAL_RESAMPLING equ 1 +CONFIG_SPEED_STATS equ 0 CONFIG_STATIC equ 1 CONFIG_WEBM_IO equ 0 DECODE_HEIGHT_LIMIT equ 0 diff --git a/media/libaom/config/win/ia32/config/aom_config.h b/media/libaom/config/win/ia32/config/aom_config.h index 090a455778..394c9d574e 100644 --- a/media/libaom/config/win/ia32/config/aom_config.h +++ b/media/libaom/config/win/ia32/config/aom_config.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, Alliance for Open Media. All rights reserved + * Copyright (c) 2019, Alliance for Open Media. All rights reserved * * This source code is subject to the terms of the BSD 2 Clause License and * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License @@ -24,18 +24,17 @@ #define CONFIG_BIG_ENDIAN 0 #define CONFIG_BITSTREAM_DEBUG 0 #define CONFIG_COEFFICIENT_RANGE_CHECKING 0 -#define CONFIG_COLLECT_INTER_MODE_RD_STATS 0 +#define CONFIG_COLLECT_PARTITION_STATS 0 #define CONFIG_COLLECT_RD_STATS 0 #define CONFIG_DEBUG 0 #define CONFIG_DENOISE 1 +#define CONFIG_DISABLE_FULL_PIXEL_SPLIT_8X8 1 #define CONFIG_DIST_8X8 0 #define CONFIG_ENTROPY_STATS 0 #define CONFIG_FILEOPTIONS 1 -#define CONFIG_FIX_GF_LENGTH 1 #define CONFIG_FP_MB_STATS 0 #define CONFIG_GCC 1 #define CONFIG_GCOV 0 -#define CONFIG_GLOBAL_MOTION_SEARCH 1 #define CONFIG_GPROF 0 #define CONFIG_INSPECTION 0 #define CONFIG_INTERNAL_STATS 0 @@ -46,15 +45,16 @@ #define CONFIG_MISMATCH_DEBUG 0 #define CONFIG_MULTITHREAD 1 #define CONFIG_NORMAL_TILE_MODE 0 +#define CONFIG_ONE_PASS_SVM 0 #define CONFIG_OS_SUPPORT 1 #define CONFIG_PIC 0 #define CONFIG_RD_DEBUG 0 -#define CONFIG_REDUCED_ENCODER_BORDER 0 #define CONFIG_RUNTIME_CPU_DETECT 1 #define CONFIG_SHARED 0 #define CONFIG_SHARP_SETTINGS 0 #define CONFIG_SIZE_LIMIT 0 #define CONFIG_SPATIAL_RESAMPLING 1 +#define CONFIG_SPEED_STATS 0 #define CONFIG_STATIC 1 #define CONFIG_WEBM_IO 0 #define DECODE_HEIGHT_LIMIT 0 @@ -76,7 +76,5 @@ #define HAVE_SSSE3 1 #define HAVE_VSX 0 #define HAVE_WXWIDGETS 0 -#define INCLUDE_INSTALL_DIR INSTALLDIR/include #define INLINE inline -#define LIB_INSTALL_DIR INSTALLDIR/lib #endif /* AOM_CONFIG_H_ */ diff --git a/media/libaom/config/win/ia32/config/aom_dsp_rtcd.h b/media/libaom/config/win/ia32/config/aom_dsp_rtcd.h index 5b3762f109..a60a7aea6e 100644 --- a/media/libaom/config/win/ia32/config/aom_dsp_rtcd.h +++ b/media/libaom/config/win/ia32/config/aom_dsp_rtcd.h @@ -472,7 +472,9 @@ void aom_h_predictor_8x8_sse2(uint8_t *dst, ptrdiff_t y_stride, const uint8_t *a RTCD_EXTERN void (*aom_h_predictor_8x8)(uint8_t *dst, ptrdiff_t y_stride, const uint8_t *above, const uint8_t *left); void aom_highbd_blend_a64_d16_mask_c(uint8_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0, uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride, const uint8_t *mask, uint32_t mask_stride, int w, int h, int subx, int suby, ConvolveParams *conv_params, const int bd); -#define aom_highbd_blend_a64_d16_mask aom_highbd_blend_a64_d16_mask_c +void aom_highbd_blend_a64_d16_mask_sse4_1(uint8_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0, uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride, const uint8_t *mask, uint32_t mask_stride, int w, int h, int subx, int suby, ConvolveParams *conv_params, const int bd); +void aom_highbd_blend_a64_d16_mask_avx2(uint8_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0, uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride, const uint8_t *mask, uint32_t mask_stride, int w, int h, int subx, int suby, ConvolveParams *conv_params, const int bd); +RTCD_EXTERN void (*aom_highbd_blend_a64_d16_mask)(uint8_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0, uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride, const uint8_t *mask, uint32_t mask_stride, int w, int h, int subx, int suby, ConvolveParams *conv_params, const int bd); void aom_highbd_blend_a64_hmask_c(uint8_t *dst, uint32_t dst_stride, const uint8_t *src0, uint32_t src0_stride, const uint8_t *src1, uint32_t src1_stride, const uint8_t *mask, int w, int h, int bd); void aom_highbd_blend_a64_hmask_sse4_1(uint8_t *dst, uint32_t dst_stride, const uint8_t *src0, uint32_t src0_stride, const uint8_t *src1, uint32_t src1_stride, const uint8_t *mask, int w, int h, int bd); @@ -487,10 +489,12 @@ void aom_highbd_blend_a64_vmask_sse4_1(uint8_t *dst, uint32_t dst_stride, const RTCD_EXTERN void (*aom_highbd_blend_a64_vmask)(uint8_t *dst, uint32_t dst_stride, const uint8_t *src0, uint32_t src0_stride, const uint8_t *src1, uint32_t src1_stride, const uint8_t *mask, int w, int h, int bd); void aom_highbd_convolve8_horiz_c(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, const int16_t *filter_y, int y_step_q4, int w, int h, int bps); +void aom_highbd_convolve8_horiz_sse2(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, const int16_t *filter_y, int y_step_q4, int w, int h, int bps); void aom_highbd_convolve8_horiz_avx2(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, const int16_t *filter_y, int y_step_q4, int w, int h, int bps); RTCD_EXTERN void (*aom_highbd_convolve8_horiz)(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, const int16_t *filter_y, int y_step_q4, int w, int h, int bps); void aom_highbd_convolve8_vert_c(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, const int16_t *filter_y, int y_step_q4, int w, int h, int bps); +void aom_highbd_convolve8_vert_sse2(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, const int16_t *filter_y, int y_step_q4, int w, int h, int bps); void aom_highbd_convolve8_vert_avx2(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, const int16_t *filter_y, int y_step_q4, int w, int h, int bps); RTCD_EXTERN void (*aom_highbd_convolve8_vert)(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, const int16_t *filter_y, int y_step_q4, int w, int h, int bps); @@ -1709,10 +1713,6 @@ void aom_v_predictor_8x8_c(uint8_t *dst, ptrdiff_t y_stride, const uint8_t *abov void aom_v_predictor_8x8_sse2(uint8_t *dst, ptrdiff_t y_stride, const uint8_t *above, const uint8_t *left); RTCD_EXTERN void (*aom_v_predictor_8x8)(uint8_t *dst, ptrdiff_t y_stride, const uint8_t *above, const uint8_t *left); -void av1_round_shift_array_c(int32_t *arr, int size, int bit); -void av1_round_shift_array_sse4_1(int32_t *arr, int size, int bit); -RTCD_EXTERN void (*av1_round_shift_array)(int32_t *arr, int size, int bit); - void aom_dsp_rtcd(void); #ifdef RTCD_C @@ -1955,6 +1955,9 @@ static void setup_rtcd_internal(void) if (flags & HAS_SSE2) aom_h_predictor_8x4 = aom_h_predictor_8x4_sse2; aom_h_predictor_8x8 = aom_h_predictor_8x8_c; if (flags & HAS_SSE2) aom_h_predictor_8x8 = aom_h_predictor_8x8_sse2; + aom_highbd_blend_a64_d16_mask = aom_highbd_blend_a64_d16_mask_c; + if (flags & HAS_SSE4_1) aom_highbd_blend_a64_d16_mask = aom_highbd_blend_a64_d16_mask_sse4_1; + if (flags & HAS_AVX2) aom_highbd_blend_a64_d16_mask = aom_highbd_blend_a64_d16_mask_avx2; aom_highbd_blend_a64_hmask = aom_highbd_blend_a64_hmask_c; if (flags & HAS_SSE4_1) aom_highbd_blend_a64_hmask = aom_highbd_blend_a64_hmask_sse4_1; aom_highbd_blend_a64_mask = aom_highbd_blend_a64_mask_c; @@ -1962,8 +1965,10 @@ static void setup_rtcd_internal(void) aom_highbd_blend_a64_vmask = aom_highbd_blend_a64_vmask_c; if (flags & HAS_SSE4_1) aom_highbd_blend_a64_vmask = aom_highbd_blend_a64_vmask_sse4_1; aom_highbd_convolve8_horiz = aom_highbd_convolve8_horiz_c; + if (flags & HAS_SSE2) aom_highbd_convolve8_horiz = aom_highbd_convolve8_horiz_sse2; if (flags & HAS_AVX2) aom_highbd_convolve8_horiz = aom_highbd_convolve8_horiz_avx2; aom_highbd_convolve8_vert = aom_highbd_convolve8_vert_c; + if (flags & HAS_SSE2) aom_highbd_convolve8_vert = aom_highbd_convolve8_vert_sse2; if (flags & HAS_AVX2) aom_highbd_convolve8_vert = aom_highbd_convolve8_vert_avx2; aom_highbd_convolve_copy = aom_highbd_convolve_copy_c; if (flags & HAS_SSE2) aom_highbd_convolve_copy = aom_highbd_convolve_copy_sse2; @@ -2367,8 +2372,6 @@ static void setup_rtcd_internal(void) if (flags & HAS_SSE2) aom_v_predictor_8x4 = aom_v_predictor_8x4_sse2; aom_v_predictor_8x8 = aom_v_predictor_8x8_c; if (flags & HAS_SSE2) aom_v_predictor_8x8 = aom_v_predictor_8x8_sse2; - av1_round_shift_array = av1_round_shift_array_c; - if (flags & HAS_SSE4_1) av1_round_shift_array = av1_round_shift_array_sse4_1; } #endif diff --git a/media/libaom/config/win/ia32/config/aom_scale_rtcd.h b/media/libaom/config/win/ia32/config/aom_scale_rtcd.h index 6b56795cd4..b6e814905b 100644 --- a/media/libaom/config/win/ia32/config/aom_scale_rtcd.h +++ b/media/libaom/config/win/ia32/config/aom_scale_rtcd.h @@ -59,13 +59,22 @@ void aom_yv12_copy_y_c(const struct yv12_buffer_config *src_ybc, struct yv12_buf void aom_yv12_extend_frame_borders_c(struct yv12_buffer_config *ybf, const int num_planes); #define aom_yv12_extend_frame_borders aom_yv12_extend_frame_borders_c -void aom_yv12_partial_copy_u_c(const struct yv12_buffer_config *src_bc, struct yv12_buffer_config *dst_bc, int hstart, int hend, int vstart, int vend); +void aom_yv12_partial_coloc_copy_u_c(const struct yv12_buffer_config *src_bc, struct yv12_buffer_config *dst_bc, int hstart, int hend, int vstart, int vend); +#define aom_yv12_partial_coloc_copy_u aom_yv12_partial_coloc_copy_u_c + +void aom_yv12_partial_coloc_copy_v_c(const struct yv12_buffer_config *src_bc, struct yv12_buffer_config *dst_bc, int hstart, int hend, int vstart, int vend); +#define aom_yv12_partial_coloc_copy_v aom_yv12_partial_coloc_copy_v_c + +void aom_yv12_partial_coloc_copy_y_c(const struct yv12_buffer_config *src_ybc, struct yv12_buffer_config *dst_ybc, int hstart, int hend, int vstart, int vend); +#define aom_yv12_partial_coloc_copy_y aom_yv12_partial_coloc_copy_y_c + +void aom_yv12_partial_copy_u_c(const struct yv12_buffer_config *src_bc, int hstart1, int hend1, int vstart1, int vend1, struct yv12_buffer_config *dst_bc, int hstart2, int vstart2); #define aom_yv12_partial_copy_u aom_yv12_partial_copy_u_c -void aom_yv12_partial_copy_v_c(const struct yv12_buffer_config *src_bc, struct yv12_buffer_config *dst_bc, int hstart, int hend, int vstart, int vend); +void aom_yv12_partial_copy_v_c(const struct yv12_buffer_config *src_bc, int hstart1, int hend1, int vstart1, int vend1, struct yv12_buffer_config *dst_bc, int hstart2, int vstart2); #define aom_yv12_partial_copy_v aom_yv12_partial_copy_v_c -void aom_yv12_partial_copy_y_c(const struct yv12_buffer_config *src_ybc, struct yv12_buffer_config *dst_ybc, int hstart, int hend, int vstart, int vend); +void aom_yv12_partial_copy_y_c(const struct yv12_buffer_config *src_ybc, int hstart1, int hend1, int vstart1, int vend1, struct yv12_buffer_config *dst_ybc, int hstart2, int vstart2); #define aom_yv12_partial_copy_y aom_yv12_partial_copy_y_c void aom_scale_rtcd(void); diff --git a/media/libaom/config/win/ia32/config/av1_rtcd.h b/media/libaom/config/win/ia32/config/av1_rtcd.h index a2a6f14ff5..8e69b643ef 100644 --- a/media/libaom/config/win/ia32/config/av1_rtcd.h +++ b/media/libaom/config/win/ia32/config/av1_rtcd.h @@ -31,6 +31,8 @@ struct txfm_param; struct aom_variance_vtable; struct search_site_config; struct yv12_buffer_config; +struct NN_CONFIG; +typedef struct NN_CONFIG NN_CONFIG; /* Function pointers return by CfL functions */ typedef void (*cfl_subsample_lbd_fn)(const uint8_t *input, int input_stride, @@ -99,14 +101,38 @@ void av1_convolve_y_sr_sse2(const uint8_t *src, int src_stride, uint8_t *dst, in void av1_convolve_y_sr_avx2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); RTCD_EXTERN void (*av1_convolve_y_sr)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_2d_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_2d_sse2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_2d_ssse3(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_2d_avx2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +RTCD_EXTERN void (*av1_dist_wtd_convolve_2d)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); + +void av1_dist_wtd_convolve_2d_copy_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_2d_copy_sse2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_2d_copy_avx2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +RTCD_EXTERN void (*av1_dist_wtd_convolve_2d_copy)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); + +void av1_dist_wtd_convolve_x_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_x_sse2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_x_avx2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +RTCD_EXTERN void (*av1_dist_wtd_convolve_x)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); + +void av1_dist_wtd_convolve_y_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_y_sse2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_y_avx2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +RTCD_EXTERN void (*av1_dist_wtd_convolve_y)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); + void av1_dr_prediction_z1_c(uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_above, int dx, int dy); -#define av1_dr_prediction_z1 av1_dr_prediction_z1_c +void av1_dr_prediction_z1_avx2(uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_above, int dx, int dy); +RTCD_EXTERN void (*av1_dr_prediction_z1)(uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_above, int dx, int dy); void av1_dr_prediction_z2_c(uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_above, int upsample_left, int dx, int dy); -#define av1_dr_prediction_z2 av1_dr_prediction_z2_c +void av1_dr_prediction_z2_avx2(uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_above, int upsample_left, int dx, int dy); +RTCD_EXTERN void (*av1_dr_prediction_z2)(uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_above, int upsample_left, int dx, int dy); void av1_dr_prediction_z3_c(uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_left, int dx, int dy); -#define av1_dr_prediction_z3 av1_dr_prediction_z3_c +void av1_dr_prediction_z3_avx2(uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_left, int dx, int dy); +RTCD_EXTERN void (*av1_dr_prediction_z3)(uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_left, int dx, int dy); void av1_filter_intra_edge_c(uint8_t *p, int sz, int strength); void av1_filter_intra_edge_sse4_1(uint8_t *p, int sz, int strength); @@ -163,40 +189,61 @@ void av1_highbd_convolve_y_sr_ssse3(const uint16_t *src, int src_stride, uint16_ void av1_highbd_convolve_y_sr_avx2(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); RTCD_EXTERN void (*av1_highbd_convolve_y_sr)(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +void av1_highbd_dist_wtd_convolve_2d_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +void av1_highbd_dist_wtd_convolve_2d_sse4_1(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +void av1_highbd_dist_wtd_convolve_2d_avx2(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +RTCD_EXTERN void (*av1_highbd_dist_wtd_convolve_2d)(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); + +void av1_highbd_dist_wtd_convolve_2d_copy_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +void av1_highbd_dist_wtd_convolve_2d_copy_sse4_1(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +void av1_highbd_dist_wtd_convolve_2d_copy_avx2(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +RTCD_EXTERN void (*av1_highbd_dist_wtd_convolve_2d_copy)(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); + +void av1_highbd_dist_wtd_convolve_x_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +void av1_highbd_dist_wtd_convolve_x_sse4_1(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +void av1_highbd_dist_wtd_convolve_x_avx2(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +RTCD_EXTERN void (*av1_highbd_dist_wtd_convolve_x)(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); + +void av1_highbd_dist_wtd_convolve_y_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +void av1_highbd_dist_wtd_convolve_y_sse4_1(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +void av1_highbd_dist_wtd_convolve_y_avx2(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +RTCD_EXTERN void (*av1_highbd_dist_wtd_convolve_y)(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); + void av1_highbd_dr_prediction_z1_c(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_above, int dx, int dy, int bd); -#define av1_highbd_dr_prediction_z1 av1_highbd_dr_prediction_z1_c +void av1_highbd_dr_prediction_z1_avx2(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_above, int dx, int dy, int bd); +RTCD_EXTERN void (*av1_highbd_dr_prediction_z1)(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_above, int dx, int dy, int bd); void av1_highbd_dr_prediction_z2_c(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_above, int upsample_left, int dx, int dy, int bd); #define av1_highbd_dr_prediction_z2 av1_highbd_dr_prediction_z2_c void av1_highbd_dr_prediction_z3_c(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_left, int dx, int dy, int bd); -#define av1_highbd_dr_prediction_z3 av1_highbd_dr_prediction_z3_c +void av1_highbd_dr_prediction_z3_avx2(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_left, int dx, int dy, int bd); +RTCD_EXTERN void (*av1_highbd_dr_prediction_z3)(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_left, int dx, int dy, int bd); void av1_highbd_inv_txfm_add_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); void av1_highbd_inv_txfm_add_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); void av1_highbd_inv_txfm_add_avx2(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); RTCD_EXTERN void (*av1_highbd_inv_txfm_add)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -void av1_highbd_inv_txfm_add_16x16_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -void av1_highbd_inv_txfm_add_16x16_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -RTCD_EXTERN void (*av1_highbd_inv_txfm_add_16x16)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +void av1_highbd_inv_txfm_add_16x4_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +void av1_highbd_inv_txfm_add_16x4_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +RTCD_EXTERN void (*av1_highbd_inv_txfm_add_16x4)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -void av1_highbd_inv_txfm_add_16x8_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -void av1_highbd_inv_txfm_add_16x8_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -RTCD_EXTERN void (*av1_highbd_inv_txfm_add_16x8)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); - -void av1_highbd_inv_txfm_add_32x32_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -void av1_highbd_inv_txfm_add_32x32_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -void av1_highbd_inv_txfm_add_32x32_avx2(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -RTCD_EXTERN void (*av1_highbd_inv_txfm_add_32x32)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +void av1_highbd_inv_txfm_add_4x16_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +void av1_highbd_inv_txfm_add_4x16_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +RTCD_EXTERN void (*av1_highbd_inv_txfm_add_4x16)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); void av1_highbd_inv_txfm_add_4x4_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); void av1_highbd_inv_txfm_add_4x4_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); RTCD_EXTERN void (*av1_highbd_inv_txfm_add_4x4)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -void av1_highbd_inv_txfm_add_8x16_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -void av1_highbd_inv_txfm_add_8x16_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -RTCD_EXTERN void (*av1_highbd_inv_txfm_add_8x16)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +void av1_highbd_inv_txfm_add_4x8_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +void av1_highbd_inv_txfm_add_4x8_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +RTCD_EXTERN void (*av1_highbd_inv_txfm_add_4x8)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); + +void av1_highbd_inv_txfm_add_8x4_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +void av1_highbd_inv_txfm_add_8x4_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +RTCD_EXTERN void (*av1_highbd_inv_txfm_add_8x4)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); void av1_highbd_inv_txfm_add_8x8_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); void av1_highbd_inv_txfm_add_8x8_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); @@ -208,26 +255,6 @@ void av1_highbd_iwht4x4_16_add_c(const tran_low_t *input, uint8_t *dest, int des void av1_highbd_iwht4x4_1_add_c(const tran_low_t *input, uint8_t *dest, int dest_stride, int bd); #define av1_highbd_iwht4x4_1_add av1_highbd_iwht4x4_1_add_c -void av1_highbd_jnt_convolve_2d_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -void av1_highbd_jnt_convolve_2d_sse4_1(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -void av1_highbd_jnt_convolve_2d_avx2(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -RTCD_EXTERN void (*av1_highbd_jnt_convolve_2d)(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); - -void av1_highbd_jnt_convolve_2d_copy_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -void av1_highbd_jnt_convolve_2d_copy_sse4_1(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -void av1_highbd_jnt_convolve_2d_copy_avx2(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -RTCD_EXTERN void (*av1_highbd_jnt_convolve_2d_copy)(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); - -void av1_highbd_jnt_convolve_x_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -void av1_highbd_jnt_convolve_x_sse4_1(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -void av1_highbd_jnt_convolve_x_avx2(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -RTCD_EXTERN void (*av1_highbd_jnt_convolve_x)(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); - -void av1_highbd_jnt_convolve_y_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -void av1_highbd_jnt_convolve_y_sse4_1(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -void av1_highbd_jnt_convolve_y_avx2(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -RTCD_EXTERN void (*av1_highbd_jnt_convolve_y)(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); - void av1_highbd_warp_affine_c(const int32_t *mat, const uint16_t *ref, int width, int height, int stride, uint16_t *pred, int p_col, int p_row, int p_width, int p_height, int p_stride, int subsampling_x, int subsampling_y, int bd, ConvolveParams *conv_params, int16_t alpha, int16_t beta, int16_t gamma, int16_t delta); void av1_highbd_warp_affine_sse4_1(const int32_t *mat, const uint16_t *ref, int width, int height, int stride, uint16_t *pred, int p_col, int p_row, int p_width, int p_height, int p_stride, int subsampling_x, int subsampling_y, int bd, ConvolveParams *conv_params, int16_t alpha, int16_t beta, int16_t gamma, int16_t delta); RTCD_EXTERN void (*av1_highbd_warp_affine)(const int32_t *mat, const uint16_t *ref, int width, int height, int stride, uint16_t *pred, int p_col, int p_row, int p_width, int p_height, int p_stride, int subsampling_x, int subsampling_y, int bd, ConvolveParams *conv_params, int16_t alpha, int16_t beta, int16_t gamma, int16_t delta); @@ -301,25 +328,9 @@ void av1_inv_txfm_add_ssse3(const tran_low_t *dqcoeff, uint8_t *dst, int stride, void av1_inv_txfm_add_avx2(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); RTCD_EXTERN void (*av1_inv_txfm_add)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -void av1_jnt_convolve_2d_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -void av1_jnt_convolve_2d_ssse3(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -void av1_jnt_convolve_2d_avx2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -RTCD_EXTERN void (*av1_jnt_convolve_2d)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); - -void av1_jnt_convolve_2d_copy_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -void av1_jnt_convolve_2d_copy_sse2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -void av1_jnt_convolve_2d_copy_avx2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -RTCD_EXTERN void (*av1_jnt_convolve_2d_copy)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); - -void av1_jnt_convolve_x_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -void av1_jnt_convolve_x_sse2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -void av1_jnt_convolve_x_avx2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -RTCD_EXTERN void (*av1_jnt_convolve_x)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); - -void av1_jnt_convolve_y_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -void av1_jnt_convolve_y_sse2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -void av1_jnt_convolve_y_avx2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -RTCD_EXTERN void (*av1_jnt_convolve_y)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_round_shift_array_c(int32_t *arr, int size, int bit); +void av1_round_shift_array_sse4_1(int32_t *arr, int size, int bit); +RTCD_EXTERN void (*av1_round_shift_array)(int32_t *arr, int size, int bit); int av1_selfguided_restoration_c(const uint8_t *dgd8, int width, int height, int dgd_stride, int32_t *flt0, int32_t *flt1, int flt_stride, @@ -351,12 +362,12 @@ void av1_wiener_convolve_add_src_sse2(const uint8_t *src, ptrdiff_t src_stride, void av1_wiener_convolve_add_src_avx2(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, const int16_t *filter_y, int y_step_q4, int w, int h, const ConvolveParams *conv_params); RTCD_EXTERN void (*av1_wiener_convolve_add_src)(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, const int16_t *filter_y, int y_step_q4, int w, int h, const ConvolveParams *conv_params); -void cdef_filter_block_c(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int max, int coeff_shift); -void cdef_filter_block_sse2(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int max, int coeff_shift); -void cdef_filter_block_ssse3(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int max, int coeff_shift); -void cdef_filter_block_sse4_1(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int max, int coeff_shift); -void cdef_filter_block_avx2(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int max, int coeff_shift); -RTCD_EXTERN void (*cdef_filter_block)(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int max, int coeff_shift); +void cdef_filter_block_c(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int coeff_shift); +void cdef_filter_block_sse2(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int coeff_shift); +void cdef_filter_block_ssse3(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int coeff_shift); +void cdef_filter_block_sse4_1(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int coeff_shift); +void cdef_filter_block_avx2(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int coeff_shift); +RTCD_EXTERN void (*cdef_filter_block)(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int coeff_shift); int cdef_find_dir_c(const uint16_t *img, int stride, int32_t *var, int coeff_shift); int cdef_find_dir_sse2(const uint16_t *img, int stride, int32_t *var, int coeff_shift); @@ -462,6 +473,25 @@ static void setup_rtcd_internal(void) av1_convolve_y_sr = av1_convolve_y_sr_c; if (flags & HAS_SSE2) av1_convolve_y_sr = av1_convolve_y_sr_sse2; if (flags & HAS_AVX2) av1_convolve_y_sr = av1_convolve_y_sr_avx2; + av1_dist_wtd_convolve_2d = av1_dist_wtd_convolve_2d_c; + if (flags & HAS_SSE2) av1_dist_wtd_convolve_2d = av1_dist_wtd_convolve_2d_sse2; + if (flags & HAS_SSSE3) av1_dist_wtd_convolve_2d = av1_dist_wtd_convolve_2d_ssse3; + if (flags & HAS_AVX2) av1_dist_wtd_convolve_2d = av1_dist_wtd_convolve_2d_avx2; + av1_dist_wtd_convolve_2d_copy = av1_dist_wtd_convolve_2d_copy_c; + if (flags & HAS_SSE2) av1_dist_wtd_convolve_2d_copy = av1_dist_wtd_convolve_2d_copy_sse2; + if (flags & HAS_AVX2) av1_dist_wtd_convolve_2d_copy = av1_dist_wtd_convolve_2d_copy_avx2; + av1_dist_wtd_convolve_x = av1_dist_wtd_convolve_x_c; + if (flags & HAS_SSE2) av1_dist_wtd_convolve_x = av1_dist_wtd_convolve_x_sse2; + if (flags & HAS_AVX2) av1_dist_wtd_convolve_x = av1_dist_wtd_convolve_x_avx2; + av1_dist_wtd_convolve_y = av1_dist_wtd_convolve_y_c; + if (flags & HAS_SSE2) av1_dist_wtd_convolve_y = av1_dist_wtd_convolve_y_sse2; + if (flags & HAS_AVX2) av1_dist_wtd_convolve_y = av1_dist_wtd_convolve_y_avx2; + av1_dr_prediction_z1 = av1_dr_prediction_z1_c; + if (flags & HAS_AVX2) av1_dr_prediction_z1 = av1_dr_prediction_z1_avx2; + av1_dr_prediction_z2 = av1_dr_prediction_z2_c; + if (flags & HAS_AVX2) av1_dr_prediction_z2 = av1_dr_prediction_z2_avx2; + av1_dr_prediction_z3 = av1_dr_prediction_z3_c; + if (flags & HAS_AVX2) av1_dr_prediction_z3 = av1_dr_prediction_z3_avx2; av1_filter_intra_edge = av1_filter_intra_edge_c; if (flags & HAS_SSE4_1) av1_filter_intra_edge = av1_filter_intra_edge_sse4_1; av1_filter_intra_edge_high = av1_filter_intra_edge_high_c; @@ -484,34 +514,37 @@ static void setup_rtcd_internal(void) av1_highbd_convolve_y_sr = av1_highbd_convolve_y_sr_c; if (flags & HAS_SSSE3) av1_highbd_convolve_y_sr = av1_highbd_convolve_y_sr_ssse3; if (flags & HAS_AVX2) av1_highbd_convolve_y_sr = av1_highbd_convolve_y_sr_avx2; + av1_highbd_dist_wtd_convolve_2d = av1_highbd_dist_wtd_convolve_2d_c; + if (flags & HAS_SSE4_1) av1_highbd_dist_wtd_convolve_2d = av1_highbd_dist_wtd_convolve_2d_sse4_1; + if (flags & HAS_AVX2) av1_highbd_dist_wtd_convolve_2d = av1_highbd_dist_wtd_convolve_2d_avx2; + av1_highbd_dist_wtd_convolve_2d_copy = av1_highbd_dist_wtd_convolve_2d_copy_c; + if (flags & HAS_SSE4_1) av1_highbd_dist_wtd_convolve_2d_copy = av1_highbd_dist_wtd_convolve_2d_copy_sse4_1; + if (flags & HAS_AVX2) av1_highbd_dist_wtd_convolve_2d_copy = av1_highbd_dist_wtd_convolve_2d_copy_avx2; + av1_highbd_dist_wtd_convolve_x = av1_highbd_dist_wtd_convolve_x_c; + if (flags & HAS_SSE4_1) av1_highbd_dist_wtd_convolve_x = av1_highbd_dist_wtd_convolve_x_sse4_1; + if (flags & HAS_AVX2) av1_highbd_dist_wtd_convolve_x = av1_highbd_dist_wtd_convolve_x_avx2; + av1_highbd_dist_wtd_convolve_y = av1_highbd_dist_wtd_convolve_y_c; + if (flags & HAS_SSE4_1) av1_highbd_dist_wtd_convolve_y = av1_highbd_dist_wtd_convolve_y_sse4_1; + if (flags & HAS_AVX2) av1_highbd_dist_wtd_convolve_y = av1_highbd_dist_wtd_convolve_y_avx2; + av1_highbd_dr_prediction_z1 = av1_highbd_dr_prediction_z1_c; + if (flags & HAS_AVX2) av1_highbd_dr_prediction_z1 = av1_highbd_dr_prediction_z1_avx2; + av1_highbd_dr_prediction_z3 = av1_highbd_dr_prediction_z3_c; + if (flags & HAS_AVX2) av1_highbd_dr_prediction_z3 = av1_highbd_dr_prediction_z3_avx2; av1_highbd_inv_txfm_add = av1_highbd_inv_txfm_add_c; if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add = av1_highbd_inv_txfm_add_sse4_1; if (flags & HAS_AVX2) av1_highbd_inv_txfm_add = av1_highbd_inv_txfm_add_avx2; - av1_highbd_inv_txfm_add_16x16 = av1_highbd_inv_txfm_add_16x16_c; - if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add_16x16 = av1_highbd_inv_txfm_add_16x16_sse4_1; - av1_highbd_inv_txfm_add_16x8 = av1_highbd_inv_txfm_add_16x8_c; - if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add_16x8 = av1_highbd_inv_txfm_add_16x8_sse4_1; - av1_highbd_inv_txfm_add_32x32 = av1_highbd_inv_txfm_add_32x32_c; - if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add_32x32 = av1_highbd_inv_txfm_add_32x32_sse4_1; - if (flags & HAS_AVX2) av1_highbd_inv_txfm_add_32x32 = av1_highbd_inv_txfm_add_32x32_avx2; + av1_highbd_inv_txfm_add_16x4 = av1_highbd_inv_txfm_add_16x4_c; + if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add_16x4 = av1_highbd_inv_txfm_add_16x4_sse4_1; + av1_highbd_inv_txfm_add_4x16 = av1_highbd_inv_txfm_add_4x16_c; + if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add_4x16 = av1_highbd_inv_txfm_add_4x16_sse4_1; av1_highbd_inv_txfm_add_4x4 = av1_highbd_inv_txfm_add_4x4_c; if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add_4x4 = av1_highbd_inv_txfm_add_4x4_sse4_1; - av1_highbd_inv_txfm_add_8x16 = av1_highbd_inv_txfm_add_8x16_c; - if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add_8x16 = av1_highbd_inv_txfm_add_8x16_sse4_1; + av1_highbd_inv_txfm_add_4x8 = av1_highbd_inv_txfm_add_4x8_c; + if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add_4x8 = av1_highbd_inv_txfm_add_4x8_sse4_1; + av1_highbd_inv_txfm_add_8x4 = av1_highbd_inv_txfm_add_8x4_c; + if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add_8x4 = av1_highbd_inv_txfm_add_8x4_sse4_1; av1_highbd_inv_txfm_add_8x8 = av1_highbd_inv_txfm_add_8x8_c; if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add_8x8 = av1_highbd_inv_txfm_add_8x8_sse4_1; - av1_highbd_jnt_convolve_2d = av1_highbd_jnt_convolve_2d_c; - if (flags & HAS_SSE4_1) av1_highbd_jnt_convolve_2d = av1_highbd_jnt_convolve_2d_sse4_1; - if (flags & HAS_AVX2) av1_highbd_jnt_convolve_2d = av1_highbd_jnt_convolve_2d_avx2; - av1_highbd_jnt_convolve_2d_copy = av1_highbd_jnt_convolve_2d_copy_c; - if (flags & HAS_SSE4_1) av1_highbd_jnt_convolve_2d_copy = av1_highbd_jnt_convolve_2d_copy_sse4_1; - if (flags & HAS_AVX2) av1_highbd_jnt_convolve_2d_copy = av1_highbd_jnt_convolve_2d_copy_avx2; - av1_highbd_jnt_convolve_x = av1_highbd_jnt_convolve_x_c; - if (flags & HAS_SSE4_1) av1_highbd_jnt_convolve_x = av1_highbd_jnt_convolve_x_sse4_1; - if (flags & HAS_AVX2) av1_highbd_jnt_convolve_x = av1_highbd_jnt_convolve_x_avx2; - av1_highbd_jnt_convolve_y = av1_highbd_jnt_convolve_y_c; - if (flags & HAS_SSE4_1) av1_highbd_jnt_convolve_y = av1_highbd_jnt_convolve_y_sse4_1; - if (flags & HAS_AVX2) av1_highbd_jnt_convolve_y = av1_highbd_jnt_convolve_y_avx2; av1_highbd_warp_affine = av1_highbd_warp_affine_c; if (flags & HAS_SSE4_1) av1_highbd_warp_affine = av1_highbd_warp_affine_sse4_1; av1_highbd_wiener_convolve_add_src = av1_highbd_wiener_convolve_add_src_c; @@ -524,18 +557,8 @@ static void setup_rtcd_internal(void) av1_inv_txfm_add = av1_inv_txfm_add_c; if (flags & HAS_SSSE3) av1_inv_txfm_add = av1_inv_txfm_add_ssse3; if (flags & HAS_AVX2) av1_inv_txfm_add = av1_inv_txfm_add_avx2; - av1_jnt_convolve_2d = av1_jnt_convolve_2d_c; - if (flags & HAS_SSSE3) av1_jnt_convolve_2d = av1_jnt_convolve_2d_ssse3; - if (flags & HAS_AVX2) av1_jnt_convolve_2d = av1_jnt_convolve_2d_avx2; - av1_jnt_convolve_2d_copy = av1_jnt_convolve_2d_copy_c; - if (flags & HAS_SSE2) av1_jnt_convolve_2d_copy = av1_jnt_convolve_2d_copy_sse2; - if (flags & HAS_AVX2) av1_jnt_convolve_2d_copy = av1_jnt_convolve_2d_copy_avx2; - av1_jnt_convolve_x = av1_jnt_convolve_x_c; - if (flags & HAS_SSE2) av1_jnt_convolve_x = av1_jnt_convolve_x_sse2; - if (flags & HAS_AVX2) av1_jnt_convolve_x = av1_jnt_convolve_x_avx2; - av1_jnt_convolve_y = av1_jnt_convolve_y_c; - if (flags & HAS_SSE2) av1_jnt_convolve_y = av1_jnt_convolve_y_sse2; - if (flags & HAS_AVX2) av1_jnt_convolve_y = av1_jnt_convolve_y_avx2; + av1_round_shift_array = av1_round_shift_array_c; + if (flags & HAS_SSE4_1) av1_round_shift_array = av1_round_shift_array_sse4_1; av1_selfguided_restoration = av1_selfguided_restoration_c; if (flags & HAS_SSE4_1) av1_selfguided_restoration = av1_selfguided_restoration_sse4_1; if (flags & HAS_AVX2) av1_selfguided_restoration = av1_selfguided_restoration_avx2; diff --git a/media/libaom/config/win/mingw32/config/aom_config.asm b/media/libaom/config/win/mingw32/config/aom_config.asm index 6ae776c7c4..b3a8c2d596 100644 --- a/media/libaom/config/win/mingw32/config/aom_config.asm +++ b/media/libaom/config/win/mingw32/config/aom_config.asm @@ -1,5 +1,5 @@ ; -; Copyright (c) 2018, Alliance for Open Media. All rights reserved +; Copyright (c) 2019, Alliance for Open Media. All rights reserved ; ; This source code is subject to the terms of the BSD 2 Clause License and ; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License @@ -22,18 +22,17 @@ CONFIG_AV1_ENCODER equ 0 CONFIG_BIG_ENDIAN equ 0 CONFIG_BITSTREAM_DEBUG equ 0 CONFIG_COEFFICIENT_RANGE_CHECKING equ 0 -CONFIG_COLLECT_INTER_MODE_RD_STATS equ 0 +CONFIG_COLLECT_PARTITION_STATS equ 0 CONFIG_COLLECT_RD_STATS equ 0 CONFIG_DEBUG equ 0 CONFIG_DENOISE equ 1 +CONFIG_DISABLE_FULL_PIXEL_SPLIT_8X8 equ 1 CONFIG_DIST_8X8 equ 0 CONFIG_ENTROPY_STATS equ 0 CONFIG_FILEOPTIONS equ 1 -CONFIG_FIX_GF_LENGTH equ 1 CONFIG_FP_MB_STATS equ 0 CONFIG_GCC equ 1 CONFIG_GCOV equ 0 -CONFIG_GLOBAL_MOTION_SEARCH equ 1 CONFIG_GPROF equ 0 CONFIG_INSPECTION equ 0 CONFIG_INTERNAL_STATS equ 0 @@ -44,15 +43,16 @@ CONFIG_MAX_DECODE_PROFILE equ 2 CONFIG_MISMATCH_DEBUG equ 0 CONFIG_MULTITHREAD equ 1 CONFIG_NORMAL_TILE_MODE equ 0 +CONFIG_ONE_PASS_SVM equ 0 CONFIG_OS_SUPPORT equ 1 CONFIG_PIC equ 0 CONFIG_RD_DEBUG equ 0 -CONFIG_REDUCED_ENCODER_BORDER equ 0 CONFIG_RUNTIME_CPU_DETECT equ 1 CONFIG_SHARED equ 0 CONFIG_SHARP_SETTINGS equ 0 CONFIG_SIZE_LIMIT equ 0 CONFIG_SPATIAL_RESAMPLING equ 1 +CONFIG_SPEED_STATS equ 0 CONFIG_STATIC equ 1 CONFIG_WEBM_IO equ 0 DECODE_HEIGHT_LIMIT equ 0 diff --git a/media/libaom/config/win/mingw32/config/aom_config.h b/media/libaom/config/win/mingw32/config/aom_config.h index 090a455778..394c9d574e 100644 --- a/media/libaom/config/win/mingw32/config/aom_config.h +++ b/media/libaom/config/win/mingw32/config/aom_config.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, Alliance for Open Media. All rights reserved + * Copyright (c) 2019, Alliance for Open Media. All rights reserved * * This source code is subject to the terms of the BSD 2 Clause License and * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License @@ -24,18 +24,17 @@ #define CONFIG_BIG_ENDIAN 0 #define CONFIG_BITSTREAM_DEBUG 0 #define CONFIG_COEFFICIENT_RANGE_CHECKING 0 -#define CONFIG_COLLECT_INTER_MODE_RD_STATS 0 +#define CONFIG_COLLECT_PARTITION_STATS 0 #define CONFIG_COLLECT_RD_STATS 0 #define CONFIG_DEBUG 0 #define CONFIG_DENOISE 1 +#define CONFIG_DISABLE_FULL_PIXEL_SPLIT_8X8 1 #define CONFIG_DIST_8X8 0 #define CONFIG_ENTROPY_STATS 0 #define CONFIG_FILEOPTIONS 1 -#define CONFIG_FIX_GF_LENGTH 1 #define CONFIG_FP_MB_STATS 0 #define CONFIG_GCC 1 #define CONFIG_GCOV 0 -#define CONFIG_GLOBAL_MOTION_SEARCH 1 #define CONFIG_GPROF 0 #define CONFIG_INSPECTION 0 #define CONFIG_INTERNAL_STATS 0 @@ -46,15 +45,16 @@ #define CONFIG_MISMATCH_DEBUG 0 #define CONFIG_MULTITHREAD 1 #define CONFIG_NORMAL_TILE_MODE 0 +#define CONFIG_ONE_PASS_SVM 0 #define CONFIG_OS_SUPPORT 1 #define CONFIG_PIC 0 #define CONFIG_RD_DEBUG 0 -#define CONFIG_REDUCED_ENCODER_BORDER 0 #define CONFIG_RUNTIME_CPU_DETECT 1 #define CONFIG_SHARED 0 #define CONFIG_SHARP_SETTINGS 0 #define CONFIG_SIZE_LIMIT 0 #define CONFIG_SPATIAL_RESAMPLING 1 +#define CONFIG_SPEED_STATS 0 #define CONFIG_STATIC 1 #define CONFIG_WEBM_IO 0 #define DECODE_HEIGHT_LIMIT 0 @@ -76,7 +76,5 @@ #define HAVE_SSSE3 1 #define HAVE_VSX 0 #define HAVE_WXWIDGETS 0 -#define INCLUDE_INSTALL_DIR INSTALLDIR/include #define INLINE inline -#define LIB_INSTALL_DIR INSTALLDIR/lib #endif /* AOM_CONFIG_H_ */ diff --git a/media/libaom/config/win/mingw32/config/aom_dsp_rtcd.h b/media/libaom/config/win/mingw32/config/aom_dsp_rtcd.h index 5b3762f109..a60a7aea6e 100644 --- a/media/libaom/config/win/mingw32/config/aom_dsp_rtcd.h +++ b/media/libaom/config/win/mingw32/config/aom_dsp_rtcd.h @@ -472,7 +472,9 @@ void aom_h_predictor_8x8_sse2(uint8_t *dst, ptrdiff_t y_stride, const uint8_t *a RTCD_EXTERN void (*aom_h_predictor_8x8)(uint8_t *dst, ptrdiff_t y_stride, const uint8_t *above, const uint8_t *left); void aom_highbd_blend_a64_d16_mask_c(uint8_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0, uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride, const uint8_t *mask, uint32_t mask_stride, int w, int h, int subx, int suby, ConvolveParams *conv_params, const int bd); -#define aom_highbd_blend_a64_d16_mask aom_highbd_blend_a64_d16_mask_c +void aom_highbd_blend_a64_d16_mask_sse4_1(uint8_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0, uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride, const uint8_t *mask, uint32_t mask_stride, int w, int h, int subx, int suby, ConvolveParams *conv_params, const int bd); +void aom_highbd_blend_a64_d16_mask_avx2(uint8_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0, uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride, const uint8_t *mask, uint32_t mask_stride, int w, int h, int subx, int suby, ConvolveParams *conv_params, const int bd); +RTCD_EXTERN void (*aom_highbd_blend_a64_d16_mask)(uint8_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0, uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride, const uint8_t *mask, uint32_t mask_stride, int w, int h, int subx, int suby, ConvolveParams *conv_params, const int bd); void aom_highbd_blend_a64_hmask_c(uint8_t *dst, uint32_t dst_stride, const uint8_t *src0, uint32_t src0_stride, const uint8_t *src1, uint32_t src1_stride, const uint8_t *mask, int w, int h, int bd); void aom_highbd_blend_a64_hmask_sse4_1(uint8_t *dst, uint32_t dst_stride, const uint8_t *src0, uint32_t src0_stride, const uint8_t *src1, uint32_t src1_stride, const uint8_t *mask, int w, int h, int bd); @@ -487,10 +489,12 @@ void aom_highbd_blend_a64_vmask_sse4_1(uint8_t *dst, uint32_t dst_stride, const RTCD_EXTERN void (*aom_highbd_blend_a64_vmask)(uint8_t *dst, uint32_t dst_stride, const uint8_t *src0, uint32_t src0_stride, const uint8_t *src1, uint32_t src1_stride, const uint8_t *mask, int w, int h, int bd); void aom_highbd_convolve8_horiz_c(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, const int16_t *filter_y, int y_step_q4, int w, int h, int bps); +void aom_highbd_convolve8_horiz_sse2(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, const int16_t *filter_y, int y_step_q4, int w, int h, int bps); void aom_highbd_convolve8_horiz_avx2(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, const int16_t *filter_y, int y_step_q4, int w, int h, int bps); RTCD_EXTERN void (*aom_highbd_convolve8_horiz)(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, const int16_t *filter_y, int y_step_q4, int w, int h, int bps); void aom_highbd_convolve8_vert_c(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, const int16_t *filter_y, int y_step_q4, int w, int h, int bps); +void aom_highbd_convolve8_vert_sse2(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, const int16_t *filter_y, int y_step_q4, int w, int h, int bps); void aom_highbd_convolve8_vert_avx2(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, const int16_t *filter_y, int y_step_q4, int w, int h, int bps); RTCD_EXTERN void (*aom_highbd_convolve8_vert)(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, const int16_t *filter_y, int y_step_q4, int w, int h, int bps); @@ -1709,10 +1713,6 @@ void aom_v_predictor_8x8_c(uint8_t *dst, ptrdiff_t y_stride, const uint8_t *abov void aom_v_predictor_8x8_sse2(uint8_t *dst, ptrdiff_t y_stride, const uint8_t *above, const uint8_t *left); RTCD_EXTERN void (*aom_v_predictor_8x8)(uint8_t *dst, ptrdiff_t y_stride, const uint8_t *above, const uint8_t *left); -void av1_round_shift_array_c(int32_t *arr, int size, int bit); -void av1_round_shift_array_sse4_1(int32_t *arr, int size, int bit); -RTCD_EXTERN void (*av1_round_shift_array)(int32_t *arr, int size, int bit); - void aom_dsp_rtcd(void); #ifdef RTCD_C @@ -1955,6 +1955,9 @@ static void setup_rtcd_internal(void) if (flags & HAS_SSE2) aom_h_predictor_8x4 = aom_h_predictor_8x4_sse2; aom_h_predictor_8x8 = aom_h_predictor_8x8_c; if (flags & HAS_SSE2) aom_h_predictor_8x8 = aom_h_predictor_8x8_sse2; + aom_highbd_blend_a64_d16_mask = aom_highbd_blend_a64_d16_mask_c; + if (flags & HAS_SSE4_1) aom_highbd_blend_a64_d16_mask = aom_highbd_blend_a64_d16_mask_sse4_1; + if (flags & HAS_AVX2) aom_highbd_blend_a64_d16_mask = aom_highbd_blend_a64_d16_mask_avx2; aom_highbd_blend_a64_hmask = aom_highbd_blend_a64_hmask_c; if (flags & HAS_SSE4_1) aom_highbd_blend_a64_hmask = aom_highbd_blend_a64_hmask_sse4_1; aom_highbd_blend_a64_mask = aom_highbd_blend_a64_mask_c; @@ -1962,8 +1965,10 @@ static void setup_rtcd_internal(void) aom_highbd_blend_a64_vmask = aom_highbd_blend_a64_vmask_c; if (flags & HAS_SSE4_1) aom_highbd_blend_a64_vmask = aom_highbd_blend_a64_vmask_sse4_1; aom_highbd_convolve8_horiz = aom_highbd_convolve8_horiz_c; + if (flags & HAS_SSE2) aom_highbd_convolve8_horiz = aom_highbd_convolve8_horiz_sse2; if (flags & HAS_AVX2) aom_highbd_convolve8_horiz = aom_highbd_convolve8_horiz_avx2; aom_highbd_convolve8_vert = aom_highbd_convolve8_vert_c; + if (flags & HAS_SSE2) aom_highbd_convolve8_vert = aom_highbd_convolve8_vert_sse2; if (flags & HAS_AVX2) aom_highbd_convolve8_vert = aom_highbd_convolve8_vert_avx2; aom_highbd_convolve_copy = aom_highbd_convolve_copy_c; if (flags & HAS_SSE2) aom_highbd_convolve_copy = aom_highbd_convolve_copy_sse2; @@ -2367,8 +2372,6 @@ static void setup_rtcd_internal(void) if (flags & HAS_SSE2) aom_v_predictor_8x4 = aom_v_predictor_8x4_sse2; aom_v_predictor_8x8 = aom_v_predictor_8x8_c; if (flags & HAS_SSE2) aom_v_predictor_8x8 = aom_v_predictor_8x8_sse2; - av1_round_shift_array = av1_round_shift_array_c; - if (flags & HAS_SSE4_1) av1_round_shift_array = av1_round_shift_array_sse4_1; } #endif diff --git a/media/libaom/config/win/mingw32/config/aom_scale_rtcd.h b/media/libaom/config/win/mingw32/config/aom_scale_rtcd.h index 6b56795cd4..b6e814905b 100644 --- a/media/libaom/config/win/mingw32/config/aom_scale_rtcd.h +++ b/media/libaom/config/win/mingw32/config/aom_scale_rtcd.h @@ -59,13 +59,22 @@ void aom_yv12_copy_y_c(const struct yv12_buffer_config *src_ybc, struct yv12_buf void aom_yv12_extend_frame_borders_c(struct yv12_buffer_config *ybf, const int num_planes); #define aom_yv12_extend_frame_borders aom_yv12_extend_frame_borders_c -void aom_yv12_partial_copy_u_c(const struct yv12_buffer_config *src_bc, struct yv12_buffer_config *dst_bc, int hstart, int hend, int vstart, int vend); +void aom_yv12_partial_coloc_copy_u_c(const struct yv12_buffer_config *src_bc, struct yv12_buffer_config *dst_bc, int hstart, int hend, int vstart, int vend); +#define aom_yv12_partial_coloc_copy_u aom_yv12_partial_coloc_copy_u_c + +void aom_yv12_partial_coloc_copy_v_c(const struct yv12_buffer_config *src_bc, struct yv12_buffer_config *dst_bc, int hstart, int hend, int vstart, int vend); +#define aom_yv12_partial_coloc_copy_v aom_yv12_partial_coloc_copy_v_c + +void aom_yv12_partial_coloc_copy_y_c(const struct yv12_buffer_config *src_ybc, struct yv12_buffer_config *dst_ybc, int hstart, int hend, int vstart, int vend); +#define aom_yv12_partial_coloc_copy_y aom_yv12_partial_coloc_copy_y_c + +void aom_yv12_partial_copy_u_c(const struct yv12_buffer_config *src_bc, int hstart1, int hend1, int vstart1, int vend1, struct yv12_buffer_config *dst_bc, int hstart2, int vstart2); #define aom_yv12_partial_copy_u aom_yv12_partial_copy_u_c -void aom_yv12_partial_copy_v_c(const struct yv12_buffer_config *src_bc, struct yv12_buffer_config *dst_bc, int hstart, int hend, int vstart, int vend); +void aom_yv12_partial_copy_v_c(const struct yv12_buffer_config *src_bc, int hstart1, int hend1, int vstart1, int vend1, struct yv12_buffer_config *dst_bc, int hstart2, int vstart2); #define aom_yv12_partial_copy_v aom_yv12_partial_copy_v_c -void aom_yv12_partial_copy_y_c(const struct yv12_buffer_config *src_ybc, struct yv12_buffer_config *dst_ybc, int hstart, int hend, int vstart, int vend); +void aom_yv12_partial_copy_y_c(const struct yv12_buffer_config *src_ybc, int hstart1, int hend1, int vstart1, int vend1, struct yv12_buffer_config *dst_ybc, int hstart2, int vstart2); #define aom_yv12_partial_copy_y aom_yv12_partial_copy_y_c void aom_scale_rtcd(void); diff --git a/media/libaom/config/win/mingw32/config/av1_rtcd.h b/media/libaom/config/win/mingw32/config/av1_rtcd.h index a2a6f14ff5..8e69b643ef 100644 --- a/media/libaom/config/win/mingw32/config/av1_rtcd.h +++ b/media/libaom/config/win/mingw32/config/av1_rtcd.h @@ -31,6 +31,8 @@ struct txfm_param; struct aom_variance_vtable; struct search_site_config; struct yv12_buffer_config; +struct NN_CONFIG; +typedef struct NN_CONFIG NN_CONFIG; /* Function pointers return by CfL functions */ typedef void (*cfl_subsample_lbd_fn)(const uint8_t *input, int input_stride, @@ -99,14 +101,38 @@ void av1_convolve_y_sr_sse2(const uint8_t *src, int src_stride, uint8_t *dst, in void av1_convolve_y_sr_avx2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); RTCD_EXTERN void (*av1_convolve_y_sr)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_2d_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_2d_sse2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_2d_ssse3(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_2d_avx2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +RTCD_EXTERN void (*av1_dist_wtd_convolve_2d)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); + +void av1_dist_wtd_convolve_2d_copy_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_2d_copy_sse2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_2d_copy_avx2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +RTCD_EXTERN void (*av1_dist_wtd_convolve_2d_copy)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); + +void av1_dist_wtd_convolve_x_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_x_sse2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_x_avx2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +RTCD_EXTERN void (*av1_dist_wtd_convolve_x)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); + +void av1_dist_wtd_convolve_y_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_y_sse2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_y_avx2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +RTCD_EXTERN void (*av1_dist_wtd_convolve_y)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); + void av1_dr_prediction_z1_c(uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_above, int dx, int dy); -#define av1_dr_prediction_z1 av1_dr_prediction_z1_c +void av1_dr_prediction_z1_avx2(uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_above, int dx, int dy); +RTCD_EXTERN void (*av1_dr_prediction_z1)(uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_above, int dx, int dy); void av1_dr_prediction_z2_c(uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_above, int upsample_left, int dx, int dy); -#define av1_dr_prediction_z2 av1_dr_prediction_z2_c +void av1_dr_prediction_z2_avx2(uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_above, int upsample_left, int dx, int dy); +RTCD_EXTERN void (*av1_dr_prediction_z2)(uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_above, int upsample_left, int dx, int dy); void av1_dr_prediction_z3_c(uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_left, int dx, int dy); -#define av1_dr_prediction_z3 av1_dr_prediction_z3_c +void av1_dr_prediction_z3_avx2(uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_left, int dx, int dy); +RTCD_EXTERN void (*av1_dr_prediction_z3)(uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_left, int dx, int dy); void av1_filter_intra_edge_c(uint8_t *p, int sz, int strength); void av1_filter_intra_edge_sse4_1(uint8_t *p, int sz, int strength); @@ -163,40 +189,61 @@ void av1_highbd_convolve_y_sr_ssse3(const uint16_t *src, int src_stride, uint16_ void av1_highbd_convolve_y_sr_avx2(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); RTCD_EXTERN void (*av1_highbd_convolve_y_sr)(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +void av1_highbd_dist_wtd_convolve_2d_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +void av1_highbd_dist_wtd_convolve_2d_sse4_1(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +void av1_highbd_dist_wtd_convolve_2d_avx2(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +RTCD_EXTERN void (*av1_highbd_dist_wtd_convolve_2d)(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); + +void av1_highbd_dist_wtd_convolve_2d_copy_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +void av1_highbd_dist_wtd_convolve_2d_copy_sse4_1(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +void av1_highbd_dist_wtd_convolve_2d_copy_avx2(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +RTCD_EXTERN void (*av1_highbd_dist_wtd_convolve_2d_copy)(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); + +void av1_highbd_dist_wtd_convolve_x_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +void av1_highbd_dist_wtd_convolve_x_sse4_1(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +void av1_highbd_dist_wtd_convolve_x_avx2(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +RTCD_EXTERN void (*av1_highbd_dist_wtd_convolve_x)(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); + +void av1_highbd_dist_wtd_convolve_y_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +void av1_highbd_dist_wtd_convolve_y_sse4_1(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +void av1_highbd_dist_wtd_convolve_y_avx2(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +RTCD_EXTERN void (*av1_highbd_dist_wtd_convolve_y)(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); + void av1_highbd_dr_prediction_z1_c(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_above, int dx, int dy, int bd); -#define av1_highbd_dr_prediction_z1 av1_highbd_dr_prediction_z1_c +void av1_highbd_dr_prediction_z1_avx2(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_above, int dx, int dy, int bd); +RTCD_EXTERN void (*av1_highbd_dr_prediction_z1)(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_above, int dx, int dy, int bd); void av1_highbd_dr_prediction_z2_c(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_above, int upsample_left, int dx, int dy, int bd); #define av1_highbd_dr_prediction_z2 av1_highbd_dr_prediction_z2_c void av1_highbd_dr_prediction_z3_c(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_left, int dx, int dy, int bd); -#define av1_highbd_dr_prediction_z3 av1_highbd_dr_prediction_z3_c +void av1_highbd_dr_prediction_z3_avx2(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_left, int dx, int dy, int bd); +RTCD_EXTERN void (*av1_highbd_dr_prediction_z3)(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_left, int dx, int dy, int bd); void av1_highbd_inv_txfm_add_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); void av1_highbd_inv_txfm_add_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); void av1_highbd_inv_txfm_add_avx2(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); RTCD_EXTERN void (*av1_highbd_inv_txfm_add)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -void av1_highbd_inv_txfm_add_16x16_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -void av1_highbd_inv_txfm_add_16x16_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -RTCD_EXTERN void (*av1_highbd_inv_txfm_add_16x16)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +void av1_highbd_inv_txfm_add_16x4_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +void av1_highbd_inv_txfm_add_16x4_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +RTCD_EXTERN void (*av1_highbd_inv_txfm_add_16x4)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -void av1_highbd_inv_txfm_add_16x8_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -void av1_highbd_inv_txfm_add_16x8_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -RTCD_EXTERN void (*av1_highbd_inv_txfm_add_16x8)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); - -void av1_highbd_inv_txfm_add_32x32_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -void av1_highbd_inv_txfm_add_32x32_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -void av1_highbd_inv_txfm_add_32x32_avx2(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -RTCD_EXTERN void (*av1_highbd_inv_txfm_add_32x32)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +void av1_highbd_inv_txfm_add_4x16_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +void av1_highbd_inv_txfm_add_4x16_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +RTCD_EXTERN void (*av1_highbd_inv_txfm_add_4x16)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); void av1_highbd_inv_txfm_add_4x4_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); void av1_highbd_inv_txfm_add_4x4_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); RTCD_EXTERN void (*av1_highbd_inv_txfm_add_4x4)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -void av1_highbd_inv_txfm_add_8x16_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -void av1_highbd_inv_txfm_add_8x16_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -RTCD_EXTERN void (*av1_highbd_inv_txfm_add_8x16)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +void av1_highbd_inv_txfm_add_4x8_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +void av1_highbd_inv_txfm_add_4x8_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +RTCD_EXTERN void (*av1_highbd_inv_txfm_add_4x8)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); + +void av1_highbd_inv_txfm_add_8x4_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +void av1_highbd_inv_txfm_add_8x4_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +RTCD_EXTERN void (*av1_highbd_inv_txfm_add_8x4)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); void av1_highbd_inv_txfm_add_8x8_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); void av1_highbd_inv_txfm_add_8x8_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); @@ -208,26 +255,6 @@ void av1_highbd_iwht4x4_16_add_c(const tran_low_t *input, uint8_t *dest, int des void av1_highbd_iwht4x4_1_add_c(const tran_low_t *input, uint8_t *dest, int dest_stride, int bd); #define av1_highbd_iwht4x4_1_add av1_highbd_iwht4x4_1_add_c -void av1_highbd_jnt_convolve_2d_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -void av1_highbd_jnt_convolve_2d_sse4_1(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -void av1_highbd_jnt_convolve_2d_avx2(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -RTCD_EXTERN void (*av1_highbd_jnt_convolve_2d)(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); - -void av1_highbd_jnt_convolve_2d_copy_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -void av1_highbd_jnt_convolve_2d_copy_sse4_1(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -void av1_highbd_jnt_convolve_2d_copy_avx2(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -RTCD_EXTERN void (*av1_highbd_jnt_convolve_2d_copy)(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); - -void av1_highbd_jnt_convolve_x_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -void av1_highbd_jnt_convolve_x_sse4_1(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -void av1_highbd_jnt_convolve_x_avx2(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -RTCD_EXTERN void (*av1_highbd_jnt_convolve_x)(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); - -void av1_highbd_jnt_convolve_y_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -void av1_highbd_jnt_convolve_y_sse4_1(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -void av1_highbd_jnt_convolve_y_avx2(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -RTCD_EXTERN void (*av1_highbd_jnt_convolve_y)(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); - void av1_highbd_warp_affine_c(const int32_t *mat, const uint16_t *ref, int width, int height, int stride, uint16_t *pred, int p_col, int p_row, int p_width, int p_height, int p_stride, int subsampling_x, int subsampling_y, int bd, ConvolveParams *conv_params, int16_t alpha, int16_t beta, int16_t gamma, int16_t delta); void av1_highbd_warp_affine_sse4_1(const int32_t *mat, const uint16_t *ref, int width, int height, int stride, uint16_t *pred, int p_col, int p_row, int p_width, int p_height, int p_stride, int subsampling_x, int subsampling_y, int bd, ConvolveParams *conv_params, int16_t alpha, int16_t beta, int16_t gamma, int16_t delta); RTCD_EXTERN void (*av1_highbd_warp_affine)(const int32_t *mat, const uint16_t *ref, int width, int height, int stride, uint16_t *pred, int p_col, int p_row, int p_width, int p_height, int p_stride, int subsampling_x, int subsampling_y, int bd, ConvolveParams *conv_params, int16_t alpha, int16_t beta, int16_t gamma, int16_t delta); @@ -301,25 +328,9 @@ void av1_inv_txfm_add_ssse3(const tran_low_t *dqcoeff, uint8_t *dst, int stride, void av1_inv_txfm_add_avx2(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); RTCD_EXTERN void (*av1_inv_txfm_add)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -void av1_jnt_convolve_2d_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -void av1_jnt_convolve_2d_ssse3(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -void av1_jnt_convolve_2d_avx2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -RTCD_EXTERN void (*av1_jnt_convolve_2d)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); - -void av1_jnt_convolve_2d_copy_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -void av1_jnt_convolve_2d_copy_sse2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -void av1_jnt_convolve_2d_copy_avx2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -RTCD_EXTERN void (*av1_jnt_convolve_2d_copy)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); - -void av1_jnt_convolve_x_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -void av1_jnt_convolve_x_sse2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -void av1_jnt_convolve_x_avx2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -RTCD_EXTERN void (*av1_jnt_convolve_x)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); - -void av1_jnt_convolve_y_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -void av1_jnt_convolve_y_sse2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -void av1_jnt_convolve_y_avx2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -RTCD_EXTERN void (*av1_jnt_convolve_y)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_round_shift_array_c(int32_t *arr, int size, int bit); +void av1_round_shift_array_sse4_1(int32_t *arr, int size, int bit); +RTCD_EXTERN void (*av1_round_shift_array)(int32_t *arr, int size, int bit); int av1_selfguided_restoration_c(const uint8_t *dgd8, int width, int height, int dgd_stride, int32_t *flt0, int32_t *flt1, int flt_stride, @@ -351,12 +362,12 @@ void av1_wiener_convolve_add_src_sse2(const uint8_t *src, ptrdiff_t src_stride, void av1_wiener_convolve_add_src_avx2(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, const int16_t *filter_y, int y_step_q4, int w, int h, const ConvolveParams *conv_params); RTCD_EXTERN void (*av1_wiener_convolve_add_src)(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, const int16_t *filter_y, int y_step_q4, int w, int h, const ConvolveParams *conv_params); -void cdef_filter_block_c(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int max, int coeff_shift); -void cdef_filter_block_sse2(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int max, int coeff_shift); -void cdef_filter_block_ssse3(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int max, int coeff_shift); -void cdef_filter_block_sse4_1(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int max, int coeff_shift); -void cdef_filter_block_avx2(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int max, int coeff_shift); -RTCD_EXTERN void (*cdef_filter_block)(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int max, int coeff_shift); +void cdef_filter_block_c(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int coeff_shift); +void cdef_filter_block_sse2(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int coeff_shift); +void cdef_filter_block_ssse3(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int coeff_shift); +void cdef_filter_block_sse4_1(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int coeff_shift); +void cdef_filter_block_avx2(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int coeff_shift); +RTCD_EXTERN void (*cdef_filter_block)(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int coeff_shift); int cdef_find_dir_c(const uint16_t *img, int stride, int32_t *var, int coeff_shift); int cdef_find_dir_sse2(const uint16_t *img, int stride, int32_t *var, int coeff_shift); @@ -462,6 +473,25 @@ static void setup_rtcd_internal(void) av1_convolve_y_sr = av1_convolve_y_sr_c; if (flags & HAS_SSE2) av1_convolve_y_sr = av1_convolve_y_sr_sse2; if (flags & HAS_AVX2) av1_convolve_y_sr = av1_convolve_y_sr_avx2; + av1_dist_wtd_convolve_2d = av1_dist_wtd_convolve_2d_c; + if (flags & HAS_SSE2) av1_dist_wtd_convolve_2d = av1_dist_wtd_convolve_2d_sse2; + if (flags & HAS_SSSE3) av1_dist_wtd_convolve_2d = av1_dist_wtd_convolve_2d_ssse3; + if (flags & HAS_AVX2) av1_dist_wtd_convolve_2d = av1_dist_wtd_convolve_2d_avx2; + av1_dist_wtd_convolve_2d_copy = av1_dist_wtd_convolve_2d_copy_c; + if (flags & HAS_SSE2) av1_dist_wtd_convolve_2d_copy = av1_dist_wtd_convolve_2d_copy_sse2; + if (flags & HAS_AVX2) av1_dist_wtd_convolve_2d_copy = av1_dist_wtd_convolve_2d_copy_avx2; + av1_dist_wtd_convolve_x = av1_dist_wtd_convolve_x_c; + if (flags & HAS_SSE2) av1_dist_wtd_convolve_x = av1_dist_wtd_convolve_x_sse2; + if (flags & HAS_AVX2) av1_dist_wtd_convolve_x = av1_dist_wtd_convolve_x_avx2; + av1_dist_wtd_convolve_y = av1_dist_wtd_convolve_y_c; + if (flags & HAS_SSE2) av1_dist_wtd_convolve_y = av1_dist_wtd_convolve_y_sse2; + if (flags & HAS_AVX2) av1_dist_wtd_convolve_y = av1_dist_wtd_convolve_y_avx2; + av1_dr_prediction_z1 = av1_dr_prediction_z1_c; + if (flags & HAS_AVX2) av1_dr_prediction_z1 = av1_dr_prediction_z1_avx2; + av1_dr_prediction_z2 = av1_dr_prediction_z2_c; + if (flags & HAS_AVX2) av1_dr_prediction_z2 = av1_dr_prediction_z2_avx2; + av1_dr_prediction_z3 = av1_dr_prediction_z3_c; + if (flags & HAS_AVX2) av1_dr_prediction_z3 = av1_dr_prediction_z3_avx2; av1_filter_intra_edge = av1_filter_intra_edge_c; if (flags & HAS_SSE4_1) av1_filter_intra_edge = av1_filter_intra_edge_sse4_1; av1_filter_intra_edge_high = av1_filter_intra_edge_high_c; @@ -484,34 +514,37 @@ static void setup_rtcd_internal(void) av1_highbd_convolve_y_sr = av1_highbd_convolve_y_sr_c; if (flags & HAS_SSSE3) av1_highbd_convolve_y_sr = av1_highbd_convolve_y_sr_ssse3; if (flags & HAS_AVX2) av1_highbd_convolve_y_sr = av1_highbd_convolve_y_sr_avx2; + av1_highbd_dist_wtd_convolve_2d = av1_highbd_dist_wtd_convolve_2d_c; + if (flags & HAS_SSE4_1) av1_highbd_dist_wtd_convolve_2d = av1_highbd_dist_wtd_convolve_2d_sse4_1; + if (flags & HAS_AVX2) av1_highbd_dist_wtd_convolve_2d = av1_highbd_dist_wtd_convolve_2d_avx2; + av1_highbd_dist_wtd_convolve_2d_copy = av1_highbd_dist_wtd_convolve_2d_copy_c; + if (flags & HAS_SSE4_1) av1_highbd_dist_wtd_convolve_2d_copy = av1_highbd_dist_wtd_convolve_2d_copy_sse4_1; + if (flags & HAS_AVX2) av1_highbd_dist_wtd_convolve_2d_copy = av1_highbd_dist_wtd_convolve_2d_copy_avx2; + av1_highbd_dist_wtd_convolve_x = av1_highbd_dist_wtd_convolve_x_c; + if (flags & HAS_SSE4_1) av1_highbd_dist_wtd_convolve_x = av1_highbd_dist_wtd_convolve_x_sse4_1; + if (flags & HAS_AVX2) av1_highbd_dist_wtd_convolve_x = av1_highbd_dist_wtd_convolve_x_avx2; + av1_highbd_dist_wtd_convolve_y = av1_highbd_dist_wtd_convolve_y_c; + if (flags & HAS_SSE4_1) av1_highbd_dist_wtd_convolve_y = av1_highbd_dist_wtd_convolve_y_sse4_1; + if (flags & HAS_AVX2) av1_highbd_dist_wtd_convolve_y = av1_highbd_dist_wtd_convolve_y_avx2; + av1_highbd_dr_prediction_z1 = av1_highbd_dr_prediction_z1_c; + if (flags & HAS_AVX2) av1_highbd_dr_prediction_z1 = av1_highbd_dr_prediction_z1_avx2; + av1_highbd_dr_prediction_z3 = av1_highbd_dr_prediction_z3_c; + if (flags & HAS_AVX2) av1_highbd_dr_prediction_z3 = av1_highbd_dr_prediction_z3_avx2; av1_highbd_inv_txfm_add = av1_highbd_inv_txfm_add_c; if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add = av1_highbd_inv_txfm_add_sse4_1; if (flags & HAS_AVX2) av1_highbd_inv_txfm_add = av1_highbd_inv_txfm_add_avx2; - av1_highbd_inv_txfm_add_16x16 = av1_highbd_inv_txfm_add_16x16_c; - if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add_16x16 = av1_highbd_inv_txfm_add_16x16_sse4_1; - av1_highbd_inv_txfm_add_16x8 = av1_highbd_inv_txfm_add_16x8_c; - if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add_16x8 = av1_highbd_inv_txfm_add_16x8_sse4_1; - av1_highbd_inv_txfm_add_32x32 = av1_highbd_inv_txfm_add_32x32_c; - if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add_32x32 = av1_highbd_inv_txfm_add_32x32_sse4_1; - if (flags & HAS_AVX2) av1_highbd_inv_txfm_add_32x32 = av1_highbd_inv_txfm_add_32x32_avx2; + av1_highbd_inv_txfm_add_16x4 = av1_highbd_inv_txfm_add_16x4_c; + if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add_16x4 = av1_highbd_inv_txfm_add_16x4_sse4_1; + av1_highbd_inv_txfm_add_4x16 = av1_highbd_inv_txfm_add_4x16_c; + if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add_4x16 = av1_highbd_inv_txfm_add_4x16_sse4_1; av1_highbd_inv_txfm_add_4x4 = av1_highbd_inv_txfm_add_4x4_c; if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add_4x4 = av1_highbd_inv_txfm_add_4x4_sse4_1; - av1_highbd_inv_txfm_add_8x16 = av1_highbd_inv_txfm_add_8x16_c; - if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add_8x16 = av1_highbd_inv_txfm_add_8x16_sse4_1; + av1_highbd_inv_txfm_add_4x8 = av1_highbd_inv_txfm_add_4x8_c; + if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add_4x8 = av1_highbd_inv_txfm_add_4x8_sse4_1; + av1_highbd_inv_txfm_add_8x4 = av1_highbd_inv_txfm_add_8x4_c; + if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add_8x4 = av1_highbd_inv_txfm_add_8x4_sse4_1; av1_highbd_inv_txfm_add_8x8 = av1_highbd_inv_txfm_add_8x8_c; if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add_8x8 = av1_highbd_inv_txfm_add_8x8_sse4_1; - av1_highbd_jnt_convolve_2d = av1_highbd_jnt_convolve_2d_c; - if (flags & HAS_SSE4_1) av1_highbd_jnt_convolve_2d = av1_highbd_jnt_convolve_2d_sse4_1; - if (flags & HAS_AVX2) av1_highbd_jnt_convolve_2d = av1_highbd_jnt_convolve_2d_avx2; - av1_highbd_jnt_convolve_2d_copy = av1_highbd_jnt_convolve_2d_copy_c; - if (flags & HAS_SSE4_1) av1_highbd_jnt_convolve_2d_copy = av1_highbd_jnt_convolve_2d_copy_sse4_1; - if (flags & HAS_AVX2) av1_highbd_jnt_convolve_2d_copy = av1_highbd_jnt_convolve_2d_copy_avx2; - av1_highbd_jnt_convolve_x = av1_highbd_jnt_convolve_x_c; - if (flags & HAS_SSE4_1) av1_highbd_jnt_convolve_x = av1_highbd_jnt_convolve_x_sse4_1; - if (flags & HAS_AVX2) av1_highbd_jnt_convolve_x = av1_highbd_jnt_convolve_x_avx2; - av1_highbd_jnt_convolve_y = av1_highbd_jnt_convolve_y_c; - if (flags & HAS_SSE4_1) av1_highbd_jnt_convolve_y = av1_highbd_jnt_convolve_y_sse4_1; - if (flags & HAS_AVX2) av1_highbd_jnt_convolve_y = av1_highbd_jnt_convolve_y_avx2; av1_highbd_warp_affine = av1_highbd_warp_affine_c; if (flags & HAS_SSE4_1) av1_highbd_warp_affine = av1_highbd_warp_affine_sse4_1; av1_highbd_wiener_convolve_add_src = av1_highbd_wiener_convolve_add_src_c; @@ -524,18 +557,8 @@ static void setup_rtcd_internal(void) av1_inv_txfm_add = av1_inv_txfm_add_c; if (flags & HAS_SSSE3) av1_inv_txfm_add = av1_inv_txfm_add_ssse3; if (flags & HAS_AVX2) av1_inv_txfm_add = av1_inv_txfm_add_avx2; - av1_jnt_convolve_2d = av1_jnt_convolve_2d_c; - if (flags & HAS_SSSE3) av1_jnt_convolve_2d = av1_jnt_convolve_2d_ssse3; - if (flags & HAS_AVX2) av1_jnt_convolve_2d = av1_jnt_convolve_2d_avx2; - av1_jnt_convolve_2d_copy = av1_jnt_convolve_2d_copy_c; - if (flags & HAS_SSE2) av1_jnt_convolve_2d_copy = av1_jnt_convolve_2d_copy_sse2; - if (flags & HAS_AVX2) av1_jnt_convolve_2d_copy = av1_jnt_convolve_2d_copy_avx2; - av1_jnt_convolve_x = av1_jnt_convolve_x_c; - if (flags & HAS_SSE2) av1_jnt_convolve_x = av1_jnt_convolve_x_sse2; - if (flags & HAS_AVX2) av1_jnt_convolve_x = av1_jnt_convolve_x_avx2; - av1_jnt_convolve_y = av1_jnt_convolve_y_c; - if (flags & HAS_SSE2) av1_jnt_convolve_y = av1_jnt_convolve_y_sse2; - if (flags & HAS_AVX2) av1_jnt_convolve_y = av1_jnt_convolve_y_avx2; + av1_round_shift_array = av1_round_shift_array_c; + if (flags & HAS_SSE4_1) av1_round_shift_array = av1_round_shift_array_sse4_1; av1_selfguided_restoration = av1_selfguided_restoration_c; if (flags & HAS_SSE4_1) av1_selfguided_restoration = av1_selfguided_restoration_sse4_1; if (flags & HAS_AVX2) av1_selfguided_restoration = av1_selfguided_restoration_avx2; diff --git a/media/libaom/config/win/mingw64/config/aom_config.asm b/media/libaom/config/win/mingw64/config/aom_config.asm index 4ab2dedb42..294d0294b6 100644 --- a/media/libaom/config/win/mingw64/config/aom_config.asm +++ b/media/libaom/config/win/mingw64/config/aom_config.asm @@ -1,5 +1,5 @@ ; -; Copyright (c) 2018, Alliance for Open Media. All rights reserved +; Copyright (c) 2019, Alliance for Open Media. All rights reserved ; ; This source code is subject to the terms of the BSD 2 Clause License and ; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License @@ -22,18 +22,17 @@ CONFIG_AV1_ENCODER equ 0 CONFIG_BIG_ENDIAN equ 0 CONFIG_BITSTREAM_DEBUG equ 0 CONFIG_COEFFICIENT_RANGE_CHECKING equ 0 -CONFIG_COLLECT_INTER_MODE_RD_STATS equ 0 +CONFIG_COLLECT_PARTITION_STATS equ 0 CONFIG_COLLECT_RD_STATS equ 0 CONFIG_DEBUG equ 0 CONFIG_DENOISE equ 1 +CONFIG_DISABLE_FULL_PIXEL_SPLIT_8X8 equ 1 CONFIG_DIST_8X8 equ 0 CONFIG_ENTROPY_STATS equ 0 CONFIG_FILEOPTIONS equ 1 -CONFIG_FIX_GF_LENGTH equ 1 CONFIG_FP_MB_STATS equ 0 CONFIG_GCC equ 1 CONFIG_GCOV equ 0 -CONFIG_GLOBAL_MOTION_SEARCH equ 1 CONFIG_GPROF equ 0 CONFIG_INSPECTION equ 0 CONFIG_INTERNAL_STATS equ 0 @@ -44,15 +43,16 @@ CONFIG_MAX_DECODE_PROFILE equ 2 CONFIG_MISMATCH_DEBUG equ 0 CONFIG_MULTITHREAD equ 1 CONFIG_NORMAL_TILE_MODE equ 0 +CONFIG_ONE_PASS_SVM equ 0 CONFIG_OS_SUPPORT equ 1 CONFIG_PIC equ 0 CONFIG_RD_DEBUG equ 0 -CONFIG_REDUCED_ENCODER_BORDER equ 0 CONFIG_RUNTIME_CPU_DETECT equ 1 CONFIG_SHARED equ 0 CONFIG_SHARP_SETTINGS equ 0 CONFIG_SIZE_LIMIT equ 0 CONFIG_SPATIAL_RESAMPLING equ 1 +CONFIG_SPEED_STATS equ 0 CONFIG_STATIC equ 1 CONFIG_WEBM_IO equ 0 DECODE_HEIGHT_LIMIT equ 0 diff --git a/media/libaom/config/win/mingw64/config/aom_config.h b/media/libaom/config/win/mingw64/config/aom_config.h index 9a0be7cd11..ffff5abe6a 100644 --- a/media/libaom/config/win/mingw64/config/aom_config.h +++ b/media/libaom/config/win/mingw64/config/aom_config.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, Alliance for Open Media. All rights reserved + * Copyright (c) 2019, Alliance for Open Media. All rights reserved * * This source code is subject to the terms of the BSD 2 Clause License and * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License @@ -24,18 +24,17 @@ #define CONFIG_BIG_ENDIAN 0 #define CONFIG_BITSTREAM_DEBUG 0 #define CONFIG_COEFFICIENT_RANGE_CHECKING 0 -#define CONFIG_COLLECT_INTER_MODE_RD_STATS 0 +#define CONFIG_COLLECT_PARTITION_STATS 0 #define CONFIG_COLLECT_RD_STATS 0 #define CONFIG_DEBUG 0 #define CONFIG_DENOISE 1 +#define CONFIG_DISABLE_FULL_PIXEL_SPLIT_8X8 1 #define CONFIG_DIST_8X8 0 #define CONFIG_ENTROPY_STATS 0 #define CONFIG_FILEOPTIONS 1 -#define CONFIG_FIX_GF_LENGTH 1 #define CONFIG_FP_MB_STATS 0 #define CONFIG_GCC 1 #define CONFIG_GCOV 0 -#define CONFIG_GLOBAL_MOTION_SEARCH 1 #define CONFIG_GPROF 0 #define CONFIG_INSPECTION 0 #define CONFIG_INTERNAL_STATS 0 @@ -46,15 +45,16 @@ #define CONFIG_MISMATCH_DEBUG 0 #define CONFIG_MULTITHREAD 1 #define CONFIG_NORMAL_TILE_MODE 0 +#define CONFIG_ONE_PASS_SVM 0 #define CONFIG_OS_SUPPORT 1 #define CONFIG_PIC 0 #define CONFIG_RD_DEBUG 0 -#define CONFIG_REDUCED_ENCODER_BORDER 0 #define CONFIG_RUNTIME_CPU_DETECT 1 #define CONFIG_SHARED 0 #define CONFIG_SHARP_SETTINGS 0 #define CONFIG_SIZE_LIMIT 0 #define CONFIG_SPATIAL_RESAMPLING 1 +#define CONFIG_SPEED_STATS 0 #define CONFIG_STATIC 1 #define CONFIG_WEBM_IO 0 #define DECODE_HEIGHT_LIMIT 0 @@ -76,7 +76,5 @@ #define HAVE_SSSE3 1 #define HAVE_VSX 0 #define HAVE_WXWIDGETS 0 -#define INCLUDE_INSTALL_DIR INSTALLDIR/include #define INLINE inline -#define LIB_INSTALL_DIR INSTALLDIR/lib #endif /* AOM_CONFIG_H_ */ diff --git a/media/libaom/config/win/mingw64/config/aom_dsp_rtcd.h b/media/libaom/config/win/mingw64/config/aom_dsp_rtcd.h index 2856d4ede7..5826faff95 100644 --- a/media/libaom/config/win/mingw64/config/aom_dsp_rtcd.h +++ b/media/libaom/config/win/mingw64/config/aom_dsp_rtcd.h @@ -472,7 +472,9 @@ void aom_h_predictor_8x8_sse2(uint8_t *dst, ptrdiff_t y_stride, const uint8_t *a #define aom_h_predictor_8x8 aom_h_predictor_8x8_sse2 void aom_highbd_blend_a64_d16_mask_c(uint8_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0, uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride, const uint8_t *mask, uint32_t mask_stride, int w, int h, int subx, int suby, ConvolveParams *conv_params, const int bd); -#define aom_highbd_blend_a64_d16_mask aom_highbd_blend_a64_d16_mask_c +void aom_highbd_blend_a64_d16_mask_sse4_1(uint8_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0, uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride, const uint8_t *mask, uint32_t mask_stride, int w, int h, int subx, int suby, ConvolveParams *conv_params, const int bd); +void aom_highbd_blend_a64_d16_mask_avx2(uint8_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0, uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride, const uint8_t *mask, uint32_t mask_stride, int w, int h, int subx, int suby, ConvolveParams *conv_params, const int bd); +RTCD_EXTERN void (*aom_highbd_blend_a64_d16_mask)(uint8_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0, uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride, const uint8_t *mask, uint32_t mask_stride, int w, int h, int subx, int suby, ConvolveParams *conv_params, const int bd); void aom_highbd_blend_a64_hmask_c(uint8_t *dst, uint32_t dst_stride, const uint8_t *src0, uint32_t src0_stride, const uint8_t *src1, uint32_t src1_stride, const uint8_t *mask, int w, int h, int bd); void aom_highbd_blend_a64_hmask_sse4_1(uint8_t *dst, uint32_t dst_stride, const uint8_t *src0, uint32_t src0_stride, const uint8_t *src1, uint32_t src1_stride, const uint8_t *mask, int w, int h, int bd); @@ -1711,10 +1713,6 @@ void aom_v_predictor_8x8_c(uint8_t *dst, ptrdiff_t y_stride, const uint8_t *abov void aom_v_predictor_8x8_sse2(uint8_t *dst, ptrdiff_t y_stride, const uint8_t *above, const uint8_t *left); #define aom_v_predictor_8x8 aom_v_predictor_8x8_sse2 -void av1_round_shift_array_c(int32_t *arr, int size, int bit); -void av1_round_shift_array_sse4_1(int32_t *arr, int size, int bit); -RTCD_EXTERN void (*av1_round_shift_array)(int32_t *arr, int size, int bit); - void aom_dsp_rtcd(void); #ifdef RTCD_C @@ -1788,6 +1786,9 @@ static void setup_rtcd_internal(void) if (flags & HAS_AVX2) aom_dc_top_predictor_64x64 = aom_dc_top_predictor_64x64_avx2; aom_h_predictor_32x32 = aom_h_predictor_32x32_sse2; if (flags & HAS_AVX2) aom_h_predictor_32x32 = aom_h_predictor_32x32_avx2; + aom_highbd_blend_a64_d16_mask = aom_highbd_blend_a64_d16_mask_c; + if (flags & HAS_SSE4_1) aom_highbd_blend_a64_d16_mask = aom_highbd_blend_a64_d16_mask_sse4_1; + if (flags & HAS_AVX2) aom_highbd_blend_a64_d16_mask = aom_highbd_blend_a64_d16_mask_avx2; aom_highbd_blend_a64_hmask = aom_highbd_blend_a64_hmask_c; if (flags & HAS_SSE4_1) aom_highbd_blend_a64_hmask = aom_highbd_blend_a64_hmask_sse4_1; aom_highbd_blend_a64_mask = aom_highbd_blend_a64_mask_c; @@ -1989,8 +1990,6 @@ static void setup_rtcd_internal(void) if (flags & HAS_AVX2) aom_v_predictor_64x32 = aom_v_predictor_64x32_avx2; aom_v_predictor_64x64 = aom_v_predictor_64x64_sse2; if (flags & HAS_AVX2) aom_v_predictor_64x64 = aom_v_predictor_64x64_avx2; - av1_round_shift_array = av1_round_shift_array_c; - if (flags & HAS_SSE4_1) av1_round_shift_array = av1_round_shift_array_sse4_1; } #endif diff --git a/media/libaom/config/win/mingw64/config/aom_scale_rtcd.h b/media/libaom/config/win/mingw64/config/aom_scale_rtcd.h index 6b56795cd4..b6e814905b 100644 --- a/media/libaom/config/win/mingw64/config/aom_scale_rtcd.h +++ b/media/libaom/config/win/mingw64/config/aom_scale_rtcd.h @@ -59,13 +59,22 @@ void aom_yv12_copy_y_c(const struct yv12_buffer_config *src_ybc, struct yv12_buf void aom_yv12_extend_frame_borders_c(struct yv12_buffer_config *ybf, const int num_planes); #define aom_yv12_extend_frame_borders aom_yv12_extend_frame_borders_c -void aom_yv12_partial_copy_u_c(const struct yv12_buffer_config *src_bc, struct yv12_buffer_config *dst_bc, int hstart, int hend, int vstart, int vend); +void aom_yv12_partial_coloc_copy_u_c(const struct yv12_buffer_config *src_bc, struct yv12_buffer_config *dst_bc, int hstart, int hend, int vstart, int vend); +#define aom_yv12_partial_coloc_copy_u aom_yv12_partial_coloc_copy_u_c + +void aom_yv12_partial_coloc_copy_v_c(const struct yv12_buffer_config *src_bc, struct yv12_buffer_config *dst_bc, int hstart, int hend, int vstart, int vend); +#define aom_yv12_partial_coloc_copy_v aom_yv12_partial_coloc_copy_v_c + +void aom_yv12_partial_coloc_copy_y_c(const struct yv12_buffer_config *src_ybc, struct yv12_buffer_config *dst_ybc, int hstart, int hend, int vstart, int vend); +#define aom_yv12_partial_coloc_copy_y aom_yv12_partial_coloc_copy_y_c + +void aom_yv12_partial_copy_u_c(const struct yv12_buffer_config *src_bc, int hstart1, int hend1, int vstart1, int vend1, struct yv12_buffer_config *dst_bc, int hstart2, int vstart2); #define aom_yv12_partial_copy_u aom_yv12_partial_copy_u_c -void aom_yv12_partial_copy_v_c(const struct yv12_buffer_config *src_bc, struct yv12_buffer_config *dst_bc, int hstart, int hend, int vstart, int vend); +void aom_yv12_partial_copy_v_c(const struct yv12_buffer_config *src_bc, int hstart1, int hend1, int vstart1, int vend1, struct yv12_buffer_config *dst_bc, int hstart2, int vstart2); #define aom_yv12_partial_copy_v aom_yv12_partial_copy_v_c -void aom_yv12_partial_copy_y_c(const struct yv12_buffer_config *src_ybc, struct yv12_buffer_config *dst_ybc, int hstart, int hend, int vstart, int vend); +void aom_yv12_partial_copy_y_c(const struct yv12_buffer_config *src_ybc, int hstart1, int hend1, int vstart1, int vend1, struct yv12_buffer_config *dst_ybc, int hstart2, int vstart2); #define aom_yv12_partial_copy_y aom_yv12_partial_copy_y_c void aom_scale_rtcd(void); diff --git a/media/libaom/config/win/mingw64/config/av1_rtcd.h b/media/libaom/config/win/mingw64/config/av1_rtcd.h index d273182086..80bacb8dd5 100644 --- a/media/libaom/config/win/mingw64/config/av1_rtcd.h +++ b/media/libaom/config/win/mingw64/config/av1_rtcd.h @@ -31,6 +31,8 @@ struct txfm_param; struct aom_variance_vtable; struct search_site_config; struct yv12_buffer_config; +struct NN_CONFIG; +typedef struct NN_CONFIG NN_CONFIG; /* Function pointers return by CfL functions */ typedef void (*cfl_subsample_lbd_fn)(const uint8_t *input, int input_stride, @@ -99,14 +101,38 @@ void av1_convolve_y_sr_sse2(const uint8_t *src, int src_stride, uint8_t *dst, in void av1_convolve_y_sr_avx2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); RTCD_EXTERN void (*av1_convolve_y_sr)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_2d_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_2d_sse2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_2d_ssse3(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_2d_avx2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +RTCD_EXTERN void (*av1_dist_wtd_convolve_2d)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); + +void av1_dist_wtd_convolve_2d_copy_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_2d_copy_sse2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_2d_copy_avx2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +RTCD_EXTERN void (*av1_dist_wtd_convolve_2d_copy)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); + +void av1_dist_wtd_convolve_x_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_x_sse2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_x_avx2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +RTCD_EXTERN void (*av1_dist_wtd_convolve_x)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); + +void av1_dist_wtd_convolve_y_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_y_sse2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_y_avx2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +RTCD_EXTERN void (*av1_dist_wtd_convolve_y)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); + void av1_dr_prediction_z1_c(uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_above, int dx, int dy); -#define av1_dr_prediction_z1 av1_dr_prediction_z1_c +void av1_dr_prediction_z1_avx2(uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_above, int dx, int dy); +RTCD_EXTERN void (*av1_dr_prediction_z1)(uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_above, int dx, int dy); void av1_dr_prediction_z2_c(uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_above, int upsample_left, int dx, int dy); -#define av1_dr_prediction_z2 av1_dr_prediction_z2_c +void av1_dr_prediction_z2_avx2(uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_above, int upsample_left, int dx, int dy); +RTCD_EXTERN void (*av1_dr_prediction_z2)(uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_above, int upsample_left, int dx, int dy); void av1_dr_prediction_z3_c(uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_left, int dx, int dy); -#define av1_dr_prediction_z3 av1_dr_prediction_z3_c +void av1_dr_prediction_z3_avx2(uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_left, int dx, int dy); +RTCD_EXTERN void (*av1_dr_prediction_z3)(uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_left, int dx, int dy); void av1_filter_intra_edge_c(uint8_t *p, int sz, int strength); void av1_filter_intra_edge_sse4_1(uint8_t *p, int sz, int strength); @@ -166,40 +192,61 @@ void av1_highbd_convolve_y_sr_ssse3(const uint16_t *src, int src_stride, uint16_ void av1_highbd_convolve_y_sr_avx2(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); RTCD_EXTERN void (*av1_highbd_convolve_y_sr)(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +void av1_highbd_dist_wtd_convolve_2d_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +void av1_highbd_dist_wtd_convolve_2d_sse4_1(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +void av1_highbd_dist_wtd_convolve_2d_avx2(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +RTCD_EXTERN void (*av1_highbd_dist_wtd_convolve_2d)(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); + +void av1_highbd_dist_wtd_convolve_2d_copy_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +void av1_highbd_dist_wtd_convolve_2d_copy_sse4_1(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +void av1_highbd_dist_wtd_convolve_2d_copy_avx2(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +RTCD_EXTERN void (*av1_highbd_dist_wtd_convolve_2d_copy)(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); + +void av1_highbd_dist_wtd_convolve_x_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +void av1_highbd_dist_wtd_convolve_x_sse4_1(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +void av1_highbd_dist_wtd_convolve_x_avx2(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +RTCD_EXTERN void (*av1_highbd_dist_wtd_convolve_x)(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); + +void av1_highbd_dist_wtd_convolve_y_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +void av1_highbd_dist_wtd_convolve_y_sse4_1(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +void av1_highbd_dist_wtd_convolve_y_avx2(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +RTCD_EXTERN void (*av1_highbd_dist_wtd_convolve_y)(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); + void av1_highbd_dr_prediction_z1_c(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_above, int dx, int dy, int bd); -#define av1_highbd_dr_prediction_z1 av1_highbd_dr_prediction_z1_c +void av1_highbd_dr_prediction_z1_avx2(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_above, int dx, int dy, int bd); +RTCD_EXTERN void (*av1_highbd_dr_prediction_z1)(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_above, int dx, int dy, int bd); void av1_highbd_dr_prediction_z2_c(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_above, int upsample_left, int dx, int dy, int bd); #define av1_highbd_dr_prediction_z2 av1_highbd_dr_prediction_z2_c void av1_highbd_dr_prediction_z3_c(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_left, int dx, int dy, int bd); -#define av1_highbd_dr_prediction_z3 av1_highbd_dr_prediction_z3_c +void av1_highbd_dr_prediction_z3_avx2(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_left, int dx, int dy, int bd); +RTCD_EXTERN void (*av1_highbd_dr_prediction_z3)(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_left, int dx, int dy, int bd); void av1_highbd_inv_txfm_add_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); void av1_highbd_inv_txfm_add_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); void av1_highbd_inv_txfm_add_avx2(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); RTCD_EXTERN void (*av1_highbd_inv_txfm_add)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -void av1_highbd_inv_txfm_add_16x16_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -void av1_highbd_inv_txfm_add_16x16_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -RTCD_EXTERN void (*av1_highbd_inv_txfm_add_16x16)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +void av1_highbd_inv_txfm_add_16x4_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +void av1_highbd_inv_txfm_add_16x4_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +RTCD_EXTERN void (*av1_highbd_inv_txfm_add_16x4)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -void av1_highbd_inv_txfm_add_16x8_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -void av1_highbd_inv_txfm_add_16x8_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -RTCD_EXTERN void (*av1_highbd_inv_txfm_add_16x8)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); - -void av1_highbd_inv_txfm_add_32x32_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -void av1_highbd_inv_txfm_add_32x32_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -void av1_highbd_inv_txfm_add_32x32_avx2(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -RTCD_EXTERN void (*av1_highbd_inv_txfm_add_32x32)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +void av1_highbd_inv_txfm_add_4x16_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +void av1_highbd_inv_txfm_add_4x16_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +RTCD_EXTERN void (*av1_highbd_inv_txfm_add_4x16)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); void av1_highbd_inv_txfm_add_4x4_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); void av1_highbd_inv_txfm_add_4x4_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); RTCD_EXTERN void (*av1_highbd_inv_txfm_add_4x4)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -void av1_highbd_inv_txfm_add_8x16_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -void av1_highbd_inv_txfm_add_8x16_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -RTCD_EXTERN void (*av1_highbd_inv_txfm_add_8x16)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +void av1_highbd_inv_txfm_add_4x8_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +void av1_highbd_inv_txfm_add_4x8_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +RTCD_EXTERN void (*av1_highbd_inv_txfm_add_4x8)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); + +void av1_highbd_inv_txfm_add_8x4_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +void av1_highbd_inv_txfm_add_8x4_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +RTCD_EXTERN void (*av1_highbd_inv_txfm_add_8x4)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); void av1_highbd_inv_txfm_add_8x8_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); void av1_highbd_inv_txfm_add_8x8_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); @@ -211,26 +258,6 @@ void av1_highbd_iwht4x4_16_add_c(const tran_low_t *input, uint8_t *dest, int des void av1_highbd_iwht4x4_1_add_c(const tran_low_t *input, uint8_t *dest, int dest_stride, int bd); #define av1_highbd_iwht4x4_1_add av1_highbd_iwht4x4_1_add_c -void av1_highbd_jnt_convolve_2d_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -void av1_highbd_jnt_convolve_2d_sse4_1(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -void av1_highbd_jnt_convolve_2d_avx2(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -RTCD_EXTERN void (*av1_highbd_jnt_convolve_2d)(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); - -void av1_highbd_jnt_convolve_2d_copy_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -void av1_highbd_jnt_convolve_2d_copy_sse4_1(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -void av1_highbd_jnt_convolve_2d_copy_avx2(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -RTCD_EXTERN void (*av1_highbd_jnt_convolve_2d_copy)(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); - -void av1_highbd_jnt_convolve_x_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -void av1_highbd_jnt_convolve_x_sse4_1(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -void av1_highbd_jnt_convolve_x_avx2(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -RTCD_EXTERN void (*av1_highbd_jnt_convolve_x)(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); - -void av1_highbd_jnt_convolve_y_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -void av1_highbd_jnt_convolve_y_sse4_1(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -void av1_highbd_jnt_convolve_y_avx2(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -RTCD_EXTERN void (*av1_highbd_jnt_convolve_y)(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); - void av1_highbd_warp_affine_c(const int32_t *mat, const uint16_t *ref, int width, int height, int stride, uint16_t *pred, int p_col, int p_row, int p_width, int p_height, int p_stride, int subsampling_x, int subsampling_y, int bd, ConvolveParams *conv_params, int16_t alpha, int16_t beta, int16_t gamma, int16_t delta); void av1_highbd_warp_affine_sse4_1(const int32_t *mat, const uint16_t *ref, int width, int height, int stride, uint16_t *pred, int p_col, int p_row, int p_width, int p_height, int p_stride, int subsampling_x, int subsampling_y, int bd, ConvolveParams *conv_params, int16_t alpha, int16_t beta, int16_t gamma, int16_t delta); RTCD_EXTERN void (*av1_highbd_warp_affine)(const int32_t *mat, const uint16_t *ref, int width, int height, int stride, uint16_t *pred, int p_col, int p_row, int p_width, int p_height, int p_stride, int subsampling_x, int subsampling_y, int bd, ConvolveParams *conv_params, int16_t alpha, int16_t beta, int16_t gamma, int16_t delta); @@ -304,25 +331,9 @@ void av1_inv_txfm_add_ssse3(const tran_low_t *dqcoeff, uint8_t *dst, int stride, void av1_inv_txfm_add_avx2(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); RTCD_EXTERN void (*av1_inv_txfm_add)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -void av1_jnt_convolve_2d_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -void av1_jnt_convolve_2d_ssse3(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -void av1_jnt_convolve_2d_avx2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -RTCD_EXTERN void (*av1_jnt_convolve_2d)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); - -void av1_jnt_convolve_2d_copy_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -void av1_jnt_convolve_2d_copy_sse2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -void av1_jnt_convolve_2d_copy_avx2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -RTCD_EXTERN void (*av1_jnt_convolve_2d_copy)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); - -void av1_jnt_convolve_x_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -void av1_jnt_convolve_x_sse2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -void av1_jnt_convolve_x_avx2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -RTCD_EXTERN void (*av1_jnt_convolve_x)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); - -void av1_jnt_convolve_y_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -void av1_jnt_convolve_y_sse2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -void av1_jnt_convolve_y_avx2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -RTCD_EXTERN void (*av1_jnt_convolve_y)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_round_shift_array_c(int32_t *arr, int size, int bit); +void av1_round_shift_array_sse4_1(int32_t *arr, int size, int bit); +RTCD_EXTERN void (*av1_round_shift_array)(int32_t *arr, int size, int bit); int av1_selfguided_restoration_c(const uint8_t *dgd8, int width, int height, int dgd_stride, int32_t *flt0, int32_t *flt1, int flt_stride, @@ -354,12 +365,12 @@ void av1_wiener_convolve_add_src_sse2(const uint8_t *src, ptrdiff_t src_stride, void av1_wiener_convolve_add_src_avx2(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, const int16_t *filter_y, int y_step_q4, int w, int h, const ConvolveParams *conv_params); RTCD_EXTERN void (*av1_wiener_convolve_add_src)(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, const int16_t *filter_y, int y_step_q4, int w, int h, const ConvolveParams *conv_params); -void cdef_filter_block_c(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int max, int coeff_shift); -void cdef_filter_block_sse2(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int max, int coeff_shift); -void cdef_filter_block_ssse3(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int max, int coeff_shift); -void cdef_filter_block_sse4_1(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int max, int coeff_shift); -void cdef_filter_block_avx2(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int max, int coeff_shift); -RTCD_EXTERN void (*cdef_filter_block)(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int max, int coeff_shift); +void cdef_filter_block_c(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int coeff_shift); +void cdef_filter_block_sse2(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int coeff_shift); +void cdef_filter_block_ssse3(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int coeff_shift); +void cdef_filter_block_sse4_1(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int coeff_shift); +void cdef_filter_block_avx2(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int coeff_shift); +RTCD_EXTERN void (*cdef_filter_block)(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int coeff_shift); int cdef_find_dir_c(const uint16_t *img, int stride, int32_t *var, int coeff_shift); int cdef_find_dir_sse2(const uint16_t *img, int stride, int32_t *var, int coeff_shift); @@ -461,6 +472,21 @@ static void setup_rtcd_internal(void) if (flags & HAS_AVX2) av1_convolve_x_sr = av1_convolve_x_sr_avx2; av1_convolve_y_sr = av1_convolve_y_sr_sse2; if (flags & HAS_AVX2) av1_convolve_y_sr = av1_convolve_y_sr_avx2; + av1_dist_wtd_convolve_2d = av1_dist_wtd_convolve_2d_sse2; + if (flags & HAS_SSSE3) av1_dist_wtd_convolve_2d = av1_dist_wtd_convolve_2d_ssse3; + if (flags & HAS_AVX2) av1_dist_wtd_convolve_2d = av1_dist_wtd_convolve_2d_avx2; + av1_dist_wtd_convolve_2d_copy = av1_dist_wtd_convolve_2d_copy_sse2; + if (flags & HAS_AVX2) av1_dist_wtd_convolve_2d_copy = av1_dist_wtd_convolve_2d_copy_avx2; + av1_dist_wtd_convolve_x = av1_dist_wtd_convolve_x_sse2; + if (flags & HAS_AVX2) av1_dist_wtd_convolve_x = av1_dist_wtd_convolve_x_avx2; + av1_dist_wtd_convolve_y = av1_dist_wtd_convolve_y_sse2; + if (flags & HAS_AVX2) av1_dist_wtd_convolve_y = av1_dist_wtd_convolve_y_avx2; + av1_dr_prediction_z1 = av1_dr_prediction_z1_c; + if (flags & HAS_AVX2) av1_dr_prediction_z1 = av1_dr_prediction_z1_avx2; + av1_dr_prediction_z2 = av1_dr_prediction_z2_c; + if (flags & HAS_AVX2) av1_dr_prediction_z2 = av1_dr_prediction_z2_avx2; + av1_dr_prediction_z3 = av1_dr_prediction_z3_c; + if (flags & HAS_AVX2) av1_dr_prediction_z3 = av1_dr_prediction_z3_avx2; av1_filter_intra_edge = av1_filter_intra_edge_c; if (flags & HAS_SSE4_1) av1_filter_intra_edge = av1_filter_intra_edge_sse4_1; av1_filter_intra_edge_high = av1_filter_intra_edge_high_c; @@ -482,34 +508,37 @@ static void setup_rtcd_internal(void) av1_highbd_convolve_y_sr = av1_highbd_convolve_y_sr_c; if (flags & HAS_SSSE3) av1_highbd_convolve_y_sr = av1_highbd_convolve_y_sr_ssse3; if (flags & HAS_AVX2) av1_highbd_convolve_y_sr = av1_highbd_convolve_y_sr_avx2; + av1_highbd_dist_wtd_convolve_2d = av1_highbd_dist_wtd_convolve_2d_c; + if (flags & HAS_SSE4_1) av1_highbd_dist_wtd_convolve_2d = av1_highbd_dist_wtd_convolve_2d_sse4_1; + if (flags & HAS_AVX2) av1_highbd_dist_wtd_convolve_2d = av1_highbd_dist_wtd_convolve_2d_avx2; + av1_highbd_dist_wtd_convolve_2d_copy = av1_highbd_dist_wtd_convolve_2d_copy_c; + if (flags & HAS_SSE4_1) av1_highbd_dist_wtd_convolve_2d_copy = av1_highbd_dist_wtd_convolve_2d_copy_sse4_1; + if (flags & HAS_AVX2) av1_highbd_dist_wtd_convolve_2d_copy = av1_highbd_dist_wtd_convolve_2d_copy_avx2; + av1_highbd_dist_wtd_convolve_x = av1_highbd_dist_wtd_convolve_x_c; + if (flags & HAS_SSE4_1) av1_highbd_dist_wtd_convolve_x = av1_highbd_dist_wtd_convolve_x_sse4_1; + if (flags & HAS_AVX2) av1_highbd_dist_wtd_convolve_x = av1_highbd_dist_wtd_convolve_x_avx2; + av1_highbd_dist_wtd_convolve_y = av1_highbd_dist_wtd_convolve_y_c; + if (flags & HAS_SSE4_1) av1_highbd_dist_wtd_convolve_y = av1_highbd_dist_wtd_convolve_y_sse4_1; + if (flags & HAS_AVX2) av1_highbd_dist_wtd_convolve_y = av1_highbd_dist_wtd_convolve_y_avx2; + av1_highbd_dr_prediction_z1 = av1_highbd_dr_prediction_z1_c; + if (flags & HAS_AVX2) av1_highbd_dr_prediction_z1 = av1_highbd_dr_prediction_z1_avx2; + av1_highbd_dr_prediction_z3 = av1_highbd_dr_prediction_z3_c; + if (flags & HAS_AVX2) av1_highbd_dr_prediction_z3 = av1_highbd_dr_prediction_z3_avx2; av1_highbd_inv_txfm_add = av1_highbd_inv_txfm_add_c; if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add = av1_highbd_inv_txfm_add_sse4_1; if (flags & HAS_AVX2) av1_highbd_inv_txfm_add = av1_highbd_inv_txfm_add_avx2; - av1_highbd_inv_txfm_add_16x16 = av1_highbd_inv_txfm_add_16x16_c; - if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add_16x16 = av1_highbd_inv_txfm_add_16x16_sse4_1; - av1_highbd_inv_txfm_add_16x8 = av1_highbd_inv_txfm_add_16x8_c; - if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add_16x8 = av1_highbd_inv_txfm_add_16x8_sse4_1; - av1_highbd_inv_txfm_add_32x32 = av1_highbd_inv_txfm_add_32x32_c; - if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add_32x32 = av1_highbd_inv_txfm_add_32x32_sse4_1; - if (flags & HAS_AVX2) av1_highbd_inv_txfm_add_32x32 = av1_highbd_inv_txfm_add_32x32_avx2; + av1_highbd_inv_txfm_add_16x4 = av1_highbd_inv_txfm_add_16x4_c; + if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add_16x4 = av1_highbd_inv_txfm_add_16x4_sse4_1; + av1_highbd_inv_txfm_add_4x16 = av1_highbd_inv_txfm_add_4x16_c; + if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add_4x16 = av1_highbd_inv_txfm_add_4x16_sse4_1; av1_highbd_inv_txfm_add_4x4 = av1_highbd_inv_txfm_add_4x4_c; if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add_4x4 = av1_highbd_inv_txfm_add_4x4_sse4_1; - av1_highbd_inv_txfm_add_8x16 = av1_highbd_inv_txfm_add_8x16_c; - if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add_8x16 = av1_highbd_inv_txfm_add_8x16_sse4_1; + av1_highbd_inv_txfm_add_4x8 = av1_highbd_inv_txfm_add_4x8_c; + if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add_4x8 = av1_highbd_inv_txfm_add_4x8_sse4_1; + av1_highbd_inv_txfm_add_8x4 = av1_highbd_inv_txfm_add_8x4_c; + if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add_8x4 = av1_highbd_inv_txfm_add_8x4_sse4_1; av1_highbd_inv_txfm_add_8x8 = av1_highbd_inv_txfm_add_8x8_c; if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add_8x8 = av1_highbd_inv_txfm_add_8x8_sse4_1; - av1_highbd_jnt_convolve_2d = av1_highbd_jnt_convolve_2d_c; - if (flags & HAS_SSE4_1) av1_highbd_jnt_convolve_2d = av1_highbd_jnt_convolve_2d_sse4_1; - if (flags & HAS_AVX2) av1_highbd_jnt_convolve_2d = av1_highbd_jnt_convolve_2d_avx2; - av1_highbd_jnt_convolve_2d_copy = av1_highbd_jnt_convolve_2d_copy_c; - if (flags & HAS_SSE4_1) av1_highbd_jnt_convolve_2d_copy = av1_highbd_jnt_convolve_2d_copy_sse4_1; - if (flags & HAS_AVX2) av1_highbd_jnt_convolve_2d_copy = av1_highbd_jnt_convolve_2d_copy_avx2; - av1_highbd_jnt_convolve_x = av1_highbd_jnt_convolve_x_c; - if (flags & HAS_SSE4_1) av1_highbd_jnt_convolve_x = av1_highbd_jnt_convolve_x_sse4_1; - if (flags & HAS_AVX2) av1_highbd_jnt_convolve_x = av1_highbd_jnt_convolve_x_avx2; - av1_highbd_jnt_convolve_y = av1_highbd_jnt_convolve_y_c; - if (flags & HAS_SSE4_1) av1_highbd_jnt_convolve_y = av1_highbd_jnt_convolve_y_sse4_1; - if (flags & HAS_AVX2) av1_highbd_jnt_convolve_y = av1_highbd_jnt_convolve_y_avx2; av1_highbd_warp_affine = av1_highbd_warp_affine_c; if (flags & HAS_SSE4_1) av1_highbd_warp_affine = av1_highbd_warp_affine_sse4_1; av1_highbd_wiener_convolve_add_src = av1_highbd_wiener_convolve_add_src_c; @@ -522,15 +551,8 @@ static void setup_rtcd_internal(void) av1_inv_txfm_add = av1_inv_txfm_add_c; if (flags & HAS_SSSE3) av1_inv_txfm_add = av1_inv_txfm_add_ssse3; if (flags & HAS_AVX2) av1_inv_txfm_add = av1_inv_txfm_add_avx2; - av1_jnt_convolve_2d = av1_jnt_convolve_2d_c; - if (flags & HAS_SSSE3) av1_jnt_convolve_2d = av1_jnt_convolve_2d_ssse3; - if (flags & HAS_AVX2) av1_jnt_convolve_2d = av1_jnt_convolve_2d_avx2; - av1_jnt_convolve_2d_copy = av1_jnt_convolve_2d_copy_sse2; - if (flags & HAS_AVX2) av1_jnt_convolve_2d_copy = av1_jnt_convolve_2d_copy_avx2; - av1_jnt_convolve_x = av1_jnt_convolve_x_sse2; - if (flags & HAS_AVX2) av1_jnt_convolve_x = av1_jnt_convolve_x_avx2; - av1_jnt_convolve_y = av1_jnt_convolve_y_sse2; - if (flags & HAS_AVX2) av1_jnt_convolve_y = av1_jnt_convolve_y_avx2; + av1_round_shift_array = av1_round_shift_array_c; + if (flags & HAS_SSE4_1) av1_round_shift_array = av1_round_shift_array_sse4_1; av1_selfguided_restoration = av1_selfguided_restoration_c; if (flags & HAS_SSE4_1) av1_selfguided_restoration = av1_selfguided_restoration_sse4_1; if (flags & HAS_AVX2) av1_selfguided_restoration = av1_selfguided_restoration_avx2; diff --git a/media/libaom/config/win/x64/config/aom_config.asm b/media/libaom/config/win/x64/config/aom_config.asm index 4ab2dedb42..294d0294b6 100644 --- a/media/libaom/config/win/x64/config/aom_config.asm +++ b/media/libaom/config/win/x64/config/aom_config.asm @@ -1,5 +1,5 @@ ; -; Copyright (c) 2018, Alliance for Open Media. All rights reserved +; Copyright (c) 2019, Alliance for Open Media. All rights reserved ; ; This source code is subject to the terms of the BSD 2 Clause License and ; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License @@ -22,18 +22,17 @@ CONFIG_AV1_ENCODER equ 0 CONFIG_BIG_ENDIAN equ 0 CONFIG_BITSTREAM_DEBUG equ 0 CONFIG_COEFFICIENT_RANGE_CHECKING equ 0 -CONFIG_COLLECT_INTER_MODE_RD_STATS equ 0 +CONFIG_COLLECT_PARTITION_STATS equ 0 CONFIG_COLLECT_RD_STATS equ 0 CONFIG_DEBUG equ 0 CONFIG_DENOISE equ 1 +CONFIG_DISABLE_FULL_PIXEL_SPLIT_8X8 equ 1 CONFIG_DIST_8X8 equ 0 CONFIG_ENTROPY_STATS equ 0 CONFIG_FILEOPTIONS equ 1 -CONFIG_FIX_GF_LENGTH equ 1 CONFIG_FP_MB_STATS equ 0 CONFIG_GCC equ 1 CONFIG_GCOV equ 0 -CONFIG_GLOBAL_MOTION_SEARCH equ 1 CONFIG_GPROF equ 0 CONFIG_INSPECTION equ 0 CONFIG_INTERNAL_STATS equ 0 @@ -44,15 +43,16 @@ CONFIG_MAX_DECODE_PROFILE equ 2 CONFIG_MISMATCH_DEBUG equ 0 CONFIG_MULTITHREAD equ 1 CONFIG_NORMAL_TILE_MODE equ 0 +CONFIG_ONE_PASS_SVM equ 0 CONFIG_OS_SUPPORT equ 1 CONFIG_PIC equ 0 CONFIG_RD_DEBUG equ 0 -CONFIG_REDUCED_ENCODER_BORDER equ 0 CONFIG_RUNTIME_CPU_DETECT equ 1 CONFIG_SHARED equ 0 CONFIG_SHARP_SETTINGS equ 0 CONFIG_SIZE_LIMIT equ 0 CONFIG_SPATIAL_RESAMPLING equ 1 +CONFIG_SPEED_STATS equ 0 CONFIG_STATIC equ 1 CONFIG_WEBM_IO equ 0 DECODE_HEIGHT_LIMIT equ 0 diff --git a/media/libaom/config/win/x64/config/aom_config.h b/media/libaom/config/win/x64/config/aom_config.h index 9a0be7cd11..ffff5abe6a 100644 --- a/media/libaom/config/win/x64/config/aom_config.h +++ b/media/libaom/config/win/x64/config/aom_config.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, Alliance for Open Media. All rights reserved + * Copyright (c) 2019, Alliance for Open Media. All rights reserved * * This source code is subject to the terms of the BSD 2 Clause License and * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License @@ -24,18 +24,17 @@ #define CONFIG_BIG_ENDIAN 0 #define CONFIG_BITSTREAM_DEBUG 0 #define CONFIG_COEFFICIENT_RANGE_CHECKING 0 -#define CONFIG_COLLECT_INTER_MODE_RD_STATS 0 +#define CONFIG_COLLECT_PARTITION_STATS 0 #define CONFIG_COLLECT_RD_STATS 0 #define CONFIG_DEBUG 0 #define CONFIG_DENOISE 1 +#define CONFIG_DISABLE_FULL_PIXEL_SPLIT_8X8 1 #define CONFIG_DIST_8X8 0 #define CONFIG_ENTROPY_STATS 0 #define CONFIG_FILEOPTIONS 1 -#define CONFIG_FIX_GF_LENGTH 1 #define CONFIG_FP_MB_STATS 0 #define CONFIG_GCC 1 #define CONFIG_GCOV 0 -#define CONFIG_GLOBAL_MOTION_SEARCH 1 #define CONFIG_GPROF 0 #define CONFIG_INSPECTION 0 #define CONFIG_INTERNAL_STATS 0 @@ -46,15 +45,16 @@ #define CONFIG_MISMATCH_DEBUG 0 #define CONFIG_MULTITHREAD 1 #define CONFIG_NORMAL_TILE_MODE 0 +#define CONFIG_ONE_PASS_SVM 0 #define CONFIG_OS_SUPPORT 1 #define CONFIG_PIC 0 #define CONFIG_RD_DEBUG 0 -#define CONFIG_REDUCED_ENCODER_BORDER 0 #define CONFIG_RUNTIME_CPU_DETECT 1 #define CONFIG_SHARED 0 #define CONFIG_SHARP_SETTINGS 0 #define CONFIG_SIZE_LIMIT 0 #define CONFIG_SPATIAL_RESAMPLING 1 +#define CONFIG_SPEED_STATS 0 #define CONFIG_STATIC 1 #define CONFIG_WEBM_IO 0 #define DECODE_HEIGHT_LIMIT 0 @@ -76,7 +76,5 @@ #define HAVE_SSSE3 1 #define HAVE_VSX 0 #define HAVE_WXWIDGETS 0 -#define INCLUDE_INSTALL_DIR INSTALLDIR/include #define INLINE inline -#define LIB_INSTALL_DIR INSTALLDIR/lib #endif /* AOM_CONFIG_H_ */ diff --git a/media/libaom/config/win/x64/config/aom_dsp_rtcd.h b/media/libaom/config/win/x64/config/aom_dsp_rtcd.h index 2856d4ede7..5826faff95 100644 --- a/media/libaom/config/win/x64/config/aom_dsp_rtcd.h +++ b/media/libaom/config/win/x64/config/aom_dsp_rtcd.h @@ -472,7 +472,9 @@ void aom_h_predictor_8x8_sse2(uint8_t *dst, ptrdiff_t y_stride, const uint8_t *a #define aom_h_predictor_8x8 aom_h_predictor_8x8_sse2 void aom_highbd_blend_a64_d16_mask_c(uint8_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0, uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride, const uint8_t *mask, uint32_t mask_stride, int w, int h, int subx, int suby, ConvolveParams *conv_params, const int bd); -#define aom_highbd_blend_a64_d16_mask aom_highbd_blend_a64_d16_mask_c +void aom_highbd_blend_a64_d16_mask_sse4_1(uint8_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0, uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride, const uint8_t *mask, uint32_t mask_stride, int w, int h, int subx, int suby, ConvolveParams *conv_params, const int bd); +void aom_highbd_blend_a64_d16_mask_avx2(uint8_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0, uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride, const uint8_t *mask, uint32_t mask_stride, int w, int h, int subx, int suby, ConvolveParams *conv_params, const int bd); +RTCD_EXTERN void (*aom_highbd_blend_a64_d16_mask)(uint8_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0, uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride, const uint8_t *mask, uint32_t mask_stride, int w, int h, int subx, int suby, ConvolveParams *conv_params, const int bd); void aom_highbd_blend_a64_hmask_c(uint8_t *dst, uint32_t dst_stride, const uint8_t *src0, uint32_t src0_stride, const uint8_t *src1, uint32_t src1_stride, const uint8_t *mask, int w, int h, int bd); void aom_highbd_blend_a64_hmask_sse4_1(uint8_t *dst, uint32_t dst_stride, const uint8_t *src0, uint32_t src0_stride, const uint8_t *src1, uint32_t src1_stride, const uint8_t *mask, int w, int h, int bd); @@ -1711,10 +1713,6 @@ void aom_v_predictor_8x8_c(uint8_t *dst, ptrdiff_t y_stride, const uint8_t *abov void aom_v_predictor_8x8_sse2(uint8_t *dst, ptrdiff_t y_stride, const uint8_t *above, const uint8_t *left); #define aom_v_predictor_8x8 aom_v_predictor_8x8_sse2 -void av1_round_shift_array_c(int32_t *arr, int size, int bit); -void av1_round_shift_array_sse4_1(int32_t *arr, int size, int bit); -RTCD_EXTERN void (*av1_round_shift_array)(int32_t *arr, int size, int bit); - void aom_dsp_rtcd(void); #ifdef RTCD_C @@ -1788,6 +1786,9 @@ static void setup_rtcd_internal(void) if (flags & HAS_AVX2) aom_dc_top_predictor_64x64 = aom_dc_top_predictor_64x64_avx2; aom_h_predictor_32x32 = aom_h_predictor_32x32_sse2; if (flags & HAS_AVX2) aom_h_predictor_32x32 = aom_h_predictor_32x32_avx2; + aom_highbd_blend_a64_d16_mask = aom_highbd_blend_a64_d16_mask_c; + if (flags & HAS_SSE4_1) aom_highbd_blend_a64_d16_mask = aom_highbd_blend_a64_d16_mask_sse4_1; + if (flags & HAS_AVX2) aom_highbd_blend_a64_d16_mask = aom_highbd_blend_a64_d16_mask_avx2; aom_highbd_blend_a64_hmask = aom_highbd_blend_a64_hmask_c; if (flags & HAS_SSE4_1) aom_highbd_blend_a64_hmask = aom_highbd_blend_a64_hmask_sse4_1; aom_highbd_blend_a64_mask = aom_highbd_blend_a64_mask_c; @@ -1989,8 +1990,6 @@ static void setup_rtcd_internal(void) if (flags & HAS_AVX2) aom_v_predictor_64x32 = aom_v_predictor_64x32_avx2; aom_v_predictor_64x64 = aom_v_predictor_64x64_sse2; if (flags & HAS_AVX2) aom_v_predictor_64x64 = aom_v_predictor_64x64_avx2; - av1_round_shift_array = av1_round_shift_array_c; - if (flags & HAS_SSE4_1) av1_round_shift_array = av1_round_shift_array_sse4_1; } #endif diff --git a/media/libaom/config/win/x64/config/aom_scale_rtcd.h b/media/libaom/config/win/x64/config/aom_scale_rtcd.h index 6b56795cd4..b6e814905b 100644 --- a/media/libaom/config/win/x64/config/aom_scale_rtcd.h +++ b/media/libaom/config/win/x64/config/aom_scale_rtcd.h @@ -59,13 +59,22 @@ void aom_yv12_copy_y_c(const struct yv12_buffer_config *src_ybc, struct yv12_buf void aom_yv12_extend_frame_borders_c(struct yv12_buffer_config *ybf, const int num_planes); #define aom_yv12_extend_frame_borders aom_yv12_extend_frame_borders_c -void aom_yv12_partial_copy_u_c(const struct yv12_buffer_config *src_bc, struct yv12_buffer_config *dst_bc, int hstart, int hend, int vstart, int vend); +void aom_yv12_partial_coloc_copy_u_c(const struct yv12_buffer_config *src_bc, struct yv12_buffer_config *dst_bc, int hstart, int hend, int vstart, int vend); +#define aom_yv12_partial_coloc_copy_u aom_yv12_partial_coloc_copy_u_c + +void aom_yv12_partial_coloc_copy_v_c(const struct yv12_buffer_config *src_bc, struct yv12_buffer_config *dst_bc, int hstart, int hend, int vstart, int vend); +#define aom_yv12_partial_coloc_copy_v aom_yv12_partial_coloc_copy_v_c + +void aom_yv12_partial_coloc_copy_y_c(const struct yv12_buffer_config *src_ybc, struct yv12_buffer_config *dst_ybc, int hstart, int hend, int vstart, int vend); +#define aom_yv12_partial_coloc_copy_y aom_yv12_partial_coloc_copy_y_c + +void aom_yv12_partial_copy_u_c(const struct yv12_buffer_config *src_bc, int hstart1, int hend1, int vstart1, int vend1, struct yv12_buffer_config *dst_bc, int hstart2, int vstart2); #define aom_yv12_partial_copy_u aom_yv12_partial_copy_u_c -void aom_yv12_partial_copy_v_c(const struct yv12_buffer_config *src_bc, struct yv12_buffer_config *dst_bc, int hstart, int hend, int vstart, int vend); +void aom_yv12_partial_copy_v_c(const struct yv12_buffer_config *src_bc, int hstart1, int hend1, int vstart1, int vend1, struct yv12_buffer_config *dst_bc, int hstart2, int vstart2); #define aom_yv12_partial_copy_v aom_yv12_partial_copy_v_c -void aom_yv12_partial_copy_y_c(const struct yv12_buffer_config *src_ybc, struct yv12_buffer_config *dst_ybc, int hstart, int hend, int vstart, int vend); +void aom_yv12_partial_copy_y_c(const struct yv12_buffer_config *src_ybc, int hstart1, int hend1, int vstart1, int vend1, struct yv12_buffer_config *dst_ybc, int hstart2, int vstart2); #define aom_yv12_partial_copy_y aom_yv12_partial_copy_y_c void aom_scale_rtcd(void); diff --git a/media/libaom/config/win/x64/config/av1_rtcd.h b/media/libaom/config/win/x64/config/av1_rtcd.h index d273182086..80bacb8dd5 100644 --- a/media/libaom/config/win/x64/config/av1_rtcd.h +++ b/media/libaom/config/win/x64/config/av1_rtcd.h @@ -31,6 +31,8 @@ struct txfm_param; struct aom_variance_vtable; struct search_site_config; struct yv12_buffer_config; +struct NN_CONFIG; +typedef struct NN_CONFIG NN_CONFIG; /* Function pointers return by CfL functions */ typedef void (*cfl_subsample_lbd_fn)(const uint8_t *input, int input_stride, @@ -99,14 +101,38 @@ void av1_convolve_y_sr_sse2(const uint8_t *src, int src_stride, uint8_t *dst, in void av1_convolve_y_sr_avx2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); RTCD_EXTERN void (*av1_convolve_y_sr)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_2d_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_2d_sse2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_2d_ssse3(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_2d_avx2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +RTCD_EXTERN void (*av1_dist_wtd_convolve_2d)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); + +void av1_dist_wtd_convolve_2d_copy_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_2d_copy_sse2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_2d_copy_avx2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +RTCD_EXTERN void (*av1_dist_wtd_convolve_2d_copy)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); + +void av1_dist_wtd_convolve_x_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_x_sse2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_x_avx2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +RTCD_EXTERN void (*av1_dist_wtd_convolve_x)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); + +void av1_dist_wtd_convolve_y_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_y_sse2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_dist_wtd_convolve_y_avx2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +RTCD_EXTERN void (*av1_dist_wtd_convolve_y)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); + void av1_dr_prediction_z1_c(uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_above, int dx, int dy); -#define av1_dr_prediction_z1 av1_dr_prediction_z1_c +void av1_dr_prediction_z1_avx2(uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_above, int dx, int dy); +RTCD_EXTERN void (*av1_dr_prediction_z1)(uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_above, int dx, int dy); void av1_dr_prediction_z2_c(uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_above, int upsample_left, int dx, int dy); -#define av1_dr_prediction_z2 av1_dr_prediction_z2_c +void av1_dr_prediction_z2_avx2(uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_above, int upsample_left, int dx, int dy); +RTCD_EXTERN void (*av1_dr_prediction_z2)(uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_above, int upsample_left, int dx, int dy); void av1_dr_prediction_z3_c(uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_left, int dx, int dy); -#define av1_dr_prediction_z3 av1_dr_prediction_z3_c +void av1_dr_prediction_z3_avx2(uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_left, int dx, int dy); +RTCD_EXTERN void (*av1_dr_prediction_z3)(uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_left, int dx, int dy); void av1_filter_intra_edge_c(uint8_t *p, int sz, int strength); void av1_filter_intra_edge_sse4_1(uint8_t *p, int sz, int strength); @@ -166,40 +192,61 @@ void av1_highbd_convolve_y_sr_ssse3(const uint16_t *src, int src_stride, uint16_ void av1_highbd_convolve_y_sr_avx2(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); RTCD_EXTERN void (*av1_highbd_convolve_y_sr)(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +void av1_highbd_dist_wtd_convolve_2d_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +void av1_highbd_dist_wtd_convolve_2d_sse4_1(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +void av1_highbd_dist_wtd_convolve_2d_avx2(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +RTCD_EXTERN void (*av1_highbd_dist_wtd_convolve_2d)(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); + +void av1_highbd_dist_wtd_convolve_2d_copy_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +void av1_highbd_dist_wtd_convolve_2d_copy_sse4_1(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +void av1_highbd_dist_wtd_convolve_2d_copy_avx2(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +RTCD_EXTERN void (*av1_highbd_dist_wtd_convolve_2d_copy)(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); + +void av1_highbd_dist_wtd_convolve_x_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +void av1_highbd_dist_wtd_convolve_x_sse4_1(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +void av1_highbd_dist_wtd_convolve_x_avx2(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +RTCD_EXTERN void (*av1_highbd_dist_wtd_convolve_x)(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); + +void av1_highbd_dist_wtd_convolve_y_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +void av1_highbd_dist_wtd_convolve_y_sse4_1(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +void av1_highbd_dist_wtd_convolve_y_avx2(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); +RTCD_EXTERN void (*av1_highbd_dist_wtd_convolve_y)(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); + void av1_highbd_dr_prediction_z1_c(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_above, int dx, int dy, int bd); -#define av1_highbd_dr_prediction_z1 av1_highbd_dr_prediction_z1_c +void av1_highbd_dr_prediction_z1_avx2(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_above, int dx, int dy, int bd); +RTCD_EXTERN void (*av1_highbd_dr_prediction_z1)(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_above, int dx, int dy, int bd); void av1_highbd_dr_prediction_z2_c(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_above, int upsample_left, int dx, int dy, int bd); #define av1_highbd_dr_prediction_z2 av1_highbd_dr_prediction_z2_c void av1_highbd_dr_prediction_z3_c(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_left, int dx, int dy, int bd); -#define av1_highbd_dr_prediction_z3 av1_highbd_dr_prediction_z3_c +void av1_highbd_dr_prediction_z3_avx2(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_left, int dx, int dy, int bd); +RTCD_EXTERN void (*av1_highbd_dr_prediction_z3)(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_left, int dx, int dy, int bd); void av1_highbd_inv_txfm_add_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); void av1_highbd_inv_txfm_add_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); void av1_highbd_inv_txfm_add_avx2(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); RTCD_EXTERN void (*av1_highbd_inv_txfm_add)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -void av1_highbd_inv_txfm_add_16x16_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -void av1_highbd_inv_txfm_add_16x16_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -RTCD_EXTERN void (*av1_highbd_inv_txfm_add_16x16)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +void av1_highbd_inv_txfm_add_16x4_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +void av1_highbd_inv_txfm_add_16x4_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +RTCD_EXTERN void (*av1_highbd_inv_txfm_add_16x4)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -void av1_highbd_inv_txfm_add_16x8_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -void av1_highbd_inv_txfm_add_16x8_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -RTCD_EXTERN void (*av1_highbd_inv_txfm_add_16x8)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); - -void av1_highbd_inv_txfm_add_32x32_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -void av1_highbd_inv_txfm_add_32x32_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -void av1_highbd_inv_txfm_add_32x32_avx2(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -RTCD_EXTERN void (*av1_highbd_inv_txfm_add_32x32)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +void av1_highbd_inv_txfm_add_4x16_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +void av1_highbd_inv_txfm_add_4x16_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +RTCD_EXTERN void (*av1_highbd_inv_txfm_add_4x16)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); void av1_highbd_inv_txfm_add_4x4_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); void av1_highbd_inv_txfm_add_4x4_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); RTCD_EXTERN void (*av1_highbd_inv_txfm_add_4x4)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -void av1_highbd_inv_txfm_add_8x16_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -void av1_highbd_inv_txfm_add_8x16_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -RTCD_EXTERN void (*av1_highbd_inv_txfm_add_8x16)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +void av1_highbd_inv_txfm_add_4x8_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +void av1_highbd_inv_txfm_add_4x8_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +RTCD_EXTERN void (*av1_highbd_inv_txfm_add_4x8)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); + +void av1_highbd_inv_txfm_add_8x4_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +void av1_highbd_inv_txfm_add_8x4_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +RTCD_EXTERN void (*av1_highbd_inv_txfm_add_8x4)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); void av1_highbd_inv_txfm_add_8x8_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); void av1_highbd_inv_txfm_add_8x8_sse4_1(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); @@ -211,26 +258,6 @@ void av1_highbd_iwht4x4_16_add_c(const tran_low_t *input, uint8_t *dest, int des void av1_highbd_iwht4x4_1_add_c(const tran_low_t *input, uint8_t *dest, int dest_stride, int bd); #define av1_highbd_iwht4x4_1_add av1_highbd_iwht4x4_1_add_c -void av1_highbd_jnt_convolve_2d_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -void av1_highbd_jnt_convolve_2d_sse4_1(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -void av1_highbd_jnt_convolve_2d_avx2(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -RTCD_EXTERN void (*av1_highbd_jnt_convolve_2d)(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); - -void av1_highbd_jnt_convolve_2d_copy_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -void av1_highbd_jnt_convolve_2d_copy_sse4_1(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -void av1_highbd_jnt_convolve_2d_copy_avx2(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -RTCD_EXTERN void (*av1_highbd_jnt_convolve_2d_copy)(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); - -void av1_highbd_jnt_convolve_x_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -void av1_highbd_jnt_convolve_x_sse4_1(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -void av1_highbd_jnt_convolve_x_avx2(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -RTCD_EXTERN void (*av1_highbd_jnt_convolve_x)(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); - -void av1_highbd_jnt_convolve_y_c(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -void av1_highbd_jnt_convolve_y_sse4_1(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -void av1_highbd_jnt_convolve_y_avx2(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); -RTCD_EXTERN void (*av1_highbd_jnt_convolve_y)(const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd); - void av1_highbd_warp_affine_c(const int32_t *mat, const uint16_t *ref, int width, int height, int stride, uint16_t *pred, int p_col, int p_row, int p_width, int p_height, int p_stride, int subsampling_x, int subsampling_y, int bd, ConvolveParams *conv_params, int16_t alpha, int16_t beta, int16_t gamma, int16_t delta); void av1_highbd_warp_affine_sse4_1(const int32_t *mat, const uint16_t *ref, int width, int height, int stride, uint16_t *pred, int p_col, int p_row, int p_width, int p_height, int p_stride, int subsampling_x, int subsampling_y, int bd, ConvolveParams *conv_params, int16_t alpha, int16_t beta, int16_t gamma, int16_t delta); RTCD_EXTERN void (*av1_highbd_warp_affine)(const int32_t *mat, const uint16_t *ref, int width, int height, int stride, uint16_t *pred, int p_col, int p_row, int p_width, int p_height, int p_stride, int subsampling_x, int subsampling_y, int bd, ConvolveParams *conv_params, int16_t alpha, int16_t beta, int16_t gamma, int16_t delta); @@ -304,25 +331,9 @@ void av1_inv_txfm_add_ssse3(const tran_low_t *dqcoeff, uint8_t *dst, int stride, void av1_inv_txfm_add_avx2(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); RTCD_EXTERN void (*av1_inv_txfm_add)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -void av1_jnt_convolve_2d_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -void av1_jnt_convolve_2d_ssse3(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -void av1_jnt_convolve_2d_avx2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -RTCD_EXTERN void (*av1_jnt_convolve_2d)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); - -void av1_jnt_convolve_2d_copy_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -void av1_jnt_convolve_2d_copy_sse2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -void av1_jnt_convolve_2d_copy_avx2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -RTCD_EXTERN void (*av1_jnt_convolve_2d_copy)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); - -void av1_jnt_convolve_x_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -void av1_jnt_convolve_x_sse2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -void av1_jnt_convolve_x_avx2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -RTCD_EXTERN void (*av1_jnt_convolve_x)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); - -void av1_jnt_convolve_y_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -void av1_jnt_convolve_y_sse2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -void av1_jnt_convolve_y_avx2(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); -RTCD_EXTERN void (*av1_jnt_convolve_y)(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params); +void av1_round_shift_array_c(int32_t *arr, int size, int bit); +void av1_round_shift_array_sse4_1(int32_t *arr, int size, int bit); +RTCD_EXTERN void (*av1_round_shift_array)(int32_t *arr, int size, int bit); int av1_selfguided_restoration_c(const uint8_t *dgd8, int width, int height, int dgd_stride, int32_t *flt0, int32_t *flt1, int flt_stride, @@ -354,12 +365,12 @@ void av1_wiener_convolve_add_src_sse2(const uint8_t *src, ptrdiff_t src_stride, void av1_wiener_convolve_add_src_avx2(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, const int16_t *filter_y, int y_step_q4, int w, int h, const ConvolveParams *conv_params); RTCD_EXTERN void (*av1_wiener_convolve_add_src)(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, const int16_t *filter_y, int y_step_q4, int w, int h, const ConvolveParams *conv_params); -void cdef_filter_block_c(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int max, int coeff_shift); -void cdef_filter_block_sse2(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int max, int coeff_shift); -void cdef_filter_block_ssse3(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int max, int coeff_shift); -void cdef_filter_block_sse4_1(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int max, int coeff_shift); -void cdef_filter_block_avx2(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int max, int coeff_shift); -RTCD_EXTERN void (*cdef_filter_block)(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int max, int coeff_shift); +void cdef_filter_block_c(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int coeff_shift); +void cdef_filter_block_sse2(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int coeff_shift); +void cdef_filter_block_ssse3(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int coeff_shift); +void cdef_filter_block_sse4_1(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int coeff_shift); +void cdef_filter_block_avx2(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int coeff_shift); +RTCD_EXTERN void (*cdef_filter_block)(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int coeff_shift); int cdef_find_dir_c(const uint16_t *img, int stride, int32_t *var, int coeff_shift); int cdef_find_dir_sse2(const uint16_t *img, int stride, int32_t *var, int coeff_shift); @@ -461,6 +472,21 @@ static void setup_rtcd_internal(void) if (flags & HAS_AVX2) av1_convolve_x_sr = av1_convolve_x_sr_avx2; av1_convolve_y_sr = av1_convolve_y_sr_sse2; if (flags & HAS_AVX2) av1_convolve_y_sr = av1_convolve_y_sr_avx2; + av1_dist_wtd_convolve_2d = av1_dist_wtd_convolve_2d_sse2; + if (flags & HAS_SSSE3) av1_dist_wtd_convolve_2d = av1_dist_wtd_convolve_2d_ssse3; + if (flags & HAS_AVX2) av1_dist_wtd_convolve_2d = av1_dist_wtd_convolve_2d_avx2; + av1_dist_wtd_convolve_2d_copy = av1_dist_wtd_convolve_2d_copy_sse2; + if (flags & HAS_AVX2) av1_dist_wtd_convolve_2d_copy = av1_dist_wtd_convolve_2d_copy_avx2; + av1_dist_wtd_convolve_x = av1_dist_wtd_convolve_x_sse2; + if (flags & HAS_AVX2) av1_dist_wtd_convolve_x = av1_dist_wtd_convolve_x_avx2; + av1_dist_wtd_convolve_y = av1_dist_wtd_convolve_y_sse2; + if (flags & HAS_AVX2) av1_dist_wtd_convolve_y = av1_dist_wtd_convolve_y_avx2; + av1_dr_prediction_z1 = av1_dr_prediction_z1_c; + if (flags & HAS_AVX2) av1_dr_prediction_z1 = av1_dr_prediction_z1_avx2; + av1_dr_prediction_z2 = av1_dr_prediction_z2_c; + if (flags & HAS_AVX2) av1_dr_prediction_z2 = av1_dr_prediction_z2_avx2; + av1_dr_prediction_z3 = av1_dr_prediction_z3_c; + if (flags & HAS_AVX2) av1_dr_prediction_z3 = av1_dr_prediction_z3_avx2; av1_filter_intra_edge = av1_filter_intra_edge_c; if (flags & HAS_SSE4_1) av1_filter_intra_edge = av1_filter_intra_edge_sse4_1; av1_filter_intra_edge_high = av1_filter_intra_edge_high_c; @@ -482,34 +508,37 @@ static void setup_rtcd_internal(void) av1_highbd_convolve_y_sr = av1_highbd_convolve_y_sr_c; if (flags & HAS_SSSE3) av1_highbd_convolve_y_sr = av1_highbd_convolve_y_sr_ssse3; if (flags & HAS_AVX2) av1_highbd_convolve_y_sr = av1_highbd_convolve_y_sr_avx2; + av1_highbd_dist_wtd_convolve_2d = av1_highbd_dist_wtd_convolve_2d_c; + if (flags & HAS_SSE4_1) av1_highbd_dist_wtd_convolve_2d = av1_highbd_dist_wtd_convolve_2d_sse4_1; + if (flags & HAS_AVX2) av1_highbd_dist_wtd_convolve_2d = av1_highbd_dist_wtd_convolve_2d_avx2; + av1_highbd_dist_wtd_convolve_2d_copy = av1_highbd_dist_wtd_convolve_2d_copy_c; + if (flags & HAS_SSE4_1) av1_highbd_dist_wtd_convolve_2d_copy = av1_highbd_dist_wtd_convolve_2d_copy_sse4_1; + if (flags & HAS_AVX2) av1_highbd_dist_wtd_convolve_2d_copy = av1_highbd_dist_wtd_convolve_2d_copy_avx2; + av1_highbd_dist_wtd_convolve_x = av1_highbd_dist_wtd_convolve_x_c; + if (flags & HAS_SSE4_1) av1_highbd_dist_wtd_convolve_x = av1_highbd_dist_wtd_convolve_x_sse4_1; + if (flags & HAS_AVX2) av1_highbd_dist_wtd_convolve_x = av1_highbd_dist_wtd_convolve_x_avx2; + av1_highbd_dist_wtd_convolve_y = av1_highbd_dist_wtd_convolve_y_c; + if (flags & HAS_SSE4_1) av1_highbd_dist_wtd_convolve_y = av1_highbd_dist_wtd_convolve_y_sse4_1; + if (flags & HAS_AVX2) av1_highbd_dist_wtd_convolve_y = av1_highbd_dist_wtd_convolve_y_avx2; + av1_highbd_dr_prediction_z1 = av1_highbd_dr_prediction_z1_c; + if (flags & HAS_AVX2) av1_highbd_dr_prediction_z1 = av1_highbd_dr_prediction_z1_avx2; + av1_highbd_dr_prediction_z3 = av1_highbd_dr_prediction_z3_c; + if (flags & HAS_AVX2) av1_highbd_dr_prediction_z3 = av1_highbd_dr_prediction_z3_avx2; av1_highbd_inv_txfm_add = av1_highbd_inv_txfm_add_c; if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add = av1_highbd_inv_txfm_add_sse4_1; if (flags & HAS_AVX2) av1_highbd_inv_txfm_add = av1_highbd_inv_txfm_add_avx2; - av1_highbd_inv_txfm_add_16x16 = av1_highbd_inv_txfm_add_16x16_c; - if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add_16x16 = av1_highbd_inv_txfm_add_16x16_sse4_1; - av1_highbd_inv_txfm_add_16x8 = av1_highbd_inv_txfm_add_16x8_c; - if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add_16x8 = av1_highbd_inv_txfm_add_16x8_sse4_1; - av1_highbd_inv_txfm_add_32x32 = av1_highbd_inv_txfm_add_32x32_c; - if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add_32x32 = av1_highbd_inv_txfm_add_32x32_sse4_1; - if (flags & HAS_AVX2) av1_highbd_inv_txfm_add_32x32 = av1_highbd_inv_txfm_add_32x32_avx2; + av1_highbd_inv_txfm_add_16x4 = av1_highbd_inv_txfm_add_16x4_c; + if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add_16x4 = av1_highbd_inv_txfm_add_16x4_sse4_1; + av1_highbd_inv_txfm_add_4x16 = av1_highbd_inv_txfm_add_4x16_c; + if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add_4x16 = av1_highbd_inv_txfm_add_4x16_sse4_1; av1_highbd_inv_txfm_add_4x4 = av1_highbd_inv_txfm_add_4x4_c; if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add_4x4 = av1_highbd_inv_txfm_add_4x4_sse4_1; - av1_highbd_inv_txfm_add_8x16 = av1_highbd_inv_txfm_add_8x16_c; - if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add_8x16 = av1_highbd_inv_txfm_add_8x16_sse4_1; + av1_highbd_inv_txfm_add_4x8 = av1_highbd_inv_txfm_add_4x8_c; + if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add_4x8 = av1_highbd_inv_txfm_add_4x8_sse4_1; + av1_highbd_inv_txfm_add_8x4 = av1_highbd_inv_txfm_add_8x4_c; + if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add_8x4 = av1_highbd_inv_txfm_add_8x4_sse4_1; av1_highbd_inv_txfm_add_8x8 = av1_highbd_inv_txfm_add_8x8_c; if (flags & HAS_SSE4_1) av1_highbd_inv_txfm_add_8x8 = av1_highbd_inv_txfm_add_8x8_sse4_1; - av1_highbd_jnt_convolve_2d = av1_highbd_jnt_convolve_2d_c; - if (flags & HAS_SSE4_1) av1_highbd_jnt_convolve_2d = av1_highbd_jnt_convolve_2d_sse4_1; - if (flags & HAS_AVX2) av1_highbd_jnt_convolve_2d = av1_highbd_jnt_convolve_2d_avx2; - av1_highbd_jnt_convolve_2d_copy = av1_highbd_jnt_convolve_2d_copy_c; - if (flags & HAS_SSE4_1) av1_highbd_jnt_convolve_2d_copy = av1_highbd_jnt_convolve_2d_copy_sse4_1; - if (flags & HAS_AVX2) av1_highbd_jnt_convolve_2d_copy = av1_highbd_jnt_convolve_2d_copy_avx2; - av1_highbd_jnt_convolve_x = av1_highbd_jnt_convolve_x_c; - if (flags & HAS_SSE4_1) av1_highbd_jnt_convolve_x = av1_highbd_jnt_convolve_x_sse4_1; - if (flags & HAS_AVX2) av1_highbd_jnt_convolve_x = av1_highbd_jnt_convolve_x_avx2; - av1_highbd_jnt_convolve_y = av1_highbd_jnt_convolve_y_c; - if (flags & HAS_SSE4_1) av1_highbd_jnt_convolve_y = av1_highbd_jnt_convolve_y_sse4_1; - if (flags & HAS_AVX2) av1_highbd_jnt_convolve_y = av1_highbd_jnt_convolve_y_avx2; av1_highbd_warp_affine = av1_highbd_warp_affine_c; if (flags & HAS_SSE4_1) av1_highbd_warp_affine = av1_highbd_warp_affine_sse4_1; av1_highbd_wiener_convolve_add_src = av1_highbd_wiener_convolve_add_src_c; @@ -522,15 +551,8 @@ static void setup_rtcd_internal(void) av1_inv_txfm_add = av1_inv_txfm_add_c; if (flags & HAS_SSSE3) av1_inv_txfm_add = av1_inv_txfm_add_ssse3; if (flags & HAS_AVX2) av1_inv_txfm_add = av1_inv_txfm_add_avx2; - av1_jnt_convolve_2d = av1_jnt_convolve_2d_c; - if (flags & HAS_SSSE3) av1_jnt_convolve_2d = av1_jnt_convolve_2d_ssse3; - if (flags & HAS_AVX2) av1_jnt_convolve_2d = av1_jnt_convolve_2d_avx2; - av1_jnt_convolve_2d_copy = av1_jnt_convolve_2d_copy_sse2; - if (flags & HAS_AVX2) av1_jnt_convolve_2d_copy = av1_jnt_convolve_2d_copy_avx2; - av1_jnt_convolve_x = av1_jnt_convolve_x_sse2; - if (flags & HAS_AVX2) av1_jnt_convolve_x = av1_jnt_convolve_x_avx2; - av1_jnt_convolve_y = av1_jnt_convolve_y_sse2; - if (flags & HAS_AVX2) av1_jnt_convolve_y = av1_jnt_convolve_y_avx2; + av1_round_shift_array = av1_round_shift_array_c; + if (flags & HAS_SSE4_1) av1_round_shift_array = av1_round_shift_array_sse4_1; av1_selfguided_restoration = av1_selfguided_restoration_c; if (flags & HAS_SSE4_1) av1_selfguided_restoration = av1_selfguided_restoration_sse4_1; if (flags & HAS_AVX2) av1_selfguided_restoration = av1_selfguided_restoration_avx2; diff --git a/media/libaom/sources.mozbuild b/media/libaom/sources.mozbuild index 9aff837c49..b939b54328 100644 --- a/media/libaom/sources.mozbuild +++ b/media/libaom/sources.mozbuild @@ -1,4 +1,4 @@ -# This file is automatically generated. Do not edit. +# This file is generated. Do not edit. files = { 'ARM_EXPORTS': [ @@ -34,6 +34,7 @@ files = { '../../third_party/aom/aom_dsp/arm/intrapred_neon.c', '../../third_party/aom/aom_dsp/arm/loopfilter_neon.c', '../../third_party/aom/aom_dsp/arm/subtract_neon.c', + '../../third_party/aom/aom_dsp/avg.c', '../../third_party/aom/aom_dsp/binary_codes_reader.c', '../../third_party/aom/aom_dsp/bitreader_buffer.c', '../../third_party/aom/aom_dsp/bitwriter_buffer.c', @@ -109,7 +110,6 @@ files = { '../../third_party/aom/av1/decoder/decoder.c', '../../third_party/aom/av1/decoder/decodetxb.c', '../../third_party/aom/av1/decoder/detokenize.c', - '../../third_party/aom/av1/decoder/dthread.c', '../../third_party/aom/av1/decoder/obu.c', '../../third_party/aom/av1/encoder/arm/neon/quantize_neon.c', ], @@ -140,6 +140,7 @@ files = { '../../third_party/aom/aom/src/aom_integer.c', '../../third_party/aom/aom_dsp/aom_convolve.c', '../../third_party/aom/aom_dsp/aom_dsp_rtcd.c', + '../../third_party/aom/aom_dsp/avg.c', '../../third_party/aom/aom_dsp/binary_codes_reader.c', '../../third_party/aom/aom_dsp/bitreader_buffer.c', '../../third_party/aom/aom_dsp/bitwriter_buffer.c', @@ -202,7 +203,6 @@ files = { '../../third_party/aom/av1/decoder/decoder.c', '../../third_party/aom/av1/decoder/decodetxb.c', '../../third_party/aom/av1/decoder/detokenize.c', - '../../third_party/aom/av1/decoder/dthread.c', '../../third_party/aom/av1/decoder/obu.c', ], 'IA32_EXPORTS': [ @@ -232,6 +232,7 @@ files = { '../../third_party/aom/aom/src/aom_integer.c', '../../third_party/aom/aom_dsp/aom_convolve.c', '../../third_party/aom/aom_dsp/aom_dsp_rtcd.c', + '../../third_party/aom/aom_dsp/avg.c', '../../third_party/aom/aom_dsp/binary_codes_reader.c', '../../third_party/aom/aom_dsp/bitreader_buffer.c', '../../third_party/aom/aom_dsp/bitwriter_buffer.c', @@ -251,11 +252,14 @@ files = { '../../third_party/aom/aom_dsp/x86/aom_high_subpixel_8t_sse2.asm', '../../third_party/aom/aom_dsp/x86/aom_high_subpixel_bilinear_sse2.asm', '../../third_party/aom/aom_dsp/x86/aom_subpixel_8t_intrin_avx2.c', + '../../third_party/aom/aom_dsp/x86/aom_subpixel_8t_intrin_sse2.c', '../../third_party/aom/aom_dsp/x86/aom_subpixel_8t_intrin_ssse3.c', '../../third_party/aom/aom_dsp/x86/aom_subpixel_8t_sse2.asm', '../../third_party/aom/aom_dsp/x86/aom_subpixel_8t_ssse3.asm', '../../third_party/aom/aom_dsp/x86/aom_subpixel_bilinear_sse2.asm', '../../third_party/aom/aom_dsp/x86/aom_subpixel_bilinear_ssse3.asm', + '../../third_party/aom/aom_dsp/x86/avg_intrin_avx2.c', + '../../third_party/aom/aom_dsp/x86/avg_intrin_sse2.c', '../../third_party/aom/aom_dsp/x86/blend_a64_hmask_sse4.c', '../../third_party/aom/aom_dsp/x86/blend_a64_mask_avx2.c', '../../third_party/aom/aom_dsp/x86/blend_a64_mask_sse4.c', @@ -263,14 +267,15 @@ files = { '../../third_party/aom/aom_dsp/x86/fft_avx2.c', '../../third_party/aom/aom_dsp/x86/fft_sse2.c', '../../third_party/aom/aom_dsp/x86/highbd_convolve_avx2.c', + '../../third_party/aom/aom_dsp/x86/highbd_convolve_sse2.c', '../../third_party/aom/aom_dsp/x86/highbd_convolve_ssse3.c', + '../../third_party/aom/aom_dsp/x86/highbd_intrapred_asm_sse2.asm', '../../third_party/aom/aom_dsp/x86/highbd_intrapred_sse2.c', - '../../third_party/aom/aom_dsp/x86/highbd_intrapred_sse2_asm.asm', '../../third_party/aom/aom_dsp/x86/highbd_loopfilter_avx2.c', '../../third_party/aom/aom_dsp/x86/highbd_loopfilter_sse2.c', + '../../third_party/aom/aom_dsp/x86/intrapred_asm_sse2.asm', '../../third_party/aom/aom_dsp/x86/intrapred_avx2.c', '../../third_party/aom/aom_dsp/x86/intrapred_sse2.c', - '../../third_party/aom/aom_dsp/x86/intrapred_sse2_asm.asm', '../../third_party/aom/aom_dsp/x86/intrapred_ssse3.c', '../../third_party/aom/aom_dsp/x86/inv_wht_sse2.asm', '../../third_party/aom/aom_dsp/x86/loopfilter_sse2.c', @@ -365,7 +370,6 @@ files = { '../../third_party/aom/av1/decoder/decoder.c', '../../third_party/aom/av1/decoder/decodetxb.c', '../../third_party/aom/av1/decoder/detokenize.c', - '../../third_party/aom/av1/decoder/dthread.c', '../../third_party/aom/av1/decoder/obu.c', ], 'X64_EXPORTS': [ @@ -395,6 +399,7 @@ files = { '../../third_party/aom/aom/src/aom_integer.c', '../../third_party/aom/aom_dsp/aom_convolve.c', '../../third_party/aom/aom_dsp/aom_dsp_rtcd.c', + '../../third_party/aom/aom_dsp/avg.c', '../../third_party/aom/aom_dsp/binary_codes_reader.c', '../../third_party/aom/aom_dsp/bitreader_buffer.c', '../../third_party/aom/aom_dsp/bitwriter_buffer.c', @@ -414,11 +419,14 @@ files = { '../../third_party/aom/aom_dsp/x86/aom_high_subpixel_8t_sse2.asm', '../../third_party/aom/aom_dsp/x86/aom_high_subpixel_bilinear_sse2.asm', '../../third_party/aom/aom_dsp/x86/aom_subpixel_8t_intrin_avx2.c', + '../../third_party/aom/aom_dsp/x86/aom_subpixel_8t_intrin_sse2.c', '../../third_party/aom/aom_dsp/x86/aom_subpixel_8t_intrin_ssse3.c', '../../third_party/aom/aom_dsp/x86/aom_subpixel_8t_sse2.asm', '../../third_party/aom/aom_dsp/x86/aom_subpixel_8t_ssse3.asm', '../../third_party/aom/aom_dsp/x86/aom_subpixel_bilinear_sse2.asm', '../../third_party/aom/aom_dsp/x86/aom_subpixel_bilinear_ssse3.asm', + '../../third_party/aom/aom_dsp/x86/avg_intrin_avx2.c', + '../../third_party/aom/aom_dsp/x86/avg_intrin_sse2.c', '../../third_party/aom/aom_dsp/x86/blend_a64_hmask_sse4.c', '../../third_party/aom/aom_dsp/x86/blend_a64_mask_avx2.c', '../../third_party/aom/aom_dsp/x86/blend_a64_mask_sse4.c', @@ -426,14 +434,15 @@ files = { '../../third_party/aom/aom_dsp/x86/fft_avx2.c', '../../third_party/aom/aom_dsp/x86/fft_sse2.c', '../../third_party/aom/aom_dsp/x86/highbd_convolve_avx2.c', + '../../third_party/aom/aom_dsp/x86/highbd_convolve_sse2.c', '../../third_party/aom/aom_dsp/x86/highbd_convolve_ssse3.c', + '../../third_party/aom/aom_dsp/x86/highbd_intrapred_asm_sse2.asm', '../../third_party/aom/aom_dsp/x86/highbd_intrapred_sse2.c', - '../../third_party/aom/aom_dsp/x86/highbd_intrapred_sse2_asm.asm', '../../third_party/aom/aom_dsp/x86/highbd_loopfilter_avx2.c', '../../third_party/aom/aom_dsp/x86/highbd_loopfilter_sse2.c', + '../../third_party/aom/aom_dsp/x86/intrapred_asm_sse2.asm', '../../third_party/aom/aom_dsp/x86/intrapred_avx2.c', '../../third_party/aom/aom_dsp/x86/intrapred_sse2.c', - '../../third_party/aom/aom_dsp/x86/intrapred_sse2_asm.asm', '../../third_party/aom/aom_dsp/x86/intrapred_ssse3.c', '../../third_party/aom/aom_dsp/x86/inv_wht_sse2.asm', '../../third_party/aom/aom_dsp/x86/loopfilter_sse2.c', @@ -527,7 +536,6 @@ files = { '../../third_party/aom/av1/decoder/decoder.c', '../../third_party/aom/av1/decoder/decodetxb.c', '../../third_party/aom/av1/decoder/detokenize.c', - '../../third_party/aom/av1/decoder/dthread.c', '../../third_party/aom/av1/decoder/obu.c', ], } diff --git a/third_party/aom/CMakeLists.txt b/third_party/aom/CMakeLists.txt index a58e54f40a..f409892d27 100644 --- a/third_party/aom/CMakeLists.txt +++ b/third_party/aom/CMakeLists.txt @@ -22,10 +22,6 @@ project(AOM C CXX) set(AOM_ROOT "${CMAKE_CURRENT_SOURCE_DIR}") set(AOM_CONFIG_DIR "${CMAKE_CURRENT_BINARY_DIR}") -set(INCLUDE_INSTALL_DIR "${CMAKE_INSTALL_PREFIX}/include" - CACHE PATH "Installation path of includes") -set(LIB_INSTALL_DIR "${CMAKE_INSTALL_PREFIX}/lib" - CACHE PATH "Installation path of libraries") if("${AOM_ROOT}" STREQUAL "${AOM_CONFIG_DIR}") message(FATAL_ERROR @@ -42,9 +38,10 @@ include("${AOM_ROOT}/aom_ports/aom_ports.cmake") include("${AOM_ROOT}/aom_scale/aom_scale.cmake") include("${AOM_ROOT}/aom_util/aom_util.cmake") include("${AOM_ROOT}/av1/av1.cmake") -include("${AOM_ROOT}/test/test.cmake") +include("${AOM_ROOT}/build/cmake/aom_install.cmake") include("${AOM_ROOT}/build/cmake/sanitizers.cmake") include("${AOM_ROOT}/build/cmake/util.cmake") +include("${AOM_ROOT}/test/test.cmake") list(APPEND AOM_RTCD_SOURCES "${AOM_CONFIG_DIR}/config/aom_dsp_rtcd.h" @@ -158,8 +155,6 @@ list(APPEND AOM_ENCODER_STATS_SOURCES "${AOM_ROOT}/stats/aomstats.c" "${AOM_ROOT}/stats/aomstats.h" "${AOM_ROOT}/stats/rate_hist.c" "${AOM_ROOT}/stats/rate_hist.h") -list(APPEND AOM_PKG_CONFIG_SOURCES "${AOM_CONFIG_DIR}/aom.pc") - list(APPEND AOM_VERSION_SOURCES "${AOM_CONFIG_DIR}/config/aom_version.h") list(APPEND AOM_WEBM_DECODER_SOURCES "${AOM_ROOT}/common/webmdec.cc" @@ -192,22 +187,6 @@ add_custom_target(aom_version_check COMMENT "Updating version info if necessary." VERBATIM) add_dependencies(aom_version aom_version_check) -if(NOT MSVC) - add_library(aom_pc ${AOM_PKG_CONFIG_SOURCES}) - add_dummy_source_file_to_target(aom_pc c) - add_custom_command(OUTPUT "${AOM_CONFIG_DIR}/aom.pc" - COMMAND ${CMAKE_COMMAND} ARGS - -DAOM_CONFIG_DIR=${AOM_CONFIG_DIR} - -DAOM_ROOT=${AOM_ROOT} - -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX} - -DCMAKE_PROJECT_NAME=${CMAKE_PROJECT_NAME} - -DCONFIG_MULTITHREAD=${CONFIG_MULTITHREAD} - -DHAVE_PTHREAD_H=${HAVE_PTHREAD_H} -P - "${AOM_ROOT}/build/cmake/pkg_config.cmake" - COMMENT "Writing aom.pc" VERBATIM) - add_dependencies(aom_pc aom_version) -endif() - # TODO(tomfinegan): Move rtcd target setup where it belongs for each rtcd # source. add_rtcd_build_step("${AOM_ROOT}/aom_dsp/aom_dsp_rtcd_defs.pl" @@ -234,8 +213,7 @@ if(NOT MSVC AND NOT APPLE) endif() # List of object and static library targets. -set(AOM_LIB_TARGETS ${AOM_LIB_TARGETS} aom_rtcd aom_encoder_stats aom_mem - aom_scale aom) +set(AOM_LIB_TARGETS ${AOM_LIB_TARGETS} aom_rtcd aom_mem aom_scale aom) # Setup dependencies. setup_aom_dsp_targets() @@ -403,7 +381,6 @@ endif() if(ENABLE_TOOLS) if(CONFIG_AV1_DECODER) - require_cxx_flag_nomsvc("-std=c++11" NO) add_executable(dump_obu "${AOM_GEN_SRC_DIR}/usage_exit.cc" "${AOM_ROOT}/tools/dump_obu.cc" "${AOM_ROOT}/tools/obu_parser.cc" @@ -414,10 +391,6 @@ if(ENABLE_TOOLS) list(APPEND AOM_TOOL_TARGETS dump_obu) list(APPEND AOM_APP_TARGETS dump_obu) - if(NOT MSVC) - target_compile_options(dump_obu PUBLIC -std=c++11) - endif() - # Maintain a separate variable listing only the examples to facilitate # installation of example programs into an tools sub directory of # $AOM_DIST_DIR/bin when building the dist target. @@ -489,8 +462,6 @@ if(ENABLE_EXAMPLES OR ENABLE_TESTS OR ENABLE_TOOLS) endif() if(CONFIG_WEBM_IO) - require_cxx_flag_nomsvc("-std=c++11" NO) - add_library(webm OBJECT ${AOM_LIBWEBM_SOURCES}) include_directories("${AOM_ROOT}/third_party/libwebm") target_compile_definitions(webm PRIVATE __STDC_CONSTANT_MACROS) @@ -498,7 +469,6 @@ if(ENABLE_EXAMPLES OR ENABLE_TESTS OR ENABLE_TOOLS) if(NOT MSVC) target_compile_options(webm PRIVATE -Wno-shadow) - target_compile_options(webm PUBLIC -std=c++11) endif() # Add to existing targets. @@ -593,46 +563,6 @@ if(ENABLE_DOCS) endif() endif() -if(NOT (MSVC OR XCODE)) - - # Aomedia install rule. - list(APPEND AOM_INSTALL_INCS "${AOM_ROOT}/aom/aom.h" - "${AOM_ROOT}/aom/aom_codec.h" - "${AOM_ROOT}/aom/aom_frame_buffer.h" - "${AOM_ROOT}/aom/aom_image.h" "${AOM_ROOT}/aom/aom_integer.h" - "${AOM_ROOT}/aom/aom.h") - - if(CONFIG_AV1_DECODER) - if(ENABLE_EXAMPLES) - list(APPEND AOM_INSTALL_BINS aomdec) - endif() - - list(APPEND AOM_INSTALL_INCS "${AOM_ROOT}/aom/aom_decoder.h" - "${AOM_ROOT}/aom/aomdx.h") - endif() - - if(CONFIG_AV1_ENCODER) - if(ENABLE_EXAMPLES) - list(APPEND AOM_INSTALL_BINS aomenc) - endif() - - list(APPEND AOM_INSTALL_INCS "${AOM_ROOT}/aom/aomcx.h" - "${AOM_ROOT}/aom/aom_encoder.h") - endif() - - set(AOM_INSTALL_LIBS aom) - - install(FILES ${AOM_INSTALL_INCS} DESTINATION "${INCLUDE_INSTALL_DIR}/aom") - install(FILES "${AOM_CONFIG_DIR}/aom.pc" DESTINATION - "${LIB_INSTALL_DIR}/pkgconfig") - install(TARGETS ${AOM_INSTALL_LIBS} DESTINATION "${LIB_INSTALL_DIR}") - - if(ENABLE_EXAMPLES) - install(TARGETS ${AOM_INSTALL_BINS} DESTINATION - "${CMAKE_INSTALL_PREFIX}/bin") - endif() -endif() - # Aomedia dist rule. if(CONFIG_AV1_DECODER AND ENABLE_EXAMPLES) list(APPEND AOM_DIST_APPS $) @@ -756,3 +686,6 @@ foreach(aom_source_var ${aom_source_vars}) file(APPEND "${libaom_srcs_gni_file}" "]\n") endif() endforeach() + +# Generate aom.pc and setup install rule. +setup_aom_install_targets() diff --git a/third_party/aom/PATENTS b/third_party/aom/PATENTS index 97842e02f1..493f616370 100644 --- a/third_party/aom/PATENTS +++ b/third_party/aom/PATENTS @@ -57,10 +57,10 @@ Alliance for Open Media Patent License 1.0 2. Definitions. -2.1. Affiliate. �Affiliate� means an entity that directly or indirectly +2.1. Affiliate. "Affiliate" means an entity that directly or indirectly Controls, is Controlled by, or is under common Control of that party. -2.2. Control. �Control� means direct or indirect control of more than 50% of +2.2. Control. "Control" means direct or indirect control of more than 50% of the voting power to elect directors of that corporation, or for any other entity, the power to direct management of such entity. @@ -70,7 +70,7 @@ Alliance for Open Media Patent License 1.0 2.4. Encoder. "Encoder" means any encoder that produces a bitstream that can be decoded by a Decoder only to the extent it produces such a bitstream. -2.5. Final Deliverable. �Final Deliverable� means the final version of a +2.5. Final Deliverable. "Final Deliverable" means the final version of a deliverable approved by the Alliance for Open Media as a Final Deliverable. @@ -79,9 +79,9 @@ Alliance for Open Media Patent License 1.0 Implementation also includes components of an Implementation only to the extent they are used as part of an Implementation. -2.7. License. �License� means this license. +2.7. License. "License" means this license. -2.8. Licensee. �Licensee� means any person or entity who exercises patent +2.8. Licensee. "Licensee" means any person or entity who exercises patent rights granted under this License. 2.9. Licensor. "Licensor" means (i) any Licensee that makes, sells, offers @@ -98,11 +98,11 @@ Alliance for Open Media Patent License 1.0 as if the Specification was a W3C Recommendation; or (ii) are infringed by the Reference Implementation. -2.11. Reference Implementation. �Reference Implementation� means an Encoder +2.11. Reference Implementation. "Reference Implementation" means an Encoder and/or Decoder released by the Alliance for Open Media as a Final Deliverable. -2.12. Specification. �Specification� means the specification designated by +2.12. Specification. "Specification" means the specification designated by the Alliance for Open Media as a Final Deliverable for which this License was issued. diff --git a/third_party/aom/README.md b/third_party/aom/README.md index cab3f9993f..6b58d35408 100644 --- a/third_party/aom/README.md +++ b/third_party/aom/README.md @@ -210,15 +210,18 @@ compiler documentation to determine which, if any, are available. ### Microsoft Visual Studio builds -Building the AV1 codec library in Microsoft Visual Studio is supported. The -following example demonstrates generating projects and a solution for the -Microsoft IDE: +Building the AV1 codec library in Microsoft Visual Studio is supported. Visual +Studio 2015 (14.0) or later is required. The following example demonstrates +generating projects and a solution for the Microsoft IDE: ~~~ # This does not require a bash shell; command.exe is fine. $ cmake path/to/aom -G "Visual Studio 15 2017" ~~~ +NOTE: The build system targets Windows 7 or later by compiling files with +`-D_WIN32_WINNT=0x0601`. + ### Xcode builds Building the AV1 codec library in Xcode is supported. The following example diff --git a/third_party/aom/aom/aom_decoder.h b/third_party/aom/aom/aom_decoder.h index 06c2dc5f7b..70420c39e7 100644 --- a/third_party/aom/aom/aom_decoder.h +++ b/third_party/aom/aom/aom_decoder.h @@ -55,8 +55,6 @@ extern "C" { #define AOM_CODEC_CAP_PUT_SLICE 0x10000 /**< Will issue put_slice callbacks */ #define AOM_CODEC_CAP_PUT_FRAME 0x20000 /**< Will issue put_frame callbacks */ #define AOM_CODEC_CAP_POSTPROC 0x40000 /**< Can postprocess decoded frame */ -/*!\brief Can receive encoded frames one fragment at a time */ -#define AOM_CODEC_CAP_INPUT_FRAGMENTS 0x100000 /*! \brief Initialization-time Feature Enabling * @@ -65,15 +63,10 @@ extern "C" { * * The available flags are specified by AOM_CODEC_USE_* defines. */ -/*!\brief Can support frame-based multi-threading */ -#define AOM_CODEC_CAP_FRAME_THREADING 0x200000 /*!brief Can support external frame buffers */ -#define AOM_CODEC_CAP_EXTERNAL_FRAME_BUFFER 0x400000 +#define AOM_CODEC_CAP_EXTERNAL_FRAME_BUFFER 0x200000 #define AOM_CODEC_USE_POSTPROC 0x10000 /**< Postprocess decoded frame */ -/*!\brief The input frame should be passed to the decoder one fragment at a - * time */ -#define AOM_CODEC_USE_INPUT_FRAGMENTS 0x40000 /*!\brief Stream properties * @@ -191,13 +184,6 @@ aom_codec_err_t aom_codec_get_stream_info(aom_codec_ctx_t *ctx, * generated, as appropriate. Encoded data \ref MUST be passed in DTS (decode * time stamp) order. Frames produced will always be in PTS (presentation * time stamp) order. - * If the decoder is configured with AOM_CODEC_USE_INPUT_FRAGMENTS enabled, - * data and data_sz can contain a fragment of the encoded frame. Fragment - * \#n must contain at least partition \#n, but can also contain subsequent - * partitions (\#n+1 - \#n+i), and if so, fragments \#n+1, .., \#n+i must - * be empty. When no more data is available, this function should be called - * with NULL as data and 0 as data_sz. The memory passed to this function - * must be available until the frame has been decoded. * * \param[in] ctx Pointer to this instance's context * \param[in] data Pointer to this block of new coded data. If diff --git a/third_party/aom/aom/aom_encoder.h b/third_party/aom/aom/aom_encoder.h index 0894ca9e31..777236f8a2 100644 --- a/third_party/aom/aom/aom_encoder.h +++ b/third_party/aom/aom/aom_encoder.h @@ -97,10 +97,14 @@ typedef uint32_t aom_codec_frame_flags_t; /*!\brief frame can be dropped without affecting the stream (no future frame * depends on this one) */ #define AOM_FRAME_IS_DROPPABLE 0x2 -/*!\brief frame should be decoded but will not be shown */ -#define AOM_FRAME_IS_INVISIBLE 0x4 -/*!\brief this is a fragment of the encoded frame */ -#define AOM_FRAME_IS_FRAGMENT 0x8 +/*!\brief this is an INTRA_ONLY frame */ +#define AOM_FRAME_IS_INTRAONLY 0x10 +/*!\brief this is an S-frame */ +#define AOM_FRAME_IS_SWITCH 0x20 +/*!\brief this is an error-resilient frame */ +#define AOM_FRAME_IS_ERROR_RESILIENT 0x40 +/*!\brief this is a key-frame dependent recovery-point frame */ +#define AOM_FRAME_IS_DELAYED_RANDOM_ACCESS_POINT 0x80 /*!\brief Error Resilient flags * diff --git a/third_party/aom/aom/aom_frame_buffer.h b/third_party/aom/aom/aom_frame_buffer.h index fba4322f85..a715645a7a 100644 --- a/third_party/aom/aom/aom_frame_buffer.h +++ b/third_party/aom/aom/aom_frame_buffer.h @@ -53,9 +53,9 @@ typedef struct aom_codec_frame_buffer { * data. The callback is triggered when the decoder needs a frame buffer to * decode a compressed image into. This function may be called more than once * for every call to aom_codec_decode. The application may set fb->priv to - * some data which will be passed back in the ximage and the release function - * call. |fb| is guaranteed to not be NULL. On success the callback must - * return 0. Any failure the callback must return a value less than 0. + * some data which will be passed back in the aom_image_t and the release + * function call. |fb| is guaranteed to not be NULL. On success the callback + * must return 0. Any failure the callback must return a value less than 0. * * \param[in] priv Callback's private data * \param[in] new_size Size in bytes needed by the buffer diff --git a/third_party/aom/aom/aom_image.h b/third_party/aom/aom/aom_image.h index a960127f15..245ef2c692 100644 --- a/third_party/aom/aom/aom_image.h +++ b/third_party/aom/aom/aom_image.h @@ -30,11 +30,11 @@ extern "C" { * types, removing or reassigning enums, adding/removing/rearranging * fields to structures */ -#define AOM_IMAGE_ABI_VERSION (4) /**<\hideinitializer*/ +#define AOM_IMAGE_ABI_VERSION (5) /**<\hideinitializer*/ -#define AOM_IMG_FMT_PLANAR 0x100 /**< Image is a planar format. */ -#define AOM_IMG_FMT_UV_FLIP 0x200 /**< V plane precedes U in memory. */ -#define AOM_IMG_FMT_HAS_ALPHA 0x400 /**< Image has an alpha channel. */ +#define AOM_IMG_FMT_PLANAR 0x100 /**< Image is a planar format. */ +#define AOM_IMG_FMT_UV_FLIP 0x200 /**< V plane precedes U in memory. */ +/** 0x400 used to signal alpha channel, skipping for backwards compatibility. */ #define AOM_IMG_FMT_HIGHBITDEPTH 0x800 /**< Image uses 16bit framebuffer. */ /*!\brief List of supported image formats */ @@ -48,8 +48,8 @@ typedef enum aom_img_fmt { AOM_IMG_FMT_AOMI420 = AOM_IMG_FMT_PLANAR | 4, AOM_IMG_FMT_I422 = AOM_IMG_FMT_PLANAR | 5, AOM_IMG_FMT_I444 = AOM_IMG_FMT_PLANAR | 6, - AOM_IMG_FMT_444A = AOM_IMG_FMT_PLANAR | AOM_IMG_FMT_HAS_ALPHA | 6, AOM_IMG_FMT_I42016 = AOM_IMG_FMT_I420 | AOM_IMG_FMT_HIGHBITDEPTH, + AOM_IMG_FMT_YV1216 = AOM_IMG_FMT_YV12 | AOM_IMG_FMT_HIGHBITDEPTH, AOM_IMG_FMT_I42216 = AOM_IMG_FMT_I422 | AOM_IMG_FMT_HIGHBITDEPTH, AOM_IMG_FMT_I44416 = AOM_IMG_FMT_I444 | AOM_IMG_FMT_HIGHBITDEPTH, } aom_img_fmt_t; /**< alias for enum aom_img_fmt */ @@ -169,9 +169,8 @@ typedef struct aom_image { #define AOM_PLANE_Y 0 /**< Y (Luminance) plane */ #define AOM_PLANE_U 1 /**< U (Chroma) plane */ #define AOM_PLANE_V 2 /**< V (Chroma) plane */ -#define AOM_PLANE_ALPHA 3 /**< A (Transparency) plane */ - unsigned char *planes[4]; /**< pointer to the top left pixel for each plane */ - int stride[4]; /**< stride between rows for each plane */ + unsigned char *planes[3]; /**< pointer to the top left pixel for each plane */ + int stride[3]; /**< stride between rows for each plane */ size_t sz; /**< data size */ int bps; /**< bits per sample (for packed formats) */ diff --git a/third_party/aom/aom/aomcx.h b/third_party/aom/aom/aomcx.h index 013ddf57e8..fc0f23315f 100644 --- a/third_party/aom/aom/aomcx.h +++ b/third_party/aom/aom/aomcx.h @@ -300,9 +300,6 @@ enum aome_enc_control_id { * 2 = 4 tile columns * ..... * n = 2**n tile columns - * The requested tile columns will be capped by encoder based on image size - * limitation (The minimum width of a tile column is 256 pixel, the maximum - * is 4096). * * By default, the value is 0, i.e. one single column tile for entire image. */ @@ -324,6 +321,13 @@ enum aome_enc_control_id { */ AV1E_SET_TILE_ROWS, + /*!\brief Codec control function to enable RDO modulated by frame temporal + * dependency. + * + * By default, this feature is off. + */ + AV1E_SET_ENABLE_TPL_MODEL, + /*!\brief Codec control function to enable frame parallel decoding feature. * * AV1 has a bitstream feature to reduce decoding dependency between frames @@ -557,6 +561,16 @@ enum aome_enc_control_id { */ AV1E_SET_ENABLE_RESTORATION, + /*!\brief Codec control function to predict with OBMC mode. + * + * 0 = do not allow OBMC mode + * 1 = allow OBMC mode + * + * By default, the encoder allows OBMC prediction mode. + * + */ + AV1E_SET_ENABLE_OBMC, + /*!\brief Codec control function to encode without trellis quantization. * * 0 = apply trellis quantization @@ -684,18 +698,6 @@ enum aome_enc_control_id { */ AV1E_SET_MTU, - /*!\brief Codec control function to set dependent_horz_tiles. - * - * In encoding and decoding, AV1 allows enabling dependent horizontal tile - * The parameter for this control describes the value of this flag, - * which has a valid range [0, 1]: - * 0 = disable dependent horizontal tile - * 1 = enable dependent horizontal tile, - * - * By default, the value is 0, i.e. disable dependent horizontal tile. - */ - AV1E_SET_TILE_DEPENDENT_ROWS, - /*!\brief Codec control function to set the number of symbols in an ANS data * window. * @@ -708,13 +710,22 @@ enum aome_enc_control_id { */ AV1E_SET_ANS_WINDOW_SIZE_LOG2, - /*!\brief Codec control function to turn on / off dual filter - * enabling/disabling. + /*!\brief Codec control function to enable/disable rectangular partitions. * - * This will enable or disable dual filter. The default value is 1 + * This will enable or disable usage of rectangular partitions. The default + * value is 1. * */ - AV1E_SET_ENABLE_DF, + AV1E_SET_ENABLE_RECT_PARTITIONS, + + /*!\brief Codec control function to turn on / off intra edge filter + * at sequence level. + * + * This will enable or disable usage of intra-edge filtering. The default + * value is 1. + * + */ + AV1E_SET_ENABLE_INTRA_EDGE_FILTER, /*!\brief Codec control function to turn on / off frame order hint for a * few tools: @@ -728,14 +739,23 @@ enum aome_enc_control_id { */ AV1E_SET_ENABLE_ORDER_HINT, - /*!\brief Codec control function to turn on / off joint compound mode - * at sequence level. + /*!\brief Codec control function to turn on / off 64-length transforms. * - * This will enable or disable joint compound mode. The default value is 1. - * If AV1E_SET_ENABLE_ORDER_HINT is 0, then this flag is forced to 0. + * This will enable or disable usage of length 64 transforms in any + * direction. The default value is 1. * */ - AV1E_SET_ENABLE_JNT_COMP, + AV1E_SET_ENABLE_TX64, + + /*!\brief Codec control function to turn on / off dist-wtd compound mode + * at sequence level. + * + * This will enable or disable distance-weighted compound mode. The default + * value is 1. If AV1E_SET_ENABLE_ORDER_HINT is 0, then this flag is forced + * to 0. + * + */ + AV1E_SET_ENABLE_DIST_WTD_COMP, /*!\brief Codec control function to turn on / off ref frame mvs (mfmv) usage * at sequence level. @@ -755,6 +775,77 @@ enum aome_enc_control_id { */ AV1E_SET_ALLOW_REF_FRAME_MVS, + /*!\brief Codec control function to turn on / off dual filter usage + * for a sequence. + * + * This will enable or disable use of dual interpolation filter. + * The default value is 1. + * + */ + AV1E_SET_ENABLE_DUAL_FILTER, + + /*!\brief Codec control function to turn on / off masked compound usage + * for a sequence. + * + * This will enable or disable usage of wedge and diff-wtd compound + * modes. The default value is 1. + * + */ + AV1E_SET_ENABLE_MASKED_COMP, + + /*!\brief Codec control function to turn on / off interintra compound + * for a sequence. + * + * This will enable or disable usage of inter-intra compound modes. + * The default value is 1. + * + */ + AV1E_SET_ENABLE_INTERINTRA_COMP, + + /*!\brief Codec control function to turn on / off smooth inter-intra + * mode for a sequence. + * + * This will enable or disable usage of smooth inter-intra mode. + * The default value is 1. + * + */ + AV1E_SET_ENABLE_SMOOTH_INTERINTRA, + + /*!\brief Codec control function to turn on / off difference weighted + * compound. + * + * This will enable or disable usage of difference weighted compound. + * The default value is 1. + * + */ + AV1E_SET_ENABLE_DIFF_WTD_COMP, + + /*!\brief Codec control function to turn on / off interinter wedge + * compound. + * + * This will enable or disable usage of interinter wedge compound. + * The default value is 1. + * + */ + AV1E_SET_ENABLE_INTERINTER_WEDGE, + + /*!\brief Codec control function to turn on / off interintra wedge + * compound. + * + * This will enable or disable usage of interintra wedge compound. + * The default value is 1. + * + */ + AV1E_SET_ENABLE_INTERINTRA_WEDGE, + + /*!\brief Codec control function to turn on / off global motion usage + * for a sequence. + * + * This will enable or disable usage of global motion. The default value is 1. + * + */ + AV1E_SET_ENABLE_GLOBAL_MOTION, + /*!\brief Codec control function to turn on / off warped motion usage * at sequence level. * @@ -772,6 +863,39 @@ enum aome_enc_control_id { */ AV1E_SET_ALLOW_WARPED_MOTION, + /*!\brief Codec control function to turn on / off filter intra usage at + * sequence level. + * + * This will enable or disable usage of filter intra. The default value is 1. + * If AV1E_SET_ENABLE_FILTER_INTRA is 0, then this flag is forced to 0. + * + */ + AV1E_SET_ENABLE_FILTER_INTRA, + + /*!\brief Codec control function to turn on / off smooth intra modes usage. + * + * This will enable or disable usage of smooth, smooth_h and smooth_v intra + * modes. The default value is 1. + * + */ + AV1E_SET_ENABLE_SMOOTH_INTRA, + + /*!\brief Codec control function to turn on / off Paeth intra mode usage. + * + * This will enable or disable usage of Paeth intra mode. The default value + * is 1. + * + */ + AV1E_SET_ENABLE_PAETH_INTRA, + + /*!\brief Codec control function to turn on / off CFL uv intra mode usage. + * + * This will enable or disable usage of chroma-from-luma intra mode. The + * default value is 1. + * + */ + AV1E_SET_ENABLE_CFL_INTRA, + /*!\brief Codec control function to turn on / off frame superresolution. * * This will enable or disable frame superresolution. The default value is 1 @@ -779,34 +903,14 @@ enum aome_enc_control_id { */ AV1E_SET_ENABLE_SUPERRES, - /*!\brief Codec control function to set loop_filter_across_tiles_v_enabled - * and loop_filter_across_tiles_h_enabled. - * In encoding and decoding, AV1 allows disabling loop filter across tile - * boundary The parameter for this control describes the value of this flag, - * which has a valid range [0, 1]: - * 0 = disable loop filter across tile boundary - * 1 = enable loop filter across tile boundary - * - * By default, the value is 1, i.e. enable loop filter across tile boundary. - * - * Experiment: LOOPFILTERING_ACROSS_TILES_EXT - */ - AV1E_SET_TILE_LOOPFILTER_V, - AV1E_SET_TILE_LOOPFILTER_H, + /*!\brief Codec control function to turn on/off palette mode */ + AV1E_SET_ENABLE_PALETTE, - /*!\brief Codec control function to set loop_filter_across_tiles_enabled. - * - * In encoding and decoding, AV1 allows disabling loop filter across tile - * boundary The parameter for this control describes the value of this flag, - * which has a valid range [0, 1]: - * 0 = disable loop filter across tile boundary - * 1 = enable loop filter across tile boundary - * - * By default, the value is 1, i.e. enable loop filter across tile boundary. - * - * Experiment: LOOPFILTERING_ACROSS_TILES - */ - AV1E_SET_TILE_LOOPFILTER, + /*!\brief Codec control function to turn on/off intra block copy mode */ + AV1E_SET_ENABLE_INTRABC, + + /*!\brief Codec control function to turn on/off intra angle delta */ + AV1E_SET_ENABLE_ANGLE_DELTA, /*!\brief Codec control function to set the delta q mode * @@ -865,6 +969,26 @@ enum aome_enc_control_id { /*!\brief Sets the chroma subsampling y value */ AV1E_SET_CHROMA_SUBSAMPLING_Y, + + /*!\brief Control to use a reduced tx type set */ + AV1E_SET_REDUCED_TX_TYPE_SET, + + /*!\brief Control to use dct only for intra modes */ + AV1E_SET_INTRA_DCT_ONLY, + + /*!\brief Control to use dct only for inter modes */ + AV1E_SET_INTER_DCT_ONLY, + + /*!\brief Control to use adaptive quantize_b */ + AV1E_SET_QUANT_B_ADAPT, + + /*!\brief Control to select maximum height for the GF group pyramid structure + * (valid values: 1 - 4) */ + AV1E_SET_GF_MAX_PYRAMID_HEIGHT, + + /*!\brief Control to select maximum reference frames allowed per frame + * (valid values: 3 - 7) */ + AV1E_SET_MAX_REFERENCE_FRAMES, }; /*!\brief aom 1-D scaling mode @@ -1006,15 +1130,8 @@ AOM_CTRL_USE_TYPE(AV1E_SET_TILE_COLUMNS, int) AOM_CTRL_USE_TYPE(AV1E_SET_TILE_ROWS, int) #define AOM_CTRL_AV1E_SET_TILE_ROWS -AOM_CTRL_USE_TYPE(AV1E_SET_TILE_DEPENDENT_ROWS, int) -#define AOM_CTRL_AV1E_SET_TILE_DEPENDENT_ROWS - -AOM_CTRL_USE_TYPE(AV1E_SET_TILE_LOOPFILTER_V, int) -#define AOM_CTRL_AV1E_SET_TILE_LOOPFILTER_V -AOM_CTRL_USE_TYPE(AV1E_SET_TILE_LOOPFILTER_H, int) -#define AOM_CTRL_AV1E_SET_TILE_LOOPFILTER_H -AOM_CTRL_USE_TYPE(AV1E_SET_TILE_LOOPFILTER, int) -#define AOM_CTRL_AV1E_SET_TILE_LOOPFILTER +AOM_CTRL_USE_TYPE(AV1E_SET_ENABLE_TPL_MODEL, unsigned int) +#define AOM_CTRL_AV1E_SET_ENABLE_TPL_MODEL AOM_CTRL_USE_TYPE(AOME_GET_LAST_QUANTIZER, int *) #define AOM_CTRL_AOME_GET_LAST_QUANTIZER @@ -1041,6 +1158,9 @@ AOM_CTRL_USE_TYPE(AV1E_SET_ENABLE_CDEF, unsigned int) AOM_CTRL_USE_TYPE(AV1E_SET_ENABLE_RESTORATION, unsigned int) #define AOM_CTRL_AV1E_SET_ENABLE_RESTORATION +AOM_CTRL_USE_TYPE(AV1E_SET_ENABLE_OBMC, unsigned int) +#define AOM_CTRL_AV1E_SET_ENABLE_OBMC + AOM_CTRL_USE_TYPE(AV1E_SET_DISABLE_TRELLIS_QUANT, unsigned int) #define AOM_CTRL_AV1E_SET_DISABLE_TRELLIS_QUANT @@ -1070,17 +1190,23 @@ AOM_CTRL_USE_TYPE(AV1E_SET_NUM_TG, unsigned int) AOM_CTRL_USE_TYPE(AV1E_SET_MTU, unsigned int) #define AOM_CTRL_AV1E_SET_MTU -AOM_CTRL_USE_TYPE(AV1E_SET_TIMING_INFO_TYPE, aom_timing_info_type_t) +AOM_CTRL_USE_TYPE(AV1E_SET_TIMING_INFO_TYPE, int) /* aom_timing_info_type_t */ #define AOM_CTRL_AV1E_SET_TIMING_INFO_TYPE -AOM_CTRL_USE_TYPE(AV1E_SET_ENABLE_DF, unsigned int) -#define AOM_CTRL_AV1E_SET_ENABLE_DF +AOM_CTRL_USE_TYPE(AV1E_SET_ENABLE_RECT_PARTITIONS, unsigned int) +#define AOM_CTRL_AV1E_SET_ENABLE_RECT_PARTITIONS + +AOM_CTRL_USE_TYPE(AV1E_SET_ENABLE_INTRA_EDGE_FILTER, unsigned int) +#define AOM_CTRL_AV1E_SET_ENABLE_INTRA_EDGE_FILTER AOM_CTRL_USE_TYPE(AV1E_SET_ENABLE_ORDER_HINT, unsigned int) #define AOM_CTRL_AV1E_SET_ENABLE_ORDER_HINT -AOM_CTRL_USE_TYPE(AV1E_SET_ENABLE_JNT_COMP, unsigned int) -#define AOM_CTRL_AV1E_SET_ENABLE_JNT_COMP +AOM_CTRL_USE_TYPE(AV1E_SET_ENABLE_TX64, unsigned int) +#define AOM_CTRL_AV1E_SET_ENABLE_TX64 + +AOM_CTRL_USE_TYPE(AV1E_SET_ENABLE_DIST_WTD_COMP, unsigned int) +#define AOM_CTRL_AV1E_SET_ENABLE_DIST_WTD_COMP AOM_CTRL_USE_TYPE(AV1E_SET_ENABLE_REF_FRAME_MVS, unsigned int) #define AOM_CTRL_AV1E_SET_ENABLE_REF_FRAME_MVS @@ -1088,15 +1214,60 @@ AOM_CTRL_USE_TYPE(AV1E_SET_ENABLE_REF_FRAME_MVS, unsigned int) AOM_CTRL_USE_TYPE(AV1E_SET_ALLOW_REF_FRAME_MVS, unsigned int) #define AOM_CTRL_AV1E_SET_ALLOW_REF_FRAME_MVS +AOM_CTRL_USE_TYPE(AV1E_SET_ENABLE_DUAL_FILTER, unsigned int) +#define AOM_CTRL_AV1E_SET_ENABLE_DUAL_FILTER + +AOM_CTRL_USE_TYPE(AV1E_SET_ENABLE_MASKED_COMP, unsigned int) +#define AOM_CTRL_AV1E_SET_ENABLE_MASKED_COMP + +AOM_CTRL_USE_TYPE(AV1E_SET_ENABLE_INTERINTRA_COMP, unsigned int) +#define AOM_CTRL_AV1E_SET_ENABLE_INTERINTRA_COMP + +AOM_CTRL_USE_TYPE(AV1E_SET_ENABLE_SMOOTH_INTERINTRA, unsigned int) +#define AOM_CTRL_AV1E_SET_ENABLE_SMOOTH_INTERINTRA + +AOM_CTRL_USE_TYPE(AV1E_SET_ENABLE_DIFF_WTD_COMP, unsigned int) +#define AOM_CTRL_AV1E_SET_ENABLE_DIFF_WTD_COMP + +AOM_CTRL_USE_TYPE(AV1E_SET_ENABLE_INTERINTER_WEDGE, unsigned int) +#define AOM_CTRL_AV1E_SET_ENABLE_INTERINTER_WEDGE + +AOM_CTRL_USE_TYPE(AV1E_SET_ENABLE_INTERINTRA_WEDGE, unsigned int) +#define AOM_CTRL_AV1E_SET_ENABLE_INTERINTRA_WEDGE + +AOM_CTRL_USE_TYPE(AV1E_SET_ENABLE_GLOBAL_MOTION, unsigned int) +#define AOM_CTRL_AV1E_SET_ENABLE_GLOBAL_MOTION + AOM_CTRL_USE_TYPE(AV1E_SET_ENABLE_WARPED_MOTION, unsigned int) #define AOM_CTRL_AV1E_SET_ENABLE_WARPED_MOTION AOM_CTRL_USE_TYPE(AV1E_SET_ALLOW_WARPED_MOTION, unsigned int) #define AOM_CTRL_AV1E_SET_ALLOW_WARPED_MOTION +AOM_CTRL_USE_TYPE(AV1E_SET_ENABLE_FILTER_INTRA, unsigned int) +#define AOM_CTRL_AV1E_SET_ENABLE_FILTER_INTRA + +AOM_CTRL_USE_TYPE(AV1E_SET_ENABLE_SMOOTH_INTRA, unsigned int) +#define AOM_CTRL_AV1E_SET_ENABLE_SMOOTH_INTRA + +AOM_CTRL_USE_TYPE(AV1E_SET_ENABLE_PAETH_INTRA, unsigned int) +#define AOM_CTRL_AV1E_SET_ENABLE_PAETH_INTRA + +AOM_CTRL_USE_TYPE(AV1E_SET_ENABLE_CFL_INTRA, unsigned int) +#define AOM_CTRL_AV1E_SET_ENABLE_CFL_INTRA + AOM_CTRL_USE_TYPE(AV1E_SET_ENABLE_SUPERRES, unsigned int) #define AOM_CTRL_AV1E_SET_ENABLE_SUPERRES +AOM_CTRL_USE_TYPE(AV1E_SET_ENABLE_PALETTE, unsigned int) +#define AOM_CTRL_AV1E_SET_ENABLE_PALETTE + +AOM_CTRL_USE_TYPE(AV1E_SET_ENABLE_INTRABC, unsigned int) +#define AOM_CTRL_AV1E_SET_ENABLE_INTRABC + +AOM_CTRL_USE_TYPE(AV1E_SET_ENABLE_ANGLE_DELTA, unsigned int) +#define AOM_CTRL_AV1E_SET_ENABLE_ANGLE_DELTA + AOM_CTRL_USE_TYPE(AV1E_SET_FRAME_PARALLEL_DECODING, unsigned int) #define AOM_CTRL_AV1E_SET_FRAME_PARALLEL_DECODING @@ -1189,6 +1360,24 @@ AOM_CTRL_USE_TYPE(AV1E_SET_CHROMA_SUBSAMPLING_X, unsigned int) AOM_CTRL_USE_TYPE(AV1E_SET_CHROMA_SUBSAMPLING_Y, unsigned int) #define AOM_CTRL_AV1E_SET_CHROMA_SUBSAMPLING_Y +AOM_CTRL_USE_TYPE(AV1E_SET_REDUCED_TX_TYPE_SET, unsigned int) +#define AOM_CTRL_AV1E_SET_REDUCED_TX_TYPE_SET + +AOM_CTRL_USE_TYPE(AV1E_SET_INTRA_DCT_ONLY, unsigned int) +#define AOM_CTRL_AV1E_SET_INTRA_DCT_ONLY + +AOM_CTRL_USE_TYPE(AV1E_SET_INTER_DCT_ONLY, unsigned int) +#define AOM_CTRL_AV1E_SET_INTER_DCT_ONLY + +AOM_CTRL_USE_TYPE(AV1E_SET_QUANT_B_ADAPT, unsigned int) +#define AOM_CTRL_AV1E_SET_QUANT_B_ADAPT + +AOM_CTRL_USE_TYPE(AV1E_SET_GF_MAX_PYRAMID_HEIGHT, unsigned int) +#define AOM_CTRL_AV1E_SET_GF_MAX_PYRAMID_HEIGHT + +AOM_CTRL_USE_TYPE(AV1E_SET_MAX_REFERENCE_FRAMES, unsigned int) +#define AOM_CTRL_AV1E_SET_MAX_REFERENCE_FRAMES + /*!\endcond */ /*! @} - end defgroup aom_encoder */ #ifdef __cplusplus diff --git a/third_party/aom/aom/aomdx.h b/third_party/aom/aom/aomdx.h index 765856a1bb..c71eaf90d6 100644 --- a/third_party/aom/aom/aomdx.h +++ b/third_party/aom/aom/aomdx.h @@ -45,6 +45,7 @@ typedef struct Accounting Accounting; /** Callback that inspects decoder frame data. */ typedef void (*aom_inspect_cb)(void *decoder, void *ctx); + #endif /*!\brief Structure to hold inspection callback and context. @@ -60,6 +61,21 @@ typedef struct aom_inspect_init { void *inspect_ctx; } aom_inspect_init; +/*!\brief Structure to collect a buffer index when inspecting. + * + * Defines a structure to hold the buffer and return an index + * when calling decode from inspect. This enables us to decode + * non showable sub frames. + */ +typedef struct { + /*! Pointer for new position in compressed buffer after decoding 1 OBU. */ + const unsigned char *buf; + /*! Index into reference buffer array to see result of decoding 1 OBU. */ + int idx; + /*! Is a show existing frame. */ + int show_existing; +} Av1DecodeReturn; + /*!\brief Structure to hold a tile's start address and size in the bitstream. * * Defines a structure to hold a tile's start address and size in the bitstream. @@ -125,6 +141,9 @@ enum aom_dec_control_id { /** control function to get the size of the tile. */ AV1D_GET_TILE_SIZE, + /** control function to get the tile count in a tile list. */ + AV1D_GET_TILE_COUNT, + /** control function to set the byte alignment of the planes in the reference * buffers. Valid values are power of 2, from 32 to 1024. A value of 0 sets * legacy alignment. I.e. Y plane is aligned to 32 bytes, U plane directly @@ -260,6 +279,8 @@ AOM_CTRL_USE_TYPE(AV1D_GET_IMG_FORMAT, aom_img_fmt_t *) #define AOM_CTRL_AV1D_GET_IMG_FORMAT AOM_CTRL_USE_TYPE(AV1D_GET_TILE_SIZE, unsigned int *) #define AOM_CTRL_AV1D_GET_TILE_SIZE +AOM_CTRL_USE_TYPE(AV1D_GET_TILE_COUNT, unsigned int *) +#define AOM_CTRL_AV1D_GET_TILE_COUNT AOM_CTRL_USE_TYPE(AV1D_GET_FRAME_SIZE, int *) #define AOM_CTRL_AV1D_GET_FRAME_SIZE AOM_CTRL_USE_TYPE(AV1_INVERT_TILE_DECODE_ORDER, int) diff --git a/third_party/aom/aom/exports_com b/third_party/aom/aom/exports_com index 2798bd51ad..cf99bc5d95 100644 --- a/third_party/aom/aom/exports_com +++ b/third_party/aom/aom/exports_com @@ -9,6 +9,7 @@ text aom_codec_iface_name text aom_codec_version text aom_codec_version_extra_str text aom_codec_version_str +text aom_free text aom_img_alloc text aom_img_alloc_with_border text aom_img_flip diff --git a/third_party/aom/aom/src/aom_decoder.c b/third_party/aom/aom/src/aom_decoder.c index 8c9111faf0..282ec8a756 100644 --- a/third_party/aom/aom/src/aom_decoder.c +++ b/third_party/aom/aom/src/aom_decoder.c @@ -37,9 +37,6 @@ aom_codec_err_t aom_codec_dec_init_ver(aom_codec_ctx_t *ctx, else if ((flags & AOM_CODEC_USE_POSTPROC) && !(iface->caps & AOM_CODEC_CAP_POSTPROC)) res = AOM_CODEC_INCAPABLE; - else if ((flags & AOM_CODEC_USE_INPUT_FRAGMENTS) && - !(iface->caps & AOM_CODEC_CAP_INPUT_FRAGMENTS)) - res = AOM_CODEC_INCAPABLE; else if (!(iface->caps & AOM_CODEC_CAP_DECODER)) res = AOM_CODEC_INCAPABLE; else { diff --git a/third_party/aom/aom/src/aom_image.c b/third_party/aom/aom/src/aom_image.c index 437f0241eb..6504cdde01 100644 --- a/third_party/aom/aom/src/aom_image.c +++ b/third_party/aom/aom/src/aom_image.c @@ -59,6 +59,7 @@ static aom_image_t *img_alloc_helper( case AOM_IMG_FMT_AOMYV12: bps = 12; break; case AOM_IMG_FMT_I422: case AOM_IMG_FMT_I444: bps = 24; break; + case AOM_IMG_FMT_YV1216: case AOM_IMG_FMT_I42016: bps = 24; break; case AOM_IMG_FMT_I42216: case AOM_IMG_FMT_I44416: bps = 48; break; @@ -73,6 +74,7 @@ static aom_image_t *img_alloc_helper( case AOM_IMG_FMT_AOMYV12: case AOM_IMG_FMT_I422: case AOM_IMG_FMT_I42016: + case AOM_IMG_FMT_YV1216: case AOM_IMG_FMT_I42216: xcs = 1; break; default: xcs = 0; break; } @@ -82,6 +84,7 @@ static aom_image_t *img_alloc_helper( case AOM_IMG_FMT_YV12: case AOM_IMG_FMT_AOMI420: case AOM_IMG_FMT_AOMYV12: + case AOM_IMG_FMT_YV1216: case AOM_IMG_FMT_I42016: ycs = 1; break; default: ycs = 0; break; } @@ -117,6 +120,7 @@ static aom_image_t *img_alloc_helper( img->img_data = (uint8_t *)aom_memalign(buf_align, (size_t)alloc_size); img->img_data_owner = 1; + img->sz = (size_t)alloc_size; } if (!img->img_data) goto fail; @@ -131,7 +135,7 @@ static aom_image_t *img_alloc_helper( img->bps = bps; /* Calculate strides */ - img->stride[AOM_PLANE_Y] = img->stride[AOM_PLANE_ALPHA] = stride_in_bytes; + img->stride[AOM_PLANE_Y] = stride_in_bytes; img->stride[AOM_PLANE_U] = img->stride[AOM_PLANE_V] = stride_in_bytes >> xcs; /* Default viewport to entire image */ @@ -185,12 +189,6 @@ int aom_img_set_rect(aom_image_t *img, unsigned int x, unsigned int y, (img->fmt & AOM_IMG_FMT_HIGHBITDEPTH) ? 2 : 1; data = img->img_data; - if (img->fmt & AOM_IMG_FMT_HAS_ALPHA) { - img->planes[AOM_PLANE_ALPHA] = - data + x * bytes_per_sample + y * img->stride[AOM_PLANE_ALPHA]; - data += (img->h + 2 * border) * img->stride[AOM_PLANE_ALPHA]; - } - img->planes[AOM_PLANE_Y] = data + x * bytes_per_sample + y * img->stride[AOM_PLANE_Y]; data += (img->h + 2 * border) * img->stride[AOM_PLANE_Y]; @@ -236,10 +234,6 @@ void aom_img_flip(aom_image_t *img) { img->planes[AOM_PLANE_V] += (signed)((img->d_h >> img->y_chroma_shift) - 1) * img->stride[AOM_PLANE_V]; img->stride[AOM_PLANE_V] = -img->stride[AOM_PLANE_V]; - - img->planes[AOM_PLANE_ALPHA] += - (signed)(img->d_h - 1) * img->stride[AOM_PLANE_ALPHA]; - img->stride[AOM_PLANE_ALPHA] = -img->stride[AOM_PLANE_ALPHA]; } void aom_img_free(aom_image_t *img) { diff --git a/third_party/aom/aom_dsp/add_noise.c b/third_party/aom/aom_dsp/add_noise.c index bfb3e7e009..43587ca315 100644 --- a/third_party/aom/aom_dsp/add_noise.c +++ b/third_party/aom/aom_dsp/add_noise.c @@ -40,7 +40,7 @@ void aom_plane_add_noise_c(uint8_t *start, char *noise, char blackclamp[16], } static double gaussian(double sigma, double mu, double x) { - return 1 / (sigma * sqrt(2.0 * 3.14159265)) * + return 1 / (sigma * sqrt(2.0 * PI)) * (exp(-(x - mu) * (x - mu) / (2 * sigma * sigma))); } diff --git a/third_party/aom/aom_dsp/aom_dsp.cmake b/third_party/aom/aom_dsp/aom_dsp.cmake index 11ff73756b..c6a6e3df59 100644 --- a/third_party/aom/aom_dsp/aom_dsp.cmake +++ b/third_party/aom/aom_dsp/aom_dsp.cmake @@ -35,6 +35,7 @@ list(APPEND AOM_DSP_COMMON_SOURCES "${AOM_ROOT}/aom_dsp/intrapred_common.h" "${AOM_ROOT}/aom_dsp/loopfilter.c" "${AOM_ROOT}/aom_dsp/prob.h" + "${AOM_ROOT}/aom_dsp/recenter.h" "${AOM_ROOT}/aom_dsp/simd/v128_intrinsics.h" "${AOM_ROOT}/aom_dsp/simd/v128_intrinsics_c.h" "${AOM_ROOT}/aom_dsp/simd/v256_intrinsics.h" @@ -43,7 +44,8 @@ list(APPEND AOM_DSP_COMMON_SOURCES "${AOM_ROOT}/aom_dsp/simd/v64_intrinsics_c.h" "${AOM_ROOT}/aom_dsp/subtract.c" "${AOM_ROOT}/aom_dsp/txfm_common.h" - "${AOM_ROOT}/aom_dsp/x86/convolve_common_intrin.h") + "${AOM_ROOT}/aom_dsp/x86/convolve_common_intrin.h" + "${AOM_ROOT}/aom_dsp/avg.c") list(APPEND AOM_DSP_COMMON_ASM_SSE2 "${AOM_ROOT}/aom_dsp/x86/aom_convolve_copy_sse2.asm" @@ -51,15 +53,17 @@ list(APPEND AOM_DSP_COMMON_ASM_SSE2 "${AOM_ROOT}/aom_dsp/x86/aom_high_subpixel_bilinear_sse2.asm" "${AOM_ROOT}/aom_dsp/x86/aom_subpixel_8t_sse2.asm" "${AOM_ROOT}/aom_dsp/x86/aom_subpixel_bilinear_sse2.asm" - "${AOM_ROOT}/aom_dsp/x86/highbd_intrapred_sse2.asm" - "${AOM_ROOT}/aom_dsp/x86/intrapred_sse2.asm" + "${AOM_ROOT}/aom_dsp/x86/highbd_intrapred_asm_sse2.asm" + "${AOM_ROOT}/aom_dsp/x86/intrapred_asm_sse2.asm" "${AOM_ROOT}/aom_dsp/x86/inv_wht_sse2.asm") list(APPEND AOM_DSP_COMMON_INTRIN_SSE2 + "${AOM_ROOT}/aom_dsp/x86/aom_subpixel_8t_intrin_sse2.c" "${AOM_ROOT}/aom_dsp/x86/aom_asm_stubs.c" "${AOM_ROOT}/aom_dsp/x86/convolve.h" "${AOM_ROOT}/aom_dsp/x86/convolve_sse2.h" "${AOM_ROOT}/aom_dsp/x86/fft_sse2.c" + "${AOM_ROOT}/aom_dsp/x86/highbd_convolve_sse2.c" "${AOM_ROOT}/aom_dsp/x86/highbd_intrapred_sse2.c" "${AOM_ROOT}/aom_dsp/x86/highbd_loopfilter_sse2.c" "${AOM_ROOT}/aom_dsp/x86/intrapred_sse2.c" @@ -68,7 +72,9 @@ list(APPEND AOM_DSP_COMMON_INTRIN_SSE2 "${AOM_ROOT}/aom_dsp/x86/mem_sse2.h" "${AOM_ROOT}/aom_dsp/x86/transpose_sse2.h" "${AOM_ROOT}/aom_dsp/x86/txfm_common_sse2.h" - "${AOM_ROOT}/aom_dsp/x86/sum_squares_sse2.h") + "${AOM_ROOT}/aom_dsp/x86/sum_squares_sse2.h" + "${AOM_ROOT}/aom_dsp/x86/avg_intrin_sse2.c" + "${AOM_ROOT}/aom_dsp/x86/bitdepth_conversion_sse2.h") list(APPEND AOM_DSP_COMMON_ASM_SSSE3 "${AOM_ROOT}/aom_dsp/x86/aom_subpixel_8t_ssse3.asm" @@ -94,7 +100,9 @@ list(APPEND AOM_DSP_COMMON_INTRIN_AVX2 "${AOM_ROOT}/aom_dsp/x86/highbd_convolve_avx2.c" "${AOM_ROOT}/aom_dsp/x86/highbd_loopfilter_avx2.c" "${AOM_ROOT}/aom_dsp/x86/intrapred_avx2.c" - "${AOM_ROOT}/aom_dsp/x86/blend_a64_mask_avx2.c") + "${AOM_ROOT}/aom_dsp/x86/blend_a64_mask_avx2.c" + "${AOM_ROOT}/aom_dsp/x86/avg_intrin_avx2.c" + "${AOM_ROOT}/aom_dsp/x86/bitdepth_conversion_avx2.h") list(APPEND AOM_DSP_COMMON_INTRIN_NEON "${AOM_ROOT}/aom_dsp/arm/fwd_txfm_neon.c" @@ -175,6 +183,9 @@ if(CONFIG_AV1_ENCODER) "${AOM_ROOT}/aom_dsp/x86/subpel_variance_sse2.asm" "${AOM_ROOT}/aom_dsp/x86/subtract_sse2.asm") + list(APPEND AOM_DSP_ENCODER_ASM_SSE2_X86_64 + "${AOM_ROOT}/aom_dsp/x86/ssim_sse2_x86_64.asm") + list(APPEND AOM_DSP_ENCODER_INTRIN_SSE2 "${AOM_ROOT}/aom_dsp/x86/fwd_txfm_impl_sse2.h" "${AOM_ROOT}/aom_dsp/x86/fwd_txfm_sse2.c" @@ -189,7 +200,7 @@ if(CONFIG_AV1_ENCODER) list(APPEND AOM_DSP_ENCODER_ASM_SSSE3_X86_64 "${AOM_ROOT}/aom_dsp/x86/fwd_txfm_ssse3_x86_64.asm" - "${AOM_ROOT}/aom_dsp/x86/ssim_opt_x86_64.asm") + "${AOM_ROOT}/aom_dsp/x86/quantize_ssse3_x86_64.asm") list(APPEND AOM_DSP_ENCODER_INTRIN_AVX2 "${AOM_ROOT}/aom_dsp/x86/masked_sad_intrin_avx2.c" @@ -207,9 +218,6 @@ if(CONFIG_AV1_ENCODER) "${AOM_ROOT}/aom_dsp/x86/obmc_variance_avx2.c" "${AOM_ROOT}/aom_dsp/x86/sum_squares_avx2.c") - list(APPEND AOM_DSP_ENCODER_ASM_SSSE3_X86_64 - "${AOM_ROOT}/aom_dsp/x86/quantize_ssse3_x86_64.asm") - list(APPEND AOM_DSP_ENCODER_AVX_ASM_X86_64 "${AOM_ROOT}/aom_dsp/x86/quantize_avx_x86_64.asm") @@ -218,6 +226,7 @@ if(CONFIG_AV1_ENCODER) "${AOM_ROOT}/aom_dsp/x86/masked_sad_intrin_ssse3.c" "${AOM_ROOT}/aom_dsp/x86/masked_variance_intrin_ssse3.h" "${AOM_ROOT}/aom_dsp/x86/masked_variance_intrin_ssse3.c" + "${AOM_ROOT}/aom_dsp/x86/quantize_ssse3.c" "${AOM_ROOT}/aom_dsp/x86/variance_impl_ssse3.c" "${AOM_ROOT}/aom_dsp/x86/jnt_variance_ssse3.c" "${AOM_ROOT}/aom_dsp/x86/jnt_sad_ssse3.c") @@ -278,6 +287,9 @@ function(setup_aom_dsp_targets) "AOM_DSP_COMMON_INTRIN_SSE2" "aom") if(CONFIG_AV1_ENCODER) + if("${AOM_TARGET_CPU}" STREQUAL "x86_64") + list(APPEND AOM_DSP_ENCODER_ASM_SSE2 ${AOM_DSP_ENCODER_ASM_SSE2_X86_64}) + endif() add_asm_library("aom_dsp_encoder_sse2" "AOM_DSP_ENCODER_ASM_SSE2" "aom") add_intrinsics_object_library("-msse2" "sse2" "aom_dsp_encoder" "AOM_DSP_ENCODER_INTRIN_SSE2" "aom") diff --git a/third_party/aom/aom_dsp/aom_dsp_rtcd_defs.pl b/third_party/aom/aom_dsp/aom_dsp_rtcd_defs.pl index 8e8a480fec..ff0f65333c 100755 --- a/third_party/aom/aom_dsp/aom_dsp_rtcd_defs.pl +++ b/third_party/aom/aom_dsp/aom_dsp_rtcd_defs.pl @@ -362,10 +362,10 @@ add_proto qw/void aom_highbd_convolve_copy/, "const uint8_t *src, ptrdiff_t src_ specialize qw/aom_highbd_convolve_copy sse2 avx2/; add_proto qw/void aom_highbd_convolve8_horiz/, "const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, const int16_t *filter_y, int y_step_q4, int w, int h, int bps"; -specialize qw/aom_highbd_convolve8_horiz avx2/, "$sse2_x86_64"; +specialize qw/aom_highbd_convolve8_horiz sse2 avx2/; add_proto qw/void aom_highbd_convolve8_vert/, "const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, const int16_t *filter_y, int y_step_q4, int w, int h, int bps"; -specialize qw/aom_highbd_convolve8_vert avx2/, "$sse2_x86_64"; +specialize qw/aom_highbd_convolve8_vert sse2 avx2/; # # Loopfilter @@ -466,10 +466,6 @@ specialize qw/aom_highbd_lpf_horizontal_4 sse2/; add_proto qw/void aom_highbd_lpf_horizontal_4_dual/, "uint16_t *s, int pitch, const uint8_t *blimit0, const uint8_t *limit0, const uint8_t *thresh0, const uint8_t *blimit1, const uint8_t *limit1, const uint8_t *thresh1, int bd"; specialize qw/aom_highbd_lpf_horizontal_4_dual sse2 avx2/; -# Helper functions. -add_proto qw/void av1_round_shift_array/, "int32_t *arr, int size, int bit"; -specialize "av1_round_shift_array", qw/sse4_1 neon/; - # # Encoder functions. # @@ -526,6 +522,7 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") { specialize qw/aom_quantize_b_32x32/, "$ssse3_x86_64", "$avx_x86_64"; add_proto qw/void aom_quantize_b_64x64/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan"; + specialize qw/aom_quantize_b_64x64 ssse3/; } # CONFIG_AV1_ENCODER if (aom_config("CONFIG_AV1_ENCODER") eq "yes") { @@ -544,7 +541,6 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") { # add_proto qw/void aom_lowbd_blend_a64_d16_mask/, "uint8_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0, uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride, const uint8_t *mask, uint32_t mask_stride, int w, int h, int subx, int suby, ConvolveParams *conv_params"; specialize qw/aom_lowbd_blend_a64_d16_mask sse4_1 avx2 neon/; -add_proto qw/void aom_highbd_blend_a64_d16_mask/, "uint8_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0, uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride, const uint8_t *mask, uint32_t mask_stride, int w, int h, int subx, int suby, ConvolveParams *conv_params, const int bd"; add_proto qw/void aom_blend_a64_mask/, "uint8_t *dst, uint32_t dst_stride, const uint8_t *src0, uint32_t src0_stride, const uint8_t *src1, uint32_t src1_stride, const uint8_t *mask, uint32_t mask_stride, int w, int h, int subx, int suby"; add_proto qw/void aom_blend_a64_hmask/, "uint8_t *dst, uint32_t dst_stride, const uint8_t *src0, uint32_t src0_stride, const uint8_t *src1, uint32_t src1_stride, const uint8_t *mask, int w, int h"; add_proto qw/void aom_blend_a64_vmask/, "uint8_t *dst, uint32_t dst_stride, const uint8_t *src0, uint32_t src0_stride, const uint8_t *src1, uint32_t src1_stride, const uint8_t *mask, int w, int h"; @@ -555,9 +551,11 @@ specialize "aom_blend_a64_vmask", qw/sse4_1 neon/; add_proto qw/void aom_highbd_blend_a64_mask/, "uint8_t *dst, uint32_t dst_stride, const uint8_t *src0, uint32_t src0_stride, const uint8_t *src1, uint32_t src1_stride, const uint8_t *mask, uint32_t mask_stride, int w, int h, int subx, int suby, int bd"; add_proto qw/void aom_highbd_blend_a64_hmask/, "uint8_t *dst, uint32_t dst_stride, const uint8_t *src0, uint32_t src0_stride, const uint8_t *src1, uint32_t src1_stride, const uint8_t *mask, int w, int h, int bd"; add_proto qw/void aom_highbd_blend_a64_vmask/, "uint8_t *dst, uint32_t dst_stride, const uint8_t *src0, uint32_t src0_stride, const uint8_t *src1, uint32_t src1_stride, const uint8_t *mask, int w, int h, int bd"; +add_proto qw/void aom_highbd_blend_a64_d16_mask/, "uint8_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0, uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride, const uint8_t *mask, uint32_t mask_stride, int w, int h, int subx, int suby, ConvolveParams *conv_params, const int bd"; specialize "aom_highbd_blend_a64_mask", qw/sse4_1/; specialize "aom_highbd_blend_a64_hmask", qw/sse4_1/; specialize "aom_highbd_blend_a64_vmask", qw/sse4_1/; +specialize "aom_highbd_blend_a64_d16_mask", qw/sse4_1 avx2/; if (aom_config("CONFIG_AV1_ENCODER") eq "yes") { # @@ -595,7 +593,7 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") { ($w, $h) = @$_; add_proto qw/unsigned int/, "aom_sad${w}x${h}", "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride"; add_proto qw/unsigned int/, "aom_sad${w}x${h}_avg", "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred"; - add_proto qw/unsigned int/, "aom_jnt_sad${w}x${h}_avg", "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred, const JNT_COMP_PARAMS *jcp_param"; + add_proto qw/unsigned int/, "aom_dist_wtd_sad${w}x${h}_avg", "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred, const DIST_WTD_COMP_PARAMS *jcp_param"; } specialize qw/aom_sad128x128 avx2 sse2/; @@ -646,29 +644,29 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") { specialize qw/aom_sad16x64_avg sse2/; specialize qw/aom_sad64x16_avg sse2/; - specialize qw/aom_jnt_sad128x128_avg ssse3/; - specialize qw/aom_jnt_sad128x64_avg ssse3/; - specialize qw/aom_jnt_sad64x128_avg ssse3/; - specialize qw/aom_jnt_sad64x64_avg ssse3/; - specialize qw/aom_jnt_sad64x32_avg ssse3/; - specialize qw/aom_jnt_sad32x64_avg ssse3/; - specialize qw/aom_jnt_sad32x32_avg ssse3/; - specialize qw/aom_jnt_sad32x16_avg ssse3/; - specialize qw/aom_jnt_sad16x32_avg ssse3/; - specialize qw/aom_jnt_sad16x16_avg ssse3/; - specialize qw/aom_jnt_sad16x8_avg ssse3/; - specialize qw/aom_jnt_sad8x16_avg ssse3/; - specialize qw/aom_jnt_sad8x8_avg ssse3/; - specialize qw/aom_jnt_sad8x4_avg ssse3/; - specialize qw/aom_jnt_sad4x8_avg ssse3/; - specialize qw/aom_jnt_sad4x4_avg ssse3/; + specialize qw/aom_dist_wtd_sad128x128_avg ssse3/; + specialize qw/aom_dist_wtd_sad128x64_avg ssse3/; + specialize qw/aom_dist_wtd_sad64x128_avg ssse3/; + specialize qw/aom_dist_wtd_sad64x64_avg ssse3/; + specialize qw/aom_dist_wtd_sad64x32_avg ssse3/; + specialize qw/aom_dist_wtd_sad32x64_avg ssse3/; + specialize qw/aom_dist_wtd_sad32x32_avg ssse3/; + specialize qw/aom_dist_wtd_sad32x16_avg ssse3/; + specialize qw/aom_dist_wtd_sad16x32_avg ssse3/; + specialize qw/aom_dist_wtd_sad16x16_avg ssse3/; + specialize qw/aom_dist_wtd_sad16x8_avg ssse3/; + specialize qw/aom_dist_wtd_sad8x16_avg ssse3/; + specialize qw/aom_dist_wtd_sad8x8_avg ssse3/; + specialize qw/aom_dist_wtd_sad8x4_avg ssse3/; + specialize qw/aom_dist_wtd_sad4x8_avg ssse3/; + specialize qw/aom_dist_wtd_sad4x4_avg ssse3/; - specialize qw/aom_jnt_sad4x16_avg ssse3/; - specialize qw/aom_jnt_sad16x4_avg ssse3/; - specialize qw/aom_jnt_sad8x32_avg ssse3/; - specialize qw/aom_jnt_sad32x8_avg ssse3/; - specialize qw/aom_jnt_sad16x64_avg ssse3/; - specialize qw/aom_jnt_sad64x16_avg ssse3/; + specialize qw/aom_dist_wtd_sad4x16_avg ssse3/; + specialize qw/aom_dist_wtd_sad16x4_avg ssse3/; + specialize qw/aom_dist_wtd_sad8x32_avg ssse3/; + specialize qw/aom_dist_wtd_sad32x8_avg ssse3/; + specialize qw/aom_dist_wtd_sad16x64_avg ssse3/; + specialize qw/aom_dist_wtd_sad64x16_avg ssse3/; add_proto qw/unsigned int/, "aom_sad4xh", "const uint8_t *a, int a_stride, const uint8_t *b, int b_stride, int width, int height"; add_proto qw/unsigned int/, "aom_sad8xh", "const uint8_t *a, int a_stride, const uint8_t *b, int b_stride, int width, int height"; @@ -693,7 +691,7 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") { specialize "aom_highbd_sad${w}x${h}", qw/sse2/; specialize "aom_highbd_sad${w}x${h}_avg", qw/sse2/; } - add_proto qw/unsigned int/, "aom_highbd_jnt_sad${w}x${h}_avg", "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred, const JNT_COMP_PARAMS* jcp_param"; + add_proto qw/unsigned int/, "aom_highbd_dist_wtd_sad${w}x${h}_avg", "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred, const DIST_WTD_COMP_PARAMS* jcp_param"; } specialize qw/aom_highbd_sad128x128 avx2/; specialize qw/aom_highbd_sad128x64 avx2/; @@ -837,6 +835,21 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") { specialize qw/aom_highbd_sad16x64x4d sse2/; specialize qw/aom_highbd_sad64x16x4d sse2/; + # + # hamadard transform and satd for implmenting temporal dependency model + # + add_proto qw/void aom_hadamard_8x8/, "const int16_t *src_diff, ptrdiff_t src_stride, tran_low_t *coeff"; + specialize qw/aom_hadamard_8x8 sse2/; + + add_proto qw/void aom_hadamard_16x16/, "const int16_t *src_diff, ptrdiff_t src_stride, tran_low_t *coeff"; + specialize qw/aom_hadamard_16x16 avx2 sse2/; + + add_proto qw/void aom_hadamard_32x32/, "const int16_t *src_diff, ptrdiff_t src_stride, tran_low_t *coeff"; + specialize qw/aom_hadamard_32x32 avx2 sse2/; + + add_proto qw/int aom_satd/, "const tran_low_t *coeff, int length"; + specialize qw/aom_satd avx2 sse2/; + # # Structured Similarity (SSIM) # @@ -903,11 +916,11 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") { int ref_stride, int subpel_search"; specialize qw/aom_comp_avg_upsampled_pred sse2/; - add_proto qw/void aom_jnt_comp_avg_upsampled_pred/, "MACROBLOCKD *xd, const struct AV1Common *const cm, int mi_row, int mi_col, + add_proto qw/void aom_dist_wtd_comp_avg_upsampled_pred/, "MACROBLOCKD *xd, const struct AV1Common *const cm, int mi_row, int mi_col, const MV *const mv, uint8_t *comp_pred, const uint8_t *pred, int width, int height, int subpel_x_q3, int subpel_y_q3, const uint8_t *ref, - int ref_stride, const JNT_COMP_PARAMS *jcp_param, int subpel_search"; - specialize qw/aom_jnt_comp_avg_upsampled_pred ssse3/; + int ref_stride, const DIST_WTD_COMP_PARAMS *jcp_param, int subpel_search"; + specialize qw/aom_dist_wtd_comp_avg_upsampled_pred ssse3/; add_proto qw/void aom_comp_mask_upsampled_pred/, "MACROBLOCKD *xd, const struct AV1Common *const cm, int mi_row, int mi_col, const MV *const mv, uint8_t *comp_pred, const uint8_t *pred, int width, @@ -916,7 +929,6 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") { int subpel_search"; specialize qw/aom_comp_mask_upsampled_pred sse2/; - add_proto qw/void aom_highbd_upsampled_pred/, "MACROBLOCKD *xd, const struct AV1Common *const cm, int mi_row, int mi_col, const MV *const mv, uint8_t *comp_pred8, int width, int height, int subpel_x_q3, int subpel_y_q3, const uint8_t *ref8, int ref_stride, int bd, int subpel_search"; @@ -927,11 +939,11 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") { int height, int subpel_x_q3, int subpel_y_q3, const uint8_t *ref8, int ref_stride, int bd, int subpel_search"; specialize qw/aom_highbd_comp_avg_upsampled_pred sse2/; - add_proto qw/void aom_highbd_jnt_comp_avg_upsampled_pred/, "MACROBLOCKD *xd, const struct AV1Common *const cm, int mi_row, int mi_col, + add_proto qw/void aom_highbd_dist_wtd_comp_avg_upsampled_pred/, "MACROBLOCKD *xd, const struct AV1Common *const cm, int mi_row, int mi_col, const MV *const mv, uint8_t *comp_pred8, const uint8_t *pred8, int width, int height, int subpel_x_q3, int subpel_y_q3, const uint8_t *ref8, - int ref_stride, int bd, const JNT_COMP_PARAMS *jcp_param, int subpel_search"; - specialize qw/aom_highbd_jnt_comp_avg_upsampled_pred sse2/; + int ref_stride, int bd, const DIST_WTD_COMP_PARAMS *jcp_param, int subpel_search"; + specialize qw/aom_highbd_dist_wtd_comp_avg_upsampled_pred sse2/; # @@ -957,7 +969,7 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") { add_proto qw/unsigned int/, "aom_variance${w}x${h}", "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse"; add_proto qw/uint32_t/, "aom_sub_pixel_variance${w}x${h}", "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse"; add_proto qw/uint32_t/, "aom_sub_pixel_avg_variance${w}x${h}", "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred"; - add_proto qw/uint32_t/, "aom_jnt_sub_pixel_avg_variance${w}x${h}", "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred, const JNT_COMP_PARAMS *jcp_param"; + add_proto qw/uint32_t/, "aom_dist_wtd_sub_pixel_avg_variance${w}x${h}", "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred, const DIST_WTD_COMP_PARAMS *jcp_param"; } specialize qw/aom_variance128x128 sse2 avx2 /; specialize qw/aom_variance128x64 sse2 avx2 /; @@ -1029,30 +1041,30 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") { specialize qw/aom_sub_pixel_avg_variance16x64 sse2 ssse3/; specialize qw/aom_sub_pixel_avg_variance64x16 sse2 ssse3/; - specialize qw/aom_jnt_sub_pixel_avg_variance64x64 ssse3/; - specialize qw/aom_jnt_sub_pixel_avg_variance64x32 ssse3/; - specialize qw/aom_jnt_sub_pixel_avg_variance32x64 ssse3/; - specialize qw/aom_jnt_sub_pixel_avg_variance32x32 ssse3/; - specialize qw/aom_jnt_sub_pixel_avg_variance32x16 ssse3/; - specialize qw/aom_jnt_sub_pixel_avg_variance16x32 ssse3/; - specialize qw/aom_jnt_sub_pixel_avg_variance16x16 ssse3/; - specialize qw/aom_jnt_sub_pixel_avg_variance16x8 ssse3/; - specialize qw/aom_jnt_sub_pixel_avg_variance8x16 ssse3/; - specialize qw/aom_jnt_sub_pixel_avg_variance8x8 ssse3/; - specialize qw/aom_jnt_sub_pixel_avg_variance8x4 ssse3/; - specialize qw/aom_jnt_sub_pixel_avg_variance4x8 ssse3/; - specialize qw/aom_jnt_sub_pixel_avg_variance4x4 ssse3/; + specialize qw/aom_dist_wtd_sub_pixel_avg_variance64x64 ssse3/; + specialize qw/aom_dist_wtd_sub_pixel_avg_variance64x32 ssse3/; + specialize qw/aom_dist_wtd_sub_pixel_avg_variance32x64 ssse3/; + specialize qw/aom_dist_wtd_sub_pixel_avg_variance32x32 ssse3/; + specialize qw/aom_dist_wtd_sub_pixel_avg_variance32x16 ssse3/; + specialize qw/aom_dist_wtd_sub_pixel_avg_variance16x32 ssse3/; + specialize qw/aom_dist_wtd_sub_pixel_avg_variance16x16 ssse3/; + specialize qw/aom_dist_wtd_sub_pixel_avg_variance16x8 ssse3/; + specialize qw/aom_dist_wtd_sub_pixel_avg_variance8x16 ssse3/; + specialize qw/aom_dist_wtd_sub_pixel_avg_variance8x8 ssse3/; + specialize qw/aom_dist_wtd_sub_pixel_avg_variance8x4 ssse3/; + specialize qw/aom_dist_wtd_sub_pixel_avg_variance4x8 ssse3/; + specialize qw/aom_dist_wtd_sub_pixel_avg_variance4x4 ssse3/; - specialize qw/aom_jnt_sub_pixel_avg_variance4x16 ssse3/; - specialize qw/aom_jnt_sub_pixel_avg_variance16x4 ssse3/; - specialize qw/aom_jnt_sub_pixel_avg_variance8x32 ssse3/; - specialize qw/aom_jnt_sub_pixel_avg_variance32x8 ssse3/; - specialize qw/aom_jnt_sub_pixel_avg_variance16x64 ssse3/; - specialize qw/aom_jnt_sub_pixel_avg_variance64x16 ssse3/; + specialize qw/aom_dist_wtd_sub_pixel_avg_variance4x16 ssse3/; + specialize qw/aom_dist_wtd_sub_pixel_avg_variance16x4 ssse3/; + specialize qw/aom_dist_wtd_sub_pixel_avg_variance8x32 ssse3/; + specialize qw/aom_dist_wtd_sub_pixel_avg_variance32x8 ssse3/; + specialize qw/aom_dist_wtd_sub_pixel_avg_variance16x64 ssse3/; + specialize qw/aom_dist_wtd_sub_pixel_avg_variance64x16 ssse3/; - specialize qw/aom_jnt_sub_pixel_avg_variance128x128 ssse3/; - specialize qw/aom_jnt_sub_pixel_avg_variance128x64 ssse3/; - specialize qw/aom_jnt_sub_pixel_avg_variance64x128 ssse3/; + specialize qw/aom_dist_wtd_sub_pixel_avg_variance128x128 ssse3/; + specialize qw/aom_dist_wtd_sub_pixel_avg_variance128x64 ssse3/; + specialize qw/aom_dist_wtd_sub_pixel_avg_variance64x128 ssse3/; foreach $bd (8, 10, 12) { @@ -1084,7 +1096,7 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") { specialize "aom_highbd_${bd}_sub_pixel_avg_variance${w}x${h}", "sse4_1"; } - add_proto qw/uint32_t/, "aom_highbd_${bd}_jnt_sub_pixel_avg_variance${w}x${h}", "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred, const JNT_COMP_PARAMS* jcp_param"; + add_proto qw/uint32_t/, "aom_highbd_${bd}_dist_wtd_sub_pixel_avg_variance${w}x${h}", "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred, const DIST_WTD_COMP_PARAMS* jcp_param"; } } @@ -1173,8 +1185,8 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") { # add_proto qw/void aom_comp_avg_pred/, "uint8_t *comp_pred, const uint8_t *pred, int width, int height, const uint8_t *ref, int ref_stride"; - add_proto qw/void aom_jnt_comp_avg_pred/, "uint8_t *comp_pred, const uint8_t *pred, int width, int height, const uint8_t *ref, int ref_stride, const JNT_COMP_PARAMS *jcp_param"; - specialize qw/aom_jnt_comp_avg_pred ssse3/; + add_proto qw/void aom_dist_wtd_comp_avg_pred/, "uint8_t *comp_pred, const uint8_t *pred, int width, int height, const uint8_t *ref, int ref_stride, const DIST_WTD_COMP_PARAMS *jcp_param"; + specialize qw/aom_dist_wtd_comp_avg_pred ssse3/; add_proto qw/unsigned int aom_highbd_12_variance128x128/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse"; specialize qw/aom_highbd_12_variance128x128 sse2/; @@ -1340,8 +1352,8 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") { add_proto qw/void aom_highbd_comp_avg_pred/, "uint8_t *comp_pred8, const uint8_t *pred8, int width, int height, const uint8_t *ref8, int ref_stride"; - add_proto qw/void aom_highbd_jnt_comp_avg_pred/, "uint8_t *comp_pred8, const uint8_t *pred8, int width, int height, const uint8_t *ref8, int ref_stride, const JNT_COMP_PARAMS *jcp_param"; - specialize qw/aom_highbd_jnt_comp_avg_pred sse2/; + add_proto qw/void aom_highbd_dist_wtd_comp_avg_pred/, "uint8_t *comp_pred8, const uint8_t *pred8, int width, int height, const uint8_t *ref8, int ref_stride, const DIST_WTD_COMP_PARAMS *jcp_param"; + specialize qw/aom_highbd_dist_wtd_comp_avg_pred sse2/; # # Subpixel Variance diff --git a/third_party/aom/aom_dsp/arm/loopfilter_neon.c b/third_party/aom/aom_dsp/arm/loopfilter_neon.c index bdc67626d6..aafac8966d 100644 --- a/third_party/aom/aom_dsp/arm/loopfilter_neon.c +++ b/third_party/aom/aom_dsp/arm/loopfilter_neon.c @@ -785,16 +785,15 @@ void aom_lpf_vertical_4_neon(uint8_t *src, int stride, const uint8_t *blimit, const uint8_t *limit, const uint8_t *thresh) { uint32x2x2_t p1q0_p0q1, p1q1_p0q0, p1p0_q1q0; uint32x2_t pq_rev; - uint8x8_t UNINITIALIZED_IS_SAFE(p1p0), q0q1, p0q0, p1q1; + uint8x8_t UNINITIALIZED_IS_SAFE(p1p0), UNINITIALIZED_IS_SAFE(q0q1); + uint8x8_t p0q0, p1q1; // row0: p1 p0 | q0 q1 // row1: p1 p0 | q0 q1 // row2: p1 p0 | q0 q1 // row3: p1 p0 | q0 q1 - load_u8_4x1(src - 2, &p1p0, 0); - load_u8_4x1((src - 2) + 1 * stride, &p1p0, 1); - load_u8_4x1((src - 2) + 2 * stride, &q0q1, 0); - load_u8_4x1((src - 2) + 3 * stride, &q0q1, 1); + load_unaligned_u8_4x4(src - 2, stride, (uint32x2_t *)&p1p0, + (uint32x2_t *)&q0q1); transpose_u8_4x4(&p1p0, &q0q1); @@ -815,10 +814,10 @@ void aom_lpf_vertical_4_neon(uint8_t *src, int stride, const uint8_t *blimit, transpose_u8_4x4(&p1p0, &q0q1); - store_u8_4x1(src - 2, p1p0, 0); - store_u8_4x1((src - 2) + 1 * stride, q0q1, 0); - store_u8_4x1((src - 2) + 2 * stride, p1p0, 1); - store_u8_4x1((src - 2) + 3 * stride, q0q1, 1); + store_unaligned_u8_4x1(src - 2, p1p0, 0); + store_unaligned_u8_4x1((src - 2) + 1 * stride, q0q1, 0); + store_unaligned_u8_4x1((src - 2) + 2 * stride, p1p0, 1); + store_unaligned_u8_4x1((src - 2) + 3 * stride, q0q1, 1); } void aom_lpf_horizontal_14_neon(uint8_t *src, int stride, const uint8_t *blimit, diff --git a/third_party/aom/aom_dsp/avg.c b/third_party/aom/aom_dsp/avg.c new file mode 100644 index 0000000000..4d78c9c569 --- /dev/null +++ b/third_party/aom/aom_dsp/avg.c @@ -0,0 +1,148 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "config/aom_dsp_rtcd.h" +#include "aom_ports/mem.h" + +// src_diff: first pass, 9 bit, dynamic range [-255, 255] +// second pass, 12 bit, dynamic range [-2040, 2040] +static void hadamard_col8(const int16_t *src_diff, ptrdiff_t src_stride, + int16_t *coeff) { + int16_t b0 = src_diff[0 * src_stride] + src_diff[1 * src_stride]; + int16_t b1 = src_diff[0 * src_stride] - src_diff[1 * src_stride]; + int16_t b2 = src_diff[2 * src_stride] + src_diff[3 * src_stride]; + int16_t b3 = src_diff[2 * src_stride] - src_diff[3 * src_stride]; + int16_t b4 = src_diff[4 * src_stride] + src_diff[5 * src_stride]; + int16_t b5 = src_diff[4 * src_stride] - src_diff[5 * src_stride]; + int16_t b6 = src_diff[6 * src_stride] + src_diff[7 * src_stride]; + int16_t b7 = src_diff[6 * src_stride] - src_diff[7 * src_stride]; + + int16_t c0 = b0 + b2; + int16_t c1 = b1 + b3; + int16_t c2 = b0 - b2; + int16_t c3 = b1 - b3; + int16_t c4 = b4 + b6; + int16_t c5 = b5 + b7; + int16_t c6 = b4 - b6; + int16_t c7 = b5 - b7; + + coeff[0] = c0 + c4; + coeff[7] = c1 + c5; + coeff[3] = c2 + c6; + coeff[4] = c3 + c7; + coeff[2] = c0 - c4; + coeff[6] = c1 - c5; + coeff[1] = c2 - c6; + coeff[5] = c3 - c7; +} + +// The order of the output coeff of the hadamard is not important. For +// optimization purposes the final transpose may be skipped. +void aom_hadamard_8x8_c(const int16_t *src_diff, ptrdiff_t src_stride, + tran_low_t *coeff) { + int idx; + int16_t buffer[64]; + int16_t buffer2[64]; + int16_t *tmp_buf = &buffer[0]; + for (idx = 0; idx < 8; ++idx) { + hadamard_col8(src_diff, src_stride, tmp_buf); // src_diff: 9 bit + // dynamic range [-255, 255] + tmp_buf += 8; + ++src_diff; + } + + tmp_buf = &buffer[0]; + for (idx = 0; idx < 8; ++idx) { + hadamard_col8(tmp_buf, 8, buffer2 + 8 * idx); // tmp_buf: 12 bit + // dynamic range [-2040, 2040] + // buffer2: 15 bit + // dynamic range [-16320, 16320] + ++tmp_buf; + } + + for (idx = 0; idx < 64; ++idx) coeff[idx] = (tran_low_t)buffer2[idx]; +} + +// In place 16x16 2D Hadamard transform +void aom_hadamard_16x16_c(const int16_t *src_diff, ptrdiff_t src_stride, + tran_low_t *coeff) { + int idx; + for (idx = 0; idx < 4; ++idx) { + // src_diff: 9 bit, dynamic range [-255, 255] + const int16_t *src_ptr = + src_diff + (idx >> 1) * 8 * src_stride + (idx & 0x01) * 8; + aom_hadamard_8x8_c(src_ptr, src_stride, coeff + idx * 64); + } + + // coeff: 15 bit, dynamic range [-16320, 16320] + for (idx = 0; idx < 64; ++idx) { + tran_low_t a0 = coeff[0]; + tran_low_t a1 = coeff[64]; + tran_low_t a2 = coeff[128]; + tran_low_t a3 = coeff[192]; + + tran_low_t b0 = (a0 + a1) >> 1; // (a0 + a1): 16 bit, [-32640, 32640] + tran_low_t b1 = (a0 - a1) >> 1; // b0-b3: 15 bit, dynamic range + tran_low_t b2 = (a2 + a3) >> 1; // [-16320, 16320] + tran_low_t b3 = (a2 - a3) >> 1; + + coeff[0] = b0 + b2; // 16 bit, [-32640, 32640] + coeff[64] = b1 + b3; + coeff[128] = b0 - b2; + coeff[192] = b1 - b3; + + ++coeff; + } +} + +void aom_hadamard_32x32_c(const int16_t *src_diff, ptrdiff_t src_stride, + tran_low_t *coeff) { + int idx; + for (idx = 0; idx < 4; ++idx) { + // src_diff: 9 bit, dynamic range [-255, 255] + const int16_t *src_ptr = + src_diff + (idx >> 1) * 16 * src_stride + (idx & 0x01) * 16; + aom_hadamard_16x16_c(src_ptr, src_stride, coeff + idx * 256); + } + + // coeff: 15 bit, dynamic range [-16320, 16320] + for (idx = 0; idx < 256; ++idx) { + tran_low_t a0 = coeff[0]; + tran_low_t a1 = coeff[256]; + tran_low_t a2 = coeff[512]; + tran_low_t a3 = coeff[768]; + + tran_low_t b0 = (a0 + a1) >> 2; // (a0 + a1): 16 bit, [-32640, 32640] + tran_low_t b1 = (a0 - a1) >> 2; // b0-b3: 15 bit, dynamic range + tran_low_t b2 = (a2 + a3) >> 2; // [-16320, 16320] + tran_low_t b3 = (a2 - a3) >> 2; + + coeff[0] = b0 + b2; // 16 bit, [-32640, 32640] + coeff[256] = b1 + b3; + coeff[512] = b0 - b2; + coeff[768] = b1 - b3; + + ++coeff; + } +} + +// coeff: 16 bits, dynamic range [-32640, 32640]. +// length: value range {16, 64, 256, 1024}. +int aom_satd_c(const tran_low_t *coeff, int length) { + int i; + int satd = 0; + for (i = 0; i < length; ++i) satd += abs(coeff[i]); + + // satd: 26 bits, dynamic range [-32640 * 1024, 32640 * 1024] + return satd; +} diff --git a/third_party/aom/aom_dsp/binary_codes_reader.c b/third_party/aom/aom_dsp/binary_codes_reader.c index 01088010a1..7cd903d821 100644 --- a/third_party/aom/aom_dsp/binary_codes_reader.c +++ b/third_party/aom/aom_dsp/binary_codes_reader.c @@ -10,29 +10,9 @@ */ #include "aom_dsp/binary_codes_reader.h" - +#include "aom_dsp/recenter.h" #include "av1/common/common.h" -// Inverse recenters a non-negative literal v around a reference r -static uint16_t inv_recenter_nonneg(uint16_t r, uint16_t v) { - if (v > (r << 1)) - return v; - else if ((v & 1) == 0) - return (v >> 1) + r; - else - return r - ((v + 1) >> 1); -} - -// Inverse recenters a non-negative literal v in [0, n-1] around a -// reference r also in [0, n-1] -static uint16_t inv_recenter_finite_nonneg(uint16_t n, uint16_t r, uint16_t v) { - if ((r << 1) <= n) { - return inv_recenter_nonneg(r, v); - } else { - return n - 1 - inv_recenter_nonneg(n - 1 - r, v); - } -} - uint16_t aom_read_primitive_quniform_(aom_reader *r, uint16_t n ACCT_STR_PARAM) { if (n <= 1) return 0; @@ -42,15 +22,6 @@ uint16_t aom_read_primitive_quniform_(aom_reader *r, return v < m ? v : (v << 1) - m + aom_read_bit(r, ACCT_STR_NAME); } -static uint16_t aom_rb_read_primitive_quniform(struct aom_read_bit_buffer *rb, - uint16_t n) { - if (n <= 1) return 0; - const int l = get_msb(n) + 1; - const int m = (1 << l) - n; - const int v = aom_rb_read_literal(rb, l - 1); - return v < m ? v : (v << 1) - m + aom_rb_read_bit(rb); -} - // Decode finite subexponential code that for a symbol v in [0, n-1] with // parameter k uint16_t aom_read_primitive_subexpfin_(aom_reader *r, uint16_t n, @@ -78,46 +49,8 @@ uint16_t aom_read_primitive_subexpfin_(aom_reader *r, uint16_t n, return 0; } -static uint16_t aom_rb_read_primitive_subexpfin(struct aom_read_bit_buffer *rb, - uint16_t n, uint16_t k) { - int i = 0; - int mk = 0; - - while (1) { - int b = (i ? k + i - 1 : k); - int a = (1 << b); - - if (n <= mk + 3 * a) { - return aom_rb_read_primitive_quniform(rb, n - mk) + mk; - } - - if (!aom_rb_read_bit(rb)) { - return aom_rb_read_literal(rb, b) + mk; - } - - i = i + 1; - mk += a; - } - - assert(0); - return 0; -} - uint16_t aom_read_primitive_refsubexpfin_(aom_reader *r, uint16_t n, uint16_t k, uint16_t ref ACCT_STR_PARAM) { return inv_recenter_finite_nonneg( n, ref, aom_read_primitive_subexpfin(r, n, k, ACCT_STR_NAME)); } - -static uint16_t aom_rb_read_primitive_refsubexpfin( - struct aom_read_bit_buffer *rb, uint16_t n, uint16_t k, uint16_t ref) { - return inv_recenter_finite_nonneg(n, ref, - aom_rb_read_primitive_subexpfin(rb, n, k)); -} - -int16_t aom_rb_read_signed_primitive_refsubexpfin( - struct aom_read_bit_buffer *rb, uint16_t n, uint16_t k, int16_t ref) { - ref += n - 1; - const uint16_t scaled_n = (n << 1) - 1; - return aom_rb_read_primitive_refsubexpfin(rb, scaled_n, k, ref) - n + 1; -} diff --git a/third_party/aom/aom_dsp/binary_codes_reader.h b/third_party/aom/aom_dsp/binary_codes_reader.h index 364a67469a..d218f0619f 100644 --- a/third_party/aom/aom_dsp/binary_codes_reader.h +++ b/third_party/aom/aom_dsp/binary_codes_reader.h @@ -37,9 +37,6 @@ uint16_t aom_read_primitive_subexpfin_(aom_reader *r, uint16_t n, uint16_t aom_read_primitive_refsubexpfin_(aom_reader *r, uint16_t n, uint16_t k, uint16_t ref ACCT_STR_PARAM); -int16_t aom_rb_read_signed_primitive_refsubexpfin( - struct aom_read_bit_buffer *rb, uint16_t n, uint16_t k, int16_t ref); - #ifdef __cplusplus } // extern "C" #endif diff --git a/third_party/aom/aom_dsp/binary_codes_writer.c b/third_party/aom/aom_dsp/binary_codes_writer.c index ee7a9f567e..adf1c1304c 100644 --- a/third_party/aom/aom_dsp/binary_codes_writer.c +++ b/third_party/aom/aom_dsp/binary_codes_writer.c @@ -11,29 +11,10 @@ #include "aom_dsp/bitwriter.h" #include "aom_dsp/binary_codes_writer.h" - +#include "aom_dsp/recenter.h" +#include "aom_ports/bitops.h" #include "av1/common/common.h" -// Recenters a non-negative literal v around a reference r -static uint16_t recenter_nonneg(uint16_t r, uint16_t v) { - if (v > (r << 1)) - return v; - else if (v >= r) - return ((v - r) << 1); - else - return ((r - v) << 1) - 1; -} - -// Recenters a non-negative literal v in [0, n-1] around a -// reference r also in [0, n-1] -static uint16_t recenter_finite_nonneg(uint16_t n, uint16_t r, uint16_t v) { - if ((r << 1) <= n) { - return recenter_nonneg(r, v); - } else { - return recenter_nonneg(n - 1 - r, n - 1 - v); - } -} - // Codes a symbol v in [-2^mag_bits, 2^mag_bits]. // mag_bits is number of bits for magnitude. The alphabet is of size // 2 * 2^mag_bits + 1, symmetric around 0, where one bit is used to @@ -69,19 +50,6 @@ void aom_write_primitive_quniform(aom_writer *w, uint16_t n, uint16_t v) { } } -static void aom_wb_write_primitive_quniform(struct aom_write_bit_buffer *wb, - uint16_t n, uint16_t v) { - if (n <= 1) return; - const int l = get_msb(n) + 1; - const int m = (1 << l) - n; - if (v < m) { - aom_wb_write_literal(wb, v, l - 1); - } else { - aom_wb_write_literal(wb, m + ((v - m) >> 1), l - 1); - aom_wb_write_bit(wb, (v - m) & 1); - } -} - int aom_count_primitive_quniform(uint16_t n, uint16_t v) { if (n <= 1) return 0; const int l = get_msb(n) + 1; @@ -114,31 +82,6 @@ void aom_write_primitive_subexpfin(aom_writer *w, uint16_t n, uint16_t k, } } -static void aom_wb_write_primitive_subexpfin(struct aom_write_bit_buffer *wb, - uint16_t n, uint16_t k, - uint16_t v) { - int i = 0; - int mk = 0; - while (1) { - int b = (i ? k + i - 1 : k); - int a = (1 << b); - if (n <= mk + 3 * a) { - aom_wb_write_primitive_quniform(wb, n - mk, v - mk); - break; - } else { - int t = (v >= mk + a); - aom_wb_write_bit(wb, t); - if (t) { - i = i + 1; - mk += a; - } else { - aom_wb_write_literal(wb, v - mk, b); - break; - } - } - } -} - int aom_count_primitive_subexpfin(uint16_t n, uint16_t k, uint16_t v) { int count = 0; int i = 0; @@ -172,12 +115,6 @@ void aom_write_primitive_refsubexpfin(aom_writer *w, uint16_t n, uint16_t k, aom_write_primitive_subexpfin(w, n, k, recenter_finite_nonneg(n, ref, v)); } -static void aom_wb_write_primitive_refsubexpfin(struct aom_write_bit_buffer *wb, - uint16_t n, uint16_t k, - uint16_t ref, uint16_t v) { - aom_wb_write_primitive_subexpfin(wb, n, k, recenter_finite_nonneg(n, ref, v)); -} - void aom_write_signed_primitive_refsubexpfin(aom_writer *w, uint16_t n, uint16_t k, int16_t ref, int16_t v) { @@ -187,15 +124,6 @@ void aom_write_signed_primitive_refsubexpfin(aom_writer *w, uint16_t n, aom_write_primitive_refsubexpfin(w, scaled_n, k, ref, v); } -void aom_wb_write_signed_primitive_refsubexpfin(struct aom_write_bit_buffer *wb, - uint16_t n, uint16_t k, - int16_t ref, int16_t v) { - ref += n - 1; - v += n - 1; - const uint16_t scaled_n = (n << 1) - 1; - aom_wb_write_primitive_refsubexpfin(wb, scaled_n, k, ref, v); -} - int aom_count_primitive_refsubexpfin(uint16_t n, uint16_t k, uint16_t ref, uint16_t v) { return aom_count_primitive_subexpfin(n, k, recenter_finite_nonneg(n, ref, v)); diff --git a/third_party/aom/aom_dsp/binary_codes_writer.h b/third_party/aom/aom_dsp/binary_codes_writer.h index c360e0e298..5ec8662139 100644 --- a/third_party/aom/aom_dsp/binary_codes_writer.h +++ b/third_party/aom/aom_dsp/binary_codes_writer.h @@ -49,10 +49,6 @@ void aom_write_signed_primitive_refsubexpfin(aom_writer *w, uint16_t n, uint16_t k, int16_t ref, int16_t v); -void aom_wb_write_signed_primitive_refsubexpfin(struct aom_write_bit_buffer *wb, - uint16_t n, uint16_t k, - int16_t ref, int16_t v); - // Functions that counts bits for the above primitives int aom_count_primitive_symmetric(int16_t v, unsigned int mag_bits); int aom_count_primitive_quniform(uint16_t n, uint16_t v); @@ -61,6 +57,7 @@ int aom_count_primitive_refsubexpfin(uint16_t n, uint16_t k, uint16_t ref, uint16_t v); int aom_count_signed_primitive_refsubexpfin(uint16_t n, uint16_t k, int16_t ref, int16_t v); + #ifdef __cplusplus } // extern "C" #endif diff --git a/third_party/aom/aom_dsp/bitreader.h b/third_party/aom/aom_dsp/bitreader.h index 7c0efcc785..38b17eae5f 100644 --- a/third_party/aom/aom_dsp/bitreader.h +++ b/third_party/aom/aom_dsp/bitreader.h @@ -65,10 +65,6 @@ static INLINE const uint8_t *aom_reader_find_end(aom_reader *r) { return aom_daala_reader_find_end(r); } -static INLINE int aom_reader_has_error(aom_reader *r) { - return aom_daala_reader_has_error(r); -} - // Returns true if the bit reader has tried to decode more data from the buffer // than was actually provided. static INLINE int aom_reader_has_overflowed(const aom_reader *r) { diff --git a/third_party/aom/aom_dsp/bitreader_buffer.c b/third_party/aom/aom_dsp/bitreader_buffer.c index b532117849..984b217738 100644 --- a/third_party/aom/aom_dsp/bitreader_buffer.c +++ b/third_party/aom/aom_dsp/bitreader_buffer.c @@ -14,6 +14,8 @@ #include "config/aom_config.h" #include "aom_dsp/bitreader_buffer.h" +#include "aom_dsp/recenter.h" +#include "aom_ports/bitops.h" size_t aom_rb_bytes_read(const struct aom_read_bit_buffer *rb) { return (rb->bit_offset + 7) >> 3; @@ -65,3 +67,50 @@ uint32_t aom_rb_read_uvlc(struct aom_read_bit_buffer *rb) { const uint32_t value = aom_rb_read_literal(rb, leading_zeros); return base + value; } + +static uint16_t aom_rb_read_primitive_quniform(struct aom_read_bit_buffer *rb, + uint16_t n) { + if (n <= 1) return 0; + const int l = get_msb(n) + 1; + const int m = (1 << l) - n; + const int v = aom_rb_read_literal(rb, l - 1); + return v < m ? v : (v << 1) - m + aom_rb_read_bit(rb); +} + +static uint16_t aom_rb_read_primitive_subexpfin(struct aom_read_bit_buffer *rb, + uint16_t n, uint16_t k) { + int i = 0; + int mk = 0; + + while (1) { + int b = (i ? k + i - 1 : k); + int a = (1 << b); + + if (n <= mk + 3 * a) { + return aom_rb_read_primitive_quniform(rb, n - mk) + mk; + } + + if (!aom_rb_read_bit(rb)) { + return aom_rb_read_literal(rb, b) + mk; + } + + i = i + 1; + mk += a; + } + + assert(0); + return 0; +} + +static uint16_t aom_rb_read_primitive_refsubexpfin( + struct aom_read_bit_buffer *rb, uint16_t n, uint16_t k, uint16_t ref) { + return inv_recenter_finite_nonneg(n, ref, + aom_rb_read_primitive_subexpfin(rb, n, k)); +} + +int16_t aom_rb_read_signed_primitive_refsubexpfin( + struct aom_read_bit_buffer *rb, uint16_t n, uint16_t k, int16_t ref) { + ref += n - 1; + const uint16_t scaled_n = (n << 1) - 1; + return aom_rb_read_primitive_refsubexpfin(rb, scaled_n, k, ref) - n + 1; +} diff --git a/third_party/aom/aom_dsp/bitreader_buffer.h b/third_party/aom/aom_dsp/bitreader_buffer.h index 725ca1ea2b..359fbe5194 100644 --- a/third_party/aom/aom_dsp/bitreader_buffer.h +++ b/third_party/aom/aom_dsp/bitreader_buffer.h @@ -43,6 +43,9 @@ int aom_rb_read_inv_signed_literal(struct aom_read_bit_buffer *rb, int bits); uint32_t aom_rb_read_uvlc(struct aom_read_bit_buffer *rb); +int16_t aom_rb_read_signed_primitive_refsubexpfin( + struct aom_read_bit_buffer *rb, uint16_t n, uint16_t k, int16_t ref); + #ifdef __cplusplus } // extern "C" #endif diff --git a/third_party/aom/aom_dsp/bitwriter_buffer.c b/third_party/aom/aom_dsp/bitwriter_buffer.c index 596246deb2..c08cc9da0b 100644 --- a/third_party/aom/aom_dsp/bitwriter_buffer.c +++ b/third_party/aom/aom_dsp/bitwriter_buffer.c @@ -16,6 +16,8 @@ #include "config/aom_config.h" #include "aom_dsp/bitwriter_buffer.h" +#include "aom_dsp/recenter.h" +#include "aom_ports/bitops.h" int aom_wb_is_byte_aligned(const struct aom_write_bit_buffer *wb) { return (wb->bit_offset % CHAR_BIT == 0); @@ -85,3 +87,56 @@ void aom_wb_write_uvlc(struct aom_write_bit_buffer *wb, uint32_t v) { aom_wb_write_literal(wb, 0, leading_zeroes >> 1); aom_wb_write_unsigned_literal(wb, v, (leading_zeroes + 1) >> 1); } + +static void aom_wb_write_primitive_quniform(struct aom_write_bit_buffer *wb, + uint16_t n, uint16_t v) { + if (n <= 1) return; + const int l = get_msb(n) + 1; + const int m = (1 << l) - n; + if (v < m) { + aom_wb_write_literal(wb, v, l - 1); + } else { + aom_wb_write_literal(wb, m + ((v - m) >> 1), l - 1); + aom_wb_write_bit(wb, (v - m) & 1); + } +} + +static void aom_wb_write_primitive_subexpfin(struct aom_write_bit_buffer *wb, + uint16_t n, uint16_t k, + uint16_t v) { + int i = 0; + int mk = 0; + while (1) { + int b = (i ? k + i - 1 : k); + int a = (1 << b); + if (n <= mk + 3 * a) { + aom_wb_write_primitive_quniform(wb, n - mk, v - mk); + break; + } else { + int t = (v >= mk + a); + aom_wb_write_bit(wb, t); + if (t) { + i = i + 1; + mk += a; + } else { + aom_wb_write_literal(wb, v - mk, b); + break; + } + } + } +} + +static void aom_wb_write_primitive_refsubexpfin(struct aom_write_bit_buffer *wb, + uint16_t n, uint16_t k, + uint16_t ref, uint16_t v) { + aom_wb_write_primitive_subexpfin(wb, n, k, recenter_finite_nonneg(n, ref, v)); +} + +void aom_wb_write_signed_primitive_refsubexpfin(struct aom_write_bit_buffer *wb, + uint16_t n, uint16_t k, + int16_t ref, int16_t v) { + ref += n - 1; + v += n - 1; + const uint16_t scaled_n = (n << 1) - 1; + aom_wb_write_primitive_refsubexpfin(wb, scaled_n, k, ref, v); +} diff --git a/third_party/aom/aom_dsp/bitwriter_buffer.h b/third_party/aom/aom_dsp/bitwriter_buffer.h index d0311284f2..fd10e01bb7 100644 --- a/third_party/aom/aom_dsp/bitwriter_buffer.h +++ b/third_party/aom/aom_dsp/bitwriter_buffer.h @@ -44,6 +44,10 @@ void aom_wb_write_inv_signed_literal(struct aom_write_bit_buffer *wb, int data, void aom_wb_write_uvlc(struct aom_write_bit_buffer *wb, uint32_t v); +void aom_wb_write_signed_primitive_refsubexpfin(struct aom_write_bit_buffer *wb, + uint16_t n, uint16_t k, + int16_t ref, int16_t v); + #ifdef __cplusplus } // extern "C" #endif diff --git a/third_party/aom/aom_dsp/blend_a64_mask.c b/third_party/aom/aom_dsp/blend_a64_mask.c index 992cc5c0c4..79956c3a57 100644 --- a/third_party/aom/aom_dsp/blend_a64_mask.c +++ b/third_party/aom/aom_dsp/blend_a64_mask.c @@ -22,7 +22,7 @@ // as described for AOM_BLEND_A64 in aom_dsp/blend.h. src0 or src1 can // be the same as dst, or dst can be different from both sources. -// NOTE(david.barker): The input and output of aom_blend_a64_d32_mask_c() are +// NOTE(david.barker): The input and output of aom_blend_a64_d16_mask_c() are // in a higher intermediate precision, and will later be rounded down to pixel // precision. // Thus, in order to avoid double-rounding, we want to use normal right shifts @@ -30,7 +30,7 @@ // This works because of the identity: // ROUND_POWER_OF_TWO(x >> y, z) == ROUND_POWER_OF_TWO(x, y+z) // -// In contrast, the output of the non-d32 functions will not be further rounded, +// In contrast, the output of the non-d16 functions will not be further rounded, // so we *should* use ROUND_POWER_OF_TWO there. void aom_lowbd_blend_a64_d16_mask_c( diff --git a/third_party/aom/aom_dsp/daalaboolreader.h b/third_party/aom/aom_dsp/daalaboolreader.h index ba78f916d7..b1810fcda0 100644 --- a/third_party/aom/aom_dsp/daalaboolreader.h +++ b/third_party/aom/aom_dsp/daalaboolreader.h @@ -101,10 +101,6 @@ static INLINE int aom_daala_read(daala_reader *r, int prob) { return bit; } -static INLINE int aom_daala_reader_has_error(daala_reader *r) { - return r->ec.error; -} - static INLINE int daala_read_symbol(daala_reader *r, const aom_cdf_prob *cdf, int nsymbs) { int symb; diff --git a/third_party/aom/aom_dsp/entcode.h b/third_party/aom/aom_dsp/entcode.h index 7ba2b1c392..7518879217 100644 --- a/third_party/aom/aom_dsp/entcode.h +++ b/third_party/aom/aom_dsp/entcode.h @@ -24,6 +24,7 @@ on a larger type, you can speed up the decoder by using it here.*/ typedef uint32_t od_ec_window; +/*The size in bits of od_ec_window.*/ #define OD_EC_WINDOW_SIZE ((int)sizeof(od_ec_window) * CHAR_BIT) /*The resolution of fractional-precision bit usage measurements, i.e., diff --git a/third_party/aom/aom_dsp/entdec.c b/third_party/aom/aom_dsp/entdec.c index d1764c47b1..da43e8a399 100644 --- a/third_party/aom/aom_dsp/entdec.c +++ b/third_party/aom/aom_dsp/entdec.c @@ -87,11 +87,25 @@ static void od_ec_dec_refill(od_ec_dec *dec) { end = dec->end; s = OD_EC_WINDOW_SIZE - 9 - (cnt + 15); for (; s >= 0 && bptr < end; s -= 8, bptr++) { + /*Each time a byte is inserted into the window (dif), bptr advances and cnt + is incremented by 8, so the total number of consumed bits (the return + value of od_ec_dec_tell) does not change.*/ assert(s <= OD_EC_WINDOW_SIZE - 8); dif ^= (od_ec_window)bptr[0] << s; cnt += 8; } if (bptr >= end) { + /*We've reached the end of the buffer. It is perfectly valid for us to need + to fill the window with additional bits past the end of the buffer (and + this happens in normal operation). These bits should all just be taken + as zero. But we cannot increment bptr past 'end' (this is undefined + behavior), so we start to increment dec->tell_offs. We also don't want + to keep testing bptr against 'end', so we set cnt to OD_EC_LOTS_OF_BITS + and adjust dec->tell_offs so that the total number of unconsumed bits in + the window (dec->cnt - dec->tell_offs) does not change. This effectively + puts lots of zero bits into the window, and means we won't try to refill + it from the buffer for a very long time (at which point we'll put lots + of zero bits into the window again).*/ dec->tell_offs += OD_EC_LOTS_OF_BITS - cnt; cnt = OD_EC_LOTS_OF_BITS; } @@ -112,8 +126,9 @@ static int od_ec_dec_normalize(od_ec_dec *dec, od_ec_window dif, unsigned rng, int ret) { int d; assert(rng <= 65535U); - // The number of leading zeros in the 16-bit binary representation of rng. + /*The number of leading zeros in the 16-bit binary representation of rng.*/ d = 16 - OD_ILOG_NZ(rng); + /*d bits in dec->dif are consumed.*/ dec->cnt -= d; /*This is equivalent to shifting in 1's instead of 0's.*/ dec->dif = ((dif + 1) << d) - 1; @@ -124,7 +139,7 @@ static int od_ec_dec_normalize(od_ec_dec *dec, od_ec_window dif, unsigned rng, /*Initializes the decoder. buf: The input buffer to use. - Return: 0 on success, or a negative value on error.*/ + storage: The size in bytes of the input buffer.*/ void od_ec_dec_init(od_ec_dec *dec, const unsigned char *buf, uint32_t storage) { dec->buf = buf; @@ -134,7 +149,6 @@ void od_ec_dec_init(od_ec_dec *dec, const unsigned char *buf, dec->dif = ((od_ec_window)1 << (OD_EC_WINDOW_SIZE - 1)) - 1; dec->rng = 0x8000; dec->cnt = -15; - dec->error = 0; od_ec_dec_refill(dec); } @@ -215,6 +229,10 @@ int od_ec_decode_cdf_q15(od_ec_dec *dec, const uint16_t *icdf, int nsyms) { This will always be slightly larger than the exact value (e.g., all rounding error is in the positive direction).*/ int od_ec_dec_tell(const od_ec_dec *dec) { + /*There is a window of bits stored in dec->dif. The difference + (dec->bptr - dec->buf) tells us how many bytes have been read into this + window. The difference (dec->cnt - dec->tell_offs) tells us how many of + the bits in that window remain unconsumed.*/ return (int)((dec->bptr - dec->buf) * 8 - dec->cnt + dec->tell_offs); } diff --git a/third_party/aom/aom_dsp/entdec.h b/third_party/aom/aom_dsp/entdec.h index 283bf18315..c746167775 100644 --- a/third_party/aom/aom_dsp/entdec.h +++ b/third_party/aom/aom_dsp/entdec.h @@ -34,7 +34,7 @@ struct od_ec_dec { const unsigned char *buf; /*An offset used to keep track of tell after reaching the end of the stream. This is constant throughout most of the decoding process, but becomes - important once we hit the end of the buffer and stop incrementing pointers + important once we hit the end of the buffer and stop incrementing bptr (and instead pretend cnt has lots of bits).*/ int32_t tell_offs; /*The end of the current input buffer.*/ @@ -53,8 +53,6 @@ struct od_ec_dec { uint16_t rng; /*The number of bits of data in the current value.*/ int16_t cnt; - /*Nonzero if an error occurred.*/ - int error; }; /*See entdec.c for further documentation.*/ diff --git a/third_party/aom/aom_dsp/entenc.c b/third_party/aom/aom_dsp/entenc.c index a61da263cb..2fd4493eab 100644 --- a/third_party/aom/aom_dsp/entenc.c +++ b/third_party/aom/aom_dsp/entenc.c @@ -60,7 +60,7 @@ static void od_ec_enc_normalize(od_ec_enc *enc, od_ec_window low, int s; c = enc->cnt; assert(rng <= 65535U); - // The number of leading zeros in the 16-bit binary representation of rng. + /*The number of leading zeros in the 16-bit binary representation of rng.*/ d = 16 - OD_ILOG_NZ(rng); s = c + d; /*TODO: Right now we flush every time we have at least one byte available. diff --git a/third_party/aom/aom_dsp/grain_synthesis.h b/third_party/aom/aom_dsp/grain_synthesis.h index 7aee6f6f44..9155b39035 100644 --- a/third_party/aom/aom_dsp/grain_synthesis.h +++ b/third_party/aom/aom_dsp/grain_synthesis.h @@ -20,6 +20,8 @@ extern "C" { #endif +#include + #include "aom_dsp/aom_dsp_common.h" #include "aom/aom_image.h" @@ -28,6 +30,9 @@ extern "C" { * This structure contains input parameters for film grain synthesis */ typedef struct { + // This structure is compared element-by-element in the function + // av1_check_grain_params_equiv: this function must be updated if any changes + // are made to this structure. int apply_grain; int update_parameters; @@ -79,8 +84,73 @@ typedef struct { int grain_scale_shift; uint16_t random_seed; + // This structure is compared element-by-element in the function + // av1_check_grain_params_equiv: this function must be updated if any changes + // are made to this structure. } aom_film_grain_t; +/*!\brief Check if two film grain parameters structs are equivalent + * + * Check if two film grain parameters are equal, except for the + * update_parameters and random_seed elements which are ignored. + * + * \param[in] pa The first set of parameters to compare + * \param[in] pb The second set of parameters to compare + * \return Returns 1 if the params are equivalent, 0 otherwise + */ +static INLINE int av1_check_grain_params_equiv( + const aom_film_grain_t *const pa, const aom_film_grain_t *const pb) { + if (pa->apply_grain != pb->apply_grain) return 0; + // Don't compare update_parameters + + if (pa->num_y_points != pb->num_y_points) return 0; + if (memcmp(pa->scaling_points_y, pb->scaling_points_y, + pa->num_y_points * 2 * sizeof(*pa->scaling_points_y)) != 0) + return 0; + + if (pa->num_cb_points != pb->num_cb_points) return 0; + if (memcmp(pa->scaling_points_cb, pb->scaling_points_cb, + pa->num_cb_points * 2 * sizeof(*pa->scaling_points_cb)) != 0) + return 0; + + if (pa->num_cr_points != pb->num_cr_points) return 0; + if (memcmp(pa->scaling_points_cr, pb->scaling_points_cr, + pa->num_cr_points * 2 * sizeof(*pa->scaling_points_cr)) != 0) + return 0; + + if (pa->scaling_shift != pb->scaling_shift) return 0; + if (pa->ar_coeff_lag != pb->ar_coeff_lag) return 0; + + const int num_pos = 2 * pa->ar_coeff_lag * (pa->ar_coeff_lag + 1); + if (memcmp(pa->ar_coeffs_y, pb->ar_coeffs_y, + num_pos * sizeof(*pa->ar_coeffs_y)) != 0) + return 0; + if (memcmp(pa->ar_coeffs_cb, pb->ar_coeffs_cb, + num_pos * sizeof(*pa->ar_coeffs_cb)) != 0) + return 0; + if (memcmp(pa->ar_coeffs_cr, pb->ar_coeffs_cr, + num_pos * sizeof(*pa->ar_coeffs_cr)) != 0) + return 0; + + if (pa->ar_coeff_shift != pb->ar_coeff_shift) return 0; + + if (pa->cb_mult != pb->cb_mult) return 0; + if (pa->cb_luma_mult != pb->cb_luma_mult) return 0; + if (pa->cb_offset != pb->cb_offset) return 0; + + if (pa->cr_mult != pb->cr_mult) return 0; + if (pa->cr_luma_mult != pb->cr_luma_mult) return 0; + if (pa->cr_offset != pb->cr_offset) return 0; + + if (pa->overlap_flag != pb->overlap_flag) return 0; + if (pa->clip_to_restricted_range != pb->clip_to_restricted_range) return 0; + if (pa->bit_depth != pb->bit_depth) return 0; + if (pa->chroma_scaling_from_luma != pb->chroma_scaling_from_luma) return 0; + if (pa->grain_scale_shift != pb->grain_scale_shift) return 0; + + return 1; +} + /*!\brief Add film grain * * Add film grain to an image diff --git a/third_party/aom/aom_dsp/grain_table.c b/third_party/aom/aom_dsp/grain_table.c index 0d6a73f558..5eb5b68b79 100644 --- a/third_party/aom/aom_dsp/grain_table.c +++ b/third_party/aom/aom_dsp/grain_table.c @@ -134,7 +134,8 @@ static void grain_table_entry_read(FILE *file, } } -void grain_table_entry_write(FILE *file, aom_film_grain_table_entry_t *entry) { +static void grain_table_entry_write(FILE *file, + aom_film_grain_table_entry_t *entry) { const aom_film_grain_t *pars = &entry->params; fprintf(file, "E %" PRId64 " %" PRId64 " %d %d %d\n", entry->start_time, entry->end_time, pars->apply_grain, pars->random_seed, diff --git a/third_party/aom/aom_dsp/intrapred.c b/third_party/aom/aom_dsp/intrapred.c index c6aa6b207b..72ccfd8358 100644 --- a/third_party/aom/aom_dsp/intrapred.c +++ b/third_party/aom/aom_dsp/intrapred.c @@ -239,23 +239,23 @@ static INLINE int divide_using_multiply_shift(int num, int shift1, return interm * multiplier >> shift2; } - // The constants (multiplier and shifts) for a given block size are obtained - // as follows: - // - Let sum_w_h = block width + block height. - // - Shift 'sum_w_h' right until we reach an odd number. Let the number of - // shifts for that block size be called 'shift1' (see the parameter in - // dc_predictor_rect() function), and let the odd number be 'd'. [d has only 2 - // possible values: d = 3 for a 1:2 rect block and d = 5 for a 1:4 rect - // block]. - // - Find multipliers for (i) dividing by 3, and (ii) dividing by 5, - // using the "Algorithm 1" in: - // http://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=1467632 - // by ensuring that m + n = 16 (in that algorithm). This ensures that our 2nd - // shift will be 16, regardless of the block size. +// The constants (multiplier and shifts) for a given block size are obtained +// as follows: +// - Let sum_w_h = block width + block height. +// - Shift 'sum_w_h' right until we reach an odd number. Let the number of +// shifts for that block size be called 'shift1' (see the parameter in +// dc_predictor_rect() function), and let the odd number be 'd'. [d has only 2 +// possible values: d = 3 for a 1:2 rect block and d = 5 for a 1:4 rect +// block]. +// - Find multipliers for (i) dividing by 3, and (ii) dividing by 5, +// using the "Algorithm 1" in: +// http://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=1467632 +// by ensuring that m + n = 16 (in that algorithm). This ensures that our 2nd +// shift will be 16, regardless of the block size. - // Note: For low bitdepth, assembly code may be optimized by using smaller - // constants for smaller block sizes, where the range of the 'sum' is - // restricted to fewer bits. +// Note: For low bitdepth, assembly code may be optimized by using smaller +// constants for smaller block sizes, where the range of the 'sum' is +// restricted to fewer bits. #define DC_MULTIPLIER_1X2 0x5556 #define DC_MULTIPLIER_1X4 0x3334 diff --git a/third_party/aom/aom_dsp/noise_model.h b/third_party/aom/aom_dsp/noise_model.h index 049d5be151..5e7de9bf2c 100644 --- a/third_party/aom/aom_dsp/noise_model.h +++ b/third_party/aom/aom_dsp/noise_model.h @@ -158,10 +158,10 @@ int aom_flat_block_finder_run(const aom_flat_block_finder_t *block_finder, int stride, uint8_t *flat_blocks); // The noise shape indicates the allowed coefficients in the AR model. -typedef enum { +enum { AOM_NOISE_SHAPE_DIAMOND = 0, AOM_NOISE_SHAPE_SQUARE = 1 -} aom_noise_shape; +} UENUM1BYTE(aom_noise_shape); // The parameters of the noise model include the shape type, lag, the // bit depth of the input images provided, and whether the input images @@ -202,13 +202,13 @@ typedef struct { } aom_noise_model_t; /*!\brief Result of a noise model update. */ -typedef enum { +enum { AOM_NOISE_STATUS_OK = 0, AOM_NOISE_STATUS_INVALID_ARGUMENT, AOM_NOISE_STATUS_INSUFFICIENT_FLAT_BLOCKS, AOM_NOISE_STATUS_DIFFERENT_NOISE_TYPE, AOM_NOISE_STATUS_INTERNAL_ERROR, -} aom_noise_status_t; +} UENUM1BYTE(aom_noise_status_t); /*!\brief Initializes a noise model with the given parameters. * diff --git a/third_party/aom/aom_dsp/prob.h b/third_party/aom/aom_dsp/prob.h index d003a986e6..20ffdeaeaf 100644 --- a/third_party/aom/aom_dsp/prob.h +++ b/third_party/aom/aom_dsp/prob.h @@ -641,7 +641,7 @@ static INLINE uint8_t get_prob(unsigned int num, unsigned int den) { } } -static INLINE void update_cdf(aom_cdf_prob *cdf, int val, int nsymbs) { +static INLINE void update_cdf(aom_cdf_prob *cdf, int8_t val, int nsymbs) { int rate; int i, tmp; diff --git a/third_party/aom/aom_dsp/quantize.c b/third_party/aom/aom_dsp/quantize.c index 62dbd86a92..4acbf20939 100644 --- a/third_party/aom/aom_dsp/quantize.c +++ b/third_party/aom/aom_dsp/quantize.c @@ -12,6 +12,68 @@ #include "aom_dsp/quantize.h" #include "aom_mem/aom_mem.h" +void quantize_b_adaptive_helper_c( + const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, + const int16_t *round_ptr, const int16_t *quant_ptr, + const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, + tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, + const int16_t *scan, const int16_t *iscan, const qm_val_t *qm_ptr, + const qm_val_t *iqm_ptr, const int log_scale) { + const int zbins[2] = { ROUND_POWER_OF_TWO(zbin_ptr[0], log_scale), + ROUND_POWER_OF_TWO(zbin_ptr[1], log_scale) }; + const int nzbins[2] = { zbins[0] * -1, zbins[1] * -1 }; + int i, non_zero_count = (int)n_coeffs, eob = -1; + (void)iscan; + + memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); + memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); + + // Pre-scan pass + for (i = (int)n_coeffs - 1; i >= 0; i--) { + const int rc = scan[i]; + const qm_val_t wt = qm_ptr != NULL ? qm_ptr[rc] : (1 << AOM_QM_BITS); + const int coeff = coeff_ptr[rc] * wt; + + int prescan_add = ROUND_POWER_OF_TWO(dequant_ptr[rc != 0] * 325, 7); + if (coeff < (zbins[rc != 0] * (1 << AOM_QM_BITS) + prescan_add) && + coeff > (nzbins[rc != 0] * (1 << AOM_QM_BITS) - prescan_add)) + non_zero_count--; + else + break; + } + + // Quantization pass: All coefficients with index >= zero_flag are + // skippable. Note: zero_flag can be zero. + for (i = 0; i < non_zero_count; i++) { + const int rc = scan[i]; + const int coeff = coeff_ptr[rc]; + const int coeff_sign = (coeff >> 31); + const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign; + int tmp32; + + const qm_val_t wt = qm_ptr != NULL ? qm_ptr[rc] : (1 << AOM_QM_BITS); + if (abs_coeff * wt >= (zbins[rc != 0] << AOM_QM_BITS)) { + int64_t tmp = + clamp(abs_coeff + ROUND_POWER_OF_TWO(round_ptr[rc != 0], log_scale), + INT16_MIN, INT16_MAX); + tmp *= wt; + tmp32 = (int)(((((tmp * quant_ptr[rc != 0]) >> 16) + tmp) * + quant_shift_ptr[rc != 0]) >> + (16 - log_scale + AOM_QM_BITS)); // quantization + qcoeff_ptr[rc] = (tmp32 ^ coeff_sign) - coeff_sign; + const int iwt = iqm_ptr != NULL ? iqm_ptr[rc] : (1 << AOM_QM_BITS); + const int dequant = + (dequant_ptr[rc != 0] * iwt + (1 << (AOM_QM_BITS - 1))) >> + AOM_QM_BITS; + const tran_low_t abs_dqcoeff = (tmp32 * dequant) >> log_scale; + dqcoeff_ptr[rc] = (tran_low_t)((abs_dqcoeff ^ coeff_sign) - coeff_sign); + + if (tmp32) eob = i; + } + } + *eob_ptr = eob + 1; +} + void quantize_b_helper_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, @@ -74,6 +136,64 @@ void quantize_b_helper_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, *eob_ptr = eob + 1; } +void highbd_quantize_b_adaptive_helper_c( + const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, + const int16_t *round_ptr, const int16_t *quant_ptr, + const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, + tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, + const int16_t *scan, const int16_t *iscan, const qm_val_t *qm_ptr, + const qm_val_t *iqm_ptr, const int log_scale) { + int i, eob = -1; + const int zbins[2] = { ROUND_POWER_OF_TWO(zbin_ptr[0], log_scale), + ROUND_POWER_OF_TWO(zbin_ptr[1], log_scale) }; + const int nzbins[2] = { zbins[0] * -1, zbins[1] * -1 }; + int dequant; + int idx_arr[4096]; + (void)iscan; + int idx = 0; + + memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); + memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); + + // Pre-scan pass + for (i = 0; i < n_coeffs; i++) { + const int rc = scan[i]; + const qm_val_t wt = qm_ptr != NULL ? qm_ptr[rc] : (1 << AOM_QM_BITS); + const int coeff = coeff_ptr[rc] * wt; + + // If the coefficient is out of the base ZBIN range, keep it for + // quantization. + int prescan_add = ROUND_POWER_OF_TWO(dequant_ptr[rc != 0] * 325, 7); + if (coeff >= (zbins[rc != 0] * (1 << AOM_QM_BITS) + prescan_add) && + coeff <= (nzbins[rc != 0] * (1 << AOM_QM_BITS) - prescan_add)) + idx_arr[idx++] = i; + } + + // Quantization pass: only process the coefficients selected in + // pre-scan pass. Note: idx can be zero. + for (i = 0; i < idx; i++) { + const int rc = scan[idx_arr[i]]; + const int coeff = coeff_ptr[rc]; + const int coeff_sign = (coeff >> 31); + const qm_val_t wt = qm_ptr != NULL ? qm_ptr[rc] : (1 << AOM_QM_BITS); + const qm_val_t iwt = iqm_ptr != NULL ? iqm_ptr[rc] : (1 << AOM_QM_BITS); + const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign; + const int64_t tmp1 = + abs_coeff + ROUND_POWER_OF_TWO(round_ptr[rc != 0], log_scale); + const int64_t tmpw = tmp1 * wt; + const int64_t tmp2 = ((tmpw * quant_ptr[rc != 0]) >> 16) + tmpw; + const int abs_qcoeff = (int)((tmp2 * quant_shift_ptr[rc != 0]) >> + (16 - log_scale + AOM_QM_BITS)); + qcoeff_ptr[rc] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign); + dequant = + (dequant_ptr[rc != 0] * iwt + (1 << (AOM_QM_BITS - 1))) >> AOM_QM_BITS; + const tran_low_t abs_dqcoeff = (abs_qcoeff * dequant) >> log_scale; + dqcoeff_ptr[rc] = (tran_low_t)((abs_dqcoeff ^ coeff_sign) - coeff_sign); + if (abs_qcoeff) eob = idx_arr[i]; + } + *eob_ptr = eob + 1; +} + void highbd_quantize_b_helper_c( const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, @@ -133,6 +253,80 @@ void highbd_quantize_b_helper_c( /* These functions should only be called when quantisation matrices are not used. */ +void aom_quantize_b_adaptive_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, + const int16_t *zbin_ptr, + const int16_t *round_ptr, + const int16_t *quant_ptr, + const int16_t *quant_shift_ptr, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, + const int16_t *dequant_ptr, uint16_t *eob_ptr, + const int16_t *scan, const int16_t *iscan) { + quantize_b_adaptive_helper_c(coeff_ptr, n_coeffs, zbin_ptr, round_ptr, + quant_ptr, quant_shift_ptr, qcoeff_ptr, + dqcoeff_ptr, dequant_ptr, eob_ptr, scan, iscan, + NULL, NULL, 0); +} + +void aom_quantize_b_32x32_adaptive_c( + const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, + const int16_t *round_ptr, const int16_t *quant_ptr, + const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, + tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, + const int16_t *scan, const int16_t *iscan) { + quantize_b_adaptive_helper_c(coeff_ptr, n_coeffs, zbin_ptr, round_ptr, + quant_ptr, quant_shift_ptr, qcoeff_ptr, + dqcoeff_ptr, dequant_ptr, eob_ptr, scan, iscan, + NULL, NULL, 1); +} + +void aom_quantize_b_64x64_adaptive_c( + const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, + const int16_t *round_ptr, const int16_t *quant_ptr, + const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, + tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, + const int16_t *scan, const int16_t *iscan) { + quantize_b_adaptive_helper_c(coeff_ptr, n_coeffs, zbin_ptr, round_ptr, + quant_ptr, quant_shift_ptr, qcoeff_ptr, + dqcoeff_ptr, dequant_ptr, eob_ptr, scan, iscan, + NULL, NULL, 2); +} + +void aom_highbd_quantize_b_adaptive_c( + const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, + const int16_t *round_ptr, const int16_t *quant_ptr, + const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, + tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, + const int16_t *scan, const int16_t *iscan) { + highbd_quantize_b_adaptive_helper_c(coeff_ptr, n_coeffs, zbin_ptr, round_ptr, + quant_ptr, quant_shift_ptr, qcoeff_ptr, + dqcoeff_ptr, dequant_ptr, eob_ptr, scan, + iscan, NULL, NULL, 0); +} + +void aom_highbd_quantize_b_32x32_adaptive_c( + const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, + const int16_t *round_ptr, const int16_t *quant_ptr, + const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, + tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, + const int16_t *scan, const int16_t *iscan) { + highbd_quantize_b_adaptive_helper_c(coeff_ptr, n_coeffs, zbin_ptr, round_ptr, + quant_ptr, quant_shift_ptr, qcoeff_ptr, + dqcoeff_ptr, dequant_ptr, eob_ptr, scan, + iscan, NULL, NULL, 1); +} + +void aom_highbd_quantize_b_64x64_adaptive_c( + const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, + const int16_t *round_ptr, const int16_t *quant_ptr, + const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, + tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, + const int16_t *scan, const int16_t *iscan) { + highbd_quantize_b_adaptive_helper_c(coeff_ptr, n_coeffs, zbin_ptr, round_ptr, + quant_ptr, quant_shift_ptr, qcoeff_ptr, + dqcoeff_ptr, dequant_ptr, eob_ptr, scan, + iscan, NULL, NULL, 2); +} + void aom_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, diff --git a/third_party/aom/aom_dsp/quantize.h b/third_party/aom/aom_dsp/quantize.h index c55ab234e7..43c30ee090 100644 --- a/third_party/aom/aom_dsp/quantize.h +++ b/third_party/aom/aom_dsp/quantize.h @@ -20,6 +20,66 @@ extern "C" { #endif +void quantize_b_adaptive_helper_c( + const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, + const int16_t *round_ptr, const int16_t *quant_ptr, + const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, + tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, + const int16_t *scan, const int16_t *iscan, const qm_val_t *qm_ptr, + const qm_val_t *iqm_ptr, const int log_scale); + +void aom_quantize_b_adaptive_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, + const int16_t *zbin_ptr, + const int16_t *round_ptr, + const int16_t *quant_ptr, + const int16_t *quant_shift_ptr, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, + const int16_t *dequant_ptr, uint16_t *eob_ptr, + const int16_t *scan, const int16_t *iscan); + +void aom_quantize_b_32x32_adaptive_c( + const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, + const int16_t *round_ptr, const int16_t *quant_ptr, + const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, + tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, + const int16_t *scan, const int16_t *iscan); + +void aom_quantize_b_64x64_adaptive_c( + const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, + const int16_t *round_ptr, const int16_t *quant_ptr, + const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, + tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, + const int16_t *scan, const int16_t *iscan); + +void highbd_quantize_b_adaptive_helper_c( + const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, + const int16_t *round_ptr, const int16_t *quant_ptr, + const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, + tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, + const int16_t *scan, const int16_t *iscan, const qm_val_t *qm_ptr, + const qm_val_t *iqm_ptr, const int log_scale); + +void aom_highbd_quantize_b_adaptive_c( + const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, + const int16_t *round_ptr, const int16_t *quant_ptr, + const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, + tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, + const int16_t *scan, const int16_t *iscan); + +void aom_highbd_quantize_b_32x32_adaptive_c( + const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, + const int16_t *round_ptr, const int16_t *quant_ptr, + const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, + tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, + const int16_t *scan, const int16_t *iscan); + +void aom_highbd_quantize_b_64x64_adaptive_c( + const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, + const int16_t *round_ptr, const int16_t *quant_ptr, + const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, + tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, + const int16_t *scan, const int16_t *iscan); + void quantize_b_helper_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, diff --git a/third_party/aom/aom_dsp/recenter.h b/third_party/aom/aom_dsp/recenter.h new file mode 100644 index 0000000000..b3fd412907 --- /dev/null +++ b/third_party/aom/aom_dsp/recenter.h @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2018, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_AOM_DSP_RECENTER_H_ +#define AOM_AOM_DSP_RECENTER_H_ + +#include "config/aom_config.h" + +#include "aom/aom_integer.h" + +// Inverse recenters a non-negative literal v around a reference r +static INLINE uint16_t inv_recenter_nonneg(uint16_t r, uint16_t v) { + if (v > (r << 1)) + return v; + else if ((v & 1) == 0) + return (v >> 1) + r; + else + return r - ((v + 1) >> 1); +} + +// Inverse recenters a non-negative literal v in [0, n-1] around a +// reference r also in [0, n-1] +static INLINE uint16_t inv_recenter_finite_nonneg(uint16_t n, uint16_t r, + uint16_t v) { + if ((r << 1) <= n) { + return inv_recenter_nonneg(r, v); + } else { + return n - 1 - inv_recenter_nonneg(n - 1 - r, v); + } +} + +// Recenters a non-negative literal v around a reference r +static INLINE uint16_t recenter_nonneg(uint16_t r, uint16_t v) { + if (v > (r << 1)) + return v; + else if (v >= r) + return ((v - r) << 1); + else + return ((r - v) << 1) - 1; +} + +// Recenters a non-negative literal v in [0, n-1] around a +// reference r also in [0, n-1] +static INLINE uint16_t recenter_finite_nonneg(uint16_t n, uint16_t r, + uint16_t v) { + if ((r << 1) <= n) { + return recenter_nonneg(r, v); + } else { + return recenter_nonneg(n - 1 - r, n - 1 - v); + } +} + +#endif // AOM_AOM_DSP_RECENTER_H_ diff --git a/third_party/aom/aom_dsp/sad.c b/third_party/aom/aom_dsp/sad.c index 1e24df4a56..9169e78cd1 100644 --- a/third_party/aom/aom_dsp/sad.c +++ b/third_party/aom/aom_dsp/sad.c @@ -25,7 +25,9 @@ static INLINE unsigned int sad(const uint8_t *a, int a_stride, const uint8_t *b, unsigned int sad = 0; for (y = 0; y < height; y++) { - for (x = 0; x < width; x++) sad += abs(a[x] - b[x]); + for (x = 0; x < width; x++) { + sad += abs(a[x] - b[x]); + } a += a_stride; b += b_stride; @@ -52,12 +54,12 @@ static INLINE unsigned int sad(const uint8_t *a, int a_stride, const uint8_t *b, aom_comp_avg_pred(comp_pred, second_pred, m, n, ref, ref_stride); \ return sad(src, src_stride, comp_pred, m, m, n); \ } \ - unsigned int aom_jnt_sad##m##x##n##_avg_c( \ + unsigned int aom_dist_wtd_sad##m##x##n##_avg_c( \ const uint8_t *src, int src_stride, const uint8_t *ref, int ref_stride, \ - const uint8_t *second_pred, const JNT_COMP_PARAMS *jcp_param) { \ + const uint8_t *second_pred, const DIST_WTD_COMP_PARAMS *jcp_param) { \ uint8_t comp_pred[m * n]; \ - aom_jnt_comp_avg_pred_c(comp_pred, second_pred, m, n, ref, ref_stride, \ - jcp_param); \ + aom_dist_wtd_comp_avg_pred_c(comp_pred, second_pred, m, n, ref, \ + ref_stride, jcp_param); \ return sad(src, src_stride, comp_pred, m, m, n); \ } @@ -67,75 +69,75 @@ static INLINE unsigned int sad(const uint8_t *a, int a_stride, const uint8_t *b, const uint8_t *const ref_array[], \ int ref_stride, uint32_t *sad_array) { \ int i; \ - for (i = 0; i < 4; ++i) \ + for (i = 0; i < 4; ++i) { \ sad_array[i] = \ aom_sad##m##x##n##_c(src, src_stride, ref_array[i], ref_stride); \ + } \ } -/* clang-format off */ // 128x128 -sadMxN(128, 128) -sadMxNx4D(128, 128) +sadMxN(128, 128); +sadMxNx4D(128, 128); // 128x64 -sadMxN(128, 64) -sadMxNx4D(128, 64) +sadMxN(128, 64); +sadMxNx4D(128, 64); // 64x128 -sadMxN(64, 128) -sadMxNx4D(64, 128) +sadMxN(64, 128); +sadMxNx4D(64, 128); // 64x64 -sadMxN(64, 64) -sadMxNx4D(64, 64) +sadMxN(64, 64); +sadMxNx4D(64, 64); // 64x32 -sadMxN(64, 32) -sadMxNx4D(64, 32) +sadMxN(64, 32); +sadMxNx4D(64, 32); // 32x64 -sadMxN(32, 64) -sadMxNx4D(32, 64) +sadMxN(32, 64); +sadMxNx4D(32, 64); // 32x32 -sadMxN(32, 32) -sadMxNx4D(32, 32) +sadMxN(32, 32); +sadMxNx4D(32, 32); // 32x16 -sadMxN(32, 16) -sadMxNx4D(32, 16) +sadMxN(32, 16); +sadMxNx4D(32, 16); // 16x32 -sadMxN(16, 32) -sadMxNx4D(16, 32) +sadMxN(16, 32); +sadMxNx4D(16, 32); // 16x16 -sadMxN(16, 16) -sadMxNx4D(16, 16) +sadMxN(16, 16); +sadMxNx4D(16, 16); // 16x8 -sadMxN(16, 8) -sadMxNx4D(16, 8) +sadMxN(16, 8); +sadMxNx4D(16, 8); // 8x16 -sadMxN(8, 16) -sadMxNx4D(8, 16) +sadMxN(8, 16); +sadMxNx4D(8, 16); // 8x8 -sadMxN(8, 8) -sadMxNx4D(8, 8) +sadMxN(8, 8); +sadMxNx4D(8, 8); // 8x4 -sadMxN(8, 4) -sadMxNx4D(8, 4) +sadMxN(8, 4); +sadMxNx4D(8, 4); // 4x8 -sadMxN(4, 8) -sadMxNx4D(4, 8) +sadMxN(4, 8); +sadMxNx4D(4, 8); // 4x4 -sadMxN(4, 4) -sadMxNx4D(4, 4) +sadMxN(4, 4); +sadMxNx4D(4, 4); sadMxh(128); sadMxh(64); @@ -144,30 +146,30 @@ sadMxh(16); sadMxh(8); sadMxh(4); -sadMxN(4, 16) -sadMxNx4D(4, 16) -sadMxN(16, 4) -sadMxNx4D(16, 4) -sadMxN(8, 32) -sadMxNx4D(8, 32) -sadMxN(32, 8) -sadMxNx4D(32, 8) -sadMxN(16, 64) -sadMxNx4D(16, 64) -sadMxN(64, 16) -sadMxNx4D(64, 16) +sadMxN(4, 16); +sadMxNx4D(4, 16); +sadMxN(16, 4); +sadMxNx4D(16, 4); +sadMxN(8, 32); +sadMxNx4D(8, 32); +sadMxN(32, 8); +sadMxNx4D(32, 8); +sadMxN(16, 64); +sadMxNx4D(16, 64); +sadMxN(64, 16); +sadMxNx4D(64, 16); - /* clang-format on */ - - static INLINE - unsigned int highbd_sad(const uint8_t *a8, int a_stride, const uint8_t *b8, - int b_stride, int width, int height) { +static INLINE unsigned int highbd_sad(const uint8_t *a8, int a_stride, + const uint8_t *b8, int b_stride, + int width, int height) { int y, x; unsigned int sad = 0; const uint16_t *a = CONVERT_TO_SHORTPTR(a8); const uint16_t *b = CONVERT_TO_SHORTPTR(b8); for (y = 0; y < height; y++) { - for (x = 0; x < width; x++) sad += abs(a[x] - b[x]); + for (x = 0; x < width; x++) { + sad += abs(a[x] - b[x]); + } a += a_stride; b += b_stride; @@ -182,7 +184,9 @@ static INLINE unsigned int highbd_sadb(const uint8_t *a8, int a_stride, unsigned int sad = 0; const uint16_t *a = CONVERT_TO_SHORTPTR(a8); for (y = 0; y < height; y++) { - for (x = 0; x < width; x++) sad += abs(a[x] - b[x]); + for (x = 0; x < width; x++) { + sad += abs(a[x] - b[x]); + } a += a_stride; b += b_stride; @@ -204,12 +208,13 @@ static INLINE unsigned int highbd_sadb(const uint8_t *a8, int a_stride, ref, ref_stride); \ return highbd_sadb(src, src_stride, comp_pred, m, m, n); \ } \ - unsigned int aom_highbd_jnt_sad##m##x##n##_avg_c( \ + unsigned int aom_highbd_dist_wtd_sad##m##x##n##_avg_c( \ const uint8_t *src, int src_stride, const uint8_t *ref, int ref_stride, \ - const uint8_t *second_pred, const JNT_COMP_PARAMS *jcp_param) { \ + const uint8_t *second_pred, const DIST_WTD_COMP_PARAMS *jcp_param) { \ uint16_t comp_pred[m * n]; \ - aom_highbd_jnt_comp_avg_pred(CONVERT_TO_BYTEPTR(comp_pred), second_pred, \ - m, n, ref, ref_stride, jcp_param); \ + aom_highbd_dist_wtd_comp_avg_pred(CONVERT_TO_BYTEPTR(comp_pred), \ + second_pred, m, n, ref, ref_stride, \ + jcp_param); \ return highbd_sadb(src, src_stride, comp_pred, m, m, n); \ } @@ -224,81 +229,79 @@ static INLINE unsigned int highbd_sadb(const uint8_t *a8, int a_stride, } \ } -/* clang-format off */ // 128x128 -highbd_sadMxN(128, 128) -highbd_sadMxNx4D(128, 128) +highbd_sadMxN(128, 128); +highbd_sadMxNx4D(128, 128); // 128x64 -highbd_sadMxN(128, 64) -highbd_sadMxNx4D(128, 64) +highbd_sadMxN(128, 64); +highbd_sadMxNx4D(128, 64); // 64x128 -highbd_sadMxN(64, 128) -highbd_sadMxNx4D(64, 128) +highbd_sadMxN(64, 128); +highbd_sadMxNx4D(64, 128); // 64x64 -highbd_sadMxN(64, 64) -highbd_sadMxNx4D(64, 64) +highbd_sadMxN(64, 64); +highbd_sadMxNx4D(64, 64); // 64x32 -highbd_sadMxN(64, 32) -highbd_sadMxNx4D(64, 32) +highbd_sadMxN(64, 32); +highbd_sadMxNx4D(64, 32); // 32x64 -highbd_sadMxN(32, 64) -highbd_sadMxNx4D(32, 64) +highbd_sadMxN(32, 64); +highbd_sadMxNx4D(32, 64); // 32x32 -highbd_sadMxN(32, 32) -highbd_sadMxNx4D(32, 32) +highbd_sadMxN(32, 32); +highbd_sadMxNx4D(32, 32); // 32x16 -highbd_sadMxN(32, 16) -highbd_sadMxNx4D(32, 16) +highbd_sadMxN(32, 16); +highbd_sadMxNx4D(32, 16); // 16x32 -highbd_sadMxN(16, 32) -highbd_sadMxNx4D(16, 32) +highbd_sadMxN(16, 32); +highbd_sadMxNx4D(16, 32); // 16x16 -highbd_sadMxN(16, 16) -highbd_sadMxNx4D(16, 16) +highbd_sadMxN(16, 16); +highbd_sadMxNx4D(16, 16); // 16x8 -highbd_sadMxN(16, 8) -highbd_sadMxNx4D(16, 8) +highbd_sadMxN(16, 8); +highbd_sadMxNx4D(16, 8); // 8x16 -highbd_sadMxN(8, 16) -highbd_sadMxNx4D(8, 16) +highbd_sadMxN(8, 16); +highbd_sadMxNx4D(8, 16); // 8x8 -highbd_sadMxN(8, 8) -highbd_sadMxNx4D(8, 8) +highbd_sadMxN(8, 8); +highbd_sadMxNx4D(8, 8); // 8x4 -highbd_sadMxN(8, 4) -highbd_sadMxNx4D(8, 4) +highbd_sadMxN(8, 4); +highbd_sadMxNx4D(8, 4); // 4x8 -highbd_sadMxN(4, 8) -highbd_sadMxNx4D(4, 8) +highbd_sadMxN(4, 8); +highbd_sadMxNx4D(4, 8); // 4x4 -highbd_sadMxN(4, 4) -highbd_sadMxNx4D(4, 4) +highbd_sadMxN(4, 4); +highbd_sadMxNx4D(4, 4); -highbd_sadMxN(4, 16) -highbd_sadMxNx4D(4, 16) -highbd_sadMxN(16, 4) -highbd_sadMxNx4D(16, 4) -highbd_sadMxN(8, 32) -highbd_sadMxNx4D(8, 32) -highbd_sadMxN(32, 8) -highbd_sadMxNx4D(32, 8) -highbd_sadMxN(16, 64) -highbd_sadMxNx4D(16, 64) -highbd_sadMxN(64, 16) -highbd_sadMxNx4D(64, 16) - /* clang-format on */ +highbd_sadMxN(4, 16); +highbd_sadMxNx4D(4, 16); +highbd_sadMxN(16, 4); +highbd_sadMxNx4D(16, 4); +highbd_sadMxN(8, 32); +highbd_sadMxNx4D(8, 32); +highbd_sadMxN(32, 8); +highbd_sadMxNx4D(32, 8); +highbd_sadMxN(16, 64); +highbd_sadMxNx4D(16, 64); +highbd_sadMxN(64, 16); +highbd_sadMxNx4D(64, 16); diff --git a/third_party/aom/aom_dsp/txfm_common.h b/third_party/aom/aom_dsp/txfm_common.h index f98242840c..f13d69092d 100644 --- a/third_party/aom/aom_dsp/txfm_common.h +++ b/third_party/aom/aom_dsp/txfm_common.h @@ -39,7 +39,7 @@ typedef struct txfm_param { // Constants: // for (int i = 1; i< 32; ++i) // printf("static const int cospi_%d_64 = %.0f;\n", i, -// round(16384 * cos(i*M_PI/64))); +// round(16384 * cos(i*PI/64))); // Note: sin(k*Pi/64) = cos((32-k)*Pi/64) static const tran_high_t cospi_1_64 = 16364; static const tran_high_t cospi_2_64 = 16305; diff --git a/third_party/aom/aom_dsp/variance.c b/third_party/aom/aom_dsp/variance.c index 23b7153098..18a33c5467 100644 --- a/third_party/aom/aom_dsp/variance.c +++ b/third_party/aom/aom_dsp/variance.c @@ -164,40 +164,40 @@ void aom_var_filter_block2d_bil_second_pass_c(const uint16_t *a, uint8_t *b, return aom_variance##W##x##H##_c(temp2, W, b, b_stride, sse); \ } -#define SUBPIX_AVG_VAR(W, H) \ - uint32_t aom_sub_pixel_avg_variance##W##x##H##_c( \ - const uint8_t *a, int a_stride, int xoffset, int yoffset, \ - const uint8_t *b, int b_stride, uint32_t *sse, \ - const uint8_t *second_pred) { \ - uint16_t fdata3[(H + 1) * W]; \ - uint8_t temp2[H * W]; \ - DECLARE_ALIGNED(16, uint8_t, temp3[H * W]); \ - \ - aom_var_filter_block2d_bil_first_pass_c(a, fdata3, a_stride, 1, H + 1, W, \ - bilinear_filters_2t[xoffset]); \ - aom_var_filter_block2d_bil_second_pass_c(fdata3, temp2, W, W, H, W, \ - bilinear_filters_2t[yoffset]); \ - \ - aom_comp_avg_pred(temp3, second_pred, W, H, temp2, W); \ - \ - return aom_variance##W##x##H##_c(temp3, W, b, b_stride, sse); \ - } \ - uint32_t aom_jnt_sub_pixel_avg_variance##W##x##H##_c( \ - const uint8_t *a, int a_stride, int xoffset, int yoffset, \ - const uint8_t *b, int b_stride, uint32_t *sse, \ - const uint8_t *second_pred, const JNT_COMP_PARAMS *jcp_param) { \ - uint16_t fdata3[(H + 1) * W]; \ - uint8_t temp2[H * W]; \ - DECLARE_ALIGNED(16, uint8_t, temp3[H * W]); \ - \ - aom_var_filter_block2d_bil_first_pass_c(a, fdata3, a_stride, 1, H + 1, W, \ - bilinear_filters_2t[xoffset]); \ - aom_var_filter_block2d_bil_second_pass_c(fdata3, temp2, W, W, H, W, \ - bilinear_filters_2t[yoffset]); \ - \ - aom_jnt_comp_avg_pred(temp3, second_pred, W, H, temp2, W, jcp_param); \ - \ - return aom_variance##W##x##H(temp3, W, b, b_stride, sse); \ +#define SUBPIX_AVG_VAR(W, H) \ + uint32_t aom_sub_pixel_avg_variance##W##x##H##_c( \ + const uint8_t *a, int a_stride, int xoffset, int yoffset, \ + const uint8_t *b, int b_stride, uint32_t *sse, \ + const uint8_t *second_pred) { \ + uint16_t fdata3[(H + 1) * W]; \ + uint8_t temp2[H * W]; \ + DECLARE_ALIGNED(16, uint8_t, temp3[H * W]); \ + \ + aom_var_filter_block2d_bil_first_pass_c(a, fdata3, a_stride, 1, H + 1, W, \ + bilinear_filters_2t[xoffset]); \ + aom_var_filter_block2d_bil_second_pass_c(fdata3, temp2, W, W, H, W, \ + bilinear_filters_2t[yoffset]); \ + \ + aom_comp_avg_pred(temp3, second_pred, W, H, temp2, W); \ + \ + return aom_variance##W##x##H##_c(temp3, W, b, b_stride, sse); \ + } \ + uint32_t aom_dist_wtd_sub_pixel_avg_variance##W##x##H##_c( \ + const uint8_t *a, int a_stride, int xoffset, int yoffset, \ + const uint8_t *b, int b_stride, uint32_t *sse, \ + const uint8_t *second_pred, const DIST_WTD_COMP_PARAMS *jcp_param) { \ + uint16_t fdata3[(H + 1) * W]; \ + uint8_t temp2[H * W]; \ + DECLARE_ALIGNED(16, uint8_t, temp3[H * W]); \ + \ + aom_var_filter_block2d_bil_first_pass_c(a, fdata3, a_stride, 1, H + 1, W, \ + bilinear_filters_2t[xoffset]); \ + aom_var_filter_block2d_bil_second_pass_c(fdata3, temp2, W, W, H, W, \ + bilinear_filters_2t[yoffset]); \ + \ + aom_dist_wtd_comp_avg_pred(temp3, second_pred, W, H, temp2, W, jcp_param); \ + \ + return aom_variance##W##x##H(temp3, W, b, b_stride, sse); \ } /* Identical to the variance call except it takes an additional parameter, sum, @@ -291,7 +291,7 @@ void aom_upsampled_pred_c(MACROBLOCKD *xd, const AV1_COMMON *const cm, const int ref_num = 0; const int is_intrabc = is_intrabc_block(mi); const struct scale_factors *const sf = - is_intrabc ? &cm->sf_identity : &xd->block_refs[ref_num]->sf; + is_intrabc ? &cm->sf_identity : xd->block_ref_scale_factors[ref_num]; const int is_scaled = av1_is_scaled(sf); if (is_scaled) { @@ -368,10 +368,7 @@ void aom_upsampled_pred_c(MACROBLOCKD *xd, const AV1_COMMON *const cm, } } - const InterpFilterParams *filter = - (subpel_search == 1) - ? av1_get_4tap_interp_filter_params(EIGHTTAP_REGULAR) - : av1_get_interp_filter_params_with_block_size(EIGHTTAP_REGULAR, 8); + const InterpFilterParams *filter = av1_get_filter(subpel_search); if (!subpel_x_q3 && !subpel_y_q3) { for (int i = 0; i < height; i++) { @@ -427,9 +424,10 @@ void aom_comp_avg_upsampled_pred_c(MACROBLOCKD *xd, const AV1_COMMON *const cm, } } -void aom_jnt_comp_avg_pred_c(uint8_t *comp_pred, const uint8_t *pred, int width, - int height, const uint8_t *ref, int ref_stride, - const JNT_COMP_PARAMS *jcp_param) { +void aom_dist_wtd_comp_avg_pred_c(uint8_t *comp_pred, const uint8_t *pred, + int width, int height, const uint8_t *ref, + int ref_stride, + const DIST_WTD_COMP_PARAMS *jcp_param) { int i, j; const int fwd_offset = jcp_param->fwd_offset; const int bck_offset = jcp_param->bck_offset; @@ -446,11 +444,11 @@ void aom_jnt_comp_avg_pred_c(uint8_t *comp_pred, const uint8_t *pred, int width, } } -void aom_jnt_comp_avg_upsampled_pred_c( +void aom_dist_wtd_comp_avg_upsampled_pred_c( MACROBLOCKD *xd, const AV1_COMMON *const cm, int mi_row, int mi_col, const MV *const mv, uint8_t *comp_pred, const uint8_t *pred, int width, int height, int subpel_x_q3, int subpel_y_q3, const uint8_t *ref, - int ref_stride, const JNT_COMP_PARAMS *jcp_param, int subpel_search) { + int ref_stride, const DIST_WTD_COMP_PARAMS *jcp_param, int subpel_search) { int i, j; const int fwd_offset = jcp_param->fwd_offset; const int bck_offset = jcp_param->bck_offset; @@ -691,125 +689,128 @@ void aom_highbd_var_filter_block2d_bil_second_pass( dst, dst_stride, sse); \ } -#define HIGHBD_SUBPIX_AVG_VAR(W, H) \ - uint32_t aom_highbd_8_sub_pixel_avg_variance##W##x##H##_c( \ - const uint8_t *src, int src_stride, int xoffset, int yoffset, \ - const uint8_t *dst, int dst_stride, uint32_t *sse, \ - const uint8_t *second_pred) { \ - uint16_t fdata3[(H + 1) * W]; \ - uint16_t temp2[H * W]; \ - DECLARE_ALIGNED(16, uint16_t, temp3[H * W]); \ - \ - aom_highbd_var_filter_block2d_bil_first_pass( \ - src, fdata3, src_stride, 1, H + 1, W, bilinear_filters_2t[xoffset]); \ - aom_highbd_var_filter_block2d_bil_second_pass( \ - fdata3, temp2, W, W, H, W, bilinear_filters_2t[yoffset]); \ - \ - aom_highbd_comp_avg_pred_c(CONVERT_TO_BYTEPTR(temp3), second_pred, W, H, \ - CONVERT_TO_BYTEPTR(temp2), W); \ - \ - return aom_highbd_8_variance##W##x##H##_c(CONVERT_TO_BYTEPTR(temp3), W, \ - dst, dst_stride, sse); \ - } \ - \ - uint32_t aom_highbd_10_sub_pixel_avg_variance##W##x##H##_c( \ - const uint8_t *src, int src_stride, int xoffset, int yoffset, \ - const uint8_t *dst, int dst_stride, uint32_t *sse, \ - const uint8_t *second_pred) { \ - uint16_t fdata3[(H + 1) * W]; \ - uint16_t temp2[H * W]; \ - DECLARE_ALIGNED(16, uint16_t, temp3[H * W]); \ - \ - aom_highbd_var_filter_block2d_bil_first_pass( \ - src, fdata3, src_stride, 1, H + 1, W, bilinear_filters_2t[xoffset]); \ - aom_highbd_var_filter_block2d_bil_second_pass( \ - fdata3, temp2, W, W, H, W, bilinear_filters_2t[yoffset]); \ - \ - aom_highbd_comp_avg_pred_c(CONVERT_TO_BYTEPTR(temp3), second_pred, W, H, \ - CONVERT_TO_BYTEPTR(temp2), W); \ - \ - return aom_highbd_10_variance##W##x##H##_c(CONVERT_TO_BYTEPTR(temp3), W, \ - dst, dst_stride, sse); \ - } \ - \ - uint32_t aom_highbd_12_sub_pixel_avg_variance##W##x##H##_c( \ - const uint8_t *src, int src_stride, int xoffset, int yoffset, \ - const uint8_t *dst, int dst_stride, uint32_t *sse, \ - const uint8_t *second_pred) { \ - uint16_t fdata3[(H + 1) * W]; \ - uint16_t temp2[H * W]; \ - DECLARE_ALIGNED(16, uint16_t, temp3[H * W]); \ - \ - aom_highbd_var_filter_block2d_bil_first_pass( \ - src, fdata3, src_stride, 1, H + 1, W, bilinear_filters_2t[xoffset]); \ - aom_highbd_var_filter_block2d_bil_second_pass( \ - fdata3, temp2, W, W, H, W, bilinear_filters_2t[yoffset]); \ - \ - aom_highbd_comp_avg_pred_c(CONVERT_TO_BYTEPTR(temp3), second_pred, W, H, \ - CONVERT_TO_BYTEPTR(temp2), W); \ - \ - return aom_highbd_12_variance##W##x##H##_c(CONVERT_TO_BYTEPTR(temp3), W, \ - dst, dst_stride, sse); \ - } \ - \ - uint32_t aom_highbd_8_jnt_sub_pixel_avg_variance##W##x##H##_c( \ - const uint8_t *src, int src_stride, int xoffset, int yoffset, \ - const uint8_t *dst, int dst_stride, uint32_t *sse, \ - const uint8_t *second_pred, const JNT_COMP_PARAMS *jcp_param) { \ - uint16_t fdata3[(H + 1) * W]; \ - uint16_t temp2[H * W]; \ - DECLARE_ALIGNED(16, uint16_t, temp3[H * W]); \ - \ - aom_highbd_var_filter_block2d_bil_first_pass( \ - src, fdata3, src_stride, 1, H + 1, W, bilinear_filters_2t[xoffset]); \ - aom_highbd_var_filter_block2d_bil_second_pass( \ - fdata3, temp2, W, W, H, W, bilinear_filters_2t[yoffset]); \ - \ - aom_highbd_jnt_comp_avg_pred(CONVERT_TO_BYTEPTR(temp3), second_pred, W, H, \ - CONVERT_TO_BYTEPTR(temp2), W, jcp_param); \ - \ - return aom_highbd_8_variance##W##x##H(CONVERT_TO_BYTEPTR(temp3), W, dst, \ - dst_stride, sse); \ - } \ - \ - uint32_t aom_highbd_10_jnt_sub_pixel_avg_variance##W##x##H##_c( \ - const uint8_t *src, int src_stride, int xoffset, int yoffset, \ - const uint8_t *dst, int dst_stride, uint32_t *sse, \ - const uint8_t *second_pred, const JNT_COMP_PARAMS *jcp_param) { \ - uint16_t fdata3[(H + 1) * W]; \ - uint16_t temp2[H * W]; \ - DECLARE_ALIGNED(16, uint16_t, temp3[H * W]); \ - \ - aom_highbd_var_filter_block2d_bil_first_pass( \ - src, fdata3, src_stride, 1, H + 1, W, bilinear_filters_2t[xoffset]); \ - aom_highbd_var_filter_block2d_bil_second_pass( \ - fdata3, temp2, W, W, H, W, bilinear_filters_2t[yoffset]); \ - \ - aom_highbd_jnt_comp_avg_pred(CONVERT_TO_BYTEPTR(temp3), second_pred, W, H, \ - CONVERT_TO_BYTEPTR(temp2), W, jcp_param); \ - \ - return aom_highbd_10_variance##W##x##H(CONVERT_TO_BYTEPTR(temp3), W, dst, \ - dst_stride, sse); \ - } \ - \ - uint32_t aom_highbd_12_jnt_sub_pixel_avg_variance##W##x##H##_c( \ - const uint8_t *src, int src_stride, int xoffset, int yoffset, \ - const uint8_t *dst, int dst_stride, uint32_t *sse, \ - const uint8_t *second_pred, const JNT_COMP_PARAMS *jcp_param) { \ - uint16_t fdata3[(H + 1) * W]; \ - uint16_t temp2[H * W]; \ - DECLARE_ALIGNED(16, uint16_t, temp3[H * W]); \ - \ - aom_highbd_var_filter_block2d_bil_first_pass( \ - src, fdata3, src_stride, 1, H + 1, W, bilinear_filters_2t[xoffset]); \ - aom_highbd_var_filter_block2d_bil_second_pass( \ - fdata3, temp2, W, W, H, W, bilinear_filters_2t[yoffset]); \ - \ - aom_highbd_jnt_comp_avg_pred(CONVERT_TO_BYTEPTR(temp3), second_pred, W, H, \ - CONVERT_TO_BYTEPTR(temp2), W, jcp_param); \ - \ - return aom_highbd_12_variance##W##x##H(CONVERT_TO_BYTEPTR(temp3), W, dst, \ - dst_stride, sse); \ +#define HIGHBD_SUBPIX_AVG_VAR(W, H) \ + uint32_t aom_highbd_8_sub_pixel_avg_variance##W##x##H##_c( \ + const uint8_t *src, int src_stride, int xoffset, int yoffset, \ + const uint8_t *dst, int dst_stride, uint32_t *sse, \ + const uint8_t *second_pred) { \ + uint16_t fdata3[(H + 1) * W]; \ + uint16_t temp2[H * W]; \ + DECLARE_ALIGNED(16, uint16_t, temp3[H * W]); \ + \ + aom_highbd_var_filter_block2d_bil_first_pass( \ + src, fdata3, src_stride, 1, H + 1, W, bilinear_filters_2t[xoffset]); \ + aom_highbd_var_filter_block2d_bil_second_pass( \ + fdata3, temp2, W, W, H, W, bilinear_filters_2t[yoffset]); \ + \ + aom_highbd_comp_avg_pred_c(CONVERT_TO_BYTEPTR(temp3), second_pred, W, H, \ + CONVERT_TO_BYTEPTR(temp2), W); \ + \ + return aom_highbd_8_variance##W##x##H##_c(CONVERT_TO_BYTEPTR(temp3), W, \ + dst, dst_stride, sse); \ + } \ + \ + uint32_t aom_highbd_10_sub_pixel_avg_variance##W##x##H##_c( \ + const uint8_t *src, int src_stride, int xoffset, int yoffset, \ + const uint8_t *dst, int dst_stride, uint32_t *sse, \ + const uint8_t *second_pred) { \ + uint16_t fdata3[(H + 1) * W]; \ + uint16_t temp2[H * W]; \ + DECLARE_ALIGNED(16, uint16_t, temp3[H * W]); \ + \ + aom_highbd_var_filter_block2d_bil_first_pass( \ + src, fdata3, src_stride, 1, H + 1, W, bilinear_filters_2t[xoffset]); \ + aom_highbd_var_filter_block2d_bil_second_pass( \ + fdata3, temp2, W, W, H, W, bilinear_filters_2t[yoffset]); \ + \ + aom_highbd_comp_avg_pred_c(CONVERT_TO_BYTEPTR(temp3), second_pred, W, H, \ + CONVERT_TO_BYTEPTR(temp2), W); \ + \ + return aom_highbd_10_variance##W##x##H##_c(CONVERT_TO_BYTEPTR(temp3), W, \ + dst, dst_stride, sse); \ + } \ + \ + uint32_t aom_highbd_12_sub_pixel_avg_variance##W##x##H##_c( \ + const uint8_t *src, int src_stride, int xoffset, int yoffset, \ + const uint8_t *dst, int dst_stride, uint32_t *sse, \ + const uint8_t *second_pred) { \ + uint16_t fdata3[(H + 1) * W]; \ + uint16_t temp2[H * W]; \ + DECLARE_ALIGNED(16, uint16_t, temp3[H * W]); \ + \ + aom_highbd_var_filter_block2d_bil_first_pass( \ + src, fdata3, src_stride, 1, H + 1, W, bilinear_filters_2t[xoffset]); \ + aom_highbd_var_filter_block2d_bil_second_pass( \ + fdata3, temp2, W, W, H, W, bilinear_filters_2t[yoffset]); \ + \ + aom_highbd_comp_avg_pred_c(CONVERT_TO_BYTEPTR(temp3), second_pred, W, H, \ + CONVERT_TO_BYTEPTR(temp2), W); \ + \ + return aom_highbd_12_variance##W##x##H##_c(CONVERT_TO_BYTEPTR(temp3), W, \ + dst, dst_stride, sse); \ + } \ + \ + uint32_t aom_highbd_8_dist_wtd_sub_pixel_avg_variance##W##x##H##_c( \ + const uint8_t *src, int src_stride, int xoffset, int yoffset, \ + const uint8_t *dst, int dst_stride, uint32_t *sse, \ + const uint8_t *second_pred, const DIST_WTD_COMP_PARAMS *jcp_param) { \ + uint16_t fdata3[(H + 1) * W]; \ + uint16_t temp2[H * W]; \ + DECLARE_ALIGNED(16, uint16_t, temp3[H * W]); \ + \ + aom_highbd_var_filter_block2d_bil_first_pass( \ + src, fdata3, src_stride, 1, H + 1, W, bilinear_filters_2t[xoffset]); \ + aom_highbd_var_filter_block2d_bil_second_pass( \ + fdata3, temp2, W, W, H, W, bilinear_filters_2t[yoffset]); \ + \ + aom_highbd_dist_wtd_comp_avg_pred(CONVERT_TO_BYTEPTR(temp3), second_pred, \ + W, H, CONVERT_TO_BYTEPTR(temp2), W, \ + jcp_param); \ + \ + return aom_highbd_8_variance##W##x##H(CONVERT_TO_BYTEPTR(temp3), W, dst, \ + dst_stride, sse); \ + } \ + \ + uint32_t aom_highbd_10_dist_wtd_sub_pixel_avg_variance##W##x##H##_c( \ + const uint8_t *src, int src_stride, int xoffset, int yoffset, \ + const uint8_t *dst, int dst_stride, uint32_t *sse, \ + const uint8_t *second_pred, const DIST_WTD_COMP_PARAMS *jcp_param) { \ + uint16_t fdata3[(H + 1) * W]; \ + uint16_t temp2[H * W]; \ + DECLARE_ALIGNED(16, uint16_t, temp3[H * W]); \ + \ + aom_highbd_var_filter_block2d_bil_first_pass( \ + src, fdata3, src_stride, 1, H + 1, W, bilinear_filters_2t[xoffset]); \ + aom_highbd_var_filter_block2d_bil_second_pass( \ + fdata3, temp2, W, W, H, W, bilinear_filters_2t[yoffset]); \ + \ + aom_highbd_dist_wtd_comp_avg_pred(CONVERT_TO_BYTEPTR(temp3), second_pred, \ + W, H, CONVERT_TO_BYTEPTR(temp2), W, \ + jcp_param); \ + \ + return aom_highbd_10_variance##W##x##H(CONVERT_TO_BYTEPTR(temp3), W, dst, \ + dst_stride, sse); \ + } \ + \ + uint32_t aom_highbd_12_dist_wtd_sub_pixel_avg_variance##W##x##H##_c( \ + const uint8_t *src, int src_stride, int xoffset, int yoffset, \ + const uint8_t *dst, int dst_stride, uint32_t *sse, \ + const uint8_t *second_pred, const DIST_WTD_COMP_PARAMS *jcp_param) { \ + uint16_t fdata3[(H + 1) * W]; \ + uint16_t temp2[H * W]; \ + DECLARE_ALIGNED(16, uint16_t, temp3[H * W]); \ + \ + aom_highbd_var_filter_block2d_bil_first_pass( \ + src, fdata3, src_stride, 1, H + 1, W, bilinear_filters_2t[xoffset]); \ + aom_highbd_var_filter_block2d_bil_second_pass( \ + fdata3, temp2, W, W, H, W, bilinear_filters_2t[yoffset]); \ + \ + aom_highbd_dist_wtd_comp_avg_pred(CONVERT_TO_BYTEPTR(temp3), second_pred, \ + W, H, CONVERT_TO_BYTEPTR(temp2), W, \ + jcp_param); \ + \ + return aom_highbd_12_variance##W##x##H(CONVERT_TO_BYTEPTR(temp3), W, dst, \ + dst_stride, sse); \ } /* All three forms of the variance are available in the same sizes. */ @@ -883,7 +884,7 @@ void aom_highbd_upsampled_pred_c(MACROBLOCKD *xd, const int ref_num = 0; const int is_intrabc = is_intrabc_block(mi); const struct scale_factors *const sf = - is_intrabc ? &cm->sf_identity : &xd->block_refs[ref_num]->sf; + is_intrabc ? &cm->sf_identity : xd->block_ref_scale_factors[ref_num]; const int is_scaled = av1_is_scaled(sf); if (is_scaled) { @@ -959,10 +960,7 @@ void aom_highbd_upsampled_pred_c(MACROBLOCKD *xd, } } - const InterpFilterParams *filter = - (subpel_search == 1) - ? av1_get_4tap_interp_filter_params(EIGHTTAP_REGULAR) - : av1_get_interp_filter_params_with_block_size(EIGHTTAP_REGULAR, 8); + const InterpFilterParams *filter = av1_get_filter(subpel_search); if (!subpel_x_q3 && !subpel_y_q3) { const uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); @@ -975,13 +973,13 @@ void aom_highbd_upsampled_pred_c(MACROBLOCKD *xd, } else if (!subpel_y_q3) { const int16_t *const kernel = av1_get_interp_filter_subpel_kernel(filter, subpel_x_q3 << 1); - aom_highbd_convolve8_horiz(ref8, ref_stride, comp_pred8, width, kernel, 16, - NULL, -1, width, height, bd); + aom_highbd_convolve8_horiz_c(ref8, ref_stride, comp_pred8, width, kernel, + 16, NULL, -1, width, height, bd); } else if (!subpel_x_q3) { const int16_t *const kernel = av1_get_interp_filter_subpel_kernel(filter, subpel_y_q3 << 1); - aom_highbd_convolve8_vert(ref8, ref_stride, comp_pred8, width, NULL, -1, - kernel, 16, width, height, bd); + aom_highbd_convolve8_vert_c(ref8, ref_stride, comp_pred8, width, NULL, -1, + kernel, 16, width, height, bd); } else { DECLARE_ALIGNED(16, uint16_t, temp[((MAX_SB_SIZE + 16) + 16) * MAX_SB_SIZE]); @@ -992,11 +990,11 @@ void aom_highbd_upsampled_pred_c(MACROBLOCKD *xd, const int intermediate_height = (((height - 1) * 8 + subpel_y_q3) >> 3) + filter->taps; assert(intermediate_height <= (MAX_SB_SIZE * 2 + 16) + 16); - aom_highbd_convolve8_horiz(ref8 - ref_stride * ((filter->taps >> 1) - 1), - ref_stride, CONVERT_TO_BYTEPTR(temp), - MAX_SB_SIZE, kernel_x, 16, NULL, -1, width, - intermediate_height, bd); - aom_highbd_convolve8_vert( + aom_highbd_convolve8_horiz_c(ref8 - ref_stride * ((filter->taps >> 1) - 1), + ref_stride, CONVERT_TO_BYTEPTR(temp), + MAX_SB_SIZE, kernel_x, 16, NULL, -1, width, + intermediate_height, bd); + aom_highbd_convolve8_vert_c( CONVERT_TO_BYTEPTR(temp + MAX_SB_SIZE * ((filter->taps >> 1) - 1)), MAX_SB_SIZE, comp_pred8, width, NULL, -1, kernel_y, 16, width, height, bd); @@ -1024,10 +1022,10 @@ void aom_highbd_comp_avg_upsampled_pred_c( } } -void aom_highbd_jnt_comp_avg_pred_c(uint8_t *comp_pred8, const uint8_t *pred8, - int width, int height, const uint8_t *ref8, - int ref_stride, - const JNT_COMP_PARAMS *jcp_param) { +void aom_highbd_dist_wtd_comp_avg_pred_c( + uint8_t *comp_pred8, const uint8_t *pred8, int width, int height, + const uint8_t *ref8, int ref_stride, + const DIST_WTD_COMP_PARAMS *jcp_param) { int i, j; const int fwd_offset = jcp_param->fwd_offset; const int bck_offset = jcp_param->bck_offset; @@ -1047,20 +1045,20 @@ void aom_highbd_jnt_comp_avg_pred_c(uint8_t *comp_pred8, const uint8_t *pred8, } } -void aom_highbd_jnt_comp_avg_upsampled_pred_c( +void aom_highbd_dist_wtd_comp_avg_upsampled_pred_c( MACROBLOCKD *xd, const struct AV1Common *const cm, int mi_row, int mi_col, const MV *const mv, uint8_t *comp_pred8, const uint8_t *pred8, int width, int height, int subpel_x_q3, int subpel_y_q3, const uint8_t *ref8, - int ref_stride, int bd, const JNT_COMP_PARAMS *jcp_param, + int ref_stride, int bd, const DIST_WTD_COMP_PARAMS *jcp_param, int subpel_search) { int i, j; const int fwd_offset = jcp_param->fwd_offset; const int bck_offset = jcp_param->bck_offset; const uint16_t *pred = CONVERT_TO_SHORTPTR(pred8); uint16_t *comp_pred = CONVERT_TO_SHORTPTR(comp_pred8); - aom_highbd_upsampled_pred(xd, cm, mi_row, mi_col, mv, comp_pred8, width, - height, subpel_x_q3, subpel_y_q3, ref8, ref_stride, - bd, subpel_search); + aom_highbd_upsampled_pred_c(xd, cm, mi_row, mi_col, mv, comp_pred8, width, + height, subpel_x_q3, subpel_y_q3, ref8, + ref_stride, bd, subpel_search); for (i = 0; i < height; i++) { for (j = 0; j < width; j++) { diff --git a/third_party/aom/aom_dsp/variance.h b/third_party/aom/aom_dsp/variance.h index 362da29d33..4550c17b34 100644 --- a/third_party/aom/aom_dsp/variance.h +++ b/third_party/aom/aom_dsp/variance.h @@ -50,15 +50,14 @@ typedef unsigned int (*aom_subp_avg_variance_fn_t)( const uint8_t *a, int a_stride, int xoffset, int yoffset, const uint8_t *b, int b_stride, unsigned int *sse, const uint8_t *second_pred); -typedef unsigned int (*aom_jnt_sad_avg_fn_t)(const uint8_t *a, int a_stride, - const uint8_t *b, int b_stride, - const uint8_t *second_pred, - const JNT_COMP_PARAMS *jcp_param); +typedef unsigned int (*aom_dist_wtd_sad_avg_fn_t)( + const uint8_t *a, int a_stride, const uint8_t *b, int b_stride, + const uint8_t *second_pred, const DIST_WTD_COMP_PARAMS *jcp_param); -typedef unsigned int (*aom_jnt_subp_avg_variance_fn_t)( +typedef unsigned int (*aom_dist_wtd_subp_avg_variance_fn_t)( const uint8_t *a, int a_stride, int xoffset, int yoffset, const uint8_t *b, int b_stride, unsigned int *sse, const uint8_t *second_pred, - const JNT_COMP_PARAMS *jcp_param); + const DIST_WTD_COMP_PARAMS *jcp_param); typedef unsigned int (*aom_masked_sad_fn_t)(const uint8_t *src, int src_stride, const uint8_t *ref, int ref_stride, @@ -101,8 +100,8 @@ typedef struct aom_variance_vtable { aom_obmc_sad_fn_t osdf; aom_obmc_variance_fn_t ovf; aom_obmc_subpixvariance_fn_t osvf; - aom_jnt_sad_avg_fn_t jsdaf; - aom_jnt_subp_avg_variance_fn_t jsvaf; + aom_dist_wtd_sad_avg_fn_t jsdaf; + aom_dist_wtd_subp_avg_variance_fn_t jsvaf; } aom_variance_fn_ptr_t; void aom_highbd_var_filter_block2d_bil_first_pass( diff --git a/third_party/aom/aom_dsp/x86/aom_asm_stubs.c b/third_party/aom/aom_dsp/x86/aom_asm_stubs.c index 5f5bf5f14e..3bf7b5597a 100644 --- a/third_party/aom/aom_dsp/x86/aom_asm_stubs.c +++ b/third_party/aom/aom_dsp/x86/aom_asm_stubs.c @@ -21,13 +21,13 @@ filter8_1dfunction aom_filter_block1d8_v8_sse2; filter8_1dfunction aom_filter_block1d8_h8_sse2; filter8_1dfunction aom_filter_block1d4_v8_sse2; filter8_1dfunction aom_filter_block1d4_h8_sse2; +filter8_1dfunction aom_filter_block1d16_v4_sse2; +filter8_1dfunction aom_filter_block1d16_h4_sse2; -#define aom_filter_block1d16_h4_sse2 aom_filter_block1d16_h8_sse2 -#define aom_filter_block1d16_v4_sse2 aom_filter_block1d16_v8_sse2 -#define aom_filter_block1d8_h4_sse2 aom_filter_block1d8_h8_sse2 -#define aom_filter_block1d8_v4_sse2 aom_filter_block1d8_v8_sse2 -#define aom_filter_block1d4_h4_sse2 aom_filter_block1d4_h8_sse2 -#define aom_filter_block1d4_v4_sse2 aom_filter_block1d4_v8_sse2 +filter8_1dfunction aom_filter_block1d8_h4_sse2; +filter8_1dfunction aom_filter_block1d8_v4_sse2; +filter8_1dfunction aom_filter_block1d4_h4_sse2; +filter8_1dfunction aom_filter_block1d4_v4_sse2; filter8_1dfunction aom_filter_block1d16_v2_sse2; filter8_1dfunction aom_filter_block1d16_h2_sse2; @@ -49,7 +49,6 @@ filter8_1dfunction aom_filter_block1d4_h2_sse2; FUN_CONV_1D(horiz, x_step_q4, filter_x, h, src, , sse2); FUN_CONV_1D(vert, y_step_q4, filter_y, v, src - src_stride * 3, , sse2); -#if ARCH_X86_64 highbd_filter8_1dfunction aom_highbd_filter_block1d16_v8_sse2; highbd_filter8_1dfunction aom_highbd_filter_block1d16_h8_sse2; highbd_filter8_1dfunction aom_highbd_filter_block1d8_v8_sse2; @@ -57,6 +56,13 @@ highbd_filter8_1dfunction aom_highbd_filter_block1d8_h8_sse2; highbd_filter8_1dfunction aom_highbd_filter_block1d4_v8_sse2; highbd_filter8_1dfunction aom_highbd_filter_block1d4_h8_sse2; +highbd_filter8_1dfunction aom_highbd_filter_block1d16_v4_sse2; +highbd_filter8_1dfunction aom_highbd_filter_block1d16_h4_sse2; +highbd_filter8_1dfunction aom_highbd_filter_block1d8_v4_sse2; +highbd_filter8_1dfunction aom_highbd_filter_block1d8_h4_sse2; +highbd_filter8_1dfunction aom_highbd_filter_block1d4_v4_sse2; +highbd_filter8_1dfunction aom_highbd_filter_block1d4_h4_sse2; + highbd_filter8_1dfunction aom_highbd_filter_block1d16_v2_sse2; highbd_filter8_1dfunction aom_highbd_filter_block1d16_h2_sse2; highbd_filter8_1dfunction aom_highbd_filter_block1d8_v2_sse2; @@ -85,5 +91,4 @@ highbd_filter8_1dfunction aom_highbd_filter_block1d4_h2_sse2; HIGH_FUN_CONV_1D(horiz, x_step_q4, filter_x, h, src, , sse2); HIGH_FUN_CONV_1D(vert, y_step_q4, filter_y, v, src - src_stride * 3, , sse2); -#endif // ARCH_X86_64 #endif // HAVE_SSE2 diff --git a/third_party/aom/aom_dsp/x86/aom_high_subpixel_bilinear_sse2.asm b/third_party/aom/aom_dsp/x86/aom_high_subpixel_bilinear_sse2.asm index 7b3fe6419a..a7152be57c 100644 --- a/third_party/aom/aom_dsp/x86/aom_high_subpixel_bilinear_sse2.asm +++ b/third_party/aom/aom_dsp/x86/aom_high_subpixel_bilinear_sse2.asm @@ -67,7 +67,6 @@ dec rcx %endm -%if ARCH_X86_64 %macro HIGH_GET_PARAM 0 mov rdx, arg(5) ;filter ptr mov rsi, arg(0) ;src_ptr @@ -86,14 +85,17 @@ mov rdx, 0x00010001 movsxd rcx, DWORD PTR arg(6) ;bps - movq xmm8, rdx + movq xmm3, rdx movq xmm5, rcx - pshufd xmm8, xmm8, 0b - movdqa xmm1, xmm8 - psllw xmm8, xmm5 - psubw xmm8, xmm1 ;max value (for clamping) + pshufd xmm3, xmm3, 0b + movdqa xmm1, xmm3 + psllw xmm3, xmm5 + psubw xmm3, xmm1 ;max value (for clamping) pxor xmm5, xmm5 ;min value (for clamping) + movdqa max, xmm3 + movdqa min, xmm5 + movsxd rax, DWORD PTR arg(1) ;pixels_per_line movsxd rdx, DWORD PTR arg(3) ;out_pitch movsxd rcx, DWORD PTR arg(4) ;output_height @@ -113,8 +115,8 @@ packssdw xmm0, xmm6 ;pack back to word ;clamp the values - pminsw xmm0, xmm8 - pmaxsw xmm0, xmm5 + pminsw xmm0, max + pmaxsw xmm0, min %if %1 movdqu xmm1, [rdi] @@ -128,36 +130,36 @@ %endm %macro HIGH_APPLY_FILTER_16 1 - movdqa xmm9, xmm0 + movdqa xmm5, xmm0 movdqa xmm6, xmm2 - punpckhwd xmm9, xmm1 + punpckhwd xmm5, xmm1 punpckhwd xmm6, xmm3 punpcklwd xmm0, xmm1 punpcklwd xmm2, xmm3 - pmaddwd xmm9, xmm7 + pmaddwd xmm5, xmm7 pmaddwd xmm6, xmm7 pmaddwd xmm0, xmm7 pmaddwd xmm2, xmm7 - paddd xmm9, xmm4 ;rounding + paddd xmm5, xmm4 ;rounding paddd xmm6, xmm4 paddd xmm0, xmm4 paddd xmm2, xmm4 - psrad xmm9, 7 ;shift + psrad xmm5, 7 ;shift psrad xmm6, 7 psrad xmm0, 7 psrad xmm2, 7 - packssdw xmm0, xmm9 ;pack back to word + packssdw xmm0, xmm5 ;pack back to word packssdw xmm2, xmm6 ;pack back to word ;clamp the values - pminsw xmm0, xmm8 - pmaxsw xmm0, xmm5 - pminsw xmm2, xmm8 - pmaxsw xmm2, xmm5 + pminsw xmm0, max + pmaxsw xmm0, min + pminsw xmm2, max + pmaxsw xmm2, min %if %1 movdqu xmm1, [rdi] @@ -172,7 +174,6 @@ lea rdi, [rdi + 2*rdx] dec rcx %endm -%endif SECTION .text @@ -200,7 +201,6 @@ sym(aom_highbd_filter_block1d4_v2_sse2): pop rbp ret -%if ARCH_X86_64 global sym(aom_highbd_filter_block1d8_v2_sse2) PRIVATE sym(aom_highbd_filter_block1d8_v2_sse2): push rbp @@ -211,6 +211,11 @@ sym(aom_highbd_filter_block1d8_v2_sse2): push rdi ; end prolog + ALIGN_STACK 16, rax + sub rsp, 16 * 2 + %define max [rsp + 16 * 0] + %define min [rsp + 16 * 1] + HIGH_GET_PARAM .loop: movdqu xmm0, [rsi] ;0 @@ -219,6 +224,9 @@ sym(aom_highbd_filter_block1d8_v2_sse2): HIGH_APPLY_FILTER_8 0 jnz .loop + add rsp, 16 * 2 + pop rsp + ; begin epilog pop rdi pop rsi @@ -237,6 +245,11 @@ sym(aom_highbd_filter_block1d16_v2_sse2): push rdi ; end prolog + ALIGN_STACK 16, rax + sub rsp, 16 * 2 + %define max [rsp + 16 * 0] + %define min [rsp + 16 * 1] + HIGH_GET_PARAM .loop: movdqu xmm0, [rsi] ;0 @@ -247,6 +260,9 @@ sym(aom_highbd_filter_block1d16_v2_sse2): HIGH_APPLY_FILTER_16 0 jnz .loop + add rsp, 16 * 2 + pop rsp + ; begin epilog pop rdi pop rsi @@ -254,7 +270,6 @@ sym(aom_highbd_filter_block1d16_v2_sse2): UNSHADOW_ARGS pop rbp ret -%endif global sym(aom_highbd_filter_block1d4_h2_sse2) PRIVATE sym(aom_highbd_filter_block1d4_h2_sse2): @@ -281,7 +296,6 @@ sym(aom_highbd_filter_block1d4_h2_sse2): pop rbp ret -%if ARCH_X86_64 global sym(aom_highbd_filter_block1d8_h2_sse2) PRIVATE sym(aom_highbd_filter_block1d8_h2_sse2): push rbp @@ -292,6 +306,11 @@ sym(aom_highbd_filter_block1d8_h2_sse2): push rdi ; end prolog + ALIGN_STACK 16, rax + sub rsp, 16 * 2 + %define max [rsp + 16 * 0] + %define min [rsp + 16 * 1] + HIGH_GET_PARAM .loop: movdqu xmm0, [rsi] ;load src @@ -300,6 +319,9 @@ sym(aom_highbd_filter_block1d8_h2_sse2): HIGH_APPLY_FILTER_8 0 jnz .loop + add rsp, 16 * 2 + pop rsp + ; begin epilog pop rdi pop rsi @@ -318,6 +340,11 @@ sym(aom_highbd_filter_block1d16_h2_sse2): push rdi ; end prolog + ALIGN_STACK 16, rax + sub rsp, 16 * 2 + %define max [rsp + 16 * 0] + %define min [rsp + 16 * 1] + HIGH_GET_PARAM .loop: movdqu xmm0, [rsi] ;load src @@ -328,6 +355,9 @@ sym(aom_highbd_filter_block1d16_h2_sse2): HIGH_APPLY_FILTER_16 0 jnz .loop + add rsp, 16 * 2 + pop rsp + ; begin epilog pop rdi pop rsi @@ -335,4 +365,3 @@ sym(aom_highbd_filter_block1d16_h2_sse2): UNSHADOW_ARGS pop rbp ret -%endif diff --git a/third_party/aom/aom_dsp/x86/aom_subpixel_8t_intrin_sse2.c b/third_party/aom/aom_dsp/x86/aom_subpixel_8t_intrin_sse2.c new file mode 100644 index 0000000000..cff7f43eee --- /dev/null +++ b/third_party/aom/aom_dsp/x86/aom_subpixel_8t_intrin_sse2.c @@ -0,0 +1,569 @@ +/* + * Copyright (c) 2018, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include // SSE2 + +#include "config/aom_dsp_rtcd.h" +#include "aom_dsp/x86/convolve.h" +#include "aom_ports/mem.h" + +void aom_filter_block1d16_h4_sse2(const uint8_t *src_ptr, + ptrdiff_t src_pixels_per_line, + uint8_t *output_ptr, ptrdiff_t output_pitch, + uint32_t output_height, + const int16_t *filter) { + __m128i filtersReg; + __m128i addFilterReg32; + __m128i secondFilters, thirdFilters; + __m128i srcRegFilt32b1_1, srcRegFilt32b1_2, srcRegFilt32b2_1, + srcRegFilt32b2_2; + __m128i srcReg32b1, srcReg32b2; + unsigned int i; + src_ptr -= 3; + addFilterReg32 = _mm_set1_epi16(32); + filtersReg = _mm_loadu_si128((const __m128i *)filter); + filtersReg = _mm_srai_epi16(filtersReg, 1); + + // coeffs 0 1 0 1 2 3 2 3 + const __m128i tmp_0 = _mm_unpacklo_epi32(filtersReg, filtersReg); + // coeffs 4 5 4 5 6 7 6 7 + const __m128i tmp_1 = _mm_unpackhi_epi32(filtersReg, filtersReg); + + secondFilters = _mm_unpackhi_epi64(tmp_0, tmp_0); // coeffs 2 3 2 3 2 3 2 3 + thirdFilters = _mm_unpacklo_epi64(tmp_1, tmp_1); // coeffs 4 5 4 5 4 5 4 5 + + for (i = output_height; i > 0; i -= 1) { + srcReg32b1 = _mm_loadu_si128((const __m128i *)src_ptr); + + __m128i ss_2 = _mm_srli_si128(srcReg32b1, 2); + __m128i ss_4 = _mm_srli_si128(srcReg32b1, 4); + __m128i ss_1_1 = _mm_unpacklo_epi8(ss_2, _mm_setzero_si128()); + __m128i ss_2_1 = _mm_unpacklo_epi8(ss_4, _mm_setzero_si128()); + __m128i d1 = _mm_madd_epi16(ss_1_1, secondFilters); + __m128i d2 = _mm_madd_epi16(ss_2_1, thirdFilters); + srcRegFilt32b1_1 = _mm_add_epi32(d1, d2); + + __m128i ss_1 = _mm_srli_si128(srcReg32b1, 3); + __m128i ss_3 = _mm_srli_si128(srcReg32b1, 5); + __m128i ss_1_2 = _mm_unpacklo_epi8(ss_1, _mm_setzero_si128()); + __m128i ss_2_2 = _mm_unpacklo_epi8(ss_3, _mm_setzero_si128()); + d1 = _mm_madd_epi16(ss_1_2, secondFilters); + d2 = _mm_madd_epi16(ss_2_2, thirdFilters); + srcRegFilt32b1_2 = _mm_add_epi32(d1, d2); + + __m128i res_lo = _mm_unpacklo_epi32(srcRegFilt32b1_1, srcRegFilt32b1_2); + __m128i res_hi = _mm_unpackhi_epi32(srcRegFilt32b1_1, srcRegFilt32b1_2); + srcRegFilt32b1_1 = _mm_packs_epi32(res_lo, res_hi); + + // reading stride of the next 16 bytes + // (part of it was being read by earlier read) + srcReg32b2 = _mm_loadu_si128((const __m128i *)(src_ptr + 8)); + + ss_2 = _mm_srli_si128(srcReg32b2, 2); + ss_4 = _mm_srli_si128(srcReg32b2, 4); + ss_1_1 = _mm_unpacklo_epi8(ss_2, _mm_setzero_si128()); + ss_2_1 = _mm_unpacklo_epi8(ss_4, _mm_setzero_si128()); + d1 = _mm_madd_epi16(ss_1_1, secondFilters); + d2 = _mm_madd_epi16(ss_2_1, thirdFilters); + srcRegFilt32b2_1 = _mm_add_epi32(d1, d2); + + ss_1 = _mm_srli_si128(srcReg32b2, 3); + ss_3 = _mm_srli_si128(srcReg32b2, 5); + ss_1_2 = _mm_unpacklo_epi8(ss_1, _mm_setzero_si128()); + ss_2_2 = _mm_unpacklo_epi8(ss_3, _mm_setzero_si128()); + d1 = _mm_madd_epi16(ss_1_2, secondFilters); + d2 = _mm_madd_epi16(ss_2_2, thirdFilters); + srcRegFilt32b2_2 = _mm_add_epi32(d1, d2); + + res_lo = _mm_unpacklo_epi32(srcRegFilt32b2_1, srcRegFilt32b2_2); + res_hi = _mm_unpackhi_epi32(srcRegFilt32b2_1, srcRegFilt32b2_2); + srcRegFilt32b2_1 = _mm_packs_epi32(res_lo, res_hi); + + // shift by 6 bit each 16 bit + srcRegFilt32b1_1 = _mm_adds_epi16(srcRegFilt32b1_1, addFilterReg32); + srcRegFilt32b2_1 = _mm_adds_epi16(srcRegFilt32b2_1, addFilterReg32); + srcRegFilt32b1_1 = _mm_srai_epi16(srcRegFilt32b1_1, 6); + srcRegFilt32b2_1 = _mm_srai_epi16(srcRegFilt32b2_1, 6); + + // shrink to 8 bit each 16 bits, the first lane contain the first + // convolve result and the second lane contain the second convolve result + srcRegFilt32b1_1 = _mm_packus_epi16(srcRegFilt32b1_1, srcRegFilt32b2_1); + + src_ptr += src_pixels_per_line; + + _mm_store_si128((__m128i *)output_ptr, srcRegFilt32b1_1); + + output_ptr += output_pitch; + } +} + +void aom_filter_block1d16_v4_sse2(const uint8_t *src_ptr, ptrdiff_t src_pitch, + uint8_t *output_ptr, ptrdiff_t out_pitch, + uint32_t output_height, + const int16_t *filter) { + __m128i filtersReg; + __m128i srcReg2, srcReg3, srcReg4, srcReg5, srcReg6; + __m128i srcReg23_lo, srcReg23_hi, srcReg34_lo, srcReg34_hi; + __m128i srcReg45_lo, srcReg45_hi, srcReg56_lo, srcReg56_hi; + __m128i resReg23_lo, resReg34_lo, resReg45_lo, resReg56_lo; + __m128i resReg23_hi, resReg34_hi, resReg45_hi, resReg56_hi; + __m128i resReg23_45_lo, resReg34_56_lo, resReg23_45_hi, resReg34_56_hi; + __m128i resReg23_45, resReg34_56; + __m128i addFilterReg32, secondFilters, thirdFilters; + __m128i tmp_0, tmp_1; + unsigned int i; + ptrdiff_t src_stride, dst_stride; + + addFilterReg32 = _mm_set1_epi16(32); + filtersReg = _mm_loadu_si128((const __m128i *)filter); + filtersReg = _mm_srai_epi16(filtersReg, 1); + + // coeffs 0 1 0 1 2 3 2 3 + const __m128i tmp0 = _mm_unpacklo_epi32(filtersReg, filtersReg); + // coeffs 4 5 4 5 6 7 6 7 + const __m128i tmp1 = _mm_unpackhi_epi32(filtersReg, filtersReg); + + secondFilters = _mm_unpackhi_epi64(tmp0, tmp0); // coeffs 2 3 2 3 2 3 2 3 + thirdFilters = _mm_unpacklo_epi64(tmp1, tmp1); // coeffs 4 5 4 5 4 5 4 5 + + // multiply the size of the source and destination stride by two + src_stride = src_pitch << 1; + dst_stride = out_pitch << 1; + + srcReg2 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 2)); + srcReg3 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 3)); + srcReg23_lo = _mm_unpacklo_epi8(srcReg2, srcReg3); + srcReg23_hi = _mm_unpackhi_epi8(srcReg2, srcReg3); + __m128i resReg23_lo_1 = _mm_unpacklo_epi8(srcReg23_lo, _mm_setzero_si128()); + __m128i resReg23_lo_2 = _mm_unpackhi_epi8(srcReg23_lo, _mm_setzero_si128()); + __m128i resReg23_hi_1 = _mm_unpacklo_epi8(srcReg23_hi, _mm_setzero_si128()); + __m128i resReg23_hi_2 = _mm_unpackhi_epi8(srcReg23_hi, _mm_setzero_si128()); + + srcReg4 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 4)); + srcReg34_lo = _mm_unpacklo_epi8(srcReg3, srcReg4); + srcReg34_hi = _mm_unpackhi_epi8(srcReg3, srcReg4); + __m128i resReg34_lo_1 = _mm_unpacklo_epi8(srcReg34_lo, _mm_setzero_si128()); + __m128i resReg34_lo_2 = _mm_unpackhi_epi8(srcReg34_lo, _mm_setzero_si128()); + __m128i resReg34_hi_1 = _mm_unpacklo_epi8(srcReg34_hi, _mm_setzero_si128()); + __m128i resReg34_hi_2 = _mm_unpackhi_epi8(srcReg34_hi, _mm_setzero_si128()); + + for (i = output_height; i > 1; i -= 2) { + srcReg5 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 5)); + + srcReg45_lo = _mm_unpacklo_epi8(srcReg4, srcReg5); + srcReg45_hi = _mm_unpackhi_epi8(srcReg4, srcReg5); + + srcReg6 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 6)); + + srcReg56_lo = _mm_unpacklo_epi8(srcReg5, srcReg6); + srcReg56_hi = _mm_unpackhi_epi8(srcReg5, srcReg6); + + // multiply 2 adjacent elements with the filter and add the result + + tmp_0 = _mm_madd_epi16(resReg23_lo_1, secondFilters); + tmp_1 = _mm_madd_epi16(resReg23_lo_2, secondFilters); + resReg23_lo = _mm_packs_epi32(tmp_0, tmp_1); + + tmp_0 = _mm_madd_epi16(resReg34_lo_1, secondFilters); + tmp_1 = _mm_madd_epi16(resReg34_lo_2, secondFilters); + resReg34_lo = _mm_packs_epi32(tmp_0, tmp_1); + + __m128i resReg45_lo_1 = _mm_unpacklo_epi8(srcReg45_lo, _mm_setzero_si128()); + __m128i resReg45_lo_2 = _mm_unpackhi_epi8(srcReg45_lo, _mm_setzero_si128()); + tmp_0 = _mm_madd_epi16(resReg45_lo_1, thirdFilters); + tmp_1 = _mm_madd_epi16(resReg45_lo_2, thirdFilters); + resReg45_lo = _mm_packs_epi32(tmp_0, tmp_1); + + __m128i resReg56_lo_1 = _mm_unpacklo_epi8(srcReg56_lo, _mm_setzero_si128()); + __m128i resReg56_lo_2 = _mm_unpackhi_epi8(srcReg56_lo, _mm_setzero_si128()); + tmp_0 = _mm_madd_epi16(resReg56_lo_1, thirdFilters); + tmp_1 = _mm_madd_epi16(resReg56_lo_2, thirdFilters); + resReg56_lo = _mm_packs_epi32(tmp_0, tmp_1); + + // add and saturate the results together + resReg23_45_lo = _mm_adds_epi16(resReg23_lo, resReg45_lo); + resReg34_56_lo = _mm_adds_epi16(resReg34_lo, resReg56_lo); + + // multiply 2 adjacent elements with the filter and add the result + + tmp_0 = _mm_madd_epi16(resReg23_hi_1, secondFilters); + tmp_1 = _mm_madd_epi16(resReg23_hi_2, secondFilters); + resReg23_hi = _mm_packs_epi32(tmp_0, tmp_1); + + tmp_0 = _mm_madd_epi16(resReg34_hi_1, secondFilters); + tmp_1 = _mm_madd_epi16(resReg34_hi_2, secondFilters); + resReg34_hi = _mm_packs_epi32(tmp_0, tmp_1); + + __m128i resReg45_hi_1 = _mm_unpacklo_epi8(srcReg45_hi, _mm_setzero_si128()); + __m128i resReg45_hi_2 = _mm_unpackhi_epi8(srcReg45_hi, _mm_setzero_si128()); + tmp_0 = _mm_madd_epi16(resReg45_hi_1, thirdFilters); + tmp_1 = _mm_madd_epi16(resReg45_hi_2, thirdFilters); + resReg45_hi = _mm_packs_epi32(tmp_0, tmp_1); + + __m128i resReg56_hi_1 = _mm_unpacklo_epi8(srcReg56_hi, _mm_setzero_si128()); + __m128i resReg56_hi_2 = _mm_unpackhi_epi8(srcReg56_hi, _mm_setzero_si128()); + tmp_0 = _mm_madd_epi16(resReg56_hi_1, thirdFilters); + tmp_1 = _mm_madd_epi16(resReg56_hi_2, thirdFilters); + resReg56_hi = _mm_packs_epi32(tmp_0, tmp_1); + + // add and saturate the results together + resReg23_45_hi = _mm_adds_epi16(resReg23_hi, resReg45_hi); + resReg34_56_hi = _mm_adds_epi16(resReg34_hi, resReg56_hi); + + // shift by 6 bit each 16 bit + resReg23_45_lo = _mm_adds_epi16(resReg23_45_lo, addFilterReg32); + resReg34_56_lo = _mm_adds_epi16(resReg34_56_lo, addFilterReg32); + resReg23_45_hi = _mm_adds_epi16(resReg23_45_hi, addFilterReg32); + resReg34_56_hi = _mm_adds_epi16(resReg34_56_hi, addFilterReg32); + resReg23_45_lo = _mm_srai_epi16(resReg23_45_lo, 6); + resReg34_56_lo = _mm_srai_epi16(resReg34_56_lo, 6); + resReg23_45_hi = _mm_srai_epi16(resReg23_45_hi, 6); + resReg34_56_hi = _mm_srai_epi16(resReg34_56_hi, 6); + + // shrink to 8 bit each 16 bits, the first lane contain the first + // convolve result and the second lane contain the second convolve + // result + resReg23_45 = _mm_packus_epi16(resReg23_45_lo, resReg23_45_hi); + resReg34_56 = _mm_packus_epi16(resReg34_56_lo, resReg34_56_hi); + + src_ptr += src_stride; + + _mm_store_si128((__m128i *)output_ptr, (resReg23_45)); + _mm_store_si128((__m128i *)(output_ptr + out_pitch), (resReg34_56)); + + output_ptr += dst_stride; + + // save part of the registers for next strides + resReg23_lo_1 = resReg45_lo_1; + resReg23_lo_2 = resReg45_lo_2; + resReg23_hi_1 = resReg45_hi_1; + resReg23_hi_2 = resReg45_hi_2; + resReg34_lo_1 = resReg56_lo_1; + resReg34_lo_2 = resReg56_lo_2; + resReg34_hi_1 = resReg56_hi_1; + resReg34_hi_2 = resReg56_hi_2; + srcReg4 = srcReg6; + } +} + +void aom_filter_block1d8_h4_sse2(const uint8_t *src_ptr, + ptrdiff_t src_pixels_per_line, + uint8_t *output_ptr, ptrdiff_t output_pitch, + uint32_t output_height, + const int16_t *filter) { + __m128i filtersReg; + __m128i addFilterReg32; + __m128i secondFilters, thirdFilters; + __m128i srcRegFilt32b1_1, srcRegFilt32b1_2; + __m128i srcReg32b1; + unsigned int i; + src_ptr -= 3; + addFilterReg32 = _mm_set1_epi16(32); + filtersReg = _mm_loadu_si128((const __m128i *)filter); + filtersReg = _mm_srai_epi16(filtersReg, 1); + + // coeffs 0 1 0 1 2 3 2 3 + const __m128i tmp_0 = _mm_unpacklo_epi32(filtersReg, filtersReg); + // coeffs 4 5 4 5 6 7 6 7 + const __m128i tmp_1 = _mm_unpackhi_epi32(filtersReg, filtersReg); + + secondFilters = _mm_unpackhi_epi64(tmp_0, tmp_0); // coeffs 2 3 2 3 2 3 2 3 + thirdFilters = _mm_unpacklo_epi64(tmp_1, tmp_1); // coeffs 4 5 4 5 4 5 4 5 + + for (i = output_height; i > 0; i -= 1) { + srcReg32b1 = _mm_loadu_si128((const __m128i *)src_ptr); + + __m128i ss_2 = _mm_srli_si128(srcReg32b1, 2); + __m128i ss_4 = _mm_srli_si128(srcReg32b1, 4); + ss_2 = _mm_unpacklo_epi8(ss_2, _mm_setzero_si128()); + ss_4 = _mm_unpacklo_epi8(ss_4, _mm_setzero_si128()); + __m128i d1 = _mm_madd_epi16(ss_2, secondFilters); + __m128i d2 = _mm_madd_epi16(ss_4, thirdFilters); + srcRegFilt32b1_1 = _mm_add_epi32(d1, d2); + + __m128i ss_3 = _mm_srli_si128(srcReg32b1, 3); + __m128i ss_5 = _mm_srli_si128(srcReg32b1, 5); + ss_3 = _mm_unpacklo_epi8(ss_3, _mm_setzero_si128()); + ss_5 = _mm_unpacklo_epi8(ss_5, _mm_setzero_si128()); + d1 = _mm_madd_epi16(ss_3, secondFilters); + d2 = _mm_madd_epi16(ss_5, thirdFilters); + srcRegFilt32b1_2 = _mm_add_epi32(d1, d2); + + __m128i res_lo = _mm_unpacklo_epi32(srcRegFilt32b1_1, srcRegFilt32b1_2); + __m128i res_hi = _mm_unpackhi_epi32(srcRegFilt32b1_1, srcRegFilt32b1_2); + srcRegFilt32b1_1 = _mm_packs_epi32(res_lo, res_hi); + + // shift by 6 bit each 16 bit + srcRegFilt32b1_1 = _mm_adds_epi16(srcRegFilt32b1_1, addFilterReg32); + srcRegFilt32b1_1 = _mm_srai_epi16(srcRegFilt32b1_1, 6); + + // shrink to 8 bit each 16 bits, the first lane contain the first + // convolve result and the second lane contain the second convolve result + srcRegFilt32b1_1 = _mm_packus_epi16(srcRegFilt32b1_1, _mm_setzero_si128()); + + src_ptr += src_pixels_per_line; + + _mm_storel_epi64((__m128i *)output_ptr, srcRegFilt32b1_1); + + output_ptr += output_pitch; + } +} + +void aom_filter_block1d8_v4_sse2(const uint8_t *src_ptr, ptrdiff_t src_pitch, + uint8_t *output_ptr, ptrdiff_t out_pitch, + uint32_t output_height, + const int16_t *filter) { + __m128i filtersReg; + __m128i srcReg2, srcReg3, srcReg4, srcReg5, srcReg6; + __m128i srcReg23_lo, srcReg34_lo; + __m128i srcReg45_lo, srcReg56_lo; + __m128i resReg23_lo, resReg34_lo, resReg45_lo, resReg56_lo; + __m128i resReg23_45_lo, resReg34_56_lo; + __m128i resReg23_45, resReg34_56; + __m128i addFilterReg32, secondFilters, thirdFilters; + __m128i tmp_0, tmp_1; + unsigned int i; + ptrdiff_t src_stride, dst_stride; + + addFilterReg32 = _mm_set1_epi16(32); + filtersReg = _mm_loadu_si128((const __m128i *)filter); + filtersReg = _mm_srai_epi16(filtersReg, 1); + + // coeffs 0 1 0 1 2 3 2 3 + const __m128i tmp0 = _mm_unpacklo_epi32(filtersReg, filtersReg); + // coeffs 4 5 4 5 6 7 6 7 + const __m128i tmp1 = _mm_unpackhi_epi32(filtersReg, filtersReg); + + secondFilters = _mm_unpackhi_epi64(tmp0, tmp0); // coeffs 2 3 2 3 2 3 2 3 + thirdFilters = _mm_unpacklo_epi64(tmp1, tmp1); // coeffs 4 5 4 5 4 5 4 5 + + // multiply the size of the source and destination stride by two + src_stride = src_pitch << 1; + dst_stride = out_pitch << 1; + + srcReg2 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 2)); + srcReg3 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 3)); + srcReg23_lo = _mm_unpacklo_epi8(srcReg2, srcReg3); + __m128i resReg23_lo_1 = _mm_unpacklo_epi8(srcReg23_lo, _mm_setzero_si128()); + __m128i resReg23_lo_2 = _mm_unpackhi_epi8(srcReg23_lo, _mm_setzero_si128()); + + srcReg4 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 4)); + srcReg34_lo = _mm_unpacklo_epi8(srcReg3, srcReg4); + __m128i resReg34_lo_1 = _mm_unpacklo_epi8(srcReg34_lo, _mm_setzero_si128()); + __m128i resReg34_lo_2 = _mm_unpackhi_epi8(srcReg34_lo, _mm_setzero_si128()); + + for (i = output_height; i > 1; i -= 2) { + srcReg5 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 5)); + srcReg45_lo = _mm_unpacklo_epi8(srcReg4, srcReg5); + + srcReg6 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 6)); + srcReg56_lo = _mm_unpacklo_epi8(srcReg5, srcReg6); + + // multiply 2 adjacent elements with the filter and add the result + + tmp_0 = _mm_madd_epi16(resReg23_lo_1, secondFilters); + tmp_1 = _mm_madd_epi16(resReg23_lo_2, secondFilters); + resReg23_lo = _mm_packs_epi32(tmp_0, tmp_1); + + tmp_0 = _mm_madd_epi16(resReg34_lo_1, secondFilters); + tmp_1 = _mm_madd_epi16(resReg34_lo_2, secondFilters); + resReg34_lo = _mm_packs_epi32(tmp_0, tmp_1); + + __m128i resReg45_lo_1 = _mm_unpacklo_epi8(srcReg45_lo, _mm_setzero_si128()); + __m128i resReg45_lo_2 = _mm_unpackhi_epi8(srcReg45_lo, _mm_setzero_si128()); + tmp_0 = _mm_madd_epi16(resReg45_lo_1, thirdFilters); + tmp_1 = _mm_madd_epi16(resReg45_lo_2, thirdFilters); + resReg45_lo = _mm_packs_epi32(tmp_0, tmp_1); + + __m128i resReg56_lo_1 = _mm_unpacklo_epi8(srcReg56_lo, _mm_setzero_si128()); + __m128i resReg56_lo_2 = _mm_unpackhi_epi8(srcReg56_lo, _mm_setzero_si128()); + tmp_0 = _mm_madd_epi16(resReg56_lo_1, thirdFilters); + tmp_1 = _mm_madd_epi16(resReg56_lo_2, thirdFilters); + resReg56_lo = _mm_packs_epi32(tmp_0, tmp_1); + + // add and saturate the results together + resReg23_45_lo = _mm_adds_epi16(resReg23_lo, resReg45_lo); + resReg34_56_lo = _mm_adds_epi16(resReg34_lo, resReg56_lo); + + // shift by 6 bit each 16 bit + resReg23_45_lo = _mm_adds_epi16(resReg23_45_lo, addFilterReg32); + resReg34_56_lo = _mm_adds_epi16(resReg34_56_lo, addFilterReg32); + resReg23_45_lo = _mm_srai_epi16(resReg23_45_lo, 6); + resReg34_56_lo = _mm_srai_epi16(resReg34_56_lo, 6); + + // shrink to 8 bit each 16 bits, the first lane contain the first + // convolve result and the second lane contain the second convolve + // result + resReg23_45 = _mm_packus_epi16(resReg23_45_lo, _mm_setzero_si128()); + resReg34_56 = _mm_packus_epi16(resReg34_56_lo, _mm_setzero_si128()); + + src_ptr += src_stride; + + _mm_storel_epi64((__m128i *)output_ptr, (resReg23_45)); + _mm_storel_epi64((__m128i *)(output_ptr + out_pitch), (resReg34_56)); + + output_ptr += dst_stride; + + // save part of the registers for next strides + resReg23_lo_1 = resReg45_lo_1; + resReg23_lo_2 = resReg45_lo_2; + resReg34_lo_1 = resReg56_lo_1; + resReg34_lo_2 = resReg56_lo_2; + srcReg4 = srcReg6; + } +} + +void aom_filter_block1d4_h4_sse2(const uint8_t *src_ptr, + ptrdiff_t src_pixels_per_line, + uint8_t *output_ptr, ptrdiff_t output_pitch, + uint32_t output_height, + const int16_t *filter) { + __m128i filtersReg; + __m128i addFilterReg32; + __m128i secondFilters, thirdFilters; + __m128i srcRegFilt32b1_1; + __m128i srcReg32b1; + unsigned int i; + src_ptr -= 3; + addFilterReg32 = _mm_set1_epi16(32); + filtersReg = _mm_loadu_si128((const __m128i *)filter); + filtersReg = _mm_srai_epi16(filtersReg, 1); + + // coeffs 0 1 0 1 2 3 2 3 + const __m128i tmp_0 = _mm_unpacklo_epi32(filtersReg, filtersReg); + // coeffs 4 5 4 5 6 7 6 7 + const __m128i tmp_1 = _mm_unpackhi_epi32(filtersReg, filtersReg); + + secondFilters = _mm_unpackhi_epi64(tmp_0, tmp_0); // coeffs 2 3 2 3 2 3 2 3 + thirdFilters = _mm_unpacklo_epi64(tmp_1, tmp_1); // coeffs 4 5 4 5 4 5 4 5 + + for (i = output_height; i > 0; i -= 1) { + srcReg32b1 = _mm_loadu_si128((const __m128i *)src_ptr); + + __m128i ss_2 = _mm_srli_si128(srcReg32b1, 2); + __m128i ss_3 = _mm_srli_si128(srcReg32b1, 3); + __m128i ss_4 = _mm_srli_si128(srcReg32b1, 4); + __m128i ss_5 = _mm_srli_si128(srcReg32b1, 5); + + ss_2 = _mm_unpacklo_epi8(ss_2, _mm_setzero_si128()); + ss_3 = _mm_unpacklo_epi8(ss_3, _mm_setzero_si128()); + ss_4 = _mm_unpacklo_epi8(ss_4, _mm_setzero_si128()); + ss_5 = _mm_unpacklo_epi8(ss_5, _mm_setzero_si128()); + + __m128i ss_1_1 = _mm_unpacklo_epi32(ss_2, ss_3); + __m128i ss_1_2 = _mm_unpacklo_epi32(ss_4, ss_5); + + __m128i d1 = _mm_madd_epi16(ss_1_1, secondFilters); + __m128i d2 = _mm_madd_epi16(ss_1_2, thirdFilters); + srcRegFilt32b1_1 = _mm_add_epi32(d1, d2); + + srcRegFilt32b1_1 = _mm_packs_epi32(srcRegFilt32b1_1, _mm_setzero_si128()); + + // shift by 6 bit each 16 bit + srcRegFilt32b1_1 = _mm_adds_epi16(srcRegFilt32b1_1, addFilterReg32); + srcRegFilt32b1_1 = _mm_srai_epi16(srcRegFilt32b1_1, 6); + + // shrink to 8 bit each 16 bits, the first lane contain the first + // convolve result and the second lane contain the second convolve result + srcRegFilt32b1_1 = _mm_packus_epi16(srcRegFilt32b1_1, _mm_setzero_si128()); + + src_ptr += src_pixels_per_line; + + *((uint32_t *)(output_ptr)) = _mm_cvtsi128_si32(srcRegFilt32b1_1); + + output_ptr += output_pitch; + } +} + +void aom_filter_block1d4_v4_sse2(const uint8_t *src_ptr, ptrdiff_t src_pitch, + uint8_t *output_ptr, ptrdiff_t out_pitch, + uint32_t output_height, + const int16_t *filter) { + __m128i filtersReg; + __m128i srcReg2, srcReg3, srcReg4, srcReg5, srcReg6; + __m128i srcReg23, srcReg34, srcReg45, srcReg56; + __m128i resReg23_34, resReg45_56; + __m128i resReg23_34_45_56; + __m128i addFilterReg32, secondFilters, thirdFilters; + __m128i tmp_0, tmp_1; + unsigned int i; + ptrdiff_t src_stride, dst_stride; + + addFilterReg32 = _mm_set1_epi16(32); + filtersReg = _mm_loadu_si128((const __m128i *)filter); + filtersReg = _mm_srai_epi16(filtersReg, 1); + + // coeffs 0 1 0 1 2 3 2 3 + const __m128i tmp0 = _mm_unpacklo_epi32(filtersReg, filtersReg); + // coeffs 4 5 4 5 6 7 6 7 + const __m128i tmp1 = _mm_unpackhi_epi32(filtersReg, filtersReg); + + secondFilters = _mm_unpackhi_epi64(tmp0, tmp0); // coeffs 2 3 2 3 2 3 2 3 + thirdFilters = _mm_unpacklo_epi64(tmp1, tmp1); // coeffs 4 5 4 5 4 5 4 5 + + // multiply the size of the source and destination stride by two + src_stride = src_pitch << 1; + dst_stride = out_pitch << 1; + + srcReg2 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 2)); + srcReg3 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 3)); + srcReg23 = _mm_unpacklo_epi8(srcReg2, srcReg3); + __m128i resReg23 = _mm_unpacklo_epi8(srcReg23, _mm_setzero_si128()); + + srcReg4 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 4)); + srcReg34 = _mm_unpacklo_epi8(srcReg3, srcReg4); + __m128i resReg34 = _mm_unpacklo_epi8(srcReg34, _mm_setzero_si128()); + + for (i = output_height; i > 1; i -= 2) { + srcReg5 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 5)); + srcReg45 = _mm_unpacklo_epi8(srcReg4, srcReg5); + srcReg6 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 6)); + srcReg56 = _mm_unpacklo_epi8(srcReg5, srcReg6); + + // multiply 2 adjacent elements with the filter and add the result + tmp_0 = _mm_madd_epi16(resReg23, secondFilters); + tmp_1 = _mm_madd_epi16(resReg34, secondFilters); + resReg23_34 = _mm_packs_epi32(tmp_0, tmp_1); + + __m128i resReg45 = _mm_unpacklo_epi8(srcReg45, _mm_setzero_si128()); + __m128i resReg56 = _mm_unpacklo_epi8(srcReg56, _mm_setzero_si128()); + + tmp_0 = _mm_madd_epi16(resReg45, thirdFilters); + tmp_1 = _mm_madd_epi16(resReg56, thirdFilters); + resReg45_56 = _mm_packs_epi32(tmp_0, tmp_1); + + // add and saturate the results together + resReg23_34_45_56 = _mm_adds_epi16(resReg23_34, resReg45_56); + + // shift by 6 bit each 16 bit + resReg23_34_45_56 = _mm_adds_epi16(resReg23_34_45_56, addFilterReg32); + resReg23_34_45_56 = _mm_srai_epi16(resReg23_34_45_56, 6); + + // shrink to 8 bit each 16 bits, the first lane contain the first + // convolve result and the second lane contain the second convolve + // result + resReg23_34_45_56 = + _mm_packus_epi16(resReg23_34_45_56, _mm_setzero_si128()); + + src_ptr += src_stride; + + *((uint32_t *)(output_ptr)) = _mm_cvtsi128_si32(resReg23_34_45_56); + *((uint32_t *)(output_ptr + out_pitch)) = + _mm_cvtsi128_si32(_mm_srli_si128(resReg23_34_45_56, 4)); + + output_ptr += dst_stride; + + // save part of the registers for next strides + resReg23 = resReg45; + resReg34 = resReg56; + srcReg4 = srcReg6; + } +} diff --git a/third_party/aom/aom_dsp/x86/aom_subpixel_8t_intrin_ssse3.c b/third_party/aom/aom_dsp/x86/aom_subpixel_8t_intrin_ssse3.c index 325a21b761..79324f5457 100644 --- a/third_party/aom/aom_dsp/x86/aom_subpixel_8t_intrin_ssse3.c +++ b/third_party/aom/aom_dsp/x86/aom_subpixel_8t_intrin_ssse3.c @@ -45,11 +45,153 @@ DECLARE_ALIGNED(16, static const uint8_t, filt4_global[16]) = { 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14 }; +DECLARE_ALIGNED(32, static const uint8_t, filt_h4[]) = { + 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 0, 1, 1, + 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 2, 3, 3, 4, 4, 5, + 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 2, 3, 3, 4, 4, 5, 5, 6, 6, + 7, 7, 8, 8, 9, 9, 10, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, + 10, 11, 11, 12, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, + 12, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 6, 7, + 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14 +}; + +DECLARE_ALIGNED(32, static const uint8_t, filtd4[]) = { + 2, 3, 4, 5, 3, 4, 5, 6, 4, 5, 6, 7, 5, 6, 7, 8, + 2, 3, 4, 5, 3, 4, 5, 6, 4, 5, 6, 7, 5, 6, 7, 8, +}; + // These are reused by the avx2 intrinsics. filter8_1dfunction aom_filter_block1d8_v8_intrin_ssse3; filter8_1dfunction aom_filter_block1d8_h8_intrin_ssse3; filter8_1dfunction aom_filter_block1d4_h8_intrin_ssse3; +static void aom_filter_block1d4_h4_ssse3( + const uint8_t *src_ptr, ptrdiff_t src_pixels_per_line, uint8_t *output_ptr, + ptrdiff_t output_pitch, uint32_t output_height, const int16_t *filter) { + __m128i filtersReg; + __m128i addFilterReg32, filt1Reg, firstFilters, srcReg32b1, srcRegFilt32b1_1; + unsigned int i; + src_ptr -= 3; + addFilterReg32 = _mm_set1_epi16(32); + filtersReg = _mm_loadu_si128((const __m128i *)filter); + filtersReg = _mm_srai_epi16(filtersReg, 1); + // converting the 16 bit (short) to 8 bit (byte) and have the same data + // in both lanes of 128 bit register. + filtersReg = _mm_packs_epi16(filtersReg, filtersReg); + + firstFilters = _mm_shuffle_epi8(filtersReg, _mm_set1_epi32(0x5040302u)); + filt1Reg = _mm_load_si128((__m128i const *)(filtd4)); + + for (i = output_height; i > 0; i -= 1) { + // load the 2 strides of source + srcReg32b1 = _mm_loadu_si128((const __m128i *)src_ptr); + + // filter the source buffer + srcRegFilt32b1_1 = _mm_shuffle_epi8(srcReg32b1, filt1Reg); + + // multiply 4 adjacent elements with the filter and add the result + srcRegFilt32b1_1 = _mm_maddubs_epi16(srcRegFilt32b1_1, firstFilters); + + srcRegFilt32b1_1 = _mm_hadds_epi16(srcRegFilt32b1_1, _mm_setzero_si128()); + + // shift by 6 bit each 16 bit + srcRegFilt32b1_1 = _mm_adds_epi16(srcRegFilt32b1_1, addFilterReg32); + srcRegFilt32b1_1 = _mm_srai_epi16(srcRegFilt32b1_1, 6); + + // shrink to 8 bit each 16 bits, the first lane contain the first + // convolve result and the second lane contain the second convolve result + srcRegFilt32b1_1 = _mm_packus_epi16(srcRegFilt32b1_1, _mm_setzero_si128()); + + src_ptr += src_pixels_per_line; + + *((uint32_t *)(output_ptr)) = _mm_cvtsi128_si32(srcRegFilt32b1_1); + output_ptr += output_pitch; + } +} + +static void aom_filter_block1d4_v4_ssse3( + const uint8_t *src_ptr, ptrdiff_t src_pitch, uint8_t *output_ptr, + ptrdiff_t out_pitch, uint32_t output_height, const int16_t *filter) { + __m128i filtersReg; + __m128i addFilterReg32; + __m128i srcReg2, srcReg3, srcReg23, srcReg4, srcReg34, srcReg5, srcReg45, + srcReg6, srcReg56; + __m128i srcReg23_34_lo, srcReg45_56_lo; + __m128i srcReg2345_3456_lo, srcReg2345_3456_hi; + __m128i resReglo, resReghi; + __m128i firstFilters; + unsigned int i; + ptrdiff_t src_stride, dst_stride; + + addFilterReg32 = _mm_set1_epi16(32); + filtersReg = _mm_loadu_si128((const __m128i *)filter); + // converting the 16 bit (short) to 8 bit (byte) and have the + // same data in both lanes of 128 bit register. + filtersReg = _mm_srai_epi16(filtersReg, 1); + filtersReg = _mm_packs_epi16(filtersReg, filtersReg); + + firstFilters = _mm_shuffle_epi8(filtersReg, _mm_set1_epi32(0x5040302u)); + + // multiple the size of the source and destination stride by two + src_stride = src_pitch << 1; + dst_stride = out_pitch << 1; + + srcReg2 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 2)); + srcReg3 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 3)); + srcReg23 = _mm_unpacklo_epi32(srcReg2, srcReg3); + + srcReg4 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 4)); + + // have consecutive loads on the same 256 register + srcReg34 = _mm_unpacklo_epi32(srcReg3, srcReg4); + + srcReg23_34_lo = _mm_unpacklo_epi8(srcReg23, srcReg34); + + for (i = output_height; i > 1; i -= 2) { + srcReg5 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 5)); + srcReg45 = _mm_unpacklo_epi32(srcReg4, srcReg5); + + srcReg6 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 6)); + srcReg56 = _mm_unpacklo_epi32(srcReg5, srcReg6); + + // merge every two consecutive registers + srcReg45_56_lo = _mm_unpacklo_epi8(srcReg45, srcReg56); + + srcReg2345_3456_lo = _mm_unpacklo_epi16(srcReg23_34_lo, srcReg45_56_lo); + srcReg2345_3456_hi = _mm_unpackhi_epi16(srcReg23_34_lo, srcReg45_56_lo); + + // multiply 2 adjacent elements with the filter and add the result + resReglo = _mm_maddubs_epi16(srcReg2345_3456_lo, firstFilters); + resReghi = _mm_maddubs_epi16(srcReg2345_3456_hi, firstFilters); + + resReglo = _mm_hadds_epi16(resReglo, _mm_setzero_si128()); + resReghi = _mm_hadds_epi16(resReghi, _mm_setzero_si128()); + + // shift by 6 bit each 16 bit + resReglo = _mm_adds_epi16(resReglo, addFilterReg32); + resReghi = _mm_adds_epi16(resReghi, addFilterReg32); + resReglo = _mm_srai_epi16(resReglo, 6); + resReghi = _mm_srai_epi16(resReghi, 6); + + // shrink to 8 bit each 16 bits, the first lane contain the first + // convolve result and the second lane contain the second convolve + // result + resReglo = _mm_packus_epi16(resReglo, resReglo); + resReghi = _mm_packus_epi16(resReghi, resReghi); + + src_ptr += src_stride; + + *((uint32_t *)(output_ptr)) = _mm_cvtsi128_si32(resReglo); + *((uint32_t *)(output_ptr + out_pitch)) = _mm_cvtsi128_si32(resReghi); + + output_ptr += dst_stride; + + // save part of the registers for next strides + srcReg23_34_lo = srcReg45_56_lo; + srcReg4 = srcReg6; + } +} + void aom_filter_block1d4_h8_intrin_ssse3( const uint8_t *src_ptr, ptrdiff_t src_pixels_per_line, uint8_t *output_ptr, ptrdiff_t output_pitch, uint32_t output_height, const int16_t *filter) { @@ -118,6 +260,145 @@ void aom_filter_block1d4_h8_intrin_ssse3( } } +static void aom_filter_block1d8_h4_ssse3( + const uint8_t *src_ptr, ptrdiff_t src_pixels_per_line, uint8_t *output_ptr, + ptrdiff_t output_pitch, uint32_t output_height, const int16_t *filter) { + __m128i filtersReg; + __m128i addFilterReg32, filt2Reg, filt3Reg; + __m128i secondFilters, thirdFilters; + __m128i srcRegFilt32b1_1, srcRegFilt32b2, srcRegFilt32b3; + __m128i srcReg32b1; + unsigned int i; + src_ptr -= 3; + addFilterReg32 = _mm_set1_epi16(32); + filtersReg = _mm_loadu_si128((const __m128i *)filter); + filtersReg = _mm_srai_epi16(filtersReg, 1); + // converting the 16 bit (short) to 8 bit (byte) and have the same data + // in both lanes of 128 bit register. + filtersReg = _mm_packs_epi16(filtersReg, filtersReg); + + // duplicate only the second 16 bits (third and forth byte) + // across 256 bit register + secondFilters = _mm_shuffle_epi8(filtersReg, _mm_set1_epi16(0x302u)); + // duplicate only the third 16 bits (fifth and sixth byte) + // across 256 bit register + thirdFilters = _mm_shuffle_epi8(filtersReg, _mm_set1_epi16(0x504u)); + + filt2Reg = _mm_load_si128((__m128i const *)(filt_h4 + 32)); + filt3Reg = _mm_load_si128((__m128i const *)(filt_h4 + 32 * 2)); + + for (i = output_height; i > 0; i -= 1) { + srcReg32b1 = _mm_loadu_si128((const __m128i *)src_ptr); + + // filter the source buffer + srcRegFilt32b3 = _mm_shuffle_epi8(srcReg32b1, filt2Reg); + srcRegFilt32b2 = _mm_shuffle_epi8(srcReg32b1, filt3Reg); + + // multiply 2 adjacent elements with the filter and add the result + srcRegFilt32b3 = _mm_maddubs_epi16(srcRegFilt32b3, secondFilters); + srcRegFilt32b2 = _mm_maddubs_epi16(srcRegFilt32b2, thirdFilters); + + srcRegFilt32b1_1 = _mm_adds_epi16(srcRegFilt32b3, srcRegFilt32b2); + + // shift by 6 bit each 16 bit + srcRegFilt32b1_1 = _mm_adds_epi16(srcRegFilt32b1_1, addFilterReg32); + srcRegFilt32b1_1 = _mm_srai_epi16(srcRegFilt32b1_1, 6); + + // shrink to 8 bit each 16 bits + srcRegFilt32b1_1 = _mm_packus_epi16(srcRegFilt32b1_1, _mm_setzero_si128()); + + src_ptr += src_pixels_per_line; + + _mm_storel_epi64((__m128i *)output_ptr, srcRegFilt32b1_1); + + output_ptr += output_pitch; + } +} + +static void aom_filter_block1d8_v4_ssse3( + const uint8_t *src_ptr, ptrdiff_t src_pitch, uint8_t *output_ptr, + ptrdiff_t out_pitch, uint32_t output_height, const int16_t *filter) { + __m128i filtersReg; + __m128i srcReg2, srcReg3, srcReg4, srcReg5, srcReg6; + __m128i srcReg23, srcReg34, srcReg45, srcReg56; + __m128i resReg23, resReg34, resReg45, resReg56; + __m128i resReg23_45, resReg34_56; + __m128i addFilterReg32, secondFilters, thirdFilters; + unsigned int i; + ptrdiff_t src_stride, dst_stride; + + addFilterReg32 = _mm_set1_epi16(32); + filtersReg = _mm_loadu_si128((const __m128i *)filter); + // converting the 16 bit (short) to 8 bit (byte) and have the + // same data in both lanes of 128 bit register. + filtersReg = _mm_srai_epi16(filtersReg, 1); + filtersReg = _mm_packs_epi16(filtersReg, filtersReg); + + // duplicate only the second 16 bits (third and forth byte) + // across 128 bit register + secondFilters = _mm_shuffle_epi8(filtersReg, _mm_set1_epi16(0x302u)); + // duplicate only the third 16 bits (fifth and sixth byte) + // across 128 bit register + thirdFilters = _mm_shuffle_epi8(filtersReg, _mm_set1_epi16(0x504u)); + + // multiple the size of the source and destination stride by two + src_stride = src_pitch << 1; + dst_stride = out_pitch << 1; + + srcReg2 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 2)); + srcReg3 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 3)); + srcReg23 = _mm_unpacklo_epi8(srcReg2, srcReg3); + + srcReg4 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 4)); + + // have consecutive loads on the same 256 register + srcReg34 = _mm_unpacklo_epi8(srcReg3, srcReg4); + + for (i = output_height; i > 1; i -= 2) { + srcReg5 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 5)); + + srcReg45 = _mm_unpacklo_epi8(srcReg4, srcReg5); + + srcReg6 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 6)); + + srcReg56 = _mm_unpacklo_epi8(srcReg5, srcReg6); + + // multiply 2 adjacent elements with the filter and add the result + resReg23 = _mm_maddubs_epi16(srcReg23, secondFilters); + resReg34 = _mm_maddubs_epi16(srcReg34, secondFilters); + resReg45 = _mm_maddubs_epi16(srcReg45, thirdFilters); + resReg56 = _mm_maddubs_epi16(srcReg56, thirdFilters); + + // add and saturate the results together + resReg23_45 = _mm_adds_epi16(resReg23, resReg45); + resReg34_56 = _mm_adds_epi16(resReg34, resReg56); + + // shift by 6 bit each 16 bit + resReg23_45 = _mm_adds_epi16(resReg23_45, addFilterReg32); + resReg34_56 = _mm_adds_epi16(resReg34_56, addFilterReg32); + resReg23_45 = _mm_srai_epi16(resReg23_45, 6); + resReg34_56 = _mm_srai_epi16(resReg34_56, 6); + + // shrink to 8 bit each 16 bits, the first lane contain the first + // convolve result and the second lane contain the second convolve + // result + resReg23_45 = _mm_packus_epi16(resReg23_45, _mm_setzero_si128()); + resReg34_56 = _mm_packus_epi16(resReg34_56, _mm_setzero_si128()); + + src_ptr += src_stride; + + _mm_storel_epi64((__m128i *)output_ptr, (resReg23_45)); + _mm_storel_epi64((__m128i *)(output_ptr + out_pitch), (resReg34_56)); + + output_ptr += dst_stride; + + // save part of the registers for next strides + srcReg23 = srcReg45; + srcReg34 = srcReg56; + srcReg4 = srcReg6; + } +} + void aom_filter_block1d8_h8_intrin_ssse3( const uint8_t *src_ptr, ptrdiff_t src_pixels_per_line, uint8_t *output_ptr, ptrdiff_t output_pitch, uint32_t output_height, const int16_t *filter) { @@ -280,6 +561,187 @@ void aom_filter_block1d8_v8_intrin_ssse3( } } +static void aom_filter_block1d16_h4_ssse3( + const uint8_t *src_ptr, ptrdiff_t src_pixels_per_line, uint8_t *output_ptr, + ptrdiff_t output_pitch, uint32_t output_height, const int16_t *filter) { + __m128i filtersReg; + __m128i addFilterReg32, filt2Reg, filt3Reg; + __m128i secondFilters, thirdFilters; + __m128i srcRegFilt32b1_1, srcRegFilt32b2_1, srcRegFilt32b2, srcRegFilt32b3; + __m128i srcReg32b1, srcReg32b2; + unsigned int i; + src_ptr -= 3; + addFilterReg32 = _mm_set1_epi16(32); + filtersReg = _mm_loadu_si128((const __m128i *)filter); + filtersReg = _mm_srai_epi16(filtersReg, 1); + // converting the 16 bit (short) to 8 bit (byte) and have the same data + // in both lanes of 128 bit register. + filtersReg = _mm_packs_epi16(filtersReg, filtersReg); + + // duplicate only the second 16 bits (third and forth byte) + // across 256 bit register + secondFilters = _mm_shuffle_epi8(filtersReg, _mm_set1_epi16(0x302u)); + // duplicate only the third 16 bits (fifth and sixth byte) + // across 256 bit register + thirdFilters = _mm_shuffle_epi8(filtersReg, _mm_set1_epi16(0x504u)); + + filt2Reg = _mm_load_si128((__m128i const *)(filt_h4 + 32)); + filt3Reg = _mm_load_si128((__m128i const *)(filt_h4 + 32 * 2)); + + for (i = output_height; i > 0; i -= 1) { + srcReg32b1 = _mm_loadu_si128((const __m128i *)src_ptr); + + // filter the source buffer + srcRegFilt32b3 = _mm_shuffle_epi8(srcReg32b1, filt2Reg); + srcRegFilt32b2 = _mm_shuffle_epi8(srcReg32b1, filt3Reg); + + // multiply 2 adjacent elements with the filter and add the result + srcRegFilt32b3 = _mm_maddubs_epi16(srcRegFilt32b3, secondFilters); + srcRegFilt32b2 = _mm_maddubs_epi16(srcRegFilt32b2, thirdFilters); + + srcRegFilt32b1_1 = _mm_adds_epi16(srcRegFilt32b3, srcRegFilt32b2); + + // reading stride of the next 16 bytes + // (part of it was being read by earlier read) + srcReg32b2 = _mm_loadu_si128((const __m128i *)(src_ptr + 8)); + + // filter the source buffer + srcRegFilt32b3 = _mm_shuffle_epi8(srcReg32b2, filt2Reg); + srcRegFilt32b2 = _mm_shuffle_epi8(srcReg32b2, filt3Reg); + + // multiply 2 adjacent elements with the filter and add the result + srcRegFilt32b3 = _mm_maddubs_epi16(srcRegFilt32b3, secondFilters); + srcRegFilt32b2 = _mm_maddubs_epi16(srcRegFilt32b2, thirdFilters); + + // add and saturate the results together + srcRegFilt32b2_1 = _mm_adds_epi16(srcRegFilt32b3, srcRegFilt32b2); + + // shift by 6 bit each 16 bit + srcRegFilt32b1_1 = _mm_adds_epi16(srcRegFilt32b1_1, addFilterReg32); + srcRegFilt32b2_1 = _mm_adds_epi16(srcRegFilt32b2_1, addFilterReg32); + srcRegFilt32b1_1 = _mm_srai_epi16(srcRegFilt32b1_1, 6); + srcRegFilt32b2_1 = _mm_srai_epi16(srcRegFilt32b2_1, 6); + + // shrink to 8 bit each 16 bits, the first lane contain the first + // convolve result and the second lane contain the second convolve result + srcRegFilt32b1_1 = _mm_packus_epi16(srcRegFilt32b1_1, srcRegFilt32b2_1); + + src_ptr += src_pixels_per_line; + + _mm_store_si128((__m128i *)output_ptr, srcRegFilt32b1_1); + + output_ptr += output_pitch; + } +} + +static void aom_filter_block1d16_v4_ssse3( + const uint8_t *src_ptr, ptrdiff_t src_pitch, uint8_t *output_ptr, + ptrdiff_t out_pitch, uint32_t output_height, const int16_t *filter) { + __m128i filtersReg; + __m128i srcReg2, srcReg3, srcReg4, srcReg5, srcReg6; + __m128i srcReg23_lo, srcReg23_hi, srcReg34_lo, srcReg34_hi; + __m128i srcReg45_lo, srcReg45_hi, srcReg56_lo, srcReg56_hi; + __m128i resReg23_lo, resReg34_lo, resReg45_lo, resReg56_lo; + __m128i resReg23_hi, resReg34_hi, resReg45_hi, resReg56_hi; + __m128i resReg23_45_lo, resReg34_56_lo, resReg23_45_hi, resReg34_56_hi; + __m128i resReg23_45, resReg34_56; + __m128i addFilterReg32, secondFilters, thirdFilters; + unsigned int i; + ptrdiff_t src_stride, dst_stride; + + addFilterReg32 = _mm_set1_epi16(32); + filtersReg = _mm_loadu_si128((const __m128i *)filter); + // converting the 16 bit (short) to 8 bit (byte) and have the + // same data in both lanes of 128 bit register. + filtersReg = _mm_srai_epi16(filtersReg, 1); + filtersReg = _mm_packs_epi16(filtersReg, filtersReg); + + // duplicate only the second 16 bits (third and forth byte) + // across 128 bit register + secondFilters = _mm_shuffle_epi8(filtersReg, _mm_set1_epi16(0x302u)); + // duplicate only the third 16 bits (fifth and sixth byte) + // across 128 bit register + thirdFilters = _mm_shuffle_epi8(filtersReg, _mm_set1_epi16(0x504u)); + + // multiple the size of the source and destination stride by two + src_stride = src_pitch << 1; + dst_stride = out_pitch << 1; + + srcReg2 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 2)); + srcReg3 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 3)); + srcReg23_lo = _mm_unpacklo_epi8(srcReg2, srcReg3); + srcReg23_hi = _mm_unpackhi_epi8(srcReg2, srcReg3); + + srcReg4 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 4)); + + // have consecutive loads on the same 256 register + srcReg34_lo = _mm_unpacklo_epi8(srcReg3, srcReg4); + srcReg34_hi = _mm_unpackhi_epi8(srcReg3, srcReg4); + + for (i = output_height; i > 1; i -= 2) { + srcReg5 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 5)); + + srcReg45_lo = _mm_unpacklo_epi8(srcReg4, srcReg5); + srcReg45_hi = _mm_unpackhi_epi8(srcReg4, srcReg5); + + srcReg6 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 6)); + + srcReg56_lo = _mm_unpacklo_epi8(srcReg5, srcReg6); + srcReg56_hi = _mm_unpackhi_epi8(srcReg5, srcReg6); + + // multiply 2 adjacent elements with the filter and add the result + resReg23_lo = _mm_maddubs_epi16(srcReg23_lo, secondFilters); + resReg34_lo = _mm_maddubs_epi16(srcReg34_lo, secondFilters); + resReg45_lo = _mm_maddubs_epi16(srcReg45_lo, thirdFilters); + resReg56_lo = _mm_maddubs_epi16(srcReg56_lo, thirdFilters); + + // add and saturate the results together + resReg23_45_lo = _mm_adds_epi16(resReg23_lo, resReg45_lo); + resReg34_56_lo = _mm_adds_epi16(resReg34_lo, resReg56_lo); + + // multiply 2 adjacent elements with the filter and add the result + + resReg23_hi = _mm_maddubs_epi16(srcReg23_hi, secondFilters); + resReg34_hi = _mm_maddubs_epi16(srcReg34_hi, secondFilters); + resReg45_hi = _mm_maddubs_epi16(srcReg45_hi, thirdFilters); + resReg56_hi = _mm_maddubs_epi16(srcReg56_hi, thirdFilters); + + // add and saturate the results together + resReg23_45_hi = _mm_adds_epi16(resReg23_hi, resReg45_hi); + resReg34_56_hi = _mm_adds_epi16(resReg34_hi, resReg56_hi); + + // shift by 6 bit each 16 bit + resReg23_45_lo = _mm_adds_epi16(resReg23_45_lo, addFilterReg32); + resReg34_56_lo = _mm_adds_epi16(resReg34_56_lo, addFilterReg32); + resReg23_45_hi = _mm_adds_epi16(resReg23_45_hi, addFilterReg32); + resReg34_56_hi = _mm_adds_epi16(resReg34_56_hi, addFilterReg32); + resReg23_45_lo = _mm_srai_epi16(resReg23_45_lo, 6); + resReg34_56_lo = _mm_srai_epi16(resReg34_56_lo, 6); + resReg23_45_hi = _mm_srai_epi16(resReg23_45_hi, 6); + resReg34_56_hi = _mm_srai_epi16(resReg34_56_hi, 6); + + // shrink to 8 bit each 16 bits, the first lane contain the first + // convolve result and the second lane contain the second convolve + // result + resReg23_45 = _mm_packus_epi16(resReg23_45_lo, resReg23_45_hi); + resReg34_56 = _mm_packus_epi16(resReg34_56_lo, resReg34_56_hi); + + src_ptr += src_stride; + + _mm_store_si128((__m128i *)output_ptr, (resReg23_45)); + _mm_store_si128((__m128i *)(output_ptr + out_pitch), (resReg34_56)); + + output_ptr += dst_stride; + + // save part of the registers for next strides + srcReg23_lo = srcReg45_lo; + srcReg34_lo = srcReg56_lo; + srcReg23_hi = srcReg45_hi; + srcReg34_hi = srcReg56_hi; + srcReg4 = srcReg6; + } +} + filter8_1dfunction aom_filter_block1d16_v8_ssse3; filter8_1dfunction aom_filter_block1d16_h8_ssse3; filter8_1dfunction aom_filter_block1d8_v8_ssse3; @@ -287,13 +749,6 @@ filter8_1dfunction aom_filter_block1d8_h8_ssse3; filter8_1dfunction aom_filter_block1d4_v8_ssse3; filter8_1dfunction aom_filter_block1d4_h8_ssse3; -#define aom_filter_block1d16_h4_ssse3 aom_filter_block1d16_h8_ssse3 -#define aom_filter_block1d16_v4_ssse3 aom_filter_block1d16_v8_ssse3 -#define aom_filter_block1d8_h4_ssse3 aom_filter_block1d8_h8_ssse3 -#define aom_filter_block1d8_v4_ssse3 aom_filter_block1d8_v8_ssse3 -#define aom_filter_block1d4_h4_ssse3 aom_filter_block1d4_h8_ssse3 -#define aom_filter_block1d4_v4_ssse3 aom_filter_block1d4_v8_ssse3 - filter8_1dfunction aom_filter_block1d16_v2_ssse3; filter8_1dfunction aom_filter_block1d16_h2_ssse3; filter8_1dfunction aom_filter_block1d8_v2_ssse3; diff --git a/third_party/aom/aom_dsp/x86/avg_intrin_avx2.c b/third_party/aom/aom_dsp/x86/avg_intrin_avx2.c new file mode 100644 index 0000000000..e0ba8d5de6 --- /dev/null +++ b/third_party/aom/aom_dsp/x86/avg_intrin_avx2.c @@ -0,0 +1,249 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "config/aom_dsp_rtcd.h" +#include "aom/aom_integer.h" +#include "aom_dsp/x86/bitdepth_conversion_avx2.h" +#include "aom_ports/mem.h" + +static void hadamard_col8x2_avx2(__m256i *in, int iter) { + __m256i a0 = in[0]; + __m256i a1 = in[1]; + __m256i a2 = in[2]; + __m256i a3 = in[3]; + __m256i a4 = in[4]; + __m256i a5 = in[5]; + __m256i a6 = in[6]; + __m256i a7 = in[7]; + + __m256i b0 = _mm256_add_epi16(a0, a1); + __m256i b1 = _mm256_sub_epi16(a0, a1); + __m256i b2 = _mm256_add_epi16(a2, a3); + __m256i b3 = _mm256_sub_epi16(a2, a3); + __m256i b4 = _mm256_add_epi16(a4, a5); + __m256i b5 = _mm256_sub_epi16(a4, a5); + __m256i b6 = _mm256_add_epi16(a6, a7); + __m256i b7 = _mm256_sub_epi16(a6, a7); + + a0 = _mm256_add_epi16(b0, b2); + a1 = _mm256_add_epi16(b1, b3); + a2 = _mm256_sub_epi16(b0, b2); + a3 = _mm256_sub_epi16(b1, b3); + a4 = _mm256_add_epi16(b4, b6); + a5 = _mm256_add_epi16(b5, b7); + a6 = _mm256_sub_epi16(b4, b6); + a7 = _mm256_sub_epi16(b5, b7); + + if (iter == 0) { + b0 = _mm256_add_epi16(a0, a4); + b7 = _mm256_add_epi16(a1, a5); + b3 = _mm256_add_epi16(a2, a6); + b4 = _mm256_add_epi16(a3, a7); + b2 = _mm256_sub_epi16(a0, a4); + b6 = _mm256_sub_epi16(a1, a5); + b1 = _mm256_sub_epi16(a2, a6); + b5 = _mm256_sub_epi16(a3, a7); + + a0 = _mm256_unpacklo_epi16(b0, b1); + a1 = _mm256_unpacklo_epi16(b2, b3); + a2 = _mm256_unpackhi_epi16(b0, b1); + a3 = _mm256_unpackhi_epi16(b2, b3); + a4 = _mm256_unpacklo_epi16(b4, b5); + a5 = _mm256_unpacklo_epi16(b6, b7); + a6 = _mm256_unpackhi_epi16(b4, b5); + a7 = _mm256_unpackhi_epi16(b6, b7); + + b0 = _mm256_unpacklo_epi32(a0, a1); + b1 = _mm256_unpacklo_epi32(a4, a5); + b2 = _mm256_unpackhi_epi32(a0, a1); + b3 = _mm256_unpackhi_epi32(a4, a5); + b4 = _mm256_unpacklo_epi32(a2, a3); + b5 = _mm256_unpacklo_epi32(a6, a7); + b6 = _mm256_unpackhi_epi32(a2, a3); + b7 = _mm256_unpackhi_epi32(a6, a7); + + in[0] = _mm256_unpacklo_epi64(b0, b1); + in[1] = _mm256_unpackhi_epi64(b0, b1); + in[2] = _mm256_unpacklo_epi64(b2, b3); + in[3] = _mm256_unpackhi_epi64(b2, b3); + in[4] = _mm256_unpacklo_epi64(b4, b5); + in[5] = _mm256_unpackhi_epi64(b4, b5); + in[6] = _mm256_unpacklo_epi64(b6, b7); + in[7] = _mm256_unpackhi_epi64(b6, b7); + } else { + in[0] = _mm256_add_epi16(a0, a4); + in[7] = _mm256_add_epi16(a1, a5); + in[3] = _mm256_add_epi16(a2, a6); + in[4] = _mm256_add_epi16(a3, a7); + in[2] = _mm256_sub_epi16(a0, a4); + in[6] = _mm256_sub_epi16(a1, a5); + in[1] = _mm256_sub_epi16(a2, a6); + in[5] = _mm256_sub_epi16(a3, a7); + } +} + +static void hadamard_8x8x2_avx2(const int16_t *src_diff, ptrdiff_t src_stride, + int16_t *coeff) { + __m256i src[8]; + src[0] = _mm256_loadu_si256((const __m256i *)src_diff); + src[1] = _mm256_loadu_si256((const __m256i *)(src_diff += src_stride)); + src[2] = _mm256_loadu_si256((const __m256i *)(src_diff += src_stride)); + src[3] = _mm256_loadu_si256((const __m256i *)(src_diff += src_stride)); + src[4] = _mm256_loadu_si256((const __m256i *)(src_diff += src_stride)); + src[5] = _mm256_loadu_si256((const __m256i *)(src_diff += src_stride)); + src[6] = _mm256_loadu_si256((const __m256i *)(src_diff += src_stride)); + src[7] = _mm256_loadu_si256((const __m256i *)(src_diff += src_stride)); + + hadamard_col8x2_avx2(src, 0); + hadamard_col8x2_avx2(src, 1); + + _mm256_storeu_si256((__m256i *)coeff, + _mm256_permute2x128_si256(src[0], src[1], 0x20)); + coeff += 16; + _mm256_storeu_si256((__m256i *)coeff, + _mm256_permute2x128_si256(src[2], src[3], 0x20)); + coeff += 16; + _mm256_storeu_si256((__m256i *)coeff, + _mm256_permute2x128_si256(src[4], src[5], 0x20)); + coeff += 16; + _mm256_storeu_si256((__m256i *)coeff, + _mm256_permute2x128_si256(src[6], src[7], 0x20)); + coeff += 16; + _mm256_storeu_si256((__m256i *)coeff, + _mm256_permute2x128_si256(src[0], src[1], 0x31)); + coeff += 16; + _mm256_storeu_si256((__m256i *)coeff, + _mm256_permute2x128_si256(src[2], src[3], 0x31)); + coeff += 16; + _mm256_storeu_si256((__m256i *)coeff, + _mm256_permute2x128_si256(src[4], src[5], 0x31)); + coeff += 16; + _mm256_storeu_si256((__m256i *)coeff, + _mm256_permute2x128_si256(src[6], src[7], 0x31)); +} + +static INLINE void hadamard_16x16_avx2(const int16_t *src_diff, + ptrdiff_t src_stride, tran_low_t *coeff, + int is_final) { + DECLARE_ALIGNED(32, int16_t, temp_coeff[16 * 16]); + int16_t *t_coeff = temp_coeff; + int16_t *coeff16 = (int16_t *)coeff; + int idx; + for (idx = 0; idx < 2; ++idx) { + const int16_t *src_ptr = src_diff + idx * 8 * src_stride; + hadamard_8x8x2_avx2(src_ptr, src_stride, t_coeff + (idx * 64 * 2)); + } + + for (idx = 0; idx < 64; idx += 16) { + const __m256i coeff0 = _mm256_loadu_si256((const __m256i *)t_coeff); + const __m256i coeff1 = _mm256_loadu_si256((const __m256i *)(t_coeff + 64)); + const __m256i coeff2 = _mm256_loadu_si256((const __m256i *)(t_coeff + 128)); + const __m256i coeff3 = _mm256_loadu_si256((const __m256i *)(t_coeff + 192)); + + __m256i b0 = _mm256_add_epi16(coeff0, coeff1); + __m256i b1 = _mm256_sub_epi16(coeff0, coeff1); + __m256i b2 = _mm256_add_epi16(coeff2, coeff3); + __m256i b3 = _mm256_sub_epi16(coeff2, coeff3); + + b0 = _mm256_srai_epi16(b0, 1); + b1 = _mm256_srai_epi16(b1, 1); + b2 = _mm256_srai_epi16(b2, 1); + b3 = _mm256_srai_epi16(b3, 1); + if (is_final) { + store_tran_low(_mm256_add_epi16(b0, b2), coeff); + store_tran_low(_mm256_add_epi16(b1, b3), coeff + 64); + store_tran_low(_mm256_sub_epi16(b0, b2), coeff + 128); + store_tran_low(_mm256_sub_epi16(b1, b3), coeff + 192); + coeff += 16; + } else { + _mm256_storeu_si256((__m256i *)coeff16, _mm256_add_epi16(b0, b2)); + _mm256_storeu_si256((__m256i *)(coeff16 + 64), _mm256_add_epi16(b1, b3)); + _mm256_storeu_si256((__m256i *)(coeff16 + 128), _mm256_sub_epi16(b0, b2)); + _mm256_storeu_si256((__m256i *)(coeff16 + 192), _mm256_sub_epi16(b1, b3)); + coeff16 += 16; + } + t_coeff += 16; + } +} + +void aom_hadamard_16x16_avx2(const int16_t *src_diff, ptrdiff_t src_stride, + tran_low_t *coeff) { + hadamard_16x16_avx2(src_diff, src_stride, coeff, 1); +} + +void aom_hadamard_32x32_avx2(const int16_t *src_diff, ptrdiff_t src_stride, + tran_low_t *coeff) { + // For high bitdepths, it is unnecessary to store_tran_low + // (mult/unpack/store), then load_tran_low (load/pack) the same memory in the + // next stage. Output to an intermediate buffer first, then store_tran_low() + // in the final stage. + DECLARE_ALIGNED(32, int16_t, temp_coeff[32 * 32]); + int16_t *t_coeff = temp_coeff; + int idx; + for (idx = 0; idx < 4; ++idx) { + // src_diff: 9 bit, dynamic range [-255, 255] + const int16_t *src_ptr = + src_diff + (idx >> 1) * 16 * src_stride + (idx & 0x01) * 16; + hadamard_16x16_avx2(src_ptr, src_stride, + (tran_low_t *)(t_coeff + idx * 256), 0); + } + + for (idx = 0; idx < 256; idx += 16) { + const __m256i coeff0 = _mm256_loadu_si256((const __m256i *)t_coeff); + const __m256i coeff1 = _mm256_loadu_si256((const __m256i *)(t_coeff + 256)); + const __m256i coeff2 = _mm256_loadu_si256((const __m256i *)(t_coeff + 512)); + const __m256i coeff3 = _mm256_loadu_si256((const __m256i *)(t_coeff + 768)); + + __m256i b0 = _mm256_add_epi16(coeff0, coeff1); + __m256i b1 = _mm256_sub_epi16(coeff0, coeff1); + __m256i b2 = _mm256_add_epi16(coeff2, coeff3); + __m256i b3 = _mm256_sub_epi16(coeff2, coeff3); + + b0 = _mm256_srai_epi16(b0, 2); + b1 = _mm256_srai_epi16(b1, 2); + b2 = _mm256_srai_epi16(b2, 2); + b3 = _mm256_srai_epi16(b3, 2); + + store_tran_low(_mm256_add_epi16(b0, b2), coeff); + store_tran_low(_mm256_add_epi16(b1, b3), coeff + 256); + store_tran_low(_mm256_sub_epi16(b0, b2), coeff + 512); + store_tran_low(_mm256_sub_epi16(b1, b3), coeff + 768); + + coeff += 16; + t_coeff += 16; + } +} + +int aom_satd_avx2(const tran_low_t *coeff, int length) { + const __m256i one = _mm256_set1_epi16(1); + __m256i accum = _mm256_setzero_si256(); + int i; + + for (i = 0; i < length; i += 16) { + const __m256i src_line = load_tran_low(coeff); + const __m256i abs = _mm256_abs_epi16(src_line); + const __m256i sum = _mm256_madd_epi16(abs, one); + accum = _mm256_add_epi32(accum, sum); + coeff += 16; + } + + { // 32 bit horizontal add + const __m256i a = _mm256_srli_si256(accum, 8); + const __m256i b = _mm256_add_epi32(accum, a); + const __m256i c = _mm256_srli_epi64(b, 32); + const __m256i d = _mm256_add_epi32(b, c); + const __m128i accum_128 = _mm_add_epi32(_mm256_castsi256_si128(d), + _mm256_extractf128_si256(d, 1)); + return _mm_cvtsi128_si32(accum_128); + } +} diff --git a/third_party/aom/aom_dsp/x86/avg_intrin_sse2.c b/third_party/aom/aom_dsp/x86/avg_intrin_sse2.c new file mode 100644 index 0000000000..969e4e195d --- /dev/null +++ b/third_party/aom/aom_dsp/x86/avg_intrin_sse2.c @@ -0,0 +1,285 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "config/aom_dsp_rtcd.h" +#include "aom/aom_integer.h" +#include "aom_dsp/x86/bitdepth_conversion_sse2.h" +#include "aom_ports/mem.h" + +static void hadamard_col8_sse2(__m128i *in, int iter) { + __m128i a0 = in[0]; + __m128i a1 = in[1]; + __m128i a2 = in[2]; + __m128i a3 = in[3]; + __m128i a4 = in[4]; + __m128i a5 = in[5]; + __m128i a6 = in[6]; + __m128i a7 = in[7]; + + __m128i b0 = _mm_add_epi16(a0, a1); + __m128i b1 = _mm_sub_epi16(a0, a1); + __m128i b2 = _mm_add_epi16(a2, a3); + __m128i b3 = _mm_sub_epi16(a2, a3); + __m128i b4 = _mm_add_epi16(a4, a5); + __m128i b5 = _mm_sub_epi16(a4, a5); + __m128i b6 = _mm_add_epi16(a6, a7); + __m128i b7 = _mm_sub_epi16(a6, a7); + + a0 = _mm_add_epi16(b0, b2); + a1 = _mm_add_epi16(b1, b3); + a2 = _mm_sub_epi16(b0, b2); + a3 = _mm_sub_epi16(b1, b3); + a4 = _mm_add_epi16(b4, b6); + a5 = _mm_add_epi16(b5, b7); + a6 = _mm_sub_epi16(b4, b6); + a7 = _mm_sub_epi16(b5, b7); + + if (iter == 0) { + b0 = _mm_add_epi16(a0, a4); + b7 = _mm_add_epi16(a1, a5); + b3 = _mm_add_epi16(a2, a6); + b4 = _mm_add_epi16(a3, a7); + b2 = _mm_sub_epi16(a0, a4); + b6 = _mm_sub_epi16(a1, a5); + b1 = _mm_sub_epi16(a2, a6); + b5 = _mm_sub_epi16(a3, a7); + + a0 = _mm_unpacklo_epi16(b0, b1); + a1 = _mm_unpacklo_epi16(b2, b3); + a2 = _mm_unpackhi_epi16(b0, b1); + a3 = _mm_unpackhi_epi16(b2, b3); + a4 = _mm_unpacklo_epi16(b4, b5); + a5 = _mm_unpacklo_epi16(b6, b7); + a6 = _mm_unpackhi_epi16(b4, b5); + a7 = _mm_unpackhi_epi16(b6, b7); + + b0 = _mm_unpacklo_epi32(a0, a1); + b1 = _mm_unpacklo_epi32(a4, a5); + b2 = _mm_unpackhi_epi32(a0, a1); + b3 = _mm_unpackhi_epi32(a4, a5); + b4 = _mm_unpacklo_epi32(a2, a3); + b5 = _mm_unpacklo_epi32(a6, a7); + b6 = _mm_unpackhi_epi32(a2, a3); + b7 = _mm_unpackhi_epi32(a6, a7); + + in[0] = _mm_unpacklo_epi64(b0, b1); + in[1] = _mm_unpackhi_epi64(b0, b1); + in[2] = _mm_unpacklo_epi64(b2, b3); + in[3] = _mm_unpackhi_epi64(b2, b3); + in[4] = _mm_unpacklo_epi64(b4, b5); + in[5] = _mm_unpackhi_epi64(b4, b5); + in[6] = _mm_unpacklo_epi64(b6, b7); + in[7] = _mm_unpackhi_epi64(b6, b7); + } else { + in[0] = _mm_add_epi16(a0, a4); + in[7] = _mm_add_epi16(a1, a5); + in[3] = _mm_add_epi16(a2, a6); + in[4] = _mm_add_epi16(a3, a7); + in[2] = _mm_sub_epi16(a0, a4); + in[6] = _mm_sub_epi16(a1, a5); + in[1] = _mm_sub_epi16(a2, a6); + in[5] = _mm_sub_epi16(a3, a7); + } +} + +static INLINE void hadamard_8x8_sse2(const int16_t *src_diff, + ptrdiff_t src_stride, tran_low_t *coeff, + int is_final) { + __m128i src[8]; + src[0] = _mm_load_si128((const __m128i *)src_diff); + src[1] = _mm_load_si128((const __m128i *)(src_diff += src_stride)); + src[2] = _mm_load_si128((const __m128i *)(src_diff += src_stride)); + src[3] = _mm_load_si128((const __m128i *)(src_diff += src_stride)); + src[4] = _mm_load_si128((const __m128i *)(src_diff += src_stride)); + src[5] = _mm_load_si128((const __m128i *)(src_diff += src_stride)); + src[6] = _mm_load_si128((const __m128i *)(src_diff += src_stride)); + src[7] = _mm_load_si128((const __m128i *)(src_diff += src_stride)); + + hadamard_col8_sse2(src, 0); + hadamard_col8_sse2(src, 1); + + if (is_final) { + store_tran_low(src[0], coeff); + coeff += 8; + store_tran_low(src[1], coeff); + coeff += 8; + store_tran_low(src[2], coeff); + coeff += 8; + store_tran_low(src[3], coeff); + coeff += 8; + store_tran_low(src[4], coeff); + coeff += 8; + store_tran_low(src[5], coeff); + coeff += 8; + store_tran_low(src[6], coeff); + coeff += 8; + store_tran_low(src[7], coeff); + } else { + int16_t *coeff16 = (int16_t *)coeff; + _mm_store_si128((__m128i *)coeff16, src[0]); + coeff16 += 8; + _mm_store_si128((__m128i *)coeff16, src[1]); + coeff16 += 8; + _mm_store_si128((__m128i *)coeff16, src[2]); + coeff16 += 8; + _mm_store_si128((__m128i *)coeff16, src[3]); + coeff16 += 8; + _mm_store_si128((__m128i *)coeff16, src[4]); + coeff16 += 8; + _mm_store_si128((__m128i *)coeff16, src[5]); + coeff16 += 8; + _mm_store_si128((__m128i *)coeff16, src[6]); + coeff16 += 8; + _mm_store_si128((__m128i *)coeff16, src[7]); + } +} + +void aom_hadamard_8x8_sse2(const int16_t *src_diff, ptrdiff_t src_stride, + tran_low_t *coeff) { + hadamard_8x8_sse2(src_diff, src_stride, coeff, 1); +} + +static INLINE void hadamard_16x16_sse2(const int16_t *src_diff, + ptrdiff_t src_stride, tran_low_t *coeff, + int is_final) { + // For high bitdepths, it is unnecessary to store_tran_low + // (mult/unpack/store), then load_tran_low (load/pack) the same memory in the + // next stage. Output to an intermediate buffer first, then store_tran_low() + // in the final stage. + DECLARE_ALIGNED(32, int16_t, temp_coeff[16 * 16]); + int16_t *t_coeff = temp_coeff; + int16_t *coeff16 = (int16_t *)coeff; + int idx; + for (idx = 0; idx < 4; ++idx) { + const int16_t *src_ptr = + src_diff + (idx >> 1) * 8 * src_stride + (idx & 0x01) * 8; + hadamard_8x8_sse2(src_ptr, src_stride, (tran_low_t *)(t_coeff + idx * 64), + 0); + } + + for (idx = 0; idx < 64; idx += 8) { + __m128i coeff0 = _mm_load_si128((const __m128i *)t_coeff); + __m128i coeff1 = _mm_load_si128((const __m128i *)(t_coeff + 64)); + __m128i coeff2 = _mm_load_si128((const __m128i *)(t_coeff + 128)); + __m128i coeff3 = _mm_load_si128((const __m128i *)(t_coeff + 192)); + + __m128i b0 = _mm_add_epi16(coeff0, coeff1); + __m128i b1 = _mm_sub_epi16(coeff0, coeff1); + __m128i b2 = _mm_add_epi16(coeff2, coeff3); + __m128i b3 = _mm_sub_epi16(coeff2, coeff3); + + b0 = _mm_srai_epi16(b0, 1); + b1 = _mm_srai_epi16(b1, 1); + b2 = _mm_srai_epi16(b2, 1); + b3 = _mm_srai_epi16(b3, 1); + + coeff0 = _mm_add_epi16(b0, b2); + coeff1 = _mm_add_epi16(b1, b3); + coeff2 = _mm_sub_epi16(b0, b2); + coeff3 = _mm_sub_epi16(b1, b3); + + if (is_final) { + store_tran_low(coeff0, coeff); + store_tran_low(coeff1, coeff + 64); + store_tran_low(coeff2, coeff + 128); + store_tran_low(coeff3, coeff + 192); + coeff += 8; + } else { + _mm_store_si128((__m128i *)coeff16, coeff0); + _mm_store_si128((__m128i *)(coeff16 + 64), coeff1); + _mm_store_si128((__m128i *)(coeff16 + 128), coeff2); + _mm_store_si128((__m128i *)(coeff16 + 192), coeff3); + coeff16 += 8; + } + + t_coeff += 8; + } +} + +void aom_hadamard_16x16_sse2(const int16_t *src_diff, ptrdiff_t src_stride, + tran_low_t *coeff) { + hadamard_16x16_sse2(src_diff, src_stride, coeff, 1); +} + +void aom_hadamard_32x32_sse2(const int16_t *src_diff, ptrdiff_t src_stride, + tran_low_t *coeff) { + // For high bitdepths, it is unnecessary to store_tran_low + // (mult/unpack/store), then load_tran_low (load/pack) the same memory in the + // next stage. Output to an intermediate buffer first, then store_tran_low() + // in the final stage. + DECLARE_ALIGNED(32, int16_t, temp_coeff[32 * 32]); + int16_t *t_coeff = temp_coeff; + int idx; + for (idx = 0; idx < 4; ++idx) { + const int16_t *src_ptr = + src_diff + (idx >> 1) * 16 * src_stride + (idx & 0x01) * 16; + hadamard_16x16_sse2(src_ptr, src_stride, + (tran_low_t *)(t_coeff + idx * 256), 0); + } + + for (idx = 0; idx < 256; idx += 8) { + __m128i coeff0 = _mm_load_si128((const __m128i *)t_coeff); + __m128i coeff1 = _mm_load_si128((const __m128i *)(t_coeff + 256)); + __m128i coeff2 = _mm_load_si128((const __m128i *)(t_coeff + 512)); + __m128i coeff3 = _mm_load_si128((const __m128i *)(t_coeff + 768)); + + __m128i b0 = _mm_add_epi16(coeff0, coeff1); + __m128i b1 = _mm_sub_epi16(coeff0, coeff1); + __m128i b2 = _mm_add_epi16(coeff2, coeff3); + __m128i b3 = _mm_sub_epi16(coeff2, coeff3); + + b0 = _mm_srai_epi16(b0, 2); + b1 = _mm_srai_epi16(b1, 2); + b2 = _mm_srai_epi16(b2, 2); + b3 = _mm_srai_epi16(b3, 2); + + coeff0 = _mm_add_epi16(b0, b2); + coeff1 = _mm_add_epi16(b1, b3); + store_tran_low(coeff0, coeff); + store_tran_low(coeff1, coeff + 256); + + coeff2 = _mm_sub_epi16(b0, b2); + coeff3 = _mm_sub_epi16(b1, b3); + store_tran_low(coeff2, coeff + 512); + store_tran_low(coeff3, coeff + 768); + + coeff += 8; + t_coeff += 8; + } +} + +int aom_satd_sse2(const tran_low_t *coeff, int length) { + int i; + const __m128i zero = _mm_setzero_si128(); + __m128i accum = zero; + + for (i = 0; i < length; i += 8) { + const __m128i src_line = load_tran_low(coeff); + const __m128i inv = _mm_sub_epi16(zero, src_line); + const __m128i abs = _mm_max_epi16(src_line, inv); // abs(src_line) + const __m128i abs_lo = _mm_unpacklo_epi16(abs, zero); + const __m128i abs_hi = _mm_unpackhi_epi16(abs, zero); + const __m128i sum = _mm_add_epi32(abs_lo, abs_hi); + accum = _mm_add_epi32(accum, sum); + coeff += 8; + } + + { // cascading summation of accum + __m128i hi = _mm_srli_si128(accum, 8); + accum = _mm_add_epi32(accum, hi); + hi = _mm_srli_epi64(accum, 32); + accum = _mm_add_epi32(accum, hi); + } + + return _mm_cvtsi128_si32(accum); +} diff --git a/third_party/aom/aom_dsp/x86/bitdepth_conversion_avx2.h b/third_party/aom/aom_dsp/x86/bitdepth_conversion_avx2.h new file mode 100644 index 0000000000..85896e2768 --- /dev/null +++ b/third_party/aom/aom_dsp/x86/bitdepth_conversion_avx2.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "config/aom_config.h" +#include "aom/aom_integer.h" +#include "aom_dsp/aom_dsp_common.h" + +static INLINE __m256i load_tran_low(const tran_low_t *a) { + const __m256i a_low = _mm256_loadu_si256((const __m256i *)a); + const __m256i a_high = _mm256_loadu_si256((const __m256i *)(a + 8)); + return _mm256_packs_epi32(a_low, a_high); +} + +static INLINE void store_tran_low(__m256i a, tran_low_t *b) { + const __m256i one = _mm256_set1_epi16(1); + const __m256i a_hi = _mm256_mulhi_epi16(a, one); + const __m256i a_lo = _mm256_mullo_epi16(a, one); + const __m256i a_1 = _mm256_unpacklo_epi16(a_lo, a_hi); + const __m256i a_2 = _mm256_unpackhi_epi16(a_lo, a_hi); + _mm256_storeu_si256((__m256i *)b, a_1); + _mm256_storeu_si256((__m256i *)(b + 8), a_2); +} diff --git a/third_party/aom/aom_dsp/x86/bitdepth_conversion_sse2.h b/third_party/aom/aom_dsp/x86/bitdepth_conversion_sse2.h new file mode 100644 index 0000000000..42bb2d1d32 --- /dev/null +++ b/third_party/aom/aom_dsp/x86/bitdepth_conversion_sse2.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "config/aom_config.h" +#include "aom/aom_integer.h" +#include "aom_dsp/aom_dsp_common.h" + +// Load 8 16 bit values. If the source is 32 bits then pack down with +// saturation. +static INLINE __m128i load_tran_low(const tran_low_t *a) { + const __m128i a_low = _mm_load_si128((const __m128i *)a); + return _mm_packs_epi32(a_low, *(const __m128i *)(a + 4)); +} + +// Store 8 16 bit values. If the destination is 32 bits then sign extend the +// values by multiplying by 1. +static INLINE void store_tran_low(__m128i a, tran_low_t *b) { + const __m128i one = _mm_set1_epi16(1); + const __m128i a_hi = _mm_mulhi_epi16(a, one); + const __m128i a_lo = _mm_mullo_epi16(a, one); + const __m128i a_1 = _mm_unpacklo_epi16(a_lo, a_hi); + const __m128i a_2 = _mm_unpackhi_epi16(a_lo, a_hi); + _mm_store_si128((__m128i *)(b), a_1); + _mm_store_si128((__m128i *)(b + 4), a_2); +} diff --git a/third_party/aom/aom_dsp/x86/blend_a64_mask_avx2.c b/third_party/aom/aom_dsp/x86/blend_a64_mask_avx2.c index 67fb4d32bd..057f615c20 100644 --- a/third_party/aom/aom_dsp/x86/blend_a64_mask_avx2.c +++ b/third_party/aom/aom_dsp/x86/blend_a64_mask_avx2.c @@ -898,3 +898,475 @@ void aom_blend_a64_mask_avx2(uint8_t *dst, uint32_t dst_stride, } } } + +////////////////////////////////////////////////////////////////////////////// +// aom_highbd_blend_a64_d16_mask_avx2() +////////////////////////////////////////////////////////////////////////////// + +static INLINE void highbd_blend_a64_d16_mask_w4_avx2( + uint16_t *dst, int dst_stride, const CONV_BUF_TYPE *src0, int src0_stride, + const CONV_BUF_TYPE *src1, int src1_stride, const __m256i *mask0, + const __m256i *round_offset, int shift, const __m256i *clip_low, + const __m256i *clip_high, const __m256i *mask_max) { + // Load 4x u16 pixels from each of 4 rows from each source + const __m256i s0 = _mm256_set_epi64x(*(uint64_t *)(src0 + 3 * src0_stride), + *(uint64_t *)(src0 + 2 * src0_stride), + *(uint64_t *)(src0 + 1 * src0_stride), + *(uint64_t *)(src0 + 0 * src0_stride)); + const __m256i s1 = _mm256_set_epi64x(*(uint64_t *)(src1 + 3 * src1_stride), + *(uint64_t *)(src1 + 2 * src1_stride), + *(uint64_t *)(src1 + 1 * src1_stride), + *(uint64_t *)(src1 + 0 * src1_stride)); + // Generate the inverse mask + const __m256i mask1 = _mm256_sub_epi16(*mask_max, *mask0); + + // Multiply each mask by the respective source + const __m256i mul0_highs = _mm256_mulhi_epu16(*mask0, s0); + const __m256i mul0_lows = _mm256_mullo_epi16(*mask0, s0); + const __m256i mul0h = _mm256_unpackhi_epi16(mul0_lows, mul0_highs); + const __m256i mul0l = _mm256_unpacklo_epi16(mul0_lows, mul0_highs); + // Note that AVX2 unpack orders 64-bit words as [3 1] [2 0] to keep within + // lanes Later, packs does the same again which cancels this out with no need + // for a permute. The intermediate values being reordered makes no difference + + const __m256i mul1_highs = _mm256_mulhi_epu16(mask1, s1); + const __m256i mul1_lows = _mm256_mullo_epi16(mask1, s1); + const __m256i mul1h = _mm256_unpackhi_epi16(mul1_lows, mul1_highs); + const __m256i mul1l = _mm256_unpacklo_epi16(mul1_lows, mul1_highs); + + const __m256i sumh = _mm256_add_epi32(mul0h, mul1h); + const __m256i suml = _mm256_add_epi32(mul0l, mul1l); + + const __m256i roundh = + _mm256_srai_epi32(_mm256_sub_epi32(sumh, *round_offset), shift); + const __m256i roundl = + _mm256_srai_epi32(_mm256_sub_epi32(suml, *round_offset), shift); + + const __m256i pack = _mm256_packs_epi32(roundl, roundh); + const __m256i clip = + _mm256_min_epi16(_mm256_max_epi16(pack, *clip_low), *clip_high); + + // _mm256_extract_epi64 doesn't exist on x86, so do it the old-fashioned way: + const __m128i cliph = _mm256_extracti128_si256(clip, 1); + xx_storel_64(dst + 3 * dst_stride, _mm_srli_si128(cliph, 8)); + xx_storel_64(dst + 2 * dst_stride, cliph); + const __m128i clipl = _mm256_castsi256_si128(clip); + xx_storel_64(dst + 1 * dst_stride, _mm_srli_si128(clipl, 8)); + xx_storel_64(dst + 0 * dst_stride, clipl); +} + +static INLINE void highbd_blend_a64_d16_mask_subw0_subh0_w4_avx2( + uint16_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0, + uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride, + const uint8_t *mask, uint32_t mask_stride, int h, + const __m256i *round_offset, int shift, const __m256i *clip_low, + const __m256i *clip_high, const __m256i *mask_max) { + do { + // Load 8x u8 pixels from each of 4 rows of the mask, pad each to u16 + const __m128i mask08 = _mm_set_epi32(*(uint32_t *)(mask + 3 * mask_stride), + *(uint32_t *)(mask + 2 * mask_stride), + *(uint32_t *)(mask + 1 * mask_stride), + *(uint32_t *)(mask + 0 * mask_stride)); + const __m256i mask0 = _mm256_cvtepu8_epi16(mask08); + + highbd_blend_a64_d16_mask_w4_avx2(dst, dst_stride, src0, src0_stride, src1, + src1_stride, &mask0, round_offset, shift, + clip_low, clip_high, mask_max); + + dst += dst_stride * 4; + src0 += src0_stride * 4; + src1 += src1_stride * 4; + mask += mask_stride * 4; + } while (h -= 4); +} + +static INLINE void highbd_blend_a64_d16_mask_subw1_subh1_w4_avx2( + uint16_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0, + uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride, + const uint8_t *mask, uint32_t mask_stride, int h, + const __m256i *round_offset, int shift, const __m256i *clip_low, + const __m256i *clip_high, const __m256i *mask_max) { + const __m256i one_b = _mm256_set1_epi8(1); + const __m256i two_w = _mm256_set1_epi16(2); + do { + // Load 8 pixels from each of 8 rows of mask, + // (saturating) add together rows then use madd to add adjacent pixels + // Finally, divide each value by 4 (with rounding) + const __m256i m0246 = + _mm256_set_epi64x(*(uint64_t *)(mask + 6 * mask_stride), + *(uint64_t *)(mask + 4 * mask_stride), + *(uint64_t *)(mask + 2 * mask_stride), + *(uint64_t *)(mask + 0 * mask_stride)); + const __m256i m1357 = + _mm256_set_epi64x(*(uint64_t *)(mask + 7 * mask_stride), + *(uint64_t *)(mask + 5 * mask_stride), + *(uint64_t *)(mask + 3 * mask_stride), + *(uint64_t *)(mask + 1 * mask_stride)); + const __m256i addrows = _mm256_adds_epu8(m0246, m1357); + const __m256i adjacent = _mm256_maddubs_epi16(addrows, one_b); + const __m256i mask0 = + _mm256_srli_epi16(_mm256_add_epi16(adjacent, two_w), 2); + + highbd_blend_a64_d16_mask_w4_avx2(dst, dst_stride, src0, src0_stride, src1, + src1_stride, &mask0, round_offset, shift, + clip_low, clip_high, mask_max); + + dst += dst_stride * 4; + src0 += src0_stride * 4; + src1 += src1_stride * 4; + mask += mask_stride * 8; + } while (h -= 4); +} + +static INLINE void highbd_blend_a64_d16_mask_w8_avx2( + uint16_t *dst, int dst_stride, const CONV_BUF_TYPE *src0, int src0_stride, + const CONV_BUF_TYPE *src1, int src1_stride, const __m256i *mask0a, + const __m256i *mask0b, const __m256i *round_offset, int shift, + const __m256i *clip_low, const __m256i *clip_high, + const __m256i *mask_max) { + // Load 8x u16 pixels from each of 4 rows from each source + const __m256i s0a = + yy_loadu2_128(src0 + 0 * src0_stride, src0 + 1 * src0_stride); + const __m256i s0b = + yy_loadu2_128(src0 + 2 * src0_stride, src0 + 3 * src0_stride); + const __m256i s1a = + yy_loadu2_128(src1 + 0 * src1_stride, src1 + 1 * src1_stride); + const __m256i s1b = + yy_loadu2_128(src1 + 2 * src1_stride, src1 + 3 * src1_stride); + + // Generate inverse masks + const __m256i mask1a = _mm256_sub_epi16(*mask_max, *mask0a); + const __m256i mask1b = _mm256_sub_epi16(*mask_max, *mask0b); + + // Multiply sources by respective masks + const __m256i mul0a_highs = _mm256_mulhi_epu16(*mask0a, s0a); + const __m256i mul0a_lows = _mm256_mullo_epi16(*mask0a, s0a); + const __m256i mul0ah = _mm256_unpackhi_epi16(mul0a_lows, mul0a_highs); + const __m256i mul0al = _mm256_unpacklo_epi16(mul0a_lows, mul0a_highs); + // Note that AVX2 unpack orders 64-bit words as [3 1] [2 0] to keep within + // lanes Later, packs does the same again which cancels this out with no need + // for a permute. The intermediate values being reordered makes no difference + + const __m256i mul1a_highs = _mm256_mulhi_epu16(mask1a, s1a); + const __m256i mul1a_lows = _mm256_mullo_epi16(mask1a, s1a); + const __m256i mul1ah = _mm256_unpackhi_epi16(mul1a_lows, mul1a_highs); + const __m256i mul1al = _mm256_unpacklo_epi16(mul1a_lows, mul1a_highs); + + const __m256i sumah = _mm256_add_epi32(mul0ah, mul1ah); + const __m256i sumal = _mm256_add_epi32(mul0al, mul1al); + + const __m256i mul0b_highs = _mm256_mulhi_epu16(*mask0b, s0b); + const __m256i mul0b_lows = _mm256_mullo_epi16(*mask0b, s0b); + const __m256i mul0bh = _mm256_unpackhi_epi16(mul0b_lows, mul0b_highs); + const __m256i mul0bl = _mm256_unpacklo_epi16(mul0b_lows, mul0b_highs); + + const __m256i mul1b_highs = _mm256_mulhi_epu16(mask1b, s1b); + const __m256i mul1b_lows = _mm256_mullo_epi16(mask1b, s1b); + const __m256i mul1bh = _mm256_unpackhi_epi16(mul1b_lows, mul1b_highs); + const __m256i mul1bl = _mm256_unpacklo_epi16(mul1b_lows, mul1b_highs); + + const __m256i sumbh = _mm256_add_epi32(mul0bh, mul1bh); + const __m256i sumbl = _mm256_add_epi32(mul0bl, mul1bl); + + // Divide down each result, with rounding + const __m256i roundah = + _mm256_srai_epi32(_mm256_sub_epi32(sumah, *round_offset), shift); + const __m256i roundal = + _mm256_srai_epi32(_mm256_sub_epi32(sumal, *round_offset), shift); + const __m256i roundbh = + _mm256_srai_epi32(_mm256_sub_epi32(sumbh, *round_offset), shift); + const __m256i roundbl = + _mm256_srai_epi32(_mm256_sub_epi32(sumbl, *round_offset), shift); + + // Pack each i32 down to an i16 with saturation, then clip to valid range + const __m256i packa = _mm256_packs_epi32(roundal, roundah); + const __m256i clipa = + _mm256_min_epi16(_mm256_max_epi16(packa, *clip_low), *clip_high); + const __m256i packb = _mm256_packs_epi32(roundbl, roundbh); + const __m256i clipb = + _mm256_min_epi16(_mm256_max_epi16(packb, *clip_low), *clip_high); + + // Store 8x u16 pixels to each of 4 rows in the destination + yy_storeu2_128(dst + 0 * dst_stride, dst + 1 * dst_stride, clipa); + yy_storeu2_128(dst + 2 * dst_stride, dst + 3 * dst_stride, clipb); +} + +static INLINE void highbd_blend_a64_d16_mask_subw0_subh0_w8_avx2( + uint16_t *dst, int dst_stride, const CONV_BUF_TYPE *src0, int src0_stride, + const CONV_BUF_TYPE *src1, int src1_stride, const uint8_t *mask, + int mask_stride, int h, const __m256i *round_offset, int shift, + const __m256i *clip_low, const __m256i *clip_high, + const __m256i *mask_max) { + do { + // Load 8x u8 pixels from each of 4 rows in the mask + const __m128i mask0a8 = + _mm_set_epi64x(*(uint64_t *)mask, *(uint64_t *)(mask + mask_stride)); + const __m128i mask0b8 = + _mm_set_epi64x(*(uint64_t *)(mask + 2 * mask_stride), + *(uint64_t *)(mask + 3 * mask_stride)); + const __m256i mask0a = _mm256_cvtepu8_epi16(mask0a8); + const __m256i mask0b = _mm256_cvtepu8_epi16(mask0b8); + + highbd_blend_a64_d16_mask_w8_avx2( + dst, dst_stride, src0, src0_stride, src1, src1_stride, &mask0a, &mask0b, + round_offset, shift, clip_low, clip_high, mask_max); + + dst += dst_stride * 4; + src0 += src0_stride * 4; + src1 += src1_stride * 4; + mask += mask_stride * 4; + } while (h -= 4); +} + +static INLINE void highbd_blend_a64_d16_mask_subw1_subh1_w8_avx2( + uint16_t *dst, int dst_stride, const CONV_BUF_TYPE *src0, int src0_stride, + const CONV_BUF_TYPE *src1, int src1_stride, const uint8_t *mask, + int mask_stride, int h, const __m256i *round_offset, int shift, + const __m256i *clip_low, const __m256i *clip_high, + const __m256i *mask_max) { + const __m256i one_b = _mm256_set1_epi8(1); + const __m256i two_w = _mm256_set1_epi16(2); + do { + // Load 16x u8 pixels from each of 8 rows in the mask, + // (saturating) add together rows then use madd to add adjacent pixels + // Finally, divide each value by 4 (with rounding) + const __m256i m02 = + yy_loadu2_128(mask + 0 * mask_stride, mask + 2 * mask_stride); + const __m256i m13 = + yy_loadu2_128(mask + 1 * mask_stride, mask + 3 * mask_stride); + const __m256i m0123 = + _mm256_maddubs_epi16(_mm256_adds_epu8(m02, m13), one_b); + const __m256i mask_0a = + _mm256_srli_epi16(_mm256_add_epi16(m0123, two_w), 2); + const __m256i m46 = + yy_loadu2_128(mask + 4 * mask_stride, mask + 6 * mask_stride); + const __m256i m57 = + yy_loadu2_128(mask + 5 * mask_stride, mask + 7 * mask_stride); + const __m256i m4567 = + _mm256_maddubs_epi16(_mm256_adds_epu8(m46, m57), one_b); + const __m256i mask_0b = + _mm256_srli_epi16(_mm256_add_epi16(m4567, two_w), 2); + + highbd_blend_a64_d16_mask_w8_avx2( + dst, dst_stride, src0, src0_stride, src1, src1_stride, &mask_0a, + &mask_0b, round_offset, shift, clip_low, clip_high, mask_max); + + dst += dst_stride * 4; + src0 += src0_stride * 4; + src1 += src1_stride * 4; + mask += mask_stride * 8; + } while (h -= 4); +} + +static INLINE void highbd_blend_a64_d16_mask_w16_avx2( + uint16_t *dst, int dst_stride, const CONV_BUF_TYPE *src0, int src0_stride, + const CONV_BUF_TYPE *src1, int src1_stride, const __m256i *mask0a, + const __m256i *mask0b, const __m256i *round_offset, int shift, + const __m256i *clip_low, const __m256i *clip_high, + const __m256i *mask_max) { + // Load 16x pixels from each of 2 rows from each source + const __m256i s0a = yy_loadu_256(src0); + const __m256i s0b = yy_loadu_256(src0 + src0_stride); + const __m256i s1a = yy_loadu_256(src1); + const __m256i s1b = yy_loadu_256(src1 + src1_stride); + + // Calculate inverse masks + const __m256i mask1a = _mm256_sub_epi16(*mask_max, *mask0a); + const __m256i mask1b = _mm256_sub_epi16(*mask_max, *mask0b); + + // Multiply each source by appropriate mask + const __m256i mul0a_highs = _mm256_mulhi_epu16(*mask0a, s0a); + const __m256i mul0a_lows = _mm256_mullo_epi16(*mask0a, s0a); + const __m256i mul0ah = _mm256_unpackhi_epi16(mul0a_lows, mul0a_highs); + const __m256i mul0al = _mm256_unpacklo_epi16(mul0a_lows, mul0a_highs); + // Note that AVX2 unpack orders 64-bit words as [3 1] [2 0] to keep within + // lanes Later, packs does the same again which cancels this out with no need + // for a permute. The intermediate values being reordered makes no difference + + const __m256i mul1a_highs = _mm256_mulhi_epu16(mask1a, s1a); + const __m256i mul1a_lows = _mm256_mullo_epi16(mask1a, s1a); + const __m256i mul1ah = _mm256_unpackhi_epi16(mul1a_lows, mul1a_highs); + const __m256i mul1al = _mm256_unpacklo_epi16(mul1a_lows, mul1a_highs); + + const __m256i mulah = _mm256_add_epi32(mul0ah, mul1ah); + const __m256i mulal = _mm256_add_epi32(mul0al, mul1al); + + const __m256i mul0b_highs = _mm256_mulhi_epu16(*mask0b, s0b); + const __m256i mul0b_lows = _mm256_mullo_epi16(*mask0b, s0b); + const __m256i mul0bh = _mm256_unpackhi_epi16(mul0b_lows, mul0b_highs); + const __m256i mul0bl = _mm256_unpacklo_epi16(mul0b_lows, mul0b_highs); + + const __m256i mul1b_highs = _mm256_mulhi_epu16(mask1b, s1b); + const __m256i mul1b_lows = _mm256_mullo_epi16(mask1b, s1b); + const __m256i mul1bh = _mm256_unpackhi_epi16(mul1b_lows, mul1b_highs); + const __m256i mul1bl = _mm256_unpacklo_epi16(mul1b_lows, mul1b_highs); + + const __m256i mulbh = _mm256_add_epi32(mul0bh, mul1bh); + const __m256i mulbl = _mm256_add_epi32(mul0bl, mul1bl); + + const __m256i resah = + _mm256_srai_epi32(_mm256_sub_epi32(mulah, *round_offset), shift); + const __m256i resal = + _mm256_srai_epi32(_mm256_sub_epi32(mulal, *round_offset), shift); + const __m256i resbh = + _mm256_srai_epi32(_mm256_sub_epi32(mulbh, *round_offset), shift); + const __m256i resbl = + _mm256_srai_epi32(_mm256_sub_epi32(mulbl, *round_offset), shift); + + // Signed saturating pack from i32 to i16: + const __m256i packa = _mm256_packs_epi32(resal, resah); + const __m256i packb = _mm256_packs_epi32(resbl, resbh); + + // Clip the values to the valid range + const __m256i clipa = + _mm256_min_epi16(_mm256_max_epi16(packa, *clip_low), *clip_high); + const __m256i clipb = + _mm256_min_epi16(_mm256_max_epi16(packb, *clip_low), *clip_high); + + // Store 16 pixels + yy_storeu_256(dst, clipa); + yy_storeu_256(dst + dst_stride, clipb); +} + +static INLINE void highbd_blend_a64_d16_mask_subw0_subh0_w16_avx2( + uint16_t *dst, int dst_stride, const CONV_BUF_TYPE *src0, int src0_stride, + const CONV_BUF_TYPE *src1, int src1_stride, const uint8_t *mask, + int mask_stride, int h, int w, const __m256i *round_offset, int shift, + const __m256i *clip_low, const __m256i *clip_high, + const __m256i *mask_max) { + for (int i = 0; i < h; i += 2) { + for (int j = 0; j < w; j += 16) { + // Load 16x u8 alpha-mask values from each of two rows and pad to u16 + const __m128i masks_a8 = xx_loadu_128(mask + j); + const __m128i masks_b8 = xx_loadu_128(mask + mask_stride + j); + const __m256i mask0a = _mm256_cvtepu8_epi16(masks_a8); + const __m256i mask0b = _mm256_cvtepu8_epi16(masks_b8); + + highbd_blend_a64_d16_mask_w16_avx2( + dst + j, dst_stride, src0 + j, src0_stride, src1 + j, src1_stride, + &mask0a, &mask0b, round_offset, shift, clip_low, clip_high, mask_max); + } + dst += dst_stride * 2; + src0 += src0_stride * 2; + src1 += src1_stride * 2; + mask += mask_stride * 2; + } +} + +static INLINE void highbd_blend_a64_d16_mask_subw1_subh1_w16_avx2( + uint16_t *dst, int dst_stride, const CONV_BUF_TYPE *src0, int src0_stride, + const CONV_BUF_TYPE *src1, int src1_stride, const uint8_t *mask, + int mask_stride, int h, int w, const __m256i *round_offset, int shift, + const __m256i *clip_low, const __m256i *clip_high, + const __m256i *mask_max) { + const __m256i one_b = _mm256_set1_epi8(1); + const __m256i two_w = _mm256_set1_epi16(2); + for (int i = 0; i < h; i += 2) { + for (int j = 0; j < w; j += 16) { + // Load 32x u8 alpha-mask values from each of four rows + // (saturating) add pairs of rows, then use madd to add adjacent values + // Finally, divide down each result with rounding + const __m256i m0 = yy_loadu_256(mask + 0 * mask_stride + 2 * j); + const __m256i m1 = yy_loadu_256(mask + 1 * mask_stride + 2 * j); + const __m256i m2 = yy_loadu_256(mask + 2 * mask_stride + 2 * j); + const __m256i m3 = yy_loadu_256(mask + 3 * mask_stride + 2 * j); + + const __m256i m01_8 = _mm256_adds_epu8(m0, m1); + const __m256i m23_8 = _mm256_adds_epu8(m2, m3); + + const __m256i m01 = _mm256_maddubs_epi16(m01_8, one_b); + const __m256i m23 = _mm256_maddubs_epi16(m23_8, one_b); + + const __m256i mask0a = _mm256_srli_epi16(_mm256_add_epi16(m01, two_w), 2); + const __m256i mask0b = _mm256_srli_epi16(_mm256_add_epi16(m23, two_w), 2); + + highbd_blend_a64_d16_mask_w16_avx2( + dst + j, dst_stride, src0 + j, src0_stride, src1 + j, src1_stride, + &mask0a, &mask0b, round_offset, shift, clip_low, clip_high, mask_max); + } + dst += dst_stride * 2; + src0 += src0_stride * 2; + src1 += src1_stride * 2; + mask += mask_stride * 4; + } +} + +void aom_highbd_blend_a64_d16_mask_avx2( + uint8_t *dst8, uint32_t dst_stride, const CONV_BUF_TYPE *src0, + uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride, + const uint8_t *mask, uint32_t mask_stride, int w, int h, int subw, int subh, + ConvolveParams *conv_params, const int bd) { + uint16_t *dst = CONVERT_TO_SHORTPTR(dst8); + const int round_bits = + 2 * FILTER_BITS - conv_params->round_0 - conv_params->round_1; + const int32_t round_offset = + ((1 << (round_bits + bd)) + (1 << (round_bits + bd - 1)) - + (1 << (round_bits - 1))) + << AOM_BLEND_A64_ROUND_BITS; + const __m256i v_round_offset = _mm256_set1_epi32(round_offset); + const int shift = round_bits + AOM_BLEND_A64_ROUND_BITS; + + const __m256i clip_low = _mm256_set1_epi16(0); + const __m256i clip_high = _mm256_set1_epi16((1 << bd) - 1); + const __m256i mask_max = _mm256_set1_epi16(AOM_BLEND_A64_MAX_ALPHA); + + assert(IMPLIES((void *)src0 == dst, src0_stride == dst_stride)); + assert(IMPLIES((void *)src1 == dst, src1_stride == dst_stride)); + + assert(h >= 4); + assert(w >= 4); + assert(IS_POWER_OF_TWO(h)); + assert(IS_POWER_OF_TWO(w)); + + if (subw == 0 && subh == 0) { + switch (w) { + case 4: + highbd_blend_a64_d16_mask_subw0_subh0_w4_avx2( + dst, dst_stride, src0, src0_stride, src1, src1_stride, mask, + mask_stride, h, &v_round_offset, shift, &clip_low, &clip_high, + &mask_max); + break; + case 8: + highbd_blend_a64_d16_mask_subw0_subh0_w8_avx2( + dst, dst_stride, src0, src0_stride, src1, src1_stride, mask, + mask_stride, h, &v_round_offset, shift, &clip_low, &clip_high, + &mask_max); + break; + default: // >= 16 + highbd_blend_a64_d16_mask_subw0_subh0_w16_avx2( + dst, dst_stride, src0, src0_stride, src1, src1_stride, mask, + mask_stride, h, w, &v_round_offset, shift, &clip_low, &clip_high, + &mask_max); + break; + } + + } else if (subw == 1 && subh == 1) { + switch (w) { + case 4: + highbd_blend_a64_d16_mask_subw1_subh1_w4_avx2( + dst, dst_stride, src0, src0_stride, src1, src1_stride, mask, + mask_stride, h, &v_round_offset, shift, &clip_low, &clip_high, + &mask_max); + break; + case 8: + highbd_blend_a64_d16_mask_subw1_subh1_w8_avx2( + dst, dst_stride, src0, src0_stride, src1, src1_stride, mask, + mask_stride, h, &v_round_offset, shift, &clip_low, &clip_high, + &mask_max); + break; + default: // >= 16 + highbd_blend_a64_d16_mask_subw1_subh1_w16_avx2( + dst, dst_stride, src0, src0_stride, src1, src1_stride, mask, + mask_stride, h, w, &v_round_offset, shift, &clip_low, &clip_high, + &mask_max); + break; + } + } else { + // Sub-sampling in only one axis doesn't seem to happen very much, so fall + // back to the vanilla C implementation instead of having all the optimised + // code for these. + aom_highbd_blend_a64_d16_mask_c(dst8, dst_stride, src0, src0_stride, src1, + src1_stride, mask, mask_stride, w, h, subw, + subh, conv_params, bd); + } +} diff --git a/third_party/aom/aom_dsp/x86/blend_a64_mask_sse4.c b/third_party/aom/aom_dsp/x86/blend_a64_mask_sse4.c index 9d6b4c2f74..b7a2468045 100644 --- a/third_party/aom/aom_dsp/x86/blend_a64_mask_sse4.c +++ b/third_party/aom/aom_dsp/x86/blend_a64_mask_sse4.c @@ -1107,3 +1107,452 @@ void aom_lowbd_blend_a64_d16_mask_sse4_1( } } } + +////////////////////////////////////////////////////////////////////////////// +// aom_highbd_blend_a64_d16_mask_sse4_1() +////////////////////////////////////////////////////////////////////////////// + +static INLINE void highbd_blend_a64_d16_mask_w4_sse4_1( + uint16_t *dst, int dst_stride, const CONV_BUF_TYPE *src0, int src0_stride, + const CONV_BUF_TYPE *src1, int src1_stride, const __m128i *mask0a, + const __m128i *mask0b, const __m128i *round_offset, int shift, + const __m128i *clip_low, const __m128i *clip_high, + const __m128i *mask_max) { + // Load 4 pixels from each of 4 rows from each source + const __m128i s0a = + _mm_set_epi64x(*(uint64_t *)src0, *(uint64_t *)(src0 + src0_stride)); + const __m128i s0b = _mm_set_epi64x(*(uint64_t *)(src0 + 2 * src0_stride), + *(uint64_t *)(src0 + 3 * src0_stride)); + const __m128i s1a = + _mm_set_epi64x(*(uint64_t *)(src1), *(uint64_t *)(src1 + src1_stride)); + const __m128i s1b = _mm_set_epi64x(*(uint64_t *)(src1 + 2 * src1_stride), + *(uint64_t *)(src1 + 3 * src1_stride)); + + // Generate the inverse masks + const __m128i mask1a = _mm_sub_epi16(*mask_max, *mask0a); + const __m128i mask1b = _mm_sub_epi16(*mask_max, *mask0b); + + // Multiply each mask by the respective source + const __m128i mul0a_highs = _mm_mulhi_epu16(*mask0a, s0a); + const __m128i mul0a_lows = _mm_mullo_epi16(*mask0a, s0a); + const __m128i mul0ah = _mm_unpackhi_epi16(mul0a_lows, mul0a_highs); + const __m128i mul0al = _mm_unpacklo_epi16(mul0a_lows, mul0a_highs); + const __m128i mul1a_highs = _mm_mulhi_epu16(mask1a, s1a); + const __m128i mul1a_lows = _mm_mullo_epi16(mask1a, s1a); + const __m128i mul1ah = _mm_unpackhi_epi16(mul1a_lows, mul1a_highs); + const __m128i mul1al = _mm_unpacklo_epi16(mul1a_lows, mul1a_highs); + + const __m128i mul0b_highs = _mm_mulhi_epu16(*mask0b, s0b); + const __m128i mul0b_lows = _mm_mullo_epi16(*mask0b, s0b); + const __m128i mul0bh = _mm_unpackhi_epi16(mul0b_lows, mul0b_highs); + const __m128i mul0bl = _mm_unpacklo_epi16(mul0b_lows, mul0b_highs); + const __m128i mul1b_highs = _mm_mulhi_epu16(mask1b, s1b); + const __m128i mul1b_lows = _mm_mullo_epi16(mask1b, s1b); + const __m128i mul1bh = _mm_unpackhi_epi16(mul1b_lows, mul1b_highs); + const __m128i mul1bl = _mm_unpacklo_epi16(mul1b_lows, mul1b_highs); + + const __m128i sumah = _mm_add_epi32(mul0ah, mul1ah); + const __m128i sumal = _mm_add_epi32(mul0al, mul1al); + const __m128i sumbh = _mm_add_epi32(mul0bh, mul1bh); + const __m128i sumbl = _mm_add_epi32(mul0bl, mul1bl); + + const __m128i roundah = + _mm_srai_epi32(_mm_sub_epi32(sumah, *round_offset), shift); + const __m128i roundbh = + _mm_srai_epi32(_mm_sub_epi32(sumbh, *round_offset), shift); + const __m128i roundal = + _mm_srai_epi32(_mm_sub_epi32(sumal, *round_offset), shift); + const __m128i roundbl = + _mm_srai_epi32(_mm_sub_epi32(sumbl, *round_offset), shift); + + const __m128i packa = _mm_packs_epi32(roundal, roundah); + const __m128i packb = _mm_packs_epi32(roundbl, roundbh); + + const __m128i clipa = + _mm_min_epi16(_mm_max_epi16(packa, *clip_low), *clip_high); + const __m128i clipb = + _mm_min_epi16(_mm_max_epi16(packb, *clip_low), *clip_high); + + xx_storel_64(dst, _mm_srli_si128(clipa, 8)); + xx_storel_64(dst + dst_stride, clipa); + xx_storel_64(dst + 2 * dst_stride, _mm_srli_si128(clipb, 8)); + xx_storel_64(dst + 3 * dst_stride, clipb); +} + +static INLINE void highbd_blend_a64_d16_mask_subw0_subh0_w4_sse4_1( + uint16_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0, + uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride, + const uint8_t *mask, uint32_t mask_stride, int h, + const __m128i *round_offset, int shift, const __m128i *clip_low, + const __m128i *clip_high, const __m128i *mask_max) { + do { + const __m128i mask0a8 = _mm_set_epi32(0, 0, *(uint32_t *)mask, + *(uint32_t *)(mask + mask_stride)); + const __m128i mask0b8 = + _mm_set_epi32(0, 0, *(uint32_t *)(mask + 2 * mask_stride), + *(uint32_t *)(mask + 3 * mask_stride)); + const __m128i mask0a = _mm_cvtepu8_epi16(mask0a8); + const __m128i mask0b = _mm_cvtepu8_epi16(mask0b8); + + highbd_blend_a64_d16_mask_w4_sse4_1( + dst, dst_stride, src0, src0_stride, src1, src1_stride, &mask0a, &mask0b, + round_offset, shift, clip_low, clip_high, mask_max); + + dst += dst_stride * 4; + src0 += src0_stride * 4; + src1 += src1_stride * 4; + mask += mask_stride * 4; + } while (h -= 4); +} + +static INLINE void highbd_blend_a64_d16_mask_subw1_subh1_w4_sse4_1( + uint16_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0, + uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride, + const uint8_t *mask, uint32_t mask_stride, int h, + const __m128i *round_offset, int shift, const __m128i *clip_low, + const __m128i *clip_high, const __m128i *mask_max) { + const __m128i one_b = _mm_set1_epi8(1); + const __m128i two_w = _mm_set1_epi16(2); + do { + // Load 8 pixels from each of 8 rows of mask, + // (saturating) add together rows then use madd to add adjacent pixels + // Finally, divide each value by 4 (with rounding) + const __m128i m02 = _mm_set_epi64x(*(uint64_t *)(mask), + *(uint64_t *)(mask + 2 * mask_stride)); + const __m128i m13 = _mm_set_epi64x(*(uint64_t *)(mask + mask_stride), + *(uint64_t *)(mask + 3 * mask_stride)); + const __m128i m0123 = _mm_maddubs_epi16(_mm_adds_epu8(m02, m13), one_b); + const __m128i mask_0a = _mm_srli_epi16(_mm_add_epi16(m0123, two_w), 2); + const __m128i m46 = _mm_set_epi64x(*(uint64_t *)(mask + 4 * mask_stride), + *(uint64_t *)(mask + 6 * mask_stride)); + const __m128i m57 = _mm_set_epi64x(*(uint64_t *)(mask + 5 * mask_stride), + *(uint64_t *)(mask + 7 * mask_stride)); + const __m128i m4567 = _mm_maddubs_epi16(_mm_adds_epu8(m46, m57), one_b); + const __m128i mask_0b = _mm_srli_epi16(_mm_add_epi16(m4567, two_w), 2); + + highbd_blend_a64_d16_mask_w4_sse4_1( + dst, dst_stride, src0, src0_stride, src1, src1_stride, &mask_0a, + &mask_0b, round_offset, shift, clip_low, clip_high, mask_max); + + dst += dst_stride * 4; + src0 += src0_stride * 4; + src1 += src1_stride * 4; + mask += mask_stride * 8; + } while (h -= 4); +} + +static INLINE void highbd_blend_a64_d16_mask_w8_sse4_1( + uint16_t *dst, int dst_stride, const CONV_BUF_TYPE *src0, int src0_stride, + const CONV_BUF_TYPE *src1, int src1_stride, const __m128i *mask0a, + const __m128i *mask0b, const __m128i *round_offset, int shift, + const __m128i *clip_low, const __m128i *clip_high, + const __m128i *max_mask) { + // Load 8x pixels from each of 2 rows from each source + const __m128i s0a = xx_loadu_128(src0); + const __m128i s0b = xx_loadu_128(src0 + src0_stride); + const __m128i s1a = xx_loadu_128(src1); + const __m128i s1b = xx_loadu_128(src1 + src1_stride); + + // Generate inverse masks + const __m128i mask1a = _mm_sub_epi16(*max_mask, *mask0a); + const __m128i mask1b = _mm_sub_epi16(*max_mask, *mask0b); + + // Multiply sources by respective masks + const __m128i mul0a_highs = _mm_mulhi_epu16(*mask0a, s0a); + const __m128i mul0a_lows = _mm_mullo_epi16(*mask0a, s0a); + const __m128i mul0ah = _mm_unpackhi_epi16(mul0a_lows, mul0a_highs); + const __m128i mul0al = _mm_unpacklo_epi16(mul0a_lows, mul0a_highs); + + const __m128i mul1a_highs = _mm_mulhi_epu16(mask1a, s1a); + const __m128i mul1a_lows = _mm_mullo_epi16(mask1a, s1a); + const __m128i mul1ah = _mm_unpackhi_epi16(mul1a_lows, mul1a_highs); + const __m128i mul1al = _mm_unpacklo_epi16(mul1a_lows, mul1a_highs); + + const __m128i sumah = _mm_add_epi32(mul0ah, mul1ah); + const __m128i sumal = _mm_add_epi32(mul0al, mul1al); + + const __m128i mul0b_highs = _mm_mulhi_epu16(*mask0b, s0b); + const __m128i mul0b_lows = _mm_mullo_epi16(*mask0b, s0b); + const __m128i mul0bh = _mm_unpackhi_epi16(mul0b_lows, mul0b_highs); + const __m128i mul0bl = _mm_unpacklo_epi16(mul0b_lows, mul0b_highs); + + const __m128i mul1b_highs = _mm_mulhi_epu16(mask1b, s1b); + const __m128i mul1b_lows = _mm_mullo_epi16(mask1b, s1b); + const __m128i mul1bh = _mm_unpackhi_epi16(mul1b_lows, mul1b_highs); + const __m128i mul1bl = _mm_unpacklo_epi16(mul1b_lows, mul1b_highs); + + const __m128i sumbh = _mm_add_epi32(mul0bh, mul1bh); + const __m128i sumbl = _mm_add_epi32(mul0bl, mul1bl); + + const __m128i roundah = + _mm_srai_epi32(_mm_sub_epi32(sumah, *round_offset), shift); + const __m128i roundal = + _mm_srai_epi32(_mm_sub_epi32(sumal, *round_offset), shift); + const __m128i roundbh = + _mm_srai_epi32(_mm_sub_epi32(sumbh, *round_offset), shift); + const __m128i roundbl = + _mm_srai_epi32(_mm_sub_epi32(sumbl, *round_offset), shift); + + const __m128i packa = _mm_packs_epi32(roundal, roundah); + const __m128i clipa = + _mm_min_epi16(_mm_max_epi16(packa, *clip_low), *clip_high); + const __m128i packb = _mm_packs_epi32(roundbl, roundbh); + const __m128i clipb = + _mm_min_epi16(_mm_max_epi16(packb, *clip_low), *clip_high); + + xx_storeu_128(dst, clipa); + xx_storeu_128(dst + dst_stride, clipb); +} + +static INLINE void highbd_blend_a64_d16_mask_subw0_subh0_w8_sse4_1( + uint16_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0, + uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride, + const uint8_t *mask, uint32_t mask_stride, int h, + const __m128i *round_offset, int shift, const __m128i *clip_low, + const __m128i *clip_high, const __m128i *max_mask) { + do { + const __m128i mask0a = _mm_cvtepu8_epi16(xx_loadl_64(mask)); + const __m128i mask0b = _mm_cvtepu8_epi16(xx_loadl_64(mask + mask_stride)); + highbd_blend_a64_d16_mask_w8_sse4_1( + dst, dst_stride, src0, src0_stride, src1, src1_stride, &mask0a, &mask0b, + round_offset, shift, clip_low, clip_high, max_mask); + + dst += dst_stride * 2; + src0 += src0_stride * 2; + src1 += src1_stride * 2; + mask += mask_stride * 2; + } while (h -= 2); +} + +static INLINE void highbd_blend_a64_d16_mask_subw1_subh1_w8_sse4_1( + uint16_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0, + uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride, + const uint8_t *mask, uint32_t mask_stride, int h, + const __m128i *round_offset, int shift, const __m128i *clip_low, + const __m128i *clip_high, const __m128i *max_mask) { + const __m128i one_b = _mm_set1_epi8(1); + const __m128i two_w = _mm_set1_epi16(2); + do { + const __m128i mask_thisrowa = xx_loadu_128(mask); + const __m128i mask_nextrowa = xx_loadu_128(mask + mask_stride); + const __m128i mask_thisrowb = xx_loadu_128(mask + 2 * mask_stride); + const __m128i mask_nextrowb = xx_loadu_128(mask + 3 * mask_stride); + const __m128i mask_bothrowsa = _mm_adds_epu8(mask_thisrowa, mask_nextrowa); + const __m128i mask_bothrowsb = _mm_adds_epu8(mask_thisrowb, mask_nextrowb); + const __m128i mask_16a = _mm_maddubs_epi16(mask_bothrowsa, one_b); + const __m128i mask_16b = _mm_maddubs_epi16(mask_bothrowsb, one_b); + const __m128i mask_sa = _mm_srli_epi16(_mm_add_epi16(mask_16a, two_w), 2); + const __m128i mask_sb = _mm_srli_epi16(_mm_add_epi16(mask_16b, two_w), 2); + + highbd_blend_a64_d16_mask_w8_sse4_1( + dst, dst_stride, src0, src0_stride, src1, src1_stride, &mask_sa, + &mask_sb, round_offset, shift, clip_low, clip_high, max_mask); + + dst += dst_stride * 2; + src0 += src0_stride * 2; + src1 += src1_stride * 2; + mask += mask_stride * 4; + } while (h -= 2); +} + +static INLINE void highbd_blend_a64_d16_mask_w16_sse4_1( + uint16_t *dst, const CONV_BUF_TYPE *src0, const CONV_BUF_TYPE *src1, + const __m128i *round_offset, int shift, const __m128i *mask0l, + const __m128i *mask0h, const __m128i *clip_low, const __m128i *clip_high, + const __m128i *mask_max) { + // Load 16x u16 pixels for this row from each src + const __m128i s0l = xx_loadu_128(src0); + const __m128i s0h = xx_loadu_128(src0 + 8); + const __m128i s1l = xx_loadu_128(src1); + const __m128i s1h = xx_loadu_128(src1 + 8); + + // Calculate inverse masks + const __m128i mask1h = _mm_sub_epi16(*mask_max, *mask0h); + const __m128i mask1l = _mm_sub_epi16(*mask_max, *mask0l); + + const __m128i mul0_highs = _mm_mulhi_epu16(*mask0h, s0h); + const __m128i mul0_lows = _mm_mullo_epi16(*mask0h, s0h); + const __m128i mul0h = _mm_unpackhi_epi16(mul0_lows, mul0_highs); + const __m128i mul0l = _mm_unpacklo_epi16(mul0_lows, mul0_highs); + + const __m128i mul1_highs = _mm_mulhi_epu16(mask1h, s1h); + const __m128i mul1_lows = _mm_mullo_epi16(mask1h, s1h); + const __m128i mul1h = _mm_unpackhi_epi16(mul1_lows, mul1_highs); + const __m128i mul1l = _mm_unpacklo_epi16(mul1_lows, mul1_highs); + + const __m128i mulhh = _mm_add_epi32(mul0h, mul1h); + const __m128i mulhl = _mm_add_epi32(mul0l, mul1l); + + const __m128i mul2_highs = _mm_mulhi_epu16(*mask0l, s0l); + const __m128i mul2_lows = _mm_mullo_epi16(*mask0l, s0l); + const __m128i mul2h = _mm_unpackhi_epi16(mul2_lows, mul2_highs); + const __m128i mul2l = _mm_unpacklo_epi16(mul2_lows, mul2_highs); + + const __m128i mul3_highs = _mm_mulhi_epu16(mask1l, s1l); + const __m128i mul3_lows = _mm_mullo_epi16(mask1l, s1l); + const __m128i mul3h = _mm_unpackhi_epi16(mul3_lows, mul3_highs); + const __m128i mul3l = _mm_unpacklo_epi16(mul3_lows, mul3_highs); + + const __m128i mullh = _mm_add_epi32(mul2h, mul3h); + const __m128i mulll = _mm_add_epi32(mul2l, mul3l); + + const __m128i reshh = + _mm_srai_epi32(_mm_sub_epi32(mulhh, *round_offset), shift); + const __m128i reshl = + _mm_srai_epi32(_mm_sub_epi32(mulhl, *round_offset), shift); + const __m128i reslh = + _mm_srai_epi32(_mm_sub_epi32(mullh, *round_offset), shift); + const __m128i resll = + _mm_srai_epi32(_mm_sub_epi32(mulll, *round_offset), shift); + + // Signed saturating pack from i32 to i16: + const __m128i packh = _mm_packs_epi32(reshl, reshh); + const __m128i packl = _mm_packs_epi32(resll, reslh); + + // Clip the values to the valid range + const __m128i cliph = + _mm_min_epi16(_mm_max_epi16(packh, *clip_low), *clip_high); + const __m128i clipl = + _mm_min_epi16(_mm_max_epi16(packl, *clip_low), *clip_high); + + // Store 16 pixels + xx_storeu_128(dst, clipl); + xx_storeu_128(dst + 8, cliph); +} + +static INLINE void highbd_blend_a64_d16_mask_subw0_subh0_w16_sse4_1( + uint16_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0, + uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride, + const uint8_t *mask, uint32_t mask_stride, int h, int w, + const __m128i *round_offset, int shift, const __m128i *clip_low, + const __m128i *clip_high, const __m128i *mask_max) { + for (int i = 0; i < h; i++) { + for (int j = 0; j < w; j += 16) { + // Load 16x u8 alpha-mask values and pad to u16 + const __m128i masks_u8 = xx_loadu_128(mask + j); + const __m128i mask0l = _mm_cvtepu8_epi16(masks_u8); + const __m128i mask0h = _mm_cvtepu8_epi16(_mm_srli_si128(masks_u8, 8)); + + highbd_blend_a64_d16_mask_w16_sse4_1( + dst + j, src0 + j, src1 + j, round_offset, shift, &mask0l, &mask0h, + clip_low, clip_high, mask_max); + } + dst += dst_stride; + src0 += src0_stride; + src1 += src1_stride; + mask += mask_stride; + } +} + +static INLINE void highbd_blend_a64_d16_mask_subw1_subh1_w16_sse4_1( + uint16_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0, + uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride, + const uint8_t *mask, uint32_t mask_stride, int h, int w, + const __m128i *round_offset, int shift, const __m128i *clip_low, + const __m128i *clip_high, const __m128i *mask_max) { + const __m128i one_b = _mm_set1_epi8(1); + const __m128i two_w = _mm_set1_epi16(2); + for (int i = 0; i < h; i++) { + for (int j = 0; j < w; j += 16) { + const __m128i m_i00 = xx_loadu_128(mask + 2 * j); + const __m128i m_i01 = xx_loadu_128(mask + 2 * j + 16); + const __m128i m_i10 = xx_loadu_128(mask + mask_stride + 2 * j); + const __m128i m_i11 = xx_loadu_128(mask + mask_stride + 2 * j + 16); + + const __m128i m0_ac = _mm_adds_epu8(m_i00, m_i10); + const __m128i m1_ac = _mm_adds_epu8(m_i01, m_i11); + const __m128i m0_acbd = _mm_maddubs_epi16(m0_ac, one_b); + const __m128i m1_acbd = _mm_maddubs_epi16(m1_ac, one_b); + const __m128i mask_l = _mm_srli_epi16(_mm_add_epi16(m0_acbd, two_w), 2); + const __m128i mask_h = _mm_srli_epi16(_mm_add_epi16(m1_acbd, two_w), 2); + + highbd_blend_a64_d16_mask_w16_sse4_1( + dst + j, src0 + j, src1 + j, round_offset, shift, &mask_l, &mask_h, + clip_low, clip_high, mask_max); + } + dst += dst_stride; + src0 += src0_stride; + src1 += src1_stride; + mask += mask_stride * 2; + } +} + +void aom_highbd_blend_a64_d16_mask_sse4_1( + uint8_t *dst8, uint32_t dst_stride, const CONV_BUF_TYPE *src0, + uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride, + const uint8_t *mask, uint32_t mask_stride, int w, int h, int subw, int subh, + ConvolveParams *conv_params, const int bd) { + uint16_t *dst = CONVERT_TO_SHORTPTR(dst8); + const int round_bits = + 2 * FILTER_BITS - conv_params->round_0 - conv_params->round_1; + const int32_t round_offset = + ((1 << (round_bits + bd)) + (1 << (round_bits + bd - 1)) - + (1 << (round_bits - 1))) + << AOM_BLEND_A64_ROUND_BITS; + const __m128i v_round_offset = _mm_set1_epi32(round_offset); + const int shift = round_bits + AOM_BLEND_A64_ROUND_BITS; + + const __m128i clip_low = _mm_set1_epi16(0); + const __m128i clip_high = _mm_set1_epi16((1 << bd) - 1); + const __m128i mask_max = _mm_set1_epi16(AOM_BLEND_A64_MAX_ALPHA); + + assert(IMPLIES((void *)src0 == dst, src0_stride == dst_stride)); + assert(IMPLIES((void *)src1 == dst, src1_stride == dst_stride)); + + assert(h >= 4); + assert(w >= 4); + assert(IS_POWER_OF_TWO(h)); + assert(IS_POWER_OF_TWO(w)); + + if (subw == 0 && subh == 0) { + switch (w) { + case 4: + highbd_blend_a64_d16_mask_subw0_subh0_w4_sse4_1( + dst, dst_stride, src0, src0_stride, src1, src1_stride, mask, + mask_stride, h, &v_round_offset, shift, &clip_low, &clip_high, + &mask_max); + break; + case 8: + highbd_blend_a64_d16_mask_subw0_subh0_w8_sse4_1( + dst, dst_stride, src0, src0_stride, src1, src1_stride, mask, + mask_stride, h, &v_round_offset, shift, &clip_low, &clip_high, + &mask_max); + break; + default: // >=16 + highbd_blend_a64_d16_mask_subw0_subh0_w16_sse4_1( + dst, dst_stride, src0, src0_stride, src1, src1_stride, mask, + mask_stride, h, w, &v_round_offset, shift, &clip_low, &clip_high, + &mask_max); + break; + } + + } else if (subw == 1 && subh == 1) { + switch (w) { + case 4: + highbd_blend_a64_d16_mask_subw1_subh1_w4_sse4_1( + dst, dst_stride, src0, src0_stride, src1, src1_stride, mask, + mask_stride, h, &v_round_offset, shift, &clip_low, &clip_high, + &mask_max); + break; + case 8: + highbd_blend_a64_d16_mask_subw1_subh1_w8_sse4_1( + dst, dst_stride, src0, src0_stride, src1, src1_stride, mask, + mask_stride, h, &v_round_offset, shift, &clip_low, &clip_high, + &mask_max); + break; + default: // >=16 + highbd_blend_a64_d16_mask_subw1_subh1_w16_sse4_1( + dst, dst_stride, src0, src0_stride, src1, src1_stride, mask, + mask_stride, h, w, &v_round_offset, shift, &clip_low, &clip_high, + &mask_max); + break; + } + } else { + // Sub-sampling in only one axis doesn't seem to happen very much, so fall + // back to the vanilla C implementation instead of having all the optimised + // code for these. + aom_highbd_blend_a64_d16_mask_c(dst8, dst_stride, src0, src0_stride, src1, + src1_stride, mask, mask_stride, w, h, subw, + subh, conv_params, bd); + } +} diff --git a/third_party/aom/aom_dsp/x86/convolve.h b/third_party/aom/aom_dsp/x86/convolve.h index 3e19682cd9..ff3224e5ad 100644 --- a/third_party/aom/aom_dsp/x86/convolve.h +++ b/third_party/aom/aom_dsp/x86/convolve.h @@ -122,7 +122,30 @@ typedef void highbd_filter8_1dfunction(const uint16_t *src_ptr, uint16_t *src = CONVERT_TO_SHORTPTR(src8); \ uint16_t *dst = CONVERT_TO_SHORTPTR(dst8); \ if (step_q4 == 16 && filter[3] != 128) { \ - if (filter[0] | filter[1] | filter[2]) { \ + if (((filter[0] | filter[1] | filter[6] | filter[7]) == 0) && \ + (filter[2] | filter[5])) { \ + while (w >= 16) { \ + aom_highbd_filter_block1d16_##dir##4_##avg##opt( \ + src_start, src_stride, dst, dst_stride, h, filter, bd); \ + src += 16; \ + dst += 16; \ + w -= 16; \ + } \ + while (w >= 8) { \ + aom_highbd_filter_block1d8_##dir##4_##avg##opt( \ + src_start, src_stride, dst, dst_stride, h, filter, bd); \ + src += 8; \ + dst += 8; \ + w -= 8; \ + } \ + while (w >= 4) { \ + aom_highbd_filter_block1d4_##dir##4_##avg##opt( \ + src_start, src_stride, dst, dst_stride, h, filter, bd); \ + src += 4; \ + dst += 4; \ + w -= 4; \ + } \ + } else if (filter[0] | filter[1] | filter[2]) { \ while (w >= 16) { \ aom_highbd_filter_block1d16_##dir##8_##avg##opt( \ src_start, src_stride, dst, dst_stride, h, filter, bd); \ diff --git a/third_party/aom/aom_dsp/x86/convolve_avx2.h b/third_party/aom/aom_dsp/x86/convolve_avx2.h index 30253f65c2..e041a6230e 100644 --- a/third_party/aom/aom_dsp/x86/convolve_avx2.h +++ b/third_party/aom/aom_dsp/x86/convolve_avx2.h @@ -34,6 +34,31 @@ DECLARE_ALIGNED(32, static const uint8_t, filt4_d4_global_avx2[]) = { 2, 3, 4, 5, 3, 4, 5, 6, 4, 5, 6, 7, 5, 6, 7, 8, }; +DECLARE_ALIGNED(32, static const uint8_t, filt_center_global_avx2[32]) = { + 3, 255, 4, 255, 5, 255, 6, 255, 7, 255, 8, 255, 9, 255, 10, 255, + 3, 255, 4, 255, 5, 255, 6, 255, 7, 255, 8, 255, 9, 255, 10, 255 +}; + +DECLARE_ALIGNED(32, static const uint8_t, filt1_global_avx2[32]) = { + 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, + 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8 +}; + +DECLARE_ALIGNED(32, static const uint8_t, filt2_global_avx2[32]) = { + 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, + 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10 +}; + +DECLARE_ALIGNED(32, static const uint8_t, filt3_global_avx2[32]) = { + 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, + 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12 +}; + +DECLARE_ALIGNED(32, static const uint8_t, filt4_global_avx2[32]) = { + 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, + 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14 +}; + static INLINE void prepare_coeffs_lowbd( const InterpFilterParams *const filter_params, const int subpel_q4, __m256i *const coeffs /* [4] */) { @@ -108,6 +133,15 @@ static INLINE __m256i convolve(const __m256i *const s, return res; } +static INLINE __m256i convolve_4tap(const __m256i *const s, + const __m256i *const coeffs) { + const __m256i res_1 = _mm256_madd_epi16(s[0], coeffs[0]); + const __m256i res_2 = _mm256_madd_epi16(s[1], coeffs[1]); + + const __m256i res = _mm256_add_epi32(res_1, res_2); + return res; +} + static INLINE __m256i convolve_lowbd_x(const __m256i data, const __m256i *const coeffs, const __m256i *const filt) { @@ -138,9 +172,9 @@ static INLINE void add_store_aligned_256(CONV_BUF_TYPE *const dst, static INLINE __m256i comp_avg(const __m256i *const data_ref_0, const __m256i *const res_unsigned, const __m256i *const wt, - const int use_jnt_comp_avg) { + const int use_dist_wtd_comp_avg) { __m256i res; - if (use_jnt_comp_avg) { + if (use_dist_wtd_comp_avg) { const __m256i data_lo = _mm256_unpacklo_epi16(*data_ref_0, *res_unsigned); const __m256i data_hi = _mm256_unpackhi_epi16(*data_ref_0, *res_unsigned); @@ -172,9 +206,9 @@ static INLINE __m256i highbd_comp_avg(const __m256i *const data_ref_0, const __m256i *const res_unsigned, const __m256i *const wt0, const __m256i *const wt1, - const int use_jnt_comp_avg) { + const int use_dist_wtd_comp_avg) { __m256i res; - if (use_jnt_comp_avg) { + if (use_dist_wtd_comp_avg) { const __m256i wt0_res = _mm256_mullo_epi32(*data_ref_0, *wt0); const __m256i wt1_res = _mm256_mullo_epi32(*res_unsigned, *wt1); const __m256i wt_res = _mm256_add_epi32(wt0_res, wt1_res); diff --git a/third_party/aom/aom_dsp/x86/convolve_sse2.h b/third_party/aom/aom_dsp/x86/convolve_sse2.h index 445d04b103..385c7c7e18 100644 --- a/third_party/aom/aom_dsp/x86/convolve_sse2.h +++ b/third_party/aom/aom_dsp/x86/convolve_sse2.h @@ -78,9 +78,9 @@ static INLINE __m128i convolve_hi_y(const __m128i *const s, static INLINE __m128i comp_avg(const __m128i *const data_ref_0, const __m128i *const res_unsigned, const __m128i *const wt, - const int use_jnt_comp_avg) { + const int use_dist_wtd_avg) { __m128i res; - if (use_jnt_comp_avg) { + if (use_dist_wtd_avg) { const __m128i data_lo = _mm_unpacklo_epi16(*data_ref_0, *res_unsigned); const __m128i data_hi = _mm_unpackhi_epi16(*data_ref_0, *res_unsigned); diff --git a/third_party/aom/aom_dsp/x86/convolve_sse4_1.h b/third_party/aom/aom_dsp/x86/convolve_sse4_1.h index 6b8388d84a..b1a3bb4664 100644 --- a/third_party/aom/aom_dsp/x86/convolve_sse4_1.h +++ b/third_party/aom/aom_dsp/x86/convolve_sse4_1.h @@ -35,9 +35,9 @@ static INLINE __m128i highbd_comp_avg_sse4_1(const __m128i *const data_ref_0, const __m128i *const res_unsigned, const __m128i *const wt0, const __m128i *const wt1, - const int use_jnt_comp_avg) { + const int use_dist_wtd_avg) { __m128i res; - if (use_jnt_comp_avg) { + if (use_dist_wtd_avg) { const __m128i wt0_res = _mm_mullo_epi32(*data_ref_0, *wt0); const __m128i wt1_res = _mm_mullo_epi32(*res_unsigned, *wt1); diff --git a/third_party/aom/aom_dsp/x86/fft_avx2.c b/third_party/aom/aom_dsp/x86/fft_avx2.c index 54da022538..4cccc5f00f 100644 --- a/third_party/aom/aom_dsp/x86/fft_avx2.c +++ b/third_party/aom/aom_dsp/x86/fft_avx2.c @@ -11,6 +11,7 @@ #include +#include "config/aom_dsp_rtcd.h" #include "aom_dsp/aom_dsp_common.h" #include "aom_dsp/fft_common.h" diff --git a/third_party/aom/aom_dsp/x86/fft_sse2.c b/third_party/aom/aom_dsp/x86/fft_sse2.c index 12bdc3e185..6f20a3cc01 100644 --- a/third_party/aom/aom_dsp/x86/fft_sse2.c +++ b/third_party/aom/aom_dsp/x86/fft_sse2.c @@ -11,6 +11,7 @@ s * PATENTS file, you can obtain it at www.aomedia.org/license/patent. #include +#include "config/aom_dsp_rtcd.h" #include "aom_dsp/aom_dsp_common.h" #include "aom_dsp/fft_common.h" diff --git a/third_party/aom/aom_dsp/x86/fwd_txfm_sse2.c b/third_party/aom/aom_dsp/x86/fwd_txfm_sse2.c index 2d8f8f71e4..6b7c11b18d 100644 --- a/third_party/aom/aom_dsp/x86/fwd_txfm_sse2.c +++ b/third_party/aom/aom_dsp/x86/fwd_txfm_sse2.c @@ -17,46 +17,6 @@ #include "aom_dsp/aom_dsp_common.h" #include "aom_dsp/x86/fwd_txfm_sse2.h" -void aom_fdct8x8_1_sse2(const int16_t *input, tran_low_t *output, int stride) { - __m128i in0 = _mm_load_si128((const __m128i *)(input + 0 * stride)); - __m128i in1 = _mm_load_si128((const __m128i *)(input + 1 * stride)); - __m128i in2 = _mm_load_si128((const __m128i *)(input + 2 * stride)); - __m128i in3 = _mm_load_si128((const __m128i *)(input + 3 * stride)); - __m128i u0, u1, sum; - - u0 = _mm_add_epi16(in0, in1); - u1 = _mm_add_epi16(in2, in3); - - in0 = _mm_load_si128((const __m128i *)(input + 4 * stride)); - in1 = _mm_load_si128((const __m128i *)(input + 5 * stride)); - in2 = _mm_load_si128((const __m128i *)(input + 6 * stride)); - in3 = _mm_load_si128((const __m128i *)(input + 7 * stride)); - - sum = _mm_add_epi16(u0, u1); - - in0 = _mm_add_epi16(in0, in1); - in2 = _mm_add_epi16(in2, in3); - sum = _mm_add_epi16(sum, in0); - - u0 = _mm_setzero_si128(); - sum = _mm_add_epi16(sum, in2); - - in0 = _mm_unpacklo_epi16(u0, sum); - in1 = _mm_unpackhi_epi16(u0, sum); - in0 = _mm_srai_epi32(in0, 16); - in1 = _mm_srai_epi32(in1, 16); - - sum = _mm_add_epi32(in0, in1); - in0 = _mm_unpacklo_epi32(sum, u0); - in1 = _mm_unpackhi_epi32(sum, u0); - - sum = _mm_add_epi32(in0, in1); - in0 = _mm_srli_si128(sum, 8); - - in1 = _mm_add_epi32(sum, in0); - output[0] = (tran_low_t)_mm_cvtsi128_si32(in1); -} - #define DCT_HIGH_BIT_DEPTH 0 #define FDCT8x8_2D aom_fdct8x8_sse2 #include "aom_dsp/x86/fwd_txfm_impl_sse2.h" diff --git a/third_party/aom/aom_dsp/x86/highbd_convolve_avx2.c b/third_party/aom/aom_dsp/x86/highbd_convolve_avx2.c index 099fcf7fc6..ebcb5ac2e6 100644 --- a/third_party/aom/aom_dsp/x86/highbd_convolve_avx2.c +++ b/third_party/aom/aom_dsp/x86/highbd_convolve_avx2.c @@ -20,6 +20,14 @@ // ----------------------------------------------------------------------------- // Copy and average +static const uint8_t ip_shuffle_f2f3[32] = { 0, 1, 2, 3, 2, 3, 4, 5, 4, 5, 6, + 7, 6, 7, 8, 9, 0, 1, 2, 3, 2, 3, + 4, 5, 4, 5, 6, 7, 6, 7, 8, 9 }; +static const uint8_t ip_shuffle_f4f5[32] = { 4, 5, 6, 7, 6, 7, 8, 9, + 8, 9, 10, 11, 10, 11, 12, 13, + 4, 5, 6, 7, 6, 7, 8, 9, + 8, 9, 10, 11, 10, 11, 12, 13 }; + void aom_highbd_convolve_copy_avx2(const uint8_t *src8, ptrdiff_t src_stride, uint8_t *dst8, ptrdiff_t dst_stride, const int16_t *filter_x, int filter_x_stride, @@ -444,6 +452,17 @@ static INLINE void pack_filters(const int16_t *filter, __m256i *f /*f[4]*/) { f[3] = _mm256_shuffle_epi8(hh, p3); } +static INLINE void pack_filters_4tap(const int16_t *filter, + __m256i *f /*f[4]*/) { + const __m128i h = _mm_loadu_si128((const __m128i *)filter); + const __m256i coeff = _mm256_broadcastsi128_si256(h); + + // coeffs 2 3 2 3 2 3 2 3 + f[0] = _mm256_shuffle_epi32(coeff, 0x55); + // coeffs 4 5 4 5 4 5 4 5 + f[1] = _mm256_shuffle_epi32(coeff, 0xaa); +} + static INLINE void filter_8x1_pixels(const __m256i *sig /*sig[4]*/, const __m256i *fil /*fil[4]*/, __m256i *y) { @@ -544,6 +563,176 @@ static void aom_highbd_filter_block1d16_h8_avx2( } while (height > 0); } +static void aom_highbd_filter_block1d4_h4_avx2( + const uint16_t *src_ptr, ptrdiff_t src_pitch, uint16_t *dst_ptr, + ptrdiff_t dst_pitch, uint32_t height, const int16_t *filter, int bd) { + const __m256i rounding = _mm256_set1_epi32(1 << (CONV8_ROUNDING_BITS - 1)); + __m256i ff[2], s[2]; + uint32_t i; + const __m256i clip_pixel = + _mm256_set1_epi16(bd == 10 ? 1023 : (bd == 12 ? 4095 : 255)); + const __m256i zero = _mm256_setzero_si256(); + + static const uint8_t shuffle_mask[32] = { 0, 1, 2, 3, 2, 3, 4, 5, 4, 5, 6, + 7, 6, 7, 8, 9, 0, 1, 2, 3, 2, 3, + 4, 5, 4, 5, 6, 7, 6, 7, 8, 9 }; + + __m256i mask = _mm256_loadu_si256((__m256i *)shuffle_mask); + __m256i ip_mask_f2f3 = _mm256_loadu_si256((__m256i *)ip_shuffle_f2f3); + __m256i ip_mask_f4f5 = _mm256_loadu_si256((__m256i *)ip_shuffle_f4f5); + + pack_filters_4tap(filter, ff); + src_ptr -= 3; + for (i = 0; i <= (height - 2); i += 2) { + __m256i row0 = _mm256_castsi128_si256( + _mm_loadu_si128((__m128i *)&src_ptr[i * src_pitch + 2])); + __m256i row1 = _mm256_castsi128_si256( + _mm_loadu_si128((__m128i *)&src_ptr[(i + 1) * src_pitch + 2])); + + s[0] = _mm256_inserti128_si256(row0, _mm256_castsi256_si128(row1), 1); + s[1] = _mm256_alignr_epi8(s[0], s[0], 4); + + s[0] = _mm256_shuffle_epi8(s[0], mask); + s[1] = _mm256_shuffle_epi8(s[1], mask); + + __m256i res = convolve_4tap(s, ff); + res = + _mm256_srai_epi32(_mm256_add_epi32(res, rounding), CONV8_ROUNDING_BITS); + + res = _mm256_packs_epi32(res, res); + res = _mm256_min_epi16(res, clip_pixel); + res = _mm256_max_epi16(res, zero); + + _mm_storel_epi64((__m128i *)&dst_ptr[i * dst_pitch], + _mm256_castsi256_si128(res)); + _mm_storel_epi64((__m128i *)&dst_ptr[(i + 1) * dst_pitch], + _mm256_extracti128_si256(res, 1)); + } + if (height % 2 != 0) { + i = height - 1; + const __m256i row0_0 = _mm256_castsi128_si256( + _mm_loadu_si128((__m128i *)&src_ptr[i * src_pitch + 2])); + const __m256i row0_1 = _mm256_castsi128_si256( + _mm_loadu_si128((__m128i *)&src_ptr[i * src_pitch + 6])); + + const __m256i r0 = + _mm256_inserti128_si256(row0_0, _mm256_castsi256_si128(row0_1), 1); + + s[0] = _mm256_shuffle_epi8(r0, ip_mask_f2f3); + s[1] = _mm256_shuffle_epi8(r0, ip_mask_f4f5); + + __m256i res = convolve_4tap(s, ff); + res = + _mm256_srai_epi32(_mm256_add_epi32(res, rounding), CONV8_ROUNDING_BITS); + + res = _mm256_packs_epi32(res, res); + res = _mm256_min_epi16(res, clip_pixel); + res = _mm256_max_epi16(res, zero); + + _mm_storel_epi64((__m128i *)&dst_ptr[i * dst_pitch], + _mm256_castsi256_si128(res)); + } +} + +static void aom_highbd_filter_block1d8_h4_avx2( + const uint16_t *src_ptr, ptrdiff_t src_pitch, uint16_t *dst_ptr, + ptrdiff_t dst_pitch, uint32_t height, const int16_t *filter, int bd) { + const __m256i rounding = _mm256_set1_epi32(1 << (CONV8_ROUNDING_BITS - 1)); + __m256i ff[2], s[2]; + uint32_t i = 0; + const __m256i clip_pixel = + _mm256_set1_epi16(bd == 10 ? 1023 : (bd == 12 ? 4095 : 255)); + const __m256i zero = _mm256_setzero_si256(); + + static const uint8_t shuffle_mask[32] = { 0, 1, 8, 9, 2, 3, 10, 11, + 4, 5, 12, 13, 6, 7, 14, 15, + 0, 1, 8, 9, 2, 3, 10, 11, + 4, 5, 12, 13, 6, 7, 14, 15 }; + + __m256i mask = _mm256_loadu_si256((__m256i *)shuffle_mask); + __m256i ip_mask_f2f3 = _mm256_loadu_si256((__m256i *)ip_shuffle_f2f3); + __m256i ip_mask_f4f5 = _mm256_loadu_si256((__m256i *)ip_shuffle_f4f5); + + pack_filters_4tap(filter, ff); + src_ptr -= 3; + + /* Horizontal filter */ + + for (i = 0; i <= (height - 2); i += 2) { + const __m256i row0 = + _mm256_loadu_si256((__m256i *)&src_ptr[i * src_pitch + 2]); + __m256i row1 = + _mm256_loadu_si256((__m256i *)&src_ptr[(i + 1) * src_pitch + 2]); + + const __m256i r0 = + _mm256_inserti128_si256(row0, _mm256_castsi256_si128(row1), 1); + const __m256i r1 = _mm256_permute2x128_si256(row0, row1, 0x31); + + // even pixels + s[0] = r0; + s[1] = _mm256_alignr_epi8(r1, r0, 4); + + __m256i res_even = convolve_4tap(s, ff); + res_even = _mm256_srai_epi32(_mm256_add_epi32(res_even, rounding), + CONV8_ROUNDING_BITS); + + // odd pixels + s[0] = _mm256_alignr_epi8(r1, r0, 2); + s[1] = _mm256_alignr_epi8(r1, r0, 6); + + __m256i res_odd = convolve_4tap(s, ff); + res_odd = _mm256_srai_epi32(_mm256_add_epi32(res_odd, rounding), + CONV8_ROUNDING_BITS); + + __m256i res = _mm256_packs_epi32(res_even, res_odd); + res = _mm256_shuffle_epi8(res, mask); + + res = _mm256_min_epi16(res, clip_pixel); + res = _mm256_max_epi16(res, zero); + + _mm_storeu_si128((__m128i *)&dst_ptr[i * dst_pitch], + _mm256_castsi256_si128(res)); + _mm_storeu_si128((__m128i *)&dst_ptr[i * dst_pitch + dst_pitch], + _mm256_extracti128_si256(res, 1)); + } + + if (height % 2 != 0) { + i = height - 1; + const __m256i row0_0 = + _mm256_loadu_si256((__m256i *)&src_ptr[i * src_pitch + 2]); + const __m256i row0_1 = + _mm256_loadu_si256((__m256i *)&src_ptr[i * src_pitch + 6]); + + const __m256i r0 = + _mm256_inserti128_si256(row0_0, _mm256_castsi256_si128(row0_1), 1); + + s[0] = _mm256_shuffle_epi8(r0, ip_mask_f2f3); + s[1] = _mm256_shuffle_epi8(r0, ip_mask_f4f5); + + __m256i res = convolve_4tap(s, ff); + res = + _mm256_srai_epi32(_mm256_add_epi32(res, rounding), CONV8_ROUNDING_BITS); + + res = _mm256_packs_epi32(res, res); + res = _mm256_min_epi16(res, clip_pixel); + res = _mm256_max_epi16(res, zero); + + _mm_storel_epi64((__m128i *)&dst_ptr[i * dst_pitch], + _mm256_castsi256_si128(res)); + _mm_storel_epi64((__m128i *)&dst_ptr[i * dst_pitch + 4], + _mm256_extracti128_si256(res, 1)); + } +} + +static void aom_highbd_filter_block1d16_h4_avx2( + const uint16_t *src_ptr, ptrdiff_t src_pitch, uint16_t *dst_ptr, + ptrdiff_t dst_pitch, uint32_t height, const int16_t *filter, int bd) { + aom_highbd_filter_block1d8_h4_avx2(src_ptr, src_pitch, dst_ptr, dst_pitch, + height, filter, bd); + aom_highbd_filter_block1d8_h4_avx2(src_ptr + 8, src_pitch, dst_ptr + 8, + dst_pitch, height, filter, bd); +} + // ----------------------------------------------------------------------------- // 2-tap horizontal filtering @@ -875,6 +1064,142 @@ static void aom_highbd_filter_block1d16_v8_avx2( } while (height > 0); } +static void aom_highbd_filter_block1d4_v4_avx2( + const uint16_t *src_ptr, ptrdiff_t src_pitch, uint16_t *dst_ptr, + ptrdiff_t dst_pitch, uint32_t height, const int16_t *filter, int bd) { + const int bits = FILTER_BITS; + + const __m128i round_shift_bits = _mm_cvtsi32_si128(bits); + const __m256i round_const_bits = _mm256_set1_epi32((1 << bits) >> 1); + const __m256i clip_pixel = + _mm256_set1_epi32(bd == 10 ? 1023 : (bd == 12 ? 4095 : 255)); + const __m256i zero = _mm256_setzero_si256(); + uint32_t i; + __m256i s[2], ff[2]; + + pack_filters_4tap(filter, ff); + + const uint16_t *data = src_ptr; + /* Vertical filter */ + { + __m128i s2 = _mm_loadl_epi64((__m128i *)(data + 2 * src_pitch)); + __m128i s3 = _mm_loadl_epi64((__m128i *)(data + 3 * src_pitch)); + + __m256i s23 = _mm256_inserti128_si256(_mm256_castsi128_si256(s2), s3, 1); + + __m128i s4 = _mm_loadl_epi64((__m128i *)(data + 4 * src_pitch)); + + __m256i s34 = _mm256_inserti128_si256(_mm256_castsi128_si256(s3), s4, 1); + + s[0] = _mm256_unpacklo_epi16(s23, s34); + + for (i = 0; i < height; i += 2) { + data = &src_ptr[i * src_pitch]; + + __m128i s5 = _mm_loadl_epi64((__m128i *)(data + 5 * src_pitch)); + __m128i s6 = _mm_loadl_epi64((__m128i *)(data + 6 * src_pitch)); + + __m256i s45 = _mm256_inserti128_si256(_mm256_castsi128_si256(s4), s5, 1); + __m256i s56 = _mm256_inserti128_si256(_mm256_castsi128_si256(s5), s6, 1); + + s[1] = _mm256_unpacklo_epi16(s45, s56); + + const __m256i res_a = convolve_4tap(s, ff); + + __m256i res_a_round = _mm256_sra_epi32( + _mm256_add_epi32(res_a, round_const_bits), round_shift_bits); + + __m256i res_16bit = _mm256_min_epi32(res_a_round, clip_pixel); + res_16bit = _mm256_max_epi32(res_16bit, zero); + res_16bit = _mm256_packs_epi32(res_16bit, res_16bit); + + _mm_storel_epi64((__m128i *)&dst_ptr[i * dst_pitch], + _mm256_castsi256_si128(res_16bit)); + _mm_storel_epi64((__m128i *)&dst_ptr[i * dst_pitch + dst_pitch], + _mm256_extracti128_si256(res_16bit, 1)); + + s[0] = s[1]; + s4 = s6; + } + } +} + +static void aom_highbd_filter_block1d8_v4_avx2( + const uint16_t *src_ptr, ptrdiff_t src_pitch, uint16_t *dst_ptr, + ptrdiff_t dst_pitch, uint32_t height, const int16_t *filter, int bd) { + const int bits = FILTER_BITS; + + const __m128i round_shift_bits = _mm_cvtsi32_si128(bits); + const __m256i round_const_bits = _mm256_set1_epi32((1 << bits) >> 1); + const __m256i clip_pixel = + _mm256_set1_epi16(bd == 10 ? 1023 : (bd == 12 ? 4095 : 255)); + const __m256i zero = _mm256_setzero_si256(); + __m256i s[4], ff[2]; + uint32_t i; + pack_filters_4tap(filter, ff); + + const uint16_t *data = src_ptr; + /* Vertical filter */ + { + __m128i s2 = _mm_loadu_si128((__m128i *)(data + 2 * src_pitch)); + __m128i s3 = _mm_loadu_si128((__m128i *)(data + 3 * src_pitch)); + + __m256i s23 = _mm256_inserti128_si256(_mm256_castsi128_si256(s2), s3, 1); + + __m128i s4 = _mm_loadu_si128((__m128i *)(data + 4 * src_pitch)); + + __m256i s34 = _mm256_inserti128_si256(_mm256_castsi128_si256(s3), s4, 1); + + s[0] = _mm256_unpacklo_epi16(s23, s34); + s[2] = _mm256_unpackhi_epi16(s23, s34); + + for (i = 0; i < height; i += 2) { + data = &src_ptr[i * src_pitch]; + + __m128i s5 = _mm_loadu_si128((__m128i *)(data + 5 * src_pitch)); + __m128i s6 = _mm_loadu_si128((__m128i *)(data + 6 * src_pitch)); + + __m256i s45 = _mm256_inserti128_si256(_mm256_castsi128_si256(s4), s5, 1); + __m256i s56 = _mm256_inserti128_si256(_mm256_castsi128_si256(s5), s6, 1); + + s[1] = _mm256_unpacklo_epi16(s45, s56); + s[3] = _mm256_unpackhi_epi16(s45, s56); + + const __m256i res_a = convolve_4tap(s, ff); + + __m256i res_a_round = _mm256_sra_epi32( + _mm256_add_epi32(res_a, round_const_bits), round_shift_bits); + + const __m256i res_b = convolve_4tap(s + 2, ff); + __m256i res_b_round = _mm256_sra_epi32( + _mm256_add_epi32(res_b, round_const_bits), round_shift_bits); + + __m256i res_16bit = _mm256_packs_epi32(res_a_round, res_b_round); + res_16bit = _mm256_min_epi16(res_16bit, clip_pixel); + res_16bit = _mm256_max_epi16(res_16bit, zero); + + _mm_storeu_si128((__m128i *)&dst_ptr[i * dst_pitch], + _mm256_castsi256_si128(res_16bit)); + _mm_storeu_si128((__m128i *)&dst_ptr[i * dst_pitch + dst_pitch], + _mm256_extracti128_si256(res_16bit, 1)); + + s[0] = s[1]; + s[2] = s[3]; + s4 = s6; + } + } +} + +static void aom_highbd_filter_block1d16_v4_avx2( + const uint16_t *src_ptr, ptrdiff_t src_pitch, uint16_t *dst_ptr, + ptrdiff_t dst_pitch, uint32_t height, const int16_t *filter, int bd) { + aom_highbd_filter_block1d8_v4_avx2(src_ptr, src_pitch, dst_ptr, dst_pitch, + height, filter, bd); + + aom_highbd_filter_block1d8_v4_avx2(src_ptr + 8, src_pitch, dst_ptr + 8, + dst_pitch, height, filter, bd); +} + // ----------------------------------------------------------------------------- // 2-tap vertical filtering diff --git a/third_party/aom/aom_dsp/x86/highbd_convolve_sse2.c b/third_party/aom/aom_dsp/x86/highbd_convolve_sse2.c new file mode 100644 index 0000000000..a2bb283222 --- /dev/null +++ b/third_party/aom/aom_dsp/x86/highbd_convolve_sse2.c @@ -0,0 +1,351 @@ +/* + * Copyright (c) 2018, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ +#include + +#include "config/aom_dsp_rtcd.h" +#include "aom_dsp/x86/convolve.h" + +// ----------------------------------------------------------------------------- + +void aom_highbd_filter_block1d4_v4_sse2(const uint16_t *src_ptr, + ptrdiff_t src_pitch, uint16_t *dst_ptr, + ptrdiff_t dst_pitch, uint32_t height, + const int16_t *filter, int bd) { + __m128i filtersReg; + __m128i srcReg2, srcReg3, srcReg4, srcReg5, srcReg6; + __m128i srcReg23_lo, srcReg34_lo; + __m128i srcReg45_lo, srcReg56_lo; + __m128i resReg23_lo, resReg34_lo, resReg45_lo, resReg56_lo; + __m128i resReg23_45_lo, resReg34_56_lo; + __m128i resReg23_45, resReg34_56; + __m128i addFilterReg64, secondFilters, thirdFilters; + unsigned int i; + ptrdiff_t src_stride, dst_stride; + + const __m128i max = _mm_set1_epi16((1 << bd) - 1); + addFilterReg64 = _mm_set1_epi32(64); + filtersReg = _mm_loadu_si128((const __m128i *)filter); + + // coeffs 0 1 0 1 2 3 2 3 + const __m128i tmp0 = _mm_unpacklo_epi32(filtersReg, filtersReg); + // coeffs 4 5 4 5 6 7 6 7 + const __m128i tmp1 = _mm_unpackhi_epi32(filtersReg, filtersReg); + + secondFilters = _mm_unpackhi_epi64(tmp0, tmp0); // coeffs 2 3 2 3 2 3 2 3 + thirdFilters = _mm_unpacklo_epi64(tmp1, tmp1); // coeffs 4 5 4 5 4 5 4 5 + + // multiply the size of the source and destination stride by two + src_stride = src_pitch << 1; + dst_stride = dst_pitch << 1; + + srcReg2 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 2)); + srcReg3 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 3)); + srcReg23_lo = _mm_unpacklo_epi16(srcReg2, srcReg3); + + srcReg4 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 4)); + srcReg34_lo = _mm_unpacklo_epi16(srcReg3, srcReg4); + + for (i = height; i > 1; i -= 2) { + srcReg5 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 5)); + srcReg45_lo = _mm_unpacklo_epi16(srcReg4, srcReg5); + + srcReg6 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 6)); + srcReg56_lo = _mm_unpacklo_epi16(srcReg5, srcReg6); + + // multiply 2 adjacent elements with the filter and add the result + + resReg23_lo = _mm_madd_epi16(srcReg23_lo, secondFilters); + resReg34_lo = _mm_madd_epi16(srcReg34_lo, secondFilters); + resReg45_lo = _mm_madd_epi16(srcReg45_lo, thirdFilters); + resReg56_lo = _mm_madd_epi16(srcReg56_lo, thirdFilters); + + resReg23_45_lo = _mm_add_epi32(resReg23_lo, resReg45_lo); + resReg34_56_lo = _mm_add_epi32(resReg34_lo, resReg56_lo); + + // shift by 7 bit each 32 bit + resReg23_45_lo = _mm_add_epi32(resReg23_45_lo, addFilterReg64); + resReg34_56_lo = _mm_add_epi32(resReg34_56_lo, addFilterReg64); + resReg23_45_lo = _mm_srai_epi32(resReg23_45_lo, 7); + resReg34_56_lo = _mm_srai_epi32(resReg34_56_lo, 7); + + // shrink to 16 bit each 32 bits, the first lane contain the first + // convolve result and the second lane contain the second convolve + // result + resReg23_45 = _mm_packs_epi32(resReg23_45_lo, _mm_setzero_si128()); + resReg34_56 = _mm_packs_epi32(resReg34_56_lo, _mm_setzero_si128()); + + resReg23_45 = _mm_max_epi16(resReg23_45, _mm_setzero_si128()); + resReg23_45 = _mm_min_epi16(resReg23_45, max); + resReg34_56 = _mm_max_epi16(resReg34_56, _mm_setzero_si128()); + resReg34_56 = _mm_min_epi16(resReg34_56, max); + + src_ptr += src_stride; + + _mm_storel_epi64((__m128i *)dst_ptr, (resReg23_45)); + _mm_storel_epi64((__m128i *)(dst_ptr + dst_pitch), (resReg34_56)); + + dst_ptr += dst_stride; + + // save part of the registers for next strides + srcReg23_lo = srcReg45_lo; + srcReg34_lo = srcReg56_lo; + srcReg4 = srcReg6; + } +} + +void aom_highbd_filter_block1d4_h4_sse2(const uint16_t *src_ptr, + ptrdiff_t src_pitch, uint16_t *dst_ptr, + ptrdiff_t dst_pitch, uint32_t height, + const int16_t *filter, int bd) { + __m128i filtersReg; + __m128i addFilterReg64; + __m128i secondFilters, thirdFilters; + __m128i srcRegFilt32b1_1; + __m128i srcReg32b1; + unsigned int i; + src_ptr -= 3; + addFilterReg64 = _mm_set1_epi32(64); + filtersReg = _mm_loadu_si128((const __m128i *)filter); + const __m128i max = _mm_set1_epi16((1 << bd) - 1); + + // coeffs 0 1 0 1 2 3 2 3 + const __m128i tmp_0 = _mm_unpacklo_epi32(filtersReg, filtersReg); + // coeffs 4 5 4 5 6 7 6 7 + const __m128i tmp_1 = _mm_unpackhi_epi32(filtersReg, filtersReg); + + secondFilters = _mm_unpackhi_epi64(tmp_0, tmp_0); // coeffs 2 3 2 3 2 3 2 3 + thirdFilters = _mm_unpacklo_epi64(tmp_1, tmp_1); // coeffs 4 5 4 5 4 5 4 5 + + for (i = height; i > 0; i -= 1) { + srcReg32b1 = _mm_loadu_si128((const __m128i *)(src_ptr + 2)); + + __m128i ss_3_1 = _mm_srli_si128(srcReg32b1, 2); + __m128i ss_4_1 = _mm_srli_si128(srcReg32b1, 4); + __m128i ss_5_1 = _mm_srli_si128(srcReg32b1, 6); + __m128i ss_23 = _mm_unpacklo_epi32(srcReg32b1, ss_3_1); + __m128i ss_45 = _mm_unpacklo_epi32(ss_4_1, ss_5_1); + + ss_23 = _mm_madd_epi16(ss_23, secondFilters); + ss_45 = _mm_madd_epi16(ss_45, thirdFilters); + srcRegFilt32b1_1 = _mm_add_epi32(ss_23, ss_45); + + // shift by 7 bit each 32 bit + srcRegFilt32b1_1 = _mm_add_epi32(srcRegFilt32b1_1, addFilterReg64); + srcRegFilt32b1_1 = _mm_srai_epi32(srcRegFilt32b1_1, 7); + + srcRegFilt32b1_1 = _mm_packs_epi32(srcRegFilt32b1_1, _mm_setzero_si128()); + srcRegFilt32b1_1 = _mm_max_epi16(srcRegFilt32b1_1, _mm_setzero_si128()); + srcRegFilt32b1_1 = _mm_min_epi16(srcRegFilt32b1_1, max); + + src_ptr += src_pitch; + + _mm_storel_epi64((__m128i *)dst_ptr, srcRegFilt32b1_1); + + dst_ptr += dst_pitch; + } +} + +void aom_highbd_filter_block1d8_v4_sse2(const uint16_t *src_ptr, + ptrdiff_t src_pitch, uint16_t *dst_ptr, + ptrdiff_t dst_pitch, uint32_t height, + const int16_t *filter, int bd) { + __m128i filtersReg; + __m128i srcReg2, srcReg3, srcReg4, srcReg5, srcReg6; + __m128i srcReg23_lo, srcReg23_hi, srcReg34_lo, srcReg34_hi; + __m128i srcReg45_lo, srcReg45_hi, srcReg56_lo, srcReg56_hi; + __m128i resReg23_lo, resReg34_lo, resReg45_lo, resReg56_lo; + __m128i resReg23_hi, resReg34_hi, resReg45_hi, resReg56_hi; + __m128i resReg23_45_lo, resReg34_56_lo, resReg23_45_hi, resReg34_56_hi; + __m128i resReg23_45, resReg34_56; + __m128i addFilterReg64, secondFilters, thirdFilters; + unsigned int i; + ptrdiff_t src_stride, dst_stride; + + const __m128i max = _mm_set1_epi16((1 << bd) - 1); + addFilterReg64 = _mm_set1_epi32(64); + filtersReg = _mm_loadu_si128((const __m128i *)filter); + + // coeffs 0 1 0 1 2 3 2 3 + const __m128i tmp0 = _mm_unpacklo_epi32(filtersReg, filtersReg); + // coeffs 4 5 4 5 6 7 6 7 + const __m128i tmp1 = _mm_unpackhi_epi32(filtersReg, filtersReg); + + secondFilters = _mm_unpackhi_epi64(tmp0, tmp0); // coeffs 2 3 2 3 2 3 2 3 + thirdFilters = _mm_unpacklo_epi64(tmp1, tmp1); // coeffs 4 5 4 5 4 5 4 5 + + // multiple the size of the source and destination stride by two + src_stride = src_pitch << 1; + dst_stride = dst_pitch << 1; + + srcReg2 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 2)); + srcReg3 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 3)); + srcReg23_lo = _mm_unpacklo_epi16(srcReg2, srcReg3); + srcReg23_hi = _mm_unpackhi_epi16(srcReg2, srcReg3); + + srcReg4 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 4)); + srcReg34_lo = _mm_unpacklo_epi16(srcReg3, srcReg4); + srcReg34_hi = _mm_unpackhi_epi16(srcReg3, srcReg4); + + for (i = height; i > 1; i -= 2) { + srcReg5 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 5)); + + srcReg45_lo = _mm_unpacklo_epi16(srcReg4, srcReg5); + srcReg45_hi = _mm_unpackhi_epi16(srcReg4, srcReg5); + + srcReg6 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 6)); + + srcReg56_lo = _mm_unpacklo_epi16(srcReg5, srcReg6); + srcReg56_hi = _mm_unpackhi_epi16(srcReg5, srcReg6); + + // multiply 2 adjacent elements with the filter and add the result + + resReg23_lo = _mm_madd_epi16(srcReg23_lo, secondFilters); + resReg34_lo = _mm_madd_epi16(srcReg34_lo, secondFilters); + resReg45_lo = _mm_madd_epi16(srcReg45_lo, thirdFilters); + resReg56_lo = _mm_madd_epi16(srcReg56_lo, thirdFilters); + + resReg23_45_lo = _mm_add_epi32(resReg23_lo, resReg45_lo); + resReg34_56_lo = _mm_add_epi32(resReg34_lo, resReg56_lo); + + // multiply 2 adjacent elements with the filter and add the result + + resReg23_hi = _mm_madd_epi16(srcReg23_hi, secondFilters); + resReg34_hi = _mm_madd_epi16(srcReg34_hi, secondFilters); + resReg45_hi = _mm_madd_epi16(srcReg45_hi, thirdFilters); + resReg56_hi = _mm_madd_epi16(srcReg56_hi, thirdFilters); + + resReg23_45_hi = _mm_add_epi32(resReg23_hi, resReg45_hi); + resReg34_56_hi = _mm_add_epi32(resReg34_hi, resReg56_hi); + + // shift by 7 bit each 32 bit + resReg23_45_lo = _mm_add_epi32(resReg23_45_lo, addFilterReg64); + resReg34_56_lo = _mm_add_epi32(resReg34_56_lo, addFilterReg64); + resReg23_45_hi = _mm_add_epi32(resReg23_45_hi, addFilterReg64); + resReg34_56_hi = _mm_add_epi32(resReg34_56_hi, addFilterReg64); + resReg23_45_lo = _mm_srai_epi32(resReg23_45_lo, 7); + resReg34_56_lo = _mm_srai_epi32(resReg34_56_lo, 7); + resReg23_45_hi = _mm_srai_epi32(resReg23_45_hi, 7); + resReg34_56_hi = _mm_srai_epi32(resReg34_56_hi, 7); + + // shrink to 16 bit each 32 bits, the first lane contain the first + // convolve result and the second lane contain the second convolve + // result + resReg23_45 = _mm_packs_epi32(resReg23_45_lo, resReg23_45_hi); + resReg34_56 = _mm_packs_epi32(resReg34_56_lo, resReg34_56_hi); + + resReg23_45 = _mm_max_epi16(resReg23_45, _mm_setzero_si128()); + resReg23_45 = _mm_min_epi16(resReg23_45, max); + resReg34_56 = _mm_max_epi16(resReg34_56, _mm_setzero_si128()); + resReg34_56 = _mm_min_epi16(resReg34_56, max); + + src_ptr += src_stride; + + _mm_store_si128((__m128i *)dst_ptr, (resReg23_45)); + _mm_store_si128((__m128i *)(dst_ptr + dst_pitch), (resReg34_56)); + + dst_ptr += dst_stride; + + // save part of the registers for next strides + srcReg23_lo = srcReg45_lo; + srcReg23_hi = srcReg45_hi; + srcReg34_lo = srcReg56_lo; + srcReg34_hi = srcReg56_hi; + srcReg4 = srcReg6; + } +} + +void aom_highbd_filter_block1d8_h4_sse2(const uint16_t *src_ptr, + ptrdiff_t src_pitch, uint16_t *dst_ptr, + ptrdiff_t dst_pitch, uint32_t height, + const int16_t *filter, int bd) { + __m128i filtersReg; + __m128i addFilterReg64; + __m128i secondFilters, thirdFilters; + __m128i srcRegFilt32b1_1, srcRegFilt32b1_2; + __m128i srcReg32b1, srcReg32b2; + unsigned int i; + src_ptr -= 3; + addFilterReg64 = _mm_set1_epi32(64); + filtersReg = _mm_loadu_si128((const __m128i *)filter); + const __m128i max = _mm_set1_epi16((1 << bd) - 1); + + // coeffs 0 1 0 1 2 3 2 3 + const __m128i tmp_0 = _mm_unpacklo_epi32(filtersReg, filtersReg); + // coeffs 4 5 4 5 6 7 6 7 + const __m128i tmp_1 = _mm_unpackhi_epi32(filtersReg, filtersReg); + + secondFilters = _mm_unpackhi_epi64(tmp_0, tmp_0); // coeffs 2 3 2 3 2 3 2 3 + thirdFilters = _mm_unpacklo_epi64(tmp_1, tmp_1); // coeffs 4 5 4 5 4 5 4 5 + + for (i = height; i > 0; i -= 1) { + srcReg32b1 = _mm_loadu_si128((const __m128i *)(src_ptr + 2)); + srcReg32b2 = _mm_loadu_si128((const __m128i *)(src_ptr + 6)); + + __m128i ss_4_1 = _mm_srli_si128(srcReg32b1, 4); + __m128i ss_4_2 = _mm_srli_si128(srcReg32b2, 4); + __m128i ss_4 = _mm_unpacklo_epi64(ss_4_1, ss_4_2); + + __m128i d1 = _mm_madd_epi16(srcReg32b1, secondFilters); + __m128i d2 = _mm_madd_epi16(ss_4, thirdFilters); + srcRegFilt32b1_1 = _mm_add_epi32(d1, d2); + + __m128i ss_3_1 = _mm_srli_si128(srcReg32b1, 2); + __m128i ss_5_1 = _mm_srli_si128(srcReg32b1, 6); + __m128i ss_3_2 = _mm_srli_si128(srcReg32b2, 2); + __m128i ss_5_2 = _mm_srli_si128(srcReg32b2, 6); + __m128i ss_3 = _mm_unpacklo_epi64(ss_3_1, ss_3_2); + __m128i ss_5 = _mm_unpacklo_epi64(ss_5_1, ss_5_2); + + d1 = _mm_madd_epi16(ss_3, secondFilters); + d2 = _mm_madd_epi16(ss_5, thirdFilters); + srcRegFilt32b1_2 = _mm_add_epi32(d1, d2); + + __m128i res_lo_1 = _mm_unpacklo_epi32(srcRegFilt32b1_1, srcRegFilt32b1_2); + __m128i res_hi_1 = _mm_unpackhi_epi32(srcRegFilt32b1_1, srcRegFilt32b1_2); + + // shift by 7 bit each 32 bit + res_lo_1 = _mm_add_epi32(res_lo_1, addFilterReg64); + res_hi_1 = _mm_add_epi32(res_hi_1, addFilterReg64); + res_lo_1 = _mm_srai_epi32(res_lo_1, 7); + res_hi_1 = _mm_srai_epi32(res_hi_1, 7); + + srcRegFilt32b1_1 = _mm_packs_epi32(res_lo_1, res_hi_1); + + srcRegFilt32b1_1 = _mm_max_epi16(srcRegFilt32b1_1, _mm_setzero_si128()); + srcRegFilt32b1_1 = _mm_min_epi16(srcRegFilt32b1_1, max); + + src_ptr += src_pitch; + + _mm_store_si128((__m128i *)dst_ptr, srcRegFilt32b1_1); + + dst_ptr += dst_pitch; + } +} + +void aom_highbd_filter_block1d16_v4_sse2(const uint16_t *src_ptr, + ptrdiff_t src_pitch, uint16_t *dst_ptr, + ptrdiff_t dst_pitch, uint32_t height, + const int16_t *filter, int bd) { + aom_highbd_filter_block1d8_v4_sse2(src_ptr, src_pitch, dst_ptr, dst_pitch, + height, filter, bd); + aom_highbd_filter_block1d8_v4_sse2((src_ptr + 8), src_pitch, (dst_ptr + 8), + dst_pitch, height, filter, bd); +} + +void aom_highbd_filter_block1d16_h4_sse2(const uint16_t *src_ptr, + ptrdiff_t src_pitch, uint16_t *dst_ptr, + ptrdiff_t dst_pitch, uint32_t height, + const int16_t *filter, int bd) { + aom_highbd_filter_block1d8_h4_sse2(src_ptr, src_pitch, dst_ptr, dst_pitch, + height, filter, bd); + aom_highbd_filter_block1d8_h4_sse2((src_ptr + 8), src_pitch, (dst_ptr + 8), + dst_pitch, height, filter, bd); +} diff --git a/third_party/aom/aom_dsp/x86/highbd_intrapred_asm_sse2.asm b/third_party/aom/aom_dsp/x86/highbd_intrapred_asm_sse2.asm new file mode 100644 index 0000000000..91b3d126ca --- /dev/null +++ b/third_party/aom/aom_dsp/x86/highbd_intrapred_asm_sse2.asm @@ -0,0 +1,259 @@ +; +; Copyright (c) 2016, Alliance for Open Media. All rights reserved +; +; This source code is subject to the terms of the BSD 2 Clause License and +; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +; was not distributed with this source code in the LICENSE file, you can +; obtain it at www.aomedia.org/license/software. If the Alliance for Open +; Media Patent License 1.0 was not distributed with this source code in the +; PATENTS file, you can obtain it at www.aomedia.org/license/patent. +; + +; + +%include "third_party/x86inc/x86inc.asm" + +SECTION_RODATA +pw_4: times 8 dw 4 +pw_8: times 8 dw 8 +pw_16: times 4 dd 16 +pw_32: times 4 dd 32 + +SECTION .text +INIT_XMM sse2 +cglobal highbd_dc_predictor_4x4, 4, 5, 4, dst, stride, above, left, goffset + GET_GOT goffsetq + + movq m0, [aboveq] + movq m2, [leftq] + paddw m0, m2 + pshuflw m1, m0, 0xe + paddw m0, m1 + pshuflw m1, m0, 0x1 + paddw m0, m1 + paddw m0, [GLOBAL(pw_4)] + psraw m0, 3 + pshuflw m0, m0, 0x0 + movq [dstq ], m0 + movq [dstq+strideq*2], m0 + lea dstq, [dstq+strideq*4] + movq [dstq ], m0 + movq [dstq+strideq*2], m0 + + RESTORE_GOT + RET + +INIT_XMM sse2 +cglobal highbd_dc_predictor_8x8, 4, 5, 4, dst, stride, above, left, goffset + GET_GOT goffsetq + + pxor m1, m1 + mova m0, [aboveq] + mova m2, [leftq] + DEFINE_ARGS dst, stride, stride3, one + mov oned, 0x00010001 + lea stride3q, [strideq*3] + movd m3, oned + pshufd m3, m3, 0x0 + paddw m0, m2 + pmaddwd m0, m3 + packssdw m0, m1 + pmaddwd m0, m3 + packssdw m0, m1 + pmaddwd m0, m3 + paddw m0, [GLOBAL(pw_8)] + psrlw m0, 4 + pshuflw m0, m0, 0x0 + punpcklqdq m0, m0 + mova [dstq ], m0 + mova [dstq+strideq*2 ], m0 + mova [dstq+strideq*4 ], m0 + mova [dstq+stride3q*2], m0 + lea dstq, [dstq+strideq*8] + mova [dstq ], m0 + mova [dstq+strideq*2 ], m0 + mova [dstq+strideq*4 ], m0 + mova [dstq+stride3q*2], m0 + + RESTORE_GOT + RET + +INIT_XMM sse2 +cglobal highbd_dc_predictor_16x16, 4, 5, 5, dst, stride, above, left, goffset + GET_GOT goffsetq + + pxor m1, m1 + mova m0, [aboveq] + mova m3, [aboveq+16] + mova m2, [leftq] + mova m4, [leftq+16] + DEFINE_ARGS dst, stride, stride3, lines4 + lea stride3q, [strideq*3] + mov lines4d, 4 + paddw m0, m2 + paddw m0, m3 + paddw m0, m4 + movhlps m2, m0 + paddw m0, m2 + punpcklwd m0, m1 + movhlps m2, m0 + paddd m0, m2 + punpckldq m0, m1 + movhlps m2, m0 + paddd m0, m2 + paddd m0, [GLOBAL(pw_16)] + psrad m0, 5 + pshuflw m0, m0, 0x0 + punpcklqdq m0, m0 +.loop: + mova [dstq ], m0 + mova [dstq +16], m0 + mova [dstq+strideq*2 ], m0 + mova [dstq+strideq*2 +16], m0 + mova [dstq+strideq*4 ], m0 + mova [dstq+strideq*4 +16], m0 + mova [dstq+stride3q*2 ], m0 + mova [dstq+stride3q*2+16], m0 + lea dstq, [dstq+strideq*8] + dec lines4d + jnz .loop + + RESTORE_GOT + REP_RET + +INIT_XMM sse2 +cglobal highbd_dc_predictor_32x32, 4, 5, 7, dst, stride, above, left, goffset + GET_GOT goffsetq + + mova m0, [aboveq] + mova m2, [aboveq+16] + mova m3, [aboveq+32] + mova m4, [aboveq+48] + paddw m0, m2 + paddw m3, m4 + mova m2, [leftq] + mova m4, [leftq+16] + mova m5, [leftq+32] + mova m6, [leftq+48] + paddw m2, m4 + paddw m5, m6 + paddw m0, m3 + paddw m2, m5 + pxor m1, m1 + paddw m0, m2 + DEFINE_ARGS dst, stride, stride3, lines4 + lea stride3q, [strideq*3] + mov lines4d, 8 + movhlps m2, m0 + paddw m0, m2 + punpcklwd m0, m1 + movhlps m2, m0 + paddd m0, m2 + punpckldq m0, m1 + movhlps m2, m0 + paddd m0, m2 + paddd m0, [GLOBAL(pw_32)] + psrad m0, 6 + pshuflw m0, m0, 0x0 + punpcklqdq m0, m0 +.loop: + mova [dstq ], m0 + mova [dstq +16 ], m0 + mova [dstq +32 ], m0 + mova [dstq +48 ], m0 + mova [dstq+strideq*2 ], m0 + mova [dstq+strideq*2+16 ], m0 + mova [dstq+strideq*2+32 ], m0 + mova [dstq+strideq*2+48 ], m0 + mova [dstq+strideq*4 ], m0 + mova [dstq+strideq*4+16 ], m0 + mova [dstq+strideq*4+32 ], m0 + mova [dstq+strideq*4+48 ], m0 + mova [dstq+stride3q*2 ], m0 + mova [dstq+stride3q*2 +16], m0 + mova [dstq+stride3q*2 +32], m0 + mova [dstq+stride3q*2 +48], m0 + lea dstq, [dstq+strideq*8] + dec lines4d + jnz .loop + + RESTORE_GOT + REP_RET + +INIT_XMM sse2 +cglobal highbd_v_predictor_4x4, 3, 3, 1, dst, stride, above + movq m0, [aboveq] + movq [dstq ], m0 + movq [dstq+strideq*2], m0 + lea dstq, [dstq+strideq*4] + movq [dstq ], m0 + movq [dstq+strideq*2], m0 + RET + +INIT_XMM sse2 +cglobal highbd_v_predictor_8x8, 3, 3, 1, dst, stride, above + mova m0, [aboveq] + DEFINE_ARGS dst, stride, stride3 + lea stride3q, [strideq*3] + mova [dstq ], m0 + mova [dstq+strideq*2 ], m0 + mova [dstq+strideq*4 ], m0 + mova [dstq+stride3q*2], m0 + lea dstq, [dstq+strideq*8] + mova [dstq ], m0 + mova [dstq+strideq*2 ], m0 + mova [dstq+strideq*4 ], m0 + mova [dstq+stride3q*2], m0 + RET + +INIT_XMM sse2 +cglobal highbd_v_predictor_16x16, 3, 4, 2, dst, stride, above + mova m0, [aboveq] + mova m1, [aboveq+16] + DEFINE_ARGS dst, stride, stride3, nlines4 + lea stride3q, [strideq*3] + mov nlines4d, 4 +.loop: + mova [dstq ], m0 + mova [dstq +16], m1 + mova [dstq+strideq*2 ], m0 + mova [dstq+strideq*2 +16], m1 + mova [dstq+strideq*4 ], m0 + mova [dstq+strideq*4 +16], m1 + mova [dstq+stride3q*2 ], m0 + mova [dstq+stride3q*2+16], m1 + lea dstq, [dstq+strideq*8] + dec nlines4d + jnz .loop + REP_RET + +INIT_XMM sse2 +cglobal highbd_v_predictor_32x32, 3, 4, 4, dst, stride, above + mova m0, [aboveq] + mova m1, [aboveq+16] + mova m2, [aboveq+32] + mova m3, [aboveq+48] + DEFINE_ARGS dst, stride, stride3, nlines4 + lea stride3q, [strideq*3] + mov nlines4d, 8 +.loop: + mova [dstq ], m0 + mova [dstq +16], m1 + mova [dstq +32], m2 + mova [dstq +48], m3 + mova [dstq+strideq*2 ], m0 + mova [dstq+strideq*2 +16], m1 + mova [dstq+strideq*2 +32], m2 + mova [dstq+strideq*2 +48], m3 + mova [dstq+strideq*4 ], m0 + mova [dstq+strideq*4 +16], m1 + mova [dstq+strideq*4 +32], m2 + mova [dstq+strideq*4 +48], m3 + mova [dstq+stride3q*2 ], m0 + mova [dstq+stride3q*2 +16], m1 + mova [dstq+stride3q*2 +32], m2 + mova [dstq+stride3q*2 +48], m3 + lea dstq, [dstq+strideq*8] + dec nlines4d + jnz .loop + REP_RET diff --git a/third_party/aom/aom_dsp/x86/highbd_loopfilter_sse2.c b/third_party/aom/aom_dsp/x86/highbd_loopfilter_sse2.c index 097e0778ff..70b91c651d 100644 --- a/third_party/aom/aom_dsp/x86/highbd_loopfilter_sse2.c +++ b/third_party/aom/aom_dsp/x86/highbd_loopfilter_sse2.c @@ -727,8 +727,8 @@ void aom_highbd_lpf_horizontal_14_dual_sse2( _limit1, _thresh1, bd); for (i = 0; i < 6; i++) { - _mm_store_si128((__m128i *)(s - (i + 1) * pitch), p[i]); - _mm_store_si128((__m128i *)(s + i * pitch), q[i]); + _mm_storeu_si128((__m128i *)(s - (i + 1) * pitch), p[i]); + _mm_storeu_si128((__m128i *)(s + i * pitch), q[i]); } } diff --git a/third_party/aom/aom_dsp/x86/highbd_subpel_variance_impl_sse2.asm b/third_party/aom/aom_dsp/x86/highbd_subpel_variance_impl_sse2.asm index 61f5b8e865..5c78933df5 100644 --- a/third_party/aom/aom_dsp/x86/highbd_subpel_variance_impl_sse2.asm +++ b/third_party/aom/aom_dsp/x86/highbd_subpel_variance_impl_sse2.asm @@ -114,45 +114,33 @@ SECTION .text cglobal highbd_sub_pixel_avg_variance%1xh, 7, 7, 13, src, src_stride, \ x_offset, y_offset, \ dst, dst_stride, \ - sec, sec_stride, height, sse, \ - g_bilin_filter, g_pw_8 + sec, sec_stride, height, sse %define block_height dword heightm %define sec_str sec_stridemp - - ; Store bilin_filter and pw_8 location in stack - %if GET_GOT_DEFINED == 1 - GET_GOT eax - add esp, 4 ; restore esp - %endif - - lea ecx, [GLOBAL(bilin_filter_m)] - mov g_bilin_filterm, ecx - - lea ecx, [GLOBAL(pw_8)] - mov g_pw_8m, ecx - - LOAD_IF_USED 0, 1 ; load eax, ecx back %else cglobal highbd_sub_pixel_variance%1xh, 7, 7, 13, src, src_stride, \ x_offset, y_offset, \ - dst, dst_stride, height, sse, \ - g_bilin_filter, g_pw_8 + dst, dst_stride, height, sse %define block_height heightd - - ; Store bilin_filter and pw_8 location in stack - %if GET_GOT_DEFINED == 1 - GET_GOT eax - add esp, 4 ; restore esp - %endif - - lea ecx, [GLOBAL(bilin_filter_m)] - mov g_bilin_filterm, ecx - - lea ecx, [GLOBAL(pw_8)] - mov g_pw_8m, ecx - - LOAD_IF_USED 0, 1 ; load eax, ecx back %endif + + ; reuse argument stack space + %define g_bilin_filterm x_offsetm + %define g_pw_8m y_offsetm + + ; Store bilin_filter and pw_8 location in stack + %if GET_GOT_DEFINED == 1 + GET_GOT eax + add esp, 4 ; restore esp + %endif + + lea ecx, [GLOBAL(bilin_filter_m)] + mov g_bilin_filterm, ecx + + lea ecx, [GLOBAL(pw_8)] + mov g_pw_8m, ecx + + LOAD_IF_USED 0, 1 ; load eax, ecx back %else %if %2 == 1 ; avg cglobal highbd_sub_pixel_avg_variance%1xh, 7, 7, 13, src, src_stride, \ diff --git a/third_party/aom/aom_dsp/x86/highbd_subtract_sse2.c b/third_party/aom/aom_dsp/x86/highbd_subtract_sse2.c index 18eb03d12c..b72d1cf8ba 100644 --- a/third_party/aom/aom_dsp/x86/highbd_subtract_sse2.c +++ b/third_party/aom/aom_dsp/x86/highbd_subtract_sse2.c @@ -29,15 +29,15 @@ static void subtract_4x4(int16_t *diff, ptrdiff_t diff_stride, __m128i x0, x1, x2, x3; int64_t *store_diff = (int64_t *)(diff + 0 * diff_stride); - u0 = _mm_loadu_si128((__m128i const *)(src + 0 * src_stride)); - u1 = _mm_loadu_si128((__m128i const *)(src + 1 * src_stride)); - u2 = _mm_loadu_si128((__m128i const *)(src + 2 * src_stride)); - u3 = _mm_loadu_si128((__m128i const *)(src + 3 * src_stride)); + u0 = _mm_loadl_epi64((__m128i const *)(src + 0 * src_stride)); + u1 = _mm_loadl_epi64((__m128i const *)(src + 1 * src_stride)); + u2 = _mm_loadl_epi64((__m128i const *)(src + 2 * src_stride)); + u3 = _mm_loadl_epi64((__m128i const *)(src + 3 * src_stride)); - v0 = _mm_loadu_si128((__m128i const *)(pred + 0 * pred_stride)); - v1 = _mm_loadu_si128((__m128i const *)(pred + 1 * pred_stride)); - v2 = _mm_loadu_si128((__m128i const *)(pred + 2 * pred_stride)); - v3 = _mm_loadu_si128((__m128i const *)(pred + 3 * pred_stride)); + v0 = _mm_loadl_epi64((__m128i const *)(pred + 0 * pred_stride)); + v1 = _mm_loadl_epi64((__m128i const *)(pred + 1 * pred_stride)); + v2 = _mm_loadl_epi64((__m128i const *)(pred + 2 * pred_stride)); + v3 = _mm_loadl_epi64((__m128i const *)(pred + 3 * pred_stride)); x0 = _mm_sub_epi16(u0, v0); x1 = _mm_sub_epi16(u1, v1); @@ -61,23 +61,23 @@ static void subtract_4x8(int16_t *diff, ptrdiff_t diff_stride, __m128i x0, x1, x2, x3, x4, x5, x6, x7; int64_t *store_diff = (int64_t *)(diff + 0 * diff_stride); - u0 = _mm_loadu_si128((__m128i const *)(src + 0 * src_stride)); - u1 = _mm_loadu_si128((__m128i const *)(src + 1 * src_stride)); - u2 = _mm_loadu_si128((__m128i const *)(src + 2 * src_stride)); - u3 = _mm_loadu_si128((__m128i const *)(src + 3 * src_stride)); - u4 = _mm_loadu_si128((__m128i const *)(src + 4 * src_stride)); - u5 = _mm_loadu_si128((__m128i const *)(src + 5 * src_stride)); - u6 = _mm_loadu_si128((__m128i const *)(src + 6 * src_stride)); - u7 = _mm_loadu_si128((__m128i const *)(src + 7 * src_stride)); + u0 = _mm_loadl_epi64((__m128i const *)(src + 0 * src_stride)); + u1 = _mm_loadl_epi64((__m128i const *)(src + 1 * src_stride)); + u2 = _mm_loadl_epi64((__m128i const *)(src + 2 * src_stride)); + u3 = _mm_loadl_epi64((__m128i const *)(src + 3 * src_stride)); + u4 = _mm_loadl_epi64((__m128i const *)(src + 4 * src_stride)); + u5 = _mm_loadl_epi64((__m128i const *)(src + 5 * src_stride)); + u6 = _mm_loadl_epi64((__m128i const *)(src + 6 * src_stride)); + u7 = _mm_loadl_epi64((__m128i const *)(src + 7 * src_stride)); - v0 = _mm_loadu_si128((__m128i const *)(pred + 0 * pred_stride)); - v1 = _mm_loadu_si128((__m128i const *)(pred + 1 * pred_stride)); - v2 = _mm_loadu_si128((__m128i const *)(pred + 2 * pred_stride)); - v3 = _mm_loadu_si128((__m128i const *)(pred + 3 * pred_stride)); - v4 = _mm_loadu_si128((__m128i const *)(pred + 4 * pred_stride)); - v5 = _mm_loadu_si128((__m128i const *)(pred + 5 * pred_stride)); - v6 = _mm_loadu_si128((__m128i const *)(pred + 6 * pred_stride)); - v7 = _mm_loadu_si128((__m128i const *)(pred + 7 * pred_stride)); + v0 = _mm_loadl_epi64((__m128i const *)(pred + 0 * pred_stride)); + v1 = _mm_loadl_epi64((__m128i const *)(pred + 1 * pred_stride)); + v2 = _mm_loadl_epi64((__m128i const *)(pred + 2 * pred_stride)); + v3 = _mm_loadl_epi64((__m128i const *)(pred + 3 * pred_stride)); + v4 = _mm_loadl_epi64((__m128i const *)(pred + 4 * pred_stride)); + v5 = _mm_loadl_epi64((__m128i const *)(pred + 5 * pred_stride)); + v6 = _mm_loadl_epi64((__m128i const *)(pred + 6 * pred_stride)); + v7 = _mm_loadl_epi64((__m128i const *)(pred + 7 * pred_stride)); x0 = _mm_sub_epi16(u0, v0); x1 = _mm_sub_epi16(u1, v1); diff --git a/third_party/aom/aom_dsp/x86/highbd_variance_sse2.c b/third_party/aom/aom_dsp/x86/highbd_variance_sse2.c index 47b052abc9..2373d3aecb 100644 --- a/third_party/aom/aom_dsp/x86/highbd_variance_sse2.c +++ b/third_party/aom/aom_dsp/x86/highbd_variance_sse2.c @@ -603,7 +603,7 @@ void aom_highbd_upsampled_pred_sse2(MACROBLOCKD *xd, const int ref_num = 0; const int is_intrabc = is_intrabc_block(mi); const struct scale_factors *const sf = - is_intrabc ? &cm->sf_identity : &xd->block_refs[ref_num]->sf; + is_intrabc ? &cm->sf_identity : xd->block_ref_scale_factors[ref_num]; const int is_scaled = av1_is_scaled(sf); if (is_scaled) { @@ -677,11 +677,8 @@ void aom_highbd_upsampled_pred_sse2(MACROBLOCKD *xd, } } - const InterpFilterParams *filter = - (subpel_search == 1) - ? av1_get_4tap_interp_filter_params(EIGHTTAP_REGULAR) - : av1_get_interp_filter_params_with_block_size(EIGHTTAP_REGULAR, 8); - + const InterpFilterParams *filter = av1_get_filter(subpel_search); + int filter_taps = (subpel_search <= USE_4_TAPS) ? 4 : SUBPEL_TAPS; if (!subpel_x_q3 && !subpel_y_q3) { uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); uint16_t *comp_pred = CONVERT_TO_SHORTPTR(comp_pred8); @@ -729,17 +726,20 @@ void aom_highbd_upsampled_pred_sse2(MACROBLOCKD *xd, av1_get_interp_filter_subpel_kernel(filter, subpel_x_q3 << 1); const int16_t *const kernel_y = av1_get_interp_filter_subpel_kernel(filter, subpel_y_q3 << 1); + const uint8_t *ref_start = ref8 - ref_stride * ((filter_taps >> 1) - 1); + uint16_t *temp_start_horiz = (subpel_search <= USE_4_TAPS) + ? temp + (filter_taps >> 1) * MAX_SB_SIZE + : temp; + uint16_t *temp_start_vert = temp + MAX_SB_SIZE * ((filter->taps >> 1) - 1); const int intermediate_height = - (((height - 1) * 8 + subpel_y_q3) >> 3) + filter->taps; + (((height - 1) * 8 + subpel_y_q3) >> 3) + filter_taps; assert(intermediate_height <= (MAX_SB_SIZE * 2 + 16) + 16); - aom_highbd_convolve8_horiz(ref8 - ref_stride * ((filter->taps >> 1) - 1), - ref_stride, CONVERT_TO_BYTEPTR(temp), - MAX_SB_SIZE, kernel_x, 16, NULL, -1, width, - intermediate_height, bd); - aom_highbd_convolve8_vert( - CONVERT_TO_BYTEPTR(temp + MAX_SB_SIZE * ((filter->taps >> 1) - 1)), - MAX_SB_SIZE, comp_pred8, width, NULL, -1, kernel_y, 16, width, height, - bd); + aom_highbd_convolve8_horiz( + ref_start, ref_stride, CONVERT_TO_BYTEPTR(temp_start_horiz), + MAX_SB_SIZE, kernel_x, 16, NULL, -1, width, intermediate_height, bd); + aom_highbd_convolve8_vert(CONVERT_TO_BYTEPTR(temp_start_vert), MAX_SB_SIZE, + comp_pred8, width, NULL, -1, kernel_y, 16, width, + height, bd); } } @@ -765,11 +765,11 @@ void aom_highbd_comp_avg_upsampled_pred_sse2( } } -static INLINE void highbd_compute_jnt_comp_avg(__m128i *p0, __m128i *p1, - const __m128i *w0, - const __m128i *w1, - const __m128i *r, - void *const result) { +static INLINE void highbd_compute_dist_wtd_comp_avg(__m128i *p0, __m128i *p1, + const __m128i *w0, + const __m128i *w1, + const __m128i *r, + void *const result) { assert(DIST_PRECISION_BITS <= 4); __m128i mult0 = _mm_mullo_epi16(*p0, *w0); __m128i mult1 = _mm_mullo_epi16(*p1, *w1); @@ -780,11 +780,10 @@ static INLINE void highbd_compute_jnt_comp_avg(__m128i *p0, __m128i *p1, xx_storeu_128(result, shift); } -void aom_highbd_jnt_comp_avg_pred_sse2(uint8_t *comp_pred8, - const uint8_t *pred8, int width, - int height, const uint8_t *ref8, - int ref_stride, - const JNT_COMP_PARAMS *jcp_param) { +void aom_highbd_dist_wtd_comp_avg_pred_sse2( + uint8_t *comp_pred8, const uint8_t *pred8, int width, int height, + const uint8_t *ref8, int ref_stride, + const DIST_WTD_COMP_PARAMS *jcp_param) { int i; const uint16_t wt0 = (uint16_t)jcp_param->fwd_offset; const uint16_t wt1 = (uint16_t)jcp_param->bck_offset; @@ -806,7 +805,7 @@ void aom_highbd_jnt_comp_avg_pred_sse2(uint8_t *comp_pred8, __m128i p0 = xx_loadu_128(ref); __m128i p1 = xx_loadu_128(pred); - highbd_compute_jnt_comp_avg(&p0, &p1, &w0, &w1, &r, comp_pred); + highbd_compute_dist_wtd_comp_avg(&p0, &p1, &w0, &w1, &r, comp_pred); comp_pred += 8; pred += 8; @@ -823,7 +822,7 @@ void aom_highbd_jnt_comp_avg_pred_sse2(uint8_t *comp_pred8, __m128i p0 = _mm_unpacklo_epi64(p0_0, p0_1); __m128i p1 = xx_loadu_128(pred); - highbd_compute_jnt_comp_avg(&p0, &p1, &w0, &w1, &r, comp_pred); + highbd_compute_dist_wtd_comp_avg(&p0, &p1, &w0, &w1, &r, comp_pred); comp_pred += 8; pred += 8; @@ -832,11 +831,11 @@ void aom_highbd_jnt_comp_avg_pred_sse2(uint8_t *comp_pred8, } } -void aom_highbd_jnt_comp_avg_upsampled_pred_sse2( +void aom_highbd_dist_wtd_comp_avg_upsampled_pred_sse2( MACROBLOCKD *xd, const struct AV1Common *const cm, int mi_row, int mi_col, const MV *const mv, uint8_t *comp_pred8, const uint8_t *pred8, int width, int height, int subpel_x_q3, int subpel_y_q3, const uint8_t *ref8, - int ref_stride, int bd, const JNT_COMP_PARAMS *jcp_param, + int ref_stride, int bd, const DIST_WTD_COMP_PARAMS *jcp_param, int subpel_search) { uint16_t *pred = CONVERT_TO_SHORTPTR(pred8); int n; @@ -860,7 +859,7 @@ void aom_highbd_jnt_comp_avg_upsampled_pred_sse2( __m128i p0 = xx_loadu_128(comp_pred16); __m128i p1 = xx_loadu_128(pred); - highbd_compute_jnt_comp_avg(&p0, &p1, &w0, &w1, &r, comp_pred16); + highbd_compute_dist_wtd_comp_avg(&p0, &p1, &w0, &w1, &r, comp_pred16); comp_pred16 += 8; pred += 8; diff --git a/third_party/aom/aom_dsp/x86/intrapred_asm_sse2.asm b/third_party/aom/aom_dsp/x86/intrapred_asm_sse2.asm new file mode 100644 index 0000000000..0eb632326b --- /dev/null +++ b/third_party/aom/aom_dsp/x86/intrapred_asm_sse2.asm @@ -0,0 +1,608 @@ +; +; Copyright (c) 2016, Alliance for Open Media. All rights reserved +; +; This source code is subject to the terms of the BSD 2 Clause License and +; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +; was not distributed with this source code in the LICENSE file, you can +; obtain it at www.aomedia.org/license/software. If the Alliance for Open +; Media Patent License 1.0 was not distributed with this source code in the +; PATENTS file, you can obtain it at www.aomedia.org/license/patent. +; + +; + +%include "third_party/x86inc/x86inc.asm" + +SECTION_RODATA +pb_1: times 16 db 1 +pw_4: times 8 dw 4 +pw_8: times 8 dw 8 +pw_16: times 8 dw 16 +pw_32: times 8 dw 32 +dc_128: times 16 db 128 +pw2_4: times 8 dw 2 +pw2_8: times 8 dw 4 +pw2_16: times 8 dw 8 +pw2_32: times 8 dw 16 + +SECTION .text + +INIT_XMM sse2 +cglobal dc_predictor_4x4, 4, 5, 3, dst, stride, above, left, goffset + GET_GOT goffsetq + + movd m2, [leftq] + movd m0, [aboveq] + pxor m1, m1 + punpckldq m0, m2 + psadbw m0, m1 + paddw m0, [GLOBAL(pw_4)] + psraw m0, 3 + pshuflw m0, m0, 0x0 + packuswb m0, m0 + movd [dstq ], m0 + movd [dstq+strideq], m0 + lea dstq, [dstq+strideq*2] + movd [dstq ], m0 + movd [dstq+strideq], m0 + + RESTORE_GOT + RET + +INIT_XMM sse2 +cglobal dc_left_predictor_4x4, 2, 5, 2, dst, stride, above, left, goffset + movifnidn leftq, leftmp + GET_GOT goffsetq + + pxor m1, m1 + movd m0, [leftq] + psadbw m0, m1 + paddw m0, [GLOBAL(pw2_4)] + psraw m0, 2 + pshuflw m0, m0, 0x0 + packuswb m0, m0 + movd [dstq ], m0 + movd [dstq+strideq], m0 + lea dstq, [dstq+strideq*2] + movd [dstq ], m0 + movd [dstq+strideq], m0 + + RESTORE_GOT + RET + +INIT_XMM sse2 +cglobal dc_top_predictor_4x4, 3, 5, 2, dst, stride, above, left, goffset + GET_GOT goffsetq + + pxor m1, m1 + movd m0, [aboveq] + psadbw m0, m1 + paddw m0, [GLOBAL(pw2_4)] + psraw m0, 2 + pshuflw m0, m0, 0x0 + packuswb m0, m0 + movd [dstq ], m0 + movd [dstq+strideq], m0 + lea dstq, [dstq+strideq*2] + movd [dstq ], m0 + movd [dstq+strideq], m0 + + RESTORE_GOT + RET + +INIT_XMM sse2 +cglobal dc_predictor_8x8, 4, 5, 3, dst, stride, above, left, goffset + GET_GOT goffsetq + + pxor m1, m1 + movq m0, [aboveq] + movq m2, [leftq] + DEFINE_ARGS dst, stride, stride3 + lea stride3q, [strideq*3] + psadbw m0, m1 + psadbw m2, m1 + paddw m0, m2 + paddw m0, [GLOBAL(pw_8)] + psraw m0, 4 + punpcklbw m0, m0 + pshuflw m0, m0, 0x0 + movq [dstq ], m0 + movq [dstq+strideq ], m0 + movq [dstq+strideq*2], m0 + movq [dstq+stride3q ], m0 + lea dstq, [dstq+strideq*4] + movq [dstq ], m0 + movq [dstq+strideq ], m0 + movq [dstq+strideq*2], m0 + movq [dstq+stride3q ], m0 + + RESTORE_GOT + RET + +INIT_XMM sse2 +cglobal dc_top_predictor_8x8, 3, 5, 2, dst, stride, above, left, goffset + GET_GOT goffsetq + + pxor m1, m1 + movq m0, [aboveq] + DEFINE_ARGS dst, stride, stride3 + lea stride3q, [strideq*3] + psadbw m0, m1 + paddw m0, [GLOBAL(pw2_8)] + psraw m0, 3 + punpcklbw m0, m0 + pshuflw m0, m0, 0x0 + movq [dstq ], m0 + movq [dstq+strideq ], m0 + movq [dstq+strideq*2], m0 + movq [dstq+stride3q ], m0 + lea dstq, [dstq+strideq*4] + movq [dstq ], m0 + movq [dstq+strideq ], m0 + movq [dstq+strideq*2], m0 + movq [dstq+stride3q ], m0 + + RESTORE_GOT + RET + +INIT_XMM sse2 +cglobal dc_left_predictor_8x8, 2, 5, 2, dst, stride, above, left, goffset + movifnidn leftq, leftmp + GET_GOT goffsetq + + pxor m1, m1 + movq m0, [leftq] + DEFINE_ARGS dst, stride, stride3 + lea stride3q, [strideq*3] + psadbw m0, m1 + paddw m0, [GLOBAL(pw2_8)] + psraw m0, 3 + punpcklbw m0, m0 + pshuflw m0, m0, 0x0 + movq [dstq ], m0 + movq [dstq+strideq ], m0 + movq [dstq+strideq*2], m0 + movq [dstq+stride3q ], m0 + lea dstq, [dstq+strideq*4] + movq [dstq ], m0 + movq [dstq+strideq ], m0 + movq [dstq+strideq*2], m0 + movq [dstq+stride3q ], m0 + + RESTORE_GOT + RET + +INIT_XMM sse2 +cglobal dc_128_predictor_4x4, 2, 5, 1, dst, stride, above, left, goffset + GET_GOT goffsetq + + DEFINE_ARGS dst, stride, stride3 + lea stride3q, [strideq*3] + movd m0, [GLOBAL(dc_128)] + movd [dstq ], m0 + movd [dstq+strideq ], m0 + movd [dstq+strideq*2], m0 + movd [dstq+stride3q ], m0 + RESTORE_GOT + RET + +INIT_XMM sse2 +cglobal dc_128_predictor_8x8, 2, 5, 1, dst, stride, above, left, goffset + GET_GOT goffsetq + + DEFINE_ARGS dst, stride, stride3 + lea stride3q, [strideq*3] + movq m0, [GLOBAL(dc_128)] + movq [dstq ], m0 + movq [dstq+strideq ], m0 + movq [dstq+strideq*2], m0 + movq [dstq+stride3q ], m0 + lea dstq, [dstq+strideq*4] + movq [dstq ], m0 + movq [dstq+strideq ], m0 + movq [dstq+strideq*2], m0 + movq [dstq+stride3q ], m0 + RESTORE_GOT + RET + +INIT_XMM sse2 +cglobal dc_predictor_16x16, 4, 5, 3, dst, stride, above, left, goffset + GET_GOT goffsetq + + pxor m1, m1 + mova m0, [aboveq] + mova m2, [leftq] + DEFINE_ARGS dst, stride, stride3, lines4 + lea stride3q, [strideq*3] + mov lines4d, 4 + psadbw m0, m1 + psadbw m2, m1 + paddw m0, m2 + movhlps m2, m0 + paddw m0, m2 + paddw m0, [GLOBAL(pw_16)] + psraw m0, 5 + pshuflw m0, m0, 0x0 + punpcklqdq m0, m0 + packuswb m0, m0 +.loop: + mova [dstq ], m0 + mova [dstq+strideq ], m0 + mova [dstq+strideq*2], m0 + mova [dstq+stride3q ], m0 + lea dstq, [dstq+strideq*4] + dec lines4d + jnz .loop + + RESTORE_GOT + REP_RET + + +INIT_XMM sse2 +cglobal dc_top_predictor_16x16, 4, 5, 3, dst, stride, above, left, goffset + GET_GOT goffsetq + + pxor m1, m1 + mova m0, [aboveq] + DEFINE_ARGS dst, stride, stride3, lines4 + lea stride3q, [strideq*3] + mov lines4d, 4 + psadbw m0, m1 + movhlps m2, m0 + paddw m0, m2 + paddw m0, [GLOBAL(pw2_16)] + psraw m0, 4 + pshuflw m0, m0, 0x0 + punpcklqdq m0, m0 + packuswb m0, m0 +.loop: + mova [dstq ], m0 + mova [dstq+strideq ], m0 + mova [dstq+strideq*2], m0 + mova [dstq+stride3q ], m0 + lea dstq, [dstq+strideq*4] + dec lines4d + jnz .loop + + RESTORE_GOT + REP_RET + +INIT_XMM sse2 +cglobal dc_left_predictor_16x16, 4, 5, 3, dst, stride, above, left, goffset + GET_GOT goffsetq + + pxor m1, m1 + mova m0, [leftq] + DEFINE_ARGS dst, stride, stride3, lines4 + lea stride3q, [strideq*3] + mov lines4d, 4 + psadbw m0, m1 + movhlps m2, m0 + paddw m0, m2 + paddw m0, [GLOBAL(pw2_16)] + psraw m0, 4 + pshuflw m0, m0, 0x0 + punpcklqdq m0, m0 + packuswb m0, m0 +.loop: + mova [dstq ], m0 + mova [dstq+strideq ], m0 + mova [dstq+strideq*2], m0 + mova [dstq+stride3q ], m0 + lea dstq, [dstq+strideq*4] + dec lines4d + jnz .loop + + RESTORE_GOT + REP_RET + +INIT_XMM sse2 +cglobal dc_128_predictor_16x16, 4, 5, 3, dst, stride, above, left, goffset + GET_GOT goffsetq + + DEFINE_ARGS dst, stride, stride3, lines4 + lea stride3q, [strideq*3] + mov lines4d, 4 + mova m0, [GLOBAL(dc_128)] +.loop: + mova [dstq ], m0 + mova [dstq+strideq ], m0 + mova [dstq+strideq*2], m0 + mova [dstq+stride3q ], m0 + lea dstq, [dstq+strideq*4] + dec lines4d + jnz .loop + RESTORE_GOT + RET + + +INIT_XMM sse2 +cglobal dc_predictor_32x32, 4, 5, 5, dst, stride, above, left, goffset + GET_GOT goffsetq + + pxor m1, m1 + mova m0, [aboveq] + mova m2, [aboveq+16] + mova m3, [leftq] + mova m4, [leftq+16] + DEFINE_ARGS dst, stride, stride3, lines4 + lea stride3q, [strideq*3] + mov lines4d, 8 + psadbw m0, m1 + psadbw m2, m1 + psadbw m3, m1 + psadbw m4, m1 + paddw m0, m2 + paddw m0, m3 + paddw m0, m4 + movhlps m2, m0 + paddw m0, m2 + paddw m0, [GLOBAL(pw_32)] + psraw m0, 6 + pshuflw m0, m0, 0x0 + punpcklqdq m0, m0 + packuswb m0, m0 +.loop: + mova [dstq ], m0 + mova [dstq +16], m0 + mova [dstq+strideq ], m0 + mova [dstq+strideq +16], m0 + mova [dstq+strideq*2 ], m0 + mova [dstq+strideq*2+16], m0 + mova [dstq+stride3q ], m0 + mova [dstq+stride3q +16], m0 + lea dstq, [dstq+strideq*4] + dec lines4d + jnz .loop + + RESTORE_GOT + REP_RET + +INIT_XMM sse2 +cglobal dc_top_predictor_32x32, 4, 5, 5, dst, stride, above, left, goffset + GET_GOT goffsetq + + pxor m1, m1 + mova m0, [aboveq] + mova m2, [aboveq+16] + DEFINE_ARGS dst, stride, stride3, lines4 + lea stride3q, [strideq*3] + mov lines4d, 8 + psadbw m0, m1 + psadbw m2, m1 + paddw m0, m2 + movhlps m2, m0 + paddw m0, m2 + paddw m0, [GLOBAL(pw2_32)] + psraw m0, 5 + pshuflw m0, m0, 0x0 + punpcklqdq m0, m0 + packuswb m0, m0 +.loop: + mova [dstq ], m0 + mova [dstq +16], m0 + mova [dstq+strideq ], m0 + mova [dstq+strideq +16], m0 + mova [dstq+strideq*2 ], m0 + mova [dstq+strideq*2+16], m0 + mova [dstq+stride3q ], m0 + mova [dstq+stride3q +16], m0 + lea dstq, [dstq+strideq*4] + dec lines4d + jnz .loop + + RESTORE_GOT + REP_RET + +INIT_XMM sse2 +cglobal dc_left_predictor_32x32, 4, 5, 5, dst, stride, above, left, goffset + GET_GOT goffsetq + + pxor m1, m1 + mova m0, [leftq] + mova m2, [leftq+16] + DEFINE_ARGS dst, stride, stride3, lines4 + lea stride3q, [strideq*3] + mov lines4d, 8 + psadbw m0, m1 + psadbw m2, m1 + paddw m0, m2 + movhlps m2, m0 + paddw m0, m2 + paddw m0, [GLOBAL(pw2_32)] + psraw m0, 5 + pshuflw m0, m0, 0x0 + punpcklqdq m0, m0 + packuswb m0, m0 +.loop: + mova [dstq ], m0 + mova [dstq +16], m0 + mova [dstq+strideq ], m0 + mova [dstq+strideq +16], m0 + mova [dstq+strideq*2 ], m0 + mova [dstq+strideq*2+16], m0 + mova [dstq+stride3q ], m0 + mova [dstq+stride3q +16], m0 + lea dstq, [dstq+strideq*4] + dec lines4d + jnz .loop + + RESTORE_GOT + REP_RET + +INIT_XMM sse2 +cglobal dc_128_predictor_32x32, 4, 5, 3, dst, stride, above, left, goffset + GET_GOT goffsetq + + DEFINE_ARGS dst, stride, stride3, lines4 + lea stride3q, [strideq*3] + mov lines4d, 8 + mova m0, [GLOBAL(dc_128)] +.loop: + mova [dstq ], m0 + mova [dstq +16], m0 + mova [dstq+strideq ], m0 + mova [dstq+strideq +16], m0 + mova [dstq+strideq*2 ], m0 + mova [dstq+strideq*2+16], m0 + mova [dstq+stride3q ], m0 + mova [dstq+stride3q +16], m0 + lea dstq, [dstq+strideq*4] + dec lines4d + jnz .loop + RESTORE_GOT + RET + +INIT_XMM sse2 +cglobal v_predictor_4x4, 3, 3, 1, dst, stride, above + movd m0, [aboveq] + movd [dstq ], m0 + movd [dstq+strideq], m0 + lea dstq, [dstq+strideq*2] + movd [dstq ], m0 + movd [dstq+strideq], m0 + RET + +INIT_XMM sse2 +cglobal v_predictor_8x8, 3, 3, 1, dst, stride, above + movq m0, [aboveq] + DEFINE_ARGS dst, stride, stride3 + lea stride3q, [strideq*3] + movq [dstq ], m0 + movq [dstq+strideq ], m0 + movq [dstq+strideq*2], m0 + movq [dstq+stride3q ], m0 + lea dstq, [dstq+strideq*4] + movq [dstq ], m0 + movq [dstq+strideq ], m0 + movq [dstq+strideq*2], m0 + movq [dstq+stride3q ], m0 + RET + +INIT_XMM sse2 +cglobal v_predictor_16x16, 3, 4, 1, dst, stride, above + mova m0, [aboveq] + DEFINE_ARGS dst, stride, stride3, nlines4 + lea stride3q, [strideq*3] + mov nlines4d, 4 +.loop: + mova [dstq ], m0 + mova [dstq+strideq ], m0 + mova [dstq+strideq*2], m0 + mova [dstq+stride3q ], m0 + lea dstq, [dstq+strideq*4] + dec nlines4d + jnz .loop + REP_RET + +INIT_XMM sse2 +cglobal v_predictor_32x32, 3, 4, 2, dst, stride, above + mova m0, [aboveq] + mova m1, [aboveq+16] + DEFINE_ARGS dst, stride, stride3, nlines4 + lea stride3q, [strideq*3] + mov nlines4d, 8 +.loop: + mova [dstq ], m0 + mova [dstq +16], m1 + mova [dstq+strideq ], m0 + mova [dstq+strideq +16], m1 + mova [dstq+strideq*2 ], m0 + mova [dstq+strideq*2+16], m1 + mova [dstq+stride3q ], m0 + mova [dstq+stride3q +16], m1 + lea dstq, [dstq+strideq*4] + dec nlines4d + jnz .loop + REP_RET + +INIT_XMM sse2 +cglobal h_predictor_4x4, 2, 4, 4, dst, stride, line, left + movifnidn leftq, leftmp + movd m0, [leftq] + punpcklbw m0, m0 + punpcklbw m0, m0 + pshufd m1, m0, 0x1 + movd [dstq ], m0 + movd [dstq+strideq], m1 + pshufd m2, m0, 0x2 + lea dstq, [dstq+strideq*2] + pshufd m3, m0, 0x3 + movd [dstq ], m2 + movd [dstq+strideq], m3 + RET + +INIT_XMM sse2 +cglobal h_predictor_8x8, 2, 5, 3, dst, stride, line, left + movifnidn leftq, leftmp + mov lineq, -2 + DEFINE_ARGS dst, stride, line, left, stride3 + lea stride3q, [strideq*3] + movq m0, [leftq ] + punpcklbw m0, m0 ; l1 l1 l2 l2 ... l8 l8 +.loop: + pshuflw m1, m0, 0x0 ; l1 l1 l1 l1 l1 l1 l1 l1 + pshuflw m2, m0, 0x55 ; l2 l2 l2 l2 l2 l2 l2 l2 + movq [dstq ], m1 + movq [dstq+strideq], m2 + pshuflw m1, m0, 0xaa + pshuflw m2, m0, 0xff + movq [dstq+strideq*2], m1 + movq [dstq+stride3q ], m2 + pshufd m0, m0, 0xe ; [63:0] l5 l5 l6 l6 l7 l7 l8 l8 + inc lineq + lea dstq, [dstq+strideq*4] + jnz .loop + REP_RET + +INIT_XMM sse2 +cglobal h_predictor_16x16, 2, 5, 3, dst, stride, line, left + movifnidn leftq, leftmp + mov lineq, -4 + DEFINE_ARGS dst, stride, line, left, stride3 + lea stride3q, [strideq*3] +.loop: + movd m0, [leftq] + punpcklbw m0, m0 + punpcklbw m0, m0 ; l1 to l4 each repeated 4 times + pshufd m1, m0, 0x0 ; l1 repeated 16 times + pshufd m2, m0, 0x55 ; l2 repeated 16 times + mova [dstq ], m1 + mova [dstq+strideq ], m2 + pshufd m1, m0, 0xaa + pshufd m2, m0, 0xff + mova [dstq+strideq*2], m1 + mova [dstq+stride3q ], m2 + inc lineq + lea leftq, [leftq+4 ] + lea dstq, [dstq+strideq*4] + jnz .loop + REP_RET + +INIT_XMM sse2 +cglobal h_predictor_32x32, 2, 5, 3, dst, stride, line, left + movifnidn leftq, leftmp + mov lineq, -8 + DEFINE_ARGS dst, stride, line, left, stride3 + lea stride3q, [strideq*3] +.loop: + movd m0, [leftq] + punpcklbw m0, m0 + punpcklbw m0, m0 ; l1 to l4 each repeated 4 times + pshufd m1, m0, 0x0 ; l1 repeated 16 times + pshufd m2, m0, 0x55 ; l2 repeated 16 times + mova [dstq ], m1 + mova [dstq+16 ], m1 + mova [dstq+strideq ], m2 + mova [dstq+strideq+16 ], m2 + pshufd m1, m0, 0xaa + pshufd m2, m0, 0xff + mova [dstq+strideq*2 ], m1 + mova [dstq+strideq*2+16], m1 + mova [dstq+stride3q ], m2 + mova [dstq+stride3q+16 ], m2 + inc lineq + lea leftq, [leftq+4 ] + lea dstq, [dstq+strideq*4] + jnz .loop + REP_RET diff --git a/third_party/aom/aom_dsp/x86/intrapred_avx2.c b/third_party/aom/aom_dsp/x86/intrapred_avx2.c index 1e67d392e8..97569b3ed1 100644 --- a/third_party/aom/aom_dsp/x86/intrapred_avx2.c +++ b/third_party/aom/aom_dsp/x86/intrapred_avx2.c @@ -12,6 +12,7 @@ #include #include "config/aom_dsp_rtcd.h" +#include "aom_dsp/x86/lpf_common_sse2.h" static INLINE __m256i dc_sum_64(const uint8_t *ref) { const __m256i x0 = _mm256_loadu_si256((const __m256i *)ref); @@ -63,6 +64,187 @@ static INLINE void row_store_64xh(const __m256i *r, int height, uint8_t *dst, } } +static INLINE void highbd_transpose16x4_8x8_sse2(__m128i *x, __m128i *d) { + __m128i r0, r1, r2, r3, r4, r5, r6, r7, r8, r9, r10, r11, r12, r13, r14, r15; + + r0 = _mm_unpacklo_epi16(x[0], x[1]); + r1 = _mm_unpacklo_epi16(x[2], x[3]); + r2 = _mm_unpacklo_epi16(x[4], x[5]); + r3 = _mm_unpacklo_epi16(x[6], x[7]); + + r4 = _mm_unpacklo_epi16(x[8], x[9]); + r5 = _mm_unpacklo_epi16(x[10], x[11]); + r6 = _mm_unpacklo_epi16(x[12], x[13]); + r7 = _mm_unpacklo_epi16(x[14], x[15]); + + r8 = _mm_unpacklo_epi32(r0, r1); + r9 = _mm_unpackhi_epi32(r0, r1); + r10 = _mm_unpacklo_epi32(r2, r3); + r11 = _mm_unpackhi_epi32(r2, r3); + + r12 = _mm_unpacklo_epi32(r4, r5); + r13 = _mm_unpackhi_epi32(r4, r5); + r14 = _mm_unpacklo_epi32(r6, r7); + r15 = _mm_unpackhi_epi32(r6, r7); + + r0 = _mm_unpacklo_epi64(r8, r9); + r1 = _mm_unpackhi_epi64(r8, r9); + r2 = _mm_unpacklo_epi64(r10, r11); + r3 = _mm_unpackhi_epi64(r10, r11); + + r4 = _mm_unpacklo_epi64(r12, r13); + r5 = _mm_unpackhi_epi64(r12, r13); + r6 = _mm_unpacklo_epi64(r14, r15); + r7 = _mm_unpackhi_epi64(r14, r15); + + d[0] = _mm_unpacklo_epi64(r0, r2); + d[1] = _mm_unpacklo_epi64(r4, r6); + d[2] = _mm_unpacklo_epi64(r1, r3); + d[3] = _mm_unpacklo_epi64(r5, r7); + + d[4] = _mm_unpackhi_epi64(r0, r2); + d[5] = _mm_unpackhi_epi64(r4, r6); + d[6] = _mm_unpackhi_epi64(r1, r3); + d[7] = _mm_unpackhi_epi64(r5, r7); +} + +static INLINE void highbd_transpose4x16_avx2(__m256i *x, __m256i *d) { + __m256i w0, w1, w2, w3, ww0, ww1; + + w0 = _mm256_unpacklo_epi16(x[0], x[1]); // 00 10 01 11 02 12 03 13 + w1 = _mm256_unpacklo_epi16(x[2], x[3]); // 20 30 21 31 22 32 23 33 + w2 = _mm256_unpackhi_epi16(x[0], x[1]); // 40 50 41 51 42 52 43 53 + w3 = _mm256_unpackhi_epi16(x[2], x[3]); // 60 70 61 71 62 72 63 73 + + ww0 = _mm256_unpacklo_epi32(w0, w1); // 00 10 20 30 01 11 21 31 + ww1 = _mm256_unpacklo_epi32(w2, w3); // 40 50 60 70 41 51 61 71 + + d[0] = _mm256_unpacklo_epi64(ww0, ww1); // 00 10 20 30 40 50 60 70 + d[1] = _mm256_unpackhi_epi64(ww0, ww1); // 01 11 21 31 41 51 61 71 + + ww0 = _mm256_unpackhi_epi32(w0, w1); // 02 12 22 32 03 13 23 33 + ww1 = _mm256_unpackhi_epi32(w2, w3); // 42 52 62 72 43 53 63 73 + + d[2] = _mm256_unpacklo_epi64(ww0, ww1); // 02 12 22 32 42 52 62 72 + d[3] = _mm256_unpackhi_epi64(ww0, ww1); // 03 13 23 33 43 53 63 73 +} + +static INLINE void highbd_transpose8x16_16x8_avx2(__m256i *x, __m256i *d) { + __m256i w0, w1, w2, w3, ww0, ww1; + + w0 = _mm256_unpacklo_epi16(x[0], x[1]); // 00 10 01 11 02 12 03 13 + w1 = _mm256_unpacklo_epi16(x[2], x[3]); // 20 30 21 31 22 32 23 33 + w2 = _mm256_unpacklo_epi16(x[4], x[5]); // 40 50 41 51 42 52 43 53 + w3 = _mm256_unpacklo_epi16(x[6], x[7]); // 60 70 61 71 62 72 63 73 + + ww0 = _mm256_unpacklo_epi32(w0, w1); // 00 10 20 30 01 11 21 31 + ww1 = _mm256_unpacklo_epi32(w2, w3); // 40 50 60 70 41 51 61 71 + + d[0] = _mm256_unpacklo_epi64(ww0, ww1); // 00 10 20 30 40 50 60 70 + d[1] = _mm256_unpackhi_epi64(ww0, ww1); // 01 11 21 31 41 51 61 71 + + ww0 = _mm256_unpackhi_epi32(w0, w1); // 02 12 22 32 03 13 23 33 + ww1 = _mm256_unpackhi_epi32(w2, w3); // 42 52 62 72 43 53 63 73 + + d[2] = _mm256_unpacklo_epi64(ww0, ww1); // 02 12 22 32 42 52 62 72 + d[3] = _mm256_unpackhi_epi64(ww0, ww1); // 03 13 23 33 43 53 63 73 + + w0 = _mm256_unpackhi_epi16(x[0], x[1]); // 04 14 05 15 06 16 07 17 + w1 = _mm256_unpackhi_epi16(x[2], x[3]); // 24 34 25 35 26 36 27 37 + w2 = _mm256_unpackhi_epi16(x[4], x[5]); // 44 54 45 55 46 56 47 57 + w3 = _mm256_unpackhi_epi16(x[6], x[7]); // 64 74 65 75 66 76 67 77 + + ww0 = _mm256_unpacklo_epi32(w0, w1); // 04 14 24 34 05 15 25 35 + ww1 = _mm256_unpacklo_epi32(w2, w3); // 44 54 64 74 45 55 65 75 + + d[4] = _mm256_unpacklo_epi64(ww0, ww1); // 04 14 24 34 44 54 64 74 + d[5] = _mm256_unpackhi_epi64(ww0, ww1); // 05 15 25 35 45 55 65 75 + + ww0 = _mm256_unpackhi_epi32(w0, w1); // 06 16 26 36 07 17 27 37 + ww1 = _mm256_unpackhi_epi32(w2, w3); // 46 56 66 76 47 57 67 77 + + d[6] = _mm256_unpacklo_epi64(ww0, ww1); // 06 16 26 36 46 56 66 76 + d[7] = _mm256_unpackhi_epi64(ww0, ww1); // 07 17 27 37 47 57 67 77 +} + +static INLINE void highbd_transpose16x16_avx2(__m256i *x, __m256i *d) { + __m256i w0, w1, w2, w3, ww0, ww1; + __m256i dd[16]; + w0 = _mm256_unpacklo_epi16(x[0], x[1]); + w1 = _mm256_unpacklo_epi16(x[2], x[3]); + w2 = _mm256_unpacklo_epi16(x[4], x[5]); + w3 = _mm256_unpacklo_epi16(x[6], x[7]); + + ww0 = _mm256_unpacklo_epi32(w0, w1); // + ww1 = _mm256_unpacklo_epi32(w2, w3); // + + dd[0] = _mm256_unpacklo_epi64(ww0, ww1); + dd[1] = _mm256_unpackhi_epi64(ww0, ww1); + + ww0 = _mm256_unpackhi_epi32(w0, w1); // + ww1 = _mm256_unpackhi_epi32(w2, w3); // + + dd[2] = _mm256_unpacklo_epi64(ww0, ww1); + dd[3] = _mm256_unpackhi_epi64(ww0, ww1); + + w0 = _mm256_unpackhi_epi16(x[0], x[1]); + w1 = _mm256_unpackhi_epi16(x[2], x[3]); + w2 = _mm256_unpackhi_epi16(x[4], x[5]); + w3 = _mm256_unpackhi_epi16(x[6], x[7]); + + ww0 = _mm256_unpacklo_epi32(w0, w1); // + ww1 = _mm256_unpacklo_epi32(w2, w3); // + + dd[4] = _mm256_unpacklo_epi64(ww0, ww1); + dd[5] = _mm256_unpackhi_epi64(ww0, ww1); + + ww0 = _mm256_unpackhi_epi32(w0, w1); // + ww1 = _mm256_unpackhi_epi32(w2, w3); // + + dd[6] = _mm256_unpacklo_epi64(ww0, ww1); + dd[7] = _mm256_unpackhi_epi64(ww0, ww1); + + w0 = _mm256_unpacklo_epi16(x[8], x[9]); + w1 = _mm256_unpacklo_epi16(x[10], x[11]); + w2 = _mm256_unpacklo_epi16(x[12], x[13]); + w3 = _mm256_unpacklo_epi16(x[14], x[15]); + + ww0 = _mm256_unpacklo_epi32(w0, w1); + ww1 = _mm256_unpacklo_epi32(w2, w3); + + dd[8] = _mm256_unpacklo_epi64(ww0, ww1); + dd[9] = _mm256_unpackhi_epi64(ww0, ww1); + + ww0 = _mm256_unpackhi_epi32(w0, w1); + ww1 = _mm256_unpackhi_epi32(w2, w3); + + dd[10] = _mm256_unpacklo_epi64(ww0, ww1); + dd[11] = _mm256_unpackhi_epi64(ww0, ww1); + + w0 = _mm256_unpackhi_epi16(x[8], x[9]); + w1 = _mm256_unpackhi_epi16(x[10], x[11]); + w2 = _mm256_unpackhi_epi16(x[12], x[13]); + w3 = _mm256_unpackhi_epi16(x[14], x[15]); + + ww0 = _mm256_unpacklo_epi32(w0, w1); + ww1 = _mm256_unpacklo_epi32(w2, w3); + + dd[12] = _mm256_unpacklo_epi64(ww0, ww1); + dd[13] = _mm256_unpackhi_epi64(ww0, ww1); + + ww0 = _mm256_unpackhi_epi32(w0, w1); + ww1 = _mm256_unpackhi_epi32(w2, w3); + + dd[14] = _mm256_unpacklo_epi64(ww0, ww1); + dd[15] = _mm256_unpackhi_epi64(ww0, ww1); + + for (int i = 0; i < 8; i++) { + d[i] = _mm256_insertf128_si256(dd[i], _mm256_castsi256_si128(dd[i + 8]), 1); + d[i + 8] = _mm256_insertf128_si256(dd[i + 8], + _mm256_extracti128_si256(dd[i], 1), 0); + } +} + void aom_dc_predictor_32x32_avx2(uint8_t *dst, ptrdiff_t stride, const uint8_t *above, const uint8_t *left) { const __m256i sum_above = dc_sum_32(above); @@ -196,7 +378,7 @@ void aom_dc_predictor_32x16_avx2(uint8_t *dst, ptrdiff_t stride, const __m128i top_sum = dc_sum_32_sse2(above); __m128i left_sum = dc_sum_16_sse2(left); left_sum = _mm_add_epi16(top_sum, left_sum); - uint32_t sum = _mm_cvtsi128_si32(left_sum); + uint16_t sum = _mm_cvtsi128_si32(left_sum); sum += 24; sum /= 48; const __m256i row = _mm256_set1_epi8((uint8_t)sum); @@ -208,7 +390,7 @@ void aom_dc_predictor_32x64_avx2(uint8_t *dst, ptrdiff_t stride, const __m256i sum_above = dc_sum_32(above); __m256i sum_left = dc_sum_64(left); sum_left = _mm256_add_epi16(sum_left, sum_above); - uint32_t sum = _mm_cvtsi128_si32(_mm256_castsi256_si128(sum_left)); + uint16_t sum = _mm_cvtsi128_si32(_mm256_castsi256_si128(sum_left)); sum += 48; sum /= 96; const __m256i row = _mm256_set1_epi8((uint8_t)sum); @@ -220,7 +402,7 @@ void aom_dc_predictor_64x64_avx2(uint8_t *dst, ptrdiff_t stride, const __m256i sum_above = dc_sum_64(above); __m256i sum_left = dc_sum_64(left); sum_left = _mm256_add_epi16(sum_left, sum_above); - uint32_t sum = _mm_cvtsi128_si32(_mm256_castsi256_si128(sum_left)); + uint16_t sum = _mm_cvtsi128_si32(_mm256_castsi256_si128(sum_left)); sum += 64; sum /= 128; const __m256i row = _mm256_set1_epi8((uint8_t)sum); @@ -232,7 +414,7 @@ void aom_dc_predictor_64x32_avx2(uint8_t *dst, ptrdiff_t stride, const __m256i sum_above = dc_sum_64(above); __m256i sum_left = dc_sum_32(left); sum_left = _mm256_add_epi16(sum_left, sum_above); - uint32_t sum = _mm_cvtsi128_si32(_mm256_castsi256_si128(sum_left)); + uint16_t sum = _mm_cvtsi128_si32(_mm256_castsi256_si128(sum_left)); sum += 48; sum /= 96; const __m256i row = _mm256_set1_epi8((uint8_t)sum); @@ -244,7 +426,7 @@ void aom_dc_predictor_64x16_avx2(uint8_t *dst, ptrdiff_t stride, const __m256i sum_above = dc_sum_64(above); __m256i sum_left = _mm256_castsi128_si256(dc_sum_16_sse2(left)); sum_left = _mm256_add_epi16(sum_left, sum_above); - uint32_t sum = _mm_cvtsi128_si32(_mm256_castsi256_si128(sum_left)); + uint16_t sum = _mm_cvtsi128_si32(_mm256_castsi256_si128(sum_left)); sum += 40; sum /= 80; const __m256i row = _mm256_set1_epi8((uint8_t)sum); @@ -809,3 +991,3384 @@ void aom_paeth_predictor_64x16_avx2(uint8_t *dst, ptrdiff_t stride, rep = _mm256_add_epi16(rep, one); } } + +#define PERM4x64(c0, c1, c2, c3) c0 + (c1 << 2) + (c2 << 4) + (c3 << 6) +#define PERM2x128(c0, c1) c0 + (c1 << 4) + +static AOM_FORCE_INLINE void highbd_dr_prediction_z1_4xN_internal_avx2( + int N, __m128i *dst, const uint16_t *above, int upsample_above, int dx) { + const int frac_bits = 6 - upsample_above; + const int max_base_x = ((N + 4) - 1) << upsample_above; + int x; + // a assert(dx > 0); + // pre-filter above pixels + // store in temp buffers: + // above[x] * 32 + 16 + // above[x+1] - above[x] + // final pixels will be caluculated as: + // (above[x] * 32 + 16 + (above[x+1] - above[x]) * shift) >> 5 + __m256i a0, a1, a32, a16; + __m256i diff; + __m128i a_mbase_x, max_base_x128, base_inc128, mask128; + + a16 = _mm256_set1_epi32(16); + a_mbase_x = _mm_set1_epi16(above[max_base_x]); + max_base_x128 = _mm_set1_epi32(max_base_x); + + x = dx; + for (int r = 0; r < N; r++) { + __m256i b, res, shift; + __m128i res1; + + int base = x >> frac_bits; + if (base >= max_base_x) { + for (int i = r; i < N; ++i) { + dst[i] = a_mbase_x; // save 4 values + } + return; + } + + a0 = _mm256_cvtepu16_epi32(_mm_loadu_si128((__m128i *)(above + base))); + a1 = _mm256_cvtepu16_epi32(_mm_loadu_si128((__m128i *)(above + base + 1))); + + if (upsample_above) { + a0 = _mm256_permutevar8x32_epi32( + a0, _mm256_set_epi32(7, 5, 3, 1, 6, 4, 2, 0)); + a1 = _mm256_castsi128_si256(_mm256_extracti128_si256(a0, 1)); + base_inc128 = _mm_setr_epi32(base, base + 2, base + 4, base + 6); + shift = _mm256_srli_epi32( + _mm256_and_si256( + _mm256_slli_epi32(_mm256_set1_epi32(x), upsample_above), + _mm256_set1_epi32(0x3f)), + 1); + } else { + base_inc128 = _mm_setr_epi32(base, base + 1, base + 2, base + 3); + shift = _mm256_srli_epi32( + _mm256_and_si256(_mm256_set1_epi32(x), _mm256_set1_epi32(0x3f)), 1); + } + + diff = _mm256_sub_epi32(a1, a0); // a[x+1] - a[x] + a32 = _mm256_slli_epi32(a0, 5); // a[x] * 32 + a32 = _mm256_add_epi32(a32, a16); // a[x] * 32 + 16 + + b = _mm256_mullo_epi32(diff, shift); + res = _mm256_add_epi32(a32, b); + res = _mm256_srli_epi32(res, 5); + + res1 = _mm256_castsi256_si128(res); + res1 = _mm_packus_epi32(res1, res1); + + mask128 = _mm_cmpgt_epi32(max_base_x128, base_inc128); + mask128 = _mm_packs_epi32(mask128, mask128); // goto 16 bit + dst[r] = _mm_blendv_epi8(a_mbase_x, res1, mask128); + x += dx; + } +} + +static void highbd_dr_prediction_z1_4xN_avx2(int N, uint16_t *dst, + ptrdiff_t stride, + const uint16_t *above, + int upsample_above, int dx) { + __m128i dstvec[16]; + + highbd_dr_prediction_z1_4xN_internal_avx2(N, dstvec, above, upsample_above, + dx); + for (int i = 0; i < N; i++) { + _mm_storel_epi64((__m128i *)(dst + stride * i), dstvec[i]); + } +} + +static AOM_FORCE_INLINE void highbd_dr_prediction_z1_8xN_internal_avx2( + int N, __m128i *dst, const uint16_t *above, int upsample_above, int dx) { + const int frac_bits = 6 - upsample_above; + const int max_base_x = ((8 + N) - 1) << upsample_above; + + int x; + // a assert(dx > 0); + // pre-filter above pixels + // store in temp buffers: + // above[x] * 32 + 16 + // above[x+1] - above[x] + // final pixels will be caluculated as: + // (above[x] * 32 + 16 + (above[x+1] - above[x]) * shift) >> 5 + __m256i a0, a1, a0_1, a1_1, a32, a16; + __m256i a_mbase_x, diff, max_base_x256, base_inc256, mask256; + + a16 = _mm256_set1_epi32(16); + a_mbase_x = _mm256_set1_epi16(above[max_base_x]); + max_base_x256 = _mm256_set1_epi32(max_base_x); + + x = dx; + for (int r = 0; r < N; r++) { + __m256i b, res, res1, shift; + + int base = x >> frac_bits; + if (base >= max_base_x) { + for (int i = r; i < N; ++i) { + dst[i] = _mm256_castsi256_si128(a_mbase_x); // save 8 values + } + return; + } + + a0 = _mm256_cvtepu16_epi32(_mm_loadu_si128((__m128i *)(above + base))); + a1 = _mm256_cvtepu16_epi32(_mm_loadu_si128((__m128i *)(above + base + 1))); + + if (upsample_above) { + a0 = _mm256_permutevar8x32_epi32( + a0, _mm256_set_epi32(7, 5, 3, 1, 6, 4, 2, 0)); + a1 = _mm256_castsi128_si256(_mm256_extracti128_si256(a0, 1)); + + a0_1 = + _mm256_cvtepu16_epi32(_mm_loadu_si128((__m128i *)(above + base + 8))); + a0_1 = _mm256_permutevar8x32_epi32( + a0_1, _mm256_set_epi32(7, 5, 3, 1, 6, 4, 2, 0)); + a1_1 = _mm256_castsi128_si256(_mm256_extracti128_si256(a0_1, 1)); + + a0 = _mm256_inserti128_si256(a0, _mm256_castsi256_si128(a0_1), 1); + a1 = _mm256_inserti128_si256(a1, _mm256_castsi256_si128(a1_1), 1); + base_inc256 = + _mm256_setr_epi32(base, base + 2, base + 4, base + 6, base + 8, + base + 10, base + 12, base + 14); + shift = _mm256_srli_epi32( + _mm256_and_si256( + _mm256_slli_epi32(_mm256_set1_epi32(x), upsample_above), + _mm256_set1_epi32(0x3f)), + 1); + } else { + base_inc256 = _mm256_setr_epi32(base, base + 1, base + 2, base + 3, + base + 4, base + 5, base + 6, base + 7); + shift = _mm256_srli_epi32( + _mm256_and_si256(_mm256_set1_epi32(x), _mm256_set1_epi32(0x3f)), 1); + } + + diff = _mm256_sub_epi32(a1, a0); // a[x+1] - a[x] + a32 = _mm256_slli_epi32(a0, 5); // a[x] * 32 + a32 = _mm256_add_epi32(a32, a16); // a[x] * 32 + 16 + + b = _mm256_mullo_epi32(diff, shift); + res = _mm256_add_epi32(a32, b); + res = _mm256_srli_epi32(res, 5); + + res1 = _mm256_packus_epi32( + res, _mm256_castsi128_si256(_mm256_extracti128_si256(res, 1))); + + mask256 = _mm256_cmpgt_epi32(max_base_x256, base_inc256); + mask256 = _mm256_packs_epi32( + mask256, _mm256_castsi128_si256( + _mm256_extracti128_si256(mask256, 1))); // goto 16 bit + res1 = _mm256_blendv_epi8(a_mbase_x, res1, mask256); + dst[r] = _mm256_castsi256_si128(res1); + x += dx; + } +} + +static void highbd_dr_prediction_z1_8xN_avx2(int N, uint16_t *dst, + ptrdiff_t stride, + const uint16_t *above, + int upsample_above, int dx) { + __m128i dstvec[32]; + + highbd_dr_prediction_z1_8xN_internal_avx2(N, dstvec, above, upsample_above, + dx); + for (int i = 0; i < N; i++) { + _mm_storeu_si128((__m128i *)(dst + stride * i), dstvec[i]); + } +} + +static AOM_FORCE_INLINE void highbd_dr_prediction_z1_16xN_internal_avx2( + int N, __m256i *dstvec, const uint16_t *above, int upsample_above, int dx) { + int x; + // here upsample_above is 0 by design of av1_use_intra_edge_upsample + (void)upsample_above; + const int frac_bits = 6; + const int max_base_x = ((16 + N) - 1); + + // pre-filter above pixels + // store in temp buffers: + // above[x] * 32 + 16 + // above[x+1] - above[x] + // final pixels will be caluculated as: + // (above[x] * 32 + 16 + (above[x+1] - above[x]) * shift) >> 5 + __m256i a0, a0_1, a1, a1_1, a32, a16; + __m256i a_mbase_x, diff, max_base_x256, base_inc256, mask256; + + a16 = _mm256_set1_epi32(16); + a_mbase_x = _mm256_set1_epi16(above[max_base_x]); + max_base_x256 = _mm256_set1_epi16(max_base_x); + + x = dx; + for (int r = 0; r < N; r++) { + __m256i b, res[2], res1; + + int base = x >> frac_bits; + if (base >= max_base_x) { + for (int i = r; i < N; ++i) { + dstvec[i] = a_mbase_x; // save 16 values + } + return; + } + __m256i shift = _mm256_srli_epi32( + _mm256_and_si256(_mm256_set1_epi32(x), _mm256_set1_epi32(0x3f)), 1); + + a0 = _mm256_cvtepu16_epi32(_mm_loadu_si128((__m128i *)(above + base))); + a1 = _mm256_cvtepu16_epi32(_mm_loadu_si128((__m128i *)(above + base + 1))); + + diff = _mm256_sub_epi32(a1, a0); // a[x+1] - a[x] + a32 = _mm256_slli_epi32(a0, 5); // a[x] * 32 + a32 = _mm256_add_epi32(a32, a16); // a[x] * 32 + 16 + b = _mm256_mullo_epi32(diff, shift); + + res[0] = _mm256_add_epi32(a32, b); + res[0] = _mm256_srli_epi32(res[0], 5); + res[0] = _mm256_packus_epi32( + res[0], _mm256_castsi128_si256(_mm256_extracti128_si256(res[0], 1))); + + int mdif = max_base_x - base; + if (mdif > 8) { + a0_1 = + _mm256_cvtepu16_epi32(_mm_loadu_si128((__m128i *)(above + base + 8))); + a1_1 = + _mm256_cvtepu16_epi32(_mm_loadu_si128((__m128i *)(above + base + 9))); + + diff = _mm256_sub_epi32(a1_1, a0_1); // a[x+1] - a[x] + a32 = _mm256_slli_epi32(a0_1, 5); // a[x] * 32 + a32 = _mm256_add_epi32(a32, a16); // a[x] * 32 + 16 + b = _mm256_mullo_epi32(diff, shift); + + res[1] = _mm256_add_epi32(a32, b); + res[1] = _mm256_srli_epi32(res[1], 5); + res[1] = _mm256_packus_epi32( + res[1], _mm256_castsi128_si256(_mm256_extracti128_si256(res[1], 1))); + } else { + res[1] = a_mbase_x; + } + res1 = _mm256_inserti128_si256(res[0], _mm256_castsi256_si128(res[1]), + 1); // 16 16bit values + + base_inc256 = _mm256_setr_epi16(base, base + 1, base + 2, base + 3, + base + 4, base + 5, base + 6, base + 7, + base + 8, base + 9, base + 10, base + 11, + base + 12, base + 13, base + 14, base + 15); + mask256 = _mm256_cmpgt_epi16(max_base_x256, base_inc256); + dstvec[r] = _mm256_blendv_epi8(a_mbase_x, res1, mask256); + x += dx; + } +} + +static void highbd_dr_prediction_z1_16xN_avx2(int N, uint16_t *dst, + ptrdiff_t stride, + const uint16_t *above, + int upsample_above, int dx) { + __m256i dstvec[64]; + highbd_dr_prediction_z1_16xN_internal_avx2(N, dstvec, above, upsample_above, + dx); + for (int i = 0; i < N; i++) { + _mm256_storeu_si256((__m256i *)(dst + stride * i), dstvec[i]); + } +} + +static AOM_FORCE_INLINE void highbd_dr_prediction_z1_32xN_internal_avx2( + int N, __m256i *dstvec, const uint16_t *above, int upsample_above, int dx) { + int x; + // here upsample_above is 0 by design of av1_use_intra_edge_upsample + (void)upsample_above; + const int frac_bits = 6; + const int max_base_x = ((32 + N) - 1); + + // pre-filter above pixels + // store in temp buffers: + // above[x] * 32 + 16 + // above[x+1] - above[x] + // final pixels will be caluculated as: + // (above[x] * 32 + 16 + (above[x+1] - above[x]) * shift) >> 5 + __m256i a0, a0_1, a1, a1_1, a32, a16; + __m256i a_mbase_x, diff, max_base_x256, base_inc256, mask256; + + a16 = _mm256_set1_epi32(16); + a_mbase_x = _mm256_set1_epi16(above[max_base_x]); + max_base_x256 = _mm256_set1_epi16(max_base_x); + + x = dx; + for (int r = 0; r < N; r++) { + __m256i b, res[2], res1; + + int base = x >> frac_bits; + if (base >= max_base_x) { + for (int i = r; i < N; ++i) { + dstvec[i] = a_mbase_x; // save 32 values + dstvec[i + N] = a_mbase_x; + } + return; + } + + __m256i shift = _mm256_srli_epi32( + _mm256_and_si256(_mm256_set1_epi32(x), _mm256_set1_epi32(0x3f)), 1); + + for (int j = 0; j < 32; j += 16) { + int mdif = max_base_x - (base + j); + if (mdif <= 0) { + res1 = a_mbase_x; + } else { + a0 = _mm256_cvtepu16_epi32( + _mm_loadu_si128((__m128i *)(above + base + j))); + a1 = _mm256_cvtepu16_epi32( + _mm_loadu_si128((__m128i *)(above + base + 1 + j))); + + diff = _mm256_sub_epi32(a1, a0); // a[x+1] - a[x] + a32 = _mm256_slli_epi32(a0, 5); // a[x] * 32 + a32 = _mm256_add_epi32(a32, a16); // a[x] * 32 + 16 + b = _mm256_mullo_epi32(diff, shift); + + res[0] = _mm256_add_epi32(a32, b); + res[0] = _mm256_srli_epi32(res[0], 5); + res[0] = _mm256_packus_epi32( + res[0], + _mm256_castsi128_si256(_mm256_extracti128_si256(res[0], 1))); + if (mdif > 8) { + a0_1 = _mm256_cvtepu16_epi32( + _mm_loadu_si128((__m128i *)(above + base + 8 + j))); + a1_1 = _mm256_cvtepu16_epi32( + _mm_loadu_si128((__m128i *)(above + base + 9 + j))); + + diff = _mm256_sub_epi32(a1_1, a0_1); // a[x+1] - a[x] + a32 = _mm256_slli_epi32(a0_1, 5); // a[x] * 32 + a32 = _mm256_add_epi32(a32, a16); // a[x] * 32 + 16 + b = _mm256_mullo_epi32(diff, shift); + + res[1] = _mm256_add_epi32(a32, b); + res[1] = _mm256_srli_epi32(res[1], 5); + res[1] = _mm256_packus_epi32( + res[1], + _mm256_castsi128_si256(_mm256_extracti128_si256(res[1], 1))); + } else { + res[1] = a_mbase_x; + } + res1 = _mm256_inserti128_si256(res[0], _mm256_castsi256_si128(res[1]), + 1); // 16 16bit values + base_inc256 = _mm256_setr_epi16( + base + j, base + j + 1, base + j + 2, base + j + 3, base + j + 4, + base + j + 5, base + j + 6, base + j + 7, base + j + 8, + base + j + 9, base + j + 10, base + j + 11, base + j + 12, + base + j + 13, base + j + 14, base + j + 15); + + mask256 = _mm256_cmpgt_epi16(max_base_x256, base_inc256); + res1 = _mm256_blendv_epi8(a_mbase_x, res1, mask256); + } + if (!j) + dstvec[r] = res1; + else + dstvec[r + N] = res1; + } + x += dx; + } +} + +static void highbd_dr_prediction_z1_32xN_avx2(int N, uint16_t *dst, + ptrdiff_t stride, + const uint16_t *above, + int upsample_above, int dx) { + __m256i dstvec[128]; + + highbd_dr_prediction_z1_32xN_internal_avx2(N, dstvec, above, upsample_above, + dx); + for (int i = 0; i < N; i++) { + _mm256_storeu_si256((__m256i *)(dst + stride * i), dstvec[i]); + _mm256_storeu_si256((__m256i *)(dst + stride * i + 16), dstvec[i + N]); + } +} + +static void highbd_dr_prediction_z1_64xN_avx2(int N, uint16_t *dst, + ptrdiff_t stride, + const uint16_t *above, + int upsample_above, int dx) { + int x; + + // here upsample_above is 0 by design of av1_use_intra_edge_upsample + (void)upsample_above; + const int frac_bits = 6; + const int max_base_x = ((64 + N) - 1); + + // pre-filter above pixels + // store in temp buffers: + // above[x] * 32 + 16 + // above[x+1] - above[x] + // final pixels will be caluculated as: + // (above[x] * 32 + 16 + (above[x+1] - above[x]) * shift) >> 5 + __m256i a0, a0_1, a1, a1_1, a32, a16; + __m256i a_mbase_x, diff, max_base_x256, base_inc256, mask256; + + a16 = _mm256_set1_epi32(16); + a_mbase_x = _mm256_set1_epi16(above[max_base_x]); + max_base_x256 = _mm256_set1_epi16(max_base_x); + + x = dx; + for (int r = 0; r < N; r++, dst += stride) { + __m256i b, res[2], res1; + + int base = x >> frac_bits; + if (base >= max_base_x) { + for (int i = r; i < N; ++i) { + _mm256_storeu_si256((__m256i *)dst, a_mbase_x); // save 32 values + _mm256_storeu_si256((__m256i *)(dst + 16), a_mbase_x); + _mm256_storeu_si256((__m256i *)(dst + 32), a_mbase_x); + _mm256_storeu_si256((__m256i *)(dst + 48), a_mbase_x); + dst += stride; + } + return; + } + + __m256i shift = _mm256_srli_epi32( + _mm256_and_si256(_mm256_set1_epi32(x), _mm256_set1_epi32(0x3f)), 1); + + __m128i a0_128, a0_1_128, a1_128, a1_1_128; + for (int j = 0; j < 64; j += 16) { + int mdif = max_base_x - (base + j); + if (mdif <= 0) { + _mm256_storeu_si256((__m256i *)(dst + j), a_mbase_x); + } else { + a0_128 = _mm_loadu_si128((__m128i *)(above + base + j)); + a1_128 = _mm_loadu_si128((__m128i *)(above + base + 1 + j)); + a0 = _mm256_cvtepu16_epi32(a0_128); + a1 = _mm256_cvtepu16_epi32(a1_128); + + diff = _mm256_sub_epi32(a1, a0); // a[x+1] - a[x] + a32 = _mm256_slli_epi32(a0, 5); // a[x] * 32 + a32 = _mm256_add_epi32(a32, a16); // a[x] * 32 + 16 + b = _mm256_mullo_epi32(diff, shift); + + res[0] = _mm256_add_epi32(a32, b); + res[0] = _mm256_srli_epi32(res[0], 5); + res[0] = _mm256_packus_epi32( + res[0], + _mm256_castsi128_si256(_mm256_extracti128_si256(res[0], 1))); + if (mdif > 8) { + a0_1_128 = _mm_loadu_si128((__m128i *)(above + base + 8 + j)); + a1_1_128 = _mm_loadu_si128((__m128i *)(above + base + 9 + j)); + a0_1 = _mm256_cvtepu16_epi32(a0_1_128); + a1_1 = _mm256_cvtepu16_epi32(a1_1_128); + + diff = _mm256_sub_epi32(a1_1, a0_1); // a[x+1] - a[x] + a32 = _mm256_slli_epi32(a0_1, 5); // a[x] * 32 + a32 = _mm256_add_epi32(a32, a16); // a[x] * 32 + 16 + b = _mm256_mullo_epi32(diff, shift); + + res[1] = _mm256_add_epi32(a32, b); + res[1] = _mm256_srli_epi32(res[1], 5); + res[1] = _mm256_packus_epi32( + res[1], + _mm256_castsi128_si256(_mm256_extracti128_si256(res[1], 1))); + } else { + res[1] = a_mbase_x; + } + res1 = _mm256_inserti128_si256(res[0], _mm256_castsi256_si128(res[1]), + 1); // 16 16bit values + base_inc256 = _mm256_setr_epi16( + base + j, base + j + 1, base + j + 2, base + j + 3, base + j + 4, + base + j + 5, base + j + 6, base + j + 7, base + j + 8, + base + j + 9, base + j + 10, base + j + 11, base + j + 12, + base + j + 13, base + j + 14, base + j + 15); + + mask256 = _mm256_cmpgt_epi16(max_base_x256, base_inc256); + res1 = _mm256_blendv_epi8(a_mbase_x, res1, mask256); + _mm256_storeu_si256((__m256i *)(dst + j), res1); + } + } + x += dx; + } +} + +// Directional prediction, zone 1: 0 < angle < 90 +void av1_highbd_dr_prediction_z1_avx2(uint16_t *dst, ptrdiff_t stride, int bw, + int bh, const uint16_t *above, + const uint16_t *left, int upsample_above, + int dx, int dy, int bd) { + (void)left; + (void)dy; + (void)bd; + + switch (bw) { + case 4: + highbd_dr_prediction_z1_4xN_avx2(bh, dst, stride, above, upsample_above, + dx); + break; + case 8: + highbd_dr_prediction_z1_8xN_avx2(bh, dst, stride, above, upsample_above, + dx); + break; + case 16: + highbd_dr_prediction_z1_16xN_avx2(bh, dst, stride, above, upsample_above, + dx); + break; + case 32: + highbd_dr_prediction_z1_32xN_avx2(bh, dst, stride, above, upsample_above, + dx); + break; + case 64: + highbd_dr_prediction_z1_64xN_avx2(bh, dst, stride, above, upsample_above, + dx); + break; + default: break; + } + return; +} + +static void highbd_transpose_TX_8X8(const uint16_t *src, ptrdiff_t pitchSrc, + uint16_t *dst, ptrdiff_t pitchDst) { + __m128i r0, r1, r2, r3, r4, r5, r6, r7, r0_Lo, r1_Lo, r2_Lo, r3_Lo, r4_Lo, + r5_Lo, r6_Lo; + r0 = _mm_load_si128( + (__m128i *)(src + 0 * pitchSrc)); // 07,06,05,04,03,02,01,00 + r1 = _mm_load_si128( + (__m128i *)(src + 1 * pitchSrc)); // 17,16,15,14,13,12,11,10 + r2 = _mm_load_si128( + (__m128i *)(src + 2 * pitchSrc)); // 27,26,25,24,23,22,21,20 + r3 = _mm_load_si128( + (__m128i *)(src + 3 * pitchSrc)); // 37,36,35,34,33,32,31,30 + r4 = _mm_load_si128( + (__m128i *)(src + 4 * pitchSrc)); // 47,46,45,44,43,42,41,40 + r5 = _mm_load_si128( + (__m128i *)(src + 5 * pitchSrc)); // 57,56,55,54,53,52,51,50 + r6 = _mm_load_si128( + (__m128i *)(src + 6 * pitchSrc)); // 67,66,65,64,63,62,61,60 + r7 = _mm_load_si128( + (__m128i *)(src + 7 * pitchSrc)); // 77,76,75,74,73,72,71,70 + + r0_Lo = _mm_unpacklo_epi16(r0, r1); + r2_Lo = _mm_unpacklo_epi16(r2, r3); + r4_Lo = _mm_unpacklo_epi16(r4, r5); + r6_Lo = _mm_unpacklo_epi16(r6, r7); + + r1_Lo = r0_Lo; + r0_Lo = _mm_unpacklo_epi32(r0_Lo, r2_Lo); + r1_Lo = _mm_unpackhi_epi32(r1_Lo, r2_Lo); + r5_Lo = r4_Lo; + r4_Lo = _mm_unpacklo_epi32(r4_Lo, r6_Lo); + r5_Lo = _mm_unpackhi_epi32(r5_Lo, r6_Lo); + r2_Lo = r0_Lo; + r0_Lo = _mm_unpacklo_epi64(r0_Lo, r4_Lo); // 64 + r2_Lo = _mm_unpackhi_epi64(r2_Lo, r4_Lo); + r3_Lo = r1_Lo; + r1_Lo = _mm_unpacklo_epi64(r1_Lo, r5_Lo); + r3_Lo = _mm_unpackhi_epi64(r3_Lo, r5_Lo); + + _mm_storeu_si128((__m128i *)(dst + 0 * pitchDst), r0_Lo); + _mm_storeu_si128((__m128i *)(dst + 1 * pitchDst), r2_Lo); + _mm_storeu_si128((__m128i *)(dst + 2 * pitchDst), r1_Lo); + _mm_storeu_si128((__m128i *)(dst + 3 * pitchDst), r3_Lo); + + r0 = _mm_unpackhi_epi16(r0, r1); + r2 = _mm_unpackhi_epi16(r2, r3); + r4 = _mm_unpackhi_epi16(r4, r5); + r6 = _mm_unpackhi_epi16(r6, r7); + + r1 = r0; + r0 = _mm_unpacklo_epi32(r0, r2); + r1 = _mm_unpackhi_epi32(r1, r2); + r5 = r4; + r4 = _mm_unpacklo_epi32(r4, r6); + r5 = _mm_unpackhi_epi32(r5, r6); + r2 = r0; + r0 = _mm_unpacklo_epi64(r0, r4); + r2 = _mm_unpackhi_epi64(r2, r4); + r3 = r1; + r1 = _mm_unpacklo_epi64(r1, r5); + r3 = _mm_unpackhi_epi64(r3, r5); + + _mm_storeu_si128((__m128i *)(dst + 4 * pitchDst), r0); + _mm_storeu_si128((__m128i *)(dst + 5 * pitchDst), r2); + _mm_storeu_si128((__m128i *)(dst + 6 * pitchDst), r1); + _mm_storeu_si128((__m128i *)(dst + 7 * pitchDst), r3); +} + +static uint8_t HighbdLoadMaskx[8][16] = { + { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }, + { 0, 1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13 }, + { 0, 1, 0, 1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 }, + { 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }, + { 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 4, 5, 6, 7 }, + { 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 4, 5 }, + { 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 2, 3 }, + { 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1 }, +}; + +static uint8_t HighbdEvenOddMaskx4[8][16] = { + { 0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, + 15 }, // 0=0,1, 1=2,3, 2=4,5, 3=6,7, 4=8,9, 5=10,11, 6=12,13, 7=14,15, + // >7=0,1 + { 0, 1, 2, 3, 6, 7, 10, 11, 14, 15, 4, 5, 8, 9, 12, 13 }, + { 0, 1, 0, 1, 4, 5, 8, 9, 12, 13, 0, 1, 6, 7, 10, 11 }, + { 0, 1, 0, 1, 0, 1, 6, 7, 10, 11, 14, 15, 0, 1, 8, 9 }, + { 0, 1, 0, 1, 0, 1, 0, 1, 8, 9, 12, 13, 0, 1, 0, 1 }, + { 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 10, 11, 14, 15, 0, 1 }, + { 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 12, 13, 0, 1 }, + { 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 14, 15 } +}; + +static uint16_t HighbdEvenOddMaskx8_2[8][16] = { + { 0, 2, 4, 6, 8, 10, 12, 14 }, { 2, 2, 4, 6, 8, 10, 12, 14 }, + { 4, 4, 4, 6, 8, 10, 12, 14 }, { 6, 6, 6, 6, 8, 10, 12, 14 }, + { 8, 8, 8, 8, 8, 10, 12, 14 }, { 10, 10, 10, 10, 10, 10, 12, 14 }, + { 12, 12, 12, 12, 12, 12, 12, 14 }, { 14, 14, 14, 14, 14, 14, 14, 14 }, +}; + +static uint16_t HighbdBaseMask[17][16] = { + { + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + }, + { 0xffff, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 0xffff, 0xffff, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 0xffff, 0xffff, 0xffff, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 0xffff, 0xffff, 0xffff, 0xffff, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0 }, + { 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0, 0, 0, 0, 0, 0, 0, + 0, 0 }, + { 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0, 0, 0, 0, + 0, 0, 0, 0 }, + { 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0, + 0, 0, 0, 0, 0, 0 }, + { 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, + 0xffff, 0, 0, 0, 0, 0, 0 }, + { 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, + 0xffff, 0xffff, 0, 0, 0, 0, 0 }, + { 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, + 0xffff, 0xffff, 0xffff, 0, 0, 0, 0 }, + { 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, + 0xffff, 0xffff, 0xffff, 0xffff, 0, 0, 0 }, + { 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, + 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0, 0 }, + { 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, + 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0 }, + { 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, + 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff } +}; + +static void highbd_dr_prediction_z2_Nx4_avx2( + int N, uint16_t *dst, ptrdiff_t stride, const uint16_t *above, + const uint16_t *left, int upsample_above, int upsample_left, int dx, + int dy) { + const int min_base_x = -(1 << upsample_above); + const int min_base_y = -(1 << upsample_left); + const int frac_bits_x = 6 - upsample_above; + const int frac_bits_y = 6 - upsample_left; + + // a assert(dx > 0); + // pre-filter above pixels + // store in temp buffers: + // above[x] * 32 + 16 + // above[x+1] - above[x] + // final pixels will be caluculated as: + // (above[x] * 32 + 16 + (above[x+1] - above[x]) * shift) >> 5 + __m256i a0_x, a1_x, a0_y, a1_y, a32, a16; + __m256i diff; + __m128i c3f, min_base_y128; + + a16 = _mm256_set1_epi32(16); + c3f = _mm_set1_epi32(0x3f); + min_base_y128 = _mm_set1_epi32(min_base_y); + + for (int r = 0; r < N; r++) { + __m256i b, res, shift; + __m128i resx, resy, resxy; + __m128i a0_x128, a1_x128; + int y = r + 1; + int base_x = (-y * dx) >> frac_bits_x; + int base_shift = 0; + if (base_x < (min_base_x - 1)) { + base_shift = (min_base_x - base_x - 1) >> upsample_above; + } + int base_min_diff = + (min_base_x - base_x + upsample_above) >> upsample_above; + if (base_min_diff > 4) { + base_min_diff = 4; + } else { + if (base_min_diff < 0) base_min_diff = 0; + } + + if (base_shift > 3) { + resx = _mm_setzero_si128(); + } else { + a0_x128 = _mm_loadu_si128((__m128i *)(above + base_x + base_shift)); + a1_x128 = _mm_loadu_si128((__m128i *)(above + base_x + 1 + base_shift)); + + if (upsample_above) { + a0_x128 = _mm_shuffle_epi8(a0_x128, + *(__m128i *)HighbdEvenOddMaskx4[base_shift]); + a1_x128 = _mm_shuffle_epi8(a1_x128, + *(__m128i *)HighbdEvenOddMaskx4[base_shift]); + shift = _mm256_castsi128_si256(_mm_srli_epi32( + _mm_and_si128( + _mm_slli_epi32( + _mm_setr_epi32(-y * dx, (1 << 6) - y * dx, + (2 << 6) - y * dx, (3 << 6) - y * dx), + upsample_above), + c3f), + 1)); + } else { + a0_x128 = + _mm_shuffle_epi8(a0_x128, *(__m128i *)HighbdLoadMaskx[base_shift]); + a1_x128 = + _mm_shuffle_epi8(a1_x128, *(__m128i *)HighbdLoadMaskx[base_shift]); + shift = _mm256_castsi128_si256(_mm_srli_epi32( + _mm_and_si128(_mm_setr_epi32(-y * dx, (1 << 6) - y * dx, + (2 << 6) - y * dx, (3 << 6) - y * dx), + c3f), + 1)); + } + a0_x = _mm256_cvtepu16_epi32(a0_x128); + a1_x = _mm256_cvtepu16_epi32(a1_x128); + + diff = _mm256_sub_epi32(a1_x, a0_x); // a[x+1] - a[x] + a32 = _mm256_slli_epi32(a0_x, 5); // a[x] * 32 + a32 = _mm256_add_epi32(a32, a16); // a[x] * 32 + 16 + + b = _mm256_mullo_epi32(diff, shift); + res = _mm256_add_epi32(a32, b); + res = _mm256_srli_epi32(res, 5); + + resx = _mm256_castsi256_si128(res); + resx = _mm_packus_epi32(resx, resx); + } + // y calc + if (base_x < min_base_x) { + DECLARE_ALIGNED(32, int, base_y_c[4]); + __m128i r6, c1234, dy128, y_c128, base_y_c128, mask128; + r6 = _mm_set1_epi32(r << 6); + dy128 = _mm_set1_epi32(dy); + c1234 = _mm_setr_epi32(1, 2, 3, 4); + y_c128 = _mm_sub_epi32(r6, _mm_mullo_epi32(c1234, dy128)); + base_y_c128 = _mm_srai_epi32(y_c128, frac_bits_y); + mask128 = _mm_cmpgt_epi32(min_base_y128, base_y_c128); + base_y_c128 = _mm_andnot_si128(mask128, base_y_c128); + _mm_store_si128((__m128i *)base_y_c, base_y_c128); + + a0_y = _mm256_castsi128_si256( + _mm_setr_epi32(left[base_y_c[0]], left[base_y_c[1]], + left[base_y_c[2]], left[base_y_c[3]])); + a1_y = _mm256_castsi128_si256( + _mm_setr_epi32(left[base_y_c[0] + 1], left[base_y_c[1] + 1], + left[base_y_c[2] + 1], left[base_y_c[3] + 1])); + + if (upsample_left) { + shift = _mm256_castsi128_si256(_mm_srli_epi32( + _mm_and_si128(_mm_slli_epi32(y_c128, upsample_left), c3f), 1)); + } else { + shift = _mm256_castsi128_si256( + _mm_srli_epi32(_mm_and_si128(y_c128, c3f), 1)); + } + diff = _mm256_sub_epi32(a1_y, a0_y); // a[x+1] - a[x] + a32 = _mm256_slli_epi32(a0_y, 5); // a[x] * 32 + a32 = _mm256_add_epi32(a32, a16); // a[x] * 32 + 16 + + b = _mm256_mullo_epi32(diff, shift); + res = _mm256_add_epi32(a32, b); + res = _mm256_srli_epi32(res, 5); + + resy = _mm256_castsi256_si128(res); + resy = _mm_packus_epi32(resy, resy); + } else { + resy = resx; + } + resxy = + _mm_blendv_epi8(resx, resy, *(__m128i *)HighbdBaseMask[base_min_diff]); + _mm_storel_epi64((__m128i *)(dst), resxy); + dst += stride; + } +} + +static void highbd_dr_prediction_z2_Nx8_avx2( + int N, uint16_t *dst, ptrdiff_t stride, const uint16_t *above, + const uint16_t *left, int upsample_above, int upsample_left, int dx, + int dy) { + const int min_base_x = -(1 << upsample_above); + const int min_base_y = -(1 << upsample_left); + const int frac_bits_x = 6 - upsample_above; + const int frac_bits_y = 6 - upsample_left; + + // pre-filter above pixels + // store in temp buffers: + // above[x] * 32 + 16 + // above[x+1] - above[x] + // final pixels will be caluculated as: + // (above[x] * 32 + 16 + (above[x+1] - above[x]) * shift) >> 5 + __m256i a0_x, a1_x, a0_y, a1_y, a32, a16, c3f, min_base_y256; + __m256i diff; + __m128i a0_x128, a1_x128; + + a16 = _mm256_set1_epi32(16); + c3f = _mm256_set1_epi32(0x3f); + min_base_y256 = _mm256_set1_epi32(min_base_y); + + for (int r = 0; r < N; r++) { + __m256i b, res, shift; + __m128i resx, resy, resxy; + int y = r + 1; + int base_x = (-y * dx) >> frac_bits_x; + int base_shift = 0; + if (base_x < (min_base_x - 1)) { + base_shift = (min_base_x - base_x - 1) >> upsample_above; + } + int base_min_diff = + (min_base_x - base_x + upsample_above) >> upsample_above; + if (base_min_diff > 8) { + base_min_diff = 8; + } else { + if (base_min_diff < 0) base_min_diff = 0; + } + + if (base_shift > 7) { + resx = _mm_setzero_si128(); + } else { + if (upsample_above) { + a0_x128 = _mm_setr_epi16( + above[base_x + HighbdEvenOddMaskx8_2[base_shift][0]], + above[base_x + HighbdEvenOddMaskx8_2[base_shift][1]], + above[base_x + HighbdEvenOddMaskx8_2[base_shift][2]], + above[base_x + HighbdEvenOddMaskx8_2[base_shift][3]], + above[base_x + HighbdEvenOddMaskx8_2[base_shift][4]], + above[base_x + HighbdEvenOddMaskx8_2[base_shift][5]], + above[base_x + HighbdEvenOddMaskx8_2[base_shift][6]], + above[base_x + HighbdEvenOddMaskx8_2[base_shift][7]]); + a1_x128 = _mm_setr_epi16( + above[base_x + 1 + HighbdEvenOddMaskx8_2[base_shift][0]], + above[base_x + 1 + HighbdEvenOddMaskx8_2[base_shift][1]], + above[base_x + 1 + HighbdEvenOddMaskx8_2[base_shift][2]], + above[base_x + 1 + HighbdEvenOddMaskx8_2[base_shift][3]], + above[base_x + 1 + HighbdEvenOddMaskx8_2[base_shift][4]], + above[base_x + 1 + HighbdEvenOddMaskx8_2[base_shift][5]], + above[base_x + 1 + HighbdEvenOddMaskx8_2[base_shift][6]], + above[base_x + 1 + HighbdEvenOddMaskx8_2[base_shift][7]]); + shift = _mm256_srli_epi32( + _mm256_and_si256( + _mm256_slli_epi32( + _mm256_setr_epi32(-y * dx, (1 << 6) - y * dx, + (2 << 6) - y * dx, (3 << 6) - y * dx, + (4 << 6) - y * dx, (5 << 6) - y * dx, + (6 << 6) - y * dx, (7 << 6) - y * dx), + upsample_above), + c3f), + 1); + } else { + a0_x128 = _mm_loadu_si128((__m128i *)(above + base_x + base_shift)); + a1_x128 = _mm_loadu_si128((__m128i *)(above + base_x + 1 + base_shift)); + a0_x128 = + _mm_shuffle_epi8(a0_x128, *(__m128i *)HighbdLoadMaskx[base_shift]); + a1_x128 = + _mm_shuffle_epi8(a1_x128, *(__m128i *)HighbdLoadMaskx[base_shift]); + + shift = _mm256_srli_epi32( + _mm256_and_si256( + _mm256_setr_epi32(-y * dx, (1 << 6) - y * dx, (2 << 6) - y * dx, + (3 << 6) - y * dx, (4 << 6) - y * dx, + (5 << 6) - y * dx, (6 << 6) - y * dx, + (7 << 6) - y * dx), + c3f), + 1); + } + + a0_x = _mm256_cvtepu16_epi32(a0_x128); + a1_x = _mm256_cvtepu16_epi32(a1_x128); + + diff = _mm256_sub_epi32(a1_x, a0_x); // a[x+1] - a[x] + a32 = _mm256_slli_epi32(a0_x, 5); // a[x] * 32 + a32 = _mm256_add_epi32(a32, a16); // a[x] * 32 + 16 + + b = _mm256_mullo_epi32(diff, shift); + res = _mm256_add_epi32(a32, b); + res = _mm256_srli_epi32(res, 5); + + resx = _mm256_castsi256_si128(_mm256_packus_epi32( + res, _mm256_castsi128_si256(_mm256_extracti128_si256(res, 1)))); + } + // y calc + if (base_x < min_base_x) { + DECLARE_ALIGNED(32, int, base_y_c[8]); + __m256i r6, c256, dy256, y_c256, base_y_c256, mask256; + r6 = _mm256_set1_epi32(r << 6); + dy256 = _mm256_set1_epi32(dy); + c256 = _mm256_setr_epi32(1, 2, 3, 4, 5, 6, 7, 8); + y_c256 = _mm256_sub_epi32(r6, _mm256_mullo_epi32(c256, dy256)); + base_y_c256 = _mm256_srai_epi32(y_c256, frac_bits_y); + mask256 = _mm256_cmpgt_epi32(min_base_y256, base_y_c256); + base_y_c256 = _mm256_andnot_si256(mask256, base_y_c256); + _mm256_store_si256((__m256i *)base_y_c, base_y_c256); + + a0_y = _mm256_cvtepu16_epi32(_mm_setr_epi16( + left[base_y_c[0]], left[base_y_c[1]], left[base_y_c[2]], + left[base_y_c[3]], left[base_y_c[4]], left[base_y_c[5]], + left[base_y_c[6]], left[base_y_c[7]])); + a1_y = _mm256_cvtepu16_epi32(_mm_setr_epi16( + left[base_y_c[0] + 1], left[base_y_c[1] + 1], left[base_y_c[2] + 1], + left[base_y_c[3] + 1], left[base_y_c[4] + 1], left[base_y_c[5] + 1], + left[base_y_c[6] + 1], left[base_y_c[7] + 1])); + + if (upsample_left) { + shift = _mm256_srli_epi32( + _mm256_and_si256(_mm256_slli_epi32((y_c256), upsample_left), c3f), + 1); + } else { + shift = _mm256_srli_epi32(_mm256_and_si256(y_c256, c3f), 1); + } + diff = _mm256_sub_epi32(a1_y, a0_y); // a[x+1] - a[x] + a32 = _mm256_slli_epi32(a0_y, 5); // a[x] * 32 + a32 = _mm256_add_epi32(a32, a16); // a[x] * 32 + 16 + + b = _mm256_mullo_epi32(diff, shift); + res = _mm256_add_epi32(a32, b); + res = _mm256_srli_epi32(res, 5); + + resy = _mm256_castsi256_si128(_mm256_packus_epi32( + res, _mm256_castsi128_si256(_mm256_extracti128_si256(res, 1)))); + } else { + resy = resx; + } + resxy = + _mm_blendv_epi8(resx, resy, *(__m128i *)HighbdBaseMask[base_min_diff]); + _mm_storeu_si128((__m128i *)(dst), resxy); + dst += stride; + } +} + +static void highbd_dr_prediction_z2_HxW_avx2( + int H, int W, uint16_t *dst, ptrdiff_t stride, const uint16_t *above, + const uint16_t *left, int upsample_above, int upsample_left, int dx, + int dy) { + // here upsample_above and upsample_left are 0 by design of + // av1_use_intra_edge_upsample + const int min_base_x = -1; + const int min_base_y = -1; + (void)upsample_above; + (void)upsample_left; + const int frac_bits_x = 6; + const int frac_bits_y = 6; + + // pre-filter above pixels + // store in temp buffers: + // above[x] * 32 + 16 + // above[x+1] - above[x] + // final pixels will be caluculated as: + // (above[x] * 32 + 16 + (above[x+1] - above[x]) * shift) >> 5 + __m256i a0_x, a1_x, a0_y, a1_y, a32, a0_1_x, a1_1_x, a16; + __m256i diff, min_base_y256, c3f; + __m128i a0_x128, a1_x128, a0_1_x128, a1_1_x128; + + a16 = _mm256_set1_epi32(16); + min_base_y256 = _mm256_set1_epi16(min_base_y); + c3f = _mm256_set1_epi32(0x3f); + for (int r = 0; r < H; r++) { + __m256i b, res, shift; + __m256i resx[2], resy[2]; + __m256i resxy; + for (int j = 0; j < W; j += 16) { + int y = r + 1; + int base_x = (-y * dx) >> frac_bits_x; + int base_shift = 0; + if ((base_x + j) < (min_base_x - 1)) { + base_shift = (min_base_x - (base_x + j) - 1); + } + int base_min_diff = (min_base_x - base_x - j); + if (base_min_diff > 16) { + base_min_diff = 16; + } else { + if (base_min_diff < 0) base_min_diff = 0; + } + + if (base_shift > 7) { + resx[0] = _mm256_setzero_si256(); + } else { + a0_x128 = _mm_loadu_si128((__m128i *)(above + base_x + base_shift + j)); + a1_x128 = + _mm_loadu_si128((__m128i *)(above + base_x + base_shift + 1 + j)); + a0_x128 = + _mm_shuffle_epi8(a0_x128, *(__m128i *)HighbdLoadMaskx[base_shift]); + a1_x128 = + _mm_shuffle_epi8(a1_x128, *(__m128i *)HighbdLoadMaskx[base_shift]); + + a0_x = _mm256_cvtepu16_epi32(a0_x128); + a1_x = _mm256_cvtepu16_epi32(a1_x128); + + shift = _mm256_srli_epi32( + _mm256_and_si256( + _mm256_setr_epi32( + ((0 + j) << 6) - y * dx, ((1 + j) << 6) - y * dx, + ((2 + j) << 6) - y * dx, ((3 + j) << 6) - y * dx, + ((4 + j) << 6) - y * dx, ((5 + j) << 6) - y * dx, + ((6 + j) << 6) - y * dx, ((7 + j) << 6) - y * dx), + _mm256_set1_epi32(0x3f)), + 1); + + diff = _mm256_sub_epi32(a1_x, a0_x); // a[x+1] - a[x] + a32 = _mm256_slli_epi32(a0_x, 5); // a[x] * 32 + a32 = _mm256_add_epi32(a32, a16); // a[x] * 32 + 16 + + b = _mm256_mullo_epi32(diff, shift); + res = _mm256_add_epi32(a32, b); + res = _mm256_srli_epi32(res, 5); + + resx[0] = _mm256_packus_epi32( + res, _mm256_castsi128_si256(_mm256_extracti128_si256(res, 1))); + } + base_shift = 0; + if ((base_x + j + 8) < (min_base_x - 1)) { + base_shift = (min_base_x - (base_x + j + 8) - 1); + } + if (base_shift > 7) { + resx[1] = _mm256_setzero_si256(); + } else { + a0_1_x128 = + _mm_loadu_si128((__m128i *)(above + base_x + base_shift + 8 + j)); + a1_1_x128 = + _mm_loadu_si128((__m128i *)(above + base_x + base_shift + 9 + j)); + a0_1_x128 = _mm_shuffle_epi8(a0_1_x128, + *(__m128i *)HighbdLoadMaskx[base_shift]); + a1_1_x128 = _mm_shuffle_epi8(a1_1_x128, + *(__m128i *)HighbdLoadMaskx[base_shift]); + + a0_1_x = _mm256_cvtepu16_epi32(a0_1_x128); + a1_1_x = _mm256_cvtepu16_epi32(a1_1_x128); + + shift = _mm256_srli_epi32( + _mm256_and_si256( + _mm256_setr_epi32( + ((8 + j) << 6) - y * dx, ((9 + j) << 6) - y * dx, + ((10 + j) << 6) - y * dx, ((11 + j) << 6) - y * dx, + ((12 + j) << 6) - y * dx, ((13 + j) << 6) - y * dx, + ((14 + j) << 6) - y * dx, ((15 + j) << 6) - y * dx), + _mm256_set1_epi32(0x3f)), + 1); + + diff = _mm256_sub_epi32(a1_1_x, a0_1_x); // a[x+1] - a[x] + a32 = _mm256_slli_epi32(a0_1_x, 5); // a[x] * 32 + a32 = _mm256_add_epi32(a32, a16); // a[x] * 32 + 16 + b = _mm256_mullo_epi32(diff, shift); + + resx[1] = _mm256_add_epi32(a32, b); + resx[1] = _mm256_srli_epi32(resx[1], 5); + resx[1] = _mm256_packus_epi32( + resx[1], + _mm256_castsi128_si256(_mm256_extracti128_si256(resx[1], 1))); + } + resx[0] = + _mm256_inserti128_si256(resx[0], _mm256_castsi256_si128(resx[1]), + 1); // 16 16bit values + + // y calc + if ((base_x < min_base_x)) { + DECLARE_ALIGNED(32, int, base_y_c[16]); + __m256i r6, c256, dy256, y_c256, y_c_1_256, base_y_c256, mask256; + r6 = _mm256_set1_epi32(r << 6); + dy256 = _mm256_set1_epi32(dy); + c256 = _mm256_setr_epi32(1 + j, 2 + j, 3 + j, 4 + j, 5 + j, 6 + j, + 7 + j, 8 + j); + y_c256 = _mm256_sub_epi32(r6, _mm256_mullo_epi32(c256, dy256)); + base_y_c256 = _mm256_srai_epi32(y_c256, frac_bits_y); + mask256 = _mm256_cmpgt_epi32(min_base_y256, base_y_c256); + base_y_c256 = _mm256_andnot_si256(mask256, base_y_c256); + _mm256_store_si256((__m256i *)base_y_c, base_y_c256); + c256 = _mm256_setr_epi32(9 + j, 10 + j, 11 + j, 12 + j, 13 + j, 14 + j, + 15 + j, 16 + j); + y_c_1_256 = _mm256_sub_epi32(r6, _mm256_mullo_epi32(c256, dy256)); + base_y_c256 = _mm256_srai_epi32(y_c_1_256, frac_bits_y); + mask256 = _mm256_cmpgt_epi32(min_base_y256, base_y_c256); + base_y_c256 = _mm256_andnot_si256(mask256, base_y_c256); + _mm256_store_si256((__m256i *)(base_y_c + 8), base_y_c256); + + a0_y = _mm256_cvtepu16_epi32(_mm_setr_epi16( + left[base_y_c[0]], left[base_y_c[1]], left[base_y_c[2]], + left[base_y_c[3]], left[base_y_c[4]], left[base_y_c[5]], + left[base_y_c[6]], left[base_y_c[7]])); + a1_y = _mm256_cvtepu16_epi32(_mm_setr_epi16( + left[base_y_c[0] + 1], left[base_y_c[1] + 1], left[base_y_c[2] + 1], + left[base_y_c[3] + 1], left[base_y_c[4] + 1], left[base_y_c[5] + 1], + left[base_y_c[6] + 1], left[base_y_c[7] + 1])); + + shift = _mm256_srli_epi32(_mm256_and_si256(y_c256, c3f), 1); + + diff = _mm256_sub_epi32(a1_y, a0_y); // a[x+1] - a[x] + a32 = _mm256_slli_epi32(a0_y, 5); // a[x] * 32 + a32 = _mm256_add_epi32(a32, a16); // a[x] * 32 + 16 + + b = _mm256_mullo_epi32(diff, shift); + res = _mm256_add_epi32(a32, b); + res = _mm256_srli_epi32(res, 5); + + resy[0] = _mm256_packus_epi32( + res, _mm256_castsi128_si256(_mm256_extracti128_si256(res, 1))); + + a0_y = _mm256_cvtepu16_epi32(_mm_setr_epi16( + left[base_y_c[8]], left[base_y_c[9]], left[base_y_c[10]], + left[base_y_c[11]], left[base_y_c[12]], left[base_y_c[13]], + left[base_y_c[14]], left[base_y_c[15]])); + a1_y = _mm256_cvtepu16_epi32( + _mm_setr_epi16(left[base_y_c[8] + 1], left[base_y_c[9] + 1], + left[base_y_c[10] + 1], left[base_y_c[11] + 1], + left[base_y_c[12] + 1], left[base_y_c[13] + 1], + left[base_y_c[14] + 1], left[base_y_c[15] + 1])); + shift = _mm256_srli_epi32(_mm256_and_si256(y_c_1_256, c3f), 1); + + diff = _mm256_sub_epi32(a1_y, a0_y); // a[x+1] - a[x] + a32 = _mm256_slli_epi32(a0_y, 5); // a[x] * 32 + a32 = _mm256_add_epi32(a32, a16); // a[x] * 32 + 16 + + b = _mm256_mullo_epi32(diff, shift); + res = _mm256_add_epi32(a32, b); + res = _mm256_srli_epi32(res, 5); + + resy[1] = _mm256_packus_epi32( + res, _mm256_castsi128_si256(_mm256_extracti128_si256(res, 1))); + + resy[0] = + _mm256_inserti128_si256(resy[0], _mm256_castsi256_si128(resy[1]), + 1); // 16 16bit values + } else { + resy[0] = resx[0]; + } + resxy = _mm256_blendv_epi8(resx[0], resy[0], + *(__m256i *)HighbdBaseMask[base_min_diff]); + _mm256_storeu_si256((__m256i *)(dst + j), resxy); + } // for j + dst += stride; + } +} + +// Directional prediction, zone 2: 90 < angle < 180 +void av1_highbd_dr_prediction_z2_avx2(uint16_t *dst, ptrdiff_t stride, int bw, + int bh, const uint16_t *above, + const uint16_t *left, int upsample_above, + int upsample_left, int dx, int dy, + int bd) { + (void)bd; + assert(dx > 0); + assert(dy > 0); + switch (bw) { + case 4: + highbd_dr_prediction_z2_Nx4_avx2(bh, dst, stride, above, left, + upsample_above, upsample_left, dx, dy); + break; + case 8: + highbd_dr_prediction_z2_Nx8_avx2(bh, dst, stride, above, left, + upsample_above, upsample_left, dx, dy); + break; + default: + highbd_dr_prediction_z2_HxW_avx2(bh, bw, dst, stride, above, left, + upsample_above, upsample_left, dx, dy); + break; + } + return; +} + +static void highbd_transpose(const uint16_t *src, ptrdiff_t pitchSrc, + uint16_t *dst, ptrdiff_t pitchDst, int width, + int height) { + for (int j = 0; j < height; j += 8) + for (int i = 0; i < width; i += 8) + highbd_transpose_TX_8X8(src + i * pitchSrc + j, pitchSrc, + dst + j * pitchDst + i, pitchDst); +} + +static void highbd_dr_prediction_z3_4x4_avx2(uint16_t *dst, ptrdiff_t stride, + const uint16_t *left, + int upsample_left, int dy) { + __m128i dstvec[4], d[4]; + + highbd_dr_prediction_z1_4xN_internal_avx2(4, dstvec, left, upsample_left, dy); + highbd_transpose4x8_8x4_low_sse2(&dstvec[0], &dstvec[1], &dstvec[2], + &dstvec[3], &d[0], &d[1], &d[2], &d[3]); + _mm_storel_epi64((__m128i *)(dst + 0 * stride), d[0]); + _mm_storel_epi64((__m128i *)(dst + 1 * stride), d[1]); + _mm_storel_epi64((__m128i *)(dst + 2 * stride), d[2]); + _mm_storel_epi64((__m128i *)(dst + 3 * stride), d[3]); + return; +} + +static void highbd_dr_prediction_z3_8x8_avx2(uint16_t *dst, ptrdiff_t stride, + const uint16_t *left, + int upsample_left, int dy) { + __m128i dstvec[8], d[8]; + + highbd_dr_prediction_z1_8xN_internal_avx2(8, dstvec, left, upsample_left, dy); + highbd_transpose8x8_sse2(&dstvec[0], &dstvec[1], &dstvec[2], &dstvec[3], + &dstvec[4], &dstvec[5], &dstvec[6], &dstvec[7], + &d[0], &d[1], &d[2], &d[3], &d[4], &d[5], &d[6], + &d[7]); + for (int i = 0; i < 8; i++) { + _mm_storeu_si128((__m128i *)(dst + i * stride), d[i]); + } +} + +static void highbd_dr_prediction_z3_4x8_avx2(uint16_t *dst, ptrdiff_t stride, + const uint16_t *left, + int upsample_left, int dy) { + __m128i dstvec[4], d[8]; + + highbd_dr_prediction_z1_8xN_internal_avx2(4, dstvec, left, upsample_left, dy); + highbd_transpose4x8_8x4_sse2(&dstvec[0], &dstvec[1], &dstvec[2], &dstvec[3], + &d[0], &d[1], &d[2], &d[3], &d[4], &d[5], &d[6], + &d[7]); + for (int i = 0; i < 8; i++) { + _mm_storel_epi64((__m128i *)(dst + i * stride), d[i]); + } +} + +static void highbd_dr_prediction_z3_8x4_avx2(uint16_t *dst, ptrdiff_t stride, + const uint16_t *left, + int upsample_left, int dy) { + __m128i dstvec[8], d[4]; + + highbd_dr_prediction_z1_4xN_internal_avx2(8, dstvec, left, upsample_left, dy); + highbd_transpose8x8_low_sse2(&dstvec[0], &dstvec[1], &dstvec[2], &dstvec[3], + &dstvec[4], &dstvec[5], &dstvec[6], &dstvec[7], + &d[0], &d[1], &d[2], &d[3]); + _mm_storeu_si128((__m128i *)(dst + 0 * stride), d[0]); + _mm_storeu_si128((__m128i *)(dst + 1 * stride), d[1]); + _mm_storeu_si128((__m128i *)(dst + 2 * stride), d[2]); + _mm_storeu_si128((__m128i *)(dst + 3 * stride), d[3]); +} + +static void highbd_dr_prediction_z3_8x16_avx2(uint16_t *dst, ptrdiff_t stride, + const uint16_t *left, + int upsample_left, int dy) { + __m256i dstvec[8], d[16]; + + highbd_dr_prediction_z1_16xN_internal_avx2(8, dstvec, left, upsample_left, + dy); + highbd_transpose8x16_16x8_avx2(dstvec, d); + for (int i = 0; i < 8; i++) { + _mm_storeu_si128((__m128i *)(dst + i * stride), + _mm256_castsi256_si128(d[i])); + } + for (int i = 8; i < 16; i++) { + _mm_storeu_si128((__m128i *)(dst + i * stride), + _mm256_extracti128_si256(d[i - 8], 1)); + } +} + +static void highbd_dr_prediction_z3_16x8_avx2(uint16_t *dst, ptrdiff_t stride, + const uint16_t *left, + int upsample_left, int dy) { + __m128i dstvec[16], d[16]; + + highbd_dr_prediction_z1_8xN_internal_avx2(16, dstvec, left, upsample_left, + dy); + for (int i = 0; i < 16; i += 8) { + highbd_transpose8x8_sse2(&dstvec[0 + i], &dstvec[1 + i], &dstvec[2 + i], + &dstvec[3 + i], &dstvec[4 + i], &dstvec[5 + i], + &dstvec[6 + i], &dstvec[7 + i], &d[0 + i], + &d[1 + i], &d[2 + i], &d[3 + i], &d[4 + i], + &d[5 + i], &d[6 + i], &d[7 + i]); + } + for (int i = 0; i < 8; i++) { + _mm_storeu_si128((__m128i *)(dst + i * stride), d[i]); + _mm_storeu_si128((__m128i *)(dst + i * stride + 8), d[i + 8]); + } +} + +static void highbd_dr_prediction_z3_4x16_avx2(uint16_t *dst, ptrdiff_t stride, + const uint16_t *left, + int upsample_left, int dy) { + __m256i dstvec[4], d[4], d1; + + highbd_dr_prediction_z1_16xN_internal_avx2(4, dstvec, left, upsample_left, + dy); + highbd_transpose4x16_avx2(dstvec, d); + for (int i = 0; i < 4; i++) { + _mm_storel_epi64((__m128i *)(dst + i * stride), + _mm256_castsi256_si128(d[i])); + d1 = _mm256_bsrli_epi128(d[i], 8); + _mm_storel_epi64((__m128i *)(dst + (i + 4) * stride), + _mm256_castsi256_si128(d1)); + _mm_storel_epi64((__m128i *)(dst + (i + 8) * stride), + _mm256_extracti128_si256(d[i], 1)); + _mm_storel_epi64((__m128i *)(dst + (i + 12) * stride), + _mm256_extracti128_si256(d1, 1)); + } +} + +static void highbd_dr_prediction_z3_16x4_avx2(uint16_t *dst, ptrdiff_t stride, + const uint16_t *left, + int upsample_left, int dy) { + __m128i dstvec[16], d[8]; + + highbd_dr_prediction_z1_4xN_internal_avx2(16, dstvec, left, upsample_left, + dy); + highbd_transpose16x4_8x8_sse2(dstvec, d); + + _mm_storeu_si128((__m128i *)(dst + 0 * stride), d[0]); + _mm_storeu_si128((__m128i *)(dst + 0 * stride + 8), d[1]); + _mm_storeu_si128((__m128i *)(dst + 1 * stride), d[2]); + _mm_storeu_si128((__m128i *)(dst + 1 * stride + 8), d[3]); + _mm_storeu_si128((__m128i *)(dst + 2 * stride), d[4]); + _mm_storeu_si128((__m128i *)(dst + 2 * stride + 8), d[5]); + _mm_storeu_si128((__m128i *)(dst + 3 * stride), d[6]); + _mm_storeu_si128((__m128i *)(dst + 3 * stride + 8), d[7]); +} + +static void highbd_dr_prediction_z3_8x32_avx2(uint16_t *dst, ptrdiff_t stride, + const uint16_t *left, + int upsample_left, int dy) { + __m256i dstvec[16], d[16]; + + highbd_dr_prediction_z1_32xN_internal_avx2(8, dstvec, left, upsample_left, + dy); + for (int i = 0; i < 16; i += 8) { + highbd_transpose8x16_16x8_avx2(dstvec + i, d + i); + } + + for (int i = 0; i < 8; i++) { + _mm_storeu_si128((__m128i *)(dst + i * stride), + _mm256_castsi256_si128(d[i])); + } + for (int i = 0; i < 8; i++) { + _mm_storeu_si128((__m128i *)(dst + (i + 8) * stride), + _mm256_extracti128_si256(d[i], 1)); + } + for (int i = 8; i < 16; i++) { + _mm_storeu_si128((__m128i *)(dst + (i + 8) * stride), + _mm256_castsi256_si128(d[i])); + } + for (int i = 8; i < 16; i++) { + _mm_storeu_si128((__m128i *)(dst + (i + 16) * stride), + _mm256_extracti128_si256(d[i], 1)); + } +} + +static void highbd_dr_prediction_z3_32x8_avx2(uint16_t *dst, ptrdiff_t stride, + const uint16_t *left, + int upsample_left, int dy) { + __m128i dstvec[32], d[32]; + + highbd_dr_prediction_z1_8xN_internal_avx2(32, dstvec, left, upsample_left, + dy); + for (int i = 0; i < 32; i += 8) { + highbd_transpose8x8_sse2(&dstvec[0 + i], &dstvec[1 + i], &dstvec[2 + i], + &dstvec[3 + i], &dstvec[4 + i], &dstvec[5 + i], + &dstvec[6 + i], &dstvec[7 + i], &d[0 + i], + &d[1 + i], &d[2 + i], &d[3 + i], &d[4 + i], + &d[5 + i], &d[6 + i], &d[7 + i]); + } + for (int i = 0; i < 8; i++) { + _mm_storeu_si128((__m128i *)(dst + i * stride), d[i]); + _mm_storeu_si128((__m128i *)(dst + i * stride + 8), d[i + 8]); + _mm_storeu_si128((__m128i *)(dst + i * stride + 16), d[i + 16]); + _mm_storeu_si128((__m128i *)(dst + i * stride + 24), d[i + 24]); + } +} + +static void highbd_dr_prediction_z3_16x16_avx2(uint16_t *dst, ptrdiff_t stride, + const uint16_t *left, + int upsample_left, int dy) { + __m256i dstvec[16], d[16]; + + highbd_dr_prediction_z1_16xN_internal_avx2(16, dstvec, left, upsample_left, + dy); + highbd_transpose16x16_avx2(dstvec, d); + + for (int i = 0; i < 16; i++) { + _mm256_storeu_si256((__m256i *)(dst + i * stride), d[i]); + } +} + +static void highbd_dr_prediction_z3_32x32_avx2(uint16_t *dst, ptrdiff_t stride, + const uint16_t *left, + int upsample_left, int dy) { + __m256i dstvec[64], d[16]; + + highbd_dr_prediction_z1_32xN_internal_avx2(32, dstvec, left, upsample_left, + dy); + + highbd_transpose16x16_avx2(dstvec, d); + for (int j = 0; j < 16; j++) { + _mm256_storeu_si256((__m256i *)(dst + j * stride), d[j]); + } + highbd_transpose16x16_avx2(dstvec + 16, d); + for (int j = 0; j < 16; j++) { + _mm256_storeu_si256((__m256i *)(dst + j * stride + 16), d[j]); + } + highbd_transpose16x16_avx2(dstvec + 32, d); + for (int j = 0; j < 16; j++) { + _mm256_storeu_si256((__m256i *)(dst + (j + 16) * stride), d[j]); + } + highbd_transpose16x16_avx2(dstvec + 48, d); + for (int j = 0; j < 16; j++) { + _mm256_storeu_si256((__m256i *)(dst + (j + 16) * stride + 16), d[j]); + } +} + +static void highbd_dr_prediction_z3_64x64_avx2(uint16_t *dst, ptrdiff_t stride, + const uint16_t *left, + int upsample_left, int dy) { + DECLARE_ALIGNED(16, uint16_t, dstT[64 * 64]); + highbd_dr_prediction_z1_64xN_avx2(64, dstT, 64, left, upsample_left, dy); + highbd_transpose(dstT, 64, dst, stride, 64, 64); +} + +static void highbd_dr_prediction_z3_16x32_avx2(uint16_t *dst, ptrdiff_t stride, + const uint16_t *left, + int upsample_left, int dy) { + __m256i dstvec[32], d[32]; + + highbd_dr_prediction_z1_32xN_internal_avx2(16, dstvec, left, upsample_left, + dy); + for (int i = 0; i < 32; i += 8) { + highbd_transpose8x16_16x8_avx2(dstvec + i, d + i); + } + // store + for (int j = 0; j < 32; j += 16) { + for (int i = 0; i < 8; i++) { + _mm_storeu_si128((__m128i *)(dst + (i + j) * stride), + _mm256_castsi256_si128(d[(i + j)])); + } + for (int i = 0; i < 8; i++) { + _mm_storeu_si128((__m128i *)(dst + (i + j) * stride + 8), + _mm256_castsi256_si128(d[(i + j) + 8])); + } + for (int i = 8; i < 16; i++) { + _mm256_storeu_si256( + (__m256i *)(dst + (i + j) * stride), + _mm256_inserti128_si256( + d[(i + j)], _mm256_extracti128_si256(d[(i + j) - 8], 1), 0)); + } + } +} + +static void highbd_dr_prediction_z3_32x16_avx2(uint16_t *dst, ptrdiff_t stride, + const uint16_t *left, + int upsample_left, int dy) { + __m256i dstvec[32], d[16]; + + highbd_dr_prediction_z1_16xN_internal_avx2(32, dstvec, left, upsample_left, + dy); + for (int i = 0; i < 32; i += 16) { + highbd_transpose16x16_avx2((dstvec + i), d); + for (int j = 0; j < 16; j++) { + _mm256_storeu_si256((__m256i *)(dst + j * stride + i), d[j]); + } + } +} + +static void highbd_dr_prediction_z3_32x64_avx2(uint16_t *dst, ptrdiff_t stride, + const uint16_t *left, + int upsample_left, int dy) { + uint16_t dstT[64 * 32]; + highbd_dr_prediction_z1_64xN_avx2(32, dstT, 64, left, upsample_left, dy); + highbd_transpose(dstT, 64, dst, stride, 32, 64); +} + +static void highbd_dr_prediction_z3_64x32_avx2(uint16_t *dst, ptrdiff_t stride, + const uint16_t *left, + int upsample_left, int dy) { + DECLARE_ALIGNED(16, uint16_t, dstT[32 * 64]); + highbd_dr_prediction_z1_32xN_avx2(64, dstT, 32, left, upsample_left, dy); + highbd_transpose(dstT, 32, dst, stride, 64, 32); + return; +} + +static void highbd_dr_prediction_z3_16x64_avx2(uint16_t *dst, ptrdiff_t stride, + const uint16_t *left, + int upsample_left, int dy) { + DECLARE_ALIGNED(16, uint16_t, dstT[64 * 16]); + highbd_dr_prediction_z1_64xN_avx2(16, dstT, 64, left, upsample_left, dy); + highbd_transpose(dstT, 64, dst, stride, 16, 64); +} + +static void highbd_dr_prediction_z3_64x16_avx2(uint16_t *dst, ptrdiff_t stride, + const uint16_t *left, + int upsample_left, int dy) { + __m256i dstvec[64], d[16]; + + highbd_dr_prediction_z1_16xN_internal_avx2(64, dstvec, left, upsample_left, + dy); + for (int i = 0; i < 64; i += 16) { + highbd_transpose16x16_avx2((dstvec + i), d); + for (int j = 0; j < 16; j++) { + _mm256_storeu_si256((__m256i *)(dst + j * stride + i), d[j]); + } + } +} + +void av1_highbd_dr_prediction_z3_avx2(uint16_t *dst, ptrdiff_t stride, int bw, + int bh, const uint16_t *above, + const uint16_t *left, int upsample_left, + int dx, int dy, int bd) { + (void)above; + (void)dx; + (void)bd; + assert(dx == 1); + assert(dy > 0); + if (bw == bh) { + switch (bw) { + case 4: + highbd_dr_prediction_z3_4x4_avx2(dst, stride, left, upsample_left, dy); + break; + case 8: + highbd_dr_prediction_z3_8x8_avx2(dst, stride, left, upsample_left, dy); + break; + case 16: + highbd_dr_prediction_z3_16x16_avx2(dst, stride, left, upsample_left, + dy); + break; + case 32: + highbd_dr_prediction_z3_32x32_avx2(dst, stride, left, upsample_left, + dy); + break; + case 64: + highbd_dr_prediction_z3_64x64_avx2(dst, stride, left, upsample_left, + dy); + break; + } + } else { + if (bw < bh) { + if (bw + bw == bh) { + switch (bw) { + case 4: + highbd_dr_prediction_z3_4x8_avx2(dst, stride, left, upsample_left, + dy); + break; + case 8: + highbd_dr_prediction_z3_8x16_avx2(dst, stride, left, upsample_left, + dy); + break; + case 16: + highbd_dr_prediction_z3_16x32_avx2(dst, stride, left, upsample_left, + dy); + break; + case 32: + highbd_dr_prediction_z3_32x64_avx2(dst, stride, left, upsample_left, + dy); + break; + } + } else { + switch (bw) { + case 4: + highbd_dr_prediction_z3_4x16_avx2(dst, stride, left, upsample_left, + dy); + break; + case 8: + highbd_dr_prediction_z3_8x32_avx2(dst, stride, left, upsample_left, + dy); + break; + case 16: + highbd_dr_prediction_z3_16x64_avx2(dst, stride, left, upsample_left, + dy); + break; + } + } + } else { + if (bh + bh == bw) { + switch (bh) { + case 4: + highbd_dr_prediction_z3_8x4_avx2(dst, stride, left, upsample_left, + dy); + break; + case 8: + highbd_dr_prediction_z3_16x8_avx2(dst, stride, left, upsample_left, + dy); + break; + case 16: + highbd_dr_prediction_z3_32x16_avx2(dst, stride, left, upsample_left, + dy); + break; + case 32: + highbd_dr_prediction_z3_64x32_avx2(dst, stride, left, upsample_left, + dy); + break; + } + } else { + switch (bh) { + case 4: + highbd_dr_prediction_z3_16x4_avx2(dst, stride, left, upsample_left, + dy); + break; + case 8: + highbd_dr_prediction_z3_32x8_avx2(dst, stride, left, upsample_left, + dy); + break; + case 16: + highbd_dr_prediction_z3_64x16_avx2(dst, stride, left, upsample_left, + dy); + break; + } + } + } + } + return; +} + +// Low bit depth functions +static uint8_t BaseMask[33][32] = { + { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 0xff, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 0xff, 0xff, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 0xff, 0xff, 0xff, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 0xff, 0xff, 0xff, 0xff, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 0xff, 0xff, 0xff, 0xff, 0xff, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0, 0, 0, 0, 0, 0, 0 }, + { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0, 0, 0, 0, 0, 0 }, + { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0, 0, 0, 0, 0 }, + { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0, 0, 0, 0 }, + { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0, 0, 0 }, + { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0, 0 }, + { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0 }, + { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, +}; + +static AOM_FORCE_INLINE void dr_prediction_z1_4xN_internal_avx2( + int N, __m128i *dst, const uint8_t *above, int upsample_above, int dx) { + const int frac_bits = 6 - upsample_above; + const int max_base_x = ((N + 4) - 1) << upsample_above; + int x; + // a assert(dx > 0); + // pre-filter above pixels + // store in temp buffers: + // above[x] * 32 + 16 + // above[x+1] - above[x] + // final pixels will be caluculated as: + // (above[x] * 32 + 16 + (above[x+1] - above[x]) * shift) >> 5 + __m256i a0, a1, a32, a16; + __m256i diff; + __m128i a_mbase_x; + + a16 = _mm256_set1_epi32(16); + a_mbase_x = _mm_set1_epi8(above[max_base_x]); + + x = dx; + for (int r = 0; r < N; r++) { + __m256i b, res, shift; + __m128i res1, a0_128, a1_128; + + int base = x >> frac_bits; + int base_max_diff = (max_base_x - base) >> upsample_above; + if (base_max_diff <= 0) { + for (int i = r; i < N; ++i) { + dst[i] = a_mbase_x; // save 4 values + } + return; + } + if (base_max_diff > 4) base_max_diff = 4; + a0_128 = _mm_loadu_si128((__m128i *)(above + base)); + a1_128 = _mm_srli_si128(a0_128, 1); + a0 = _mm256_cvtepu8_epi32(a0_128); + a1 = _mm256_cvtepu8_epi32(a1_128); + + if (upsample_above) { + a0 = _mm256_permutevar8x32_epi32( + a0, _mm256_set_epi32(7, 5, 3, 1, 6, 4, 2, 0)); + a1 = _mm256_castsi128_si256(_mm256_extracti128_si256(a0, 1)); + shift = _mm256_srli_epi32( + _mm256_and_si256( + _mm256_slli_epi32(_mm256_set1_epi32(x), upsample_above), + _mm256_set1_epi32(0x3f)), + 1); + } else { + shift = _mm256_srli_epi32( + _mm256_and_si256(_mm256_set1_epi32(x), _mm256_set1_epi32(0x3f)), 1); + } + + diff = _mm256_sub_epi32(a1, a0); // a[x+1] - a[x] + a32 = _mm256_slli_epi32(a0, 5); // a[x] * 32 + a32 = _mm256_add_epi32(a32, a16); // a[x] * 32 + 16 + + b = _mm256_mullo_epi32(diff, shift); + res = _mm256_add_epi32(a32, b); + res = _mm256_srli_epi32(res, 5); + + res1 = _mm256_castsi256_si128(res); + res1 = _mm_packus_epi32(res1, res1); + res1 = _mm_packus_epi16(res1, res1); + + dst[r] = + _mm_blendv_epi8(a_mbase_x, res1, *(__m128i *)BaseMask[base_max_diff]); + x += dx; + } +} + +static void dr_prediction_z1_4xN_avx2(int N, uint8_t *dst, ptrdiff_t stride, + const uint8_t *above, int upsample_above, + int dx) { + __m128i dstvec[16]; + + dr_prediction_z1_4xN_internal_avx2(N, dstvec, above, upsample_above, dx); + for (int i = 0; i < N; i++) { + *(uint32_t *)(dst + stride * i) = _mm_cvtsi128_si32(dstvec[i]); + } +} + +static AOM_FORCE_INLINE void dr_prediction_z1_8xN_internal_avx2( + int N, __m128i *dst, const uint8_t *above, int upsample_above, int dx) { + const int frac_bits = 6 - upsample_above; + const int max_base_x = ((8 + N) - 1) << upsample_above; + + int x; + // pre-filter above pixels + // store in temp buffers: + // above[x] * 32 + 16 + // above[x+1] - above[x] + // final pixels will be caluculated as: + // (above[x] * 32 + 16 + (above[x+1] - above[x]) * shift) >> 5 + __m256i a0, a1, a0_1, a1_1, a32, a16, diff; + __m128i a_mbase_x; + + a16 = _mm256_set1_epi32(16); + a_mbase_x = _mm_set1_epi8(above[max_base_x]); + + x = dx; + for (int r = 0; r < N; r++) { + __m256i b, res, res1, shift; + __m128i res128; + + int base = x >> frac_bits; + int base_max_diff = (max_base_x - base) >> upsample_above; + if (base_max_diff <= 0) { + for (int i = r; i < N; ++i) { + dst[i] = a_mbase_x; // save 16 values, 8 to be used furter + } + return; + } + if (base_max_diff > 8) base_max_diff = 8; + + a0 = _mm256_cvtepu8_epi32(_mm_loadu_si128((__m128i *)(above + base))); + a1 = _mm256_cvtepu8_epi32(_mm_loadu_si128((__m128i *)(above + base + 1))); + + if (upsample_above) { + a0 = _mm256_permutevar8x32_epi32( + a0, _mm256_set_epi32(7, 5, 3, 1, 6, 4, 2, 0)); + a1 = _mm256_castsi128_si256(_mm256_extracti128_si256(a0, 1)); + + a0_1 = + _mm256_cvtepu8_epi32(_mm_loadu_si128((__m128i *)(above + base + 8))); + a0_1 = _mm256_permutevar8x32_epi32( + a0_1, _mm256_set_epi32(7, 5, 3, 1, 6, 4, 2, 0)); + a1_1 = _mm256_castsi128_si256(_mm256_extracti128_si256(a0_1, 1)); + + a0 = _mm256_inserti128_si256(a0, _mm256_castsi256_si128(a0_1), 1); + a1 = _mm256_inserti128_si256(a1, _mm256_castsi256_si128(a1_1), 1); + + shift = _mm256_srli_epi32( + _mm256_and_si256( + _mm256_slli_epi32(_mm256_set1_epi32(x), upsample_above), + _mm256_set1_epi32(0x3f)), + 1); + } else { + shift = _mm256_srli_epi32( + _mm256_and_si256(_mm256_set1_epi32(x), _mm256_set1_epi32(0x3f)), 1); + } + + diff = _mm256_sub_epi32(a1, a0); // a[x+1] - a[x] + a32 = _mm256_slli_epi32(a0, 5); // a[x] * 32 + a32 = _mm256_add_epi32(a32, a16); // a[x] * 32 + 16 + + b = _mm256_mullo_epi32(diff, shift); + res = _mm256_add_epi32(a32, b); + res = _mm256_srli_epi32(res, 5); + + res1 = _mm256_packus_epi32( + res, _mm256_castsi128_si256( + _mm256_extracti128_si256(res, 1))); // goto 16 bit + + res128 = _mm_packus_epi16(_mm256_castsi256_si128(res1), + _mm256_castsi256_si128(res1)); // goto 8 bit + + res128 = + _mm_blendv_epi8(a_mbase_x, res128, *(__m128i *)BaseMask[base_max_diff]); + dst[r] = res128; + x += dx; + } +} + +static void dr_prediction_z1_8xN_avx2(int N, uint8_t *dst, ptrdiff_t stride, + const uint8_t *above, int upsample_above, + int dx) { + __m128i dstvec[32]; + + dr_prediction_z1_8xN_internal_avx2(N, dstvec, above, upsample_above, dx); + for (int i = 0; i < N; i++) { + _mm_storel_epi64((__m128i *)(dst + stride * i), dstvec[i]); + } +} + +static AOM_FORCE_INLINE void dr_prediction_z1_16xN_internal_avx2( + int N, __m128i *dstvec, const uint8_t *above, int upsample_above, int dx) { + int x; + // here upsample_above is 0 by design of av1_use_intra_edge_upsample + (void)upsample_above; + const int frac_bits = 6; + const int max_base_x = ((16 + N) - 1); + + // pre-filter above pixels + // store in temp buffers: + // above[x] * 32 + 16 + // above[x+1] - above[x] + // final pixels will be caluculated as: + // (above[x] * 32 + 16 + (above[x+1] - above[x]) * shift) >> 5 + __m256i a0, a0_1, a1, a1_1, diff, a32, a16; + __m128i a_mbase_x; + + a16 = _mm256_set1_epi32(16); + a_mbase_x = _mm_set1_epi8((uint8_t)above[max_base_x]); + + x = dx; + for (int r = 0; r < N; r++) { + __m256i b, res[2]; + __m128i res128[2]; + int base = x >> frac_bits; + int base_max_diff = (max_base_x - base); + if (base_max_diff <= 0) { + for (int i = r; i < N; ++i) { + dstvec[i] = a_mbase_x; // save 16 values + } + return; + } + __m256i shift = _mm256_srli_epi32( + _mm256_and_si256(_mm256_set1_epi32(x), _mm256_set1_epi32(0x3f)), 1); + + a0 = _mm256_cvtepu8_epi32(_mm_loadu_si128((__m128i *)(above + base))); + a1 = _mm256_cvtepu8_epi32(_mm_loadu_si128((__m128i *)(above + base + 1))); + + diff = _mm256_sub_epi32(a1, a0); // a[x+1] - a[x] + a32 = _mm256_slli_epi32(a0, 5); // a[x] * 32 + a32 = _mm256_add_epi32(a32, a16); // a[x] * 32 + 16 + b = _mm256_mullo_epi32(diff, shift); + + res[0] = _mm256_add_epi32(a32, b); + res[0] = _mm256_srli_epi32(res[0], 5); + res[0] = _mm256_packus_epi32( + res[0], _mm256_castsi128_si256(_mm256_extracti128_si256(res[0], 1))); + res128[0] = _mm_packus_epi16(_mm256_castsi256_si128(res[0]), + _mm256_castsi256_si128(res[0])); // goto 8 bit + + if (base_max_diff > 8) { + if (base_max_diff > 16) base_max_diff = 16; + a0_1 = + _mm256_cvtepu8_epi32(_mm_loadu_si128((__m128i *)(above + base + 8))); + a1_1 = + _mm256_cvtepu8_epi32(_mm_loadu_si128((__m128i *)(above + base + 9))); + + diff = _mm256_sub_epi32(a1_1, a0_1); // a[x+1] - a[x] + a32 = _mm256_slli_epi32(a0_1, 5); // a[x] * 32 + a32 = _mm256_add_epi32(a32, a16); // a[x] * 32 + 16 + b = _mm256_mullo_epi32(diff, shift); + + res[1] = _mm256_add_epi32(a32, b); + res[1] = _mm256_srli_epi32(res[1], 5); + res[1] = _mm256_packus_epi32( + res[1], _mm256_castsi128_si256(_mm256_extracti128_si256(res[1], 1))); + res128[1] = + _mm_packus_epi16(_mm256_castsi256_si128(res[1]), + _mm256_castsi256_si128(res[1])); // goto 8 bit + + } else { + res128[1] = a_mbase_x; + } + res128[0] = _mm_unpacklo_epi64(res128[0], res128[1]); // 16 8bit values + + dstvec[r] = _mm_blendv_epi8(a_mbase_x, res128[0], + *(__m128i *)BaseMask[base_max_diff]); + x += dx; + } +} +static void dr_prediction_z1_16xN_avx2(int N, uint8_t *dst, ptrdiff_t stride, + const uint8_t *above, int upsample_above, + int dx) { + __m128i dstvec[64]; + + dr_prediction_z1_16xN_internal_avx2(N, dstvec, above, upsample_above, dx); + for (int i = 0; i < N; i++) { + _mm_storeu_si128((__m128i *)(dst + stride * i), dstvec[i]); + } +} + +static AOM_FORCE_INLINE void dr_prediction_z1_32xN_internal_avx2( + int N, __m256i *dstvec, const uint8_t *above, int upsample_above, int dx) { + int x; + // here upsample_above is 0 by design of av1_use_intra_edge_upsample + (void)upsample_above; + const int frac_bits = 6; + const int max_base_x = ((32 + N) - 1); + + // pre-filter above pixels + // store in temp buffers: + // above[x] * 32 + 16 + // above[x+1] - above[x] + // final pixels will be caluculated as: + // (above[x] * 32 + 16 + (above[x+1] - above[x]) * shift) >> 5 + __m256i a0, a0_1, a1, a1_1, a32, a16; + __m256i a_mbase_x, diff; + + a16 = _mm256_set1_epi32(16); + a_mbase_x = _mm256_set1_epi8(above[max_base_x]); + + x = dx; + for (int r = 0; r < N; r++) { + __m256i b, res[2], res16[2]; + + int base = x >> frac_bits; + int base_max_diff = (max_base_x - base); + if (base_max_diff <= 0) { + for (int i = r; i < N; ++i) { + dstvec[i] = a_mbase_x; // save 32 values + } + return; + } + if (base_max_diff > 32) base_max_diff = 32; + __m256i shift = _mm256_srli_epi32( + _mm256_and_si256(_mm256_set1_epi32(x), _mm256_set1_epi32(0x3f)), 1); + + for (int j = 0, jj = 0; j < 32; j += 16, jj++) { + int mdiff = base_max_diff - j; + if (mdiff <= 0) { + res16[jj] = a_mbase_x; + } else { + a0 = _mm256_cvtepu8_epi32( + _mm_loadu_si128((__m128i *)(above + base + j))); + a1 = _mm256_cvtepu8_epi32( + _mm_loadu_si128((__m128i *)(above + base + 1 + j))); + + diff = _mm256_sub_epi32(a1, a0); // a[x+1] - a[x] + a32 = _mm256_slli_epi32(a0, 5); // a[x] * 32 + a32 = _mm256_add_epi32(a32, a16); // a[x] * 32 + 16 + b = _mm256_mullo_epi32(diff, shift); + + res[0] = _mm256_add_epi32(a32, b); + res[0] = _mm256_srli_epi32(res[0], 5); + res[0] = _mm256_packus_epi32( + res[0], + _mm256_castsi128_si256(_mm256_extracti128_si256(res[0], 1))); + + // goto 8 bit + res[0] = _mm256_packus_epi16(res[0], res[0]); + + if (mdiff > 8) { + a0_1 = _mm256_cvtepu8_epi32( + _mm_loadu_si128((__m128i *)(above + base + 8 + j))); + a1_1 = _mm256_cvtepu8_epi32( + _mm_loadu_si128((__m128i *)(above + base + 9 + j))); + + diff = _mm256_sub_epi32(a1_1, a0_1); // a[x+1] - a[x] + a32 = _mm256_slli_epi32(a0_1, 5); // a[x] * 32 + a32 = _mm256_add_epi32(a32, a16); // a[x] * 32 + 16 + b = _mm256_mullo_epi32(diff, shift); + + res[1] = _mm256_add_epi32(a32, b); + res[1] = _mm256_srli_epi32(res[1], 5); + res[1] = _mm256_packus_epi32( + res[1], + _mm256_castsi128_si256(_mm256_extracti128_si256(res[1], 1))); + res[1] = _mm256_packus_epi16(res[1], res[1]); + // goto 8 bit + } else { + res[1] = a_mbase_x; + } + res16[jj] = _mm256_unpacklo_epi64(res[0], res[1]); // 16 8bit values + } + } + res16[1] = + _mm256_inserti128_si256(res16[0], _mm256_castsi256_si128(res16[1]), + 1); // 32 8bit values + + dstvec[r] = _mm256_blendv_epi8( + a_mbase_x, res16[1], + *(__m256i *)BaseMask[base_max_diff]); // 32 8bit values + x += dx; + } +} + +static void dr_prediction_z1_32xN_avx2(int N, uint8_t *dst, ptrdiff_t stride, + const uint8_t *above, int upsample_above, + int dx) { + __m256i dstvec[64]; + dr_prediction_z1_32xN_internal_avx2(N, dstvec, above, upsample_above, dx); + for (int i = 0; i < N; i++) { + _mm256_storeu_si256((__m256i *)(dst + stride * i), dstvec[i]); + } +} + +static void dr_prediction_z1_64xN_avx2(int N, uint8_t *dst, ptrdiff_t stride, + const uint8_t *above, int upsample_above, + int dx) { + int x; + + // here upsample_above is 0 by design of av1_use_intra_edge_upsample + (void)upsample_above; + const int frac_bits = 6; + const int max_base_x = ((64 + N) - 1); + + // pre-filter above pixels + // store in temp buffers: + // above[x] * 32 + 16 + // above[x+1] - above[x] + // final pixels will be caluculated as: + // (above[x] * 32 + 16 + (above[x+1] - above[x]) * shift) >> 5 + __m256i a0, a0_1, a1, a1_1, a32, a16; + __m256i a_mbase_x, diff; + __m128i max_base_x128, base_inc128, mask128; + + a16 = _mm256_set1_epi32(16); + a_mbase_x = _mm256_set1_epi8(above[max_base_x]); + max_base_x128 = _mm_set1_epi8(max_base_x); + + x = dx; + for (int r = 0; r < N; r++, dst += stride) { + __m256i b, res[2]; + __m128i res1; + + int base = x >> frac_bits; + if (base >= max_base_x) { + for (int i = r; i < N; ++i) { + _mm256_storeu_si256((__m256i *)dst, a_mbase_x); // save 32 values + _mm256_storeu_si256((__m256i *)(dst + 32), a_mbase_x); + dst += stride; + } + return; + } + + __m256i shift = _mm256_srli_epi32( + _mm256_and_si256(_mm256_set1_epi32(x), _mm256_set1_epi32(0x3f)), 1); + + __m128i a0_128, a0_1_128, a1_128, a1_1_128; + for (int j = 0; j < 64; j += 16) { + int mdif = max_base_x - (base + j); + if (mdif <= 0) { + _mm_storeu_si128((__m128i *)(dst + j), + _mm256_castsi256_si128(a_mbase_x)); + } else { + a0_128 = _mm_loadu_si128((__m128i *)(above + base + j)); + a1_128 = _mm_loadu_si128((__m128i *)(above + base + 1 + j)); + a0 = _mm256_cvtepu8_epi32(a0_128); + a1 = _mm256_cvtepu8_epi32(a1_128); + + diff = _mm256_sub_epi32(a1, a0); // a[x+1] - a[x] + a32 = _mm256_slli_epi32(a0, 5); // a[x] * 32 + a32 = _mm256_add_epi32(a32, a16); // a[x] * 32 + 16 + b = _mm256_mullo_epi32(diff, shift); + + res[0] = _mm256_add_epi32(a32, b); + res[0] = _mm256_srli_epi32(res[0], 5); + res[0] = _mm256_packus_epi32( + res[0], + _mm256_castsi128_si256(_mm256_extracti128_si256(res[0], 1))); + // goto 8 bit + res[0] = _mm256_packus_epi16(res[0], res[0]); + + if (mdif > 8) { + a0_1_128 = _mm_loadu_si128((__m128i *)(above + base + 8 + j)); + a1_1_128 = _mm_loadu_si128((__m128i *)(above + base + 9 + j)); + a0_1 = _mm256_cvtepu8_epi32(a0_1_128); + a1_1 = _mm256_cvtepu8_epi32(a1_1_128); + + diff = _mm256_sub_epi32(a1_1, a0_1); // a[x+1] - a[x] + a32 = _mm256_slli_epi32(a0_1, 5); // a[x] * 32 + a32 = _mm256_add_epi32(a32, a16); // a[x] * 32 + 16 + b = _mm256_mullo_epi32(diff, shift); + + res[1] = _mm256_add_epi32(a32, b); + res[1] = _mm256_srli_epi32(res[1], 5); + res[1] = _mm256_packus_epi32( + res[1], + _mm256_castsi128_si256(_mm256_extracti128_si256(res[1], 1))); + res[1] = _mm256_packus_epi16(res[1], res[1]); + + } else { + res[1] = a_mbase_x; + } + res1 = _mm_unpacklo_epi64( + _mm256_castsi256_si128(res[0]), + _mm256_castsi256_si128(res[1])); // 16 8bit values + + base_inc128 = _mm_setr_epi8( + base + j, base + j + 1, base + j + 2, base + j + 3, base + j + 4, + base + j + 5, base + j + 6, base + j + 7, base + j + 8, + base + j + 9, base + j + 10, base + j + 11, base + j + 12, + base + j + 13, base + j + 14, base + j + 15); + + mask128 = _mm_cmpgt_epi8(_mm_subs_epu8(max_base_x128, base_inc128), + _mm_setzero_si128()); + res1 = + _mm_blendv_epi8(_mm256_castsi256_si128(a_mbase_x), res1, mask128); + _mm_storeu_si128((__m128i *)(dst + j), res1); + } + } + x += dx; + } +} + +// Directional prediction, zone 1: 0 < angle < 90 +void av1_dr_prediction_z1_avx2(uint8_t *dst, ptrdiff_t stride, int bw, int bh, + const uint8_t *above, const uint8_t *left, + int upsample_above, int dx, int dy) { + (void)left; + (void)dy; + switch (bw) { + case 4: + dr_prediction_z1_4xN_avx2(bh, dst, stride, above, upsample_above, dx); + break; + case 8: + dr_prediction_z1_8xN_avx2(bh, dst, stride, above, upsample_above, dx); + break; + case 16: + dr_prediction_z1_16xN_avx2(bh, dst, stride, above, upsample_above, dx); + break; + case 32: + dr_prediction_z1_32xN_avx2(bh, dst, stride, above, upsample_above, dx); + break; + case 64: + dr_prediction_z1_64xN_avx2(bh, dst, stride, above, upsample_above, dx); + break; + default: break; + } + return; +} + +static uint8_t LoadMaskx[8][16] = { + { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }, + { 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14 }, + { 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13 }, + { 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 }, + { 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 }, + { 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }, + { 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }, + { 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8 }, +}; + +static uint8_t EvenOddMaskx4[8][16] = { + { 0, 2, 4, 6, 1, 3, 5, 7, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 0, 1, 3, 5, 7, 2, 4, 6, 8, 0, 0, 0, 0, 0, 0, 0 }, + { 0, 0, 2, 4, 6, 8, 3, 5, 7, 9, 0, 0, 0, 0, 0, 0 }, + { 0, 0, 0, 3, 5, 7, 9, 4, 6, 8, 10, 0, 0, 0, 0, 0 }, + { 0, 0, 0, 0, 4, 6, 8, 10, 5, 7, 9, 11, 0, 0, 0, 0 }, + { 0, 0, 0, 0, 0, 5, 7, 9, 11, 6, 8, 10, 12, 0, 0, 0 }, + { 0, 0, 0, 0, 0, 0, 6, 8, 10, 12, 7, 9, 11, 13, 0, 0 }, + { 0, 0, 0, 0, 0, 0, 0, 7, 9, 11, 13, 8, 10, 12, 14, 0 } +}; + +static uint8_t EvenOddMaskx[8][16] = { + { 0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 0, 0, 0, 0 }, + { 0, 1, 3, 5, 7, 9, 11, 13, 15, 2, 4, 6, 8, 0, 0, 0 }, + { 0, 0, 2, 4, 6, 8, 10, 12, 14, 3, 5, 7, 9, 0, 0, 0 }, + { 0, 0, 0, 3, 5, 7, 9, 11, 13, 15, 4, 6, 8, 10, 0 }, + { 0, 0, 0, 0, 4, 6, 8, 10, 12, 14, 5, 7, 9, 11, 0, 0 }, + { 0, 0, 0, 0, 0, 5, 7, 9, 11, 13, 15, 6, 8, 10, 12, 0 }, + { 0, 0, 0, 0, 0, 0, 6, 8, 10, 12, 14, 7, 9, 11, 13, 0 }, + { 0, 0, 0, 0, 0, 0, 0, 7, 9, 11, 13, 15, 8, 10, 12, 14 } +}; + +static void dr_prediction_z2_Nx4_avx2(int N, uint8_t *dst, ptrdiff_t stride, + const uint8_t *above, const uint8_t *left, + int upsample_above, int upsample_left, + int dx, int dy) { + const int min_base_x = -(1 << upsample_above); + const int min_base_y = -(1 << upsample_left); + const int frac_bits_x = 6 - upsample_above; + const int frac_bits_y = 6 - upsample_left; + + // a assert(dx > 0); + // pre-filter above pixels + // store in temp buffers: + // above[x] * 32 + 16 + // above[x+1] - above[x] + // final pixels will be caluculated as: + // (above[x] * 32 + 16 + (above[x+1] - above[x]) * shift) >> 5 + __m256i a0_x, a1_x, a0_y, a1_y, a32, a16, diff; + __m128i c3f, min_base_y128; + + a16 = _mm256_set1_epi32(16); + c3f = _mm_set1_epi32(0x3f); + min_base_y128 = _mm_set1_epi32(min_base_y); + + for (int r = 0; r < N; r++) { + __m256i b, res, shift; + __m128i resx, resy, resxy; + __m128i a0_x128, a1_x128; + int y = r + 1; + int base_x = (-y * dx) >> frac_bits_x; + int base_shift = 0; + if (base_x < (min_base_x - 1)) { + base_shift = (min_base_x - base_x - 1) >> upsample_above; + } + int base_min_diff = + (min_base_x - base_x + upsample_above) >> upsample_above; + if (base_min_diff > 4) { + base_min_diff = 4; + } else { + if (base_min_diff < 0) base_min_diff = 0; + } + + if (base_shift > 3) { + resx = _mm_setzero_si128(); + } else { + a0_x128 = _mm_loadu_si128((__m128i *)(above + base_x + base_shift)); + if (upsample_above) { + a0_x128 = + _mm_shuffle_epi8(a0_x128, *(__m128i *)EvenOddMaskx4[base_shift]); + a1_x128 = _mm_srli_si128(a0_x128, 4); + + shift = _mm256_castsi128_si256(_mm_srli_epi32( + _mm_and_si128( + _mm_slli_epi32( + _mm_setr_epi32(-y * dx, (1 << 6) - y * dx, + (2 << 6) - y * dx, (3 << 6) - y * dx), + upsample_above), + c3f), + 1)); + } else { + a0_x128 = _mm_shuffle_epi8(a0_x128, *(__m128i *)LoadMaskx[base_shift]); + a1_x128 = _mm_srli_si128(a0_x128, 1); + shift = _mm256_castsi128_si256(_mm_srli_epi32( + _mm_and_si128(_mm_setr_epi32(-y * dx, (1 << 6) - y * dx, + (2 << 6) - y * dx, (3 << 6) - y * dx), + c3f), + 1)); + } + a0_x = _mm256_cvtepu8_epi32(a0_x128); + a1_x = _mm256_cvtepu8_epi32(a1_x128); + + diff = _mm256_sub_epi32(a1_x, a0_x); // a[x+1] - a[x] + a32 = _mm256_slli_epi32(a0_x, 5); // a[x] * 32 + a32 = _mm256_add_epi32(a32, a16); // a[x] * 32 + 16 + + b = _mm256_mullo_epi32(diff, shift); + res = _mm256_add_epi32(a32, b); + res = _mm256_srli_epi32(res, 5); + + resx = _mm256_castsi256_si128(res); + resx = _mm_packus_epi32(resx, resx); + resx = _mm_packus_epi16(resx, resx); + } + // y calc + if (base_x < min_base_x) { + DECLARE_ALIGNED(32, int, base_y_c[4]); + __m128i r6, c1234, dy128, y_c128, base_y_c128, mask128; + r6 = _mm_set1_epi32(r << 6); + dy128 = _mm_set1_epi32(dy); + c1234 = _mm_setr_epi32(1, 2, 3, 4); + y_c128 = _mm_sub_epi32(r6, _mm_mullo_epi32(c1234, dy128)); + base_y_c128 = _mm_srai_epi32(y_c128, frac_bits_y); + mask128 = _mm_cmpgt_epi32(min_base_y128, base_y_c128); + base_y_c128 = _mm_andnot_si128(mask128, base_y_c128); + _mm_store_si128((__m128i *)base_y_c, base_y_c128); + + a0_y = _mm256_castsi128_si256( + _mm_setr_epi32(left[base_y_c[0]], left[base_y_c[1]], + left[base_y_c[2]], left[base_y_c[3]])); + a1_y = _mm256_castsi128_si256( + _mm_setr_epi32(left[base_y_c[0] + 1], left[base_y_c[1] + 1], + left[base_y_c[2] + 1], left[base_y_c[3] + 1])); + + if (upsample_left) { + shift = _mm256_castsi128_si256(_mm_srli_epi32( + _mm_and_si128(_mm_slli_epi32(y_c128, upsample_left), c3f), 1)); + } else { + shift = _mm256_castsi128_si256( + _mm_srli_epi32(_mm_and_si128(y_c128, c3f), 1)); + } + diff = _mm256_sub_epi32(a1_y, a0_y); // a[x+1] - a[x] + a32 = _mm256_slli_epi32(a0_y, 5); // a[x] * 32 + a32 = _mm256_add_epi32(a32, a16); // a[x] * 32 + 16 + + b = _mm256_mullo_epi32(diff, shift); + res = _mm256_add_epi32(a32, b); + res = _mm256_srli_epi32(res, 5); + + resy = _mm256_castsi256_si128(res); + resy = _mm_packus_epi32(resy, resy); + resy = _mm_packus_epi16(resy, resy); + } else { + resy = resx; + } + resxy = _mm_blendv_epi8(resx, resy, *(__m128i *)BaseMask[base_min_diff]); + *(uint32_t *)(dst) = _mm_cvtsi128_si32(resxy); + dst += stride; + } +} + +static void dr_prediction_z2_Nx8_avx2(int N, uint8_t *dst, ptrdiff_t stride, + const uint8_t *above, const uint8_t *left, + int upsample_above, int upsample_left, + int dx, int dy) { + const int min_base_x = -(1 << upsample_above); + const int min_base_y = -(1 << upsample_left); + const int frac_bits_x = 6 - upsample_above; + const int frac_bits_y = 6 - upsample_left; + + // pre-filter above pixels + // store in temp buffers: + // above[x] * 32 + 16 + // above[x+1] - above[x] + // final pixels will be caluculated as: + // (above[x] * 32 + 16 + (above[x+1] - above[x]) * shift) >> 5 + __m256i a0_x, a1_x, a0_y, a1_y, a32, a16, c3f; + __m256i diff, min_base_y256; + __m128i a0_x128, a1_x128; + + a16 = _mm256_set1_epi32(16); + c3f = _mm256_set1_epi32(0x3f); + min_base_y256 = _mm256_set1_epi32(min_base_y); + + for (int r = 0; r < N; r++) { + __m256i b, res, shift; + __m128i resx, resy, resxy; + + int y = r + 1; + int base_x = (-y * dx) >> frac_bits_x; + int base_shift = 0; + if (base_x < (min_base_x - 1)) { + base_shift = (min_base_x - base_x - 1) >> upsample_above; + } + int base_min_diff = + (min_base_x - base_x + upsample_above) >> upsample_above; + if (base_min_diff > 8) { + base_min_diff = 8; + } else { + if (base_min_diff < 0) base_min_diff = 0; + } + + if (base_shift > 7) { + resx = _mm_setzero_si128(); + } else { + a0_x128 = _mm_loadu_si128((__m128i *)(above + base_x + base_shift)); + a1_x128 = _mm_loadu_si128((__m128i *)(above + base_x + 1 + base_shift)); + if (upsample_above) { + a0_x128 = + _mm_shuffle_epi8(a0_x128, *(__m128i *)EvenOddMaskx[base_shift]); + a1_x128 = + _mm_shuffle_epi8(a1_x128, *(__m128i *)EvenOddMaskx[base_shift]); + + shift = _mm256_srli_epi32( + _mm256_and_si256( + _mm256_slli_epi32( + _mm256_setr_epi32(-y * dx, (1 << 6) - y * dx, + (2 << 6) - y * dx, (3 << 6) - y * dx, + (4 << 6) - y * dx, (5 << 6) - y * dx, + (6 << 6) - y * dx, (7 << 6) - y * dx), + upsample_above), + c3f), + 1); + } else { + a0_x128 = _mm_shuffle_epi8(a0_x128, *(__m128i *)LoadMaskx[base_shift]); + a1_x128 = _mm_shuffle_epi8(a1_x128, *(__m128i *)LoadMaskx[base_shift]); + + shift = _mm256_srli_epi32( + _mm256_and_si256( + _mm256_setr_epi32(-y * dx, (1 << 6) - y * dx, (2 << 6) - y * dx, + (3 << 6) - y * dx, (4 << 6) - y * dx, + (5 << 6) - y * dx, (6 << 6) - y * dx, + (7 << 6) - y * dx), + c3f), + 1); + } + a0_x = _mm256_cvtepu8_epi32(a0_x128); + a1_x = _mm256_cvtepu8_epi32(a1_x128); + + diff = _mm256_sub_epi32(a1_x, a0_x); // a[x+1] - a[x] + a32 = _mm256_slli_epi32(a0_x, 5); // a[x] * 32 + a32 = _mm256_add_epi32(a32, a16); // a[x] * 32 + 16 + + b = _mm256_mullo_epi32(diff, shift); + res = _mm256_add_epi32(a32, b); + res = _mm256_srli_epi32(res, 5); + + res = _mm256_packus_epi32( + res, _mm256_castsi128_si256(_mm256_extracti128_si256(res, 1))); + resx = _mm_packus_epi16(_mm256_castsi256_si128(res), + _mm256_castsi256_si128(res)); + } + // y calc + if (base_x < min_base_x) { + DECLARE_ALIGNED(32, int, base_y_c[8]); + __m256i r6, c256, dy256, y_c256, base_y_c256, mask256; + r6 = _mm256_set1_epi32(r << 6); + dy256 = _mm256_set1_epi32(dy); + c256 = _mm256_setr_epi32(1, 2, 3, 4, 5, 6, 7, 8); + y_c256 = _mm256_sub_epi32(r6, _mm256_mullo_epi32(c256, dy256)); + base_y_c256 = _mm256_srai_epi32(y_c256, frac_bits_y); + mask256 = _mm256_cmpgt_epi32(min_base_y256, base_y_c256); + base_y_c256 = _mm256_andnot_si256(mask256, base_y_c256); + _mm256_store_si256((__m256i *)base_y_c, base_y_c256); + + a0_y = _mm256_setr_epi32(left[base_y_c[0]], left[base_y_c[1]], + left[base_y_c[2]], left[base_y_c[3]], + left[base_y_c[4]], left[base_y_c[5]], + left[base_y_c[6]], left[base_y_c[7]]); + a1_y = _mm256_setr_epi32(left[base_y_c[0] + 1], left[base_y_c[1] + 1], + left[base_y_c[2] + 1], left[base_y_c[3] + 1], + left[base_y_c[4] + 1], left[base_y_c[5] + 1], + left[base_y_c[6] + 1], left[base_y_c[7] + 1]); + + if (upsample_left) { + shift = _mm256_srli_epi32( + _mm256_and_si256(_mm256_slli_epi32(y_c256, upsample_left), c3f), 1); + } else { + shift = _mm256_srli_epi32(_mm256_and_si256(y_c256, c3f), 1); + } + diff = _mm256_sub_epi32(a1_y, a0_y); // a[x+1] - a[x] + a32 = _mm256_slli_epi32(a0_y, 5); // a[x] * 32 + a32 = _mm256_add_epi32(a32, a16); // a[x] * 32 + 16 + + b = _mm256_mullo_epi32(diff, shift); + res = _mm256_add_epi32(a32, b); + res = _mm256_srli_epi32(res, 5); + + res = _mm256_packus_epi32( + res, _mm256_castsi128_si256(_mm256_extracti128_si256(res, 1))); + resy = _mm_packus_epi16(_mm256_castsi256_si128(res), + _mm256_castsi256_si128(res)); + } else { + resy = resx; + } + resxy = _mm_blendv_epi8(resx, resy, *(__m128i *)BaseMask[base_min_diff]); + _mm_storel_epi64((__m128i *)(dst), resxy); + dst += stride; + } +} + +static void dr_prediction_z2_HxW_avx2(int H, int W, uint8_t *dst, + ptrdiff_t stride, const uint8_t *above, + const uint8_t *left, int upsample_above, + int upsample_left, int dx, int dy) { + // here upsample_above and upsample_left are 0 by design of + // av1_use_intra_edge_upsample + const int min_base_x = -1; + const int min_base_y = -1; + (void)upsample_above; + (void)upsample_left; + const int frac_bits_x = 6; + const int frac_bits_y = 6; + + // pre-filter above pixels + // store in temp buffers: + // above[x] * 32 + 16 + // above[x+1] - above[x] + // final pixels will be caluculated as: + // (above[x] * 32 + 16 + (above[x+1] - above[x]) * shift) >> 5 + __m256i a0_x, a1_x, a0_y, a1_y, a32, a0_1_x, a1_1_x, a16; + __m256i diff, min_base_y256, c3f; + __m128i a0_x128, a1_x128, a0_1_x128, a1_1_x128; + + a16 = _mm256_set1_epi32(16); + min_base_y256 = _mm256_set1_epi32(min_base_y); + c3f = _mm256_set1_epi32(0x3f); + + for (int r = 0; r < H; r++) { + __m256i b, res, shift; + __m128i resx[2], resy[2]; + __m128i resxy; + for (int j = 0; j < W; j += 16) { + int y = r + 1; + int base_x = (-y * dx) >> frac_bits_x; + + int base_shift = 0; + if ((base_x + j) < (min_base_x - 1)) { + base_shift = (min_base_x - (base_x + j) - 1); + } + int base_min_diff = (min_base_x - base_x - j); + if (base_min_diff > 16) { + base_min_diff = 16; + } else { + if (base_min_diff < 0) base_min_diff = 0; + } + if (base_shift > 7) { + resx[0] = _mm_setzero_si128(); + } else { + a0_x128 = _mm_loadu_si128((__m128i *)(above + base_x + base_shift + j)); + a1_x128 = + _mm_loadu_si128((__m128i *)(above + base_x + base_shift + 1 + j)); + a0_x128 = _mm_shuffle_epi8(a0_x128, *(__m128i *)LoadMaskx[base_shift]); + a1_x128 = _mm_shuffle_epi8(a1_x128, *(__m128i *)LoadMaskx[base_shift]); + + a0_x = _mm256_cvtepu8_epi32(a0_x128); + a1_x = _mm256_cvtepu8_epi32(a1_x128); + + shift = _mm256_srli_epi32( + _mm256_and_si256( + _mm256_setr_epi32( + ((0 + j) << 6) - y * dx, ((1 + j) << 6) - y * dx, + ((2 + j) << 6) - y * dx, ((3 + j) << 6) - y * dx, + ((4 + j) << 6) - y * dx, ((5 + j) << 6) - y * dx, + ((6 + j) << 6) - y * dx, ((7 + j) << 6) - y * dx), + c3f), + 1); + + diff = _mm256_sub_epi32(a1_x, a0_x); // a[x+1] - a[x] + a32 = _mm256_slli_epi32(a0_x, 5); // a[x] * 32 + a32 = _mm256_add_epi32(a32, a16); // a[x] * 32 + 16 + + b = _mm256_mullo_epi32(diff, shift); + res = _mm256_add_epi32(a32, b); + res = _mm256_srli_epi32(res, 5); + + res = _mm256_packus_epi32( + res, _mm256_castsi128_si256(_mm256_extracti128_si256(res, 1))); + resx[0] = _mm_packus_epi16(_mm256_castsi256_si128(res), + _mm256_castsi256_si128(res)); + } + base_shift = 0; + if ((base_x + j + 8) < (min_base_x - 1)) { + base_shift = (min_base_x - (base_x + j + 8) - 1); + } + if (base_shift > 7) { + resx[1] = _mm_setzero_si128(); + } else { + a0_1_x128 = + _mm_loadu_si128((__m128i *)(above + base_x + base_shift + 8 + j)); + a1_1_x128 = + _mm_loadu_si128((__m128i *)(above + base_x + base_shift + 9 + j)); + a0_1_x128 = + _mm_shuffle_epi8(a0_1_x128, *(__m128i *)LoadMaskx[base_shift]); + a1_1_x128 = + _mm_shuffle_epi8(a1_1_x128, *(__m128i *)LoadMaskx[base_shift]); + + a0_1_x = _mm256_cvtepu8_epi32(a0_1_x128); + a1_1_x = _mm256_cvtepu8_epi32(a1_1_x128); + + shift = _mm256_srli_epi32( + _mm256_and_si256( + _mm256_setr_epi32( + ((8 + j) << 6) - y * dx, ((9 + j) << 6) - y * dx, + ((10 + j) << 6) - y * dx, ((11 + j) << 6) - y * dx, + ((12 + j) << 6) - y * dx, ((13 + j) << 6) - y * dx, + ((14 + j) << 6) - y * dx, ((15 + j) << 6) - y * dx), + _mm256_set1_epi32(0x3f)), + 1); + + diff = _mm256_sub_epi32(a1_1_x, a0_1_x); // a[x+1] - a[x] + a32 = _mm256_slli_epi32(a0_1_x, 5); // a[x] * 32 + a32 = _mm256_add_epi32(a32, a16); // a[x] * 32 + 16 + b = _mm256_mullo_epi32(diff, shift); + + res = _mm256_add_epi32(a32, b); + res = _mm256_srli_epi32(res, 5); + res = _mm256_packus_epi32( + res, _mm256_castsi128_si256(_mm256_extracti128_si256(res, 1))); + resx[1] = _mm_packus_epi16(_mm256_castsi256_si128(res), + _mm256_castsi256_si128(res)); + } + resx[0] = _mm_unpacklo_epi64(resx[0], resx[1]); + + // y calc + if ((base_x < min_base_x)) { + DECLARE_ALIGNED(32, int, base_y_c[16]); + __m256i r6, c256, dy256, y_c256, y_c_1_256, base_y_c256, mask256; + r6 = _mm256_set1_epi32(r << 6); + dy256 = _mm256_set1_epi32(dy); + c256 = _mm256_setr_epi32(1 + j, 2 + j, 3 + j, 4 + j, 5 + j, 6 + j, + 7 + j, 8 + j); + y_c256 = _mm256_sub_epi32(r6, _mm256_mullo_epi32(c256, dy256)); + base_y_c256 = _mm256_srai_epi32(y_c256, frac_bits_y); + mask256 = _mm256_cmpgt_epi32(min_base_y256, base_y_c256); + base_y_c256 = _mm256_andnot_si256(mask256, base_y_c256); + _mm256_store_si256((__m256i *)base_y_c, base_y_c256); + c256 = _mm256_setr_epi32(9 + j, 10 + j, 11 + j, 12 + j, 13 + j, 14 + j, + 15 + j, 16 + j); + y_c_1_256 = _mm256_sub_epi32(r6, _mm256_mullo_epi32(c256, dy256)); + base_y_c256 = _mm256_srai_epi32(y_c_1_256, frac_bits_y); + mask256 = _mm256_cmpgt_epi32(min_base_y256, base_y_c256); + base_y_c256 = _mm256_andnot_si256(mask256, base_y_c256); + _mm256_store_si256((__m256i *)(base_y_c + 8), base_y_c256); + + a0_y = _mm256_cvtepu8_epi32(_mm_setr_epi8( + left[base_y_c[0]], left[base_y_c[1]], left[base_y_c[2]], + left[base_y_c[3]], left[base_y_c[4]], left[base_y_c[5]], + left[base_y_c[6]], left[base_y_c[7]], 0, 0, 0, 0, 0, 0, 0, 0)); + a1_y = _mm256_cvtepu8_epi32(_mm_setr_epi8( + left[base_y_c[0] + 1], left[base_y_c[1] + 1], left[base_y_c[2] + 1], + left[base_y_c[3] + 1], left[base_y_c[4] + 1], left[base_y_c[5] + 1], + left[base_y_c[6] + 1], left[base_y_c[7] + 1], 0, 0, 0, 0, 0, 0, 0, + 0)); + + shift = _mm256_srli_epi32(_mm256_and_si256(y_c256, c3f), 1); + + diff = _mm256_sub_epi32(a1_y, a0_y); // a[x+1] - a[x] + a32 = _mm256_slli_epi32(a0_y, 5); // a[x] * 32 + a32 = _mm256_add_epi32(a32, a16); // a[x] * 32 + 16 + + b = _mm256_mullo_epi32(diff, shift); + res = _mm256_add_epi32(a32, b); + res = _mm256_srli_epi32(res, 5); + + res = _mm256_packus_epi32( + res, _mm256_castsi128_si256(_mm256_extracti128_si256(res, 1))); + resy[0] = _mm_packus_epi16(_mm256_castsi256_si128(res), + _mm256_castsi256_si128(res)); + + a0_y = _mm256_cvtepu8_epi32(_mm_setr_epi8( + left[base_y_c[8]], left[base_y_c[9]], left[base_y_c[10]], + left[base_y_c[11]], left[base_y_c[12]], left[base_y_c[13]], + left[base_y_c[14]], left[base_y_c[15]], 0, 0, 0, 0, 0, 0, 0, 0)); + a1_y = _mm256_cvtepu8_epi32( + _mm_setr_epi8(left[base_y_c[8] + 1], left[base_y_c[9] + 1], + left[base_y_c[10] + 1], left[base_y_c[11] + 1], + left[base_y_c[12] + 1], left[base_y_c[13] + 1], + left[base_y_c[14] + 1], left[base_y_c[15] + 1], 0, 0, + 0, 0, 0, 0, 0, 0)); + shift = _mm256_srli_epi32(_mm256_and_si256(y_c_1_256, c3f), 1); + + diff = _mm256_sub_epi32(a1_y, a0_y); // a[x+1] - a[x] + a32 = _mm256_slli_epi32(a0_y, 5); // a[x] * 32 + a32 = _mm256_add_epi32(a32, a16); // a[x] * 32 + 16 + + b = _mm256_mullo_epi32(diff, shift); + res = _mm256_add_epi32(a32, b); + res = _mm256_srli_epi32(res, 5); + + res = _mm256_packus_epi32( + res, _mm256_castsi128_si256(_mm256_extracti128_si256(res, 1))); + resy[1] = + _mm_packus_epi16(_mm256_castsi256_si128(res), + _mm256_castsi256_si128(res)); // 8 16bit values + resy[0] = _mm_unpacklo_epi64(resy[0], resy[1]); + } else { + resy[0] = resx[0]; + } + resxy = _mm_blendv_epi8(resx[0], resy[0], + *(__m128i *)BaseMask[base_min_diff]); + _mm_storeu_si128((__m128i *)(dst + j), resxy); + } // for j + dst += stride; + } +} + +// Directional prediction, zone 2: 90 < angle < 180 +void av1_dr_prediction_z2_avx2(uint8_t *dst, ptrdiff_t stride, int bw, int bh, + const uint8_t *above, const uint8_t *left, + int upsample_above, int upsample_left, int dx, + int dy) { + assert(dx > 0); + assert(dy > 0); + switch (bw) { + case 4: + dr_prediction_z2_Nx4_avx2(bh, dst, stride, above, left, upsample_above, + upsample_left, dx, dy); + break; + case 8: + dr_prediction_z2_Nx8_avx2(bh, dst, stride, above, left, upsample_above, + upsample_left, dx, dy); + + break; + default: + dr_prediction_z2_HxW_avx2(bh, bw, dst, stride, above, left, + upsample_above, upsample_left, dx, dy); + break; + } + return; +} + +// z3 functions +static INLINE void transpose4x16_sse2(__m128i *x, __m128i *d) { + __m128i w0, w1, w2, w3, ww0, ww1, ww2, ww3; + w0 = _mm_unpacklo_epi8(x[0], x[1]); + w1 = _mm_unpacklo_epi8(x[2], x[3]); + w2 = _mm_unpackhi_epi8(x[0], x[1]); + w3 = _mm_unpackhi_epi8(x[2], x[3]); + + ww0 = _mm_unpacklo_epi16(w0, w1); + ww1 = _mm_unpacklo_epi16(w2, w3); + ww2 = _mm_unpackhi_epi16(w0, w1); + ww3 = _mm_unpackhi_epi16(w2, w3); + + w0 = _mm_unpacklo_epi32(ww0, ww1); + w2 = _mm_unpacklo_epi32(ww2, ww3); + w1 = _mm_unpackhi_epi32(ww0, ww1); + w3 = _mm_unpackhi_epi32(ww2, ww3); + + d[0] = _mm_unpacklo_epi64(w0, w2); + d[1] = _mm_unpackhi_epi64(w0, w2); + d[2] = _mm_unpacklo_epi64(w1, w3); + d[3] = _mm_unpackhi_epi64(w1, w3); + + d[4] = _mm_srli_si128(d[0], 8); + d[5] = _mm_srli_si128(d[1], 8); + d[6] = _mm_srli_si128(d[2], 8); + d[7] = _mm_srli_si128(d[3], 8); + + d[8] = _mm_srli_si128(d[0], 4); + d[9] = _mm_srli_si128(d[1], 4); + d[10] = _mm_srli_si128(d[2], 4); + d[11] = _mm_srli_si128(d[3], 4); + + d[12] = _mm_srli_si128(d[0], 12); + d[13] = _mm_srli_si128(d[1], 12); + d[14] = _mm_srli_si128(d[2], 12); + d[15] = _mm_srli_si128(d[3], 12); +} + +static INLINE void transpose16x32_avx2(__m256i *x, __m256i *d) { + __m256i w0, w1, w2, w3, w4, w5, w6, w7, w8, w9; + __m256i w10, w11, w12, w13, w14, w15; + + w0 = _mm256_unpacklo_epi8(x[0], x[1]); + w1 = _mm256_unpacklo_epi8(x[2], x[3]); + w2 = _mm256_unpacklo_epi8(x[4], x[5]); + w3 = _mm256_unpacklo_epi8(x[6], x[7]); + + w8 = _mm256_unpacklo_epi8(x[8], x[9]); + w9 = _mm256_unpacklo_epi8(x[10], x[11]); + w10 = _mm256_unpacklo_epi8(x[12], x[13]); + w11 = _mm256_unpacklo_epi8(x[14], x[15]); + + w4 = _mm256_unpacklo_epi16(w0, w1); + w5 = _mm256_unpacklo_epi16(w2, w3); + w12 = _mm256_unpacklo_epi16(w8, w9); + w13 = _mm256_unpacklo_epi16(w10, w11); + + w6 = _mm256_unpacklo_epi32(w4, w5); + w7 = _mm256_unpackhi_epi32(w4, w5); + w14 = _mm256_unpacklo_epi32(w12, w13); + w15 = _mm256_unpackhi_epi32(w12, w13); + + // Store first 4-line result + d[0] = _mm256_unpacklo_epi64(w6, w14); + d[1] = _mm256_unpackhi_epi64(w6, w14); + d[2] = _mm256_unpacklo_epi64(w7, w15); + d[3] = _mm256_unpackhi_epi64(w7, w15); + + w4 = _mm256_unpackhi_epi16(w0, w1); + w5 = _mm256_unpackhi_epi16(w2, w3); + w12 = _mm256_unpackhi_epi16(w8, w9); + w13 = _mm256_unpackhi_epi16(w10, w11); + + w6 = _mm256_unpacklo_epi32(w4, w5); + w7 = _mm256_unpackhi_epi32(w4, w5); + w14 = _mm256_unpacklo_epi32(w12, w13); + w15 = _mm256_unpackhi_epi32(w12, w13); + + // Store second 4-line result + d[4] = _mm256_unpacklo_epi64(w6, w14); + d[5] = _mm256_unpackhi_epi64(w6, w14); + d[6] = _mm256_unpacklo_epi64(w7, w15); + d[7] = _mm256_unpackhi_epi64(w7, w15); + + // upper half + w0 = _mm256_unpackhi_epi8(x[0], x[1]); + w1 = _mm256_unpackhi_epi8(x[2], x[3]); + w2 = _mm256_unpackhi_epi8(x[4], x[5]); + w3 = _mm256_unpackhi_epi8(x[6], x[7]); + + w8 = _mm256_unpackhi_epi8(x[8], x[9]); + w9 = _mm256_unpackhi_epi8(x[10], x[11]); + w10 = _mm256_unpackhi_epi8(x[12], x[13]); + w11 = _mm256_unpackhi_epi8(x[14], x[15]); + + w4 = _mm256_unpacklo_epi16(w0, w1); + w5 = _mm256_unpacklo_epi16(w2, w3); + w12 = _mm256_unpacklo_epi16(w8, w9); + w13 = _mm256_unpacklo_epi16(w10, w11); + + w6 = _mm256_unpacklo_epi32(w4, w5); + w7 = _mm256_unpackhi_epi32(w4, w5); + w14 = _mm256_unpacklo_epi32(w12, w13); + w15 = _mm256_unpackhi_epi32(w12, w13); + + // Store first 4-line result + d[8] = _mm256_unpacklo_epi64(w6, w14); + d[9] = _mm256_unpackhi_epi64(w6, w14); + d[10] = _mm256_unpacklo_epi64(w7, w15); + d[11] = _mm256_unpackhi_epi64(w7, w15); + + w4 = _mm256_unpackhi_epi16(w0, w1); + w5 = _mm256_unpackhi_epi16(w2, w3); + w12 = _mm256_unpackhi_epi16(w8, w9); + w13 = _mm256_unpackhi_epi16(w10, w11); + + w6 = _mm256_unpacklo_epi32(w4, w5); + w7 = _mm256_unpackhi_epi32(w4, w5); + w14 = _mm256_unpacklo_epi32(w12, w13); + w15 = _mm256_unpackhi_epi32(w12, w13); + + // Store second 4-line result + d[12] = _mm256_unpacklo_epi64(w6, w14); + d[13] = _mm256_unpackhi_epi64(w6, w14); + d[14] = _mm256_unpacklo_epi64(w7, w15); + d[15] = _mm256_unpackhi_epi64(w7, w15); +} + +static INLINE void transpose16x16_sse2(__m128i *x, __m128i *d) { + __m128i w0, w1, w2, w3, w4, w5, w6, w7, w8, w9; + __m128i w10, w11, w12, w13, w14, w15; + + w0 = _mm_unpacklo_epi8(x[0], x[1]); + w1 = _mm_unpacklo_epi8(x[2], x[3]); + w2 = _mm_unpacklo_epi8(x[4], x[5]); + w3 = _mm_unpacklo_epi8(x[6], x[7]); + + w8 = _mm_unpacklo_epi8(x[8], x[9]); + w9 = _mm_unpacklo_epi8(x[10], x[11]); + w10 = _mm_unpacklo_epi8(x[12], x[13]); + w11 = _mm_unpacklo_epi8(x[14], x[15]); + + w4 = _mm_unpacklo_epi16(w0, w1); + w5 = _mm_unpacklo_epi16(w2, w3); + w12 = _mm_unpacklo_epi16(w8, w9); + w13 = _mm_unpacklo_epi16(w10, w11); + + w6 = _mm_unpacklo_epi32(w4, w5); + w7 = _mm_unpackhi_epi32(w4, w5); + w14 = _mm_unpacklo_epi32(w12, w13); + w15 = _mm_unpackhi_epi32(w12, w13); + + // Store first 4-line result + d[0] = _mm_unpacklo_epi64(w6, w14); + d[1] = _mm_unpackhi_epi64(w6, w14); + d[2] = _mm_unpacklo_epi64(w7, w15); + d[3] = _mm_unpackhi_epi64(w7, w15); + + w4 = _mm_unpackhi_epi16(w0, w1); + w5 = _mm_unpackhi_epi16(w2, w3); + w12 = _mm_unpackhi_epi16(w8, w9); + w13 = _mm_unpackhi_epi16(w10, w11); + + w6 = _mm_unpacklo_epi32(w4, w5); + w7 = _mm_unpackhi_epi32(w4, w5); + w14 = _mm_unpacklo_epi32(w12, w13); + w15 = _mm_unpackhi_epi32(w12, w13); + + // Store second 4-line result + d[4] = _mm_unpacklo_epi64(w6, w14); + d[5] = _mm_unpackhi_epi64(w6, w14); + d[6] = _mm_unpacklo_epi64(w7, w15); + d[7] = _mm_unpackhi_epi64(w7, w15); + + // upper half + w0 = _mm_unpackhi_epi8(x[0], x[1]); + w1 = _mm_unpackhi_epi8(x[2], x[3]); + w2 = _mm_unpackhi_epi8(x[4], x[5]); + w3 = _mm_unpackhi_epi8(x[6], x[7]); + + w8 = _mm_unpackhi_epi8(x[8], x[9]); + w9 = _mm_unpackhi_epi8(x[10], x[11]); + w10 = _mm_unpackhi_epi8(x[12], x[13]); + w11 = _mm_unpackhi_epi8(x[14], x[15]); + + w4 = _mm_unpacklo_epi16(w0, w1); + w5 = _mm_unpacklo_epi16(w2, w3); + w12 = _mm_unpacklo_epi16(w8, w9); + w13 = _mm_unpacklo_epi16(w10, w11); + + w6 = _mm_unpacklo_epi32(w4, w5); + w7 = _mm_unpackhi_epi32(w4, w5); + w14 = _mm_unpacklo_epi32(w12, w13); + w15 = _mm_unpackhi_epi32(w12, w13); + + // Store first 4-line result + d[8] = _mm_unpacklo_epi64(w6, w14); + d[9] = _mm_unpackhi_epi64(w6, w14); + d[10] = _mm_unpacklo_epi64(w7, w15); + d[11] = _mm_unpackhi_epi64(w7, w15); + + w4 = _mm_unpackhi_epi16(w0, w1); + w5 = _mm_unpackhi_epi16(w2, w3); + w12 = _mm_unpackhi_epi16(w8, w9); + w13 = _mm_unpackhi_epi16(w10, w11); + + w6 = _mm_unpacklo_epi32(w4, w5); + w7 = _mm_unpackhi_epi32(w4, w5); + w14 = _mm_unpacklo_epi32(w12, w13); + w15 = _mm_unpackhi_epi32(w12, w13); + + // Store second 4-line result + d[12] = _mm_unpacklo_epi64(w6, w14); + d[13] = _mm_unpackhi_epi64(w6, w14); + d[14] = _mm_unpacklo_epi64(w7, w15); + d[15] = _mm_unpackhi_epi64(w7, w15); +} + +static void transpose_TX_8X8(const uint8_t *src, ptrdiff_t pitchSrc, + uint8_t *dst, ptrdiff_t pitchDst) { + __m128i r0, r1, r2, r3, r4, r5, r6, r7; + __m128i d0d1, d2d3, d4d5, d6d7; + r0 = _mm_loadl_epi64((__m128i *)(src + 0 * pitchSrc)); + r1 = _mm_loadl_epi64((__m128i *)(src + 1 * pitchSrc)); + r2 = _mm_loadl_epi64((__m128i *)(src + 2 * pitchSrc)); + r3 = _mm_loadl_epi64((__m128i *)(src + 3 * pitchSrc)); + r4 = _mm_loadl_epi64((__m128i *)(src + 4 * pitchSrc)); + r5 = _mm_loadl_epi64((__m128i *)(src + 5 * pitchSrc)); + r6 = _mm_loadl_epi64((__m128i *)(src + 6 * pitchSrc)); + r7 = _mm_loadl_epi64((__m128i *)(src + 7 * pitchSrc)); + + transpose8x8_sse2(&r0, &r1, &r2, &r3, &r4, &r5, &r6, &r7, &d0d1, &d2d3, &d4d5, + &d6d7); + + _mm_storel_epi64((__m128i *)(dst + 0 * pitchDst), d0d1); + _mm_storel_epi64((__m128i *)(dst + 1 * pitchDst), _mm_srli_si128(d0d1, 8)); + _mm_storel_epi64((__m128i *)(dst + 2 * pitchDst), d2d3); + _mm_storel_epi64((__m128i *)(dst + 3 * pitchDst), _mm_srli_si128(d2d3, 8)); + _mm_storel_epi64((__m128i *)(dst + 4 * pitchDst), d4d5); + _mm_storel_epi64((__m128i *)(dst + 5 * pitchDst), _mm_srli_si128(d4d5, 8)); + _mm_storel_epi64((__m128i *)(dst + 6 * pitchDst), d6d7); + _mm_storel_epi64((__m128i *)(dst + 7 * pitchDst), _mm_srli_si128(d6d7, 8)); +} + +static void transpose(const uint8_t *src, ptrdiff_t pitchSrc, uint8_t *dst, + ptrdiff_t pitchDst, int width, int height) { + for (int j = 0; j < height; j += 8) + for (int i = 0; i < width; i += 8) + transpose_TX_8X8(src + i * pitchSrc + j, pitchSrc, dst + j * pitchDst + i, + pitchDst); +} + +static void dr_prediction_z3_4x4_avx2(uint8_t *dst, ptrdiff_t stride, + const uint8_t *left, int upsample_left, + int dy) { + __m128i dstvec[4], d[4]; + + dr_prediction_z1_4xN_internal_avx2(4, dstvec, left, upsample_left, dy); + transpose4x8_8x4_low_sse2(&dstvec[0], &dstvec[1], &dstvec[2], &dstvec[3], + &d[0], &d[1], &d[2], &d[3]); + + *(uint32_t *)(dst + stride * 0) = _mm_cvtsi128_si32(d[0]); + *(uint32_t *)(dst + stride * 1) = _mm_cvtsi128_si32(d[1]); + *(uint32_t *)(dst + stride * 2) = _mm_cvtsi128_si32(d[2]); + *(uint32_t *)(dst + stride * 3) = _mm_cvtsi128_si32(d[3]); + return; +} + +static void dr_prediction_z3_8x8_avx2(uint8_t *dst, ptrdiff_t stride, + const uint8_t *left, int upsample_left, + int dy) { + __m128i dstvec[8], d[8]; + + dr_prediction_z1_8xN_internal_avx2(8, dstvec, left, upsample_left, dy); + transpose8x8_sse2(&dstvec[0], &dstvec[1], &dstvec[2], &dstvec[3], &dstvec[4], + &dstvec[5], &dstvec[6], &dstvec[7], &d[0], &d[1], &d[2], + &d[3]); + + _mm_storel_epi64((__m128i *)(dst + 0 * stride), d[0]); + _mm_storel_epi64((__m128i *)(dst + 1 * stride), _mm_srli_si128(d[0], 8)); + _mm_storel_epi64((__m128i *)(dst + 2 * stride), d[1]); + _mm_storel_epi64((__m128i *)(dst + 3 * stride), _mm_srli_si128(d[1], 8)); + _mm_storel_epi64((__m128i *)(dst + 4 * stride), d[2]); + _mm_storel_epi64((__m128i *)(dst + 5 * stride), _mm_srli_si128(d[2], 8)); + _mm_storel_epi64((__m128i *)(dst + 6 * stride), d[3]); + _mm_storel_epi64((__m128i *)(dst + 7 * stride), _mm_srli_si128(d[3], 8)); +} + +static void dr_prediction_z3_4x8_avx2(uint8_t *dst, ptrdiff_t stride, + const uint8_t *left, int upsample_left, + int dy) { + __m128i dstvec[4], d[8]; + + dr_prediction_z1_8xN_internal_avx2(4, dstvec, left, upsample_left, dy); + transpose4x8_8x4_sse2(&dstvec[0], &dstvec[1], &dstvec[2], &dstvec[3], &d[0], + &d[1], &d[2], &d[3], &d[4], &d[5], &d[6], &d[7]); + for (int i = 0; i < 8; i++) { + *(uint32_t *)(dst + stride * i) = _mm_cvtsi128_si32(d[i]); + } +} + +static void dr_prediction_z3_8x4_avx2(uint8_t *dst, ptrdiff_t stride, + const uint8_t *left, int upsample_left, + int dy) { + __m128i dstvec[8], d[4]; + + dr_prediction_z1_4xN_internal_avx2(8, dstvec, left, upsample_left, dy); + transpose8x8_low_sse2(&dstvec[0], &dstvec[1], &dstvec[2], &dstvec[3], + &dstvec[4], &dstvec[5], &dstvec[6], &dstvec[7], &d[0], + &d[1], &d[2], &d[3]); + _mm_storel_epi64((__m128i *)(dst + 0 * stride), d[0]); + _mm_storel_epi64((__m128i *)(dst + 1 * stride), d[1]); + _mm_storel_epi64((__m128i *)(dst + 2 * stride), d[2]); + _mm_storel_epi64((__m128i *)(dst + 3 * stride), d[3]); +} + +static void dr_prediction_z3_8x16_avx2(uint8_t *dst, ptrdiff_t stride, + const uint8_t *left, int upsample_left, + int dy) { + __m128i dstvec[8], d[8]; + + dr_prediction_z1_16xN_internal_avx2(8, dstvec, left, upsample_left, dy); + transpose8x16_16x8_sse2(dstvec, dstvec + 1, dstvec + 2, dstvec + 3, + dstvec + 4, dstvec + 5, dstvec + 6, dstvec + 7, d, + d + 1, d + 2, d + 3, d + 4, d + 5, d + 6, d + 7); + for (int i = 0; i < 8; i++) { + _mm_storel_epi64((__m128i *)(dst + i * stride), d[i]); + _mm_storel_epi64((__m128i *)(dst + (i + 8) * stride), + _mm_srli_si128(d[i], 8)); + } +} + +static void dr_prediction_z3_16x8_avx2(uint8_t *dst, ptrdiff_t stride, + const uint8_t *left, int upsample_left, + int dy) { + __m128i dstvec[16], d[16]; + + dr_prediction_z1_8xN_internal_avx2(16, dstvec, left, upsample_left, dy); + transpose16x8_8x16_sse2( + &dstvec[0], &dstvec[1], &dstvec[2], &dstvec[3], &dstvec[4], &dstvec[5], + &dstvec[6], &dstvec[7], &dstvec[8], &dstvec[9], &dstvec[10], &dstvec[11], + &dstvec[12], &dstvec[13], &dstvec[14], &dstvec[15], &d[0], &d[1], &d[2], + &d[3], &d[4], &d[5], &d[6], &d[7]); + + for (int i = 0; i < 8; i++) { + _mm_storeu_si128((__m128i *)(dst + i * stride), d[i]); + } +} + +static void dr_prediction_z3_4x16_avx2(uint8_t *dst, ptrdiff_t stride, + const uint8_t *left, int upsample_left, + int dy) { + __m128i dstvec[4], d[16]; + + dr_prediction_z1_16xN_internal_avx2(4, dstvec, left, upsample_left, dy); + transpose4x16_sse2(dstvec, d); + for (int i = 0; i < 16; i++) { + *(uint32_t *)(dst + stride * i) = _mm_cvtsi128_si32(d[i]); + } +} + +static void dr_prediction_z3_16x4_avx2(uint8_t *dst, ptrdiff_t stride, + const uint8_t *left, int upsample_left, + int dy) { + __m128i dstvec[16], d[8]; + + dr_prediction_z1_4xN_internal_avx2(16, dstvec, left, upsample_left, dy); + for (int i = 4; i < 8; i++) { + d[i] = _mm_setzero_si128(); + } + transpose16x8_8x16_sse2( + &dstvec[0], &dstvec[1], &dstvec[2], &dstvec[3], &dstvec[4], &dstvec[5], + &dstvec[6], &dstvec[7], &dstvec[8], &dstvec[9], &dstvec[10], &dstvec[11], + &dstvec[12], &dstvec[13], &dstvec[14], &dstvec[15], &d[0], &d[1], &d[2], + &d[3], &d[4], &d[5], &d[6], &d[7]); + + for (int i = 0; i < 4; i++) { + _mm_storeu_si128((__m128i *)(dst + i * stride), d[i]); + } +} + +static void dr_prediction_z3_8x32_avx2(uint8_t *dst, ptrdiff_t stride, + const uint8_t *left, int upsample_left, + int dy) { + __m256i dstvec[16], d[16]; + + dr_prediction_z1_32xN_internal_avx2(8, dstvec, left, upsample_left, dy); + for (int i = 8; i < 16; i++) { + dstvec[i] = _mm256_setzero_si256(); + } + transpose16x32_avx2(dstvec, d); + + for (int i = 0; i < 16; i++) { + _mm_storel_epi64((__m128i *)(dst + i * stride), + _mm256_castsi256_si128(d[i])); + } + for (int i = 0; i < 16; i++) { + _mm_storel_epi64((__m128i *)(dst + (i + 16) * stride), + _mm256_extracti128_si256(d[i], 1)); + } +} + +static void dr_prediction_z3_32x8_avx2(uint8_t *dst, ptrdiff_t stride, + const uint8_t *left, int upsample_left, + int dy) { + __m128i dstvec[32], d[16]; + + dr_prediction_z1_8xN_internal_avx2(32, dstvec, left, upsample_left, dy); + + transpose16x8_8x16_sse2( + &dstvec[0], &dstvec[1], &dstvec[2], &dstvec[3], &dstvec[4], &dstvec[5], + &dstvec[6], &dstvec[7], &dstvec[8], &dstvec[9], &dstvec[10], &dstvec[11], + &dstvec[12], &dstvec[13], &dstvec[14], &dstvec[15], &d[0], &d[1], &d[2], + &d[3], &d[4], &d[5], &d[6], &d[7]); + transpose16x8_8x16_sse2( + &dstvec[0 + 16], &dstvec[1 + 16], &dstvec[2 + 16], &dstvec[3 + 16], + &dstvec[4 + 16], &dstvec[5 + 16], &dstvec[6 + 16], &dstvec[7 + 16], + &dstvec[8 + 16], &dstvec[9 + 16], &dstvec[10 + 16], &dstvec[11 + 16], + &dstvec[12 + 16], &dstvec[13 + 16], &dstvec[14 + 16], &dstvec[15 + 16], + &d[0 + 8], &d[1 + 8], &d[2 + 8], &d[3 + 8], &d[4 + 8], &d[5 + 8], + &d[6 + 8], &d[7 + 8]); + + for (int i = 0; i < 8; i++) { + _mm_storeu_si128((__m128i *)(dst + i * stride), d[i]); + _mm_storeu_si128((__m128i *)(dst + i * stride + 16), d[i + 8]); + } +} + +static void dr_prediction_z3_16x16_avx2(uint8_t *dst, ptrdiff_t stride, + const uint8_t *left, int upsample_left, + int dy) { + __m128i dstvec[16], d[16]; + + dr_prediction_z1_16xN_internal_avx2(16, dstvec, left, upsample_left, dy); + transpose16x16_sse2(dstvec, d); + + for (int i = 0; i < 16; i++) { + _mm_storeu_si128((__m128i *)(dst + i * stride), d[i]); + } +} + +static void dr_prediction_z3_32x32_avx2(uint8_t *dst, ptrdiff_t stride, + const uint8_t *left, int upsample_left, + int dy) { + __m256i dstvec[32], d[32]; + + dr_prediction_z1_32xN_internal_avx2(32, dstvec, left, upsample_left, dy); + transpose16x32_avx2(dstvec, d); + transpose16x32_avx2(dstvec + 16, d + 16); + for (int j = 0; j < 16; j++) { + _mm_storeu_si128((__m128i *)(dst + j * stride), + _mm256_castsi256_si128(d[j])); + _mm_storeu_si128((__m128i *)(dst + j * stride + 16), + _mm256_castsi256_si128(d[j + 16])); + } + for (int j = 0; j < 16; j++) { + _mm_storeu_si128((__m128i *)(dst + (j + 16) * stride), + _mm256_extracti128_si256(d[j], 1)); + _mm_storeu_si128((__m128i *)(dst + (j + 16) * stride + 16), + _mm256_extracti128_si256(d[j + 16], 1)); + } +} + +static void dr_prediction_z3_64x64_avx2(uint8_t *dst, ptrdiff_t stride, + const uint8_t *left, int upsample_left, + int dy) { + DECLARE_ALIGNED(16, uint8_t, dstT[64 * 64]); + dr_prediction_z1_64xN_avx2(64, dstT, 64, left, upsample_left, dy); + transpose(dstT, 64, dst, stride, 64, 64); +} + +static void dr_prediction_z3_16x32_avx2(uint8_t *dst, ptrdiff_t stride, + const uint8_t *left, int upsample_left, + int dy) { + __m256i dstvec[16], d[16]; + + dr_prediction_z1_32xN_internal_avx2(16, dstvec, left, upsample_left, dy); + transpose16x32_avx2(dstvec, d); + // store + for (int j = 0; j < 16; j++) { + _mm_storeu_si128((__m128i *)(dst + j * stride), + _mm256_castsi256_si128(d[j])); + _mm_storeu_si128((__m128i *)(dst + (j + 16) * stride), + _mm256_extracti128_si256(d[j], 1)); + } +} + +static void dr_prediction_z3_32x16_avx2(uint8_t *dst, ptrdiff_t stride, + const uint8_t *left, int upsample_left, + int dy) { + __m128i dstvec[32], d[16]; + + dr_prediction_z1_16xN_internal_avx2(32, dstvec, left, upsample_left, dy); + for (int i = 0; i < 32; i += 16) { + transpose16x16_sse2((dstvec + i), d); + for (int j = 0; j < 16; j++) { + _mm_storeu_si128((__m128i *)(dst + j * stride + i), d[j]); + } + } +} + +static void dr_prediction_z3_32x64_avx2(uint8_t *dst, ptrdiff_t stride, + const uint8_t *left, int upsample_left, + int dy) { + uint8_t dstT[64 * 32]; + dr_prediction_z1_64xN_avx2(32, dstT, 64, left, upsample_left, dy); + transpose(dstT, 64, dst, stride, 32, 64); +} + +static void dr_prediction_z3_64x32_avx2(uint8_t *dst, ptrdiff_t stride, + const uint8_t *left, int upsample_left, + int dy) { + uint8_t dstT[32 * 64]; + dr_prediction_z1_32xN_avx2(64, dstT, 32, left, upsample_left, dy); + transpose(dstT, 32, dst, stride, 64, 32); + return; +} + +static void dr_prediction_z3_16x64_avx2(uint8_t *dst, ptrdiff_t stride, + const uint8_t *left, int upsample_left, + int dy) { + uint8_t dstT[64 * 16]; + dr_prediction_z1_64xN_avx2(16, dstT, 64, left, upsample_left, dy); + transpose(dstT, 64, dst, stride, 16, 64); +} + +static void dr_prediction_z3_64x16_avx2(uint8_t *dst, ptrdiff_t stride, + const uint8_t *left, int upsample_left, + int dy) { + __m128i dstvec[64], d[16]; + + dr_prediction_z1_16xN_internal_avx2(64, dstvec, left, upsample_left, dy); + for (int i = 0; i < 64; i += 16) { + transpose16x16_sse2((dstvec + i), d); + for (int j = 0; j < 16; j++) { + _mm_storeu_si128((__m128i *)(dst + j * stride + i), d[j]); + } + } +} + +void av1_dr_prediction_z3_avx2(uint8_t *dst, ptrdiff_t stride, int bw, int bh, + const uint8_t *above, const uint8_t *left, + int upsample_left, int dx, int dy) { + (void)above; + (void)dx; + assert(dx == 1); + assert(dy > 0); + + if (bw == bh) { + switch (bw) { + case 4: + dr_prediction_z3_4x4_avx2(dst, stride, left, upsample_left, dy); + break; + case 8: + dr_prediction_z3_8x8_avx2(dst, stride, left, upsample_left, dy); + break; + case 16: + dr_prediction_z3_16x16_avx2(dst, stride, left, upsample_left, dy); + break; + case 32: + dr_prediction_z3_32x32_avx2(dst, stride, left, upsample_left, dy); + break; + case 64: + dr_prediction_z3_64x64_avx2(dst, stride, left, upsample_left, dy); + break; + } + } else { + if (bw < bh) { + if (bw + bw == bh) { + switch (bw) { + case 4: + dr_prediction_z3_4x8_avx2(dst, stride, left, upsample_left, dy); + break; + case 8: + dr_prediction_z3_8x16_avx2(dst, stride, left, upsample_left, dy); + break; + case 16: + dr_prediction_z3_16x32_avx2(dst, stride, left, upsample_left, dy); + break; + case 32: + dr_prediction_z3_32x64_avx2(dst, stride, left, upsample_left, dy); + break; + } + } else { + switch (bw) { + case 4: + dr_prediction_z3_4x16_avx2(dst, stride, left, upsample_left, dy); + break; + case 8: + dr_prediction_z3_8x32_avx2(dst, stride, left, upsample_left, dy); + break; + case 16: + dr_prediction_z3_16x64_avx2(dst, stride, left, upsample_left, dy); + break; + } + } + } else { + if (bh + bh == bw) { + switch (bh) { + case 4: + dr_prediction_z3_8x4_avx2(dst, stride, left, upsample_left, dy); + break; + case 8: + dr_prediction_z3_16x8_avx2(dst, stride, left, upsample_left, dy); + break; + case 16: + dr_prediction_z3_32x16_avx2(dst, stride, left, upsample_left, dy); + break; + case 32: + dr_prediction_z3_64x32_avx2(dst, stride, left, upsample_left, dy); + break; + } + } else { + switch (bh) { + case 4: + dr_prediction_z3_16x4_avx2(dst, stride, left, upsample_left, dy); + break; + case 8: + dr_prediction_z3_32x8_avx2(dst, stride, left, upsample_left, dy); + break; + case 16: + dr_prediction_z3_64x16_avx2(dst, stride, left, upsample_left, dy); + break; + } + } + } + } + return; +} diff --git a/third_party/aom/aom_dsp/x86/jnt_sad_ssse3.c b/third_party/aom/aom_dsp/x86/jnt_sad_ssse3.c index c3c88245a4..2e3e2be105 100644 --- a/third_party/aom/aom_dsp/x86/jnt_sad_ssse3.c +++ b/third_party/aom/aom_dsp/x86/jnt_sad_ssse3.c @@ -192,47 +192,47 @@ unsigned int aom_sad128xh_sse2(const uint8_t *a, int a_stride, const uint8_t *b, return res; } -#define jnt_sadMxN_sse2(m, n) \ - unsigned int aom_jnt_sad##m##x##n##_avg_ssse3( \ +#define dist_wtd_sadMxN_sse2(m, n) \ + unsigned int aom_dist_wtd_sad##m##x##n##_avg_ssse3( \ const uint8_t *src, int src_stride, const uint8_t *ref, int ref_stride, \ - const uint8_t *second_pred, const JNT_COMP_PARAMS *jcp_param) { \ + const uint8_t *second_pred, const DIST_WTD_COMP_PARAMS *jcp_param) { \ uint8_t comp_pred[m * n]; \ - aom_jnt_comp_avg_pred(comp_pred, second_pred, m, n, ref, ref_stride, \ - jcp_param); \ + aom_dist_wtd_comp_avg_pred(comp_pred, second_pred, m, n, ref, ref_stride, \ + jcp_param); \ return aom_sad##m##xh_sse2(src, src_stride, comp_pred, m, m, n); \ } -#define jnt_sadMxN_avx2(m, n) \ - unsigned int aom_jnt_sad##m##x##n##_avg_avx2( \ +#define dist_wtd_sadMxN_avx2(m, n) \ + unsigned int aom_dist_wtd_sad##m##x##n##_avg_avx2( \ const uint8_t *src, int src_stride, const uint8_t *ref, int ref_stride, \ - const uint8_t *second_pred, const JNT_COMP_PARAMS *jcp_param) { \ + const uint8_t *second_pred, const DIST_WTD_COMP_PARAMS *jcp_param) { \ uint8_t comp_pred[m * n]; \ - aom_jnt_comp_avg_pred(comp_pred, second_pred, m, n, ref, ref_stride, \ - jcp_param); \ + aom_dist_wtd_comp_avg_pred(comp_pred, second_pred, m, n, ref, ref_stride, \ + jcp_param); \ return aom_sad##m##xh_avx2(src, src_stride, comp_pred, m, m, n); \ } /* clang-format off */ -jnt_sadMxN_sse2(128, 128) -jnt_sadMxN_sse2(128, 64) -jnt_sadMxN_sse2(64, 128) -jnt_sadMxN_sse2(64, 64) -jnt_sadMxN_sse2(64, 32) -jnt_sadMxN_sse2(32, 64) -jnt_sadMxN_sse2(32, 32) -jnt_sadMxN_sse2(32, 16) -jnt_sadMxN_sse2(16, 32) -jnt_sadMxN_sse2(16, 16) -jnt_sadMxN_sse2(16, 8) -jnt_sadMxN_sse2(8, 16) -jnt_sadMxN_sse2(8, 8) -jnt_sadMxN_sse2(8, 4) -jnt_sadMxN_sse2(4, 8) -jnt_sadMxN_sse2(4, 4) -jnt_sadMxN_sse2(4, 16) -jnt_sadMxN_sse2(16, 4) -jnt_sadMxN_sse2(8, 32) -jnt_sadMxN_sse2(32, 8) -jnt_sadMxN_sse2(16, 64) -jnt_sadMxN_sse2(64, 16) +dist_wtd_sadMxN_sse2(128, 128) +dist_wtd_sadMxN_sse2(128, 64) +dist_wtd_sadMxN_sse2(64, 128) +dist_wtd_sadMxN_sse2(64, 64) +dist_wtd_sadMxN_sse2(64, 32) +dist_wtd_sadMxN_sse2(32, 64) +dist_wtd_sadMxN_sse2(32, 32) +dist_wtd_sadMxN_sse2(32, 16) +dist_wtd_sadMxN_sse2(16, 32) +dist_wtd_sadMxN_sse2(16, 16) +dist_wtd_sadMxN_sse2(16, 8) +dist_wtd_sadMxN_sse2(8, 16) +dist_wtd_sadMxN_sse2(8, 8) +dist_wtd_sadMxN_sse2(8, 4) +dist_wtd_sadMxN_sse2(4, 8) +dist_wtd_sadMxN_sse2(4, 4) +dist_wtd_sadMxN_sse2(4, 16) +dist_wtd_sadMxN_sse2(16, 4) +dist_wtd_sadMxN_sse2(8, 32) +dist_wtd_sadMxN_sse2(32, 8) +dist_wtd_sadMxN_sse2(16, 64) +dist_wtd_sadMxN_sse2(64, 16) /* clang-format on */ diff --git a/third_party/aom/aom_dsp/x86/jnt_variance_ssse3.c b/third_party/aom/aom_dsp/x86/jnt_variance_ssse3.c index f9a41a210b..c8b02f5560 100644 --- a/third_party/aom/aom_dsp/x86/jnt_variance_ssse3.c +++ b/third_party/aom/aom_dsp/x86/jnt_variance_ssse3.c @@ -29,7 +29,7 @@ void aom_var_filter_block2d_bil_second_pass_ssse3( unsigned int pixel_step, unsigned int output_height, unsigned int output_width, const uint8_t *filter); -static INLINE void compute_jnt_comp_avg(__m128i *p0, __m128i *p1, +static INLINE void compute_dist_wtd_avg(__m128i *p0, __m128i *p1, const __m128i *w, const __m128i *r, void *const result) { __m128i p_lo = _mm_unpacklo_epi8(*p0, *p1); @@ -45,10 +45,10 @@ static INLINE void compute_jnt_comp_avg(__m128i *p0, __m128i *p1, xx_storeu_128(result, _mm_packus_epi16(shift_lo, shift_hi)); } -void aom_jnt_comp_avg_pred_ssse3(uint8_t *comp_pred, const uint8_t *pred, - int width, int height, const uint8_t *ref, - int ref_stride, - const JNT_COMP_PARAMS *jcp_param) { +void aom_dist_wtd_comp_avg_pred_ssse3(uint8_t *comp_pred, const uint8_t *pred, + int width, int height, const uint8_t *ref, + int ref_stride, + const DIST_WTD_COMP_PARAMS *jcp_param) { int i; const uint8_t w0 = (uint8_t)jcp_param->fwd_offset; const uint8_t w1 = (uint8_t)jcp_param->bck_offset; @@ -67,7 +67,7 @@ void aom_jnt_comp_avg_pred_ssse3(uint8_t *comp_pred, const uint8_t *pred, __m128i p0 = xx_loadu_128(ref); __m128i p1 = xx_loadu_128(pred); - compute_jnt_comp_avg(&p0, &p1, &w, &r, comp_pred); + compute_dist_wtd_avg(&p0, &p1, &w, &r, comp_pred); comp_pred += 16; pred += 16; @@ -85,7 +85,7 @@ void aom_jnt_comp_avg_pred_ssse3(uint8_t *comp_pred, const uint8_t *pred, __m128i p0 = _mm_unpacklo_epi64(p0_0, p0_1); __m128i p1 = xx_loadu_128(pred); - compute_jnt_comp_avg(&p0, &p1, &w, &r, comp_pred); + compute_dist_wtd_avg(&p0, &p1, &w, &r, comp_pred); comp_pred += 16; pred += 16; @@ -107,7 +107,7 @@ void aom_jnt_comp_avg_pred_ssse3(uint8_t *comp_pred, const uint8_t *pred, row3[0], row3[1], row3[2], row3[3]); __m128i p1 = xx_loadu_128(pred); - compute_jnt_comp_avg(&p0, &p1, &w, &r, comp_pred); + compute_dist_wtd_avg(&p0, &p1, &w, &r, comp_pred); comp_pred += 16; pred += 16; @@ -116,11 +116,11 @@ void aom_jnt_comp_avg_pred_ssse3(uint8_t *comp_pred, const uint8_t *pred, } } -void aom_jnt_comp_avg_upsampled_pred_ssse3( +void aom_dist_wtd_comp_avg_upsampled_pred_ssse3( MACROBLOCKD *xd, const struct AV1Common *const cm, int mi_row, int mi_col, const MV *const mv, uint8_t *comp_pred, const uint8_t *pred, int width, int height, int subpel_x_q3, int subpel_y_q3, const uint8_t *ref, - int ref_stride, const JNT_COMP_PARAMS *jcp_param, int subpel_search) { + int ref_stride, const DIST_WTD_COMP_PARAMS *jcp_param, int subpel_search) { int n; int i; aom_upsampled_pred(xd, cm, mi_row, mi_col, mv, comp_pred, width, height, @@ -141,52 +141,52 @@ void aom_jnt_comp_avg_upsampled_pred_ssse3( __m128i p0 = xx_loadu_128(comp_pred); __m128i p1 = xx_loadu_128(pred); - compute_jnt_comp_avg(&p0, &p1, &w, &r, comp_pred); + compute_dist_wtd_avg(&p0, &p1, &w, &r, comp_pred); comp_pred += 16; pred += 16; } } -#define JNT_SUBPIX_AVG_VAR(W, H) \ - uint32_t aom_jnt_sub_pixel_avg_variance##W##x##H##_ssse3( \ - const uint8_t *a, int a_stride, int xoffset, int yoffset, \ - const uint8_t *b, int b_stride, uint32_t *sse, \ - const uint8_t *second_pred, const JNT_COMP_PARAMS *jcp_param) { \ - uint16_t fdata3[(H + 1) * W]; \ - uint8_t temp2[H * W]; \ - DECLARE_ALIGNED(16, uint8_t, temp3[H * W]); \ - \ - aom_var_filter_block2d_bil_first_pass_ssse3( \ - a, fdata3, a_stride, 1, H + 1, W, bilinear_filters_2t[xoffset]); \ - aom_var_filter_block2d_bil_second_pass_ssse3( \ - fdata3, temp2, W, W, H, W, bilinear_filters_2t[yoffset]); \ - \ - aom_jnt_comp_avg_pred_ssse3(temp3, second_pred, W, H, temp2, W, \ - jcp_param); \ - \ - return aom_variance##W##x##H(temp3, W, b, b_stride, sse); \ +#define DIST_WTD_SUBPIX_AVG_VAR(W, H) \ + uint32_t aom_dist_wtd_sub_pixel_avg_variance##W##x##H##_ssse3( \ + const uint8_t *a, int a_stride, int xoffset, int yoffset, \ + const uint8_t *b, int b_stride, uint32_t *sse, \ + const uint8_t *second_pred, const DIST_WTD_COMP_PARAMS *jcp_param) { \ + uint16_t fdata3[(H + 1) * W]; \ + uint8_t temp2[H * W]; \ + DECLARE_ALIGNED(16, uint8_t, temp3[H * W]); \ + \ + aom_var_filter_block2d_bil_first_pass_ssse3( \ + a, fdata3, a_stride, 1, H + 1, W, bilinear_filters_2t[xoffset]); \ + aom_var_filter_block2d_bil_second_pass_ssse3( \ + fdata3, temp2, W, W, H, W, bilinear_filters_2t[yoffset]); \ + \ + aom_dist_wtd_comp_avg_pred_ssse3(temp3, second_pred, W, H, temp2, W, \ + jcp_param); \ + \ + return aom_variance##W##x##H(temp3, W, b, b_stride, sse); \ } -JNT_SUBPIX_AVG_VAR(128, 128) -JNT_SUBPIX_AVG_VAR(128, 64) -JNT_SUBPIX_AVG_VAR(64, 128) -JNT_SUBPIX_AVG_VAR(64, 64) -JNT_SUBPIX_AVG_VAR(64, 32) -JNT_SUBPIX_AVG_VAR(32, 64) -JNT_SUBPIX_AVG_VAR(32, 32) -JNT_SUBPIX_AVG_VAR(32, 16) -JNT_SUBPIX_AVG_VAR(16, 32) -JNT_SUBPIX_AVG_VAR(16, 16) -JNT_SUBPIX_AVG_VAR(16, 8) -JNT_SUBPIX_AVG_VAR(8, 16) -JNT_SUBPIX_AVG_VAR(8, 8) -JNT_SUBPIX_AVG_VAR(8, 4) -JNT_SUBPIX_AVG_VAR(4, 8) -JNT_SUBPIX_AVG_VAR(4, 4) -JNT_SUBPIX_AVG_VAR(4, 16) -JNT_SUBPIX_AVG_VAR(16, 4) -JNT_SUBPIX_AVG_VAR(8, 32) -JNT_SUBPIX_AVG_VAR(32, 8) -JNT_SUBPIX_AVG_VAR(16, 64) -JNT_SUBPIX_AVG_VAR(64, 16) +DIST_WTD_SUBPIX_AVG_VAR(128, 128) +DIST_WTD_SUBPIX_AVG_VAR(128, 64) +DIST_WTD_SUBPIX_AVG_VAR(64, 128) +DIST_WTD_SUBPIX_AVG_VAR(64, 64) +DIST_WTD_SUBPIX_AVG_VAR(64, 32) +DIST_WTD_SUBPIX_AVG_VAR(32, 64) +DIST_WTD_SUBPIX_AVG_VAR(32, 32) +DIST_WTD_SUBPIX_AVG_VAR(32, 16) +DIST_WTD_SUBPIX_AVG_VAR(16, 32) +DIST_WTD_SUBPIX_AVG_VAR(16, 16) +DIST_WTD_SUBPIX_AVG_VAR(16, 8) +DIST_WTD_SUBPIX_AVG_VAR(8, 16) +DIST_WTD_SUBPIX_AVG_VAR(8, 8) +DIST_WTD_SUBPIX_AVG_VAR(8, 4) +DIST_WTD_SUBPIX_AVG_VAR(4, 8) +DIST_WTD_SUBPIX_AVG_VAR(4, 4) +DIST_WTD_SUBPIX_AVG_VAR(4, 16) +DIST_WTD_SUBPIX_AVG_VAR(16, 4) +DIST_WTD_SUBPIX_AVG_VAR(8, 32) +DIST_WTD_SUBPIX_AVG_VAR(32, 8) +DIST_WTD_SUBPIX_AVG_VAR(16, 64) +DIST_WTD_SUBPIX_AVG_VAR(64, 16) diff --git a/third_party/aom/aom_dsp/x86/loopfilter_sse2.c b/third_party/aom/aom_dsp/x86/loopfilter_sse2.c index 9d88b5e493..e1981e555a 100644 --- a/third_party/aom/aom_dsp/x86/loopfilter_sse2.c +++ b/third_party/aom/aom_dsp/x86/loopfilter_sse2.c @@ -16,347 +16,12 @@ #include "aom_dsp/x86/synonyms.h" #include "aom_ports/mem.h" #include "aom_ports/emmintrin_compat.h" +#include "aom_dsp/x86/lpf_common_sse2.h" static INLINE __m128i abs_diff(__m128i a, __m128i b) { return _mm_or_si128(_mm_subs_epu8(a, b), _mm_subs_epu8(b, a)); } -static INLINE void transpose4x8_8x4_low_sse2(__m128i *x0, __m128i *x1, - __m128i *x2, __m128i *x3, - __m128i *d0, __m128i *d1, - __m128i *d2, __m128i *d3) { - // input - // x0 00 01 02 03 04 05 06 07 xx xx xx xx xx xx xx xx - // x1 10 11 12 13 14 15 16 17 xx xx xx xx xx xx xx xx - // x2 20 21 22 23 24 25 26 27 xx xx xx xx xx xx xx xx - // x3 30 31 32 33 34 35 36 37 xx xx xx xx xx xx xx xx - // output - // 00 10 20 30 xx xx xx xx xx xx xx xx xx xx xx xx - // 01 11 21 31 xx xx xx xx xx xx xx xx xx xx xx xx - // 02 12 22 32 xx xx xx xx xx xx xx xx xx xx xx xx - // 03 13 23 33 xx xx xx xx xx xx xx xx xx xx xx xx - - __m128i w0, w1; - - w0 = _mm_unpacklo_epi8( - *x0, *x1); // 00 10 01 11 02 12 03 13 04 14 05 15 06 16 07 17 - w1 = _mm_unpacklo_epi8( - *x2, *x3); // 20 30 21 31 22 32 23 33 24 34 25 35 26 36 27 37 - - *d0 = _mm_unpacklo_epi16( - w0, w1); // 00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33 - - *d1 = _mm_srli_si128(*d0, - 4); // 01 11 21 31 xx xx xx xx xx xx xx xx xx xx xx xx - *d2 = _mm_srli_si128(*d0, - 8); // 02 12 22 32 xx xx xx xx xx xx xx xx xx xx xx xx - *d3 = _mm_srli_si128(*d0, - 12); // 03 13 23 33 xx xx xx xx xx xx xx xx xx xx xx xx -} - -static INLINE void transpose4x8_8x4_sse2(__m128i *x0, __m128i *x1, __m128i *x2, - __m128i *x3, __m128i *d0, __m128i *d1, - __m128i *d2, __m128i *d3, __m128i *d4, - __m128i *d5, __m128i *d6, - __m128i *d7) { - // input - // x0 00 01 02 03 04 05 06 07 xx xx xx xx xx xx xx xx - // x1 10 11 12 13 14 15 16 17 xx xx xx xx xx xx xx xx - // x2 20 21 22 23 24 25 26 27 xx xx xx xx xx xx xx xx - // x3 30 31 32 33 34 35 36 37 xx xx xx xx xx xx xx xx - // output - // 00 10 20 30 xx xx xx xx xx xx xx xx xx xx xx xx - // 01 11 21 31 xx xx xx xx xx xx xx xx xx xx xx xx - // 02 12 22 32 xx xx xx xx xx xx xx xx xx xx xx xx - // 03 13 23 33 xx xx xx xx xx xx xx xx xx xx xx xx - // 04 14 24 34 xx xx xx xx xx xx xx xx xx xx xx xx - // 05 15 25 35 xx xx xx xx xx xx xx xx xx xx xx xx - // 06 16 26 36 xx xx xx xx xx xx xx xx xx xx xx xx - // 07 17 27 37 xx xx xx xx xx xx xx xx xx xx xx xx - - __m128i w0, w1, ww0, ww1; - - w0 = _mm_unpacklo_epi8( - *x0, *x1); // 00 10 01 11 02 12 03 13 04 14 05 15 06 16 07 17 - w1 = _mm_unpacklo_epi8( - *x2, *x3); // 20 30 21 31 22 32 23 33 24 34 25 35 26 36 27 37 - - ww0 = _mm_unpacklo_epi16( - w0, w1); // 00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33 - ww1 = _mm_unpackhi_epi16( - w0, w1); // 04 14 24 34 05 15 25 35 06 16 26 36 07 17 27 37 - - *d0 = ww0; // 00 10 20 30 xx xx xx xx xx xx xx xx xx xx xx xx - *d1 = _mm_srli_si128(ww0, - 4); // 01 11 21 31 xx xx xx xx xx xx xx xx xx xx xx xx - *d2 = _mm_srli_si128(ww0, - 8); // 02 12 22 32 xx xx xx xx xx xx xx xx xx xx xx xx - *d3 = _mm_srli_si128(ww0, - 12); // 03 13 23 33 xx xx xx xx xx xx xx xx xx xx xx xx - - *d4 = ww1; // 04 14 24 34 xx xx xx xx xx xx xx xx xx xx xx xx - *d5 = _mm_srli_si128(ww1, - 4); // 05 15 25 35 xx xx xx xx xx xx xx xx xx xx xx xx - *d6 = _mm_srli_si128(ww1, - 8); // 06 16 26 36 xx xx xx xx xx xx xx xx xx xx xx xx - *d7 = _mm_srli_si128(ww1, - 12); // 07 17 27 37 xx xx xx xx xx xx xx xx xx xx xx xx -} - -static INLINE void transpose8x8_low_sse2(__m128i *x0, __m128i *x1, __m128i *x2, - __m128i *x3, __m128i *x4, __m128i *x5, - __m128i *x6, __m128i *x7, __m128i *d0, - __m128i *d1, __m128i *d2, - __m128i *d3) { - // input - // x0 00 01 02 03 04 05 06 07 - // x1 10 11 12 13 14 15 16 17 - // x2 20 21 22 23 24 25 26 27 - // x3 30 31 32 33 34 35 36 37 - // x4 40 41 42 43 44 45 46 47 - // x5 50 51 52 53 54 55 56 57 - // x6 60 61 62 63 64 65 66 67 - // x7 70 71 72 73 74 75 76 77 - // output - // d0 00 10 20 30 40 50 60 70 xx xx xx xx xx xx xx - // d1 01 11 21 31 41 51 61 71 xx xx xx xx xx xx xx xx - // d2 02 12 22 32 42 52 62 72 xx xx xx xx xx xx xx xx - // d3 03 13 23 33 43 53 63 73 xx xx xx xx xx xx xx xx - - __m128i w0, w1, w2, w3, w4, w5; - - w0 = _mm_unpacklo_epi8( - *x0, *x1); // 00 10 01 11 02 12 03 13 04 14 05 15 06 16 07 17 - - w1 = _mm_unpacklo_epi8( - *x2, *x3); // 20 30 21 31 22 32 23 33 24 34 25 35 26 36 27 37 - - w2 = _mm_unpacklo_epi8( - *x4, *x5); // 40 50 41 51 42 52 43 53 44 54 45 55 46 56 47 57 - - w3 = _mm_unpacklo_epi8( - *x6, *x7); // 60 70 61 71 62 72 63 73 64 74 65 75 66 76 67 77 - - w4 = _mm_unpacklo_epi16( - w0, w1); // 00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33 - w5 = _mm_unpacklo_epi16( - w2, w3); // 40 50 60 70 41 51 61 71 42 52 62 72 43 53 63 73 - - *d0 = _mm_unpacklo_epi32( - w4, w5); // 00 10 20 30 40 50 60 70 01 11 21 31 41 51 61 71 - *d1 = _mm_srli_si128(*d0, 8); - *d2 = _mm_unpackhi_epi32( - w4, w5); // 02 12 22 32 42 52 62 72 03 13 23 33 43 53 63 73 - *d3 = _mm_srli_si128(*d2, 8); -} - -static INLINE void transpose8x8_sse2(__m128i *x0, __m128i *x1, __m128i *x2, - __m128i *x3, __m128i *x4, __m128i *x5, - __m128i *x6, __m128i *x7, __m128i *d0d1, - __m128i *d2d3, __m128i *d4d5, - __m128i *d6d7) { - __m128i w0, w1, w2, w3, w4, w5, w6, w7; - // x0 00 01 02 03 04 05 06 07 - // x1 10 11 12 13 14 15 16 17 - w0 = _mm_unpacklo_epi8( - *x0, *x1); // 00 10 01 11 02 12 03 13 04 14 05 15 06 16 07 17 - - // x2 20 21 22 23 24 25 26 27 - // x3 30 31 32 33 34 35 36 37 - w1 = _mm_unpacklo_epi8( - *x2, *x3); // 20 30 21 31 22 32 23 33 24 34 25 35 26 36 27 37 - - // x4 40 41 42 43 44 45 46 47 - // x5 50 51 52 53 54 55 56 57 - w2 = _mm_unpacklo_epi8( - *x4, *x5); // 40 50 41 51 42 52 43 53 44 54 45 55 46 56 47 57 - - // x6 60 61 62 63 64 65 66 67 - // x7 70 71 72 73 74 75 76 77 - w3 = _mm_unpacklo_epi8( - *x6, *x7); // 60 70 61 71 62 72 63 73 64 74 65 75 66 76 67 77 - - w4 = _mm_unpacklo_epi16( - w0, w1); // 00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33 - w5 = _mm_unpacklo_epi16( - w2, w3); // 40 50 60 70 41 51 61 71 42 52 62 72 43 53 63 73 - - *d0d1 = _mm_unpacklo_epi32( - w4, w5); // 00 10 20 30 40 50 60 70 01 11 21 31 41 51 61 71 - *d2d3 = _mm_unpackhi_epi32( - w4, w5); // 02 12 22 32 42 52 62 72 03 13 23 33 43 53 63 73 - - w6 = _mm_unpackhi_epi16( - w0, w1); // 04 14 24 34 05 15 25 35 06 16 26 36 07 17 27 37 - w7 = _mm_unpackhi_epi16( - w2, w3); // 44 54 64 74 45 55 65 75 46 56 66 76 47 57 67 77 - - *d4d5 = _mm_unpacklo_epi32( - w6, w7); // 04 14 24 34 44 54 64 74 05 15 25 35 45 55 65 75 - *d6d7 = _mm_unpackhi_epi32( - w6, w7); // 06 16 26 36 46 56 66 76 07 17 27 37 47 57 67 77 -} - -static INLINE void transpose16x8_8x16_sse2( - __m128i *x0, __m128i *x1, __m128i *x2, __m128i *x3, __m128i *x4, - __m128i *x5, __m128i *x6, __m128i *x7, __m128i *x8, __m128i *x9, - __m128i *x10, __m128i *x11, __m128i *x12, __m128i *x13, __m128i *x14, - __m128i *x15, __m128i *d0, __m128i *d1, __m128i *d2, __m128i *d3, - __m128i *d4, __m128i *d5, __m128i *d6, __m128i *d7) { - __m128i w0, w1, w2, w3, w4, w5, w6, w7, w8, w9; - __m128i w10, w11, w12, w13, w14, w15; - - w0 = _mm_unpacklo_epi8(*x0, *x1); - w1 = _mm_unpacklo_epi8(*x2, *x3); - w2 = _mm_unpacklo_epi8(*x4, *x5); - w3 = _mm_unpacklo_epi8(*x6, *x7); - - w8 = _mm_unpacklo_epi8(*x8, *x9); - w9 = _mm_unpacklo_epi8(*x10, *x11); - w10 = _mm_unpacklo_epi8(*x12, *x13); - w11 = _mm_unpacklo_epi8(*x14, *x15); - - w4 = _mm_unpacklo_epi16(w0, w1); - w5 = _mm_unpacklo_epi16(w2, w3); - w12 = _mm_unpacklo_epi16(w8, w9); - w13 = _mm_unpacklo_epi16(w10, w11); - - w6 = _mm_unpacklo_epi32(w4, w5); - w7 = _mm_unpackhi_epi32(w4, w5); - w14 = _mm_unpacklo_epi32(w12, w13); - w15 = _mm_unpackhi_epi32(w12, w13); - - // Store first 4-line result - *d0 = _mm_unpacklo_epi64(w6, w14); - *d1 = _mm_unpackhi_epi64(w6, w14); - *d2 = _mm_unpacklo_epi64(w7, w15); - *d3 = _mm_unpackhi_epi64(w7, w15); - - w4 = _mm_unpackhi_epi16(w0, w1); - w5 = _mm_unpackhi_epi16(w2, w3); - w12 = _mm_unpackhi_epi16(w8, w9); - w13 = _mm_unpackhi_epi16(w10, w11); - - w6 = _mm_unpacklo_epi32(w4, w5); - w7 = _mm_unpackhi_epi32(w4, w5); - w14 = _mm_unpacklo_epi32(w12, w13); - w15 = _mm_unpackhi_epi32(w12, w13); - - // Store second 4-line result - *d4 = _mm_unpacklo_epi64(w6, w14); - *d5 = _mm_unpackhi_epi64(w6, w14); - *d6 = _mm_unpacklo_epi64(w7, w15); - *d7 = _mm_unpackhi_epi64(w7, w15); -} - -// this function treats its input as 2 parallel 8x4 matrices, transposes each of -// them independently while flipping the second matrix horizontaly Used for 14 -// taps filter pq pairs inverse -static INLINE void transpose_pq_14_inv_sse2(__m128i *x0, __m128i *x1, - __m128i *x2, __m128i *x3, - __m128i *x4, __m128i *x5, - __m128i *x6, __m128i *x7, - __m128i *pq0, __m128i *pq1, - __m128i *pq2, __m128i *pq3) { - __m128i w10, w11, w12, w13; - __m128i w0, w1, w2, w3, w4, w5; - __m128i d0, d1, d2, d3; - - w0 = _mm_unpacklo_epi8( - *x0, *x1); // p 00 10 01 11 02 12 03 13 04 14 05 15 06 16 07 17 - w1 = _mm_unpacklo_epi8( - *x2, *x3); // p 20 30 21 31 22 32 23 33 24 34 25 35 26 36 27 37 - w2 = _mm_unpacklo_epi8( - *x4, *x5); // p 40 50 41 51 42 52 43 53 44 54 45 55 46 56 47 57 - w3 = _mm_unpacklo_epi8( - *x6, *x7); // p 60 70 61 71 62 72 63 73 64 74 65 75 66 76 67 77 - - w4 = _mm_unpacklo_epi16( - w0, w1); // 00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33 - w5 = _mm_unpacklo_epi16( - w2, w3); // 40 50 60 70 41 51 61 71 42 52 62 72 43 53 63 73 - - d0 = _mm_unpacklo_epi32( - w4, w5); // 00 10 20 30 40 50 60 70 01 11 21 31 41 51 61 71 - d2 = _mm_unpackhi_epi32( - w4, w5); // 02 12 22 32 42 52 62 72 03 13 23 33 43 53 63 73 - - w10 = _mm_unpacklo_epi8( - *x7, *x6); // q xx xx xx xx xx xx xx xx 00 10 01 11 02 12 03 13 - w11 = _mm_unpacklo_epi8( - *x5, *x4); // q xx xx xx xx xx xx xx xx 20 30 21 31 22 32 23 33 - w12 = _mm_unpacklo_epi8( - *x3, *x2); // q xx xx xx xx xx xx xx xx 40 50 41 51 42 52 43 53 - w13 = _mm_unpacklo_epi8( - *x1, *x0); // q xx xx xx xx xx xx xx xx 60 70 61 71 62 72 63 73 - - w4 = _mm_unpackhi_epi16( - w10, w11); // 00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33 - w5 = _mm_unpackhi_epi16( - w12, w13); // 40 50 60 70 41 51 61 71 42 52 62 72 43 53 63 73 - - d1 = _mm_unpacklo_epi32( - w4, w5); // 00 10 20 30 40 50 60 70 01 11 21 31 41 51 61 71 - d3 = _mm_unpackhi_epi32( - w4, w5); // 02 12 22 32 42 52 62 72 03 13 23 33 43 53 63 73 - - *pq0 = _mm_unpacklo_epi64(d0, d1); // pq - *pq1 = _mm_unpackhi_epi64(d0, d1); // pq - *pq2 = _mm_unpacklo_epi64(d2, d3); // pq - *pq3 = _mm_unpackhi_epi64(d2, d3); // pq -} - -static INLINE void transpose8x16_16x8_sse2( - __m128i *x0, __m128i *x1, __m128i *x2, __m128i *x3, __m128i *x4, - __m128i *x5, __m128i *x6, __m128i *x7, __m128i *d0d1, __m128i *d2d3, - __m128i *d4d5, __m128i *d6d7, __m128i *d8d9, __m128i *d10d11, - __m128i *d12d13, __m128i *d14d15) { - __m128i w0, w1, w2, w3, w4, w5, w6, w7, w8, w9; - __m128i w10, w11, w12, w13, w14, w15; - - w0 = _mm_unpacklo_epi8(*x0, *x1); - w1 = _mm_unpacklo_epi8(*x2, *x3); - w2 = _mm_unpacklo_epi8(*x4, *x5); - w3 = _mm_unpacklo_epi8(*x6, *x7); - - w8 = _mm_unpackhi_epi8(*x0, *x1); - w9 = _mm_unpackhi_epi8(*x2, *x3); - w10 = _mm_unpackhi_epi8(*x4, *x5); - w11 = _mm_unpackhi_epi8(*x6, *x7); - - w4 = _mm_unpacklo_epi16(w0, w1); - w5 = _mm_unpacklo_epi16(w2, w3); - w12 = _mm_unpacklo_epi16(w8, w9); - w13 = _mm_unpacklo_epi16(w10, w11); - - w6 = _mm_unpacklo_epi32(w4, w5); - w7 = _mm_unpackhi_epi32(w4, w5); - w14 = _mm_unpacklo_epi32(w12, w13); - w15 = _mm_unpackhi_epi32(w12, w13); - - // Store first 4-line result - *d0d1 = _mm_unpacklo_epi64(w6, w14); - *d2d3 = _mm_unpackhi_epi64(w6, w14); - *d4d5 = _mm_unpacklo_epi64(w7, w15); - *d6d7 = _mm_unpackhi_epi64(w7, w15); - - w4 = _mm_unpackhi_epi16(w0, w1); - w5 = _mm_unpackhi_epi16(w2, w3); - w12 = _mm_unpackhi_epi16(w8, w9); - w13 = _mm_unpackhi_epi16(w10, w11); - - w6 = _mm_unpacklo_epi32(w4, w5); - w7 = _mm_unpackhi_epi32(w4, w5); - w14 = _mm_unpacklo_epi32(w12, w13); - w15 = _mm_unpackhi_epi32(w12, w13); - - // Store second 4-line result - *d8d9 = _mm_unpacklo_epi64(w6, w14); - *d10d11 = _mm_unpackhi_epi64(w6, w14); - *d12d13 = _mm_unpacklo_epi64(w7, w15); - *d14d15 = _mm_unpackhi_epi64(w7, w15); -} - // this function treats its input as 2 parallel 8x4 matrices, transposes each of // them to 4x8 independently while flipping the second matrix horizontaly. Used // for 14 taps pq pairs creation @@ -416,6 +81,63 @@ static INLINE void transpose_pq_14_sse2(__m128i *x0, __m128i *x1, __m128i *x2, ww2); // 07 17 27 37 08 18 28 38 xx xx xx xx xx xx xx xx } +// this function treats its input as 2 parallel 8x4 matrices, transposes each of +// them independently while flipping the second matrix horizontaly Used for 14 +// taps filter pq pairs inverse +static INLINE void transpose_pq_14_inv_sse2(__m128i *x0, __m128i *x1, + __m128i *x2, __m128i *x3, + __m128i *x4, __m128i *x5, + __m128i *x6, __m128i *x7, + __m128i *pq0, __m128i *pq1, + __m128i *pq2, __m128i *pq3) { + __m128i w10, w11, w12, w13; + __m128i w0, w1, w2, w3, w4, w5; + __m128i d0, d1, d2, d3; + + w0 = _mm_unpacklo_epi8( + *x0, *x1); // p 00 10 01 11 02 12 03 13 04 14 05 15 06 16 07 17 + w1 = _mm_unpacklo_epi8( + *x2, *x3); // p 20 30 21 31 22 32 23 33 24 34 25 35 26 36 27 37 + w2 = _mm_unpacklo_epi8( + *x4, *x5); // p 40 50 41 51 42 52 43 53 44 54 45 55 46 56 47 57 + w3 = _mm_unpacklo_epi8( + *x6, *x7); // p 60 70 61 71 62 72 63 73 64 74 65 75 66 76 67 77 + + w4 = _mm_unpacklo_epi16( + w0, w1); // 00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33 + w5 = _mm_unpacklo_epi16( + w2, w3); // 40 50 60 70 41 51 61 71 42 52 62 72 43 53 63 73 + + d0 = _mm_unpacklo_epi32( + w4, w5); // 00 10 20 30 40 50 60 70 01 11 21 31 41 51 61 71 + d2 = _mm_unpackhi_epi32( + w4, w5); // 02 12 22 32 42 52 62 72 03 13 23 33 43 53 63 73 + + w10 = _mm_unpacklo_epi8( + *x7, *x6); // q xx xx xx xx xx xx xx xx 00 10 01 11 02 12 03 13 + w11 = _mm_unpacklo_epi8( + *x5, *x4); // q xx xx xx xx xx xx xx xx 20 30 21 31 22 32 23 33 + w12 = _mm_unpacklo_epi8( + *x3, *x2); // q xx xx xx xx xx xx xx xx 40 50 41 51 42 52 43 53 + w13 = _mm_unpacklo_epi8( + *x1, *x0); // q xx xx xx xx xx xx xx xx 60 70 61 71 62 72 63 73 + + w4 = _mm_unpackhi_epi16( + w10, w11); // 00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33 + w5 = _mm_unpackhi_epi16( + w12, w13); // 40 50 60 70 41 51 61 71 42 52 62 72 43 53 63 73 + + d1 = _mm_unpacklo_epi32( + w4, w5); // 00 10 20 30 40 50 60 70 01 11 21 31 41 51 61 71 + d3 = _mm_unpackhi_epi32( + w4, w5); // 02 12 22 32 42 52 62 72 03 13 23 33 43 53 63 73 + + *pq0 = _mm_unpacklo_epi64(d0, d1); // pq + *pq1 = _mm_unpackhi_epi64(d0, d1); // pq + *pq2 = _mm_unpacklo_epi64(d2, d3); // pq + *pq3 = _mm_unpackhi_epi64(d2, d3); // pq +} + static AOM_FORCE_INLINE void filter4_sse2(__m128i *p1p0, __m128i *q1q0, __m128i *hev, __m128i *mask, __m128i *qs1qs0, __m128i *ps1ps0) { @@ -616,10 +338,10 @@ void aom_lpf_horizontal_4_sse2(uint8_t *s, int p /* pitch */, __m128i qs1qs0, ps1ps0; __m128i p1, p0, q0, q1; - p1 = _mm_cvtsi32_si128(*(int *)(s - 2 * p)); - p0 = _mm_cvtsi32_si128(*(int *)(s - 1 * p)); - q0 = _mm_cvtsi32_si128(*(int *)(s + 0 * p)); - q1 = _mm_cvtsi32_si128(*(int *)(s + 1 * p)); + p1 = xx_loadl_32(s - 2 * p); + p0 = xx_loadl_32(s - 1 * p); + q0 = xx_loadl_32(s - 0 * p); + q1 = xx_loadl_32(s + 1 * p); lpf_internal_4_sse2(&p1, &p0, &q0, &q1, &limit, &thresh, &qs1qs0, &ps1ps0); @@ -1241,23 +963,16 @@ void aom_lpf_horizontal_14_sse2(unsigned char *s, int p, __m128i limit = _mm_load_si128((const __m128i *)_limit); __m128i thresh = _mm_load_si128((const __m128i *)_thresh); - q4p4 = _mm_unpacklo_epi32(_mm_cvtsi32_si128(*(int *)(s - 5 * p)), - _mm_cvtsi32_si128(*(int *)(s + 4 * p))); - q3p3 = _mm_unpacklo_epi32(_mm_cvtsi32_si128(*(int *)(s - 4 * p)), - _mm_cvtsi32_si128(*(int *)(s + 3 * p))); - q2p2 = _mm_unpacklo_epi32(_mm_cvtsi32_si128(*(int *)(s - 3 * p)), - _mm_cvtsi32_si128(*(int *)(s + 2 * p))); - q1p1 = _mm_unpacklo_epi32(_mm_cvtsi32_si128(*(int *)(s - 2 * p)), - _mm_cvtsi32_si128(*(int *)(s + 1 * p))); + q4p4 = _mm_unpacklo_epi32(xx_loadl_32(s - 5 * p), xx_loadl_32(s + 4 * p)); + q3p3 = _mm_unpacklo_epi32(xx_loadl_32(s - 4 * p), xx_loadl_32(s + 3 * p)); + q2p2 = _mm_unpacklo_epi32(xx_loadl_32(s - 3 * p), xx_loadl_32(s + 2 * p)); + q1p1 = _mm_unpacklo_epi32(xx_loadl_32(s - 2 * p), xx_loadl_32(s + 1 * p)); - q0p0 = _mm_unpacklo_epi32(_mm_cvtsi32_si128(*(int *)(s - 1 * p)), - _mm_cvtsi32_si128(*(int *)(s - 0 * p))); + q0p0 = _mm_unpacklo_epi32(xx_loadl_32(s - 1 * p), xx_loadl_32(s - 0 * p)); - q5p5 = _mm_unpacklo_epi32(_mm_cvtsi32_si128(*(int *)(s - 6 * p)), - _mm_cvtsi32_si128(*(int *)(s + 5 * p))); + q5p5 = _mm_unpacklo_epi32(xx_loadl_32(s - 6 * p), xx_loadl_32(s + 5 * p)); - q6p6 = _mm_unpacklo_epi32(_mm_cvtsi32_si128(*(int *)(s - 7 * p)), - _mm_cvtsi32_si128(*(int *)(s + 6 * p))); + q6p6 = _mm_unpacklo_epi32(xx_loadl_32(s - 7 * p), xx_loadl_32(s + 6 * p)); lpf_internal_14_sse2(&q6p6, &q5p5, &q4p4, &q3p3, &q2p2, &q1p1, &q0p0, &blimit, &limit, &thresh); @@ -1543,12 +1258,12 @@ void aom_lpf_horizontal_6_sse2(unsigned char *s, int p, __m128i limit = _mm_load_si128((__m128i *)_limit); __m128i thresh = _mm_load_si128((__m128i *)_thresh); - p2 = _mm_cvtsi32_si128(*(int *)(s - 3 * p)); - p1 = _mm_cvtsi32_si128(*(int *)(s - 2 * p)); - p0 = _mm_cvtsi32_si128(*(int *)(s - 1 * p)); - q0 = _mm_cvtsi32_si128(*(int *)(s - 0 * p)); - q1 = _mm_cvtsi32_si128(*(int *)(s + 1 * p)); - q2 = _mm_cvtsi32_si128(*(int *)(s + 2 * p)); + p2 = xx_loadl_32(s - 3 * p); + p1 = xx_loadl_32(s - 2 * p); + p0 = xx_loadl_32(s - 1 * p); + q0 = xx_loadl_32(s - 0 * p); + q1 = xx_loadl_32(s + 1 * p); + q2 = xx_loadl_32(s + 2 * p); lpf_internal_6_sse2(&p2, &q2, &p1, &q1, &p0, &q0, &q1q0, &p1p0, &blimit, &limit, &thresh); @@ -1895,20 +1610,20 @@ void aom_lpf_horizontal_8_sse2(unsigned char *s, int p, const unsigned char *_blimit, const unsigned char *_limit, const unsigned char *_thresh) { - __m128i p2, p1, p0, q0, q1, q2, p3, q3; + __m128i p3, p2, p1, p0, q0, q1, q2, q3; __m128i q1q0, p1p0; __m128i blimit = _mm_load_si128((const __m128i *)_blimit); __m128i limit = _mm_load_si128((const __m128i *)_limit); __m128i thresh = _mm_load_si128((const __m128i *)_thresh); - p3 = _mm_cvtsi32_si128(*(int *)(s - 4 * p)); - p2 = _mm_cvtsi32_si128(*(int *)(s - 3 * p)); - p1 = _mm_cvtsi32_si128(*(int *)(s - 2 * p)); - p0 = _mm_cvtsi32_si128(*(int *)(s - 1 * p)); - q0 = _mm_cvtsi32_si128(*(int *)(s - 0 * p)); - q1 = _mm_cvtsi32_si128(*(int *)(s + 1 * p)); - q2 = _mm_cvtsi32_si128(*(int *)(s + 2 * p)); - q3 = _mm_cvtsi32_si128(*(int *)(s + 3 * p)); + p3 = xx_loadl_32(s - 4 * p); + p2 = xx_loadl_32(s - 3 * p); + p1 = xx_loadl_32(s - 2 * p); + p0 = xx_loadl_32(s - 1 * p); + q0 = xx_loadl_32(s - 0 * p); + q1 = xx_loadl_32(s + 1 * p); + q2 = xx_loadl_32(s + 2 * p); + q3 = xx_loadl_32(s + 3 * p); lpf_internal_8_sse2(&p3, &q3, &p2, &q2, &p1, &q1, &p0, &q0, &q1q0, &p1p0, &blimit, &limit, &thresh); diff --git a/third_party/aom/aom_dsp/x86/lpf_common_sse2.h b/third_party/aom/aom_dsp/x86/lpf_common_sse2.h index 8970fe7dd6..6ed2cbfdf4 100644 --- a/third_party/aom/aom_dsp/x86/lpf_common_sse2.h +++ b/third_party/aom/aom_dsp/x86/lpf_common_sse2.h @@ -212,4 +212,284 @@ static INLINE void highbd_transpose8x16_sse2( d4 + 1, d5 + 1, d6 + 1, d7 + 1); } +// Low bit depth functions +static INLINE void transpose4x8_8x4_low_sse2(__m128i *x0, __m128i *x1, + __m128i *x2, __m128i *x3, + __m128i *d0, __m128i *d1, + __m128i *d2, __m128i *d3) { + // input + // x0 00 01 02 03 04 05 06 07 xx xx xx xx xx xx xx xx + // x1 10 11 12 13 14 15 16 17 xx xx xx xx xx xx xx xx + // x2 20 21 22 23 24 25 26 27 xx xx xx xx xx xx xx xx + // x3 30 31 32 33 34 35 36 37 xx xx xx xx xx xx xx xx + // output + // 00 10 20 30 xx xx xx xx xx xx xx xx xx xx xx xx + // 01 11 21 31 xx xx xx xx xx xx xx xx xx xx xx xx + // 02 12 22 32 xx xx xx xx xx xx xx xx xx xx xx xx + // 03 13 23 33 xx xx xx xx xx xx xx xx xx xx xx xx + + __m128i w0, w1; + + w0 = _mm_unpacklo_epi8( + *x0, *x1); // 00 10 01 11 02 12 03 13 04 14 05 15 06 16 07 17 + w1 = _mm_unpacklo_epi8( + *x2, *x3); // 20 30 21 31 22 32 23 33 24 34 25 35 26 36 27 37 + + *d0 = _mm_unpacklo_epi16( + w0, w1); // 00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33 + + *d1 = _mm_srli_si128(*d0, + 4); // 01 11 21 31 xx xx xx xx xx xx xx xx xx xx xx xx + *d2 = _mm_srli_si128(*d0, + 8); // 02 12 22 32 xx xx xx xx xx xx xx xx xx xx xx xx + *d3 = _mm_srli_si128(*d0, + 12); // 03 13 23 33 xx xx xx xx xx xx xx xx xx xx xx xx +} + +static INLINE void transpose4x8_8x4_sse2(__m128i *x0, __m128i *x1, __m128i *x2, + __m128i *x3, __m128i *d0, __m128i *d1, + __m128i *d2, __m128i *d3, __m128i *d4, + __m128i *d5, __m128i *d6, + __m128i *d7) { + // input + // x0 00 01 02 03 04 05 06 07 xx xx xx xx xx xx xx xx + // x1 10 11 12 13 14 15 16 17 xx xx xx xx xx xx xx xx + // x2 20 21 22 23 24 25 26 27 xx xx xx xx xx xx xx xx + // x3 30 31 32 33 34 35 36 37 xx xx xx xx xx xx xx xx + // output + // 00 10 20 30 xx xx xx xx xx xx xx xx xx xx xx xx + // 01 11 21 31 xx xx xx xx xx xx xx xx xx xx xx xx + // 02 12 22 32 xx xx xx xx xx xx xx xx xx xx xx xx + // 03 13 23 33 xx xx xx xx xx xx xx xx xx xx xx xx + // 04 14 24 34 xx xx xx xx xx xx xx xx xx xx xx xx + // 05 15 25 35 xx xx xx xx xx xx xx xx xx xx xx xx + // 06 16 26 36 xx xx xx xx xx xx xx xx xx xx xx xx + // 07 17 27 37 xx xx xx xx xx xx xx xx xx xx xx xx + + __m128i w0, w1, ww0, ww1; + + w0 = _mm_unpacklo_epi8( + *x0, *x1); // 00 10 01 11 02 12 03 13 04 14 05 15 06 16 07 17 + w1 = _mm_unpacklo_epi8( + *x2, *x3); // 20 30 21 31 22 32 23 33 24 34 25 35 26 36 27 37 + + ww0 = _mm_unpacklo_epi16( + w0, w1); // 00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33 + ww1 = _mm_unpackhi_epi16( + w0, w1); // 04 14 24 34 05 15 25 35 06 16 26 36 07 17 27 37 + + *d0 = ww0; // 00 10 20 30 xx xx xx xx xx xx xx xx xx xx xx xx + *d1 = _mm_srli_si128(ww0, + 4); // 01 11 21 31 xx xx xx xx xx xx xx xx xx xx xx xx + *d2 = _mm_srli_si128(ww0, + 8); // 02 12 22 32 xx xx xx xx xx xx xx xx xx xx xx xx + *d3 = _mm_srli_si128(ww0, + 12); // 03 13 23 33 xx xx xx xx xx xx xx xx xx xx xx xx + + *d4 = ww1; // 04 14 24 34 xx xx xx xx xx xx xx xx xx xx xx xx + *d5 = _mm_srli_si128(ww1, + 4); // 05 15 25 35 xx xx xx xx xx xx xx xx xx xx xx xx + *d6 = _mm_srli_si128(ww1, + 8); // 06 16 26 36 xx xx xx xx xx xx xx xx xx xx xx xx + *d7 = _mm_srli_si128(ww1, + 12); // 07 17 27 37 xx xx xx xx xx xx xx xx xx xx xx xx +} + +static INLINE void transpose8x8_low_sse2(__m128i *x0, __m128i *x1, __m128i *x2, + __m128i *x3, __m128i *x4, __m128i *x5, + __m128i *x6, __m128i *x7, __m128i *d0, + __m128i *d1, __m128i *d2, + __m128i *d3) { + // input + // x0 00 01 02 03 04 05 06 07 + // x1 10 11 12 13 14 15 16 17 + // x2 20 21 22 23 24 25 26 27 + // x3 30 31 32 33 34 35 36 37 + // x4 40 41 42 43 44 45 46 47 + // x5 50 51 52 53 54 55 56 57 + // x6 60 61 62 63 64 65 66 67 + // x7 70 71 72 73 74 75 76 77 + // output + // d0 00 10 20 30 40 50 60 70 xx xx xx xx xx xx xx + // d1 01 11 21 31 41 51 61 71 xx xx xx xx xx xx xx xx + // d2 02 12 22 32 42 52 62 72 xx xx xx xx xx xx xx xx + // d3 03 13 23 33 43 53 63 73 xx xx xx xx xx xx xx xx + + __m128i w0, w1, w2, w3, w4, w5; + + w0 = _mm_unpacklo_epi8( + *x0, *x1); // 00 10 01 11 02 12 03 13 04 14 05 15 06 16 07 17 + + w1 = _mm_unpacklo_epi8( + *x2, *x3); // 20 30 21 31 22 32 23 33 24 34 25 35 26 36 27 37 + + w2 = _mm_unpacklo_epi8( + *x4, *x5); // 40 50 41 51 42 52 43 53 44 54 45 55 46 56 47 57 + + w3 = _mm_unpacklo_epi8( + *x6, *x7); // 60 70 61 71 62 72 63 73 64 74 65 75 66 76 67 77 + + w4 = _mm_unpacklo_epi16( + w0, w1); // 00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33 + w5 = _mm_unpacklo_epi16( + w2, w3); // 40 50 60 70 41 51 61 71 42 52 62 72 43 53 63 73 + + *d0 = _mm_unpacklo_epi32( + w4, w5); // 00 10 20 30 40 50 60 70 01 11 21 31 41 51 61 71 + *d1 = _mm_srli_si128(*d0, 8); + *d2 = _mm_unpackhi_epi32( + w4, w5); // 02 12 22 32 42 52 62 72 03 13 23 33 43 53 63 73 + *d3 = _mm_srli_si128(*d2, 8); +} + +static INLINE void transpose8x8_sse2(__m128i *x0, __m128i *x1, __m128i *x2, + __m128i *x3, __m128i *x4, __m128i *x5, + __m128i *x6, __m128i *x7, __m128i *d0d1, + __m128i *d2d3, __m128i *d4d5, + __m128i *d6d7) { + __m128i w0, w1, w2, w3, w4, w5, w6, w7; + // x0 00 01 02 03 04 05 06 07 + // x1 10 11 12 13 14 15 16 17 + w0 = _mm_unpacklo_epi8( + *x0, *x1); // 00 10 01 11 02 12 03 13 04 14 05 15 06 16 07 17 + + // x2 20 21 22 23 24 25 26 27 + // x3 30 31 32 33 34 35 36 37 + w1 = _mm_unpacklo_epi8( + *x2, *x3); // 20 30 21 31 22 32 23 33 24 34 25 35 26 36 27 37 + + // x4 40 41 42 43 44 45 46 47 + // x5 50 51 52 53 54 55 56 57 + w2 = _mm_unpacklo_epi8( + *x4, *x5); // 40 50 41 51 42 52 43 53 44 54 45 55 46 56 47 57 + + // x6 60 61 62 63 64 65 66 67 + // x7 70 71 72 73 74 75 76 77 + w3 = _mm_unpacklo_epi8( + *x6, *x7); // 60 70 61 71 62 72 63 73 64 74 65 75 66 76 67 77 + + w4 = _mm_unpacklo_epi16( + w0, w1); // 00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33 + w5 = _mm_unpacklo_epi16( + w2, w3); // 40 50 60 70 41 51 61 71 42 52 62 72 43 53 63 73 + + *d0d1 = _mm_unpacklo_epi32( + w4, w5); // 00 10 20 30 40 50 60 70 01 11 21 31 41 51 61 71 + *d2d3 = _mm_unpackhi_epi32( + w4, w5); // 02 12 22 32 42 52 62 72 03 13 23 33 43 53 63 73 + + w6 = _mm_unpackhi_epi16( + w0, w1); // 04 14 24 34 05 15 25 35 06 16 26 36 07 17 27 37 + w7 = _mm_unpackhi_epi16( + w2, w3); // 44 54 64 74 45 55 65 75 46 56 66 76 47 57 67 77 + + *d4d5 = _mm_unpacklo_epi32( + w6, w7); // 04 14 24 34 44 54 64 74 05 15 25 35 45 55 65 75 + *d6d7 = _mm_unpackhi_epi32( + w6, w7); // 06 16 26 36 46 56 66 76 07 17 27 37 47 57 67 77 +} + +static INLINE void transpose16x8_8x16_sse2( + __m128i *x0, __m128i *x1, __m128i *x2, __m128i *x3, __m128i *x4, + __m128i *x5, __m128i *x6, __m128i *x7, __m128i *x8, __m128i *x9, + __m128i *x10, __m128i *x11, __m128i *x12, __m128i *x13, __m128i *x14, + __m128i *x15, __m128i *d0, __m128i *d1, __m128i *d2, __m128i *d3, + __m128i *d4, __m128i *d5, __m128i *d6, __m128i *d7) { + __m128i w0, w1, w2, w3, w4, w5, w6, w7, w8, w9; + __m128i w10, w11, w12, w13, w14, w15; + + w0 = _mm_unpacklo_epi8(*x0, *x1); + w1 = _mm_unpacklo_epi8(*x2, *x3); + w2 = _mm_unpacklo_epi8(*x4, *x5); + w3 = _mm_unpacklo_epi8(*x6, *x7); + + w8 = _mm_unpacklo_epi8(*x8, *x9); + w9 = _mm_unpacklo_epi8(*x10, *x11); + w10 = _mm_unpacklo_epi8(*x12, *x13); + w11 = _mm_unpacklo_epi8(*x14, *x15); + + w4 = _mm_unpacklo_epi16(w0, w1); + w5 = _mm_unpacklo_epi16(w2, w3); + w12 = _mm_unpacklo_epi16(w8, w9); + w13 = _mm_unpacklo_epi16(w10, w11); + + w6 = _mm_unpacklo_epi32(w4, w5); + w7 = _mm_unpackhi_epi32(w4, w5); + w14 = _mm_unpacklo_epi32(w12, w13); + w15 = _mm_unpackhi_epi32(w12, w13); + + // Store first 4-line result + *d0 = _mm_unpacklo_epi64(w6, w14); + *d1 = _mm_unpackhi_epi64(w6, w14); + *d2 = _mm_unpacklo_epi64(w7, w15); + *d3 = _mm_unpackhi_epi64(w7, w15); + + w4 = _mm_unpackhi_epi16(w0, w1); + w5 = _mm_unpackhi_epi16(w2, w3); + w12 = _mm_unpackhi_epi16(w8, w9); + w13 = _mm_unpackhi_epi16(w10, w11); + + w6 = _mm_unpacklo_epi32(w4, w5); + w7 = _mm_unpackhi_epi32(w4, w5); + w14 = _mm_unpacklo_epi32(w12, w13); + w15 = _mm_unpackhi_epi32(w12, w13); + + // Store second 4-line result + *d4 = _mm_unpacklo_epi64(w6, w14); + *d5 = _mm_unpackhi_epi64(w6, w14); + *d6 = _mm_unpacklo_epi64(w7, w15); + *d7 = _mm_unpackhi_epi64(w7, w15); +} + +static INLINE void transpose8x16_16x8_sse2( + __m128i *x0, __m128i *x1, __m128i *x2, __m128i *x3, __m128i *x4, + __m128i *x5, __m128i *x6, __m128i *x7, __m128i *d0d1, __m128i *d2d3, + __m128i *d4d5, __m128i *d6d7, __m128i *d8d9, __m128i *d10d11, + __m128i *d12d13, __m128i *d14d15) { + __m128i w0, w1, w2, w3, w4, w5, w6, w7, w8, w9; + __m128i w10, w11, w12, w13, w14, w15; + + w0 = _mm_unpacklo_epi8(*x0, *x1); + w1 = _mm_unpacklo_epi8(*x2, *x3); + w2 = _mm_unpacklo_epi8(*x4, *x5); + w3 = _mm_unpacklo_epi8(*x6, *x7); + + w8 = _mm_unpackhi_epi8(*x0, *x1); + w9 = _mm_unpackhi_epi8(*x2, *x3); + w10 = _mm_unpackhi_epi8(*x4, *x5); + w11 = _mm_unpackhi_epi8(*x6, *x7); + + w4 = _mm_unpacklo_epi16(w0, w1); + w5 = _mm_unpacklo_epi16(w2, w3); + w12 = _mm_unpacklo_epi16(w8, w9); + w13 = _mm_unpacklo_epi16(w10, w11); + + w6 = _mm_unpacklo_epi32(w4, w5); + w7 = _mm_unpackhi_epi32(w4, w5); + w14 = _mm_unpacklo_epi32(w12, w13); + w15 = _mm_unpackhi_epi32(w12, w13); + + // Store first 4-line result + *d0d1 = _mm_unpacklo_epi64(w6, w14); + *d2d3 = _mm_unpackhi_epi64(w6, w14); + *d4d5 = _mm_unpacklo_epi64(w7, w15); + *d6d7 = _mm_unpackhi_epi64(w7, w15); + + w4 = _mm_unpackhi_epi16(w0, w1); + w5 = _mm_unpackhi_epi16(w2, w3); + w12 = _mm_unpackhi_epi16(w8, w9); + w13 = _mm_unpackhi_epi16(w10, w11); + + w6 = _mm_unpacklo_epi32(w4, w5); + w7 = _mm_unpackhi_epi32(w4, w5); + w14 = _mm_unpacklo_epi32(w12, w13); + w15 = _mm_unpackhi_epi32(w12, w13); + + // Store second 4-line result + *d8d9 = _mm_unpacklo_epi64(w6, w14); + *d10d11 = _mm_unpackhi_epi64(w6, w14); + *d12d13 = _mm_unpacklo_epi64(w7, w15); + *d14d15 = _mm_unpackhi_epi64(w7, w15); +} + #endif // AOM_AOM_DSP_X86_LPF_COMMON_SSE2_H_ diff --git a/third_party/aom/aom_dsp/x86/masked_variance_intrin_ssse3.c b/third_party/aom/aom_dsp/x86/masked_variance_intrin_ssse3.c index d7dbefd7d9..ebf4631ba7 100644 --- a/third_party/aom/aom_dsp/x86/masked_variance_intrin_ssse3.c +++ b/third_party/aom/aom_dsp/x86/masked_variance_intrin_ssse3.c @@ -218,15 +218,15 @@ static void bilinear_filter(const uint8_t *src, int src_stride, int xoffset, } } -static INLINE __m128i filter_block_2rows(const __m128i a0, const __m128i b0, - const __m128i a1, const __m128i b1, - const __m128i filter) { - __m128i v0 = _mm_unpacklo_epi8(a0, b0); - v0 = _mm_maddubs_epi16(v0, filter); +static INLINE __m128i filter_block_2rows(const __m128i *a0, const __m128i *b0, + const __m128i *a1, const __m128i *b1, + const __m128i *filter) { + __m128i v0 = _mm_unpacklo_epi8(*a0, *b0); + v0 = _mm_maddubs_epi16(v0, *filter); v0 = xx_roundn_epu16(v0, FILTER_BITS); - __m128i v1 = _mm_unpacklo_epi8(a1, b1); - v1 = _mm_maddubs_epi16(v1, filter); + __m128i v1 = _mm_unpacklo_epi8(*a1, *b1); + v1 = _mm_maddubs_epi16(v1, *filter); v1 = xx_roundn_epu16(v1, FILTER_BITS); return _mm_packus_epi16(v0, v1); @@ -262,7 +262,7 @@ static void bilinear_filter8xh(const uint8_t *src, int src_stride, int xoffset, const __m128i z0 = _mm_srli_si128(x0, 1); const __m128i x1 = _mm_loadu_si128((__m128i *)&src[src_stride]); const __m128i z1 = _mm_srli_si128(x1, 1); - const __m128i res = filter_block_2rows(x0, z0, x1, z1, hfilter_vec); + const __m128i res = filter_block_2rows(&x0, &z0, &x1, &z1, &hfilter_vec); _mm_storeu_si128((__m128i *)b, res); src += src_stride * 2; @@ -296,7 +296,7 @@ static void bilinear_filter8xh(const uint8_t *src, int src_stride, int xoffset, const __m128i x = _mm_loadl_epi64((__m128i *)dst); const __m128i y = _mm_loadl_epi64((__m128i *)&dst[8]); const __m128i z = _mm_loadl_epi64((__m128i *)&dst[16]); - const __m128i res = filter_block_2rows(x, y, y, z, vfilter_vec); + const __m128i res = filter_block_2rows(&x, &y, &y, &z, &vfilter_vec); _mm_storeu_si128((__m128i *)dst, res); dst += 16; @@ -343,7 +343,7 @@ static void bilinear_filter4xh(const uint8_t *src, int src_stride, int xoffset, const __m128i b0 = _mm_unpacklo_epi32(z0, z1); const __m128i a1 = _mm_unpacklo_epi32(x2, x3); const __m128i b1 = _mm_unpacklo_epi32(z2, z3); - const __m128i res = filter_block_2rows(a0, b0, a1, b1, hfilter_vec); + const __m128i res = filter_block_2rows(&a0, &b0, &a1, &b1, &hfilter_vec); _mm_storeu_si128((__m128i *)b, res); src += src_stride * 4; @@ -384,7 +384,7 @@ static void bilinear_filter4xh(const uint8_t *src, int src_stride, int xoffset, const __m128i b0 = _mm_unpacklo_epi32(b, c); const __m128i a1 = _mm_unpacklo_epi32(c, d); const __m128i b1 = _mm_unpacklo_epi32(d, e); - const __m128i res = filter_block_2rows(a0, b0, a1, b1, vfilter_vec); + const __m128i res = filter_block_2rows(&a0, &b0, &a1, &b1, &vfilter_vec); _mm_storeu_si128((__m128i *)dst, res); dst += 16; @@ -392,29 +392,29 @@ static void bilinear_filter4xh(const uint8_t *src, int src_stride, int xoffset, } } -static INLINE void accumulate_block(const __m128i src, const __m128i a, - const __m128i b, const __m128i m, +static INLINE void accumulate_block(const __m128i *src, const __m128i *a, + const __m128i *b, const __m128i *m, __m128i *sum, __m128i *sum_sq) { const __m128i zero = _mm_setzero_si128(); const __m128i one = _mm_set1_epi16(1); const __m128i mask_max = _mm_set1_epi8((1 << AOM_BLEND_A64_ROUND_BITS)); - const __m128i m_inv = _mm_sub_epi8(mask_max, m); + const __m128i m_inv = _mm_sub_epi8(mask_max, *m); // Calculate 16 predicted pixels. // Note that the maximum value of any entry of 'pred_l' or 'pred_r' // is 64 * 255, so we have plenty of space to add rounding constants. - const __m128i data_l = _mm_unpacklo_epi8(a, b); - const __m128i mask_l = _mm_unpacklo_epi8(m, m_inv); + const __m128i data_l = _mm_unpacklo_epi8(*a, *b); + const __m128i mask_l = _mm_unpacklo_epi8(*m, m_inv); __m128i pred_l = _mm_maddubs_epi16(data_l, mask_l); pred_l = xx_roundn_epu16(pred_l, AOM_BLEND_A64_ROUND_BITS); - const __m128i data_r = _mm_unpackhi_epi8(a, b); - const __m128i mask_r = _mm_unpackhi_epi8(m, m_inv); + const __m128i data_r = _mm_unpackhi_epi8(*a, *b); + const __m128i mask_r = _mm_unpackhi_epi8(*m, m_inv); __m128i pred_r = _mm_maddubs_epi16(data_r, mask_r); pred_r = xx_roundn_epu16(pred_r, AOM_BLEND_A64_ROUND_BITS); - const __m128i src_l = _mm_unpacklo_epi8(src, zero); - const __m128i src_r = _mm_unpackhi_epi8(src, zero); + const __m128i src_l = _mm_unpacklo_epi8(*src, zero); + const __m128i src_r = _mm_unpackhi_epi8(*src, zero); const __m128i diff_l = _mm_sub_epi16(pred_l, src_l); const __m128i diff_r = _mm_sub_epi16(pred_r, src_r); @@ -440,7 +440,7 @@ static void masked_variance(const uint8_t *src_ptr, int src_stride, const __m128i a = _mm_loadu_si128((const __m128i *)&a_ptr[x]); const __m128i b = _mm_loadu_si128((const __m128i *)&b_ptr[x]); const __m128i m = _mm_loadu_si128((const __m128i *)&m_ptr[x]); - accumulate_block(src, a, b, m, &sum, &sum_sq); + accumulate_block(&src, &a, &b, &m, &sum, &sum_sq); } src_ptr += src_stride; @@ -471,7 +471,7 @@ static void masked_variance8xh(const uint8_t *src_ptr, int src_stride, const __m128i m = _mm_unpacklo_epi64(_mm_loadl_epi64((const __m128i *)m_ptr), _mm_loadl_epi64((const __m128i *)&m_ptr[m_stride])); - accumulate_block(src, a, b, m, &sum, &sum_sq); + accumulate_block(&src, &a, &b, &m, &sum, &sum_sq); src_ptr += src_stride * 2; a_ptr += 16; @@ -503,7 +503,7 @@ static void masked_variance4xh(const uint8_t *src_ptr, int src_stride, const __m128i m = _mm_setr_epi32( *(uint32_t *)m_ptr, *(uint32_t *)&m_ptr[m_stride], *(uint32_t *)&m_ptr[m_stride * 2], *(uint32_t *)&m_ptr[m_stride * 3]); - accumulate_block(src, a, b, m, &sum, &sum_sq); + accumulate_block(&src, &a, &b, &m, &sum, &sum_sq); src_ptr += src_stride * 4; a_ptr += 16; @@ -797,17 +797,17 @@ static void highbd_bilinear_filter(const uint16_t *src, int src_stride, } } -static INLINE __m128i highbd_filter_block_2rows(const __m128i a0, - const __m128i b0, - const __m128i a1, - const __m128i b1, - const __m128i filter) { - __m128i v0 = _mm_unpacklo_epi16(a0, b0); - v0 = _mm_madd_epi16(v0, filter); +static INLINE __m128i highbd_filter_block_2rows(const __m128i *a0, + const __m128i *b0, + const __m128i *a1, + const __m128i *b1, + const __m128i *filter) { + __m128i v0 = _mm_unpacklo_epi16(*a0, *b0); + v0 = _mm_madd_epi16(v0, *filter); v0 = xx_roundn_epu32(v0, FILTER_BITS); - __m128i v1 = _mm_unpacklo_epi16(a1, b1); - v1 = _mm_madd_epi16(v1, filter); + __m128i v1 = _mm_unpacklo_epi16(*a1, *b1); + v1 = _mm_madd_epi16(v1, *filter); v1 = xx_roundn_epu32(v1, FILTER_BITS); return _mm_packs_epi32(v0, v1); @@ -845,7 +845,7 @@ static void highbd_bilinear_filter4xh(const uint16_t *src, int src_stride, const __m128i x1 = _mm_loadu_si128((__m128i *)&src[src_stride]); const __m128i z1 = _mm_srli_si128(x1, 2); const __m128i res = - highbd_filter_block_2rows(x0, z0, x1, z1, hfilter_vec); + highbd_filter_block_2rows(&x0, &z0, &x1, &z1, &hfilter_vec); _mm_storeu_si128((__m128i *)b, res); src += src_stride * 2; @@ -879,7 +879,8 @@ static void highbd_bilinear_filter4xh(const uint16_t *src, int src_stride, const __m128i x = _mm_loadl_epi64((__m128i *)dst); const __m128i y = _mm_loadl_epi64((__m128i *)&dst[4]); const __m128i z = _mm_loadl_epi64((__m128i *)&dst[8]); - const __m128i res = highbd_filter_block_2rows(x, y, y, z, vfilter_vec); + const __m128i res = + highbd_filter_block_2rows(&x, &y, &y, &z, &vfilter_vec); _mm_storeu_si128((__m128i *)dst, res); dst += 8; diff --git a/third_party/aom/aom_dsp/x86/quantize_avx_x86_64.asm b/third_party/aom/aom_dsp/x86/quantize_avx_x86_64.asm index 216a0bd8f9..d6e15c4be5 100644 --- a/third_party/aom/aom_dsp/x86/quantize_avx_x86_64.asm +++ b/third_party/aom/aom_dsp/x86/quantize_avx_x86_64.asm @@ -126,7 +126,7 @@ cglobal quantize_%1, 0, %2, 15, coeff, ncoeff, zbin, round, quant, \ punpckhqdq m3, m3 pmullw m13, m3 ; dqc[i] = qc[i] * q - ; Store 16bit numbers as 32bit numbers in array pointed to by qcoeff + ; Store 16bit numbers as 32bit numbers in array pointed to by dqcoeff pcmpgtw m6, m5, m8 punpckhwd m6, m8, m6 pmovsxwd m11, m8 @@ -198,10 +198,7 @@ DEFINE_ARGS coeff, ncoeff, zbin, round, quant, shift, \ mova m4, [r2] ; m4 = shift mov r4, dqcoeffmp mov r5, iscanmp -%ifidn %1, b_32x32 - psllw m4, 1 -%endif - pxor m5, m5 ; m5 = dedicated zero + pxor m5, m5 ; m5 = dedicated zero DEFINE_ARGS coeff, ncoeff, d1, qcoeff, dqcoeff, iscan, d2, d3, d4, eob @@ -255,9 +252,26 @@ DEFINE_ARGS coeff, ncoeff, zbin, round, quant, shift, \ pmulhw m13, m11, m2 ; m13 = m11*q>>16 paddw m8, m6 ; m8 += m6 paddw m13, m11 ; m13 += m11 + %ifidn %1, b_32x32 + pmullw m5, m8, m4 ; store the lower 16 bits of m8*qsh + %endif pmulhw m8, m4 ; m8 = m8*qsh>>16 + %ifidn %1, b_32x32 + psllw m8, 1 + psrlw m5, 15 + por m8, m5 + %endif punpckhqdq m4, m4 + %ifidn %1, b_32x32 + pmullw m5, m13, m4 ; store the lower 16 bits of m13*qsh + %endif pmulhw m13, m4 ; m13 = m13*qsh>>16 + %ifidn %1, b_32x32 + psllw m13, 1 + psrlw m5, 15 + por m13, m5 + pxor m5, m5 ; reset m5 to zero register + %endif psignw m8, m9 ; m8 = reinsert sign psignw m13, m10 ; m13 = reinsert sign pand m8, m7 @@ -289,7 +303,7 @@ DEFINE_ARGS coeff, ncoeff, zbin, round, quant, shift, \ psignw m13, m10 %endif - ; store 16bit numbers as 32bit numbers in array pointed to by qcoeff + ; store 16bit numbers as 32bit numbers in array pointed to by dqcoeff pcmpgtw m6, m5, m8 punpckhwd m6, m8, m6 pmovsxwd m11, m8 @@ -359,8 +373,23 @@ DEFINE_ARGS coeff, ncoeff, zbin, round, quant, shift, \ pmulhw m13, m11, m2 ; m13 = m11*q>>16 paddw m14, m6 ; m14 += m6 paddw m13, m11 ; m13 += m11 + %ifidn %1, b_32x32 + pmullw m5, m14, m4 ; store the lower 16 bits of m14*qsh + %endif pmulhw m14, m4 ; m14 = m14*qsh>>16 + %ifidn %1, b_32x32 + psllw m14, 1 + psrlw m5, 15 + por m14, m5 + pmullw m5, m13, m4 ; store the lower 16 bits of m13*qsh + %endif pmulhw m13, m4 ; m13 = m13*qsh>>16 + %ifidn %1, b_32x32 + psllw m13, 1 + psrlw m5, 15 + por m13, m5 + pxor m5, m5 ; reset m5 to zero register + %endif psignw m14, m9 ; m14 = reinsert sign psignw m13, m10 ; m13 = reinsert sign pand m14, m7 @@ -391,7 +420,7 @@ DEFINE_ARGS coeff, ncoeff, zbin, round, quant, shift, \ psignw m13, m10 %endif - ; store 16bit numbers as 32bit numbers in array pointed to by qcoeff + ; store 16bit numbers as 32bit numbers in array pointed to by dqcoeff pcmpgtw m6, m5, m14 punpckhwd m6, m14, m6 pmovsxwd m11, m14 diff --git a/third_party/aom/aom_dsp/x86/quantize_sse2.c b/third_party/aom/aom_dsp/x86/quantize_sse2.c index d3de6e24db..ebef1fbac2 100644 --- a/third_party/aom/aom_dsp/x86/quantize_sse2.c +++ b/third_party/aom/aom_dsp/x86/quantize_sse2.c @@ -18,28 +18,6 @@ #include "aom/aom_integer.h" #include "aom_dsp/x86/quantize_x86.h" -static INLINE __m128i load_coefficients(const tran_low_t *coeff_ptr) { - assert(sizeof(tran_low_t) == 4); - - return _mm_setr_epi16((int16_t)coeff_ptr[0], (int16_t)coeff_ptr[1], - (int16_t)coeff_ptr[2], (int16_t)coeff_ptr[3], - (int16_t)coeff_ptr[4], (int16_t)coeff_ptr[5], - (int16_t)coeff_ptr[6], (int16_t)coeff_ptr[7]); -} - -static INLINE void store_coefficients(__m128i coeff_vals, - tran_low_t *coeff_ptr) { - assert(sizeof(tran_low_t) == 4); - - __m128i one = _mm_set1_epi16(1); - __m128i coeff_vals_hi = _mm_mulhi_epi16(coeff_vals, one); - __m128i coeff_vals_lo = _mm_mullo_epi16(coeff_vals, one); - __m128i coeff_vals_1 = _mm_unpacklo_epi16(coeff_vals_lo, coeff_vals_hi); - __m128i coeff_vals_2 = _mm_unpackhi_epi16(coeff_vals_lo, coeff_vals_hi); - _mm_store_si128((__m128i *)(coeff_ptr), coeff_vals_1); - _mm_store_si128((__m128i *)(coeff_ptr + 4), coeff_vals_2); -} - void aom_quantize_b_sse2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, diff --git a/third_party/aom/aom_dsp/x86/quantize_ssse3.c b/third_party/aom/aom_dsp/x86/quantize_ssse3.c new file mode 100644 index 0000000000..25980a055a --- /dev/null +++ b/third_party/aom/aom_dsp/x86/quantize_ssse3.c @@ -0,0 +1,192 @@ +/* + * Copyright (c) 2019, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include +#include +#include + +#include "config/aom_dsp_rtcd.h" + +#include "aom/aom_integer.h" +#include "aom_dsp/x86/quantize_x86.h" + +static INLINE void calculate_qcoeff_64x64(__m128i *coeff, const __m128i round, + const __m128i quant, + const __m128i *shift) { + __m128i tmp, qcoeff, tmp1; + qcoeff = _mm_adds_epi16(*coeff, round); + tmp = _mm_mulhi_epi16(qcoeff, quant); + qcoeff = _mm_add_epi16(tmp, qcoeff); + tmp = _mm_mullo_epi16(qcoeff, *shift); + tmp = _mm_srli_epi16(tmp, 14); + tmp1 = _mm_mulhi_epi16(qcoeff, *shift); + tmp1 = _mm_slli_epi16(tmp1, 2); + *coeff = _mm_or_si128(tmp, tmp1); +} + +static INLINE void calculate_dqcoeff_and_store_64x64(const __m128i qcoeff, + const __m128i dequant, + const __m128i zero, + tran_low_t *dqcoeff) { + // Un-sign to bias rounding like C. + const __m128i coeff = _mm_abs_epi16(qcoeff); + + const __m128i sign_0 = _mm_unpacklo_epi16(zero, qcoeff); + const __m128i sign_1 = _mm_unpackhi_epi16(zero, qcoeff); + + const __m128i low = _mm_mullo_epi16(coeff, dequant); + const __m128i high = _mm_mulhi_epi16(coeff, dequant); + __m128i dqcoeff32_0 = _mm_unpacklo_epi16(low, high); + __m128i dqcoeff32_1 = _mm_unpackhi_epi16(low, high); + + // "Divide" by 4. + dqcoeff32_0 = _mm_srli_epi32(dqcoeff32_0, 2); + dqcoeff32_1 = _mm_srli_epi32(dqcoeff32_1, 2); + + dqcoeff32_0 = _mm_sign_epi32(dqcoeff32_0, sign_0); + dqcoeff32_1 = _mm_sign_epi32(dqcoeff32_1, sign_1); + + _mm_store_si128((__m128i *)(dqcoeff), dqcoeff32_0); + _mm_store_si128((__m128i *)(dqcoeff + 4), dqcoeff32_1); +} + +void aom_quantize_b_64x64_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, + const int16_t *zbin_ptr, + const int16_t *round_ptr, + const int16_t *quant_ptr, + const int16_t *quant_shift_ptr, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, + const int16_t *dequant_ptr, uint16_t *eob_ptr, + const int16_t *scan, const int16_t *iscan) { + const __m128i zero = _mm_setzero_si128(); + const __m128i one = _mm_set1_epi16(1); + const __m128i two = _mm_set1_epi16(2); + int index; + + __m128i zbin, round, quant, dequant, shift; + __m128i coeff0, coeff1, qcoeff0, qcoeff1; + __m128i cmp_mask0, cmp_mask1, all_zero; + __m128i eob = zero, eob0; + + (void)scan; + (void)n_coeffs; + + // Setup global values. + zbin = _mm_load_si128((const __m128i *)zbin_ptr); + round = _mm_load_si128((const __m128i *)round_ptr); + quant = _mm_load_si128((const __m128i *)quant_ptr); + dequant = _mm_load_si128((const __m128i *)dequant_ptr); + shift = _mm_load_si128((const __m128i *)quant_shift_ptr); + + // Shift with rounding. + zbin = _mm_add_epi16(zbin, two); + round = _mm_add_epi16(round, two); + zbin = _mm_srli_epi16(zbin, 2); + round = _mm_srli_epi16(round, 2); + zbin = _mm_sub_epi16(zbin, one); + // Do DC and first 15 AC. + coeff0 = load_coefficients(coeff_ptr); + coeff1 = load_coefficients(coeff_ptr + 8); + + qcoeff0 = _mm_abs_epi16(coeff0); + qcoeff1 = _mm_abs_epi16(coeff1); + + cmp_mask0 = _mm_cmpgt_epi16(qcoeff0, zbin); + zbin = _mm_unpackhi_epi64(zbin, zbin); + cmp_mask1 = _mm_cmpgt_epi16(qcoeff1, zbin); + all_zero = _mm_or_si128(cmp_mask0, cmp_mask1); + if (_mm_movemask_epi8(all_zero) == 0) { + _mm_store_si128((__m128i *)(qcoeff_ptr), zero); + _mm_store_si128((__m128i *)(qcoeff_ptr + 4), zero); + _mm_store_si128((__m128i *)(qcoeff_ptr + 8), zero); + _mm_store_si128((__m128i *)(qcoeff_ptr + 12), zero); + _mm_store_si128((__m128i *)(dqcoeff_ptr), zero); + _mm_store_si128((__m128i *)(dqcoeff_ptr + 4), zero); + _mm_store_si128((__m128i *)(dqcoeff_ptr + 8), zero); + _mm_store_si128((__m128i *)(dqcoeff_ptr + 12), zero); + round = _mm_unpackhi_epi64(round, round); + quant = _mm_unpackhi_epi64(quant, quant); + shift = _mm_unpackhi_epi64(shift, shift); + dequant = _mm_unpackhi_epi64(dequant, dequant); + } else { + calculate_qcoeff_64x64(&qcoeff0, round, quant, &shift); + round = _mm_unpackhi_epi64(round, round); + quant = _mm_unpackhi_epi64(quant, quant); + shift = _mm_unpackhi_epi64(shift, shift); + calculate_qcoeff_64x64(&qcoeff1, round, quant, &shift); + + // Reinsert signs. + qcoeff0 = _mm_sign_epi16(qcoeff0, coeff0); + qcoeff1 = _mm_sign_epi16(qcoeff1, coeff1); + + // Mask out zbin threshold coeffs. + qcoeff0 = _mm_and_si128(qcoeff0, cmp_mask0); + qcoeff1 = _mm_and_si128(qcoeff1, cmp_mask1); + + store_coefficients(qcoeff0, qcoeff_ptr); + store_coefficients(qcoeff1, qcoeff_ptr + 8); + + calculate_dqcoeff_and_store_64x64(qcoeff0, dequant, zero, dqcoeff_ptr); + dequant = _mm_unpackhi_epi64(dequant, dequant); + calculate_dqcoeff_and_store_64x64(qcoeff1, dequant, zero, dqcoeff_ptr + 8); + + eob = + scan_for_eob(&qcoeff0, &qcoeff1, cmp_mask0, cmp_mask1, iscan, 0, zero); + } + + // AC only loop. + for (index = 16; index < 1024; index += 16) { + coeff0 = load_coefficients(coeff_ptr + index); + coeff1 = load_coefficients(coeff_ptr + index + 8); + + qcoeff0 = _mm_abs_epi16(coeff0); + qcoeff1 = _mm_abs_epi16(coeff1); + + cmp_mask0 = _mm_cmpgt_epi16(qcoeff0, zbin); + cmp_mask1 = _mm_cmpgt_epi16(qcoeff1, zbin); + + all_zero = _mm_or_si128(cmp_mask0, cmp_mask1); + if (_mm_movemask_epi8(all_zero) == 0) { + _mm_store_si128((__m128i *)(qcoeff_ptr + index), zero); + _mm_store_si128((__m128i *)(qcoeff_ptr + index + 4), zero); + _mm_store_si128((__m128i *)(qcoeff_ptr + index + 8), zero); + _mm_store_si128((__m128i *)(qcoeff_ptr + index + 12), zero); + _mm_store_si128((__m128i *)(dqcoeff_ptr + index), zero); + _mm_store_si128((__m128i *)(dqcoeff_ptr + index + 4), zero); + _mm_store_si128((__m128i *)(dqcoeff_ptr + index + 8), zero); + _mm_store_si128((__m128i *)(dqcoeff_ptr + index + 12), zero); + continue; + } + calculate_qcoeff_64x64(&qcoeff0, round, quant, &shift); + calculate_qcoeff_64x64(&qcoeff1, round, quant, &shift); + + qcoeff0 = _mm_sign_epi16(qcoeff0, coeff0); + qcoeff1 = _mm_sign_epi16(qcoeff1, coeff1); + + qcoeff0 = _mm_and_si128(qcoeff0, cmp_mask0); + qcoeff1 = _mm_and_si128(qcoeff1, cmp_mask1); + + store_coefficients(qcoeff0, qcoeff_ptr + index); + store_coefficients(qcoeff1, qcoeff_ptr + index + 8); + + calculate_dqcoeff_and_store_64x64(qcoeff0, dequant, zero, + dqcoeff_ptr + index); + calculate_dqcoeff_and_store_64x64(qcoeff1, dequant, zero, + dqcoeff_ptr + 8 + index); + + eob0 = scan_for_eob(&qcoeff0, &qcoeff1, cmp_mask0, cmp_mask1, iscan, index, + zero); + eob = _mm_max_epi16(eob, eob0); + } + + *eob_ptr = accumulate_eob(eob); +} diff --git a/third_party/aom/aom_dsp/x86/quantize_ssse3_x86_64.asm b/third_party/aom/aom_dsp/x86/quantize_ssse3_x86_64.asm index 39d4ca674c..fa616a6f1a 100644 --- a/third_party/aom/aom_dsp/x86/quantize_ssse3_x86_64.asm +++ b/third_party/aom/aom_dsp/x86/quantize_ssse3_x86_64.asm @@ -48,9 +48,6 @@ cglobal quantize_%1, 0, %2, 15, coeff, ncoeff, zbin, round, quant, \ mov r3, qcoeffmp mov r4, dqcoeffmp mov r5, iscanmp -%ifidn %1, b_32x32 - psllw m4, 1 -%endif pxor m5, m5 ; m5 = dedicated zero DEFINE_ARGS coeff, ncoeff, d1, qcoeff, dqcoeff, iscan, d2, d3, d4, eob lea coeffq, [ coeffq+ncoeffq*4] @@ -78,9 +75,26 @@ cglobal quantize_%1, 0, %2, 15, coeff, ncoeff, zbin, round, quant, \ pmulhw m13, m11, m2 ; m13 = m11*q>>16 paddw m8, m6 ; m8 += m6 paddw m13, m11 ; m13 += m11 + %ifidn %1, b_32x32 + pmullw m5, m8, m4 ; store the lower 16 bits of m8*qsh + %endif pmulhw m8, m4 ; m8 = m8*qsh>>16 + %ifidn %1, b_32x32 + psllw m8, 1 + psrlw m5, 15 + por m8, m5 + %endif punpckhqdq m4, m4 + %ifidn %1, b_32x32 + pmullw m5, m13, m4 ; store the lower 16 bits of m13*qsh + %endif pmulhw m13, m4 ; m13 = m13*qsh>>16 + %ifidn %1, b_32x32 + psllw m13, 1 + psrlw m5, 15 + por m13, m5 + pxor m5, m5 ; reset m5 to zero register + %endif psignw m8, m9 ; m8 = reinsert sign psignw m13, m10 ; m13 = reinsert sign pand m8, m7 @@ -117,7 +131,7 @@ cglobal quantize_%1, 0, %2, 15, coeff, ncoeff, zbin, round, quant, \ psignw m8, m9 psignw m13, m10 %endif - ; store 16bit numbers as 32bit numbers in array pointed to by qcoeff + ; store 16bit numbers as 32bit numbers in array pointed to by dqcoeff mova m11, m8 mova m6, m8 pcmpgtw m5, m8 @@ -169,12 +183,28 @@ cglobal quantize_%1, 0, %2, 15, coeff, ncoeff, zbin, round, quant, \ pmulhw m13, m11, m2 ; m13 = m11*q>>16 paddw m14, m6 ; m14 += m6 paddw m13, m11 ; m13 += m11 + %ifidn %1, b_32x32 + pmullw m5, m14, m4 ; store the lower 16 bits of m14*qsh + %endif pmulhw m14, m4 ; m14 = m14*qsh>>16 + %ifidn %1, b_32x32 + psllw m14, 1 + psrlw m5, 15 + por m14, m5 + pmullw m5, m13, m4 ; store the lower 16 bits of m13*qsh + %endif pmulhw m13, m4 ; m13 = m13*qsh>>16 + %ifidn %1, b_32x32 + psllw m13, 1 + psrlw m5, 15 + por m13, m5 + pxor m5, m5 ; reset m5 to zero register + %endif psignw m14, m9 ; m14 = reinsert sign psignw m13, m10 ; m13 = reinsert sign pand m14, m7 pand m13, m12 + ; store 16bit numbers as 32bit numbers in array pointed to by qcoeff pxor m11, m11 mova m11, m14 @@ -207,7 +237,7 @@ cglobal quantize_%1, 0, %2, 15, coeff, ncoeff, zbin, round, quant, \ psignw m13, m10 %endif - ; store 16bit numbers as 32bit numbers in array pointed to by qcoeff + ; store 16bit numbers as 32bit numbers in array pointed to by dqcoeff mova m11, m14 mova m6, m14 pcmpgtw m5, m14 diff --git a/third_party/aom/aom_dsp/x86/quantize_x86.h b/third_party/aom/aom_dsp/x86/quantize_x86.h index 4eed7dd29a..3b94bb05e3 100644 --- a/third_party/aom/aom_dsp/x86/quantize_x86.h +++ b/third_party/aom/aom_dsp/x86/quantize_x86.h @@ -75,3 +75,23 @@ static INLINE int16_t accumulate_eob(__m128i eob) { eob = _mm_max_epi16(eob, eob_shuffled); return _mm_extract_epi16(eob, 1); } + +static INLINE __m128i load_coefficients(const tran_low_t *coeff_ptr) { + assert(sizeof(tran_low_t) == 4); + const __m128i coeff1 = _mm_load_si128((__m128i *)(coeff_ptr)); + const __m128i coeff2 = _mm_load_si128((__m128i *)(coeff_ptr + 4)); + return _mm_packs_epi32(coeff1, coeff2); +} + +static INLINE void store_coefficients(__m128i coeff_vals, + tran_low_t *coeff_ptr) { + assert(sizeof(tran_low_t) == 4); + + __m128i one = _mm_set1_epi16(1); + __m128i coeff_vals_hi = _mm_mulhi_epi16(coeff_vals, one); + __m128i coeff_vals_lo = _mm_mullo_epi16(coeff_vals, one); + __m128i coeff_vals_1 = _mm_unpacklo_epi16(coeff_vals_lo, coeff_vals_hi); + __m128i coeff_vals_2 = _mm_unpackhi_epi16(coeff_vals_lo, coeff_vals_hi); + _mm_store_si128((__m128i *)(coeff_ptr), coeff_vals_1); + _mm_store_si128((__m128i *)(coeff_ptr + 4), coeff_vals_2); +} diff --git a/third_party/aom/aom_dsp/x86/sse_avx2.c b/third_party/aom/aom_dsp/x86/sse_avx2.c index 305dde5c08..42df9815cf 100644 --- a/third_party/aom/aom_dsp/x86/sse_avx2.c +++ b/third_party/aom/aom_dsp/x86/sse_avx2.c @@ -21,12 +21,11 @@ static INLINE void sse_w32_avx2(__m256i *sum, const uint8_t *a, const uint8_t *b) { const __m256i v_a0 = yy_loadu_256(a); const __m256i v_b0 = yy_loadu_256(b); - const __m256i v_a00_w = _mm256_cvtepu8_epi16(_mm256_castsi256_si128(v_a0)); - const __m256i v_a01_w = - _mm256_cvtepu8_epi16(_mm256_extracti128_si256(v_a0, 1)); - const __m256i v_b00_w = _mm256_cvtepu8_epi16(_mm256_castsi256_si128(v_b0)); - const __m256i v_b01_w = - _mm256_cvtepu8_epi16(_mm256_extracti128_si256(v_b0, 1)); + const __m256i zero = _mm256_setzero_si256(); + const __m256i v_a00_w = _mm256_unpacklo_epi8(v_a0, zero); + const __m256i v_a01_w = _mm256_unpackhi_epi8(v_a0, zero); + const __m256i v_b00_w = _mm256_unpacklo_epi8(v_b0, zero); + const __m256i v_b01_w = _mm256_unpackhi_epi8(v_b0, zero); const __m256i v_d00_w = _mm256_sub_epi16(v_a00_w, v_b00_w); const __m256i v_d01_w = _mm256_sub_epi16(v_a01_w, v_b01_w); *sum = _mm256_add_epi32(*sum, _mm256_madd_epi16(v_d00_w, v_d00_w)); @@ -35,43 +34,76 @@ static INLINE void sse_w32_avx2(__m256i *sum, const uint8_t *a, static INLINE int64_t summary_all_avx2(const __m256i *sum_all) { int64_t sum; - const __m256i sum0_4x64 = - _mm256_cvtepu32_epi64(_mm256_castsi256_si128(*sum_all)); - const __m256i sum1_4x64 = - _mm256_cvtepu32_epi64(_mm256_extracti128_si256(*sum_all, 1)); + __m256i zero = _mm256_setzero_si256(); + const __m256i sum0_4x64 = _mm256_unpacklo_epi32(*sum_all, zero); + const __m256i sum1_4x64 = _mm256_unpackhi_epi32(*sum_all, zero); const __m256i sum_4x64 = _mm256_add_epi64(sum0_4x64, sum1_4x64); const __m128i sum_2x64 = _mm_add_epi64(_mm256_castsi256_si128(sum_4x64), _mm256_extracti128_si256(sum_4x64, 1)); const __m128i sum_1x64 = _mm_add_epi64(sum_2x64, _mm_srli_si128(sum_2x64, 8)); + xx_storel_64(&sum, sum_1x64); + return sum; +} + +static INLINE void summary_32_avx2(const __m256i *sum32, __m256i *sum) { + const __m256i sum0_4x64 = + _mm256_cvtepu32_epi64(_mm256_castsi256_si128(*sum32)); + const __m256i sum1_4x64 = + _mm256_cvtepu32_epi64(_mm256_extracti128_si256(*sum32, 1)); + const __m256i sum_4x64 = _mm256_add_epi64(sum0_4x64, sum1_4x64); + *sum = _mm256_add_epi64(*sum, sum_4x64); +} + +static INLINE int64_t summary_4x64_avx2(const __m256i sum_4x64) { + int64_t sum; + const __m128i sum_2x64 = _mm_add_epi64(_mm256_castsi256_si128(sum_4x64), + _mm256_extracti128_si256(sum_4x64, 1)); + const __m128i sum_1x64 = _mm_add_epi64(sum_2x64, _mm_srli_si128(sum_2x64, 8)); xx_storel_64(&sum, sum_1x64); return sum; } +static INLINE void sse_w4x4_avx2(const uint8_t *a, int a_stride, + const uint8_t *b, int b_stride, __m256i *sum) { + const __m128i v_a0 = xx_loadl_32(a); + const __m128i v_a1 = xx_loadl_32(a + a_stride); + const __m128i v_a2 = xx_loadl_32(a + a_stride * 2); + const __m128i v_a3 = xx_loadl_32(a + a_stride * 3); + const __m128i v_b0 = xx_loadl_32(b); + const __m128i v_b1 = xx_loadl_32(b + b_stride); + const __m128i v_b2 = xx_loadl_32(b + b_stride * 2); + const __m128i v_b3 = xx_loadl_32(b + b_stride * 3); + const __m128i v_a0123 = _mm_unpacklo_epi64(_mm_unpacklo_epi32(v_a0, v_a1), + _mm_unpacklo_epi32(v_a2, v_a3)); + const __m128i v_b0123 = _mm_unpacklo_epi64(_mm_unpacklo_epi32(v_b0, v_b1), + _mm_unpacklo_epi32(v_b2, v_b3)); + const __m256i v_a_w = _mm256_cvtepu8_epi16(v_a0123); + const __m256i v_b_w = _mm256_cvtepu8_epi16(v_b0123); + const __m256i v_d_w = _mm256_sub_epi16(v_a_w, v_b_w); + *sum = _mm256_add_epi32(*sum, _mm256_madd_epi16(v_d_w, v_d_w)); +} +static INLINE void sse_w8x2_avx2(const uint8_t *a, int a_stride, + const uint8_t *b, int b_stride, __m256i *sum) { + const __m128i v_a0 = xx_loadl_64(a); + const __m128i v_a1 = xx_loadl_64(a + a_stride); + const __m128i v_b0 = xx_loadl_64(b); + const __m128i v_b1 = xx_loadl_64(b + b_stride); + const __m256i v_a_w = _mm256_cvtepu8_epi16(_mm_unpacklo_epi64(v_a0, v_a1)); + const __m256i v_b_w = _mm256_cvtepu8_epi16(_mm_unpacklo_epi64(v_b0, v_b1)); + const __m256i v_d_w = _mm256_sub_epi16(v_a_w, v_b_w); + *sum = _mm256_add_epi32(*sum, _mm256_madd_epi16(v_d_w, v_d_w)); +} int64_t aom_sse_avx2(const uint8_t *a, int a_stride, const uint8_t *b, int b_stride, int width, int height) { int32_t y = 0; int64_t sse = 0; __m256i sum = _mm256_setzero_si256(); + __m256i zero = _mm256_setzero_si256(); switch (width) { case 4: do { - const __m128i v_a0 = xx_loadl_32(a); - const __m128i v_a1 = xx_loadl_32(a + a_stride); - const __m128i v_a2 = xx_loadl_32(a + a_stride * 2); - const __m128i v_a3 = xx_loadl_32(a + a_stride * 3); - const __m128i v_b0 = xx_loadl_32(b); - const __m128i v_b1 = xx_loadl_32(b + b_stride); - const __m128i v_b2 = xx_loadl_32(b + b_stride * 2); - const __m128i v_b3 = xx_loadl_32(b + b_stride * 3); - const __m128i v_a0123 = _mm_unpacklo_epi64( - _mm_unpacklo_epi32(v_a0, v_a1), _mm_unpacklo_epi32(v_a2, v_a3)); - const __m128i v_b0123 = _mm_unpacklo_epi64( - _mm_unpacklo_epi32(v_b0, v_b1), _mm_unpacklo_epi32(v_b2, v_b3)); - const __m256i v_a_w = _mm256_cvtepu8_epi16(v_a0123); - const __m256i v_b_w = _mm256_cvtepu8_epi16(v_b0123); - const __m256i v_d_w = _mm256_sub_epi16(v_a_w, v_b_w); - sum = _mm256_add_epi32(sum, _mm256_madd_epi16(v_d_w, v_d_w)); + sse_w4x4_avx2(a, a_stride, b, b_stride, &sum); a += a_stride << 2; b += b_stride << 2; y += 4; @@ -80,16 +112,7 @@ int64_t aom_sse_avx2(const uint8_t *a, int a_stride, const uint8_t *b, break; case 8: do { - const __m128i v_a0 = xx_loadl_64(a); - const __m128i v_a1 = xx_loadl_64(a + a_stride); - const __m128i v_b0 = xx_loadl_64(b); - const __m128i v_b1 = xx_loadl_64(b + b_stride); - const __m256i v_a_w = - _mm256_cvtepu8_epi16(_mm_unpacklo_epi64(v_a0, v_a1)); - const __m256i v_b_w = - _mm256_cvtepu8_epi16(_mm_unpacklo_epi64(v_b0, v_b1)); - const __m256i v_d_w = _mm256_sub_epi16(v_a_w, v_b_w); - sum = _mm256_add_epi32(sum, _mm256_madd_epi16(v_d_w, v_d_w)); + sse_w8x2_avx2(a, a_stride, b, b_stride, &sum); a += a_stride << 1; b += b_stride << 1; y += 2; @@ -99,14 +122,26 @@ int64_t aom_sse_avx2(const uint8_t *a, int a_stride, const uint8_t *b, case 16: do { const __m128i v_a0 = xx_loadu_128(a); + const __m128i v_a1 = xx_loadu_128(a + a_stride); const __m128i v_b0 = xx_loadu_128(b); - const __m256i v_a_w = _mm256_cvtepu8_epi16(v_a0); - const __m256i v_b_w = _mm256_cvtepu8_epi16(v_b0); - const __m256i v_d_w = _mm256_sub_epi16(v_a_w, v_b_w); - sum = _mm256_add_epi32(sum, _mm256_madd_epi16(v_d_w, v_d_w)); - a += a_stride; - b += b_stride; - y += 1; + const __m128i v_b1 = xx_loadu_128(b + b_stride); + const __m256i v_a = + _mm256_insertf128_si256(_mm256_castsi128_si256(v_a0), v_a1, 0x01); + const __m256i v_b = + _mm256_insertf128_si256(_mm256_castsi128_si256(v_b0), v_b1, 0x01); + const __m256i v_al = _mm256_unpacklo_epi8(v_a, zero); + const __m256i v_au = _mm256_unpackhi_epi8(v_a, zero); + const __m256i v_bl = _mm256_unpacklo_epi8(v_b, zero); + const __m256i v_bu = _mm256_unpackhi_epi8(v_b, zero); + const __m256i v_asub = _mm256_sub_epi16(v_al, v_bl); + const __m256i v_bsub = _mm256_sub_epi16(v_au, v_bu); + const __m256i temp = + _mm256_add_epi32(_mm256_madd_epi16(v_asub, v_asub), + _mm256_madd_epi16(v_bsub, v_bsub)); + sum = _mm256_add_epi32(sum, temp); + a += a_stride << 1; + b += b_stride << 1; + y += 2; } while (y < height); sse = summary_all_avx2(&sum); break; @@ -141,7 +176,36 @@ int64_t aom_sse_avx2(const uint8_t *a, int a_stride, const uint8_t *b, } while (y < height); sse = summary_all_avx2(&sum); break; - default: break; + default: + if ((width & 0x07) == 0) { + do { + int i = 0; + do { + sse_w8x2_avx2(a + i, a_stride, b + i, b_stride, &sum); + i += 8; + } while (i < width); + a += a_stride << 1; + b += b_stride << 1; + y += 2; + } while (y < height); + } else { + do { + int i = 0; + do { + sse_w8x2_avx2(a + i, a_stride, b + i, b_stride, &sum); + const uint8_t *a2 = a + i + (a_stride << 1); + const uint8_t *b2 = b + i + (b_stride << 1); + sse_w8x2_avx2(a2, a_stride, b2, b_stride, &sum); + i += 8; + } while (i + 4 < width); + sse_w4x4_avx2(a + i, a_stride, b + i, b_stride, &sum); + a += a_stride << 2; + b += b_stride << 2; + y += 4; + } while (y < height); + } + sse = summary_all_avx2(&sum); + break; } return sse; @@ -155,6 +219,33 @@ static INLINE void highbd_sse_w16_avx2(__m256i *sum, const uint16_t *a, *sum = _mm256_add_epi32(*sum, _mm256_madd_epi16(v_d_w, v_d_w)); } +static INLINE void highbd_sse_w4x4_avx2(__m256i *sum, const uint16_t *a, + int a_stride, const uint16_t *b, + int b_stride) { + const __m128i v_a0 = xx_loadl_64(a); + const __m128i v_a1 = xx_loadl_64(a + a_stride); + const __m128i v_a2 = xx_loadl_64(a + a_stride * 2); + const __m128i v_a3 = xx_loadl_64(a + a_stride * 3); + const __m128i v_b0 = xx_loadl_64(b); + const __m128i v_b1 = xx_loadl_64(b + b_stride); + const __m128i v_b2 = xx_loadl_64(b + b_stride * 2); + const __m128i v_b3 = xx_loadl_64(b + b_stride * 3); + const __m256i v_a_w = yy_set_m128i(_mm_unpacklo_epi64(v_a0, v_a1), + _mm_unpacklo_epi64(v_a2, v_a3)); + const __m256i v_b_w = yy_set_m128i(_mm_unpacklo_epi64(v_b0, v_b1), + _mm_unpacklo_epi64(v_b2, v_b3)); + const __m256i v_d_w = _mm256_sub_epi16(v_a_w, v_b_w); + *sum = _mm256_add_epi32(*sum, _mm256_madd_epi16(v_d_w, v_d_w)); +} + +static INLINE void highbd_sse_w8x2_avx2(__m256i *sum, const uint16_t *a, + int a_stride, const uint16_t *b, + int b_stride) { + const __m256i v_a_w = yy_loadu2_128(a + a_stride, a); + const __m256i v_b_w = yy_loadu2_128(b + b_stride, b); + const __m256i v_d_w = _mm256_sub_epi16(v_a_w, v_b_w); + *sum = _mm256_add_epi32(*sum, _mm256_madd_epi16(v_d_w, v_d_w)); +} int64_t aom_highbd_sse_avx2(const uint8_t *a8, int a_stride, const uint8_t *b8, int b_stride, int width, int height) { int32_t y = 0; @@ -165,20 +256,7 @@ int64_t aom_highbd_sse_avx2(const uint8_t *a8, int a_stride, const uint8_t *b8, switch (width) { case 4: do { - const __m128i v_a0 = xx_loadl_64(a); - const __m128i v_a1 = xx_loadl_64(a + a_stride); - const __m128i v_a2 = xx_loadl_64(a + a_stride * 2); - const __m128i v_a3 = xx_loadl_64(a + a_stride * 3); - const __m128i v_b0 = xx_loadl_64(b); - const __m128i v_b1 = xx_loadl_64(b + b_stride); - const __m128i v_b2 = xx_loadl_64(b + b_stride * 2); - const __m128i v_b3 = xx_loadl_64(b + b_stride * 3); - const __m256i v_a_w = yy_set_m128i(_mm_unpacklo_epi64(v_a0, v_a1), - _mm_unpacklo_epi64(v_a2, v_a3)); - const __m256i v_b_w = yy_set_m128i(_mm_unpacklo_epi64(v_b0, v_b1), - _mm_unpacklo_epi64(v_b2, v_b3)); - const __m256i v_d_w = _mm256_sub_epi16(v_a_w, v_b_w); - sum = _mm256_add_epi32(sum, _mm256_madd_epi16(v_d_w, v_d_w)); + highbd_sse_w4x4_avx2(&sum, a, a_stride, b, b_stride); a += a_stride << 2; b += b_stride << 2; y += 4; @@ -187,10 +265,7 @@ int64_t aom_highbd_sse_avx2(const uint8_t *a8, int a_stride, const uint8_t *b8, break; case 8: do { - const __m256i v_a_w = yy_loadu2_128(a + a_stride, a); - const __m256i v_b_w = yy_loadu2_128(b + b_stride, b); - const __m256i v_d_w = _mm256_sub_epi16(v_a_w, v_b_w); - sum = _mm256_add_epi32(sum, _mm256_madd_epi16(v_d_w, v_d_w)); + highbd_sse_w8x2_avx2(&sum, a, a_stride, b, b_stride); a += a_stride << 1; b += b_stride << 1; y += 2; @@ -208,43 +283,98 @@ int64_t aom_highbd_sse_avx2(const uint8_t *a8, int a_stride, const uint8_t *b8, break; case 32: do { - highbd_sse_w16_avx2(&sum, a, b); - highbd_sse_w16_avx2(&sum, a + 16, b + 16); - a += a_stride; - b += b_stride; - y += 1; + int l = 0; + __m256i sum32 = _mm256_setzero_si256(); + do { + highbd_sse_w16_avx2(&sum32, a, b); + highbd_sse_w16_avx2(&sum32, a + 16, b + 16); + a += a_stride; + b += b_stride; + l += 1; + } while (l < 64 && l < (height - y)); + summary_32_avx2(&sum32, &sum); + y += 64; } while (y < height); - sse = summary_all_avx2(&sum); + sse = summary_4x64_avx2(sum); break; case 64: do { - highbd_sse_w16_avx2(&sum, a, b); - highbd_sse_w16_avx2(&sum, a + 16 * 1, b + 16 * 1); - highbd_sse_w16_avx2(&sum, a + 16 * 2, b + 16 * 2); - highbd_sse_w16_avx2(&sum, a + 16 * 3, b + 16 * 3); - a += a_stride; - b += b_stride; - y += 1; + int l = 0; + __m256i sum32 = _mm256_setzero_si256(); + do { + highbd_sse_w16_avx2(&sum32, a, b); + highbd_sse_w16_avx2(&sum32, a + 16 * 1, b + 16 * 1); + highbd_sse_w16_avx2(&sum32, a + 16 * 2, b + 16 * 2); + highbd_sse_w16_avx2(&sum32, a + 16 * 3, b + 16 * 3); + a += a_stride; + b += b_stride; + l += 1; + } while (l < 32 && l < (height - y)); + summary_32_avx2(&sum32, &sum); + y += 32; } while (y < height); - sse = summary_all_avx2(&sum); + sse = summary_4x64_avx2(sum); break; case 128: do { - highbd_sse_w16_avx2(&sum, a, b); - highbd_sse_w16_avx2(&sum, a + 16 * 1, b + 16 * 1); - highbd_sse_w16_avx2(&sum, a + 16 * 2, b + 16 * 2); - highbd_sse_w16_avx2(&sum, a + 16 * 3, b + 16 * 3); - highbd_sse_w16_avx2(&sum, a + 16 * 4, b + 16 * 4); - highbd_sse_w16_avx2(&sum, a + 16 * 5, b + 16 * 5); - highbd_sse_w16_avx2(&sum, a + 16 * 6, b + 16 * 6); - highbd_sse_w16_avx2(&sum, a + 16 * 7, b + 16 * 7); - a += a_stride; - b += b_stride; - y += 1; + int l = 0; + __m256i sum32 = _mm256_setzero_si256(); + do { + highbd_sse_w16_avx2(&sum32, a, b); + highbd_sse_w16_avx2(&sum32, a + 16 * 1, b + 16 * 1); + highbd_sse_w16_avx2(&sum32, a + 16 * 2, b + 16 * 2); + highbd_sse_w16_avx2(&sum32, a + 16 * 3, b + 16 * 3); + highbd_sse_w16_avx2(&sum32, a + 16 * 4, b + 16 * 4); + highbd_sse_w16_avx2(&sum32, a + 16 * 5, b + 16 * 5); + highbd_sse_w16_avx2(&sum32, a + 16 * 6, b + 16 * 6); + highbd_sse_w16_avx2(&sum32, a + 16 * 7, b + 16 * 7); + a += a_stride; + b += b_stride; + l += 1; + } while (l < 16 && l < (height - y)); + summary_32_avx2(&sum32, &sum); + y += 16; } while (y < height); - sse = summary_all_avx2(&sum); + sse = summary_4x64_avx2(sum); + break; + default: + if (width & 0x7) { + do { + int i = 0; + __m256i sum32 = _mm256_setzero_si256(); + do { + highbd_sse_w8x2_avx2(&sum32, a + i, a_stride, b + i, b_stride); + const uint16_t *a2 = a + i + (a_stride << 1); + const uint16_t *b2 = b + i + (b_stride << 1); + highbd_sse_w8x2_avx2(&sum32, a2, a_stride, b2, b_stride); + i += 8; + } while (i + 4 < width); + highbd_sse_w4x4_avx2(&sum32, a + i, a_stride, b + i, b_stride); + summary_32_avx2(&sum32, &sum); + a += a_stride << 2; + b += b_stride << 2; + y += 4; + } while (y < height); + } else { + do { + int l = 0; + __m256i sum32 = _mm256_setzero_si256(); + do { + int i = 0; + do { + highbd_sse_w8x2_avx2(&sum32, a + i, a_stride, b + i, b_stride); + i += 8; + } while (i < width); + a += a_stride << 1; + b += b_stride << 1; + l += 2; + } while (l < 8 && l < (height - y)); + summary_32_avx2(&sum32, &sum); + y += 8; + } while (y < height); + } + sse = summary_4x64_avx2(sum); break; - default: break; } return sse; } diff --git a/third_party/aom/aom_dsp/x86/sse_sse4.c b/third_party/aom/aom_dsp/x86/sse_sse4.c index 8b5af84691..0d45003e92 100644 --- a/third_party/aom/aom_dsp/x86/sse_sse4.c +++ b/third_party/aom/aom_dsp/x86/sse_sse4.c @@ -28,6 +28,13 @@ static INLINE int64_t summary_all_sse4(const __m128i *sum_all) { return sum; } +static INLINE void summary_32_sse4(const __m128i *sum32, __m128i *sum64) { + const __m128i sum0 = _mm_cvtepu32_epi64(*sum32); + const __m128i sum1 = _mm_cvtepu32_epi64(_mm_srli_si128(*sum32, 8)); + *sum64 = _mm_add_epi64(sum0, *sum64); + *sum64 = _mm_add_epi64(sum1, *sum64); +} + static INLINE void sse_w16_sse4_1(__m128i *sum, const uint8_t *a, const uint8_t *b) { const __m128i v_a0 = xx_loadu_128(a); @@ -42,6 +49,28 @@ static INLINE void sse_w16_sse4_1(__m128i *sum, const uint8_t *a, *sum = _mm_add_epi32(*sum, _mm_madd_epi16(v_d01_w, v_d01_w)); } +static INLINE void aom_sse4x2_sse4_1(const uint8_t *a, int a_stride, + const uint8_t *b, int b_stride, + __m128i *sum) { + const __m128i v_a0 = xx_loadl_32(a); + const __m128i v_a1 = xx_loadl_32(a + a_stride); + const __m128i v_b0 = xx_loadl_32(b); + const __m128i v_b1 = xx_loadl_32(b + b_stride); + const __m128i v_a_w = _mm_cvtepu8_epi16(_mm_unpacklo_epi32(v_a0, v_a1)); + const __m128i v_b_w = _mm_cvtepu8_epi16(_mm_unpacklo_epi32(v_b0, v_b1)); + const __m128i v_d_w = _mm_sub_epi16(v_a_w, v_b_w); + *sum = _mm_add_epi32(*sum, _mm_madd_epi16(v_d_w, v_d_w)); +} +static INLINE void aom_sse8_sse4_1(const uint8_t *a, const uint8_t *b, + __m128i *sum) { + const __m128i v_a0 = xx_loadl_64(a); + const __m128i v_b0 = xx_loadl_64(b); + const __m128i v_a_w = _mm_cvtepu8_epi16(v_a0); + const __m128i v_b_w = _mm_cvtepu8_epi16(v_b0); + const __m128i v_d_w = _mm_sub_epi16(v_a_w, v_b_w); + *sum = _mm_add_epi32(*sum, _mm_madd_epi16(v_d_w, v_d_w)); +} + int64_t aom_sse_sse4_1(const uint8_t *a, int a_stride, const uint8_t *b, int b_stride, int width, int height) { int y = 0; @@ -50,14 +79,7 @@ int64_t aom_sse_sse4_1(const uint8_t *a, int a_stride, const uint8_t *b, switch (width) { case 4: do { - const __m128i v_a0 = xx_loadl_32(a); - const __m128i v_a1 = xx_loadl_32(a + a_stride); - const __m128i v_b0 = xx_loadl_32(b); - const __m128i v_b1 = xx_loadl_32(b + b_stride); - const __m128i v_a_w = _mm_cvtepu8_epi16(_mm_unpacklo_epi32(v_a0, v_a1)); - const __m128i v_b_w = _mm_cvtepu8_epi16(_mm_unpacklo_epi32(v_b0, v_b1)); - const __m128i v_d_w = _mm_sub_epi16(v_a_w, v_b_w); - sum = _mm_add_epi32(sum, _mm_madd_epi16(v_d_w, v_d_w)); + aom_sse4x2_sse4_1(a, a_stride, b, b_stride, &sum); a += a_stride << 1; b += b_stride << 1; y += 2; @@ -66,12 +88,7 @@ int64_t aom_sse_sse4_1(const uint8_t *a, int a_stride, const uint8_t *b, break; case 8: do { - const __m128i v_a0 = xx_loadl_64(a); - const __m128i v_b0 = xx_loadl_64(b); - const __m128i v_a_w = _mm_cvtepu8_epi16(v_a0); - const __m128i v_b_w = _mm_cvtepu8_epi16(v_b0); - const __m128i v_d_w = _mm_sub_epi16(v_a_w, v_b_w); - sum = _mm_add_epi32(sum, _mm_madd_epi16(v_d_w, v_d_w)); + aom_sse8_sse4_1(a, b, &sum); a += a_stride; b += b_stride; y += 1; @@ -125,12 +142,52 @@ int64_t aom_sse_sse4_1(const uint8_t *a, int a_stride, const uint8_t *b, } while (y < height); sse = summary_all_sse4(&sum); break; - default: break; + default: + if (width & 0x07) { + do { + int i = 0; + do { + aom_sse8_sse4_1(a + i, b + i, &sum); + aom_sse8_sse4_1(a + i + a_stride, b + i + b_stride, &sum); + i += 8; + } while (i + 4 < width); + aom_sse4x2_sse4_1(a + i, a_stride, b + i, b_stride, &sum); + a += (a_stride << 1); + b += (b_stride << 1); + y += 2; + } while (y < height); + } else { + do { + int i = 0; + do { + aom_sse8_sse4_1(a + i, b + i, &sum); + i += 8; + } while (i < width); + a += a_stride; + b += b_stride; + y += 1; + } while (y < height); + } + sse = summary_all_sse4(&sum); + break; } return sse; } +static INLINE void highbd_sse_w4x2_sse4_1(__m128i *sum, const uint16_t *a, + int a_stride, const uint16_t *b, + int b_stride) { + const __m128i v_a0 = xx_loadl_64(a); + const __m128i v_a1 = xx_loadl_64(a + a_stride); + const __m128i v_b0 = xx_loadl_64(b); + const __m128i v_b1 = xx_loadl_64(b + b_stride); + const __m128i v_a_w = _mm_unpacklo_epi64(v_a0, v_a1); + const __m128i v_b_w = _mm_unpacklo_epi64(v_b0, v_b1); + const __m128i v_d_w = _mm_sub_epi16(v_a_w, v_b_w); + *sum = _mm_add_epi32(*sum, _mm_madd_epi16(v_d_w, v_d_w)); +} + static INLINE void highbd_sse_w8_sse4_1(__m128i *sum, const uint16_t *a, const uint16_t *b) { const __m128i v_a_w = xx_loadu_128(a); @@ -150,14 +207,7 @@ int64_t aom_highbd_sse_sse4_1(const uint8_t *a8, int a_stride, switch (width) { case 4: do { - const __m128i v_a0 = xx_loadl_64(a); - const __m128i v_a1 = xx_loadl_64(a + a_stride); - const __m128i v_b0 = xx_loadl_64(b); - const __m128i v_b1 = xx_loadl_64(b + b_stride); - const __m128i v_a_w = _mm_unpacklo_epi64(v_a0, v_a1); - const __m128i v_b_w = _mm_unpacklo_epi64(v_b0, v_b1); - const __m128i v_d_w = _mm_sub_epi16(v_a_w, v_b_w); - sum = _mm_add_epi32(sum, _mm_madd_epi16(v_d_w, v_d_w)); + highbd_sse_w4x2_sse4_1(&sum, a, a_stride, b, b_stride); a += a_stride << 1; b += b_stride << 1; y += 2; @@ -175,67 +225,126 @@ int64_t aom_highbd_sse_sse4_1(const uint8_t *a8, int a_stride, break; case 16: do { - highbd_sse_w8_sse4_1(&sum, a, b); - highbd_sse_w8_sse4_1(&sum, a + 8, b + 8); - a += a_stride; - b += b_stride; - y += 1; + int l = 0; + __m128i sum32 = _mm_setzero_si128(); + do { + highbd_sse_w8_sse4_1(&sum32, a, b); + highbd_sse_w8_sse4_1(&sum32, a + 8, b + 8); + a += a_stride; + b += b_stride; + l += 1; + } while (l < 64 && l < (height - y)); + summary_32_sse4(&sum32, &sum); + y += 64; } while (y < height); - sse = summary_all_sse4(&sum); + xx_storel_64(&sse, _mm_add_epi64(sum, _mm_srli_si128(sum, 8))); break; case 32: do { - highbd_sse_w8_sse4_1(&sum, a, b); - highbd_sse_w8_sse4_1(&sum, a + 8 * 1, b + 8 * 1); - highbd_sse_w8_sse4_1(&sum, a + 8 * 2, b + 8 * 2); - highbd_sse_w8_sse4_1(&sum, a + 8 * 3, b + 8 * 3); - a += a_stride; - b += b_stride; - y += 1; + int l = 0; + __m128i sum32 = _mm_setzero_si128(); + do { + highbd_sse_w8_sse4_1(&sum32, a, b); + highbd_sse_w8_sse4_1(&sum32, a + 8 * 1, b + 8 * 1); + highbd_sse_w8_sse4_1(&sum32, a + 8 * 2, b + 8 * 2); + highbd_sse_w8_sse4_1(&sum32, a + 8 * 3, b + 8 * 3); + a += a_stride; + b += b_stride; + l += 1; + } while (l < 32 && l < (height - y)); + summary_32_sse4(&sum32, &sum); + y += 32; } while (y < height); - sse = summary_all_sse4(&sum); + xx_storel_64(&sse, _mm_add_epi64(sum, _mm_srli_si128(sum, 8))); break; case 64: do { - highbd_sse_w8_sse4_1(&sum, a, b); - highbd_sse_w8_sse4_1(&sum, a + 8 * 1, b + 8 * 1); - highbd_sse_w8_sse4_1(&sum, a + 8 * 2, b + 8 * 2); - highbd_sse_w8_sse4_1(&sum, a + 8 * 3, b + 8 * 3); - highbd_sse_w8_sse4_1(&sum, a + 8 * 4, b + 8 * 4); - highbd_sse_w8_sse4_1(&sum, a + 8 * 5, b + 8 * 5); - highbd_sse_w8_sse4_1(&sum, a + 8 * 6, b + 8 * 6); - highbd_sse_w8_sse4_1(&sum, a + 8 * 7, b + 8 * 7); - a += a_stride; - b += b_stride; - y += 1; + int l = 0; + __m128i sum32 = _mm_setzero_si128(); + do { + highbd_sse_w8_sse4_1(&sum32, a, b); + highbd_sse_w8_sse4_1(&sum32, a + 8 * 1, b + 8 * 1); + highbd_sse_w8_sse4_1(&sum32, a + 8 * 2, b + 8 * 2); + highbd_sse_w8_sse4_1(&sum32, a + 8 * 3, b + 8 * 3); + highbd_sse_w8_sse4_1(&sum32, a + 8 * 4, b + 8 * 4); + highbd_sse_w8_sse4_1(&sum32, a + 8 * 5, b + 8 * 5); + highbd_sse_w8_sse4_1(&sum32, a + 8 * 6, b + 8 * 6); + highbd_sse_w8_sse4_1(&sum32, a + 8 * 7, b + 8 * 7); + a += a_stride; + b += b_stride; + l += 1; + } while (l < 16 && l < (height - y)); + summary_32_sse4(&sum32, &sum); + y += 16; } while (y < height); - sse = summary_all_sse4(&sum); + xx_storel_64(&sse, _mm_add_epi64(sum, _mm_srli_si128(sum, 8))); break; case 128: do { - highbd_sse_w8_sse4_1(&sum, a, b); - highbd_sse_w8_sse4_1(&sum, a + 8 * 1, b + 8 * 1); - highbd_sse_w8_sse4_1(&sum, a + 8 * 2, b + 8 * 2); - highbd_sse_w8_sse4_1(&sum, a + 8 * 3, b + 8 * 3); - highbd_sse_w8_sse4_1(&sum, a + 8 * 4, b + 8 * 4); - highbd_sse_w8_sse4_1(&sum, a + 8 * 5, b + 8 * 5); - highbd_sse_w8_sse4_1(&sum, a + 8 * 6, b + 8 * 6); - highbd_sse_w8_sse4_1(&sum, a + 8 * 7, b + 8 * 7); - highbd_sse_w8_sse4_1(&sum, a + 8 * 8, b + 8 * 8); - highbd_sse_w8_sse4_1(&sum, a + 8 * 9, b + 8 * 9); - highbd_sse_w8_sse4_1(&sum, a + 8 * 10, b + 8 * 10); - highbd_sse_w8_sse4_1(&sum, a + 8 * 11, b + 8 * 11); - highbd_sse_w8_sse4_1(&sum, a + 8 * 12, b + 8 * 12); - highbd_sse_w8_sse4_1(&sum, a + 8 * 13, b + 8 * 13); - highbd_sse_w8_sse4_1(&sum, a + 8 * 14, b + 8 * 14); - highbd_sse_w8_sse4_1(&sum, a + 8 * 15, b + 8 * 15); - a += a_stride; - b += b_stride; - y += 1; + int l = 0; + __m128i sum32 = _mm_setzero_si128(); + do { + highbd_sse_w8_sse4_1(&sum32, a, b); + highbd_sse_w8_sse4_1(&sum32, a + 8 * 1, b + 8 * 1); + highbd_sse_w8_sse4_1(&sum32, a + 8 * 2, b + 8 * 2); + highbd_sse_w8_sse4_1(&sum32, a + 8 * 3, b + 8 * 3); + highbd_sse_w8_sse4_1(&sum32, a + 8 * 4, b + 8 * 4); + highbd_sse_w8_sse4_1(&sum32, a + 8 * 5, b + 8 * 5); + highbd_sse_w8_sse4_1(&sum32, a + 8 * 6, b + 8 * 6); + highbd_sse_w8_sse4_1(&sum32, a + 8 * 7, b + 8 * 7); + highbd_sse_w8_sse4_1(&sum32, a + 8 * 8, b + 8 * 8); + highbd_sse_w8_sse4_1(&sum32, a + 8 * 9, b + 8 * 9); + highbd_sse_w8_sse4_1(&sum32, a + 8 * 10, b + 8 * 10); + highbd_sse_w8_sse4_1(&sum32, a + 8 * 11, b + 8 * 11); + highbd_sse_w8_sse4_1(&sum32, a + 8 * 12, b + 8 * 12); + highbd_sse_w8_sse4_1(&sum32, a + 8 * 13, b + 8 * 13); + highbd_sse_w8_sse4_1(&sum32, a + 8 * 14, b + 8 * 14); + highbd_sse_w8_sse4_1(&sum32, a + 8 * 15, b + 8 * 15); + a += a_stride; + b += b_stride; + l += 1; + } while (l < 8 && l < (height - y)); + summary_32_sse4(&sum32, &sum); + y += 8; } while (y < height); - sse = summary_all_sse4(&sum); + xx_storel_64(&sse, _mm_add_epi64(sum, _mm_srli_si128(sum, 8))); + break; + default: + if (width & 0x7) { + do { + __m128i sum32 = _mm_setzero_si128(); + int i = 0; + do { + highbd_sse_w8_sse4_1(&sum32, a + i, b + i); + highbd_sse_w8_sse4_1(&sum32, a + i + a_stride, b + i + b_stride); + i += 8; + } while (i + 4 < width); + highbd_sse_w4x2_sse4_1(&sum32, a + i, a_stride, b + i, b_stride); + a += (a_stride << 1); + b += (b_stride << 1); + y += 2; + summary_32_sse4(&sum32, &sum); + } while (y < height); + } else { + do { + int l = 0; + __m128i sum32 = _mm_setzero_si128(); + do { + int i = 0; + do { + highbd_sse_w8_sse4_1(&sum32, a + i, b + i); + i += 8; + } while (i < width); + a += a_stride; + b += b_stride; + l += 1; + } while (l < 8 && l < (height - y)); + summary_32_sse4(&sum32, &sum); + y += 8; + } while (y < height); + } + xx_storel_64(&sse, _mm_add_epi64(sum, _mm_srli_si128(sum, 8))); break; - default: break; } return sse; } diff --git a/third_party/aom/aom_dsp/x86/ssim_sse2_x86_64.asm b/third_party/aom/aom_dsp/x86/ssim_sse2_x86_64.asm new file mode 100644 index 0000000000..6d9b5a12f1 --- /dev/null +++ b/third_party/aom/aom_dsp/x86/ssim_sse2_x86_64.asm @@ -0,0 +1,222 @@ +; +; Copyright (c) 2016, Alliance for Open Media. All rights reserved +; +; This source code is subject to the terms of the BSD 2 Clause License and +; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +; was not distributed with this source code in the LICENSE file, you can +; obtain it at www.aomedia.org/license/software. If the Alliance for Open +; Media Patent License 1.0 was not distributed with this source code in the +; PATENTS file, you can obtain it at www.aomedia.org/license/patent. +; + +; + +%include "aom_ports/x86_abi_support.asm" + +; tabulate_ssim - sums sum_s,sum_r,sum_sq_s,sum_sq_r, sum_sxr +%macro TABULATE_SSIM 0 + paddusw xmm15, xmm3 ; sum_s + paddusw xmm14, xmm4 ; sum_r + movdqa xmm1, xmm3 + pmaddwd xmm1, xmm1 + paddd xmm13, xmm1 ; sum_sq_s + movdqa xmm2, xmm4 + pmaddwd xmm2, xmm2 + paddd xmm12, xmm2 ; sum_sq_r + pmaddwd xmm3, xmm4 + paddd xmm11, xmm3 ; sum_sxr +%endmacro + +; Sum across the register %1 starting with q words +%macro SUM_ACROSS_Q 1 + movdqa xmm2,%1 + punpckldq %1,xmm0 + punpckhdq xmm2,xmm0 + paddq %1,xmm2 + movdqa xmm2,%1 + punpcklqdq %1,xmm0 + punpckhqdq xmm2,xmm0 + paddq %1,xmm2 +%endmacro + +; Sum across the register %1 starting with q words +%macro SUM_ACROSS_W 1 + movdqa xmm1, %1 + punpcklwd %1,xmm0 + punpckhwd xmm1,xmm0 + paddd %1, xmm1 + SUM_ACROSS_Q %1 +%endmacro + +SECTION .text + +;void ssim_parms_sse2( +; unsigned char *s, +; int sp, +; unsigned char *r, +; int rp +; uint32_t *sum_s, +; uint32_t *sum_r, +; uint32_t *sum_sq_s, +; uint32_t *sum_sq_r, +; uint32_t *sum_sxr); +; +; TODO: Use parm passing through structure, probably don't need the pxors +; ( calling app will initialize to 0 ) could easily fit everything in sse2 +; without too much hastle, and can probably do better estimates with psadw +; or pavgb At this point this is just meant to be first pass for calculating +; all the parms needed for 16x16 ssim so we can play with dssim as distortion +; in mode selection code. +global sym(aom_ssim_parms_16x16_sse2) PRIVATE +sym(aom_ssim_parms_16x16_sse2): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 9 + SAVE_XMM 15 + push rsi + push rdi + ; end prolog + + mov rsi, arg(0) ;s + mov rcx, arg(1) ;sp + mov rdi, arg(2) ;r + mov rax, arg(3) ;rp + + pxor xmm0, xmm0 + pxor xmm15,xmm15 ;sum_s + pxor xmm14,xmm14 ;sum_r + pxor xmm13,xmm13 ;sum_sq_s + pxor xmm12,xmm12 ;sum_sq_r + pxor xmm11,xmm11 ;sum_sxr + + mov rdx, 16 ;row counter +.NextRow: + + ;grab source and reference pixels + movdqu xmm5, [rsi] + movdqu xmm6, [rdi] + movdqa xmm3, xmm5 + movdqa xmm4, xmm6 + punpckhbw xmm3, xmm0 ; high_s + punpckhbw xmm4, xmm0 ; high_r + + TABULATE_SSIM + + movdqa xmm3, xmm5 + movdqa xmm4, xmm6 + punpcklbw xmm3, xmm0 ; low_s + punpcklbw xmm4, xmm0 ; low_r + + TABULATE_SSIM + + add rsi, rcx ; next s row + add rdi, rax ; next r row + + dec rdx ; counter + jnz .NextRow + + SUM_ACROSS_W xmm15 + SUM_ACROSS_W xmm14 + SUM_ACROSS_Q xmm13 + SUM_ACROSS_Q xmm12 + SUM_ACROSS_Q xmm11 + + mov rdi,arg(4) + movd [rdi], xmm15; + mov rdi,arg(5) + movd [rdi], xmm14; + mov rdi,arg(6) + movd [rdi], xmm13; + mov rdi,arg(7) + movd [rdi], xmm12; + mov rdi,arg(8) + movd [rdi], xmm11; + + ; begin epilog + pop rdi + pop rsi + RESTORE_XMM + UNSHADOW_ARGS + pop rbp + ret + +;void ssim_parms_sse2( +; unsigned char *s, +; int sp, +; unsigned char *r, +; int rp +; uint32_t *sum_s, +; uint32_t *sum_r, +; uint32_t *sum_sq_s, +; uint32_t *sum_sq_r, +; uint32_t *sum_sxr); +; +; TODO: Use parm passing through structure, probably don't need the pxors +; ( calling app will initialize to 0 ) could easily fit everything in sse2 +; without too much hastle, and can probably do better estimates with psadw +; or pavgb At this point this is just meant to be first pass for calculating +; all the parms needed for 16x16 ssim so we can play with dssim as distortion +; in mode selection code. +global sym(aom_ssim_parms_8x8_sse2) PRIVATE +sym(aom_ssim_parms_8x8_sse2): + push rbp + mov rbp, rsp + SHADOW_ARGS_TO_STACK 9 + SAVE_XMM 15 + push rsi + push rdi + ; end prolog + + mov rsi, arg(0) ;s + mov rcx, arg(1) ;sp + mov rdi, arg(2) ;r + mov rax, arg(3) ;rp + + pxor xmm0, xmm0 + pxor xmm15,xmm15 ;sum_s + pxor xmm14,xmm14 ;sum_r + pxor xmm13,xmm13 ;sum_sq_s + pxor xmm12,xmm12 ;sum_sq_r + pxor xmm11,xmm11 ;sum_sxr + + mov rdx, 8 ;row counter +.NextRow: + + ;grab source and reference pixels + movq xmm3, [rsi] + movq xmm4, [rdi] + punpcklbw xmm3, xmm0 ; low_s + punpcklbw xmm4, xmm0 ; low_r + + TABULATE_SSIM + + add rsi, rcx ; next s row + add rdi, rax ; next r row + + dec rdx ; counter + jnz .NextRow + + SUM_ACROSS_W xmm15 + SUM_ACROSS_W xmm14 + SUM_ACROSS_Q xmm13 + SUM_ACROSS_Q xmm12 + SUM_ACROSS_Q xmm11 + + mov rdi,arg(4) + movd [rdi], xmm15; + mov rdi,arg(5) + movd [rdi], xmm14; + mov rdi,arg(6) + movd [rdi], xmm13; + mov rdi,arg(7) + movd [rdi], xmm12; + mov rdi,arg(8) + movd [rdi], xmm11; + + ; begin epilog + pop rdi + pop rsi + RESTORE_XMM + UNSHADOW_ARGS + pop rbp + ret diff --git a/third_party/aom/aom_dsp/x86/subpel_variance_sse2.asm b/third_party/aom/aom_dsp/x86/subpel_variance_sse2.asm index 45bf6ec3c5..cbf28901be 100644 --- a/third_party/aom/aom_dsp/x86/subpel_variance_sse2.asm +++ b/third_party/aom/aom_dsp/x86/subpel_variance_sse2.asm @@ -135,44 +135,33 @@ SECTION .text %if %2 == 1 ; avg cglobal sub_pixel_avg_variance%1xh, 7, 7, 13, src, src_stride, \ x_offset, y_offset, dst, dst_stride, \ - sec, sec_stride, height, sse, \ - g_bilin_filter, g_pw_8 + sec, sec_stride, height, sse %define block_height dword heightm %define sec_str sec_stridemp - - ;Store bilin_filter and pw_8 location in stack - %if GET_GOT_DEFINED == 1 - GET_GOT eax - add esp, 4 ; restore esp - %endif - - lea ecx, [GLOBAL(bilin_filter_m)] - mov g_bilin_filterm, ecx - - lea ecx, [GLOBAL(pw_8)] - mov g_pw_8m, ecx - - LOAD_IF_USED 0, 1 ; load eax, ecx back %else cglobal sub_pixel_variance%1xh, 7, 7, 13, src, src_stride, \ x_offset, y_offset, dst, dst_stride, \ - height, sse, g_bilin_filter, g_pw_8 + height, sse %define block_height heightd - - ;Store bilin_filter and pw_8 location in stack - %if GET_GOT_DEFINED == 1 - GET_GOT eax - add esp, 4 ; restore esp - %endif - - lea ecx, [GLOBAL(bilin_filter_m)] - mov g_bilin_filterm, ecx - - lea ecx, [GLOBAL(pw_8)] - mov g_pw_8m, ecx - - LOAD_IF_USED 0, 1 ; load eax, ecx back %endif + + ; reuse argument stack space + %define g_bilin_filterm x_offsetm + %define g_pw_8m y_offsetm + + ;Store bilin_filter and pw_8 location in stack + %if GET_GOT_DEFINED == 1 + GET_GOT eax + add esp, 4 ; restore esp + %endif + + lea ecx, [GLOBAL(bilin_filter_m)] + mov g_bilin_filterm, ecx + + lea ecx, [GLOBAL(pw_8)] + mov g_pw_8m, ecx + + LOAD_IF_USED 0, 1 ; load eax, ecx back %else %if %2 == 1 ; avg cglobal sub_pixel_avg_variance%1xh, 7, 7, 13, src, src_stride, \ diff --git a/third_party/aom/aom_dsp/x86/synonyms.h b/third_party/aom/aom_dsp/x86/synonyms.h index 1e9f1e27b8..2e99bee3e9 100644 --- a/third_party/aom/aom_dsp/x86/synonyms.h +++ b/third_party/aom/aom_dsp/x86/synonyms.h @@ -13,6 +13,7 @@ #define AOM_AOM_DSP_X86_SYNONYMS_H_ #include +#include #include "config/aom_config.h" @@ -28,7 +29,9 @@ // Loads and stores to do away with the tedium of casting the address // to the right type. static INLINE __m128i xx_loadl_32(const void *a) { - return _mm_cvtsi32_si128(*(const uint32_t *)a); + int val; + memcpy(&val, a, sizeof(val)); + return _mm_cvtsi32_si128(val); } static INLINE __m128i xx_loadl_64(const void *a) { @@ -44,7 +47,8 @@ static INLINE __m128i xx_loadu_128(const void *a) { } static INLINE void xx_storel_32(void *const a, const __m128i v) { - *(uint32_t *)a = _mm_cvtsi128_si32(v); + const int val = _mm_cvtsi128_si32(v); + memcpy(a, &val, sizeof(val)); } static INLINE void xx_storel_64(void *const a, const __m128i v) { diff --git a/third_party/aom/aom_dsp/x86/synonyms_avx2.h b/third_party/aom/aom_dsp/x86/synonyms_avx2.h index 3f69b120ea..4d6ee6ad64 100644 --- a/third_party/aom/aom_dsp/x86/synonyms_avx2.h +++ b/third_party/aom/aom_dsp/x86/synonyms_avx2.h @@ -67,6 +67,11 @@ static INLINE __m256i yy_loadu2_128(const void *hi, const void *lo) { return yy_set_m128i(mhi, mlo); } +static INLINE void yy_storeu2_128(void *hi, void *lo, const __m256i a) { + _mm_storeu_si128((__m128i *)hi, _mm256_extracti128_si256(a, 1)); + _mm_storeu_si128((__m128i *)lo, _mm256_castsi256_si128(a)); +} + static INLINE __m256i yy_roundn_epu16(__m256i v_val_w, int bits) { const __m256i v_s_w = _mm256_srli_epi16(v_val_w, bits - 1); return _mm256_avg_epu16(v_s_w, _mm256_setzero_si256()); diff --git a/third_party/aom/aom_dsp/x86/txfm_common_avx2.h b/third_party/aom/aom_dsp/x86/txfm_common_avx2.h index b1611ba870..8a40508986 100644 --- a/third_party/aom/aom_dsp/x86/txfm_common_avx2.h +++ b/third_party/aom/aom_dsp/x86/txfm_common_avx2.h @@ -20,9 +20,6 @@ extern "C" { #endif -typedef void (*transform_1d_avx2)(const __m256i *input, __m256i *output, - int8_t cos_bit); - static INLINE __m256i pair_set_w16_epi16(int16_t a, int16_t b) { return _mm256_set1_epi32( (int32_t)(((uint16_t)(a)) | (((uint32_t)(b)) << 16))); @@ -192,6 +189,53 @@ static INLINE void round_shift_16bit_w16_avx2(__m256i *in, int size, int bit) { } } +static INLINE __m256i av1_round_shift_32_avx2(__m256i vec, int bit) { + __m256i tmp, round; + round = _mm256_set1_epi32(1 << (bit - 1)); + tmp = _mm256_add_epi32(vec, round); + return _mm256_srai_epi32(tmp, bit); +} + +static INLINE void av1_round_shift_array_32_avx2(__m256i *input, + __m256i *output, + const int size, + const int bit) { + if (bit > 0) { + int i; + for (i = 0; i < size; i++) { + output[i] = av1_round_shift_32_avx2(input[i], bit); + } + } else { + int i; + for (i = 0; i < size; i++) { + output[i] = _mm256_slli_epi32(input[i], -bit); + } + } +} + +static INLINE void av1_round_shift_rect_array_32_avx2(__m256i *input, + __m256i *output, + const int size, + const int bit, + const int val) { + const __m256i sqrt2 = _mm256_set1_epi32(val); + if (bit > 0) { + int i; + for (i = 0; i < size; i++) { + const __m256i r0 = av1_round_shift_32_avx2(input[i], bit); + const __m256i r1 = _mm256_mullo_epi32(sqrt2, r0); + output[i] = av1_round_shift_32_avx2(r1, NewSqrt2Bits); + } + } else { + int i; + for (i = 0; i < size; i++) { + const __m256i r0 = _mm256_slli_epi32(input[i], -bit); + const __m256i r1 = _mm256_mullo_epi32(sqrt2, r0); + output[i] = av1_round_shift_32_avx2(r1, NewSqrt2Bits); + } + } +} + #ifdef __cplusplus } #endif diff --git a/third_party/aom/aom_dsp/x86/variance_sse2.c b/third_party/aom/aom_dsp/x86/variance_sse2.c index 3c37e77c06..f3efc157f9 100644 --- a/third_party/aom/aom_dsp/x86/variance_sse2.c +++ b/third_party/aom/aom_dsp/x86/variance_sse2.c @@ -494,7 +494,7 @@ void aom_upsampled_pred_sse2(MACROBLOCKD *xd, const struct AV1Common *const cm, const int ref_num = 0; const int is_intrabc = is_intrabc_block(mi); const struct scale_factors *const sf = - is_intrabc ? &cm->sf_identity : &xd->block_refs[ref_num]->sf; + is_intrabc ? &cm->sf_identity : xd->block_ref_scale_factors[ref_num]; const int is_scaled = av1_is_scaled(sf); if (is_scaled) { @@ -571,11 +571,10 @@ void aom_upsampled_pred_sse2(MACROBLOCKD *xd, const struct AV1Common *const cm, } } - const InterpFilterParams *filter = - (subpel_search == 1) - ? av1_get_4tap_interp_filter_params(EIGHTTAP_REGULAR) - : av1_get_interp_filter_params_with_block_size(EIGHTTAP_REGULAR, 8); - int filter_taps = (subpel_search == 1) ? 4 : SUBPEL_TAPS; + const InterpFilterParams *filter = av1_get_filter(subpel_search); + // (TODO:yunqing) 2-tap case uses 4-tap functions since there is no SIMD for + // 2-tap yet. + int filter_taps = (subpel_search <= USE_4_TAPS) ? 4 : SUBPEL_TAPS; if (!subpel_x_q3 && !subpel_y_q3) { if (width >= 16) { @@ -638,20 +637,13 @@ void aom_upsampled_pred_sse2(MACROBLOCKD *xd, const struct AV1Common *const cm, const int16_t *const kernel_y = av1_get_interp_filter_subpel_kernel(filter, subpel_y_q3 << 1); const uint8_t *ref_start = ref - ref_stride * ((filter_taps >> 1) - 1); - uint8_t *temp_start_horiz = - (subpel_search == 1) ? temp + (filter_taps >> 1) * MAX_SB_SIZE : temp; + uint8_t *temp_start_horiz = (subpel_search <= USE_4_TAPS) + ? temp + (filter_taps >> 1) * MAX_SB_SIZE + : temp; uint8_t *temp_start_vert = temp + MAX_SB_SIZE * ((filter->taps >> 1) - 1); int intermediate_height = (((height - 1) * 8 + subpel_y_q3) >> 3) + filter_taps; assert(intermediate_height <= (MAX_SB_SIZE * 2 + 16) + 16); - // TODO(Deepa): Remove the memset below when we have - // 4 tap simd for sse2 and ssse3. - if (subpel_search == 1) { - memset(temp_start_vert - 3 * MAX_SB_SIZE, 0, width); - memset(temp_start_vert - 2 * MAX_SB_SIZE, 0, width); - memset(temp_start_vert + (height + 2) * MAX_SB_SIZE, 0, width); - memset(temp_start_vert + (height + 3) * MAX_SB_SIZE, 0, width); - } aom_convolve8_horiz(ref_start, ref_stride, temp_start_horiz, MAX_SB_SIZE, kernel_x, 16, NULL, -1, width, intermediate_height); aom_convolve8_vert(temp_start_vert, MAX_SB_SIZE, comp_pred, width, NULL, -1, diff --git a/third_party/aom/aom_ports/mem.h b/third_party/aom/aom_ports/mem.h index 3ffea3cd6d..9e3d424037 100644 --- a/third_party/aom/aom_ports/mem.h +++ b/third_party/aom/aom_ports/mem.h @@ -66,4 +66,34 @@ #define CONVERT_TO_SHORTPTR(x) ((uint16_t *)(((uintptr_t)(x)) << 1)) #define CONVERT_TO_BYTEPTR(x) ((uint8_t *)(((uintptr_t)(x)) >> 1)) +/*!\brief force enum to be unsigned 1 byte*/ +#define UENUM1BYTE(enumvar) \ + ; \ + typedef uint8_t enumvar + +/*!\brief force enum to be signed 1 byte*/ +#define SENUM1BYTE(enumvar) \ + ; \ + typedef int8_t enumvar + +/*!\brief force enum to be unsigned 2 byte*/ +#define UENUM2BYTE(enumvar) \ + ; \ + typedef uint16_t enumvar + +/*!\brief force enum to be signed 2 byte*/ +#define SENUM2BYTE(enumvar) \ + ; \ + typedef int16_t enumvar + +/*!\brief force enum to be unsigned 4 byte*/ +#define UENUM4BYTE(enumvar) \ + ; \ + typedef uint32_t enumvar + +/*!\brief force enum to be unsigned 4 byte*/ +#define SENUM4BYTE(enumvar) \ + ; \ + typedef int32_t enumvar + #endif // AOM_AOM_PORTS_MEM_H_ diff --git a/third_party/aom/aom_ports/x86.h b/third_party/aom/aom_ports/x86.h index 52ee49cb3c..8c18448714 100644 --- a/third_party/aom/aom_ports/x86.h +++ b/third_party/aom/aom_ports/x86.h @@ -222,11 +222,26 @@ static INLINE int x86_simd_caps(void) { return flags & mask; } -// Note: -// 32-bit CPU cycle counter is light-weighted for most function performance -// measurement. For large function (CPU time > a couple of seconds), 64-bit -// counter should be used. -// 32-bit CPU cycle counter +// Fine-Grain Measurement Functions +// +// If you are a timing a small region of code, access the timestamp counter +// (TSC) via: +// +// unsigned int start = x86_tsc_start(); +// ... +// unsigned int end = x86_tsc_end(); +// unsigned int diff = end - start; +// +// The start/end functions introduce a few more instructions than using +// x86_readtsc directly, but prevent the CPU's out-of-order execution from +// affecting the measurement (by having earlier/later instructions be evaluated +// in the time interval). See the white paper, "How to Benchmark Code +// Execution Times on Intel® IA-32 and IA-64 Instruction Set Architectures" by +// Gabriele Paoloni for more information. +// +// If you are timing a large function (CPU time > a couple of seconds), use +// x86_readtsc64 to read the timestamp counter in a 64-bit integer. The +// out-of-order leakage that can occur is minimal compared to total runtime. static INLINE unsigned int x86_readtsc(void) { #if defined(__GNUC__) && __GNUC__ unsigned int tsc; @@ -263,6 +278,41 @@ static INLINE uint64_t x86_readtsc64(void) { #endif } +// 32-bit CPU cycle counter with a partial fence against out-of-order execution. +static INLINE unsigned int x86_readtscp(void) { +#if defined(__GNUC__) && __GNUC__ + unsigned int tscp; + __asm__ __volatile__("rdtscp\n\t" : "=a"(tscp) :); + return tscp; +#elif defined(__SUNPRO_C) || defined(__SUNPRO_CC) + unsigned int tscp; + asm volatile("rdtscp\n\t" : "=a"(tscp) :); + return tscp; +#elif defined(_MSC_VER) + unsigned int ui; + return (unsigned int)__rdtscp(&ui); +#else +#if ARCH_X86_64 + return (unsigned int)__rdtscp(); +#else + __asm rdtscp; +#endif +#endif +} + +static INLINE unsigned int x86_tsc_start(void) { + unsigned int reg_eax, reg_ebx, reg_ecx, reg_edx; + cpuid(0, 0, reg_eax, reg_ebx, reg_ecx, reg_edx); + return x86_readtsc(); +} + +static INLINE unsigned int x86_tsc_end(void) { + uint32_t v = x86_readtscp(); + unsigned int reg_eax, reg_ebx, reg_ecx, reg_edx; + cpuid(0, 0, reg_eax, reg_ebx, reg_ecx, reg_edx); + return v; +} + #if defined(__GNUC__) && __GNUC__ #define x86_pause_hint() __asm__ __volatile__("pause \n\t") #elif defined(__SUNPRO_C) || defined(__SUNPRO_CC) diff --git a/third_party/aom/aom_scale/aom_scale_rtcd.pl b/third_party/aom/aom_scale/aom_scale_rtcd.pl index c5990b1bb3..27378c7ef4 100644 --- a/third_party/aom/aom_scale/aom_scale_rtcd.pl +++ b/third_party/aom/aom_scale/aom_scale_rtcd.pl @@ -36,11 +36,12 @@ add_proto qw/void aom_yv12_copy_u/, "const struct yv12_buffer_config *src_bc, st add_proto qw/void aom_yv12_copy_v/, "const struct yv12_buffer_config *src_bc, struct yv12_buffer_config *dst_bc"; -add_proto qw/void aom_yv12_partial_copy_y/, "const struct yv12_buffer_config *src_ybc, struct yv12_buffer_config *dst_ybc, int hstart, int hend, int vstart, int vend"; - -add_proto qw/void aom_yv12_partial_copy_u/, "const struct yv12_buffer_config *src_bc, struct yv12_buffer_config *dst_bc, int hstart, int hend, int vstart, int vend"; - -add_proto qw/void aom_yv12_partial_copy_v/, "const struct yv12_buffer_config *src_bc, struct yv12_buffer_config *dst_bc, int hstart, int hend, int vstart, int vend"; +add_proto qw/void aom_yv12_partial_copy_y/, "const struct yv12_buffer_config *src_ybc, int hstart1, int hend1, int vstart1, int vend1, struct yv12_buffer_config *dst_ybc, int hstart2, int vstart2"; +add_proto qw/void aom_yv12_partial_coloc_copy_y/, "const struct yv12_buffer_config *src_ybc, struct yv12_buffer_config *dst_ybc, int hstart, int hend, int vstart, int vend"; +add_proto qw/void aom_yv12_partial_copy_u/, "const struct yv12_buffer_config *src_bc, int hstart1, int hend1, int vstart1, int vend1, struct yv12_buffer_config *dst_bc, int hstart2, int vstart2"; +add_proto qw/void aom_yv12_partial_coloc_copy_u/, "const struct yv12_buffer_config *src_bc, struct yv12_buffer_config *dst_bc, int hstart, int hend, int vstart, int vend"; +add_proto qw/void aom_yv12_partial_copy_v/, "const struct yv12_buffer_config *src_bc, int hstart1, int hend1, int vstart1, int vend1, struct yv12_buffer_config *dst_bc, int hstart2, int vstart2"; +add_proto qw/void aom_yv12_partial_coloc_copy_v/, "const struct yv12_buffer_config *src_bc, struct yv12_buffer_config *dst_bc, int hstart, int hend, int vstart, int vend"; add_proto qw/void aom_extend_frame_borders/, "struct yv12_buffer_config *ybf, const int num_planes"; specialize qw/aom_extend_frame_borders dspr2/; diff --git a/third_party/aom/aom_scale/generic/yv12config.c b/third_party/aom/aom_scale/generic/yv12config.c index 84705e2d8b..dafe02b171 100644 --- a/third_party/aom/aom_scale/generic/yv12config.c +++ b/third_party/aom/aom_scale/generic/yv12config.c @@ -55,6 +55,13 @@ int aom_realloc_frame_buffer(YV12_BUFFER_CONFIG *ybf, int width, int height, if (width > DECODE_WIDTH_LIMIT || height > DECODE_HEIGHT_LIMIT) return -1; #endif + /* Only support allocating buffers that have a border that's a multiple + * of 32. The border restriction is required to get 16-byte alignment of + * the start of the chroma rows without introducing an arbitrary gap + * between planes, which would break the semantics of things like + * aom_img_set_rect(). */ + if (border & 0x1f) return -3; + if (ybf) { const int aom_byte_align = (byte_alignment == 0) ? 1 : byte_alignment; const int aligned_width = (width + 7) & ~7; @@ -109,10 +116,11 @@ int aom_realloc_frame_buffer(YV12_BUFFER_CONFIG *ybf, int width, int height, memset(ybf->buffer_alloc, 0, (size_t)frame_size); #endif #endif - } else if (frame_size > (size_t)ybf->buffer_alloc_sz) { + } else if (frame_size > ybf->buffer_alloc_sz) { // Allocation to hold larger frame, or first allocation. aom_free(ybf->buffer_alloc); ybf->buffer_alloc = NULL; + ybf->buffer_alloc_sz = 0; if (frame_size != (size_t)frame_size) return -1; @@ -127,13 +135,6 @@ int aom_realloc_frame_buffer(YV12_BUFFER_CONFIG *ybf, int width, int height, memset(ybf->buffer_alloc, 0, ybf->buffer_alloc_sz); } - /* Only support allocating buffers that have a border that's a multiple - * of 32. The border restriction is required to get 16-byte alignment of - * the start of the chroma rows without introducing an arbitrary gap - * between planes, which would break the semantics of things like - * aom_img_set_rect(). */ - if (border & 0x1f) return -3; - ybf->y_crop_width = width; ybf->y_crop_height = height; ybf->y_width = aligned_width; diff --git a/third_party/aom/aom_scale/generic/yv12extend.c b/third_party/aom/aom_scale/generic/yv12extend.c index ba183520a0..127ca23116 100644 --- a/third_party/aom/aom_scale/generic/yv12extend.c +++ b/third_party/aom/aom_scale/generic/yv12extend.c @@ -321,91 +321,116 @@ void aom_yv12_copy_v_c(const YV12_BUFFER_CONFIG *src_bc, } } -void aom_yv12_partial_copy_y_c(const YV12_BUFFER_CONFIG *src_ybc, - YV12_BUFFER_CONFIG *dst_ybc, int hstart, - int hend, int vstart, int vend) { +void aom_yv12_partial_copy_y_c(const YV12_BUFFER_CONFIG *src_ybc, int hstart1, + int hend1, int vstart1, int vend1, + YV12_BUFFER_CONFIG *dst_ybc, int hstart2, + int vstart2) { int row; const uint8_t *src = src_ybc->y_buffer; uint8_t *dst = dst_ybc->y_buffer; if (src_ybc->flags & YV12_FLAG_HIGHBITDEPTH) { const uint16_t *src16 = - CONVERT_TO_SHORTPTR(src + vstart * src_ybc->y_stride + hstart); + CONVERT_TO_SHORTPTR(src + vstart1 * src_ybc->y_stride + hstart1); uint16_t *dst16 = - CONVERT_TO_SHORTPTR(dst + vstart * dst_ybc->y_stride + hstart); - for (row = vstart; row < vend; ++row) { - memcpy(dst16, src16, (hend - hstart) * sizeof(uint16_t)); + CONVERT_TO_SHORTPTR(dst + vstart2 * dst_ybc->y_stride + hstart2); + + for (row = vstart1; row < vend1; ++row) { + memcpy(dst16, src16, (hend1 - hstart1) * sizeof(uint16_t)); src16 += src_ybc->y_stride; dst16 += dst_ybc->y_stride; } return; } - src = (src + vstart * src_ybc->y_stride + hstart); - dst = (dst + vstart * dst_ybc->y_stride + hstart); + src = (src + vstart1 * src_ybc->y_stride + hstart1); + dst = (dst + vstart2 * dst_ybc->y_stride + hstart2); - for (row = vstart; row < vend; ++row) { - memcpy(dst, src, (hend - hstart)); + for (row = vstart1; row < vend1; ++row) { + memcpy(dst, src, (hend1 - hstart1)); src += src_ybc->y_stride; dst += dst_ybc->y_stride; } } -void aom_yv12_partial_copy_u_c(const YV12_BUFFER_CONFIG *src_bc, - YV12_BUFFER_CONFIG *dst_bc, int hstart, int hend, - int vstart, int vend) { +void aom_yv12_partial_coloc_copy_y_c(const YV12_BUFFER_CONFIG *src_ybc, + YV12_BUFFER_CONFIG *dst_ybc, int hstart, + int hend, int vstart, int vend) { + aom_yv12_partial_copy_y_c(src_ybc, hstart, hend, vstart, vend, dst_ybc, + hstart, vstart); +} + +void aom_yv12_partial_copy_u_c(const YV12_BUFFER_CONFIG *src_bc, int hstart1, + int hend1, int vstart1, int vend1, + YV12_BUFFER_CONFIG *dst_bc, int hstart2, + int vstart2) { int row; const uint8_t *src = src_bc->u_buffer; uint8_t *dst = dst_bc->u_buffer; if (src_bc->flags & YV12_FLAG_HIGHBITDEPTH) { const uint16_t *src16 = - CONVERT_TO_SHORTPTR(src + vstart * src_bc->uv_stride + hstart); + CONVERT_TO_SHORTPTR(src + vstart1 * src_bc->uv_stride + hstart1); uint16_t *dst16 = - CONVERT_TO_SHORTPTR(dst + vstart * dst_bc->uv_stride + hstart); - for (row = vstart; row < vend; ++row) { - memcpy(dst16, src16, (hend - hstart) * sizeof(uint16_t)); + CONVERT_TO_SHORTPTR(dst + vstart2 * dst_bc->uv_stride + hstart2); + for (row = vstart1; row < vend1; ++row) { + memcpy(dst16, src16, (hend1 - hstart1) * sizeof(uint16_t)); src16 += src_bc->uv_stride; dst16 += dst_bc->uv_stride; } return; } - src = (src + vstart * src_bc->uv_stride + hstart); - dst = (dst + vstart * dst_bc->uv_stride + hstart); + src = (src + vstart1 * src_bc->uv_stride + hstart1); + dst = (dst + vstart2 * dst_bc->uv_stride + hstart2); - for (row = vstart; row < vend; ++row) { - memcpy(dst, src, (hend - hstart)); + for (row = vstart1; row < vend1; ++row) { + memcpy(dst, src, (hend1 - hstart1)); src += src_bc->uv_stride; dst += dst_bc->uv_stride; } } -void aom_yv12_partial_copy_v_c(const YV12_BUFFER_CONFIG *src_bc, - YV12_BUFFER_CONFIG *dst_bc, int hstart, int hend, - int vstart, int vend) { +void aom_yv12_partial_coloc_copy_u_c(const YV12_BUFFER_CONFIG *src_bc, + YV12_BUFFER_CONFIG *dst_bc, int hstart, + int hend, int vstart, int vend) { + aom_yv12_partial_copy_u_c(src_bc, hstart, hend, vstart, vend, dst_bc, hstart, + vstart); +} + +void aom_yv12_partial_copy_v_c(const YV12_BUFFER_CONFIG *src_bc, int hstart1, + int hend1, int vstart1, int vend1, + YV12_BUFFER_CONFIG *dst_bc, int hstart2, + int vstart2) { int row; const uint8_t *src = src_bc->v_buffer; uint8_t *dst = dst_bc->v_buffer; if (src_bc->flags & YV12_FLAG_HIGHBITDEPTH) { const uint16_t *src16 = - CONVERT_TO_SHORTPTR(src + vstart * src_bc->uv_stride + hstart); + CONVERT_TO_SHORTPTR(src + vstart1 * src_bc->uv_stride + hstart1); uint16_t *dst16 = - CONVERT_TO_SHORTPTR(dst + vstart * dst_bc->uv_stride + hstart); - for (row = vstart; row < vend; ++row) { - memcpy(dst16, src16, (hend - hstart) * sizeof(uint16_t)); + CONVERT_TO_SHORTPTR(dst + vstart2 * dst_bc->uv_stride + hstart2); + for (row = vstart1; row < vend1; ++row) { + memcpy(dst16, src16, (hend1 - hstart1) * sizeof(uint16_t)); src16 += src_bc->uv_stride; dst16 += dst_bc->uv_stride; } return; } - src = (src + vstart * src_bc->uv_stride + hstart); - dst = (dst + vstart * dst_bc->uv_stride + hstart); + src = (src + vstart1 * src_bc->uv_stride + hstart1); + dst = (dst + vstart2 * dst_bc->uv_stride + hstart2); - for (row = vstart; row < vend; ++row) { - memcpy(dst, src, (hend - hstart)); + for (row = vstart1; row < vend1; ++row) { + memcpy(dst, src, (hend1 - hstart1)); src += src_bc->uv_stride; dst += dst_bc->uv_stride; } } + +void aom_yv12_partial_coloc_copy_v_c(const YV12_BUFFER_CONFIG *src_bc, + YV12_BUFFER_CONFIG *dst_bc, int hstart, + int hend, int vstart, int vend) { + aom_yv12_partial_copy_v_c(src_bc, hstart, hend, vstart, vend, dst_bc, hstart, + vstart); +} diff --git a/third_party/aom/aom_scale/yv12config.h b/third_party/aom/aom_scale/yv12config.h index 2fb81acd7c..bb0d56430b 100644 --- a/third_party/aom/aom_scale/yv12config.h +++ b/third_party/aom/aom_scale/yv12config.h @@ -24,32 +24,24 @@ extern "C" { #define AOMINNERBORDERINPIXELS 160 #define AOM_INTERP_EXTEND 4 - -// TODO(jingning): Use unified inter predictor for encoder and -// decoder during the development process. Revisit the frame border -// to improve the decoder performance. -#if CONFIG_REDUCED_ENCODER_BORDER -#define AOM_BORDER_IN_PIXELS 160 -#else #define AOM_BORDER_IN_PIXELS 288 -#endif // CONFIG_REDUCED_ENCODER_BORDER +#define AOM_ENC_NO_SCALE_BORDER 160 +#define AOM_DEC_BORDER_IN_PIXELS 64 typedef struct yv12_buffer_config { union { struct { int y_width; int uv_width; - int alpha_width; }; - int widths[3]; + int widths[2]; }; union { struct { int y_height; int uv_height; - int alpha_height; }; - int heights[3]; + int heights[2]; }; union { struct { @@ -69,18 +61,16 @@ typedef struct yv12_buffer_config { struct { int y_stride; int uv_stride; - int alpha_stride; }; - int strides[3]; + int strides[2]; }; union { struct { uint8_t *y_buffer; uint8_t *u_buffer; uint8_t *v_buffer; - uint8_t *alpha_buffer; }; - uint8_t *buffers[4]; + uint8_t *buffers[3]; }; // Indicate whether y_buffer, u_buffer, and v_buffer points to the internally @@ -106,7 +96,7 @@ typedef struct yv12_buffer_config { aom_color_primaries_t color_primaries; aom_transfer_characteristics_t transfer_characteristics; aom_matrix_coefficients_t matrix_coefficients; - int monochrome; + uint8_t monochrome; aom_chroma_sample_position_t chroma_sample_position; aom_color_range_t color_range; int render_width; diff --git a/third_party/aom/aom_util/aom_thread.c b/third_party/aom/aom_util/aom_thread.c index cae9f5e250..244ac3bc29 100644 --- a/third_party/aom/aom_util/aom_thread.c +++ b/third_party/aom/aom_util/aom_thread.c @@ -14,6 +14,12 @@ // Original source: // https://chromium.googlesource.com/webm/libwebp +// Enable GNU extensions in glibc so that we can call pthread_setname_np(). +// This must be before any #include statements. +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif + #include #include // for memset() @@ -34,6 +40,28 @@ static void execute(AVxWorker *const worker); // Forward declaration. static THREADFN thread_loop(void *ptr) { AVxWorker *const worker = (AVxWorker *)ptr; +#ifdef __APPLE__ + if (worker->thread_name != NULL) { + // Apple's version of pthread_setname_np takes one argument and operates on + // the current thread only. The maximum size of the thread_name buffer was + // noted in the Chromium source code and was confirmed by experiments. If + // thread_name is too long, pthread_setname_np returns -1 with errno + // ENAMETOOLONG (63). + char thread_name[64]; + strncpy(thread_name, worker->thread_name, sizeof(thread_name)); + thread_name[sizeof(thread_name) - 1] = '\0'; + pthread_setname_np(thread_name); + } +#elif defined(__GLIBC__) || defined(__BIONIC__) + if (worker->thread_name != NULL) { + // Linux and Android require names (with nul) fit in 16 chars, otherwise + // pthread_setname_np() returns ERANGE (34). + char thread_name[16]; + strncpy(thread_name, worker->thread_name, sizeof(thread_name)); + thread_name[sizeof(thread_name) - 1] = '\0'; + pthread_setname_np(pthread_self(), thread_name); + } +#endif int done = 0; while (!done) { pthread_mutex_lock(&worker->impl_->mutex_); diff --git a/third_party/aom/aom_util/aom_thread.h b/third_party/aom/aom_util/aom_thread.h index f14c1ac18c..bda8b756ec 100644 --- a/third_party/aom/aom_util/aom_thread.h +++ b/third_party/aom/aom_util/aom_thread.h @@ -26,6 +26,7 @@ extern "C" { // Set maximum decode threads to be 8 due to the limit of frame buffers // and not enough semaphores in the emulation layer on windows. #define MAX_DECODE_THREADS 8 +#define MAX_NUM_THREADS 64 #if CONFIG_MULTITHREAD @@ -380,6 +381,10 @@ typedef struct AVxWorkerImpl AVxWorkerImpl; typedef struct { AVxWorkerImpl *impl_; AVxWorkerStatus status_; + // Thread name for the debugger. If not NULL, must point to a string that + // outlives the worker thread. For portability, use a name <= 15 characters + // long (not including the terminating NUL character). + const char *thread_name; AVxWorkerHook hook; // hook to call void *data1; // first argument passed to 'hook' void *data2; // second argument passed to 'hook' diff --git a/third_party/aom/apps/aomdec.c b/third_party/aom/apps/aomdec.c index ff13b6f500..c054425777 100644 --- a/third_party/aom/apps/aomdec.c +++ b/third_party/aom/apps/aomdec.c @@ -143,7 +143,7 @@ static INLINE int libyuv_scale(aom_image_t *src, aom_image_t *dst, } #endif -void show_help(FILE *fout, int shorthelp) { +static void show_help(FILE *fout, int shorthelp) { fprintf(fout, "Usage: %s filename\n\n", exec_name); if (shorthelp) { @@ -427,13 +427,6 @@ static FILE *open_outfile(const char *name) { } } -static int img_shifted_realloc_required(const aom_image_t *img, - const aom_image_t *shifted, - aom_img_fmt_t required_fmt) { - return img->d_w != shifted->d_w || img->d_h != shifted->d_h || - required_fmt != shifted->fmt; -} - static int main_loop(int argc, const char **argv_) { aom_codec_ctx_t decoder; char *fn = NULL; @@ -459,7 +452,7 @@ static int main_loop(int argc, const char **argv_) { int opt_i420 = 0; int opt_raw = 0; aom_codec_dec_cfg_t cfg = { 0, 0, 0, CONFIG_LOWBITDEPTH, { 1 } }; - unsigned int output_bit_depth = 0; + unsigned int fixed_output_bit_depth = 0; unsigned int is_annexb = 0; int frames_corrupted = 0; int dec_flags = 0; @@ -491,6 +484,7 @@ static int main_loop(int argc, const char **argv_) { input.webm_ctx = &webm_ctx; #endif struct ObuDecInputContext obu_ctx = { NULL, NULL, 0, 0, 0 }; + int is_ivf = 0; obu_ctx.avx_ctx = &aom_input_ctx; input.obu_ctx = &obu_ctx; @@ -572,7 +566,7 @@ static int main_loop(int argc, const char **argv_) { } else if (arg_match(&arg, &continuearg, argi)) { keep_going = 1; } else if (arg_match(&arg, &outbitdeptharg, argi)) { - output_bit_depth = arg_parse_uint(&arg); + fixed_output_bit_depth = arg_parse_uint(&arg); } else if (arg_match(&arg, &isannexb, argi)) { is_annexb = 1; input.obu_ctx->is_annexb = 1; @@ -617,8 +611,10 @@ static int main_loop(int argc, const char **argv_) { #endif input.aom_input_ctx->filename = fn; input.aom_input_ctx->file = infile; - if (file_is_ivf(input.aom_input_ctx)) + if (file_is_ivf(input.aom_input_ctx)) { input.aom_input_ctx->file_type = FILE_TYPE_IVF; + is_ivf = 1; + } #if CONFIG_WEBM_IO else if (file_is_webm(input.webm_ctx, input.aom_input_ctx)) input.aom_input_ctx->file_type = FILE_TYPE_WEBM; @@ -668,6 +664,10 @@ static int main_loop(int argc, const char **argv_) { } fourcc_interface = get_aom_decoder_by_fourcc(aom_input_ctx.fourcc); + + if (is_ivf && !fourcc_interface) + fatal("Unsupported fourcc: %x\n", aom_input_ctx.fourcc); + if (interface && fourcc_interface && interface != fourcc_interface) warn("Header indicates codec: %s\n", fourcc_interface->name); else @@ -779,7 +779,7 @@ static int main_loop(int argc, const char **argv_) { aom_usec_timer_start(&timer); if (flush_decoder) { - // Flush the decoder in frame parallel decode. + // Flush the decoder. if (aom_codec_decode(&decoder, NULL, 0, NULL)) { warn("Failed to flush decoder: %s", aom_codec_error(&decoder)); } @@ -850,41 +850,15 @@ static int main_loop(int argc, const char **argv_) { } } // Default to codec bit depth if output bit depth not set - if (!output_bit_depth && single_file && !do_md5) { + unsigned int output_bit_depth; + if (!fixed_output_bit_depth && single_file && !do_md5) { output_bit_depth = img->bit_depth; + } else { + output_bit_depth = fixed_output_bit_depth; } // Shift up or down if necessary - if (output_bit_depth != 0) { - const aom_img_fmt_t shifted_fmt = - output_bit_depth == 8 ? img->fmt & ~AOM_IMG_FMT_HIGHBITDEPTH - : img->fmt | AOM_IMG_FMT_HIGHBITDEPTH; - - if (shifted_fmt != img->fmt || output_bit_depth != img->bit_depth) { - if (img_shifted && - img_shifted_realloc_required(img, img_shifted, shifted_fmt)) { - aom_img_free(img_shifted); - img_shifted = NULL; - } - if (img_shifted) { - img_shifted->monochrome = img->monochrome; - } - if (!img_shifted) { - img_shifted = - aom_img_alloc(NULL, shifted_fmt, img->d_w, img->d_h, 16); - img_shifted->bit_depth = output_bit_depth; - img_shifted->monochrome = img->monochrome; - img_shifted->csp = img->csp; - } - if (output_bit_depth > img->bit_depth) { - aom_img_upshift(img_shifted, img, - output_bit_depth - img->bit_depth); - } else { - aom_img_downshift(img_shifted, img, - img->bit_depth - output_bit_depth); - } - img = img_shifted; - } - } + if (output_bit_depth != 0) + aom_shift_img(output_bit_depth, &img, &img_shifted); aom_input_ctx.width = img->d_w; aom_input_ctx.height = img->d_h; @@ -900,6 +874,11 @@ static int main_loop(int argc, const char **argv_) { y4m_buf, sizeof(y4m_buf), aom_input_ctx.width, aom_input_ctx.height, &aom_input_ctx.framerate, img->monochrome, img->csp, img->fmt, img->bit_depth); + if (img->csp == AOM_CSP_COLOCATED) { + fprintf(stderr, + "Warning: Y4M lacks a colorspace for colocated " + "chroma. Using a placeholder.\n"); + } if (do_md5) { MD5Update(&md5_ctx, (md5byte *)y4m_buf, (unsigned int)len); } else { diff --git a/third_party/aom/apps/aomenc.c b/third_party/aom/apps/aomenc.c index 2e5d35cfef..83c1353de5 100644 --- a/third_party/aom/apps/aomenc.c +++ b/third_party/aom/apps/aomenc.c @@ -263,9 +263,9 @@ static const arg_def_t global_error_resilient = "Enable global error resiliency features"); static const arg_def_t lag_in_frames = ARG_DEF(NULL, "lag-in-frames", 1, "Max number of frames to lag"); -static const arg_def_t large_scale_tile = - ARG_DEF(NULL, "large-scale-tile", 1, - "Large scale tile coding (0: off (default), 1: on)"); +static const arg_def_t large_scale_tile = ARG_DEF( + NULL, "large-scale-tile", 1, + "Large scale tile coding (0: off (default), 1: on (ivf output only))"); static const arg_def_t monochrome = ARG_DEF(NULL, "monochrome", 0, "Monochrome video (no chroma planes)"); static const arg_def_t full_still_picture_hdr = ARG_DEF( @@ -415,11 +415,14 @@ static const arg_def_t cpu_used_av1 = ARG_DEF(NULL, "cpu-used", 1, "CPU Used (0..8)"); static const arg_def_t rowmtarg = ARG_DEF(NULL, "row-mt", 1, - "Enable row based multi-threading (0: off (default), 1: on)"); + "Enable row based multi-threading (0: off, 1: on (default))"); static const arg_def_t tile_cols = ARG_DEF(NULL, "tile-columns", 1, "Number of tile columns to use, log2"); static const arg_def_t tile_rows = ARG_DEF(NULL, "tile-rows", 1, "Number of tile rows to use, log2"); +static const arg_def_t enable_tpl_model = + ARG_DEF(NULL, "enable-tpl-model", 1, + "RDO modulation based on frame temporal dependency"); static const arg_def_t tile_width = ARG_DEF(NULL, "tile-width", 1, "Tile widths (comma separated)"); static const arg_def_t tile_height = @@ -434,6 +437,88 @@ static const arg_def_t enable_restoration = ARG_DEF(NULL, "enable-restoration", 1, "Enable the loop restoration filter (0: false, " "1: true (default))"); +static const arg_def_t enable_rect_partitions = + ARG_DEF(NULL, "enable-rect-partitions", 1, + "Enable rectangular partitions " + "(0: false, 1: true (default))"); +static const arg_def_t enable_dual_filter = + ARG_DEF(NULL, "enable-dual-filter", 1, + "Enable dual filter " + "(0: false, 1: true (default))"); +static const arg_def_t enable_intra_edge_filter = + ARG_DEF(NULL, "enable-intra-edge-filter", 1, + "Enable intra edge filtering " + "(0: false, 1: true (default))"); +static const arg_def_t enable_order_hint = + ARG_DEF(NULL, "enable-order-hint", 1, + "Enable order hint " + "(0: false, 1: true (default))"); +static const arg_def_t enable_tx64 = + ARG_DEF(NULL, "enable-tx64", 1, + "Enable 64-pt transform (0: false, 1: true (default))"); +static const arg_def_t enable_dist_wtd_comp = + ARG_DEF(NULL, "enable-dist-wtd-comp", 1, + "Enable distance-weighted compound " + "(0: false, 1: true (default))"); +static const arg_def_t enable_masked_comp = + ARG_DEF(NULL, "enable-masked-comp", 1, + "Enable masked (wedge/diff-wtd) compound " + "(0: false, 1: true (default))"); +static const arg_def_t enable_interintra_comp = + ARG_DEF(NULL, "enable-interintra-comp", 1, + "Enable interintra compound " + "(0: false, 1: true (default))"); +static const arg_def_t enable_smooth_interintra = + ARG_DEF(NULL, "enable-smooth-interintra", 1, + "Enable smooth interintra mode " + "(0: false, 1: true (default))"); +static const arg_def_t enable_diff_wtd_comp = + ARG_DEF(NULL, "enable-diff-wtd-comp", 1, + "Enable difference-weighted compound " + "(0: false, 1: true (default))"); +static const arg_def_t enable_interinter_wedge = + ARG_DEF(NULL, "enable-interinter-wedge", 1, + "Enable interinter wedge compound " + "(0: false, 1: true (default))"); +static const arg_def_t enable_interintra_wedge = + ARG_DEF(NULL, "enable-interintra-wedge", 1, + "Enable interintra wedge compound " + "(0: false, 1: true (default))"); +static const arg_def_t enable_global_motion = + ARG_DEF(NULL, "enable-global-motion", 1, + "Enable global motion " + "(0: false, 1: true (default))"); +static const arg_def_t enable_warped_motion = + ARG_DEF(NULL, "enable-warped-motion", 1, + "Enable local warped motion " + "(0: false, 1: true (default))"); +static const arg_def_t enable_filter_intra = + ARG_DEF(NULL, "enable-filter-intra", 1, + "Enable filter intra prediction mode " + "(0: false, 1: true (default))"); +static const arg_def_t enable_smooth_intra = + ARG_DEF(NULL, "enable-smooth-intra", 1, + "Enable smooth intra prediction modes " + "(0: false, 1: true (default))"); +static const arg_def_t enable_paeth_intra = + ARG_DEF(NULL, "enable-paeth-intra", 1, + "Enable Paeth intra prediction mode (0: false, 1: true (default))"); +static const arg_def_t enable_cfl_intra = + ARG_DEF(NULL, "enable-cfl-intra", 1, + "Enable chroma from luma intra prediction mode " + "(0: false, 1: true (default))"); +static const arg_def_t enable_obmc = ARG_DEF( + NULL, "enable-obmc", 1, "Enable OBMC (0: false, 1: true (default))"); +static const arg_def_t enable_palette = + ARG_DEF(NULL, "enable-palette", 1, + "Enable palette prediction mode (0: false, 1: true (default))"); +static const arg_def_t enable_intrabc = + ARG_DEF(NULL, "enable-intrabc", 1, + "Enable intra block copy prediction mode " + "(0: false, 1: true (default))"); +static const arg_def_t enable_angle_delta = + ARG_DEF(NULL, "enable-angle-delta", 1, + "Enable intra angle delta (0: false, 1: true (default))"); static const arg_def_t disable_trellis_quant = ARG_DEF(NULL, "disable-trellis-quant", 1, "Disable trellis optimization of quantized coefficients (0: false (" @@ -445,6 +530,14 @@ static const arg_def_t qm_min = ARG_DEF( NULL, "qm-min", 1, "Min quant matrix flatness (0..15), default is 8"); static const arg_def_t qm_max = ARG_DEF( NULL, "qm-max", 1, "Max quant matrix flatness (0..15), default is 15"); +static const arg_def_t reduced_tx_type_set = ARG_DEF( + NULL, "reduced-tx-type-set", 1, "Use reduced set of transform types"); +static const arg_def_t use_intra_dct_only = + ARG_DEF(NULL, "use-intra-dct-only", 1, "Use DCT only for INTRA modes"); +static const arg_def_t use_inter_dct_only = + ARG_DEF(NULL, "use-inter-dct-only", 1, "Use DCT only for INTER modes"); +static const arg_def_t quant_b_adapt = + ARG_DEF(NULL, "quant-b-adapt", 1, "Use adaptive quantize_b"); #if CONFIG_DIST_8X8 static const arg_def_t enable_dist_8x8 = ARG_DEF(NULL, "enable-dist-8x8", 1, @@ -512,6 +605,12 @@ static const arg_def_t min_gf_interval = ARG_DEF( static const arg_def_t max_gf_interval = ARG_DEF( NULL, "max-gf-interval", 1, "max gf/arf frame interval (default 0, indicating in-built behavior)"); +static const arg_def_t gf_max_pyr_height = + ARG_DEF(NULL, "gf-max-pyr-height", 1, + "maximum height for GF group pyramid structure (1 to 4 (default))"); +static const arg_def_t max_reference_frames = ARG_DEF( + NULL, "max-reference-frames", 1, + "maximum number of reference frames allowed per frame (3 to 7 (default))"); static const struct arg_enum_list color_primaries_enum[] = { { "bt709", AOM_CICP_CP_BT_709 }, @@ -624,6 +723,7 @@ static const arg_def_t *av1_args[] = { &cpu_used_av1, &rowmtarg, &tile_cols, &tile_rows, + &enable_tpl_model, &arnr_maxframes, &arnr_strength, &tune_metric, @@ -634,10 +734,36 @@ static const arg_def_t *av1_args[] = { &cpu_used_av1, &lossless, &enable_cdef, &enable_restoration, + &enable_rect_partitions, + &enable_dual_filter, + &enable_intra_edge_filter, + &enable_order_hint, + &enable_tx64, + &enable_dist_wtd_comp, + &enable_masked_comp, + &enable_interintra_comp, + &enable_smooth_interintra, + &enable_diff_wtd_comp, + &enable_interinter_wedge, + &enable_interintra_wedge, + &enable_global_motion, + &enable_warped_motion, + &enable_filter_intra, + &enable_smooth_intra, + &enable_paeth_intra, + &enable_cfl_intra, + &enable_obmc, + &enable_palette, + &enable_intrabc, + &enable_angle_delta, &disable_trellis_quant, &enable_qm, &qm_min, &qm_max, + &reduced_tx_type_set, + &use_intra_dct_only, + &use_inter_dct_only, + &quant_b_adapt, #if CONFIG_DIST_8X8 &enable_dist_8x8, #endif @@ -655,6 +781,7 @@ static const arg_def_t *av1_args[] = { &cpu_used_av1, &input_chroma_sample_position, &min_gf_interval, &max_gf_interval, + &gf_max_pyr_height, &superblock_size, &num_tg, &mtu_size, @@ -664,7 +791,8 @@ static const arg_def_t *av1_args[] = { &cpu_used_av1, #if CONFIG_DENOISE &denoise_noise_level, &denoise_block_size, -#endif +#endif // CONFIG_DENOISE + &max_reference_frames, &enable_ref_frame_mvs, &bitdeptharg, &inbitdeptharg, @@ -681,6 +809,7 @@ static const int av1_arg_ctrl_map[] = { AOME_SET_CPUUSED, AV1E_SET_ROW_MT, AV1E_SET_TILE_COLUMNS, AV1E_SET_TILE_ROWS, + AV1E_SET_ENABLE_TPL_MODEL, AOME_SET_ARNR_MAXFRAMES, AOME_SET_ARNR_STRENGTH, AOME_SET_TUNING, @@ -691,10 +820,36 @@ static const int av1_arg_ctrl_map[] = { AOME_SET_CPUUSED, AV1E_SET_LOSSLESS, AV1E_SET_ENABLE_CDEF, AV1E_SET_ENABLE_RESTORATION, + AV1E_SET_ENABLE_RECT_PARTITIONS, + AV1E_SET_ENABLE_DUAL_FILTER, + AV1E_SET_ENABLE_INTRA_EDGE_FILTER, + AV1E_SET_ENABLE_ORDER_HINT, + AV1E_SET_ENABLE_TX64, + AV1E_SET_ENABLE_DIST_WTD_COMP, + AV1E_SET_ENABLE_MASKED_COMP, + AV1E_SET_ENABLE_INTERINTRA_COMP, + AV1E_SET_ENABLE_SMOOTH_INTERINTRA, + AV1E_SET_ENABLE_DIFF_WTD_COMP, + AV1E_SET_ENABLE_INTERINTER_WEDGE, + AV1E_SET_ENABLE_INTERINTRA_WEDGE, + AV1E_SET_ENABLE_GLOBAL_MOTION, + AV1E_SET_ENABLE_WARPED_MOTION, + AV1E_SET_ENABLE_FILTER_INTRA, + AV1E_SET_ENABLE_SMOOTH_INTRA, + AV1E_SET_ENABLE_PAETH_INTRA, + AV1E_SET_ENABLE_CFL_INTRA, + AV1E_SET_ENABLE_OBMC, + AV1E_SET_ENABLE_PALETTE, + AV1E_SET_ENABLE_INTRABC, + AV1E_SET_ENABLE_ANGLE_DELTA, AV1E_SET_DISABLE_TRELLIS_QUANT, AV1E_SET_ENABLE_QM, AV1E_SET_QM_MIN, AV1E_SET_QM_MAX, + AV1E_SET_REDUCED_TX_TYPE_SET, + AV1E_SET_INTRA_DCT_ONLY, + AV1E_SET_INTER_DCT_ONLY, + AV1E_SET_QUANT_B_ADAPT, #if CONFIG_DIST_8X8 AV1E_SET_ENABLE_DIST_8X8, #endif @@ -712,6 +867,7 @@ static const int av1_arg_ctrl_map[] = { AOME_SET_CPUUSED, AV1E_SET_CHROMA_SAMPLE_POSITION, AV1E_SET_MIN_GF_INTERVAL, AV1E_SET_MAX_GF_INTERVAL, + AV1E_SET_GF_MAX_PYRAMID_HEIGHT, AV1E_SET_SUPERBLOCK_SIZE, AV1E_SET_NUM_TG, AV1E_SET_MTU, @@ -721,18 +877,15 @@ static const int av1_arg_ctrl_map[] = { AOME_SET_CPUUSED, #if CONFIG_DENOISE AV1E_SET_DENOISE_NOISE_LEVEL, AV1E_SET_DENOISE_BLOCK_SIZE, -#endif +#endif // CONFIG_DENOISE + AV1E_SET_MAX_REFERENCE_FRAMES, AV1E_SET_ENABLE_REF_FRAME_MVS, - AV1E_SET_ENABLE_DF, - AV1E_SET_ENABLE_ORDER_HINT, - AV1E_SET_ENABLE_JNT_COMP, - AV1E_SET_ENABLE_SUPERRES, 0 }; #endif // CONFIG_AV1_ENCODER static const arg_def_t *no_args[] = { NULL }; -void show_help(FILE *fout, int shorthelp) { +static void show_help(FILE *fout, int shorthelp) { fprintf(fout, "Usage: %s -o dst_filename src_filename \n", exec_name); @@ -845,14 +998,17 @@ static void validate_positive_rational(const char *msg, if (!rat->den) die("Error: %s has zero denominator\n", msg); } -static void parse_global_config(struct AvxEncoderConfig *global, int *argc, +/* Parses global config arguments into the AvxEncoderConfig. Note that + * argv is modified and overwrites all parsed arguments. + */ +static void parse_global_config(struct AvxEncoderConfig *global, int argc, char ***argv) { char **argi, **argj; struct arg arg; const int num_encoder = get_aom_encoder_count(); char **argv_local = (char **)*argv; #if CONFIG_FILEOPTIONS - int argc_local = *argc; + int argc_local = argc; #endif if (num_encoder < 1) die("Error: no valid encoder available\n"); @@ -861,6 +1017,7 @@ static void parse_global_config(struct AvxEncoderConfig *global, int *argc, global->codec = get_aom_encoder_by_index(num_encoder - 1); global->passes = 0; global->color_type = I420; + global->csp = AOM_CSP_UNKNOWN; #if CONFIG_FILEOPTIONS const char *cfg = NULL; @@ -900,6 +1057,10 @@ static void parse_global_config(struct AvxEncoderConfig *global, int *argc, if (global->pass < 1 || global->pass > 2) die("Error: Invalid pass selected (%d)\n", global->pass); + } else if (arg_match(&arg, &input_chroma_sample_position, argi)) { + global->csp = arg_parse_enum(&arg); + /* Flag is used by later code as well, preserve it. */ + argj++; } else if (arg_match(&arg, &usage, argi)) global->usage = arg_parse_uint(&arg); else if (arg_match(&arg, &good_dl, argi)) @@ -963,7 +1124,8 @@ static void parse_global_config(struct AvxEncoderConfig *global, int *argc, } } -static void open_input_file(struct AvxInputContext *input) { +static void open_input_file(struct AvxInputContext *input, + aom_chroma_sample_position_t csp) { /* Parse certain options from the input file, if possible */ input->file = strcmp(input->filename, "-") ? fopen(input->filename, "rb") : set_binary_mode(stdin); @@ -989,7 +1151,7 @@ static void open_input_file(struct AvxInputContext *input) { input->detect.position = 0; if (input->detect.buf_read == 4 && file_is_y4m(input->detect.buf)) { - if (y4m_input_open(&input->y4m, input->file, input->detect.buf, 4, + if (y4m_input_open(&input->y4m, input->file, input->detect.buf, 4, csp, input->only_i420) >= 0) { input->file_type = FILE_TYPE_Y4M; input->width = input->y4m.pic_w; @@ -1195,6 +1357,7 @@ static int parse_stream_params(struct AvxEncoderConfig *global, config->cfg.g_lag_in_frames = arg_parse_uint(&arg); } else if (arg_match(&arg, &large_scale_tile, argi)) { config->cfg.large_scale_tile = arg_parse_uint(&arg); + if (config->cfg.large_scale_tile) global->codec = get_aom_lst_encoder(); } else if (arg_match(&arg, &monochrome, argi)) { config->cfg.monochrome = 1; } else if (arg_match(&arg, &full_still_picture_hdr, argi)) { @@ -1306,13 +1469,11 @@ static void validate_stream_config(const struct stream_state *stream, " and --height (-h)", stream->index); - // Check that the codec bit depth is greater than the input bit depth. - if (stream->config.cfg.g_input_bit_depth > - (unsigned int)stream->config.cfg.g_bit_depth) { - fatal("Stream %d: codec bit depth (%d) less than input bit depth (%d)", - stream->index, (int)stream->config.cfg.g_bit_depth, - stream->config.cfg.g_input_bit_depth); - } + /* Even if bit depth is set on the command line flag to be lower, + * it is upgraded to at least match the input bit depth. + */ + assert(stream->config.cfg.g_input_bit_depth <= + (unsigned int)stream->config.cfg.g_bit_depth); for (streami = stream; streami; streami = streami->next) { /* All streams require output files */ @@ -1378,6 +1539,7 @@ static const char *image_format_to_string(aom_img_fmt_t f) { case AOM_IMG_FMT_I422: return "I422"; case AOM_IMG_FMT_I444: return "I444"; case AOM_IMG_FMT_YV12: return "YV12"; + case AOM_IMG_FMT_YV1216: return "YV1216"; case AOM_IMG_FMT_I42016: return "I42016"; case AOM_IMG_FMT_I42216: return "I42216"; case AOM_IMG_FMT_I44416: return "I44416"; @@ -1463,8 +1625,11 @@ static void open_output_file(struct stream_state *stream, #if CONFIG_WEBM_IO if (stream->config.write_webm) { stream->webm_ctx.stream = stream->file; - write_webm_file_header(&stream->webm_ctx, cfg, stream->config.stereo_fmt, - global->codec->fourcc, pixel_aspect_ratio); + if (write_webm_file_header(&stream->webm_ctx, &stream->encoder, cfg, + stream->config.stereo_fmt, global->codec->fourcc, + pixel_aspect_ratio) != 0) { + fatal("WebM writer initialization failed."); + } } #else (void)pixel_aspect_ratio; @@ -1483,7 +1648,9 @@ static void close_output_file(struct stream_state *stream, #if CONFIG_WEBM_IO if (stream->config.write_webm) { - write_webm_file_footer(&stream->webm_ctx); + if (write_webm_file_footer(&stream->webm_ctx) != 0) { + fatal("WebM writer finalization failed."); + } } #endif @@ -1699,16 +1866,16 @@ static void get_cx_data(struct stream_state *stream, switch (pkt->kind) { case AOM_CODEC_CX_FRAME_PKT: - if (!(pkt->data.frame.flags & AOM_FRAME_IS_FRAGMENT)) { - stream->frames_out++; - } + ++stream->frames_out; if (!global->quiet) fprintf(stderr, " %6luF", (unsigned long)pkt->data.frame.sz); update_rate_histogram(stream->rate_hist, cfg, pkt); #if CONFIG_WEBM_IO if (stream->config.write_webm) { - write_webm_block(&stream->webm_ctx, cfg, pkt); + if (write_webm_block(&stream->webm_ctx, cfg, pkt) != 0) { + fatal("WebM writer failed."); + } } #endif if (!stream->config.write_webm) { @@ -1721,12 +1888,10 @@ static void get_cx_data(struct stream_state *stream, } else { fsize += pkt->data.frame.sz; - if (!(pkt->data.frame.flags & AOM_FRAME_IS_FRAGMENT)) { - const FileOffset currpos = ftello(stream->file); - fseeko(stream->file, ivf_header_pos, SEEK_SET); - ivf_write_frame_size(stream->file, fsize); - fseeko(stream->file, currpos, SEEK_SET); - } + const FileOffset currpos = ftello(stream->file); + fseeko(stream->file, ivf_header_pos, SEEK_SET); + ivf_write_frame_size(stream->file, fsize); + fseeko(stream->file, currpos, SEEK_SET); } } @@ -1913,7 +2078,7 @@ int main(int argc, const char **argv_) { * codec. */ argv = argv_dup(argc - 1, argv_ + 1); - parse_global_config(&global, &argc, &argv); + parse_global_config(&global, argc, &argv); #if CONFIG_FILEOPTIONS if (argc < 2) usage_exit(); @@ -1950,6 +2115,10 @@ int main(int argc, const char **argv_) { FOREACH_STREAM(stream, streams) { check_encoder_config(global.disable_warning_prompt, &global, &stream->config.cfg); + + // If large_scale_tile = 1, only support to output to ivf format. + if (stream->config.cfg.large_scale_tile && !stream->config.write_ivf) + die("only support ivf output format while large-scale-tile=1\n"); } /* Handle non-option arguments */ @@ -1969,7 +2138,7 @@ int main(int argc, const char **argv_) { int64_t average_rate = -1; int64_t lagged_count = 0; - open_input_file(&input); + open_input_file(&input, global.csp); /* If the input file doesn't specify its w/h (raw files), try to get * the data from the first stream's configuration. @@ -2069,6 +2238,18 @@ int main(int argc, const char **argv_) { default: break; } } + /* Automatically set the codec bit depth to match the input bit depth. + * Upgrade the profile if required. */ + if (stream->config.cfg.g_input_bit_depth > + (unsigned int)stream->config.cfg.g_bit_depth) { + stream->config.cfg.g_bit_depth = stream->config.cfg.g_input_bit_depth; + if (!global.quiet) { + fprintf(stderr, + "Warning: automatically updating bit depth to %d to " + "match input format.\n", + stream->config.cfg.g_input_bit_depth); + } + } if (stream->config.cfg.g_bit_depth > 10) { switch (stream->config.cfg.g_profile) { case 0: @@ -2153,10 +2334,11 @@ int main(int argc, const char **argv_) { } FOREACH_STREAM(stream, streams) { setup_pass(stream, &global, pass); } + FOREACH_STREAM(stream, streams) { initialize_encoder(stream, &global); } FOREACH_STREAM(stream, streams) { open_output_file(stream, &global, &input.pixel_aspect_ratio); } - FOREACH_STREAM(stream, streams) { initialize_encoder(stream, &global); } + if (strcmp(global.codec->name, "av1") == 0 || strcmp(global.codec->name, "av1") == 0) { // Check to see if at least one stream uses 16 bit internal. diff --git a/third_party/aom/apps/aomenc.h b/third_party/aom/apps/aomenc.h index 7c23df0068..5e59c1ac5d 100644 --- a/third_party/aom/apps/aomenc.h +++ b/third_party/aom/apps/aomenc.h @@ -53,6 +53,7 @@ struct AvxEncoderConfig { int disable_warnings; int disable_warning_prompt; int experimental_bitstream; + aom_chroma_sample_position_t csp; }; #ifdef __cplusplus diff --git a/third_party/aom/av1/av1.cmake b/third_party/aom/av1/av1.cmake index 3a7cd7ee13..0b4901a619 100644 --- a/third_party/aom/av1/av1.cmake +++ b/third_party/aom/av1/av1.cmake @@ -100,7 +100,6 @@ list(APPEND AOM_AV1_DECODER_SOURCES "${AOM_ROOT}/av1/decoder/decodetxb.h" "${AOM_ROOT}/av1/decoder/detokenize.c" "${AOM_ROOT}/av1/decoder/detokenize.h" - "${AOM_ROOT}/av1/decoder/dthread.c" "${AOM_ROOT}/av1/decoder/dthread.h" "${AOM_ROOT}/av1/decoder/obu.h" "${AOM_ROOT}/av1/decoder/obu.c") @@ -117,6 +116,8 @@ list(APPEND AOM_AV1_ENCODER_SOURCES "${AOM_ROOT}/av1/encoder/av1_fwd_txfm1d.h" "${AOM_ROOT}/av1/encoder/av1_fwd_txfm1d_cfg.h" "${AOM_ROOT}/av1/encoder/av1_fwd_txfm2d.c" + "${AOM_ROOT}/av1/encoder/av1_multi_thread.c" + "${AOM_ROOT}/av1/encoder/av1_multi_thread.h" "${AOM_ROOT}/av1/encoder/av1_quantize.c" "${AOM_ROOT}/av1/encoder/av1_quantize.h" "${AOM_ROOT}/av1/encoder/bitstream.c" @@ -136,6 +137,8 @@ list(APPEND AOM_AV1_ENCODER_SOURCES "${AOM_ROOT}/av1/encoder/encodemb.h" "${AOM_ROOT}/av1/encoder/encodemv.c" "${AOM_ROOT}/av1/encoder/encodemv.h" + "${AOM_ROOT}/av1/encoder/encode_strategy.c" + "${AOM_ROOT}/av1/encoder/encode_strategy.h" "${AOM_ROOT}/av1/encoder/encoder.c" "${AOM_ROOT}/av1/encoder/encoder.h" "${AOM_ROOT}/av1/encoder/encodetxb.c" @@ -188,6 +191,8 @@ list(APPEND AOM_AV1_ENCODER_SOURCES "${AOM_ROOT}/av1/encoder/temporal_filter.h" "${AOM_ROOT}/av1/encoder/tokenize.c" "${AOM_ROOT}/av1/encoder/tokenize.h" + "${AOM_ROOT}/av1/encoder/tpl_model.c" + "${AOM_ROOT}/av1/encoder/tpl_model.h" "${AOM_ROOT}/av1/encoder/wedge_utils.c" "${AOM_ROOT}/third_party/fastfeat/fast.c" "${AOM_ROOT}/third_party/fastfeat/fast.h" @@ -263,6 +268,8 @@ list(APPEND AOM_AV1_ENCODER_INTRIN_SSE2 "${AOM_ROOT}/av1/encoder/x86/highbd_block_error_intrin_sse2.c" "${AOM_ROOT}/av1/encoder/x86/wedge_utils_sse2.c") +list(APPEND AOM_AV1_ENCODER_INTRIN_SSE3 "${AOM_ROOT}/av1/encoder/x86/ml_sse3.c") + list(APPEND AOM_AV1_ENCODER_ASM_SSSE3_X86_64 "${AOM_ROOT}/av1/encoder/x86/av1_quantize_ssse3_x86_64.asm") @@ -273,16 +280,20 @@ list(APPEND AOM_AV1_ENCODER_INTRIN_SSE4_1 "${AOM_ROOT}/av1/encoder/x86/corner_match_sse4.c" "${AOM_ROOT}/av1/encoder/x86/encodetxb_sse4.c" "${AOM_ROOT}/av1/encoder/x86/highbd_fwd_txfm_sse4.c" + "${AOM_ROOT}/av1/encoder/x86/rdopt_sse4.c" "${AOM_ROOT}/av1/encoder/x86/pickrst_sse4.c") list(APPEND AOM_AV1_ENCODER_INTRIN_AVX2 "${AOM_ROOT}/av1/encoder/x86/av1_quantize_avx2.c" "${AOM_ROOT}/av1/encoder/x86/av1_highbd_quantize_avx2.c" + "${AOM_ROOT}/av1/encoder/x86/corner_match_avx2.c" "${AOM_ROOT}/av1/encoder/x86/error_intrin_avx2.c" "${AOM_ROOT}/av1/encoder/x86/av1_fwd_txfm_avx2.h" "${AOM_ROOT}/av1/encoder/x86/av1_fwd_txfm2d_avx2.c" + "${AOM_ROOT}/av1/encoder/x86/highbd_fwd_txfm_avx2.c" "${AOM_ROOT}/av1/encoder/x86/wedge_utils_avx2.c" "${AOM_ROOT}/av1/encoder/x86/encodetxb_avx2.c" + "${AOM_ROOT}/av1/encoder/x86/rdopt_avx2.c" "${AOM_ROOT}/av1/encoder/x86/pickrst_avx2.c") list(APPEND AOM_AV1_ENCODER_INTRIN_NEON @@ -379,6 +390,14 @@ function(setup_av1_targets) endif() endif() + if(HAVE_SSE3) + require_compiler_flag_nomsvc("-msse3" NO) + if(CONFIG_AV1_ENCODER) + add_intrinsics_object_library("-msse3" "sse3" "aom_av1_encoder" + "AOM_AV1_ENCODER_INTRIN_SSE3" "aom") + endif() + endif() + if(HAVE_SSSE3) require_compiler_flag_nomsvc("-mssse3" NO) add_intrinsics_object_library("-mssse3" "ssse3" "aom_av1_common" diff --git a/third_party/aom/av1/av1_cx_iface.c b/third_party/aom/av1/av1_cx_iface.c index 3295f618aa..285e6a5086 100644 --- a/third_party/aom/av1/av1_cx_iface.c +++ b/third_party/aom/av1/av1_cx_iface.c @@ -39,10 +39,12 @@ struct av1_extracfg { unsigned int row_mt; unsigned int tile_columns; // log2 number of tile columns unsigned int tile_rows; // log2 number of tile rows + unsigned int enable_tpl_model; unsigned int arnr_max_frames; unsigned int arnr_strength; unsigned int min_gf_interval; unsigned int max_gf_interval; + unsigned int gf_max_pyr_height; aom_tune_metric tuning; unsigned int cq_level; // constrained quality level unsigned int rc_max_intra_bitrate_pct; @@ -51,6 +53,7 @@ struct av1_extracfg { unsigned int lossless; unsigned int enable_cdef; unsigned int enable_restoration; + unsigned int enable_obmc; unsigned int disable_trellis_quant; unsigned int enable_qm; unsigned int qm_y; @@ -66,7 +69,7 @@ struct av1_extracfg { aom_timing_info_type_t timing_info_type; unsigned int frame_parallel_decoding_mode; - int use_dual_filter; + int enable_dual_filter; AQ_MODE aq_mode; DELTAQ_MODE deltaq_mode; unsigned int frame_periodic_boost; @@ -88,13 +91,31 @@ struct av1_extracfg { const char *film_grain_table_filename; unsigned int motion_vector_unit_test; unsigned int cdf_update_mode; - int enable_order_hint; - int enable_jnt_comp; - int enable_ref_frame_mvs; // sequence level - int allow_ref_frame_mvs; // frame level - int enable_warped_motion; // sequence level - int allow_warped_motion; // frame level + int enable_rect_partitions; // enable rectangular partitions for sequence + int enable_intra_edge_filter; // enable intra-edge filter for sequence + int enable_order_hint; // enable order hint for sequence + int enable_tx64; // enable 64-pt transform usage for sequence + int enable_dist_wtd_comp; // enable dist wtd compound for sequence + int max_reference_frames; // maximum number of references per frame + int enable_ref_frame_mvs; // sequence level + int allow_ref_frame_mvs; // frame level + int enable_masked_comp; // enable masked compound for sequence + int enable_interintra_comp; // enable interintra compound for sequence + int enable_smooth_interintra; // enable smooth interintra mode usage + int enable_diff_wtd_comp; // enable diff-wtd compound usage + int enable_interinter_wedge; // enable interinter-wedge compound usage + int enable_interintra_wedge; // enable interintra-wedge compound usage + int enable_global_motion; // enable global motion usage for sequence + int enable_warped_motion; // sequence level + int allow_warped_motion; // frame level + int enable_filter_intra; // enable filter intra for sequence + int enable_smooth_intra; // enable smooth intra modes for sequence + int enable_paeth_intra; // enable Peeth intra mode for sequence + int enable_cfl_intra; // enable CFL uv intra mode for sequence int enable_superres; + int enable_palette; + int enable_intrabc; + int enable_angle_delta; #if CONFIG_DENOISE float noise_level; int noise_block_size; @@ -102,6 +123,10 @@ struct av1_extracfg { unsigned int chroma_subsampling_x; unsigned int chroma_subsampling_y; + int reduced_tx_type_set; + int use_intra_dct_only; + int use_inter_dct_only; + int quant_b_adapt; }; static struct av1_extracfg default_extra_cfg = { @@ -111,13 +136,15 @@ static struct av1_extracfg default_extra_cfg = { 0, // noise_sensitivity CONFIG_SHARP_SETTINGS, // sharpness 0, // static_thresh - 0, // row_mt + 1, // row_mt 0, // tile_columns 0, // tile_rows + 0, // enable_tpl_model 7, // arnr_max_frames 5, // arnr_strength 0, // min_gf_interval; 0 -> default decision 0, // max_gf_interval; 0 -> default decision + 4, // gf_max_pyr_height AOM_TUNE_PSNR, // tuning 10, // cq_level 0, // rc_max_intra_bitrate_pct @@ -126,6 +153,7 @@ static struct av1_extracfg default_extra_cfg = { 0, // lossless !CONFIG_SHARP_SETTINGS, // enable_cdef 1, // enable_restoration + 1, // enable_obmc 0, // disable_trellis_quant 0, // enable_qm DEFAULT_QM_Y, // qm_y @@ -161,19 +189,41 @@ static struct av1_extracfg default_extra_cfg = { 0, // film_grain_table_filename 0, // motion_vector_unit_test 1, // CDF update mode + 1, // enable rectangular partitions + 1, // enable intra edge filter 1, // frame order hint - 1, // jnt_comp + 1, // enable 64-pt transform usage + 1, // dist-wtd compound + 7, // max_reference_frames 1, // enable_ref_frame_mvs sequence level 1, // allow ref_frame_mvs frame level + 1, // enable masked compound at sequence level + 1, // enable interintra compound at sequence level + 1, // enable smooth interintra mode + 1, // enable difference-weighted compound + 1, // enable interinter wedge compound + 1, // enable interintra wedge compound + 1, // enable_global_motion usage 1, // enable_warped_motion at sequence level 1, // allow_warped_motion at frame level + 1, // enable filter intra at sequence level + 1, // enable smooth intra modes usage for sequence + 1, // enable Paeth intra mode usage for sequence + 1, // enable CFL uv intra mode usage for sequence 1, // superres + 1, // enable palette + 1, // enable intrabc + 1, // enable angle delta #if CONFIG_DENOISE 0, // noise_level 32, // noise_block_size #endif 0, // chroma_subsampling_x 0, // chroma_subsampling_y + 0, // reduced_tx_type_set + 0, // use_intra_dct_only + 0, // use_inter_dct_only + 0, // quant_b_adapt }; struct aom_codec_alg_priv { @@ -245,7 +295,7 @@ static aom_codec_err_t validate_config(aom_codec_alg_priv_t *ctx, RANGE_CHECK_HI(extra_cfg, aq_mode, AQ_MODE_COUNT - 1); RANGE_CHECK_HI(extra_cfg, deltaq_mode, DELTAQ_MODE_COUNT - 1); RANGE_CHECK_HI(extra_cfg, frame_periodic_boost, 1); - RANGE_CHECK_HI(cfg, g_threads, 64); + RANGE_CHECK_HI(cfg, g_threads, MAX_NUM_THREADS); RANGE_CHECK_HI(cfg, g_lag_in_frames, MAX_LAG_BUFFERS); RANGE_CHECK(cfg, rc_end_usage, AOM_VBR, AOM_Q); RANGE_CHECK_HI(cfg, rc_undershoot_pct, 100); @@ -260,6 +310,7 @@ static aom_codec_err_t validate_config(aom_codec_alg_priv_t *ctx, RANGE_CHECK(extra_cfg, max_gf_interval, MAX(2, extra_cfg->min_gf_interval), (MAX_LAG_BUFFERS - 1)); } + RANGE_CHECK(extra_cfg, gf_max_pyr_height, 1, 4); RANGE_CHECK_HI(cfg, rc_resize_mode, RESIZE_MODES - 1); RANGE_CHECK(cfg, rc_resize_denominator, SCALE_NUMERATOR, @@ -284,7 +335,7 @@ static aom_codec_err_t validate_config(aom_codec_alg_priv_t *ctx, "or kf_max_dist instead."); RANGE_CHECK_HI(extra_cfg, motion_vector_unit_test, 2); - RANGE_CHECK_HI(extra_cfg, enable_auto_alt_ref, 2); + RANGE_CHECK_HI(extra_cfg, enable_auto_alt_ref, 1); RANGE_CHECK_HI(extra_cfg, enable_auto_bwd_ref, 2); RANGE_CHECK(extra_cfg, cpu_used, 0, 8); RANGE_CHECK_HI(extra_cfg, noise_sensitivity, 6); @@ -376,6 +427,7 @@ static aom_codec_err_t validate_config(aom_codec_alg_priv_t *ctx, #endif } + RANGE_CHECK(extra_cfg, max_reference_frames, 3, 7); RANGE_CHECK_HI(extra_cfg, chroma_subsampling_x, 1); RANGE_CHECK_HI(extra_cfg, chroma_subsampling_y, 1); @@ -387,6 +439,7 @@ static aom_codec_err_t validate_img(aom_codec_alg_priv_t *ctx, switch (img->fmt) { case AOM_IMG_FMT_YV12: case AOM_IMG_FMT_I420: + case AOM_IMG_FMT_YV1216: case AOM_IMG_FMT_I42016: break; case AOM_IMG_FMT_I444: case AOM_IMG_FMT_I44416: @@ -420,6 +473,7 @@ static int get_image_bps(const aom_image_t *img) { case AOM_IMG_FMT_I420: return 12; case AOM_IMG_FMT_I422: return 16; case AOM_IMG_FMT_I444: return 24; + case AOM_IMG_FMT_YV1216: case AOM_IMG_FMT_I42016: return 24; case AOM_IMG_FMT_I42216: return 32; case AOM_IMG_FMT_I44416: return 48; @@ -514,6 +568,10 @@ static aom_codec_err_t set_encoder_config( oxcf->enable_cdef = extra_cfg->enable_cdef; oxcf->enable_restoration = extra_cfg->enable_restoration; + oxcf->enable_obmc = extra_cfg->enable_obmc; + oxcf->enable_palette = extra_cfg->enable_palette; + oxcf->enable_intrabc = extra_cfg->enable_intrabc; + oxcf->enable_angle_delta = extra_cfg->enable_angle_delta; oxcf->disable_trellis_quant = extra_cfg->disable_trellis_quant; oxcf->using_qm = extra_cfg->enable_qm; oxcf->qm_y = extra_cfg->qm_y; @@ -521,6 +579,10 @@ static aom_codec_err_t set_encoder_config( oxcf->qm_v = extra_cfg->qm_v; oxcf->qm_minlevel = extra_cfg->qm_min; oxcf->qm_maxlevel = extra_cfg->qm_max; + oxcf->reduced_tx_type_set = extra_cfg->reduced_tx_type_set; + oxcf->use_intra_dct_only = extra_cfg->use_intra_dct_only; + oxcf->use_inter_dct_only = extra_cfg->use_inter_dct_only; + oxcf->quant_b_adapt = extra_cfg->quant_b_adapt; #if CONFIG_DIST_8X8 oxcf->using_dist_8x8 = extra_cfg->enable_dist_8x8; if (extra_cfg->tuning == AOM_TUNE_CDEF_DIST || @@ -570,6 +632,9 @@ static aom_codec_err_t set_encoder_config( } } + oxcf->enable_tpl_model = + extra_cfg->enable_tpl_model && (oxcf->superres_mode == SUPERRES_NONE); + oxcf->maximum_buffer_size_ms = is_vbr ? 240000 : cfg->rc_buf_sz; oxcf->starting_buffer_level_ms = is_vbr ? 60000 : cfg->rc_buf_initial_sz; oxcf->optimal_buffer_level_ms = is_vbr ? 60000 : cfg->rc_buf_optimal_sz; @@ -614,6 +679,7 @@ static aom_codec_err_t set_encoder_config( oxcf->arnr_strength = extra_cfg->arnr_strength; oxcf->min_gf_interval = extra_cfg->min_gf_interval; oxcf->max_gf_interval = extra_cfg->max_gf_interval; + oxcf->gf_max_pyr_height = extra_cfg->gf_max_pyr_height; oxcf->tuning = extra_cfg->tuning; oxcf->content = extra_cfg->content; @@ -650,16 +716,39 @@ static aom_codec_err_t set_encoder_config( oxcf->monochrome = cfg->monochrome; oxcf->full_still_picture_hdr = cfg->full_still_picture_hdr; - oxcf->enable_dual_filter = extra_cfg->use_dual_filter; + oxcf->enable_dual_filter = extra_cfg->enable_dual_filter; + oxcf->enable_rect_partitions = extra_cfg->enable_rect_partitions; + oxcf->enable_intra_edge_filter = extra_cfg->enable_intra_edge_filter; + oxcf->enable_tx64 = extra_cfg->enable_tx64; oxcf->enable_order_hint = extra_cfg->enable_order_hint; - oxcf->enable_jnt_comp = - extra_cfg->enable_jnt_comp & extra_cfg->enable_order_hint; + oxcf->enable_dist_wtd_comp = + extra_cfg->enable_dist_wtd_comp & extra_cfg->enable_order_hint; + oxcf->max_reference_frames = extra_cfg->max_reference_frames; + if (oxcf->max_reference_frames > 3 && oxcf->max_reference_frames < 7) { + // TODO(urvang): Enable all possible values, after they work properly. + oxcf->max_reference_frames = 3; + } + oxcf->enable_masked_comp = extra_cfg->enable_masked_comp; + oxcf->enable_diff_wtd_comp = + extra_cfg->enable_masked_comp & extra_cfg->enable_diff_wtd_comp; + oxcf->enable_interinter_wedge = + extra_cfg->enable_masked_comp & extra_cfg->enable_interinter_wedge; + oxcf->enable_interintra_comp = extra_cfg->enable_interintra_comp; + oxcf->enable_smooth_interintra = + extra_cfg->enable_interintra_comp && extra_cfg->enable_smooth_interintra; + oxcf->enable_interintra_wedge = + extra_cfg->enable_interintra_comp & extra_cfg->enable_interintra_wedge; oxcf->enable_ref_frame_mvs = extra_cfg->enable_ref_frame_mvs & extra_cfg->enable_order_hint; + oxcf->enable_global_motion = extra_cfg->enable_global_motion; oxcf->enable_warped_motion = extra_cfg->enable_warped_motion; oxcf->allow_warped_motion = extra_cfg->allow_warped_motion & extra_cfg->enable_warped_motion; + oxcf->enable_filter_intra = extra_cfg->enable_filter_intra; + oxcf->enable_smooth_intra = extra_cfg->enable_smooth_intra; + oxcf->enable_paeth_intra = extra_cfg->enable_paeth_intra; + oxcf->enable_cfl_intra = extra_cfg->enable_cfl_intra; oxcf->enable_superres = (oxcf->superres_mode != SUPERRES_NONE) && extra_cfg->enable_superres; @@ -701,23 +790,11 @@ static aom_codec_err_t set_encoder_config( oxcf->frame_periodic_boost = extra_cfg->frame_periodic_boost; oxcf->motion_vector_unit_test = extra_cfg->motion_vector_unit_test; -#if CONFIG_REDUCED_ENCODER_BORDER - if (oxcf->superres_mode != SUPERRES_NONE || - oxcf->resize_mode != RESIZE_NONE) { - warn( - "Superres / resize cannot be used with CONFIG_REDUCED_ENCODER_BORDER. " - "Disabling superres/resize.\n"); - // return AOM_CODEC_INVALID_PARAM; - disable_superres(oxcf); - oxcf->resize_mode = RESIZE_NONE; - oxcf->resize_scale_denominator = SCALE_NUMERATOR; - oxcf->resize_kf_scale_denominator = SCALE_NUMERATOR; - } -#endif // CONFIG_REDUCED_ENCODER_BORDER - oxcf->chroma_subsampling_x = extra_cfg->chroma_subsampling_x; oxcf->chroma_subsampling_y = extra_cfg->chroma_subsampling_y; - + oxcf->border_in_pixels = (oxcf->resize_mode || oxcf->superres_mode) + ? AOM_BORDER_IN_PIXELS + : AOM_ENC_NO_SCALE_BORDER; return AOM_CODEC_OK; } @@ -851,6 +928,13 @@ static aom_codec_err_t ctrl_set_tile_rows(aom_codec_alg_priv_t *ctx, return update_extra_cfg(ctx, &extra_cfg); } +static aom_codec_err_t ctrl_set_enable_tpl_model(aom_codec_alg_priv_t *ctx, + va_list args) { + struct av1_extracfg extra_cfg = ctx->extra_cfg; + extra_cfg.enable_tpl_model = CAST(AV1E_SET_ENABLE_TPL_MODEL, args); + return update_extra_cfg(ctx, &extra_cfg); +} + static aom_codec_err_t ctrl_set_arnr_max_frames(aom_codec_alg_priv_t *ctx, va_list args) { struct av1_extracfg extra_cfg = ctx->extra_cfg; @@ -923,6 +1007,13 @@ static aom_codec_err_t ctrl_set_enable_restoration(aom_codec_alg_priv_t *ctx, return update_extra_cfg(ctx, &extra_cfg); } +static aom_codec_err_t ctrl_set_enable_obmc(aom_codec_alg_priv_t *ctx, + va_list args) { + struct av1_extracfg extra_cfg = ctx->extra_cfg; + extra_cfg.enable_obmc = CAST(AV1E_SET_ENABLE_OBMC, args); + return update_extra_cfg(ctx, &extra_cfg); +} + static aom_codec_err_t ctrl_set_disable_trellis_quant(aom_codec_alg_priv_t *ctx, va_list args) { struct av1_extracfg extra_cfg = ctx->extra_cfg; @@ -991,10 +1082,26 @@ static aom_codec_err_t ctrl_set_timing_info_type(aom_codec_alg_priv_t *ctx, return update_extra_cfg(ctx, &extra_cfg); } -static aom_codec_err_t ctrl_set_enable_df(aom_codec_alg_priv_t *ctx, - va_list args) { +static aom_codec_err_t ctrl_set_enable_dual_filter(aom_codec_alg_priv_t *ctx, + va_list args) { struct av1_extracfg extra_cfg = ctx->extra_cfg; - extra_cfg.use_dual_filter = CAST(AV1E_SET_ENABLE_DF, args); + extra_cfg.enable_dual_filter = CAST(AV1E_SET_ENABLE_DUAL_FILTER, args); + return update_extra_cfg(ctx, &extra_cfg); +} + +static aom_codec_err_t ctrl_set_enable_rect_partitions( + aom_codec_alg_priv_t *ctx, va_list args) { + struct av1_extracfg extra_cfg = ctx->extra_cfg; + extra_cfg.enable_rect_partitions = + CAST(AV1E_SET_ENABLE_RECT_PARTITIONS, args); + return update_extra_cfg(ctx, &extra_cfg); +} + +static aom_codec_err_t ctrl_set_enable_intra_edge_filter( + aom_codec_alg_priv_t *ctx, va_list args) { + struct av1_extracfg extra_cfg = ctx->extra_cfg; + extra_cfg.enable_intra_edge_filter = + CAST(AV1E_SET_ENABLE_INTRA_EDGE_FILTER, args); return update_extra_cfg(ctx, &extra_cfg); } @@ -1005,10 +1112,24 @@ static aom_codec_err_t ctrl_set_enable_order_hint(aom_codec_alg_priv_t *ctx, return update_extra_cfg(ctx, &extra_cfg); } -static aom_codec_err_t ctrl_set_enable_jnt_comp(aom_codec_alg_priv_t *ctx, - va_list args) { +static aom_codec_err_t ctrl_set_enable_tx64(aom_codec_alg_priv_t *ctx, + va_list args) { struct av1_extracfg extra_cfg = ctx->extra_cfg; - extra_cfg.enable_jnt_comp = CAST(AV1E_SET_ENABLE_JNT_COMP, args); + extra_cfg.enable_tx64 = CAST(AV1E_SET_ENABLE_TX64, args); + return update_extra_cfg(ctx, &extra_cfg); +} + +static aom_codec_err_t ctrl_set_enable_dist_wtd_comp(aom_codec_alg_priv_t *ctx, + va_list args) { + struct av1_extracfg extra_cfg = ctx->extra_cfg; + extra_cfg.enable_dist_wtd_comp = CAST(AV1E_SET_ENABLE_DIST_WTD_COMP, args); + return update_extra_cfg(ctx, &extra_cfg); +} + +static aom_codec_err_t ctrl_set_max_reference_frames(aom_codec_alg_priv_t *ctx, + va_list args) { + struct av1_extracfg extra_cfg = ctx->extra_cfg; + extra_cfg.max_reference_frames = CAST(AV1E_SET_MAX_REFERENCE_FRAMES, args); return update_extra_cfg(ctx, &extra_cfg); } @@ -1026,6 +1147,59 @@ static aom_codec_err_t ctrl_set_allow_ref_frame_mvs(aom_codec_alg_priv_t *ctx, return update_extra_cfg(ctx, &extra_cfg); } +static aom_codec_err_t ctrl_set_enable_masked_comp(aom_codec_alg_priv_t *ctx, + va_list args) { + struct av1_extracfg extra_cfg = ctx->extra_cfg; + extra_cfg.enable_masked_comp = CAST(AV1E_SET_ENABLE_MASKED_COMP, args); + return update_extra_cfg(ctx, &extra_cfg); +} + +static aom_codec_err_t ctrl_set_enable_interintra_comp( + aom_codec_alg_priv_t *ctx, va_list args) { + struct av1_extracfg extra_cfg = ctx->extra_cfg; + extra_cfg.enable_interintra_comp = + CAST(AV1E_SET_ENABLE_INTERINTRA_COMP, args); + return update_extra_cfg(ctx, &extra_cfg); +} + +static aom_codec_err_t ctrl_set_enable_smooth_interintra( + aom_codec_alg_priv_t *ctx, va_list args) { + struct av1_extracfg extra_cfg = ctx->extra_cfg; + extra_cfg.enable_smooth_interintra = + CAST(AV1E_SET_ENABLE_SMOOTH_INTERINTRA, args); + return update_extra_cfg(ctx, &extra_cfg); +} + +static aom_codec_err_t ctrl_set_enable_diff_wtd_comp(aom_codec_alg_priv_t *ctx, + va_list args) { + struct av1_extracfg extra_cfg = ctx->extra_cfg; + extra_cfg.enable_diff_wtd_comp = CAST(AV1E_SET_ENABLE_DIFF_WTD_COMP, args); + return update_extra_cfg(ctx, &extra_cfg); +} + +static aom_codec_err_t ctrl_set_enable_interinter_wedge( + aom_codec_alg_priv_t *ctx, va_list args) { + struct av1_extracfg extra_cfg = ctx->extra_cfg; + extra_cfg.enable_interinter_wedge = + CAST(AV1E_SET_ENABLE_INTERINTER_WEDGE, args); + return update_extra_cfg(ctx, &extra_cfg); +} + +static aom_codec_err_t ctrl_set_enable_interintra_wedge( + aom_codec_alg_priv_t *ctx, va_list args) { + struct av1_extracfg extra_cfg = ctx->extra_cfg; + extra_cfg.enable_interintra_wedge = + CAST(AV1E_SET_ENABLE_INTERINTRA_WEDGE, args); + return update_extra_cfg(ctx, &extra_cfg); +} + +static aom_codec_err_t ctrl_set_enable_global_motion(aom_codec_alg_priv_t *ctx, + va_list args) { + struct av1_extracfg extra_cfg = ctx->extra_cfg; + extra_cfg.enable_global_motion = CAST(AV1E_SET_ENABLE_GLOBAL_MOTION, args); + return update_extra_cfg(ctx, &extra_cfg); +} + static aom_codec_err_t ctrl_set_enable_warped_motion(aom_codec_alg_priv_t *ctx, va_list args) { struct av1_extracfg extra_cfg = ctx->extra_cfg; @@ -1040,6 +1214,34 @@ static aom_codec_err_t ctrl_set_allow_warped_motion(aom_codec_alg_priv_t *ctx, return update_extra_cfg(ctx, &extra_cfg); } +static aom_codec_err_t ctrl_set_enable_filter_intra(aom_codec_alg_priv_t *ctx, + va_list args) { + struct av1_extracfg extra_cfg = ctx->extra_cfg; + extra_cfg.enable_filter_intra = CAST(AV1E_SET_ENABLE_FILTER_INTRA, args); + return update_extra_cfg(ctx, &extra_cfg); +} + +static aom_codec_err_t ctrl_set_enable_smooth_intra(aom_codec_alg_priv_t *ctx, + va_list args) { + struct av1_extracfg extra_cfg = ctx->extra_cfg; + extra_cfg.enable_smooth_intra = CAST(AV1E_SET_ENABLE_SMOOTH_INTRA, args); + return update_extra_cfg(ctx, &extra_cfg); +} + +static aom_codec_err_t ctrl_set_enable_paeth_intra(aom_codec_alg_priv_t *ctx, + va_list args) { + struct av1_extracfg extra_cfg = ctx->extra_cfg; + extra_cfg.enable_paeth_intra = CAST(AV1E_SET_ENABLE_PAETH_INTRA, args); + return update_extra_cfg(ctx, &extra_cfg); +} + +static aom_codec_err_t ctrl_set_enable_cfl_intra(aom_codec_alg_priv_t *ctx, + va_list args) { + struct av1_extracfg extra_cfg = ctx->extra_cfg; + extra_cfg.enable_cfl_intra = CAST(AV1E_SET_ENABLE_CFL_INTRA, args); + return update_extra_cfg(ctx, &extra_cfg); +} + static aom_codec_err_t ctrl_set_enable_superres(aom_codec_alg_priv_t *ctx, va_list args) { struct av1_extracfg extra_cfg = ctx->extra_cfg; @@ -1047,6 +1249,27 @@ static aom_codec_err_t ctrl_set_enable_superres(aom_codec_alg_priv_t *ctx, return update_extra_cfg(ctx, &extra_cfg); } +static aom_codec_err_t ctrl_set_enable_palette(aom_codec_alg_priv_t *ctx, + va_list args) { + struct av1_extracfg extra_cfg = ctx->extra_cfg; + extra_cfg.enable_palette = CAST(AV1E_SET_ENABLE_PALETTE, args); + return update_extra_cfg(ctx, &extra_cfg); +} + +static aom_codec_err_t ctrl_set_enable_intrabc(aom_codec_alg_priv_t *ctx, + va_list args) { + struct av1_extracfg extra_cfg = ctx->extra_cfg; + extra_cfg.enable_intrabc = CAST(AV1E_SET_ENABLE_INTRABC, args); + return update_extra_cfg(ctx, &extra_cfg); +} + +static aom_codec_err_t ctrl_set_enable_angle_delta(aom_codec_alg_priv_t *ctx, + va_list args) { + struct av1_extracfg extra_cfg = ctx->extra_cfg; + extra_cfg.enable_angle_delta = CAST(AV1E_SET_ENABLE_ANGLE_DELTA, args); + return update_extra_cfg(ctx, &extra_cfg); +} + static aom_codec_err_t ctrl_set_error_resilient_mode(aom_codec_alg_priv_t *ctx, va_list args) { struct av1_extracfg extra_cfg = ctx->extra_cfg; @@ -1083,6 +1306,34 @@ static aom_codec_err_t ctrl_set_aq_mode(aom_codec_alg_priv_t *ctx, return update_extra_cfg(ctx, &extra_cfg); } +static aom_codec_err_t ctrl_set_reduced_tx_type_set(aom_codec_alg_priv_t *ctx, + va_list args) { + struct av1_extracfg extra_cfg = ctx->extra_cfg; + extra_cfg.reduced_tx_type_set = CAST(AV1E_SET_REDUCED_TX_TYPE_SET, args); + return update_extra_cfg(ctx, &extra_cfg); +} + +static aom_codec_err_t ctrl_set_intra_dct_only(aom_codec_alg_priv_t *ctx, + va_list args) { + struct av1_extracfg extra_cfg = ctx->extra_cfg; + extra_cfg.use_intra_dct_only = CAST(AV1E_SET_INTRA_DCT_ONLY, args); + return update_extra_cfg(ctx, &extra_cfg); +} + +static aom_codec_err_t ctrl_set_inter_dct_only(aom_codec_alg_priv_t *ctx, + va_list args) { + struct av1_extracfg extra_cfg = ctx->extra_cfg; + extra_cfg.use_inter_dct_only = CAST(AV1E_SET_INTER_DCT_ONLY, args); + return update_extra_cfg(ctx, &extra_cfg); +} + +static aom_codec_err_t ctrl_set_quant_b_adapt(aom_codec_alg_priv_t *ctx, + va_list args) { + struct av1_extracfg extra_cfg = ctx->extra_cfg; + extra_cfg.quant_b_adapt = CAST(AV1E_SET_QUANT_B_ADAPT, args); + return update_extra_cfg(ctx, &extra_cfg); +} + static aom_codec_err_t ctrl_set_film_grain_test_vector( aom_codec_alg_priv_t *ctx, va_list args) { struct av1_extracfg extra_cfg = ctx->extra_cfg; @@ -1136,6 +1387,13 @@ static aom_codec_err_t ctrl_set_max_gf_interval(aom_codec_alg_priv_t *ctx, return update_extra_cfg(ctx, &extra_cfg); } +static aom_codec_err_t ctrl_set_gf_max_pyr_height(aom_codec_alg_priv_t *ctx, + va_list args) { + struct av1_extracfg extra_cfg = ctx->extra_cfg; + extra_cfg.gf_max_pyr_height = CAST(AV1E_SET_GF_MAX_PYRAMID_HEIGHT, args); + return update_extra_cfg(ctx, &extra_cfg); +} + static aom_codec_err_t ctrl_set_frame_periodic_boost(aom_codec_alg_priv_t *ctx, va_list args) { struct av1_extracfg extra_cfg = ctx->extra_cfg; @@ -1214,7 +1472,10 @@ static aom_codec_frame_flags_t get_frame_pkt_flags(const AV1_COMP *cpi, aom_codec_frame_flags_t flags = lib_flags << 16; if (lib_flags & FRAMEFLAGS_KEY) flags |= AOM_FRAME_IS_KEY; - + if (lib_flags & FRAMEFLAGS_INTRAONLY) flags |= AOM_FRAME_IS_INTRAONLY; + if (lib_flags & FRAMEFLAGS_SWITCH) flags |= AOM_FRAME_IS_SWITCH; + if (lib_flags & FRAMEFLAGS_ERROR_RESILIENT) + flags |= AOM_FRAME_IS_ERROR_RESILIENT; if (cpi->droppable) flags |= AOM_FRAME_IS_DROPPABLE; return flags; @@ -1329,18 +1590,13 @@ static aom_codec_err_t encoder_encode(aom_codec_alg_priv_t *ctx, unsigned int lib_flags = 0; int is_frame_visible = 0; int index_size = 0; + int has_fwd_keyframe = 0; // invisible frames get packed with the next visible frame while (cx_data_sz - index_size >= ctx->cx_data_sz / 2 && !is_frame_visible && -1 != av1_get_compressed_data(cpi, &lib_flags, &frame_size, cx_data, &dst_time_stamp, &dst_end_time_stamp, !img, timebase)) { - if (cpi->common.seq_params.frame_id_numbers_present_flag) { - if (cpi->common.invalid_delta_frame_id_minus_1) { - aom_internal_error(&cpi->common.error, AOM_CODEC_ERROR, - "Invalid delta_frame_id_minus_1"); - } - } cpi->seq_params_locked = 1; if (frame_size) { if (ctx->pending_cx_data == 0) ctx->pending_cx_data = cx_data; @@ -1403,6 +1659,9 @@ static aom_codec_err_t encoder_encode(aom_codec_alg_priv_t *ctx, index_size = MAG_SIZE * (ctx->pending_frame_count - 1) + 2; is_frame_visible = cpi->common.show_frame; + + has_fwd_keyframe |= (!is_frame_visible && + cpi->common.current_frame.frame_type == KEY_FRAME); } } if (is_frame_visible) { @@ -1434,6 +1693,11 @@ static aom_codec_err_t encoder_encode(aom_codec_alg_priv_t *ctx, pkt.data.frame.pts = ticks_to_timebase_units(timebase, dst_time_stamp); pkt.data.frame.flags = get_frame_pkt_flags(cpi, lib_flags); + if (has_fwd_keyframe) { + // If one of the invisible frames in the packet is a keyframe, set + // the delayed random access point flag. + pkt.data.frame.flags |= AOM_FRAME_IS_DELAYED_RANDOM_ACCESS_POINT; + } pkt.data.frame.duration = (uint32_t)ticks_to_timebase_units( timebase, dst_end_time_stamp - dst_time_stamp); @@ -1733,6 +1997,7 @@ static aom_codec_ctrl_fn_map_t encoder_ctrl_maps[] = { { AV1E_SET_ROW_MT, ctrl_set_row_mt }, { AV1E_SET_TILE_COLUMNS, ctrl_set_tile_columns }, { AV1E_SET_TILE_ROWS, ctrl_set_tile_rows }, + { AV1E_SET_ENABLE_TPL_MODEL, ctrl_set_enable_tpl_model }, { AOME_SET_ARNR_MAXFRAMES, ctrl_set_arnr_max_frames }, { AOME_SET_ARNR_STRENGTH, ctrl_set_arnr_strength }, { AOME_SET_TUNING, ctrl_set_tuning }, @@ -1744,6 +2009,7 @@ static aom_codec_ctrl_fn_map_t encoder_ctrl_maps[] = { { AV1E_SET_LOSSLESS, ctrl_set_lossless }, { AV1E_SET_ENABLE_CDEF, ctrl_set_enable_cdef }, { AV1E_SET_ENABLE_RESTORATION, ctrl_set_enable_restoration }, + { AV1E_SET_ENABLE_OBMC, ctrl_set_enable_obmc }, { AV1E_SET_DISABLE_TRELLIS_QUANT, ctrl_set_disable_trellis_quant }, { AV1E_SET_ENABLE_QM, ctrl_set_enable_qm }, { AV1E_SET_QM_Y, ctrl_set_qm_y }, @@ -1760,15 +2026,37 @@ static aom_codec_ctrl_fn_map_t encoder_ctrl_maps[] = { { AV1E_SET_FRAME_PARALLEL_DECODING, ctrl_set_frame_parallel_decoding_mode }, { AV1E_SET_ERROR_RESILIENT_MODE, ctrl_set_error_resilient_mode }, { AV1E_SET_S_FRAME_MODE, ctrl_set_s_frame_mode }, - { AV1E_SET_ENABLE_DF, ctrl_set_enable_df }, + { AV1E_SET_ENABLE_RECT_PARTITIONS, ctrl_set_enable_rect_partitions }, + { AV1E_SET_ENABLE_DUAL_FILTER, ctrl_set_enable_dual_filter }, + { AV1E_SET_ENABLE_INTRA_EDGE_FILTER, ctrl_set_enable_intra_edge_filter }, { AV1E_SET_ENABLE_ORDER_HINT, ctrl_set_enable_order_hint }, - { AV1E_SET_ENABLE_JNT_COMP, ctrl_set_enable_jnt_comp }, + { AV1E_SET_ENABLE_TX64, ctrl_set_enable_tx64 }, + { AV1E_SET_ENABLE_DIST_WTD_COMP, ctrl_set_enable_dist_wtd_comp }, + { AV1E_SET_MAX_REFERENCE_FRAMES, ctrl_set_max_reference_frames }, { AV1E_SET_ENABLE_REF_FRAME_MVS, ctrl_set_enable_ref_frame_mvs }, { AV1E_SET_ALLOW_REF_FRAME_MVS, ctrl_set_allow_ref_frame_mvs }, + { AV1E_SET_ENABLE_MASKED_COMP, ctrl_set_enable_masked_comp }, + { AV1E_SET_ENABLE_INTERINTRA_COMP, ctrl_set_enable_interintra_comp }, + { AV1E_SET_ENABLE_SMOOTH_INTERINTRA, ctrl_set_enable_smooth_interintra }, + { AV1E_SET_ENABLE_DIFF_WTD_COMP, ctrl_set_enable_diff_wtd_comp }, + { AV1E_SET_ENABLE_INTERINTER_WEDGE, ctrl_set_enable_interinter_wedge }, + { AV1E_SET_ENABLE_INTERINTRA_WEDGE, ctrl_set_enable_interintra_wedge }, + { AV1E_SET_ENABLE_GLOBAL_MOTION, ctrl_set_enable_global_motion }, { AV1E_SET_ENABLE_WARPED_MOTION, ctrl_set_enable_warped_motion }, { AV1E_SET_ALLOW_WARPED_MOTION, ctrl_set_allow_warped_motion }, + { AV1E_SET_ENABLE_FILTER_INTRA, ctrl_set_enable_filter_intra }, + { AV1E_SET_ENABLE_SMOOTH_INTRA, ctrl_set_enable_smooth_intra }, + { AV1E_SET_ENABLE_PAETH_INTRA, ctrl_set_enable_paeth_intra }, + { AV1E_SET_ENABLE_CFL_INTRA, ctrl_set_enable_cfl_intra }, { AV1E_SET_ENABLE_SUPERRES, ctrl_set_enable_superres }, + { AV1E_SET_ENABLE_PALETTE, ctrl_set_enable_palette }, + { AV1E_SET_ENABLE_INTRABC, ctrl_set_enable_intrabc }, + { AV1E_SET_ENABLE_ANGLE_DELTA, ctrl_set_enable_angle_delta }, { AV1E_SET_AQ_MODE, ctrl_set_aq_mode }, + { AV1E_SET_REDUCED_TX_TYPE_SET, ctrl_set_reduced_tx_type_set }, + { AV1E_SET_INTRA_DCT_ONLY, ctrl_set_intra_dct_only }, + { AV1E_SET_INTER_DCT_ONLY, ctrl_set_inter_dct_only }, + { AV1E_SET_QUANT_B_ADAPT, ctrl_set_quant_b_adapt }, { AV1E_SET_DELTAQ_MODE, ctrl_set_deltaq_mode }, { AV1E_SET_FRAME_PERIODIC_BOOST, ctrl_set_frame_periodic_boost }, { AV1E_SET_TUNE_CONTENT, ctrl_set_tune_content }, @@ -1781,6 +2069,7 @@ static aom_codec_ctrl_fn_map_t encoder_ctrl_maps[] = { { AV1E_SET_NOISE_SENSITIVITY, ctrl_set_noise_sensitivity }, { AV1E_SET_MIN_GF_INTERVAL, ctrl_set_min_gf_interval }, { AV1E_SET_MAX_GF_INTERVAL, ctrl_set_max_gf_interval }, + { AV1E_SET_GF_MAX_PYRAMID_HEIGHT, ctrl_set_gf_max_pyr_height }, { AV1E_SET_RENDER_SIZE, ctrl_set_render_size }, { AV1E_SET_SUPERBLOCK_SIZE, ctrl_set_superblock_size }, { AV1E_SET_SINGLE_TILE_DECODING, ctrl_set_single_tile_decoding }, @@ -1837,7 +2126,7 @@ static aom_codec_enc_cfg_map_t encoder_usage_cfg_map[] = { SCALE_NUMERATOR, // rc_superres_denominator SCALE_NUMERATOR, // rc_superres_kf_denominator 63, // rc_superres_qthresh - 63, // rc_superres_kf_qthresh + 32, // rc_superres_kf_qthresh AOM_VBR, // rc_end_usage { NULL, 0 }, // rc_twopass_stats_in diff --git a/third_party/aom/av1/av1_dx_iface.c b/third_party/aom/av1/av1_dx_iface.c index 4a66310471..87a6e3dca0 100644 --- a/third_party/aom/av1/av1_dx_iface.c +++ b/third_party/aom/av1/av1_dx_iface.c @@ -44,7 +44,7 @@ struct aom_codec_alg_priv { int img_avail; int flushed; int invert_tile_order; - int last_show_frame; // Index of last output frame. + RefCntBuffer *last_show_frame; // Last output frame buffer int byte_alignment; int skip_loop_filter; int skip_film_grain; @@ -58,12 +58,13 @@ struct aom_codec_alg_priv { int operating_point; int output_all_layers; + // TODO(wtc): This can be simplified. num_frame_workers is always 1, and + // next_output_worker_id is always 0. The frame_workers array of size 1 can + // be replaced by a single AVxWorker. AVxWorker *frame_workers; int num_frame_workers; - int next_submit_worker_id; - int last_submit_worker_id; int next_output_worker_id; - int available_threads; + aom_image_t *image_with_grain[MAX_NUM_SPATIAL_LAYERS]; int need_resync; // wait for key/intra-only frame // BufferPool that holds all reference frames. Shared by all the FrameWorkers. @@ -132,11 +133,6 @@ static aom_codec_err_t decoder_destroy(aom_codec_alg_priv_t *ctx) { av1_remove_common(&frame_worker_data->pbi->common); av1_free_restoration_buffers(&frame_worker_data->pbi->common); av1_decoder_remove(frame_worker_data->pbi); - aom_free(frame_worker_data->scratch_buffer); -#if CONFIG_MULTITHREAD - pthread_mutex_destroy(&frame_worker_data->stats_mutex); - pthread_cond_destroy(&frame_worker_data->stats_cond); -#endif aom_free(frame_worker_data); } #if CONFIG_MULTITHREAD @@ -207,7 +203,7 @@ static aom_codec_err_t decoder_peek_si_internal(const uint8_t *data, memset(&obu_header, 0, sizeof(obu_header)); size_t payload_size = 0; size_t bytes_read = 0; - int reduced_still_picture_hdr = 0; + uint8_t reduced_still_picture_hdr = 0; aom_codec_err_t status = aom_read_obu_header_and_size( data, data_sz, si->is_annexb, &obu_header, &payload_size, &bytes_read); if (status != AOM_CODEC_OK) return status; @@ -236,7 +232,7 @@ static aom_codec_err_t decoder_peek_si_internal(const uint8_t *data, struct aom_read_bit_buffer rb = { data, data + data_sz, 0, NULL, NULL }; av1_read_profile(&rb); // profile - const int still_picture = aom_rb_read_bit(&rb); + const uint8_t still_picture = aom_rb_read_bit(&rb); reduced_still_picture_hdr = aom_rb_read_bit(&rb); if (!still_picture && reduced_still_picture_hdr) { @@ -321,7 +317,7 @@ static void init_buffer_callbacks(aom_codec_alg_priv_t *ctx) { AV1_COMMON *const cm = &frame_worker_data->pbi->common; BufferPool *const pool = cm->buffer_pool; - cm->new_fb_idx = INVALID_IDX; + cm->cur_frame = NULL; cm->byte_alignment = ctx->byte_alignment; cm->skip_loop_filter = ctx->skip_loop_filter; cm->skip_film_grain = ctx->skip_film_grain; @@ -360,7 +356,6 @@ static int frame_worker_hook(void *arg1, void *arg2) { if (result != 0) { // Check decode result in serial decode. - frame_worker_data->pbi->cur_buf->buf.corrupted = 1; frame_worker_data->pbi->need_resync = 1; } return !result; @@ -370,15 +365,12 @@ static aom_codec_err_t init_decoder(aom_codec_alg_priv_t *ctx) { int i; const AVxWorkerInterface *const winterface = aom_get_worker_interface(); - ctx->last_show_frame = -1; - ctx->next_submit_worker_id = 0; - ctx->last_submit_worker_id = 0; + ctx->last_show_frame = NULL; ctx->next_output_worker_id = 0; ctx->need_resync = 1; ctx->num_frame_workers = 1; if (ctx->num_frame_workers > MAX_DECODE_THREADS) ctx->num_frame_workers = MAX_DECODE_THREADS; - ctx->available_threads = ctx->num_frame_workers; ctx->flushed = 0; ctx->buffer_pool = (BufferPool *)aom_calloc(1, sizeof(BufferPool)); @@ -402,6 +394,7 @@ static aom_codec_err_t init_decoder(aom_codec_alg_priv_t *ctx) { AVxWorker *const worker = &ctx->frame_workers[i]; FrameWorkerData *frame_worker_data = NULL; winterface->init(worker); + worker->thread_name = "aom frameworker"; worker->data1 = aom_memalign(32, sizeof(FrameWorkerData)); if (worker->data1 == NULL) { set_error_detail(ctx, "Failed to allocate frame_worker_data"); @@ -414,23 +407,9 @@ static aom_codec_err_t init_decoder(aom_codec_alg_priv_t *ctx) { return AOM_CODEC_MEM_ERROR; } frame_worker_data->pbi->common.options = &ctx->cfg.cfg; - frame_worker_data->pbi->frame_worker_owner = worker; frame_worker_data->worker_id = i; - frame_worker_data->scratch_buffer = NULL; - frame_worker_data->scratch_buffer_size = 0; frame_worker_data->frame_context_ready = 0; frame_worker_data->received_frame = 0; -#if CONFIG_MULTITHREAD - if (pthread_mutex_init(&frame_worker_data->stats_mutex, NULL)) { - set_error_detail(ctx, "Failed to allocate frame_worker_data mutex"); - return AOM_CODEC_MEM_ERROR; - } - - if (pthread_cond_init(&frame_worker_data->stats_cond, NULL)) { - set_error_detail(ctx, "Failed to allocate frame_worker_data cond"); - return AOM_CODEC_MEM_ERROR; - } -#endif frame_worker_data->pbi->allow_lowbitdepth = ctx->cfg.allow_lowbitdepth; // If decoding in serial mode, FrameWorker thread could create tile worker @@ -447,7 +426,8 @@ static aom_codec_err_t init_decoder(aom_codec_alg_priv_t *ctx) { frame_worker_data->pbi->row_mt = ctx->row_mt; worker->hook = frame_worker_hook; - if (!winterface->reset(worker)) { + // The main thread acts as Frame Worker 0. + if (i != 0 && !winterface->reset(worker)) { set_error_detail(ctx, "Frame Worker thread creation failed"); return AOM_CODEC_MEM_ERROR; } @@ -467,7 +447,7 @@ static INLINE void check_resync(aom_codec_alg_priv_t *const ctx, const AV1Decoder *const pbi) { // Clear resync flag if worker got a key frame or intra only frame. if (ctx->need_resync == 1 && pbi->need_resync == 0 && - (pbi->common.intra_only || pbi->common.frame_type == KEY_FRAME)) + frame_is_intra_only(&pbi->common)) ctx->need_resync = 0; } @@ -496,11 +476,6 @@ static aom_codec_err_t decode_one(aom_codec_alg_priv_t *ctx, frame_worker_data->user_priv = user_priv; frame_worker_data->received_frame = 1; -#if CONFIG_INSPECTION - frame_worker_data->pbi->inspect_cb = ctx->inspect_cb; - frame_worker_data->pbi->inspect_ctx = ctx->inspect_ctx; -#endif - frame_worker_data->pbi->common.large_scale_tile = ctx->tile_mode; frame_worker_data->pbi->dec_tile_row = ctx->decode_tile_row; frame_worker_data->pbi->dec_tile_col = ctx->decode_tile_col; @@ -524,17 +499,55 @@ static aom_codec_err_t decode_one(aom_codec_alg_priv_t *ctx, return AOM_CODEC_OK; } +#if CONFIG_INSPECTION +// This function enables the inspector to inspect non visible frames. +static aom_codec_err_t decoder_inspect(aom_codec_alg_priv_t *ctx, + const uint8_t *data, size_t data_sz, + void *user_priv) { + aom_codec_err_t res = AOM_CODEC_OK; + + Av1DecodeReturn *data2 = (Av1DecodeReturn *)user_priv; + + if (ctx->frame_workers == NULL) { + res = init_decoder(ctx); + if (res != AOM_CODEC_OK) return res; + } + FrameWorkerData *const frame_worker_data = + (FrameWorkerData *)ctx->frame_workers[0].data1; + AV1Decoder *const pbi = frame_worker_data->pbi; + AV1_COMMON *const cm = &pbi->common; + frame_worker_data->pbi->inspect_cb = ctx->inspect_cb; + frame_worker_data->pbi->inspect_ctx = ctx->inspect_ctx; + res = av1_receive_compressed_data(frame_worker_data->pbi, data_sz, &data); + check_resync(ctx, frame_worker_data->pbi); + + if (ctx->frame_workers->had_error) + return update_error_state(ctx, &frame_worker_data->pbi->common.error); + + data2->idx = -1; + for (int i = 0; i < REF_FRAMES; ++i) + if (cm->ref_frame_map[i] == cm->cur_frame) data2->idx = i; + data2->buf = data; + data2->show_existing = cm->show_existing_frame; + return res; +} +#endif + static aom_codec_err_t decoder_decode(aom_codec_alg_priv_t *ctx, const uint8_t *data, size_t data_sz, void *user_priv) { aom_codec_err_t res = AOM_CODEC_OK; +#if CONFIG_INSPECTION + if (user_priv != 0) { + return decoder_inspect(ctx, data, data_sz, user_priv); + } +#endif // Release any pending output frames from the previous decoder_decode call. // We need to do this even if the decoder is being flushed or the input // arguments are invalid. if (ctx->frame_workers) { BufferPool *const pool = ctx->buffer_pool; - RefCntBuffer *const frame_bufs = pool->frame_bufs; lock_buffer_pool(pool); for (int i = 0; i < ctx->num_frame_workers; ++i) { AVxWorker *const worker = &ctx->frame_workers[i]; @@ -542,7 +555,7 @@ static aom_codec_err_t decoder_decode(aom_codec_alg_priv_t *ctx, (FrameWorkerData *)worker->data1; struct AV1Decoder *pbi = frame_worker_data->pbi; for (size_t j = 0; j < pbi->num_output_frames; j++) { - decrease_ref_count((int)pbi->output_frame_index[j], frame_bufs, pool); + decrease_ref_count(pbi->output_frames[j], pool); } pbi->num_output_frames = 0; } @@ -644,6 +657,7 @@ static aom_image_t *add_grain_if_needed(aom_image_t *img, if (grain_img_buf) { grain_img_buf->user_priv = img->user_priv; + grain_img_buf->fb_priv = img->fb_priv; if (av1_add_film_grain(grain_params, img, grain_img_buf)) { aom_img_free(grain_img_buf); grain_img_buf = NULL; @@ -676,39 +690,40 @@ static aom_image_t *decoder_get_frame(aom_codec_alg_priv_t *ctx, AVxWorker *const worker = &ctx->frame_workers[ctx->next_output_worker_id]; FrameWorkerData *const frame_worker_data = (FrameWorkerData *)worker->data1; + AV1Decoder *const pbi = frame_worker_data->pbi; + AV1_COMMON *const cm = &pbi->common; ctx->next_output_worker_id = (ctx->next_output_worker_id + 1) % ctx->num_frame_workers; // Wait for the frame from worker thread. if (winterface->sync(worker)) { // Check if worker has received any frames. if (frame_worker_data->received_frame == 1) { - ++ctx->available_threads; frame_worker_data->received_frame = 0; check_resync(ctx, frame_worker_data->pbi); } aom_film_grain_t *grain_params; if (av1_get_raw_frame(frame_worker_data->pbi, *index, &sd, &grain_params) == 0) { - AV1Decoder *const pbi = frame_worker_data->pbi; - AV1_COMMON *const cm = &pbi->common; - RefCntBuffer *const frame_bufs = cm->buffer_pool->frame_bufs; - ctx->last_show_frame = cm->new_fb_idx; + RefCntBuffer *const output_frame_buf = pbi->output_frames[*index]; + ctx->last_show_frame = output_frame_buf; if (ctx->need_resync) return NULL; yuvconfig2image(&ctx->img, sd, frame_worker_data->user_priv); if (!pbi->ext_tile_debug && cm->large_scale_tile) { *index += 1; // Advance the iterator to point to the next image + + yuvconfig2image(&ctx->img, &pbi->tile_list_outbuf, NULL); img = &ctx->img; - img->img_data = pbi->tile_list_output; - img->sz = pbi->tile_list_size; return img; } const int num_planes = av1_num_planes(cm); if (pbi->ext_tile_debug && cm->single_tile_decoding && pbi->dec_tile_row >= 0) { + int tile_width, tile_height; + av1_get_uniform_tile_size(cm, &tile_width, &tile_height); const int tile_row = AOMMIN(pbi->dec_tile_row, cm->tile_rows - 1); - const int mi_row = tile_row * cm->tile_height; + const int mi_row = tile_row * tile_height; const int ssy = ctx->img.y_chroma_shift; int plane; ctx->img.planes[0] += mi_row * MI_SIZE * ctx->img.stride[0]; @@ -718,14 +733,15 @@ static aom_image_t *decoder_get_frame(aom_codec_alg_priv_t *ctx, mi_row * (MI_SIZE >> ssy) * ctx->img.stride[plane]; } } - ctx->img.d_h = - AOMMIN(cm->tile_height, cm->mi_rows - mi_row) * MI_SIZE; + ctx->img.d_h = AOMMIN(tile_height, cm->mi_rows - mi_row) * MI_SIZE; } if (pbi->ext_tile_debug && cm->single_tile_decoding && pbi->dec_tile_col >= 0) { + int tile_width, tile_height; + av1_get_uniform_tile_size(cm, &tile_width, &tile_height); const int tile_col = AOMMIN(pbi->dec_tile_col, cm->tile_cols - 1); - const int mi_col = tile_col * cm->tile_width; + const int mi_col = tile_col * tile_width; const int ssx = ctx->img.x_chroma_shift; const int is_hbd = (ctx->img.fmt & AOM_IMG_FMT_HIGHBITDEPTH) ? 1 : 0; @@ -737,11 +753,10 @@ static aom_image_t *decoder_get_frame(aom_codec_alg_priv_t *ctx, mi_col * (MI_SIZE >> ssx) * (1 + is_hbd); } } - ctx->img.d_w = - AOMMIN(cm->tile_width, cm->mi_cols - mi_col) * MI_SIZE; + ctx->img.d_w = AOMMIN(tile_width, cm->mi_cols - mi_col) * MI_SIZE; } - ctx->img.fb_priv = frame_bufs[cm->new_fb_idx].raw_frame_buffer.priv; + ctx->img.fb_priv = output_frame_buf->raw_frame_buffer.priv; img = &ctx->img; img->temporal_id = cm->temporal_layer_id; img->spatial_id = cm->spatial_layer_id; @@ -758,11 +773,10 @@ static aom_image_t *decoder_get_frame(aom_codec_alg_priv_t *ctx, } else { // Decoding failed. Release the worker thread. frame_worker_data->received_frame = 0; - ++ctx->available_threads; ctx->need_resync = 1; if (ctx->flushed != 1) return NULL; } - } while (ctx->next_output_worker_id != ctx->next_submit_worker_id); + } while (ctx->next_output_worker_id != 0); } return NULL; } @@ -894,7 +908,8 @@ static aom_codec_err_t ctrl_get_last_ref_updates(aom_codec_alg_priv_t *ctx, AVxWorker *const worker = ctx->frame_workers; FrameWorkerData *const frame_worker_data = (FrameWorkerData *)worker->data1; - *update_info = frame_worker_data->pbi->refresh_frame_flags; + *update_info = + frame_worker_data->pbi->common.current_frame.refresh_frame_flags; return AOM_CODEC_OK; } else { return AOM_CODEC_ERROR; @@ -923,11 +938,10 @@ static aom_codec_err_t ctrl_get_frame_corrupted(aom_codec_alg_priv_t *ctx, FrameWorkerData *const frame_worker_data = (FrameWorkerData *)worker->data1; AV1Decoder *const pbi = frame_worker_data->pbi; - RefCntBuffer *const frame_bufs = pbi->common.buffer_pool->frame_bufs; if (pbi->seen_frame_header && pbi->num_output_frames == 0) return AOM_CODEC_ERROR; - if (ctx->last_show_frame >= 0) - *corrupted = frame_bufs[ctx->last_show_frame].buf.corrupted; + if (ctx->last_show_frame != NULL) + *corrupted = ctx->last_show_frame->buf.corrupted; return AOM_CODEC_OK; } else { return AOM_CODEC_ERROR; @@ -1107,8 +1121,27 @@ static aom_codec_err_t ctrl_get_tile_size(aom_codec_alg_priv_t *ctx, FrameWorkerData *const frame_worker_data = (FrameWorkerData *)worker->data1; const AV1_COMMON *const cm = &frame_worker_data->pbi->common; - *tile_size = - ((cm->tile_width * MI_SIZE) << 16) + cm->tile_height * MI_SIZE; + int tile_width, tile_height; + av1_get_uniform_tile_size(cm, &tile_width, &tile_height); + *tile_size = ((tile_width * MI_SIZE) << 16) + tile_height * MI_SIZE; + return AOM_CODEC_OK; + } else { + return AOM_CODEC_ERROR; + } + } + return AOM_CODEC_INVALID_PARAM; +} + +static aom_codec_err_t ctrl_get_tile_count(aom_codec_alg_priv_t *ctx, + va_list args) { + unsigned int *const tile_count = va_arg(args, unsigned int *); + + if (tile_count) { + AVxWorker *const worker = &ctx->frame_workers[ctx->next_output_worker_id]; + if (worker) { + FrameWorkerData *const frame_worker_data = + (FrameWorkerData *)worker->data1; + *tile_count = frame_worker_data->pbi->tile_count_minus_1 + 1; return AOM_CODEC_OK; } else { return AOM_CODEC_ERROR; @@ -1283,6 +1316,7 @@ static aom_codec_ctrl_fn_map_t decoder_ctrl_maps[] = { { AV1D_GET_BIT_DEPTH, ctrl_get_bit_depth }, { AV1D_GET_IMG_FORMAT, ctrl_get_img_format }, { AV1D_GET_TILE_SIZE, ctrl_get_tile_size }, + { AV1D_GET_TILE_COUNT, ctrl_get_tile_count }, { AV1D_GET_DISPLAY_SIZE, ctrl_get_render_size }, { AV1D_GET_FRAME_SIZE, ctrl_get_frame_size }, { AV1_GET_ACCOUNTING, ctrl_get_accounting }, diff --git a/third_party/aom/av1/av1_iface_common.h b/third_party/aom/av1/av1_iface_common.h index 4a7af580b8..5568c89611 100644 --- a/third_party/aom/av1/av1_iface_common.h +++ b/third_party/aom/av1/av1_iface_common.h @@ -52,11 +52,9 @@ static void yuvconfig2image(aom_image_t *img, const YV12_BUFFER_CONFIG *yv12, img->planes[AOM_PLANE_Y] = yv12->y_buffer; img->planes[AOM_PLANE_U] = yv12->u_buffer; img->planes[AOM_PLANE_V] = yv12->v_buffer; - img->planes[AOM_PLANE_ALPHA] = NULL; img->stride[AOM_PLANE_Y] = yv12->y_stride; img->stride[AOM_PLANE_U] = yv12->uv_stride; img->stride[AOM_PLANE_V] = yv12->uv_stride; - img->stride[AOM_PLANE_ALPHA] = yv12->y_stride; if (yv12->flags & YV12_FLAG_HIGHBITDEPTH) { // aom_image_t uses byte strides and a pointer to the first byte // of the image. @@ -65,17 +63,16 @@ static void yuvconfig2image(aom_image_t *img, const YV12_BUFFER_CONFIG *yv12, img->planes[AOM_PLANE_Y] = (uint8_t *)CONVERT_TO_SHORTPTR(yv12->y_buffer); img->planes[AOM_PLANE_U] = (uint8_t *)CONVERT_TO_SHORTPTR(yv12->u_buffer); img->planes[AOM_PLANE_V] = (uint8_t *)CONVERT_TO_SHORTPTR(yv12->v_buffer); - img->planes[AOM_PLANE_ALPHA] = NULL; img->stride[AOM_PLANE_Y] = 2 * yv12->y_stride; img->stride[AOM_PLANE_U] = 2 * yv12->uv_stride; img->stride[AOM_PLANE_V] = 2 * yv12->uv_stride; - img->stride[AOM_PLANE_ALPHA] = 2 * yv12->y_stride; } img->bps = bps; img->user_priv = user_priv; img->img_data = yv12->buffer_alloc; img->img_data_owner = 0; img->self_allocd = 0; + img->sz = yv12->frame_size; } static aom_codec_err_t image2yuvconfig(const aom_image_t *img, @@ -127,7 +124,12 @@ static aom_codec_err_t image2yuvconfig(const aom_image_t *img, } else { yv12->flags = 0; } - yv12->border = (yv12->y_stride - img->w) / 2; + + // Note(yunqing): if img is allocated the same as the frame buffer, y_stride + // is 32-byte aligned. Also, handle the cases while allocating img without a + // border or stride_align is less than 32. + int border = (yv12->y_stride - (int)((img->w + 31) & ~31)) / 2; + yv12->border = (border < 0) ? 0 : border; yv12->subsampling_x = img->x_chroma_shift; yv12->subsampling_y = img->y_chroma_shift; return AOM_CODEC_OK; diff --git a/third_party/aom/av1/common/alloccommon.c b/third_party/aom/av1/common/alloccommon.c index 1bf81c91d4..1c8528a468 100644 --- a/third_party/aom/av1/common/alloccommon.c +++ b/third_party/aom/av1/common/alloccommon.c @@ -92,6 +92,9 @@ void av1_free_ref_frame_buffers(BufferPool *pool) { if (pool->frame_bufs[i].ref_count > 0 && pool->frame_bufs[i].raw_frame_buffer.data != NULL) { pool->release_fb_cb(pool->cb_priv, &pool->frame_bufs[i].raw_frame_buffer); + pool->frame_bufs[i].raw_frame_buffer.data = NULL; + pool->frame_bufs[i].raw_frame_buffer.size = 0; + pool->frame_bufs[i].raw_frame_buffer.priv = NULL; pool->frame_bufs[i].ref_count = 0; } aom_free(pool->frame_bufs[i].mvs); @@ -131,13 +134,12 @@ void av1_alloc_restoration_buffers(AV1_COMMON *cm) { const int ext_h = RESTORATION_UNIT_OFFSET + (mi_h << MI_SIZE_LOG2); const int tile_stripes = (ext_h + 63) / 64; num_stripes += tile_stripes; - cm->rst_end_stripe[i] = num_stripes; } // Now we need to allocate enough space to store the line buffers for the // stripes const int frame_w = cm->superres_upscaled_width; - const int use_highbd = cm->seq_params.use_highbitdepth ? 1 : 0; + const int use_highbd = cm->seq_params.use_highbitdepth; for (int p = 0; p < num_planes; ++p) { const int is_uv = p > 0; @@ -293,8 +295,8 @@ void av1_remove_common(AV1_COMMON *cm) { aom_free(cm->fc); cm->fc = NULL; - aom_free(cm->frame_contexts); - cm->frame_contexts = NULL; + aom_free(cm->default_frame_context); + cm->default_frame_context = NULL; } void av1_init_context_buffers(AV1_COMMON *cm) { cm->setup_mi(cm); } diff --git a/third_party/aom/av1/common/arm/av1_inv_txfm_neon.c b/third_party/aom/av1/common/arm/av1_inv_txfm_neon.c index bad411743d..7a23174be2 100644 --- a/third_party/aom/av1/common/arm/av1_inv_txfm_neon.c +++ b/third_party/aom/av1/common/arm/av1_inv_txfm_neon.c @@ -2086,33 +2086,1570 @@ static INLINE void idct32_low16_new_neon(int16x8_t *in, int16x8_t *out, out[30] = vqsubq_s16(step2[1], step2[30]); out[31] = vqsubq_s16(step2[0], step2[31]); } +static INLINE void idct64_stage9_neon(int16x8_t *step2, int16x8_t *step1, + int8_t cos_bit) { + const int32_t *cospi = cospi_arr(cos_bit); + const int16x4_t c3 = + create_s16x4_neon((int16_t *)(cospi + 32), (int16_t *)(cospi + 32), + (int16_t *)(cospi + 16), (int16_t *)(cospi + 48)); + + btf_16_lane_0_1_neon(step2[27], step2[20], c3, &step1[27], &step1[20]); + btf_16_lane_0_1_neon(step2[26], step2[21], c3, &step1[26], &step1[21]); + btf_16_lane_0_1_neon(step2[25], step2[22], c3, &step1[25], &step1[22]); + btf_16_lane_0_1_neon(step2[24], step2[23], c3, &step1[24], &step1[23]); + + step1[0] = vqaddq_s16(step2[0], step2[15]); + step1[1] = vqaddq_s16(step2[1], step2[14]); + step1[2] = vqaddq_s16(step2[2], step2[13]); + step1[3] = vqaddq_s16(step2[3], step2[12]); + step1[4] = vqaddq_s16(step2[4], step2[11]); + step1[5] = vqaddq_s16(step2[5], step2[10]); + step1[6] = vqaddq_s16(step2[6], step2[9]); + step1[7] = vqaddq_s16(step2[7], step2[8]); + step1[8] = vqsubq_s16(step2[7], step2[8]); + step1[9] = vqsubq_s16(step2[6], step2[9]); + step1[10] = vqsubq_s16(step2[5], step2[10]); + step1[11] = vqsubq_s16(step2[4], step2[11]); + step1[12] = vqsubq_s16(step2[3], step2[12]); + step1[13] = vqsubq_s16(step2[2], step2[13]); + step1[14] = vqsubq_s16(step2[1], step2[14]); + step1[15] = vqsubq_s16(step2[0], step2[15]); + step1[16] = step2[16]; + step1[17] = step2[17]; + step1[18] = step2[18]; + step1[19] = step2[19]; + step1[28] = step2[28]; + step1[29] = step2[29]; + step1[30] = step2[30]; + step1[31] = step2[31]; + step1[32] = vqaddq_s16(step2[32], step2[47]); + step1[33] = vqaddq_s16(step2[33], step2[46]); + step1[34] = vqaddq_s16(step2[34], step2[45]); + step1[35] = vqaddq_s16(step2[35], step2[44]); + step1[36] = vqaddq_s16(step2[36], step2[43]); + step1[37] = vqaddq_s16(step2[37], step2[42]); + step1[38] = vqaddq_s16(step2[38], step2[41]); + step1[39] = vqaddq_s16(step2[39], step2[40]); + step1[40] = vqsubq_s16(step2[39], step2[40]); + step1[41] = vqsubq_s16(step2[38], step2[41]); + step1[42] = vqsubq_s16(step2[37], step2[42]); + step1[43] = vqsubq_s16(step2[36], step2[43]); + step1[44] = vqsubq_s16(step2[35], step2[44]); + step1[45] = vqsubq_s16(step2[34], step2[45]); + step1[46] = vqsubq_s16(step2[33], step2[46]); + step1[47] = vqsubq_s16(step2[32], step2[47]); + step1[48] = vqsubq_s16(step2[63], step2[48]); + step1[49] = vqsubq_s16(step2[62], step2[49]); + step1[50] = vqsubq_s16(step2[61], step2[50]); + step1[51] = vqsubq_s16(step2[60], step2[51]); + step1[52] = vqsubq_s16(step2[59], step2[52]); + step1[53] = vqsubq_s16(step2[58], step2[53]); + step1[54] = vqsubq_s16(step2[57], step2[54]); + step1[55] = vqsubq_s16(step2[56], step2[55]); + step1[56] = vqaddq_s16(step2[56], step2[55]); + step1[57] = vqaddq_s16(step2[57], step2[54]); + step1[58] = vqaddq_s16(step2[58], step2[53]); + step1[59] = vqaddq_s16(step2[59], step2[52]); + step1[60] = vqaddq_s16(step2[60], step2[51]); + step1[61] = vqaddq_s16(step2[61], step2[50]); + step1[62] = vqaddq_s16(step2[62], step2[49]); + step1[63] = vqaddq_s16(step2[63], step2[48]); +} + +static INLINE void idct64_stage10_neon(int16x8_t *step1, int16x8_t *step2, + int8_t cos_bit) { + const int32_t *cospi = cospi_arr(cos_bit); + const int16x4_t c3 = + create_s16x4_neon((int16_t *)(cospi + 32), (int16_t *)(cospi + 32), + (int16_t *)(cospi + 16), (int16_t *)(cospi + 48)); + + btf_16_lane_0_1_neon(step1[55], step1[40], c3, &step2[55], &step2[40]); + btf_16_lane_0_1_neon(step1[54], step1[41], c3, &step2[54], &step2[41]); + btf_16_lane_0_1_neon(step1[53], step1[42], c3, &step2[53], &step2[42]); + btf_16_lane_0_1_neon(step1[52], step1[43], c3, &step2[52], &step2[43]); + btf_16_lane_0_1_neon(step1[51], step1[44], c3, &step2[51], &step2[44]); + btf_16_lane_0_1_neon(step1[50], step1[45], c3, &step2[50], &step2[45]); + btf_16_lane_0_1_neon(step1[49], step1[46], c3, &step2[49], &step2[46]); + btf_16_lane_0_1_neon(step1[48], step1[47], c3, &step2[48], &step2[47]); + + step2[0] = vqaddq_s16(step1[0], step1[31]); + step2[1] = vqaddq_s16(step1[1], step1[30]); + step2[2] = vqaddq_s16(step1[2], step1[29]); + step2[3] = vqaddq_s16(step1[3], step1[28]); + step2[4] = vqaddq_s16(step1[4], step1[27]); + step2[5] = vqaddq_s16(step1[5], step1[26]); + step2[6] = vqaddq_s16(step1[6], step1[25]); + step2[7] = vqaddq_s16(step1[7], step1[24]); + step2[8] = vqaddq_s16(step1[8], step1[23]); + step2[9] = vqaddq_s16(step1[9], step1[22]); + step2[10] = vqaddq_s16(step1[10], step1[21]); + step2[11] = vqaddq_s16(step1[11], step1[20]); + step2[12] = vqaddq_s16(step1[12], step1[19]); + step2[13] = vqaddq_s16(step1[13], step1[18]); + step2[14] = vqaddq_s16(step1[14], step1[17]); + step2[15] = vqaddq_s16(step1[15], step1[16]); + step2[16] = vqsubq_s16(step1[15], step1[16]); + step2[17] = vqsubq_s16(step1[14], step1[17]); + step2[18] = vqsubq_s16(step1[13], step1[18]); + step2[19] = vqsubq_s16(step1[12], step1[19]); + step2[20] = vqsubq_s16(step1[11], step1[20]); + step2[21] = vqsubq_s16(step1[10], step1[21]); + step2[22] = vqsubq_s16(step1[9], step1[22]); + step2[23] = vqsubq_s16(step1[8], step1[23]); + step2[24] = vqsubq_s16(step1[7], step1[24]); + step2[25] = vqsubq_s16(step1[6], step1[25]); + step2[26] = vqsubq_s16(step1[5], step1[26]); + step2[27] = vqsubq_s16(step1[4], step1[27]); + step2[28] = vqsubq_s16(step1[3], step1[28]); + step2[29] = vqsubq_s16(step1[2], step1[29]); + step2[30] = vqsubq_s16(step1[1], step1[30]); + step2[31] = vqsubq_s16(step1[0], step1[31]); + step2[32] = step1[32]; + step2[33] = step1[33]; + step2[34] = step1[34]; + step2[35] = step1[35]; + step2[36] = step1[36]; + step2[37] = step1[37]; + step2[38] = step1[38]; + step2[39] = step1[39]; + step2[56] = step1[56]; + step2[57] = step1[57]; + step2[58] = step1[58]; + step2[59] = step1[59]; + step2[60] = step1[60]; + step2[61] = step1[61]; + step2[62] = step1[62]; + step2[63] = step1[63]; +} + +static INLINE void idct64_low32_new_neon(int16x8_t *in, int16x8_t *out, + int8_t cos_bit, int bit) { + (void)bit; + const int32_t *cospi = cospi_arr(cos_bit); + int16x8_t step2[64], step1[64]; + const int16x4_t c0 = + create_s16x4_neon((int16_t *)(cospi + 4), (int16_t *)(cospi + 60), + (int16_t *)(cospi + 36), (int16_t *)(cospi + 28)); + const int16x4_t c1 = + create_s16x4_neon((int16_t *)(cospi + 20), (int16_t *)(cospi + 44), + (int16_t *)(cospi + 52), (int16_t *)(cospi + 12)); + const int16x4_t c2 = + create_s16x4_neon((int16_t *)(cospi + 8), (int16_t *)(cospi + 56), + (int16_t *)(cospi + 40), (int16_t *)(cospi + 24)); + const int16x4_t c3 = + create_s16x4_neon((int16_t *)(cospi + 32), (int16_t *)(cospi + 32), + (int16_t *)(cospi + 16), (int16_t *)(cospi + 48)); + + // stage 1 + // stage 2 + + step2[0] = in[0]; + step2[2] = in[16]; + step2[4] = in[8]; + step2[6] = in[24]; + step2[8] = in[4]; + step2[10] = in[20]; + step2[12] = in[12]; + step2[14] = in[28]; + step2[16] = in[2]; + step2[18] = in[18]; + step2[20] = in[10]; + step2[22] = in[26]; + step2[24] = in[6]; + step2[26] = in[22]; + step2[28] = in[14]; + step2[30] = in[30]; + + btf_16_neon(in[1], cospi[63], cospi[1], &step2[32], &step2[63]); + btf_16_neon(in[31], -cospi[33], cospi[31], &step2[33], &step2[62]); + btf_16_neon(in[17], cospi[47], cospi[17], &step2[34], &step2[61]); + btf_16_neon(in[15], -cospi[49], cospi[15], &step2[35], &step2[60]); + btf_16_neon(in[9], cospi[55], cospi[9], &step2[36], &step2[59]); + btf_16_neon(in[23], -cospi[41], cospi[23], &step2[37], &step2[58]); + btf_16_neon(in[25], cospi[39], cospi[25], &step2[38], &step2[57]); + btf_16_neon(in[7], -cospi[57], cospi[7], &step2[39], &step2[56]); + btf_16_neon(in[5], cospi[59], cospi[5], &step2[40], &step2[55]); + btf_16_neon(in[27], -cospi[37], cospi[27], &step2[41], &step2[54]); + btf_16_neon(in[21], cospi[43], cospi[21], &step2[42], &step2[53]); + btf_16_neon(in[11], -cospi[53], cospi[11], &step2[43], &step2[52]); + btf_16_neon(in[13], cospi[51], cospi[13], &step2[44], &step2[51]); + btf_16_neon(in[19], -cospi[45], cospi[19], &step2[45], &step2[50]); + btf_16_neon(in[29], cospi[35], cospi[29], &step2[46], &step2[49]); + btf_16_neon(in[3], -cospi[61], cospi[3], &step2[47], &step2[48]); + + // stage 3 + + step1[0] = step2[0]; + step1[2] = step2[2]; + step1[4] = step2[4]; + step1[6] = step2[6]; + step1[8] = step2[8]; + step1[10] = step2[10]; + step1[12] = step2[12]; + step1[14] = step2[14]; + + btf_16_neon(step2[16], cospi[62], cospi[2], &step1[16], &step1[31]); + btf_16_neon(step2[30], -cospi[34], cospi[30], &step1[17], &step1[30]); + btf_16_neon(step2[18], cospi[46], cospi[18], &step1[18], &step1[29]); + btf_16_neon(step2[28], -cospi[50], cospi[14], &step1[19], &step1[28]); + btf_16_neon(step2[20], cospi[54], cospi[10], &step1[20], &step1[27]); + btf_16_neon(step2[26], -cospi[42], cospi[22], &step1[21], &step1[26]); + btf_16_neon(step2[22], cospi[38], cospi[26], &step1[22], &step1[25]); + btf_16_neon(step2[24], -cospi[58], cospi[6], &step1[23], &step1[24]); + + step1[32] = vqaddq_s16(step2[32], step2[33]); + step1[33] = vqsubq_s16(step2[32], step2[33]); + step1[34] = vqsubq_s16(step2[35], step2[34]); + step1[35] = vqaddq_s16(step2[35], step2[34]); + step1[36] = vqaddq_s16(step2[36], step2[37]); + step1[37] = vqsubq_s16(step2[36], step2[37]); + step1[38] = vqsubq_s16(step2[39], step2[38]); + step1[39] = vqaddq_s16(step2[39], step2[38]); + step1[40] = vqaddq_s16(step2[40], step2[41]); + step1[41] = vqsubq_s16(step2[40], step2[41]); + step1[42] = vqsubq_s16(step2[43], step2[42]); + step1[43] = vqaddq_s16(step2[43], step2[42]); + step1[44] = vqaddq_s16(step2[44], step2[45]); + step1[45] = vqsubq_s16(step2[44], step2[45]); + step1[46] = vqsubq_s16(step2[47], step2[46]); + step1[47] = vqaddq_s16(step2[47], step2[46]); + step1[48] = vqaddq_s16(step2[48], step2[49]); + step1[49] = vqsubq_s16(step2[48], step2[49]); + step1[50] = vqsubq_s16(step2[51], step2[50]); + step1[51] = vqaddq_s16(step2[51], step2[50]); + step1[52] = vqaddq_s16(step2[52], step2[53]); + step1[53] = vqsubq_s16(step2[52], step2[53]); + step1[54] = vqsubq_s16(step2[55], step2[54]); + step1[55] = vqaddq_s16(step2[55], step2[54]); + step1[56] = vqaddq_s16(step2[56], step2[57]); + step1[57] = vqsubq_s16(step2[56], step2[57]); + step1[58] = vqsubq_s16(step2[59], step2[58]); + step1[59] = vqaddq_s16(step2[59], step2[58]); + step1[60] = vqaddq_s16(step2[60], step2[61]); + step1[61] = vqsubq_s16(step2[60], step2[61]); + step1[62] = vqsubq_s16(step2[63], step2[62]); + step1[63] = vqaddq_s16(step2[63], step2[62]); + + // stage 4 + + step2[0] = step1[0]; + step2[2] = step1[2]; + step2[4] = step1[4]; + step2[6] = step1[6]; + + btf_16_neon(step1[8], cospi[60], cospi[4], &step2[8], &step2[15]); + btf_16_neon(step1[14], -cospi[36], cospi[28], &step2[9], &step2[14]); + btf_16_neon(step1[10], cospi[44], cospi[20], &step2[10], &step2[13]); + btf_16_neon(step1[12], -cospi[52], cospi[12], &step2[11], &step2[12]); + btf_16_lane_0_1_neon(step1[62], step1[33], c0, &step2[62], &step2[33]); + btf_16_lane_1_0_neon(vnegq_s16(step1[34]), vnegq_s16(step1[61]), c0, + &step2[34], &step2[61]); + btf_16_lane_2_3_neon(step1[58], step1[37], c0, &step2[58], &step2[37]); + btf_16_lane_3_2_neon(vnegq_s16(step1[38]), vnegq_s16(step1[57]), c0, + &step2[38], &step2[57]); + btf_16_lane_0_1_neon(step1[54], step1[41], c1, &step2[54], &step2[41]); + btf_16_lane_1_0_neon(vnegq_s16(step1[42]), vnegq_s16(step1[53]), c1, + &step2[42], &step2[53]); + btf_16_lane_2_3_neon(step1[50], step1[45], c1, &step2[50], &step2[45]); + btf_16_lane_3_2_neon(vnegq_s16(step1[46]), vnegq_s16(step1[49]), c1, + &step2[46], &step2[49]); + + step2[16] = vqaddq_s16(step1[16], step1[17]); + step2[17] = vqsubq_s16(step1[16], step1[17]); + step2[18] = vqsubq_s16(step1[19], step1[18]); + step2[19] = vqaddq_s16(step1[19], step1[18]); + step2[20] = vqaddq_s16(step1[20], step1[21]); + step2[21] = vqsubq_s16(step1[20], step1[21]); + step2[22] = vqsubq_s16(step1[23], step1[22]); + step2[23] = vqaddq_s16(step1[23], step1[22]); + step2[24] = vqaddq_s16(step1[24], step1[25]); + step2[25] = vqsubq_s16(step1[24], step1[25]); + step2[26] = vqsubq_s16(step1[27], step1[26]); + step2[27] = vqaddq_s16(step1[27], step1[26]); + step2[28] = vqaddq_s16(step1[28], step1[29]); + step2[29] = vqsubq_s16(step1[28], step1[29]); + step2[30] = vqsubq_s16(step1[31], step1[30]); + step2[31] = vqaddq_s16(step1[31], step1[30]); + step2[32] = step1[32]; + step2[35] = step1[35]; + step2[36] = step1[36]; + step2[39] = step1[39]; + step2[40] = step1[40]; + step2[43] = step1[43]; + step2[44] = step1[44]; + step2[47] = step1[47]; + step2[48] = step1[48]; + step2[51] = step1[51]; + step2[52] = step1[52]; + step2[55] = step1[55]; + step2[56] = step1[56]; + step2[59] = step1[59]; + step2[60] = step1[60]; + step2[63] = step1[63]; + + // stage 5 + + step1[0] = step2[0]; + step1[2] = step2[2]; + + btf_16_neon(step2[4], cospi[56], cospi[8], &step1[4], &step1[7]); + btf_16_neon(step2[6], -cospi[40], cospi[24], &step1[5], &step1[6]); + btf_16_lane_0_1_neon(step2[30], step2[17], c2, &step1[30], &step1[17]); + btf_16_lane_1_0_neon(vnegq_s16(step2[18]), vnegq_s16(step2[29]), c2, + &step1[18], &step1[29]); + btf_16_lane_2_3_neon(step2[26], step2[21], c2, &step1[26], &step1[21]); + btf_16_lane_3_2_neon(vnegq_s16(step2[22]), vnegq_s16(step2[25]), c2, + &step1[22], &step1[25]); + + step1[8] = vqaddq_s16(step2[8], step2[9]); + step1[9] = vqsubq_s16(step2[8], step2[9]); + step1[10] = vqsubq_s16(step2[11], step2[10]); + step1[11] = vqaddq_s16(step2[11], step2[10]); + step1[12] = vqaddq_s16(step2[12], step2[13]); + step1[13] = vqsubq_s16(step2[12], step2[13]); + step1[14] = vqsubq_s16(step2[15], step2[14]); + step1[15] = vqaddq_s16(step2[15], step2[14]); + step1[16] = step2[16]; + step1[19] = step2[19]; + step1[20] = step2[20]; + step1[23] = step2[23]; + step1[24] = step2[24]; + step1[27] = step2[27]; + step1[28] = step2[28]; + step1[31] = step2[31]; + step1[32] = vqaddq_s16(step2[32], step2[35]); + step1[33] = vqaddq_s16(step2[33], step2[34]); + step1[34] = vqsubq_s16(step2[33], step2[34]); + step1[35] = vqsubq_s16(step2[32], step2[35]); + step1[36] = vqsubq_s16(step2[39], step2[36]); + step1[37] = vqsubq_s16(step2[38], step2[37]); + step1[38] = vqaddq_s16(step2[38], step2[37]); + step1[39] = vqaddq_s16(step2[39], step2[36]); + step1[40] = vqaddq_s16(step2[40], step2[43]); + step1[41] = vqaddq_s16(step2[41], step2[42]); + step1[42] = vqsubq_s16(step2[41], step2[42]); + step1[43] = vqsubq_s16(step2[40], step2[43]); + step1[44] = vqsubq_s16(step2[47], step2[44]); + step1[45] = vqsubq_s16(step2[46], step2[45]); + step1[46] = vqaddq_s16(step2[46], step2[45]); + step1[47] = vqaddq_s16(step2[47], step2[44]); + step1[48] = vqaddq_s16(step2[48], step2[51]); + step1[49] = vqaddq_s16(step2[49], step2[50]); + step1[50] = vqsubq_s16(step2[49], step2[50]); + step1[51] = vqsubq_s16(step2[48], step2[51]); + step1[52] = vqsubq_s16(step2[55], step2[52]); + step1[53] = vqsubq_s16(step2[54], step2[53]); + step1[54] = vqaddq_s16(step2[54], step2[53]); + step1[55] = vqaddq_s16(step2[55], step2[52]); + step1[56] = vqaddq_s16(step2[56], step2[59]); + step1[57] = vqaddq_s16(step2[57], step2[58]); + step1[58] = vqsubq_s16(step2[57], step2[58]); + step1[59] = vqsubq_s16(step2[56], step2[59]); + step1[60] = vqsubq_s16(step2[63], step2[60]); + step1[61] = vqsubq_s16(step2[62], step2[61]); + step1[62] = vqaddq_s16(step2[62], step2[61]); + step1[63] = vqaddq_s16(step2[63], step2[60]); + + // stage 6 + + btf_16_neon(step1[0], cospi[32], cospi[32], &step2[0], &step2[1]); + btf_16_neon(step1[2], cospi[48], cospi[16], &step2[2], &step2[3]); + btf_16_lane_2_3_neon(step1[14], step1[9], c3, &step2[14], &step2[9]); + btf_16_lane_3_2_neon(vnegq_s16(step1[10]), vnegq_s16(step1[13]), c3, + &step2[10], &step2[13]); + btf_16_lane_0_1_neon(step1[61], step1[34], c2, &step2[61], &step2[34]); + btf_16_lane_0_1_neon(step1[60], step1[35], c2, &step2[60], &step2[35]); + btf_16_lane_1_0_neon(vnegq_s16(step1[36]), vnegq_s16(step1[59]), c2, + &step2[36], &step2[59]); + btf_16_lane_1_0_neon(vnegq_s16(step1[37]), vnegq_s16(step1[58]), c2, + &step2[37], &step2[58]); + btf_16_lane_2_3_neon(step1[53], step1[42], c2, &step2[53], &step2[42]); + btf_16_lane_2_3_neon(step1[52], step1[43], c2, &step2[52], &step2[43]); + btf_16_lane_3_2_neon(vnegq_s16(step1[44]), vnegq_s16(step1[51]), c2, + &step2[44], &step2[51]); + btf_16_lane_3_2_neon(vnegq_s16(step1[45]), vnegq_s16(step1[50]), c2, + &step2[45], &step2[50]); + + step2[4] = vqaddq_s16(step1[4], step1[5]); + step2[5] = vqsubq_s16(step1[4], step1[5]); + step2[6] = vqsubq_s16(step1[7], step1[6]); + step2[7] = vqaddq_s16(step1[7], step1[6]); + step2[8] = step1[8]; + step2[11] = step1[11]; + step2[12] = step1[12]; + step2[15] = step1[15]; + step2[16] = vqaddq_s16(step1[16], step1[19]); + step2[17] = vqaddq_s16(step1[17], step1[18]); + step2[18] = vqsubq_s16(step1[17], step1[18]); + step2[19] = vqsubq_s16(step1[16], step1[19]); + step2[20] = vqsubq_s16(step1[23], step1[20]); + step2[21] = vqsubq_s16(step1[22], step1[21]); + step2[22] = vqaddq_s16(step1[22], step1[21]); + step2[23] = vqaddq_s16(step1[23], step1[20]); + step2[24] = vqaddq_s16(step1[24], step1[27]); + step2[25] = vqaddq_s16(step1[25], step1[26]); + step2[26] = vqsubq_s16(step1[25], step1[26]); + step2[27] = vqsubq_s16(step1[24], step1[27]); + step2[28] = vqsubq_s16(step1[31], step1[28]); + step2[29] = vqsubq_s16(step1[30], step1[29]); + step2[30] = vqaddq_s16(step1[30], step1[29]); + step2[31] = vqaddq_s16(step1[31], step1[28]); + step2[32] = step1[32]; + step2[33] = step1[33]; + step2[38] = step1[38]; + step2[39] = step1[39]; + step2[40] = step1[40]; + step2[41] = step1[41]; + step2[46] = step1[46]; + step2[47] = step1[47]; + step2[48] = step1[48]; + step2[49] = step1[49]; + step2[54] = step1[54]; + step2[55] = step1[55]; + step2[56] = step1[56]; + step2[57] = step1[57]; + step2[62] = step1[62]; + step2[63] = step1[63]; + + // stage 7 + + btf_16_lane_0_1_neon(step2[6], step2[5], c3, &step1[6], &step1[5]); + btf_16_lane_2_3_neon(step2[29], step2[18], c3, &step1[29], &step1[18]); + btf_16_lane_2_3_neon(step2[28], step2[19], c3, &step1[28], &step1[19]); + btf_16_lane_3_2_neon(vnegq_s16(step2[20]), vnegq_s16(step2[27]), c3, + &step1[20], &step1[27]); + btf_16_lane_3_2_neon(vnegq_s16(step2[21]), vnegq_s16(step2[26]), c3, + &step1[21], &step1[26]); + + step1[0] = vqaddq_s16(step2[0], step2[3]); + step1[1] = vqaddq_s16(step2[1], step2[2]); + step1[2] = vqsubq_s16(step2[1], step2[2]); + step1[3] = vqsubq_s16(step2[0], step2[3]); + step1[4] = step2[4]; + step1[7] = step2[7]; + step1[8] = vqaddq_s16(step2[8], step2[11]); + step1[9] = vqaddq_s16(step2[9], step2[10]); + step1[10] = vqsubq_s16(step2[9], step2[10]); + step1[11] = vqsubq_s16(step2[8], step2[11]); + step1[12] = vqsubq_s16(step2[15], step2[12]); + step1[13] = vqsubq_s16(step2[14], step2[13]); + step1[14] = vqaddq_s16(step2[14], step2[13]); + step1[15] = vqaddq_s16(step2[15], step2[12]); + step1[16] = step2[16]; + step1[17] = step2[17]; + step1[22] = step2[22]; + step1[23] = step2[23]; + step1[24] = step2[24]; + step1[25] = step2[25]; + step1[30] = step2[30]; + step1[31] = step2[31]; + step1[32] = vqaddq_s16(step2[32], step2[39]); + step1[33] = vqaddq_s16(step2[33], step2[38]); + step1[34] = vqaddq_s16(step2[34], step2[37]); + step1[35] = vqaddq_s16(step2[35], step2[36]); + step1[36] = vqsubq_s16(step2[35], step2[36]); + step1[37] = vqsubq_s16(step2[34], step2[37]); + step1[38] = vqsubq_s16(step2[33], step2[38]); + step1[39] = vqsubq_s16(step2[32], step2[39]); + step1[40] = vqsubq_s16(step2[47], step2[40]); + step1[41] = vqsubq_s16(step2[46], step2[41]); + step1[42] = vqsubq_s16(step2[45], step2[42]); + step1[43] = vqsubq_s16(step2[44], step2[43]); + step1[44] = vqaddq_s16(step2[43], step2[44]); + step1[45] = vqaddq_s16(step2[42], step2[45]); + step1[46] = vqaddq_s16(step2[41], step2[46]); + step1[47] = vqaddq_s16(step2[40], step2[47]); + step1[48] = vqaddq_s16(step2[48], step2[55]); + step1[49] = vqaddq_s16(step2[49], step2[54]); + step1[50] = vqaddq_s16(step2[50], step2[53]); + step1[51] = vqaddq_s16(step2[51], step2[52]); + step1[52] = vqsubq_s16(step2[51], step2[52]); + step1[53] = vqsubq_s16(step2[50], step2[53]); + step1[54] = vqsubq_s16(step2[49], step2[54]); + step1[55] = vqsubq_s16(step2[48], step2[55]); + step1[56] = vqsubq_s16(step2[63], step2[56]); + step1[57] = vqsubq_s16(step2[62], step2[57]); + step1[58] = vqsubq_s16(step2[61], step2[58]); + step1[59] = vqsubq_s16(step2[60], step2[59]); + step1[60] = vqaddq_s16(step2[59], step2[60]); + step1[61] = vqaddq_s16(step2[58], step2[61]); + step1[62] = vqaddq_s16(step2[57], step2[62]); + step1[63] = vqaddq_s16(step2[56], step2[63]); + + // stage 8 + + btf_16_lane_0_1_neon(step1[13], step1[10], c3, &step2[13], &step2[10]); + btf_16_lane_0_1_neon(step1[12], step1[11], c3, &step2[12], &step2[11]); + btf_16_lane_2_3_neon(step1[59], step1[36], c3, &step2[59], &step2[36]); + btf_16_lane_2_3_neon(step1[58], step1[37], c3, &step2[58], &step2[37]); + btf_16_lane_2_3_neon(step1[57], step1[38], c3, &step2[57], &step2[38]); + btf_16_lane_2_3_neon(step1[56], step1[39], c3, &step2[56], &step2[39]); + btf_16_lane_3_2_neon(vnegq_s16(step1[40]), vnegq_s16(step1[55]), c3, + &step2[40], &step2[55]); + btf_16_lane_3_2_neon(vnegq_s16(step1[41]), vnegq_s16(step1[54]), c3, + &step2[41], &step2[54]); + btf_16_lane_3_2_neon(vnegq_s16(step1[42]), vnegq_s16(step1[53]), c3, + &step2[42], &step2[53]); + btf_16_lane_3_2_neon(vnegq_s16(step1[43]), vnegq_s16(step1[52]), c3, + &step2[43], &step2[52]); + + step2[0] = vqaddq_s16(step1[0], step1[7]); + step2[1] = vqaddq_s16(step1[1], step1[6]); + step2[2] = vqaddq_s16(step1[2], step1[5]); + step2[3] = vqaddq_s16(step1[3], step1[4]); + step2[4] = vqsubq_s16(step1[3], step1[4]); + step2[5] = vqsubq_s16(step1[2], step1[5]); + step2[6] = vqsubq_s16(step1[1], step1[6]); + step2[7] = vqsubq_s16(step1[0], step1[7]); + step2[8] = step1[8]; + step2[9] = step1[9]; + step2[14] = step1[14]; + step2[15] = step1[15]; + step2[16] = vqaddq_s16(step1[16], step1[23]); + step2[17] = vqaddq_s16(step1[17], step1[22]); + step2[18] = vqaddq_s16(step1[18], step1[21]); + step2[19] = vqaddq_s16(step1[19], step1[20]); + step2[20] = vqsubq_s16(step1[19], step1[20]); + step2[21] = vqsubq_s16(step1[18], step1[21]); + step2[22] = vqsubq_s16(step1[17], step1[22]); + step2[23] = vqsubq_s16(step1[16], step1[23]); + step2[24] = vqsubq_s16(step1[31], step1[24]); + step2[25] = vqsubq_s16(step1[30], step1[25]); + step2[26] = vqsubq_s16(step1[29], step1[26]); + step2[27] = vqsubq_s16(step1[28], step1[27]); + step2[28] = vqaddq_s16(step1[28], step1[27]); + step2[29] = vqaddq_s16(step1[29], step1[26]); + step2[30] = vqaddq_s16(step1[30], step1[25]); + step2[31] = vqaddq_s16(step1[31], step1[24]); + step2[32] = step1[32]; + step2[33] = step1[33]; + step2[34] = step1[34]; + step2[35] = step1[35]; + step2[44] = step1[44]; + step2[45] = step1[45]; + step2[46] = step1[46]; + step2[47] = step1[47]; + step2[48] = step1[48]; + step2[49] = step1[49]; + step2[50] = step1[50]; + step2[51] = step1[51]; + step2[60] = step1[60]; + step2[61] = step1[61]; + step2[62] = step1[62]; + step2[63] = step1[63]; + + // stage 9 + idct64_stage9_neon(step2, step1, cos_bit); + + // stage 10 + idct64_stage10_neon(step1, step2, cos_bit); + + // stage 11 + + out[0] = vqaddq_s16(step2[0], step2[63]); + out[1] = vqaddq_s16(step2[1], step2[62]); + out[2] = vqaddq_s16(step2[2], step2[61]); + out[3] = vqaddq_s16(step2[3], step2[60]); + out[4] = vqaddq_s16(step2[4], step2[59]); + out[5] = vqaddq_s16(step2[5], step2[58]); + out[6] = vqaddq_s16(step2[6], step2[57]); + out[7] = vqaddq_s16(step2[7], step2[56]); + out[8] = vqaddq_s16(step2[8], step2[55]); + out[9] = vqaddq_s16(step2[9], step2[54]); + out[10] = vqaddq_s16(step2[10], step2[53]); + out[11] = vqaddq_s16(step2[11], step2[52]); + out[12] = vqaddq_s16(step2[12], step2[51]); + out[13] = vqaddq_s16(step2[13], step2[50]); + out[14] = vqaddq_s16(step2[14], step2[49]); + out[15] = vqaddq_s16(step2[15], step2[48]); + out[16] = vqaddq_s16(step2[16], step2[47]); + out[17] = vqaddq_s16(step2[17], step2[46]); + out[18] = vqaddq_s16(step2[18], step2[45]); + out[19] = vqaddq_s16(step2[19], step2[44]); + out[20] = vqaddq_s16(step2[20], step2[43]); + out[21] = vqaddq_s16(step2[21], step2[42]); + out[22] = vqaddq_s16(step2[22], step2[41]); + out[23] = vqaddq_s16(step2[23], step2[40]); + out[24] = vqaddq_s16(step2[24], step2[39]); + out[25] = vqaddq_s16(step2[25], step2[38]); + out[26] = vqaddq_s16(step2[26], step2[37]); + out[27] = vqaddq_s16(step2[27], step2[36]); + out[28] = vqaddq_s16(step2[28], step2[35]); + out[29] = vqaddq_s16(step2[29], step2[34]); + out[30] = vqaddq_s16(step2[30], step2[33]); + out[31] = vqaddq_s16(step2[31], step2[32]); + out[32] = vqsubq_s16(step2[31], step2[32]); + out[33] = vqsubq_s16(step2[30], step2[33]); + out[34] = vqsubq_s16(step2[29], step2[34]); + out[35] = vqsubq_s16(step2[28], step2[35]); + out[36] = vqsubq_s16(step2[27], step2[36]); + out[37] = vqsubq_s16(step2[26], step2[37]); + out[38] = vqsubq_s16(step2[25], step2[38]); + out[39] = vqsubq_s16(step2[24], step2[39]); + out[40] = vqsubq_s16(step2[23], step2[40]); + out[41] = vqsubq_s16(step2[22], step2[41]); + out[42] = vqsubq_s16(step2[21], step2[42]); + out[43] = vqsubq_s16(step2[20], step2[43]); + out[44] = vqsubq_s16(step2[19], step2[44]); + out[45] = vqsubq_s16(step2[18], step2[45]); + out[46] = vqsubq_s16(step2[17], step2[46]); + out[47] = vqsubq_s16(step2[16], step2[47]); + out[48] = vqsubq_s16(step2[15], step2[48]); + out[49] = vqsubq_s16(step2[14], step2[49]); + out[50] = vqsubq_s16(step2[13], step2[50]); + out[51] = vqsubq_s16(step2[12], step2[51]); + out[52] = vqsubq_s16(step2[11], step2[52]); + out[53] = vqsubq_s16(step2[10], step2[53]); + out[54] = vqsubq_s16(step2[9], step2[54]); + out[55] = vqsubq_s16(step2[8], step2[55]); + out[56] = vqsubq_s16(step2[7], step2[56]); + out[57] = vqsubq_s16(step2[6], step2[57]); + out[58] = vqsubq_s16(step2[5], step2[58]); + out[59] = vqsubq_s16(step2[4], step2[59]); + out[60] = vqsubq_s16(step2[3], step2[60]); + out[61] = vqsubq_s16(step2[2], step2[61]); + out[62] = vqsubq_s16(step2[1], step2[62]); + out[63] = vqsubq_s16(step2[0], step2[63]); +} + +static INLINE void idct64_low1_new_neon(int16x8_t *input, int16x8_t *out, + int8_t cos_bit, int bit) { + (void)bit; + const int32_t *cospi = cospi_arr(cos_bit); + int16x8_t step1; + int32x4_t t32[2]; + + // stage 1 + // stage 2 + // stage 3 + // stage 4 + // stage 5 + // stage 6 + + t32[0] = vmull_n_s16(vget_low_s16(input[0]), cospi[32]); + t32[1] = vmull_n_s16(vget_high_s16(input[0]), cospi[32]); + + step1 = vcombine_s16(vrshrn_n_s32(t32[0], INV_COS_BIT), + vrshrn_n_s32(t32[1], INV_COS_BIT)); + // stage 7 + // stage 8 + // stage 9 + // stage 10 + // stage 11 + out[0] = step1; + out[1] = step1; + out[2] = step1; + out[3] = step1; + out[4] = step1; + out[5] = step1; + out[6] = step1; + out[7] = step1; + out[8] = step1; + out[9] = step1; + out[10] = step1; + out[11] = step1; + out[12] = step1; + out[13] = step1; + out[14] = step1; + out[15] = step1; + out[16] = step1; + out[17] = step1; + out[18] = step1; + out[19] = step1; + out[20] = step1; + out[21] = step1; + out[22] = step1; + out[23] = step1; + out[24] = step1; + out[25] = step1; + out[26] = step1; + out[27] = step1; + out[28] = step1; + out[29] = step1; + out[30] = step1; + out[31] = step1; + out[32] = step1; + out[33] = step1; + out[34] = step1; + out[35] = step1; + out[36] = step1; + out[37] = step1; + out[38] = step1; + out[39] = step1; + out[40] = step1; + out[41] = step1; + out[42] = step1; + out[43] = step1; + out[44] = step1; + out[45] = step1; + out[46] = step1; + out[47] = step1; + out[48] = step1; + out[49] = step1; + out[50] = step1; + out[51] = step1; + out[52] = step1; + out[53] = step1; + out[54] = step1; + out[55] = step1; + out[56] = step1; + out[57] = step1; + out[58] = step1; + out[59] = step1; + out[60] = step1; + out[61] = step1; + out[62] = step1; + out[63] = step1; +} + +static INLINE void idct64_low8_new_neon(int16x8_t *in, int16x8_t *out, + int8_t cos_bit, int bit) { + (void)bit; + const int32_t *cospi = cospi_arr(cos_bit); + int16x8_t step2[64], step1[64]; + + const int16x4_t c0 = + create_s16x4_neon((int16_t *)(cospi + 4), (int16_t *)(cospi + 60), + (int16_t *)(cospi + 36), (int16_t *)(cospi + 28)); + const int16x4_t c1 = + create_s16x4_neon((int16_t *)(cospi + 20), (int16_t *)(cospi + 44), + (int16_t *)(cospi + 52), (int16_t *)(cospi + 12)); + const int16x4_t c2 = + create_s16x4_neon((int16_t *)(cospi + 8), (int16_t *)(cospi + 56), + (int16_t *)(cospi + 40), (int16_t *)(cospi + 24)); + const int16x4_t c3 = + create_s16x4_neon((int16_t *)(cospi + 32), (int16_t *)(cospi + 32), + (int16_t *)(cospi + 16), (int16_t *)(cospi + 48)); + + // stage 1 + // stage 2 + + step2[0] = in[0]; + step2[8] = in[4]; + step2[16] = in[2]; + step2[24] = in[6]; + + btf_16_neon(in[1], cospi[63], cospi[1], &step2[32], &step2[63]); + btf_16_neon(in[7], -cospi[57], cospi[7], &step2[39], &step2[56]); + btf_16_neon(in[5], cospi[59], cospi[5], &step2[40], &step2[55]); + btf_16_neon(in[3], -cospi[61], cospi[3], &step2[47], &step2[48]); + + // stage 3 + + step1[0] = step2[0]; + step1[8] = step2[8]; + + btf_16_neon(step2[16], cospi[62], cospi[2], &step1[16], &step1[31]); + btf_16_neon(step2[24], -cospi[58], cospi[6], &step1[23], &step1[24]); + + step1[32] = step2[32]; + step1[33] = step2[32]; + step1[38] = step2[39]; + step1[39] = step2[39]; + step1[40] = step2[40]; + step1[41] = step2[40]; + step1[46] = step2[47]; + step1[47] = step2[47]; + step1[48] = step2[48]; + step1[49] = step2[48]; + step1[54] = step2[55]; + step1[55] = step2[55]; + step1[56] = step2[56]; + step1[57] = step2[56]; + step1[62] = step2[63]; + step1[63] = step2[63]; + + // stage 4 + + step2[0] = step1[0]; + + btf_16_neon(step1[8], cospi[60], cospi[4], &step2[8], &step2[15]); + btf_16_lane_0_1_neon(step1[62], step1[33], c0, &step2[62], &step2[33]); + btf_16_lane_3_2_neon(vnegq_s16(step1[38]), vnegq_s16(step1[57]), c0, + &step2[38], &step2[57]); + btf_16_lane_0_1_neon(step1[54], step1[41], c1, &step2[54], &step2[41]); + btf_16_lane_3_2_neon(vnegq_s16(step1[46]), vnegq_s16(step1[49]), c1, + &step2[46], &step2[49]); + + step2[16] = step1[16]; + step2[17] = step1[16]; + step2[22] = step1[23]; + step2[23] = step1[23]; + step2[24] = step1[24]; + step2[25] = step1[24]; + step2[30] = step1[31]; + step2[31] = step1[31]; + step2[32] = step1[32]; + step2[39] = step1[39]; + step2[40] = step1[40]; + step2[47] = step1[47]; + step2[48] = step1[48]; + step2[55] = step1[55]; + step2[56] = step1[56]; + step2[63] = step1[63]; + + // stage 5 + + step1[0] = step2[0]; + + btf_16_lane_0_1_neon(step2[30], step2[17], c2, &step1[30], &step1[17]); + btf_16_lane_3_2_neon(vnegq_s16(step2[22]), vnegq_s16(step2[25]), c2, + &step1[22], &step1[25]); + + step1[8] = step2[8]; + step1[9] = step2[8]; + step1[14] = step2[15]; + step1[15] = step2[15]; + + step1[16] = step2[16]; + step1[23] = step2[23]; + step1[24] = step2[24]; + step1[31] = step2[31]; + step1[32] = step2[32]; + step1[33] = step2[33]; + step1[34] = step2[33]; + step1[35] = step2[32]; + step1[36] = step2[39]; + step1[37] = step2[38]; + step1[38] = step2[38]; + step1[39] = step2[39]; + step1[40] = step2[40]; + step1[41] = step2[41]; + step1[42] = step2[41]; + step1[43] = step2[40]; + step1[44] = step2[47]; + step1[45] = step2[46]; + step1[46] = step2[46]; + step1[47] = step2[47]; + step1[48] = step2[48]; + step1[49] = step2[49]; + step1[50] = step2[49]; + step1[51] = step2[48]; + step1[52] = step2[55]; + step1[53] = step2[54]; + step1[54] = step2[54]; + step1[55] = step2[55]; + step1[56] = step2[56]; + step1[57] = step2[57]; + step1[58] = step2[57]; + step1[59] = step2[56]; + step1[60] = step2[63]; + step1[61] = step2[62]; + step1[62] = step2[62]; + step1[63] = step2[63]; + + // stage 6 + + btf_16_neon(step1[0], cospi[32], cospi[32], &step2[0], &step2[1]); + btf_16_lane_2_3_neon(step1[14], step1[9], c3, &step2[14], &step2[9]); + btf_16_lane_0_1_neon(step1[61], step1[34], c2, &step2[61], &step2[34]); + btf_16_lane_0_1_neon(step1[60], step1[35], c2, &step2[60], &step2[35]); + btf_16_lane_1_0_neon(vnegq_s16(step1[36]), vnegq_s16(step1[59]), c2, + &step2[36], &step2[59]); + btf_16_lane_1_0_neon(vnegq_s16(step1[37]), vnegq_s16(step1[58]), c2, + &step2[37], &step2[58]); + btf_16_lane_2_3_neon(step1[53], step1[42], c2, &step2[53], &step2[42]); + btf_16_lane_2_3_neon(step1[52], step1[43], c2, &step2[52], &step2[43]); + btf_16_lane_3_2_neon(vnegq_s16(step1[44]), vnegq_s16(step1[51]), c2, + &step2[44], &step2[51]); + btf_16_lane_3_2_neon(vnegq_s16(step1[45]), vnegq_s16(step1[50]), c2, + &step2[45], &step2[50]); + + step2[8] = step1[8]; + step2[15] = step1[15]; + step2[16] = step1[16]; + step2[17] = step1[17]; + step2[18] = step1[17]; + step2[19] = step1[16]; + step2[20] = step1[23]; + step2[21] = step1[22]; + step2[22] = step1[22]; + step2[23] = step1[23]; + step2[24] = step1[24]; + step2[25] = step1[25]; + step2[26] = step1[25]; + step2[27] = step1[24]; + step2[28] = step1[31]; + step2[29] = step1[30]; + step2[30] = step1[30]; + step2[31] = step1[31]; + step2[32] = step1[32]; + step2[33] = step1[33]; + step2[38] = step1[38]; + step2[39] = step1[39]; + step2[40] = step1[40]; + step2[41] = step1[41]; + step2[46] = step1[46]; + step2[47] = step1[47]; + step2[48] = step1[48]; + step2[49] = step1[49]; + step2[54] = step1[54]; + step2[55] = step1[55]; + step2[56] = step1[56]; + step2[57] = step1[57]; + step2[62] = step1[62]; + step2[63] = step1[63]; + + // stage 7 + + btf_16_lane_2_3_neon(step2[29], step2[18], c3, &step1[29], &step1[18]); + btf_16_lane_2_3_neon(step2[28], step2[19], c3, &step1[28], &step1[19]); + btf_16_lane_3_2_neon(vnegq_s16(step2[20]), vnegq_s16(step2[27]), c3, + &step1[20], &step1[27]); + btf_16_lane_3_2_neon(vnegq_s16(step2[21]), vnegq_s16(step2[26]), c3, + &step1[21], &step1[26]); + + step1[0] = step2[0]; + step1[1] = step2[1]; + step1[2] = step2[1]; + step1[3] = step2[0]; + step1[8] = step2[8]; + step1[9] = step2[9]; + step1[10] = step2[9]; + step1[11] = step2[8]; + step1[12] = step2[15]; + step1[13] = step2[14]; + step1[14] = step2[14]; + step1[15] = step2[15]; + step1[16] = step2[16]; + step1[17] = step2[17]; + step1[22] = step2[22]; + step1[23] = step2[23]; + step1[24] = step2[24]; + step1[25] = step2[25]; + step1[30] = step2[30]; + step1[31] = step2[31]; + step1[32] = vqaddq_s16(step2[32], step2[39]); + step1[33] = vqaddq_s16(step2[33], step2[38]); + step1[34] = vqaddq_s16(step2[34], step2[37]); + step1[35] = vqaddq_s16(step2[35], step2[36]); + step1[36] = vqsubq_s16(step2[35], step2[36]); + step1[37] = vqsubq_s16(step2[34], step2[37]); + step1[38] = vqsubq_s16(step2[33], step2[38]); + step1[39] = vqsubq_s16(step2[32], step2[39]); + step1[40] = vqsubq_s16(step2[47], step2[40]); + step1[41] = vqsubq_s16(step2[46], step2[41]); + step1[42] = vqsubq_s16(step2[45], step2[42]); + step1[43] = vqsubq_s16(step2[44], step2[43]); + step1[44] = vqaddq_s16(step2[43], step2[44]); + step1[45] = vqaddq_s16(step2[42], step2[45]); + step1[46] = vqaddq_s16(step2[41], step2[46]); + step1[47] = vqaddq_s16(step2[40], step2[47]); + step1[48] = vqaddq_s16(step2[48], step2[55]); + step1[49] = vqaddq_s16(step2[49], step2[54]); + step1[50] = vqaddq_s16(step2[50], step2[53]); + step1[51] = vqaddq_s16(step2[51], step2[52]); + step1[52] = vqsubq_s16(step2[51], step2[52]); + step1[53] = vqsubq_s16(step2[50], step2[53]); + step1[54] = vqsubq_s16(step2[49], step2[54]); + step1[55] = vqsubq_s16(step2[48], step2[55]); + step1[56] = vqsubq_s16(step2[63], step2[56]); + step1[57] = vqsubq_s16(step2[62], step2[57]); + step1[58] = vqsubq_s16(step2[61], step2[58]); + step1[59] = vqsubq_s16(step2[60], step2[59]); + step1[60] = vqaddq_s16(step2[59], step2[60]); + step1[61] = vqaddq_s16(step2[58], step2[61]); + step1[62] = vqaddq_s16(step2[57], step2[62]); + step1[63] = vqaddq_s16(step2[56], step2[63]); + + // stage 8 + + btf_16_lane_0_1_neon(step1[13], step1[10], c3, &step2[13], &step2[10]); + btf_16_lane_0_1_neon(step1[12], step1[11], c3, &step2[12], &step2[11]); + btf_16_lane_2_3_neon(step1[59], step1[36], c3, &step2[59], &step2[36]); + btf_16_lane_2_3_neon(step1[58], step1[37], c3, &step2[58], &step2[37]); + btf_16_lane_2_3_neon(step1[57], step1[38], c3, &step2[57], &step2[38]); + btf_16_lane_2_3_neon(step1[56], step1[39], c3, &step2[56], &step2[39]); + btf_16_lane_3_2_neon(vnegq_s16(step1[40]), vnegq_s16(step1[55]), c3, + &step2[40], &step2[55]); + btf_16_lane_3_2_neon(vnegq_s16(step1[41]), vnegq_s16(step1[54]), c3, + &step2[41], &step2[54]); + btf_16_lane_3_2_neon(vnegq_s16(step1[42]), vnegq_s16(step1[53]), c3, + &step2[42], &step2[53]); + btf_16_lane_3_2_neon(vnegq_s16(step1[43]), vnegq_s16(step1[52]), c3, + &step2[43], &step2[52]); + + step2[0] = step1[0]; + step2[1] = step1[1]; + step2[2] = step1[2]; + step2[3] = step1[3]; + step2[4] = step1[3]; + step2[5] = step1[2]; + step2[6] = step1[1]; + step2[7] = step1[0]; + step2[8] = step1[8]; + step2[9] = step1[9]; + step2[14] = step1[14]; + step2[15] = step1[15]; + step2[16] = vqaddq_s16(step1[16], step1[23]); + step2[17] = vqaddq_s16(step1[17], step1[22]); + step2[18] = vqaddq_s16(step1[18], step1[21]); + step2[19] = vqaddq_s16(step1[19], step1[20]); + step2[20] = vqsubq_s16(step1[19], step1[20]); + step2[21] = vqsubq_s16(step1[18], step1[21]); + step2[22] = vqsubq_s16(step1[17], step1[22]); + step2[23] = vqsubq_s16(step1[16], step1[23]); + step2[24] = vqsubq_s16(step1[31], step1[24]); + step2[25] = vqsubq_s16(step1[30], step1[25]); + step2[26] = vqsubq_s16(step1[29], step1[26]); + step2[27] = vqsubq_s16(step1[28], step1[27]); + step2[28] = vqaddq_s16(step1[28], step1[27]); + step2[29] = vqaddq_s16(step1[29], step1[26]); + step2[30] = vqaddq_s16(step1[30], step1[25]); + step2[31] = vqaddq_s16(step1[31], step1[24]); + step2[32] = step1[32]; + step2[33] = step1[33]; + step2[34] = step1[34]; + step2[35] = step1[35]; + step2[44] = step1[44]; + step2[45] = step1[45]; + step2[46] = step1[46]; + step2[47] = step1[47]; + step2[48] = step1[48]; + step2[49] = step1[49]; + step2[50] = step1[50]; + step2[51] = step1[51]; + step2[60] = step1[60]; + step2[61] = step1[61]; + step2[62] = step1[62]; + step2[63] = step1[63]; + + // stage 9 + idct64_stage9_neon(step2, step1, cos_bit); + + // stage 10 + idct64_stage10_neon(step1, step2, cos_bit); + + // stage 11 + + out[0] = vqaddq_s16(step2[0], step2[63]); + out[1] = vqaddq_s16(step2[1], step2[62]); + out[2] = vqaddq_s16(step2[2], step2[61]); + out[3] = vqaddq_s16(step2[3], step2[60]); + out[4] = vqaddq_s16(step2[4], step2[59]); + out[5] = vqaddq_s16(step2[5], step2[58]); + out[6] = vqaddq_s16(step2[6], step2[57]); + out[7] = vqaddq_s16(step2[7], step2[56]); + out[8] = vqaddq_s16(step2[8], step2[55]); + out[9] = vqaddq_s16(step2[9], step2[54]); + out[10] = vqaddq_s16(step2[10], step2[53]); + out[11] = vqaddq_s16(step2[11], step2[52]); + out[12] = vqaddq_s16(step2[12], step2[51]); + out[13] = vqaddq_s16(step2[13], step2[50]); + out[14] = vqaddq_s16(step2[14], step2[49]); + out[15] = vqaddq_s16(step2[15], step2[48]); + out[16] = vqaddq_s16(step2[16], step2[47]); + out[17] = vqaddq_s16(step2[17], step2[46]); + out[18] = vqaddq_s16(step2[18], step2[45]); + out[19] = vqaddq_s16(step2[19], step2[44]); + out[20] = vqaddq_s16(step2[20], step2[43]); + out[21] = vqaddq_s16(step2[21], step2[42]); + out[22] = vqaddq_s16(step2[22], step2[41]); + out[23] = vqaddq_s16(step2[23], step2[40]); + out[24] = vqaddq_s16(step2[24], step2[39]); + out[25] = vqaddq_s16(step2[25], step2[38]); + out[26] = vqaddq_s16(step2[26], step2[37]); + out[27] = vqaddq_s16(step2[27], step2[36]); + out[28] = vqaddq_s16(step2[28], step2[35]); + out[29] = vqaddq_s16(step2[29], step2[34]); + out[30] = vqaddq_s16(step2[30], step2[33]); + out[31] = vqaddq_s16(step2[31], step2[32]); + out[32] = vqsubq_s16(step2[31], step2[32]); + out[33] = vqsubq_s16(step2[30], step2[33]); + out[34] = vqsubq_s16(step2[29], step2[34]); + out[35] = vqsubq_s16(step2[28], step2[35]); + out[36] = vqsubq_s16(step2[27], step2[36]); + out[37] = vqsubq_s16(step2[26], step2[37]); + out[38] = vqsubq_s16(step2[25], step2[38]); + out[39] = vqsubq_s16(step2[24], step2[39]); + out[40] = vqsubq_s16(step2[23], step2[40]); + out[41] = vqsubq_s16(step2[22], step2[41]); + out[42] = vqsubq_s16(step2[21], step2[42]); + out[43] = vqsubq_s16(step2[20], step2[43]); + out[44] = vqsubq_s16(step2[19], step2[44]); + out[45] = vqsubq_s16(step2[18], step2[45]); + out[46] = vqsubq_s16(step2[17], step2[46]); + out[47] = vqsubq_s16(step2[16], step2[47]); + out[48] = vqsubq_s16(step2[15], step2[48]); + out[49] = vqsubq_s16(step2[14], step2[49]); + out[50] = vqsubq_s16(step2[13], step2[50]); + out[51] = vqsubq_s16(step2[12], step2[51]); + out[52] = vqsubq_s16(step2[11], step2[52]); + out[53] = vqsubq_s16(step2[10], step2[53]); + out[54] = vqsubq_s16(step2[9], step2[54]); + out[55] = vqsubq_s16(step2[8], step2[55]); + out[56] = vqsubq_s16(step2[7], step2[56]); + out[57] = vqsubq_s16(step2[6], step2[57]); + out[58] = vqsubq_s16(step2[5], step2[58]); + out[59] = vqsubq_s16(step2[4], step2[59]); + out[60] = vqsubq_s16(step2[3], step2[60]); + out[61] = vqsubq_s16(step2[2], step2[61]); + out[62] = vqsubq_s16(step2[1], step2[62]); + out[63] = vqsubq_s16(step2[0], step2[63]); +} + +static INLINE void idct64_low16_new_neon(int16x8_t *in, int16x8_t *out, + int8_t cos_bit, int bit) { + (void)bit; + const int32_t *cospi = cospi_arr(cos_bit); + int16x8_t step2[64], step1[64]; + + const int16x4_t c0 = + create_s16x4_neon((int16_t *)(cospi + 4), (int16_t *)(cospi + 60), + (int16_t *)(cospi + 36), (int16_t *)(cospi + 28)); + const int16x4_t c1 = + create_s16x4_neon((int16_t *)(cospi + 20), (int16_t *)(cospi + 44), + (int16_t *)(cospi + 52), (int16_t *)(cospi + 12)); + const int16x4_t c2 = + create_s16x4_neon((int16_t *)(cospi + 8), (int16_t *)(cospi + 56), + (int16_t *)(cospi + 40), (int16_t *)(cospi + 24)); + const int16x4_t c3 = + create_s16x4_neon((int16_t *)(cospi + 32), (int16_t *)(cospi + 32), + (int16_t *)(cospi + 16), (int16_t *)(cospi + 48)); + + // stage 1 + // stage 2 + + step2[0] = in[0]; + step2[4] = in[8]; + step2[8] = in[4]; + step2[12] = in[12]; + step2[16] = in[2]; + step2[20] = in[10]; + step2[24] = in[6]; + step2[28] = in[14]; + + btf_16_neon(in[1], cospi[63], cospi[1], &step2[32], &step2[63]); + btf_16_neon(in[15], -cospi[49], cospi[15], &step2[35], &step2[60]); + btf_16_neon(in[9], cospi[55], cospi[9], &step2[36], &step2[59]); + btf_16_neon(in[7], -cospi[57], cospi[7], &step2[39], &step2[56]); + btf_16_neon(in[5], cospi[59], cospi[5], &step2[40], &step2[55]); + btf_16_neon(in[11], -cospi[53], cospi[11], &step2[43], &step2[52]); + btf_16_neon(in[13], cospi[51], cospi[13], &step2[44], &step2[51]); + btf_16_neon(in[3], -cospi[61], cospi[3], &step2[47], &step2[48]); + + // stage 3 + + step1[0] = step2[0]; + step1[4] = step2[4]; + step1[8] = step2[8]; + step1[12] = step2[12]; + + btf_16_neon(step2[16], cospi[62], cospi[2], &step1[16], &step1[31]); + btf_16_neon(step2[20], cospi[54], cospi[10], &step1[20], &step1[27]); + btf_16_neon(step2[24], -cospi[58], cospi[6], &step1[23], &step1[24]); + btf_16_neon(step2[28], -cospi[50], cospi[14], &step1[19], &step1[28]); + + step1[32] = step2[32]; + step1[33] = step2[32]; + step1[34] = step2[35]; + step1[35] = step2[35]; + step1[36] = step2[36]; + step1[37] = step2[36]; + step1[38] = step2[39]; + step1[39] = step2[39]; + step1[40] = step2[40]; + step1[41] = step2[40]; + step1[42] = step2[43]; + step1[43] = step2[43]; + step1[44] = step2[44]; + step1[45] = step2[44]; + step1[46] = step2[47]; + step1[47] = step2[47]; + step1[48] = step2[48]; + step1[49] = step2[48]; + step1[50] = step2[51]; + step1[51] = step2[51]; + step1[52] = step2[52]; + step1[53] = step2[52]; + step1[54] = step2[55]; + step1[55] = step2[55]; + step1[56] = step2[56]; + step1[57] = step2[56]; + step1[58] = step2[59]; + step1[59] = step2[59]; + step1[60] = step2[60]; + step1[61] = step2[60]; + step1[62] = step2[63]; + step1[63] = step2[63]; + + // stage 4 + + step2[0] = step1[0]; + step2[4] = step1[4]; + + btf_16_neon(step1[8], cospi[60], cospi[4], &step2[8], &step2[15]); + btf_16_neon(step1[12], -cospi[52], cospi[12], &step2[11], &step2[12]); + btf_16_lane_0_1_neon(step1[62], step1[33], c0, &step2[62], &step2[33]); + btf_16_lane_1_0_neon(vnegq_s16(step1[34]), vnegq_s16(step1[61]), c0, + &step2[34], &step2[61]); + btf_16_lane_2_3_neon(step1[58], step1[37], c0, &step2[58], &step2[37]); + btf_16_lane_3_2_neon(vnegq_s16(step1[38]), vnegq_s16(step1[57]), c0, + &step2[38], &step2[57]); + btf_16_lane_0_1_neon(step1[54], step1[41], c1, &step2[54], &step2[41]); + btf_16_lane_1_0_neon(vnegq_s16(step1[42]), vnegq_s16(step1[53]), c1, + &step2[42], &step2[53]); + btf_16_lane_2_3_neon(step1[50], step1[45], c1, &step2[50], &step2[45]); + btf_16_lane_3_2_neon(vnegq_s16(step1[46]), vnegq_s16(step1[49]), c1, + &step2[46], &step2[49]); + + step2[16] = step1[16]; + step2[17] = step1[16]; + step2[18] = step1[19]; + step2[19] = step1[19]; + step2[20] = step1[20]; + step2[21] = step1[20]; + step2[22] = step1[23]; + step2[23] = step1[23]; + step2[24] = step1[24]; + step2[25] = step1[24]; + step2[26] = step1[27]; + step2[27] = step1[27]; + step2[28] = step1[28]; + step2[29] = step1[28]; + step2[30] = step1[31]; + step2[31] = step1[31]; + step2[32] = step1[32]; + step2[35] = step1[35]; + step2[36] = step1[36]; + step2[39] = step1[39]; + step2[40] = step1[40]; + step2[43] = step1[43]; + step2[44] = step1[44]; + step2[47] = step1[47]; + step2[48] = step1[48]; + step2[51] = step1[51]; + step2[52] = step1[52]; + step2[55] = step1[55]; + step2[56] = step1[56]; + step2[59] = step1[59]; + step2[60] = step1[60]; + step2[63] = step1[63]; + + // stage 5 + + step1[0] = step2[0]; + + btf_16_neon(step2[4], cospi[56], cospi[8], &step1[4], &step1[7]); + btf_16_lane_0_1_neon(step2[30], step2[17], c2, &step1[30], &step1[17]); + btf_16_lane_1_0_neon(vnegq_s16(step2[18]), vnegq_s16(step2[29]), c2, + &step1[18], &step1[29]); + btf_16_lane_2_3_neon(step2[26], step2[21], c2, &step1[26], &step1[21]); + btf_16_lane_3_2_neon(vnegq_s16(step2[22]), vnegq_s16(step2[25]), c2, + &step1[22], &step1[25]); + + step1[8] = step2[8]; + step1[9] = step2[8]; + step1[10] = step2[11]; + step1[11] = step2[11]; + step1[12] = step2[12]; + step1[13] = step2[12]; + step1[14] = step2[15]; + step1[15] = step2[15]; + step1[16] = step2[16]; + step1[19] = step2[19]; + step1[20] = step2[20]; + step1[23] = step2[23]; + step1[24] = step2[24]; + step1[27] = step2[27]; + step1[28] = step2[28]; + step1[31] = step2[31]; + step1[32] = vqaddq_s16(step2[32], step2[35]); + step1[33] = vqaddq_s16(step2[33], step2[34]); + step1[34] = vqsubq_s16(step2[33], step2[34]); + step1[35] = vqsubq_s16(step2[32], step2[35]); + step1[36] = vqsubq_s16(step2[39], step2[36]); + step1[37] = vqsubq_s16(step2[38], step2[37]); + step1[38] = vqaddq_s16(step2[38], step2[37]); + step1[39] = vqaddq_s16(step2[39], step2[36]); + step1[40] = vqaddq_s16(step2[40], step2[43]); + step1[41] = vqaddq_s16(step2[41], step2[42]); + step1[42] = vqsubq_s16(step2[41], step2[42]); + step1[43] = vqsubq_s16(step2[40], step2[43]); + step1[44] = vqsubq_s16(step2[47], step2[44]); + step1[45] = vqsubq_s16(step2[46], step2[45]); + step1[46] = vqaddq_s16(step2[46], step2[45]); + step1[47] = vqaddq_s16(step2[47], step2[44]); + step1[48] = vqaddq_s16(step2[48], step2[51]); + step1[49] = vqaddq_s16(step2[49], step2[50]); + step1[50] = vqsubq_s16(step2[49], step2[50]); + step1[51] = vqsubq_s16(step2[48], step2[51]); + step1[52] = vqsubq_s16(step2[55], step2[52]); + step1[53] = vqsubq_s16(step2[54], step2[53]); + step1[54] = vqaddq_s16(step2[54], step2[53]); + step1[55] = vqaddq_s16(step2[55], step2[52]); + step1[56] = vqaddq_s16(step2[56], step2[59]); + step1[57] = vqaddq_s16(step2[57], step2[58]); + step1[58] = vqsubq_s16(step2[57], step2[58]); + step1[59] = vqsubq_s16(step2[56], step2[59]); + step1[60] = vqsubq_s16(step2[63], step2[60]); + step1[61] = vqsubq_s16(step2[62], step2[61]); + step1[62] = vqaddq_s16(step2[62], step2[61]); + step1[63] = vqaddq_s16(step2[63], step2[60]); + + // stage 6 + + btf_16_neon(step1[0], cospi[32], cospi[32], &step2[0], &step2[1]); + btf_16_lane_2_3_neon(step1[14], step1[9], c3, &step2[14], &step2[9]); + btf_16_lane_3_2_neon(vnegq_s16(step1[10]), vnegq_s16(step1[13]), c3, + &step2[10], &step2[13]); + btf_16_lane_0_1_neon(step1[61], step1[34], c2, &step2[61], &step2[34]); + btf_16_lane_0_1_neon(step1[60], step1[35], c2, &step2[60], &step2[35]); + btf_16_lane_1_0_neon(vnegq_s16(step1[36]), vnegq_s16(step1[59]), c2, + &step2[36], &step2[59]); + btf_16_lane_1_0_neon(vnegq_s16(step1[37]), vnegq_s16(step1[58]), c2, + &step2[37], &step2[58]); + btf_16_lane_2_3_neon(step1[53], step1[42], c2, &step2[53], &step2[42]); + btf_16_lane_2_3_neon(step1[52], step1[43], c2, &step2[52], &step2[43]); + btf_16_lane_3_2_neon(vnegq_s16(step1[44]), vnegq_s16(step1[51]), c2, + &step2[44], &step2[51]); + btf_16_lane_3_2_neon(vnegq_s16(step1[45]), vnegq_s16(step1[50]), c2, + &step2[45], &step2[50]); + + step2[4] = step1[4]; + step2[5] = step1[4]; + step2[6] = step1[7]; + step2[7] = step1[7]; + step2[8] = step1[8]; + step2[11] = step1[11]; + step2[12] = step1[12]; + step2[15] = step1[15]; + step2[16] = vqaddq_s16(step1[16], step1[19]); + step2[17] = vqaddq_s16(step1[17], step1[18]); + step2[18] = vqsubq_s16(step1[17], step1[18]); + step2[19] = vqsubq_s16(step1[16], step1[19]); + step2[20] = vqsubq_s16(step1[23], step1[20]); + step2[21] = vqsubq_s16(step1[22], step1[21]); + step2[22] = vqaddq_s16(step1[22], step1[21]); + step2[23] = vqaddq_s16(step1[23], step1[20]); + step2[24] = vqaddq_s16(step1[24], step1[27]); + step2[25] = vqaddq_s16(step1[25], step1[26]); + step2[26] = vqsubq_s16(step1[25], step1[26]); + step2[27] = vqsubq_s16(step1[24], step1[27]); + step2[28] = vqsubq_s16(step1[31], step1[28]); + step2[29] = vqsubq_s16(step1[30], step1[29]); + step2[30] = vqaddq_s16(step1[30], step1[29]); + step2[31] = vqaddq_s16(step1[31], step1[28]); + step2[32] = step1[32]; + step2[33] = step1[33]; + step2[38] = step1[38]; + step2[39] = step1[39]; + step2[40] = step1[40]; + step2[41] = step1[41]; + step2[46] = step1[46]; + step2[47] = step1[47]; + step2[48] = step1[48]; + step2[49] = step1[49]; + step2[54] = step1[54]; + step2[55] = step1[55]; + step2[56] = step1[56]; + step2[57] = step1[57]; + step2[62] = step1[62]; + step2[63] = step1[63]; + + // stage 7 + + btf_16_lane_0_1_neon(step2[6], step2[5], c3, &step1[6], &step1[5]); + btf_16_lane_2_3_neon(step2[29], step2[18], c3, &step1[29], &step1[18]); + btf_16_lane_2_3_neon(step2[28], step2[19], c3, &step1[28], &step1[19]); + btf_16_lane_3_2_neon(vnegq_s16(step2[20]), vnegq_s16(step2[27]), c3, + &step1[20], &step1[27]); + btf_16_lane_3_2_neon(vnegq_s16(step2[21]), vnegq_s16(step2[26]), c3, + &step1[21], &step1[26]); + + step1[0] = step2[0]; + step1[1] = step2[1]; + step1[2] = step2[1]; + step1[3] = step2[0]; + step1[4] = step2[4]; + step1[7] = step2[7]; + step1[8] = vqaddq_s16(step2[8], step2[11]); + step1[9] = vqaddq_s16(step2[9], step2[10]); + step1[10] = vqsubq_s16(step2[9], step2[10]); + step1[11] = vqsubq_s16(step2[8], step2[11]); + step1[12] = vqsubq_s16(step2[15], step2[12]); + step1[13] = vqsubq_s16(step2[14], step2[13]); + step1[14] = vqaddq_s16(step2[14], step2[13]); + step1[15] = vqaddq_s16(step2[15], step2[12]); + step1[16] = step2[16]; + step1[17] = step2[17]; + step1[22] = step2[22]; + step1[23] = step2[23]; + step1[24] = step2[24]; + step1[25] = step2[25]; + step1[30] = step2[30]; + step1[31] = step2[31]; + step1[32] = vqaddq_s16(step2[32], step2[39]); + step1[33] = vqaddq_s16(step2[33], step2[38]); + step1[34] = vqaddq_s16(step2[34], step2[37]); + step1[35] = vqaddq_s16(step2[35], step2[36]); + step1[36] = vqsubq_s16(step2[35], step2[36]); + step1[37] = vqsubq_s16(step2[34], step2[37]); + step1[38] = vqsubq_s16(step2[33], step2[38]); + step1[39] = vqsubq_s16(step2[32], step2[39]); + step1[40] = vqsubq_s16(step2[47], step2[40]); + step1[41] = vqsubq_s16(step2[46], step2[41]); + step1[42] = vqsubq_s16(step2[45], step2[42]); + step1[43] = vqsubq_s16(step2[44], step2[43]); + step1[44] = vqaddq_s16(step2[43], step2[44]); + step1[45] = vqaddq_s16(step2[42], step2[45]); + step1[46] = vqaddq_s16(step2[41], step2[46]); + step1[47] = vqaddq_s16(step2[40], step2[47]); + step1[48] = vqaddq_s16(step2[48], step2[55]); + step1[49] = vqaddq_s16(step2[49], step2[54]); + step1[50] = vqaddq_s16(step2[50], step2[53]); + step1[51] = vqaddq_s16(step2[51], step2[52]); + step1[52] = vqsubq_s16(step2[51], step2[52]); + step1[53] = vqsubq_s16(step2[50], step2[53]); + step1[54] = vqsubq_s16(step2[49], step2[54]); + step1[55] = vqsubq_s16(step2[48], step2[55]); + step1[56] = vqsubq_s16(step2[63], step2[56]); + step1[57] = vqsubq_s16(step2[62], step2[57]); + step1[58] = vqsubq_s16(step2[61], step2[58]); + step1[59] = vqsubq_s16(step2[60], step2[59]); + step1[60] = vqaddq_s16(step2[59], step2[60]); + step1[61] = vqaddq_s16(step2[58], step2[61]); + step1[62] = vqaddq_s16(step2[57], step2[62]); + step1[63] = vqaddq_s16(step2[56], step2[63]); + + // stage 8 + + btf_16_lane_0_1_neon(step1[13], step1[10], c3, &step2[13], &step2[10]); + btf_16_lane_0_1_neon(step1[12], step1[11], c3, &step2[12], &step2[11]); + btf_16_lane_2_3_neon(step1[59], step1[36], c3, &step2[59], &step2[36]); + btf_16_lane_2_3_neon(step1[58], step1[37], c3, &step2[58], &step2[37]); + btf_16_lane_2_3_neon(step1[57], step1[38], c3, &step2[57], &step2[38]); + btf_16_lane_2_3_neon(step1[56], step1[39], c3, &step2[56], &step2[39]); + btf_16_lane_3_2_neon(vnegq_s16(step1[40]), vnegq_s16(step1[55]), c3, + &step2[40], &step2[55]); + btf_16_lane_3_2_neon(vnegq_s16(step1[41]), vnegq_s16(step1[54]), c3, + &step2[41], &step2[54]); + btf_16_lane_3_2_neon(vnegq_s16(step1[42]), vnegq_s16(step1[53]), c3, + &step2[42], &step2[53]); + btf_16_lane_3_2_neon(vnegq_s16(step1[43]), vnegq_s16(step1[52]), c3, + &step2[43], &step2[52]); + + step2[0] = vqaddq_s16(step1[0], step1[7]); + step2[1] = vqaddq_s16(step1[1], step1[6]); + step2[2] = vqaddq_s16(step1[2], step1[5]); + step2[3] = vqaddq_s16(step1[3], step1[4]); + step2[4] = vqsubq_s16(step1[3], step1[4]); + step2[5] = vqsubq_s16(step1[2], step1[5]); + step2[6] = vqsubq_s16(step1[1], step1[6]); + step2[7] = vqsubq_s16(step1[0], step1[7]); + step2[8] = step1[8]; + step2[9] = step1[9]; + step2[14] = step1[14]; + step2[15] = step1[15]; + step2[16] = vqaddq_s16(step1[16], step1[23]); + step2[17] = vqaddq_s16(step1[17], step1[22]); + step2[18] = vqaddq_s16(step1[18], step1[21]); + step2[19] = vqaddq_s16(step1[19], step1[20]); + step2[20] = vqsubq_s16(step1[19], step1[20]); + step2[21] = vqsubq_s16(step1[18], step1[21]); + step2[22] = vqsubq_s16(step1[17], step1[22]); + step2[23] = vqsubq_s16(step1[16], step1[23]); + step2[24] = vqsubq_s16(step1[31], step1[24]); + step2[25] = vqsubq_s16(step1[30], step1[25]); + step2[26] = vqsubq_s16(step1[29], step1[26]); + step2[27] = vqsubq_s16(step1[28], step1[27]); + step2[28] = vqaddq_s16(step1[28], step1[27]); + step2[29] = vqaddq_s16(step1[29], step1[26]); + step2[30] = vqaddq_s16(step1[30], step1[25]); + step2[31] = vqaddq_s16(step1[31], step1[24]); + step2[32] = step1[32]; + step2[33] = step1[33]; + step2[34] = step1[34]; + step2[35] = step1[35]; + step2[44] = step1[44]; + step2[45] = step1[45]; + step2[46] = step1[46]; + step2[47] = step1[47]; + step2[48] = step1[48]; + step2[49] = step1[49]; + step2[50] = step1[50]; + step2[51] = step1[51]; + step2[60] = step1[60]; + step2[61] = step1[61]; + step2[62] = step1[62]; + step2[63] = step1[63]; + + // stage 9 + idct64_stage9_neon(step2, step1, cos_bit); + + // stage 10 + idct64_stage10_neon(step1, step2, cos_bit); + + // stage 11 + + out[0] = vqaddq_s16(step2[0], step2[63]); + out[1] = vqaddq_s16(step2[1], step2[62]); + out[2] = vqaddq_s16(step2[2], step2[61]); + out[3] = vqaddq_s16(step2[3], step2[60]); + out[4] = vqaddq_s16(step2[4], step2[59]); + out[5] = vqaddq_s16(step2[5], step2[58]); + out[6] = vqaddq_s16(step2[6], step2[57]); + out[7] = vqaddq_s16(step2[7], step2[56]); + out[8] = vqaddq_s16(step2[8], step2[55]); + out[9] = vqaddq_s16(step2[9], step2[54]); + out[10] = vqaddq_s16(step2[10], step2[53]); + out[11] = vqaddq_s16(step2[11], step2[52]); + out[12] = vqaddq_s16(step2[12], step2[51]); + out[13] = vqaddq_s16(step2[13], step2[50]); + out[14] = vqaddq_s16(step2[14], step2[49]); + out[15] = vqaddq_s16(step2[15], step2[48]); + out[16] = vqaddq_s16(step2[16], step2[47]); + out[17] = vqaddq_s16(step2[17], step2[46]); + out[18] = vqaddq_s16(step2[18], step2[45]); + out[19] = vqaddq_s16(step2[19], step2[44]); + out[20] = vqaddq_s16(step2[20], step2[43]); + out[21] = vqaddq_s16(step2[21], step2[42]); + out[22] = vqaddq_s16(step2[22], step2[41]); + out[23] = vqaddq_s16(step2[23], step2[40]); + out[24] = vqaddq_s16(step2[24], step2[39]); + out[25] = vqaddq_s16(step2[25], step2[38]); + out[26] = vqaddq_s16(step2[26], step2[37]); + out[27] = vqaddq_s16(step2[27], step2[36]); + out[28] = vqaddq_s16(step2[28], step2[35]); + out[29] = vqaddq_s16(step2[29], step2[34]); + out[30] = vqaddq_s16(step2[30], step2[33]); + out[31] = vqaddq_s16(step2[31], step2[32]); + out[32] = vqsubq_s16(step2[31], step2[32]); + out[33] = vqsubq_s16(step2[30], step2[33]); + out[34] = vqsubq_s16(step2[29], step2[34]); + out[35] = vqsubq_s16(step2[28], step2[35]); + out[36] = vqsubq_s16(step2[27], step2[36]); + out[37] = vqsubq_s16(step2[26], step2[37]); + out[38] = vqsubq_s16(step2[25], step2[38]); + out[39] = vqsubq_s16(step2[24], step2[39]); + out[40] = vqsubq_s16(step2[23], step2[40]); + out[41] = vqsubq_s16(step2[22], step2[41]); + out[42] = vqsubq_s16(step2[21], step2[42]); + out[43] = vqsubq_s16(step2[20], step2[43]); + out[44] = vqsubq_s16(step2[19], step2[44]); + out[45] = vqsubq_s16(step2[18], step2[45]); + out[46] = vqsubq_s16(step2[17], step2[46]); + out[47] = vqsubq_s16(step2[16], step2[47]); + out[48] = vqsubq_s16(step2[15], step2[48]); + out[49] = vqsubq_s16(step2[14], step2[49]); + out[50] = vqsubq_s16(step2[13], step2[50]); + out[51] = vqsubq_s16(step2[12], step2[51]); + out[52] = vqsubq_s16(step2[11], step2[52]); + out[53] = vqsubq_s16(step2[10], step2[53]); + out[54] = vqsubq_s16(step2[9], step2[54]); + out[55] = vqsubq_s16(step2[8], step2[55]); + out[56] = vqsubq_s16(step2[7], step2[56]); + out[57] = vqsubq_s16(step2[6], step2[57]); + out[58] = vqsubq_s16(step2[5], step2[58]); + out[59] = vqsubq_s16(step2[4], step2[59]); + out[60] = vqsubq_s16(step2[3], step2[60]); + out[61] = vqsubq_s16(step2[2], step2[61]); + out[62] = vqsubq_s16(step2[1], step2[62]); + out[63] = vqsubq_s16(step2[0], step2[63]); +} // Functions for blocks with eob at DC and within // topleft 8x8, 16x16, 32x32 corner -static const transform_1d_neon - lowbd_txfm_all_1d_zeros_w8_arr[TX_SIZES][ITX_TYPES_1D][4] = { - { - { av1_idct4_new, av1_idct4_new, NULL, NULL }, - { av1_iadst4_new, av1_iadst4_new, NULL, NULL }, - { av1_iidentity4_c, av1_iidentity4_c, NULL, NULL }, - }, - { { av1_idct8_new, av1_idct8_new, NULL, NULL }, - { av1_iadst8_new, av1_iadst8_new, NULL, NULL }, - { av1_iidentity8_c, av1_iidentity8_c, NULL, NULL } }, - { - { av1_idct16_new, av1_idct16_new, av1_idct16_new, NULL }, - { av1_iadst16_new, av1_iadst16_new, av1_iadst16_new, NULL }, - { av1_iidentity16_c, av1_iidentity16_c, av1_iidentity16_c, NULL }, - }, - { { av1_idct32_new, av1_idct32_new, av1_idct32_new, av1_idct32_new }, - { NULL, NULL, NULL, NULL }, - { av1_iidentity32_c, av1_iidentity32_c, av1_iidentity32_c, - av1_iidentity32_c } }, - { { av1_idct64_new, av1_idct64_new, av1_idct64_new, av1_idct64_new }, - { NULL, NULL, NULL, NULL }, - { NULL, NULL, NULL, NULL } } - }; - static const transform_neon lowbd_txfm_all_1d_zeros_w_arr[TX_SIZES][ITX_TYPES_1D][4] = { { @@ -2135,80 +3672,12 @@ static const transform_neon { NULL, NULL, NULL, NULL }, { identity32_new_neon, identity32_new_neon, identity32_new_neon, identity32_new_neon } }, - { { NULL, NULL, NULL, NULL }, + { { idct64_low1_new_neon, idct64_low8_new_neon, idct64_low16_new_neon, + idct64_low32_new_neon }, { NULL, NULL, NULL, NULL }, { NULL, NULL, NULL, NULL } } }; -static INLINE void lowbd_inv_txfm2d_add_wxh_idtx_neon( - const int32_t *input, uint8_t *output, int stride, TX_TYPE tx_type, - TX_SIZE tx_size, int eob) { - DECLARE_ALIGNED(32, int, txfm_buf[32 * 32 + 32 + 32]); - int32_t *temp_in = txfm_buf; - - int eobx, eoby; - get_eobx_eoby_scan_default(&eobx, &eoby, tx_size, eob); - const int8_t *shift = inv_txfm_shift_ls[tx_size]; - const int txw_idx = get_txw_idx(tx_size); - const int txh_idx = get_txh_idx(tx_size); - const int cos_bit_col = inv_cos_bit_col[txw_idx][txh_idx]; - const int cos_bit_row = inv_cos_bit_row[txw_idx][txh_idx]; - const int txfm_size_col = tx_size_wide[tx_size]; - const int txfm_size_row = tx_size_high[tx_size]; - const int buf_size_nonzero_h_div8 = (eoby + 8) >> 3; - - const int rect_type = get_rect_tx_log_ratio(txfm_size_col, txfm_size_row); - const int buf_offset = AOMMAX(txfm_size_row, txfm_size_col); - - int32_t *temp_out = temp_in + buf_offset; - int32_t *buf = temp_out + buf_offset; - int32_t *buf_ptr = buf; - const int8_t stage_range[MAX_TXFM_STAGE_NUM] = { 16 }; - int r, bd = 8; - - const int fun_idx_x = lowbd_txfm_all_1d_zeros_idx[eobx]; - const int fun_idx_y = lowbd_txfm_all_1d_zeros_idx[eoby]; - const transform_1d_neon row_txfm = - lowbd_txfm_all_1d_zeros_w8_arr[txw_idx][hitx_1d_tab[tx_type]][fun_idx_x]; - const transform_1d_neon col_txfm = - lowbd_txfm_all_1d_zeros_w8_arr[txh_idx][vitx_1d_tab[tx_type]][fun_idx_y]; - - assert(col_txfm != NULL); - assert(row_txfm != NULL); - - // row tx - int row_start = (buf_size_nonzero_h_div8 * 8); - for (int i = 0; i < row_start; i++) { - if (abs(rect_type) == 1) { - for (int j = 0; j < txfm_size_col; j++) - temp_in[j] = round_shift((int64_t)input[j] * NewInvSqrt2, NewSqrt2Bits); - row_txfm(temp_in, buf_ptr, cos_bit_row, stage_range); - } else { - row_txfm(input, buf_ptr, cos_bit_row, stage_range); - } - av1_round_shift_array(buf_ptr, txfm_size_col, -shift[0]); - input += txfm_size_col; - buf_ptr += txfm_size_col; - } - - // Doing memset for the rows which are not processed in row transform. - memset(buf_ptr, 0, - sizeof(int32_t) * txfm_size_col * (txfm_size_row - row_start)); - - // col tx - for (int c = 0; c < txfm_size_col; c++) { - for (r = 0; r < txfm_size_row; ++r) temp_in[r] = buf[r * txfm_size_col + c]; - - col_txfm(temp_in, temp_out, cos_bit_col, stage_range); - av1_round_shift_array(temp_out, txfm_size_row, -shift[1]); - - for (r = 0; r < txfm_size_row; ++r) { - output[r * stride + c] = - highbd_clip_pixel_add(output[r * stride + c], temp_out[r], bd); - } - } -} - static INLINE void lowbd_inv_txfm2d_add_idtx_neon(const int32_t *input, uint8_t *output, int stride, TX_TYPE tx_type, @@ -2281,90 +3750,6 @@ static INLINE void lowbd_inv_txfm2d_add_idtx_neon(const int32_t *input, } } -static INLINE void lowbd_inv_txfm2d_add_v_wxh_identity_neon( - const int32_t *input, uint8_t *output, int stride, TX_TYPE tx_type, - TX_SIZE tx_size, int eob) { - DECLARE_ALIGNED(32, int, txfm_buf[32 * 32 + 32 + 32]); - int32_t *temp_in = txfm_buf; - - int eobx, eoby; - get_eobx_eoby_scan_v_identity(&eobx, &eoby, tx_size, eob); - const int8_t *shift = inv_txfm_shift_ls[tx_size]; - const int txw_idx = get_txw_idx(tx_size); - const int txh_idx = get_txh_idx(tx_size); - const int cos_bit_col = inv_cos_bit_col[txw_idx][txh_idx]; - const int cos_bit_row = inv_cos_bit_row[txw_idx][txh_idx]; - const int txfm_size_col = tx_size_wide[tx_size]; - const int txfm_size_row = tx_size_high[tx_size]; - const int buf_size_nonzero_h_div8 = (eoby + 8) >> 3; - - const int rect_type = get_rect_tx_log_ratio(txfm_size_col, txfm_size_row); - const int buf_offset = AOMMAX(txfm_size_row, txfm_size_col); - - int32_t *temp_out = temp_in + buf_offset; - int32_t *buf = temp_out + buf_offset; - int32_t *buf_ptr = buf; - const int8_t stage_range[MAX_TXFM_STAGE_NUM] = { 16 }; - int r, bd = 8; - - const int fun_idx_x = lowbd_txfm_all_1d_zeros_idx[eobx]; - const int fun_idx_y = lowbd_txfm_all_1d_zeros_idx[eoby]; - const transform_1d_neon row_txfm = - lowbd_txfm_all_1d_zeros_w8_arr[txw_idx][hitx_1d_tab[tx_type]][fun_idx_x]; - const transform_1d_neon col_txfm = - lowbd_txfm_all_1d_zeros_w8_arr[txh_idx][vitx_1d_tab[tx_type]][fun_idx_y]; - - assert(col_txfm != NULL); - assert(row_txfm != NULL); - int ud_flip, lr_flip; - get_flip_cfg(tx_type, &ud_flip, &lr_flip); - - // row tx - int row_start = (buf_size_nonzero_h_div8 * 8); - for (int i = 0; i < row_start; i++) { - if (abs(rect_type) == 1) { - for (int j = 0; j < txfm_size_col; j++) - temp_in[j] = round_shift((int64_t)input[j] * NewInvSqrt2, NewSqrt2Bits); - row_txfm(temp_in, buf_ptr, cos_bit_row, stage_range); - } else { - row_txfm(input, buf_ptr, cos_bit_row, stage_range); - } - av1_round_shift_array(buf_ptr, txfm_size_col, -shift[0]); - input += txfm_size_col; - buf_ptr += txfm_size_col; - } - // Doing memset for the rows which are not processed in row transform. - memset(buf_ptr, 0, - sizeof(int32_t) * txfm_size_col * (txfm_size_row - row_start)); - - // col tx - for (int c = 0; c < txfm_size_col; c++) { - if (lr_flip == 0) { - for (r = 0; r < txfm_size_row; ++r) - temp_in[r] = buf[r * txfm_size_col + c]; - } else { - // flip left right - for (r = 0; r < txfm_size_row; ++r) - temp_in[r] = buf[r * txfm_size_col + (txfm_size_col - c - 1)]; - } - col_txfm(temp_in, temp_out, cos_bit_col, stage_range); - av1_round_shift_array(temp_out, txfm_size_row, -shift[1]); - - if (ud_flip == 0) { - for (r = 0; r < txfm_size_row; ++r) { - output[r * stride + c] = - highbd_clip_pixel_add(output[r * stride + c], temp_out[r], bd); - } - } else { - // flip upside down - for (r = 0; r < txfm_size_row; ++r) { - output[r * stride + c] = highbd_clip_pixel_add( - output[r * stride + c], temp_out[txfm_size_row - r - 1], bd); - } - } - } -} - static INLINE void lowbd_inv_txfm2d_add_v_identity_neon( const int32_t *input, uint8_t *output, int stride, TX_TYPE tx_type, TX_SIZE tx_size, int eob) { @@ -2446,90 +3831,6 @@ static INLINE void lowbd_inv_txfm2d_add_v_identity_neon( } } -static INLINE void lowbd_inv_txfm2d_add_h_wxh_identity_neon( - const int32_t *input, uint8_t *output, int stride, TX_TYPE tx_type, - TX_SIZE tx_size, int eob) { - DECLARE_ALIGNED(32, int, txfm_buf[32 * 32 + 32 + 32]); - int32_t *temp_in = txfm_buf; - - int eobx, eoby; - get_eobx_eoby_scan_h_identity(&eobx, &eoby, tx_size, eob); - const int8_t *shift = inv_txfm_shift_ls[tx_size]; - const int txw_idx = get_txw_idx(tx_size); - const int txh_idx = get_txh_idx(tx_size); - const int cos_bit_col = inv_cos_bit_col[txw_idx][txh_idx]; - const int cos_bit_row = inv_cos_bit_row[txw_idx][txh_idx]; - const int txfm_size_col = tx_size_wide[tx_size]; - const int txfm_size_row = tx_size_high[tx_size]; - const int buf_size_nonzero_h_div8 = (eoby + 8) >> 3; - - const int rect_type = get_rect_tx_log_ratio(txfm_size_col, txfm_size_row); - const int buf_offset = AOMMAX(txfm_size_row, txfm_size_col); - - int32_t *temp_out = temp_in + buf_offset; - int32_t *buf = temp_out + buf_offset; - int32_t *buf_ptr = buf; - const int8_t stage_range[MAX_TXFM_STAGE_NUM] = { 16 }; - int r, bd = 8; - - const int fun_idx_x = lowbd_txfm_all_1d_zeros_idx[eobx]; - const int fun_idx_y = lowbd_txfm_all_1d_zeros_idx[eoby]; - const transform_1d_neon row_txfm = - lowbd_txfm_all_1d_zeros_w8_arr[txw_idx][hitx_1d_tab[tx_type]][fun_idx_x]; - const transform_1d_neon col_txfm = - lowbd_txfm_all_1d_zeros_w8_arr[txh_idx][vitx_1d_tab[tx_type]][fun_idx_y]; - - assert(col_txfm != NULL); - assert(row_txfm != NULL); - int ud_flip, lr_flip; - get_flip_cfg(tx_type, &ud_flip, &lr_flip); - - // row tx - int row_start = (buf_size_nonzero_h_div8 * 8); - for (int i = 0; i < row_start; i++) { - if (abs(rect_type) == 1) { - for (int j = 0; j < txfm_size_col; j++) - temp_in[j] = round_shift((int64_t)input[j] * NewInvSqrt2, NewSqrt2Bits); - row_txfm(temp_in, buf_ptr, cos_bit_row, stage_range); - } else { - row_txfm(input, buf_ptr, cos_bit_row, stage_range); - } - av1_round_shift_array(buf_ptr, txfm_size_col, -shift[0]); - input += txfm_size_col; - buf_ptr += txfm_size_col; - } - // Doing memset for the rows which are not processed in row transform. - memset(buf_ptr, 0, - sizeof(int32_t) * txfm_size_col * (txfm_size_row - row_start)); - - // col tx - for (int c = 0; c < txfm_size_col; c++) { - if (lr_flip == 0) { - for (r = 0; r < txfm_size_row; ++r) - temp_in[r] = buf[r * txfm_size_col + c]; - } else { - // flip left right - for (r = 0; r < txfm_size_row; ++r) - temp_in[r] = buf[r * txfm_size_col + (txfm_size_col - c - 1)]; - } - col_txfm(temp_in, temp_out, cos_bit_col, stage_range); - av1_round_shift_array(temp_out, txfm_size_row, -shift[1]); - - if (ud_flip == 0) { - for (r = 0; r < txfm_size_row; ++r) { - output[r * stride + c] = - highbd_clip_pixel_add(output[r * stride + c], temp_out[r], bd); - } - } else { - // flip upside down - for (r = 0; r < txfm_size_row; ++r) { - output[r * stride + c] = highbd_clip_pixel_add( - output[r * stride + c], temp_out[txfm_size_row - r - 1], bd); - } - } - } -} - static INLINE void lowbd_inv_txfm2d_add_h_identity_neon( const int32_t *input, uint8_t *output, int stride, TX_TYPE tx_type, TX_SIZE tx_size, int eob) { @@ -2604,9 +3905,9 @@ static INLINE void lowbd_inv_txfm2d_add_h_identity_neon( static INLINE void lowbd_inv_txfm2d_add_4x4_neon(const int32_t *input, uint8_t *output, int stride, - TX_TYPE tx_type, - TX_SIZE tx_size, int eob) { + TX_TYPE tx_type, int eob) { (void)eob; + TX_SIZE tx_size = TX_4X4; DECLARE_ALIGNED(32, int, txfm_buf[4 * 4 + 8 + 8]); int32_t *temp_in = txfm_buf; @@ -2666,9 +3967,9 @@ static INLINE void lowbd_inv_txfm2d_add_4x4_neon(const int32_t *input, } void lowbd_inv_txfm2d_add_4x8_neon(const int32_t *input, uint8_t *output, - int stride, TX_TYPE tx_type, TX_SIZE tx_size, - int eob) { + int stride, TX_TYPE tx_type, int eob) { (void)eob; + TX_SIZE tx_size = TX_4X8; DECLARE_ALIGNED(32, int, txfm_buf[4 * 8 + 8 + 8]); int32_t *temp_in = txfm_buf; @@ -2730,9 +4031,9 @@ void lowbd_inv_txfm2d_add_4x8_neon(const int32_t *input, uint8_t *output, } void lowbd_inv_txfm2d_add_8x4_neon(const int32_t *input, uint8_t *output, - int stride, TX_TYPE tx_type, TX_SIZE tx_size, - int eob) { + int stride, TX_TYPE tx_type, int eob) { (void)eob; + TX_SIZE tx_size = TX_8X4; DECLARE_ALIGNED(32, int, txfm_buf[8 * 4 + 8 + 8]); int32_t *temp_in = txfm_buf; @@ -2794,9 +4095,9 @@ void lowbd_inv_txfm2d_add_8x4_neon(const int32_t *input, uint8_t *output, } void lowbd_inv_txfm2d_add_4x16_neon(const int32_t *input, uint8_t *output, - int stride, TX_TYPE tx_type, - TX_SIZE tx_size, int eob) { + int stride, TX_TYPE tx_type, int eob) { (void)eob; + TX_SIZE tx_size = TX_4X16; DECLARE_ALIGNED(32, int, txfm_buf[4 * 16 + 16 + 16]); int32_t *temp_in = txfm_buf; @@ -2856,10 +4157,9 @@ void lowbd_inv_txfm2d_add_4x16_neon(const int32_t *input, uint8_t *output, } void lowbd_inv_txfm2d_add_16x4_neon(const int32_t *input, uint8_t *output, - int stride, TX_TYPE tx_type, - TX_SIZE tx_size, int eob) { + int stride, TX_TYPE tx_type, int eob) { (void)eob; - + TX_SIZE tx_size = TX_16X4; DECLARE_ALIGNED(32, int, txfm_buf[16 * 4 + 16 + 16]); int32_t *temp_in = txfm_buf; @@ -2918,89 +4218,6 @@ void lowbd_inv_txfm2d_add_16x4_neon(const int32_t *input, uint8_t *output, } } -static INLINE void lowbd_inv_txfm2d_add_wxh_no_identity_neon( - const int32_t *input, uint8_t *output, int stride, TX_TYPE tx_type, - TX_SIZE tx_size, int eob) { - DECLARE_ALIGNED(32, int, txfm_buf[64 * 64 + 64 + 64]); - int32_t *temp_in = txfm_buf; - - int eobx, eoby, ud_flip, lr_flip, row_start; - get_eobx_eoby_scan_default(&eobx, &eoby, tx_size, eob); - const int8_t *shift = inv_txfm_shift_ls[tx_size]; - const int txw_idx = get_txw_idx(tx_size); - const int txh_idx = get_txh_idx(tx_size); - const int cos_bit_col = inv_cos_bit_col[txw_idx][txh_idx]; - const int cos_bit_row = inv_cos_bit_row[txw_idx][txh_idx]; - const int txfm_size_col = tx_size_wide[tx_size]; - const int txfm_size_row = tx_size_high[tx_size]; - const int buf_size_nonzero_h_div8 = (eoby + 8) >> 3; - const int rect_type = get_rect_tx_log_ratio(txfm_size_col, txfm_size_row); - const int buf_offset = AOMMAX(txfm_size_row, txfm_size_col); - - int32_t *temp_out = temp_in + buf_offset; - int32_t *buf = temp_out + buf_offset; - int32_t *buf_ptr = buf; - const int8_t stage_range[MAX_TXFM_STAGE_NUM] = { 16 }; - const int bd = 8; - int r; - - const int fun_idx_x = lowbd_txfm_all_1d_zeros_idx[eobx]; - const int fun_idx_y = lowbd_txfm_all_1d_zeros_idx[eoby]; - const transform_1d_neon row_txfm = - lowbd_txfm_all_1d_zeros_w8_arr[txw_idx][hitx_1d_tab[tx_type]][fun_idx_x]; - const transform_1d_neon col_txfm = - lowbd_txfm_all_1d_zeros_w8_arr[txh_idx][vitx_1d_tab[tx_type]][fun_idx_y]; - - assert(col_txfm != NULL); - assert(row_txfm != NULL); - - get_flip_cfg(tx_type, &ud_flip, &lr_flip); - row_start = (buf_size_nonzero_h_div8 << 3); - - for (int i = 0; i < row_start; i++) { - if (abs(rect_type) == 1) { - for (int j = 0; j < txfm_size_col; j++) - temp_in[j] = round_shift((int64_t)input[j] * NewInvSqrt2, NewSqrt2Bits); - row_txfm(temp_in, buf_ptr, cos_bit_row, stage_range); - } else { - row_txfm(input, buf_ptr, cos_bit_row, stage_range); - } - av1_round_shift_array(buf_ptr, txfm_size_col, -shift[0]); - input += txfm_size_col; - buf_ptr += txfm_size_col; - } - - // Doing memset for the rows which are not processed in row transform. - memset(buf_ptr, 0, - sizeof(int32_t) * txfm_size_col * (txfm_size_row - row_start)); - - for (int c = 0; c < txfm_size_col; c++) { - if (lr_flip == 0) { - for (r = 0; r < txfm_size_row; ++r) - temp_in[r] = buf[r * txfm_size_col + c]; - } else { - // flip left right - for (r = 0; r < txfm_size_row; ++r) - temp_in[r] = buf[r * txfm_size_col + (txfm_size_col - c - 1)]; - } - col_txfm(temp_in, temp_out, cos_bit_col, stage_range); - av1_round_shift_array(temp_out, txfm_size_row, -shift[1]); - - if (ud_flip == 0) { - for (r = 0; r < txfm_size_row; ++r) { - output[r * stride + c] = - highbd_clip_pixel_add(output[r * stride + c], temp_out[r], bd); - } - } else { - // flip upside down - for (r = 0; r < txfm_size_row; ++r) { - output[r * stride + c] = highbd_clip_pixel_add( - output[r * stride + c], temp_out[txfm_size_row - r - 1], bd); - } - } - } -} - static INLINE void lowbd_inv_txfm2d_add_no_identity_neon( const int32_t *input, uint8_t *output, int stride, TX_TYPE tx_type, TX_SIZE tx_size, int eob) { @@ -3019,6 +4236,7 @@ static INLINE void lowbd_inv_txfm2d_add_no_identity_neon( const int buf_size_w_div8 = txfm_size_col >> 3; const int buf_size_nonzero_h_div8 = (eoby + 8) >> 3; const int buf_size_nonzero_w_div8 = (eobx + 8) >> 3; + const int input_stride = AOMMIN(32, txfm_size_col); const int fun_idx_x = lowbd_txfm_all_1d_zeros_idx[eobx]; const int fun_idx_y = lowbd_txfm_all_1d_zeros_idx[eoby]; const int32_t *input_1; @@ -3038,14 +4256,14 @@ static INLINE void lowbd_inv_txfm2d_add_no_identity_neon( input_1 = input; for (int j = 0; j < buf_size_nonzero_w_div8; ++j) { int k = j * 8 + i * txfm_size_col; - load_buffer_32bit_to_16bit_neon(input_1, &a[k], txfm_size_col); + load_buffer_32bit_to_16bit_neon(input_1, &a[k], input_stride); transpose_s16_8x8q(&a[k], &a[k]); input_1 += 8; } - input += (txfm_size_col * 8); + input += (input_stride * 8); if (abs(rect_type) == 1) { int y = i * txfm_size_col; - round_shift_for_rect(&a[y], &a[y], txfm_size_col); + round_shift_for_rect(&a[y], &a[y], input_stride); } row_txfm(&a[i * txfm_size_col], &a[i * txfm_size_col], cos_bit_row, 0); av1_round_shift_array_16_neon(&a[i * txfm_size_col], txfm_size_col, @@ -3083,36 +4301,6 @@ static INLINE void lowbd_inv_txfm2d_add_no_identity_neon( } } -static INLINE void lowbd_inv_txfm2d_add_wxh_universe_neon( - const int32_t *input, uint8_t *output, int stride, TX_TYPE tx_type, - TX_SIZE tx_size, int eob) { - switch (tx_type) { - case IDTX: - lowbd_inv_txfm2d_add_wxh_idtx_neon(input, output, stride, tx_type, - tx_size, eob); - break; - - case H_DCT: - case H_ADST: - case H_FLIPADST: - lowbd_inv_txfm2d_add_v_wxh_identity_neon(input, output, stride, tx_type, - tx_size, eob); - break; - - case V_DCT: - case V_ADST: - case V_FLIPADST: - lowbd_inv_txfm2d_add_h_wxh_identity_neon(input, output, stride, tx_type, - tx_size, eob); - break; - - default: - lowbd_inv_txfm2d_add_wxh_no_identity_neon(input, output, stride, tx_type, - tx_size, eob); - break; - } -} - static INLINE void lowbd_inv_txfm2d_add_universe_neon( const int32_t *input, uint8_t *output, int stride, TX_TYPE tx_type, TX_SIZE tx_size, int eob) { @@ -3146,73 +4334,27 @@ static INLINE void lowbd_inv_txfm2d_add_universe_neon( void av1_lowbd_inv_txfm2d_add_neon(const int32_t *input, uint8_t *output, int stride, TX_TYPE tx_type, TX_SIZE tx_size, int eob) { - int row; switch (tx_size) { case TX_4X4: - lowbd_inv_txfm2d_add_4x4_neon(input, output, stride, tx_type, tx_size, - eob); + lowbd_inv_txfm2d_add_4x4_neon(input, output, stride, tx_type, eob); break; case TX_4X8: - lowbd_inv_txfm2d_add_4x8_neon(input, output, stride, tx_type, tx_size, - eob); + lowbd_inv_txfm2d_add_4x8_neon(input, output, stride, tx_type, eob); break; case TX_8X4: - lowbd_inv_txfm2d_add_8x4_neon(input, output, stride, tx_type, tx_size, - eob); + lowbd_inv_txfm2d_add_8x4_neon(input, output, stride, tx_type, eob); break; case TX_4X16: - lowbd_inv_txfm2d_add_4x16_neon(input, output, stride, tx_type, tx_size, - eob); + lowbd_inv_txfm2d_add_4x16_neon(input, output, stride, tx_type, eob); break; case TX_16X4: - lowbd_inv_txfm2d_add_16x4_neon(input, output, stride, tx_type, tx_size, - eob); + lowbd_inv_txfm2d_add_16x4_neon(input, output, stride, tx_type, eob); break; - case TX_16X64: { - lowbd_inv_txfm2d_add_wxh_universe_neon(input, output, stride, tx_type, - tx_size, eob); - } break; - - case TX_64X16: { - int32_t mod_input[64 * 16]; - for (row = 0; row < 16; ++row) { - memcpy(mod_input + row * 64, input + row * 32, 32 * sizeof(*mod_input)); - memset(mod_input + row * 64 + 32, 0, 32 * sizeof(*mod_input)); - } - lowbd_inv_txfm2d_add_wxh_universe_neon(mod_input, output, stride, tx_type, - tx_size, eob); - } break; - - case TX_32X64: { - lowbd_inv_txfm2d_add_wxh_universe_neon(input, output, stride, tx_type, - tx_size, eob); - } break; - - case TX_64X32: { - int32_t mod_input[64 * 32]; - for (row = 0; row < 32; ++row) { - memcpy(mod_input + row * 64, input + row * 32, 32 * sizeof(*mod_input)); - memset(mod_input + row * 64 + 32, 0, 32 * sizeof(*mod_input)); - } - lowbd_inv_txfm2d_add_wxh_universe_neon(mod_input, output, stride, tx_type, - tx_size, eob); - } break; - - case TX_64X64: { - int32_t mod_input[64 * 64]; - for (row = 0; row < 32; ++row) { - memcpy(mod_input + row * 64, input + row * 32, 32 * sizeof(*mod_input)); - memset(mod_input + row * 64 + 32, 0, 32 * sizeof(*mod_input)); - } - lowbd_inv_txfm2d_add_wxh_universe_neon(mod_input, output, stride, tx_type, - tx_size, eob); - } break; - default: lowbd_inv_txfm2d_add_universe_neon(input, output, stride, tx_type, tx_size, eob); diff --git a/third_party/aom/av1/common/arm/av1_txfm_neon.c b/third_party/aom/av1/common/arm/av1_txfm_neon.c index de3c547248..7e3a05ab7e 100644 --- a/third_party/aom/av1/common/arm/av1_txfm_neon.c +++ b/third_party/aom/av1/common/arm/av1_txfm_neon.c @@ -12,6 +12,8 @@ #include #include +#include "config/av1_rtcd.h" + #include "aom_ports/mem.h" #include "av1/common/arm/mem_neon.h" diff --git a/third_party/aom/av1/common/arm/jnt_convolve_neon.c b/third_party/aom/av1/common/arm/jnt_convolve_neon.c index e5674ef7c2..379ff9899d 100644 --- a/third_party/aom/av1/common/arm/jnt_convolve_neon.c +++ b/third_party/aom/av1/common/arm/jnt_convolve_neon.c @@ -23,19 +23,17 @@ #include "av1/common/arm/transpose_neon.h" #if !defined(__aarch64__) -static INLINE void compute_avg_4x1(uint16x4_t res0, uint16x4_t d0, - const uint16_t fwd_offset, - const uint16_t bck_offset, - const int16x4_t sub_const_vec, - const int16_t round_bits, - const int use_jnt_comp_avg, uint8x8_t *t0) { +static INLINE void compute_avg_4x1( + uint16x4_t res0, uint16x4_t d0, const uint16_t fwd_offset, + const uint16_t bck_offset, const int16x4_t sub_const_vec, + const int16_t round_bits, const int use_dist_wtd_comp_avg, uint8x8_t *t0) { int16x4_t tmp0; uint16x4_t tmp_u0; uint32x4_t sum0; int32x4_t dst0; int16x8_t tmp4; - if (use_jnt_comp_avg) { + if (use_dist_wtd_comp_avg) { const int32x4_t round_bits_vec = vdupq_n_s32((int32_t)(-round_bits)); sum0 = vmull_n_u16(res0, fwd_offset); @@ -65,12 +63,10 @@ static INLINE void compute_avg_4x1(uint16x4_t res0, uint16x4_t d0, } } -static INLINE void compute_avg_8x1(uint16x8_t res0, uint16x8_t d0, - const uint16_t fwd_offset, - const uint16_t bck_offset, - const int16x4_t sub_const, - const int16_t round_bits, - const int use_jnt_comp_avg, uint8x8_t *t0) { +static INLINE void compute_avg_8x1( + uint16x8_t res0, uint16x8_t d0, const uint16_t fwd_offset, + const uint16_t bck_offset, const int16x4_t sub_const, + const int16_t round_bits, const int use_dist_wtd_comp_avg, uint8x8_t *t0) { int16x4_t tmp0, tmp2; int16x8_t f0; uint32x4_t sum0, sum2; @@ -78,7 +74,7 @@ static INLINE void compute_avg_8x1(uint16x8_t res0, uint16x8_t d0, uint16x8_t tmp_u0; - if (use_jnt_comp_avg) { + if (use_dist_wtd_comp_avg) { const int32x4_t sub_const_vec = vmovl_s16(sub_const); const int32x4_t round_bits_vec = vdupq_n_s32(-(int32_t)round_bits); @@ -123,7 +119,7 @@ static INLINE void compute_avg_4x4( uint16x4_t d0, uint16x4_t d1, uint16x4_t d2, uint16x4_t d3, const uint16_t fwd_offset, const uint16_t bck_offset, const int16x4_t sub_const_vec, const int16_t round_bits, - const int use_jnt_comp_avg, uint8x8_t *t0, uint8x8_t *t1) { + const int use_dist_wtd_comp_avg, uint8x8_t *t0, uint8x8_t *t1) { int16x4_t tmp0, tmp1, tmp2, tmp3; uint16x4_t tmp_u0, tmp_u1, tmp_u2, tmp_u3; uint32x4_t sum0, sum1, sum2, sum3; @@ -132,7 +128,7 @@ static INLINE void compute_avg_4x4( int16x8_t tmp4, tmp5; const int16x8_t zero = vdupq_n_s16(0); - if (use_jnt_comp_avg) { + if (use_dist_wtd_comp_avg) { const int32x4_t round_bits_vec = vdupq_n_s32((int32_t)(-round_bits)); const int32x4_t const_vec = vmovl_s16(sub_const_vec); @@ -203,8 +199,8 @@ static INLINE void compute_avg_8x4( uint16x8_t d0, uint16x8_t d1, uint16x8_t d2, uint16x8_t d3, const uint16_t fwd_offset, const uint16_t bck_offset, const int16x4_t sub_const, const int16_t round_bits, - const int use_jnt_comp_avg, uint8x8_t *t0, uint8x8_t *t1, uint8x8_t *t2, - uint8x8_t *t3) { + const int use_dist_wtd_comp_avg, uint8x8_t *t0, uint8x8_t *t1, + uint8x8_t *t2, uint8x8_t *t3) { int16x4_t tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; int16x8_t f0, f1, f2, f3; uint32x4_t sum0, sum1, sum2, sum3; @@ -214,7 +210,7 @@ static INLINE void compute_avg_8x4( uint16x8_t tmp_u0, tmp_u1, tmp_u2, tmp_u3; const int16x8_t zero = vdupq_n_s16(0); - if (use_jnt_comp_avg) { + if (use_dist_wtd_comp_avg) { const int32x4_t sub_const_vec = vmovl_s16(sub_const); const int32x4_t round_bits_vec = vdupq_n_s32(-(int32_t)round_bits); @@ -319,7 +315,7 @@ static INLINE void compute_avg_8x4( } } -static INLINE void jnt_convolve_2d_horiz_neon( +static INLINE void dist_wtd_convolve_2d_horiz_neon( const uint8_t *src, int src_stride, int16_t *im_block, const int im_stride, int16_t *x_filter_tmp, const int im_h, int w, const int round_0) { const int bd = 8; @@ -563,7 +559,7 @@ static INLINE void jnt_convolve_2d_horiz_neon( } } -static INLINE void jnt_convolve_2d_vert_neon( +static INLINE void dist_wtd_convolve_2d_vert_neon( int16_t *im_block, const int im_stride, uint8_t *dst8, int dst8_stride, ConvolveParams *conv_params, const int16_t *y_filter, int h, int w) { uint8_t *dst_u8_ptr, *d_u8; @@ -587,7 +583,7 @@ static INLINE void jnt_convolve_2d_vert_neon( const uint16_t fwd_offset = conv_params->fwd_offset; const uint16_t bck_offset = conv_params->bck_offset; const int do_average = conv_params->do_average; - const int use_jnt_comp_avg = conv_params->use_jnt_comp_avg; + const int use_dist_wtd_comp_avg = conv_params->use_dist_wtd_comp_avg; int16x4_t s0, s1, s2, s3, s4, s5, s6, s7; uint16x4_t res4, d0; @@ -652,8 +648,8 @@ static INLINE void jnt_convolve_2d_vert_neon( d += (dst_stride << 2); compute_avg_4x4(res4, res5, res6, res7, d0, d1, d2, d3, fwd_offset, - bck_offset, sub_const_vec, round_bits, use_jnt_comp_avg, - &t0, &t1); + bck_offset, sub_const_vec, round_bits, + use_dist_wtd_comp_avg, &t0, &t1); vst1_lane_u32((uint32_t *)d_u8, vreinterpret_u32_u8(t0), 0); d_u8 += dst8_stride; @@ -691,7 +687,7 @@ static INLINE void jnt_convolve_2d_vert_neon( d += (dst_stride); compute_avg_4x1(res4, d0, fwd_offset, bck_offset, sub_const_vec, - round_bits, use_jnt_comp_avg, &t0); + round_bits, use_dist_wtd_comp_avg, &t0); vst1_lane_u32((uint32_t *)d_u8, vreinterpret_u32_u8(t0), 0); d_u8 += dst8_stride; @@ -717,12 +713,12 @@ static INLINE void jnt_convolve_2d_vert_neon( } while (w > 0); } -void av1_jnt_convolve_2d_neon(const uint8_t *src, int src_stride, uint8_t *dst8, - int dst8_stride, int w, int h, - const InterpFilterParams *filter_params_x, - const InterpFilterParams *filter_params_y, - const int subpel_x_q4, const int subpel_y_q4, - ConvolveParams *conv_params) { +void av1_dist_wtd_convolve_2d_neon(const uint8_t *src, int src_stride, + uint8_t *dst8, int dst8_stride, int w, int h, + const InterpFilterParams *filter_params_x, + const InterpFilterParams *filter_params_y, + const int subpel_x_q4, const int subpel_y_q4, + ConvolveParams *conv_params) { assert(!(w % 4)); assert(!(h % 4)); @@ -748,19 +744,18 @@ void av1_jnt_convolve_2d_neon(const uint8_t *src, int src_stride, uint8_t *dst8, filter_x_coef = vshrq_n_s16(filter_x_coef, 1); vst1q_s16(&x_filter_tmp[0], filter_x_coef); - jnt_convolve_2d_horiz_neon(src_ptr, src_stride, im_block, im_stride, - x_filter_tmp, im_h, w, round_0); + dist_wtd_convolve_2d_horiz_neon(src_ptr, src_stride, im_block, im_stride, + x_filter_tmp, im_h, w, round_0); - jnt_convolve_2d_vert_neon(im_block, im_stride, dst8, dst8_stride, conv_params, - y_filter, h, w); + dist_wtd_convolve_2d_vert_neon(im_block, im_stride, dst8, dst8_stride, + conv_params, y_filter, h, w); } -void av1_jnt_convolve_2d_copy_neon(const uint8_t *src, int src_stride, - uint8_t *dst8, int dst8_stride, int w, int h, - const InterpFilterParams *filter_params_x, - const InterpFilterParams *filter_params_y, - const int subpel_x_q4, const int subpel_y_q4, - ConvolveParams *conv_params) { +void av1_dist_wtd_convolve_2d_copy_neon( + const uint8_t *src, int src_stride, uint8_t *dst8, int dst8_stride, int w, + int h, const InterpFilterParams *filter_params_x, + const InterpFilterParams *filter_params_y, const int subpel_x_q4, + const int subpel_y_q4, ConvolveParams *conv_params) { uint8x8_t res0_8, res1_8, res2_8, res3_8, tmp_shift0, tmp_shift1, tmp_shift2, tmp_shift3; uint16x8_t res_q0, res_q1, res_q2, res_q3, tmp_q0, tmp_q1, tmp_q2, tmp_q3; @@ -811,7 +806,7 @@ void av1_jnt_convolve_2d_copy_neon(const uint8_t *src, int src_stride, compute_avg_8x4(tmp_q0, tmp_q1, tmp_q2, tmp_q3, res_q0, res_q1, res_q2, res_q3, conv_params->fwd_offset, conv_params->bck_offset, sub_const_vec, bits, - conv_params->use_jnt_comp_avg, &tmp_shift0, + conv_params->use_dist_wtd_comp_avg, &tmp_shift0, &tmp_shift1, &tmp_shift2, &tmp_shift3); vst1_u8(dst8_1 + (0 * dst8_stride), tmp_shift0); @@ -854,7 +849,7 @@ void av1_jnt_convolve_2d_copy_neon(const uint8_t *src, int src_stride, compute_avg_4x4(tmp4, tmp5, tmp6, tmp7, res4, res5, res6, res7, conv_params->fwd_offset, conv_params->bck_offset, - sub_const_vec, bits, conv_params->use_jnt_comp_avg, + sub_const_vec, bits, conv_params->use_dist_wtd_comp_avg, &tmp_shift0, &tmp_shift1); vst1_lane_u32((uint32_t *)(dst8_1), vreinterpret_u32_u8(tmp_shift0), 0); @@ -881,12 +876,12 @@ void av1_jnt_convolve_2d_copy_neon(const uint8_t *src, int src_stride, } } -void av1_jnt_convolve_x_neon(const uint8_t *src, int src_stride, uint8_t *dst8, - int dst8_stride, int w, int h, - const InterpFilterParams *filter_params_x, - const InterpFilterParams *filter_params_y, - const int subpel_x_q4, const int subpel_y_q4, - ConvolveParams *conv_params) { +void av1_dist_wtd_convolve_x_neon(const uint8_t *src, int src_stride, + uint8_t *dst8, int dst8_stride, int w, int h, + const InterpFilterParams *filter_params_x, + const InterpFilterParams *filter_params_y, + const int subpel_x_q4, const int subpel_y_q4, + ConvolveParams *conv_params) { assert(!(w % 4)); assert(!(h % 4)); @@ -902,7 +897,7 @@ void av1_jnt_convolve_x_neon(const uint8_t *src, int src_stride, uint8_t *dst8, 2 * FILTER_BITS - conv_params->round_0 - conv_params->round_1; const uint16_t fwd_offset = conv_params->fwd_offset; const uint16_t bck_offset = conv_params->bck_offset; - const int use_jnt_comp_avg = conv_params->use_jnt_comp_avg; + const int use_dist_wtd_comp_avg = conv_params->use_dist_wtd_comp_avg; (void)filter_params_y; (void)subpel_y_q4; @@ -1031,8 +1026,8 @@ void av1_jnt_convolve_x_neon(const uint8_t *src, int src_stride, uint8_t *dst8, compute_avg_4x4(res4, res5, res6, res7, vreinterpret_u16_s16(d0), vreinterpret_u16_s16(d1), vreinterpret_u16_s16(d2), vreinterpret_u16_s16(d3), fwd_offset, bck_offset, - round_offset_vec, round_bits, use_jnt_comp_avg, &t0, - &t1); + round_offset_vec, round_bits, use_dist_wtd_comp_avg, + &t0, &t1); vst1_lane_u32((uint32_t *)d_u8, vreinterpret_u32_u8(t0), 0); // 00 01 02 03 @@ -1103,7 +1098,7 @@ void av1_jnt_convolve_x_neon(const uint8_t *src, int src_stride, uint8_t *dst8, compute_avg_4x1(res4, vreinterpret_u16_s16(d0), fwd_offset, bck_offset, round_offset_vec, round_bits, - use_jnt_comp_avg, &t0); + use_dist_wtd_comp_avg, &t0); vst1_lane_u32((uint32_t *)d_u8, vreinterpret_u32_u8(t0), 0); // 00 01 02 03 @@ -1231,11 +1226,12 @@ void av1_jnt_convolve_x_neon(const uint8_t *src, int src_stride, uint8_t *dst8, load_u16_8x4(d_tmp, dst_stride, &res8, &res9, &res10, &res11); d_tmp += (dst_stride << 2); - compute_avg_8x4( - res8, res9, res10, res11, vreinterpretq_u16_s16(res0), - vreinterpretq_u16_s16(res1), vreinterpretq_u16_s16(res2), - vreinterpretq_u16_s16(res3), fwd_offset, bck_offset, - round_offset64, round_bits, use_jnt_comp_avg, &t0, &t1, &t2, &t3); + compute_avg_8x4(res8, res9, res10, res11, vreinterpretq_u16_s16(res0), + vreinterpretq_u16_s16(res1), + vreinterpretq_u16_s16(res2), + vreinterpretq_u16_s16(res3), fwd_offset, bck_offset, + round_offset64, round_bits, use_dist_wtd_comp_avg, + &t0, &t1, &t2, &t3); store_u8_8x4(d_u8, dst8_stride, t0, t1, t2, t3); d_u8 += (dst8_stride << 2); @@ -1243,11 +1239,12 @@ void av1_jnt_convolve_x_neon(const uint8_t *src, int src_stride, uint8_t *dst8, load_u16_8x4(d_tmp, dst_stride, &res8, &res9, &res10, &res11); d_tmp += (dst_stride << 2); - compute_avg_8x4( - res8, res9, res10, res11, vreinterpretq_u16_s16(res4), - vreinterpretq_u16_s16(res5), vreinterpretq_u16_s16(res6), - vreinterpretq_u16_s16(res7), fwd_offset, bck_offset, - round_offset64, round_bits, use_jnt_comp_avg, &t0, &t1, &t2, &t3); + compute_avg_8x4(res8, res9, res10, res11, vreinterpretq_u16_s16(res4), + vreinterpretq_u16_s16(res5), + vreinterpretq_u16_s16(res6), + vreinterpretq_u16_s16(res7), fwd_offset, bck_offset, + round_offset64, round_bits, use_dist_wtd_comp_avg, + &t0, &t1, &t2, &t3); store_u8_8x4(d_u8, dst8_stride, t0, t1, t2, t3); d_u8 += (dst8_stride << 2); @@ -1319,7 +1316,7 @@ void av1_jnt_convolve_x_neon(const uint8_t *src, int src_stride, uint8_t *dst8, compute_avg_8x1(res8, vreinterpretq_u16_s16(res0), fwd_offset, bck_offset, round_offset64, round_bits, - use_jnt_comp_avg, &t0); + use_dist_wtd_comp_avg, &t0); vst1_u8(d_u8, t0); d_u8 += (dst8_stride); @@ -1342,12 +1339,12 @@ void av1_jnt_convolve_x_neon(const uint8_t *src, int src_stride, uint8_t *dst8, } } -void av1_jnt_convolve_y_neon(const uint8_t *src, int src_stride, uint8_t *dst8, - int dst8_stride, int w, int h, - const InterpFilterParams *filter_params_x, - const InterpFilterParams *filter_params_y, - const int subpel_x_q4, const int subpel_y_q4, - ConvolveParams *conv_params) { +void av1_dist_wtd_convolve_y_neon(const uint8_t *src, int src_stride, + uint8_t *dst8, int dst8_stride, int w, int h, + const InterpFilterParams *filter_params_x, + const InterpFilterParams *filter_params_y, + const int subpel_x_q4, const int subpel_y_q4, + ConvolveParams *conv_params) { assert(!(w % 4)); assert(!(h % 4)); @@ -1363,7 +1360,7 @@ void av1_jnt_convolve_y_neon(const uint8_t *src, int src_stride, uint8_t *dst8, 2 * FILTER_BITS - conv_params->round_0 - conv_params->round_1; const uint16_t fwd_offset = conv_params->fwd_offset; const uint16_t bck_offset = conv_params->bck_offset; - const int use_jnt_comp_avg = conv_params->use_jnt_comp_avg; + const int use_dist_wtd_comp_avg = conv_params->use_dist_wtd_comp_avg; const int shift_value = (conv_params->round_1 - 1 - bits); (void)filter_params_x; @@ -1489,8 +1486,8 @@ void av1_jnt_convolve_y_neon(const uint8_t *src, int src_stride, uint8_t *dst8, compute_avg_4x4(res4, res5, res6, res7, vreinterpret_u16_s16(d0), vreinterpret_u16_s16(d1), vreinterpret_u16_s16(d2), vreinterpret_u16_s16(d3), fwd_offset, bck_offset, - round_offset64, round_bits, use_jnt_comp_avg, &t0, - &t1); + round_offset64, round_bits, use_dist_wtd_comp_avg, + &t0, &t1); vst1_lane_u32((uint32_t *)d_u8, vreinterpret_u32_u8(t0), 0); d_u8 += dst8_stride; @@ -1535,7 +1532,7 @@ void av1_jnt_convolve_y_neon(const uint8_t *src, int src_stride, uint8_t *dst8, compute_avg_4x1(res4, vreinterpret_u16_s16(d0), fwd_offset, bck_offset, round_offset64, round_bits, - use_jnt_comp_avg, &t0); + use_dist_wtd_comp_avg, &t0); vst1_lane_u32((uint32_t *)d_u8, vreinterpret_u32_u8(t0), 0); d_u8 += dst8_stride; @@ -1654,11 +1651,12 @@ void av1_jnt_convolve_y_neon(const uint8_t *src, int src_stride, uint8_t *dst8, load_u16_8x4(d_tmp, dst_stride, &res8, &res9, &res10, &res11); d_tmp += (dst_stride << 2); - compute_avg_8x4( - res8, res9, res10, res11, vreinterpretq_u16_s16(res0), - vreinterpretq_u16_s16(res1), vreinterpretq_u16_s16(res2), - vreinterpretq_u16_s16(res3), fwd_offset, bck_offset, - round_offset64, round_bits, use_jnt_comp_avg, &t0, &t1, &t2, &t3); + compute_avg_8x4(res8, res9, res10, res11, vreinterpretq_u16_s16(res0), + vreinterpretq_u16_s16(res1), + vreinterpretq_u16_s16(res2), + vreinterpretq_u16_s16(res3), fwd_offset, bck_offset, + round_offset64, round_bits, use_dist_wtd_comp_avg, + &t0, &t1, &t2, &t3); store_u8_8x4(d_u8, dst8_stride, t0, t1, t2, t3); d_u8 += (dst8_stride << 2); @@ -1666,11 +1664,12 @@ void av1_jnt_convolve_y_neon(const uint8_t *src, int src_stride, uint8_t *dst8, load_u16_8x4(d_tmp, dst_stride, &res8, &res9, &res10, &res11); d_tmp += (dst_stride << 2); - compute_avg_8x4( - res8, res9, res10, res11, vreinterpretq_u16_s16(res4), - vreinterpretq_u16_s16(res5), vreinterpretq_u16_s16(res6), - vreinterpretq_u16_s16(res7), fwd_offset, bck_offset, - round_offset64, round_bits, use_jnt_comp_avg, &t0, &t1, &t2, &t3); + compute_avg_8x4(res8, res9, res10, res11, vreinterpretq_u16_s16(res4), + vreinterpretq_u16_s16(res5), + vreinterpretq_u16_s16(res6), + vreinterpretq_u16_s16(res7), fwd_offset, bck_offset, + round_offset64, round_bits, use_dist_wtd_comp_avg, + &t0, &t1, &t2, &t3); store_u8_8x4(d_u8, dst8_stride, t0, t1, t2, t3); d_u8 += (dst8_stride << 2); @@ -1718,7 +1717,7 @@ void av1_jnt_convolve_y_neon(const uint8_t *src, int src_stride, uint8_t *dst8, compute_avg_8x1(res8, vreinterpretq_u16_s16(res0), fwd_offset, bck_offset, round_offset64, round_bits, - use_jnt_comp_avg, &t0); + use_dist_wtd_comp_avg, &t0); vst1_u8(d_u8, t0); d_u8 += (dst8_stride); diff --git a/third_party/aom/av1/common/arm/mem_neon.h b/third_party/aom/av1/common/arm/mem_neon.h index c4ae2e7849..beae4ed0db 100644 --- a/third_party/aom/av1/common/arm/mem_neon.h +++ b/third_party/aom/av1/common/arm/mem_neon.h @@ -383,6 +383,15 @@ static INLINE void load_unaligned_u8_4x2(const uint8_t *buf, int stride, *tu0 = vset_lane_u32(a, *tu0, 1); } +/* These intrinsics require immediate values, so we must use #defines + to enforce that. */ +#define store_unaligned_u8_4x1(dst, src, lane) \ + do { \ + uint32_t a; \ + a = vget_lane_u32(vreinterpret_u32_u8(src), lane); \ + memcpy(dst, &a, 4); \ + } while (0) + static INLINE void load_unaligned_u8_2x2(const uint8_t *buf, int stride, uint16x4_t *tu0) { uint16_t a; diff --git a/third_party/aom/av1/common/arm/warp_plane_neon.c b/third_party/aom/av1/common/arm/warp_plane_neon.c index 7f02d42a73..1062cc3f9d 100644 --- a/third_party/aom/av1/common/arm/warp_plane_neon.c +++ b/third_party/aom/av1/common/arm/warp_plane_neon.c @@ -640,7 +640,7 @@ void av1_warp_affine_neon(const int32_t *mat, const uint8_t *ref, int width, uint16x4_t tmp16_lo = vld1_u16(p); int32x4_t tmp32_lo = vreinterpretq_s32_u32(vmovl_u16(tmp16_lo)); int16x4_t tmp16_low; - if (conv_params->use_jnt_comp_avg) { + if (conv_params->use_dist_wtd_comp_avg) { res_lo = vmulq_s32(res_lo, bwd); tmp32_lo = vmulq_s32(tmp32_lo, fwd); tmp32_lo = vaddq_s32(tmp32_lo, res_lo); @@ -671,7 +671,7 @@ void av1_warp_affine_neon(const int32_t *mat, const uint8_t *ref, int width, uint16x4_t tmp16_hi = vld1_u16(p4); int32x4_t tmp32_hi = vreinterpretq_s32_u32(vmovl_u16(tmp16_hi)); int16x4_t tmp16_high; - if (conv_params->use_jnt_comp_avg) { + if (conv_params->use_dist_wtd_comp_avg) { res_hi = vmulq_s32(res_hi, bwd); tmp32_hi = vmulq_s32(tmp32_hi, fwd); tmp32_hi = vaddq_s32(tmp32_hi, res_hi); diff --git a/third_party/aom/av1/common/av1_inv_txfm2d.c b/third_party/aom/av1/common/av1_inv_txfm2d.c index 4e69443145..4f2d57b802 100644 --- a/third_party/aom/av1/common/av1_inv_txfm2d.c +++ b/third_party/aom/av1/common/av1_inv_txfm2d.c @@ -183,7 +183,6 @@ void av1_get_inv_txfm_cfg(TX_TYPE tx_type, TX_SIZE tx_size, TXFM_2D_FLIP_CFG *cfg) { assert(cfg != NULL); cfg->tx_size = tx_size; - set_flip_cfg(tx_type, cfg); av1_zero(cfg->stage_range_col); av1_zero(cfg->stage_range_row); set_flip_cfg(tx_type, cfg); diff --git a/third_party/aom/av1/common/av1_loopfilter.c b/third_party/aom/av1/common/av1_loopfilter.c index 537d8dfe92..0aa1f9b907 100644 --- a/third_party/aom/av1/common/av1_loopfilter.c +++ b/third_party/aom/av1/common/av1_loopfilter.c @@ -32,7 +32,7 @@ static const int delta_lf_id_lut[MAX_MB_PLANE][2] = { { 0, 1 }, { 2, 2 }, { 3, 3 } }; -typedef enum EDGE_DIR { VERT_EDGE = 0, HORZ_EDGE = 1, NUM_EDGE_DIRS } EDGE_DIR; +enum { VERT_EDGE = 0, HORZ_EDGE = 1, NUM_EDGE_DIRS } UENUM1BYTE(EDGE_DIR); static const int mode_lf_lut[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // INTRA_MODES @@ -40,7 +40,6 @@ static const int mode_lf_lut[] = { 1, 1, 1, 1, 1, 1, 0, 1 // INTER_COMPOUND_MODES (GLOBAL_GLOBALMV == 0) }; -#if LOOP_FILTER_BITMASK // 256 bit masks (64x64 / 4x4) for left transform size for Y plane. // We use 4 uint64_t to represent the 256 bit. // Each 1 represents a position where we should apply a loop filter @@ -113,6 +112,10 @@ const int mask_id_table_tx_32x32[BLOCK_SIZES_ALL] = { -1, -1, -1, -1, -1, -1, -1, -1, -1, 0, 1, 2, 3, -1, -1, -1, -1, -1, -1, -1, -1, -1 }; +const int mask_id_table_vert_border[BLOCK_SIZES_ALL] = { 0, 47, 49, 19, 51, 53, + 33, 55, 57, 42, 59, 60, + 46, -1, -1, -1, 61, 62, + 63, 64, 65, 66 }; const FilterMask left_mask_univariant_reordered[67] = { // TX_4X4 @@ -402,6 +405,7 @@ const FilterMask above_mask_univariant_reordered[67] = { 0x0000000000000000ULL } }, // block size 64X16, TX_64X16 }; +#if LOOP_FILTER_BITMASK LoopFilterMask *get_loop_filter_mask(const AV1_COMMON *const cm, int mi_row, int mi_col) { assert(cm->lf.lfm != NULL); @@ -452,9 +456,9 @@ uint8_t get_filter_level(const AV1_COMMON *cm, const loop_filter_info_n *lfi_n, const int dir_idx, int plane, const MB_MODE_INFO *mbmi) { const int segment_id = mbmi->segment_id; - if (cm->delta_lf_present_flag) { + if (cm->delta_q_info.delta_lf_present_flag) { int delta_lf; - if (cm->delta_lf_multi) { + if (cm->delta_q_info.delta_lf_multi) { const int delta_lf_idx = delta_lf_id_lut[plane][dir_idx]; delta_lf = mbmi->delta_lf[delta_lf_idx]; } else { @@ -531,6 +535,9 @@ void av1_loop_filter_frame_init(AV1_COMMON *cm, int plane_start, filt_lvl_r[1] = cm->lf.filter_level_u; filt_lvl_r[2] = cm->lf.filter_level_v; + assert(plane_start >= AOM_PLANE_Y); + assert(plane_end <= MAX_MB_PLANE); + for (plane = plane_start; plane < plane_end; plane++) { if (plane == 0 && !filt_lvl[0] && !filt_lvl_r[0]) break; @@ -542,7 +549,6 @@ void av1_loop_filter_frame_init(AV1_COMMON *cm, int plane_start, for (seg_id = 0; seg_id < MAX_SEGMENTS; seg_id++) { for (int dir = 0; dir < 2; ++dir) { int lvl_seg = (dir == 0) ? filt_lvl[plane] : filt_lvl_r[plane]; - assert(plane >= 0 && plane <= 2); const int seg_lf_feature_id = seg_lvl_lf_lut[plane][dir]; if (segfeature_active(seg, seg_id, seg_lf_feature_id)) { const int data = get_segdata(&cm->seg, seg_id, seg_lf_feature_id); @@ -763,9 +769,11 @@ static void setup_masks(AV1_COMMON *const cm, int mi_row, int mi_col, int plane, else lfm->lfl_y_hor[row][col] = level; } else if (plane == 1) { - lfm->lfl_u[row][col] = level; + lfm->lfl_u_ver[row][col] = level; + lfm->lfl_u_hor[row][col] = level; } else { - lfm->lfl_v[row][col] = level; + lfm->lfl_v_ver[row][col] = level; + lfm->lfl_v_hor[row][col] = level; } } } @@ -1053,11 +1061,13 @@ void av1_setup_bitmask(AV1_COMMON *const cm, int mi_row, int mi_col, int plane, } else if (plane == 1) { av1_zero(lfm->left_u); av1_zero(lfm->above_u); - av1_zero(lfm->lfl_u); + av1_zero(lfm->lfl_u_ver); + av1_zero(lfm->lfl_u_hor); } else { av1_zero(lfm->left_v); av1_zero(lfm->above_v); - av1_zero(lfm->lfl_v); + av1_zero(lfm->lfl_v_ver); + av1_zero(lfm->lfl_v_hor); } } } @@ -1311,11 +1321,16 @@ static void filter_selectively_horiz(uint8_t *s, int pitch, int plane, int count; const int step = 1 << subsampling; const unsigned int two_block_mask = subsampling ? 5 : 3; + int offset = 0; for (mask = mask_16x16 | mask_8x8 | mask_4x4; mask; mask >>= step * count) { const loop_filter_thresh *lfi = lfi_n->lfthr + *lfl; - // Next block's thresholds. - const loop_filter_thresh *lfin = lfi_n->lfthr + *(lfl + step); + // Next block's thresholds, when it is within current 64x64 block. + // If it is out of bound, its mask is zero, and it points to current edge's + // filter parameters, instead of next edge's. + int next_edge = step; + if (offset + next_edge >= MI_SIZE_64X64) next_edge = 0; + const loop_filter_thresh *lfin = lfi_n->lfthr + *(lfl + next_edge); count = 1; if (mask & 1) { @@ -1376,6 +1391,7 @@ static void filter_selectively_horiz(uint8_t *s, int pitch, int plane, mask_16x16 >>= step * count; mask_8x8 >>= step * count; mask_4x4 >>= step * count; + offset += step * count; } } @@ -1387,11 +1403,16 @@ static void highbd_filter_selectively_horiz( int count; const int step = 1 << subsampling; const unsigned int two_block_mask = subsampling ? 5 : 3; + int offset = 0; for (mask = mask_16x16 | mask_8x8 | mask_4x4; mask; mask >>= step * count) { const loop_filter_thresh *lfi = lfi_n->lfthr + *lfl; - // Next block's thresholds. - const loop_filter_thresh *lfin = lfi_n->lfthr + *(lfl + step); + // Next block's thresholds, when it is within current 64x64 block. + // If it is out of bound, its mask is zero, and it points to current edge's + // filter parameters, instead of next edge's. + int next_edge = step; + if (offset + next_edge >= MI_SIZE_64X64) next_edge = 0; + const loop_filter_thresh *lfin = lfi_n->lfthr + *(lfl + next_edge); count = 1; if (mask & 1) { @@ -1401,9 +1422,9 @@ static void highbd_filter_selectively_horiz( if ((mask_16x16 & two_block_mask) == two_block_mask) { if (plane) { - aom_highbd_lpf_horizontal_6_dual(s, pitch, lfi->mblim, lfi->lim, - lfi->hev_thr, lfin->mblim, - lfin->lim, lfin->hev_thr, bd); + aom_highbd_lpf_horizontal_6_dual_c(s, pitch, lfi->mblim, lfi->lim, + lfi->hev_thr, lfin->mblim, + lfin->lim, lfin->hev_thr, bd); } else { aom_highbd_lpf_horizontal_14_dual(s, pitch, lfi->mblim, lfi->lim, lfi->hev_thr, lfin->mblim, @@ -1420,13 +1441,13 @@ static void highbd_filter_selectively_horiz( if ((mask_8x8 & two_block_mask) == two_block_mask) { if (plane) { - aom_highbd_lpf_horizontal_6_dual(s, pitch, lfi->mblim, lfi->lim, - lfi->hev_thr, lfin->mblim, - lfin->lim, lfin->hev_thr, bd); + aom_highbd_lpf_horizontal_6_dual_c(s, pitch, lfi->mblim, lfi->lim, + lfi->hev_thr, lfin->mblim, + lfin->lim, lfin->hev_thr, bd); } else { - aom_highbd_lpf_horizontal_8_dual(s, pitch, lfi->mblim, lfi->lim, - lfi->hev_thr, lfin->mblim, - lfin->lim, lfin->hev_thr, bd); + aom_highbd_lpf_horizontal_8_dual_c(s, pitch, lfi->mblim, lfi->lim, + lfi->hev_thr, lfin->mblim, + lfin->lim, lfin->hev_thr, bd); } count = 2; } else { @@ -1435,9 +1456,9 @@ static void highbd_filter_selectively_horiz( } } else if (mask_4x4 & 1) { if ((mask_4x4 & two_block_mask) == two_block_mask) { - aom_highbd_lpf_horizontal_4_dual(s, pitch, lfi->mblim, lfi->lim, - lfi->hev_thr, lfin->mblim, lfin->lim, - lfin->hev_thr, bd); + aom_highbd_lpf_horizontal_4_dual_c(s, pitch, lfi->mblim, lfi->lim, + lfi->hev_thr, lfin->mblim, + lfin->lim, lfin->hev_thr, bd); count = 2; } else { aom_highbd_lpf_horizontal_4(s, pitch, lfi->mblim, lfi->lim, @@ -1451,6 +1472,7 @@ static void highbd_filter_selectively_horiz( mask_16x16 >>= step * count; mask_8x8 >>= step * count; mask_4x4 >>= step * count; + offset += step * count; } } @@ -1463,12 +1485,13 @@ void av1_build_bitmask_vert_info( const int is_uv = plane > 0; TX_SIZE tx_size = TX_16X16, prev_tx_size = TX_16X16; uint8_t level, prev_level = 1; - int skip, prev_skip = 0; - int is_coding_block_border; + uint64_t skip, prev_skip = 0; + uint64_t is_coding_block_border; for (int r = 0; (r << MI_SIZE_LOG2) < plane_ptr->dst.height; r += row_step) { const int mi_row = r << subsampling_y; const int row = mi_row % MI_SIZE_64X64; + const int row_uv = row | subsampling_y; int index = 0; const int shift = get_index_shift(0, row, &index); @@ -1482,13 +1505,14 @@ void av1_build_bitmask_vert_info( const int x = (c + col_in_unit) << MI_SIZE_LOG2; if (x >= plane_ptr->dst.width) break; const int col = col_in_unit << subsampling_x; + const int col_uv = col | subsampling_x; const uint64_t mask = ((uint64_t)1 << (shift | col)); skip = lfm->skip.bits[index] & mask; is_coding_block_border = lfm->is_vert_border.bits[index] & mask; switch (plane) { - case 0: level = lfm->lfl_y_ver[row][col]; break; - case 1: level = lfm->lfl_u[row][col]; break; - case 2: level = lfm->lfl_v[row][col]; break; + case 0: level = lfm->lfl_y_ver[row_uv][col_uv]; break; + case 1: level = lfm->lfl_u_ver[row_uv][col_uv]; break; + case 2: level = lfm->lfl_v_ver[row_uv][col_uv]; break; default: assert(plane >= 0 && plane <= 2); return; } for (TX_SIZE ts = TX_4X4; ts <= TX_64X64; ++ts) { @@ -1502,9 +1526,7 @@ void av1_build_bitmask_vert_info( (!prev_skip || !skip || is_coding_block_border)) { const TX_SIZE min_tx_size = AOMMIN(TX_16X16, AOMMIN(tx_size, prev_tx_size)); - const int tmp_row = (mi_row | subsampling_y) % MI_SIZE_64X64; - const int tmp_col = (col | subsampling_x) % MI_SIZE_64X64; - const int shift_1 = get_index_shift(tmp_col, tmp_row, &index); + const int shift_1 = get_index_shift(col_uv, row_uv, &index); const uint64_t mask_1 = ((uint64_t)1 << shift_1); switch (plane) { case 0: lfm->left_y[min_tx_size].bits[index] |= mask_1; break; @@ -1512,6 +1534,14 @@ void av1_build_bitmask_vert_info( case 2: lfm->left_v[min_tx_size].bits[index] |= mask_1; break; default: assert(plane >= 0 && plane <= 2); return; } + if (level == 0 && prev_level != 0) { + switch (plane) { + case 0: lfm->lfl_y_ver[row_uv][col_uv] = prev_level; break; + case 1: lfm->lfl_u_ver[row_uv][col_uv] = prev_level; break; + case 2: lfm->lfl_v_ver[row_uv][col_uv] = prev_level; break; + default: assert(plane >= 0 && plane <= 2); return; + } + } } // update prev info @@ -1534,12 +1564,13 @@ void av1_build_bitmask_horz_info( const int is_uv = plane > 0; TX_SIZE tx_size = TX_16X16, prev_tx_size = TX_16X16; uint8_t level, prev_level = 1; - int skip, prev_skip = 0; - int is_coding_block_border; + uint64_t skip, prev_skip = 0; + uint64_t is_coding_block_border; for (int c = 0; (c << MI_SIZE_LOG2) < plane_ptr->dst.width; c += col_step) { const int mi_col = c << subsampling_x; const int col = mi_col % MI_SIZE_64X64; + const int col_uv = col | subsampling_x; for (int r = 0; (r << MI_SIZE_LOG2) < plane_ptr->dst.height; r += (tx_size_high_unit[TX_64X64] >> subsampling_y)) { @@ -1551,15 +1582,16 @@ void av1_build_bitmask_horz_info( const int y = (r + r_in_unit) << MI_SIZE_LOG2; if (y >= plane_ptr->dst.height) break; const int row = r_in_unit << subsampling_y; + const int row_uv = row | subsampling_y; int index = 0; const int shift = get_index_shift(col, row, &index); const uint64_t mask = ((uint64_t)1 << shift); skip = lfm->skip.bits[index] & mask; is_coding_block_border = lfm->is_horz_border.bits[index] & mask; switch (plane) { - case 0: level = lfm->lfl_y_hor[row][col]; break; - case 1: level = lfm->lfl_u[row][col]; break; - case 2: level = lfm->lfl_v[row][col]; break; + case 0: level = lfm->lfl_y_hor[row_uv][col_uv]; break; + case 1: level = lfm->lfl_u_hor[row_uv][col_uv]; break; + case 2: level = lfm->lfl_v_hor[row_uv][col_uv]; break; default: assert(plane >= 0 && plane <= 2); return; } for (TX_SIZE ts = TX_4X4; ts <= TX_64X64; ++ts) { @@ -1573,9 +1605,7 @@ void av1_build_bitmask_horz_info( (!prev_skip || !skip || is_coding_block_border)) { const TX_SIZE min_tx_size = AOMMIN(TX_16X16, AOMMIN(tx_size, prev_tx_size)); - const int tmp_row = (row | subsampling_y) % MI_SIZE_64X64; - const int tmp_col = (mi_col | subsampling_x) % MI_SIZE_64X64; - const int shift_1 = get_index_shift(tmp_col, tmp_row, &index); + const int shift_1 = get_index_shift(col_uv, row_uv, &index); const uint64_t mask_1 = ((uint64_t)1 << shift_1); switch (plane) { @@ -1584,6 +1614,14 @@ void av1_build_bitmask_horz_info( case 2: lfm->above_v[min_tx_size].bits[index] |= mask_1; break; default: assert(plane >= 0 && plane <= 2); return; } + if (level == 0 && prev_level != 0) { + switch (plane) { + case 0: lfm->lfl_y_hor[row_uv][col_uv] = prev_level; break; + case 1: lfm->lfl_u_hor[row_uv][col_uv] = prev_level; break; + case 2: lfm->lfl_v_hor[row_uv][col_uv] = prev_level; break; + default: assert(plane >= 0 && plane <= 2); return; + } + } } // update prev info @@ -1628,6 +1666,7 @@ void av1_filter_block_plane_bitmask_vert( const int shift = get_index_shift(col, row, &index); int index_next = 0; const int shift_next = get_index_shift(col, row_next, &index_next); + const int has_next_row = row_next < cm->mi_rows; switch (pl) { case 0: mask_16x16 = lfm->left_y[TX_16X16].bits[index]; @@ -1640,15 +1679,15 @@ void av1_filter_block_plane_bitmask_vert( mask_16x16 = lfm->left_u[TX_16X16].bits[index]; mask_8x8 = lfm->left_u[TX_8X8].bits[index]; mask_4x4 = lfm->left_u[TX_4X4].bits[index]; - lfl = &lfm->lfl_u[row][col]; - lfl2 = &lfm->lfl_u[row_next][col]; + lfl = &lfm->lfl_u_ver[row][col]; + lfl2 = &lfm->lfl_u_ver[row_next][col]; break; case 2: mask_16x16 = lfm->left_v[TX_16X16].bits[index]; mask_8x8 = lfm->left_v[TX_8X8].bits[index]; mask_4x4 = lfm->left_v[TX_4X4].bits[index]; - lfl = &lfm->lfl_v[row][col]; - lfl2 = &lfm->lfl_v[row_next][col]; + lfl = &lfm->lfl_v_ver[row][col]; + lfl2 = &lfm->lfl_v_ver[row_next][col]; break; default: assert(pl >= 0 && pl <= 2); return; } @@ -1658,6 +1697,11 @@ void av1_filter_block_plane_bitmask_vert( uint64_t mask_16x16_1 = (mask_16x16 >> shift_next) & mask_cutoff; uint64_t mask_8x8_1 = (mask_8x8 >> shift_next) & mask_cutoff; uint64_t mask_4x4_1 = (mask_4x4 >> shift_next) & mask_cutoff; + if (!has_next_row) { + mask_16x16_1 = 0; + mask_8x8_1 = 0; + mask_4x4_1 = 0; + } if (cm->seq_params.use_highbitdepth) highbd_filter_selectively_vert_row2( @@ -1712,13 +1756,13 @@ void av1_filter_block_plane_bitmask_horz( mask_16x16 = lfm->above_u[TX_16X16].bits[index]; mask_8x8 = lfm->above_u[TX_8X8].bits[index]; mask_4x4 = lfm->above_u[TX_4X4].bits[index]; - lfl = &lfm->lfl_u[row][col]; + lfl = &lfm->lfl_u_hor[row][col]; break; case 2: mask_16x16 = lfm->above_v[TX_16X16].bits[index]; mask_8x8 = lfm->above_v[TX_8X8].bits[index]; mask_4x4 = lfm->above_v[TX_4X4].bits[index]; - lfl = &lfm->lfl_v[row][col]; + lfl = &lfm->lfl_v_hor[row][col]; break; default: assert(pl >= 0 && pl <= 2); return; } @@ -1786,15 +1830,15 @@ void av1_filter_block_plane_ver(AV1_COMMON *const cm, mask_16x16 = lfm->left_u[TX_16X16].bits[index]; mask_8x8 = lfm->left_u[TX_8X8].bits[index]; mask_4x4 = lfm->left_u[TX_4X4].bits[index]; - lfl = &lfm->lfl_u[row][col]; - lfl2 = &lfm->lfl_u[row_next][col]; + lfl = &lfm->lfl_u_ver[row][col]; + lfl2 = &lfm->lfl_u_ver[row_next][col]; break; case 2: mask_16x16 = lfm->left_v[TX_16X16].bits[index]; mask_8x8 = lfm->left_v[TX_8X8].bits[index]; mask_4x4 = lfm->left_v[TX_4X4].bits[index]; - lfl = &lfm->lfl_v[row][col]; - lfl2 = &lfm->lfl_v[row_next][col]; + lfl = &lfm->lfl_v_ver[row][col]; + lfl2 = &lfm->lfl_v_ver[row_next][col]; break; default: assert(pl >= 0 && pl <= 2); return; } @@ -1861,13 +1905,13 @@ void av1_filter_block_plane_hor(AV1_COMMON *const cm, mask_16x16 = lfm->above_u[TX_16X16].bits[index]; mask_8x8 = lfm->above_u[TX_8X8].bits[index]; mask_4x4 = lfm->above_u[TX_4X4].bits[index]; - lfl = &lfm->lfl_u[row][col]; + lfl = &lfm->lfl_u_hor[row][col]; break; case 2: mask_16x16 = lfm->above_v[TX_16X16].bits[index]; mask_8x8 = lfm->above_v[TX_8X8].bits[index]; mask_4x4 = lfm->above_v[TX_4X4].bits[index]; - lfl = &lfm->lfl_v[row][col]; + lfl = &lfm->lfl_v_hor[row][col]; break; default: assert(pl >= 0 && pl <= 2); return; } @@ -2219,6 +2263,92 @@ void av1_filter_block_plane_horz(const AV1_COMMON *const cm, } } +void av1_filter_block_plane_vert_test(const AV1_COMMON *const cm, + const MACROBLOCKD *const xd, + const int plane, + const MACROBLOCKD_PLANE *const plane_ptr, + const uint32_t mi_row, + const uint32_t mi_col) { + const int row_step = MI_SIZE >> MI_SIZE_LOG2; + const uint32_t scale_horz = plane_ptr->subsampling_x; + const uint32_t scale_vert = plane_ptr->subsampling_y; + uint8_t *const dst_ptr = plane_ptr->dst.buf; + const int dst_stride = plane_ptr->dst.stride; + const int y_range = cm->mi_rows >> scale_vert; + const int x_range = cm->mi_cols >> scale_horz; + for (int y = 0; y < y_range; y += row_step) { + uint8_t *p = dst_ptr + y * MI_SIZE * dst_stride; + for (int x = 0; x < x_range;) { + // inner loop always filter vertical edges in a MI block. If MI size + // is 8x8, it will filter the vertical edge aligned with a 8x8 block. + // If 4x4 trasnform is used, it will then filter the internal edge + // aligned with a 4x4 block + const uint32_t curr_x = ((mi_col * MI_SIZE) >> scale_horz) + x * MI_SIZE; + const uint32_t curr_y = ((mi_row * MI_SIZE) >> scale_vert) + y * MI_SIZE; + uint32_t advance_units; + TX_SIZE tx_size; + AV1_DEBLOCKING_PARAMETERS params; + memset(¶ms, 0, sizeof(params)); + + tx_size = + set_lpf_parameters(¶ms, ((ptrdiff_t)1 << scale_horz), cm, xd, + VERT_EDGE, curr_x, curr_y, plane, plane_ptr); + if (tx_size == TX_INVALID) { + params.filter_length = 0; + tx_size = TX_4X4; + } + + // advance the destination pointer + advance_units = tx_size_wide_unit[tx_size]; + x += advance_units; + p += advance_units * MI_SIZE; + } + } +} + +void av1_filter_block_plane_horz_test(const AV1_COMMON *const cm, + const MACROBLOCKD *const xd, + const int plane, + const MACROBLOCKD_PLANE *const plane_ptr, + const uint32_t mi_row, + const uint32_t mi_col) { + const int col_step = MI_SIZE >> MI_SIZE_LOG2; + const uint32_t scale_horz = plane_ptr->subsampling_x; + const uint32_t scale_vert = plane_ptr->subsampling_y; + uint8_t *const dst_ptr = plane_ptr->dst.buf; + const int dst_stride = plane_ptr->dst.stride; + const int y_range = cm->mi_rows >> scale_vert; + const int x_range = cm->mi_cols >> scale_horz; + for (int x = 0; x < x_range; x += col_step) { + uint8_t *p = dst_ptr + x * MI_SIZE; + for (int y = 0; y < y_range;) { + // inner loop always filter vertical edges in a MI block. If MI size + // is 8x8, it will first filter the vertical edge aligned with a 8x8 + // block. If 4x4 trasnform is used, it will then filter the internal + // edge aligned with a 4x4 block + const uint32_t curr_x = ((mi_col * MI_SIZE) >> scale_horz) + x * MI_SIZE; + const uint32_t curr_y = ((mi_row * MI_SIZE) >> scale_vert) + y * MI_SIZE; + uint32_t advance_units; + TX_SIZE tx_size; + AV1_DEBLOCKING_PARAMETERS params; + memset(¶ms, 0, sizeof(params)); + + tx_size = + set_lpf_parameters(¶ms, (cm->mi_stride << scale_vert), cm, xd, + HORZ_EDGE, curr_x, curr_y, plane, plane_ptr); + if (tx_size == TX_INVALID) { + params.filter_length = 0; + tx_size = TX_4X4; + } + + // advance the destination pointer + advance_units = tx_size_high_unit[tx_size]; + y += advance_units; + p += advance_units * dst_stride * MI_SIZE; + } + } +} + static void loop_filter_rows(YV12_BUFFER_CONFIG *frame_buffer, AV1_COMMON *cm, MACROBLOCKD *xd, int start, int stop, #if LOOP_FILTER_BITMASK @@ -2233,6 +2363,7 @@ static void loop_filter_rows(YV12_BUFFER_CONFIG *frame_buffer, AV1_COMMON *cm, #if LOOP_FILTER_BITMASK if (is_decoding) { + cm->is_decoding = is_decoding; for (plane = plane_start; plane < plane_end; plane++) { if (plane == 0 && !(cm->lf.filter_level[0]) && !(cm->lf.filter_level[1])) break; @@ -2243,24 +2374,25 @@ static void loop_filter_rows(YV12_BUFFER_CONFIG *frame_buffer, AV1_COMMON *cm, av1_setup_dst_planes(pd, cm->seq_params.sb_size, frame_buffer, 0, 0, plane, plane + 1); + av1_build_bitmask_vert_info(cm, &pd[plane], plane); av1_build_bitmask_horz_info(cm, &pd[plane], plane); // apply loop filtering which only goes through buffer once for (mi_row = start; mi_row < stop; mi_row += MI_SIZE_64X64) { for (mi_col = col_start; mi_col < col_end; mi_col += MI_SIZE_64X64) { - av1_setup_dst_planes(pd, MI_SIZE_64X64, frame_buffer, mi_row, mi_col, + av1_setup_dst_planes(pd, BLOCK_64X64, frame_buffer, mi_row, mi_col, plane, plane + 1); av1_filter_block_plane_bitmask_vert(cm, &pd[plane], plane, mi_row, mi_col); if (mi_col - MI_SIZE_64X64 >= 0) { - av1_setup_dst_planes(pd, MI_SIZE_64X64, frame_buffer, mi_row, + av1_setup_dst_planes(pd, BLOCK_64X64, frame_buffer, mi_row, mi_col - MI_SIZE_64X64, plane, plane + 1); av1_filter_block_plane_bitmask_horz(cm, &pd[plane], plane, mi_row, mi_col - MI_SIZE_64X64); } } - av1_setup_dst_planes(pd, MI_SIZE_64X64, frame_buffer, mi_row, + av1_setup_dst_planes(pd, BLOCK_64X64, frame_buffer, mi_row, mi_col - MI_SIZE_64X64, plane, plane + 1); av1_filter_block_plane_bitmask_horz(cm, &pd[plane], plane, mi_row, mi_col - MI_SIZE_64X64); @@ -2278,31 +2410,6 @@ static void loop_filter_rows(YV12_BUFFER_CONFIG *frame_buffer, AV1_COMMON *cm, else if (plane == 2 && !(cm->lf.filter_level_v)) continue; -#if LOOP_FILTER_BITMASK - // filter all vertical edges every superblock (could be 128x128 or 64x64) - for (mi_row = start; mi_row < stop; mi_row += cm->seq_params.mib_size) { - for (mi_col = col_start; mi_col < col_end; - mi_col += cm->seq_params.mib_size) { - av1_setup_dst_planes(pd, cm->seq_params.sb_size, frame_buffer, mi_row, - mi_col, plane, plane + 1); - - av1_setup_bitmask(cm, mi_row, mi_col, plane, pd[plane].subsampling_x, - pd[plane].subsampling_y, stop, col_end); - av1_filter_block_plane_ver(cm, &pd[plane], plane, mi_row, mi_col); - } - } - - // filter all horizontal edges every superblock - for (mi_row = start; mi_row < stop; mi_row += cm->seq_params.mib_size) { - for (mi_col = col_start; mi_col < col_end; - mi_col += cm->seq_params.mib_size) { - av1_setup_dst_planes(pd, cm->seq_params.sb_size, frame_buffer, mi_row, - mi_col, plane, plane + 1); - - av1_filter_block_plane_hor(cm, &pd[plane], plane, mi_row, mi_col); - } - } -#else if (cm->lf.combine_vert_horz_lf) { // filter all vertical and horizontal edges in every 128x128 super block for (mi_row = start; mi_row < stop; mi_row += MAX_MIB_SIZE) { @@ -2348,7 +2455,6 @@ static void loop_filter_rows(YV12_BUFFER_CONFIG *frame_buffer, AV1_COMMON *cm, } } } -#endif // LOOP_FILTER_BITMASK } } diff --git a/third_party/aom/av1/common/av1_loopfilter.h b/third_party/aom/av1/common/av1_loopfilter.h index 80ac611781..ae4d372fad 100644 --- a/third_party/aom/av1/common/av1_loopfilter.h +++ b/third_party/aom/av1/common/av1_loopfilter.h @@ -33,11 +33,11 @@ enum lf_path { LF_PATH_SLOW, }; -#if LOOP_FILTER_BITMASK typedef struct { uint64_t bits[4]; } FilterMask; +#if LOOP_FILTER_BITMASK // This structure holds bit masks for all 4x4 blocks in a 64x64 region. // Each 1 bit represents a position in which we want to apply the loop filter. // For Y plane, 4x4 in 64x64 requires 16x16 = 256 bit, therefore we use 4 @@ -61,10 +61,12 @@ typedef struct { uint8_t lfl_y_ver[MI_SIZE_64X64][MI_SIZE_64X64]; // U plane filter level - uint8_t lfl_u[MI_SIZE_64X64][MI_SIZE_64X64]; + uint8_t lfl_u_ver[MI_SIZE_64X64][MI_SIZE_64X64]; + uint8_t lfl_u_hor[MI_SIZE_64X64][MI_SIZE_64X64]; // V plane filter level - uint8_t lfl_v[MI_SIZE_64X64][MI_SIZE_64X64]; + uint8_t lfl_v_ver[MI_SIZE_64X64][MI_SIZE_64X64]; + uint8_t lfl_v_hor[MI_SIZE_64X64][MI_SIZE_64X64]; // other info FilterMask skip; @@ -173,39 +175,23 @@ LoopFilterMask *get_loop_filter_mask(const struct AV1Common *const cm, int mi_row, int mi_col); int get_index_shift(int mi_col, int mi_row, int *index); -static const FilterMask left_txform_mask[TX_SIZES] = { - { { 0x0000000000000001ULL, // TX_4X4, - 0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL } }, +void av1_build_bitmask_vert_info( + struct AV1Common *const cm, const struct macroblockd_plane *const plane_ptr, + int plane); - { { 0x0000000000010001ULL, // TX_8X8, - 0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL } }, +void av1_build_bitmask_horz_info( + struct AV1Common *const cm, const struct macroblockd_plane *const plane_ptr, + int plane); - { { 0x0001000100010001ULL, // TX_16X16, - 0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL } }, +void av1_filter_block_plane_bitmask_vert( + struct AV1Common *const cm, struct macroblockd_plane *const plane_ptr, + int pl, int mi_row, int mi_col); - { { 0x0001000100010001ULL, // TX_32X32, - 0x0001000100010001ULL, 0x0000000000000000ULL, 0x0000000000000000ULL } }, +void av1_filter_block_plane_bitmask_horz( + struct AV1Common *const cm, struct macroblockd_plane *const plane_ptr, + int pl, int mi_row, int mi_col); - { { 0x0001000100010001ULL, // TX_64X64, - 0x0001000100010001ULL, 0x0001000100010001ULL, 0x0001000100010001ULL } }, -}; - -static const uint64_t above_txform_mask[2][TX_SIZES] = { - { - 0x0000000000000001ULL, // TX_4X4 - 0x0000000000000003ULL, // TX_8X8 - 0x000000000000000fULL, // TX_16X16 - 0x00000000000000ffULL, // TX_32X32 - 0x000000000000ffffULL, // TX_64X64 - }, - { - 0x0000000000000001ULL, // TX_4X4 - 0x0000000000000005ULL, // TX_8X8 - 0x0000000000000055ULL, // TX_16X16 - 0x0000000000005555ULL, // TX_32X32 - 0x0000000055555555ULL, // TX_64X64 - }, -}; +#endif // LOOP_FILTER_BITMASK extern const int mask_id_table_tx_4x4[BLOCK_SIZES_ALL]; @@ -215,10 +201,13 @@ extern const int mask_id_table_tx_16x16[BLOCK_SIZES_ALL]; extern const int mask_id_table_tx_32x32[BLOCK_SIZES_ALL]; +// corresponds to entry id in table left_mask_univariant_reordered, +// of block size mxn and TX_mxn. +extern const int mask_id_table_vert_border[BLOCK_SIZES_ALL]; + extern const FilterMask left_mask_univariant_reordered[67]; extern const FilterMask above_mask_univariant_reordered[67]; -#endif #ifdef __cplusplus } // extern "C" diff --git a/third_party/aom/av1/common/av1_rtcd_defs.pl b/third_party/aom/av1/common/av1_rtcd_defs.pl index dee1f1c79b..7d67d37c2b 100755 --- a/third_party/aom/av1/common/av1_rtcd_defs.pl +++ b/third_party/aom/av1/common/av1_rtcd_defs.pl @@ -33,6 +33,8 @@ struct txfm_param; struct aom_variance_vtable; struct search_site_config; struct yv12_buffer_config; +struct NN_CONFIG; +typedef struct NN_CONFIG NN_CONFIG; /* Function pointers return by CfL functions */ typedef void (*cfl_subsample_lbd_fn)(const uint8_t *input, int input_stride, @@ -79,8 +81,11 @@ specialize qw/av1_highbd_wiener_convolve_add_src avx2/; # directional intra predictor functions add_proto qw/void av1_dr_prediction_z1/, "uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_above, int dx, int dy"; +specialize qw/av1_dr_prediction_z1 avx2/; add_proto qw/void av1_dr_prediction_z2/, "uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_above, int upsample_left, int dx, int dy"; +specialize qw/av1_dr_prediction_z2 avx2/; add_proto qw/void av1_dr_prediction_z3/, "uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_left, int dx, int dy"; +specialize qw/av1_dr_prediction_z3 avx2/; # FILTER_INTRA predictor functions add_proto qw/void av1_filter_intra_predictor/, "uint8_t *dst, ptrdiff_t stride, TX_SIZE tx_size, const uint8_t *above, const uint8_t *left, int mode"; @@ -115,14 +120,14 @@ add_proto qw/void av1_highbd_inv_txfm_add_4x4/, "const tran_low_t *dqcoeff, uin specialize qw/av1_highbd_inv_txfm_add_4x4 sse4_1/; add_proto qw/void av1_highbd_inv_txfm_add_8x8/, "const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param"; specialize qw/av1_highbd_inv_txfm_add_8x8 sse4_1/; -add_proto qw/void av1_highbd_inv_txfm_add_16x8/, "const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param"; -specialize qw/av1_highbd_inv_txfm_add_16x8 sse4_1/; -add_proto qw/void av1_highbd_inv_txfm_add_8x16/, "const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param"; -specialize qw/av1_highbd_inv_txfm_add_8x16 sse4_1/; -add_proto qw/void av1_highbd_inv_txfm_add_16x16/, "const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param"; -specialize qw/av1_highbd_inv_txfm_add_16x16 sse4_1/; -add_proto qw/void av1_highbd_inv_txfm_add_32x32/, "const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param"; -specialize qw/av1_highbd_inv_txfm_add_32x32 sse4_1 avx2/; +add_proto qw/void av1_highbd_inv_txfm_add_4x8/, "const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param"; +specialize qw/av1_highbd_inv_txfm_add_4x8 sse4_1/; +add_proto qw/void av1_highbd_inv_txfm_add_8x4/, "const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param"; +specialize qw/av1_highbd_inv_txfm_add_8x4 sse4_1/; +add_proto qw/void av1_highbd_inv_txfm_add_4x16/, "const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param"; +specialize qw/av1_highbd_inv_txfm_add_4x16 sse4_1/; +add_proto qw/void av1_highbd_inv_txfm_add_16x4/, "const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param"; +specialize qw/av1_highbd_inv_txfm_add_16x4 sse4_1/; add_proto qw/void av1_highbd_iwht4x4_1_add/, "const tran_low_t *input, uint8_t *dest, int dest_stride, int bd"; add_proto qw/void av1_highbd_iwht4x4_16_add/, "const tran_low_t *input, uint8_t *dest, int dest_stride, int bd"; @@ -153,8 +158,12 @@ add_proto qw/void av1_inv_txfm2d_add_32x8/, "const int32_t *input, uint16_t *out # directional intra predictor functions add_proto qw/void av1_highbd_dr_prediction_z1/, "uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_above, int dx, int dy, int bd"; +specialize qw/av1_highbd_dr_prediction_z1 avx2/; add_proto qw/void av1_highbd_dr_prediction_z2/, "uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_above, int upsample_left, int dx, int dy, int bd"; +# TODO(https://crbug.com/aomedia/2288) +#specialize qw/av1_highbd_dr_prediction_z2 avx2/; add_proto qw/void av1_highbd_dr_prediction_z3/, "uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_left, int dx, int dy, int bd"; +specialize qw/av1_highbd_dr_prediction_z3 avx2/; # build compound seg mask functions add_proto qw/void av1_build_compound_diffwtd_mask/, "uint8_t *mask, DIFFWTD_MASK_TYPE mask_type, const uint8_t *src0, int src0_stride, const uint8_t *src1, int src1_stride, int h, int w"; @@ -166,6 +175,10 @@ specialize qw/av1_build_compound_diffwtd_mask_highbd ssse3 avx2/; add_proto qw/void av1_build_compound_diffwtd_mask_d16/, "uint8_t *mask, DIFFWTD_MASK_TYPE mask_type, const CONV_BUF_TYPE *src0, int src0_stride, const CONV_BUF_TYPE *src1, int src1_stride, int h, int w, ConvolveParams *conv_params, int bd"; specialize qw/av1_build_compound_diffwtd_mask_d16 sse4_1 avx2 neon/; +# Helper functions. +add_proto qw/void av1_round_shift_array/, "int32_t *arr, int size, int bit"; +specialize "av1_round_shift_array", qw/sse4_1 neon/; + # # Encoder functions below this point. # @@ -196,32 +209,44 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") { specialize qw/av1_lowbd_fwd_txfm sse2 sse4_1 avx2/; add_proto qw/void av1_fwd_txfm2d_4x8/, "const int16_t *input, int32_t *output, int stride, TX_TYPE tx_type, int bd"; + specialize qw/av1_fwd_txfm2d_4x8 sse4_1/; add_proto qw/void av1_fwd_txfm2d_8x4/, "const int16_t *input, int32_t *output, int stride, TX_TYPE tx_type, int bd"; + specialize qw/av1_fwd_txfm2d_8x4 sse4_1/; add_proto qw/void av1_fwd_txfm2d_8x16/, "const int16_t *input, int32_t *output, int stride, TX_TYPE tx_type, int bd"; - specialize qw/av1_fwd_txfm2d_8x16 sse4_1/; + specialize qw/av1_fwd_txfm2d_8x16 sse4_1 avx2/; add_proto qw/void av1_fwd_txfm2d_16x8/, "const int16_t *input, int32_t *output, int stride, TX_TYPE tx_type, int bd"; - specialize qw/av1_fwd_txfm2d_16x8 sse4_1/; + specialize qw/av1_fwd_txfm2d_16x8 sse4_1 avx2/; add_proto qw/void av1_fwd_txfm2d_16x32/, "const int16_t *input, int32_t *output, int stride, TX_TYPE tx_type, int bd"; + specialize qw/av1_fwd_txfm2d_16x32 sse4_1/; add_proto qw/void av1_fwd_txfm2d_32x16/, "const int16_t *input, int32_t *output, int stride, TX_TYPE tx_type, int bd"; + specialize qw/av1_fwd_txfm2d_32x16 sse4_1/; add_proto qw/void av1_fwd_txfm2d_4x16/, "const int16_t *input, int32_t *output, int stride, TX_TYPE tx_type, int bd"; + specialize qw/av1_fwd_txfm2d_4x16 sse4_1/; add_proto qw/void av1_fwd_txfm2d_16x4/, "const int16_t *input, int32_t *output, int stride, TX_TYPE tx_type, int bd"; + specialize qw/av1_fwd_txfm2d_16x4 sse4_1/; add_proto qw/void av1_fwd_txfm2d_8x32/, "const int16_t *input, int32_t *output, int stride, TX_TYPE tx_type, int bd"; + specialize qw/av1_fwd_txfm2d_8x32 sse4_1/; add_proto qw/void av1_fwd_txfm2d_32x8/, "const int16_t *input, int32_t *output, int stride, TX_TYPE tx_type, int bd"; + specialize qw/av1_fwd_txfm2d_32x8 sse4_1/; add_proto qw/void av1_fwd_txfm2d_4x4/, "const int16_t *input, int32_t *output, int stride, TX_TYPE tx_type, int bd"; specialize qw/av1_fwd_txfm2d_4x4 sse4_1/; add_proto qw/void av1_fwd_txfm2d_8x8/, "const int16_t *input, int32_t *output, int stride, TX_TYPE tx_type, int bd"; - specialize qw/av1_fwd_txfm2d_8x8 sse4_1/; + specialize qw/av1_fwd_txfm2d_8x8 sse4_1 avx2/; add_proto qw/void av1_fwd_txfm2d_16x16/, "const int16_t *input, int32_t *output, int stride, TX_TYPE tx_type, int bd"; - specialize qw/av1_fwd_txfm2d_16x16 sse4_1/; + specialize qw/av1_fwd_txfm2d_16x16 sse4_1 avx2/; add_proto qw/void av1_fwd_txfm2d_32x32/, "const int16_t *input, int32_t *output, int stride, TX_TYPE tx_type, int bd"; - specialize qw/av1_fwd_txfm2d_32x32 sse4_1/; + specialize qw/av1_fwd_txfm2d_32x32 sse4_1 avx2/; add_proto qw/void av1_fwd_txfm2d_64x64/, "const int16_t *input, int32_t *output, int stride, TX_TYPE tx_type, int bd"; - specialize qw/av1_fwd_txfm2d_64x64 sse4_1/; + specialize qw/av1_fwd_txfm2d_64x64 sse4_1 avx2/; add_proto qw/void av1_fwd_txfm2d_32x64/, "const int16_t *input, int32_t *output, int stride, TX_TYPE tx_type, int bd"; + specialize qw/av1_fwd_txfm2d_32x64 sse4_1/; add_proto qw/void av1_fwd_txfm2d_64x32/, "const int16_t *input, int32_t *output, int stride, TX_TYPE tx_type, int bd"; + specialize qw/av1_fwd_txfm2d_64x32 sse4_1/; add_proto qw/void av1_fwd_txfm2d_16x64/, "const int16_t *input, int32_t *output, int stride, TX_TYPE tx_type, int bd"; + specialize qw/av1_fwd_txfm2d_16x64 sse4_1/; add_proto qw/void av1_fwd_txfm2d_64x16/, "const int16_t *input, int32_t *output, int stride, TX_TYPE tx_type, int bd"; + specialize qw/av1_fwd_txfm2d_64x16 sse4_1/; # # Motion search @@ -230,8 +255,9 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") { add_proto qw/int av1_full_range_search/, "const struct macroblock *x, const struct search_site_config *cfg, MV *ref_mv, MV *best_mv, int search_param, int sad_per_bit, int *num00, const struct aom_variance_vtable *fn_ptr, const MV *center_mv"; - add_proto qw/void av1_temporal_filter_apply/, "uint8_t *frame1, unsigned int stride, uint8_t *frame2, unsigned int block_width, unsigned int block_height, int strength, int filter_weight, unsigned int *accumulator, uint16_t *count"; - specialize qw/av1_temporal_filter_apply sse2 msa/; + # TODO(yunqing): Add back the optimizations. + # add_proto qw/void av1_temporal_filter_apply/, "uint8_t *frame1, unsigned int stride, uint8_t *frame2, unsigned int block_width, unsigned int block_height, int strength, int filter_weight, unsigned int *accumulator, uint16_t *count"; + # specialize qw/av1_temporal_filter_apply sse2 msa/; add_proto qw/void av1_quantize_b/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan, const qm_val_t * qm_ptr, const qm_val_t * iqm_ptr, int log_scale"; @@ -240,7 +266,7 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") { add_proto qw/int64_t av1_highbd_block_error/, "const tran_low_t *coeff, const tran_low_t *dqcoeff, intptr_t block_size, int64_t *ssz, int bd"; specialize qw/av1_highbd_block_error sse2/; - add_proto qw/void av1_highbd_temporal_filter_apply/, "uint8_t *frame1, unsigned int stride, uint8_t *frame2, unsigned int block_width, unsigned int block_height, int strength, int filter_weight, unsigned int *accumulator, uint16_t *count"; + # add_proto qw/void av1_highbd_temporal_filter_apply/, "uint8_t *frame1, unsigned int stride, uint8_t *frame2, unsigned int block_width, unsigned int block_height, int strength, int filter_weight, unsigned int *accumulator, uint16_t *count"; add_proto qw/void av1_highbd_quantize_fp/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan, int log_scale"; specialize qw/av1_highbd_quantize_fp sse4_1 avx2/; @@ -266,18 +292,30 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") { add_proto qw/uint32_t av1_get_crc32c_value/, "void *crc_calculator, uint8_t *p, int length"; specialize qw/av1_get_crc32c_value sse4_2/; - add_proto qw/void av1_compute_stats/, "int wiener_win, const uint8_t *dgd8, const uint8_t *src8, int h_start, int h_end, int v_start, int v_end, int dgd_stride, int src_stride, double *M, double *H"; + add_proto qw/void av1_compute_stats/, "int wiener_win, const uint8_t *dgd8, const uint8_t *src8, int h_start, int h_end, int v_start, int v_end, int dgd_stride, int src_stride, int64_t *M, int64_t *H"; specialize qw/av1_compute_stats sse4_1 avx2/; + add_proto qw/void av1_compute_stats_highbd/, "int wiener_win, const uint8_t *dgd8, const uint8_t *src8, int h_start, int h_end, int v_start, int v_end, int dgd_stride, int src_stride, int64_t *M, int64_t *H, aom_bit_depth_t bit_depth"; + specialize qw/av1_compute_stats_highbd sse4_1 avx2/; + add_proto qw/int64_t av1_lowbd_pixel_proj_error/, " const uint8_t *src8, int width, int height, int src_stride, const uint8_t *dat8, int dat_stride, int32_t *flt0, int flt0_stride, int32_t *flt1, int flt1_stride, int xq[2], const sgr_params_type *params"; specialize qw/av1_lowbd_pixel_proj_error sse4_1 avx2/; + + add_proto qw/int64_t av1_highbd_pixel_proj_error/, " const uint8_t *src8, int width, int height, int src_stride, const uint8_t *dat8, int dat_stride, int32_t *flt0, int flt0_stride, int32_t *flt1, int flt1_stride, int xq[2], const sgr_params_type *params"; + specialize qw/av1_highbd_pixel_proj_error sse4_1 avx2/; + + add_proto qw/void av1_get_horver_correlation_full/, " const int16_t *diff, int stride, int w, int h, float *hcorr, float *vcorr"; + specialize qw/av1_get_horver_correlation_full sse4_1 avx2/; + + add_proto qw/void av1_nn_predict/, " const float *input_nodes, const NN_CONFIG *const nn_config, float *const output"; + specialize qw/av1_nn_predict sse3/; } # end encoder functions # Deringing Functions add_proto qw/int cdef_find_dir/, "const uint16_t *img, int stride, int32_t *var, int coeff_shift"; -add_proto qw/void cdef_filter_block/, "uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int max, int coeff_shift"; +add_proto qw/void cdef_filter_block/, "uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int coeff_shift"; add_proto qw/void copy_rect8_8bit_to_16bit/, "uint16_t *dst, int dstride, const uint8_t *src, int sstride, int v, int h"; add_proto qw/void copy_rect8_16bit_to_16bit/, "uint16_t *dst, int dstride, const uint16_t *src, int sstride, int v, int h"; @@ -302,7 +340,7 @@ specialize qw/av1_highbd_warp_affine sse4_1/; if (aom_config("CONFIG_AV1_ENCODER") eq "yes") { add_proto qw/double compute_cross_correlation/, "unsigned char *im1, int stride1, int x1, int y1, unsigned char *im2, int stride2, int x2, int y2"; - specialize qw/compute_cross_correlation sse4_1/; + specialize qw/compute_cross_correlation sse4_1 avx2/; } # LOOP_RESTORATION functions @@ -321,18 +359,18 @@ add_proto qw/void av1_convolve_2d_sr/, "const uint8_t *src, int src_stride, uint add_proto qw/void av1_convolve_2d_copy_sr/, "const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params"; add_proto qw/void av1_convolve_x_sr/, "const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params"; add_proto qw/void av1_convolve_y_sr/, "const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params"; -add_proto qw/void av1_jnt_convolve_2d/, "const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params"; -add_proto qw/void av1_jnt_convolve_2d_copy/, "const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params"; -add_proto qw/void av1_jnt_convolve_x/, "const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params"; -add_proto qw/void av1_jnt_convolve_y/, "const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params"; +add_proto qw/void av1_dist_wtd_convolve_2d/, "const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params"; +add_proto qw/void av1_dist_wtd_convolve_2d_copy/, "const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params"; +add_proto qw/void av1_dist_wtd_convolve_x/, "const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params"; +add_proto qw/void av1_dist_wtd_convolve_y/, "const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params"; add_proto qw/void av1_highbd_convolve_2d_copy_sr/, "const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd"; add_proto qw/void av1_highbd_convolve_2d_sr/, "const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd"; add_proto qw/void av1_highbd_convolve_x_sr/, "const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd"; add_proto qw/void av1_highbd_convolve_y_sr/, "const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd"; -add_proto qw/void av1_highbd_jnt_convolve_2d/, "const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd"; -add_proto qw/void av1_highbd_jnt_convolve_x/, "const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd"; -add_proto qw/void av1_highbd_jnt_convolve_y/, "const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd"; -add_proto qw/void av1_highbd_jnt_convolve_2d_copy/, "const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd"; +add_proto qw/void av1_highbd_dist_wtd_convolve_2d/, "const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd"; +add_proto qw/void av1_highbd_dist_wtd_convolve_x/, "const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd"; +add_proto qw/void av1_highbd_dist_wtd_convolve_y/, "const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd"; +add_proto qw/void av1_highbd_dist_wtd_convolve_2d_copy/, "const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params, int bd"; add_proto qw/void av1_convolve_2d_scale/, "const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_qn, const int x_step_qn, const int subpel_y_q4, const int y_step_qn, ConvolveParams *conv_params"; add_proto qw/void av1_highbd_convolve_2d_scale/, "const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, const int x_step_qn, const int subpel_y_q4, const int y_step_qn, ConvolveParams *conv_params, int bd"; @@ -342,19 +380,19 @@ add_proto qw/void av1_highbd_jnt_convolve_2d_copy/, "const uint16_t *src, int sr specialize qw/av1_convolve_x_sr sse2 avx2 neon/; specialize qw/av1_convolve_y_sr sse2 avx2 neon/; specialize qw/av1_convolve_2d_scale sse4_1/; - specialize qw/av1_jnt_convolve_2d ssse3 avx2 neon/; - specialize qw/av1_jnt_convolve_2d_copy sse2 avx2 neon/; - specialize qw/av1_jnt_convolve_x sse2 avx2 neon/; - specialize qw/av1_jnt_convolve_y sse2 avx2 neon/; + specialize qw/av1_dist_wtd_convolve_2d sse2 ssse3 avx2 neon/; + specialize qw/av1_dist_wtd_convolve_2d_copy sse2 avx2 neon/; + specialize qw/av1_dist_wtd_convolve_x sse2 avx2 neon/; + specialize qw/av1_dist_wtd_convolve_y sse2 avx2 neon/; specialize qw/av1_highbd_convolve_2d_copy_sr sse2 avx2/; specialize qw/av1_highbd_convolve_2d_sr ssse3 avx2/; specialize qw/av1_highbd_convolve_x_sr ssse3 avx2/; specialize qw/av1_highbd_convolve_y_sr ssse3 avx2/; specialize qw/av1_highbd_convolve_2d_scale sse4_1/; - specialize qw/av1_highbd_jnt_convolve_2d sse4_1 avx2/; - specialize qw/av1_highbd_jnt_convolve_x sse4_1 avx2/; - specialize qw/av1_highbd_jnt_convolve_y sse4_1 avx2/; - specialize qw/av1_highbd_jnt_convolve_2d_copy sse4_1 avx2/; + specialize qw/av1_highbd_dist_wtd_convolve_2d sse4_1 avx2/; + specialize qw/av1_highbd_dist_wtd_convolve_x sse4_1 avx2/; + specialize qw/av1_highbd_dist_wtd_convolve_y sse4_1 avx2/; + specialize qw/av1_highbd_dist_wtd_convolve_2d_copy sse4_1 avx2/; # INTRA_EDGE functions add_proto qw/void av1_filter_intra_edge/, "uint8_t *p, int sz, int strength"; diff --git a/third_party/aom/av1/common/av1_txfm.c b/third_party/aom/av1/common/av1_txfm.c index bb70eab703..ac43402f49 100644 --- a/third_party/aom/av1/common/av1_txfm.c +++ b/third_party/aom/av1/common/av1_txfm.c @@ -10,10 +10,11 @@ */ #include "config/aom_dsp_rtcd.h" +#include "config/av1_rtcd.h" #include "av1/common/av1_txfm.h" -// av1_cospi_arr[i][j] = (int)round(cos(M_PI*j/128) * (1<<(cos_bit_min+i))); +// av1_cospi_arr[i][j] = (int)round(cos(PI*j/128) * (1<<(cos_bit_min+i))); const int32_t av1_cospi_arr_data[7][64] = { { 1024, 1024, 1023, 1021, 1019, 1016, 1013, 1009, 1004, 999, 993, 987, 980, 972, 964, 955, 946, 936, 926, 915, 903, 891, 878, 865, 851, 837, diff --git a/third_party/aom/av1/common/av1_txfm.h b/third_party/aom/av1/common/av1_txfm.h index 59d64ca4af..14e2c0e9c2 100644 --- a/third_party/aom/av1/common/av1_txfm.h +++ b/third_party/aom/av1/common/av1_txfm.h @@ -110,7 +110,7 @@ typedef void (*TxfmFunc)(const int32_t *input, int32_t *output, int8_t cos_bit, typedef void (*FwdTxfm2dFunc)(const int16_t *input, int32_t *output, int stride, TX_TYPE tx_type, int bd); -typedef enum TXFM_TYPE { +enum { TXFM_TYPE_DCT4, TXFM_TYPE_DCT8, TXFM_TYPE_DCT16, @@ -125,7 +125,7 @@ typedef enum TXFM_TYPE { TXFM_TYPE_IDENTITY32, TXFM_TYPES, TXFM_TYPE_INVALID, -} TXFM_TYPE; +} UENUM1BYTE(TXFM_TYPE); typedef struct TXFM_2D_FLIP_CFG { TX_SIZE tx_size; diff --git a/third_party/aom/av1/common/blockd.h b/third_party/aom/av1/common/blockd.h index a2311c1b00..1cb669e29b 100644 --- a/third_party/aom/av1/common/blockd.h +++ b/third_party/aom/av1/common/blockd.h @@ -38,19 +38,19 @@ extern "C" { #define MAX_DIFFWTD_MASK_BITS 1 // DIFFWTD_MASK_TYPES should not surpass 1 << MAX_DIFFWTD_MASK_BITS -typedef enum ATTRIBUTE_PACKED { +enum { DIFFWTD_38 = 0, DIFFWTD_38_INV, DIFFWTD_MASK_TYPES, -} DIFFWTD_MASK_TYPE; +} UENUM1BYTE(DIFFWTD_MASK_TYPE); -typedef enum ATTRIBUTE_PACKED { +enum { KEY_FRAME = 0, INTER_FRAME = 1, INTRA_ONLY_FRAME = 2, // replaces intra-only S_FRAME = 3, FRAME_TYPES, -} FRAME_TYPE; +} UENUM1BYTE(FRAME_TYPE); static INLINE int is_comp_ref_allowed(BLOCK_SIZE bsize) { return AOMMIN(block_size_wide[bsize], block_size_high[bsize]) >= 8; @@ -156,18 +156,16 @@ static INLINE int is_masked_compound_type(COMPOUND_TYPE type) { modes for the Y blocks to the left and above us; for interframes, there is a single probability table. */ -typedef int8_t MV_REFERENCE_FRAME; - typedef struct { - // Number of base colors for Y (0) and UV (1) - uint8_t palette_size[2]; // Value of base colors for Y, U, and V uint16_t palette_colors[3 * PALETTE_MAX_SIZE]; + // Number of base colors for Y (0) and UV (1) + uint8_t palette_size[2]; } PALETTE_MODE_INFO; typedef struct { - uint8_t use_filter_intra; FILTER_INTRA_MODE filter_intra_mode; + uint8_t use_filter_intra; } FILTER_INTRA_MODE_INFO; static const PREDICTION_MODE fimode_to_intradir[FILTER_INTRA_MODES] = { @@ -192,6 +190,11 @@ typedef struct RD_STATS { int64_t ref_rdcost; int zero_rate; uint8_t invalid_rate; +#if CONFIG_ONE_PASS_SVM + int eob, eob_0, eob_1, eob_2, eob_3; + int64_t rd, rd_0, rd_1, rd_2, rd_3; + int64_t y_sse, sse_0, sse_1, sse_2, sse_3; +#endif #if CONFIG_RD_DEBUG int txb_coeff_cost[MAX_MB_PLANE]; int txb_coeff_cost_map[MAX_MB_PLANE][TXB_COEFF_COST_MAP_SIZE] @@ -202,10 +205,10 @@ typedef struct RD_STATS { // This struct is used to group function args that are commonly // sent together in functions related to interinter compound modes typedef struct { + uint8_t *seg_mask; int wedge_index; int wedge_sign; DIFFWTD_MASK_TYPE mask_type; - uint8_t *seg_mask; COMPOUND_TYPE type; } INTERINTER_COMPOUND_DATA; @@ -213,48 +216,18 @@ typedef struct { #define TXK_TYPE_BUF_LEN 64 // This structure now relates to 4x4 block regions. typedef struct MB_MODE_INFO { - // Common for both INTER and INTRA blocks - BLOCK_SIZE sb_type; - PREDICTION_MODE mode; - TX_SIZE tx_size; - uint8_t inter_tx_size[INTER_TX_SIZE_BUF_LEN]; - int8_t skip; - int8_t skip_mode; - int8_t segment_id; - int8_t seg_id_predicted; // valid only when temporal_update is enabled - - // Only for INTRA blocks - UV_PREDICTION_MODE uv_mode; - PALETTE_MODE_INFO palette_mode_info; - uint8_t use_intrabc; - - // Only for INTER blocks - InterpFilters interp_filters; - MV_REFERENCE_FRAME ref_frame[2]; - - TX_TYPE txk_type[TXK_TYPE_BUF_LEN]; - - FILTER_INTRA_MODE_INFO filter_intra_mode_info; - - // The actual prediction angle is the base angle + (angle_delta * step). - int8_t angle_delta[PLANE_TYPES]; - - // interintra members - INTERINTRA_MODE interintra_mode; - // TODO(debargha): Consolidate these flags - int use_wedge_interintra; - int interintra_wedge_index; - int interintra_wedge_sign; + WarpedMotionParams wm_params; // interinter members INTERINTER_COMPOUND_DATA interinter_comp; - MOTION_MODE motion_mode; - int overlappable_neighbors[2]; + FILTER_INTRA_MODE_INFO filter_intra_mode_info; int_mv mv[2]; - uint8_t ref_mv_idx; - PARTITION_TYPE partition; - /* deringing gain *per-superblock* */ - int8_t cdef_strength; + // Only for INTER blocks + InterpFilters interp_filters; + // TODO(debargha): Consolidate these flags + int interintra_wedge_index; + int interintra_wedge_sign; + int overlappable_neighbors[2]; int current_qindex; int delta_lf_from_base; int delta_lf[FRAME_LF_COUNT]; @@ -264,15 +237,43 @@ typedef struct MB_MODE_INFO { int mi_col; #endif int num_proj_ref; - WarpedMotionParams wm_params; // Index of the alpha Cb and alpha Cr combination int cfl_alpha_idx; // Joint sign of alpha Cb and alpha Cr int cfl_alpha_signs; - int compound_idx; + // Indicate if masked compound is used(1) or not(0). int comp_group_idx; + // If comp_group_idx=0, indicate if dist_wtd_comp(0) or avg_comp(1) is used. + int compound_idx; +#if CONFIG_INSPECTION + int16_t tx_skip[TXK_TYPE_BUF_LEN]; +#endif + // Common for both INTER and INTRA blocks + BLOCK_SIZE sb_type; + PREDICTION_MODE mode; + // Only for INTRA blocks + UV_PREDICTION_MODE uv_mode; + // interintra members + INTERINTRA_MODE interintra_mode; + MOTION_MODE motion_mode; + PARTITION_TYPE partition; + TX_TYPE txk_type[TXK_TYPE_BUF_LEN]; + MV_REFERENCE_FRAME ref_frame[2]; + int8_t use_wedge_interintra; + int8_t skip; + int8_t skip_mode; + uint8_t inter_tx_size[INTER_TX_SIZE_BUF_LEN]; + TX_SIZE tx_size; + int8_t segment_id; + int8_t seg_id_predicted; // valid only when temporal_update is enabled + uint8_t use_intrabc; + // The actual prediction angle is the base angle + (angle_delta * step). + int8_t angle_delta[PLANE_TYPES]; + /* deringing gain *per-superblock* */ + int8_t cdef_strength; + uint8_t ref_mv_idx; } MB_MODE_INFO; static INLINE int is_intrabc_block(const MB_MODE_INFO *mbmi) { @@ -372,7 +373,7 @@ static INLINE void mi_to_pixel_loc(int *pixel_c, int *pixel_r, int mi_col, } #endif -enum ATTRIBUTE_PACKED mv_precision { MV_PRECISION_Q3, MV_PRECISION_Q4 }; +enum { MV_PRECISION_Q3, MV_PRECISION_Q4 } UENUM1BYTE(mv_precision); struct buf_2d { uint8_t *buf; @@ -406,7 +407,7 @@ typedef struct macroblockd_plane { ENTROPY_CONTEXT *above_context; ENTROPY_CONTEXT *left_context; - // The dequantizers below are true dequntizers used only in the + // The dequantizers below are true dequantizers used only in the // dequantization process. They have the same coefficient // shift/scale as TX. int16_t seg_dequant_QTX[MAX_SEGMENTS][2]; @@ -428,13 +429,6 @@ typedef struct macroblockd_plane { #define BLOCK_OFFSET(x, i) \ ((x) + (i) * (1 << (tx_size_wide_log2[0] + tx_size_high_log2[0]))) -typedef struct RefBuffer { - int idx; // frame buf idx - int map_idx; // frame map idx - YV12_BUFFER_CONFIG *buf; - struct scale_factors sf; -} RefBuffer; - typedef struct { DECLARE_ALIGNED(16, InterpKernel, vfilter); DECLARE_ALIGNED(16, InterpKernel, hfilter); @@ -490,11 +484,13 @@ typedef struct cfl_ctx { int is_chroma_reference; } CFL_CTX; -typedef struct jnt_comp_params { - int use_jnt_comp_avg; +typedef struct dist_wtd_comp_params { + int use_dist_wtd_comp_avg; int fwd_offset; int bck_offset; -} JNT_COMP_PARAMS; +} DIST_WTD_COMP_PARAMS; + +struct scale_factors; // Most/all of the pointers are mere pointers to actual arrays are allocated // elsewhere. This is mostly for coding convenience. @@ -522,8 +518,8 @@ typedef struct macroblockd { int mb_to_top_edge; int mb_to_bottom_edge; - /* pointers to reference frames */ - const RefBuffer *block_refs[2]; + /* pointers to reference frame scale factors */ + const struct scale_factors *block_ref_scale_factors[2]; /* pointer to current frame */ const YV12_BUFFER_CONFIG *cur_buf; @@ -592,7 +588,7 @@ typedef struct macroblockd { uint8_t *mc_buf[2]; CFL_CTX cfl; - JNT_COMP_PARAMS jcp_param; + DIST_WTD_COMP_PARAMS jcp_param; uint16_t cb_offset[MAX_MB_PLANE]; uint16_t txb_offset[MAX_MB_PLANE]; @@ -602,7 +598,7 @@ typedef struct macroblockd { uint8_t *tmp_obmc_bufs[2]; } MACROBLOCKD; -static INLINE int get_bitdepth_data_path_index(const MACROBLOCKD *xd) { +static INLINE int is_cur_buf_hbd(const MACROBLOCKD *xd) { return xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH ? 1 : 0; } @@ -646,19 +642,19 @@ static INLINE BLOCK_SIZE get_partition_subsize(BLOCK_SIZE bsize, static TX_TYPE intra_mode_to_tx_type(const MB_MODE_INFO *mbmi, PLANE_TYPE plane_type) { static const TX_TYPE _intra_mode_to_tx_type[INTRA_MODES] = { - DCT_DCT, // DC - ADST_DCT, // V - DCT_ADST, // H - DCT_DCT, // D45 - ADST_ADST, // D135 - ADST_DCT, // D117 - DCT_ADST, // D153 - DCT_ADST, // D207 - ADST_DCT, // D63 - ADST_ADST, // SMOOTH - ADST_DCT, // SMOOTH_V - DCT_ADST, // SMOOTH_H - ADST_ADST, // PAETH + DCT_DCT, // DC_PRED + ADST_DCT, // V_PRED + DCT_ADST, // H_PRED + DCT_DCT, // D45_PRED + ADST_ADST, // D135_PRED + ADST_DCT, // D113_PRED + DCT_ADST, // D157_PRED + DCT_ADST, // D203_PRED + ADST_DCT, // D67_PRED + ADST_ADST, // SMOOTH_PRED + ADST_DCT, // SMOOTH_V_PRED + DCT_ADST, // SMOOTH_H_PRED + ADST_ADST, // PAETH_PRED }; const PREDICTION_MODE mode = (plane_type == PLANE_TYPE_Y) ? mbmi->mode : get_uv_mode(mbmi->uv_mode); @@ -777,11 +773,13 @@ static INLINE int av1_raster_order_to_block_index(TX_SIZE tx_size, static INLINE TX_TYPE get_default_tx_type(PLANE_TYPE plane_type, const MACROBLOCKD *xd, - TX_SIZE tx_size) { + TX_SIZE tx_size, + int is_screen_content_type) { const MB_MODE_INFO *const mbmi = xd->mi[0]; if (is_inter_block(mbmi) || plane_type != PLANE_TYPE_Y || - xd->lossless[mbmi->segment_id] || tx_size >= TX_32X32) + xd->lossless[mbmi->segment_id] || tx_size >= TX_32X32 || + is_screen_content_type) return DCT_DCT; return intra_mode_to_tx_type(mbmi, plane_type); @@ -1045,7 +1043,8 @@ motion_mode_allowed(const WarpedMotionParams *gm_params, const MACROBLOCKD *xd, if (!check_num_overlappable_neighbors(mbmi)) return SIMPLE_TRANSLATION; assert(!has_second_ref(mbmi)); if (mbmi->num_proj_ref >= 1 && - (allow_warped_motion && !av1_is_scaled(&(xd->block_refs[0]->sf)))) { + (allow_warped_motion && + !av1_is_scaled(xd->block_ref_scale_factors[0]))) { if (xd->cur_frame_force_integer_mv) { return OBMC_CAUSAL; } diff --git a/third_party/aom/av1/common/cdef.c b/third_party/aom/av1/common/cdef.c index e9e2b0e42c..556deded96 100644 --- a/third_party/aom/av1/common/cdef.c +++ b/third_party/aom/av1/common/cdef.c @@ -107,7 +107,7 @@ void copy_rect8_16bit_to_16bit_c(uint16_t *dst, int dstride, } } -static void copy_sb8_16(AOM_UNUSED AV1_COMMON *cm, uint16_t *dst, int dstride, +static void copy_sb8_16(AV1_COMMON *cm, uint16_t *dst, int dstride, const uint8_t *src, int src_voffset, int src_hoffset, int sstride, int vsize, int hsize) { if (cm->seq_params.use_highbitdepth) { @@ -140,6 +140,7 @@ static INLINE void copy_rect(uint16_t *dst, int dstride, const uint16_t *src, void av1_cdef_frame(YV12_BUFFER_CONFIG *frame, AV1_COMMON *cm, MACROBLOCKD *xd) { + const CdefInfo *const cdef_info = &cm->cdef_info; const int num_planes = av1_num_planes(cm); DECLARE_ALIGNED(16, uint16_t, src[CDEF_INBUF_SIZE]); uint16_t *linebuf[3]; @@ -231,13 +232,15 @@ void av1_cdef_frame(YV12_BUFFER_CONFIG *frame, AV1_COMMON *cm, cm->mi_grid_visible[MI_SIZE_64X64 * fbr * cm->mi_stride + MI_SIZE_64X64 * fbc] ->cdef_strength; - level = cm->cdef_strengths[mbmi_cdef_strength] / CDEF_SEC_STRENGTHS; + level = + cdef_info->cdef_strengths[mbmi_cdef_strength] / CDEF_SEC_STRENGTHS; sec_strength = - cm->cdef_strengths[mbmi_cdef_strength] % CDEF_SEC_STRENGTHS; + cdef_info->cdef_strengths[mbmi_cdef_strength] % CDEF_SEC_STRENGTHS; sec_strength += sec_strength == 3; - uv_level = cm->cdef_uv_strengths[mbmi_cdef_strength] / CDEF_SEC_STRENGTHS; + uv_level = + cdef_info->cdef_uv_strengths[mbmi_cdef_strength] / CDEF_SEC_STRENGTHS; uv_sec_strength = - cm->cdef_uv_strengths[mbmi_cdef_strength] % CDEF_SEC_STRENGTHS; + cdef_info->cdef_uv_strengths[mbmi_cdef_strength] % CDEF_SEC_STRENGTHS; uv_sec_strength += uv_sec_strength == 3; if ((level == 0 && sec_strength == 0 && uv_level == 0 && uv_sec_strength == 0) || @@ -252,8 +255,8 @@ void av1_cdef_frame(YV12_BUFFER_CONFIG *frame, AV1_COMMON *cm, for (int pli = 0; pli < num_planes; pli++) { int coffset; int rend, cend; - int pri_damping = cm->cdef_pri_damping; - int sec_damping = cm->cdef_sec_damping; + int pri_damping = cdef_info->cdef_pri_damping; + int sec_damping = cdef_info->cdef_sec_damping; int hsize = nhb << mi_wide_l2[pli]; int vsize = nvb << mi_high_l2[pli]; diff --git a/third_party/aom/av1/common/cdef_block.c b/third_party/aom/av1/common/cdef_block.c index df1de89be3..845df377b7 100644 --- a/third_party/aom/av1/common/cdef_block.c +++ b/third_party/aom/av1/common/cdef_block.c @@ -114,7 +114,7 @@ const int cdef_sec_taps[2][2] = { { 2, 1 }, { 2, 1 } }; void cdef_filter_block_c(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, - AOM_UNUSED int max_unused, int coeff_shift) { + int coeff_shift) { int i, j, k; const int s = CDEF_BSTRIDE; const int *pri_taps = cdef_pri_taps[(pri_strength >> coeff_shift) & 1]; @@ -237,12 +237,11 @@ void cdef_filter_fb(uint8_t *dst8, uint16_t *dst16, int dstride, uint16_t *in, by = dlist[bi].by; bx = dlist[bi].bx; if (dst8) - cdef_filter_block(&dst8[(by << bsizey) * dstride + (bx << bsizex)], NULL, - dstride, - &in[(by * CDEF_BSTRIDE << bsizey) + (bx << bsizex)], - (pli ? t : adjust_strength(t, var[by][bx])), s, - t ? dir[by][bx] : 0, pri_damping, sec_damping, bsize, - (256 << coeff_shift) - 1, coeff_shift); + cdef_filter_block( + &dst8[(by << bsizey) * dstride + (bx << bsizex)], NULL, dstride, + &in[(by * CDEF_BSTRIDE << bsizey) + (bx << bsizex)], + (pli ? t : adjust_strength(t, var[by][bx])), s, t ? dir[by][bx] : 0, + pri_damping, sec_damping, bsize, coeff_shift); else cdef_filter_block( NULL, @@ -251,7 +250,6 @@ void cdef_filter_fb(uint8_t *dst8, uint16_t *dst16, int dstride, uint16_t *in, dirinit ? 1 << bsizex : dstride, &in[(by * CDEF_BSTRIDE << bsizey) + (bx << bsizex)], (pli ? t : adjust_strength(t, var[by][bx])), s, t ? dir[by][bx] : 0, - pri_damping, sec_damping, bsize, (256 << coeff_shift) - 1, - coeff_shift); + pri_damping, sec_damping, bsize, coeff_shift); } } diff --git a/third_party/aom/av1/common/cdef_block.h b/third_party/aom/av1/common/cdef_block.h index 6b4452cd64..0e921e0005 100644 --- a/third_party/aom/av1/common/cdef_block.h +++ b/third_party/aom/av1/common/cdef_block.h @@ -45,7 +45,7 @@ typedef void (*cdef_filter_block_func)(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, - int sec_damping, int bsize, int max, + int sec_damping, int bsize, int coeff_shift); void copy_cdef_16bit_to_16bit(uint16_t *dst, int dstride, uint16_t *src, cdef_list *dlist, int cdef_count, int bsize); diff --git a/third_party/aom/av1/common/cdef_block_simd.h b/third_party/aom/av1/common/cdef_block_simd.h index 14587a023a..a3368ec40b 100644 --- a/third_party/aom/av1/common/cdef_block_simd.h +++ b/third_party/aom/av1/common/cdef_block_simd.h @@ -226,7 +226,6 @@ void SIMD_FUNC(cdef_filter_block_4x4_8)(uint8_t *dst, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, - AOM_UNUSED int max_unused, int coeff_shift) { v128 p0, p1, p2, p3; v256 sum, row, tap, res; @@ -393,7 +392,6 @@ void SIMD_FUNC(cdef_filter_block_8x8_8)(uint8_t *dst, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, - AOM_UNUSED int max_unused, int coeff_shift) { int i; v128 p0, p1, p2, p3; @@ -541,7 +539,6 @@ void SIMD_FUNC(cdef_filter_block_4x4_16)(uint16_t *dst, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, - AOM_UNUSED int max_unused, int coeff_shift) { int i; v256 p0, p1, p2, p3, sum, row, res; @@ -699,7 +696,6 @@ void SIMD_FUNC(cdef_filter_block_8x8_16)(uint16_t *dst, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, - AOM_UNUSED int max_unused, int coeff_shift) { int i; v256 sum, p0, p1, p2, p3, row, res; @@ -833,56 +829,55 @@ void SIMD_FUNC(cdef_filter_block_8x8_16)(uint16_t *dst, int dstride, void SIMD_FUNC(cdef_filter_block)(uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, - int sec_damping, int bsize, int max, - int coeff_shift) { + int sec_damping, int bsize, int coeff_shift) { if (dst8) { if (bsize == BLOCK_8X8) { SIMD_FUNC(cdef_filter_block_8x8_8) (dst8, dstride, in, pri_strength, sec_strength, dir, pri_damping, - sec_damping, max, coeff_shift); + sec_damping, coeff_shift); } else if (bsize == BLOCK_4X8) { SIMD_FUNC(cdef_filter_block_4x4_8) (dst8, dstride, in, pri_strength, sec_strength, dir, pri_damping, - sec_damping, max, coeff_shift); + sec_damping, coeff_shift); SIMD_FUNC(cdef_filter_block_4x4_8) (dst8 + 4 * dstride, dstride, in + 4 * CDEF_BSTRIDE, pri_strength, - sec_strength, dir, pri_damping, sec_damping, max, coeff_shift); + sec_strength, dir, pri_damping, sec_damping, coeff_shift); } else if (bsize == BLOCK_8X4) { SIMD_FUNC(cdef_filter_block_4x4_8) (dst8, dstride, in, pri_strength, sec_strength, dir, pri_damping, - sec_damping, max, coeff_shift); + sec_damping, coeff_shift); SIMD_FUNC(cdef_filter_block_4x4_8) (dst8 + 4, dstride, in + 4, pri_strength, sec_strength, dir, pri_damping, - sec_damping, max, coeff_shift); + sec_damping, coeff_shift); } else { SIMD_FUNC(cdef_filter_block_4x4_8) (dst8, dstride, in, pri_strength, sec_strength, dir, pri_damping, - sec_damping, max, coeff_shift); + sec_damping, coeff_shift); } } else { if (bsize == BLOCK_8X8) { SIMD_FUNC(cdef_filter_block_8x8_16) (dst16, dstride, in, pri_strength, sec_strength, dir, pri_damping, - sec_damping, max, coeff_shift); + sec_damping, coeff_shift); } else if (bsize == BLOCK_4X8) { SIMD_FUNC(cdef_filter_block_4x4_16) (dst16, dstride, in, pri_strength, sec_strength, dir, pri_damping, - sec_damping, max, coeff_shift); + sec_damping, coeff_shift); SIMD_FUNC(cdef_filter_block_4x4_16) (dst16 + 4 * dstride, dstride, in + 4 * CDEF_BSTRIDE, pri_strength, - sec_strength, dir, pri_damping, sec_damping, max, coeff_shift); + sec_strength, dir, pri_damping, sec_damping, coeff_shift); } else if (bsize == BLOCK_8X4) { SIMD_FUNC(cdef_filter_block_4x4_16) (dst16, dstride, in, pri_strength, sec_strength, dir, pri_damping, - sec_damping, max, coeff_shift); + sec_damping, coeff_shift); SIMD_FUNC(cdef_filter_block_4x4_16) (dst16 + 4, dstride, in + 4, pri_strength, sec_strength, dir, pri_damping, - sec_damping, max, coeff_shift); + sec_damping, coeff_shift); } else { assert(bsize == BLOCK_4X4); SIMD_FUNC(cdef_filter_block_4x4_16) (dst16, dstride, in, pri_strength, sec_strength, dir, pri_damping, - sec_damping, max, coeff_shift); + sec_damping, coeff_shift); } } } diff --git a/third_party/aom/av1/common/cfl.c b/third_party/aom/av1/common/cfl.c index ccc59b4eb7..65e18e89da 100644 --- a/third_party/aom/av1/common/cfl.c +++ b/third_party/aom/av1/common/cfl.c @@ -37,7 +37,7 @@ void cfl_store_dc_pred(MACROBLOCKD *const xd, const uint8_t *input, assert(pred_plane < CFL_PRED_PLANES); assert(width <= CFL_BUF_LINE); - if (get_bitdepth_data_path_index(xd)) { + if (is_cur_buf_hbd(xd)) { uint16_t *const input_16 = CONVERT_TO_SHORTPTR(input); memcpy(xd->cfl.dc_pred_cache[pred_plane], input_16, width << 1); return; @@ -69,7 +69,7 @@ void cfl_load_dc_pred(MACROBLOCKD *const xd, uint8_t *dst, int dst_stride, assert(pred_plane < CFL_PRED_PLANES); assert(width <= CFL_BUF_LINE); assert(height <= CFL_BUF_LINE); - if (get_bitdepth_data_path_index(xd)) { + if (is_cur_buf_hbd(xd)) { uint16_t *dst_16 = CONVERT_TO_SHORTPTR(dst); cfl_load_dc_pred_hbd(xd->cfl.dc_pred_cache[pred_plane], dst_16, dst_stride, width, height); @@ -158,16 +158,6 @@ static INLINE void cfl_predict_lbd_c(const int16_t *ac_buf_q3, uint8_t *dst, } } -// Null function used for invalid tx_sizes -void cfl_predict_lbd_null(const int16_t *ac_buf_q3, uint8_t *dst, - int dst_stride, int alpha_q3) { - (void)ac_buf_q3; - (void)dst; - (void)dst_stride; - (void)alpha_q3; - assert(0); -} - CFL_PREDICT_FN(c, lbd) void cfl_predict_hbd_c(const int16_t *ac_buf_q3, uint16_t *dst, int dst_stride, @@ -182,17 +172,6 @@ void cfl_predict_hbd_c(const int16_t *ac_buf_q3, uint16_t *dst, int dst_stride, } } -// Null function used for invalid tx_sizes -void cfl_predict_hbd_null(const int16_t *ac_buf_q3, uint16_t *dst, - int dst_stride, int alpha_q3, int bd) { - (void)ac_buf_q3; - (void)dst; - (void)dst_stride; - (void)alpha_q3; - (void)bd; - assert(0); -} - CFL_PREDICT_FN(c, hbd) static void cfl_compute_parameters(MACROBLOCKD *const xd, TX_SIZE tx_size) { @@ -217,7 +196,7 @@ void cfl_predict_block(MACROBLOCKD *const xd, uint8_t *dst, int dst_stride, cfl_idx_to_alpha(mbmi->cfl_alpha_idx, mbmi->cfl_alpha_signs, plane - 1); assert((tx_size_high[tx_size] - 1) * CFL_BUF_LINE + tx_size_wide[tx_size] <= CFL_BUF_SQUARE); - if (get_bitdepth_data_path_index(xd)) { + if (is_cur_buf_hbd(xd)) { uint16_t *dst_16 = CONVERT_TO_SHORTPTR(dst); get_predict_hbd_fn(tx_size)(cfl->ac_buf_q3, dst_16, dst_stride, alpha_q3, xd->bd); @@ -226,24 +205,6 @@ void cfl_predict_block(MACROBLOCKD *const xd, uint8_t *dst, int dst_stride, get_predict_lbd_fn(tx_size)(cfl->ac_buf_q3, dst, dst_stride, alpha_q3); } -// Null function used for invalid tx_sizes -void cfl_subsample_lbd_null(const uint8_t *input, int input_stride, - uint16_t *output_q3) { - (void)input; - (void)input_stride; - (void)output_q3; - assert(0); -} - -// Null function used for invalid tx_sizes -void cfl_subsample_hbd_null(const uint16_t *input, int input_stride, - uint16_t *output_q3) { - (void)input; - (void)input_stride; - (void)output_q3; - assert(0); -} - static void cfl_luma_subsampling_420_lbd_c(const uint8_t *input, int input_stride, uint16_t *output_q3, int width, @@ -427,8 +388,7 @@ void cfl_store_tx(MACROBLOCKD *const xd, int row, int col, TX_SIZE tx_size, assert(!((row & 1) && tx_size_high[tx_size] != 4)); sub8x8_adjust_offset(cfl, &row, &col); } - cfl_store(cfl, dst, pd->dst.stride, row, col, tx_size, - get_bitdepth_data_path_index(xd)); + cfl_store(cfl, dst, pd->dst.stride, row, col, tx_size, is_cur_buf_hbd(xd)); } void cfl_store_block(MACROBLOCKD *const xd, BLOCK_SIZE bsize, TX_SIZE tx_size) { @@ -444,5 +404,5 @@ void cfl_store_block(MACROBLOCKD *const xd, BLOCK_SIZE bsize, TX_SIZE tx_size) { const int height = max_intra_block_height(xd, bsize, AOM_PLANE_Y, tx_size); tx_size = get_tx_size(width, height); cfl_store(cfl, pd->dst.buf, pd->dst.stride, row, col, tx_size, - get_bitdepth_data_path_index(xd)); + is_cur_buf_hbd(xd)); } diff --git a/third_party/aom/av1/common/cfl.h b/third_party/aom/av1/common/cfl.h index d627891bf5..3b91d85d1d 100644 --- a/third_party/aom/av1/common/cfl.h +++ b/third_party/aom/av1/common/cfl.h @@ -80,14 +80,6 @@ void cfl_store_dc_pred(MACROBLOCKD *const xd, const uint8_t *input, void cfl_load_dc_pred(MACROBLOCKD *const xd, uint8_t *dst, int dst_stride, TX_SIZE tx_size, CFL_PRED_TYPE pred_plane); -// Null function used for invalid tx_sizes -void cfl_subsample_lbd_null(const uint8_t *input, int input_stride, - uint16_t *output_q3); - -// Null function used for invalid tx_sizes -void cfl_subsample_hbd_null(const uint16_t *input, int input_stride, - uint16_t *output_q3); - // Allows the CFL_SUBSAMPLE function to switch types depending on the bitdepth. #define CFL_lbd_TYPE uint8_t *cfl_type #define CFL_hbd_TYPE uint16_t *cfl_type @@ -133,21 +125,21 @@ void cfl_subsample_hbd_null(const uint16_t *input, int input_stride, subsample_##bd##_##sub##_8x8_##arch, /* 8x8 */ \ subsample_##bd##_##sub##_16x16_##arch, /* 16x16 */ \ subsample_##bd##_##sub##_32x32_##arch, /* 32x32 */ \ - cfl_subsample_##bd##_null, /* 64x64 (invalid CFL size) */ \ + NULL, /* 64x64 (invalid CFL size) */ \ subsample_##bd##_##sub##_4x8_##arch, /* 4x8 */ \ subsample_##bd##_##sub##_8x4_##arch, /* 8x4 */ \ subsample_##bd##_##sub##_8x16_##arch, /* 8x16 */ \ subsample_##bd##_##sub##_16x8_##arch, /* 16x8 */ \ subsample_##bd##_##sub##_16x32_##arch, /* 16x32 */ \ subsample_##bd##_##sub##_32x16_##arch, /* 32x16 */ \ - cfl_subsample_##bd##_null, /* 32x64 (invalid CFL size) */ \ - cfl_subsample_##bd##_null, /* 64x32 (invalid CFL size) */ \ + NULL, /* 32x64 (invalid CFL size) */ \ + NULL, /* 64x32 (invalid CFL size) */ \ subsample_##bd##_##sub##_4x16_##arch, /* 4x16 */ \ subsample_##bd##_##sub##_16x4_##arch, /* 16x4 */ \ subsample_##bd##_##sub##_8x32_##arch, /* 8x32 */ \ subsample_##bd##_##sub##_32x8_##arch, /* 32x8 */ \ - cfl_subsample_##bd##_null, /* 16x64 (invalid CFL size) */ \ - cfl_subsample_##bd##_null, /* 64x16 (invalid CFL size) */ \ + NULL, /* 16x64 (invalid CFL size) */ \ + NULL, /* 64x16 (invalid CFL size) */ \ }; // The RTCD script does not support passing in an array, so we wrap it in this @@ -160,14 +152,6 @@ void cfl_subsample_hbd_null(const uint16_t *input, int input_stride, CFL_SUBSAMPLE_FUNCTIONS(arch, 422, hbd) \ CFL_SUBSAMPLE_FUNCTIONS(arch, 444, hbd) -// Null function used for invalid tx_sizes -static INLINE void cfl_subtract_average_null(const uint16_t *src, - int16_t *dst) { - (void)dst; - (void)src; - assert(0); -} - // Declare a size-specific wrapper for the size-generic function. The compiler // will inline the size generic function in here, the advantage is that the size // will be constant allowing for loop unrolling and other constant propagated @@ -201,21 +185,21 @@ static INLINE void cfl_subtract_average_null(const uint16_t *src, subtract_average_8x8_##arch, /* 8x8 */ \ subtract_average_16x16_##arch, /* 16x16 */ \ subtract_average_32x32_##arch, /* 32x32 */ \ - cfl_subtract_average_null, /* 64x64 (invalid CFL size) */ \ + NULL, /* 64x64 (invalid CFL size) */ \ subtract_average_4x8_##arch, /* 4x8 */ \ subtract_average_8x4_##arch, /* 8x4 */ \ subtract_average_8x16_##arch, /* 8x16 */ \ subtract_average_16x8_##arch, /* 16x8 */ \ subtract_average_16x32_##arch, /* 16x32 */ \ subtract_average_32x16_##arch, /* 32x16 */ \ - cfl_subtract_average_null, /* 32x64 (invalid CFL size) */ \ - cfl_subtract_average_null, /* 64x32 (invalid CFL size) */ \ + NULL, /* 32x64 (invalid CFL size) */ \ + NULL, /* 64x32 (invalid CFL size) */ \ subtract_average_4x16_##arch, /* 4x16 (invalid CFL size) */ \ subtract_average_16x4_##arch, /* 16x4 (invalid CFL size) */ \ subtract_average_8x32_##arch, /* 8x32 (invalid CFL size) */ \ subtract_average_32x8_##arch, /* 32x8 (invalid CFL size) */ \ - cfl_subtract_average_null, /* 16x64 (invalid CFL size) */ \ - cfl_subtract_average_null, /* 64x16 (invalid CFL size) */ \ + NULL, /* 16x64 (invalid CFL size) */ \ + NULL, /* 64x16 (invalid CFL size) */ \ }; \ /* Modulo TX_SIZES_ALL to ensure that an attacker won't be able to */ \ /* index the function pointer array out of bounds. */ \ @@ -249,14 +233,6 @@ void subtract_average_4x16_c(const uint16_t *src, int16_t *dst); #define CFL_PREDICT_X(arch, width, height, bd) \ CFL_PREDICT_##bd(arch, width, height) -// Null function used for invalid tx_sizes -void cfl_predict_lbd_null(const int16_t *pred_buf_q3, uint8_t *dst, - int dst_stride, int alpha_q3); - -// Null function used for invalid tx_sizes -void cfl_predict_hbd_null(const int16_t *pred_buf_q3, uint16_t *dst, - int dst_stride, int alpha_q3, int bd); - #define CFL_PREDICT_FN(arch, bd) \ CFL_PREDICT_X(arch, 4, 4, bd) \ CFL_PREDICT_X(arch, 4, 8, bd) \ @@ -278,21 +254,21 @@ void cfl_predict_hbd_null(const int16_t *pred_buf_q3, uint16_t *dst, predict_##bd##_8x8_##arch, /* 8x8 */ \ predict_##bd##_16x16_##arch, /* 16x16 */ \ predict_##bd##_32x32_##arch, /* 32x32 */ \ - cfl_predict_##bd##_null, /* 64x64 (invalid CFL size) */ \ + NULL, /* 64x64 (invalid CFL size) */ \ predict_##bd##_4x8_##arch, /* 4x8 */ \ predict_##bd##_8x4_##arch, /* 8x4 */ \ predict_##bd##_8x16_##arch, /* 8x16 */ \ predict_##bd##_16x8_##arch, /* 16x8 */ \ predict_##bd##_16x32_##arch, /* 16x32 */ \ predict_##bd##_32x16_##arch, /* 32x16 */ \ - cfl_predict_##bd##_null, /* 32x64 (invalid CFL size) */ \ - cfl_predict_##bd##_null, /* 64x32 (invalid CFL size) */ \ + NULL, /* 32x64 (invalid CFL size) */ \ + NULL, /* 64x32 (invalid CFL size) */ \ predict_##bd##_4x16_##arch, /* 4x16 */ \ predict_##bd##_16x4_##arch, /* 16x4 */ \ predict_##bd##_8x32_##arch, /* 8x32 */ \ predict_##bd##_32x8_##arch, /* 32x8 */ \ - cfl_predict_##bd##_null, /* 16x64 (invalid CFL size) */ \ - cfl_predict_##bd##_null, /* 64x16 (invalid CFL size) */ \ + NULL, /* 16x64 (invalid CFL size) */ \ + NULL, /* 64x16 (invalid CFL size) */ \ }; \ /* Modulo TX_SIZES_ALL to ensure that an attacker won't be able to */ \ /* index the function pointer array out of bounds. */ \ diff --git a/third_party/aom/av1/common/convolve.c b/third_party/aom/av1/common/convolve.c index 1f11126fc3..5a55ece671 100644 --- a/third_party/aom/av1/common/convolve.c +++ b/third_party/aom/av1/common/convolve.c @@ -73,6 +73,45 @@ void av1_highbd_convolve_horiz_rs_c(const uint16_t *src, int src_stride, } } +void av1_convolve_2d_sobel_y_c(const uint8_t *src, int src_stride, double *dst, + int dst_stride, int w, int h, int dir, + double norm) { + int16_t im_block[(MAX_SB_SIZE + MAX_FILTER_TAP - 1) * MAX_SB_SIZE]; + DECLARE_ALIGNED(256, static const int16_t, sobel_a[3]) = { 1, 0, -1 }; + DECLARE_ALIGNED(256, static const int16_t, sobel_b[3]) = { 1, 2, 1 }; + const int taps = 3; + int im_h = h + taps - 1; + int im_stride = w; + const int fo_vert = 1; + const int fo_horiz = 1; + + // horizontal filter + const uint8_t *src_horiz = src - fo_vert * src_stride; + const int16_t *x_filter = dir ? sobel_a : sobel_b; + for (int y = 0; y < im_h; ++y) { + for (int x = 0; x < w; ++x) { + int16_t sum = 0; + for (int k = 0; k < taps; ++k) { + sum += x_filter[k] * src_horiz[y * src_stride + x - fo_horiz + k]; + } + im_block[y * im_stride + x] = sum; + } + } + + // vertical filter + int16_t *src_vert = im_block + fo_vert * im_stride; + const int16_t *y_filter = dir ? sobel_b : sobel_a; + for (int y = 0; y < h; ++y) { + for (int x = 0; x < w; ++x) { + int16_t sum = 0; + for (int k = 0; k < taps; ++k) { + sum += y_filter[k] * src_vert[(y - fo_vert + k) * im_stride + x]; + } + dst[y * dst_stride + x] = sum * norm; + } + } +} + void av1_convolve_2d_sr_c(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const InterpFilterParams *filter_params_x, @@ -199,16 +238,16 @@ void av1_convolve_2d_copy_sr_c(const uint8_t *src, int src_stride, uint8_t *dst, (void)conv_params; for (int y = 0; y < h; ++y) { - memcpy(dst + y * dst_stride, src + y * src_stride, w * sizeof(src[0])); + memmove(dst + y * dst_stride, src + y * src_stride, w * sizeof(src[0])); } } -void av1_jnt_convolve_2d_c(const uint8_t *src, int src_stride, uint8_t *dst8, - int dst8_stride, int w, int h, - const InterpFilterParams *filter_params_x, - const InterpFilterParams *filter_params_y, - const int subpel_x_q4, const int subpel_y_q4, - ConvolveParams *conv_params) { +void av1_dist_wtd_convolve_2d_c(const uint8_t *src, int src_stride, + uint8_t *dst8, int dst8_stride, int w, int h, + const InterpFilterParams *filter_params_x, + const InterpFilterParams *filter_params_y, + const int subpel_x_q4, const int subpel_y_q4, + ConvolveParams *conv_params) { CONV_BUF_TYPE *dst = conv_params->dst; int dst_stride = conv_params->dst_stride; int16_t im_block[(MAX_SB_SIZE + MAX_FILTER_TAP - 1) * MAX_SB_SIZE]; @@ -251,7 +290,7 @@ void av1_jnt_convolve_2d_c(const uint8_t *src, int src_stride, uint8_t *dst8, CONV_BUF_TYPE res = ROUND_POWER_OF_TWO(sum, conv_params->round_1); if (conv_params->do_average) { int32_t tmp = dst[y * dst_stride + x]; - if (conv_params->use_jnt_comp_avg) { + if (conv_params->use_dist_wtd_comp_avg) { tmp = tmp * conv_params->fwd_offset + res * conv_params->bck_offset; tmp = tmp >> DIST_PRECISION_BITS; } else { @@ -269,12 +308,12 @@ void av1_jnt_convolve_2d_c(const uint8_t *src, int src_stride, uint8_t *dst8, } } -void av1_jnt_convolve_y_c(const uint8_t *src, int src_stride, uint8_t *dst8, - int dst8_stride, int w, int h, - const InterpFilterParams *filter_params_x, - const InterpFilterParams *filter_params_y, - const int subpel_x_q4, const int subpel_y_q4, - ConvolveParams *conv_params) { +void av1_dist_wtd_convolve_y_c(const uint8_t *src, int src_stride, + uint8_t *dst8, int dst8_stride, int w, int h, + const InterpFilterParams *filter_params_x, + const InterpFilterParams *filter_params_y, + const int subpel_x_q4, const int subpel_y_q4, + ConvolveParams *conv_params) { CONV_BUF_TYPE *dst = conv_params->dst; int dst_stride = conv_params->dst_stride; const int fo_vert = filter_params_y->taps / 2 - 1; @@ -302,7 +341,7 @@ void av1_jnt_convolve_y_c(const uint8_t *src, int src_stride, uint8_t *dst8, if (conv_params->do_average) { int32_t tmp = dst[y * dst_stride + x]; - if (conv_params->use_jnt_comp_avg) { + if (conv_params->use_dist_wtd_comp_avg) { tmp = tmp * conv_params->fwd_offset + res * conv_params->bck_offset; tmp = tmp >> DIST_PRECISION_BITS; } else { @@ -319,12 +358,12 @@ void av1_jnt_convolve_y_c(const uint8_t *src, int src_stride, uint8_t *dst8, } } -void av1_jnt_convolve_x_c(const uint8_t *src, int src_stride, uint8_t *dst8, - int dst8_stride, int w, int h, - const InterpFilterParams *filter_params_x, - const InterpFilterParams *filter_params_y, - const int subpel_x_q4, const int subpel_y_q4, - ConvolveParams *conv_params) { +void av1_dist_wtd_convolve_x_c(const uint8_t *src, int src_stride, + uint8_t *dst8, int dst8_stride, int w, int h, + const InterpFilterParams *filter_params_x, + const InterpFilterParams *filter_params_y, + const int subpel_x_q4, const int subpel_y_q4, + ConvolveParams *conv_params) { CONV_BUF_TYPE *dst = conv_params->dst; int dst_stride = conv_params->dst_stride; const int fo_horiz = filter_params_x->taps / 2 - 1; @@ -352,7 +391,7 @@ void av1_jnt_convolve_x_c(const uint8_t *src, int src_stride, uint8_t *dst8, if (conv_params->do_average) { int32_t tmp = dst[y * dst_stride + x]; - if (conv_params->use_jnt_comp_avg) { + if (conv_params->use_dist_wtd_comp_avg) { tmp = tmp * conv_params->fwd_offset + res * conv_params->bck_offset; tmp = tmp >> DIST_PRECISION_BITS; } else { @@ -369,12 +408,11 @@ void av1_jnt_convolve_x_c(const uint8_t *src, int src_stride, uint8_t *dst8, } } -void av1_jnt_convolve_2d_copy_c(const uint8_t *src, int src_stride, - uint8_t *dst8, int dst8_stride, int w, int h, - const InterpFilterParams *filter_params_x, - const InterpFilterParams *filter_params_y, - const int subpel_x_q4, const int subpel_y_q4, - ConvolveParams *conv_params) { +void av1_dist_wtd_convolve_2d_copy_c( + const uint8_t *src, int src_stride, uint8_t *dst8, int dst8_stride, int w, + int h, const InterpFilterParams *filter_params_x, + const InterpFilterParams *filter_params_y, const int subpel_x_q4, + const int subpel_y_q4, ConvolveParams *conv_params) { CONV_BUF_TYPE *dst = conv_params->dst; int dst_stride = conv_params->dst_stride; const int bits = @@ -395,7 +433,7 @@ void av1_jnt_convolve_2d_copy_c(const uint8_t *src, int src_stride, if (conv_params->do_average) { int32_t tmp = dst[y * dst_stride + x]; - if (conv_params->use_jnt_comp_avg) { + if (conv_params->use_dist_wtd_comp_avg) { tmp = tmp * conv_params->fwd_offset + res * conv_params->bck_offset; tmp = tmp >> DIST_PRECISION_BITS; } else { @@ -472,7 +510,7 @@ void av1_convolve_2d_scale_c(const uint8_t *src, int src_stride, uint8_t *dst8, if (conv_params->is_compound) { if (conv_params->do_average) { int32_t tmp = dst16[y * dst16_stride + x]; - if (conv_params->use_jnt_comp_avg) { + if (conv_params->use_dist_wtd_comp_avg) { tmp = tmp * conv_params->fwd_offset + res * conv_params->bck_offset; tmp = tmp >> DIST_PRECISION_BITS; } else { @@ -593,7 +631,7 @@ void av1_highbd_convolve_2d_copy_sr_c( (void)bd; for (int y = 0; y < h; ++y) { - memcpy(dst + y * dst_stride, src + y * src_stride, w * sizeof(src[0])); + memmove(dst + y * dst_stride, src + y * src_stride, w * sizeof(src[0])); } } @@ -709,13 +747,11 @@ void av1_highbd_convolve_2d_sr_c(const uint16_t *src, int src_stride, } } -void av1_highbd_jnt_convolve_2d_c(const uint16_t *src, int src_stride, - uint16_t *dst16, int dst16_stride, int w, - int h, - const InterpFilterParams *filter_params_x, - const InterpFilterParams *filter_params_y, - const int subpel_x_q4, const int subpel_y_q4, - ConvolveParams *conv_params, int bd) { +void av1_highbd_dist_wtd_convolve_2d_c( + const uint16_t *src, int src_stride, uint16_t *dst16, int dst16_stride, + int w, int h, const InterpFilterParams *filter_params_x, + const InterpFilterParams *filter_params_y, const int subpel_x_q4, + const int subpel_y_q4, ConvolveParams *conv_params, int bd) { int x, y, k; int16_t im_block[(MAX_SB_SIZE + MAX_FILTER_TAP - 1) * MAX_SB_SIZE]; CONV_BUF_TYPE *dst = conv_params->dst; @@ -760,7 +796,7 @@ void av1_highbd_jnt_convolve_2d_c(const uint16_t *src, int src_stride, CONV_BUF_TYPE res = ROUND_POWER_OF_TWO(sum, conv_params->round_1); if (conv_params->do_average) { int32_t tmp = dst[y * dst_stride + x]; - if (conv_params->use_jnt_comp_avg) { + if (conv_params->use_dist_wtd_comp_avg) { tmp = tmp * conv_params->fwd_offset + res * conv_params->bck_offset; tmp = tmp >> DIST_PRECISION_BITS; } else { @@ -778,13 +814,11 @@ void av1_highbd_jnt_convolve_2d_c(const uint16_t *src, int src_stride, } } -void av1_highbd_jnt_convolve_x_c(const uint16_t *src, int src_stride, - uint16_t *dst16, int dst16_stride, int w, - int h, - const InterpFilterParams *filter_params_x, - const InterpFilterParams *filter_params_y, - const int subpel_x_q4, const int subpel_y_q4, - ConvolveParams *conv_params, int bd) { +void av1_highbd_dist_wtd_convolve_x_c( + const uint16_t *src, int src_stride, uint16_t *dst16, int dst16_stride, + int w, int h, const InterpFilterParams *filter_params_x, + const InterpFilterParams *filter_params_y, const int subpel_x_q4, + const int subpel_y_q4, ConvolveParams *conv_params, int bd) { CONV_BUF_TYPE *dst = conv_params->dst; int dst_stride = conv_params->dst_stride; const int fo_horiz = filter_params_x->taps / 2 - 1; @@ -812,7 +846,7 @@ void av1_highbd_jnt_convolve_x_c(const uint16_t *src, int src_stride, if (conv_params->do_average) { int32_t tmp = dst[y * dst_stride + x]; - if (conv_params->use_jnt_comp_avg) { + if (conv_params->use_dist_wtd_comp_avg) { tmp = tmp * conv_params->fwd_offset + res * conv_params->bck_offset; tmp = tmp >> DIST_PRECISION_BITS; } else { @@ -829,13 +863,11 @@ void av1_highbd_jnt_convolve_x_c(const uint16_t *src, int src_stride, } } -void av1_highbd_jnt_convolve_y_c(const uint16_t *src, int src_stride, - uint16_t *dst16, int dst16_stride, int w, - int h, - const InterpFilterParams *filter_params_x, - const InterpFilterParams *filter_params_y, - const int subpel_x_q4, const int subpel_y_q4, - ConvolveParams *conv_params, int bd) { +void av1_highbd_dist_wtd_convolve_y_c( + const uint16_t *src, int src_stride, uint16_t *dst16, int dst16_stride, + int w, int h, const InterpFilterParams *filter_params_x, + const InterpFilterParams *filter_params_y, const int subpel_x_q4, + const int subpel_y_q4, ConvolveParams *conv_params, int bd) { CONV_BUF_TYPE *dst = conv_params->dst; int dst_stride = conv_params->dst_stride; const int fo_vert = filter_params_y->taps / 2 - 1; @@ -863,7 +895,7 @@ void av1_highbd_jnt_convolve_y_c(const uint16_t *src, int src_stride, if (conv_params->do_average) { int32_t tmp = dst[y * dst_stride + x]; - if (conv_params->use_jnt_comp_avg) { + if (conv_params->use_dist_wtd_comp_avg) { tmp = tmp * conv_params->fwd_offset + res * conv_params->bck_offset; tmp = tmp >> DIST_PRECISION_BITS; } else { @@ -880,7 +912,7 @@ void av1_highbd_jnt_convolve_y_c(const uint16_t *src, int src_stride, } } -void av1_highbd_jnt_convolve_2d_copy_c( +void av1_highbd_dist_wtd_convolve_2d_copy_c( const uint16_t *src, int src_stride, uint16_t *dst16, int dst16_stride, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, @@ -904,7 +936,7 @@ void av1_highbd_jnt_convolve_2d_copy_c( res += round_offset; if (conv_params->do_average) { int32_t tmp = dst[y * dst_stride + x]; - if (conv_params->use_jnt_comp_avg) { + if (conv_params->use_dist_wtd_comp_avg) { tmp = tmp * conv_params->fwd_offset + res * conv_params->bck_offset; tmp = tmp >> DIST_PRECISION_BITS; } else { @@ -980,7 +1012,7 @@ void av1_highbd_convolve_2d_scale_c(const uint16_t *src, int src_stride, if (conv_params->is_compound) { if (conv_params->do_average) { int32_t tmp = dst16[y * dst16_stride + x]; - if (conv_params->use_jnt_comp_avg) { + if (conv_params->use_dist_wtd_comp_avg) { tmp = tmp * conv_params->fwd_offset + res * conv_params->bck_offset; tmp = tmp >> DIST_PRECISION_BITS; } else { diff --git a/third_party/aom/av1/common/convolve.h b/third_party/aom/av1/common/convolve.h index 4109dd8433..e5479e6664 100644 --- a/third_party/aom/av1/common/convolve.h +++ b/third_party/aom/av1/common/convolve.h @@ -26,7 +26,7 @@ typedef struct ConvolveParams { int round_1; int plane; int is_compound; - int use_jnt_comp_avg; + int use_dist_wtd_comp_avg; int fwd_offset; int bck_offset; } ConvolveParams; @@ -118,6 +118,11 @@ void av1_highbd_convolve_2d_facade(const uint8_t *src8, int src_stride, const struct scale_factors *sf, int is_intrabc, int bd); +// TODO(sarahparker) This will need to be integerized and optimized +void av1_convolve_2d_sobel_y_c(const uint8_t *src, int src_stride, double *dst, + int dst_stride, int w, int h, int dir, + double norm); + #ifdef __cplusplus } // extern "C" #endif diff --git a/third_party/aom/av1/common/debugmodes.c b/third_party/aom/av1/common/debugmodes.c index 868f341b5b..b26c7ddaa2 100644 --- a/third_party/aom/av1/common/debugmodes.c +++ b/third_party/aom/av1/common/debugmodes.c @@ -17,7 +17,7 @@ static void log_frame_info(AV1_COMMON *cm, const char *str, FILE *f) { fprintf(f, "%s", str); - fprintf(f, "(Frame %d, Show:%d, Q:%d): \n", cm->current_video_frame, + fprintf(f, "(Frame %d, Show:%d, Q:%d): \n", cm->current_frame.frame_number, cm->show_frame, cm->base_qindex); } /* This function dereferences a pointer to the mbmi structure @@ -40,7 +40,7 @@ static void print_mi_data(AV1_COMMON *cm, FILE *file, const char *descriptor, mi++; } fprintf(file, "\n"); - mi += MAX_MIB_SIZE; + mi += cm->mi_stride - cols; } fprintf(file, "\n"); } @@ -68,7 +68,7 @@ void av1_print_modes_and_motion_vectors(AV1_COMMON *cm, const char *file) { mi++; } fprintf(mvs, "\n"); - mi += MAX_MIB_SIZE; + mi += cm->mi_stride - cols; } fprintf(mvs, "\n"); @@ -82,7 +82,7 @@ void av1_print_modes_and_motion_vectors(AV1_COMMON *cm, const char *file) { mi++; } fprintf(mvs, "\n"); - mi += MAX_MIB_SIZE; + mi += cm->mi_stride - cols; } fprintf(mvs, "\n"); diff --git a/third_party/aom/av1/common/entropy.h b/third_party/aom/av1/common/entropy.h index 991692c2f0..41218d30e7 100644 --- a/third_party/aom/av1/common/entropy.h +++ b/third_party/aom/av1/common/entropy.h @@ -54,12 +54,12 @@ extern "C" { #define BASE_CONTEXT_POSITION_NUM 12 -typedef enum TX_CLASS { +enum { TX_CLASS_2D = 0, TX_CLASS_HORIZ = 1, TX_CLASS_VERT = 2, TX_CLASSES = 3, -} TX_CLASS; +} UENUM1BYTE(TX_CLASS); #define DCT_MAX_VALUE 16384 #define DCT_MAX_VALUE_HIGH10 65536 diff --git a/third_party/aom/av1/common/entropymode.c b/third_party/aom/av1/common/entropymode.c index 41dc30ddb8..8e7e952e2c 100644 --- a/third_party/aom/av1/common/entropymode.c +++ b/third_party/aom/av1/common/entropymode.c @@ -1068,9 +1068,16 @@ void av1_setup_frame_contexts(AV1_COMMON *cm) { // This function must ONLY be called when cm->fc has been initialized with // default probs, either by av1_setup_past_independence or after manually // initializing them - cm->frame_contexts[FRAME_CONTEXT_DEFAULTS] = *cm->fc; + *cm->default_frame_context = *cm->fc; + // TODO(jack.haughton@argondesign.com): don't think this should be necessary, + // but could do with fuller testing if (cm->large_scale_tile) { - for (int i = 0; i < FRAME_CONTEXTS; ++i) cm->frame_contexts[i] = *cm->fc; + for (int i = LAST_FRAME; i <= ALTREF_FRAME; ++i) { + RefCntBuffer *const buf = get_ref_frame_buf(cm, i); + if (buf != NULL) buf->frame_context = *cm->fc; + } + for (int i = 0; i < FRAME_BUFFERS; ++i) + cm->buffer_pool->frame_bufs[i].frame_context = *cm->fc; } } @@ -1079,10 +1086,8 @@ void av1_setup_past_independence(AV1_COMMON *cm) { // Features disabled, 0, with delta coding (Default state). av1_clearall_segfeatures(&cm->seg); - cm->current_frame_seg_map = cm->cur_frame->seg_map; - - if (cm->current_frame_seg_map) - memset(cm->current_frame_seg_map, 0, (cm->mi_rows * cm->mi_cols)); + if (cm->cur_frame->seg_map) + memset(cm->cur_frame->seg_map, 0, (cm->mi_rows * cm->mi_cols)); // reset mode ref deltas av1_set_default_ref_deltas(cm->cur_frame->ref_deltas); @@ -1092,7 +1097,6 @@ void av1_setup_past_independence(AV1_COMMON *cm) { av1_default_coef_probs(cm); init_mode_probs(cm->fc); av1_init_mv_probs(cm); - av1_init_lv_map(cm); cm->fc->initialized = 1; av1_setup_frame_contexts(cm); diff --git a/third_party/aom/av1/common/entropymv.h b/third_party/aom/av1/common/entropymv.h index fa818a2c16..cddc80768c 100644 --- a/third_party/aom/av1/common/entropymv.h +++ b/third_party/aom/av1/common/entropymv.h @@ -30,12 +30,12 @@ void av1_init_mv_probs(struct AV1Common *cm); /* Symbols for coding which components are zero jointly */ #define MV_JOINTS 4 -typedef enum { +enum { MV_JOINT_ZERO = 0, /* Zero vector */ MV_JOINT_HNZVZ = 1, /* Vert zero, hor nonzero */ MV_JOINT_HZVNZ = 2, /* Hor zero, vert nonzero */ MV_JOINT_HNZVNZ = 3, /* Both components nonzero */ -} MV_JOINT_TYPE; +} UENUM1BYTE(MV_JOINT_TYPE); static INLINE int mv_joint_vertical(MV_JOINT_TYPE type) { return type == MV_JOINT_HZVNZ || type == MV_JOINT_HNZVNZ; @@ -47,7 +47,7 @@ static INLINE int mv_joint_horizontal(MV_JOINT_TYPE type) { /* Symbols for coding magnitude class of nonzero components */ #define MV_CLASSES 11 -typedef enum { +enum { MV_CLASS_0 = 0, /* (0, 2] integer pel */ MV_CLASS_1 = 1, /* (2, 4] integer pel */ MV_CLASS_2 = 2, /* (4, 8] integer pel */ @@ -59,7 +59,7 @@ typedef enum { MV_CLASS_8 = 8, /* (256, 512] integer pel */ MV_CLASS_9 = 9, /* (512, 1024] integer pel */ MV_CLASS_10 = 10, /* (1024,2048] integer pel */ -} MV_CLASS_TYPE; +} UENUM1BYTE(MV_CLASS_TYPE); #define CLASS0_BITS 1 /* bits at integer precision for class 0 */ #define CLASS0_SIZE (1 << CLASS0_BITS) @@ -91,11 +91,11 @@ typedef struct { nmv_component comps[2]; } nmv_context; -typedef enum { +enum { MV_SUBPEL_NONE = -1, MV_SUBPEL_LOW_PRECISION = 0, MV_SUBPEL_HIGH_PRECISION, -} MvSubpelPrecision; +} SENUM1BYTE(MvSubpelPrecision); #ifdef __cplusplus } // extern "C" diff --git a/third_party/aom/av1/common/enums.h b/third_party/aom/av1/common/enums.h index 869c06ef2f..9af7b0f366 100644 --- a/third_party/aom/av1/common/enums.h +++ b/third_party/aom/av1/common/enums.h @@ -16,6 +16,7 @@ #include "aom/aom_codec.h" #include "aom/aom_integer.h" +#include "aom_ports/mem.h" #ifdef __cplusplus extern "C" { @@ -63,17 +64,6 @@ extern "C" { #define FRAME_OFFSET_BITS 5 #define MAX_FRAME_DISTANCE ((1 << FRAME_OFFSET_BITS) - 1) -#define REF_FRAMES_LOG2 3 -#define REF_FRAMES (1 << REF_FRAMES_LOG2) - -// 4 scratch frames for the new frames to support a maximum of 4 cores decoding -// in parallel, 3 for scaled references on the encoder. -// TODO(hkuang): Add ondemand frame buffers instead of hardcoding the number -// of framebuffers. -// TODO(jkoleszar): These 3 extra references could probably come from the -// normal reference pool. -#define FRAME_BUFFERS (REF_FRAMES + 7) - // 4 frame filter levels: y plane vertical, y plane horizontal, // u plane, and v plane #define FRAME_LF_COUNT 4 @@ -95,12 +85,12 @@ extern "C" { // Profile 2. 8-bit and 10-bit 4:2:2 // 12-bit 4:0:0, 4:2:2 and 4:4:4 // Since we have three bits for the profiles, it can be extended later. -typedef enum BITSTREAM_PROFILE { +enum { PROFILE_0, PROFILE_1, PROFILE_2, MAX_PROFILES, -} BITSTREAM_PROFILE; +} SENUM1BYTE(BITSTREAM_PROFILE); #define LEVEL_MAJOR_BITS 3 #define LEVEL_MINOR_BITS 2 @@ -149,7 +139,7 @@ typedef enum ATTRIBUTE_PACKED { // 4X4, 8X8, 16X16, 32X32, 64X64, 128X128 #define SQR_BLOCK_SIZES 6 -typedef enum ATTRIBUTE_PACKED { +enum { PARTITION_NONE, PARTITION_HORZ, PARTITION_VERT, @@ -163,7 +153,7 @@ typedef enum ATTRIBUTE_PACKED { EXT_PARTITION_TYPES, PARTITION_TYPES = PARTITION_SPLIT + 1, PARTITION_INVALID = 255 -} PARTITION_TYPE; +} UENUM1BYTE(PARTITION_TYPE); typedef char PARTITION_CONTEXT; #define PARTITION_PLOFFSET 4 // number of probability models per block size @@ -171,12 +161,7 @@ typedef char PARTITION_CONTEXT; #define PARTITION_CONTEXTS (PARTITION_BLOCK_SIZES * PARTITION_PLOFFSET) // block transform size -#if defined(_MSC_VER) -typedef uint8_t TX_SIZE; -enum ATTRIBUTE_PACKED { -#else -typedef enum ATTRIBUTE_PACKED { -#endif +enum { TX_4X4, // 4x4 transform TX_8X8, // 8x8 transform TX_16X16, // 16x16 transform @@ -200,11 +185,7 @@ typedef enum ATTRIBUTE_PACKED { TX_SIZES = TX_4X8, // Does NOT include rectangular transforms TX_SIZES_LARGEST = TX_64X64, TX_INVALID = 255 // Invalid transform size -#if defined(_MSC_VER) -}; -#else -} TX_SIZE; -#endif +} UENUM1BYTE(TX_SIZE); #define TX_SIZE_LUMA_MIN (TX_4X4) /* We don't need to code a transform size unless the allowed size is at least @@ -226,7 +207,7 @@ typedef enum ATTRIBUTE_PACKED { #define TX_PAD_HOR 4 // Pad 6 extra rows (2 on top and 4 on bottom) to remove vertical availability // check. -#define TX_PAD_TOP 2 +#define TX_PAD_TOP 0 #define TX_PAD_BOTTOM 4 #define TX_PAD_VER (TX_PAD_TOP + TX_PAD_BOTTOM) // Pad 16 extra bytes to avoid reading overflow in SIMD optimization. @@ -238,43 +219,43 @@ typedef enum ATTRIBUTE_PACKED { #define MAX_TX_BLOCKS_IN_MAX_SB (1 << MAX_TX_BLOCKS_IN_MAX_SB_LOG2) // frame transform mode -typedef enum ATTRIBUTE_PACKED { +enum { ONLY_4X4, // use only 4x4 transform TX_MODE_LARGEST, // transform size is the largest possible for pu size TX_MODE_SELECT, // transform specified for each block TX_MODES, -} TX_MODE; +} UENUM1BYTE(TX_MODE); // 1D tx types -typedef enum ATTRIBUTE_PACKED { +enum { DCT_1D, ADST_1D, FLIPADST_1D, IDTX_1D, TX_TYPES_1D, -} TX_TYPE_1D; +} UENUM1BYTE(TX_TYPE_1D); -typedef enum ATTRIBUTE_PACKED { - DCT_DCT, // DCT in both horizontal and vertical - ADST_DCT, // ADST in vertical, DCT in horizontal - DCT_ADST, // DCT in vertical, ADST in horizontal - ADST_ADST, // ADST in both directions - FLIPADST_DCT, - DCT_FLIPADST, - FLIPADST_FLIPADST, - ADST_FLIPADST, - FLIPADST_ADST, - IDTX, - V_DCT, - H_DCT, - V_ADST, - H_ADST, - V_FLIPADST, - H_FLIPADST, +enum { + DCT_DCT, // DCT in both horizontal and vertical + ADST_DCT, // ADST in vertical, DCT in horizontal + DCT_ADST, // DCT in vertical, ADST in horizontal + ADST_ADST, // ADST in both directions + FLIPADST_DCT, // FLIPADST in vertical, DCT in horizontal + DCT_FLIPADST, // DCT in vertical, FLIPADST in horizontal + FLIPADST_FLIPADST, // FLIPADST in both directions + ADST_FLIPADST, // ADST in vertical, FLIPADST in horizontal + FLIPADST_ADST, // FLIPADST in vertical, ADST in horizontal + IDTX, // Identity in both directions + V_DCT, // DCT in vertical, identity in horizontal + H_DCT, // Identity in vertical, DCT in horizontal + V_ADST, // ADST in vertical, identity in horizontal + H_ADST, // Identity in vertical, ADST in horizontal + V_FLIPADST, // FLIPADST in vertical, identity in horizontal + H_FLIPADST, // Identity in vertical, FLIPADST in horizontal TX_TYPES, -} TX_TYPE; +} UENUM1BYTE(TX_TYPE); -typedef enum ATTRIBUTE_PACKED { +enum { REG_REG, REG_SMOOTH, REG_SHARP, @@ -284,9 +265,9 @@ typedef enum ATTRIBUTE_PACKED { SHARP_REG, SHARP_SMOOTH, SHARP_SHARP, -} DUAL_FILTER_TYPE; +} UENUM1BYTE(DUAL_FILTER_TYPE); -typedef enum ATTRIBUTE_PACKED { +enum { // DCT only EXT_TX_SET_DCTONLY, // DCT + Identity only @@ -300,7 +281,7 @@ typedef enum ATTRIBUTE_PACKED { // Discrete Trig transforms w/ flip (9) + Identity (1) + 1D Hor/Ver (6) EXT_TX_SET_ALL16, EXT_TX_SET_TYPES -} TxSetType; +} UENUM1BYTE(TxSetType); #define IS_2D_TRANSFORM(tx_type) (tx_type < IDTX) @@ -308,7 +289,7 @@ typedef enum ATTRIBUTE_PACKED { #define EXT_TX_SETS_INTER 4 // Sets of transform selections for INTER #define EXT_TX_SETS_INTRA 3 // Sets of transform selections for INTRA -typedef enum ATTRIBUTE_PACKED { +enum { AOM_LAST_FLAG = 1 << 0, AOM_LAST2_FLAG = 1 << 1, AOM_LAST3_FLAG = 1 << 2, @@ -317,19 +298,15 @@ typedef enum ATTRIBUTE_PACKED { AOM_ALT2_FLAG = 1 << 5, AOM_ALT_FLAG = 1 << 6, AOM_REFFRAME_ALL = (1 << 7) - 1 -} AOM_REFFRAME; +} UENUM1BYTE(AOM_REFFRAME); -typedef enum ATTRIBUTE_PACKED { +enum { UNIDIR_COMP_REFERENCE, BIDIR_COMP_REFERENCE, COMP_REFERENCE_TYPES, -} COMP_REFERENCE_TYPE; +} UENUM1BYTE(COMP_REFERENCE_TYPE); -typedef enum ATTRIBUTE_PACKED { - PLANE_TYPE_Y, - PLANE_TYPE_UV, - PLANE_TYPES -} PLANE_TYPE; +enum { PLANE_TYPE_Y, PLANE_TYPE_UV, PLANE_TYPES } UENUM1BYTE(PLANE_TYPE); #define CFL_ALPHABET_SIZE_LOG2 4 #define CFL_ALPHABET_SIZE (1 << CFL_ALPHABET_SIZE_LOG2) @@ -337,24 +314,20 @@ typedef enum ATTRIBUTE_PACKED { #define CFL_IDX_U(idx) (idx >> CFL_ALPHABET_SIZE_LOG2) #define CFL_IDX_V(idx) (idx & (CFL_ALPHABET_SIZE - 1)) -typedef enum ATTRIBUTE_PACKED { - CFL_PRED_U, - CFL_PRED_V, - CFL_PRED_PLANES -} CFL_PRED_TYPE; +enum { CFL_PRED_U, CFL_PRED_V, CFL_PRED_PLANES } UENUM1BYTE(CFL_PRED_TYPE); -typedef enum ATTRIBUTE_PACKED { +enum { CFL_SIGN_ZERO, CFL_SIGN_NEG, CFL_SIGN_POS, CFL_SIGNS -} CFL_SIGN_TYPE; +} UENUM1BYTE(CFL_SIGN_TYPE); -typedef enum ATTRIBUTE_PACKED { +enum { CFL_DISALLOWED, CFL_ALLOWED, CFL_ALLOWED_TYPES -} CFL_ALLOWED_TYPE; +} UENUM1BYTE(CFL_ALLOWED_TYPE); // CFL_SIGN_ZERO,CFL_SIGN_ZERO is invalid #define CFL_JOINT_SIGNS (CFL_SIGNS * CFL_SIGNS - 1) @@ -371,12 +344,12 @@ typedef enum ATTRIBUTE_PACKED { #define CFL_CONTEXT_V(js) \ (CFL_SIGN_V(js) * CFL_SIGNS + CFL_SIGN_U(js) - CFL_SIGNS) -typedef enum ATTRIBUTE_PACKED { +enum { PALETTE_MAP, COLOR_MAP_TYPES, -} COLOR_MAP_TYPE; +} UENUM1BYTE(COLOR_MAP_TYPE); -typedef enum ATTRIBUTE_PACKED { +enum { TWO_COLORS, THREE_COLORS, FOUR_COLORS, @@ -385,9 +358,9 @@ typedef enum ATTRIBUTE_PACKED { SEVEN_COLORS, EIGHT_COLORS, PALETTE_SIZES -} PALETTE_SIZE; +} UENUM1BYTE(PALETTE_SIZE); -typedef enum ATTRIBUTE_PACKED { +enum { PALETTE_COLOR_ONE, PALETTE_COLOR_TWO, PALETTE_COLOR_THREE, @@ -397,11 +370,11 @@ typedef enum ATTRIBUTE_PACKED { PALETTE_COLOR_SEVEN, PALETTE_COLOR_EIGHT, PALETTE_COLORS -} PALETTE_COLOR; +} UENUM1BYTE(PALETTE_COLOR); // Note: All directional predictors must be between V_PRED and D67_PRED (both // inclusive). -typedef enum ATTRIBUTE_PACKED { +enum { DC_PRED, // Average of above and left pixels V_PRED, // Vertical H_PRED, // Horizontal @@ -442,11 +415,11 @@ typedef enum ATTRIBUTE_PACKED { INTER_MODE_END = MB_MODE_COUNT, INTRA_MODES = PAETH_PRED + 1, // PAETH_PRED has to be the last intra mode. INTRA_INVALID = MB_MODE_COUNT // For uv_mode in inter blocks -} PREDICTION_MODE; +} UENUM1BYTE(PREDICTION_MODE); // TODO(ltrudeau) Do we really want to pack this? // TODO(ltrudeau) Do we match with PREDICTION_MODE? -typedef enum ATTRIBUTE_PACKED { +enum { UV_DC_PRED, // Average of above and left pixels UV_V_PRED, // Vertical UV_H_PRED, // Horizontal @@ -463,38 +436,38 @@ typedef enum ATTRIBUTE_PACKED { UV_CFL_PRED, // Chroma-from-Luma UV_INTRA_MODES, UV_MODE_INVALID, // For uv_mode in inter blocks -} UV_PREDICTION_MODE; +} UENUM1BYTE(UV_PREDICTION_MODE); -typedef enum ATTRIBUTE_PACKED { +enum { SIMPLE_TRANSLATION, OBMC_CAUSAL, // 2-sided OBMC WARPED_CAUSAL, // 2-sided WARPED MOTION_MODES -} MOTION_MODE; +} UENUM1BYTE(MOTION_MODE); -typedef enum ATTRIBUTE_PACKED { +enum { II_DC_PRED, II_V_PRED, II_H_PRED, II_SMOOTH_PRED, INTERINTRA_MODES -} INTERINTRA_MODE; +} UENUM1BYTE(INTERINTRA_MODE); -typedef enum ATTRIBUTE_PACKED { +enum { COMPOUND_AVERAGE, COMPOUND_WEDGE, COMPOUND_DIFFWTD, COMPOUND_TYPES, -} COMPOUND_TYPE; +} UENUM1BYTE(COMPOUND_TYPE); -typedef enum ATTRIBUTE_PACKED { +enum { FILTER_DC_PRED, FILTER_V_PRED, FILTER_H_PRED, FILTER_D157_PRED, FILTER_PAETH_PRED, FILTER_INTRA_MODES, -} FILTER_INTRA_MODE; +} UENUM1BYTE(FILTER_INTRA_MODE); #define DIRECTIONAL_MODES 8 #define MAX_ANGLE_DELTA 3 @@ -550,28 +523,47 @@ typedef enum ATTRIBUTE_PACKED { #define TXFM_PARTITION_CONTEXTS ((TX_SIZES - TX_8X8) * 6 - 3) typedef uint8_t TXFM_CONTEXT; -#define NONE_FRAME -1 -#define INTRA_FRAME 0 -#define LAST_FRAME 1 -#define LAST2_FRAME 2 -#define LAST3_FRAME 3 -#define GOLDEN_FRAME 4 -#define BWDREF_FRAME 5 -#define ALTREF2_FRAME 6 -#define ALTREF_FRAME 7 -#define EXTREF_FRAME REF_FRAMES -#define LAST_REF_FRAMES (LAST3_FRAME - LAST_FRAME + 1) +// An enum for single reference types (and some derived values). +enum { + NONE_FRAME = -1, + INTRA_FRAME, + LAST_FRAME, + LAST2_FRAME, + LAST3_FRAME, + GOLDEN_FRAME, + BWDREF_FRAME, + ALTREF2_FRAME, + ALTREF_FRAME, + REF_FRAMES, -#define INTER_REFS_PER_FRAME (ALTREF_FRAME - LAST_FRAME + 1) + // Extra/scratch reference frame. It may be: + // - used to update the ALTREF2_FRAME ref (see lshift_bwd_ref_frames()), or + // - updated from ALTREF2_FRAME ref (see rshift_bwd_ref_frames()). + EXTREF_FRAME = REF_FRAMES, + + // Number of inter (non-intra) reference types. + INTER_REFS_PER_FRAME = ALTREF_FRAME - LAST_FRAME + 1, + + // Number of forward (aka past) reference types. + FWD_REFS = GOLDEN_FRAME - LAST_FRAME + 1, + + // Number of backward (aka future) reference types. + BWD_REFS = ALTREF_FRAME - BWDREF_FRAME + 1, + + SINGLE_REFS = FWD_REFS + BWD_REFS, +}; + +#define REF_FRAMES_LOG2 3 + +// REF_FRAMES for the cm->ref_frame_map array, 1 scratch frame for the new +// frame in cm->cur_frame, INTER_REFS_PER_FRAME for scaled references on the +// encoder in the cpi->scaled_ref_buf array. +#define FRAME_BUFFERS (REF_FRAMES + 1 + INTER_REFS_PER_FRAME) -#define FWD_REFS (GOLDEN_FRAME - LAST_FRAME + 1) #define FWD_RF_OFFSET(ref) (ref - LAST_FRAME) -#define BWD_REFS (ALTREF_FRAME - BWDREF_FRAME + 1) #define BWD_RF_OFFSET(ref) (ref - BWDREF_FRAME) -#define SINGLE_REFS (FWD_REFS + BWD_REFS) - -typedef enum ATTRIBUTE_PACKED { +enum { LAST_LAST2_FRAMES, // { LAST_FRAME, LAST2_FRAME } LAST_LAST3_FRAMES, // { LAST_FRAME, LAST3_FRAME } LAST_GOLDEN_FRAMES, // { LAST_FRAME, GOLDEN_FRAME } @@ -585,7 +577,7 @@ typedef enum ATTRIBUTE_PACKED { // NOTE: UNIDIR_COMP_REFS is the number of uni-directional reference pairs // that are explicitly signaled. UNIDIR_COMP_REFS = BWDREF_ALTREF_FRAMES + 1, -} UNIDIR_COMP_REF; +} UENUM1BYTE(UNIDIR_COMP_REF); #define TOTAL_COMP_REFS (FWD_REFS * BWD_REFS + TOTAL_UNIDIR_COMP_REFS) @@ -596,14 +588,18 @@ typedef enum ATTRIBUTE_PACKED { // possible to have a reference pair not listed for explicit signaling. #define MODE_CTX_REF_FRAMES (REF_FRAMES + TOTAL_COMP_REFS) -typedef enum ATTRIBUTE_PACKED { +// Note: It includes single and compound references. So, it can take values from +// NONE_FRAME to (MODE_CTX_REF_FRAMES - 1). Hence, it is not defined as an enum. +typedef int8_t MV_REFERENCE_FRAME; + +enum { RESTORE_NONE, RESTORE_WIENER, RESTORE_SGRPROJ, RESTORE_SWITCHABLE, RESTORE_SWITCHABLE_TYPES = RESTORE_SWITCHABLE, RESTORE_TYPES = 4, -} RestorationType; +} UENUM1BYTE(RestorationType); #define SUPERRES_SCALE_BITS 3 #define SUPERRES_SCALE_DENOMINATOR_MIN (SCALE_NUMERATOR + 1) diff --git a/third_party/aom/av1/common/filter.h b/third_party/aom/av1/common/filter.h index 571422d114..184f5b2d0a 100644 --- a/third_party/aom/av1/common/filter.h +++ b/third_party/aom/av1/common/filter.h @@ -37,12 +37,16 @@ typedef enum ATTRIBUTE_PACKED { EXTRA_FILTERS = INTERP_FILTERS_ALL - SWITCHABLE_FILTERS, } InterpFilter; -// With CONFIG_DUAL_FILTER, pack two InterpFilter's into a uint32_t: since -// there are at most 10 filters, we can use 16 bits for each and have more than -// enough space. This reduces argument passing and unifies the operation of -// setting a (pair of) filters. -// -// Without CONFIG_DUAL_FILTER, +enum { + USE_2_TAPS_ORIG = 0, // This is used in temporal filtering. + USE_2_TAPS, + USE_4_TAPS, + USE_8_TAPS, +} UENUM1BYTE(SUBPEL_SEARCH_TYPE); + +// Pack two InterpFilter's into a uint32_t: since there are at most 10 filters, +// we can use 16 bits for each and have more than enough space. This reduces +// argument passing and unifies the operation of setting a (pair of) filters. typedef uint32_t InterpFilters; static INLINE InterpFilter av1_extract_interp_filter(InterpFilters filters, int x_filter) { @@ -192,14 +196,19 @@ av1_get_interp_filter_params_with_block_size(const InterpFilter interp_filter, return &av1_interp_filter_params_list[interp_filter]; } -static INLINE const InterpFilterParams *av1_get_4tap_interp_filter_params( +static INLINE const InterpFilterParams *get_4tap_interp_filter_params( const InterpFilter interp_filter) { return &av1_interp_4tap[interp_filter]; } static INLINE const int16_t *av1_get_interp_filter_kernel( - const InterpFilter interp_filter) { - return av1_interp_filter_params_list[interp_filter].filter_ptr; + const InterpFilter interp_filter, int subpel_search) { + assert(subpel_search >= USE_2_TAPS); + return (subpel_search == USE_2_TAPS) + ? av1_interp_4tap[BILINEAR].filter_ptr + : ((subpel_search == USE_4_TAPS) + ? av1_interp_4tap[interp_filter].filter_ptr + : av1_interp_filter_params_list[interp_filter].filter_ptr); } static INLINE const int16_t *av1_get_interp_filter_subpel_kernel( @@ -207,6 +216,17 @@ static INLINE const int16_t *av1_get_interp_filter_subpel_kernel( return filter_params->filter_ptr + filter_params->taps * subpel; } +static INLINE const InterpFilterParams *av1_get_filter(int subpel_search) { + assert(subpel_search >= USE_2_TAPS); + + switch (subpel_search) { + case USE_2_TAPS: return get_4tap_interp_filter_params(BILINEAR); + case USE_4_TAPS: return get_4tap_interp_filter_params(EIGHTTAP_REGULAR); + case USE_8_TAPS: return &av1_interp_filter_params_list[EIGHTTAP_REGULAR]; + default: assert(0); return NULL; + } +} + #ifdef __cplusplus } // extern "C" #endif diff --git a/third_party/aom/av1/common/frame_buffers.c b/third_party/aom/av1/common/frame_buffers.c index fd6c4bc799..f10ccd5942 100644 --- a/third_party/aom/av1/common/frame_buffers.c +++ b/third_party/aom/av1/common/frame_buffers.c @@ -22,7 +22,11 @@ int av1_alloc_internal_frame_buffers(InternalFrameBufferList *list) { AOM_MAXIMUM_REF_BUFFERS + AOM_MAXIMUM_WORK_BUFFERS; list->int_fb = (InternalFrameBuffer *)aom_calloc( list->num_internal_frame_buffers, sizeof(*list->int_fb)); - return (list->int_fb == NULL); + if (list->int_fb == NULL) { + list->num_internal_frame_buffers = 0; + return 1; + } + return 0; } void av1_free_internal_frame_buffers(InternalFrameBufferList *list) { @@ -36,6 +40,7 @@ void av1_free_internal_frame_buffers(InternalFrameBufferList *list) { } aom_free(list->int_fb); list->int_fb = NULL; + list->num_internal_frame_buffers = 0; } void av1_zero_unused_internal_frame_buffers(InternalFrameBufferList *list) { @@ -69,7 +74,10 @@ int av1_get_frame_buffer(void *cb_priv, size_t min_size, // due to access uninitialized memory in frame border. It could be // skipped if border were totally removed. int_fb_list->int_fb[i].data = (uint8_t *)aom_calloc(1, min_size); - if (!int_fb_list->int_fb[i].data) return -1; + if (!int_fb_list->int_fb[i].data) { + int_fb_list->int_fb[i].size = 0; + return -1; + } int_fb_list->int_fb[i].size = min_size; } @@ -86,6 +94,5 @@ int av1_release_frame_buffer(void *cb_priv, aom_codec_frame_buffer_t *fb) { InternalFrameBuffer *const int_fb = (InternalFrameBuffer *)fb->priv; (void)cb_priv; if (int_fb) int_fb->in_use = 0; - fb->priv = NULL; return 0; } diff --git a/third_party/aom/av1/common/idct.c b/third_party/aom/av1/common/idct.c index 2c1cb98271..bff438f3c6 100644 --- a/third_party/aom/av1/common/idct.c +++ b/third_party/aom/av1/common/idct.c @@ -56,87 +56,87 @@ void av1_highbd_inv_txfm_add_4x4_c(const tran_low_t *input, uint8_t *dest, av1_inv_txfm2d_add_4x4_c(src, CONVERT_TO_SHORTPTR(dest), stride, tx_type, bd); } -void av1_highbd_inv_txfm_add_4x8(const tran_low_t *input, uint8_t *dest, - int stride, const TxfmParam *txfm_param) { +void av1_highbd_inv_txfm_add_4x8_c(const tran_low_t *input, uint8_t *dest, + int stride, const TxfmParam *txfm_param) { assert(av1_ext_tx_used[txfm_param->tx_set_type][txfm_param->tx_type]); const int32_t *src = cast_to_int32(input); av1_inv_txfm2d_add_4x8_c(src, CONVERT_TO_SHORTPTR(dest), stride, txfm_param->tx_type, txfm_param->bd); } -void av1_highbd_inv_txfm_add_8x4(const tran_low_t *input, uint8_t *dest, - int stride, const TxfmParam *txfm_param) { +void av1_highbd_inv_txfm_add_8x4_c(const tran_low_t *input, uint8_t *dest, + int stride, const TxfmParam *txfm_param) { assert(av1_ext_tx_used[txfm_param->tx_set_type][txfm_param->tx_type]); const int32_t *src = cast_to_int32(input); av1_inv_txfm2d_add_8x4_c(src, CONVERT_TO_SHORTPTR(dest), stride, txfm_param->tx_type, txfm_param->bd); } -void av1_highbd_inv_txfm_add_16x32(const tran_low_t *input, uint8_t *dest, - int stride, const TxfmParam *txfm_param) { +void av1_highbd_inv_txfm_add_16x32_c(const tran_low_t *input, uint8_t *dest, + int stride, const TxfmParam *txfm_param) { const int32_t *src = cast_to_int32(input); av1_inv_txfm2d_add_16x32_c(src, CONVERT_TO_SHORTPTR(dest), stride, txfm_param->tx_type, txfm_param->bd); } -void av1_highbd_inv_txfm_add_32x16(const tran_low_t *input, uint8_t *dest, - int stride, const TxfmParam *txfm_param) { +void av1_highbd_inv_txfm_add_32x16_c(const tran_low_t *input, uint8_t *dest, + int stride, const TxfmParam *txfm_param) { const int32_t *src = cast_to_int32(input); av1_inv_txfm2d_add_32x16_c(src, CONVERT_TO_SHORTPTR(dest), stride, txfm_param->tx_type, txfm_param->bd); } -void av1_highbd_inv_txfm_add_16x4(const tran_low_t *input, uint8_t *dest, - int stride, const TxfmParam *txfm_param) { +void av1_highbd_inv_txfm_add_16x4_c(const tran_low_t *input, uint8_t *dest, + int stride, const TxfmParam *txfm_param) { const int32_t *src = cast_to_int32(input); av1_inv_txfm2d_add_16x4_c(src, CONVERT_TO_SHORTPTR(dest), stride, txfm_param->tx_type, txfm_param->bd); } -void av1_highbd_inv_txfm_add_4x16(const tran_low_t *input, uint8_t *dest, - int stride, const TxfmParam *txfm_param) { +void av1_highbd_inv_txfm_add_4x16_c(const tran_low_t *input, uint8_t *dest, + int stride, const TxfmParam *txfm_param) { const int32_t *src = cast_to_int32(input); av1_inv_txfm2d_add_4x16_c(src, CONVERT_TO_SHORTPTR(dest), stride, txfm_param->tx_type, txfm_param->bd); } -void av1_highbd_inv_txfm_add_32x8(const tran_low_t *input, uint8_t *dest, - int stride, const TxfmParam *txfm_param) { +void av1_highbd_inv_txfm_add_32x8_c(const tran_low_t *input, uint8_t *dest, + int stride, const TxfmParam *txfm_param) { const int32_t *src = cast_to_int32(input); av1_inv_txfm2d_add_32x8_c(src, CONVERT_TO_SHORTPTR(dest), stride, txfm_param->tx_type, txfm_param->bd); } -void av1_highbd_inv_txfm_add_8x32(const tran_low_t *input, uint8_t *dest, - int stride, const TxfmParam *txfm_param) { +void av1_highbd_inv_txfm_add_8x32_c(const tran_low_t *input, uint8_t *dest, + int stride, const TxfmParam *txfm_param) { const int32_t *src = cast_to_int32(input); av1_inv_txfm2d_add_8x32_c(src, CONVERT_TO_SHORTPTR(dest), stride, txfm_param->tx_type, txfm_param->bd); } -void av1_highbd_inv_txfm_add_32x64(const tran_low_t *input, uint8_t *dest, - int stride, const TxfmParam *txfm_param) { +void av1_highbd_inv_txfm_add_32x64_c(const tran_low_t *input, uint8_t *dest, + int stride, const TxfmParam *txfm_param) { const int32_t *src = cast_to_int32(input); av1_inv_txfm2d_add_32x64_c(src, CONVERT_TO_SHORTPTR(dest), stride, txfm_param->tx_type, txfm_param->bd); } -void av1_highbd_inv_txfm_add_64x32(const tran_low_t *input, uint8_t *dest, - int stride, const TxfmParam *txfm_param) { +void av1_highbd_inv_txfm_add_64x32_c(const tran_low_t *input, uint8_t *dest, + int stride, const TxfmParam *txfm_param) { const int32_t *src = cast_to_int32(input); av1_inv_txfm2d_add_64x32_c(src, CONVERT_TO_SHORTPTR(dest), stride, txfm_param->tx_type, txfm_param->bd); } -void av1_highbd_inv_txfm_add_16x64(const tran_low_t *input, uint8_t *dest, - int stride, const TxfmParam *txfm_param) { +void av1_highbd_inv_txfm_add_16x64_c(const tran_low_t *input, uint8_t *dest, + int stride, const TxfmParam *txfm_param) { const int32_t *src = cast_to_int32(input); av1_inv_txfm2d_add_16x64_c(src, CONVERT_TO_SHORTPTR(dest), stride, txfm_param->tx_type, txfm_param->bd); } -void av1_highbd_inv_txfm_add_64x16(const tran_low_t *input, uint8_t *dest, - int stride, const TxfmParam *txfm_param) { +void av1_highbd_inv_txfm_add_64x16_c(const tran_low_t *input, uint8_t *dest, + int stride, const TxfmParam *txfm_param) { const int32_t *src = cast_to_int32(input); av1_inv_txfm2d_add_64x16_c(src, CONVERT_TO_SHORTPTR(dest), stride, txfm_param->tx_type, txfm_param->bd); @@ -204,7 +204,7 @@ static void init_txfm_param(const MACROBLOCKD *xd, int plane, TX_SIZE tx_size, txfm_param->eob = eob; txfm_param->lossless = xd->lossless[xd->mi[0]->segment_id]; txfm_param->bd = xd->bd; - txfm_param->is_hbd = get_bitdepth_data_path_index(xd); + txfm_param->is_hbd = is_cur_buf_hbd(xd); txfm_param->tx_set_type = av1_get_ext_tx_set_type( txfm_param->tx_size, is_inter_block(xd->mi[0]), reduced_tx_set); } @@ -224,10 +224,10 @@ void av1_highbd_inv_txfm_add_c(const tran_low_t *input, uint8_t *dest, av1_highbd_inv_txfm_add_8x8_c(input, dest, stride, txfm_param); break; case TX_4X8: - av1_highbd_inv_txfm_add_4x8(input, dest, stride, txfm_param); + av1_highbd_inv_txfm_add_4x8_c(input, dest, stride, txfm_param); break; case TX_8X4: - av1_highbd_inv_txfm_add_8x4(input, dest, stride, txfm_param); + av1_highbd_inv_txfm_add_8x4_c(input, dest, stride, txfm_param); break; case TX_8X16: av1_highbd_inv_txfm_add_8x16_c(input, dest, stride, txfm_param); @@ -236,25 +236,25 @@ void av1_highbd_inv_txfm_add_c(const tran_low_t *input, uint8_t *dest, av1_highbd_inv_txfm_add_16x8_c(input, dest, stride, txfm_param); break; case TX_16X32: - av1_highbd_inv_txfm_add_16x32(input, dest, stride, txfm_param); + av1_highbd_inv_txfm_add_16x32_c(input, dest, stride, txfm_param); break; case TX_32X16: - av1_highbd_inv_txfm_add_32x16(input, dest, stride, txfm_param); + av1_highbd_inv_txfm_add_32x16_c(input, dest, stride, txfm_param); break; case TX_64X64: av1_highbd_inv_txfm_add_64x64_c(input, dest, stride, txfm_param); break; case TX_32X64: - av1_highbd_inv_txfm_add_32x64(input, dest, stride, txfm_param); + av1_highbd_inv_txfm_add_32x64_c(input, dest, stride, txfm_param); break; case TX_64X32: - av1_highbd_inv_txfm_add_64x32(input, dest, stride, txfm_param); + av1_highbd_inv_txfm_add_64x32_c(input, dest, stride, txfm_param); break; case TX_16X64: - av1_highbd_inv_txfm_add_16x64(input, dest, stride, txfm_param); + av1_highbd_inv_txfm_add_16x64_c(input, dest, stride, txfm_param); break; case TX_64X16: - av1_highbd_inv_txfm_add_64x16(input, dest, stride, txfm_param); + av1_highbd_inv_txfm_add_64x16_c(input, dest, stride, txfm_param); break; case TX_4X4: // this is like av1_short_idct4x4 but has a special case around eob<=1 @@ -263,16 +263,16 @@ void av1_highbd_inv_txfm_add_c(const tran_low_t *input, uint8_t *dest, av1_highbd_inv_txfm_add_4x4_c(input, dest, stride, txfm_param); break; case TX_16X4: - av1_highbd_inv_txfm_add_16x4(input, dest, stride, txfm_param); + av1_highbd_inv_txfm_add_16x4_c(input, dest, stride, txfm_param); break; case TX_4X16: - av1_highbd_inv_txfm_add_4x16(input, dest, stride, txfm_param); + av1_highbd_inv_txfm_add_4x16_c(input, dest, stride, txfm_param); break; case TX_8X32: - av1_highbd_inv_txfm_add_8x32(input, dest, stride, txfm_param); + av1_highbd_inv_txfm_add_8x32_c(input, dest, stride, txfm_param); break; case TX_32X8: - av1_highbd_inv_txfm_add_32x8(input, dest, stride, txfm_param); + av1_highbd_inv_txfm_add_32x8_c(input, dest, stride, txfm_param); break; default: assert(0 && "Invalid transform size"); break; } diff --git a/third_party/aom/av1/common/idct.h b/third_party/aom/av1/common/idct.h index d9454e73fe..004d25d49a 100644 --- a/third_party/aom/av1/common/idct.h +++ b/third_party/aom/av1/common/idct.h @@ -44,22 +44,6 @@ static INLINE const int32_t *cast_to_int32(const tran_low_t *input) { return (const int32_t *)input; } -typedef void(highbd_inv_txfm_add)(const tran_low_t *input, uint8_t *dest, - int stride, const TxfmParam *param); - -highbd_inv_txfm_add av1_highbd_inv_txfm_add_4x8; -highbd_inv_txfm_add av1_highbd_inv_txfm_add_8x4; -highbd_inv_txfm_add av1_highbd_inv_txfm_add_16x32; -highbd_inv_txfm_add av1_highbd_inv_txfm_add_32x16; -highbd_inv_txfm_add av1_highbd_inv_txfm_add_32x64; -highbd_inv_txfm_add av1_highbd_inv_txfm_add_64x32; -highbd_inv_txfm_add av1_highbd_inv_txfm_add_16x64; -highbd_inv_txfm_add av1_highbd_inv_txfm_add_64x16; -highbd_inv_txfm_add av1_highbd_inv_txfm_add_16x4; -highbd_inv_txfm_add av1_highbd_inv_txfm_add_4x16; -highbd_inv_txfm_add av1_highbd_inv_txfm_add_8x32; -highbd_inv_txfm_add av1_highbd_inv_txfm_add_32x8; - #ifdef __cplusplus } // extern "C" #endif diff --git a/third_party/aom/av1/common/mv.h b/third_party/aom/av1/common/mv.h index 5b0225192a..d097f9e5f1 100644 --- a/third_party/aom/av1/common/mv.h +++ b/third_party/aom/av1/common/mv.h @@ -56,13 +56,13 @@ typedef struct mv32 { #define WARPEDDIFF_PREC_BITS (WARPEDMODEL_PREC_BITS - WARPEDPIXEL_PREC_BITS) /* clang-format off */ -typedef enum ATTRIBUTE_PACKED { +enum { IDENTITY = 0, // identity transformation, 0-parameter TRANSLATION = 1, // translational motion 2-parameter ROTZOOM = 2, // simplified affine with rotation + zoom only, 4-parameter AFFINE = 3, // affine, 6-parameter TRANS_TYPES, -} TransformationType; +} UENUM1BYTE(TransformationType); /* clang-format on */ // Number of types used for global motion (must be >= 3 and <= TRANS_TYPES) @@ -87,18 +87,18 @@ static const int trans_model_params[TRANS_TYPES] = { 0, 2, 4, 6 }; // z . y' = m4 m5 m1 * y // 1] m6 m7 1) 1] typedef struct { - TransformationType wmtype; int32_t wmmat[8]; int16_t alpha, beta, gamma, delta; + TransformationType wmtype; int8_t invalid; } WarpedMotionParams; /* clang-format off */ static const WarpedMotionParams default_warp_params = { - IDENTITY, { 0, 0, (1 << WARPEDMODEL_PREC_BITS), 0, 0, (1 << WARPEDMODEL_PREC_BITS), 0, 0 }, 0, 0, 0, 0, + IDENTITY, 0, }; /* clang-format on */ @@ -263,7 +263,7 @@ static INLINE int_mv gm_get_motion_vector(const WarpedMotionParams *gm, return res; } -static INLINE TransformationType get_gmtype(const WarpedMotionParams *gm) { +static INLINE TransformationType get_wmtype(const WarpedMotionParams *gm) { if (gm->wmmat[5] == (1 << WARPEDMODEL_PREC_BITS) && !gm->wmmat[4] && gm->wmmat[2] == (1 << WARPEDMODEL_PREC_BITS) && !gm->wmmat[3]) { return ((!gm->wmmat[1] && !gm->wmmat[0]) ? IDENTITY : TRANSLATION); diff --git a/third_party/aom/av1/common/mvref_common.c b/third_party/aom/av1/common/mvref_common.c index 7f24ab4e6d..3bdffe4ad4 100644 --- a/third_party/aom/av1/common/mvref_common.c +++ b/third_party/aom/av1/common/mvref_common.c @@ -346,10 +346,11 @@ static int add_tpl_ref_mv(const AV1_COMMON *cm, const MACROBLOCKD *xd, av1_set_ref_frame(rf, ref_frame); if (rf[1] == NONE_FRAME) { - int cur_frame_index = cm->cur_frame->cur_frame_offset; - int buf_idx_0 = cm->frame_refs[FWD_RF_OFFSET(rf[0])].idx; - int frame0_index = cm->buffer_pool->frame_bufs[buf_idx_0].cur_frame_offset; - int cur_offset_0 = get_relative_dist(cm, cur_frame_index, frame0_index); + int cur_frame_index = cm->cur_frame->order_hint; + const RefCntBuffer *const buf_0 = get_ref_frame_buf(cm, rf[0]); + int frame0_index = buf_0->order_hint; + int cur_offset_0 = get_relative_dist(&cm->seq_params.order_hint_info, + cur_frame_index, frame0_index); CANDIDATE_MV *ref_mv_stack = ref_mv_stacks[rf[0]]; if (prev_frame_mvs->mfmv0.as_int != INVALID_MV) { @@ -380,14 +381,16 @@ static int add_tpl_ref_mv(const AV1_COMMON *cm, const MACROBLOCKD *xd, } } else { // Process compound inter mode - int cur_frame_index = cm->cur_frame->cur_frame_offset; - int buf_idx_0 = cm->frame_refs[FWD_RF_OFFSET(rf[0])].idx; - int frame0_index = cm->buffer_pool->frame_bufs[buf_idx_0].cur_frame_offset; + int cur_frame_index = cm->cur_frame->order_hint; + const RefCntBuffer *const buf_0 = get_ref_frame_buf(cm, rf[0]); + int frame0_index = buf_0->order_hint; - int cur_offset_0 = get_relative_dist(cm, cur_frame_index, frame0_index); - int buf_idx_1 = cm->frame_refs[FWD_RF_OFFSET(rf[1])].idx; - int frame1_index = cm->buffer_pool->frame_bufs[buf_idx_1].cur_frame_offset; - int cur_offset_1 = get_relative_dist(cm, cur_frame_index, frame1_index); + int cur_offset_0 = get_relative_dist(&cm->seq_params.order_hint_info, + cur_frame_index, frame0_index); + const RefCntBuffer *const buf_1 = get_ref_frame_buf(cm, rf[1]); + int frame1_index = buf_1->order_hint; + int cur_offset_1 = get_relative_dist(&cm->seq_params.order_hint_info, + cur_frame_index, frame1_index); CANDIDATE_MV *ref_mv_stack = ref_mv_stacks[ref_frame]; if (prev_frame_mvs->mfmv0.as_int != INVALID_MV) { @@ -861,26 +864,25 @@ void av1_find_best_ref_mvs(int allow_hp, int_mv *mvlist, int_mv *nearest_mv, } void av1_setup_frame_buf_refs(AV1_COMMON *cm) { - cm->cur_frame->cur_frame_offset = cm->frame_offset; + cm->cur_frame->order_hint = cm->current_frame.order_hint; MV_REFERENCE_FRAME ref_frame; for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) { - const int buf_idx = cm->frame_refs[ref_frame - LAST_FRAME].idx; - if (buf_idx >= 0) - cm->cur_frame->ref_frame_offset[ref_frame - LAST_FRAME] = - cm->buffer_pool->frame_bufs[buf_idx].cur_frame_offset; + const RefCntBuffer *const buf = get_ref_frame_buf(cm, ref_frame); + if (buf != NULL) + cm->cur_frame->ref_order_hints[ref_frame - LAST_FRAME] = buf->order_hint; } } void av1_setup_frame_sign_bias(AV1_COMMON *cm) { MV_REFERENCE_FRAME ref_frame; for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) { - const int buf_idx = cm->frame_refs[ref_frame - LAST_FRAME].idx; - if (cm->seq_params.enable_order_hint && buf_idx != INVALID_IDX) { - const int ref_frame_offset = - cm->buffer_pool->frame_bufs[buf_idx].cur_frame_offset; + const RefCntBuffer *const buf = get_ref_frame_buf(cm, ref_frame); + if (cm->seq_params.order_hint_info.enable_order_hint && buf != NULL) { + const int ref_order_hint = buf->order_hint; cm->ref_frame_sign_bias[ref_frame] = - (get_relative_dist(cm, ref_frame_offset, (int)cm->frame_offset) <= 0) + (get_relative_dist(&cm->seq_params.order_hint_info, ref_order_hint, + (int)cm->current_frame.order_hint) <= 0) ? 0 : 1; } else { @@ -935,33 +937,34 @@ static int motion_field_projection(AV1_COMMON *cm, TPL_MV_REF *tpl_mvs_base = cm->tpl_mvs; int ref_offset[REF_FRAMES] = { 0 }; - (void)dir; + const RefCntBuffer *const start_frame_buf = + get_ref_frame_buf(cm, start_frame); + if (start_frame_buf == NULL) return 0; - const int start_frame_idx = cm->frame_refs[FWD_RF_OFFSET(start_frame)].idx; - if (start_frame_idx < 0) return 0; - - if (cm->buffer_pool->frame_bufs[start_frame_idx].intra_only) return 0; - - if (cm->buffer_pool->frame_bufs[start_frame_idx].mi_rows != cm->mi_rows || - cm->buffer_pool->frame_bufs[start_frame_idx].mi_cols != cm->mi_cols) + if (start_frame_buf->frame_type == KEY_FRAME || + start_frame_buf->frame_type == INTRA_ONLY_FRAME) return 0; - const int start_frame_offset = - cm->buffer_pool->frame_bufs[start_frame_idx].cur_frame_offset; - const unsigned int *const ref_frame_offsets = - &cm->buffer_pool->frame_bufs[start_frame_idx].ref_frame_offset[0]; - const int cur_frame_offset = cm->cur_frame->cur_frame_offset; - int start_to_current_frame_offset = - get_relative_dist(cm, start_frame_offset, cur_frame_offset); + if (start_frame_buf->mi_rows != cm->mi_rows || + start_frame_buf->mi_cols != cm->mi_cols) + return 0; + + const int start_frame_order_hint = start_frame_buf->order_hint; + const unsigned int *const ref_order_hints = + &start_frame_buf->ref_order_hints[0]; + const int cur_order_hint = cm->cur_frame->order_hint; + int start_to_current_frame_offset = get_relative_dist( + &cm->seq_params.order_hint_info, start_frame_order_hint, cur_order_hint); for (MV_REFERENCE_FRAME rf = LAST_FRAME; rf <= INTER_REFS_PER_FRAME; ++rf) { - ref_offset[rf] = get_relative_dist(cm, start_frame_offset, - ref_frame_offsets[rf - LAST_FRAME]); + ref_offset[rf] = get_relative_dist(&cm->seq_params.order_hint_info, + start_frame_order_hint, + ref_order_hints[rf - LAST_FRAME]); } if (dir == 2) start_to_current_frame_offset = -start_to_current_frame_offset; - MV_REF *mv_ref_base = cm->buffer_pool->frame_bufs[start_frame_idx].mvs; + MV_REF *mv_ref_base = start_frame_buf->mvs; const int mvs_rows = (cm->mi_rows + 1) >> 1; const int mvs_cols = (cm->mi_cols + 1) >> 1; @@ -1002,8 +1005,10 @@ static int motion_field_projection(AV1_COMMON *cm, } void av1_setup_motion_field(AV1_COMMON *cm) { + const OrderHintInfo *const order_hint_info = &cm->seq_params.order_hint_info; + memset(cm->ref_frame_side, 0, sizeof(cm->ref_frame_side)); - if (!cm->seq_params.enable_order_hint) return; + if (!order_hint_info->enable_order_hint) return; TPL_MV_REF *tpl_mvs_base = cm->tpl_mvs; int size = ((cm->mi_rows + MAX_MIB_SIZE) >> 1) * (cm->mi_stride >> 1); @@ -1012,23 +1017,22 @@ void av1_setup_motion_field(AV1_COMMON *cm) { tpl_mvs_base[idx].ref_frame_offset = 0; } - const int cur_order_hint = cm->cur_frame->cur_frame_offset; - RefCntBuffer *const frame_bufs = cm->buffer_pool->frame_bufs; + const int cur_order_hint = cm->cur_frame->order_hint; - int ref_buf_idx[INTER_REFS_PER_FRAME]; + const RefCntBuffer *ref_buf[INTER_REFS_PER_FRAME]; int ref_order_hint[INTER_REFS_PER_FRAME]; for (int ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ref_frame++) { const int ref_idx = ref_frame - LAST_FRAME; - const int buf_idx = cm->frame_refs[ref_idx].idx; + const RefCntBuffer *const buf = get_ref_frame_buf(cm, ref_frame); int order_hint = 0; - if (buf_idx >= 0) order_hint = frame_bufs[buf_idx].cur_frame_offset; + if (buf != NULL) order_hint = buf->order_hint; - ref_buf_idx[ref_idx] = buf_idx; + ref_buf[ref_idx] = buf; ref_order_hint[ref_idx] = order_hint; - if (get_relative_dist(cm, order_hint, cur_order_hint) > 0) + if (get_relative_dist(order_hint_info, order_hint, cur_order_hint) > 0) cm->ref_frame_side[ref_frame] = 1; else if (order_hint == cur_order_hint) cm->ref_frame_side[ref_frame] = -1; @@ -1036,10 +1040,10 @@ void av1_setup_motion_field(AV1_COMMON *cm) { int ref_stamp = MFMV_STACK_SIZE - 1; - if (ref_buf_idx[LAST_FRAME - LAST_FRAME] >= 0) { + if (ref_buf[LAST_FRAME - LAST_FRAME] != NULL) { const int alt_of_lst_order_hint = - frame_bufs[ref_buf_idx[LAST_FRAME - LAST_FRAME]] - .ref_frame_offset[ALTREF_FRAME - LAST_FRAME]; + ref_buf[LAST_FRAME - LAST_FRAME] + ->ref_order_hints[ALTREF_FRAME - LAST_FRAME]; const int is_lst_overlay = (alt_of_lst_order_hint == ref_order_hint[GOLDEN_FRAME - LAST_FRAME]); @@ -1047,23 +1051,25 @@ void av1_setup_motion_field(AV1_COMMON *cm) { --ref_stamp; } - if (get_relative_dist(cm, ref_order_hint[BWDREF_FRAME - LAST_FRAME], + if (get_relative_dist(order_hint_info, + ref_order_hint[BWDREF_FRAME - LAST_FRAME], cur_order_hint) > 0) { if (motion_field_projection(cm, BWDREF_FRAME, 0)) --ref_stamp; } - if (get_relative_dist(cm, ref_order_hint[ALTREF2_FRAME - LAST_FRAME], + if (get_relative_dist(order_hint_info, + ref_order_hint[ALTREF2_FRAME - LAST_FRAME], cur_order_hint) > 0) { if (motion_field_projection(cm, ALTREF2_FRAME, 0)) --ref_stamp; } - if (get_relative_dist(cm, ref_order_hint[ALTREF_FRAME - LAST_FRAME], + if (get_relative_dist(order_hint_info, + ref_order_hint[ALTREF_FRAME - LAST_FRAME], cur_order_hint) > 0 && ref_stamp >= 0) if (motion_field_projection(cm, ALTREF_FRAME, 0)) --ref_stamp; - if (ref_stamp >= 0 && ref_buf_idx[LAST2_FRAME - LAST_FRAME] >= 0) - if (motion_field_projection(cm, LAST2_FRAME, 2)) --ref_stamp; + if (ref_stamp >= 0) motion_field_projection(cm, LAST2_FRAME, 2); } static INLINE void record_samples(MB_MODE_INFO *mbmi, int *pts, int *pts_inref, @@ -1264,36 +1270,43 @@ int findSamples(const AV1_COMMON *cm, MACROBLOCKD *xd, int mi_row, int mi_col, } void av1_setup_skip_mode_allowed(AV1_COMMON *cm) { - cm->is_skip_mode_allowed = 0; - cm->ref_frame_idx_0 = cm->ref_frame_idx_1 = INVALID_IDX; + const OrderHintInfo *const order_hint_info = &cm->seq_params.order_hint_info; + SkipModeInfo *const skip_mode_info = &cm->current_frame.skip_mode_info; - if (!cm->seq_params.enable_order_hint || frame_is_intra_only(cm) || - cm->reference_mode == SINGLE_REFERENCE) + skip_mode_info->skip_mode_allowed = 0; + skip_mode_info->ref_frame_idx_0 = INVALID_IDX; + skip_mode_info->ref_frame_idx_1 = INVALID_IDX; + + if (!order_hint_info->enable_order_hint || frame_is_intra_only(cm) || + cm->current_frame.reference_mode == SINGLE_REFERENCE) return; - RefCntBuffer *const frame_bufs = cm->buffer_pool->frame_bufs; - const int cur_frame_offset = cm->frame_offset; - int ref_frame_offset[2] = { -1, INT_MAX }; + const int cur_order_hint = cm->current_frame.order_hint; + int ref_order_hints[2] = { -1, INT_MAX }; int ref_idx[2] = { INVALID_IDX, INVALID_IDX }; // Identify the nearest forward and backward references. for (int i = 0; i < INTER_REFS_PER_FRAME; ++i) { - const int buf_idx = cm->frame_refs[i].idx; - if (buf_idx == INVALID_IDX) continue; + const RefCntBuffer *const buf = get_ref_frame_buf(cm, LAST_FRAME + i); + if (buf == NULL) continue; - const int ref_offset = frame_bufs[buf_idx].cur_frame_offset; - if (get_relative_dist(cm, ref_offset, cur_frame_offset) < 0) { + const int ref_order_hint = buf->order_hint; + if (get_relative_dist(order_hint_info, ref_order_hint, cur_order_hint) < + 0) { // Forward reference - if (ref_frame_offset[0] == -1 || - get_relative_dist(cm, ref_offset, ref_frame_offset[0]) > 0) { - ref_frame_offset[0] = ref_offset; + if (ref_order_hints[0] == -1 || + get_relative_dist(order_hint_info, ref_order_hint, + ref_order_hints[0]) > 0) { + ref_order_hints[0] = ref_order_hint; ref_idx[0] = i; } - } else if (get_relative_dist(cm, ref_offset, cur_frame_offset) > 0) { + } else if (get_relative_dist(order_hint_info, ref_order_hint, + cur_order_hint) > 0) { // Backward reference - if (ref_frame_offset[1] == INT_MAX || - get_relative_dist(cm, ref_offset, ref_frame_offset[1]) < 0) { - ref_frame_offset[1] = ref_offset; + if (ref_order_hints[1] == INT_MAX || + get_relative_dist(order_hint_info, ref_order_hint, + ref_order_hints[1]) < 0) { + ref_order_hints[1] = ref_order_hint; ref_idx[1] = i; } } @@ -1301,75 +1314,71 @@ void av1_setup_skip_mode_allowed(AV1_COMMON *cm) { if (ref_idx[0] != INVALID_IDX && ref_idx[1] != INVALID_IDX) { // == Bi-directional prediction == - cm->is_skip_mode_allowed = 1; - cm->ref_frame_idx_0 = AOMMIN(ref_idx[0], ref_idx[1]); - cm->ref_frame_idx_1 = AOMMAX(ref_idx[0], ref_idx[1]); + skip_mode_info->skip_mode_allowed = 1; + skip_mode_info->ref_frame_idx_0 = AOMMIN(ref_idx[0], ref_idx[1]); + skip_mode_info->ref_frame_idx_1 = AOMMAX(ref_idx[0], ref_idx[1]); } else if (ref_idx[0] != INVALID_IDX && ref_idx[1] == INVALID_IDX) { // == Forward prediction only == // Identify the second nearest forward reference. - ref_frame_offset[1] = -1; + ref_order_hints[1] = -1; for (int i = 0; i < INTER_REFS_PER_FRAME; ++i) { - const int buf_idx = cm->frame_refs[i].idx; - if (buf_idx == INVALID_IDX) continue; + const RefCntBuffer *const buf = get_ref_frame_buf(cm, LAST_FRAME + i); + if (buf == NULL) continue; - const int ref_offset = frame_bufs[buf_idx].cur_frame_offset; - if ((ref_frame_offset[0] != -1 && - get_relative_dist(cm, ref_offset, ref_frame_offset[0]) < 0) && - (ref_frame_offset[1] == -1 || - get_relative_dist(cm, ref_offset, ref_frame_offset[1]) > 0)) { + const int ref_order_hint = buf->order_hint; + if ((ref_order_hints[0] != -1 && + get_relative_dist(order_hint_info, ref_order_hint, + ref_order_hints[0]) < 0) && + (ref_order_hints[1] == -1 || + get_relative_dist(order_hint_info, ref_order_hint, + ref_order_hints[1]) > 0)) { // Second closest forward reference - ref_frame_offset[1] = ref_offset; + ref_order_hints[1] = ref_order_hint; ref_idx[1] = i; } } - if (ref_frame_offset[1] != -1) { - cm->is_skip_mode_allowed = 1; - cm->ref_frame_idx_0 = AOMMIN(ref_idx[0], ref_idx[1]); - cm->ref_frame_idx_1 = AOMMAX(ref_idx[0], ref_idx[1]); + if (ref_order_hints[1] != -1) { + skip_mode_info->skip_mode_allowed = 1; + skip_mode_info->ref_frame_idx_0 = AOMMIN(ref_idx[0], ref_idx[1]); + skip_mode_info->ref_frame_idx_1 = AOMMAX(ref_idx[0], ref_idx[1]); } } } typedef struct { - int map_idx; // frame map index - int buf_idx; // frame buffer index - int sort_idx; // index based on the offset to be used for sorting + int map_idx; // frame map index + RefCntBuffer *buf; // frame buffer + int sort_idx; // index based on the offset to be used for sorting } REF_FRAME_INFO; +// Compares the sort_idx fields. If they are equal, then compares the map_idx +// fields to break the tie. This ensures a stable sort. static int compare_ref_frame_info(const void *arg_a, const void *arg_b) { const REF_FRAME_INFO *info_a = (REF_FRAME_INFO *)arg_a; const REF_FRAME_INFO *info_b = (REF_FRAME_INFO *)arg_b; - if (info_a->sort_idx < info_b->sort_idx) return -1; - if (info_a->sort_idx > info_b->sort_idx) return 1; - return (info_a->map_idx < info_b->map_idx) - ? -1 - : ((info_a->map_idx > info_b->map_idx) ? 1 : 0); + const int sort_idx_diff = info_a->sort_idx - info_b->sort_idx; + if (sort_idx_diff != 0) return sort_idx_diff; + return info_a->map_idx - info_b->map_idx; } static void set_ref_frame_info(AV1_COMMON *const cm, int frame_idx, REF_FRAME_INFO *ref_info) { assert(frame_idx >= 0 && frame_idx < INTER_REFS_PER_FRAME); - const int buf_idx = ref_info->buf_idx; - - cm->frame_refs[frame_idx].idx = buf_idx; - cm->frame_refs[frame_idx].buf = &cm->buffer_pool->frame_bufs[buf_idx].buf; - cm->frame_refs[frame_idx].map_idx = ref_info->map_idx; + cm->remapped_ref_idx[frame_idx] = ref_info->map_idx; } void av1_set_frame_refs(AV1_COMMON *const cm, int lst_map_idx, int gld_map_idx) { - BufferPool *const pool = cm->buffer_pool; - RefCntBuffer *const frame_bufs = pool->frame_bufs; - int lst_frame_sort_idx = -1; int gld_frame_sort_idx = -1; - assert(cm->seq_params.enable_order_hint); - assert(cm->seq_params.order_hint_bits_minus_1 >= 0); - const int cur_frame_offset = (int)cm->frame_offset; - const int cur_frame_sort_idx = 1 << cm->seq_params.order_hint_bits_minus_1; + assert(cm->seq_params.order_hint_info.enable_order_hint); + assert(cm->seq_params.order_hint_info.order_hint_bits_minus_1 >= 0); + const int cur_order_hint = (int)cm->current_frame.order_hint; + const int cur_frame_sort_idx = + 1 << cm->seq_params.order_hint_info.order_hint_bits_minus_1; REF_FRAME_INFO ref_frame_info[REF_FRAMES]; int ref_flag_list[INTER_REFS_PER_FRAME] = { 0, 0, 0, 0, 0, 0, 0 }; @@ -1380,18 +1389,21 @@ void av1_set_frame_refs(AV1_COMMON *const cm, int lst_map_idx, ref_frame_info[i].map_idx = map_idx; ref_frame_info[i].sort_idx = -1; - const int buf_idx = cm->ref_frame_map[map_idx]; - ref_frame_info[i].buf_idx = buf_idx; + RefCntBuffer *const buf = cm->ref_frame_map[map_idx]; + ref_frame_info[i].buf = buf; - if (buf_idx < 0 || buf_idx >= FRAME_BUFFERS) continue; - // TODO(zoeliu@google.com): To verify the checking on ref_count. - if (frame_bufs[buf_idx].ref_count <= 0) continue; + if (buf == NULL) continue; + // If this assertion fails, there is a reference leak. + assert(buf->ref_count > 0); + // TODO(wtc@google.com): Remove the checking on ref_count after 2019-03-01. + if (buf->ref_count <= 0) continue; - const int offset = (int)frame_bufs[buf_idx].cur_frame_offset; + const int offset = (int)buf->order_hint; ref_frame_info[i].sort_idx = (offset == -1) ? -1 : cur_frame_sort_idx + - get_relative_dist(cm, offset, cur_frame_offset); + get_relative_dist(&cm->seq_params.order_hint_info, + offset, cur_order_hint); assert(ref_frame_info[i].sort_idx >= -1); if (map_idx == lst_map_idx) lst_frame_sort_idx = ref_frame_info[i].sort_idx; @@ -1414,8 +1426,8 @@ void av1_set_frame_refs(AV1_COMMON *const cm, int lst_map_idx, compare_ref_frame_info); // Identify forward and backward reference frames. - // Forward reference: offset < cur_frame_offset - // Backward reference: offset >= cur_frame_offset + // Forward reference: offset < order_hint + // Backward reference: offset >= order_hint int fwd_start_idx = 0, fwd_end_idx = REF_FRAMES - 1; for (int i = 0; i < REF_FRAMES; i++) { diff --git a/third_party/aom/av1/common/mvref_common.h b/third_party/aom/av1/common/mvref_common.h index 83f7a1ac0d..1ece84a8b1 100644 --- a/third_party/aom/av1/common/mvref_common.h +++ b/third_party/aom/av1/common/mvref_common.h @@ -34,10 +34,10 @@ typedef struct position { // clamp_mv_ref #define MV_BORDER (16 << 3) // Allow 16 pels in 1/8th pel units -static INLINE int get_relative_dist(const AV1_COMMON *cm, int a, int b) { - if (!cm->seq_params.enable_order_hint) return 0; +static INLINE int get_relative_dist(const OrderHintInfo *oh, int a, int b) { + if (!oh->enable_order_hint) return 0; - const int bits = cm->seq_params.order_hint_bits_minus_1 + 1; + const int bits = oh->order_hint_bits_minus_1 + 1; assert(bits >= 1); assert(a >= 0 && a < (1 << bits)); @@ -70,18 +70,6 @@ static INLINE int_mv get_sub_block_pred_mv(const MB_MODE_INFO *candidate, return candidate->mv[which_mv]; } -// Performs mv sign inversion if indicated by the reference frame combination. -static INLINE int_mv scale_mv(const MB_MODE_INFO *mbmi, int ref, - const MV_REFERENCE_FRAME this_ref_frame, - const int *ref_sign_bias) { - int_mv mv = mbmi->mv[ref]; - if (ref_sign_bias[mbmi->ref_frame[ref]] != ref_sign_bias[this_ref_frame]) { - mv.as_mv.row *= -1; - mv.as_mv.col *= -1; - } - return mv; -} - // Checks that the given mi_row, mi_col and search point // are inside the borders of the tile. static INLINE int is_inside(const TileInfo *const tile, int mi_col, int mi_row, @@ -169,14 +157,14 @@ static MV_REFERENCE_FRAME ref_frame_map[TOTAL_COMP_REFS][2] = { // clang-format on static INLINE void av1_set_ref_frame(MV_REFERENCE_FRAME *rf, - int8_t ref_frame_type) { + MV_REFERENCE_FRAME ref_frame_type) { if (ref_frame_type >= REF_FRAMES) { rf[0] = ref_frame_map[ref_frame_type - REF_FRAMES][0]; rf[1] = ref_frame_map[ref_frame_type - REF_FRAMES][1]; } else { + assert(ref_frame_type > NONE_FRAME); rf[0] = ref_frame_type; rf[1] = NONE_FRAME; - assert(ref_frame_type > NONE_FRAME); } } diff --git a/third_party/aom/av1/common/obu_util.c b/third_party/aom/av1/common/obu_util.c index 823b700b13..7d2694b89b 100644 --- a/third_party/aom/av1/common/obu_util.c +++ b/third_party/aom/av1/common/obu_util.c @@ -8,6 +8,8 @@ * Media Patent License 1.0 was not distributed with this source code in the * PATENTS file, you can obtain it at www.aomedia.org/license/patent. */ +#include + #include "av1/common/obu_util.h" #include "aom_dsp/bitreader_buffer.h" @@ -112,36 +114,41 @@ aom_codec_err_t aom_read_obu_header_and_size(const uint8_t *data, ObuHeader *obu_header, size_t *const payload_size, size_t *const bytes_read) { - size_t length_field_size = 0, obu_size = 0; + size_t length_field_size_obu = 0; + size_t length_field_size_payload = 0; + size_t obu_size = 0; aom_codec_err_t status; if (is_annexb) { // Size field comes before the OBU header, and includes the OBU header status = - read_obu_size(data, bytes_available, &obu_size, &length_field_size); + read_obu_size(data, bytes_available, &obu_size, &length_field_size_obu); if (status != AOM_CODEC_OK) return status; } - struct aom_read_bit_buffer rb = { data + length_field_size, + struct aom_read_bit_buffer rb = { data + length_field_size_obu, data + bytes_available, 0, NULL, NULL }; status = read_obu_header(&rb, is_annexb, obu_header); if (status != AOM_CODEC_OK) return status; - if (is_annexb) { + if (!obu_header->has_size_field) { + assert(is_annexb); // Derive the payload size from the data we've already read if (obu_size < obu_header->size) return AOM_CODEC_CORRUPT_FRAME; *payload_size = obu_size - obu_header->size; } else { // Size field comes after the OBU header, and is just the payload size - status = read_obu_size(data + obu_header->size, - bytes_available - obu_header->size, payload_size, - &length_field_size); + status = read_obu_size( + data + length_field_size_obu + obu_header->size, + bytes_available - length_field_size_obu - obu_header->size, + payload_size, &length_field_size_payload); if (status != AOM_CODEC_OK) return status; } - *bytes_read = length_field_size + obu_header->size; + *bytes_read = + length_field_size_obu + obu_header->size + length_field_size_payload; return AOM_CODEC_OK; } diff --git a/third_party/aom/av1/common/onyxc_int.h b/third_party/aom/av1/common/onyxc_int.h index ff011c89e9..56f823ec92 100644 --- a/third_party/aom/av1/common/onyxc_int.h +++ b/third_party/aom/av1/common/onyxc_int.h @@ -79,14 +79,14 @@ extern "C" { #define TXCOEFF_TIMER 0 #define TXCOEFF_COST_TIMER 0 -typedef enum { +enum { SINGLE_REFERENCE = 0, COMPOUND_REFERENCE = 1, REFERENCE_MODE_SELECT = 2, REFERENCE_MODES = 3, -} REFERENCE_MODE; +} UENUM1BYTE(REFERENCE_MODE); -typedef enum { +enum { /** * Frame context updates are disabled */ @@ -96,7 +96,7 @@ typedef enum { * updates based on entropy/counts in the decoded frame */ REFRESH_FRAME_CONTEXT_BACKWARD, -} REFRESH_FRAME_CONTEXT_MODE; +} UENUM1BYTE(REFRESH_FRAME_CONTEXT_MODE); #define MFMV_STACK_SIZE 3 typedef struct { @@ -109,11 +109,39 @@ typedef struct { MV_REFERENCE_FRAME ref_frame; } MV_REF; -typedef struct { +// FIXME(jack.haughton@argondesign.com): This enum was originally in +// encoder/ratectrl.h, and is encoder specific. When we move to C++, this +// should go back there and BufferPool should be templatized. +enum { + INTER_NORMAL = 0, + INTER_LOW = 1, + INTER_HIGH = 2, + GF_ARF_LOW = 3, + GF_ARF_STD = 4, + KF_STD = 5, + RATE_FACTOR_LEVELS = 6 +} UENUM1BYTE(RATE_FACTOR_LEVEL); + +typedef struct RefCntBuffer { + // For a RefCntBuffer, the following are reference-holding variables: + // - cm->ref_frame_map[] + // - cm->cur_frame + // - cm->scaled_ref_buf[] (encoder only) + // - cm->next_ref_frame_map[] (decoder only) + // - pbi->output_frame_index[] (decoder only) + // With that definition, 'ref_count' is the number of reference-holding + // variables that are currently referencing this buffer. + // For example: + // - suppose this buffer is at index 'k' in the buffer pool, and + // - Total 'n' of the variables / array elements above have value 'k' (that + // is, they are pointing to buffer at index 'k'). + // Then, pool->frame_bufs[k].ref_count = n. + // TODO(david.turner@argondesign.com) Check whether this helpful comment is + // still correct after we finish restructuring int ref_count; - unsigned int cur_frame_offset; - unsigned int ref_frame_offset[INTER_REFS_PER_FRAME]; + unsigned int order_hint; + unsigned int ref_order_hints[INTER_REFS_PER_FRAME]; MV_REF *mvs; uint8_t *seg_map; @@ -126,36 +154,29 @@ typedef struct { int height; WarpedMotionParams global_motion[REF_FRAMES]; int showable_frame; // frame can be used as show existing frame in future - int film_grain_params_present; + uint8_t film_grain_params_present; aom_film_grain_t film_grain_params; aom_codec_frame_buffer_t raw_frame_buffer; YV12_BUFFER_CONFIG buf; hash_table hash_table; - uint8_t intra_only; FRAME_TYPE frame_type; - // The Following variables will only be used in frame parallel decode. - - // frame_worker_owner indicates which FrameWorker owns this buffer. NULL means - // that no FrameWorker owns, or is decoding, this buffer. - AVxWorker *frame_worker_owner; - - // row and col indicate which position frame has been decoded to in real - // pixel unit. They are reset to -1 when decoding begins and set to INT_MAX - // when the frame is fully decoded. - int row; - int col; // Inter frame reference frame delta for loop filter int8_t ref_deltas[REF_FRAMES]; // 0 = ZERO_MV, MV int8_t mode_deltas[MAX_MODE_LF_DELTAS]; + + FRAME_CONTEXT frame_context; + RATE_FACTOR_LEVEL frame_rf_level; } RefCntBuffer; typedef struct BufferPool { // Protect BufferPool from being accessed by several FrameWorkers at // the same time during frame parallel decode. // TODO(hkuang): Try to use atomic variable instead of locking the whole pool. +// TODO(wtc): Remove this. See +// https://chromium-review.googlesource.com/c/webm/libvpx/+/560630. #if CONFIG_MULTITHREAD pthread_mutex_t pool_mutex; #endif @@ -172,18 +193,45 @@ typedef struct BufferPool { InternalFrameBufferList int_frame_buffers; } BufferPool; -typedef struct { - int base_ctx_table[2 /*row*/][2 /*col*/][3 /*sig_map*/] - [BASE_CONTEXT_POSITION_NUM + 1]; -} LV_MAP_CTX_TABLE; -typedef int BASE_CTX_TABLE[2 /*col*/][3 /*sig_map*/] - [BASE_CONTEXT_POSITION_NUM + 1]; - typedef struct BitstreamLevel { uint8_t major; uint8_t minor; } BitstreamLevel; +typedef struct { + int cdef_pri_damping; + int cdef_sec_damping; + int nb_cdef_strengths; + int cdef_strengths[CDEF_MAX_STRENGTHS]; + int cdef_uv_strengths[CDEF_MAX_STRENGTHS]; + int cdef_bits; +} CdefInfo; + +typedef struct { + int delta_q_present_flag; + // Resolution of delta quant + int delta_q_res; + int delta_lf_present_flag; + // Resolution of delta lf level + int delta_lf_res; + // This is a flag for number of deltas of loop filter level + // 0: use 1 delta, for y_vertical, y_horizontal, u, and v + // 1: use separate deltas for each filter level + int delta_lf_multi; +} DeltaQInfo; + +typedef struct { + int enable_order_hint; // 0 - disable order hint, and related tools + int order_hint_bits_minus_1; // dist_wtd_comp, ref_frame_mvs, + // frame_sign_bias + // if 0, enable_dist_wtd_comp and + // enable_ref_frame_mvs must be set as 0. + int enable_dist_wtd_comp; // 0 - disable dist-wtd compound modes + // 1 - enable it + int enable_ref_frame_mvs; // 0 - disable ref frame mvs + // 1 - enable it +} OrderHintInfo; + // Sequence header structure. // Note: All syntax elements of sequence_header_obu that need to be // bit-identical across multiple sequence headers must be part of this struct, @@ -193,51 +241,44 @@ typedef struct SequenceHeader { int num_bits_height; int max_frame_width; int max_frame_height; - int frame_id_numbers_present_flag; + uint8_t frame_id_numbers_present_flag; int frame_id_length; int delta_frame_id_length; BLOCK_SIZE sb_size; // Size of the superblock used for this frame int mib_size; // Size of the superblock in units of MI blocks int mib_size_log2; // Log 2 of above. - int order_hint_bits_minus_1; - int force_screen_content_tools; // 0 - force off - // 1 - force on - // 2 - adaptive - int force_integer_mv; // 0 - Not to force. MV can be in 1/4 or 1/8 - // 1 - force to integer - // 2 - adaptive - int still_picture; // Video is a single frame still picture - int reduced_still_picture_hdr; // Use reduced header for still picture - int enable_filter_intra; // enables/disables filterintra - int enable_intra_edge_filter; // enables/disables corner/edge/upsampling - int enable_interintra_compound; // enables/disables interintra_compound - int enable_masked_compound; // enables/disables masked compound - int enable_dual_filter; // 0 - disable dual interpolation filter - // 1 - enable vert/horiz filter selection - int enable_order_hint; // 0 - disable order hint, and related tools - // jnt_comp, ref_frame_mvs, frame_sign_bias - // if 0, enable_jnt_comp and - // enable_ref_frame_mvs must be set zs 0. - int enable_jnt_comp; // 0 - disable joint compound modes - // 1 - enable it - int enable_ref_frame_mvs; // 0 - disable ref frame mvs - // 1 - enable it - int enable_warped_motion; // 0 - disable warped motion for sequence - // 1 - enable it for the sequence - int enable_superres; // 0 - Disable superres for the sequence, and disable - // transmitting per-frame superres enabled flag. - // 1 - Enable superres for the sequence, and also - // enable per-frame flag to denote if superres is - // enabled for that frame. - int enable_cdef; // To turn on/off CDEF - int enable_restoration; // To turn on/off loop restoration + + OrderHintInfo order_hint_info; + + uint8_t force_screen_content_tools; // 0 - force off + // 1 - force on + // 2 - adaptive + uint8_t still_picture; // Video is a single frame still picture + uint8_t reduced_still_picture_hdr; // Use reduced header for still picture + uint8_t force_integer_mv; // 0 - Don't force. MV can use subpel + // 1 - force to integer + // 2 - adaptive + uint8_t enable_filter_intra; // enables/disables filterintra + uint8_t enable_intra_edge_filter; // enables/disables edge upsampling + uint8_t enable_interintra_compound; // enables/disables interintra_compound + uint8_t enable_masked_compound; // enables/disables masked compound + uint8_t enable_dual_filter; // 0 - disable dual interpolation filter + // 1 - enable vert/horz filter selection + uint8_t enable_warped_motion; // 0 - disable warp for the sequence + // 1 - enable warp for the sequence + uint8_t enable_superres; // 0 - Disable superres for the sequence + // and no frame level superres flag + // 1 - Enable superres for the sequence + // enable per-frame superres flag + uint8_t enable_cdef; // To turn on/off CDEF + uint8_t enable_restoration; // To turn on/off loop restoration BITSTREAM_PROFILE profile; // Operating point info. int operating_points_cnt_minus_1; int operating_point_idc[MAX_NUM_OPERATING_POINTS]; - int display_model_info_present_flag; - int decoder_model_info_present_flag; + uint8_t display_model_info_present_flag; + uint8_t decoder_model_info_present_flag; BitstreamLevel level[MAX_NUM_OPERATING_POINTS]; uint8_t tier[MAX_NUM_OPERATING_POINTS]; // seq_tier in the spec. One bit: 0 // or 1. @@ -245,8 +286,8 @@ typedef struct SequenceHeader { // Color config. aom_bit_depth_t bit_depth; // AOM_BITS_8 in profile 0 or 1, // AOM_BITS_10 or AOM_BITS_12 in profile 2 or 3. - int use_highbitdepth; // If true, we need to use 16bit frame buffers. - int monochrome; // Monochorme video + uint8_t use_highbitdepth; // If true, we need to use 16bit frame buffers. + uint8_t monochrome; // Monochorme video aom_color_primaries_t color_primaries; aom_transfer_characteristics_t transfer_characteristics; aom_matrix_coefficients_t matrix_coefficients; @@ -254,19 +295,34 @@ typedef struct SequenceHeader { int subsampling_x; // Chroma subsampling for x int subsampling_y; // Chroma subsampling for y aom_chroma_sample_position_t chroma_sample_position; - int separate_uv_delta_q; - - int film_grain_params_present; + uint8_t separate_uv_delta_q; + uint8_t film_grain_params_present; } SequenceHeader; +typedef struct { + int skip_mode_allowed; + int skip_mode_flag; + int ref_frame_idx_0; + int ref_frame_idx_1; +} SkipModeInfo; + +typedef struct { + FRAME_TYPE frame_type; + REFERENCE_MODE reference_mode; + + unsigned int order_hint; + unsigned int frame_number; + SkipModeInfo skip_mode_info; + int refresh_frame_flags; // Which ref frames are overwritten by this frame +} CurrentFrame; + typedef struct AV1Common { + CurrentFrame current_frame; struct aom_internal_error_info error; int width; int height; int render_width; int render_height; - int last_width; - int last_height; int timing_info_present; aom_timing_info_t timing_info; int buffer_removal_time_present; @@ -275,56 +331,59 @@ typedef struct AV1Common { aom_op_timing_info_t op_frame_timing[MAX_NUM_OPERATING_POINTS + 1]; uint32_t frame_presentation_time; - int largest_tile_id; - size_t largest_tile_size; int context_update_tile_id; // Scale of the current frame with respect to itself. struct scale_factors sf_identity; - YV12_BUFFER_CONFIG *frame_to_show; RefCntBuffer *prev_frame; // TODO(hkuang): Combine this with cur_buf in macroblockd. RefCntBuffer *cur_frame; - int ref_frame_map[REF_FRAMES]; /* maps fb_idx to reference slot */ + // For encoder, we have a two-level mapping from reference frame type to the + // corresponding buffer in the buffer pool: + // * 'remapped_ref_idx[i - 1]' maps reference type ‘i’ (range: LAST_FRAME ... + // EXTREF_FRAME) to a remapped index ‘j’ (in range: 0 ... REF_FRAMES - 1) + // * Later, 'cm->ref_frame_map[j]' maps the remapped index ‘j’ to a pointer to + // the reference counted buffer structure RefCntBuffer, taken from the buffer + // pool cm->buffer_pool->frame_bufs. + // + // LAST_FRAME, ..., EXTREF_FRAME + // | | + // v v + // remapped_ref_idx[LAST_FRAME - 1], ..., remapped_ref_idx[EXTREF_FRAME - 1] + // | | + // v v + // ref_frame_map[], ..., ref_frame_map[] + // + // Note: INTRA_FRAME always refers to the current frame, so there's no need to + // have a remapped index for the same. + int remapped_ref_idx[REF_FRAMES]; + + struct scale_factors ref_scale_factors[REF_FRAMES]; + + // For decoder, ref_frame_map[i] maps reference type 'i' to a pointer to + // the buffer in the buffer pool ‘cm->buffer_pool.frame_bufs’. + // For encoder, ref_frame_map[j] (where j = remapped_ref_idx[i]) maps + // remapped reference index 'j' (that is, original reference type 'i') to + // a pointer to the buffer in the buffer pool ‘cm->buffer_pool.frame_bufs’. + RefCntBuffer *ref_frame_map[REF_FRAMES]; // Prepare ref_frame_map for the next frame. // Only used in frame parallel decode. - int next_ref_frame_map[REF_FRAMES]; - - // TODO(jkoleszar): could expand active_ref_idx to 4, with 0 as intra, and - // roll new_fb_idx into it. - - // Each Inter frame can reference INTER_REFS_PER_FRAME buffers - RefBuffer frame_refs[INTER_REFS_PER_FRAME]; - int is_skip_mode_allowed; - int skip_mode_flag; - int ref_frame_idx_0; - int ref_frame_idx_1; - - int new_fb_idx; - + RefCntBuffer *next_ref_frame_map[REF_FRAMES]; FRAME_TYPE last_frame_type; /* last frame's frame type for motion search.*/ - FRAME_TYPE frame_type; int show_frame; int showable_frame; // frame can be used as show existing frame in future - int last_show_frame; int show_existing_frame; - // Flag for a frame used as a reference - not written to the bitstream - int is_reference_frame; - int reset_decoder_state; - // Flag signaling that the frame is encoded using only INTRA modes. - uint8_t intra_only; - uint8_t last_intra_only; uint8_t disable_cdf_update; int allow_high_precision_mv; - int cur_frame_force_integer_mv; // 0 the default in AOM, 1 only integer + uint8_t cur_frame_force_integer_mv; // 0 the default in AOM, 1 only integer - int allow_screen_content_tools; + uint8_t allow_screen_content_tools; int allow_intrabc; int allow_warped_motion; @@ -349,7 +408,7 @@ typedef struct AV1Common { int u_ac_delta_q; int v_ac_delta_q; - // The dequantizers below are true dequntizers used only in the + // The dequantizers below are true dequantizers used only in the // dequantization process. They have the same coefficient // shift/scale as TX. int16_t y_dequant_QTX[MAX_SEGMENTS][2]; @@ -372,6 +431,7 @@ typedef struct AV1Common { int qm_v; int min_qmlevel; int max_qmlevel; + int use_quant_b_adapt; /* We allocate a MB_MODE_INFO struct for each macroblock, together with an extra row on top and column on the left to simplify prediction. */ @@ -400,8 +460,6 @@ typedef struct AV1Common { int allow_ref_frame_mvs; uint8_t *last_frame_seg_map; - uint8_t *current_frame_seg_map; - int seg_map_alloc_size; InterpFilter interp_filter; @@ -414,10 +472,6 @@ typedef struct AV1Common { int superres_upscaled_height; RestorationInfo rst_info[MAX_MB_PLANE]; - // rst_end_stripe[i] is one more than the index of the bottom stripe - // for tile row i. - int rst_end_stripe[MAX_TILE_ROWS]; - // Pointer to a scratch buffer used by self-guided restoration int32_t *rst_tmpbuf; RestorationLineBuffers *rlbs; @@ -441,25 +495,14 @@ typedef struct AV1Common { // Context probabilities for reference frame prediction MV_REFERENCE_FRAME comp_fwd_ref[FWD_REFS]; MV_REFERENCE_FRAME comp_bwd_ref[BWD_REFS]; - REFERENCE_MODE reference_mode; FRAME_CONTEXT *fc; /* this frame entropy */ - FRAME_CONTEXT *frame_contexts; // FRAME_CONTEXTS - unsigned int frame_context_idx; /* Context to use/update */ - int fb_of_context_type[REF_FRAMES]; + FRAME_CONTEXT *default_frame_context; int primary_ref_frame; - unsigned int frame_offset; - - unsigned int current_video_frame; - - aom_bit_depth_t dequant_bit_depth; // bit_depth of current dequantizer - int error_resilient_mode; - int force_primary_ref_none; int tile_cols, tile_rows; - int last_tile_cols, last_tile_rows; int max_tile_width_sb; int min_log2_tile_cols; @@ -482,14 +525,6 @@ typedef struct AV1Common { int skip_loop_filter; int skip_film_grain; - // Private data associated with the frame buffer callbacks. - void *cb_priv; - aom_get_frame_buffer_cb_fn_t get_fb_cb; - aom_release_frame_buffer_cb_fn_t release_fb_cb; - - // Handles memory for the codec. - InternalFrameBufferList int_frame_buffers; - // External BufferPool passed from outside. BufferPool *buffer_pool; @@ -499,30 +534,14 @@ typedef struct AV1Common { WarpedMotionParams global_motion[REF_FRAMES]; aom_film_grain_t film_grain_params; - int cdef_pri_damping; - int cdef_sec_damping; - int nb_cdef_strengths; - int cdef_strengths[CDEF_MAX_STRENGTHS]; - int cdef_uv_strengths[CDEF_MAX_STRENGTHS]; - int cdef_bits; + CdefInfo cdef_info; + DeltaQInfo delta_q_info; // Delta Q and Delta LF parameters - int delta_q_present_flag; - // Resolution of delta quant - int delta_q_res; - int delta_lf_present_flag; - // Resolution of delta lf level - int delta_lf_res; - // This is a flag for number of deltas of loop filter level - // 0: use 1 delta, for y_vertical, y_horizontal, u, and v - // 1: use separate deltas for each filter level - int delta_lf_multi; int num_tg; SequenceHeader seq_params; int current_frame_id; int ref_frame_id[REF_FRAMES]; int valid_for_referencing[REF_FRAMES]; - int invalid_delta_frame_id_minus_1; - LV_MAP_CTX_TABLE coeff_ctx_table; TPL_MV_REF *tpl_mvs; int tpl_mvs_mem_size; // TODO(jingning): This can be combined with sign_bias later. @@ -530,7 +549,6 @@ typedef struct AV1Common { int is_annexb; - int frame_refs_short_signaling; int temporal_layer_id; int spatial_layer_id; unsigned int number_temporal_layers; @@ -551,6 +569,7 @@ typedef struct AV1Common { int64_t txcoeff_cost_count; #endif const cfg_options_t *options; + int is_decoding; } AV1_COMMON; // TODO(hkuang): Don't need to lock the whole pool after implementing atomic @@ -573,14 +592,8 @@ static void unlock_buffer_pool(BufferPool *const pool) { static INLINE YV12_BUFFER_CONFIG *get_ref_frame(AV1_COMMON *cm, int index) { if (index < 0 || index >= REF_FRAMES) return NULL; - if (cm->ref_frame_map[index] < 0) return NULL; - assert(cm->ref_frame_map[index] < FRAME_BUFFERS); - return &cm->buffer_pool->frame_bufs[cm->ref_frame_map[index]].buf; -} - -static INLINE YV12_BUFFER_CONFIG *get_frame_new_buffer( - const AV1_COMMON *const cm) { - return &cm->buffer_pool->frame_bufs[cm->new_fb_idx].buf; + if (cm->ref_frame_map[index] == NULL) return NULL; + return &cm->ref_frame_map[index]->buf; } static INLINE int get_free_fb(AV1_COMMON *cm) { @@ -605,6 +618,9 @@ static INLINE int get_free_fb(AV1_COMMON *cm) { frame_bufs[i].ref_count = 1; } else { + // We should never run out of free buffers. If this assertion fails, there + // is a reference leak. + assert(0 && "Ran out of free frame buffers. Likely a reference leak."); // Reset i to be INVALID_IDX to indicate no free buffer found. i = INVALID_IDX; } @@ -613,39 +629,90 @@ static INLINE int get_free_fb(AV1_COMMON *cm) { return i; } -static INLINE void ref_cnt_fb(RefCntBuffer *bufs, int *idx, int new_idx) { - const int ref_index = *idx; +static INLINE RefCntBuffer *assign_cur_frame_new_fb(AV1_COMMON *const cm) { + // Release the previously-used frame-buffer + if (cm->cur_frame != NULL) { + --cm->cur_frame->ref_count; + cm->cur_frame = NULL; + } - if (ref_index >= 0 && bufs[ref_index].ref_count > 0) - bufs[ref_index].ref_count--; + // Assign a new framebuffer + const int new_fb_idx = get_free_fb(cm); + if (new_fb_idx == INVALID_IDX) return NULL; - *idx = new_idx; + cm->cur_frame = &cm->buffer_pool->frame_bufs[new_fb_idx]; + cm->cur_frame->buf.buf_8bit_valid = 0; + return cm->cur_frame; +} - bufs[new_idx].ref_count++; +// Modify 'lhs_ptr' to reference the buffer at 'rhs_ptr', and update the ref +// counts accordingly. +static INLINE void assign_frame_buffer_p(RefCntBuffer **lhs_ptr, + RefCntBuffer *rhs_ptr) { + RefCntBuffer *const old_ptr = *lhs_ptr; + if (old_ptr != NULL) { + assert(old_ptr->ref_count > 0); + // One less reference to the buffer at 'old_ptr', so decrease ref count. + --old_ptr->ref_count; + } + + *lhs_ptr = rhs_ptr; + // One more reference to the buffer at 'rhs_ptr', so increase ref count. + ++rhs_ptr->ref_count; } static INLINE int frame_is_intra_only(const AV1_COMMON *const cm) { - return cm->frame_type == KEY_FRAME || cm->intra_only; + return cm->current_frame.frame_type == KEY_FRAME || + cm->current_frame.frame_type == INTRA_ONLY_FRAME; } static INLINE int frame_is_sframe(const AV1_COMMON *cm) { - return cm->frame_type == S_FRAME; + return cm->current_frame.frame_type == S_FRAME; } -static INLINE RefCntBuffer *get_prev_frame(const AV1_COMMON *const cm) { - if (cm->primary_ref_frame == PRIMARY_REF_NONE || - cm->frame_refs[cm->primary_ref_frame].idx == INVALID_IDX) { - return NULL; - } else { - return &cm->buffer_pool - ->frame_bufs[cm->frame_refs[cm->primary_ref_frame].idx]; - } +// These functions take a reference frame label between LAST_FRAME and +// EXTREF_FRAME inclusive. Note that this is different to the indexing +// previously used by the frame_refs[] array. +static INLINE int get_ref_frame_map_idx(const AV1_COMMON *const cm, + const MV_REFERENCE_FRAME ref_frame) { + return (ref_frame >= LAST_FRAME && ref_frame <= EXTREF_FRAME) + ? cm->remapped_ref_idx[ref_frame - LAST_FRAME] + : INVALID_IDX; +} + +static INLINE RefCntBuffer *get_ref_frame_buf( + const AV1_COMMON *const cm, const MV_REFERENCE_FRAME ref_frame) { + const int map_idx = get_ref_frame_map_idx(cm, ref_frame); + return (map_idx != INVALID_IDX) ? cm->ref_frame_map[map_idx] : NULL; +} + +// Both const and non-const versions of this function are provided so that it +// can be used with a const AV1_COMMON if needed. +static INLINE const struct scale_factors *get_ref_scale_factors_const( + const AV1_COMMON *const cm, const MV_REFERENCE_FRAME ref_frame) { + const int map_idx = get_ref_frame_map_idx(cm, ref_frame); + return (map_idx != INVALID_IDX) ? &cm->ref_scale_factors[map_idx] : NULL; +} + +static INLINE struct scale_factors *get_ref_scale_factors( + AV1_COMMON *const cm, const MV_REFERENCE_FRAME ref_frame) { + const int map_idx = get_ref_frame_map_idx(cm, ref_frame); + return (map_idx != INVALID_IDX) ? &cm->ref_scale_factors[map_idx] : NULL; +} + +static INLINE RefCntBuffer *get_primary_ref_frame_buf( + const AV1_COMMON *const cm) { + if (cm->primary_ref_frame == PRIMARY_REF_NONE) return NULL; + const int map_idx = get_ref_frame_map_idx(cm, cm->primary_ref_frame + 1); + return (map_idx != INVALID_IDX) ? cm->ref_frame_map[map_idx] : NULL; } // Returns 1 if this frame might allow mvs from some reference frame. static INLINE int frame_might_allow_ref_frame_mvs(const AV1_COMMON *cm) { - return !cm->error_resilient_mode && cm->seq_params.enable_ref_frame_mvs && - cm->seq_params.enable_order_hint && !frame_is_intra_only(cm); + return !cm->error_resilient_mode && + cm->seq_params.order_hint_info.enable_ref_frame_mvs && + cm->seq_params.order_hint_info.enable_order_hint && + !frame_is_intra_only(cm); } // Returns 1 if this frame might use warped_motion @@ -1193,8 +1260,8 @@ static INLINE TX_SIZE get_tx_size(int width, int height) { return TX_4X4; } -static INLINE int txfm_partition_context(TXFM_CONTEXT *above_ctx, - TXFM_CONTEXT *left_ctx, +static INLINE int txfm_partition_context(const TXFM_CONTEXT *const above_ctx, + const TXFM_CONTEXT *const left_ctx, BLOCK_SIZE bsize, TX_SIZE tx_size) { const uint8_t txw = tx_size_wide[tx_size]; const uint8_t txh = tx_size_high[tx_size]; @@ -1292,10 +1359,6 @@ static INLINE PARTITION_TYPE get_partition(const AV1_COMMON *const cm, return base_partitions[split_idx]; } -static INLINE void set_use_reference_buffer(AV1_COMMON *const cm, int use) { - cm->seq_params.frame_id_numbers_present_flag = use; -} - static INLINE void set_sb_size(SequenceHeader *const seq_params, BLOCK_SIZE sb_size) { seq_params->sb_size = sb_size; diff --git a/third_party/aom/av1/common/ppc/cfl_ppc.c b/third_party/aom/av1/common/ppc/cfl_ppc.c index 026a078095..61d8dc112c 100644 --- a/third_party/aom/av1/common/ppc/cfl_ppc.c +++ b/third_party/aom/av1/common/ppc/cfl_ppc.c @@ -130,21 +130,21 @@ cfl_subtract_average_fn get_subtract_average_fn_vsx(TX_SIZE tx_size) { subtract_average_8x8_vsx, /* 8x8 */ subtract_average_16x16_vsx, /* 16x16 */ subtract_average_32x32_vsx, /* 32x32 */ - cfl_subtract_average_null, /* 64x64 (invalid CFL size) */ + NULL, /* 64x64 (invalid CFL size) */ subtract_average_4x8_c, /* 4x8 */ subtract_average_8x4_vsx, /* 8x4 */ subtract_average_8x16_vsx, /* 8x16 */ subtract_average_16x8_vsx, /* 16x8 */ subtract_average_16x32_vsx, /* 16x32 */ subtract_average_32x16_vsx, /* 32x16 */ - cfl_subtract_average_null, /* 32x64 (invalid CFL size) */ - cfl_subtract_average_null, /* 64x32 (invalid CFL size) */ + NULL, /* 32x64 (invalid CFL size) */ + NULL, /* 64x32 (invalid CFL size) */ subtract_average_4x16_c, /* 4x16 */ subtract_average_16x4_vsx, /* 16x4 */ subtract_average_8x32_vsx, /* 8x32 */ subtract_average_32x8_vsx, /* 32x8 */ - cfl_subtract_average_null, /* 16x64 (invalid CFL size) */ - cfl_subtract_average_null, /* 64x16 (invalid CFL size) */ + NULL, /* 16x64 (invalid CFL size) */ + NULL, /* 64x16 (invalid CFL size) */ }; // Modulo TX_SIZES_ALL to ensure that an attacker won't be able to // index the function pointer array out of bounds. diff --git a/third_party/aom/av1/common/pred_common.h b/third_party/aom/av1/common/pred_common.h index 6dba2322d6..1d69ef003e 100644 --- a/third_party/aom/av1/common/pred_common.h +++ b/third_party/aom/av1/common/pred_common.h @@ -48,16 +48,16 @@ static INLINE int av1_get_spatial_seg_pred(const AV1_COMMON *const cm, int prev_l = -1; // left segment_id int prev_u = -1; // top segment_id if ((xd->up_available) && (xd->left_available)) { - prev_ul = get_segment_id(cm, cm->current_frame_seg_map, BLOCK_4X4, - mi_row - 1, mi_col - 1); + prev_ul = get_segment_id(cm, cm->cur_frame->seg_map, BLOCK_4X4, mi_row - 1, + mi_col - 1); } if (xd->up_available) { - prev_u = get_segment_id(cm, cm->current_frame_seg_map, BLOCK_4X4, - mi_row - 1, mi_col - 0); + prev_u = get_segment_id(cm, cm->cur_frame->seg_map, BLOCK_4X4, mi_row - 1, + mi_col - 0); } if (xd->left_available) { - prev_l = get_segment_id(cm, cm->current_frame_seg_map, BLOCK_4X4, - mi_row - 0, mi_col - 1); + prev_l = get_segment_id(cm, cm->cur_frame->seg_map, BLOCK_4X4, mi_row - 0, + mi_col - 1); } // Pick CDF index based on number of matching/out-of-bounds segment IDs. @@ -90,18 +90,18 @@ static INLINE int av1_get_pred_context_seg_id(const MACROBLOCKD *xd) { static INLINE int get_comp_index_context(const AV1_COMMON *cm, const MACROBLOCKD *xd) { MB_MODE_INFO *mbmi = xd->mi[0]; - int bck_idx = cm->frame_refs[mbmi->ref_frame[0] - LAST_FRAME].idx; - int fwd_idx = cm->frame_refs[mbmi->ref_frame[1] - LAST_FRAME].idx; + const RefCntBuffer *const bck_buf = get_ref_frame_buf(cm, mbmi->ref_frame[0]); + const RefCntBuffer *const fwd_buf = get_ref_frame_buf(cm, mbmi->ref_frame[1]); int bck_frame_index = 0, fwd_frame_index = 0; - int cur_frame_index = cm->cur_frame->cur_frame_offset; + int cur_frame_index = cm->cur_frame->order_hint; - if (bck_idx >= 0) - bck_frame_index = cm->buffer_pool->frame_bufs[bck_idx].cur_frame_offset; + if (bck_buf != NULL) bck_frame_index = bck_buf->order_hint; + if (fwd_buf != NULL) fwd_frame_index = fwd_buf->order_hint; - if (fwd_idx >= 0) - fwd_frame_index = cm->buffer_pool->frame_bufs[fwd_idx].cur_frame_offset; - int fwd = abs(get_relative_dist(cm, fwd_frame_index, cur_frame_index)); - int bck = abs(get_relative_dist(cm, cur_frame_index, bck_frame_index)); + int fwd = abs(get_relative_dist(&cm->seq_params.order_hint_info, + fwd_frame_index, cur_frame_index)); + int bck = abs(get_relative_dist(&cm->seq_params.order_hint_info, + cur_frame_index, bck_frame_index)); const MB_MODE_INFO *const above_mi = xd->above_mbmi; const MB_MODE_INFO *const left_mi = xd->left_mbmi; @@ -109,14 +109,14 @@ static INLINE int get_comp_index_context(const AV1_COMMON *cm, int above_ctx = 0, left_ctx = 0; const int offset = (fwd == bck); - if (above_mi) { + if (above_mi != NULL) { if (has_second_ref(above_mi)) above_ctx = above_mi->compound_idx; else if (above_mi->ref_frame[0] == ALTREF_FRAME) above_ctx = 1; } - if (left_mi) { + if (left_mi != NULL) { if (has_second_ref(left_mi)) left_ctx = left_mi->compound_idx; else if (left_mi->ref_frame[0] == ALTREF_FRAME) diff --git a/third_party/aom/av1/common/quant_common.c b/third_party/aom/av1/common/quant_common.c index 0e14da7a38..d4bdb98062 100644 --- a/third_party/aom/av1/common/quant_common.c +++ b/third_party/aom/av1/common/quant_common.c @@ -191,10 +191,11 @@ static const int16_t ac_qlookup_12_Q3[QINDEX_RANGE] = { // underflow to 0 in the actual quantization routines. int16_t av1_dc_quant_Q3(int qindex, int delta, aom_bit_depth_t bit_depth) { + const int q_clamped = clamp(qindex + delta, 0, MAXQ); switch (bit_depth) { - case AOM_BITS_8: return dc_qlookup_Q3[clamp(qindex + delta, 0, MAXQ)]; - case AOM_BITS_10: return dc_qlookup_10_Q3[clamp(qindex + delta, 0, MAXQ)]; - case AOM_BITS_12: return dc_qlookup_12_Q3[clamp(qindex + delta, 0, MAXQ)]; + case AOM_BITS_8: return dc_qlookup_Q3[q_clamped]; + case AOM_BITS_10: return dc_qlookup_10_Q3[q_clamped]; + case AOM_BITS_12: return dc_qlookup_12_Q3[q_clamped]; default: assert(0 && "bit_depth should be AOM_BITS_8, AOM_BITS_10 or AOM_BITS_12"); return -1; @@ -202,10 +203,11 @@ int16_t av1_dc_quant_Q3(int qindex, int delta, aom_bit_depth_t bit_depth) { } int16_t av1_ac_quant_Q3(int qindex, int delta, aom_bit_depth_t bit_depth) { + const int q_clamped = clamp(qindex + delta, 0, MAXQ); switch (bit_depth) { - case AOM_BITS_8: return ac_qlookup_Q3[clamp(qindex + delta, 0, MAXQ)]; - case AOM_BITS_10: return ac_qlookup_10_Q3[clamp(qindex + delta, 0, MAXQ)]; - case AOM_BITS_12: return ac_qlookup_12_Q3[clamp(qindex + delta, 0, MAXQ)]; + case AOM_BITS_8: return ac_qlookup_Q3[q_clamped]; + case AOM_BITS_10: return ac_qlookup_10_Q3[q_clamped]; + case AOM_BITS_12: return ac_qlookup_12_Q3[q_clamped]; default: assert(0 && "bit_depth should be AOM_BITS_8, AOM_BITS_10 or AOM_BITS_12"); return -1; @@ -244,8 +246,10 @@ const qm_val_t *av1_qmatrix(AV1_COMMON *cm, int qmlevel, int plane, } #define QM_TOTAL_SIZE 3344 -static const qm_val_t wt_matrix_ref[NUM_QM_LEVELS][2][QM_TOTAL_SIZE]; -static const qm_val_t iwt_matrix_ref[NUM_QM_LEVELS][2][QM_TOTAL_SIZE]; +// We only use wt_matrix_ref[q] and iwt_matrix_ref[q] +// for q = 0, ..., NUM_QM_LEVELS - 2. +static const qm_val_t wt_matrix_ref[NUM_QM_LEVELS - 1][2][QM_TOTAL_SIZE]; +static const qm_val_t iwt_matrix_ref[NUM_QM_LEVELS - 1][2][QM_TOTAL_SIZE]; void av1_qm_init(AV1_COMMON *cm) { const int num_planes = av1_num_planes(cm); @@ -274,7 +278,7 @@ void av1_qm_init(AV1_COMMON *cm) { } } -/* Provide 16 sets of quantization matrices for chroma and luma +/* Provide 15 sets of quantization matrices for chroma and luma and each TX size. Matrices for different TX sizes are in fact sub-sampled from the 32x32 and 16x16 sizes, but explicitly defined here for convenience. Intra and inter matrix sets are the @@ -283,9 +287,10 @@ void av1_qm_init(AV1_COMMON *cm) { frame. Matrices for different QM levels have been rescaled in the frequency domain according to different nominal viewing - distances. + distances. Matrices for QM level 15 are omitted because they are + not used. */ -static const qm_val_t iwt_matrix_ref[NUM_QM_LEVELS][2][QM_TOTAL_SIZE] = { +static const qm_val_t iwt_matrix_ref[NUM_QM_LEVELS - 1][2][QM_TOTAL_SIZE] = { { { /* Luma */ /* Size 4x4 */ @@ -6633,427 +6638,9 @@ static const qm_val_t iwt_matrix_ref[NUM_QM_LEVELS][2][QM_TOTAL_SIZE] = { 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32 }, }, - { - { /* Luma */ - /* Size 4x4 */ - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - /* Size 8x8 */ - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - /* Size 16x16 */ - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, - /* Size 32x32 */ - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - /* Size 4x8 */ - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - /* Size 8x4 */ - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - /* Size 8x16 */ - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, - /* Size 16x8 */ - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, - /* Size 16x32 */ - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, - /* Size 32x16 */ - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, - /* Size 4x16 */ - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - /* Size 16x4 */ - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - /* Size 8x32 */ - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, - /* Size 32x8 */ - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32 }, - { /* Chroma */ - /* Size 4x4 */ - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - /* Size 8x8 */ - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - /* Size 16x16 */ - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, - /* Size 32x32 */ - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - /* Size 4x8 */ - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - /* Size 8x4 */ - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - /* Size 8x16 */ - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, - /* Size 16x8 */ - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, - /* Size 16x32 */ - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, - /* Size 32x16 */ - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, - /* Size 4x16 */ - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - /* Size 16x4 */ - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - /* Size 8x32 */ - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, - /* Size 32x8 */ - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32 }, - }, }; -static const qm_val_t wt_matrix_ref[NUM_QM_LEVELS][2][QM_TOTAL_SIZE] = { +static const qm_val_t wt_matrix_ref[NUM_QM_LEVELS - 1][2][QM_TOTAL_SIZE] = { { { /* Luma */ /* Size 4x4 */ @@ -13255,422 +12842,4 @@ static const qm_val_t wt_matrix_ref[NUM_QM_LEVELS][2][QM_TOTAL_SIZE] = { 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 32, 32, 32, 32, 32, 32, 32, 32, 32 }, }, - { - { /* Luma */ - /* Size 4x4 */ - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - /* Size 8x8 */ - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - /* Size 16x16 */ - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, - /* Size 32x32 */ - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - /* Size 4x8 */ - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - /* Size 8x4 */ - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - /* Size 8x16 */ - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, - /* Size 16x8 */ - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, - /* Size 16x32 */ - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, - /* Size 32x16 */ - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, - /* Size 4x16 */ - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - /* Size 16x4 */ - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - /* Size 8x32 */ - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, - /* Size 32x8 */ - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32 }, - { /* Chroma */ - /* Size 4x4 */ - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - /* Size 8x8 */ - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - /* Size 16x16 */ - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, - /* Size 32x32 */ - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - /* Size 4x8 */ - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - /* Size 8x4 */ - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - /* Size 8x16 */ - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, - /* Size 16x8 */ - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, - /* Size 16x32 */ - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, - /* Size 32x16 */ - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, - /* Size 4x16 */ - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - /* Size 16x4 */ - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - /* Size 8x32 */ - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, - /* Size 32x8 */ - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32 }, - }, }; diff --git a/third_party/aom/av1/common/reconinter.c b/third_party/aom/av1/common/reconinter.c index 3203efce4b..ea351cf698 100644 --- a/third_party/aom/av1/common/reconinter.c +++ b/third_party/aom/av1/common/reconinter.c @@ -35,10 +35,12 @@ int av1_allow_warp(const MB_MODE_INFO *const mbmi, const WarpTypesAllowed *const warp_types, const WarpedMotionParams *const gm_params, - int build_for_obmc, int x_scale, int y_scale, + int build_for_obmc, const struct scale_factors *const sf, WarpedMotionParams *final_warp_params) { - if (x_scale != SCALE_SUBPEL_SHIFTS || y_scale != SCALE_SUBPEL_SHIFTS) - return 0; + // Note: As per the spec, we must test the fixed point scales here, which are + // at a higher precision (1 << 14) than the xs and ys in subpel_params (that + // have 1 << 10 precision). + if (av1_is_scaled(sf)) return 0; if (final_warp_params != NULL) *final_warp_params = default_warp_params; @@ -75,20 +77,18 @@ void av1_make_inter_predictor(const uint8_t *src, int src_stride, uint8_t *dst, const int do_warp = (w >= 8 && h >= 8 && av1_allow_warp(mi, warp_types, &xd->global_motion[mi->ref_frame[ref]], - build_for_obmc, subpel_params->xs, subpel_params->ys, - &final_warp_params)); + build_for_obmc, sf, &final_warp_params)); const int is_intrabc = mi->use_intrabc; assert(IMPLIES(is_intrabc, !do_warp)); if (do_warp && xd->cur_frame_force_integer_mv == 0) { const struct macroblockd_plane *const pd = &xd->plane[plane]; const struct buf_2d *const pre_buf = &pd->pre[ref]; - av1_warp_plane(&final_warp_params, - xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH, xd->bd, + av1_warp_plane(&final_warp_params, is_cur_buf_hbd(xd), xd->bd, pre_buf->buf0, pre_buf->width, pre_buf->height, pre_buf->stride, dst, p_col, p_row, w, h, dst_stride, pd->subsampling_x, pd->subsampling_y, conv_params); - } else if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + } else if (is_cur_buf_hbd(xd)) { highbd_inter_predictor(src, src_stride, dst, dst_stride, subpel_params, sf, w, h, conv_params, interp_filters, is_intrabc, xd->bd); @@ -567,14 +567,15 @@ static void build_masked_compound_no_round( const int subh = (2 << mi_size_high_log2[sb_type]) == h; const int subw = (2 << mi_size_wide_log2[sb_type]) == w; const uint8_t *mask = av1_get_compound_type_mask(comp_data, sb_type); - if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) + if (is_cur_buf_hbd(xd)) { aom_highbd_blend_a64_d16_mask(dst, dst_stride, src0, src0_stride, src1, src1_stride, mask, block_size_wide[sb_type], w, h, subw, subh, conv_params, xd->bd); - else + } else { aom_lowbd_blend_a64_d16_mask(dst, dst_stride, src0, src0_stride, src1, src1_stride, mask, block_size_wide[sb_type], w, h, subw, subh, conv_params); + } } void av1_make_masked_inter_predictor( @@ -625,32 +626,31 @@ void av1_make_masked_inter_predictor( mi->sb_type, h, w, conv_params, xd); } -void av1_jnt_comp_weight_assign(const AV1_COMMON *cm, const MB_MODE_INFO *mbmi, - int order_idx, int *fwd_offset, int *bck_offset, - int *use_jnt_comp_avg, int is_compound) { +void av1_dist_wtd_comp_weight_assign(const AV1_COMMON *cm, + const MB_MODE_INFO *mbmi, int order_idx, + int *fwd_offset, int *bck_offset, + int *use_dist_wtd_comp_avg, + int is_compound) { assert(fwd_offset != NULL && bck_offset != NULL); if (!is_compound || mbmi->compound_idx) { - *use_jnt_comp_avg = 0; + *use_dist_wtd_comp_avg = 0; return; } - *use_jnt_comp_avg = 1; - const int bck_idx = cm->frame_refs[mbmi->ref_frame[0] - LAST_FRAME].idx; - const int fwd_idx = cm->frame_refs[mbmi->ref_frame[1] - LAST_FRAME].idx; - const int cur_frame_index = cm->cur_frame->cur_frame_offset; + *use_dist_wtd_comp_avg = 1; + const RefCntBuffer *const bck_buf = get_ref_frame_buf(cm, mbmi->ref_frame[0]); + const RefCntBuffer *const fwd_buf = get_ref_frame_buf(cm, mbmi->ref_frame[1]); + const int cur_frame_index = cm->cur_frame->order_hint; int bck_frame_index = 0, fwd_frame_index = 0; - if (bck_idx >= 0) { - bck_frame_index = cm->buffer_pool->frame_bufs[bck_idx].cur_frame_offset; - } + if (bck_buf != NULL) bck_frame_index = bck_buf->order_hint; + if (fwd_buf != NULL) fwd_frame_index = fwd_buf->order_hint; - if (fwd_idx >= 0) { - fwd_frame_index = cm->buffer_pool->frame_bufs[fwd_idx].cur_frame_offset; - } - - int d0 = clamp(abs(get_relative_dist(cm, fwd_frame_index, cur_frame_index)), + int d0 = clamp(abs(get_relative_dist(&cm->seq_params.order_hint_info, + fwd_frame_index, cur_frame_index)), 0, MAX_FRAME_DISTANCE); - int d1 = clamp(abs(get_relative_dist(cm, cur_frame_index, bck_frame_index)), + int d1 = clamp(abs(get_relative_dist(&cm->seq_params.order_hint_info, + cur_frame_index, bck_frame_index)), 0, MAX_FRAME_DISTANCE); const int order = d0 <= d1; @@ -708,10 +708,9 @@ void av1_setup_pre_planes(MACROBLOCKD *xd, int idx, // obmc_mask_N[overlap_position] static const uint8_t obmc_mask_1[1] = { 64 }; +DECLARE_ALIGNED(2, static const uint8_t, obmc_mask_2[2]) = { 45, 64 }; -static const uint8_t obmc_mask_2[2] = { 45, 64 }; - -static const uint8_t obmc_mask_4[4] = { 39, 50, 59, 64 }; +DECLARE_ALIGNED(4, static const uint8_t, obmc_mask_4[4]) = { 39, 50, 59, 64 }; static const uint8_t obmc_mask_8[8] = { 36, 42, 48, 53, 57, 61, 64, 64 }; @@ -814,7 +813,7 @@ static INLINE void build_obmc_inter_pred_above(MACROBLOCKD *xd, int rel_mi_col, (void)above_mi; struct obmc_inter_pred_ctxt *ctxt = (struct obmc_inter_pred_ctxt *)fun_ctxt; const BLOCK_SIZE bsize = xd->mi[0]->sb_type; - const int is_hbd = (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) ? 1 : 0; + const int is_hbd = is_cur_buf_hbd(xd); const int overlap = AOMMIN(block_size_high[bsize], block_size_high[BLOCK_64X64]) >> 1; @@ -851,7 +850,7 @@ static INLINE void build_obmc_inter_pred_left(MACROBLOCKD *xd, int rel_mi_row, const BLOCK_SIZE bsize = xd->mi[0]->sb_type; const int overlap = AOMMIN(block_size_wide[bsize], block_size_wide[BLOCK_64X64]) >> 1; - const int is_hbd = (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) ? 1 : 0; + const int is_hbd = is_cur_buf_hbd(xd); for (int plane = 0; plane < num_planes; ++plane) { const struct macroblockd_plane *pd = &xd->plane[plane]; @@ -922,14 +921,15 @@ void av1_setup_build_prediction_by_above_pred( for (int ref = 0; ref < num_refs; ++ref) { const MV_REFERENCE_FRAME frame = above_mbmi->ref_frame[ref]; - const RefBuffer *const ref_buf = &ctxt->cm->frame_refs[frame - LAST_FRAME]; - - xd->block_refs[ref] = ref_buf; - if ((!av1_is_valid_scale(&ref_buf->sf))) + const RefCntBuffer *const ref_buf = get_ref_frame_buf(ctxt->cm, frame); + const struct scale_factors *const sf = + get_ref_scale_factors_const(ctxt->cm, frame); + xd->block_ref_scale_factors[ref] = sf; + if ((!av1_is_valid_scale(sf))) aom_internal_error(xd->error_info, AOM_CODEC_UNSUP_BITSTREAM, "Reference frame has invalid dimensions"); - av1_setup_pre_planes(xd, ref, ref_buf->buf, ctxt->mi_row, above_mi_col, - &ref_buf->sf, num_planes); + av1_setup_pre_planes(xd, ref, &ref_buf->buf, ctxt->mi_row, above_mi_col, sf, + num_planes); } xd->mb_to_left_edge = 8 * MI_SIZE * (-above_mi_col); @@ -959,14 +959,16 @@ void av1_setup_build_prediction_by_left_pred(MACROBLOCKD *xd, int rel_mi_row, for (int ref = 0; ref < num_refs; ++ref) { const MV_REFERENCE_FRAME frame = left_mbmi->ref_frame[ref]; - const RefBuffer *const ref_buf = &ctxt->cm->frame_refs[frame - LAST_FRAME]; + const RefCntBuffer *const ref_buf = get_ref_frame_buf(ctxt->cm, frame); + const struct scale_factors *const ref_scale_factors = + get_ref_scale_factors_const(ctxt->cm, frame); - xd->block_refs[ref] = ref_buf; - if ((!av1_is_valid_scale(&ref_buf->sf))) + xd->block_ref_scale_factors[ref] = ref_scale_factors; + if ((!av1_is_valid_scale(ref_scale_factors))) aom_internal_error(xd->error_info, AOM_CODEC_UNSUP_BITSTREAM, "Reference frame has invalid dimensions"); - av1_setup_pre_planes(xd, ref, ref_buf->buf, left_mi_row, ctxt->mi_col, - &ref_buf->sf, num_planes); + av1_setup_pre_planes(xd, ref, &ref_buf->buf, left_mi_row, ctxt->mi_col, + ref_scale_factors, num_planes); } xd->mb_to_top_edge = 8 * MI_SIZE * (-left_mi_row); @@ -1033,12 +1035,13 @@ static void build_smooth_interintra_mask(uint8_t *mask, int stride, } } -static void combine_interintra(INTERINTRA_MODE mode, int use_wedge_interintra, - int wedge_index, int wedge_sign, - BLOCK_SIZE bsize, BLOCK_SIZE plane_bsize, - uint8_t *comppred, int compstride, - const uint8_t *interpred, int interstride, - const uint8_t *intrapred, int intrastride) { +static void combine_interintra(INTERINTRA_MODE mode, + int8_t use_wedge_interintra, int wedge_index, + int wedge_sign, BLOCK_SIZE bsize, + BLOCK_SIZE plane_bsize, uint8_t *comppred, + int compstride, const uint8_t *interpred, + int interstride, const uint8_t *intrapred, + int intrastride) { const int bw = block_size_wide[plane_bsize]; const int bh = block_size_high[plane_bsize]; @@ -1062,7 +1065,7 @@ static void combine_interintra(INTERINTRA_MODE mode, int use_wedge_interintra, } static void combine_interintra_highbd( - INTERINTRA_MODE mode, int use_wedge_interintra, int wedge_index, + INTERINTRA_MODE mode, int8_t use_wedge_interintra, int wedge_index, int wedge_sign, BLOCK_SIZE bsize, BLOCK_SIZE plane_bsize, uint8_t *comppred8, int compstride, const uint8_t *interpred8, int interstride, const uint8_t *intrapred8, int intrastride, int bd) { @@ -1092,8 +1095,8 @@ static void combine_interintra_highbd( void av1_build_intra_predictors_for_interintra(const AV1_COMMON *cm, MACROBLOCKD *xd, BLOCK_SIZE bsize, int plane, - BUFFER_SET *ctx, uint8_t *dst, - int dst_stride) { + const BUFFER_SET *ctx, + uint8_t *dst, int dst_stride) { struct macroblockd_plane *const pd = &xd->plane[plane]; const int ssx = xd->plane[plane].subsampling_x; const int ssy = xd->plane[plane].subsampling_y; @@ -1116,7 +1119,7 @@ void av1_combine_interintra(MACROBLOCKD *xd, BLOCK_SIZE bsize, int plane, const int ssx = xd->plane[plane].subsampling_x; const int ssy = xd->plane[plane].subsampling_y; const BLOCK_SIZE plane_bsize = get_plane_block_size(bsize, ssx, ssy); - if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + if (is_cur_buf_hbd(xd)) { combine_interintra_highbd( xd->mi[0]->interintra_mode, xd->mi[0]->use_wedge_interintra, xd->mi[0]->interintra_wedge_index, xd->mi[0]->interintra_wedge_sign, @@ -1135,9 +1138,9 @@ void av1_combine_interintra(MACROBLOCKD *xd, BLOCK_SIZE bsize, int plane, // build interintra_predictors for one plane void av1_build_interintra_predictors_sbp(const AV1_COMMON *cm, MACROBLOCKD *xd, uint8_t *pred, int stride, - BUFFER_SET *ctx, int plane, + const BUFFER_SET *ctx, int plane, BLOCK_SIZE bsize) { - if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + if (is_cur_buf_hbd(xd)) { DECLARE_ALIGNED(16, uint16_t, intrapredictor[MAX_SB_SQUARE]); av1_build_intra_predictors_for_interintra( cm, xd, bsize, plane, ctx, CONVERT_TO_BYTEPTR(intrapredictor), @@ -1156,7 +1159,8 @@ void av1_build_interintra_predictors_sbp(const AV1_COMMON *cm, MACROBLOCKD *xd, void av1_build_interintra_predictors_sbuv(const AV1_COMMON *cm, MACROBLOCKD *xd, uint8_t *upred, uint8_t *vpred, int ustride, int vstride, - BUFFER_SET *ctx, BLOCK_SIZE bsize) { + const BUFFER_SET *ctx, + BLOCK_SIZE bsize) { av1_build_interintra_predictors_sbp(cm, xd, upred, ustride, ctx, 1, bsize); av1_build_interintra_predictors_sbp(cm, xd, vpred, vstride, ctx, 2, bsize); } diff --git a/third_party/aom/av1/common/reconinter.h b/third_party/aom/av1/common/reconinter.h index db86c777e3..1385be50f9 100644 --- a/third_party/aom/av1/common/reconinter.h +++ b/third_party/aom/av1/common/reconinter.h @@ -47,7 +47,7 @@ extern "C" { #define WEDGE_NONE -1 // Angles are with respect to horizontal anti-clockwise -typedef enum { +enum { WEDGE_HORIZONTAL = 0, WEDGE_VERTICAL = 1, WEDGE_OBLIQUE27 = 2, @@ -55,7 +55,7 @@ typedef enum { WEDGE_OBLIQUE117 = 4, WEDGE_OBLIQUE153 = 5, WEDGE_DIRECTIONS -} WedgeDirectionType; +} UENUM1BYTE(WedgeDirectionType); // 3-tuple: {direction, x_offset, y_offset} typedef struct { @@ -333,29 +333,32 @@ const uint8_t *av1_get_compound_type_mask( // build interintra_predictors for one plane void av1_build_interintra_predictors_sbp(const AV1_COMMON *cm, MACROBLOCKD *xd, uint8_t *pred, int stride, - BUFFER_SET *ctx, int plane, + const BUFFER_SET *ctx, int plane, BLOCK_SIZE bsize); void av1_build_interintra_predictors_sbuv(const AV1_COMMON *cm, MACROBLOCKD *xd, uint8_t *upred, uint8_t *vpred, int ustride, int vstride, - BUFFER_SET *ctx, BLOCK_SIZE bsize); + const BUFFER_SET *ctx, + BLOCK_SIZE bsize); void av1_build_intra_predictors_for_interintra( const AV1_COMMON *cm, MACROBLOCKD *xd, BLOCK_SIZE bsize, int plane, - BUFFER_SET *ctx, uint8_t *intra_pred, int intra_stride); + const BUFFER_SET *ctx, uint8_t *intra_pred, int intra_stride); void av1_combine_interintra(MACROBLOCKD *xd, BLOCK_SIZE bsize, int plane, const uint8_t *inter_pred, int inter_stride, const uint8_t *intra_pred, int intra_stride); -void av1_jnt_comp_weight_assign(const AV1_COMMON *cm, const MB_MODE_INFO *mbmi, - int order_idx, int *fwd_offset, int *bck_offset, - int *use_jnt_comp_avg, int is_compound); +void av1_dist_wtd_comp_weight_assign(const AV1_COMMON *cm, + const MB_MODE_INFO *mbmi, int order_idx, + int *fwd_offset, int *bck_offset, + int *use_dist_wtd_comp_avg, + int is_compound); int av1_allow_warp(const MB_MODE_INFO *const mbmi, const WarpTypesAllowed *const warp_types, const WarpedMotionParams *const gm_params, - int build_for_obmc, int x_scale, int y_scale, + int build_for_obmc, const struct scale_factors *const sf, WarpedMotionParams *final_warp_params); #ifdef __cplusplus diff --git a/third_party/aom/av1/common/reconintra.c b/third_party/aom/av1/common/reconintra.c index 71a52e73e5..559e49921b 100644 --- a/third_party/aom/av1/common/reconintra.c +++ b/third_party/aom/av1/common/reconintra.c @@ -556,33 +556,37 @@ void av1_dr_prediction_z2_c(uint8_t *dst, ptrdiff_t stride, int bw, int bh, const uint8_t *above, const uint8_t *left, int upsample_above, int upsample_left, int dx, int dy) { - int r, c, x, y, shift1, shift2, val, base1, base2; - assert(dx > 0); assert(dy > 0); const int min_base_x = -(1 << upsample_above); + const int min_base_y = -(1 << upsample_left); + (void)min_base_y; const int frac_bits_x = 6 - upsample_above; const int frac_bits_y = 6 - upsample_left; - const int base_inc_x = 1 << upsample_above; - x = -dx; - for (r = 0; r < bh; ++r, x -= dx, dst += stride) { - base1 = x >> frac_bits_x; - y = (r << 6) - dy; - for (c = 0; c < bw; ++c, base1 += base_inc_x, y -= dy) { - if (base1 >= min_base_x) { - shift1 = ((x * (1 << upsample_above)) & 0x3F) >> 1; - val = above[base1] * (32 - shift1) + above[base1 + 1] * shift1; + + for (int r = 0; r < bh; ++r) { + for (int c = 0; c < bw; ++c) { + int val; + int y = r + 1; + int x = (c << 6) - y * dx; + const int base_x = x >> frac_bits_x; + if (base_x >= min_base_x) { + const int shift = ((x * (1 << upsample_above)) & 0x3F) >> 1; + val = above[base_x] * (32 - shift) + above[base_x + 1] * shift; val = ROUND_POWER_OF_TWO(val, 5); } else { - base2 = y >> frac_bits_y; - assert(base2 >= -(1 << upsample_left)); - shift2 = ((y * (1 << upsample_left)) & 0x3F) >> 1; - val = left[base2] * (32 - shift2) + left[base2 + 1] * shift2; + x = c + 1; + y = (r << 6) - x * dy; + const int base_y = y >> frac_bits_y; + assert(base_y >= min_base_y); + const int shift = ((y * (1 << upsample_left)) & 0x3F) >> 1; + val = left[base_y] * (32 - shift) + left[base_y + 1] * shift; val = ROUND_POWER_OF_TWO(val, 5); } dst[c] = val; } + dst += stride; } } @@ -688,30 +692,33 @@ void av1_highbd_dr_prediction_z2_c(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_above, int upsample_left, int dx, int dy, int bd) { - int r, c, x, y, shift, val, base; - (void)bd; assert(dx > 0); assert(dy > 0); const int min_base_x = -(1 << upsample_above); + const int min_base_y = -(1 << upsample_left); + (void)min_base_y; const int frac_bits_x = 6 - upsample_above; const int frac_bits_y = 6 - upsample_left; - for (r = 0; r < bh; ++r) { - for (c = 0; c < bw; ++c) { - y = r + 1; - x = (c << 6) - y * dx; - base = x >> frac_bits_x; - if (base >= min_base_x) { - shift = ((x * (1 << upsample_above)) & 0x3F) >> 1; - val = above[base] * (32 - shift) + above[base + 1] * shift; + + for (int r = 0; r < bh; ++r) { + for (int c = 0; c < bw; ++c) { + int val; + int y = r + 1; + int x = (c << 6) - y * dx; + const int base_x = x >> frac_bits_x; + if (base_x >= min_base_x) { + const int shift = ((x * (1 << upsample_above)) & 0x3F) >> 1; + val = above[base_x] * (32 - shift) + above[base_x + 1] * shift; val = ROUND_POWER_OF_TWO(val, 5); } else { x = c + 1; y = (r << 6) - x * dy; - base = y >> frac_bits_y; - shift = ((y * (1 << upsample_left)) & 0x3F) >> 1; - val = left[base] * (32 - shift) + left[base + 1] * shift; + const int base_y = y >> frac_bits_y; + assert(base_y >= min_base_y); + const int shift = ((y * (1 << upsample_left)) & 0x3F) >> 1; + val = left[base_y] * (32 - shift) + left[base_y + 1] * shift; val = ROUND_POWER_OF_TWO(val, 5); } dst[c] = val; @@ -1503,7 +1510,7 @@ void av1_predict_intra_block( xd->color_index_map_offset[plane != 0]; const uint16_t *const palette = mbmi->palette_mode_info.palette_colors + plane * PALETTE_MAX_SIZE; - if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + if (is_cur_buf_hbd(xd)) { uint16_t *dst16 = CONVERT_TO_SHORTPTR(dst); for (r = 0; r < txhpx; ++r) { for (c = 0; c < txwpx; ++c) { @@ -1562,7 +1569,7 @@ void av1_predict_intra_block( tx_size, row_off, col_off, pd->subsampling_x, pd->subsampling_y); const int disable_edge_filter = !cm->seq_params.enable_intra_edge_filter; - if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + if (is_cur_buf_hbd(xd)) { build_intra_predictors_high( xd, ref, ref_stride, dst, dst_stride, mode, angle_delta, filter_intra_mode, tx_size, disable_edge_filter, diff --git a/third_party/aom/av1/common/reconintra.h b/third_party/aom/av1/common/reconintra.h index 07853aba01..3c781abb52 100644 --- a/third_party/aom/av1/common/reconintra.h +++ b/third_party/aom/av1/common/reconintra.h @@ -110,7 +110,7 @@ static INLINE int av1_use_intra_edge_upsample(int bs0, int bs1, int delta, int type) { const int d = abs(delta); const int blk_wh = bs0 + bs1; - if (d <= 0 || d >= 40) return 0; + if (d == 0 || d >= 40) return 0; return type ? (blk_wh <= 8) : (blk_wh <= 16); } #ifdef __cplusplus diff --git a/third_party/aom/av1/common/resize.c b/third_party/aom/av1/common/resize.c index d61a20aa2d..8b24ed01b9 100644 --- a/third_party/aom/av1/common/resize.c +++ b/third_party/aom/av1/common/resize.c @@ -313,6 +313,91 @@ static void interpolate_core(const uint8_t *const input, int in_length, } } +static void interpolate_core_double_prec(const double *const input, + int in_length, double *output, + int out_length, + const int16_t *interp_filters, + int interp_taps) { + const int32_t delta = + (((uint32_t)in_length << RS_SCALE_SUBPEL_BITS) + out_length / 2) / + out_length; + const int32_t offset = + in_length > out_length + ? (((int32_t)(in_length - out_length) << (RS_SCALE_SUBPEL_BITS - 1)) + + out_length / 2) / + out_length + : -(((int32_t)(out_length - in_length) + << (RS_SCALE_SUBPEL_BITS - 1)) + + out_length / 2) / + out_length; + double *optr = output; + int x, x1, x2, k, int_pel, sub_pel; + double sum; + int32_t y; + + x = 0; + y = offset + RS_SCALE_EXTRA_OFF; + while ((y >> RS_SCALE_SUBPEL_BITS) < (interp_taps / 2 - 1)) { + x++; + y += delta; + } + x1 = x; + x = out_length - 1; + y = delta * x + offset + RS_SCALE_EXTRA_OFF; + while ((y >> RS_SCALE_SUBPEL_BITS) + (int32_t)(interp_taps / 2) >= + in_length) { + x--; + y -= delta; + } + x2 = x; + if (x1 > x2) { + for (x = 0, y = offset + RS_SCALE_EXTRA_OFF; x < out_length; + ++x, y += delta) { + int_pel = y >> RS_SCALE_SUBPEL_BITS; + sub_pel = (y >> RS_SCALE_EXTRA_BITS) & RS_SUBPEL_MASK; + const int16_t *filter = &interp_filters[sub_pel * interp_taps]; + sum = 0; + for (k = 0; k < interp_taps; ++k) { + const int pk = int_pel - interp_taps / 2 + 1 + k; + sum += filter[k] * input[AOMMAX(AOMMIN(pk, in_length - 1), 0)]; + } + *optr++ = sum / (1 << FILTER_BITS); + } + } else { + // Initial part. + for (x = 0, y = offset + RS_SCALE_EXTRA_OFF; x < x1; ++x, y += delta) { + int_pel = y >> RS_SCALE_SUBPEL_BITS; + sub_pel = (y >> RS_SCALE_EXTRA_BITS) & RS_SUBPEL_MASK; + const int16_t *filter = &interp_filters[sub_pel * interp_taps]; + sum = 0; + for (k = 0; k < interp_taps; ++k) + sum += filter[k] * input[AOMMAX(int_pel - interp_taps / 2 + 1 + k, 0)]; + *optr++ = sum / (1 << FILTER_BITS); + } + // Middle part. + for (; x <= x2; ++x, y += delta) { + int_pel = y >> RS_SCALE_SUBPEL_BITS; + sub_pel = (y >> RS_SCALE_EXTRA_BITS) & RS_SUBPEL_MASK; + const int16_t *filter = &interp_filters[sub_pel * interp_taps]; + sum = 0; + for (k = 0; k < interp_taps; ++k) + sum += filter[k] * input[int_pel - interp_taps / 2 + 1 + k]; + *optr++ = sum / (1 << FILTER_BITS); + } + // End part. + for (; x < out_length; ++x, y += delta) { + int_pel = y >> RS_SCALE_SUBPEL_BITS; + sub_pel = (y >> RS_SCALE_EXTRA_BITS) & RS_SUBPEL_MASK; + const int16_t *filter = &interp_filters[sub_pel * interp_taps]; + sum = 0; + for (k = 0; k < interp_taps; ++k) + sum += filter[k] * + input[AOMMIN(int_pel - interp_taps / 2 + 1 + k, in_length - 1)]; + *optr++ = sum / (1 << FILTER_BITS); + } + } +} + static void interpolate(const uint8_t *const input, int in_length, uint8_t *output, int out_length) { const InterpKernel *interp_filters = @@ -322,6 +407,15 @@ static void interpolate(const uint8_t *const input, int in_length, SUBPEL_TAPS); } +static void interpolate_double_prec(const double *const input, int in_length, + double *output, int out_length) { + const InterpKernel *interp_filters = + choose_interp_filter(in_length, out_length); + + interpolate_core_double_prec(input, in_length, output, out_length, + &interp_filters[0][0], SUBPEL_TAPS); +} + int32_t av1_get_upscale_convolve_step(int in_length, int out_length) { return ((in_length << RS_SCALE_SUBPEL_BITS) + out_length / 2) / out_length; } @@ -505,6 +599,12 @@ static void resize_multistep(const uint8_t *const input, int length, } } +static void upscale_multistep_double_prec(const double *const input, int length, + double *output, int olength) { + assert(length < olength); + interpolate_double_prec(input, length, output, olength); +} + static void fill_col_to_arr(uint8_t *img, int stride, int len, uint8_t *arr) { int i; uint8_t *iptr = img; @@ -523,9 +623,29 @@ static void fill_arr_to_col(uint8_t *img, int stride, int len, uint8_t *arr) { } } -static void resize_plane(const uint8_t *const input, int height, int width, - int in_stride, uint8_t *output, int height2, - int width2, int out_stride) { +static void fill_col_to_arr_double_prec(double *img, int stride, int len, + double *arr) { + int i; + double *iptr = img; + double *aptr = arr; + for (i = 0; i < len; ++i, iptr += stride) { + *aptr++ = *iptr; + } +} + +static void fill_arr_to_col_double_prec(double *img, int stride, int len, + double *arr) { + int i; + double *iptr = img; + double *aptr = arr; + for (i = 0; i < len; ++i, iptr += stride) { + *iptr = *aptr++; + } +} + +void av1_resize_plane(const uint8_t *const input, int height, int width, + int in_stride, uint8_t *output, int height2, int width2, + int out_stride) { int i; uint8_t *intbuf = (uint8_t *)aom_malloc(sizeof(uint8_t) * width2 * height); uint8_t *tmpbuf = @@ -554,6 +674,33 @@ Error: aom_free(arrbuf2); } +void av1_upscale_plane_double_prec(const double *const input, int height, + int width, int in_stride, double *output, + int height2, int width2, int out_stride) { + int i; + double *intbuf = (double *)aom_malloc(sizeof(double) * width2 * height); + double *arrbuf = (double *)aom_malloc(sizeof(double) * height); + double *arrbuf2 = (double *)aom_malloc(sizeof(double) * height2); + if (intbuf == NULL || arrbuf == NULL || arrbuf2 == NULL) goto Error; + assert(width > 0); + assert(height > 0); + assert(width2 > 0); + assert(height2 > 0); + for (i = 0; i < height; ++i) + upscale_multistep_double_prec(input + in_stride * i, width, + intbuf + width2 * i, width2); + for (i = 0; i < width2; ++i) { + fill_col_to_arr_double_prec(intbuf + i, width2, height, arrbuf); + upscale_multistep_double_prec(arrbuf, height, arrbuf2, height2); + fill_arr_to_col_double_prec(output + i, out_stride, height2, arrbuf2); + } + +Error: + aom_free(intbuf); + aom_free(arrbuf); + aom_free(arrbuf2); +} + static void upscale_normative_rect(const uint8_t *const input, int height, int width, int in_stride, uint8_t *output, int height2, int width2, int out_stride, @@ -871,10 +1018,9 @@ static void highbd_fill_arr_to_col(uint16_t *img, int stride, int len, } } -static void highbd_resize_plane(const uint8_t *const input, int height, - int width, int in_stride, uint8_t *output, - int height2, int width2, int out_stride, - int bd) { +void av1_highbd_resize_plane(const uint8_t *const input, int height, int width, + int in_stride, uint8_t *output, int height2, + int width2, int out_stride, int bd) { int i; uint16_t *intbuf = (uint16_t *)aom_malloc(sizeof(uint16_t) * width2 * height); uint16_t *tmpbuf = @@ -969,11 +1115,11 @@ void av1_resize_frame420(const uint8_t *const y, int y_stride, int uv_stride, int height, int width, uint8_t *oy, int oy_stride, uint8_t *ou, uint8_t *ov, int ouv_stride, int oheight, int owidth) { - resize_plane(y, height, width, y_stride, oy, oheight, owidth, oy_stride); - resize_plane(u, height / 2, width / 2, uv_stride, ou, oheight / 2, owidth / 2, - ouv_stride); - resize_plane(v, height / 2, width / 2, uv_stride, ov, oheight / 2, owidth / 2, - ouv_stride); + av1_resize_plane(y, height, width, y_stride, oy, oheight, owidth, oy_stride); + av1_resize_plane(u, height / 2, width / 2, uv_stride, ou, oheight / 2, + owidth / 2, ouv_stride); + av1_resize_plane(v, height / 2, width / 2, uv_stride, ov, oheight / 2, + owidth / 2, ouv_stride); } void av1_resize_frame422(const uint8_t *const y, int y_stride, @@ -981,11 +1127,11 @@ void av1_resize_frame422(const uint8_t *const y, int y_stride, int uv_stride, int height, int width, uint8_t *oy, int oy_stride, uint8_t *ou, uint8_t *ov, int ouv_stride, int oheight, int owidth) { - resize_plane(y, height, width, y_stride, oy, oheight, owidth, oy_stride); - resize_plane(u, height, width / 2, uv_stride, ou, oheight, owidth / 2, - ouv_stride); - resize_plane(v, height, width / 2, uv_stride, ov, oheight, owidth / 2, - ouv_stride); + av1_resize_plane(y, height, width, y_stride, oy, oheight, owidth, oy_stride); + av1_resize_plane(u, height, width / 2, uv_stride, ou, oheight, owidth / 2, + ouv_stride); + av1_resize_plane(v, height, width / 2, uv_stride, ov, oheight, owidth / 2, + ouv_stride); } void av1_resize_frame444(const uint8_t *const y, int y_stride, @@ -993,9 +1139,11 @@ void av1_resize_frame444(const uint8_t *const y, int y_stride, int uv_stride, int height, int width, uint8_t *oy, int oy_stride, uint8_t *ou, uint8_t *ov, int ouv_stride, int oheight, int owidth) { - resize_plane(y, height, width, y_stride, oy, oheight, owidth, oy_stride); - resize_plane(u, height, width, uv_stride, ou, oheight, owidth, ouv_stride); - resize_plane(v, height, width, uv_stride, ov, oheight, owidth, ouv_stride); + av1_resize_plane(y, height, width, y_stride, oy, oheight, owidth, oy_stride); + av1_resize_plane(u, height, width, uv_stride, ou, oheight, owidth, + ouv_stride); + av1_resize_plane(v, height, width, uv_stride, ov, oheight, owidth, + ouv_stride); } void av1_highbd_resize_frame420(const uint8_t *const y, int y_stride, @@ -1004,12 +1152,12 @@ void av1_highbd_resize_frame420(const uint8_t *const y, int y_stride, uint8_t *oy, int oy_stride, uint8_t *ou, uint8_t *ov, int ouv_stride, int oheight, int owidth, int bd) { - highbd_resize_plane(y, height, width, y_stride, oy, oheight, owidth, - oy_stride, bd); - highbd_resize_plane(u, height / 2, width / 2, uv_stride, ou, oheight / 2, - owidth / 2, ouv_stride, bd); - highbd_resize_plane(v, height / 2, width / 2, uv_stride, ov, oheight / 2, - owidth / 2, ouv_stride, bd); + av1_highbd_resize_plane(y, height, width, y_stride, oy, oheight, owidth, + oy_stride, bd); + av1_highbd_resize_plane(u, height / 2, width / 2, uv_stride, ou, oheight / 2, + owidth / 2, ouv_stride, bd); + av1_highbd_resize_plane(v, height / 2, width / 2, uv_stride, ov, oheight / 2, + owidth / 2, ouv_stride, bd); } void av1_highbd_resize_frame422(const uint8_t *const y, int y_stride, @@ -1018,12 +1166,12 @@ void av1_highbd_resize_frame422(const uint8_t *const y, int y_stride, uint8_t *oy, int oy_stride, uint8_t *ou, uint8_t *ov, int ouv_stride, int oheight, int owidth, int bd) { - highbd_resize_plane(y, height, width, y_stride, oy, oheight, owidth, - oy_stride, bd); - highbd_resize_plane(u, height, width / 2, uv_stride, ou, oheight, owidth / 2, - ouv_stride, bd); - highbd_resize_plane(v, height, width / 2, uv_stride, ov, oheight, owidth / 2, - ouv_stride, bd); + av1_highbd_resize_plane(y, height, width, y_stride, oy, oheight, owidth, + oy_stride, bd); + av1_highbd_resize_plane(u, height, width / 2, uv_stride, ou, oheight, + owidth / 2, ouv_stride, bd); + av1_highbd_resize_plane(v, height, width / 2, uv_stride, ov, oheight, + owidth / 2, ouv_stride, bd); } void av1_highbd_resize_frame444(const uint8_t *const y, int y_stride, @@ -1032,12 +1180,12 @@ void av1_highbd_resize_frame444(const uint8_t *const y, int y_stride, uint8_t *oy, int oy_stride, uint8_t *ou, uint8_t *ov, int ouv_stride, int oheight, int owidth, int bd) { - highbd_resize_plane(y, height, width, y_stride, oy, oheight, owidth, - oy_stride, bd); - highbd_resize_plane(u, height, width, uv_stride, ou, oheight, owidth, - ouv_stride, bd); - highbd_resize_plane(v, height, width, uv_stride, ov, oheight, owidth, - ouv_stride, bd); + av1_highbd_resize_plane(y, height, width, y_stride, oy, oheight, owidth, + oy_stride, bd); + av1_highbd_resize_plane(u, height, width, uv_stride, ou, oheight, owidth, + ouv_stride, bd); + av1_highbd_resize_plane(v, height, width, uv_stride, ov, oheight, owidth, + ouv_stride, bd); } void av1_resize_and_extend_frame(const YV12_BUFFER_CONFIG *src, @@ -1050,15 +1198,15 @@ void av1_resize_and_extend_frame(const YV12_BUFFER_CONFIG *src, for (int i = 0; i < AOMMIN(num_planes, MAX_MB_PLANE); ++i) { const int is_uv = i > 0; if (src->flags & YV12_FLAG_HIGHBITDEPTH) - highbd_resize_plane(src->buffers[i], src->crop_heights[is_uv], - src->crop_widths[is_uv], src->strides[is_uv], - dst->buffers[i], dst->crop_heights[is_uv], - dst->crop_widths[is_uv], dst->strides[is_uv], bd); + av1_highbd_resize_plane(src->buffers[i], src->crop_heights[is_uv], + src->crop_widths[is_uv], src->strides[is_uv], + dst->buffers[i], dst->crop_heights[is_uv], + dst->crop_widths[is_uv], dst->strides[is_uv], bd); else - resize_plane(src->buffers[i], src->crop_heights[is_uv], - src->crop_widths[is_uv], src->strides[is_uv], - dst->buffers[i], dst->crop_heights[is_uv], - dst->crop_widths[is_uv], dst->strides[is_uv]); + av1_resize_plane(src->buffers[i], src->crop_heights[is_uv], + src->crop_widths[is_uv], src->strides[is_uv], + dst->buffers[i], dst->crop_heights[is_uv], + dst->crop_widths[is_uv], dst->strides[is_uv]); } aom_extend_frame_borders(dst, num_planes); } @@ -1155,10 +1303,19 @@ YV12_BUFFER_CONFIG *av1_scale_if_required(AV1_COMMON *cm, // denominator. static void calculate_scaled_size_helper(int *dim, int denom) { if (denom != SCALE_NUMERATOR) { + // We need to ensure the constraint in "Appendix A" of the spec: + // * FrameWidth is greater than or equal to 16 + // * FrameHeight is greater than or equal to 16 + // For this, we clamp the downscaled dimension to at least 16. One + // exception: if original dimension itself was < 16, then we keep the + // downscaled dimension to be same as the original, to ensure that resizing + // is valid. + const int min_dim = AOMMIN(16, *dim); // Use this version if we need *dim to be even // *width = (*width * SCALE_NUMERATOR + denom) / (2 * denom); // *width <<= 1; *dim = (*dim * SCALE_NUMERATOR + denom / 2) / (denom); + *dim = AOMMAX(*dim, min_dim); } } @@ -1205,7 +1362,7 @@ void av1_superres_upscale(AV1_COMMON *cm, BufferPool *const pool) { YV12_BUFFER_CONFIG copy_buffer; memset(©_buffer, 0, sizeof(copy_buffer)); - YV12_BUFFER_CONFIG *const frame_to_show = get_frame_new_buffer(cm); + YV12_BUFFER_CONFIG *const frame_to_show = &cm->cur_frame->buf; const int aligned_width = ALIGN_POWER_OF_TWO(cm->width, 3); if (aom_alloc_frame_buffer( @@ -1225,8 +1382,7 @@ void av1_superres_upscale(AV1_COMMON *cm, BufferPool *const pool) { // Realloc the current frame buffer at a higher resolution in place. if (pool != NULL) { // Use callbacks if on the decoder. - aom_codec_frame_buffer_t *fb = - &pool->frame_bufs[cm->new_fb_idx].raw_frame_buffer; + aom_codec_frame_buffer_t *fb = &cm->cur_frame->raw_frame_buffer; aom_release_frame_buffer_cb_fn_t release_fb_cb = pool->release_fb_cb; aom_get_frame_buffer_cb_fn_t cb = pool->get_fb_cb; void *cb_priv = pool->cb_priv; diff --git a/third_party/aom/av1/common/resize.h b/third_party/aom/av1/common/resize.h index 9a59a8d63b..43bea58b82 100644 --- a/third_party/aom/av1/common/resize.h +++ b/third_party/aom/av1/common/resize.h @@ -23,6 +23,9 @@ extern "C" { void av1_resize_plane(const uint8_t *const input, int height, int width, int in_stride, uint8_t *output, int height2, int width2, int out_stride); +void av1_upscale_plane_double_prec(const double *const input, int height, + int width, int in_stride, double *output, + int height2, int width2, int out_stride); void av1_resize_frame420(const uint8_t *const y, int y_stride, const uint8_t *const u, const uint8_t *const v, int uv_stride, int height, int width, uint8_t *oy, diff --git a/third_party/aom/av1/common/restoration.c b/third_party/aom/av1/common/restoration.c index d276a915b5..9e472b8a6e 100644 --- a/third_party/aom/av1/common/restoration.c +++ b/third_party/aom/av1/common/restoration.c @@ -1072,13 +1072,6 @@ void av1_loop_restoration_filter_unit( } } -static void filter_frame_on_tile(int tile_row, int tile_col, void *priv, - AV1_COMMON *cm) { - (void)tile_col; - FilterFrameCtxt *ctxt = (FilterFrameCtxt *)priv; - ctxt->tile_stripe0 = (tile_row == 0) ? 0 : cm->rst_end_stripe[tile_row - 1]; -} - static void filter_frame_on_unit(const RestorationTileLimits *limits, const AV1PixelRect *tile_rect, int rest_unit_idx, void *priv, int32_t *tmpbuf, @@ -1106,7 +1099,7 @@ void av1_loop_restoration_filter_frame_init(AV1LrStruct *lr_ctxt, const int frame_height = frame->crop_heights[0]; if (aom_realloc_frame_buffer( lr_ctxt->dst, frame_width, frame_height, seq_params->subsampling_x, - seq_params->subsampling_y, highbd, AOM_BORDER_IN_PIXELS, + seq_params->subsampling_y, highbd, AOM_RESTORATION_FRAME_BORDER, cm->byte_alignment, NULL, NULL, NULL) < 0) aom_internal_error(&cm->error, AOM_CODEC_MEM_ERROR, "Failed to allocate restoration dst buffer"); @@ -1141,7 +1134,7 @@ void av1_loop_restoration_filter_frame_init(AV1LrStruct *lr_ctxt, lr_plane_ctxt->data_stride = frame->strides[is_uv]; lr_plane_ctxt->dst_stride = lr_ctxt->dst->strides[is_uv]; lr_plane_ctxt->tile_rect = av1_whole_frame_rect(cm, is_uv); - filter_frame_on_tile(LR_TILE_ROW, LR_TILE_COL, lr_plane_ctxt, cm); + lr_plane_ctxt->tile_stripe0 = 0; } } @@ -1150,9 +1143,9 @@ void av1_loop_restoration_copy_planes(AV1LrStruct *loop_rest_ctxt, typedef void (*copy_fun)(const YV12_BUFFER_CONFIG *src_ybc, YV12_BUFFER_CONFIG *dst_ybc, int hstart, int hend, int vstart, int vend); - static const copy_fun copy_funs[3] = { - aom_yv12_partial_copy_y, aom_yv12_partial_copy_u, aom_yv12_partial_copy_v - }; + static const copy_fun copy_funs[3] = { aom_yv12_partial_coloc_copy_y, + aom_yv12_partial_coloc_copy_u, + aom_yv12_partial_coloc_copy_v }; for (int plane = 0; plane < num_planes; ++plane) { if (cm->rst_info[plane].frame_restoration_type == RESTORE_NONE) continue; diff --git a/third_party/aom/av1/common/restoration.h b/third_party/aom/av1/common/restoration.h index d834f9270f..5d196a9a59 100644 --- a/third_party/aom/av1/common/restoration.h +++ b/third_party/aom/av1/common/restoration.h @@ -22,6 +22,8 @@ extern "C" { #endif +// Boarder for Loop restoration buffer +#define AOM_RESTORATION_FRAME_BORDER 32 #define CLIP(x, lo, hi) ((x) < (lo) ? (lo) : (x) > (hi) ? (hi) : (x)) #define RINT(x) ((x) < 0 ? (int)((x)-0.5) : (int)((x) + 0.5)) diff --git a/third_party/aom/av1/common/scale.c b/third_party/aom/av1/common/scale.c index c525fe2296..bac7bd9cbd 100644 --- a/third_party/aom/av1/common/scale.c +++ b/third_party/aom/av1/common/scale.c @@ -97,13 +97,13 @@ void av1_setup_scale_factors_for_frame(struct scale_factors *sf, int other_w, // subpel_x_q4 != 0 && subpel_y_q4 != 0 sf->convolve[1][1][0] = av1_convolve_2d_sr; // subpel_x_q4 == 0 && subpel_y_q4 == 0 - sf->convolve[0][0][1] = av1_jnt_convolve_2d_copy; + sf->convolve[0][0][1] = av1_dist_wtd_convolve_2d_copy; // subpel_x_q4 == 0 - sf->convolve[0][1][1] = av1_jnt_convolve_y; + sf->convolve[0][1][1] = av1_dist_wtd_convolve_y; // subpel_y_q4 == 0 - sf->convolve[1][0][1] = av1_jnt_convolve_x; + sf->convolve[1][0][1] = av1_dist_wtd_convolve_x; // subpel_x_q4 != 0 && subpel_y_q4 != 0 - sf->convolve[1][1][1] = av1_jnt_convolve_2d; + sf->convolve[1][1][1] = av1_dist_wtd_convolve_2d; // AV1 High BD convolve functions // Special case convolve functions should produce the same result as // av1_highbd_convolve_2d. @@ -116,11 +116,11 @@ void av1_setup_scale_factors_for_frame(struct scale_factors *sf, int other_w, // subpel_x_q4 != 0 && subpel_y_q4 != 0 sf->highbd_convolve[1][1][0] = av1_highbd_convolve_2d_sr; // subpel_x_q4 == 0 && subpel_y_q4 == 0 - sf->highbd_convolve[0][0][1] = av1_highbd_jnt_convolve_2d_copy; + sf->highbd_convolve[0][0][1] = av1_highbd_dist_wtd_convolve_2d_copy; // subpel_x_q4 == 0 - sf->highbd_convolve[0][1][1] = av1_highbd_jnt_convolve_y; + sf->highbd_convolve[0][1][1] = av1_highbd_dist_wtd_convolve_y; // subpel_y_q4 == 0 - sf->highbd_convolve[1][0][1] = av1_highbd_jnt_convolve_x; + sf->highbd_convolve[1][0][1] = av1_highbd_dist_wtd_convolve_x; // subpel_x_q4 != 0 && subpel_y_q4 != 0 - sf->highbd_convolve[1][1][1] = av1_highbd_jnt_convolve_2d; + sf->highbd_convolve[1][1][1] = av1_highbd_dist_wtd_convolve_2d; } diff --git a/third_party/aom/av1/common/scan.h b/third_party/aom/av1/common/scan.h index 233dc0efa2..f9c339215b 100644 --- a/third_party/aom/av1/common/scan.h +++ b/third_party/aom/av1/common/scan.h @@ -25,14 +25,14 @@ extern "C" { #define MAX_NEIGHBORS 2 -typedef enum SCAN_MODE { +enum { SCAN_MODE_ZIG_ZAG, SCAN_MODE_COL_DIAG, SCAN_MODE_ROW_DIAG, SCAN_MODE_COL_1D, SCAN_MODE_ROW_1D, SCAN_MODES -} SCAN_MODE; +} UENUM1BYTE(SCAN_MODE); extern const SCAN_ORDER av1_default_scan_orders[TX_SIZES]; extern const SCAN_ORDER av1_scan_orders[TX_SIZES_ALL][TX_TYPES]; diff --git a/third_party/aom/av1/common/seg_common.c b/third_party/aom/av1/common/seg_common.c index cd189ad769..4650903b91 100644 --- a/third_party/aom/av1/common/seg_common.c +++ b/third_party/aom/av1/common/seg_common.c @@ -16,12 +16,19 @@ #include "av1/common/seg_common.h" #include "av1/common/quant_common.h" -static const int seg_feature_data_signed[SEG_LVL_MAX] = { 1, 1, 1, 1, 1, 0, 0 }; - -static const int seg_feature_data_max[SEG_LVL_MAX] = { - MAXQ, MAX_LOOP_FILTER, MAX_LOOP_FILTER, MAX_LOOP_FILTER, MAX_LOOP_FILTER, 7, 0 +static const int seg_feature_data_signed[SEG_LVL_MAX] = { + 1, 1, 1, 1, 1, 0, 0, 0 }; +static const int seg_feature_data_max[SEG_LVL_MAX] = { MAXQ, + MAX_LOOP_FILTER, + MAX_LOOP_FILTER, + MAX_LOOP_FILTER, + MAX_LOOP_FILTER, + 7, + 0, + 0 }; + // These functions provide access to new segment level features. // Eventually these function may be "optimized out" but for the moment, // the coding mechanism is still subject to change so these provide a diff --git a/third_party/aom/av1/common/seg_common.h b/third_party/aom/av1/common/seg_common.h index 8c35bba86c..fa7894cf61 100644 --- a/third_party/aom/av1/common/seg_common.h +++ b/third_party/aom/av1/common/seg_common.h @@ -24,7 +24,7 @@ extern "C" { #define SEG_TEMPORAL_PRED_CTXS 3 #define SPATIAL_PREDICTION_PROBS 3 -typedef enum { +enum { SEG_LVL_ALT_Q, // Use alternate Quantizer .... SEG_LVL_ALT_LF_Y_V, // Use alternate loop filter value on y plane vertical SEG_LVL_ALT_LF_Y_H, // Use alternate loop filter value on y plane horizontal @@ -34,7 +34,7 @@ typedef enum { SEG_LVL_SKIP, // Optional Segment (0,0) + skip mode SEG_LVL_GLOBALMV, SEG_LVL_MAX -} SEG_LVL_FEATURES; +} UENUM1BYTE(SEG_LVL_FEATURES); struct segmentation { uint8_t enabled; diff --git a/third_party/aom/av1/common/thread_common.c b/third_party/aom/av1/common/thread_common.c index 8df4c9a09d..14406e6444 100644 --- a/third_party/aom/av1/common/thread_common.c +++ b/third_party/aom/av1/common/thread_common.c @@ -205,7 +205,11 @@ static INLINE void sync_write(AV1LfSync *const lf_sync, int r, int c, } static void enqueue_lf_jobs(AV1LfSync *lf_sync, AV1_COMMON *cm, int start, - int stop, int plane_start, int plane_end) { + int stop, +#if LOOP_FILTER_BITMASK + int is_decoding, +#endif + int plane_start, int plane_end) { int mi_row, plane, dir; AV1LfMTInfo *lf_job_queue = lf_sync->job_queue; lf_sync->jobs_enqueued = 0; @@ -219,7 +223,16 @@ static void enqueue_lf_jobs(AV1LfSync *lf_sync, AV1_COMMON *cm, int start, continue; else if (plane == 2 && !(cm->lf.filter_level_v)) continue; - for (mi_row = start; mi_row < stop; mi_row += MAX_MIB_SIZE) { +#if LOOP_FILTER_BITMASK + int step = MAX_MIB_SIZE; + if (is_decoding) { + step = MI_SIZE_64X64; + } + for (mi_row = start; mi_row < stop; mi_row += step) +#else + for (mi_row = start; mi_row < stop; mi_row += MAX_MIB_SIZE) +#endif + { lf_job_queue->mi_row = mi_row; lf_job_queue->plane = plane; lf_job_queue->dir = dir; @@ -230,7 +243,7 @@ static void enqueue_lf_jobs(AV1LfSync *lf_sync, AV1_COMMON *cm, int start, } } -AV1LfMTInfo *get_lf_job_info(AV1LfSync *lf_sync) { +static AV1LfMTInfo *get_lf_job_info(AV1LfSync *lf_sync) { AV1LfMTInfo *cur_job_info = NULL; #if CONFIG_MULTITHREAD @@ -312,15 +325,94 @@ static int loop_filter_row_worker(void *arg1, void *arg2) { return 1; } +#if LOOP_FILTER_BITMASK +static INLINE void thread_loop_filter_bitmask_rows( + const YV12_BUFFER_CONFIG *const frame_buffer, AV1_COMMON *const cm, + struct macroblockd_plane *planes, MACROBLOCKD *xd, + AV1LfSync *const lf_sync) { + const int sb_cols = + ALIGN_POWER_OF_TWO(cm->mi_cols, MIN_MIB_SIZE_LOG2) >> MIN_MIB_SIZE_LOG2; + int mi_row, mi_col, plane, dir; + int r, c; + (void)xd; + + while (1) { + AV1LfMTInfo *cur_job_info = get_lf_job_info(lf_sync); + + if (cur_job_info != NULL) { + mi_row = cur_job_info->mi_row; + plane = cur_job_info->plane; + dir = cur_job_info->dir; + r = mi_row >> MIN_MIB_SIZE_LOG2; + + if (dir == 0) { + for (mi_col = 0; mi_col < cm->mi_cols; mi_col += MI_SIZE_64X64) { + c = mi_col >> MIN_MIB_SIZE_LOG2; + + av1_setup_dst_planes(planes, BLOCK_64X64, frame_buffer, mi_row, + mi_col, plane, plane + 1); + + av1_filter_block_plane_bitmask_vert(cm, &planes[plane], plane, mi_row, + mi_col); + sync_write(lf_sync, r, c, sb_cols, plane); + } + } else if (dir == 1) { + for (mi_col = 0; mi_col < cm->mi_cols; mi_col += MI_SIZE_64X64) { + c = mi_col >> MIN_MIB_SIZE_LOG2; + + // Wait for vertical edge filtering of the top-right block to be + // completed + sync_read(lf_sync, r, c, plane); + + // Wait for vertical edge filtering of the right block to be + // completed + sync_read(lf_sync, r + 1, c, plane); + + av1_setup_dst_planes(planes, BLOCK_64X64, frame_buffer, mi_row, + mi_col, plane, plane + 1); + av1_filter_block_plane_bitmask_horz(cm, &planes[plane], plane, mi_row, + mi_col); + } + } + } else { + break; + } + } +} + +// Row-based multi-threaded loopfilter hook +static int loop_filter_bitmask_row_worker(void *arg1, void *arg2) { + AV1LfSync *const lf_sync = (AV1LfSync *)arg1; + LFWorkerData *const lf_data = (LFWorkerData *)arg2; + thread_loop_filter_bitmask_rows(lf_data->frame_buffer, lf_data->cm, + lf_data->planes, lf_data->xd, lf_sync); + return 1; +} +#endif // LOOP_FILTER_BITMASK + static void loop_filter_rows_mt(YV12_BUFFER_CONFIG *frame, AV1_COMMON *cm, MACROBLOCKD *xd, int start, int stop, int plane_start, int plane_end, +#if LOOP_FILTER_BITMASK + int is_decoding, +#endif AVxWorker *workers, int nworkers, AV1LfSync *lf_sync) { const AVxWorkerInterface *const winterface = aom_get_worker_interface(); +#if LOOP_FILTER_BITMASK + int sb_rows; + if (is_decoding) { + sb_rows = + ALIGN_POWER_OF_TWO(cm->mi_rows, MIN_MIB_SIZE_LOG2) >> MIN_MIB_SIZE_LOG2; + } else { + sb_rows = + ALIGN_POWER_OF_TWO(cm->mi_rows, MAX_MIB_SIZE_LOG2) >> MAX_MIB_SIZE_LOG2; + } +#else // Number of superblock rows and cols const int sb_rows = ALIGN_POWER_OF_TWO(cm->mi_rows, MAX_MIB_SIZE_LOG2) >> MAX_MIB_SIZE_LOG2; +#endif const int num_workers = nworkers; int i; @@ -336,14 +428,26 @@ static void loop_filter_rows_mt(YV12_BUFFER_CONFIG *frame, AV1_COMMON *cm, sizeof(*(lf_sync->cur_sb_col[i])) * sb_rows); } - enqueue_lf_jobs(lf_sync, cm, start, stop, plane_start, plane_end); + enqueue_lf_jobs(lf_sync, cm, start, stop, +#if LOOP_FILTER_BITMASK + is_decoding, +#endif + plane_start, plane_end); // Set up loopfilter thread data. for (i = 0; i < num_workers; ++i) { AVxWorker *const worker = &workers[i]; LFWorkerData *const lf_data = &lf_sync->lfdata[i]; +#if LOOP_FILTER_BITMASK + if (is_decoding) { + worker->hook = loop_filter_bitmask_row_worker; + } else { + worker->hook = loop_filter_row_worker; + } +#else worker->hook = loop_filter_row_worker; +#endif worker->data1 = lf_sync; worker->data2 = lf_data; @@ -366,8 +470,12 @@ static void loop_filter_rows_mt(YV12_BUFFER_CONFIG *frame, AV1_COMMON *cm, void av1_loop_filter_frame_mt(YV12_BUFFER_CONFIG *frame, AV1_COMMON *cm, MACROBLOCKD *xd, int plane_start, int plane_end, - int partial_frame, AVxWorker *workers, - int num_workers, AV1LfSync *lf_sync) { + int partial_frame, +#if LOOP_FILTER_BITMASK + int is_decoding, +#endif + AVxWorker *workers, int num_workers, + AV1LfSync *lf_sync) { int start_mi_row, end_mi_row, mi_rows_to_filter; start_mi_row = 0; @@ -380,8 +488,37 @@ void av1_loop_filter_frame_mt(YV12_BUFFER_CONFIG *frame, AV1_COMMON *cm, end_mi_row = start_mi_row + mi_rows_to_filter; av1_loop_filter_frame_init(cm, plane_start, plane_end); +#if LOOP_FILTER_BITMASK + if (is_decoding) { + cm->is_decoding = is_decoding; + // TODO(chengchen): currently use one thread to build bitmasks for the + // frame. Make it support multi-thread later. + for (int plane = plane_start; plane < plane_end; plane++) { + if (plane == 0 && !(cm->lf.filter_level[0]) && !(cm->lf.filter_level[1])) + break; + else if (plane == 1 && !(cm->lf.filter_level_u)) + continue; + else if (plane == 2 && !(cm->lf.filter_level_v)) + continue; + + // TODO(chengchen): can we remove this? + struct macroblockd_plane *pd = xd->plane; + av1_setup_dst_planes(pd, cm->seq_params.sb_size, frame, 0, 0, plane, + plane + 1); + + av1_build_bitmask_vert_info(cm, &pd[plane], plane); + av1_build_bitmask_horz_info(cm, &pd[plane], plane); + } + loop_filter_rows_mt(frame, cm, xd, start_mi_row, end_mi_row, plane_start, + plane_end, 1, workers, num_workers, lf_sync); + } else { + loop_filter_rows_mt(frame, cm, xd, start_mi_row, end_mi_row, plane_start, + plane_end, 0, workers, num_workers, lf_sync); + } +#else loop_filter_rows_mt(frame, cm, xd, start_mi_row, end_mi_row, plane_start, plane_end, workers, num_workers, lf_sync); +#endif } static INLINE void lr_sync_read(void *const lr_sync, int r, int c, int plane) { @@ -630,7 +767,7 @@ static void enqueue_lr_jobs(AV1LrSync *lr_sync, AV1LrStruct *lr_ctxt, } } -AV1LrMTInfo *get_lr_job_info(AV1LrSync *lr_sync) { +static AV1LrMTInfo *get_lr_job_info(AV1LrSync *lr_sync) { AV1LrMTInfo *cur_job_info = NULL; #if CONFIG_MULTITHREAD @@ -664,9 +801,9 @@ static int loop_restoration_row_worker(void *arg1, void *arg2) { typedef void (*copy_fun)(const YV12_BUFFER_CONFIG *src_ybc, YV12_BUFFER_CONFIG *dst_ybc, int hstart, int hend, int vstart, int vend); - static const copy_fun copy_funs[3] = { - aom_yv12_partial_copy_y, aom_yv12_partial_copy_u, aom_yv12_partial_copy_v - }; + static const copy_fun copy_funs[3] = { aom_yv12_partial_coloc_copy_y, + aom_yv12_partial_coloc_copy_u, + aom_yv12_partial_coloc_copy_v }; while (1) { AV1LrMTInfo *cur_job_info = get_lr_job_info(lr_sync); diff --git a/third_party/aom/av1/common/thread_common.h b/third_party/aom/av1/common/thread_common.h index 23d61d72a3..e7dbb8bd33 100644 --- a/third_party/aom/av1/common/thread_common.h +++ b/third_party/aom/av1/common/thread_common.h @@ -103,6 +103,9 @@ void av1_loop_filter_dealloc(AV1LfSync *lf_sync); void av1_loop_filter_frame_mt(YV12_BUFFER_CONFIG *frame, struct AV1Common *cm, struct macroblockd *mbd, int plane_start, int plane_end, int partial_frame, +#if LOOP_FILTER_BITMASK + int is_decoding, +#endif AVxWorker *workers, int num_workers, AV1LfSync *lf_sync); void av1_loop_restoration_filter_frame_mt(YV12_BUFFER_CONFIG *frame, diff --git a/third_party/aom/av1/common/tile_common.c b/third_party/aom/av1/common/tile_common.c index 1b413487f0..4d905924ac 100644 --- a/third_party/aom/av1/common/tile_common.c +++ b/third_party/aom/av1/common/tile_common.c @@ -143,30 +143,6 @@ int av1_get_sb_cols_in_tile(AV1_COMMON *cm, TileInfo tile) { return sb_cols; } -int get_tile_size(int mi_frame_size, int log2_tile_num, int *ntiles) { - // Round the frame up to a whole number of max superblocks - mi_frame_size = ALIGN_POWER_OF_TWO(mi_frame_size, MAX_MIB_SIZE_LOG2); - - // Divide by the signalled number of tiles, rounding up to the multiple of - // the max superblock size. To do this, shift right (and round up) to get the - // tile size in max super-blocks and then shift left again to convert it to - // mi units. - const int shift = log2_tile_num + MAX_MIB_SIZE_LOG2; - const int max_sb_tile_size = - ALIGN_POWER_OF_TWO(mi_frame_size, shift) >> shift; - const int mi_tile_size = max_sb_tile_size << MAX_MIB_SIZE_LOG2; - - // The actual number of tiles is the ceiling of the frame size in mi units - // divided by mi_size. This is at most 1 << log2_tile_num but might be - // strictly less if max_sb_tile_size got rounded up significantly. - if (ntiles) { - *ntiles = (mi_frame_size + mi_tile_size - 1) / mi_tile_size; - assert(*ntiles <= (1 << log2_tile_num)); - } - - return mi_tile_size; -} - AV1PixelRect av1_get_tile_rect(const TileInfo *tile_info, const AV1_COMMON *cm, int is_uv) { AV1PixelRect r; @@ -205,3 +181,26 @@ AV1PixelRect av1_get_tile_rect(const TileInfo *tile_info, const AV1_COMMON *cm, return r; } + +void av1_get_uniform_tile_size(const AV1_COMMON *cm, int *w, int *h) { + if (cm->uniform_tile_spacing_flag) { + *w = cm->tile_width; + *h = cm->tile_height; + } else { + for (int i = 0; i < cm->tile_cols; ++i) { + const int tile_width_sb = + cm->tile_col_start_sb[i + 1] - cm->tile_col_start_sb[i]; + const int tile_w = tile_width_sb * cm->seq_params.mib_size; + assert(i == 0 || tile_w == *w); // ensure all tiles have same dimension + *w = tile_w; + } + + for (int i = 0; i < cm->tile_rows; ++i) { + const int tile_height_sb = + cm->tile_row_start_sb[i + 1] - cm->tile_row_start_sb[i]; + const int tile_h = tile_height_sb * cm->seq_params.mib_size; + assert(i == 0 || tile_h == *h); // ensure all tiles have same dimension + *h = tile_h; + } + } +} diff --git a/third_party/aom/av1/common/tile_common.h b/third_party/aom/av1/common/tile_common.h index c03553dc6f..b7203cef75 100644 --- a/third_party/aom/av1/common/tile_common.h +++ b/third_party/aom/av1/common/tile_common.h @@ -25,7 +25,6 @@ struct AV1Common; typedef struct TileInfo { int mi_row_start, mi_row_end; int mi_col_start, mi_col_end; - int tg_horz_boundary; int tile_row; int tile_col; } TileInfo; @@ -37,12 +36,6 @@ void av1_tile_init(TileInfo *tile, const struct AV1Common *cm, int row, void av1_tile_set_row(TileInfo *tile, const struct AV1Common *cm, int row); void av1_tile_set_col(TileInfo *tile, const struct AV1Common *cm, int col); -void av1_get_tile_n_bits(int mi_cols, int *min_log2_tile_cols, - int *max_log2_tile_cols); - -// Calculate the correct tile size (width or height) for (1 << log2_tile_num) -// tiles horizontally or vertically in the frame. -int get_tile_size(int mi_frame_size, int log2_tile_num, int *ntiles); int av1_get_sb_rows_in_tile(struct AV1Common *cm, TileInfo tile); int av1_get_sb_cols_in_tile(struct AV1Common *cm, TileInfo tile); @@ -61,6 +54,7 @@ AV1PixelRect av1_get_tile_rect(const TileInfo *tile_info, #define MAX_TILE_WIDTH (4096) // Max Tile width in pixels #define MAX_TILE_AREA (4096 * 2304) // Maximum tile area in pixels +void av1_get_uniform_tile_size(const struct AV1Common *cm, int *w, int *h); void av1_get_tile_limits(struct AV1Common *const cm); void av1_calculate_tile_cols(struct AV1Common *const cm); void av1_calculate_tile_rows(struct AV1Common *const cm); diff --git a/third_party/aom/av1/common/txb_common.c b/third_party/aom/av1/common/txb_common.c index c96d37cca7..cb92bd8c6e 100644 --- a/third_party/aom/av1/common/txb_common.c +++ b/third_party/aom/av1/common/txb_common.c @@ -453,23 +453,6 @@ const int8_t *av1_nz_map_ctx_offset[19] = { av1_nz_map_ctx_offset_64x32, // TX_64x16 }; -void av1_init_lv_map(AV1_COMMON *cm) { - LV_MAP_CTX_TABLE *coeff_ctx_table = &cm->coeff_ctx_table; - for (int row = 0; row < 2; ++row) { - for (int col = 0; col < 2; ++col) { - for (int sig_mag = 0; sig_mag < 3; ++sig_mag) { - for (int count = 0; count < BASE_CONTEXT_POSITION_NUM + 1; ++count) { - if (row == 0 && col == 0 && count > 5) continue; - if ((row == 0 || col == 0) && count > 8) continue; - - coeff_ctx_table->base_ctx_table[row][col][sig_mag][count] = - get_base_ctx_from_count_mag(row, col, count, sig_mag); - } - } - } - } -} - const int16_t k_eob_group_start[12] = { 0, 1, 2, 3, 5, 9, 17, 33, 65, 129, 257, 513 }; const int16_t k_eob_offset_bits[12] = { 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; diff --git a/third_party/aom/av1/common/txb_common.h b/third_party/aom/av1/common/txb_common.h index 1dda51f8b4..8a3932d91d 100644 --- a/third_party/aom/av1/common/txb_common.h +++ b/third_party/aom/av1/common/txb_common.h @@ -12,6 +12,8 @@ #ifndef AOM_AV1_COMMON_TXB_COMMON_H_ #define AOM_AV1_COMMON_TXB_COMMON_H_ +#include "av1/common/onyxc_int.h" + extern const int16_t k_eob_group_start[12]; extern const int16_t k_eob_offset_bits[12]; @@ -157,6 +159,19 @@ static INLINE int get_br_ctx_2d(const uint8_t *const levels, return mag + 14; } +static AOM_FORCE_INLINE int get_br_ctx_eob(const int c, // raster order + const int bwl, + const TX_CLASS tx_class) { + const int row = c >> bwl; + const int col = c - (row << bwl); + if (c == 0) return 0; + if ((tx_class == TX_CLASS_2D && row < 2 && col < 2) || + (tx_class == TX_CLASS_HORIZ && col == 0) || + (tx_class == TX_CLASS_VERT && row == 0)) + return 7; + return 14; +} + static AOM_FORCE_INLINE int get_br_ctx(const uint8_t *const levels, const int c, // raster order const int bwl, const TX_CLASS tx_class) { @@ -270,12 +285,10 @@ static AOM_FORCE_INLINE int get_nz_map_ctx_from_stats( const int row = coeff_idx >> bwl; const int col = coeff_idx - (row << bwl); return ctx + nz_map_ctx_offset_1d[col]; - break; } case TX_CLASS_VERT: { const int row = coeff_idx >> bwl; return ctx + nz_map_ctx_offset_1d[row]; - break; } default: break; } @@ -419,6 +432,4 @@ static INLINE void get_txb_ctx(const BLOCK_SIZE plane_bsize, #undef MAX_TX_SIZE_UNIT } -void av1_init_lv_map(AV1_COMMON *cm); - #endif // AOM_AV1_COMMON_TXB_COMMON_H_ diff --git a/third_party/aom/av1/common/warped_motion.c b/third_party/aom/av1/common/warped_motion.c index 4144c43896..e232e10828 100644 --- a/third_party/aom/av1/common/warped_motion.c +++ b/third_party/aom/av1/common/warped_motion.c @@ -485,7 +485,7 @@ void av1_highbd_warp_affine_c(const int32_t *mat, const uint16_t *ref, uint16_t *dst16 = &pred[(i - p_row + k + 4) * p_stride + (j - p_col + l + 4)]; int32_t tmp32 = *p; - if (conv_params->use_jnt_comp_avg) { + if (conv_params->use_dist_wtd_comp_avg) { tmp32 = tmp32 * conv_params->fwd_offset + sum * conv_params->bck_offset; tmp32 = tmp32 >> DIST_PRECISION_BITS; @@ -563,7 +563,7 @@ static int64_t highbd_warp_error( uint16_t tmp[WARP_ERROR_BLOCK * WARP_ERROR_BLOCK]; ConvolveParams conv_params = get_conv_params(0, 0, bd); - conv_params.use_jnt_comp_avg = 0; + conv_params.use_dist_wtd_comp_avg = 0; for (int i = p_row; i < p_row + p_height; i += WARP_ERROR_BLOCK) { for (int j = p_col; j < p_col + p_width; j += WARP_ERROR_BLOCK) { // avoid warping extra 8x8 blocks in the padded region of the frame @@ -773,7 +773,7 @@ void av1_warp_affine_c(const int32_t *mat, const uint8_t *ref, int width, uint8_t *dst8 = &pred[(i - p_row + k + 4) * p_stride + (j - p_col + l + 4)]; int32_t tmp32 = *p; - if (conv_params->use_jnt_comp_avg) { + if (conv_params->use_dist_wtd_comp_avg) { tmp32 = tmp32 * conv_params->fwd_offset + sum * conv_params->bck_offset; tmp32 = tmp32 >> DIST_PRECISION_BITS; @@ -846,7 +846,7 @@ static int64_t warp_error(WarpedMotionParams *wm, const uint8_t *const ref, int error_bsize_h = AOMMIN(p_height, WARP_ERROR_BLOCK); uint8_t tmp[WARP_ERROR_BLOCK * WARP_ERROR_BLOCK]; ConvolveParams conv_params = get_conv_params(0, 0, 8); - conv_params.use_jnt_comp_avg = 0; + conv_params.use_dist_wtd_comp_avg = 0; for (int i = p_row; i < p_row + p_height; i += WARP_ERROR_BLOCK) { for (int j = p_col; j < p_col + p_width; j += WARP_ERROR_BLOCK) { diff --git a/third_party/aom/av1/common/x86/av1_convolve_scale_sse4.c b/third_party/aom/av1/common/x86/av1_convolve_scale_sse4.c index d9fb537856..8f442383ca 100644 --- a/third_party/aom/av1/common/x86/av1_convolve_scale_sse4.c +++ b/third_party/aom/av1/common/x86/av1_convolve_scale_sse4.c @@ -175,7 +175,7 @@ static void vfilter8(const int16_t *src, int src_stride, uint8_t *dst, if (conv_params->is_compound) { if (conv_params->do_average) { const __m128i p_16 = _mm_loadl_epi64((__m128i *)dst_16_x); - if (conv_params->use_jnt_comp_avg) { + if (conv_params->use_dist_wtd_comp_avg) { const __m128i p_16_lo = _mm_unpacklo_epi16(p_16, shifted_16); const __m128i wt_res_lo = _mm_madd_epi16(p_16_lo, wt); const __m128i shifted_32 = @@ -207,7 +207,7 @@ static void vfilter8(const int16_t *src, int src_stride, uint8_t *dst, if (conv_params->is_compound) { if (conv_params->do_average) { int32_t tmp = dst16[y * dst16_stride + x]; - if (conv_params->use_jnt_comp_avg) { + if (conv_params->use_dist_wtd_comp_avg) { tmp = tmp * conv_params->fwd_offset + res * conv_params->bck_offset; tmp = tmp >> DIST_PRECISION_BITS; } else { @@ -408,7 +408,7 @@ static void highbd_vfilter8(const int16_t *src, int src_stride, uint16_t *dst, __m128i p_32 = _mm_cvtepu16_epi32(_mm_loadl_epi64((__m128i *)dst_16_x)); - if (conv_params->use_jnt_comp_avg) { + if (conv_params->use_dist_wtd_comp_avg) { shifted = _mm_add_epi32(_mm_mullo_epi32(p_32, wt0), _mm_mullo_epi32(shifted, wt1)); shifted = _mm_srai_epi32(shifted, DIST_PRECISION_BITS); @@ -443,7 +443,7 @@ static void highbd_vfilter8(const int16_t *src, int src_stride, uint16_t *dst, if (conv_params->is_compound) { if (conv_params->do_average) { int32_t tmp = dst16[y * dst16_stride + x]; - if (conv_params->use_jnt_comp_avg) { + if (conv_params->use_dist_wtd_comp_avg) { tmp = tmp * conv_params->fwd_offset + res * conv_params->bck_offset; tmp = tmp >> DIST_PRECISION_BITS; } else { diff --git a/third_party/aom/av1/common/x86/av1_inv_txfm_avx2.c b/third_party/aom/av1/common/x86/av1_inv_txfm_avx2.c index 5db2ccf6c3..cf1f947d30 100644 --- a/third_party/aom/av1/common/x86/av1_inv_txfm_avx2.c +++ b/third_party/aom/av1/common/x86/av1_inv_txfm_avx2.c @@ -1577,6 +1577,9 @@ static void idct64_low32_new_avx2(const __m256i *input, __m256i *output, idct64_stage11_avx2(output, x); } +typedef void (*transform_1d_avx2)(const __m256i *input, __m256i *output, + int8_t cos_bit); + // 1D functions process 16 pixels at one time. static const transform_1d_avx2 lowbd_txfm_all_1d_zeros_w16_arr[TX_SIZES][ITX_TYPES_1D][4] = { diff --git a/third_party/aom/av1/common/x86/av1_inv_txfm_ssse3.c b/third_party/aom/av1/common/x86/av1_inv_txfm_ssse3.c index 995bc3da44..9841bf3e24 100644 --- a/third_party/aom/av1/common/x86/av1_inv_txfm_ssse3.c +++ b/third_party/aom/av1/common/x86/av1_inv_txfm_ssse3.c @@ -2437,9 +2437,10 @@ static INLINE void lowbd_inv_txfm2d_add_idtx_ssse3(const int32_t *input, } } -void lowbd_inv_txfm2d_add_4x4_ssse3(const int32_t *input, uint8_t *output, - int stride, TX_TYPE tx_type, - TX_SIZE tx_size_, int eob) { +static void lowbd_inv_txfm2d_add_4x4_ssse3(const int32_t *input, + uint8_t *output, int stride, + TX_TYPE tx_type, TX_SIZE tx_size_, + int eob) { (void)tx_size_; (void)eob; __m128i buf[4]; @@ -2708,9 +2709,10 @@ static INLINE void lowbd_inv_txfm2d_add_universe_ssse3( } } -void lowbd_inv_txfm2d_add_4x8_ssse3(const int32_t *input, uint8_t *output, - int stride, TX_TYPE tx_type, - TX_SIZE tx_size_, int eob) { +static void lowbd_inv_txfm2d_add_4x8_ssse3(const int32_t *input, + uint8_t *output, int stride, + TX_TYPE tx_type, TX_SIZE tx_size_, + int eob) { (void)tx_size_; (void)eob; __m128i buf[8]; @@ -2747,9 +2749,10 @@ void lowbd_inv_txfm2d_add_4x8_ssse3(const int32_t *input, uint8_t *output, lowbd_write_buffer_4xn_sse2(buf, output, stride, ud_flip, txfm_size_row); } -void lowbd_inv_txfm2d_add_8x4_ssse3(const int32_t *input, uint8_t *output, - int stride, TX_TYPE tx_type, - TX_SIZE tx_size_, int eob) { +static void lowbd_inv_txfm2d_add_8x4_ssse3(const int32_t *input, + uint8_t *output, int stride, + TX_TYPE tx_type, TX_SIZE tx_size_, + int eob) { (void)tx_size_; (void)eob; __m128i buf[8]; @@ -2786,9 +2789,10 @@ void lowbd_inv_txfm2d_add_8x4_ssse3(const int32_t *input, uint8_t *output, lowbd_write_buffer_8xn_sse2(buf, output, stride, ud_flip, txfm_size_row); } -void lowbd_inv_txfm2d_add_4x16_ssse3(const int32_t *input, uint8_t *output, - int stride, TX_TYPE tx_type, - TX_SIZE tx_size_, int eob) { +static void lowbd_inv_txfm2d_add_4x16_ssse3(const int32_t *input, + uint8_t *output, int stride, + TX_TYPE tx_type, TX_SIZE tx_size_, + int eob) { (void)tx_size_; (void)eob; __m128i buf[16]; @@ -2831,9 +2835,10 @@ void lowbd_inv_txfm2d_add_4x16_ssse3(const int32_t *input, uint8_t *output, lowbd_write_buffer_4xn_sse2(buf, output, stride, ud_flip, txfm_size_row); } -void lowbd_inv_txfm2d_add_16x4_ssse3(const int32_t *input, uint8_t *output, - int stride, TX_TYPE tx_type, - TX_SIZE tx_size_, int eob) { +static void lowbd_inv_txfm2d_add_16x4_ssse3(const int32_t *input, + uint8_t *output, int stride, + TX_TYPE tx_type, TX_SIZE tx_size_, + int eob) { (void)tx_size_; (void)eob; __m128i buf[16]; diff --git a/third_party/aom/av1/common/x86/av1_inv_txfm_ssse3.h b/third_party/aom/av1/common/x86/av1_inv_txfm_ssse3.h index 66bd339d11..7d5055debe 100644 --- a/third_party/aom/av1/common/x86/av1_inv_txfm_ssse3.h +++ b/third_party/aom/av1/common/x86/av1_inv_txfm_ssse3.h @@ -72,13 +72,13 @@ static INLINE void round_shift_16bit_ssse3(__m128i *in, int size, int bit) { } // 1D itx types -typedef enum ATTRIBUTE_PACKED { +enum { IDCT_1D, IADST_1D, IFLIPADST_1D = IADST_1D, IIDENTITY_1D, ITX_TYPES_1D, -} ITX_TYPE_1D; +} UENUM1BYTE(ITX_TYPE_1D); static const ITX_TYPE_1D vitx_1d_tab[TX_TYPES] = { IDCT_1D, IADST_1D, IDCT_1D, IADST_1D, diff --git a/third_party/aom/av1/common/x86/av1_txfm_sse4.c b/third_party/aom/av1/common/x86/av1_txfm_sse4.c index 90b9879cc4..65ccd1952e 100644 --- a/third_party/aom/av1/common/x86/av1_txfm_sse4.c +++ b/third_party/aom/av1/common/x86/av1_txfm_sse4.c @@ -9,7 +9,7 @@ * PATENTS file, you can obtain it at www.aomedia.org/license/patent. */ -#include "config/aom_dsp_rtcd.h" +#include "config/av1_rtcd.h" #include "av1/common/av1_txfm.h" #include "av1/common/x86/av1_txfm_sse4.h" diff --git a/third_party/aom/av1/common/x86/cfl_avx2.c b/third_party/aom/av1/common/x86/cfl_avx2.c index a8bfdcce6b..d9bdf60320 100644 --- a/third_party/aom/av1/common/x86/cfl_avx2.c +++ b/third_party/aom/av1/common/x86/cfl_avx2.c @@ -27,21 +27,21 @@ subsample_##bd##_##sub##_8x8_ssse3, /* 8x8 */ \ subsample_##bd##_##sub##_16x16_ssse3, /* 16x16 */ \ subsample_##bd##_##sub##_32x32_avx2, /* 32x32 */ \ - cfl_subsample_##bd##_null, /* 64x64 (invalid CFL size) */ \ + NULL, /* 64x64 (invalid CFL size) */ \ subsample_##bd##_##sub##_4x8_ssse3, /* 4x8 */ \ subsample_##bd##_##sub##_8x4_ssse3, /* 8x4 */ \ subsample_##bd##_##sub##_8x16_ssse3, /* 8x16 */ \ subsample_##bd##_##sub##_16x8_ssse3, /* 16x8 */ \ subsample_##bd##_##sub##_16x32_ssse3, /* 16x32 */ \ subsample_##bd##_##sub##_32x16_avx2, /* 32x16 */ \ - cfl_subsample_##bd##_null, /* 32x64 (invalid CFL size) */ \ - cfl_subsample_##bd##_null, /* 64x32 (invalid CFL size) */ \ + NULL, /* 32x64 (invalid CFL size) */ \ + NULL, /* 64x32 (invalid CFL size) */ \ subsample_##bd##_##sub##_4x16_ssse3, /* 4x16 */ \ subsample_##bd##_##sub##_16x4_ssse3, /* 16x4 */ \ subsample_##bd##_##sub##_8x32_ssse3, /* 8x32 */ \ subsample_##bd##_##sub##_32x8_avx2, /* 32x8 */ \ - cfl_subsample_##bd##_null, /* 16x64 (invalid CFL size) */ \ - cfl_subsample_##bd##_null, /* 64x16 (invalid CFL size) */ \ + NULL, /* 16x64 (invalid CFL size) */ \ + NULL, /* 64x16 (invalid CFL size) */ \ }; \ return subfn_##sub[tx_size]; \ } @@ -279,21 +279,21 @@ cfl_predict_lbd_fn get_predict_lbd_fn_avx2(TX_SIZE tx_size) { predict_lbd_8x8_ssse3, /* 8x8 */ predict_lbd_16x16_ssse3, /* 16x16 */ predict_lbd_32x32_avx2, /* 32x32 */ - cfl_predict_lbd_null, /* 64x64 (invalid CFL size) */ + NULL, /* 64x64 (invalid CFL size) */ predict_lbd_4x8_ssse3, /* 4x8 */ predict_lbd_8x4_ssse3, /* 8x4 */ predict_lbd_8x16_ssse3, /* 8x16 */ predict_lbd_16x8_ssse3, /* 16x8 */ predict_lbd_16x32_ssse3, /* 16x32 */ predict_lbd_32x16_avx2, /* 32x16 */ - cfl_predict_lbd_null, /* 32x64 (invalid CFL size) */ - cfl_predict_lbd_null, /* 64x32 (invalid CFL size) */ + NULL, /* 32x64 (invalid CFL size) */ + NULL, /* 64x32 (invalid CFL size) */ predict_lbd_4x16_ssse3, /* 4x16 */ predict_lbd_16x4_ssse3, /* 16x4 */ predict_lbd_8x32_ssse3, /* 8x32 */ predict_lbd_32x8_avx2, /* 32x8 */ - cfl_predict_lbd_null, /* 16x64 (invalid CFL size) */ - cfl_predict_lbd_null, /* 64x16 (invalid CFL size) */ + NULL, /* 16x64 (invalid CFL size) */ + NULL, /* 64x16 (invalid CFL size) */ }; // Modulo TX_SIZES_ALL to ensure that an attacker won't be able to index the // function pointer array out of bounds. @@ -352,21 +352,21 @@ cfl_predict_hbd_fn get_predict_hbd_fn_avx2(TX_SIZE tx_size) { predict_hbd_8x8_ssse3, /* 8x8 */ predict_hbd_16x16_avx2, /* 16x16 */ predict_hbd_32x32_avx2, /* 32x32 */ - cfl_predict_hbd_null, /* 64x64 (invalid CFL size) */ + NULL, /* 64x64 (invalid CFL size) */ predict_hbd_4x8_ssse3, /* 4x8 */ predict_hbd_8x4_ssse3, /* 8x4 */ predict_hbd_8x16_ssse3, /* 8x16 */ predict_hbd_16x8_avx2, /* 16x8 */ predict_hbd_16x32_avx2, /* 16x32 */ predict_hbd_32x16_avx2, /* 32x16 */ - cfl_predict_hbd_null, /* 32x64 (invalid CFL size) */ - cfl_predict_hbd_null, /* 64x32 (invalid CFL size) */ + NULL, /* 32x64 (invalid CFL size) */ + NULL, /* 64x32 (invalid CFL size) */ predict_hbd_4x16_ssse3, /* 4x16 */ predict_hbd_16x4_avx2, /* 16x4 */ predict_hbd_8x32_ssse3, /* 8x32 */ predict_hbd_32x8_avx2, /* 32x8 */ - cfl_predict_hbd_null, /* 16x64 (invalid CFL size) */ - cfl_predict_hbd_null, /* 64x16 (invalid CFL size) */ + NULL, /* 16x64 (invalid CFL size) */ + NULL, /* 64x16 (invalid CFL size) */ }; // Modulo TX_SIZES_ALL to ensure that an attacker won't be able to index the // function pointer array out of bounds. @@ -469,21 +469,21 @@ cfl_subtract_average_fn get_subtract_average_fn_avx2(TX_SIZE tx_size) { subtract_average_8x8_sse2, /* 8x8 */ subtract_average_16x16_avx2, /* 16x16 */ subtract_average_32x32_avx2, /* 32x32 */ - cfl_subtract_average_null, /* 64x64 (invalid CFL size) */ + NULL, /* 64x64 (invalid CFL size) */ subtract_average_4x8_sse2, /* 4x8 */ subtract_average_8x4_sse2, /* 8x4 */ subtract_average_8x16_sse2, /* 8x16 */ subtract_average_16x8_avx2, /* 16x8 */ subtract_average_16x32_avx2, /* 16x32 */ subtract_average_32x16_avx2, /* 32x16 */ - cfl_subtract_average_null, /* 32x64 (invalid CFL size) */ - cfl_subtract_average_null, /* 64x32 (invalid CFL size) */ + NULL, /* 32x64 (invalid CFL size) */ + NULL, /* 64x32 (invalid CFL size) */ subtract_average_4x16_sse2, /* 4x16 */ subtract_average_16x4_avx2, /* 16x4 */ subtract_average_8x32_sse2, /* 8x32 */ subtract_average_32x8_avx2, /* 32x8 */ - cfl_subtract_average_null, /* 16x64 (invalid CFL size) */ - cfl_subtract_average_null, /* 64x16 (invalid CFL size) */ + NULL, /* 16x64 (invalid CFL size) */ + NULL, /* 64x16 (invalid CFL size) */ }; // Modulo TX_SIZES_ALL to ensure that an attacker won't be able to // index the function pointer array out of bounds. diff --git a/third_party/aom/av1/common/x86/convolve_2d_avx2.c b/third_party/aom/av1/common/x86/convolve_2d_avx2.c index 0acafd0446..5a69ef66a1 100644 --- a/third_party/aom/av1/common/x86/convolve_2d_avx2.c +++ b/third_party/aom/av1/common/x86/convolve_2d_avx2.c @@ -195,20 +195,20 @@ void av1_convolve_2d_copy_sr_avx2(const uint8_t *src, int src_stride, if (w == 2) { do { - memcpy(dst, src, 2 * sizeof(*src)); + memmove(dst, src, 2 * sizeof(*src)); src += src_stride; dst += dst_stride; - memcpy(dst, src, 2 * sizeof(*src)); + memmove(dst, src, 2 * sizeof(*src)); src += src_stride; dst += dst_stride; h -= 2; } while (h); } else if (w == 4) { do { - memcpy(dst, src, 4 * sizeof(*src)); + memmove(dst, src, 4 * sizeof(*src)); src += src_stride; dst += dst_stride; - memcpy(dst, src, 4 * sizeof(*src)); + memmove(dst, src, 4 * sizeof(*src)); src += src_stride; dst += dst_stride; h -= 2; diff --git a/third_party/aom/av1/common/x86/convolve_2d_sse2.c b/third_party/aom/av1/common/x86/convolve_2d_sse2.c index b1a62a4f69..369922bbdf 100644 --- a/third_party/aom/av1/common/x86/convolve_2d_sse2.c +++ b/third_party/aom/av1/common/x86/convolve_2d_sse2.c @@ -255,20 +255,20 @@ void av1_convolve_2d_copy_sr_sse2(const uint8_t *src, int src_stride, if (w == 2) { do { - memcpy(dst, src, 2 * sizeof(*src)); + memmove(dst, src, 2 * sizeof(*src)); src += src_stride; dst += dst_stride; - memcpy(dst, src, 2 * sizeof(*src)); + memmove(dst, src, 2 * sizeof(*src)); src += src_stride; dst += dst_stride; h -= 2; } while (h); } else if (w == 4) { do { - memcpy(dst, src, 4 * sizeof(*src)); + memmove(dst, src, 4 * sizeof(*src)); src += src_stride; dst += dst_stride; - memcpy(dst, src, 4 * sizeof(*src)); + memmove(dst, src, 4 * sizeof(*src)); src += src_stride; dst += dst_stride; h -= 2; @@ -354,12 +354,11 @@ void av1_convolve_2d_copy_sr_sse2(const uint8_t *src, int src_stride, } } -void av1_jnt_convolve_2d_copy_sse2(const uint8_t *src, int src_stride, - uint8_t *dst0, int dst_stride0, int w, int h, - const InterpFilterParams *filter_params_x, - const InterpFilterParams *filter_params_y, - const int subpel_x_q4, const int subpel_y_q4, - ConvolveParams *conv_params) { +void av1_dist_wtd_convolve_2d_copy_sse2( + const uint8_t *src, int src_stride, uint8_t *dst0, int dst_stride0, int w, + int h, const InterpFilterParams *filter_params_x, + const InterpFilterParams *filter_params_y, const int subpel_x_q4, + const int subpel_y_q4, ConvolveParams *conv_params) { const int bd = 8; CONV_BUF_TYPE *dst = conv_params->dst; int dst_stride = conv_params->dst_stride; @@ -371,7 +370,7 @@ void av1_jnt_convolve_2d_copy_sse2(const uint8_t *src, int src_stride, const int bits = FILTER_BITS * 2 - conv_params->round_1 - conv_params->round_0; const int do_average = conv_params->do_average; - const int use_jnt_comp_avg = conv_params->use_jnt_comp_avg; + const int use_dist_wtd_comp_avg = conv_params->use_dist_wtd_comp_avg; const __m128i zero = _mm_setzero_si128(); const __m128i left_shift = _mm_cvtsi32_si128(bits); int i, j; @@ -411,14 +410,14 @@ void av1_jnt_convolve_2d_copy_sse2(const uint8_t *src, int src_stride, const __m128i data_ref_0_hi = _mm_loadu_si128((__m128i *)(&dst[j + 8])); - const __m128i comp_avg_res_lo = - comp_avg(&data_ref_0_lo, &res_unsigned_lo, &wt, use_jnt_comp_avg); + const __m128i comp_avg_res_lo = comp_avg( + &data_ref_0_lo, &res_unsigned_lo, &wt, use_dist_wtd_comp_avg); const __m128i round_result_lo = convolve_rounding( &comp_avg_res_lo, &offset_const, &rounding_const, rounding_shift); - const __m128i comp_avg_res_hi = - comp_avg(&data_ref_0_hi, &res_unsigned_hi, &wt, use_jnt_comp_avg); + const __m128i comp_avg_res_hi = comp_avg( + &data_ref_0_hi, &res_unsigned_hi, &wt, use_dist_wtd_comp_avg); const __m128i round_result_hi = convolve_rounding( &comp_avg_res_hi, &offset_const, &rounding_const, rounding_shift); @@ -449,7 +448,7 @@ void av1_jnt_convolve_2d_copy_sse2(const uint8_t *src, int src_stride, const __m128i data_ref_0 = _mm_loadu_si128((__m128i *)(&dst[j])); const __m128i comp_avg_res = - comp_avg(&data_ref_0, &res_unsigned, &wt, use_jnt_comp_avg); + comp_avg(&data_ref_0, &res_unsigned, &wt, use_dist_wtd_comp_avg); const __m128i round_result = convolve_rounding( &comp_avg_res, &offset_const, &rounding_const, rounding_shift); diff --git a/third_party/aom/av1/common/x86/highbd_convolve_2d_avx2.c b/third_party/aom/av1/common/x86/highbd_convolve_2d_avx2.c index ae68f0bbb3..357df12656 100644 --- a/third_party/aom/av1/common/x86/highbd_convolve_2d_avx2.c +++ b/third_party/aom/av1/common/x86/highbd_convolve_2d_avx2.c @@ -238,10 +238,10 @@ void av1_highbd_convolve_2d_copy_sr_avx2( if (w == 2) { do { - memcpy(dst, src, 2 * sizeof(*src)); + memmove(dst, src, 2 * sizeof(*src)); src += src_stride; dst += dst_stride; - memcpy(dst, src, 2 * sizeof(*src)); + memmove(dst, src, 2 * sizeof(*src)); src += src_stride; dst += dst_stride; h -= 2; diff --git a/third_party/aom/av1/common/x86/highbd_convolve_2d_sse4.c b/third_party/aom/av1/common/x86/highbd_convolve_2d_sse4.c index 3f8dafb4b3..3c1d5d1418 100644 --- a/third_party/aom/av1/common/x86/highbd_convolve_2d_sse4.c +++ b/third_party/aom/av1/common/x86/highbd_convolve_2d_sse4.c @@ -21,7 +21,7 @@ #include "aom_dsp/x86/convolve_sse4_1.h" #include "av1/common/convolve.h" -void av1_highbd_jnt_convolve_2d_copy_sse4_1( +void av1_highbd_dist_wtd_convolve_2d_copy_sse4_1( const uint16_t *src, int src_stride, uint16_t *dst0, int dst_stride0, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, @@ -37,7 +37,7 @@ void av1_highbd_jnt_convolve_2d_copy_sse4_1( FILTER_BITS * 2 - conv_params->round_1 - conv_params->round_0; const __m128i left_shift = _mm_cvtsi32_si128(bits); const int do_average = conv_params->do_average; - const int use_jnt_comp_avg = conv_params->use_jnt_comp_avg; + const int use_dist_wtd_comp_avg = conv_params->use_dist_wtd_comp_avg; const int w0 = conv_params->fwd_offset; const int w1 = conv_params->bck_offset; const __m128i wt0 = _mm_set1_epi32(w0); @@ -75,15 +75,17 @@ void av1_highbd_jnt_convolve_2d_copy_sse4_1( const __m128i res_unsigned_lo = _mm_add_epi32(res_32b_lo, offset_const); - const __m128i comp_avg_res_lo = highbd_comp_avg_sse4_1( - &data_ref_0_lo, &res_unsigned_lo, &wt0, &wt1, use_jnt_comp_avg); + const __m128i comp_avg_res_lo = + highbd_comp_avg_sse4_1(&data_ref_0_lo, &res_unsigned_lo, &wt0, + &wt1, use_dist_wtd_comp_avg); const __m128i res_32b_hi = _mm_unpackhi_epi16(res, zero); const __m128i res_unsigned_hi = _mm_add_epi32(res_32b_hi, offset_const); - const __m128i comp_avg_res_hi = highbd_comp_avg_sse4_1( - &data_ref_0_hi, &res_unsigned_hi, &wt0, &wt1, use_jnt_comp_avg); + const __m128i comp_avg_res_hi = + highbd_comp_avg_sse4_1(&data_ref_0_hi, &res_unsigned_hi, &wt0, + &wt1, use_dist_wtd_comp_avg); const __m128i round_result_lo = highbd_convolve_rounding_sse2( &comp_avg_res_lo, &offset_const, &rounding_const, rounding_shift); @@ -132,9 +134,9 @@ void av1_highbd_jnt_convolve_2d_copy_sse4_1( _mm_add_epi32(res_32b_hi, offset_const); const __m128i comp_avg_res_lo = highbd_comp_avg_sse4_1( - &data_ref_0, &res_unsigned_lo, &wt0, &wt1, use_jnt_comp_avg); + &data_ref_0, &res_unsigned_lo, &wt0, &wt1, use_dist_wtd_comp_avg); const __m128i comp_avg_res_hi = highbd_comp_avg_sse4_1( - &data_ref_1, &res_unsigned_hi, &wt0, &wt1, use_jnt_comp_avg); + &data_ref_1, &res_unsigned_hi, &wt0, &wt1, use_dist_wtd_comp_avg); const __m128i round_result_lo = highbd_convolve_rounding_sse2( &comp_avg_res_lo, &offset_const, &rounding_const, rounding_shift); @@ -166,7 +168,7 @@ void av1_highbd_jnt_convolve_2d_copy_sse4_1( } } -void av1_highbd_jnt_convolve_2d_sse4_1( +void av1_highbd_dist_wtd_convolve_2d_sse4_1( const uint16_t *src, int src_stride, uint16_t *dst0, int dst_stride0, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, @@ -179,7 +181,7 @@ void av1_highbd_jnt_convolve_2d_sse4_1( int im_stride = MAX_SB_SIZE; int i, j; const int do_average = conv_params->do_average; - const int use_jnt_comp_avg = conv_params->use_jnt_comp_avg; + const int use_dist_wtd_comp_avg = conv_params->use_dist_wtd_comp_avg; const int fo_vert = filter_params_y->taps / 2 - 1; const int fo_horiz = filter_params_x->taps / 2 - 1; const uint16_t *const src_ptr = src - fo_vert * src_stride - fo_horiz; @@ -359,8 +361,9 @@ void av1_highbd_jnt_convolve_2d_sse4_1( const __m128i data_ref_0 = _mm_cvtepu16_epi32(data_0); - const __m128i comp_avg_res = highbd_comp_avg_sse4_1( - &data_ref_0, &res_unsigned_lo, &wt0, &wt1, use_jnt_comp_avg); + const __m128i comp_avg_res = + highbd_comp_avg_sse4_1(&data_ref_0, &res_unsigned_lo, &wt0, + &wt1, use_dist_wtd_comp_avg); const __m128i round_result = highbd_convolve_rounding_sse2( &comp_avg_res, &offset_const, &rounding_const, rounding_shift); @@ -391,10 +394,12 @@ void av1_highbd_jnt_convolve_2d_sse4_1( const __m128i data_ref_0_lo = _mm_cvtepu16_epi32(data_lo); const __m128i data_ref_0_hi = _mm_cvtepu16_epi32(data_hi); - const __m128i comp_avg_res_lo = highbd_comp_avg_sse4_1( - &data_ref_0_lo, &res_unsigned_lo, &wt0, &wt1, use_jnt_comp_avg); - const __m128i comp_avg_res_hi = highbd_comp_avg_sse4_1( - &data_ref_0_hi, &res_unsigned_hi, &wt0, &wt1, use_jnt_comp_avg); + const __m128i comp_avg_res_lo = + highbd_comp_avg_sse4_1(&data_ref_0_lo, &res_unsigned_lo, &wt0, + &wt1, use_dist_wtd_comp_avg); + const __m128i comp_avg_res_hi = + highbd_comp_avg_sse4_1(&data_ref_0_hi, &res_unsigned_hi, &wt0, + &wt1, use_dist_wtd_comp_avg); const __m128i round_result_lo = highbd_convolve_rounding_sse2(&comp_avg_res_lo, &offset_const, diff --git a/third_party/aom/av1/common/x86/highbd_inv_txfm_avx2.c b/third_party/aom/av1/common/x86/highbd_inv_txfm_avx2.c index ade2af03e4..fe22465592 100644 --- a/third_party/aom/av1/common/x86/highbd_inv_txfm_avx2.c +++ b/third_party/aom/av1/common/x86/highbd_inv_txfm_avx2.c @@ -18,6 +18,7 @@ #include "av1/common/idct.h" #include "av1/common/x86/av1_inv_txfm_ssse3.h" #include "av1/common/x86/highbd_txfm_utility_sse4.h" +#include "aom_dsp/x86/txfm_common_avx2.h" // Note: // Total 32x4 registers to represent 32x32 block coefficients. @@ -72,30 +73,48 @@ static INLINE void highbd_write_buffer_16xn_avx2(__m256i *in, uint16_t *output, _mm256_storeu_si256((__m256i *)(output + i * stride), u); } } - -static INLINE __m256i av1_round_shift_32_avx2(__m256i vec, int bit) { - __m256i tmp, round; - round = _mm256_set1_epi32(1 << (bit - 1)); - tmp = _mm256_add_epi32(vec, round); - return _mm256_srai_epi32(tmp, bit); +static INLINE __m256i highbd_get_recon_8x8_avx2(const __m256i pred, __m256i res, + const int bd) { + __m256i x0 = pred; + x0 = _mm256_add_epi32(res, x0); + x0 = _mm256_packus_epi32(x0, x0); + x0 = _mm256_permute4x64_epi64(x0, 0xd8); + x0 = highbd_clamp_epi16_avx2(x0, bd); + return x0; } -static INLINE void av1_round_shift_array_32_avx2(__m256i *input, - __m256i *output, - const int size, - const int bit) { - if (bit > 0) { - int i; - for (i = 0; i < size; i++) { - output[i] = av1_round_shift_32_avx2(input[i], bit); - } - } else { - int i; - for (i = 0; i < size; i++) { - output[i] = _mm256_slli_epi32(input[i], -bit); - } +static INLINE void highbd_write_buffer_8xn_avx2(__m256i *in, uint16_t *output, + int stride, int flipud, + int height, const int bd) { + int j = flipud ? (height - 1) : 0; + __m128i temp; + const int step = flipud ? -1 : 1; + for (int i = 0; i < height; ++i, j += step) { + temp = _mm_loadu_si128((__m128i const *)(output + i * stride)); + __m256i v = _mm256_cvtepi16_epi32(temp); + __m256i u = highbd_get_recon_8x8_avx2(v, in[j], bd); + __m128i u1 = _mm256_castsi256_si128(u); + _mm_storeu_si128((__m128i *)(output + i * stride), u1); } } +static void neg_shift_avx2(const __m256i in0, const __m256i in1, __m256i *out0, + __m256i *out1, const __m256i *clamp_lo, + const __m256i *clamp_hi, int shift) { + __m256i offset = _mm256_set1_epi32((1 << shift) >> 1); + __m256i a0 = _mm256_add_epi32(offset, in0); + __m256i a1 = _mm256_sub_epi32(offset, in1); + + a0 = _mm256_sra_epi32(a0, _mm_cvtsi32_si128(shift)); + a1 = _mm256_sra_epi32(a1, _mm_cvtsi32_si128(shift)); + + a0 = _mm256_max_epi32(a0, *clamp_lo); + a0 = _mm256_min_epi32(a0, *clamp_hi); + a1 = _mm256_max_epi32(a1, *clamp_lo); + a1 = _mm256_min_epi32(a1, *clamp_hi); + + *out0 = a0; + *out1 = a1; +} static void transpose_8x8_avx2(const __m256i *in, __m256i *out) { __m256i u0, u1, u2, u3, u4, u5, u6, u7; @@ -134,6 +153,43 @@ static void transpose_8x8_avx2(const __m256i *in, __m256i *out) { out[7] = _mm256_permute2f128_si256(x0, x1, 0x31); } +static void transpose_8x8_flip_avx2(const __m256i *in, __m256i *out) { + __m256i u0, u1, u2, u3, u4, u5, u6, u7; + __m256i x0, x1; + + u0 = _mm256_unpacklo_epi32(in[7], in[6]); + u1 = _mm256_unpackhi_epi32(in[7], in[6]); + + u2 = _mm256_unpacklo_epi32(in[5], in[4]); + u3 = _mm256_unpackhi_epi32(in[5], in[4]); + + u4 = _mm256_unpacklo_epi32(in[3], in[2]); + u5 = _mm256_unpackhi_epi32(in[3], in[2]); + + u6 = _mm256_unpacklo_epi32(in[1], in[0]); + u7 = _mm256_unpackhi_epi32(in[1], in[0]); + + x0 = _mm256_unpacklo_epi64(u0, u2); + x1 = _mm256_unpacklo_epi64(u4, u6); + out[0] = _mm256_permute2f128_si256(x0, x1, 0x20); + out[4] = _mm256_permute2f128_si256(x0, x1, 0x31); + + x0 = _mm256_unpackhi_epi64(u0, u2); + x1 = _mm256_unpackhi_epi64(u4, u6); + out[1] = _mm256_permute2f128_si256(x0, x1, 0x20); + out[5] = _mm256_permute2f128_si256(x0, x1, 0x31); + + x0 = _mm256_unpacklo_epi64(u1, u3); + x1 = _mm256_unpacklo_epi64(u5, u7); + out[2] = _mm256_permute2f128_si256(x0, x1, 0x20); + out[6] = _mm256_permute2f128_si256(x0, x1, 0x31); + + x0 = _mm256_unpackhi_epi64(u1, u3); + x1 = _mm256_unpackhi_epi64(u5, u7); + out[3] = _mm256_permute2f128_si256(x0, x1, 0x20); + out[7] = _mm256_permute2f128_si256(x0, x1, 0x31); +} + static void load_buffer_32x32(const int32_t *coeff, __m256i *in, int input_stiride, int size) { int i; @@ -1153,7 +1209,2979 @@ static void idct32_avx2(__m256i *in, __m256i *out, int bit, int do_cols, int bd, } } } +static void idct16_low1_avx2(__m256i *in, __m256i *out, int bit, int do_cols, + int bd, int out_shift) { + const int32_t *cospi = cospi_arr(bit); + const __m256i cospi32 = _mm256_set1_epi32(cospi[32]); + const __m256i rnding = _mm256_set1_epi32(1 << (bit - 1)); + const int log_range = AOMMAX(16, bd + (do_cols ? 6 : 8)); + const __m256i clamp_lo = _mm256_set1_epi32(-(1 << (log_range - 1))); + const __m256i clamp_hi = _mm256_set1_epi32((1 << (log_range - 1)) - 1); + { + // stage 0 + // stage 1 + // stage 2 + // stage 3 + // stage 4 + in[0] = _mm256_mullo_epi32(in[0], cospi32); + in[0] = _mm256_add_epi32(in[0], rnding); + in[0] = _mm256_srai_epi32(in[0], bit); + + // stage 5 + // stage 6 + // stage 7 + if (do_cols) { + in[0] = _mm256_max_epi32(in[0], clamp_lo); + in[0] = _mm256_min_epi32(in[0], clamp_hi); + } else { + const int log_range_out = AOMMAX(16, bd + 6); + const __m256i clamp_lo_out = _mm256_set1_epi32(AOMMAX( + -(1 << (log_range_out - 1)), -(1 << (log_range - 1 - out_shift)))); + const __m256i clamp_hi_out = _mm256_set1_epi32(AOMMIN( + (1 << (log_range_out - 1)) - 1, (1 << (log_range - 1 - out_shift)))); + __m256i offset = _mm256_set1_epi32((1 << out_shift) >> 1); + in[0] = _mm256_add_epi32(in[0], offset); + in[0] = _mm256_sra_epi32(in[0], _mm_cvtsi32_si128(out_shift)); + in[0] = _mm256_max_epi32(in[0], clamp_lo_out); + in[0] = _mm256_min_epi32(in[0], clamp_hi_out); + } + + out[0] = in[0]; + out[1] = in[0]; + out[2] = in[0]; + out[3] = in[0]; + out[4] = in[0]; + out[5] = in[0]; + out[6] = in[0]; + out[7] = in[0]; + out[8] = in[0]; + out[9] = in[0]; + out[10] = in[0]; + out[11] = in[0]; + out[12] = in[0]; + out[13] = in[0]; + out[14] = in[0]; + out[15] = in[0]; + } +} + +static void idct16_low8_avx2(__m256i *in, __m256i *out, int bit, int do_cols, + int bd, int out_shift) { + const int32_t *cospi = cospi_arr(bit); + const __m256i cospi60 = _mm256_set1_epi32(cospi[60]); + const __m256i cospi28 = _mm256_set1_epi32(cospi[28]); + const __m256i cospi44 = _mm256_set1_epi32(cospi[44]); + const __m256i cospi20 = _mm256_set1_epi32(cospi[20]); + const __m256i cospi12 = _mm256_set1_epi32(cospi[12]); + const __m256i cospi4 = _mm256_set1_epi32(cospi[4]); + const __m256i cospi56 = _mm256_set1_epi32(cospi[56]); + const __m256i cospi24 = _mm256_set1_epi32(cospi[24]); + const __m256i cospim40 = _mm256_set1_epi32(-cospi[40]); + const __m256i cospi8 = _mm256_set1_epi32(cospi[8]); + const __m256i cospi32 = _mm256_set1_epi32(cospi[32]); + const __m256i cospi48 = _mm256_set1_epi32(cospi[48]); + const __m256i cospi16 = _mm256_set1_epi32(cospi[16]); + const __m256i cospim16 = _mm256_set1_epi32(-cospi[16]); + const __m256i cospim48 = _mm256_set1_epi32(-cospi[48]); + const __m256i cospim36 = _mm256_set1_epi32(-cospi[36]); + const __m256i cospim52 = _mm256_set1_epi32(-cospi[52]); + const __m256i rnding = _mm256_set1_epi32(1 << (bit - 1)); + const int log_range = AOMMAX(16, bd + (do_cols ? 6 : 8)); + const __m256i clamp_lo = _mm256_set1_epi32(-(1 << (log_range - 1))); + const __m256i clamp_hi = _mm256_set1_epi32((1 << (log_range - 1)) - 1); + __m256i u[16], x, y; + + { + // stage 0 + // stage 1 + u[0] = in[0]; + u[2] = in[4]; + u[4] = in[2]; + u[6] = in[6]; + u[8] = in[1]; + u[10] = in[5]; + u[12] = in[3]; + u[14] = in[7]; + + // stage 2 + u[15] = half_btf_0_avx2(&cospi4, &u[8], &rnding, bit); + u[8] = half_btf_0_avx2(&cospi60, &u[8], &rnding, bit); + + u[9] = half_btf_0_avx2(&cospim36, &u[14], &rnding, bit); + u[14] = half_btf_0_avx2(&cospi28, &u[14], &rnding, bit); + + u[13] = half_btf_0_avx2(&cospi20, &u[10], &rnding, bit); + u[10] = half_btf_0_avx2(&cospi44, &u[10], &rnding, bit); + + u[11] = half_btf_0_avx2(&cospim52, &u[12], &rnding, bit); + u[12] = half_btf_0_avx2(&cospi12, &u[12], &rnding, bit); + + // stage 3 + u[7] = half_btf_0_avx2(&cospi8, &u[4], &rnding, bit); + u[4] = half_btf_0_avx2(&cospi56, &u[4], &rnding, bit); + u[5] = half_btf_0_avx2(&cospim40, &u[6], &rnding, bit); + u[6] = half_btf_0_avx2(&cospi24, &u[6], &rnding, bit); + + addsub_avx2(u[8], u[9], &u[8], &u[9], &clamp_lo, &clamp_hi); + addsub_avx2(u[11], u[10], &u[11], &u[10], &clamp_lo, &clamp_hi); + addsub_avx2(u[12], u[13], &u[12], &u[13], &clamp_lo, &clamp_hi); + addsub_avx2(u[15], u[14], &u[15], &u[14], &clamp_lo, &clamp_hi); + + // stage 4 + x = _mm256_mullo_epi32(u[0], cospi32); + u[0] = _mm256_add_epi32(x, rnding); + u[0] = _mm256_srai_epi32(u[0], bit); + u[1] = u[0]; + + u[3] = half_btf_0_avx2(&cospi16, &u[2], &rnding, bit); + u[2] = half_btf_0_avx2(&cospi48, &u[2], &rnding, bit); + + addsub_avx2(u[4], u[5], &u[4], &u[5], &clamp_lo, &clamp_hi); + addsub_avx2(u[7], u[6], &u[7], &u[6], &clamp_lo, &clamp_hi); + + x = half_btf_avx2(&cospim16, &u[9], &cospi48, &u[14], &rnding, bit); + u[14] = half_btf_avx2(&cospi48, &u[9], &cospi16, &u[14], &rnding, bit); + u[9] = x; + y = half_btf_avx2(&cospim48, &u[10], &cospim16, &u[13], &rnding, bit); + u[13] = half_btf_avx2(&cospim16, &u[10], &cospi48, &u[13], &rnding, bit); + u[10] = y; + + // stage 5 + addsub_avx2(u[0], u[3], &u[0], &u[3], &clamp_lo, &clamp_hi); + addsub_avx2(u[1], u[2], &u[1], &u[2], &clamp_lo, &clamp_hi); + + x = _mm256_mullo_epi32(u[5], cospi32); + y = _mm256_mullo_epi32(u[6], cospi32); + u[5] = _mm256_sub_epi32(y, x); + u[5] = _mm256_add_epi32(u[5], rnding); + u[5] = _mm256_srai_epi32(u[5], bit); + + u[6] = _mm256_add_epi32(y, x); + u[6] = _mm256_add_epi32(u[6], rnding); + u[6] = _mm256_srai_epi32(u[6], bit); + + addsub_avx2(u[8], u[11], &u[8], &u[11], &clamp_lo, &clamp_hi); + addsub_avx2(u[9], u[10], &u[9], &u[10], &clamp_lo, &clamp_hi); + addsub_avx2(u[15], u[12], &u[15], &u[12], &clamp_lo, &clamp_hi); + addsub_avx2(u[14], u[13], &u[14], &u[13], &clamp_lo, &clamp_hi); + + // stage 6 + addsub_avx2(u[0], u[7], &u[0], &u[7], &clamp_lo, &clamp_hi); + addsub_avx2(u[1], u[6], &u[1], &u[6], &clamp_lo, &clamp_hi); + addsub_avx2(u[2], u[5], &u[2], &u[5], &clamp_lo, &clamp_hi); + addsub_avx2(u[3], u[4], &u[3], &u[4], &clamp_lo, &clamp_hi); + + x = _mm256_mullo_epi32(u[10], cospi32); + y = _mm256_mullo_epi32(u[13], cospi32); + u[10] = _mm256_sub_epi32(y, x); + u[10] = _mm256_add_epi32(u[10], rnding); + u[10] = _mm256_srai_epi32(u[10], bit); + + u[13] = _mm256_add_epi32(x, y); + u[13] = _mm256_add_epi32(u[13], rnding); + u[13] = _mm256_srai_epi32(u[13], bit); + + x = _mm256_mullo_epi32(u[11], cospi32); + y = _mm256_mullo_epi32(u[12], cospi32); + u[11] = _mm256_sub_epi32(y, x); + u[11] = _mm256_add_epi32(u[11], rnding); + u[11] = _mm256_srai_epi32(u[11], bit); + + u[12] = _mm256_add_epi32(x, y); + u[12] = _mm256_add_epi32(u[12], rnding); + u[12] = _mm256_srai_epi32(u[12], bit); + // stage 7 + if (do_cols) { + addsub_no_clamp_avx2(u[0], u[15], out + 0, out + 15); + addsub_no_clamp_avx2(u[1], u[14], out + 1, out + 14); + addsub_no_clamp_avx2(u[2], u[13], out + 2, out + 13); + addsub_no_clamp_avx2(u[3], u[12], out + 3, out + 12); + addsub_no_clamp_avx2(u[4], u[11], out + 4, out + 11); + addsub_no_clamp_avx2(u[5], u[10], out + 5, out + 10); + addsub_no_clamp_avx2(u[6], u[9], out + 6, out + 9); + addsub_no_clamp_avx2(u[7], u[8], out + 7, out + 8); + } else { + const int log_range_out = AOMMAX(16, bd + 6); + const __m256i clamp_lo_out = _mm256_set1_epi32(AOMMAX( + -(1 << (log_range_out - 1)), -(1 << (log_range - 1 - out_shift)))); + const __m256i clamp_hi_out = _mm256_set1_epi32(AOMMIN( + (1 << (log_range_out - 1)) - 1, (1 << (log_range - 1 - out_shift)))); + + addsub_shift_avx2(u[0], u[15], out + 0, out + 15, &clamp_lo_out, + &clamp_hi_out, out_shift); + addsub_shift_avx2(u[1], u[14], out + 1, out + 14, &clamp_lo_out, + &clamp_hi_out, out_shift); + addsub_shift_avx2(u[2], u[13], out + 2, out + 13, &clamp_lo_out, + &clamp_hi_out, out_shift); + addsub_shift_avx2(u[3], u[12], out + 3, out + 12, &clamp_lo_out, + &clamp_hi_out, out_shift); + addsub_shift_avx2(u[4], u[11], out + 4, out + 11, &clamp_lo_out, + &clamp_hi_out, out_shift); + addsub_shift_avx2(u[5], u[10], out + 5, out + 10, &clamp_lo_out, + &clamp_hi_out, out_shift); + addsub_shift_avx2(u[6], u[9], out + 6, out + 9, &clamp_lo_out, + &clamp_hi_out, out_shift); + addsub_shift_avx2(u[7], u[8], out + 7, out + 8, &clamp_lo_out, + &clamp_hi_out, out_shift); + } + } +} + +static void idct16_avx2(__m256i *in, __m256i *out, int bit, int do_cols, int bd, + int out_shift) { + const int32_t *cospi = cospi_arr(bit); + const __m256i cospi60 = _mm256_set1_epi32(cospi[60]); + const __m256i cospim4 = _mm256_set1_epi32(-cospi[4]); + const __m256i cospi28 = _mm256_set1_epi32(cospi[28]); + const __m256i cospim36 = _mm256_set1_epi32(-cospi[36]); + const __m256i cospi44 = _mm256_set1_epi32(cospi[44]); + const __m256i cospi20 = _mm256_set1_epi32(cospi[20]); + const __m256i cospim20 = _mm256_set1_epi32(-cospi[20]); + const __m256i cospi12 = _mm256_set1_epi32(cospi[12]); + const __m256i cospim52 = _mm256_set1_epi32(-cospi[52]); + const __m256i cospi52 = _mm256_set1_epi32(cospi[52]); + const __m256i cospi36 = _mm256_set1_epi32(cospi[36]); + const __m256i cospi4 = _mm256_set1_epi32(cospi[4]); + const __m256i cospi56 = _mm256_set1_epi32(cospi[56]); + const __m256i cospim8 = _mm256_set1_epi32(-cospi[8]); + const __m256i cospi24 = _mm256_set1_epi32(cospi[24]); + const __m256i cospim40 = _mm256_set1_epi32(-cospi[40]); + const __m256i cospi40 = _mm256_set1_epi32(cospi[40]); + const __m256i cospi8 = _mm256_set1_epi32(cospi[8]); + const __m256i cospi32 = _mm256_set1_epi32(cospi[32]); + const __m256i cospi48 = _mm256_set1_epi32(cospi[48]); + const __m256i cospi16 = _mm256_set1_epi32(cospi[16]); + const __m256i cospim16 = _mm256_set1_epi32(-cospi[16]); + const __m256i cospim48 = _mm256_set1_epi32(-cospi[48]); + const __m256i rnding = _mm256_set1_epi32(1 << (bit - 1)); + const int log_range = AOMMAX(16, bd + (do_cols ? 6 : 8)); + const __m256i clamp_lo = _mm256_set1_epi32(-(1 << (log_range - 1))); + const __m256i clamp_hi = _mm256_set1_epi32((1 << (log_range - 1)) - 1); + __m256i u[16], v[16], x, y; + + { + // stage 0 + // stage 1 + u[0] = in[0]; + u[1] = in[8]; + u[2] = in[4]; + u[3] = in[12]; + u[4] = in[2]; + u[5] = in[10]; + u[6] = in[6]; + u[7] = in[14]; + u[8] = in[1]; + u[9] = in[9]; + u[10] = in[5]; + u[11] = in[13]; + u[12] = in[3]; + u[13] = in[11]; + u[14] = in[7]; + u[15] = in[15]; + + // stage 2 + v[0] = u[0]; + v[1] = u[1]; + v[2] = u[2]; + v[3] = u[3]; + v[4] = u[4]; + v[5] = u[5]; + v[6] = u[6]; + v[7] = u[7]; + + v[8] = half_btf_avx2(&cospi60, &u[8], &cospim4, &u[15], &rnding, bit); + v[9] = half_btf_avx2(&cospi28, &u[9], &cospim36, &u[14], &rnding, bit); + v[10] = half_btf_avx2(&cospi44, &u[10], &cospim20, &u[13], &rnding, bit); + v[11] = half_btf_avx2(&cospi12, &u[11], &cospim52, &u[12], &rnding, bit); + v[12] = half_btf_avx2(&cospi52, &u[11], &cospi12, &u[12], &rnding, bit); + v[13] = half_btf_avx2(&cospi20, &u[10], &cospi44, &u[13], &rnding, bit); + v[14] = half_btf_avx2(&cospi36, &u[9], &cospi28, &u[14], &rnding, bit); + v[15] = half_btf_avx2(&cospi4, &u[8], &cospi60, &u[15], &rnding, bit); + + // stage 3 + u[0] = v[0]; + u[1] = v[1]; + u[2] = v[2]; + u[3] = v[3]; + u[4] = half_btf_avx2(&cospi56, &v[4], &cospim8, &v[7], &rnding, bit); + u[5] = half_btf_avx2(&cospi24, &v[5], &cospim40, &v[6], &rnding, bit); + u[6] = half_btf_avx2(&cospi40, &v[5], &cospi24, &v[6], &rnding, bit); + u[7] = half_btf_avx2(&cospi8, &v[4], &cospi56, &v[7], &rnding, bit); + addsub_avx2(v[8], v[9], &u[8], &u[9], &clamp_lo, &clamp_hi); + addsub_avx2(v[11], v[10], &u[11], &u[10], &clamp_lo, &clamp_hi); + addsub_avx2(v[12], v[13], &u[12], &u[13], &clamp_lo, &clamp_hi); + addsub_avx2(v[15], v[14], &u[15], &u[14], &clamp_lo, &clamp_hi); + + // stage 4 + x = _mm256_mullo_epi32(u[0], cospi32); + y = _mm256_mullo_epi32(u[1], cospi32); + v[0] = _mm256_add_epi32(x, y); + v[0] = _mm256_add_epi32(v[0], rnding); + v[0] = _mm256_srai_epi32(v[0], bit); + + v[1] = _mm256_sub_epi32(x, y); + v[1] = _mm256_add_epi32(v[1], rnding); + v[1] = _mm256_srai_epi32(v[1], bit); + + v[2] = half_btf_avx2(&cospi48, &u[2], &cospim16, &u[3], &rnding, bit); + v[3] = half_btf_avx2(&cospi16, &u[2], &cospi48, &u[3], &rnding, bit); + addsub_avx2(u[4], u[5], &v[4], &v[5], &clamp_lo, &clamp_hi); + addsub_avx2(u[7], u[6], &v[7], &v[6], &clamp_lo, &clamp_hi); + v[8] = u[8]; + v[9] = half_btf_avx2(&cospim16, &u[9], &cospi48, &u[14], &rnding, bit); + v[10] = half_btf_avx2(&cospim48, &u[10], &cospim16, &u[13], &rnding, bit); + v[11] = u[11]; + v[12] = u[12]; + v[13] = half_btf_avx2(&cospim16, &u[10], &cospi48, &u[13], &rnding, bit); + v[14] = half_btf_avx2(&cospi48, &u[9], &cospi16, &u[14], &rnding, bit); + v[15] = u[15]; + + // stage 5 + addsub_avx2(v[0], v[3], &u[0], &u[3], &clamp_lo, &clamp_hi); + addsub_avx2(v[1], v[2], &u[1], &u[2], &clamp_lo, &clamp_hi); + u[4] = v[4]; + + x = _mm256_mullo_epi32(v[5], cospi32); + y = _mm256_mullo_epi32(v[6], cospi32); + u[5] = _mm256_sub_epi32(y, x); + u[5] = _mm256_add_epi32(u[5], rnding); + u[5] = _mm256_srai_epi32(u[5], bit); + + u[6] = _mm256_add_epi32(y, x); + u[6] = _mm256_add_epi32(u[6], rnding); + u[6] = _mm256_srai_epi32(u[6], bit); + + u[7] = v[7]; + addsub_avx2(v[8], v[11], &u[8], &u[11], &clamp_lo, &clamp_hi); + addsub_avx2(v[9], v[10], &u[9], &u[10], &clamp_lo, &clamp_hi); + addsub_avx2(v[15], v[12], &u[15], &u[12], &clamp_lo, &clamp_hi); + addsub_avx2(v[14], v[13], &u[14], &u[13], &clamp_lo, &clamp_hi); + + // stage 6 + addsub_avx2(u[0], u[7], &v[0], &v[7], &clamp_lo, &clamp_hi); + addsub_avx2(u[1], u[6], &v[1], &v[6], &clamp_lo, &clamp_hi); + addsub_avx2(u[2], u[5], &v[2], &v[5], &clamp_lo, &clamp_hi); + addsub_avx2(u[3], u[4], &v[3], &v[4], &clamp_lo, &clamp_hi); + v[8] = u[8]; + v[9] = u[9]; + + x = _mm256_mullo_epi32(u[10], cospi32); + y = _mm256_mullo_epi32(u[13], cospi32); + v[10] = _mm256_sub_epi32(y, x); + v[10] = _mm256_add_epi32(v[10], rnding); + v[10] = _mm256_srai_epi32(v[10], bit); + + v[13] = _mm256_add_epi32(x, y); + v[13] = _mm256_add_epi32(v[13], rnding); + v[13] = _mm256_srai_epi32(v[13], bit); + + x = _mm256_mullo_epi32(u[11], cospi32); + y = _mm256_mullo_epi32(u[12], cospi32); + v[11] = _mm256_sub_epi32(y, x); + v[11] = _mm256_add_epi32(v[11], rnding); + v[11] = _mm256_srai_epi32(v[11], bit); + + v[12] = _mm256_add_epi32(x, y); + v[12] = _mm256_add_epi32(v[12], rnding); + v[12] = _mm256_srai_epi32(v[12], bit); + + v[14] = u[14]; + v[15] = u[15]; + + // stage 7 + if (do_cols) { + addsub_no_clamp_avx2(v[0], v[15], out + 0, out + 15); + addsub_no_clamp_avx2(v[1], v[14], out + 1, out + 14); + addsub_no_clamp_avx2(v[2], v[13], out + 2, out + 13); + addsub_no_clamp_avx2(v[3], v[12], out + 3, out + 12); + addsub_no_clamp_avx2(v[4], v[11], out + 4, out + 11); + addsub_no_clamp_avx2(v[5], v[10], out + 5, out + 10); + addsub_no_clamp_avx2(v[6], v[9], out + 6, out + 9); + addsub_no_clamp_avx2(v[7], v[8], out + 7, out + 8); + } else { + const int log_range_out = AOMMAX(16, bd + 6); + const __m256i clamp_lo_out = _mm256_set1_epi32(AOMMAX( + -(1 << (log_range_out - 1)), -(1 << (log_range - 1 - out_shift)))); + const __m256i clamp_hi_out = _mm256_set1_epi32(AOMMIN( + (1 << (log_range_out - 1)) - 1, (1 << (log_range - 1 - out_shift)))); + + addsub_shift_avx2(v[0], v[15], out + 0, out + 15, &clamp_lo_out, + &clamp_hi_out, out_shift); + addsub_shift_avx2(v[1], v[14], out + 1, out + 14, &clamp_lo_out, + &clamp_hi_out, out_shift); + addsub_shift_avx2(v[2], v[13], out + 2, out + 13, &clamp_lo_out, + &clamp_hi_out, out_shift); + addsub_shift_avx2(v[3], v[12], out + 3, out + 12, &clamp_lo_out, + &clamp_hi_out, out_shift); + addsub_shift_avx2(v[4], v[11], out + 4, out + 11, &clamp_lo_out, + &clamp_hi_out, out_shift); + addsub_shift_avx2(v[5], v[10], out + 5, out + 10, &clamp_lo_out, + &clamp_hi_out, out_shift); + addsub_shift_avx2(v[6], v[9], out + 6, out + 9, &clamp_lo_out, + &clamp_hi_out, out_shift); + addsub_shift_avx2(v[7], v[8], out + 7, out + 8, &clamp_lo_out, + &clamp_hi_out, out_shift); + } + } +} + +static void iadst16_low1_avx2(__m256i *in, __m256i *out, int bit, int do_cols, + int bd, int out_shift) { + const int32_t *cospi = cospi_arr(bit); + const __m256i cospi2 = _mm256_set1_epi32(cospi[2]); + const __m256i cospi62 = _mm256_set1_epi32(cospi[62]); + const __m256i cospi8 = _mm256_set1_epi32(cospi[8]); + const __m256i cospi56 = _mm256_set1_epi32(cospi[56]); + const __m256i cospi48 = _mm256_set1_epi32(cospi[48]); + const __m256i cospi16 = _mm256_set1_epi32(cospi[16]); + const __m256i cospi32 = _mm256_set1_epi32(cospi[32]); + const __m256i rnding = _mm256_set1_epi32(1 << (bit - 1)); + const __m256i zero = _mm256_setzero_si256(); + __m256i v[16], x, y, temp1, temp2; + + // Calculate the column 0, 1, 2, 3 + { + // stage 0 + // stage 1 + // stage 2 + x = _mm256_mullo_epi32(in[0], cospi62); + v[0] = _mm256_add_epi32(x, rnding); + v[0] = _mm256_srai_epi32(v[0], bit); + + x = _mm256_mullo_epi32(in[0], cospi2); + v[1] = _mm256_sub_epi32(zero, x); + v[1] = _mm256_add_epi32(v[1], rnding); + v[1] = _mm256_srai_epi32(v[1], bit); + + // stage 3 + v[8] = v[0]; + v[9] = v[1]; + + // stage 4 + temp1 = _mm256_mullo_epi32(v[8], cospi8); + x = _mm256_mullo_epi32(v[9], cospi56); + temp1 = _mm256_add_epi32(temp1, x); + temp1 = _mm256_add_epi32(temp1, rnding); + temp1 = _mm256_srai_epi32(temp1, bit); + + temp2 = _mm256_mullo_epi32(v[8], cospi56); + x = _mm256_mullo_epi32(v[9], cospi8); + temp2 = _mm256_sub_epi32(temp2, x); + temp2 = _mm256_add_epi32(temp2, rnding); + temp2 = _mm256_srai_epi32(temp2, bit); + v[8] = temp1; + v[9] = temp2; + + // stage 5 + v[4] = v[0]; + v[5] = v[1]; + v[12] = v[8]; + v[13] = v[9]; + + // stage 6 + temp1 = _mm256_mullo_epi32(v[4], cospi16); + x = _mm256_mullo_epi32(v[5], cospi48); + temp1 = _mm256_add_epi32(temp1, x); + temp1 = _mm256_add_epi32(temp1, rnding); + temp1 = _mm256_srai_epi32(temp1, bit); + + temp2 = _mm256_mullo_epi32(v[4], cospi48); + x = _mm256_mullo_epi32(v[5], cospi16); + temp2 = _mm256_sub_epi32(temp2, x); + temp2 = _mm256_add_epi32(temp2, rnding); + temp2 = _mm256_srai_epi32(temp2, bit); + v[4] = temp1; + v[5] = temp2; + + temp1 = _mm256_mullo_epi32(v[12], cospi16); + x = _mm256_mullo_epi32(v[13], cospi48); + temp1 = _mm256_add_epi32(temp1, x); + temp1 = _mm256_add_epi32(temp1, rnding); + temp1 = _mm256_srai_epi32(temp1, bit); + + temp2 = _mm256_mullo_epi32(v[12], cospi48); + x = _mm256_mullo_epi32(v[13], cospi16); + temp2 = _mm256_sub_epi32(temp2, x); + temp2 = _mm256_add_epi32(temp2, rnding); + temp2 = _mm256_srai_epi32(temp2, bit); + v[12] = temp1; + v[13] = temp2; + + // stage 7 + v[2] = v[0]; + v[3] = v[1]; + v[6] = v[4]; + v[7] = v[5]; + v[10] = v[8]; + v[11] = v[9]; + v[14] = v[12]; + v[15] = v[13]; + + // stage 8 + y = _mm256_mullo_epi32(v[2], cospi32); + x = _mm256_mullo_epi32(v[3], cospi32); + v[2] = _mm256_add_epi32(y, x); + v[2] = _mm256_add_epi32(v[2], rnding); + v[2] = _mm256_srai_epi32(v[2], bit); + + v[3] = _mm256_sub_epi32(y, x); + v[3] = _mm256_add_epi32(v[3], rnding); + v[3] = _mm256_srai_epi32(v[3], bit); + + y = _mm256_mullo_epi32(v[6], cospi32); + x = _mm256_mullo_epi32(v[7], cospi32); + v[6] = _mm256_add_epi32(y, x); + v[6] = _mm256_add_epi32(v[6], rnding); + v[6] = _mm256_srai_epi32(v[6], bit); + + v[7] = _mm256_sub_epi32(y, x); + v[7] = _mm256_add_epi32(v[7], rnding); + v[7] = _mm256_srai_epi32(v[7], bit); + + y = _mm256_mullo_epi32(v[10], cospi32); + x = _mm256_mullo_epi32(v[11], cospi32); + v[10] = _mm256_add_epi32(y, x); + v[10] = _mm256_add_epi32(v[10], rnding); + v[10] = _mm256_srai_epi32(v[10], bit); + + v[11] = _mm256_sub_epi32(y, x); + v[11] = _mm256_add_epi32(v[11], rnding); + v[11] = _mm256_srai_epi32(v[11], bit); + + y = _mm256_mullo_epi32(v[14], cospi32); + x = _mm256_mullo_epi32(v[15], cospi32); + v[14] = _mm256_add_epi32(y, x); + v[14] = _mm256_add_epi32(v[14], rnding); + v[14] = _mm256_srai_epi32(v[14], bit); + + v[15] = _mm256_sub_epi32(y, x); + v[15] = _mm256_add_epi32(v[15], rnding); + v[15] = _mm256_srai_epi32(v[15], bit); + + // stage 9 + if (do_cols) { + out[0] = v[0]; + out[1] = _mm256_sub_epi32(_mm256_setzero_si256(), v[8]); + out[2] = v[12]; + out[3] = _mm256_sub_epi32(_mm256_setzero_si256(), v[4]); + out[4] = v[6]; + out[5] = _mm256_sub_epi32(_mm256_setzero_si256(), v[14]); + out[6] = v[10]; + out[7] = _mm256_sub_epi32(_mm256_setzero_si256(), v[2]); + out[8] = v[3]; + out[9] = _mm256_sub_epi32(_mm256_setzero_si256(), v[11]); + out[10] = v[15]; + out[11] = _mm256_sub_epi32(_mm256_setzero_si256(), v[7]); + out[12] = v[5]; + out[13] = _mm256_sub_epi32(_mm256_setzero_si256(), v[13]); + out[14] = v[9]; + out[15] = _mm256_sub_epi32(_mm256_setzero_si256(), v[1]); + } else { + const int log_range_out = AOMMAX(16, bd + 6); + const __m256i clamp_lo_out = + _mm256_set1_epi32(-(1 << (log_range_out - 1))); + const __m256i clamp_hi_out = + _mm256_set1_epi32((1 << (log_range_out - 1)) - 1); + + neg_shift_avx2(v[0], v[8], out + 0, out + 1, &clamp_lo_out, &clamp_hi_out, + out_shift); + neg_shift_avx2(v[12], v[4], out + 2, out + 3, &clamp_lo_out, + &clamp_hi_out, out_shift); + neg_shift_avx2(v[6], v[14], out + 4, out + 5, &clamp_lo_out, + &clamp_hi_out, out_shift); + neg_shift_avx2(v[10], v[2], out + 6, out + 7, &clamp_lo_out, + &clamp_hi_out, out_shift); + neg_shift_avx2(v[3], v[11], out + 8, out + 9, &clamp_lo_out, + &clamp_hi_out, out_shift); + neg_shift_avx2(v[15], v[7], out + 10, out + 11, &clamp_lo_out, + &clamp_hi_out, out_shift); + neg_shift_avx2(v[5], v[13], out + 12, out + 13, &clamp_lo_out, + &clamp_hi_out, out_shift); + neg_shift_avx2(v[9], v[1], out + 14, out + 15, &clamp_lo_out, + &clamp_hi_out, out_shift); + } + } +} + +static void iadst16_low8_avx2(__m256i *in, __m256i *out, int bit, int do_cols, + int bd, int out_shift) { + const int32_t *cospi = cospi_arr(bit); + const __m256i cospi2 = _mm256_set1_epi32(cospi[2]); + const __m256i cospi62 = _mm256_set1_epi32(cospi[62]); + const __m256i cospi10 = _mm256_set1_epi32(cospi[10]); + const __m256i cospi54 = _mm256_set1_epi32(cospi[54]); + const __m256i cospi18 = _mm256_set1_epi32(cospi[18]); + const __m256i cospi46 = _mm256_set1_epi32(cospi[46]); + const __m256i cospi26 = _mm256_set1_epi32(cospi[26]); + const __m256i cospi38 = _mm256_set1_epi32(cospi[38]); + const __m256i cospi34 = _mm256_set1_epi32(cospi[34]); + const __m256i cospi30 = _mm256_set1_epi32(cospi[30]); + const __m256i cospi42 = _mm256_set1_epi32(cospi[42]); + const __m256i cospi22 = _mm256_set1_epi32(cospi[22]); + const __m256i cospi50 = _mm256_set1_epi32(cospi[50]); + const __m256i cospi14 = _mm256_set1_epi32(cospi[14]); + const __m256i cospi58 = _mm256_set1_epi32(cospi[58]); + const __m256i cospi6 = _mm256_set1_epi32(cospi[6]); + const __m256i cospi8 = _mm256_set1_epi32(cospi[8]); + const __m256i cospi56 = _mm256_set1_epi32(cospi[56]); + const __m256i cospi40 = _mm256_set1_epi32(cospi[40]); + const __m256i cospi24 = _mm256_set1_epi32(cospi[24]); + const __m256i cospim56 = _mm256_set1_epi32(-cospi[56]); + const __m256i cospim24 = _mm256_set1_epi32(-cospi[24]); + const __m256i cospi48 = _mm256_set1_epi32(cospi[48]); + const __m256i cospi16 = _mm256_set1_epi32(cospi[16]); + const __m256i cospim48 = _mm256_set1_epi32(-cospi[48]); + const __m256i cospi32 = _mm256_set1_epi32(cospi[32]); + const __m256i rnding = _mm256_set1_epi32(1 << (bit - 1)); + const int log_range = AOMMAX(16, bd + (do_cols ? 6 : 8)); + const __m256i clamp_lo = _mm256_set1_epi32(-(1 << (log_range - 1))); + const __m256i clamp_hi = _mm256_set1_epi32((1 << (log_range - 1)) - 1); + __m256i u[16], x, y; + + { + // stage 0 + // stage 1 + // stage 2 + __m256i zero = _mm256_setzero_si256(); + x = _mm256_mullo_epi32(in[0], cospi62); + u[0] = _mm256_add_epi32(x, rnding); + u[0] = _mm256_srai_epi32(u[0], bit); + + x = _mm256_mullo_epi32(in[0], cospi2); + u[1] = _mm256_sub_epi32(zero, x); + u[1] = _mm256_add_epi32(u[1], rnding); + u[1] = _mm256_srai_epi32(u[1], bit); + + x = _mm256_mullo_epi32(in[2], cospi54); + u[2] = _mm256_add_epi32(x, rnding); + u[2] = _mm256_srai_epi32(u[2], bit); + + x = _mm256_mullo_epi32(in[2], cospi10); + u[3] = _mm256_sub_epi32(zero, x); + u[3] = _mm256_add_epi32(u[3], rnding); + u[3] = _mm256_srai_epi32(u[3], bit); + + x = _mm256_mullo_epi32(in[4], cospi46); + u[4] = _mm256_add_epi32(x, rnding); + u[4] = _mm256_srai_epi32(u[4], bit); + + x = _mm256_mullo_epi32(in[4], cospi18); + u[5] = _mm256_sub_epi32(zero, x); + u[5] = _mm256_add_epi32(u[5], rnding); + u[5] = _mm256_srai_epi32(u[5], bit); + + x = _mm256_mullo_epi32(in[6], cospi38); + u[6] = _mm256_add_epi32(x, rnding); + u[6] = _mm256_srai_epi32(u[6], bit); + + x = _mm256_mullo_epi32(in[6], cospi26); + u[7] = _mm256_sub_epi32(zero, x); + u[7] = _mm256_add_epi32(u[7], rnding); + u[7] = _mm256_srai_epi32(u[7], bit); + + u[8] = _mm256_mullo_epi32(in[7], cospi34); + u[8] = _mm256_add_epi32(u[8], rnding); + u[8] = _mm256_srai_epi32(u[8], bit); + + u[9] = _mm256_mullo_epi32(in[7], cospi30); + u[9] = _mm256_add_epi32(u[9], rnding); + u[9] = _mm256_srai_epi32(u[9], bit); + + u[10] = _mm256_mullo_epi32(in[5], cospi42); + u[10] = _mm256_add_epi32(u[10], rnding); + u[10] = _mm256_srai_epi32(u[10], bit); + + u[11] = _mm256_mullo_epi32(in[5], cospi22); + u[11] = _mm256_add_epi32(u[11], rnding); + u[11] = _mm256_srai_epi32(u[11], bit); + + u[12] = _mm256_mullo_epi32(in[3], cospi50); + u[12] = _mm256_add_epi32(u[12], rnding); + u[12] = _mm256_srai_epi32(u[12], bit); + + u[13] = _mm256_mullo_epi32(in[3], cospi14); + u[13] = _mm256_add_epi32(u[13], rnding); + u[13] = _mm256_srai_epi32(u[13], bit); + + u[14] = _mm256_mullo_epi32(in[1], cospi58); + u[14] = _mm256_add_epi32(u[14], rnding); + u[14] = _mm256_srai_epi32(u[14], bit); + + u[15] = _mm256_mullo_epi32(in[1], cospi6); + u[15] = _mm256_add_epi32(u[15], rnding); + u[15] = _mm256_srai_epi32(u[15], bit); + + // stage 3 + addsub_avx2(u[0], u[8], &u[0], &u[8], &clamp_lo, &clamp_hi); + addsub_avx2(u[1], u[9], &u[1], &u[9], &clamp_lo, &clamp_hi); + addsub_avx2(u[2], u[10], &u[2], &u[10], &clamp_lo, &clamp_hi); + addsub_avx2(u[3], u[11], &u[3], &u[11], &clamp_lo, &clamp_hi); + addsub_avx2(u[4], u[12], &u[4], &u[12], &clamp_lo, &clamp_hi); + addsub_avx2(u[5], u[13], &u[5], &u[13], &clamp_lo, &clamp_hi); + addsub_avx2(u[6], u[14], &u[6], &u[14], &clamp_lo, &clamp_hi); + addsub_avx2(u[7], u[15], &u[7], &u[15], &clamp_lo, &clamp_hi); + + // stage 4 + y = _mm256_mullo_epi32(u[8], cospi56); + x = _mm256_mullo_epi32(u[9], cospi56); + u[8] = _mm256_mullo_epi32(u[8], cospi8); + u[8] = _mm256_add_epi32(u[8], x); + u[8] = _mm256_add_epi32(u[8], rnding); + u[8] = _mm256_srai_epi32(u[8], bit); + + x = _mm256_mullo_epi32(u[9], cospi8); + u[9] = _mm256_sub_epi32(y, x); + u[9] = _mm256_add_epi32(u[9], rnding); + u[9] = _mm256_srai_epi32(u[9], bit); + + x = _mm256_mullo_epi32(u[11], cospi24); + y = _mm256_mullo_epi32(u[10], cospi24); + u[10] = _mm256_mullo_epi32(u[10], cospi40); + u[10] = _mm256_add_epi32(u[10], x); + u[10] = _mm256_add_epi32(u[10], rnding); + u[10] = _mm256_srai_epi32(u[10], bit); + + x = _mm256_mullo_epi32(u[11], cospi40); + u[11] = _mm256_sub_epi32(y, x); + u[11] = _mm256_add_epi32(u[11], rnding); + u[11] = _mm256_srai_epi32(u[11], bit); + + x = _mm256_mullo_epi32(u[13], cospi8); + y = _mm256_mullo_epi32(u[12], cospi8); + u[12] = _mm256_mullo_epi32(u[12], cospim56); + u[12] = _mm256_add_epi32(u[12], x); + u[12] = _mm256_add_epi32(u[12], rnding); + u[12] = _mm256_srai_epi32(u[12], bit); + + x = _mm256_mullo_epi32(u[13], cospim56); + u[13] = _mm256_sub_epi32(y, x); + u[13] = _mm256_add_epi32(u[13], rnding); + u[13] = _mm256_srai_epi32(u[13], bit); + + x = _mm256_mullo_epi32(u[15], cospi40); + y = _mm256_mullo_epi32(u[14], cospi40); + u[14] = _mm256_mullo_epi32(u[14], cospim24); + u[14] = _mm256_add_epi32(u[14], x); + u[14] = _mm256_add_epi32(u[14], rnding); + u[14] = _mm256_srai_epi32(u[14], bit); + + x = _mm256_mullo_epi32(u[15], cospim24); + u[15] = _mm256_sub_epi32(y, x); + u[15] = _mm256_add_epi32(u[15], rnding); + u[15] = _mm256_srai_epi32(u[15], bit); + + // stage 5 + addsub_avx2(u[0], u[4], &u[0], &u[4], &clamp_lo, &clamp_hi); + addsub_avx2(u[1], u[5], &u[1], &u[5], &clamp_lo, &clamp_hi); + addsub_avx2(u[2], u[6], &u[2], &u[6], &clamp_lo, &clamp_hi); + addsub_avx2(u[3], u[7], &u[3], &u[7], &clamp_lo, &clamp_hi); + addsub_avx2(u[8], u[12], &u[8], &u[12], &clamp_lo, &clamp_hi); + addsub_avx2(u[9], u[13], &u[9], &u[13], &clamp_lo, &clamp_hi); + addsub_avx2(u[10], u[14], &u[10], &u[14], &clamp_lo, &clamp_hi); + addsub_avx2(u[11], u[15], &u[11], &u[15], &clamp_lo, &clamp_hi); + + // stage 6 + x = _mm256_mullo_epi32(u[5], cospi48); + y = _mm256_mullo_epi32(u[4], cospi48); + u[4] = _mm256_mullo_epi32(u[4], cospi16); + u[4] = _mm256_add_epi32(u[4], x); + u[4] = _mm256_add_epi32(u[4], rnding); + u[4] = _mm256_srai_epi32(u[4], bit); + + x = _mm256_mullo_epi32(u[5], cospi16); + u[5] = _mm256_sub_epi32(y, x); + u[5] = _mm256_add_epi32(u[5], rnding); + u[5] = _mm256_srai_epi32(u[5], bit); + + x = _mm256_mullo_epi32(u[7], cospi16); + y = _mm256_mullo_epi32(u[6], cospi16); + u[6] = _mm256_mullo_epi32(u[6], cospim48); + u[6] = _mm256_add_epi32(u[6], x); + u[6] = _mm256_add_epi32(u[6], rnding); + u[6] = _mm256_srai_epi32(u[6], bit); + + x = _mm256_mullo_epi32(u[7], cospim48); + u[7] = _mm256_sub_epi32(y, x); + u[7] = _mm256_add_epi32(u[7], rnding); + u[7] = _mm256_srai_epi32(u[7], bit); + + x = _mm256_mullo_epi32(u[13], cospi48); + y = _mm256_mullo_epi32(u[12], cospi48); + u[12] = _mm256_mullo_epi32(u[12], cospi16); + u[12] = _mm256_add_epi32(u[12], x); + u[12] = _mm256_add_epi32(u[12], rnding); + u[12] = _mm256_srai_epi32(u[12], bit); + + x = _mm256_mullo_epi32(u[13], cospi16); + u[13] = _mm256_sub_epi32(y, x); + u[13] = _mm256_add_epi32(u[13], rnding); + u[13] = _mm256_srai_epi32(u[13], bit); + + x = _mm256_mullo_epi32(u[15], cospi16); + y = _mm256_mullo_epi32(u[14], cospi16); + u[14] = _mm256_mullo_epi32(u[14], cospim48); + u[14] = _mm256_add_epi32(u[14], x); + u[14] = _mm256_add_epi32(u[14], rnding); + u[14] = _mm256_srai_epi32(u[14], bit); + + x = _mm256_mullo_epi32(u[15], cospim48); + u[15] = _mm256_sub_epi32(y, x); + u[15] = _mm256_add_epi32(u[15], rnding); + u[15] = _mm256_srai_epi32(u[15], bit); + + // stage 7 + addsub_avx2(u[0], u[2], &u[0], &u[2], &clamp_lo, &clamp_hi); + addsub_avx2(u[1], u[3], &u[1], &u[3], &clamp_lo, &clamp_hi); + addsub_avx2(u[4], u[6], &u[4], &u[6], &clamp_lo, &clamp_hi); + addsub_avx2(u[5], u[7], &u[5], &u[7], &clamp_lo, &clamp_hi); + addsub_avx2(u[8], u[10], &u[8], &u[10], &clamp_lo, &clamp_hi); + addsub_avx2(u[9], u[11], &u[9], &u[11], &clamp_lo, &clamp_hi); + addsub_avx2(u[12], u[14], &u[12], &u[14], &clamp_lo, &clamp_hi); + addsub_avx2(u[13], u[15], &u[13], &u[15], &clamp_lo, &clamp_hi); + + // stage 8 + y = _mm256_mullo_epi32(u[2], cospi32); + x = _mm256_mullo_epi32(u[3], cospi32); + u[2] = _mm256_add_epi32(y, x); + u[2] = _mm256_add_epi32(u[2], rnding); + u[2] = _mm256_srai_epi32(u[2], bit); + + u[3] = _mm256_sub_epi32(y, x); + u[3] = _mm256_add_epi32(u[3], rnding); + u[3] = _mm256_srai_epi32(u[3], bit); + y = _mm256_mullo_epi32(u[6], cospi32); + x = _mm256_mullo_epi32(u[7], cospi32); + u[6] = _mm256_add_epi32(y, x); + u[6] = _mm256_add_epi32(u[6], rnding); + u[6] = _mm256_srai_epi32(u[6], bit); + + u[7] = _mm256_sub_epi32(y, x); + u[7] = _mm256_add_epi32(u[7], rnding); + u[7] = _mm256_srai_epi32(u[7], bit); + + y = _mm256_mullo_epi32(u[10], cospi32); + x = _mm256_mullo_epi32(u[11], cospi32); + u[10] = _mm256_add_epi32(y, x); + u[10] = _mm256_add_epi32(u[10], rnding); + u[10] = _mm256_srai_epi32(u[10], bit); + + u[11] = _mm256_sub_epi32(y, x); + u[11] = _mm256_add_epi32(u[11], rnding); + u[11] = _mm256_srai_epi32(u[11], bit); + + y = _mm256_mullo_epi32(u[14], cospi32); + x = _mm256_mullo_epi32(u[15], cospi32); + u[14] = _mm256_add_epi32(y, x); + u[14] = _mm256_add_epi32(u[14], rnding); + u[14] = _mm256_srai_epi32(u[14], bit); + + u[15] = _mm256_sub_epi32(y, x); + u[15] = _mm256_add_epi32(u[15], rnding); + u[15] = _mm256_srai_epi32(u[15], bit); + + // stage 9 + if (do_cols) { + out[0] = u[0]; + out[1] = _mm256_sub_epi32(_mm256_setzero_si256(), u[8]); + out[2] = u[12]; + out[3] = _mm256_sub_epi32(_mm256_setzero_si256(), u[4]); + out[4] = u[6]; + out[5] = _mm256_sub_epi32(_mm256_setzero_si256(), u[14]); + out[6] = u[10]; + out[7] = _mm256_sub_epi32(_mm256_setzero_si256(), u[2]); + out[8] = u[3]; + out[9] = _mm256_sub_epi32(_mm256_setzero_si256(), u[11]); + out[10] = u[15]; + out[11] = _mm256_sub_epi32(_mm256_setzero_si256(), u[7]); + out[12] = u[5]; + out[13] = _mm256_sub_epi32(_mm256_setzero_si256(), u[13]); + out[14] = u[9]; + out[15] = _mm256_sub_epi32(_mm256_setzero_si256(), u[1]); + } else { + const int log_range_out = AOMMAX(16, bd + 6); + const __m256i clamp_lo_out = + _mm256_set1_epi32(-(1 << (log_range_out - 1))); + const __m256i clamp_hi_out = + _mm256_set1_epi32((1 << (log_range_out - 1)) - 1); + + neg_shift_avx2(u[0], u[8], out + 0, out + 1, &clamp_lo_out, &clamp_hi_out, + out_shift); + neg_shift_avx2(u[12], u[4], out + 2, out + 3, &clamp_lo_out, + &clamp_hi_out, out_shift); + neg_shift_avx2(u[6], u[14], out + 4, out + 5, &clamp_lo_out, + &clamp_hi_out, out_shift); + neg_shift_avx2(u[10], u[2], out + 6, out + 7, &clamp_lo_out, + &clamp_hi_out, out_shift); + neg_shift_avx2(u[3], u[11], out + 8, out + 9, &clamp_lo_out, + &clamp_hi_out, out_shift); + neg_shift_avx2(u[15], u[7], out + 10, out + 11, &clamp_lo_out, + &clamp_hi_out, out_shift); + neg_shift_avx2(u[5], u[13], out + 12, out + 13, &clamp_lo_out, + &clamp_hi_out, out_shift); + neg_shift_avx2(u[9], u[1], out + 14, out + 15, &clamp_lo_out, + &clamp_hi_out, out_shift); + } + } +} + +static void iadst16_avx2(__m256i *in, __m256i *out, int bit, int do_cols, + int bd, int out_shift) { + const int32_t *cospi = cospi_arr(bit); + const __m256i cospi2 = _mm256_set1_epi32(cospi[2]); + const __m256i cospi62 = _mm256_set1_epi32(cospi[62]); + const __m256i cospi10 = _mm256_set1_epi32(cospi[10]); + const __m256i cospi54 = _mm256_set1_epi32(cospi[54]); + const __m256i cospi18 = _mm256_set1_epi32(cospi[18]); + const __m256i cospi46 = _mm256_set1_epi32(cospi[46]); + const __m256i cospi26 = _mm256_set1_epi32(cospi[26]); + const __m256i cospi38 = _mm256_set1_epi32(cospi[38]); + const __m256i cospi34 = _mm256_set1_epi32(cospi[34]); + const __m256i cospi30 = _mm256_set1_epi32(cospi[30]); + const __m256i cospi42 = _mm256_set1_epi32(cospi[42]); + const __m256i cospi22 = _mm256_set1_epi32(cospi[22]); + const __m256i cospi50 = _mm256_set1_epi32(cospi[50]); + const __m256i cospi14 = _mm256_set1_epi32(cospi[14]); + const __m256i cospi58 = _mm256_set1_epi32(cospi[58]); + const __m256i cospi6 = _mm256_set1_epi32(cospi[6]); + const __m256i cospi8 = _mm256_set1_epi32(cospi[8]); + const __m256i cospi56 = _mm256_set1_epi32(cospi[56]); + const __m256i cospi40 = _mm256_set1_epi32(cospi[40]); + const __m256i cospi24 = _mm256_set1_epi32(cospi[24]); + const __m256i cospim56 = _mm256_set1_epi32(-cospi[56]); + const __m256i cospim24 = _mm256_set1_epi32(-cospi[24]); + const __m256i cospi48 = _mm256_set1_epi32(cospi[48]); + const __m256i cospi16 = _mm256_set1_epi32(cospi[16]); + const __m256i cospim48 = _mm256_set1_epi32(-cospi[48]); + const __m256i cospi32 = _mm256_set1_epi32(cospi[32]); + const __m256i rnding = _mm256_set1_epi32(1 << (bit - 1)); + const int log_range = AOMMAX(16, bd + (do_cols ? 6 : 8)); + const __m256i clamp_lo = _mm256_set1_epi32(-(1 << (log_range - 1))); + const __m256i clamp_hi = _mm256_set1_epi32((1 << (log_range - 1)) - 1); + __m256i u[16], v[16], x, y; + + { + // stage 0 + // stage 1 + // stage 2 + v[0] = _mm256_mullo_epi32(in[15], cospi2); + x = _mm256_mullo_epi32(in[0], cospi62); + v[0] = _mm256_add_epi32(v[0], x); + v[0] = _mm256_add_epi32(v[0], rnding); + v[0] = _mm256_srai_epi32(v[0], bit); + + v[1] = _mm256_mullo_epi32(in[15], cospi62); + x = _mm256_mullo_epi32(in[0], cospi2); + v[1] = _mm256_sub_epi32(v[1], x); + v[1] = _mm256_add_epi32(v[1], rnding); + v[1] = _mm256_srai_epi32(v[1], bit); + + v[2] = _mm256_mullo_epi32(in[13], cospi10); + x = _mm256_mullo_epi32(in[2], cospi54); + v[2] = _mm256_add_epi32(v[2], x); + v[2] = _mm256_add_epi32(v[2], rnding); + v[2] = _mm256_srai_epi32(v[2], bit); + + v[3] = _mm256_mullo_epi32(in[13], cospi54); + x = _mm256_mullo_epi32(in[2], cospi10); + v[3] = _mm256_sub_epi32(v[3], x); + v[3] = _mm256_add_epi32(v[3], rnding); + v[3] = _mm256_srai_epi32(v[3], bit); + + v[4] = _mm256_mullo_epi32(in[11], cospi18); + x = _mm256_mullo_epi32(in[4], cospi46); + v[4] = _mm256_add_epi32(v[4], x); + v[4] = _mm256_add_epi32(v[4], rnding); + v[4] = _mm256_srai_epi32(v[4], bit); + + v[5] = _mm256_mullo_epi32(in[11], cospi46); + x = _mm256_mullo_epi32(in[4], cospi18); + v[5] = _mm256_sub_epi32(v[5], x); + v[5] = _mm256_add_epi32(v[5], rnding); + v[5] = _mm256_srai_epi32(v[5], bit); + + v[6] = _mm256_mullo_epi32(in[9], cospi26); + x = _mm256_mullo_epi32(in[6], cospi38); + v[6] = _mm256_add_epi32(v[6], x); + v[6] = _mm256_add_epi32(v[6], rnding); + v[6] = _mm256_srai_epi32(v[6], bit); + + v[7] = _mm256_mullo_epi32(in[9], cospi38); + x = _mm256_mullo_epi32(in[6], cospi26); + v[7] = _mm256_sub_epi32(v[7], x); + v[7] = _mm256_add_epi32(v[7], rnding); + v[7] = _mm256_srai_epi32(v[7], bit); + + v[8] = _mm256_mullo_epi32(in[7], cospi34); + x = _mm256_mullo_epi32(in[8], cospi30); + v[8] = _mm256_add_epi32(v[8], x); + v[8] = _mm256_add_epi32(v[8], rnding); + v[8] = _mm256_srai_epi32(v[8], bit); + + v[9] = _mm256_mullo_epi32(in[7], cospi30); + x = _mm256_mullo_epi32(in[8], cospi34); + v[9] = _mm256_sub_epi32(v[9], x); + v[9] = _mm256_add_epi32(v[9], rnding); + v[9] = _mm256_srai_epi32(v[9], bit); + + v[10] = _mm256_mullo_epi32(in[5], cospi42); + x = _mm256_mullo_epi32(in[10], cospi22); + v[10] = _mm256_add_epi32(v[10], x); + v[10] = _mm256_add_epi32(v[10], rnding); + v[10] = _mm256_srai_epi32(v[10], bit); + + v[11] = _mm256_mullo_epi32(in[5], cospi22); + x = _mm256_mullo_epi32(in[10], cospi42); + v[11] = _mm256_sub_epi32(v[11], x); + v[11] = _mm256_add_epi32(v[11], rnding); + v[11] = _mm256_srai_epi32(v[11], bit); + + v[12] = _mm256_mullo_epi32(in[3], cospi50); + x = _mm256_mullo_epi32(in[12], cospi14); + v[12] = _mm256_add_epi32(v[12], x); + v[12] = _mm256_add_epi32(v[12], rnding); + v[12] = _mm256_srai_epi32(v[12], bit); + + v[13] = _mm256_mullo_epi32(in[3], cospi14); + x = _mm256_mullo_epi32(in[12], cospi50); + v[13] = _mm256_sub_epi32(v[13], x); + v[13] = _mm256_add_epi32(v[13], rnding); + v[13] = _mm256_srai_epi32(v[13], bit); + + v[14] = _mm256_mullo_epi32(in[1], cospi58); + x = _mm256_mullo_epi32(in[14], cospi6); + v[14] = _mm256_add_epi32(v[14], x); + v[14] = _mm256_add_epi32(v[14], rnding); + v[14] = _mm256_srai_epi32(v[14], bit); + + v[15] = _mm256_mullo_epi32(in[1], cospi6); + x = _mm256_mullo_epi32(in[14], cospi58); + v[15] = _mm256_sub_epi32(v[15], x); + v[15] = _mm256_add_epi32(v[15], rnding); + v[15] = _mm256_srai_epi32(v[15], bit); + + // stage 3 + addsub_avx2(v[0], v[8], &u[0], &u[8], &clamp_lo, &clamp_hi); + addsub_avx2(v[1], v[9], &u[1], &u[9], &clamp_lo, &clamp_hi); + addsub_avx2(v[2], v[10], &u[2], &u[10], &clamp_lo, &clamp_hi); + addsub_avx2(v[3], v[11], &u[3], &u[11], &clamp_lo, &clamp_hi); + addsub_avx2(v[4], v[12], &u[4], &u[12], &clamp_lo, &clamp_hi); + addsub_avx2(v[5], v[13], &u[5], &u[13], &clamp_lo, &clamp_hi); + addsub_avx2(v[6], v[14], &u[6], &u[14], &clamp_lo, &clamp_hi); + addsub_avx2(v[7], v[15], &u[7], &u[15], &clamp_lo, &clamp_hi); + + // stage 4 + v[0] = u[0]; + v[1] = u[1]; + v[2] = u[2]; + v[3] = u[3]; + v[4] = u[4]; + v[5] = u[5]; + v[6] = u[6]; + v[7] = u[7]; + + v[8] = _mm256_mullo_epi32(u[8], cospi8); + x = _mm256_mullo_epi32(u[9], cospi56); + v[8] = _mm256_add_epi32(v[8], x); + v[8] = _mm256_add_epi32(v[8], rnding); + v[8] = _mm256_srai_epi32(v[8], bit); + + v[9] = _mm256_mullo_epi32(u[8], cospi56); + x = _mm256_mullo_epi32(u[9], cospi8); + v[9] = _mm256_sub_epi32(v[9], x); + v[9] = _mm256_add_epi32(v[9], rnding); + v[9] = _mm256_srai_epi32(v[9], bit); + + v[10] = _mm256_mullo_epi32(u[10], cospi40); + x = _mm256_mullo_epi32(u[11], cospi24); + v[10] = _mm256_add_epi32(v[10], x); + v[10] = _mm256_add_epi32(v[10], rnding); + v[10] = _mm256_srai_epi32(v[10], bit); + + v[11] = _mm256_mullo_epi32(u[10], cospi24); + x = _mm256_mullo_epi32(u[11], cospi40); + v[11] = _mm256_sub_epi32(v[11], x); + v[11] = _mm256_add_epi32(v[11], rnding); + v[11] = _mm256_srai_epi32(v[11], bit); + + v[12] = _mm256_mullo_epi32(u[12], cospim56); + x = _mm256_mullo_epi32(u[13], cospi8); + v[12] = _mm256_add_epi32(v[12], x); + v[12] = _mm256_add_epi32(v[12], rnding); + v[12] = _mm256_srai_epi32(v[12], bit); + + v[13] = _mm256_mullo_epi32(u[12], cospi8); + x = _mm256_mullo_epi32(u[13], cospim56); + v[13] = _mm256_sub_epi32(v[13], x); + v[13] = _mm256_add_epi32(v[13], rnding); + v[13] = _mm256_srai_epi32(v[13], bit); + + v[14] = _mm256_mullo_epi32(u[14], cospim24); + x = _mm256_mullo_epi32(u[15], cospi40); + v[14] = _mm256_add_epi32(v[14], x); + v[14] = _mm256_add_epi32(v[14], rnding); + v[14] = _mm256_srai_epi32(v[14], bit); + + v[15] = _mm256_mullo_epi32(u[14], cospi40); + x = _mm256_mullo_epi32(u[15], cospim24); + v[15] = _mm256_sub_epi32(v[15], x); + v[15] = _mm256_add_epi32(v[15], rnding); + v[15] = _mm256_srai_epi32(v[15], bit); + + // stage 5 + addsub_avx2(v[0], v[4], &u[0], &u[4], &clamp_lo, &clamp_hi); + addsub_avx2(v[1], v[5], &u[1], &u[5], &clamp_lo, &clamp_hi); + addsub_avx2(v[2], v[6], &u[2], &u[6], &clamp_lo, &clamp_hi); + addsub_avx2(v[3], v[7], &u[3], &u[7], &clamp_lo, &clamp_hi); + addsub_avx2(v[8], v[12], &u[8], &u[12], &clamp_lo, &clamp_hi); + addsub_avx2(v[9], v[13], &u[9], &u[13], &clamp_lo, &clamp_hi); + addsub_avx2(v[10], v[14], &u[10], &u[14], &clamp_lo, &clamp_hi); + addsub_avx2(v[11], v[15], &u[11], &u[15], &clamp_lo, &clamp_hi); + + // stage 6 + v[0] = u[0]; + v[1] = u[1]; + v[2] = u[2]; + v[3] = u[3]; + + v[4] = _mm256_mullo_epi32(u[4], cospi16); + x = _mm256_mullo_epi32(u[5], cospi48); + v[4] = _mm256_add_epi32(v[4], x); + v[4] = _mm256_add_epi32(v[4], rnding); + v[4] = _mm256_srai_epi32(v[4], bit); + + v[5] = _mm256_mullo_epi32(u[4], cospi48); + x = _mm256_mullo_epi32(u[5], cospi16); + v[5] = _mm256_sub_epi32(v[5], x); + v[5] = _mm256_add_epi32(v[5], rnding); + v[5] = _mm256_srai_epi32(v[5], bit); + + v[6] = _mm256_mullo_epi32(u[6], cospim48); + x = _mm256_mullo_epi32(u[7], cospi16); + v[6] = _mm256_add_epi32(v[6], x); + v[6] = _mm256_add_epi32(v[6], rnding); + v[6] = _mm256_srai_epi32(v[6], bit); + + v[7] = _mm256_mullo_epi32(u[6], cospi16); + x = _mm256_mullo_epi32(u[7], cospim48); + v[7] = _mm256_sub_epi32(v[7], x); + v[7] = _mm256_add_epi32(v[7], rnding); + v[7] = _mm256_srai_epi32(v[7], bit); + + v[8] = u[8]; + v[9] = u[9]; + v[10] = u[10]; + v[11] = u[11]; + + v[12] = _mm256_mullo_epi32(u[12], cospi16); + x = _mm256_mullo_epi32(u[13], cospi48); + v[12] = _mm256_add_epi32(v[12], x); + v[12] = _mm256_add_epi32(v[12], rnding); + v[12] = _mm256_srai_epi32(v[12], bit); + + v[13] = _mm256_mullo_epi32(u[12], cospi48); + x = _mm256_mullo_epi32(u[13], cospi16); + v[13] = _mm256_sub_epi32(v[13], x); + v[13] = _mm256_add_epi32(v[13], rnding); + v[13] = _mm256_srai_epi32(v[13], bit); + + v[14] = _mm256_mullo_epi32(u[14], cospim48); + x = _mm256_mullo_epi32(u[15], cospi16); + v[14] = _mm256_add_epi32(v[14], x); + v[14] = _mm256_add_epi32(v[14], rnding); + v[14] = _mm256_srai_epi32(v[14], bit); + + v[15] = _mm256_mullo_epi32(u[14], cospi16); + x = _mm256_mullo_epi32(u[15], cospim48); + v[15] = _mm256_sub_epi32(v[15], x); + v[15] = _mm256_add_epi32(v[15], rnding); + v[15] = _mm256_srai_epi32(v[15], bit); + + // stage 7 + addsub_avx2(v[0], v[2], &u[0], &u[2], &clamp_lo, &clamp_hi); + addsub_avx2(v[1], v[3], &u[1], &u[3], &clamp_lo, &clamp_hi); + addsub_avx2(v[4], v[6], &u[4], &u[6], &clamp_lo, &clamp_hi); + addsub_avx2(v[5], v[7], &u[5], &u[7], &clamp_lo, &clamp_hi); + addsub_avx2(v[8], v[10], &u[8], &u[10], &clamp_lo, &clamp_hi); + addsub_avx2(v[9], v[11], &u[9], &u[11], &clamp_lo, &clamp_hi); + addsub_avx2(v[12], v[14], &u[12], &u[14], &clamp_lo, &clamp_hi); + addsub_avx2(v[13], v[15], &u[13], &u[15], &clamp_lo, &clamp_hi); + + // stage 8 + v[0] = u[0]; + v[1] = u[1]; + + y = _mm256_mullo_epi32(u[2], cospi32); + x = _mm256_mullo_epi32(u[3], cospi32); + v[2] = _mm256_add_epi32(y, x); + v[2] = _mm256_add_epi32(v[2], rnding); + v[2] = _mm256_srai_epi32(v[2], bit); + + v[3] = _mm256_sub_epi32(y, x); + v[3] = _mm256_add_epi32(v[3], rnding); + v[3] = _mm256_srai_epi32(v[3], bit); + + v[4] = u[4]; + v[5] = u[5]; + + y = _mm256_mullo_epi32(u[6], cospi32); + x = _mm256_mullo_epi32(u[7], cospi32); + v[6] = _mm256_add_epi32(y, x); + v[6] = _mm256_add_epi32(v[6], rnding); + v[6] = _mm256_srai_epi32(v[6], bit); + + v[7] = _mm256_sub_epi32(y, x); + v[7] = _mm256_add_epi32(v[7], rnding); + v[7] = _mm256_srai_epi32(v[7], bit); + + v[8] = u[8]; + v[9] = u[9]; + + y = _mm256_mullo_epi32(u[10], cospi32); + x = _mm256_mullo_epi32(u[11], cospi32); + v[10] = _mm256_add_epi32(y, x); + v[10] = _mm256_add_epi32(v[10], rnding); + v[10] = _mm256_srai_epi32(v[10], bit); + + v[11] = _mm256_sub_epi32(y, x); + v[11] = _mm256_add_epi32(v[11], rnding); + v[11] = _mm256_srai_epi32(v[11], bit); + + v[12] = u[12]; + v[13] = u[13]; + + y = _mm256_mullo_epi32(u[14], cospi32); + x = _mm256_mullo_epi32(u[15], cospi32); + v[14] = _mm256_add_epi32(y, x); + v[14] = _mm256_add_epi32(v[14], rnding); + v[14] = _mm256_srai_epi32(v[14], bit); + + v[15] = _mm256_sub_epi32(y, x); + v[15] = _mm256_add_epi32(v[15], rnding); + v[15] = _mm256_srai_epi32(v[15], bit); + + // stage 9 + if (do_cols) { + out[0] = v[0]; + out[1] = _mm256_sub_epi32(_mm256_setzero_si256(), v[8]); + out[2] = v[12]; + out[3] = _mm256_sub_epi32(_mm256_setzero_si256(), v[4]); + out[4] = v[6]; + out[5] = _mm256_sub_epi32(_mm256_setzero_si256(), v[14]); + out[6] = v[10]; + out[7] = _mm256_sub_epi32(_mm256_setzero_si256(), v[2]); + out[8] = v[3]; + out[9] = _mm256_sub_epi32(_mm256_setzero_si256(), v[11]); + out[10] = v[15]; + out[11] = _mm256_sub_epi32(_mm256_setzero_si256(), v[7]); + out[12] = v[5]; + out[13] = _mm256_sub_epi32(_mm256_setzero_si256(), v[13]); + out[14] = v[9]; + out[15] = _mm256_sub_epi32(_mm256_setzero_si256(), v[1]); + } else { + const int log_range_out = AOMMAX(16, bd + 6); + const __m256i clamp_lo_out = + _mm256_set1_epi32(-(1 << (log_range_out - 1))); + const __m256i clamp_hi_out = + _mm256_set1_epi32((1 << (log_range_out - 1)) - 1); + + neg_shift_avx2(v[0], v[8], out + 0, out + 1, &clamp_lo_out, &clamp_hi_out, + out_shift); + neg_shift_avx2(v[12], v[4], out + 2, out + 3, &clamp_lo_out, + &clamp_hi_out, out_shift); + neg_shift_avx2(v[6], v[14], out + 4, out + 5, &clamp_lo_out, + &clamp_hi_out, out_shift); + neg_shift_avx2(v[10], v[2], out + 6, out + 7, &clamp_lo_out, + &clamp_hi_out, out_shift); + neg_shift_avx2(v[3], v[11], out + 8, out + 9, &clamp_lo_out, + &clamp_hi_out, out_shift); + neg_shift_avx2(v[15], v[7], out + 10, out + 11, &clamp_lo_out, + &clamp_hi_out, out_shift); + neg_shift_avx2(v[5], v[13], out + 12, out + 13, &clamp_lo_out, + &clamp_hi_out, out_shift); + neg_shift_avx2(v[9], v[1], out + 14, out + 15, &clamp_lo_out, + &clamp_hi_out, out_shift); + } + } +} +static void idct8x8_low1_avx2(__m256i *in, __m256i *out, int bit, int do_cols, + int bd, int out_shift) { + const int32_t *cospi = cospi_arr(bit); + const __m256i cospi32 = _mm256_set1_epi32(cospi[32]); + const __m256i rnding = _mm256_set1_epi32(1 << (bit - 1)); + const int log_range = AOMMAX(16, bd + (do_cols ? 6 : 8)); + __m256i x; + + // stage 0 + // stage 1 + // stage 2 + // stage 3 + x = _mm256_mullo_epi32(in[0], cospi32); + x = _mm256_add_epi32(x, rnding); + x = _mm256_srai_epi32(x, bit); + + // stage 4 + // stage 5 + if (!do_cols) { + const int log_range_out = AOMMAX(16, bd + 6); + const __m256i clamp_lo_out = _mm256_set1_epi32(AOMMAX( + -(1 << (log_range_out - 1)), -(1 << (log_range - 1 - out_shift)))); + const __m256i clamp_hi_out = _mm256_set1_epi32(AOMMIN( + (1 << (log_range_out - 1)) - 1, (1 << (log_range - 1 - out_shift)))); + + __m256i offset = _mm256_set1_epi32((1 << out_shift) >> 1); + x = _mm256_add_epi32(x, offset); + x = _mm256_sra_epi32(x, _mm_cvtsi32_si128(out_shift)); + x = _mm256_max_epi32(x, clamp_lo_out); + x = _mm256_min_epi32(x, clamp_hi_out); + } + + out[0] = x; + out[1] = x; + out[2] = x; + out[3] = x; + out[4] = x; + out[5] = x; + out[6] = x; + out[7] = x; +} +static void idct8x8_avx2(__m256i *in, __m256i *out, int bit, int do_cols, + int bd, int out_shift) { + const int32_t *cospi = cospi_arr(bit); + const __m256i cospi56 = _mm256_set1_epi32(cospi[56]); + const __m256i cospim8 = _mm256_set1_epi32(-cospi[8]); + const __m256i cospi24 = _mm256_set1_epi32(cospi[24]); + const __m256i cospim40 = _mm256_set1_epi32(-cospi[40]); + const __m256i cospi40 = _mm256_set1_epi32(cospi[40]); + const __m256i cospi8 = _mm256_set1_epi32(cospi[8]); + const __m256i cospi32 = _mm256_set1_epi32(cospi[32]); + const __m256i cospi48 = _mm256_set1_epi32(cospi[48]); + const __m256i cospim16 = _mm256_set1_epi32(-cospi[16]); + const __m256i cospi16 = _mm256_set1_epi32(cospi[16]); + const __m256i rnding = _mm256_set1_epi32(1 << (bit - 1)); + const int log_range = AOMMAX(16, bd + (do_cols ? 6 : 8)); + const __m256i clamp_lo = _mm256_set1_epi32(-(1 << (log_range - 1))); + const __m256i clamp_hi = _mm256_set1_epi32((1 << (log_range - 1)) - 1); + __m256i u0, u1, u2, u3, u4, u5, u6, u7; + __m256i v0, v1, v2, v3, v4, v5, v6, v7; + __m256i x, y; + + // stage 0 + // stage 1 + // stage 2 + u0 = in[0]; + u1 = in[4]; + u2 = in[2]; + u3 = in[6]; + + x = _mm256_mullo_epi32(in[1], cospi56); + y = _mm256_mullo_epi32(in[7], cospim8); + u4 = _mm256_add_epi32(x, y); + u4 = _mm256_add_epi32(u4, rnding); + u4 = _mm256_srai_epi32(u4, bit); + + x = _mm256_mullo_epi32(in[1], cospi8); + y = _mm256_mullo_epi32(in[7], cospi56); + u7 = _mm256_add_epi32(x, y); + u7 = _mm256_add_epi32(u7, rnding); + u7 = _mm256_srai_epi32(u7, bit); + + x = _mm256_mullo_epi32(in[5], cospi24); + y = _mm256_mullo_epi32(in[3], cospim40); + u5 = _mm256_add_epi32(x, y); + u5 = _mm256_add_epi32(u5, rnding); + u5 = _mm256_srai_epi32(u5, bit); + + x = _mm256_mullo_epi32(in[5], cospi40); + y = _mm256_mullo_epi32(in[3], cospi24); + u6 = _mm256_add_epi32(x, y); + u6 = _mm256_add_epi32(u6, rnding); + u6 = _mm256_srai_epi32(u6, bit); + + // stage 3 + x = _mm256_mullo_epi32(u0, cospi32); + y = _mm256_mullo_epi32(u1, cospi32); + v0 = _mm256_add_epi32(x, y); + v0 = _mm256_add_epi32(v0, rnding); + v0 = _mm256_srai_epi32(v0, bit); + + v1 = _mm256_sub_epi32(x, y); + v1 = _mm256_add_epi32(v1, rnding); + v1 = _mm256_srai_epi32(v1, bit); + + x = _mm256_mullo_epi32(u2, cospi48); + y = _mm256_mullo_epi32(u3, cospim16); + v2 = _mm256_add_epi32(x, y); + v2 = _mm256_add_epi32(v2, rnding); + v2 = _mm256_srai_epi32(v2, bit); + + x = _mm256_mullo_epi32(u2, cospi16); + y = _mm256_mullo_epi32(u3, cospi48); + v3 = _mm256_add_epi32(x, y); + v3 = _mm256_add_epi32(v3, rnding); + v3 = _mm256_srai_epi32(v3, bit); + + addsub_avx2(u4, u5, &v4, &v5, &clamp_lo, &clamp_hi); + addsub_avx2(u7, u6, &v7, &v6, &clamp_lo, &clamp_hi); + + // stage 4 + addsub_avx2(v0, v3, &u0, &u3, &clamp_lo, &clamp_hi); + addsub_avx2(v1, v2, &u1, &u2, &clamp_lo, &clamp_hi); + u4 = v4; + u7 = v7; + + x = _mm256_mullo_epi32(v5, cospi32); + y = _mm256_mullo_epi32(v6, cospi32); + u6 = _mm256_add_epi32(y, x); + u6 = _mm256_add_epi32(u6, rnding); + u6 = _mm256_srai_epi32(u6, bit); + + u5 = _mm256_sub_epi32(y, x); + u5 = _mm256_add_epi32(u5, rnding); + u5 = _mm256_srai_epi32(u5, bit); + + // stage 5 + if (do_cols) { + addsub_no_clamp_avx2(u0, u7, out + 0, out + 7); + addsub_no_clamp_avx2(u1, u6, out + 1, out + 6); + addsub_no_clamp_avx2(u2, u5, out + 2, out + 5); + addsub_no_clamp_avx2(u3, u4, out + 3, out + 4); + } else { + const int log_range_out = AOMMAX(16, bd + 6); + const __m256i clamp_lo_out = _mm256_set1_epi32(AOMMAX( + -(1 << (log_range_out - 1)), -(1 << (log_range - 1 - out_shift)))); + const __m256i clamp_hi_out = _mm256_set1_epi32(AOMMIN( + (1 << (log_range_out - 1)) - 1, (1 << (log_range - 1 - out_shift)))); + addsub_shift_avx2(u0, u7, out + 0, out + 7, &clamp_lo_out, &clamp_hi_out, + out_shift); + addsub_shift_avx2(u1, u6, out + 1, out + 6, &clamp_lo_out, &clamp_hi_out, + out_shift); + addsub_shift_avx2(u2, u5, out + 2, out + 5, &clamp_lo_out, &clamp_hi_out, + out_shift); + addsub_shift_avx2(u3, u4, out + 3, out + 4, &clamp_lo_out, &clamp_hi_out, + out_shift); + } +} +static void iadst8x8_low1_avx2(__m256i *in, __m256i *out, int bit, int do_cols, + int bd, int out_shift) { + const int32_t *cospi = cospi_arr(bit); + const __m256i cospi4 = _mm256_set1_epi32(cospi[4]); + const __m256i cospi60 = _mm256_set1_epi32(cospi[60]); + const __m256i cospi16 = _mm256_set1_epi32(cospi[16]); + const __m256i cospi48 = _mm256_set1_epi32(cospi[48]); + const __m256i cospi32 = _mm256_set1_epi32(cospi[32]); + const __m256i rnding = _mm256_set1_epi32(1 << (bit - 1)); + const __m256i kZero = _mm256_setzero_si256(); + __m256i u[8], x; + + // stage 0 + // stage 1 + // stage 2 + + x = _mm256_mullo_epi32(in[0], cospi60); + u[0] = _mm256_add_epi32(x, rnding); + u[0] = _mm256_srai_epi32(u[0], bit); + + x = _mm256_mullo_epi32(in[0], cospi4); + u[1] = _mm256_sub_epi32(kZero, x); + u[1] = _mm256_add_epi32(u[1], rnding); + u[1] = _mm256_srai_epi32(u[1], bit); + + // stage 3 + // stage 4 + __m256i temp1, temp2; + temp1 = _mm256_mullo_epi32(u[0], cospi16); + x = _mm256_mullo_epi32(u[1], cospi48); + temp1 = _mm256_add_epi32(temp1, x); + temp1 = _mm256_add_epi32(temp1, rnding); + temp1 = _mm256_srai_epi32(temp1, bit); + u[4] = temp1; + + temp2 = _mm256_mullo_epi32(u[0], cospi48); + x = _mm256_mullo_epi32(u[1], cospi16); + u[5] = _mm256_sub_epi32(temp2, x); + u[5] = _mm256_add_epi32(u[5], rnding); + u[5] = _mm256_srai_epi32(u[5], bit); + + // stage 5 + // stage 6 + temp1 = _mm256_mullo_epi32(u[0], cospi32); + x = _mm256_mullo_epi32(u[1], cospi32); + u[2] = _mm256_add_epi32(temp1, x); + u[2] = _mm256_add_epi32(u[2], rnding); + u[2] = _mm256_srai_epi32(u[2], bit); + + u[3] = _mm256_sub_epi32(temp1, x); + u[3] = _mm256_add_epi32(u[3], rnding); + u[3] = _mm256_srai_epi32(u[3], bit); + + temp1 = _mm256_mullo_epi32(u[4], cospi32); + x = _mm256_mullo_epi32(u[5], cospi32); + u[6] = _mm256_add_epi32(temp1, x); + u[6] = _mm256_add_epi32(u[6], rnding); + u[6] = _mm256_srai_epi32(u[6], bit); + + u[7] = _mm256_sub_epi32(temp1, x); + u[7] = _mm256_add_epi32(u[7], rnding); + u[7] = _mm256_srai_epi32(u[7], bit); + + // stage 7 + if (do_cols) { + out[0] = u[0]; + out[1] = _mm256_sub_epi32(kZero, u[4]); + out[2] = u[6]; + out[3] = _mm256_sub_epi32(kZero, u[2]); + out[4] = u[3]; + out[5] = _mm256_sub_epi32(kZero, u[7]); + out[6] = u[5]; + out[7] = _mm256_sub_epi32(kZero, u[1]); + } else { + const int log_range_out = AOMMAX(16, bd + 6); + const __m256i clamp_lo_out = _mm256_set1_epi32(-(1 << (log_range_out - 1))); + const __m256i clamp_hi_out = + _mm256_set1_epi32((1 << (log_range_out - 1)) - 1); + + neg_shift_avx2(u[0], u[4], out + 0, out + 1, &clamp_lo_out, &clamp_hi_out, + out_shift); + neg_shift_avx2(u[6], u[2], out + 2, out + 3, &clamp_lo_out, &clamp_hi_out, + out_shift); + neg_shift_avx2(u[3], u[7], out + 4, out + 5, &clamp_lo_out, &clamp_hi_out, + out_shift); + neg_shift_avx2(u[5], u[1], out + 6, out + 7, &clamp_lo_out, &clamp_hi_out, + out_shift); + } +} + +static void iadst8x8_avx2(__m256i *in, __m256i *out, int bit, int do_cols, + int bd, int out_shift) { + const int32_t *cospi = cospi_arr(bit); + const __m256i cospi4 = _mm256_set1_epi32(cospi[4]); + const __m256i cospi60 = _mm256_set1_epi32(cospi[60]); + const __m256i cospi20 = _mm256_set1_epi32(cospi[20]); + const __m256i cospi44 = _mm256_set1_epi32(cospi[44]); + const __m256i cospi36 = _mm256_set1_epi32(cospi[36]); + const __m256i cospi28 = _mm256_set1_epi32(cospi[28]); + const __m256i cospi52 = _mm256_set1_epi32(cospi[52]); + const __m256i cospi12 = _mm256_set1_epi32(cospi[12]); + const __m256i cospi16 = _mm256_set1_epi32(cospi[16]); + const __m256i cospi48 = _mm256_set1_epi32(cospi[48]); + const __m256i cospim48 = _mm256_set1_epi32(-cospi[48]); + const __m256i cospi32 = _mm256_set1_epi32(cospi[32]); + const __m256i rnding = _mm256_set1_epi32(1 << (bit - 1)); + const __m256i kZero = _mm256_setzero_si256(); + const int log_range = AOMMAX(16, bd + (do_cols ? 6 : 8)); + const __m256i clamp_lo = _mm256_set1_epi32(-(1 << (log_range - 1))); + const __m256i clamp_hi = _mm256_set1_epi32((1 << (log_range - 1)) - 1); + __m256i u[8], v[8], x; + + // stage 0 + // stage 1 + // stage 2 + + u[0] = _mm256_mullo_epi32(in[7], cospi4); + x = _mm256_mullo_epi32(in[0], cospi60); + u[0] = _mm256_add_epi32(u[0], x); + u[0] = _mm256_add_epi32(u[0], rnding); + u[0] = _mm256_srai_epi32(u[0], bit); + + u[1] = _mm256_mullo_epi32(in[7], cospi60); + x = _mm256_mullo_epi32(in[0], cospi4); + u[1] = _mm256_sub_epi32(u[1], x); + u[1] = _mm256_add_epi32(u[1], rnding); + u[1] = _mm256_srai_epi32(u[1], bit); + + u[2] = _mm256_mullo_epi32(in[5], cospi20); + x = _mm256_mullo_epi32(in[2], cospi44); + u[2] = _mm256_add_epi32(u[2], x); + u[2] = _mm256_add_epi32(u[2], rnding); + u[2] = _mm256_srai_epi32(u[2], bit); + + u[3] = _mm256_mullo_epi32(in[5], cospi44); + x = _mm256_mullo_epi32(in[2], cospi20); + u[3] = _mm256_sub_epi32(u[3], x); + u[3] = _mm256_add_epi32(u[3], rnding); + u[3] = _mm256_srai_epi32(u[3], bit); + + u[4] = _mm256_mullo_epi32(in[3], cospi36); + x = _mm256_mullo_epi32(in[4], cospi28); + u[4] = _mm256_add_epi32(u[4], x); + u[4] = _mm256_add_epi32(u[4], rnding); + u[4] = _mm256_srai_epi32(u[4], bit); + + u[5] = _mm256_mullo_epi32(in[3], cospi28); + x = _mm256_mullo_epi32(in[4], cospi36); + u[5] = _mm256_sub_epi32(u[5], x); + u[5] = _mm256_add_epi32(u[5], rnding); + u[5] = _mm256_srai_epi32(u[5], bit); + + u[6] = _mm256_mullo_epi32(in[1], cospi52); + x = _mm256_mullo_epi32(in[6], cospi12); + u[6] = _mm256_add_epi32(u[6], x); + u[6] = _mm256_add_epi32(u[6], rnding); + u[6] = _mm256_srai_epi32(u[6], bit); + + u[7] = _mm256_mullo_epi32(in[1], cospi12); + x = _mm256_mullo_epi32(in[6], cospi52); + u[7] = _mm256_sub_epi32(u[7], x); + u[7] = _mm256_add_epi32(u[7], rnding); + u[7] = _mm256_srai_epi32(u[7], bit); + + // stage 3 + addsub_avx2(u[0], u[4], &v[0], &v[4], &clamp_lo, &clamp_hi); + addsub_avx2(u[1], u[5], &v[1], &v[5], &clamp_lo, &clamp_hi); + addsub_avx2(u[2], u[6], &v[2], &v[6], &clamp_lo, &clamp_hi); + addsub_avx2(u[3], u[7], &v[3], &v[7], &clamp_lo, &clamp_hi); + + // stage 4 + u[0] = v[0]; + u[1] = v[1]; + u[2] = v[2]; + u[3] = v[3]; + + u[4] = _mm256_mullo_epi32(v[4], cospi16); + x = _mm256_mullo_epi32(v[5], cospi48); + u[4] = _mm256_add_epi32(u[4], x); + u[4] = _mm256_add_epi32(u[4], rnding); + u[4] = _mm256_srai_epi32(u[4], bit); + + u[5] = _mm256_mullo_epi32(v[4], cospi48); + x = _mm256_mullo_epi32(v[5], cospi16); + u[5] = _mm256_sub_epi32(u[5], x); + u[5] = _mm256_add_epi32(u[5], rnding); + u[5] = _mm256_srai_epi32(u[5], bit); + + u[6] = _mm256_mullo_epi32(v[6], cospim48); + x = _mm256_mullo_epi32(v[7], cospi16); + u[6] = _mm256_add_epi32(u[6], x); + u[6] = _mm256_add_epi32(u[6], rnding); + u[6] = _mm256_srai_epi32(u[6], bit); + + u[7] = _mm256_mullo_epi32(v[6], cospi16); + x = _mm256_mullo_epi32(v[7], cospim48); + u[7] = _mm256_sub_epi32(u[7], x); + u[7] = _mm256_add_epi32(u[7], rnding); + u[7] = _mm256_srai_epi32(u[7], bit); + + // stage 5 + addsub_avx2(u[0], u[2], &v[0], &v[2], &clamp_lo, &clamp_hi); + addsub_avx2(u[1], u[3], &v[1], &v[3], &clamp_lo, &clamp_hi); + addsub_avx2(u[4], u[6], &v[4], &v[6], &clamp_lo, &clamp_hi); + addsub_avx2(u[5], u[7], &v[5], &v[7], &clamp_lo, &clamp_hi); + + // stage 6 + u[0] = v[0]; + u[1] = v[1]; + u[4] = v[4]; + u[5] = v[5]; + + v[0] = _mm256_mullo_epi32(v[2], cospi32); + x = _mm256_mullo_epi32(v[3], cospi32); + u[2] = _mm256_add_epi32(v[0], x); + u[2] = _mm256_add_epi32(u[2], rnding); + u[2] = _mm256_srai_epi32(u[2], bit); + + u[3] = _mm256_sub_epi32(v[0], x); + u[3] = _mm256_add_epi32(u[3], rnding); + u[3] = _mm256_srai_epi32(u[3], bit); + + v[0] = _mm256_mullo_epi32(v[6], cospi32); + x = _mm256_mullo_epi32(v[7], cospi32); + u[6] = _mm256_add_epi32(v[0], x); + u[6] = _mm256_add_epi32(u[6], rnding); + u[6] = _mm256_srai_epi32(u[6], bit); + + u[7] = _mm256_sub_epi32(v[0], x); + u[7] = _mm256_add_epi32(u[7], rnding); + u[7] = _mm256_srai_epi32(u[7], bit); + + // stage 7 + if (do_cols) { + out[0] = u[0]; + out[1] = _mm256_sub_epi32(kZero, u[4]); + out[2] = u[6]; + out[3] = _mm256_sub_epi32(kZero, u[2]); + out[4] = u[3]; + out[5] = _mm256_sub_epi32(kZero, u[7]); + out[6] = u[5]; + out[7] = _mm256_sub_epi32(kZero, u[1]); + } else { + const int log_range_out = AOMMAX(16, bd + 6); + const __m256i clamp_lo_out = _mm256_set1_epi32(-(1 << (log_range_out - 1))); + const __m256i clamp_hi_out = + _mm256_set1_epi32((1 << (log_range_out - 1)) - 1); + + neg_shift_avx2(u[0], u[4], out + 0, out + 1, &clamp_lo_out, &clamp_hi_out, + out_shift); + neg_shift_avx2(u[6], u[2], out + 2, out + 3, &clamp_lo_out, &clamp_hi_out, + out_shift); + neg_shift_avx2(u[3], u[7], out + 4, out + 5, &clamp_lo_out, &clamp_hi_out, + out_shift); + neg_shift_avx2(u[5], u[1], out + 6, out + 7, &clamp_lo_out, &clamp_hi_out, + out_shift); + } +} +static INLINE void idct64_stage8_avx2( + __m256i *u, const __m256i *cospim32, const __m256i *cospi32, + const __m256i *cospim16, const __m256i *cospi48, const __m256i *cospi16, + const __m256i *cospim48, const __m256i *clamp_lo, const __m256i *clamp_hi, + const __m256i *rnding, int bit) { + int i; + __m256i temp1, temp2, temp3, temp4; + temp1 = half_btf_avx2(cospim32, &u[10], cospi32, &u[13], rnding, bit); + u[13] = half_btf_avx2(cospi32, &u[10], cospi32, &u[13], rnding, bit); + u[10] = temp1; + temp2 = half_btf_avx2(cospim32, &u[11], cospi32, &u[12], rnding, bit); + u[12] = half_btf_avx2(cospi32, &u[11], cospi32, &u[12], rnding, bit); + u[11] = temp2; + + for (i = 16; i < 20; ++i) { + addsub_avx2(u[i], u[i ^ 7], &u[i], &u[i ^ 7], clamp_lo, clamp_hi); + addsub_avx2(u[i ^ 15], u[i ^ 8], &u[i ^ 15], &u[i ^ 8], clamp_lo, clamp_hi); + } + + temp1 = half_btf_avx2(cospim16, &u[36], cospi48, &u[59], rnding, bit); + temp2 = half_btf_avx2(cospim16, &u[37], cospi48, &u[58], rnding, bit); + temp3 = half_btf_avx2(cospim16, &u[38], cospi48, &u[57], rnding, bit); + temp4 = half_btf_avx2(cospim16, &u[39], cospi48, &u[56], rnding, bit); + u[56] = half_btf_avx2(cospi48, &u[39], cospi16, &u[56], rnding, bit); + u[57] = half_btf_avx2(cospi48, &u[38], cospi16, &u[57], rnding, bit); + u[58] = half_btf_avx2(cospi48, &u[37], cospi16, &u[58], rnding, bit); + u[59] = half_btf_avx2(cospi48, &u[36], cospi16, &u[59], rnding, bit); + u[36] = temp1; + u[37] = temp2; + u[38] = temp3; + u[39] = temp4; + + temp1 = half_btf_avx2(cospim48, &u[40], cospim16, &u[55], rnding, bit); + temp2 = half_btf_avx2(cospim48, &u[41], cospim16, &u[54], rnding, bit); + temp3 = half_btf_avx2(cospim48, &u[42], cospim16, &u[53], rnding, bit); + temp4 = half_btf_avx2(cospim48, &u[43], cospim16, &u[52], rnding, bit); + u[52] = half_btf_avx2(cospim16, &u[43], cospi48, &u[52], rnding, bit); + u[53] = half_btf_avx2(cospim16, &u[42], cospi48, &u[53], rnding, bit); + u[54] = half_btf_avx2(cospim16, &u[41], cospi48, &u[54], rnding, bit); + u[55] = half_btf_avx2(cospim16, &u[40], cospi48, &u[55], rnding, bit); + u[40] = temp1; + u[41] = temp2; + u[42] = temp3; + u[43] = temp4; +} + +static INLINE void idct64_stage9_avx2(__m256i *u, const __m256i *cospim32, + const __m256i *cospi32, + const __m256i *clamp_lo, + const __m256i *clamp_hi, + const __m256i *rnding, int bit) { + int i; + __m256i temp1, temp2, temp3, temp4; + for (i = 0; i < 8; ++i) { + addsub_avx2(u[i], u[15 - i], &u[i], &u[15 - i], clamp_lo, clamp_hi); + } + + temp1 = half_btf_avx2(cospim32, &u[20], cospi32, &u[27], rnding, bit); + temp2 = half_btf_avx2(cospim32, &u[21], cospi32, &u[26], rnding, bit); + temp3 = half_btf_avx2(cospim32, &u[22], cospi32, &u[25], rnding, bit); + temp4 = half_btf_avx2(cospim32, &u[23], cospi32, &u[24], rnding, bit); + u[24] = half_btf_avx2(cospi32, &u[23], cospi32, &u[24], rnding, bit); + u[25] = half_btf_avx2(cospi32, &u[22], cospi32, &u[25], rnding, bit); + u[26] = half_btf_avx2(cospi32, &u[21], cospi32, &u[26], rnding, bit); + u[27] = half_btf_avx2(cospi32, &u[20], cospi32, &u[27], rnding, bit); + u[20] = temp1; + u[21] = temp2; + u[22] = temp3; + u[23] = temp4; + for (i = 32; i < 40; i++) { + addsub_avx2(u[i], u[i ^ 15], &u[i], &u[i ^ 15], clamp_lo, clamp_hi); + } + + for (i = 48; i < 56; i++) { + addsub_avx2(u[i ^ 15], u[i], &u[i ^ 15], &u[i], clamp_lo, clamp_hi); + } +} + +static INLINE void idct64_stage10_avx2(__m256i *u, const __m256i *cospim32, + const __m256i *cospi32, + const __m256i *clamp_lo, + const __m256i *clamp_hi, + const __m256i *rnding, int bit) { + __m256i temp1, temp2, temp3, temp4; + for (int i = 0; i < 16; i++) { + addsub_avx2(u[i], u[31 - i], &u[i], &u[31 - i], clamp_lo, clamp_hi); + } + + temp1 = half_btf_avx2(cospim32, &u[40], cospi32, &u[55], rnding, bit); + temp2 = half_btf_avx2(cospim32, &u[41], cospi32, &u[54], rnding, bit); + temp3 = half_btf_avx2(cospim32, &u[42], cospi32, &u[53], rnding, bit); + temp4 = half_btf_avx2(cospim32, &u[43], cospi32, &u[52], rnding, bit); + u[52] = half_btf_avx2(cospi32, &u[43], cospi32, &u[52], rnding, bit); + u[53] = half_btf_avx2(cospi32, &u[42], cospi32, &u[53], rnding, bit); + u[54] = half_btf_avx2(cospi32, &u[41], cospi32, &u[54], rnding, bit); + u[55] = half_btf_avx2(cospi32, &u[40], cospi32, &u[55], rnding, bit); + u[40] = temp1; + u[41] = temp2; + u[42] = temp3; + u[43] = temp4; + + temp1 = half_btf_avx2(cospim32, &u[44], cospi32, &u[51], rnding, bit); + temp2 = half_btf_avx2(cospim32, &u[45], cospi32, &u[50], rnding, bit); + temp3 = half_btf_avx2(cospim32, &u[46], cospi32, &u[49], rnding, bit); + temp4 = half_btf_avx2(cospim32, &u[47], cospi32, &u[48], rnding, bit); + u[48] = half_btf_avx2(cospi32, &u[47], cospi32, &u[48], rnding, bit); + u[49] = half_btf_avx2(cospi32, &u[46], cospi32, &u[49], rnding, bit); + u[50] = half_btf_avx2(cospi32, &u[45], cospi32, &u[50], rnding, bit); + u[51] = half_btf_avx2(cospi32, &u[44], cospi32, &u[51], rnding, bit); + u[44] = temp1; + u[45] = temp2; + u[46] = temp3; + u[47] = temp4; +} + +static INLINE void idct64_stage11_avx2(__m256i *u, __m256i *out, int do_cols, + int bd, int out_shift, + const int log_range) { + if (do_cols) { + for (int i = 0; i < 32; i++) { + addsub_no_clamp_avx2(u[i], u[63 - i], &out[(i)], &out[(63 - i)]); + } + } else { + const int log_range_out = AOMMAX(16, bd + 6); + const __m256i clamp_lo_out = _mm256_set1_epi32(AOMMAX( + -(1 << (log_range_out - 1)), -(1 << (log_range - 1 - out_shift)))); + const __m256i clamp_hi_out = _mm256_set1_epi32(AOMMIN( + (1 << (log_range_out - 1)) - 1, (1 << (log_range - 1 - out_shift)))); + + for (int i = 0; i < 32; i++) { + addsub_shift_avx2(u[i], u[63 - i], &out[(i)], &out[(63 - i)], + &clamp_lo_out, &clamp_hi_out, out_shift); + } + } +} + +static void idct64_low1_avx2(__m256i *in, __m256i *out, int bit, int do_cols, + int bd, int out_shift) { + const int32_t *cospi = cospi_arr(bit); + const __m256i rnding = _mm256_set1_epi32(1 << (bit - 1)); + const int log_range = AOMMAX(16, bd + (do_cols ? 6 : 8)); + const __m256i clamp_lo = _mm256_set1_epi32(-(1 << (log_range - 1))); + const __m256i clamp_hi = _mm256_set1_epi32((1 << (log_range - 1)) - 1); + + const __m256i cospi32 = _mm256_set1_epi32(cospi[32]); + + { + __m256i x; + + // stage 1 + // stage 2 + // stage 3 + // stage 4 + // stage 5 + // stage 6 + x = half_btf_0_avx2(&cospi32, &in[0], &rnding, bit); + + // stage 8 + // stage 9 + // stage 10 + // stage 11 + if (do_cols) { + x = _mm256_max_epi32(x, clamp_lo); + x = _mm256_min_epi32(x, clamp_hi); + } else { + const int log_range_out = AOMMAX(16, bd + 6); + const __m256i clamp_lo_out = _mm256_set1_epi32(AOMMAX( + -(1 << (log_range_out - 1)), -(1 << (log_range - 1 - out_shift)))); + const __m256i clamp_hi_out = _mm256_set1_epi32(AOMMIN( + (1 << (log_range_out - 1)) - 1, (1 << (log_range - 1 - out_shift)))); + + __m256i offset = _mm256_set1_epi32((1 << out_shift) >> 1); + x = _mm256_add_epi32(x, offset); + x = _mm256_sra_epi32(x, _mm_cvtsi32_si128(out_shift)); + + x = _mm256_max_epi32(x, clamp_lo_out); + x = _mm256_min_epi32(x, clamp_hi_out); + } + + out[0] = x; + out[1] = x; + out[2] = x; + out[3] = x; + out[4] = x; + out[5] = x; + out[6] = x; + out[7] = x; + out[8] = x; + out[9] = x; + out[10] = x; + out[11] = x; + out[12] = x; + out[13] = x; + out[14] = x; + out[15] = x; + out[16] = x; + out[17] = x; + out[18] = x; + out[19] = x; + out[20] = x; + out[21] = x; + out[22] = x; + out[23] = x; + out[24] = x; + out[25] = x; + out[26] = x; + out[27] = x; + out[28] = x; + out[29] = x; + out[30] = x; + out[31] = x; + out[32] = x; + out[33] = x; + out[34] = x; + out[35] = x; + out[36] = x; + out[37] = x; + out[38] = x; + out[39] = x; + out[40] = x; + out[41] = x; + out[42] = x; + out[43] = x; + out[44] = x; + out[45] = x; + out[46] = x; + out[47] = x; + out[48] = x; + out[49] = x; + out[50] = x; + out[51] = x; + out[52] = x; + out[53] = x; + out[54] = x; + out[55] = x; + out[56] = x; + out[57] = x; + out[58] = x; + out[59] = x; + out[60] = x; + out[61] = x; + out[62] = x; + out[63] = x; + } +} +static void idct64_low8_avx2(__m256i *in, __m256i *out, int bit, int do_cols, + int bd, int out_shift) { + int i, j; + const int32_t *cospi = cospi_arr(bit); + const __m256i rnding = _mm256_set1_epi32(1 << (bit - 1)); + const int log_range = AOMMAX(16, bd + (do_cols ? 6 : 8)); + const __m256i clamp_lo = _mm256_set1_epi32(-(1 << (log_range - 1))); + const __m256i clamp_hi = _mm256_set1_epi32((1 << (log_range - 1)) - 1); + + const __m256i cospi1 = _mm256_set1_epi32(cospi[1]); + const __m256i cospi2 = _mm256_set1_epi32(cospi[2]); + const __m256i cospi3 = _mm256_set1_epi32(cospi[3]); + const __m256i cospi4 = _mm256_set1_epi32(cospi[4]); + const __m256i cospi6 = _mm256_set1_epi32(cospi[6]); + const __m256i cospi8 = _mm256_set1_epi32(cospi[8]); + const __m256i cospi12 = _mm256_set1_epi32(cospi[12]); + const __m256i cospi16 = _mm256_set1_epi32(cospi[16]); + const __m256i cospi20 = _mm256_set1_epi32(cospi[20]); + const __m256i cospi24 = _mm256_set1_epi32(cospi[24]); + const __m256i cospi28 = _mm256_set1_epi32(cospi[28]); + const __m256i cospi32 = _mm256_set1_epi32(cospi[32]); + const __m256i cospi40 = _mm256_set1_epi32(cospi[40]); + const __m256i cospi44 = _mm256_set1_epi32(cospi[44]); + const __m256i cospi48 = _mm256_set1_epi32(cospi[48]); + const __m256i cospi56 = _mm256_set1_epi32(cospi[56]); + const __m256i cospi60 = _mm256_set1_epi32(cospi[60]); + const __m256i cospim4 = _mm256_set1_epi32(-cospi[4]); + const __m256i cospim8 = _mm256_set1_epi32(-cospi[8]); + const __m256i cospim12 = _mm256_set1_epi32(-cospi[12]); + const __m256i cospim16 = _mm256_set1_epi32(-cospi[16]); + const __m256i cospim20 = _mm256_set1_epi32(-cospi[20]); + const __m256i cospim24 = _mm256_set1_epi32(-cospi[24]); + const __m256i cospim28 = _mm256_set1_epi32(-cospi[28]); + const __m256i cospim32 = _mm256_set1_epi32(-cospi[32]); + const __m256i cospim36 = _mm256_set1_epi32(-cospi[36]); + const __m256i cospim40 = _mm256_set1_epi32(-cospi[40]); + const __m256i cospim48 = _mm256_set1_epi32(-cospi[48]); + const __m256i cospim52 = _mm256_set1_epi32(-cospi[52]); + const __m256i cospim56 = _mm256_set1_epi32(-cospi[56]); + const __m256i cospi63 = _mm256_set1_epi32(cospi[63]); + const __m256i cospim57 = _mm256_set1_epi32(-cospi[57]); + const __m256i cospi7 = _mm256_set1_epi32(cospi[7]); + const __m256i cospi5 = _mm256_set1_epi32(cospi[5]); + const __m256i cospi59 = _mm256_set1_epi32(cospi[59]); + const __m256i cospim61 = _mm256_set1_epi32(-cospi[61]); + const __m256i cospim58 = _mm256_set1_epi32(-cospi[58]); + const __m256i cospi62 = _mm256_set1_epi32(cospi[62]); + + { + __m256i u[64]; + + // stage 1 + u[0] = in[0]; + u[8] = in[4]; + u[16] = in[2]; + u[24] = in[6]; + u[32] = in[1]; + u[40] = in[5]; + u[48] = in[3]; + u[56] = in[7]; + + // stage 2 + u[63] = half_btf_0_avx2(&cospi1, &u[32], &rnding, bit); + u[32] = half_btf_0_avx2(&cospi63, &u[32], &rnding, bit); + u[39] = half_btf_0_avx2(&cospim57, &u[56], &rnding, bit); + u[56] = half_btf_0_avx2(&cospi7, &u[56], &rnding, bit); + u[55] = half_btf_0_avx2(&cospi5, &u[40], &rnding, bit); + u[40] = half_btf_0_avx2(&cospi59, &u[40], &rnding, bit); + u[47] = half_btf_0_avx2(&cospim61, &u[48], &rnding, bit); + u[48] = half_btf_0_avx2(&cospi3, &u[48], &rnding, bit); + + // stage 3 + u[31] = half_btf_0_avx2(&cospi2, &u[16], &rnding, bit); + u[16] = half_btf_0_avx2(&cospi62, &u[16], &rnding, bit); + u[23] = half_btf_0_avx2(&cospim58, &u[24], &rnding, bit); + u[24] = half_btf_0_avx2(&cospi6, &u[24], &rnding, bit); + u[33] = u[32]; + u[38] = u[39]; + u[41] = u[40]; + u[46] = u[47]; + u[49] = u[48]; + u[54] = u[55]; + u[57] = u[56]; + u[62] = u[63]; + + // stage 4 + __m256i temp1, temp2; + u[15] = half_btf_0_avx2(&cospi4, &u[8], &rnding, bit); + u[8] = half_btf_0_avx2(&cospi60, &u[8], &rnding, bit); + u[17] = u[16]; + u[22] = u[23]; + u[25] = u[24]; + u[30] = u[31]; + + temp1 = half_btf_avx2(&cospim4, &u[33], &cospi60, &u[62], &rnding, bit); + u[62] = half_btf_avx2(&cospi60, &u[33], &cospi4, &u[62], &rnding, bit); + u[33] = temp1; + + temp2 = half_btf_avx2(&cospim36, &u[38], &cospi28, &u[57], &rnding, bit); + u[38] = half_btf_avx2(&cospim28, &u[38], &cospim36, &u[57], &rnding, bit); + u[57] = temp2; + + temp1 = half_btf_avx2(&cospim20, &u[41], &cospi44, &u[54], &rnding, bit); + u[54] = half_btf_avx2(&cospi44, &u[41], &cospi20, &u[54], &rnding, bit); + u[41] = temp1; + + temp2 = half_btf_avx2(&cospim12, &u[46], &cospim52, &u[49], &rnding, bit); + u[49] = half_btf_avx2(&cospim52, &u[46], &cospi12, &u[49], &rnding, bit); + u[46] = temp2; + + // stage 5 + u[9] = u[8]; + u[14] = u[15]; + + temp1 = half_btf_avx2(&cospim8, &u[17], &cospi56, &u[30], &rnding, bit); + u[30] = half_btf_avx2(&cospi56, &u[17], &cospi8, &u[30], &rnding, bit); + u[17] = temp1; + + temp2 = half_btf_avx2(&cospim24, &u[22], &cospim40, &u[25], &rnding, bit); + u[25] = half_btf_avx2(&cospim40, &u[22], &cospi24, &u[25], &rnding, bit); + u[22] = temp2; + + u[35] = u[32]; + u[34] = u[33]; + u[36] = u[39]; + u[37] = u[38]; + u[43] = u[40]; + u[42] = u[41]; + u[44] = u[47]; + u[45] = u[46]; + u[51] = u[48]; + u[50] = u[49]; + u[52] = u[55]; + u[53] = u[54]; + u[59] = u[56]; + u[58] = u[57]; + u[60] = u[63]; + u[61] = u[62]; + + // stage 6 + temp1 = half_btf_0_avx2(&cospi32, &u[0], &rnding, bit); + u[1] = half_btf_0_avx2(&cospi32, &u[0], &rnding, bit); + u[0] = temp1; + + temp2 = half_btf_avx2(&cospim16, &u[9], &cospi48, &u[14], &rnding, bit); + u[14] = half_btf_avx2(&cospi48, &u[9], &cospi16, &u[14], &rnding, bit); + u[9] = temp2; + u[19] = u[16]; + u[18] = u[17]; + u[20] = u[23]; + u[21] = u[22]; + u[27] = u[24]; + u[26] = u[25]; + u[28] = u[31]; + u[29] = u[30]; + + temp1 = half_btf_avx2(&cospim8, &u[34], &cospi56, &u[61], &rnding, bit); + u[61] = half_btf_avx2(&cospi56, &u[34], &cospi8, &u[61], &rnding, bit); + u[34] = temp1; + temp2 = half_btf_avx2(&cospim8, &u[35], &cospi56, &u[60], &rnding, bit); + u[60] = half_btf_avx2(&cospi56, &u[35], &cospi8, &u[60], &rnding, bit); + u[35] = temp2; + temp1 = half_btf_avx2(&cospim56, &u[36], &cospim8, &u[59], &rnding, bit); + u[59] = half_btf_avx2(&cospim8, &u[36], &cospi56, &u[59], &rnding, bit); + u[36] = temp1; + temp2 = half_btf_avx2(&cospim56, &u[37], &cospim8, &u[58], &rnding, bit); + u[58] = half_btf_avx2(&cospim8, &u[37], &cospi56, &u[58], &rnding, bit); + u[37] = temp2; + temp1 = half_btf_avx2(&cospim40, &u[42], &cospi24, &u[53], &rnding, bit); + u[53] = half_btf_avx2(&cospi24, &u[42], &cospi40, &u[53], &rnding, bit); + u[42] = temp1; + temp2 = half_btf_avx2(&cospim40, &u[43], &cospi24, &u[52], &rnding, bit); + u[52] = half_btf_avx2(&cospi24, &u[43], &cospi40, &u[52], &rnding, bit); + u[43] = temp2; + temp1 = half_btf_avx2(&cospim24, &u[44], &cospim40, &u[51], &rnding, bit); + u[51] = half_btf_avx2(&cospim40, &u[44], &cospi24, &u[51], &rnding, bit); + u[44] = temp1; + temp2 = half_btf_avx2(&cospim24, &u[45], &cospim40, &u[50], &rnding, bit); + u[50] = half_btf_avx2(&cospim40, &u[45], &cospi24, &u[50], &rnding, bit); + u[45] = temp2; + + // stage 7 + u[3] = u[0]; + u[2] = u[1]; + u[11] = u[8]; + u[10] = u[9]; + u[12] = u[15]; + u[13] = u[14]; + + temp1 = half_btf_avx2(&cospim16, &u[18], &cospi48, &u[29], &rnding, bit); + u[29] = half_btf_avx2(&cospi48, &u[18], &cospi16, &u[29], &rnding, bit); + u[18] = temp1; + temp2 = half_btf_avx2(&cospim16, &u[19], &cospi48, &u[28], &rnding, bit); + u[28] = half_btf_avx2(&cospi48, &u[19], &cospi16, &u[28], &rnding, bit); + u[19] = temp2; + temp1 = half_btf_avx2(&cospim48, &u[20], &cospim16, &u[27], &rnding, bit); + u[27] = half_btf_avx2(&cospim16, &u[20], &cospi48, &u[27], &rnding, bit); + u[20] = temp1; + temp2 = half_btf_avx2(&cospim48, &u[21], &cospim16, &u[26], &rnding, bit); + u[26] = half_btf_avx2(&cospim16, &u[21], &cospi48, &u[26], &rnding, bit); + u[21] = temp2; + for (i = 32; i < 64; i += 16) { + for (j = i; j < i + 4; j++) { + addsub_avx2(u[j], u[j ^ 7], &u[j], &u[j ^ 7], &clamp_lo, &clamp_hi); + addsub_avx2(u[j ^ 15], u[j ^ 8], &u[j ^ 15], &u[j ^ 8], &clamp_lo, + &clamp_hi); + } + } + + // stage 8 + u[7] = u[0]; + u[6] = u[1]; + u[5] = u[2]; + u[4] = u[3]; + u[9] = u[9]; + + idct64_stage8_avx2(u, &cospim32, &cospi32, &cospim16, &cospi48, &cospi16, + &cospim48, &clamp_lo, &clamp_hi, &rnding, bit); + + // stage 9 + idct64_stage9_avx2(u, &cospim32, &cospi32, &clamp_lo, &clamp_hi, &rnding, + bit); + + // stage 10 + idct64_stage10_avx2(u, &cospim32, &cospi32, &clamp_lo, &clamp_hi, &rnding, + bit); + + // stage 11 + idct64_stage11_avx2(u, out, do_cols, bd, out_shift, log_range); + } +} +static void idct64_low16_avx2(__m256i *in, __m256i *out, int bit, int do_cols, + int bd, int out_shift) { + int i, j; + const int32_t *cospi = cospi_arr(bit); + const __m256i rnding = _mm256_set1_epi32(1 << (bit - 1)); + const int log_range = AOMMAX(16, bd + (do_cols ? 6 : 8)); + const __m256i clamp_lo = _mm256_set1_epi32(-(1 << (log_range - 1))); + const __m256i clamp_hi = _mm256_set1_epi32((1 << (log_range - 1)) - 1); + + const __m256i cospi1 = _mm256_set1_epi32(cospi[1]); + const __m256i cospi2 = _mm256_set1_epi32(cospi[2]); + const __m256i cospi3 = _mm256_set1_epi32(cospi[3]); + const __m256i cospi4 = _mm256_set1_epi32(cospi[4]); + const __m256i cospi5 = _mm256_set1_epi32(cospi[5]); + const __m256i cospi6 = _mm256_set1_epi32(cospi[6]); + const __m256i cospi7 = _mm256_set1_epi32(cospi[7]); + const __m256i cospi8 = _mm256_set1_epi32(cospi[8]); + const __m256i cospi9 = _mm256_set1_epi32(cospi[9]); + const __m256i cospi10 = _mm256_set1_epi32(cospi[10]); + const __m256i cospi11 = _mm256_set1_epi32(cospi[11]); + const __m256i cospi12 = _mm256_set1_epi32(cospi[12]); + const __m256i cospi13 = _mm256_set1_epi32(cospi[13]); + const __m256i cospi14 = _mm256_set1_epi32(cospi[14]); + const __m256i cospi15 = _mm256_set1_epi32(cospi[15]); + const __m256i cospi16 = _mm256_set1_epi32(cospi[16]); + const __m256i cospi20 = _mm256_set1_epi32(cospi[20]); + const __m256i cospi24 = _mm256_set1_epi32(cospi[24]); + const __m256i cospi28 = _mm256_set1_epi32(cospi[28]); + const __m256i cospi32 = _mm256_set1_epi32(cospi[32]); + const __m256i cospi36 = _mm256_set1_epi32(cospi[36]); + const __m256i cospi40 = _mm256_set1_epi32(cospi[40]); + const __m256i cospi44 = _mm256_set1_epi32(cospi[44]); + const __m256i cospi48 = _mm256_set1_epi32(cospi[48]); + const __m256i cospi51 = _mm256_set1_epi32(cospi[51]); + const __m256i cospi52 = _mm256_set1_epi32(cospi[52]); + const __m256i cospi54 = _mm256_set1_epi32(cospi[54]); + const __m256i cospi55 = _mm256_set1_epi32(cospi[55]); + const __m256i cospi56 = _mm256_set1_epi32(cospi[56]); + const __m256i cospi59 = _mm256_set1_epi32(cospi[59]); + const __m256i cospi60 = _mm256_set1_epi32(cospi[60]); + const __m256i cospi62 = _mm256_set1_epi32(cospi[62]); + const __m256i cospi63 = _mm256_set1_epi32(cospi[63]); + + const __m256i cospim4 = _mm256_set1_epi32(-cospi[4]); + const __m256i cospim8 = _mm256_set1_epi32(-cospi[8]); + const __m256i cospim12 = _mm256_set1_epi32(-cospi[12]); + const __m256i cospim16 = _mm256_set1_epi32(-cospi[16]); + const __m256i cospim20 = _mm256_set1_epi32(-cospi[20]); + const __m256i cospim24 = _mm256_set1_epi32(-cospi[24]); + const __m256i cospim28 = _mm256_set1_epi32(-cospi[28]); + const __m256i cospim32 = _mm256_set1_epi32(-cospi[32]); + const __m256i cospim36 = _mm256_set1_epi32(-cospi[36]); + const __m256i cospim40 = _mm256_set1_epi32(-cospi[40]); + const __m256i cospim44 = _mm256_set1_epi32(-cospi[44]); + const __m256i cospim48 = _mm256_set1_epi32(-cospi[48]); + const __m256i cospim49 = _mm256_set1_epi32(-cospi[49]); + const __m256i cospim50 = _mm256_set1_epi32(-cospi[50]); + const __m256i cospim52 = _mm256_set1_epi32(-cospi[52]); + const __m256i cospim53 = _mm256_set1_epi32(-cospi[53]); + const __m256i cospim56 = _mm256_set1_epi32(-cospi[56]); + const __m256i cospim57 = _mm256_set1_epi32(-cospi[57]); + const __m256i cospim58 = _mm256_set1_epi32(-cospi[58]); + const __m256i cospim60 = _mm256_set1_epi32(-cospi[60]); + const __m256i cospim61 = _mm256_set1_epi32(-cospi[61]); + + { + __m256i u[64]; + __m256i tmp1, tmp2, tmp3, tmp4; + // stage 1 + u[0] = in[0]; + u[32] = in[1]; + u[36] = in[9]; + u[40] = in[5]; + u[44] = in[13]; + u[48] = in[3]; + u[52] = in[11]; + u[56] = in[7]; + u[60] = in[15]; + u[16] = in[2]; + u[20] = in[10]; + u[24] = in[6]; + u[28] = in[14]; + u[4] = in[8]; + u[8] = in[4]; + u[12] = in[12]; + + // stage 2 + u[63] = half_btf_0_avx2(&cospi1, &u[32], &rnding, bit); + u[32] = half_btf_0_avx2(&cospi63, &u[32], &rnding, bit); + u[35] = half_btf_0_avx2(&cospim49, &u[60], &rnding, bit); + u[60] = half_btf_0_avx2(&cospi15, &u[60], &rnding, bit); + u[59] = half_btf_0_avx2(&cospi9, &u[36], &rnding, bit); + u[36] = half_btf_0_avx2(&cospi55, &u[36], &rnding, bit); + u[39] = half_btf_0_avx2(&cospim57, &u[56], &rnding, bit); + u[56] = half_btf_0_avx2(&cospi7, &u[56], &rnding, bit); + u[55] = half_btf_0_avx2(&cospi5, &u[40], &rnding, bit); + u[40] = half_btf_0_avx2(&cospi59, &u[40], &rnding, bit); + u[43] = half_btf_0_avx2(&cospim53, &u[52], &rnding, bit); + u[52] = half_btf_0_avx2(&cospi11, &u[52], &rnding, bit); + u[47] = half_btf_0_avx2(&cospim61, &u[48], &rnding, bit); + u[48] = half_btf_0_avx2(&cospi3, &u[48], &rnding, bit); + u[51] = half_btf_0_avx2(&cospi13, &u[44], &rnding, bit); + u[44] = half_btf_0_avx2(&cospi51, &u[44], &rnding, bit); + + // stage 3 + u[31] = half_btf_0_avx2(&cospi2, &u[16], &rnding, bit); + u[16] = half_btf_0_avx2(&cospi62, &u[16], &rnding, bit); + u[19] = half_btf_0_avx2(&cospim50, &u[28], &rnding, bit); + u[28] = half_btf_0_avx2(&cospi14, &u[28], &rnding, bit); + u[27] = half_btf_0_avx2(&cospi10, &u[20], &rnding, bit); + u[20] = half_btf_0_avx2(&cospi54, &u[20], &rnding, bit); + u[23] = half_btf_0_avx2(&cospim58, &u[24], &rnding, bit); + u[24] = half_btf_0_avx2(&cospi6, &u[24], &rnding, bit); + u[33] = u[32]; + u[34] = u[35]; + u[37] = u[36]; + u[38] = u[39]; + u[41] = u[40]; + u[42] = u[43]; + u[45] = u[44]; + u[46] = u[47]; + u[49] = u[48]; + u[50] = u[51]; + u[53] = u[52]; + u[54] = u[55]; + u[57] = u[56]; + u[58] = u[59]; + u[61] = u[60]; + u[62] = u[63]; + + // stage 4 + u[15] = half_btf_0_avx2(&cospi4, &u[8], &rnding, bit); + u[8] = half_btf_0_avx2(&cospi60, &u[8], &rnding, bit); + u[11] = half_btf_0_avx2(&cospim52, &u[12], &rnding, bit); + u[12] = half_btf_0_avx2(&cospi12, &u[12], &rnding, bit); + + u[17] = u[16]; + u[18] = u[19]; + u[21] = u[20]; + u[22] = u[23]; + u[25] = u[24]; + u[26] = u[27]; + u[29] = u[28]; + u[30] = u[31]; + + tmp1 = half_btf_avx2(&cospim4, &u[33], &cospi60, &u[62], &rnding, bit); + tmp2 = half_btf_avx2(&cospim60, &u[34], &cospim4, &u[61], &rnding, bit); + tmp3 = half_btf_avx2(&cospim36, &u[37], &cospi28, &u[58], &rnding, bit); + tmp4 = half_btf_avx2(&cospim28, &u[38], &cospim36, &u[57], &rnding, bit); + u[57] = half_btf_avx2(&cospim36, &u[38], &cospi28, &u[57], &rnding, bit); + u[58] = half_btf_avx2(&cospi28, &u[37], &cospi36, &u[58], &rnding, bit); + u[61] = half_btf_avx2(&cospim4, &u[34], &cospi60, &u[61], &rnding, bit); + u[62] = half_btf_avx2(&cospi60, &u[33], &cospi4, &u[62], &rnding, bit); + u[33] = tmp1; + u[34] = tmp2; + u[37] = tmp3; + u[38] = tmp4; + + tmp1 = half_btf_avx2(&cospim20, &u[41], &cospi44, &u[54], &rnding, bit); + tmp2 = half_btf_avx2(&cospim44, &u[42], &cospim20, &u[53], &rnding, bit); + tmp3 = half_btf_avx2(&cospim52, &u[45], &cospi12, &u[50], &rnding, bit); + tmp4 = half_btf_avx2(&cospim12, &u[46], &cospim52, &u[49], &rnding, bit); + u[49] = half_btf_avx2(&cospim52, &u[46], &cospi12, &u[49], &rnding, bit); + u[50] = half_btf_avx2(&cospi12, &u[45], &cospi52, &u[50], &rnding, bit); + u[53] = half_btf_avx2(&cospim20, &u[42], &cospi44, &u[53], &rnding, bit); + u[54] = half_btf_avx2(&cospi44, &u[41], &cospi20, &u[54], &rnding, bit); + u[41] = tmp1; + u[42] = tmp2; + u[45] = tmp3; + u[46] = tmp4; + + // stage 5 + u[7] = half_btf_0_avx2(&cospi8, &u[4], &rnding, bit); + u[4] = half_btf_0_avx2(&cospi56, &u[4], &rnding, bit); + + u[9] = u[8]; + u[10] = u[11]; + u[13] = u[12]; + u[14] = u[15]; + + tmp1 = half_btf_avx2(&cospim8, &u[17], &cospi56, &u[30], &rnding, bit); + tmp2 = half_btf_avx2(&cospim56, &u[18], &cospim8, &u[29], &rnding, bit); + tmp3 = half_btf_avx2(&cospim40, &u[21], &cospi24, &u[26], &rnding, bit); + tmp4 = half_btf_avx2(&cospim24, &u[22], &cospim40, &u[25], &rnding, bit); + u[25] = half_btf_avx2(&cospim40, &u[22], &cospi24, &u[25], &rnding, bit); + u[26] = half_btf_avx2(&cospi24, &u[21], &cospi40, &u[26], &rnding, bit); + u[29] = half_btf_avx2(&cospim8, &u[18], &cospi56, &u[29], &rnding, bit); + u[30] = half_btf_avx2(&cospi56, &u[17], &cospi8, &u[30], &rnding, bit); + u[17] = tmp1; + u[18] = tmp2; + u[21] = tmp3; + u[22] = tmp4; + + for (i = 32; i < 64; i += 8) { + addsub_avx2(u[i + 0], u[i + 3], &u[i + 0], &u[i + 3], &clamp_lo, + &clamp_hi); + addsub_avx2(u[i + 1], u[i + 2], &u[i + 1], &u[i + 2], &clamp_lo, + &clamp_hi); + + addsub_avx2(u[i + 7], u[i + 4], &u[i + 7], &u[i + 4], &clamp_lo, + &clamp_hi); + addsub_avx2(u[i + 6], u[i + 5], &u[i + 6], &u[i + 5], &clamp_lo, + &clamp_hi); + } + + // stage 6 + tmp1 = half_btf_0_avx2(&cospi32, &u[0], &rnding, bit); + u[1] = half_btf_0_avx2(&cospi32, &u[0], &rnding, bit); + u[0] = tmp1; + u[5] = u[4]; + u[6] = u[7]; + + tmp1 = half_btf_avx2(&cospim16, &u[9], &cospi48, &u[14], &rnding, bit); + u[14] = half_btf_avx2(&cospi48, &u[9], &cospi16, &u[14], &rnding, bit); + u[9] = tmp1; + tmp2 = half_btf_avx2(&cospim48, &u[10], &cospim16, &u[13], &rnding, bit); + u[13] = half_btf_avx2(&cospim16, &u[10], &cospi48, &u[13], &rnding, bit); + u[10] = tmp2; + + for (i = 16; i < 32; i += 8) { + addsub_avx2(u[i + 0], u[i + 3], &u[i + 0], &u[i + 3], &clamp_lo, + &clamp_hi); + addsub_avx2(u[i + 1], u[i + 2], &u[i + 1], &u[i + 2], &clamp_lo, + &clamp_hi); + + addsub_avx2(u[i + 7], u[i + 4], &u[i + 7], &u[i + 4], &clamp_lo, + &clamp_hi); + addsub_avx2(u[i + 6], u[i + 5], &u[i + 6], &u[i + 5], &clamp_lo, + &clamp_hi); + } + + tmp1 = half_btf_avx2(&cospim8, &u[34], &cospi56, &u[61], &rnding, bit); + tmp2 = half_btf_avx2(&cospim8, &u[35], &cospi56, &u[60], &rnding, bit); + tmp3 = half_btf_avx2(&cospim56, &u[36], &cospim8, &u[59], &rnding, bit); + tmp4 = half_btf_avx2(&cospim56, &u[37], &cospim8, &u[58], &rnding, bit); + u[58] = half_btf_avx2(&cospim8, &u[37], &cospi56, &u[58], &rnding, bit); + u[59] = half_btf_avx2(&cospim8, &u[36], &cospi56, &u[59], &rnding, bit); + u[60] = half_btf_avx2(&cospi56, &u[35], &cospi8, &u[60], &rnding, bit); + u[61] = half_btf_avx2(&cospi56, &u[34], &cospi8, &u[61], &rnding, bit); + u[34] = tmp1; + u[35] = tmp2; + u[36] = tmp3; + u[37] = tmp4; + + tmp1 = half_btf_avx2(&cospim40, &u[42], &cospi24, &u[53], &rnding, bit); + tmp2 = half_btf_avx2(&cospim40, &u[43], &cospi24, &u[52], &rnding, bit); + tmp3 = half_btf_avx2(&cospim24, &u[44], &cospim40, &u[51], &rnding, bit); + tmp4 = half_btf_avx2(&cospim24, &u[45], &cospim40, &u[50], &rnding, bit); + u[50] = half_btf_avx2(&cospim40, &u[45], &cospi24, &u[50], &rnding, bit); + u[51] = half_btf_avx2(&cospim40, &u[44], &cospi24, &u[51], &rnding, bit); + u[52] = half_btf_avx2(&cospi24, &u[43], &cospi40, &u[52], &rnding, bit); + u[53] = half_btf_avx2(&cospi24, &u[42], &cospi40, &u[53], &rnding, bit); + u[42] = tmp1; + u[43] = tmp2; + u[44] = tmp3; + u[45] = tmp4; + + // stage 7 + u[3] = u[0]; + u[2] = u[1]; + tmp1 = half_btf_avx2(&cospim32, &u[5], &cospi32, &u[6], &rnding, bit); + u[6] = half_btf_avx2(&cospi32, &u[5], &cospi32, &u[6], &rnding, bit); + u[5] = tmp1; + addsub_avx2(u[8], u[11], &u[8], &u[11], &clamp_lo, &clamp_hi); + addsub_avx2(u[9], u[10], &u[9], &u[10], &clamp_lo, &clamp_hi); + addsub_avx2(u[15], u[12], &u[15], &u[12], &clamp_lo, &clamp_hi); + addsub_avx2(u[14], u[13], &u[14], &u[13], &clamp_lo, &clamp_hi); + + tmp1 = half_btf_avx2(&cospim16, &u[18], &cospi48, &u[29], &rnding, bit); + tmp2 = half_btf_avx2(&cospim16, &u[19], &cospi48, &u[28], &rnding, bit); + tmp3 = half_btf_avx2(&cospim48, &u[20], &cospim16, &u[27], &rnding, bit); + tmp4 = half_btf_avx2(&cospim48, &u[21], &cospim16, &u[26], &rnding, bit); + u[26] = half_btf_avx2(&cospim16, &u[21], &cospi48, &u[26], &rnding, bit); + u[27] = half_btf_avx2(&cospim16, &u[20], &cospi48, &u[27], &rnding, bit); + u[28] = half_btf_avx2(&cospi48, &u[19], &cospi16, &u[28], &rnding, bit); + u[29] = half_btf_avx2(&cospi48, &u[18], &cospi16, &u[29], &rnding, bit); + u[18] = tmp1; + u[19] = tmp2; + u[20] = tmp3; + u[21] = tmp4; + + for (i = 32; i < 64; i += 16) { + for (j = i; j < i + 4; j++) { + addsub_avx2(u[j], u[j ^ 7], &u[j], &u[j ^ 7], &clamp_lo, &clamp_hi); + addsub_avx2(u[j ^ 15], u[j ^ 8], &u[j ^ 15], &u[j ^ 8], &clamp_lo, + &clamp_hi); + } + } + + // stage 8 + for (i = 0; i < 4; ++i) { + addsub_avx2(u[i], u[7 - i], &u[i], &u[7 - i], &clamp_lo, &clamp_hi); + } + + idct64_stage8_avx2(u, &cospim32, &cospi32, &cospim16, &cospi48, &cospi16, + &cospim48, &clamp_lo, &clamp_hi, &rnding, bit); + + // stage 9 + idct64_stage9_avx2(u, &cospim32, &cospi32, &clamp_lo, &clamp_hi, &rnding, + bit); + + // stage 10 + idct64_stage10_avx2(u, &cospim32, &cospi32, &clamp_lo, &clamp_hi, &rnding, + bit); + + // stage 11 + idct64_stage11_avx2(u, out, do_cols, bd, out_shift, log_range); + } +} +static void idct64_avx2(__m256i *in, __m256i *out, int bit, int do_cols, int bd, + int out_shift) { + int i, j; + const int32_t *cospi = cospi_arr(bit); + const __m256i rnding = _mm256_set1_epi32(1 << (bit - 1)); + const int log_range = AOMMAX(16, bd + (do_cols ? 6 : 8)); + const __m256i clamp_lo = _mm256_set1_epi32(-(1 << (log_range - 1))); + const __m256i clamp_hi = _mm256_set1_epi32((1 << (log_range - 1)) - 1); + + const __m256i cospi1 = _mm256_set1_epi32(cospi[1]); + const __m256i cospi2 = _mm256_set1_epi32(cospi[2]); + const __m256i cospi3 = _mm256_set1_epi32(cospi[3]); + const __m256i cospi4 = _mm256_set1_epi32(cospi[4]); + const __m256i cospi5 = _mm256_set1_epi32(cospi[5]); + const __m256i cospi6 = _mm256_set1_epi32(cospi[6]); + const __m256i cospi7 = _mm256_set1_epi32(cospi[7]); + const __m256i cospi8 = _mm256_set1_epi32(cospi[8]); + const __m256i cospi9 = _mm256_set1_epi32(cospi[9]); + const __m256i cospi10 = _mm256_set1_epi32(cospi[10]); + const __m256i cospi11 = _mm256_set1_epi32(cospi[11]); + const __m256i cospi12 = _mm256_set1_epi32(cospi[12]); + const __m256i cospi13 = _mm256_set1_epi32(cospi[13]); + const __m256i cospi14 = _mm256_set1_epi32(cospi[14]); + const __m256i cospi15 = _mm256_set1_epi32(cospi[15]); + const __m256i cospi16 = _mm256_set1_epi32(cospi[16]); + const __m256i cospi17 = _mm256_set1_epi32(cospi[17]); + const __m256i cospi18 = _mm256_set1_epi32(cospi[18]); + const __m256i cospi19 = _mm256_set1_epi32(cospi[19]); + const __m256i cospi20 = _mm256_set1_epi32(cospi[20]); + const __m256i cospi21 = _mm256_set1_epi32(cospi[21]); + const __m256i cospi22 = _mm256_set1_epi32(cospi[22]); + const __m256i cospi23 = _mm256_set1_epi32(cospi[23]); + const __m256i cospi24 = _mm256_set1_epi32(cospi[24]); + const __m256i cospi25 = _mm256_set1_epi32(cospi[25]); + const __m256i cospi26 = _mm256_set1_epi32(cospi[26]); + const __m256i cospi27 = _mm256_set1_epi32(cospi[27]); + const __m256i cospi28 = _mm256_set1_epi32(cospi[28]); + const __m256i cospi29 = _mm256_set1_epi32(cospi[29]); + const __m256i cospi30 = _mm256_set1_epi32(cospi[30]); + const __m256i cospi31 = _mm256_set1_epi32(cospi[31]); + const __m256i cospi32 = _mm256_set1_epi32(cospi[32]); + const __m256i cospi35 = _mm256_set1_epi32(cospi[35]); + const __m256i cospi36 = _mm256_set1_epi32(cospi[36]); + const __m256i cospi38 = _mm256_set1_epi32(cospi[38]); + const __m256i cospi39 = _mm256_set1_epi32(cospi[39]); + const __m256i cospi40 = _mm256_set1_epi32(cospi[40]); + const __m256i cospi43 = _mm256_set1_epi32(cospi[43]); + const __m256i cospi44 = _mm256_set1_epi32(cospi[44]); + const __m256i cospi46 = _mm256_set1_epi32(cospi[46]); + const __m256i cospi47 = _mm256_set1_epi32(cospi[47]); + const __m256i cospi48 = _mm256_set1_epi32(cospi[48]); + const __m256i cospi51 = _mm256_set1_epi32(cospi[51]); + const __m256i cospi52 = _mm256_set1_epi32(cospi[52]); + const __m256i cospi54 = _mm256_set1_epi32(cospi[54]); + const __m256i cospi55 = _mm256_set1_epi32(cospi[55]); + const __m256i cospi56 = _mm256_set1_epi32(cospi[56]); + const __m256i cospi59 = _mm256_set1_epi32(cospi[59]); + const __m256i cospi60 = _mm256_set1_epi32(cospi[60]); + const __m256i cospi62 = _mm256_set1_epi32(cospi[62]); + const __m256i cospi63 = _mm256_set1_epi32(cospi[63]); + + const __m256i cospim4 = _mm256_set1_epi32(-cospi[4]); + const __m256i cospim8 = _mm256_set1_epi32(-cospi[8]); + const __m256i cospim12 = _mm256_set1_epi32(-cospi[12]); + const __m256i cospim16 = _mm256_set1_epi32(-cospi[16]); + const __m256i cospim20 = _mm256_set1_epi32(-cospi[20]); + const __m256i cospim24 = _mm256_set1_epi32(-cospi[24]); + const __m256i cospim28 = _mm256_set1_epi32(-cospi[28]); + const __m256i cospim32 = _mm256_set1_epi32(-cospi[32]); + const __m256i cospim33 = _mm256_set1_epi32(-cospi[33]); + const __m256i cospim34 = _mm256_set1_epi32(-cospi[34]); + const __m256i cospim36 = _mm256_set1_epi32(-cospi[36]); + const __m256i cospim37 = _mm256_set1_epi32(-cospi[37]); + const __m256i cospim40 = _mm256_set1_epi32(-cospi[40]); + const __m256i cospim41 = _mm256_set1_epi32(-cospi[41]); + const __m256i cospim42 = _mm256_set1_epi32(-cospi[42]); + const __m256i cospim44 = _mm256_set1_epi32(-cospi[44]); + const __m256i cospim45 = _mm256_set1_epi32(-cospi[45]); + const __m256i cospim48 = _mm256_set1_epi32(-cospi[48]); + const __m256i cospim49 = _mm256_set1_epi32(-cospi[49]); + const __m256i cospim50 = _mm256_set1_epi32(-cospi[50]); + const __m256i cospim52 = _mm256_set1_epi32(-cospi[52]); + const __m256i cospim53 = _mm256_set1_epi32(-cospi[53]); + const __m256i cospim56 = _mm256_set1_epi32(-cospi[56]); + const __m256i cospim57 = _mm256_set1_epi32(-cospi[57]); + const __m256i cospim58 = _mm256_set1_epi32(-cospi[58]); + const __m256i cospim60 = _mm256_set1_epi32(-cospi[60]); + const __m256i cospim61 = _mm256_set1_epi32(-cospi[61]); + + { + __m256i u[64], v[64]; + + // stage 1 + u[32] = in[1]; + u[34] = in[17]; + u[36] = in[9]; + u[38] = in[25]; + u[40] = in[5]; + u[42] = in[21]; + u[44] = in[13]; + u[46] = in[29]; + u[48] = in[3]; + u[50] = in[19]; + u[52] = in[11]; + u[54] = in[27]; + u[56] = in[7]; + u[58] = in[23]; + u[60] = in[15]; + u[62] = in[31]; + + v[16] = in[2]; + v[18] = in[18]; + v[20] = in[10]; + v[22] = in[26]; + v[24] = in[6]; + v[26] = in[22]; + v[28] = in[14]; + v[30] = in[30]; + + u[8] = in[4]; + u[10] = in[20]; + u[12] = in[12]; + u[14] = in[28]; + + v[4] = in[8]; + v[6] = in[24]; + + u[0] = in[0]; + u[2] = in[16]; + + // stage 2 + v[32] = half_btf_0_avx2(&cospi63, &u[32], &rnding, bit); + v[33] = half_btf_0_avx2(&cospim33, &u[62], &rnding, bit); + v[34] = half_btf_0_avx2(&cospi47, &u[34], &rnding, bit); + v[35] = half_btf_0_avx2(&cospim49, &u[60], &rnding, bit); + v[36] = half_btf_0_avx2(&cospi55, &u[36], &rnding, bit); + v[37] = half_btf_0_avx2(&cospim41, &u[58], &rnding, bit); + v[38] = half_btf_0_avx2(&cospi39, &u[38], &rnding, bit); + v[39] = half_btf_0_avx2(&cospim57, &u[56], &rnding, bit); + v[40] = half_btf_0_avx2(&cospi59, &u[40], &rnding, bit); + v[41] = half_btf_0_avx2(&cospim37, &u[54], &rnding, bit); + v[42] = half_btf_0_avx2(&cospi43, &u[42], &rnding, bit); + v[43] = half_btf_0_avx2(&cospim53, &u[52], &rnding, bit); + v[44] = half_btf_0_avx2(&cospi51, &u[44], &rnding, bit); + v[45] = half_btf_0_avx2(&cospim45, &u[50], &rnding, bit); + v[46] = half_btf_0_avx2(&cospi35, &u[46], &rnding, bit); + v[47] = half_btf_0_avx2(&cospim61, &u[48], &rnding, bit); + v[48] = half_btf_0_avx2(&cospi3, &u[48], &rnding, bit); + v[49] = half_btf_0_avx2(&cospi29, &u[46], &rnding, bit); + v[50] = half_btf_0_avx2(&cospi19, &u[50], &rnding, bit); + v[51] = half_btf_0_avx2(&cospi13, &u[44], &rnding, bit); + v[52] = half_btf_0_avx2(&cospi11, &u[52], &rnding, bit); + v[53] = half_btf_0_avx2(&cospi21, &u[42], &rnding, bit); + v[54] = half_btf_0_avx2(&cospi27, &u[54], &rnding, bit); + v[55] = half_btf_0_avx2(&cospi5, &u[40], &rnding, bit); + v[56] = half_btf_0_avx2(&cospi7, &u[56], &rnding, bit); + v[57] = half_btf_0_avx2(&cospi25, &u[38], &rnding, bit); + v[58] = half_btf_0_avx2(&cospi23, &u[58], &rnding, bit); + v[59] = half_btf_0_avx2(&cospi9, &u[36], &rnding, bit); + v[60] = half_btf_0_avx2(&cospi15, &u[60], &rnding, bit); + v[61] = half_btf_0_avx2(&cospi17, &u[34], &rnding, bit); + v[62] = half_btf_0_avx2(&cospi31, &u[62], &rnding, bit); + v[63] = half_btf_0_avx2(&cospi1, &u[32], &rnding, bit); + + // stage 3 + u[16] = half_btf_0_avx2(&cospi62, &v[16], &rnding, bit); + u[17] = half_btf_0_avx2(&cospim34, &v[30], &rnding, bit); + u[18] = half_btf_0_avx2(&cospi46, &v[18], &rnding, bit); + u[19] = half_btf_0_avx2(&cospim50, &v[28], &rnding, bit); + u[20] = half_btf_0_avx2(&cospi54, &v[20], &rnding, bit); + u[21] = half_btf_0_avx2(&cospim42, &v[26], &rnding, bit); + u[22] = half_btf_0_avx2(&cospi38, &v[22], &rnding, bit); + u[23] = half_btf_0_avx2(&cospim58, &v[24], &rnding, bit); + u[24] = half_btf_0_avx2(&cospi6, &v[24], &rnding, bit); + u[25] = half_btf_0_avx2(&cospi26, &v[22], &rnding, bit); + u[26] = half_btf_0_avx2(&cospi22, &v[26], &rnding, bit); + u[27] = half_btf_0_avx2(&cospi10, &v[20], &rnding, bit); + u[28] = half_btf_0_avx2(&cospi14, &v[28], &rnding, bit); + u[29] = half_btf_0_avx2(&cospi18, &v[18], &rnding, bit); + u[30] = half_btf_0_avx2(&cospi30, &v[30], &rnding, bit); + u[31] = half_btf_0_avx2(&cospi2, &v[16], &rnding, bit); + + for (i = 32; i < 64; i += 4) { + addsub_avx2(v[i + 0], v[i + 1], &u[i + 0], &u[i + 1], &clamp_lo, + &clamp_hi); + addsub_avx2(v[i + 3], v[i + 2], &u[i + 3], &u[i + 2], &clamp_lo, + &clamp_hi); + } + + // stage 4 + v[8] = half_btf_0_avx2(&cospi60, &u[8], &rnding, bit); + v[9] = half_btf_0_avx2(&cospim36, &u[14], &rnding, bit); + v[10] = half_btf_0_avx2(&cospi44, &u[10], &rnding, bit); + v[11] = half_btf_0_avx2(&cospim52, &u[12], &rnding, bit); + v[12] = half_btf_0_avx2(&cospi12, &u[12], &rnding, bit); + v[13] = half_btf_0_avx2(&cospi20, &u[10], &rnding, bit); + v[14] = half_btf_0_avx2(&cospi28, &u[14], &rnding, bit); + v[15] = half_btf_0_avx2(&cospi4, &u[8], &rnding, bit); + + for (i = 16; i < 32; i += 4) { + addsub_avx2(u[i + 0], u[i + 1], &v[i + 0], &v[i + 1], &clamp_lo, + &clamp_hi); + addsub_avx2(u[i + 3], u[i + 2], &v[i + 3], &v[i + 2], &clamp_lo, + &clamp_hi); + } + + for (i = 32; i < 64; i += 4) { + v[i + 0] = u[i + 0]; + v[i + 3] = u[i + 3]; + } + + v[33] = half_btf_avx2(&cospim4, &u[33], &cospi60, &u[62], &rnding, bit); + v[34] = half_btf_avx2(&cospim60, &u[34], &cospim4, &u[61], &rnding, bit); + v[37] = half_btf_avx2(&cospim36, &u[37], &cospi28, &u[58], &rnding, bit); + v[38] = half_btf_avx2(&cospim28, &u[38], &cospim36, &u[57], &rnding, bit); + v[41] = half_btf_avx2(&cospim20, &u[41], &cospi44, &u[54], &rnding, bit); + v[42] = half_btf_avx2(&cospim44, &u[42], &cospim20, &u[53], &rnding, bit); + v[45] = half_btf_avx2(&cospim52, &u[45], &cospi12, &u[50], &rnding, bit); + v[46] = half_btf_avx2(&cospim12, &u[46], &cospim52, &u[49], &rnding, bit); + v[49] = half_btf_avx2(&cospim52, &u[46], &cospi12, &u[49], &rnding, bit); + v[50] = half_btf_avx2(&cospi12, &u[45], &cospi52, &u[50], &rnding, bit); + v[53] = half_btf_avx2(&cospim20, &u[42], &cospi44, &u[53], &rnding, bit); + v[54] = half_btf_avx2(&cospi44, &u[41], &cospi20, &u[54], &rnding, bit); + v[57] = half_btf_avx2(&cospim36, &u[38], &cospi28, &u[57], &rnding, bit); + v[58] = half_btf_avx2(&cospi28, &u[37], &cospi36, &u[58], &rnding, bit); + v[61] = half_btf_avx2(&cospim4, &u[34], &cospi60, &u[61], &rnding, bit); + v[62] = half_btf_avx2(&cospi60, &u[33], &cospi4, &u[62], &rnding, bit); + + // stage 5 + u[4] = half_btf_0_avx2(&cospi56, &v[4], &rnding, bit); + u[5] = half_btf_0_avx2(&cospim40, &v[6], &rnding, bit); + u[6] = half_btf_0_avx2(&cospi24, &v[6], &rnding, bit); + u[7] = half_btf_0_avx2(&cospi8, &v[4], &rnding, bit); + + for (i = 8; i < 16; i += 4) { + addsub_avx2(v[i + 0], v[i + 1], &u[i + 0], &u[i + 1], &clamp_lo, + &clamp_hi); + addsub_avx2(v[i + 3], v[i + 2], &u[i + 3], &u[i + 2], &clamp_lo, + &clamp_hi); + } + + for (i = 16; i < 32; i += 4) { + u[i + 0] = v[i + 0]; + u[i + 3] = v[i + 3]; + } + + u[17] = half_btf_avx2(&cospim8, &v[17], &cospi56, &v[30], &rnding, bit); + u[18] = half_btf_avx2(&cospim56, &v[18], &cospim8, &v[29], &rnding, bit); + u[21] = half_btf_avx2(&cospim40, &v[21], &cospi24, &v[26], &rnding, bit); + u[22] = half_btf_avx2(&cospim24, &v[22], &cospim40, &v[25], &rnding, bit); + u[25] = half_btf_avx2(&cospim40, &v[22], &cospi24, &v[25], &rnding, bit); + u[26] = half_btf_avx2(&cospi24, &v[21], &cospi40, &v[26], &rnding, bit); + u[29] = half_btf_avx2(&cospim8, &v[18], &cospi56, &v[29], &rnding, bit); + u[30] = half_btf_avx2(&cospi56, &v[17], &cospi8, &v[30], &rnding, bit); + + for (i = 32; i < 64; i += 8) { + addsub_avx2(v[i + 0], v[i + 3], &u[i + 0], &u[i + 3], &clamp_lo, + &clamp_hi); + addsub_avx2(v[i + 1], v[i + 2], &u[i + 1], &u[i + 2], &clamp_lo, + &clamp_hi); + + addsub_avx2(v[i + 7], v[i + 4], &u[i + 7], &u[i + 4], &clamp_lo, + &clamp_hi); + addsub_avx2(v[i + 6], v[i + 5], &u[i + 6], &u[i + 5], &clamp_lo, + &clamp_hi); + } + + // stage 6 + v[0] = half_btf_0_avx2(&cospi32, &u[0], &rnding, bit); + v[1] = half_btf_0_avx2(&cospi32, &u[0], &rnding, bit); + v[2] = half_btf_0_avx2(&cospi48, &u[2], &rnding, bit); + v[3] = half_btf_0_avx2(&cospi16, &u[2], &rnding, bit); + + addsub_avx2(u[4], u[5], &v[4], &v[5], &clamp_lo, &clamp_hi); + addsub_avx2(u[7], u[6], &v[7], &v[6], &clamp_lo, &clamp_hi); + + for (i = 8; i < 16; i += 4) { + v[i + 0] = u[i + 0]; + v[i + 3] = u[i + 3]; + } + + v[9] = half_btf_avx2(&cospim16, &u[9], &cospi48, &u[14], &rnding, bit); + v[10] = half_btf_avx2(&cospim48, &u[10], &cospim16, &u[13], &rnding, bit); + v[13] = half_btf_avx2(&cospim16, &u[10], &cospi48, &u[13], &rnding, bit); + v[14] = half_btf_avx2(&cospi48, &u[9], &cospi16, &u[14], &rnding, bit); + + for (i = 16; i < 32; i += 8) { + addsub_avx2(u[i + 0], u[i + 3], &v[i + 0], &v[i + 3], &clamp_lo, + &clamp_hi); + addsub_avx2(u[i + 1], u[i + 2], &v[i + 1], &v[i + 2], &clamp_lo, + &clamp_hi); + + addsub_avx2(u[i + 7], u[i + 4], &v[i + 7], &v[i + 4], &clamp_lo, + &clamp_hi); + addsub_avx2(u[i + 6], u[i + 5], &v[i + 6], &v[i + 5], &clamp_lo, + &clamp_hi); + } + + for (i = 32; i < 64; i += 8) { + v[i + 0] = u[i + 0]; + v[i + 1] = u[i + 1]; + v[i + 6] = u[i + 6]; + v[i + 7] = u[i + 7]; + } + + v[34] = half_btf_avx2(&cospim8, &u[34], &cospi56, &u[61], &rnding, bit); + v[35] = half_btf_avx2(&cospim8, &u[35], &cospi56, &u[60], &rnding, bit); + v[36] = half_btf_avx2(&cospim56, &u[36], &cospim8, &u[59], &rnding, bit); + v[37] = half_btf_avx2(&cospim56, &u[37], &cospim8, &u[58], &rnding, bit); + v[42] = half_btf_avx2(&cospim40, &u[42], &cospi24, &u[53], &rnding, bit); + v[43] = half_btf_avx2(&cospim40, &u[43], &cospi24, &u[52], &rnding, bit); + v[44] = half_btf_avx2(&cospim24, &u[44], &cospim40, &u[51], &rnding, bit); + v[45] = half_btf_avx2(&cospim24, &u[45], &cospim40, &u[50], &rnding, bit); + v[50] = half_btf_avx2(&cospim40, &u[45], &cospi24, &u[50], &rnding, bit); + v[51] = half_btf_avx2(&cospim40, &u[44], &cospi24, &u[51], &rnding, bit); + v[52] = half_btf_avx2(&cospi24, &u[43], &cospi40, &u[52], &rnding, bit); + v[53] = half_btf_avx2(&cospi24, &u[42], &cospi40, &u[53], &rnding, bit); + v[58] = half_btf_avx2(&cospim8, &u[37], &cospi56, &u[58], &rnding, bit); + v[59] = half_btf_avx2(&cospim8, &u[36], &cospi56, &u[59], &rnding, bit); + v[60] = half_btf_avx2(&cospi56, &u[35], &cospi8, &u[60], &rnding, bit); + v[61] = half_btf_avx2(&cospi56, &u[34], &cospi8, &u[61], &rnding, bit); + + // stage 7 + addsub_avx2(v[0], v[3], &u[0], &u[3], &clamp_lo, &clamp_hi); + addsub_avx2(v[1], v[2], &u[1], &u[2], &clamp_lo, &clamp_hi); + + u[4] = v[4]; + u[7] = v[7]; + u[5] = half_btf_avx2(&cospim32, &v[5], &cospi32, &v[6], &rnding, bit); + u[6] = half_btf_avx2(&cospi32, &v[5], &cospi32, &v[6], &rnding, bit); + + addsub_avx2(v[8], v[11], &u[8], &u[11], &clamp_lo, &clamp_hi); + addsub_avx2(v[9], v[10], &u[9], &u[10], &clamp_lo, &clamp_hi); + addsub_avx2(v[15], v[12], &u[15], &u[12], &clamp_lo, &clamp_hi); + addsub_avx2(v[14], v[13], &u[14], &u[13], &clamp_lo, &clamp_hi); + + for (i = 16; i < 32; i += 8) { + u[i + 0] = v[i + 0]; + u[i + 1] = v[i + 1]; + u[i + 6] = v[i + 6]; + u[i + 7] = v[i + 7]; + } + + u[18] = half_btf_avx2(&cospim16, &v[18], &cospi48, &v[29], &rnding, bit); + u[19] = half_btf_avx2(&cospim16, &v[19], &cospi48, &v[28], &rnding, bit); + u[20] = half_btf_avx2(&cospim48, &v[20], &cospim16, &v[27], &rnding, bit); + u[21] = half_btf_avx2(&cospim48, &v[21], &cospim16, &v[26], &rnding, bit); + u[26] = half_btf_avx2(&cospim16, &v[21], &cospi48, &v[26], &rnding, bit); + u[27] = half_btf_avx2(&cospim16, &v[20], &cospi48, &v[27], &rnding, bit); + u[28] = half_btf_avx2(&cospi48, &v[19], &cospi16, &v[28], &rnding, bit); + u[29] = half_btf_avx2(&cospi48, &v[18], &cospi16, &v[29], &rnding, bit); + + for (i = 32; i < 64; i += 16) { + for (j = i; j < i + 4; j++) { + addsub_avx2(v[j], v[j ^ 7], &u[j], &u[j ^ 7], &clamp_lo, &clamp_hi); + addsub_avx2(v[j ^ 15], v[j ^ 8], &u[j ^ 15], &u[j ^ 8], &clamp_lo, + &clamp_hi); + } + } + + // stage 8 + for (i = 0; i < 4; ++i) { + addsub_avx2(u[i], u[7 - i], &v[i], &v[7 - i], &clamp_lo, &clamp_hi); + } + + v[8] = u[8]; + v[9] = u[9]; + v[14] = u[14]; + v[15] = u[15]; + + v[10] = half_btf_avx2(&cospim32, &u[10], &cospi32, &u[13], &rnding, bit); + v[11] = half_btf_avx2(&cospim32, &u[11], &cospi32, &u[12], &rnding, bit); + v[12] = half_btf_avx2(&cospi32, &u[11], &cospi32, &u[12], &rnding, bit); + v[13] = half_btf_avx2(&cospi32, &u[10], &cospi32, &u[13], &rnding, bit); + + for (i = 16; i < 20; ++i) { + addsub_avx2(u[i], u[i ^ 7], &v[i], &v[i ^ 7], &clamp_lo, &clamp_hi); + addsub_avx2(u[i ^ 15], u[i ^ 8], &v[i ^ 15], &v[i ^ 8], &clamp_lo, + &clamp_hi); + } + + for (i = 32; i < 36; ++i) { + v[i] = u[i]; + v[i + 12] = u[i + 12]; + v[i + 16] = u[i + 16]; + v[i + 28] = u[i + 28]; + } + + v[36] = half_btf_avx2(&cospim16, &u[36], &cospi48, &u[59], &rnding, bit); + v[37] = half_btf_avx2(&cospim16, &u[37], &cospi48, &u[58], &rnding, bit); + v[38] = half_btf_avx2(&cospim16, &u[38], &cospi48, &u[57], &rnding, bit); + v[39] = half_btf_avx2(&cospim16, &u[39], &cospi48, &u[56], &rnding, bit); + v[40] = half_btf_avx2(&cospim48, &u[40], &cospim16, &u[55], &rnding, bit); + v[41] = half_btf_avx2(&cospim48, &u[41], &cospim16, &u[54], &rnding, bit); + v[42] = half_btf_avx2(&cospim48, &u[42], &cospim16, &u[53], &rnding, bit); + v[43] = half_btf_avx2(&cospim48, &u[43], &cospim16, &u[52], &rnding, bit); + v[52] = half_btf_avx2(&cospim16, &u[43], &cospi48, &u[52], &rnding, bit); + v[53] = half_btf_avx2(&cospim16, &u[42], &cospi48, &u[53], &rnding, bit); + v[54] = half_btf_avx2(&cospim16, &u[41], &cospi48, &u[54], &rnding, bit); + v[55] = half_btf_avx2(&cospim16, &u[40], &cospi48, &u[55], &rnding, bit); + v[56] = half_btf_avx2(&cospi48, &u[39], &cospi16, &u[56], &rnding, bit); + v[57] = half_btf_avx2(&cospi48, &u[38], &cospi16, &u[57], &rnding, bit); + v[58] = half_btf_avx2(&cospi48, &u[37], &cospi16, &u[58], &rnding, bit); + v[59] = half_btf_avx2(&cospi48, &u[36], &cospi16, &u[59], &rnding, bit); + + // stage 9 + for (i = 0; i < 8; ++i) { + addsub_avx2(v[i], v[15 - i], &u[i], &u[15 - i], &clamp_lo, &clamp_hi); + } + + for (i = 16; i < 20; ++i) { + u[i] = v[i]; + u[i + 12] = v[i + 12]; + } + + u[20] = half_btf_avx2(&cospim32, &v[20], &cospi32, &v[27], &rnding, bit); + u[21] = half_btf_avx2(&cospim32, &v[21], &cospi32, &v[26], &rnding, bit); + u[22] = half_btf_avx2(&cospim32, &v[22], &cospi32, &v[25], &rnding, bit); + u[23] = half_btf_avx2(&cospim32, &v[23], &cospi32, &v[24], &rnding, bit); + u[24] = half_btf_avx2(&cospi32, &v[23], &cospi32, &v[24], &rnding, bit); + u[25] = half_btf_avx2(&cospi32, &v[22], &cospi32, &v[25], &rnding, bit); + u[26] = half_btf_avx2(&cospi32, &v[21], &cospi32, &v[26], &rnding, bit); + u[27] = half_btf_avx2(&cospi32, &v[20], &cospi32, &v[27], &rnding, bit); + + for (i = 32; i < 40; i++) { + addsub_avx2(v[i], v[i ^ 15], &u[i], &u[i ^ 15], &clamp_lo, &clamp_hi); + } + + for (i = 48; i < 56; i++) { + addsub_avx2(v[i ^ 15], v[i], &u[i ^ 15], &u[i], &clamp_lo, &clamp_hi); + } + + // stage 10 + for (i = 0; i < 16; i++) { + addsub_avx2(u[i], u[31 - i], &v[i], &v[31 - i], &clamp_lo, &clamp_hi); + } + + for (i = 32; i < 40; i++) v[i] = u[i]; + + v[40] = half_btf_avx2(&cospim32, &u[40], &cospi32, &u[55], &rnding, bit); + v[41] = half_btf_avx2(&cospim32, &u[41], &cospi32, &u[54], &rnding, bit); + v[42] = half_btf_avx2(&cospim32, &u[42], &cospi32, &u[53], &rnding, bit); + v[43] = half_btf_avx2(&cospim32, &u[43], &cospi32, &u[52], &rnding, bit); + v[44] = half_btf_avx2(&cospim32, &u[44], &cospi32, &u[51], &rnding, bit); + v[45] = half_btf_avx2(&cospim32, &u[45], &cospi32, &u[50], &rnding, bit); + v[46] = half_btf_avx2(&cospim32, &u[46], &cospi32, &u[49], &rnding, bit); + v[47] = half_btf_avx2(&cospim32, &u[47], &cospi32, &u[48], &rnding, bit); + v[48] = half_btf_avx2(&cospi32, &u[47], &cospi32, &u[48], &rnding, bit); + v[49] = half_btf_avx2(&cospi32, &u[46], &cospi32, &u[49], &rnding, bit); + v[50] = half_btf_avx2(&cospi32, &u[45], &cospi32, &u[50], &rnding, bit); + v[51] = half_btf_avx2(&cospi32, &u[44], &cospi32, &u[51], &rnding, bit); + v[52] = half_btf_avx2(&cospi32, &u[43], &cospi32, &u[52], &rnding, bit); + v[53] = half_btf_avx2(&cospi32, &u[42], &cospi32, &u[53], &rnding, bit); + v[54] = half_btf_avx2(&cospi32, &u[41], &cospi32, &u[54], &rnding, bit); + v[55] = half_btf_avx2(&cospi32, &u[40], &cospi32, &u[55], &rnding, bit); + + for (i = 56; i < 64; i++) v[i] = u[i]; + + // stage 11 + if (do_cols) { + for (i = 0; i < 32; i++) { + addsub_no_clamp_avx2(v[i], v[63 - i], &out[(i)], &out[(63 - i)]); + } + } else { + const int log_range_out = AOMMAX(16, bd + 6); + const __m256i clamp_lo_out = _mm256_set1_epi32(AOMMAX( + -(1 << (log_range_out - 1)), -(1 << (log_range - 1 - out_shift)))); + const __m256i clamp_hi_out = _mm256_set1_epi32(AOMMIN( + (1 << (log_range_out - 1)) - 1, (1 << (log_range - 1 - out_shift)))); + + for (i = 0; i < 32; i++) { + addsub_shift_avx2(v[i], v[63 - i], &out[(i)], &out[(63 - i)], + &clamp_lo_out, &clamp_hi_out, out_shift); + } + } + } +} typedef void (*transform_1d_avx2)(__m256i *in, __m256i *out, int bit, int do_cols, int bd, int out_shift); @@ -1164,19 +4192,21 @@ static const transform_1d_avx2 { NULL, NULL, NULL, NULL }, { NULL, NULL, NULL, NULL }, }, - { { NULL, NULL, NULL, NULL }, - { NULL, NULL, NULL, NULL }, - { NULL, NULL, NULL, NULL } }, { + { idct8x8_low1_avx2, idct8x8_avx2, NULL, NULL }, + { iadst8x8_low1_avx2, iadst8x8_avx2, NULL, NULL }, { NULL, NULL, NULL, NULL }, - { NULL, NULL, NULL, NULL }, + }, + { + { idct16_low1_avx2, idct16_low8_avx2, idct16_avx2, NULL }, + { iadst16_low1_avx2, iadst16_low8_avx2, iadst16_avx2, NULL }, { NULL, NULL, NULL, NULL }, }, { { idct32_low1_avx2, idct32_low8_avx2, idct32_low16_avx2, idct32_avx2 }, { NULL, NULL, NULL, NULL }, { NULL, NULL, NULL, NULL } }, - { { NULL, NULL, NULL, NULL }, + { { idct64_low1_avx2, idct64_low8_avx2, idct64_low16_avx2, idct64_avx2 }, { NULL, NULL, NULL, NULL }, { NULL, NULL, NULL, NULL } } }; @@ -1186,7 +4216,7 @@ static void highbd_inv_txfm2d_add_no_identity_avx2(const int32_t *input, TX_TYPE tx_type, TX_SIZE tx_size, int eob, const int bd) { - __m256i buf1[64 * 2]; + __m256i buf1[64 * 8]; int eobx, eoby; get_eobx_eoby_scan_default(&eobx, &eoby, tx_size, eob); const int8_t *shift = inv_txfm_shift_ls[tx_size]; @@ -1198,7 +4228,7 @@ static void highbd_inv_txfm2d_add_no_identity_avx2(const int32_t *input, const int buf_size_nonzero_w_div8 = (eobx + 8) >> 3; const int buf_size_nonzero_h_div8 = (eoby + 8) >> 3; const int input_stride = AOMMIN(32, txfm_size_col); - + const int rect_type = get_rect_tx_log_ratio(txfm_size_col, txfm_size_row); const int fun_idx_x = lowbd_txfm_all_1d_zeros_idx[eobx]; const int fun_idx_y = lowbd_txfm_all_1d_zeros_idx[eoby]; const transform_1d_avx2 row_txfm = @@ -1213,7 +4243,7 @@ static void highbd_inv_txfm2d_add_no_identity_avx2(const int32_t *input, // 1st stage: column transform for (int i = 0; i < buf_size_nonzero_h_div8; i++) { - __m256i buf0[32]; + __m256i buf0[64]; const int32_t *input_row = input + i * input_stride * 8; for (int j = 0; j < buf_size_nonzero_w_div8; ++j) { __m256i *buf0_cur = buf0 + j * 8; @@ -1221,12 +4251,22 @@ static void highbd_inv_txfm2d_add_no_identity_avx2(const int32_t *input, transpose_8x8_avx2(&buf0_cur[0], &buf0_cur[0]); } - + if (rect_type == 1 || rect_type == -1) { + av1_round_shift_rect_array_32_avx2( + buf0, buf0, buf_size_nonzero_w_div8 << 3, 0, NewInvSqrt2); + } row_txfm(buf0, buf0, inv_cos_bit_row[txw_idx][txh_idx], 0, bd, -shift[0]); __m256i *_buf1 = buf1 + i * 8; - for (int j = 0; j < buf_size_w_div8; ++j) { - transpose_8x8_avx2(&buf0[j * 8], &_buf1[j * txfm_size_row]); + if (lr_flip) { + for (int j = 0; j < buf_size_w_div8; ++j) { + transpose_8x8_flip_avx2( + &buf0[j * 8], &_buf1[(buf_size_w_div8 - 1 - j) * txfm_size_row]); + } + } else { + for (int j = 0; j < buf_size_w_div8; ++j) { + transpose_8x8_avx2(&buf0[j * 8], &_buf1[j * txfm_size_row]); + } } } // 2nd stage: column transform @@ -1240,12 +4280,15 @@ static void highbd_inv_txfm2d_add_no_identity_avx2(const int32_t *input, } // write to buffer - { + if (txfm_size_col >= 16) { for (int i = 0; i < (txfm_size_col >> 4); i++) { highbd_write_buffer_16xn_avx2(buf1 + i * txfm_size_row * 2, output + 16 * i, stride, ud_flip, txfm_size_row, bd); } + } else if (txfm_size_col == 8) { + highbd_write_buffer_8xn_avx2(buf1, output, stride, ud_flip, txfm_size_row, + bd); } } @@ -1255,95 +4298,54 @@ void av1_highbd_inv_txfm2d_add_universe_avx2(const int32_t *input, int eob, const int bd) { switch (tx_type) { case DCT_DCT: + case ADST_DCT: + case DCT_ADST: + case ADST_ADST: + case FLIPADST_DCT: + case DCT_FLIPADST: + case FLIPADST_FLIPADST: + case ADST_FLIPADST: + case FLIPADST_ADST: highbd_inv_txfm2d_add_no_identity_avx2(input, CONVERT_TO_SHORTPTR(output), stride, tx_type, tx_size, eob, bd); break; + case IDTX: + case H_DCT: + case H_ADST: + case H_FLIPADST: + case V_DCT: + case V_ADST: + case V_FLIPADST: + av1_highbd_inv_txfm2d_add_universe_sse4_1(input, output, stride, tx_type, + tx_size, eob, bd); + break; default: assert(0); break; } } - -void av1_highbd_inv_txfm_add_32x32_avx2(const tran_low_t *input, uint8_t *dest, - int stride, - const TxfmParam *txfm_param) { - const int bd = txfm_param->bd; - const TX_TYPE tx_type = txfm_param->tx_type; - const int32_t *src = cast_to_int32(input); - switch (tx_type) { - case DCT_DCT: - av1_highbd_inv_txfm2d_add_universe_avx2(input, dest, stride, tx_type, - txfm_param->tx_size, - txfm_param->eob, bd); - break; - // Assembly version doesn't support IDTX, so use C version for it. - case IDTX: - av1_inv_txfm2d_add_32x32_c(src, CONVERT_TO_SHORTPTR(dest), stride, - tx_type, bd); - break; - - default: assert(0); - } -} - void av1_highbd_inv_txfm_add_avx2(const tran_low_t *input, uint8_t *dest, int stride, const TxfmParam *txfm_param) { assert(av1_ext_tx_used[txfm_param->tx_set_type][txfm_param->tx_type]); const TX_SIZE tx_size = txfm_param->tx_size; switch (tx_size) { - case TX_32X32: - av1_highbd_inv_txfm_add_32x32_avx2(input, dest, stride, txfm_param); - break; - case TX_16X16: - av1_highbd_inv_txfm_add_16x16_sse4_1(input, dest, stride, txfm_param); - break; - case TX_8X8: - av1_highbd_inv_txfm_add_8x8_sse4_1(input, dest, stride, txfm_param); - break; case TX_4X8: - av1_highbd_inv_txfm_add_4x8(input, dest, stride, txfm_param); + av1_highbd_inv_txfm_add_4x8_sse4_1(input, dest, stride, txfm_param); break; case TX_8X4: - av1_highbd_inv_txfm_add_8x4(input, dest, stride, txfm_param); - break; - case TX_8X16: - av1_highbd_inv_txfm_add_8x16_sse4_1(input, dest, stride, txfm_param); - break; - case TX_16X8: - av1_highbd_inv_txfm_add_16x8_sse4_1(input, dest, stride, txfm_param); - break; - case TX_16X32: - av1_highbd_inv_txfm_add_16x32(input, dest, stride, txfm_param); - break; - case TX_32X16: - av1_highbd_inv_txfm_add_32x16(input, dest, stride, txfm_param); - break; - case TX_32X64: - av1_highbd_inv_txfm_add_32x64(input, dest, stride, txfm_param); - break; - case TX_64X32: - av1_highbd_inv_txfm_add_64x32(input, dest, stride, txfm_param); + av1_highbd_inv_txfm_add_8x4_sse4_1(input, dest, stride, txfm_param); break; case TX_4X4: av1_highbd_inv_txfm_add_4x4_sse4_1(input, dest, stride, txfm_param); break; case TX_16X4: - av1_highbd_inv_txfm_add_16x4(input, dest, stride, txfm_param); + av1_highbd_inv_txfm_add_16x4_sse4_1(input, dest, stride, txfm_param); break; case TX_4X16: - av1_highbd_inv_txfm_add_4x16(input, dest, stride, txfm_param); + av1_highbd_inv_txfm_add_4x16_sse4_1(input, dest, stride, txfm_param); break; - case TX_8X32: - av1_highbd_inv_txfm_add_8x32(input, dest, stride, txfm_param); - break; - case TX_32X8: - av1_highbd_inv_txfm_add_32x8(input, dest, stride, txfm_param); - break; - case TX_64X64: - case TX_16X64: - case TX_64X16: - av1_highbd_inv_txfm2d_add_universe_sse4_1( + default: + av1_highbd_inv_txfm2d_add_universe_avx2( input, dest, stride, txfm_param->tx_type, txfm_param->tx_size, txfm_param->eob, txfm_param->bd); break; - default: assert(0 && "Invalid transform size"); break; } } diff --git a/third_party/aom/av1/common/x86/highbd_inv_txfm_sse4.c b/third_party/aom/av1/common/x86/highbd_inv_txfm_sse4.c index e29e0baf50..2f0eecb8bc 100644 --- a/third_party/aom/av1/common/x86/highbd_inv_txfm_sse4.c +++ b/third_party/aom/av1/common/x86/highbd_inv_txfm_sse4.c @@ -17,6 +17,7 @@ #include "av1/common/av1_inv_txfm1d_cfg.h" #include "av1/common/idct.h" #include "av1/common/x86/av1_inv_txfm_ssse3.h" +#include "av1/common/x86/av1_txfm_sse2.h" #include "av1/common/x86/av1_txfm_sse4.h" #include "av1/common/x86/highbd_txfm_utility_sse4.h" @@ -49,6 +50,29 @@ static INLINE __m128i highbd_get_recon_8x8_sse4_1(const __m128i pred, return x0; } +static INLINE __m128i highbd_get_recon_4xn_sse4_1(const __m128i pred, + __m128i res0, const int bd) { + __m128i x0 = _mm_cvtepi16_epi32(pred); + + x0 = _mm_add_epi32(res0, x0); + x0 = _mm_packus_epi32(x0, x0); + x0 = highbd_clamp_epi16(x0, bd); + return x0; +} + +static INLINE void highbd_write_buffer_4xn_sse4_1(__m128i *in, uint16_t *output, + int stride, int flipud, + int height, const int bd) { + int j = flipud ? (height - 1) : 0; + const int step = flipud ? -1 : 1; + for (int i = 0; i < height; ++i, j += step) { + __m128i v = _mm_loadl_epi64((__m128i const *)(output + i * stride)); + __m128i u = highbd_get_recon_4xn_sse4_1(v, in[j], bd); + + _mm_storel_epi64((__m128i *)(output + i * stride), u); + } +} + static INLINE void highbd_write_buffer_8xn_sse4_1(__m128i *in, uint16_t *output, int stride, int flipud, int height, const int bd) { @@ -354,7 +378,9 @@ static void neg_shift_sse4_1(const __m128i in0, const __m128i in1, *out1 = a1; } -static void idct4x4_sse4_1(__m128i *in, int bit, int do_cols, int bd) { +static void idct4x4_sse4_1(__m128i *in, __m128i *out, int bit, int do_cols, + int bd, int out_shift) { + (void)out_shift; const int32_t *cospi = cospi_arr(bit); const __m128i cospi32 = _mm_set1_epi32(cospi[32]); const __m128i cospi48 = _mm_set1_epi32(cospi[48]); @@ -398,18 +424,20 @@ static void idct4x4_sse4_1(__m128i *in, int bit, int do_cols, int bd) { v3 = _mm_srai_epi32(v3, bit); if (do_cols) { - addsub_no_clamp_sse4_1(v0, v3, in + 0, in + 3); - addsub_no_clamp_sse4_1(v1, v2, in + 1, in + 2); + addsub_no_clamp_sse4_1(v0, v3, out + 0, out + 3); + addsub_no_clamp_sse4_1(v1, v2, out + 1, out + 2); } else { const int log_range = AOMMAX(16, bd + 6); const __m128i clamp_lo = _mm_set1_epi32(-(1 << (log_range - 1))); const __m128i clamp_hi = _mm_set1_epi32((1 << (log_range - 1)) - 1); - addsub_sse4_1(v0, v3, in + 0, in + 3, &clamp_lo, &clamp_hi); - addsub_sse4_1(v1, v2, in + 1, in + 2, &clamp_lo, &clamp_hi); + addsub_sse4_1(v0, v3, out + 0, out + 3, &clamp_lo, &clamp_hi); + addsub_sse4_1(v1, v2, out + 1, out + 2, &clamp_lo, &clamp_hi); } } -static void iadst4x4_sse4_1(__m128i *in, int bit, int do_cols, int bd) { +static void iadst4x4_sse4_1(__m128i *in, __m128i *out, int bit, int do_cols, + int bd, int out_shift) { + (void)out_shift; const int32_t *sinpi = sinpi_arr(bit); const __m128i rnding = _mm_set1_epi32(1 << (bit - 1)); const __m128i sinpi1 = _mm_set1_epi32((int)sinpi[1]); @@ -482,10 +510,10 @@ static void iadst4x4_sse4_1(__m128i *in, int bit, int do_cols, int bd) { u3 = _mm_min_epi32(u3, clamp_hi); } - in[0] = u0; - in[1] = u1; - in[2] = u2; - in[3] = u3; + out[0] = u0; + out[1] = u1; + out[2] = u2; + out[3] = u3; } static INLINE void round_shift_4x4(__m128i *in, int shift) { @@ -555,7 +583,66 @@ static void write_buffer_4x4(__m128i *in, uint16_t *output, int stride, _mm_storel_epi64((__m128i *)(output + 2 * stride), v2); _mm_storel_epi64((__m128i *)(output + 3 * stride), v3); } +static void highbd_clamp_epi32_sse4_1(const __m128i *in, __m128i *out, + const __m128i *clamp_lo, + const __m128i *clamp_hi, int size) { + __m128i a0, a1; + for (int i = 0; i < size; i += 4) { + a0 = _mm_max_epi32(in[i], *clamp_lo); + out[i] = _mm_min_epi32(a0, *clamp_hi); + a1 = _mm_max_epi32(in[i + 1], *clamp_lo); + out[i + 1] = _mm_min_epi32(a1, *clamp_hi); + + a0 = _mm_max_epi32(in[i + 2], *clamp_lo); + out[i + 2] = _mm_min_epi32(a0, *clamp_hi); + + a1 = _mm_max_epi32(in[i + 3], *clamp_lo); + out[i + 3] = _mm_min_epi32(a1, *clamp_hi); + } +} +static void iidentity4_sse4_1(__m128i *in, __m128i *out, int bit, int do_cols, + int bd, int out_shift) { + (void)bit; + (void)out_shift; + __m128i v[4]; + __m128i fact = _mm_set1_epi32(NewSqrt2); + __m128i offset = _mm_set1_epi32(1 << (NewSqrt2Bits - 1)); + __m128i a0, a1; + + a0 = _mm_mullo_epi32(in[0], fact); + a1 = _mm_mullo_epi32(in[1], fact); + a0 = _mm_add_epi32(a0, offset); + a1 = _mm_add_epi32(a1, offset); + out[0] = _mm_srai_epi32(a0, NewSqrt2Bits); + out[1] = _mm_srai_epi32(a1, NewSqrt2Bits); + + a0 = _mm_mullo_epi32(in[2], fact); + a1 = _mm_mullo_epi32(in[3], fact); + a0 = _mm_add_epi32(a0, offset); + a1 = _mm_add_epi32(a1, offset); + out[2] = _mm_srai_epi32(a0, NewSqrt2Bits); + out[3] = _mm_srai_epi32(a1, NewSqrt2Bits); + + if (!do_cols) { + const int log_range = AOMMAX(16, bd + 6); + const __m128i clamp_lo = _mm_set1_epi32(-(1 << (log_range - 1))); + const __m128i clamp_hi = _mm_set1_epi32((1 << (log_range - 1)) - 1); + + highbd_clamp_epi32_sse4_1(out, out, &clamp_lo, &clamp_hi, 4); + } + + // Transpose for 4x4 + v[0] = _mm_unpacklo_epi32(out[0], out[1]); + v[1] = _mm_unpackhi_epi32(out[0], out[1]); + v[2] = _mm_unpacklo_epi32(out[2], out[3]); + v[3] = _mm_unpackhi_epi32(out[2], out[3]); + + out[0] = _mm_unpacklo_epi64(v[0], v[2]); + out[1] = _mm_unpackhi_epi64(v[0], v[2]); + out[2] = _mm_unpacklo_epi64(v[1], v[3]); + out[3] = _mm_unpackhi_epi64(v[1], v[3]); +} void av1_inv_txfm2d_add_4x4_sse4_1(const int32_t *coeff, uint16_t *output, int stride, TX_TYPE tx_type, int bd) { __m128i in[4]; @@ -566,58 +653,100 @@ void av1_inv_txfm2d_add_4x4_sse4_1(const int32_t *coeff, uint16_t *output, switch (tx_type) { case DCT_DCT: load_buffer_4x4(coeff, in); - idct4x4_sse4_1(in, inv_cos_bit_row[txw_idx][txh_idx], 0, bd); - idct4x4_sse4_1(in, inv_cos_bit_col[txw_idx][txh_idx], 1, bd); + idct4x4_sse4_1(in, in, inv_cos_bit_row[txw_idx][txh_idx], 0, bd, 0); + idct4x4_sse4_1(in, in, inv_cos_bit_col[txw_idx][txh_idx], 1, bd, 0); write_buffer_4x4(in, output, stride, 0, 0, -shift[1], bd); break; case ADST_DCT: load_buffer_4x4(coeff, in); - idct4x4_sse4_1(in, inv_cos_bit_row[txw_idx][txh_idx], 0, bd); - iadst4x4_sse4_1(in, inv_cos_bit_col[txw_idx][txh_idx], 1, bd); + idct4x4_sse4_1(in, in, inv_cos_bit_row[txw_idx][txh_idx], 0, bd, 0); + iadst4x4_sse4_1(in, in, inv_cos_bit_col[txw_idx][txh_idx], 1, bd, 0); write_buffer_4x4(in, output, stride, 0, 0, -shift[1], bd); break; case DCT_ADST: load_buffer_4x4(coeff, in); - iadst4x4_sse4_1(in, inv_cos_bit_row[txw_idx][txh_idx], 0, bd); - idct4x4_sse4_1(in, inv_cos_bit_col[txw_idx][txh_idx], 1, bd); + iadst4x4_sse4_1(in, in, inv_cos_bit_row[txw_idx][txh_idx], 0, bd, 0); + idct4x4_sse4_1(in, in, inv_cos_bit_col[txw_idx][txh_idx], 1, bd, 0); write_buffer_4x4(in, output, stride, 0, 0, -shift[1], bd); break; case ADST_ADST: load_buffer_4x4(coeff, in); - iadst4x4_sse4_1(in, inv_cos_bit_row[txw_idx][txh_idx], 0, bd); - iadst4x4_sse4_1(in, inv_cos_bit_col[txw_idx][txh_idx], 1, bd); + iadst4x4_sse4_1(in, in, inv_cos_bit_row[txw_idx][txh_idx], 0, bd, 0); + iadst4x4_sse4_1(in, in, inv_cos_bit_col[txw_idx][txh_idx], 1, bd, 0); write_buffer_4x4(in, output, stride, 0, 0, -shift[1], bd); break; case FLIPADST_DCT: load_buffer_4x4(coeff, in); - idct4x4_sse4_1(in, inv_cos_bit_row[txw_idx][txh_idx], 0, bd); - iadst4x4_sse4_1(in, inv_cos_bit_col[txw_idx][txh_idx], 1, bd); + idct4x4_sse4_1(in, in, inv_cos_bit_row[txw_idx][txh_idx], 0, bd, 0); + iadst4x4_sse4_1(in, in, inv_cos_bit_col[txw_idx][txh_idx], 1, bd, 0); write_buffer_4x4(in, output, stride, 0, 1, -shift[1], bd); break; case DCT_FLIPADST: load_buffer_4x4(coeff, in); - iadst4x4_sse4_1(in, inv_cos_bit_row[txw_idx][txh_idx], 0, bd); - idct4x4_sse4_1(in, inv_cos_bit_col[txw_idx][txh_idx], 1, bd); + iadst4x4_sse4_1(in, in, inv_cos_bit_row[txw_idx][txh_idx], 0, bd, 0); + idct4x4_sse4_1(in, in, inv_cos_bit_col[txw_idx][txh_idx], 1, bd, 0); write_buffer_4x4(in, output, stride, 1, 0, -shift[1], bd); break; case FLIPADST_FLIPADST: load_buffer_4x4(coeff, in); - iadst4x4_sse4_1(in, inv_cos_bit_row[txw_idx][txh_idx], 0, bd); - iadst4x4_sse4_1(in, inv_cos_bit_col[txw_idx][txh_idx], 1, bd); + iadst4x4_sse4_1(in, in, inv_cos_bit_row[txw_idx][txh_idx], 0, bd, 0); + iadst4x4_sse4_1(in, in, inv_cos_bit_col[txw_idx][txh_idx], 1, bd, 0); write_buffer_4x4(in, output, stride, 1, 1, -shift[1], bd); break; case ADST_FLIPADST: load_buffer_4x4(coeff, in); - iadst4x4_sse4_1(in, inv_cos_bit_row[txw_idx][txh_idx], 0, bd); - iadst4x4_sse4_1(in, inv_cos_bit_col[txw_idx][txh_idx], 1, bd); + iadst4x4_sse4_1(in, in, inv_cos_bit_row[txw_idx][txh_idx], 0, bd, 0); + iadst4x4_sse4_1(in, in, inv_cos_bit_col[txw_idx][txh_idx], 1, bd, 0); write_buffer_4x4(in, output, stride, 1, 0, -shift[1], bd); break; case FLIPADST_ADST: load_buffer_4x4(coeff, in); - iadst4x4_sse4_1(in, inv_cos_bit_row[txw_idx][txh_idx], 0, bd); - iadst4x4_sse4_1(in, inv_cos_bit_col[txw_idx][txh_idx], 1, bd); + iadst4x4_sse4_1(in, in, inv_cos_bit_row[txw_idx][txh_idx], 0, bd, 0); + iadst4x4_sse4_1(in, in, inv_cos_bit_col[txw_idx][txh_idx], 1, bd, 0); write_buffer_4x4(in, output, stride, 0, 1, -shift[1], bd); break; + case IDTX: + load_buffer_4x4(coeff, in); + iidentity4_sse4_1(in, in, inv_cos_bit_row[txw_idx][txh_idx], 0, bd, 0); + iidentity4_sse4_1(in, in, inv_cos_bit_col[txw_idx][txh_idx], 1, bd, 0); + write_buffer_4x4(in, output, stride, 0, 0, -shift[1], bd); + break; + case V_DCT: + load_buffer_4x4(coeff, in); + iidentity4_sse4_1(in, in, inv_cos_bit_row[txw_idx][txh_idx], 0, bd, 0); + idct4x4_sse4_1(in, in, inv_cos_bit_col[txw_idx][txh_idx], 1, bd, 0); + write_buffer_4x4(in, output, stride, 0, 0, -shift[1], bd); + break; + case H_DCT: + load_buffer_4x4(coeff, in); + idct4x4_sse4_1(in, in, inv_cos_bit_row[txw_idx][txh_idx], 0, bd, 0); + iidentity4_sse4_1(in, in, inv_cos_bit_col[txw_idx][txh_idx], 1, bd, 0); + write_buffer_4x4(in, output, stride, 0, 0, -shift[1], bd); + break; + case V_ADST: + load_buffer_4x4(coeff, in); + iidentity4_sse4_1(in, in, inv_cos_bit_row[txw_idx][txh_idx], 0, bd, 0); + iadst4x4_sse4_1(in, in, inv_cos_bit_col[txw_idx][txh_idx], 1, bd, 0); + write_buffer_4x4(in, output, stride, 0, 0, -shift[1], bd); + break; + case H_ADST: + load_buffer_4x4(coeff, in); + iadst4x4_sse4_1(in, in, inv_cos_bit_row[txw_idx][txh_idx], 0, bd, 0); + iidentity4_sse4_1(in, in, inv_cos_bit_col[txw_idx][txh_idx], 1, bd, 0); + write_buffer_4x4(in, output, stride, 0, 0, -shift[1], bd); + break; + case V_FLIPADST: + load_buffer_4x4(coeff, in); + iidentity4_sse4_1(in, in, inv_cos_bit_row[txw_idx][txh_idx], 0, bd, 0); + iadst4x4_sse4_1(in, in, inv_cos_bit_col[txw_idx][txh_idx], 1, bd, 0); + write_buffer_4x4(in, output, stride, 0, 1, -shift[1], bd); + break; + case H_FLIPADST: + load_buffer_4x4(coeff, in); + iadst4x4_sse4_1(in, in, inv_cos_bit_row[txw_idx][txh_idx], 0, bd, 0); + iidentity4_sse4_1(in, in, inv_cos_bit_col[txw_idx][txh_idx], 1, bd, 0); + write_buffer_4x4(in, output, stride, 1, 0, -shift[1], bd); + break; default: assert(0); } } @@ -1088,6 +1217,61 @@ static void iadst8x8_sse4_1(__m128i *in, __m128i *out, int bit, int do_cols, &clamp_hi_out, out_shift); } } +static void shift_sse4_1(const __m128i *in, __m128i *out, + const __m128i *clamp_lo, const __m128i *clamp_hi, + int shift, int size) { + __m128i offset = _mm_set1_epi32((1 << shift) >> 1); + __m128i shift_vec = _mm_cvtsi32_si128(shift); + __m128i a0, a1; + for (int i = 0; i < size; i += 4) { + a0 = _mm_add_epi32(in[i], offset); + a1 = _mm_add_epi32(in[i + 1], offset); + a0 = _mm_sra_epi32(a0, shift_vec); + a1 = _mm_sra_epi32(a1, shift_vec); + a0 = _mm_max_epi32(a0, *clamp_lo); + a1 = _mm_max_epi32(a1, *clamp_lo); + out[i] = _mm_min_epi32(a0, *clamp_hi); + out[i + 1] = _mm_min_epi32(a1, *clamp_hi); + + a0 = _mm_add_epi32(in[i + 2], offset); + a1 = _mm_add_epi32(in[i + 3], offset); + a0 = _mm_sra_epi32(a0, shift_vec); + a1 = _mm_sra_epi32(a1, shift_vec); + a0 = _mm_max_epi32(a0, *clamp_lo); + a1 = _mm_max_epi32(a1, *clamp_lo); + out[i + 2] = _mm_min_epi32(a0, *clamp_hi); + out[i + 3] = _mm_min_epi32(a1, *clamp_hi); + } +} + +static void iidentity8_sse4_1(__m128i *in, __m128i *out, int bit, int do_cols, + int bd, int out_shift) { + (void)bit; + const int log_range = AOMMAX(16, bd + (do_cols ? 6 : 8)); + const __m128i clamp_lo = _mm_set1_epi32(-(1 << (log_range - 1))); + const __m128i clamp_hi = _mm_set1_epi32((1 << (log_range - 1)) - 1); + __m128i v[8]; + v[0] = _mm_add_epi32(in[0], in[0]); + v[1] = _mm_add_epi32(in[1], in[1]); + v[2] = _mm_add_epi32(in[2], in[2]); + v[3] = _mm_add_epi32(in[3], in[3]); + v[4] = _mm_add_epi32(in[4], in[4]); + v[5] = _mm_add_epi32(in[5], in[5]); + v[6] = _mm_add_epi32(in[6], in[6]); + v[7] = _mm_add_epi32(in[7], in[7]); + + if (!do_cols) { + const int log_range_out = AOMMAX(16, bd + 6); + const __m128i clamp_lo_out = _mm_set1_epi32(AOMMAX( + -(1 << (log_range_out - 1)), -(1 << (log_range - 1 - out_shift)))); + const __m128i clamp_hi_out = _mm_set1_epi32(AOMMIN( + (1 << (log_range_out - 1)) - 1, (1 << (log_range - 1 - out_shift)))); + + shift_sse4_1(v, out, &clamp_lo_out, &clamp_hi_out, out_shift, 8); + } else { + highbd_clamp_epi32_sse4_1(v, out, &clamp_lo, &clamp_hi, 8); + } +} static void round_shift_8x8(__m128i *in, int shift) { round_shift_4x4(&in[0], shift); @@ -2972,7 +3156,59 @@ static void iadst16x16_sse4_1(__m128i *in, __m128i *out, int bit, int do_cols, } } } +static void iidentity16_sse4_1(__m128i *in, __m128i *out, int bit, int do_cols, + int bd, int out_shift) { + (void)bit; + const int log_range = AOMMAX(16, bd + (do_cols ? 6 : 8)); + const __m128i clamp_lo = _mm_set1_epi32(-(1 << (log_range - 1))); + const __m128i clamp_hi = _mm_set1_epi32((1 << (log_range - 1)) - 1); + __m128i v[16]; + __m128i fact = _mm_set1_epi32(2 * NewSqrt2); + __m128i offset = _mm_set1_epi32(1 << (NewSqrt2Bits - 1)); + __m128i a0, a1, a2, a3; + for (int i = 0; i < 16; i += 8) { + a0 = _mm_mullo_epi32(in[i], fact); + a1 = _mm_mullo_epi32(in[i + 1], fact); + a0 = _mm_add_epi32(a0, offset); + a1 = _mm_add_epi32(a1, offset); + v[i] = _mm_srai_epi32(a0, NewSqrt2Bits); + v[i + 1] = _mm_srai_epi32(a1, NewSqrt2Bits); + + a2 = _mm_mullo_epi32(in[i + 2], fact); + a3 = _mm_mullo_epi32(in[i + 3], fact); + a2 = _mm_add_epi32(a2, offset); + a3 = _mm_add_epi32(a3, offset); + v[i + 2] = _mm_srai_epi32(a2, NewSqrt2Bits); + v[i + 3] = _mm_srai_epi32(a3, NewSqrt2Bits); + + a0 = _mm_mullo_epi32(in[i + 4], fact); + a1 = _mm_mullo_epi32(in[i + 5], fact); + a0 = _mm_add_epi32(a0, offset); + a1 = _mm_add_epi32(a1, offset); + v[i + 4] = _mm_srai_epi32(a0, NewSqrt2Bits); + v[i + 5] = _mm_srai_epi32(a1, NewSqrt2Bits); + + a2 = _mm_mullo_epi32(in[i + 6], fact); + a3 = _mm_mullo_epi32(in[i + 7], fact); + a2 = _mm_add_epi32(a2, offset); + a3 = _mm_add_epi32(a3, offset); + v[i + 6] = _mm_srai_epi32(a2, NewSqrt2Bits); + v[i + 7] = _mm_srai_epi32(a3, NewSqrt2Bits); + } + + if (!do_cols) { + const int log_range_out = AOMMAX(16, bd + 6); + const __m128i clamp_lo_out = _mm_set1_epi32(AOMMAX( + -(1 << (log_range_out - 1)), -(1 << (log_range - 1 - out_shift)))); + const __m128i clamp_hi_out = _mm_set1_epi32(AOMMIN( + (1 << (log_range_out - 1)) - 1, (1 << (log_range - 1 - out_shift)))); + + shift_sse4_1(v, out, &clamp_lo_out, &clamp_hi_out, out_shift, 16); + } else { + highbd_clamp_epi32_sse4_1(v, out, &clamp_lo, &clamp_hi, 16); + } +} static INLINE void idct64_stage8_sse4_1( __m128i *u, const __m128i *cospim32, const __m128i *cospi32, const __m128i *cospim16, const __m128i *cospi48, const __m128i *cospi16, @@ -3154,69 +3390,69 @@ static void idct64x64_low1_sse4_1(__m128i *in, __m128i *out, int bit, } out[0] = x; - out[63] = x; out[1] = x; - out[62] = x; out[2] = x; - out[61] = x; out[3] = x; - out[60] = x; out[4] = x; - out[59] = x; out[5] = x; - out[58] = x; out[6] = x; - out[57] = x; out[7] = x; - out[56] = x; out[8] = x; - out[55] = x; out[9] = x; - out[54] = x; out[10] = x; - out[53] = x; out[11] = x; - out[52] = x; out[12] = x; - out[51] = x; out[13] = x; - out[50] = x; out[14] = x; - out[49] = x; out[15] = x; - out[48] = x; out[16] = x; - out[47] = x; out[17] = x; - out[46] = x; out[18] = x; - out[45] = x; out[19] = x; - out[44] = x; out[20] = x; - out[43] = x; out[21] = x; - out[42] = x; out[22] = x; - out[41] = x; out[23] = x; - out[40] = x; out[24] = x; - out[39] = x; out[25] = x; - out[38] = x; out[26] = x; - out[37] = x; out[27] = x; - out[36] = x; out[28] = x; - out[35] = x; out[29] = x; - out[34] = x; out[30] = x; - out[33] = x; out[31] = x; out[32] = x; + out[33] = x; + out[34] = x; + out[35] = x; + out[36] = x; + out[37] = x; + out[38] = x; + out[39] = x; + out[40] = x; + out[41] = x; + out[42] = x; + out[43] = x; + out[44] = x; + out[45] = x; + out[46] = x; + out[47] = x; + out[48] = x; + out[49] = x; + out[50] = x; + out[51] = x; + out[52] = x; + out[53] = x; + out[54] = x; + out[55] = x; + out[56] = x; + out[57] = x; + out[58] = x; + out[59] = x; + out[60] = x; + out[61] = x; + out[62] = x; + out[63] = x; } } @@ -4992,17 +5228,16 @@ void av1_highbd_inv_txfm_add_8x8_sse4_1(const tran_low_t *input, uint8_t *dest, const TX_TYPE tx_type = txfm_param->tx_type; const int32_t *src = cast_to_int32(input); switch (tx_type) { - // Assembly version doesn't support some transform types, so use C version - // for those. - case V_DCT: - case H_DCT: - case V_ADST: - case H_ADST: - case V_FLIPADST: - case H_FLIPADST: case IDTX: - av1_inv_txfm2d_add_8x8_c(src, CONVERT_TO_SHORTPTR(dest), stride, tx_type, - bd); + case H_DCT: + case H_ADST: + case H_FLIPADST: + case V_DCT: + case V_ADST: + case V_FLIPADST: + av1_highbd_inv_txfm2d_add_universe_sse4_1(input, dest, stride, tx_type, + txfm_param->tx_size, + txfm_param->eob, bd); break; default: av1_inv_txfm2d_add_8x8_sse4_1(src, CONVERT_TO_SHORTPTR(dest), stride, @@ -5010,109 +5245,6 @@ void av1_highbd_inv_txfm_add_8x8_sse4_1(const tran_low_t *input, uint8_t *dest, break; } } - -void av1_highbd_inv_txfm_add_16x8_sse4_1(const tran_low_t *input, uint8_t *dest, - int stride, - const TxfmParam *txfm_param) { - int bd = txfm_param->bd; - const TX_TYPE tx_type = txfm_param->tx_type; - const int32_t *src = cast_to_int32(input); - switch (tx_type) { - // Assembly version doesn't support some transform types, so use C version - // for those. - case V_DCT: - case H_DCT: - case V_ADST: - case H_ADST: - case V_FLIPADST: - case H_FLIPADST: - case IDTX: - av1_inv_txfm2d_add_16x8_c(src, CONVERT_TO_SHORTPTR(dest), stride, - txfm_param->tx_type, txfm_param->bd); - break; - default: - av1_highbd_inv_txfm2d_add_universe_sse4_1(input, dest, stride, tx_type, - txfm_param->tx_size, - txfm_param->eob, bd); - break; - } -} - -void av1_highbd_inv_txfm_add_8x16_sse4_1(const tran_low_t *input, uint8_t *dest, - int stride, - const TxfmParam *txfm_param) { - int bd = txfm_param->bd; - const TX_TYPE tx_type = txfm_param->tx_type; - const int32_t *src = cast_to_int32(input); - switch (tx_type) { - // Assembly version doesn't support some transform types, so use C version - // for those. - case V_DCT: - case H_DCT: - case V_ADST: - case H_ADST: - case V_FLIPADST: - case H_FLIPADST: - case IDTX: - av1_inv_txfm2d_add_8x16_c(src, CONVERT_TO_SHORTPTR(dest), stride, - txfm_param->tx_type, txfm_param->bd); - break; - default: - av1_highbd_inv_txfm2d_add_universe_sse4_1(input, dest, stride, tx_type, - txfm_param->tx_size, - txfm_param->eob, bd); - break; - } -} - -void av1_highbd_inv_txfm_add_16x16_sse4_1(const tran_low_t *input, - uint8_t *dest, int stride, - const TxfmParam *txfm_param) { - int bd = txfm_param->bd; - const TX_TYPE tx_type = txfm_param->tx_type; - const int32_t *src = cast_to_int32(input); - switch (tx_type) { - // Assembly version doesn't support some transform types, so use C version - // for those. - case V_DCT: - case H_DCT: - case V_ADST: - case H_ADST: - case V_FLIPADST: - case H_FLIPADST: - case IDTX: - av1_inv_txfm2d_add_16x16_c(src, CONVERT_TO_SHORTPTR(dest), stride, - tx_type, bd); - break; - default: - av1_highbd_inv_txfm2d_add_universe_sse4_1(input, dest, stride, tx_type, - txfm_param->tx_size, - txfm_param->eob, bd); - break; - } -} - -void av1_highbd_inv_txfm_add_32x32_sse4_1(const tran_low_t *input, - uint8_t *dest, int stride, - const TxfmParam *txfm_param) { - int bd = txfm_param->bd; - const TX_TYPE tx_type = txfm_param->tx_type; - const int32_t *src = cast_to_int32(input); - switch (tx_type) { - case DCT_DCT: - av1_highbd_inv_txfm2d_add_universe_sse4_1(input, dest, stride, tx_type, - txfm_param->tx_size, - txfm_param->eob, bd); - break; - // Assembly version doesn't support IDTX, so use C version for it. - case IDTX: - av1_inv_txfm2d_add_32x32_c(src, CONVERT_TO_SHORTPTR(dest), stride, - tx_type, bd); - break; - default: assert(0); - } -} - void av1_highbd_inv_txfm_add_4x4_sse4_1(const tran_low_t *input, uint8_t *dest, int stride, const TxfmParam *txfm_param) { @@ -5127,53 +5259,271 @@ void av1_highbd_inv_txfm_add_4x4_sse4_1(const tran_low_t *input, uint8_t *dest, av1_highbd_iwht4x4_add(input, dest, stride, eob, bd); return; } - switch (tx_type) { - // Assembly version doesn't support some transform types, so use C version - // for those. - case V_DCT: - case H_DCT: - case V_ADST: - case H_ADST: - case V_FLIPADST: - case H_FLIPADST: - case IDTX: - av1_inv_txfm2d_add_4x4_c(src, CONVERT_TO_SHORTPTR(dest), stride, tx_type, - bd); - break; - default: - av1_inv_txfm2d_add_4x4_sse4_1(src, CONVERT_TO_SHORTPTR(dest), stride, - tx_type, bd); - break; + av1_inv_txfm2d_add_4x4_sse4_1(src, CONVERT_TO_SHORTPTR(dest), stride, tx_type, + bd); +} +static void iidentity32_sse4_1(__m128i *in, __m128i *out, int bit, int do_cols, + int bd, int out_shift) { + (void)bit; + const int log_range = AOMMAX(16, bd + (do_cols ? 6 : 8)); + const __m128i clamp_lo = _mm_set1_epi32(-(1 << (log_range - 1))); + const __m128i clamp_hi = _mm_set1_epi32((1 << (log_range - 1)) - 1); + __m128i v[32]; + for (int i = 0; i < 32; i += 16) { + v[i] = _mm_slli_epi32(in[i], 2); + v[i + 1] = _mm_slli_epi32(in[i + 1], 2); + v[i + 2] = _mm_slli_epi32(in[i + 2], 2); + v[i + 3] = _mm_slli_epi32(in[i + 3], 2); + v[i + 4] = _mm_slli_epi32(in[i + 4], 2); + v[i + 5] = _mm_slli_epi32(in[i + 5], 2); + v[i + 6] = _mm_slli_epi32(in[i + 6], 2); + v[i + 7] = _mm_slli_epi32(in[i + 7], 2); + v[i + 8] = _mm_slli_epi32(in[i + 8], 2); + v[i + 9] = _mm_slli_epi32(in[i + 9], 2); + v[i + 10] = _mm_slli_epi32(in[i + 10], 2); + v[i + 11] = _mm_slli_epi32(in[i + 11], 2); + v[i + 12] = _mm_slli_epi32(in[i + 12], 2); + v[i + 13] = _mm_slli_epi32(in[i + 13], 2); + v[i + 14] = _mm_slli_epi32(in[i + 14], 2); + v[i + 15] = _mm_slli_epi32(in[i + 15], 2); + } + + if (!do_cols) { + const int log_range_out = AOMMAX(16, bd + 6); + const __m128i clamp_lo_out = _mm_set1_epi32(AOMMAX( + -(1 << (log_range_out - 1)), -(1 << (log_range - 1 - out_shift)))); + const __m128i clamp_hi_out = _mm_set1_epi32(AOMMIN( + (1 << (log_range_out - 1)) - 1, (1 << (log_range - 1 - out_shift)))); + shift_sse4_1(v, out, &clamp_lo_out, &clamp_hi_out, out_shift, 32); + } else { + highbd_clamp_epi32_sse4_1(v, out, &clamp_lo, &clamp_hi, 32); } } - static const transform_1d_sse4_1 highbd_txfm_all_1d_zeros_w8_arr[TX_SIZES][ITX_TYPES_1D][4] = { { - { NULL, NULL, NULL, NULL }, - { NULL, NULL, NULL, NULL }, - { NULL, NULL, NULL, NULL }, + { idct4x4_sse4_1, NULL, NULL, NULL }, + { iadst4x4_sse4_1, NULL, NULL, NULL }, + { iidentity4_sse4_1, iidentity4_sse4_1, iidentity4_sse4_1, NULL }, }, { { idct8x8_low1_sse4_1, idct8x8_new_sse4_1, NULL, NULL }, { iadst8x8_low1_sse4_1, iadst8x8_new_sse4_1, NULL, NULL }, - { NULL, NULL, NULL, NULL } }, + { iidentity8_sse4_1, iidentity8_sse4_1, NULL, NULL } }, { { idct16x16_low1_sse4_1, idct16x16_low8_sse4_1, idct16x16_sse4_1, NULL }, { iadst16x16_low1_sse4_1, iadst16x16_low8_sse4_1, iadst16x16_sse4_1, NULL }, - { NULL, NULL, NULL, NULL }, + { iidentity16_sse4_1, NULL, iidentity16_sse4_1, NULL }, }, { { idct32x32_low1_sse4_1, idct32x32_low8_sse4_1, idct32x32_low16_sse4_1, idct32x32_sse4_1 }, { NULL, NULL, NULL, NULL }, - { NULL, NULL, NULL, NULL } }, + { iidentity32_sse4_1, NULL, NULL, NULL } }, { { idct64x64_low1_sse4_1, idct64x64_low8_sse4_1, idct64x64_low16_sse4_1, idct64x64_sse4_1 }, { NULL, NULL, NULL, NULL }, { NULL, NULL, NULL, NULL } } }; +static void highbd_inv_txfm2d_add_h_identity_ssse41(const int32_t *input, + uint16_t *output, + int stride, TX_TYPE tx_type, + TX_SIZE tx_size, int eob, + const int bd) { + __m128i buf1[64]; + int eobx, eoby; + get_eobx_eoby_scan_v_identity(&eobx, &eoby, tx_size, eob); + const int8_t *shift = inv_txfm_shift_ls[tx_size]; + const int txw_idx = get_txw_idx(tx_size); + const int txh_idx = get_txh_idx(tx_size); + const int txfm_size_col = tx_size_wide[tx_size]; + const int txfm_size_row = tx_size_high[tx_size]; + const int input_stride = AOMMIN(32, txfm_size_col); + const int buf_size_w_div4 = input_stride >> 2; + const int buf_size_h_div8 = (eoby + 8) >> 3; + const int rect_type = get_rect_tx_log_ratio(txfm_size_col, txfm_size_row); + const int fun_idx = lowbd_txfm_all_1d_zeros_idx[eoby]; + const transform_1d_sse4_1 row_txfm = + highbd_txfm_all_1d_zeros_w8_arr[txw_idx][hitx_1d_tab[tx_type]][0]; + const transform_1d_sse4_1 col_txfm = + highbd_txfm_all_1d_zeros_w8_arr[txh_idx][vitx_1d_tab[tx_type]][fun_idx]; + int ud_flip, lr_flip; + get_flip_cfg(tx_type, &ud_flip, &lr_flip); + for (int i = 0; i < (buf_size_h_div8 << 1); ++i) { + __m128i buf0[16]; + const int32_t *input_row = input + i * input_stride * 4; + for (int j = 0; j < buf_size_w_div4; ++j) { + __m128i *buf0_cur = buf0 + j * 4; + load_buffer_32bit_input(input_row + j * 4, input_stride, buf0_cur, 4); + } + if (rect_type == 1 || rect_type == -1) { + av1_round_shift_rect_array_32_sse4_1(buf0, buf0, input_stride, 0, + NewInvSqrt2); + } + row_txfm(buf0, buf0, inv_cos_bit_row[txw_idx][txh_idx], 0, bd, -shift[0]); + + __m128i *_buf1 = buf1 + i * 4; + + for (int j = 0; j < buf_size_w_div4; ++j) { + _buf1[j * txfm_size_row + 0] = buf0[j * 4 + 0]; + _buf1[j * txfm_size_row + 1] = buf0[j * 4 + 1]; + _buf1[j * txfm_size_row + 2] = buf0[j * 4 + 2]; + _buf1[j * txfm_size_row + 3] = buf0[j * 4 + 3]; + } + } + for (int i = 0; i < buf_size_w_div4; i++) { + col_txfm(buf1 + i * txfm_size_row, buf1 + i * txfm_size_row, + inv_cos_bit_col[txw_idx][txh_idx], 1, bd, 0); + + av1_round_shift_array_32_sse4_1(buf1 + i * txfm_size_row, + buf1 + i * txfm_size_row, txfm_size_row, + -shift[1]); + } + + // write to buffer + for (int i = 0; i < (txfm_size_col >> 3); i++) { + highbd_write_buffer_8xn_sse4_1(buf1 + i * txfm_size_row * 2, output + 8 * i, + stride, ud_flip, txfm_size_row, bd); + } +} +static void highbd_inv_txfm2d_add_v_identity_ssse41(const int32_t *input, + uint16_t *output, + int stride, TX_TYPE tx_type, + TX_SIZE tx_size, int eob, + const int bd) { + __m128i buf1[64]; + int eobx, eoby; + get_eobx_eoby_scan_h_identity(&eobx, &eoby, tx_size, eob); + const int8_t *shift = inv_txfm_shift_ls[tx_size]; + const int txw_idx = get_txw_idx(tx_size); + const int txh_idx = get_txh_idx(tx_size); + const int txfm_size_col = tx_size_wide[tx_size]; + const int txfm_size_row = tx_size_high[tx_size]; + const int input_stride = AOMMIN(32, txfm_size_col); + const int buf_size_w_div8 = input_stride >> 2; + const int row_max = AOMMIN(32, txfm_size_row); + const int buf_size_nonzero_w_div8 = (eobx + 8) >> 3; + const int rect_type = get_rect_tx_log_ratio(txfm_size_col, txfm_size_row); + const int fun_idx = lowbd_txfm_all_1d_zeros_idx[eobx]; + const transform_1d_sse4_1 row_txfm = + highbd_txfm_all_1d_zeros_w8_arr[txw_idx][hitx_1d_tab[tx_type]][fun_idx]; + const transform_1d_sse4_1 col_txfm = + highbd_txfm_all_1d_zeros_w8_arr[txh_idx][vitx_1d_tab[tx_type]][0]; + int ud_flip, lr_flip; + get_flip_cfg(tx_type, &ud_flip, &lr_flip); + + for (int i = 0; i < (row_max >> 2); ++i) { + __m128i buf0[16]; + const int32_t *input_row = input + i * input_stride * 4; + for (int j = 0; j < (buf_size_nonzero_w_div8 << 1); ++j) { + __m128i *buf0_cur = buf0 + j * 4; + load_buffer_32bit_input(input_row + j * 4, input_stride, buf0_cur, 4); + + TRANSPOSE_4X4(buf0_cur[0], buf0_cur[1], buf0_cur[2], buf0_cur[3], + buf0_cur[0], buf0_cur[1], buf0_cur[2], buf0_cur[3]); + } + if (rect_type == 1 || rect_type == -1) { + av1_round_shift_rect_array_32_sse4_1( + buf0, buf0, (buf_size_nonzero_w_div8 << 3), 0, NewInvSqrt2); + } + row_txfm(buf0, buf0, inv_cos_bit_row[txw_idx][txh_idx], 0, bd, -shift[0]); + + __m128i *_buf1 = buf1 + i * 4; + if (lr_flip) { + for (int j = 0; j < buf_size_w_div8; ++j) { + TRANSPOSE_4X4(buf0[4 * j + 3], buf0[4 * j + 2], buf0[4 * j + 1], + buf0[4 * j], + _buf1[txfm_size_row * (buf_size_w_div8 - 1 - j) + 0], + _buf1[txfm_size_row * (buf_size_w_div8 - 1 - j) + 1], + _buf1[txfm_size_row * (buf_size_w_div8 - 1 - j) + 2], + _buf1[txfm_size_row * (buf_size_w_div8 - 1 - j) + 3]); + } + } else { + for (int j = 0; j < buf_size_w_div8; ++j) { + TRANSPOSE_4X4( + buf0[j * 4 + 0], buf0[j * 4 + 1], buf0[j * 4 + 2], buf0[j * 4 + 3], + _buf1[j * txfm_size_row + 0], _buf1[j * txfm_size_row + 1], + _buf1[j * txfm_size_row + 2], _buf1[j * txfm_size_row + 3]); + } + } + } + for (int i = 0; i < buf_size_w_div8; i++) { + col_txfm(buf1 + i * txfm_size_row, buf1 + i * txfm_size_row, + inv_cos_bit_col[txw_idx][txh_idx], 1, bd, 0); + + av1_round_shift_array_32_sse4_1(buf1 + i * txfm_size_row, + buf1 + i * txfm_size_row, txfm_size_row, + -shift[1]); + } + + // write to buffer + { + for (int i = 0; i < (txfm_size_col >> 3); i++) { + highbd_write_buffer_8xn_sse4_1(buf1 + i * txfm_size_row * 2, + output + 8 * i, stride, ud_flip, + txfm_size_row, bd); + } + } +} +static void highbd_inv_txfm2d_add_idtx_ssse41(const int32_t *input, + uint16_t *output, int stride, + TX_TYPE tx_type, TX_SIZE tx_size, + int eob, const int bd) { + (void)eob; + __m128i buf1[64 * 4]; + const int8_t *shift = inv_txfm_shift_ls[tx_size]; + const int txw_idx = get_txw_idx(tx_size); + const int txh_idx = get_txh_idx(tx_size); + const int txfm_size_col = tx_size_wide[tx_size]; + const int txfm_size_row = tx_size_high[tx_size]; + const int input_stride = AOMMIN(32, txfm_size_col); + const int row_max = AOMMIN(32, txfm_size_row); + const int rect_type = get_rect_tx_log_ratio(txfm_size_col, txfm_size_row); + const transform_1d_sse4_1 row_txfm = + highbd_txfm_all_1d_zeros_w8_arr[txw_idx][hitx_1d_tab[tx_type]][0]; + const transform_1d_sse4_1 col_txfm = + highbd_txfm_all_1d_zeros_w8_arr[txh_idx][vitx_1d_tab[tx_type]][0]; + + for (int i = 0; i < (row_max >> 2); ++i) { + __m128i buf0[32]; + const int32_t *input_row = input + i * input_stride * 4; + for (int j = 0; j < (input_stride >> 2); ++j) { + __m128i *buf0_cur = buf0 + j * 4; + load_buffer_32bit_input(input_row + j * 4, input_stride, buf0_cur, 4); + } + if (rect_type == 1 || rect_type == -1) { + av1_round_shift_rect_array_32_sse4_1(buf0, buf0, input_stride, 0, + NewInvSqrt2); + } + row_txfm(buf0, buf0, inv_cos_bit_row[txw_idx][txh_idx], 0, bd, -shift[0]); + + __m128i *_buf1 = buf1 + i * 4; + for (int j = 0; j < (input_stride >> 2); ++j) { + _buf1[j * txfm_size_row + 0] = buf0[j * 4 + 0]; + _buf1[j * txfm_size_row + 1] = buf0[j * 4 + 1]; + _buf1[j * txfm_size_row + 2] = buf0[j * 4 + 2]; + _buf1[j * txfm_size_row + 3] = buf0[j * 4 + 3]; + } + } + for (int i = 0; i < (input_stride >> 2); i++) { + col_txfm(buf1 + i * txfm_size_row, buf1 + i * txfm_size_row, + inv_cos_bit_col[txw_idx][txh_idx], 1, bd, 0); + + av1_round_shift_array_32_sse4_1(buf1 + i * txfm_size_row, + buf1 + i * txfm_size_row, txfm_size_row, + -shift[1]); + } + + // write to buffer + { + for (int i = 0; i < (txfm_size_col >> 3); i++) { + highbd_write_buffer_8xn_sse4_1(buf1 + i * txfm_size_row * 2, + output + 8 * i, stride, 0, txfm_size_row, + bd); + } + } +} static void highbd_inv_txfm2d_add_no_identity_sse41(const int32_t *input, uint16_t *output, int stride, TX_TYPE tx_type, @@ -5261,6 +5611,232 @@ static void highbd_inv_txfm2d_add_no_identity_sse41(const int32_t *input, } } +static void highbd_inv_txfm2d_add_4x8_sse41(const int32_t *input, + uint16_t *output, int stride, + TX_TYPE tx_type, TX_SIZE tx_size, + int eob, const int bd) { + (void)eob; + __m128i buf1[8]; + const int8_t *shift = inv_txfm_shift_ls[tx_size]; + const int txw_idx = get_txw_idx(tx_size); + const int txh_idx = get_txh_idx(tx_size); + const int txfm_size_col = tx_size_wide[tx_size]; + const int txfm_size_row = tx_size_high[tx_size]; + const transform_1d_sse4_1 row_txfm = + highbd_txfm_all_1d_zeros_w8_arr[txw_idx][hitx_1d_tab[tx_type]][0]; + const transform_1d_sse4_1 col_txfm = + highbd_txfm_all_1d_zeros_w8_arr[txh_idx][vitx_1d_tab[tx_type]][1]; + const int input_stride = AOMMIN(32, txfm_size_col); + + assert(col_txfm != NULL); + assert(row_txfm != NULL); + int ud_flip, lr_flip; + get_flip_cfg(tx_type, &ud_flip, &lr_flip); + + // 1st stage: column transform + __m128i buf0[8]; + const int32_t *input_row = input; + __m128i *buf0_cur = buf0; + load_buffer_32bit_input(input_row, input_stride, buf0_cur, txfm_size_row); + av1_round_shift_rect_array_32_sse4_1(buf0, buf0, txfm_size_row, 0, + NewInvSqrt2); + row_txfm(buf0, buf0, inv_cos_bit_row[txw_idx][txh_idx], 0, bd, -shift[0]); + row_txfm(buf0 + 4, buf0 + 4, inv_cos_bit_row[txw_idx][txh_idx], 0, bd, + -shift[0]); + + if (lr_flip) { + TRANSPOSE_4X4(buf0[3], buf0[2], buf0[1], buf0[0], buf1[0], buf1[1], buf1[2], + buf1[3]); + + TRANSPOSE_4X4(buf0[7], buf0[6], buf0[5], buf0[4], buf1[4], buf1[5], buf1[6], + buf1[7]); + } else { + TRANSPOSE_4X4(buf0[0], buf0[1], buf0[2], buf0[3], buf1[0], buf1[1], buf1[2], + buf1[3]); + + TRANSPOSE_4X4(buf0[4], buf0[5], buf0[6], buf0[7], buf1[4], buf1[5], buf1[6], + buf1[7]); + } + + // 2nd stage: column transform + col_txfm(buf1, buf1, inv_cos_bit_col[txw_idx][txh_idx], 1, bd, 0); + + av1_round_shift_array_32_sse4_1(buf1, buf1, txfm_size_row, -shift[1]); + + // write to buffer + highbd_write_buffer_4xn_sse4_1(buf1, output, stride, ud_flip, txfm_size_row, + bd); +} + +static void highbd_inv_txfm2d_add_8x4_sse41(const int32_t *input, + uint16_t *output, int stride, + TX_TYPE tx_type, TX_SIZE tx_size, + int eob, const int bd) { + (void)eob; + __m128i buf1[8]; + const int8_t *shift = inv_txfm_shift_ls[tx_size]; + const int txw_idx = get_txw_idx(tx_size); + const int txh_idx = get_txh_idx(tx_size); + const int txfm_size_col = tx_size_wide[tx_size]; + const int txfm_size_row = tx_size_high[tx_size]; + const transform_1d_sse4_1 row_txfm = + highbd_txfm_all_1d_zeros_w8_arr[txw_idx][hitx_1d_tab[tx_type]][1]; + const transform_1d_sse4_1 col_txfm = + highbd_txfm_all_1d_zeros_w8_arr[txh_idx][vitx_1d_tab[tx_type]][0]; + + assert(col_txfm != NULL); + assert(row_txfm != NULL); + int ud_flip, lr_flip; + get_flip_cfg(tx_type, &ud_flip, &lr_flip); + + // 1st stage: column transform + __m128i buf0[8]; + const int32_t *input_row = input; + load_buffer_32bit_input(input_row, 4, buf0, txfm_size_col); + + TRANSPOSE_4X4(buf0[0], buf0[2], buf0[4], buf0[6], buf1[0], buf1[1], buf1[2], + buf1[3]); + TRANSPOSE_4X4(buf0[1], buf0[3], buf0[5], buf0[7], buf1[4], buf1[5], buf1[6], + buf1[7]); + + av1_round_shift_rect_array_32_sse4_1(buf1, buf0, txfm_size_col, 0, + NewInvSqrt2); + row_txfm(buf0, buf0, inv_cos_bit_row[txw_idx][txh_idx], 0, bd, -shift[0]); + + __m128i *buf1_ptr; + if (lr_flip) { + flip_buf_sse2(buf0, buf1, txfm_size_col); + buf1_ptr = buf1; + } else { + buf1_ptr = buf0; + } + + // 2nd stage: column transform + for (int i = 0; i < 2; i++) { + col_txfm(buf1_ptr + i * txfm_size_row, buf1_ptr + i * txfm_size_row, + inv_cos_bit_col[txw_idx][txh_idx], 1, bd, 0); + } + av1_round_shift_array_32_sse4_1(buf1_ptr, buf1_ptr, txfm_size_col, -shift[1]); + // write to buffer + highbd_write_buffer_8xn_sse4_1(buf1_ptr, output, stride, ud_flip, + txfm_size_row, bd); +} + +static void highbd_inv_txfm2d_add_4x16_sse4_1(const int32_t *input, + uint16_t *output, int stride, + TX_TYPE tx_type, TX_SIZE tx_size, + int eob, const int bd) { + (void)eob; + __m128i buf1[16]; + const int8_t *shift = inv_txfm_shift_ls[tx_size]; + const int txw_idx = get_txw_idx(tx_size); + const int txh_idx = get_txh_idx(tx_size); + const int txfm_size_col = tx_size_wide[tx_size]; + const int txfm_size_row = tx_size_high[tx_size]; + const int buf_size_h_div8 = txfm_size_row >> 2; + const transform_1d_sse4_1 row_txfm = + highbd_txfm_all_1d_zeros_w8_arr[txw_idx][hitx_1d_tab[tx_type]][0]; + const transform_1d_sse4_1 col_txfm = + highbd_txfm_all_1d_zeros_w8_arr[txh_idx][vitx_1d_tab[tx_type]][2]; + const int input_stride = AOMMIN(32, txfm_size_col); + + assert(col_txfm != NULL); + assert(row_txfm != NULL); + int ud_flip, lr_flip; + get_flip_cfg(tx_type, &ud_flip, &lr_flip); + + // 1st stage: column transform + __m128i buf0[16]; + const int32_t *input_row = input; + __m128i *buf0_cur = buf0; + load_buffer_32bit_input(input_row, input_stride, buf0_cur, txfm_size_row); + for (int i = 0; i < (txfm_size_row >> 2); i++) { + row_txfm(buf0 + (i << 2), buf0 + (i << 2), + inv_cos_bit_row[txw_idx][txh_idx], 0, bd, -shift[0]); + } + + av1_round_shift_array_32_sse4_1(buf0, buf0, txfm_size_row, -shift[0]); + + if (lr_flip) { + for (int j = 0; j < buf_size_h_div8; ++j) { + TRANSPOSE_4X4(buf0[4 * j + 3], buf0[4 * j + 2], buf0[4 * j + 1], + buf0[4 * j], buf1[4 * j], buf1[4 * j + 1], buf1[4 * j + 2], + buf1[4 * j + 3]); + } + } else { + for (int j = 0; j < buf_size_h_div8; ++j) { + TRANSPOSE_4X4(buf0[4 * j], buf0[4 * j + 1], buf0[4 * j + 2], + buf0[4 * j + 3], buf1[4 * j], buf1[4 * j + 1], + buf1[4 * j + 2], buf1[4 * j + 3]); + } + } + + // 2nd stage: column transform + col_txfm(buf1, buf1, inv_cos_bit_col[txw_idx][txh_idx], 1, bd, 0); + + av1_round_shift_array_32_sse4_1(buf1, buf1, txfm_size_row, -shift[1]); + + // write to buffer + highbd_write_buffer_4xn_sse4_1(buf1, output, stride, ud_flip, txfm_size_row, + bd); +} + +static void highbd_inv_txfm2d_add_16x4_sse4_1(const int32_t *input, + uint16_t *output, int stride, + TX_TYPE tx_type, TX_SIZE tx_size, + int eob, const int bd) { + (void)eob; + __m128i buf1[16]; + const int8_t *shift = inv_txfm_shift_ls[tx_size]; + const int txw_idx = get_txw_idx(tx_size); + const int txh_idx = get_txh_idx(tx_size); + const int txfm_size_col = tx_size_wide[tx_size]; + const int txfm_size_row = tx_size_high[tx_size]; + const int buf_size_w_div8 = txfm_size_col >> 2; + const transform_1d_sse4_1 row_txfm = + highbd_txfm_all_1d_zeros_w8_arr[txw_idx][hitx_1d_tab[tx_type]][2]; + const transform_1d_sse4_1 col_txfm = + highbd_txfm_all_1d_zeros_w8_arr[txh_idx][vitx_1d_tab[tx_type]][0]; + + assert(col_txfm != NULL); + assert(row_txfm != NULL); + int ud_flip, lr_flip; + get_flip_cfg(tx_type, &ud_flip, &lr_flip); + + // 1st stage: column transform + __m128i buf0[16]; + const int32_t *input_row = input; + load_buffer_32bit_input(input_row, 4, buf0, txfm_size_col); + + for (int j = 0; j < buf_size_w_div8; j++) { + TRANSPOSE_4X4(buf0[j], buf0[j + 4], buf0[j + 8], buf0[j + 12], buf1[4 * j], + buf1[4 * j + 1], buf1[4 * j + 2], buf1[4 * j + 3]); + } + row_txfm(buf1, buf0, inv_cos_bit_row[txw_idx][txh_idx], 0, bd, -shift[0]); + + __m128i *buf1_ptr; + if (lr_flip) { + flip_buf_sse2(buf0, buf1, txfm_size_col); + buf1_ptr = buf1; + } else { + buf1_ptr = buf0; + } + + // 2nd stage: column transform + for (int i = 0; i < buf_size_w_div8; i++) { + col_txfm(buf1_ptr + i * txfm_size_row, buf1_ptr + i * txfm_size_row, + inv_cos_bit_col[txw_idx][txh_idx], 1, bd, 0); + } + av1_round_shift_array_32_sse4_1(buf1_ptr, buf1_ptr, txfm_size_col, -shift[1]); + + // write to buffer + for (int i = 0; i < (txfm_size_col >> 3); i++) { + highbd_write_buffer_8xn_sse4_1(buf1_ptr + i * txfm_size_row * 2, + output + 8 * i, stride, ud_flip, + txfm_size_row, bd); + } +} + void av1_highbd_inv_txfm2d_add_universe_sse4_1(const int32_t *input, uint8_t *output, int stride, TX_TYPE tx_type, TX_SIZE tx_size, @@ -5279,70 +5855,99 @@ void av1_highbd_inv_txfm2d_add_universe_sse4_1(const int32_t *input, input, CONVERT_TO_SHORTPTR(output), stride, tx_type, tx_size, eob, bd); break; + case V_DCT: + case V_ADST: + case V_FLIPADST: + highbd_inv_txfm2d_add_h_identity_ssse41( + input, CONVERT_TO_SHORTPTR(output), stride, tx_type, tx_size, eob, + bd); + break; + case H_DCT: + case H_ADST: + case H_FLIPADST: + highbd_inv_txfm2d_add_v_identity_ssse41( + input, CONVERT_TO_SHORTPTR(output), stride, tx_type, tx_size, eob, + bd); + break; + case IDTX: + highbd_inv_txfm2d_add_idtx_ssse41(input, CONVERT_TO_SHORTPTR(output), + stride, tx_type, tx_size, eob, bd); + break; default: assert(0); break; } } +void av1_highbd_inv_txfm_add_4x8_sse4_1(const tran_low_t *input, uint8_t *dest, + int stride, + const TxfmParam *txfm_param) { + int bd = txfm_param->bd; + const TX_TYPE tx_type = txfm_param->tx_type; + const TX_SIZE tx_size = txfm_param->tx_size; + int eob = txfm_param->eob; + highbd_inv_txfm2d_add_4x8_sse41(input, CONVERT_TO_SHORTPTR(dest), stride, + tx_type, tx_size, eob, bd); +} + +void av1_highbd_inv_txfm_add_8x4_sse4_1(const tran_low_t *input, uint8_t *dest, + int stride, + const TxfmParam *txfm_param) { + int bd = txfm_param->bd; + const TX_TYPE tx_type = txfm_param->tx_type; + const TX_SIZE tx_size = txfm_param->tx_size; + int eob = txfm_param->eob; + highbd_inv_txfm2d_add_8x4_sse41(input, CONVERT_TO_SHORTPTR(dest), stride, + tx_type, tx_size, eob, bd); +} + +void av1_highbd_inv_txfm_add_4x16_sse4_1(const tran_low_t *input, uint8_t *dest, + int stride, + const TxfmParam *txfm_param) { + int bd = txfm_param->bd; + const TX_TYPE tx_type = txfm_param->tx_type; + const TX_SIZE tx_size = txfm_param->tx_size; + int eob = txfm_param->eob; + highbd_inv_txfm2d_add_4x16_sse4_1(input, CONVERT_TO_SHORTPTR(dest), stride, + tx_type, tx_size, eob, bd); +} + +void av1_highbd_inv_txfm_add_16x4_sse4_1(const tran_low_t *input, uint8_t *dest, + int stride, + const TxfmParam *txfm_param) { + int bd = txfm_param->bd; + const TX_TYPE tx_type = txfm_param->tx_type; + const TX_SIZE tx_size = txfm_param->tx_size; + int eob = txfm_param->eob; + highbd_inv_txfm2d_add_16x4_sse4_1(input, CONVERT_TO_SHORTPTR(dest), stride, + tx_type, tx_size, eob, bd); +} + void av1_highbd_inv_txfm_add_sse4_1(const tran_low_t *input, uint8_t *dest, int stride, const TxfmParam *txfm_param) { assert(av1_ext_tx_used[txfm_param->tx_set_type][txfm_param->tx_type]); const TX_SIZE tx_size = txfm_param->tx_size; switch (tx_size) { - case TX_32X32: - av1_highbd_inv_txfm_add_32x32_sse4_1(input, dest, stride, txfm_param); - break; - case TX_16X16: - av1_highbd_inv_txfm_add_16x16_sse4_1(input, dest, stride, txfm_param); - break; case TX_8X8: av1_highbd_inv_txfm_add_8x8_sse4_1(input, dest, stride, txfm_param); break; case TX_4X8: - av1_highbd_inv_txfm_add_4x8(input, dest, stride, txfm_param); + av1_highbd_inv_txfm_add_4x8_sse4_1(input, dest, stride, txfm_param); break; case TX_8X4: - av1_highbd_inv_txfm_add_8x4(input, dest, stride, txfm_param); - break; - case TX_8X16: - av1_highbd_inv_txfm_add_8x16_sse4_1(input, dest, stride, txfm_param); - break; - case TX_16X8: - av1_highbd_inv_txfm_add_16x8_sse4_1(input, dest, stride, txfm_param); - break; - case TX_16X32: - av1_highbd_inv_txfm_add_16x32(input, dest, stride, txfm_param); - break; - case TX_32X16: - av1_highbd_inv_txfm_add_32x16(input, dest, stride, txfm_param); - break; - case TX_32X64: - av1_highbd_inv_txfm_add_32x64(input, dest, stride, txfm_param); - break; - case TX_64X32: - av1_highbd_inv_txfm_add_64x32(input, dest, stride, txfm_param); + av1_highbd_inv_txfm_add_8x4_sse4_1(input, dest, stride, txfm_param); break; case TX_4X4: av1_highbd_inv_txfm_add_4x4_sse4_1(input, dest, stride, txfm_param); break; case TX_16X4: - av1_highbd_inv_txfm_add_16x4(input, dest, stride, txfm_param); + av1_highbd_inv_txfm_add_16x4_sse4_1(input, dest, stride, txfm_param); break; case TX_4X16: - av1_highbd_inv_txfm_add_4x16(input, dest, stride, txfm_param); + av1_highbd_inv_txfm_add_4x16_sse4_1(input, dest, stride, txfm_param); break; - case TX_8X32: - av1_highbd_inv_txfm_add_8x32(input, dest, stride, txfm_param); - break; - case TX_32X8: - av1_highbd_inv_txfm_add_32x8(input, dest, stride, txfm_param); - break; - case TX_64X64: - case TX_16X64: - case TX_64X16: + default: av1_highbd_inv_txfm2d_add_universe_sse4_1( - input, dest, stride, txfm_param->tx_type, txfm_param->tx_size, - txfm_param->eob, txfm_param->bd); + input, dest, stride, txfm_param->tx_type, tx_size, txfm_param->eob, + txfm_param->bd); break; - default: assert(0 && "Invalid transform size"); break; } } diff --git a/third_party/aom/av1/common/x86/highbd_jnt_convolve_avx2.c b/third_party/aom/av1/common/x86/highbd_jnt_convolve_avx2.c index e298cf6531..c5040c414e 100644 --- a/third_party/aom/av1/common/x86/highbd_jnt_convolve_avx2.c +++ b/third_party/aom/av1/common/x86/highbd_jnt_convolve_avx2.c @@ -22,7 +22,7 @@ #include "aom_dsp/aom_filter.h" #include "av1/common/convolve.h" -void av1_highbd_jnt_convolve_2d_copy_avx2( +void av1_highbd_dist_wtd_convolve_2d_copy_avx2( const uint16_t *src, int src_stride, uint16_t *dst0, int dst_stride0, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, @@ -38,7 +38,7 @@ void av1_highbd_jnt_convolve_2d_copy_avx2( FILTER_BITS * 2 - conv_params->round_1 - conv_params->round_0; const __m128i left_shift = _mm_cvtsi32_si128(bits); const int do_average = conv_params->do_average; - const int use_jnt_comp_avg = conv_params->use_jnt_comp_avg; + const int use_dist_wtd_comp_avg = conv_params->use_dist_wtd_comp_avg; const int w0 = conv_params->fwd_offset; const int w1 = conv_params->bck_offset; const __m256i wt0 = _mm256_set1_epi32(w0); @@ -78,15 +78,17 @@ void av1_highbd_jnt_convolve_2d_copy_avx2( const __m256i res_unsigned_lo = _mm256_add_epi32(res_32b_lo, offset_const); - const __m256i comp_avg_res_lo = highbd_comp_avg( - &data_ref_0_lo, &res_unsigned_lo, &wt0, &wt1, use_jnt_comp_avg); + const __m256i comp_avg_res_lo = + highbd_comp_avg(&data_ref_0_lo, &res_unsigned_lo, &wt0, &wt1, + use_dist_wtd_comp_avg); const __m256i res_32b_hi = _mm256_unpackhi_epi16(res, zero); const __m256i res_unsigned_hi = _mm256_add_epi32(res_32b_hi, offset_const); - const __m256i comp_avg_res_hi = highbd_comp_avg( - &data_ref_0_hi, &res_unsigned_hi, &wt0, &wt1, use_jnt_comp_avg); + const __m256i comp_avg_res_hi = + highbd_comp_avg(&data_ref_0_hi, &res_unsigned_hi, &wt0, &wt1, + use_dist_wtd_comp_avg); const __m256i round_result_lo = highbd_convolve_rounding( &comp_avg_res_lo, &offset_const, &rounding_const, rounding_shift); @@ -135,8 +137,9 @@ void av1_highbd_jnt_convolve_2d_copy_avx2( const __m256i res_unsigned_lo = _mm256_add_epi32(res_32b, offset_const); - const __m256i comp_avg_res = highbd_comp_avg( - &data_ref_0, &res_unsigned_lo, &wt0, &wt1, use_jnt_comp_avg); + const __m256i comp_avg_res = + highbd_comp_avg(&data_ref_0, &res_unsigned_lo, &wt0, &wt1, + use_dist_wtd_comp_avg); const __m256i round_result = highbd_convolve_rounding( &comp_avg_res, &offset_const, &rounding_const, rounding_shift); @@ -179,15 +182,17 @@ void av1_highbd_jnt_convolve_2d_copy_avx2( const __m256i res_unsigned_lo = _mm256_add_epi32(res_32b_lo, offset_const); - const __m256i comp_avg_res_lo = highbd_comp_avg( - &data_ref_0_lo, &res_unsigned_lo, &wt0, &wt1, use_jnt_comp_avg); + const __m256i comp_avg_res_lo = + highbd_comp_avg(&data_ref_0_lo, &res_unsigned_lo, &wt0, &wt1, + use_dist_wtd_comp_avg); const __m256i res_32b_hi = _mm256_unpackhi_epi16(res, zero); const __m256i res_unsigned_hi = _mm256_add_epi32(res_32b_hi, offset_const); - const __m256i comp_avg_res_hi = highbd_comp_avg( - &data_ref_0_hi, &res_unsigned_hi, &wt0, &wt1, use_jnt_comp_avg); + const __m256i comp_avg_res_hi = + highbd_comp_avg(&data_ref_0_hi, &res_unsigned_hi, &wt0, &wt1, + use_dist_wtd_comp_avg); const __m256i round_result_lo = highbd_convolve_rounding(&comp_avg_res_lo, &offset_const, @@ -223,7 +228,7 @@ void av1_highbd_jnt_convolve_2d_copy_avx2( } } -void av1_highbd_jnt_convolve_2d_avx2( +void av1_highbd_dist_wtd_convolve_2d_avx2( const uint16_t *src, int src_stride, uint16_t *dst0, int dst_stride0, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, @@ -244,7 +249,7 @@ void av1_highbd_jnt_convolve_2d_avx2( __m256i s[8], coeffs_y[4], coeffs_x[4]; const int do_average = conv_params->do_average; - const int use_jnt_comp_avg = conv_params->use_jnt_comp_avg; + const int use_dist_wtd_comp_avg = conv_params->use_dist_wtd_comp_avg; const int w0 = conv_params->fwd_offset; const int w1 = conv_params->bck_offset; @@ -364,8 +369,9 @@ void av1_highbd_jnt_convolve_2d_avx2( const __m256i data_ref_0 = _mm256_unpacklo_epi16(data_01, zero); - const __m256i comp_avg_res = highbd_comp_avg( - &data_ref_0, &res_unsigned_lo, &wt0, &wt1, use_jnt_comp_avg); + const __m256i comp_avg_res = + highbd_comp_avg(&data_ref_0, &res_unsigned_lo, &wt0, &wt1, + use_dist_wtd_comp_avg); const __m256i round_result = highbd_convolve_rounding( &comp_avg_res, &offset_const, &rounding_const, rounding_shift); @@ -409,10 +415,12 @@ void av1_highbd_jnt_convolve_2d_avx2( const __m256i data_ref_0_lo = _mm256_unpacklo_epi16(data_01, zero); const __m256i data_ref_0_hi = _mm256_unpackhi_epi16(data_01, zero); - const __m256i comp_avg_res_lo = highbd_comp_avg( - &data_ref_0_lo, &res_unsigned_lo, &wt0, &wt1, use_jnt_comp_avg); - const __m256i comp_avg_res_hi = highbd_comp_avg( - &data_ref_0_hi, &res_unsigned_hi, &wt0, &wt1, use_jnt_comp_avg); + const __m256i comp_avg_res_lo = + highbd_comp_avg(&data_ref_0_lo, &res_unsigned_lo, &wt0, &wt1, + use_dist_wtd_comp_avg); + const __m256i comp_avg_res_hi = + highbd_comp_avg(&data_ref_0_hi, &res_unsigned_hi, &wt0, &wt1, + use_dist_wtd_comp_avg); const __m256i round_result_lo = highbd_convolve_rounding(&comp_avg_res_lo, &offset_const, @@ -456,7 +464,7 @@ void av1_highbd_jnt_convolve_2d_avx2( } } -void av1_highbd_jnt_convolve_x_avx2( +void av1_highbd_dist_wtd_convolve_x_avx2( const uint16_t *src, int src_stride, uint16_t *dst0, int dst_stride0, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, @@ -473,7 +481,7 @@ void av1_highbd_jnt_convolve_x_avx2( __m256i s[4], coeffs_x[4]; const int do_average = conv_params->do_average; - const int use_jnt_comp_avg = conv_params->use_jnt_comp_avg; + const int use_dist_wtd_comp_avg = conv_params->use_dist_wtd_comp_avg; const int w0 = conv_params->fwd_offset; const int w1 = conv_params->bck_offset; const __m256i wt0 = _mm256_set1_epi32(w0); @@ -548,7 +556,7 @@ void av1_highbd_jnt_convolve_x_avx2( const __m256i data_ref_0 = _mm256_unpacklo_epi16(data_01, zero); const __m256i comp_avg_res = highbd_comp_avg( - &data_ref_0, &res_unsigned_lo, &wt0, &wt1, use_jnt_comp_avg); + &data_ref_0, &res_unsigned_lo, &wt0, &wt1, use_dist_wtd_comp_avg); const __m256i round_result = highbd_convolve_rounding( &comp_avg_res, &offset_const, &rounding_const, rounding_shift); @@ -588,10 +596,12 @@ void av1_highbd_jnt_convolve_x_avx2( const __m256i data_ref_0_lo = _mm256_unpacklo_epi16(data_01, zero); const __m256i data_ref_0_hi = _mm256_unpackhi_epi16(data_01, zero); - const __m256i comp_avg_res_lo = highbd_comp_avg( - &data_ref_0_lo, &res_unsigned_lo, &wt0, &wt1, use_jnt_comp_avg); - const __m256i comp_avg_res_hi = highbd_comp_avg( - &data_ref_0_hi, &res_unsigned_hi, &wt0, &wt1, use_jnt_comp_avg); + const __m256i comp_avg_res_lo = + highbd_comp_avg(&data_ref_0_lo, &res_unsigned_lo, &wt0, &wt1, + use_dist_wtd_comp_avg); + const __m256i comp_avg_res_hi = + highbd_comp_avg(&data_ref_0_hi, &res_unsigned_hi, &wt0, &wt1, + use_dist_wtd_comp_avg); const __m256i round_result_lo = highbd_convolve_rounding( &comp_avg_res_lo, &offset_const, &rounding_const, rounding_shift); @@ -623,7 +633,7 @@ void av1_highbd_jnt_convolve_x_avx2( } } -void av1_highbd_jnt_convolve_y_avx2( +void av1_highbd_dist_wtd_convolve_y_avx2( const uint16_t *src, int src_stride, uint16_t *dst0, int dst_stride0, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, @@ -640,7 +650,7 @@ void av1_highbd_jnt_convolve_y_avx2( int i, j; __m256i s[8], coeffs_y[4]; const int do_average = conv_params->do_average; - const int use_jnt_comp_avg = conv_params->use_jnt_comp_avg; + const int use_dist_wtd_comp_avg = conv_params->use_dist_wtd_comp_avg; const int w0 = conv_params->fwd_offset; const int w1 = conv_params->bck_offset; @@ -753,8 +763,9 @@ void av1_highbd_jnt_convolve_y_avx2( const __m256i data_ref_0 = _mm256_unpacklo_epi16(data_01, zero); - const __m256i comp_avg_res = highbd_comp_avg( - &data_ref_0, &res_unsigned_lo, &wt0, &wt1, use_jnt_comp_avg); + const __m256i comp_avg_res = + highbd_comp_avg(&data_ref_0, &res_unsigned_lo, &wt0, &wt1, + use_dist_wtd_comp_avg); const __m256i round_result = highbd_convolve_rounding( &comp_avg_res, &offset_const, &rounding_const, rounding_shift); @@ -799,10 +810,12 @@ void av1_highbd_jnt_convolve_y_avx2( const __m256i data_ref_0_lo = _mm256_unpacklo_epi16(data_01, zero); const __m256i data_ref_0_hi = _mm256_unpackhi_epi16(data_01, zero); - const __m256i comp_avg_res_lo = highbd_comp_avg( - &data_ref_0_lo, &res_unsigned_lo, &wt0, &wt1, use_jnt_comp_avg); - const __m256i comp_avg_res_hi = highbd_comp_avg( - &data_ref_0_hi, &res_unsigned_hi, &wt0, &wt1, use_jnt_comp_avg); + const __m256i comp_avg_res_lo = + highbd_comp_avg(&data_ref_0_lo, &res_unsigned_lo, &wt0, &wt1, + use_dist_wtd_comp_avg); + const __m256i comp_avg_res_hi = + highbd_comp_avg(&data_ref_0_hi, &res_unsigned_hi, &wt0, &wt1, + use_dist_wtd_comp_avg); const __m256i round_result_lo = highbd_convolve_rounding(&comp_avg_res_lo, &offset_const, diff --git a/third_party/aom/av1/common/x86/highbd_jnt_convolve_sse4.c b/third_party/aom/av1/common/x86/highbd_jnt_convolve_sse4.c index 1a29985b56..7fea36a8ac 100644 --- a/third_party/aom/av1/common/x86/highbd_jnt_convolve_sse4.c +++ b/third_party/aom/av1/common/x86/highbd_jnt_convolve_sse4.c @@ -17,7 +17,7 @@ #include "aom_dsp/x86/convolve_sse2.h" #include "aom_dsp/x86/convolve_sse4_1.h" -void av1_highbd_jnt_convolve_y_sse4_1( +void av1_highbd_dist_wtd_convolve_y_sse4_1( const uint16_t *src, int src_stride, uint16_t *dst0, int dst_stride0, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, @@ -33,7 +33,7 @@ void av1_highbd_jnt_convolve_y_sse4_1( assert(bits >= 0); int i, j; const int do_average = conv_params->do_average; - const int use_jnt_comp_avg = conv_params->use_jnt_comp_avg; + const int use_dist_wtd_comp_avg = conv_params->use_dist_wtd_comp_avg; const int w0 = conv_params->fwd_offset; const int w1 = conv_params->bck_offset; @@ -121,10 +121,12 @@ void av1_highbd_jnt_convolve_y_sse4_1( const __m128i data_ref_0 = _mm_unpacklo_epi16(data_0, zero); const __m128i data_ref_1 = _mm_unpacklo_epi16(data_1, zero); - const __m128i comp_avg_res_0 = highbd_comp_avg_sse4_1( - &data_ref_0, &res_unsigned_lo_0, &wt0, &wt1, use_jnt_comp_avg); - const __m128i comp_avg_res_1 = highbd_comp_avg_sse4_1( - &data_ref_1, &res_unsigned_lo_1, &wt0, &wt1, use_jnt_comp_avg); + const __m128i comp_avg_res_0 = + highbd_comp_avg_sse4_1(&data_ref_0, &res_unsigned_lo_0, &wt0, + &wt1, use_dist_wtd_comp_avg); + const __m128i comp_avg_res_1 = + highbd_comp_avg_sse4_1(&data_ref_1, &res_unsigned_lo_1, &wt0, + &wt1, use_dist_wtd_comp_avg); const __m128i round_result_0 = highbd_convolve_rounding_sse2(&comp_avg_res_0, &offset_const, @@ -186,16 +188,16 @@ void av1_highbd_jnt_convolve_y_sse4_1( const __m128i comp_avg_res_lo_0 = highbd_comp_avg_sse4_1(&data_ref_0_lo_0, &res_unsigned_lo_0, - &wt0, &wt1, use_jnt_comp_avg); + &wt0, &wt1, use_dist_wtd_comp_avg); const __m128i comp_avg_res_lo_1 = highbd_comp_avg_sse4_1(&data_ref_0_lo_1, &res_unsigned_lo_1, - &wt0, &wt1, use_jnt_comp_avg); + &wt0, &wt1, use_dist_wtd_comp_avg); const __m128i comp_avg_res_hi_0 = highbd_comp_avg_sse4_1(&data_ref_0_hi_0, &res_unsigned_hi_0, - &wt0, &wt1, use_jnt_comp_avg); + &wt0, &wt1, use_dist_wtd_comp_avg); const __m128i comp_avg_res_hi_1 = highbd_comp_avg_sse4_1(&data_ref_0_hi_1, &res_unsigned_hi_1, - &wt0, &wt1, use_jnt_comp_avg); + &wt0, &wt1, use_dist_wtd_comp_avg); const __m128i round_result_lo_0 = highbd_convolve_rounding_sse2(&comp_avg_res_lo_0, &offset_const, @@ -257,7 +259,7 @@ void av1_highbd_jnt_convolve_y_sse4_1( } } -void av1_highbd_jnt_convolve_x_sse4_1( +void av1_highbd_dist_wtd_convolve_x_sse4_1( const uint16_t *src, int src_stride, uint16_t *dst0, int dst_stride0, int w, int h, const InterpFilterParams *filter_params_x, const InterpFilterParams *filter_params_y, const int subpel_x_q4, @@ -274,7 +276,7 @@ void av1_highbd_jnt_convolve_x_sse4_1( __m128i s[4], coeffs_x[4]; const int do_average = conv_params->do_average; - const int use_jnt_comp_avg = conv_params->use_jnt_comp_avg; + const int use_dist_wtd_comp_avg = conv_params->use_dist_wtd_comp_avg; const int w0 = conv_params->fwd_offset; const int w1 = conv_params->bck_offset; const __m128i wt0 = _mm_set1_epi32(w0); @@ -339,7 +341,7 @@ void av1_highbd_jnt_convolve_x_sse4_1( const __m128i data_ref_0 = _mm_unpacklo_epi16(data_0, zero); const __m128i comp_avg_res = highbd_comp_avg_sse4_1( - &data_ref_0, &res_unsigned_lo, &wt0, &wt1, use_jnt_comp_avg); + &data_ref_0, &res_unsigned_lo, &wt0, &wt1, use_dist_wtd_comp_avg); const __m128i round_result = highbd_convolve_rounding_sse2( &comp_avg_res, &offset_const, &rounding_const, rounding_shift); @@ -359,10 +361,12 @@ void av1_highbd_jnt_convolve_x_sse4_1( const __m128i data_ref_0_lo = _mm_unpacklo_epi16(data_0, zero); const __m128i data_ref_0_hi = _mm_unpackhi_epi16(data_0, zero); - const __m128i comp_avg_res_lo = highbd_comp_avg_sse4_1( - &data_ref_0_lo, &res_unsigned_lo, &wt0, &wt1, use_jnt_comp_avg); - const __m128i comp_avg_res_hi = highbd_comp_avg_sse4_1( - &data_ref_0_hi, &res_unsigned_hi, &wt0, &wt1, use_jnt_comp_avg); + const __m128i comp_avg_res_lo = + highbd_comp_avg_sse4_1(&data_ref_0_lo, &res_unsigned_lo, &wt0, + &wt1, use_dist_wtd_comp_avg); + const __m128i comp_avg_res_hi = + highbd_comp_avg_sse4_1(&data_ref_0_hi, &res_unsigned_hi, &wt0, + &wt1, use_dist_wtd_comp_avg); const __m128i round_result_lo = highbd_convolve_rounding_sse2( &comp_avg_res_lo, &offset_const, &rounding_const, rounding_shift); diff --git a/third_party/aom/av1/common/x86/highbd_txfm_utility_sse4.h b/third_party/aom/av1/common/x86/highbd_txfm_utility_sse4.h index 6f24e5948a..5734810f52 100644 --- a/third_party/aom/av1/common/x86/highbd_txfm_utility_sse4.h +++ b/third_party/aom/av1/common/x86/highbd_txfm_utility_sse4.h @@ -75,13 +75,20 @@ static INLINE void transpose_16x16(const __m128i *in, __m128i *out) { out[63]); } -static INLINE void transpose_32x32(const __m128i *input, __m128i *output) { - for (int j = 0; j < 8; j++) { - for (int i = 0; i < 8; i++) { - TRANSPOSE_4X4(input[i * 32 + j + 0], input[i * 32 + j + 8], - input[i * 32 + j + 16], input[i * 32 + j + 24], - output[j * 32 + i + 0], output[j * 32 + i + 8], - output[j * 32 + i + 16], output[j * 32 + i + 24]); +static INLINE void transpose_8nx8n(const __m128i *input, __m128i *output, + const int width, const int height) { + const int numcol = height >> 2; + const int numrow = width >> 2; + for (int j = 0; j < numrow; j++) { + for (int i = 0; i < numcol; i++) { + TRANSPOSE_4X4(input[i * width + j + (numrow * 0)], + input[i * width + j + (numrow * 1)], + input[i * width + j + (numrow * 2)], + input[i * width + j + (numrow * 3)], + output[j * height + i + (numcol * 0)], + output[j * height + i + (numcol * 1)], + output[j * height + i + (numcol * 2)], + output[j * height + i + (numcol * 3)]); } } } diff --git a/third_party/aom/av1/common/x86/highbd_warp_plane_sse4.c b/third_party/aom/av1/common/x86/highbd_warp_plane_sse4.c index 4bcab05645..3765c5ef19 100644 --- a/third_party/aom/av1/common/x86/highbd_warp_plane_sse4.c +++ b/third_party/aom/av1/common/x86/highbd_warp_plane_sse4.c @@ -537,7 +537,7 @@ void av1_highbd_warp_affine_sse4_1(const int32_t *mat, const uint16_t *ref, __m128i *const dst16 = (__m128i *)&pred[(i + k + 4) * p_stride + j]; __m128i p_32 = _mm_cvtepu16_epi32(_mm_loadl_epi64(p)); - if (conv_params->use_jnt_comp_avg) { + if (conv_params->use_dist_wtd_comp_avg) { res_lo = _mm_add_epi32(_mm_mullo_epi32(p_32, wt0), _mm_mullo_epi32(res_lo, wt1)); res_lo = _mm_srai_epi32(res_lo, DIST_PRECISION_BITS); @@ -570,7 +570,7 @@ void av1_highbd_warp_affine_sse4_1(const int32_t *mat, const uint16_t *ref, (__m128i *)&pred[(i + k + 4) * p_stride + j + 4]; __m128i p4_32 = _mm_cvtepu16_epi32(_mm_loadl_epi64(p4)); - if (conv_params->use_jnt_comp_avg) { + if (conv_params->use_dist_wtd_comp_avg) { res_hi = _mm_add_epi32(_mm_mullo_epi32(p4_32, wt0), _mm_mullo_epi32(res_hi, wt1)); res_hi = _mm_srai_epi32(res_hi, DIST_PRECISION_BITS); diff --git a/third_party/aom/av1/common/x86/jnt_convolve_avx2.c b/third_party/aom/av1/common/x86/jnt_convolve_avx2.c index 9f2e2b4578..cc46ef35bc 100644 --- a/third_party/aom/av1/common/x86/jnt_convolve_avx2.c +++ b/third_party/aom/av1/common/x86/jnt_convolve_avx2.c @@ -35,12 +35,12 @@ static INLINE __m256i load_line2_avx2(const void *a, const void *b) { _mm256_castsi128_si256(_mm_loadu_si128((__m128i *)b)), 0x20); } -void av1_jnt_convolve_x_avx2(const uint8_t *src, int src_stride, uint8_t *dst0, - int dst_stride0, int w, int h, - const InterpFilterParams *filter_params_x, - const InterpFilterParams *filter_params_y, - const int subpel_x_q4, const int subpel_y_q4, - ConvolveParams *conv_params) { +void av1_dist_wtd_convolve_x_avx2(const uint8_t *src, int src_stride, + uint8_t *dst0, int dst_stride0, int w, int h, + const InterpFilterParams *filter_params_x, + const InterpFilterParams *filter_params_y, + const int subpel_x_q4, const int subpel_y_q4, + ConvolveParams *conv_params) { CONV_BUF_TYPE *dst = conv_params->dst; int dst_stride = conv_params->dst_stride; const int bd = 8; @@ -50,7 +50,7 @@ void av1_jnt_convolve_x_avx2(const uint8_t *src, int src_stride, uint8_t *dst0, const int bits = FILTER_BITS - conv_params->round_1; const __m256i wt = unpack_weights_avx2(conv_params); const int do_average = conv_params->do_average; - const int use_jnt_comp_avg = conv_params->use_jnt_comp_avg; + const int use_dist_wtd_comp_avg = conv_params->use_dist_wtd_comp_avg; const int offset_0 = bd + 2 * FILTER_BITS - conv_params->round_0 - conv_params->round_1; const int offset = (1 << offset_0) + (1 << (offset_0 - 1)); @@ -97,7 +97,7 @@ void av1_jnt_convolve_x_avx2(const uint8_t *src, int src_stride, uint8_t *dst0, const __m256i data_ref_0 = load_line2_avx2(&dst_data[j], &dst_data[j + dst_stride]); const __m256i comp_avg_res = - comp_avg(&data_ref_0, &res_unsigned, &wt, use_jnt_comp_avg); + comp_avg(&data_ref_0, &res_unsigned, &wt, use_dist_wtd_comp_avg); const __m256i round_result = convolve_rounding( &comp_avg_res, &offset_const, &rounding_const, rounding_shift); @@ -127,12 +127,12 @@ void av1_jnt_convolve_x_avx2(const uint8_t *src, int src_stride, uint8_t *dst0, } } -void av1_jnt_convolve_y_avx2(const uint8_t *src, int src_stride, uint8_t *dst0, - int dst_stride0, int w, int h, - const InterpFilterParams *filter_params_x, - const InterpFilterParams *filter_params_y, - const int subpel_x_q4, const int subpel_y_q4, - ConvolveParams *conv_params) { +void av1_dist_wtd_convolve_y_avx2(const uint8_t *src, int src_stride, + uint8_t *dst0, int dst_stride0, int w, int h, + const InterpFilterParams *filter_params_x, + const InterpFilterParams *filter_params_y, + const int subpel_x_q4, const int subpel_y_q4, + ConvolveParams *conv_params) { CONV_BUF_TYPE *dst = conv_params->dst; int dst_stride = conv_params->dst_stride; const int bd = 8; @@ -146,7 +146,7 @@ void av1_jnt_convolve_y_avx2(const uint8_t *src, int src_stride, uint8_t *dst0, const __m128i round_shift = _mm_cvtsi32_si128(conv_params->round_1); const __m256i wt = unpack_weights_avx2(conv_params); const int do_average = conv_params->do_average; - const int use_jnt_comp_avg = conv_params->use_jnt_comp_avg; + const int use_dist_wtd_comp_avg = conv_params->use_dist_wtd_comp_avg; const int offset_0 = bd + 2 * FILTER_BITS - conv_params->round_0 - conv_params->round_1; const int offset = (1 << offset_0) + (1 << (offset_0 - 1)); @@ -230,8 +230,8 @@ void av1_jnt_convolve_y_avx2(const uint8_t *src, int src_stride, uint8_t *dst0, if (do_average) { const __m256i data_ref_0 = load_line2_avx2( &dst[i * dst_stride + j], &dst[i * dst_stride + j + dst_stride]); - const __m256i comp_avg_res = - comp_avg(&data_ref_0, &res_lo_unsigned, &wt, use_jnt_comp_avg); + const __m256i comp_avg_res = comp_avg(&data_ref_0, &res_lo_unsigned, + &wt, use_dist_wtd_comp_avg); const __m256i round_result = convolve_rounding( &comp_avg_res, &offset_const, &rounding_const, rounding_shift); @@ -289,11 +289,11 @@ void av1_jnt_convolve_y_avx2(const uint8_t *src, int src_stride, uint8_t *dst0, load_line2_avx2(&dst[i * dst_stride + j + 8], &dst[i * dst_stride + j + 8 + dst_stride]); - const __m256i comp_avg_res_lo = - comp_avg(&data_ref_0_lo, &res_lo_unsigned, &wt, use_jnt_comp_avg); + const __m256i comp_avg_res_lo = comp_avg( + &data_ref_0_lo, &res_lo_unsigned, &wt, use_dist_wtd_comp_avg); - const __m256i comp_avg_res_hi = - comp_avg(&data_ref_0_hi, &res_hi_unsigned, &wt, use_jnt_comp_avg); + const __m256i comp_avg_res_hi = comp_avg( + &data_ref_0_hi, &res_hi_unsigned, &wt, use_dist_wtd_comp_avg); const __m256i round_result_lo = convolve_rounding( &comp_avg_res_lo, &offset_const, &rounding_const, rounding_shift); @@ -337,12 +337,12 @@ void av1_jnt_convolve_y_avx2(const uint8_t *src, int src_stride, uint8_t *dst0, } } -void av1_jnt_convolve_2d_avx2(const uint8_t *src, int src_stride, uint8_t *dst0, - int dst_stride0, int w, int h, - const InterpFilterParams *filter_params_x, - const InterpFilterParams *filter_params_y, - const int subpel_x_q4, const int subpel_y_q4, - ConvolveParams *conv_params) { +void av1_dist_wtd_convolve_2d_avx2(const uint8_t *src, int src_stride, + uint8_t *dst0, int dst_stride0, int w, int h, + const InterpFilterParams *filter_params_x, + const InterpFilterParams *filter_params_y, + const int subpel_x_q4, const int subpel_y_q4, + ConvolveParams *conv_params) { CONV_BUF_TYPE *dst = conv_params->dst; int dst_stride = conv_params->dst_stride; const int bd = 8; @@ -356,7 +356,7 @@ void av1_jnt_convolve_2d_avx2(const uint8_t *src, int src_stride, uint8_t *dst0, const uint8_t *const src_ptr = src - fo_vert * src_stride - fo_horiz; const __m256i wt = unpack_weights_avx2(conv_params); const int do_average = conv_params->do_average; - const int use_jnt_comp_avg = conv_params->use_jnt_comp_avg; + const int use_dist_wtd_comp_avg = conv_params->use_dist_wtd_comp_avg; const int offset_0 = bd + 2 * FILTER_BITS - conv_params->round_0 - conv_params->round_1; const int offset = (1 << offset_0) + (1 << (offset_0 - 1)); @@ -448,8 +448,8 @@ void av1_jnt_convolve_2d_avx2(const uint8_t *src, int src_stride, uint8_t *dst0, const __m256i data_ref_0 = load_line2_avx2(&dst[i * dst_stride + j], &dst[i * dst_stride + j + dst_stride]); - const __m256i comp_avg_res = - comp_avg(&data_ref_0, &res_unsigned, &wt, use_jnt_comp_avg); + const __m256i comp_avg_res = comp_avg(&data_ref_0, &res_unsigned, + &wt, use_dist_wtd_comp_avg); const __m256i round_result = convolve_rounding( &comp_avg_res, &offset_const, &rounding_const, rounding_shift); @@ -479,8 +479,8 @@ void av1_jnt_convolve_2d_avx2(const uint8_t *src, int src_stride, uint8_t *dst0, load_line2_avx2(&dst[i * dst_stride + j], &dst[i * dst_stride + j + dst_stride]); - const __m256i comp_avg_res = - comp_avg(&data_ref_0, &res_unsigned, &wt, use_jnt_comp_avg); + const __m256i comp_avg_res = comp_avg(&data_ref_0, &res_unsigned, + &wt, use_dist_wtd_comp_avg); const __m256i round_result = convolve_rounding( &comp_avg_res, &offset_const, &rounding_const, rounding_shift); @@ -517,12 +517,11 @@ void av1_jnt_convolve_2d_avx2(const uint8_t *src, int src_stride, uint8_t *dst0, } } -void av1_jnt_convolve_2d_copy_avx2(const uint8_t *src, int src_stride, - uint8_t *dst0, int dst_stride0, int w, int h, - const InterpFilterParams *filter_params_x, - const InterpFilterParams *filter_params_y, - const int subpel_x_q4, const int subpel_y_q4, - ConvolveParams *conv_params) { +void av1_dist_wtd_convolve_2d_copy_avx2( + const uint8_t *src, int src_stride, uint8_t *dst0, int dst_stride0, int w, + int h, const InterpFilterParams *filter_params_x, + const InterpFilterParams *filter_params_y, const int subpel_x_q4, + const int subpel_y_q4, ConvolveParams *conv_params) { const int bd = 8; CONV_BUF_TYPE *dst = conv_params->dst; int dst_stride = conv_params->dst_stride; @@ -535,7 +534,7 @@ void av1_jnt_convolve_2d_copy_avx2(const uint8_t *src, int src_stride, FILTER_BITS * 2 - conv_params->round_1 - conv_params->round_0; const __m128i left_shift = _mm_cvtsi32_si128(bits); const int do_average = conv_params->do_average; - const int use_jnt_comp_avg = conv_params->use_jnt_comp_avg; + const int use_dist_wtd_comp_avg = conv_params->use_dist_wtd_comp_avg; const __m256i wt = unpack_weights_avx2(conv_params); const __m256i zero = _mm256_setzero_si256(); @@ -562,7 +561,7 @@ void av1_jnt_convolve_2d_copy_avx2(const uint8_t *src, int src_stride, _mm256_loadu_si256((__m256i *)(&dst[i * dst_stride + j])); const __m256i comp_avg_res = - comp_avg(&data_ref_0, &res_unsigned, &wt, use_jnt_comp_avg); + comp_avg(&data_ref_0, &res_unsigned, &wt, use_dist_wtd_comp_avg); const __m256i round_result = convolve_rounding( &comp_avg_res, &offset_const, &rounding_const, rounding_shift); @@ -600,7 +599,7 @@ void av1_jnt_convolve_2d_copy_avx2(const uint8_t *src, int src_stride, const __m256i data_ref_0 = load_line2_avx2( &dst[i * dst_stride + j], &dst[i * dst_stride + j + dst_stride]); const __m256i comp_avg_res = - comp_avg(&data_ref_0, &res_unsigned, &wt, use_jnt_comp_avg); + comp_avg(&data_ref_0, &res_unsigned, &wt, use_dist_wtd_comp_avg); const __m256i round_result = convolve_rounding( &comp_avg_res, &offset_const, &rounding_const, rounding_shift); diff --git a/third_party/aom/av1/common/x86/jnt_convolve_sse2.c b/third_party/aom/av1/common/x86/jnt_convolve_sse2.c index 87dc3242e8..641cd02457 100644 --- a/third_party/aom/av1/common/x86/jnt_convolve_sse2.c +++ b/third_party/aom/av1/common/x86/jnt_convolve_sse2.c @@ -16,12 +16,12 @@ #include "aom_dsp/aom_filter.h" #include "aom_dsp/x86/convolve_sse2.h" -void av1_jnt_convolve_x_sse2(const uint8_t *src, int src_stride, uint8_t *dst0, - int dst_stride0, int w, int h, - const InterpFilterParams *filter_params_x, - const InterpFilterParams *filter_params_y, - const int subpel_x_q4, const int subpel_y_q4, - ConvolveParams *conv_params) { +void av1_dist_wtd_convolve_x_sse2(const uint8_t *src, int src_stride, + uint8_t *dst0, int dst_stride0, int w, int h, + const InterpFilterParams *filter_params_x, + const InterpFilterParams *filter_params_y, + const int subpel_x_q4, const int subpel_y_q4, + ConvolveParams *conv_params) { const int bd = 8; CONV_BUF_TYPE *dst = conv_params->dst; const int dst_stride = conv_params->dst_stride; @@ -37,7 +37,7 @@ void av1_jnt_convolve_x_sse2(const uint8_t *src, int src_stride, uint8_t *dst0, const __m128i wt1 = _mm_set1_epi16(w1); const __m128i wt = _mm_unpacklo_epi16(wt0, wt1); const int do_average = conv_params->do_average; - const int use_jnt_comp_avg = conv_params->use_jnt_comp_avg; + const int use_dist_wtd_comp_avg = conv_params->use_dist_wtd_comp_avg; const int offset_0 = bd + 2 * FILTER_BITS - conv_params->round_0 - conv_params->round_1; const int offset = (1 << offset_0) + (1 << (offset_0 - 1)); @@ -77,7 +77,7 @@ void av1_jnt_convolve_x_sse2(const uint8_t *src, int src_stride, uint8_t *dst0, const __m128i data_ref_0 = _mm_loadu_si128((__m128i *)dst); const __m128i comp_avg_res = - comp_avg(&data_ref_0, &res_unsigned, &wt, use_jnt_comp_avg); + comp_avg(&data_ref_0, &res_unsigned, &wt, use_dist_wtd_comp_avg); const __m128i round_result = convolve_rounding( &comp_avg_res, &offset_const, &rounding_const, rounding_shift); @@ -134,7 +134,7 @@ void av1_jnt_convolve_x_sse2(const uint8_t *src, int src_stride, uint8_t *dst0, _mm_loadu_si128((__m128i *)(&dst[i * dst_stride + j])); const __m128i comp_avg_res = - comp_avg(&data_ref_0, &res_unsigned, &wt, use_jnt_comp_avg); + comp_avg(&data_ref_0, &res_unsigned, &wt, use_dist_wtd_comp_avg); const __m128i round_result = convolve_rounding( &comp_avg_res, &offset_const, &rounding_const, rounding_shift); @@ -150,12 +150,12 @@ void av1_jnt_convolve_x_sse2(const uint8_t *src, int src_stride, uint8_t *dst0, } } -void av1_jnt_convolve_y_sse2(const uint8_t *src, int src_stride, uint8_t *dst0, - int dst_stride0, int w, int h, - const InterpFilterParams *filter_params_x, - const InterpFilterParams *filter_params_y, - const int subpel_x_q4, const int subpel_y_q4, - ConvolveParams *conv_params) { +void av1_dist_wtd_convolve_y_sse2(const uint8_t *src, int src_stride, + uint8_t *dst0, int dst_stride0, int w, int h, + const InterpFilterParams *filter_params_x, + const InterpFilterParams *filter_params_y, + const int subpel_x_q4, const int subpel_y_q4, + ConvolveParams *conv_params) { const int bd = 8; CONV_BUF_TYPE *dst = conv_params->dst; const int dst_stride = conv_params->dst_stride; @@ -167,7 +167,7 @@ void av1_jnt_convolve_y_sse2(const uint8_t *src, int src_stride, uint8_t *dst0, const __m128i wt1 = _mm_set1_epi16(conv_params->bck_offset); const __m128i wt = _mm_unpacklo_epi16(wt0, wt1); const int do_average = conv_params->do_average; - const int use_jnt_comp_avg = conv_params->use_jnt_comp_avg; + const int use_dist_wtd_comp_avg = conv_params->use_dist_wtd_comp_avg; const int offset_0 = bd + 2 * FILTER_BITS - conv_params->round_0 - conv_params->round_1; const int offset = (1 << offset_0) + (1 << (offset_0 - 1)); @@ -225,7 +225,7 @@ void av1_jnt_convolve_y_sse2(const uint8_t *src, int src_stride, uint8_t *dst0, const __m128i data_ref_0 = _mm_loadu_si128((__m128i *)dst); const __m128i comp_avg_res = - comp_avg(&data_ref_0, &res_unsigned, &wt, use_jnt_comp_avg); + comp_avg(&data_ref_0, &res_unsigned, &wt, use_dist_wtd_comp_avg); const __m128i round_result = convolve_rounding( &comp_avg_res, &offset_const, &rounding_const, rounding_shift); @@ -254,7 +254,7 @@ void av1_jnt_convolve_y_sse2(const uint8_t *src, int src_stride, uint8_t *dst0, const __m128i data_ref_0 = _mm_loadu_si128((__m128i *)dst); const __m128i comp_avg_res = - comp_avg(&data_ref_0, &res_unsigned, &wt, use_jnt_comp_avg); + comp_avg(&data_ref_0, &res_unsigned, &wt, use_dist_wtd_comp_avg); const __m128i round_result = convolve_rounding( &comp_avg_res, &offset_const, &rounding_const, rounding_shift); @@ -331,7 +331,7 @@ void av1_jnt_convolve_y_sse2(const uint8_t *src, int src_stride, uint8_t *dst0, _mm_loadu_si128((__m128i *)(&dst[i * dst_stride + j])); const __m128i comp_avg_res = - comp_avg(&data_ref_0, &res_unsigned, &wt, use_jnt_comp_avg); + comp_avg(&data_ref_0, &res_unsigned, &wt, use_dist_wtd_comp_avg); const __m128i round_result = convolve_rounding( &comp_avg_res, &offset_const, &rounding_const, rounding_shift); @@ -360,7 +360,7 @@ void av1_jnt_convolve_y_sse2(const uint8_t *src, int src_stride, uint8_t *dst0, _mm_loadu_si128((__m128i *)(&dst[i * dst_stride + j])); const __m128i comp_avg_res = - comp_avg(&data_ref_0, &res_unsigned, &wt, use_jnt_comp_avg); + comp_avg(&data_ref_0, &res_unsigned, &wt, use_dist_wtd_comp_avg); const __m128i round_result = convolve_rounding( &comp_avg_res, &offset_const, &rounding_const, rounding_shift); @@ -383,3 +383,233 @@ void av1_jnt_convolve_y_sse2(const uint8_t *src, int src_stride, uint8_t *dst0, } while (j < w); } } + +void av1_dist_wtd_convolve_2d_sse2(const uint8_t *src, int src_stride, + uint8_t *dst0, int dst_stride0, int w, int h, + const InterpFilterParams *filter_params_x, + const InterpFilterParams *filter_params_y, + const int subpel_x_q4, const int subpel_y_q4, + ConvolveParams *conv_params) { + CONV_BUF_TYPE *dst = conv_params->dst; + int dst_stride = conv_params->dst_stride; + const int bd = 8; + + DECLARE_ALIGNED(16, int16_t, + im_block[(MAX_SB_SIZE + MAX_FILTER_TAP - 1) * MAX_SB_SIZE]); + int im_h = h + filter_params_y->taps - 1; + int im_stride = MAX_SB_SIZE; + int i, j; + const int fo_vert = filter_params_y->taps / 2 - 1; + const int fo_horiz = filter_params_x->taps / 2 - 1; + const int do_average = conv_params->do_average; + const int use_dist_wtd_comp_avg = conv_params->use_dist_wtd_comp_avg; + const uint8_t *const src_ptr = src - fo_vert * src_stride - fo_horiz; + + const __m128i zero = _mm_setzero_si128(); + + const int w0 = conv_params->fwd_offset; + const int w1 = conv_params->bck_offset; + const __m128i wt0 = _mm_set1_epi16(w0); + const __m128i wt1 = _mm_set1_epi16(w1); + const __m128i wt = _mm_unpacklo_epi16(wt0, wt1); + + const int offset_0 = + bd + 2 * FILTER_BITS - conv_params->round_0 - conv_params->round_1; + const int offset = (1 << offset_0) + (1 << (offset_0 - 1)); + const __m128i offset_const = _mm_set1_epi16(offset); + const int rounding_shift = + 2 * FILTER_BITS - conv_params->round_0 - conv_params->round_1; + const __m128i rounding_const = _mm_set1_epi16((1 << rounding_shift) >> 1); + + /* Horizontal filter */ + { + const int16_t *x_filter = av1_get_interp_filter_subpel_kernel( + filter_params_x, subpel_x_q4 & SUBPEL_MASK); + const __m128i coeffs_x = _mm_loadu_si128((__m128i *)x_filter); + + // coeffs 0 1 0 1 2 3 2 3 + const __m128i tmp_0 = _mm_unpacklo_epi32(coeffs_x, coeffs_x); + // coeffs 4 5 4 5 6 7 6 7 + const __m128i tmp_1 = _mm_unpackhi_epi32(coeffs_x, coeffs_x); + + // coeffs 0 1 0 1 0 1 0 1 + const __m128i coeff_01 = _mm_unpacklo_epi64(tmp_0, tmp_0); + // coeffs 2 3 2 3 2 3 2 3 + const __m128i coeff_23 = _mm_unpackhi_epi64(tmp_0, tmp_0); + // coeffs 4 5 4 5 4 5 4 5 + const __m128i coeff_45 = _mm_unpacklo_epi64(tmp_1, tmp_1); + // coeffs 6 7 6 7 6 7 6 7 + const __m128i coeff_67 = _mm_unpackhi_epi64(tmp_1, tmp_1); + + const __m128i round_const = _mm_set1_epi32( + ((1 << conv_params->round_0) >> 1) + (1 << (bd + FILTER_BITS - 1))); + const __m128i round_shift = _mm_cvtsi32_si128(conv_params->round_0); + + for (i = 0; i < im_h; ++i) { + for (j = 0; j < w; j += 8) { + __m128i temp_lo, temp_hi; + const __m128i data = + _mm_loadu_si128((__m128i *)&src_ptr[i * src_stride + j]); + + const __m128i src_lo = _mm_unpacklo_epi8(data, zero); + const __m128i src_hi = _mm_unpackhi_epi8(data, zero); + + // Filter even-index pixels + const __m128i res_0 = _mm_madd_epi16(src_lo, coeff_01); + temp_lo = _mm_srli_si128(src_lo, 4); + temp_hi = _mm_slli_si128(src_hi, 12); + const __m128i src_2 = _mm_or_si128(temp_hi, temp_lo); + const __m128i res_2 = _mm_madd_epi16(src_2, coeff_23); + temp_lo = _mm_srli_si128(src_lo, 8); + temp_hi = _mm_slli_si128(src_hi, 8); + const __m128i src_4 = _mm_or_si128(temp_hi, temp_lo); + const __m128i res_4 = _mm_madd_epi16(src_4, coeff_45); + temp_lo = _mm_srli_si128(src_lo, 12); + temp_hi = _mm_slli_si128(src_hi, 4); + const __m128i src_6 = _mm_or_si128(temp_hi, temp_lo); + const __m128i res_6 = _mm_madd_epi16(src_6, coeff_67); + + __m128i res_even = _mm_add_epi32(_mm_add_epi32(res_0, res_4), + _mm_add_epi32(res_2, res_6)); + res_even = + _mm_sra_epi32(_mm_add_epi32(res_even, round_const), round_shift); + + // Filter odd-index pixels + temp_lo = _mm_srli_si128(src_lo, 2); + temp_hi = _mm_slli_si128(src_hi, 14); + const __m128i src_1 = _mm_or_si128(temp_hi, temp_lo); + const __m128i res_1 = _mm_madd_epi16(src_1, coeff_01); + temp_lo = _mm_srli_si128(src_lo, 6); + temp_hi = _mm_slli_si128(src_hi, 10); + const __m128i src_3 = _mm_or_si128(temp_hi, temp_lo); + const __m128i res_3 = _mm_madd_epi16(src_3, coeff_23); + temp_lo = _mm_srli_si128(src_lo, 10); + temp_hi = _mm_slli_si128(src_hi, 6); + const __m128i src_5 = _mm_or_si128(temp_hi, temp_lo); + const __m128i res_5 = _mm_madd_epi16(src_5, coeff_45); + temp_lo = _mm_srli_si128(src_lo, 14); + temp_hi = _mm_slli_si128(src_hi, 2); + const __m128i src_7 = _mm_or_si128(temp_hi, temp_lo); + const __m128i res_7 = _mm_madd_epi16(src_7, coeff_67); + + __m128i res_odd = _mm_add_epi32(_mm_add_epi32(res_1, res_5), + _mm_add_epi32(res_3, res_7)); + res_odd = + _mm_sra_epi32(_mm_add_epi32(res_odd, round_const), round_shift); + + // Pack in the column order 0, 2, 4, 6, 1, 3, 5, 7 + __m128i res = _mm_packs_epi32(res_even, res_odd); + _mm_store_si128((__m128i *)&im_block[i * im_stride + j], res); + } + } + } + + /* Vertical filter */ + { + const int16_t *y_filter = av1_get_interp_filter_subpel_kernel( + filter_params_y, subpel_y_q4 & SUBPEL_MASK); + const __m128i coeffs_y = _mm_loadu_si128((__m128i *)y_filter); + + // coeffs 0 1 0 1 2 3 2 3 + const __m128i tmp_0 = _mm_unpacklo_epi32(coeffs_y, coeffs_y); + // coeffs 4 5 4 5 6 7 6 7 + const __m128i tmp_1 = _mm_unpackhi_epi32(coeffs_y, coeffs_y); + + // coeffs 0 1 0 1 0 1 0 1 + const __m128i coeff_01 = _mm_unpacklo_epi64(tmp_0, tmp_0); + // coeffs 2 3 2 3 2 3 2 3 + const __m128i coeff_23 = _mm_unpackhi_epi64(tmp_0, tmp_0); + // coeffs 4 5 4 5 4 5 4 5 + const __m128i coeff_45 = _mm_unpacklo_epi64(tmp_1, tmp_1); + // coeffs 6 7 6 7 6 7 6 7 + const __m128i coeff_67 = _mm_unpackhi_epi64(tmp_1, tmp_1); + + const __m128i round_const = _mm_set1_epi32( + ((1 << conv_params->round_1) >> 1) - + (1 << (bd + 2 * FILTER_BITS - conv_params->round_0 - 1))); + const __m128i round_shift = _mm_cvtsi32_si128(conv_params->round_1); + + for (i = 0; i < h; ++i) { + for (j = 0; j < w; j += 8) { + // Filter even-index pixels + const int16_t *data = &im_block[i * im_stride + j]; + const __m128i src_0 = + _mm_unpacklo_epi16(*(__m128i *)(data + 0 * im_stride), + *(__m128i *)(data + 1 * im_stride)); + const __m128i src_2 = + _mm_unpacklo_epi16(*(__m128i *)(data + 2 * im_stride), + *(__m128i *)(data + 3 * im_stride)); + const __m128i src_4 = + _mm_unpacklo_epi16(*(__m128i *)(data + 4 * im_stride), + *(__m128i *)(data + 5 * im_stride)); + const __m128i src_6 = + _mm_unpacklo_epi16(*(__m128i *)(data + 6 * im_stride), + *(__m128i *)(data + 7 * im_stride)); + + const __m128i res_0 = _mm_madd_epi16(src_0, coeff_01); + const __m128i res_2 = _mm_madd_epi16(src_2, coeff_23); + const __m128i res_4 = _mm_madd_epi16(src_4, coeff_45); + const __m128i res_6 = _mm_madd_epi16(src_6, coeff_67); + + const __m128i res_even = _mm_add_epi32(_mm_add_epi32(res_0, res_2), + _mm_add_epi32(res_4, res_6)); + + // Filter odd-index pixels + const __m128i src_1 = + _mm_unpackhi_epi16(*(__m128i *)(data + 0 * im_stride), + *(__m128i *)(data + 1 * im_stride)); + const __m128i src_3 = + _mm_unpackhi_epi16(*(__m128i *)(data + 2 * im_stride), + *(__m128i *)(data + 3 * im_stride)); + const __m128i src_5 = + _mm_unpackhi_epi16(*(__m128i *)(data + 4 * im_stride), + *(__m128i *)(data + 5 * im_stride)); + const __m128i src_7 = + _mm_unpackhi_epi16(*(__m128i *)(data + 6 * im_stride), + *(__m128i *)(data + 7 * im_stride)); + + const __m128i res_1 = _mm_madd_epi16(src_1, coeff_01); + const __m128i res_3 = _mm_madd_epi16(src_3, coeff_23); + const __m128i res_5 = _mm_madd_epi16(src_5, coeff_45); + const __m128i res_7 = _mm_madd_epi16(src_7, coeff_67); + + const __m128i res_odd = _mm_add_epi32(_mm_add_epi32(res_1, res_3), + _mm_add_epi32(res_5, res_7)); + + // Rearrange pixels back into the order 0 ... 7 + const __m128i res_lo = _mm_unpacklo_epi32(res_even, res_odd); + const __m128i res_hi = _mm_unpackhi_epi32(res_even, res_odd); + + const __m128i res_lo_round = + _mm_sra_epi32(_mm_add_epi32(res_lo, round_const), round_shift); + const __m128i res_hi_round = + _mm_sra_epi32(_mm_add_epi32(res_hi, round_const), round_shift); + + const __m128i res_16b = _mm_packs_epi32(res_lo_round, res_hi_round); + const __m128i res_unsigned = _mm_add_epi16(res_16b, offset_const); + + // Accumulate values into the destination buffer + if (do_average) { + const __m128i data_ref_0 = + _mm_loadu_si128((__m128i *)(&dst[i * dst_stride + j])); + + const __m128i comp_avg_res = + comp_avg(&data_ref_0, &res_unsigned, &wt, use_dist_wtd_comp_avg); + + const __m128i round_result = convolve_rounding( + &comp_avg_res, &offset_const, &rounding_const, rounding_shift); + + const __m128i res_8 = _mm_packus_epi16(round_result, round_result); + + if (w > 4) + _mm_storel_epi64((__m128i *)(&dst0[i * dst_stride0 + j]), res_8); + else + *(uint32_t *)(&dst0[i * dst_stride0 + j]) = + _mm_cvtsi128_si32(res_8); + } else { + _mm_store_si128((__m128i *)(&dst[i * dst_stride + j]), res_unsigned); + } + } + } + } +} diff --git a/third_party/aom/av1/common/x86/jnt_convolve_ssse3.c b/third_party/aom/av1/common/x86/jnt_convolve_ssse3.c index 822772782b..9aeab29424 100644 --- a/third_party/aom/av1/common/x86/jnt_convolve_ssse3.c +++ b/third_party/aom/av1/common/x86/jnt_convolve_ssse3.c @@ -16,12 +16,11 @@ #include "aom_dsp/aom_filter.h" #include "aom_dsp/x86/convolve_sse2.h" -void av1_jnt_convolve_2d_ssse3(const uint8_t *src, int src_stride, - uint8_t *dst0, int dst_stride0, int w, int h, - const InterpFilterParams *filter_params_x, - const InterpFilterParams *filter_params_y, - const int subpel_x_q4, const int subpel_y_q4, - ConvolveParams *conv_params) { +void av1_dist_wtd_convolve_2d_ssse3( + const uint8_t *src, int src_stride, uint8_t *dst0, int dst_stride0, int w, + int h, const InterpFilterParams *filter_params_x, + const InterpFilterParams *filter_params_y, const int subpel_x_q4, + const int subpel_y_q4, ConvolveParams *conv_params) { CONV_BUF_TYPE *dst = conv_params->dst; int dst_stride = conv_params->dst_stride; const int bd = 8; @@ -34,7 +33,7 @@ void av1_jnt_convolve_2d_ssse3(const uint8_t *src, int src_stride, const int fo_vert = filter_params_y->taps / 2 - 1; const int fo_horiz = filter_params_x->taps / 2 - 1; const int do_average = conv_params->do_average; - const int use_jnt_comp_avg = conv_params->use_jnt_comp_avg; + const int use_dist_wtd_comp_avg = conv_params->use_dist_wtd_comp_avg; const uint8_t *const src_ptr = src - fo_vert * src_stride - fo_horiz; const __m128i zero = _mm_setzero_si128(); @@ -211,7 +210,7 @@ void av1_jnt_convolve_2d_ssse3(const uint8_t *src, int src_stride, _mm_loadu_si128((__m128i *)(&dst[i * dst_stride + j])); const __m128i comp_avg_res = - comp_avg(&data_ref_0, &res_unsigned, &wt, use_jnt_comp_avg); + comp_avg(&data_ref_0, &res_unsigned, &wt, use_dist_wtd_comp_avg); const __m128i round_result = convolve_rounding( &comp_avg_res, &offset_const, &rounding_const, rounding_shift); diff --git a/third_party/aom/av1/common/x86/warp_plane_sse4.c b/third_party/aom/av1/common/x86/warp_plane_sse4.c index b810cea2e5..4532d17ced 100644 --- a/third_party/aom/av1/common/x86/warp_plane_sse4.c +++ b/third_party/aom/av1/common/x86/warp_plane_sse4.c @@ -577,7 +577,7 @@ static INLINE void store_vertical_filter_output( __m128i *const dst8 = (__m128i *)&pred[(i + k + 4) * p_stride + j]; const __m128i p_16 = _mm_loadl_epi64(p); - if (conv_params->use_jnt_comp_avg) { + if (conv_params->use_dist_wtd_comp_avg) { const __m128i p_16_lo = _mm_unpacklo_epi16(p_16, temp_lo_16); const __m128i wt_res_lo = _mm_madd_epi16(p_16_lo, *wt); const __m128i shifted_32 = @@ -610,7 +610,7 @@ static INLINE void store_vertical_filter_output( (__m128i *)&pred[(i + k + 4) * p_stride + j + 4]; const __m128i p4_16 = _mm_loadl_epi64(p4); - if (conv_params->use_jnt_comp_avg) { + if (conv_params->use_dist_wtd_comp_avg) { const __m128i p_16_hi = _mm_unpacklo_epi16(p4_16, temp_hi_16); const __m128i wt_res_hi = _mm_madd_epi16(p_16_hi, *wt); const __m128i shifted_32 = diff --git a/third_party/aom/av1/common/x86/wiener_convolve_avx2.c b/third_party/aom/av1/common/x86/wiener_convolve_avx2.c index 87a6e12396..1f13e2f92e 100644 --- a/third_party/aom/av1/common/x86/wiener_convolve_avx2.c +++ b/third_party/aom/av1/common/x86/wiener_convolve_avx2.c @@ -17,6 +17,7 @@ #include "av1/common/convolve.h" #include "aom_dsp/aom_dsp_common.h" #include "aom_dsp/aom_filter.h" +#include "aom_dsp/x86/convolve_avx2.h" #include "aom_dsp/x86/synonyms.h" #include "aom_dsp/x86/synonyms_avx2.h" @@ -25,236 +26,207 @@ // on the left. // A row of, say, 8-bit pixels with values p0, p1, p2, ..., p30, p31 will be // loaded and stored as [ p31 ... p17 p16 ][ p15 ... p1 p0 ]. + +// Exploiting the range of wiener filter coefficients, +// horizontal filtering can be done in 16 bit intermediate precision. +// The details are as follows : +// Consider the horizontal wiener filter coefficients of the following form : +// [C0, C1, C2, 2^(FILTER_BITS) -2 * (C0 + C1 + C2), C2, C1, C0] +// Subtracting 2^(FILTER_BITS) from the centre tap we get the following : +// [C0, C1, C2, -2 * (C0 + C1 + C2), C2, C1, C0] +// The sum of the product "C0 * p0 + C1 * p1 + C2 * p2 -2 * (C0 + C1 + C2) * p3 +// + C2 * p4 + C1 * p5 + C0 * p6" would be in the range of signed 16 bit +// precision. Finally, after rounding the above result by round_0, we multiply +// the centre pixel by 2^(FILTER_BITS - round_0) and add it to get the +// horizontal filter output. + void av1_wiener_convolve_add_src_avx2(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, const int16_t *filter_y, int y_step_q4, int w, int h, const ConvolveParams *conv_params) { - const int bd = 8; assert(x_step_q4 == 16 && y_step_q4 == 16); assert(!(w & 7)); (void)x_step_q4; (void)y_step_q4; - DECLARE_ALIGNED(32, uint16_t, - temp[(MAX_SB_SIZE + SUBPEL_TAPS - 1) * MAX_SB_SIZE]); - int intermediate_height = h + SUBPEL_TAPS - 2; - memset(temp + (intermediate_height * MAX_SB_SIZE), 0, MAX_SB_SIZE); - const int center_tap = ((SUBPEL_TAPS - 1) / 2); + DECLARE_ALIGNED(32, int16_t, im_block[(MAX_SB_SIZE + SUBPEL_TAPS) * 8]); + int im_h = h + SUBPEL_TAPS - 2; + int im_stride = 8; + memset(im_block + (im_h * im_stride), 0, MAX_SB_SIZE); + int i, j; + const int center_tap = (SUBPEL_TAPS - 1) / 2; const uint8_t *const src_ptr = src - center_tap * src_stride - center_tap; - const __m128i zero_128 = _mm_setzero_si128(); - const __m256i zero_256 = _mm256_setzero_si256(); + __m256i filt[4], coeffs_h[4], coeffs_v[4], filt_center; + + assert(conv_params->round_0 > 0); + + filt[0] = _mm256_load_si256((__m256i const *)filt1_global_avx2); + filt[1] = _mm256_load_si256((__m256i const *)filt2_global_avx2); + filt[2] = _mm256_load_si256((__m256i const *)filt3_global_avx2); + filt[3] = _mm256_load_si256((__m256i const *)filt4_global_avx2); + + filt_center = _mm256_load_si256((__m256i const *)filt_center_global_avx2); + + const __m128i coeffs_x = _mm_loadu_si128((__m128i *)filter_x); + const __m256i filter_coeffs_x = _mm256_broadcastsi128_si256(coeffs_x); + + // coeffs 0 1 0 1 0 1 0 1 + coeffs_h[0] = + _mm256_shuffle_epi8(filter_coeffs_x, _mm256_set1_epi16(0x0200u)); + // coeffs 2 3 2 3 2 3 2 3 + coeffs_h[1] = + _mm256_shuffle_epi8(filter_coeffs_x, _mm256_set1_epi16(0x0604u)); + // coeffs 4 5 4 5 4 5 4 5 + coeffs_h[2] = + _mm256_shuffle_epi8(filter_coeffs_x, _mm256_set1_epi16(0x0a08u)); + // coeffs 6 7 6 7 6 7 6 7 + coeffs_h[3] = + _mm256_shuffle_epi8(filter_coeffs_x, _mm256_set1_epi16(0x0e0cu)); + + const __m256i round_const_h = + _mm256_set1_epi16((1 << (conv_params->round_0 - 1))); + const __m128i round_shift_h = _mm_cvtsi32_si128(conv_params->round_0); // Add an offset to account for the "add_src" part of the convolve function. - const __m128i offset = _mm_insert_epi16(zero_128, 1 << FILTER_BITS, 3); + const __m128i zero_128 = _mm_setzero_si128(); + const __m128i offset_0 = _mm_insert_epi16(zero_128, 1 << FILTER_BITS, 3); + const __m128i coeffs_y = _mm_add_epi16(xx_loadu_128(filter_y), offset_0); - const __m256i clamp_low = zero_256; - const __m256i clamp_high = - _mm256_set1_epi16(WIENER_CLAMP_LIMIT(conv_params->round_0, bd) - 1); + const __m256i filter_coeffs_y = _mm256_broadcastsi128_si256(coeffs_y); - /* Horizontal filter */ - { - // coeffs [ f7 f6 f5 f4 f3 f2 f1 f0 ] - const __m128i coeffs_x = _mm_add_epi16(xx_loadu_128(filter_x), offset); + // coeffs 0 1 0 1 0 1 0 1 + coeffs_v[0] = _mm256_shuffle_epi32(filter_coeffs_y, 0x00); + // coeffs 2 3 2 3 2 3 2 3 + coeffs_v[1] = _mm256_shuffle_epi32(filter_coeffs_y, 0x55); + // coeffs 4 5 4 5 4 5 4 5 + coeffs_v[2] = _mm256_shuffle_epi32(filter_coeffs_y, 0xaa); + // coeffs 6 7 6 7 6 7 6 7 + coeffs_v[3] = _mm256_shuffle_epi32(filter_coeffs_y, 0xff); - // coeffs [ f3 f2 f3 f2 f1 f0 f1 f0 ] - const __m128i coeffs_0123 = _mm_unpacklo_epi32(coeffs_x, coeffs_x); - // coeffs [ f7 f6 f7 f6 f5 f4 f5 f4 ] - const __m128i coeffs_4567 = _mm_unpackhi_epi32(coeffs_x, coeffs_x); + const __m256i round_const_v = + _mm256_set1_epi32((1 << (conv_params->round_1 - 1))); + const __m128i round_shift_v = _mm_cvtsi32_si128(conv_params->round_1); - // coeffs [ f1 f0 f1 f0 f1 f0 f1 f0 ] - const __m128i coeffs_01_128 = _mm_unpacklo_epi64(coeffs_0123, coeffs_0123); - // coeffs [ f3 f2 f3 f2 f3 f2 f3 f2 ] - const __m128i coeffs_23_128 = _mm_unpackhi_epi64(coeffs_0123, coeffs_0123); - // coeffs [ f5 f4 f5 f4 f5 f4 f5 f4 ] - const __m128i coeffs_45_128 = _mm_unpacklo_epi64(coeffs_4567, coeffs_4567); - // coeffs [ f7 f6 f7 f6 f7 f6 f7 f6 ] - const __m128i coeffs_67_128 = _mm_unpackhi_epi64(coeffs_4567, coeffs_4567); + for (j = 0; j < w; j += 8) { + for (i = 0; i < im_h; i += 2) { + __m256i data = _mm256_castsi128_si256( + _mm_loadu_si128((__m128i *)&src_ptr[(i * src_stride) + j])); - // coeffs [ f1 f0 f1 f0 f1 f0 f1 f0 ][ f1 f0 f1 f0 f1 f0 f1 f0 ] - const __m256i coeffs_01 = yy_set_m128i(coeffs_01_128, coeffs_01_128); - // coeffs [ f3 f2 f3 f2 f3 f2 f3 f2 ][ f3 f2 f3 f2 f3 f2 f3 f2 ] - const __m256i coeffs_23 = yy_set_m128i(coeffs_23_128, coeffs_23_128); - // coeffs [ f5 f4 f5 f4 f5 f4 f5 f4 ][ f5 f4 f5 f4 f5 f4 f5 f4 ] - const __m256i coeffs_45 = yy_set_m128i(coeffs_45_128, coeffs_45_128); - // coeffs [ f7 f6 f7 f6 f7 f6 f7 f6 ][ f7 f6 f7 f6 f7 f6 f7 f6 ] - const __m256i coeffs_67 = yy_set_m128i(coeffs_67_128, coeffs_67_128); + // Load the next line + if (i + 1 < im_h) + data = _mm256_inserti128_si256( + data, + _mm_loadu_si128( + (__m128i *)&src_ptr[(i * src_stride) + j + src_stride]), + 1); - const __m256i round_const = _mm256_set1_epi32( - (1 << (conv_params->round_0 - 1)) + (1 << (bd + FILTER_BITS - 1))); + __m256i res = convolve_lowbd_x(data, coeffs_h, filt); - for (int i = 0; i < intermediate_height; ++i) { - for (int j = 0; j < w; j += 16) { - const uint8_t *data_ij = src_ptr + i * src_stride + j; + res = + _mm256_sra_epi16(_mm256_add_epi16(res, round_const_h), round_shift_h); - // Load 8-bit src data - const __m128i data_0 = xx_loadu_128(data_ij + 0); - const __m128i data_1 = xx_loadu_128(data_ij + 1); - const __m128i data_2 = xx_loadu_128(data_ij + 2); - const __m128i data_3 = xx_loadu_128(data_ij + 3); - const __m128i data_4 = xx_loadu_128(data_ij + 4); - const __m128i data_5 = xx_loadu_128(data_ij + 5); - const __m128i data_6 = xx_loadu_128(data_ij + 6); - const __m128i data_7 = xx_loadu_128(data_ij + 7); + __m256i data_0 = _mm256_shuffle_epi8(data, filt_center); - // (Zero-)Extend 8-bit data to 16-bit data - const __m256i src_0 = _mm256_cvtepu8_epi16(data_0); - const __m256i src_1 = _mm256_cvtepu8_epi16(data_1); - const __m256i src_2 = _mm256_cvtepu8_epi16(data_2); - const __m256i src_3 = _mm256_cvtepu8_epi16(data_3); - const __m256i src_4 = _mm256_cvtepu8_epi16(data_4); - const __m256i src_5 = _mm256_cvtepu8_epi16(data_5); - const __m256i src_6 = _mm256_cvtepu8_epi16(data_6); - const __m256i src_7 = _mm256_cvtepu8_epi16(data_7); + // multiply the center pixel by 2^(FILTER_BITS - round_0) and add it to + // the result + data_0 = _mm256_slli_epi16(data_0, FILTER_BITS - conv_params->round_0); + res = _mm256_add_epi16(res, data_0); - // Multiply src data by filter coeffs and sum pairs - const __m256i res_0 = _mm256_madd_epi16(src_0, coeffs_01); - const __m256i res_1 = _mm256_madd_epi16(src_1, coeffs_01); - const __m256i res_2 = _mm256_madd_epi16(src_2, coeffs_23); - const __m256i res_3 = _mm256_madd_epi16(src_3, coeffs_23); - const __m256i res_4 = _mm256_madd_epi16(src_4, coeffs_45); - const __m256i res_5 = _mm256_madd_epi16(src_5, coeffs_45); - const __m256i res_6 = _mm256_madd_epi16(src_6, coeffs_67); - const __m256i res_7 = _mm256_madd_epi16(src_7, coeffs_67); - - // Calculate scalar product for even- and odd-indices separately, - // increasing to 32-bit precision - const __m256i res_even_sum = _mm256_add_epi32( - _mm256_add_epi32(res_0, res_4), _mm256_add_epi32(res_2, res_6)); - const __m256i res_odd_sum = _mm256_add_epi32( - _mm256_add_epi32(res_1, res_5), _mm256_add_epi32(res_3, res_7)); - - const __m256i res_even = _mm256_srai_epi32( - _mm256_add_epi32(res_even_sum, round_const), conv_params->round_0); - const __m256i res_odd = _mm256_srai_epi32( - _mm256_add_epi32(res_odd_sum, round_const), conv_params->round_0); - - // Reduce to 16-bit precision and pack even- and odd-index results - // back into one register. The _mm256_packs_epi32 intrinsic returns - // a register with the pixels ordered as follows: - // [ 15 13 11 9 14 12 10 8 ] [ 7 5 3 1 6 4 2 0 ] - const __m256i res = _mm256_packs_epi32(res_even, res_odd); - const __m256i res_clamped = - _mm256_min_epi16(_mm256_max_epi16(res, clamp_low), clamp_high); - - // Store in a temporary array - yy_storeu_256(temp + i * MAX_SB_SIZE + j, res_clamped); - } + _mm256_store_si256((__m256i *)&im_block[i * im_stride], res); } - } - /* Vertical filter */ - { - // coeffs [ g7 g6 g5 g4 g3 g2 g1 g0 ] - const __m128i coeffs_y = _mm_add_epi16(xx_loadu_128(filter_y), offset); + /* Vertical filter */ + { + __m256i src_0 = _mm256_loadu_si256((__m256i *)(im_block + 0 * im_stride)); + __m256i src_1 = _mm256_loadu_si256((__m256i *)(im_block + 1 * im_stride)); + __m256i src_2 = _mm256_loadu_si256((__m256i *)(im_block + 2 * im_stride)); + __m256i src_3 = _mm256_loadu_si256((__m256i *)(im_block + 3 * im_stride)); + __m256i src_4 = _mm256_loadu_si256((__m256i *)(im_block + 4 * im_stride)); + __m256i src_5 = _mm256_loadu_si256((__m256i *)(im_block + 5 * im_stride)); - // coeffs [ g3 g2 g3 g2 g1 g0 g1 g0 ] - const __m128i coeffs_0123 = _mm_unpacklo_epi32(coeffs_y, coeffs_y); - // coeffs [ g7 g6 g7 g6 g5 g4 g5 g4 ] - const __m128i coeffs_4567 = _mm_unpackhi_epi32(coeffs_y, coeffs_y); + __m256i s[8]; + s[0] = _mm256_unpacklo_epi16(src_0, src_1); + s[1] = _mm256_unpacklo_epi16(src_2, src_3); + s[2] = _mm256_unpacklo_epi16(src_4, src_5); - // coeffs [ g1 g0 g1 g0 g1 g0 g1 g0 ] - const __m128i coeffs_01_128 = _mm_unpacklo_epi64(coeffs_0123, coeffs_0123); - // coeffs [ g3 g2 g3 g2 g3 g2 g3 g2 ] - const __m128i coeffs_23_128 = _mm_unpackhi_epi64(coeffs_0123, coeffs_0123); - // coeffs [ g5 g4 g5 g4 g5 g4 g5 g4 ] - const __m128i coeffs_45_128 = _mm_unpacklo_epi64(coeffs_4567, coeffs_4567); - // coeffs [ g7 g6 g7 g6 g7 g6 g7 g6 ] - const __m128i coeffs_67_128 = _mm_unpackhi_epi64(coeffs_4567, coeffs_4567); + s[4] = _mm256_unpackhi_epi16(src_0, src_1); + s[5] = _mm256_unpackhi_epi16(src_2, src_3); + s[6] = _mm256_unpackhi_epi16(src_4, src_5); - // coeffs [ g1 g0 g1 g0 g1 g0 g1 g0 ][ g1 g0 g1 g0 g1 g0 g1 g0 ] - const __m256i coeffs_01 = yy_set_m128i(coeffs_01_128, coeffs_01_128); - // coeffs [ g3 g2 g3 g2 g3 g2 g3 g2 ][ g3 g2 g3 g2 g3 g2 g3 g2 ] - const __m256i coeffs_23 = yy_set_m128i(coeffs_23_128, coeffs_23_128); - // coeffs [ g5 g4 g5 g4 g5 g4 g5 g4 ][ g5 g4 g5 g4 g5 g4 g5 g4 ] - const __m256i coeffs_45 = yy_set_m128i(coeffs_45_128, coeffs_45_128); - // coeffs [ g7 g6 g7 g6 g7 g6 g7 g6 ][ g7 g6 g7 g6 g7 g6 g7 g6 ] - const __m256i coeffs_67 = yy_set_m128i(coeffs_67_128, coeffs_67_128); + for (i = 0; i < h - 1; i += 2) { + const int16_t *data = &im_block[i * im_stride]; - const __m256i round_const = - _mm256_set1_epi32((1 << (conv_params->round_1 - 1)) - - (1 << (bd + conv_params->round_1 - 1))); + const __m256i s6 = + _mm256_loadu_si256((__m256i *)(data + 6 * im_stride)); + const __m256i s7 = + _mm256_loadu_si256((__m256i *)(data + 7 * im_stride)); - for (int i = 0; i < h; ++i) { - for (int j = 0; j < w; j += 16) { - const uint16_t *data_ij = temp + i * MAX_SB_SIZE + j; + s[3] = _mm256_unpacklo_epi16(s6, s7); + s[7] = _mm256_unpackhi_epi16(s6, s7); - // Load 16-bit data from the output of the horizontal filter in - // which the pixels are ordered as follows: - // [ 15 13 11 9 14 12 10 8 ] [ 7 5 3 1 6 4 2 0 ] - const __m256i data_0 = yy_loadu_256(data_ij + 0 * MAX_SB_SIZE); - const __m256i data_1 = yy_loadu_256(data_ij + 1 * MAX_SB_SIZE); - const __m256i data_2 = yy_loadu_256(data_ij + 2 * MAX_SB_SIZE); - const __m256i data_3 = yy_loadu_256(data_ij + 3 * MAX_SB_SIZE); - const __m256i data_4 = yy_loadu_256(data_ij + 4 * MAX_SB_SIZE); - const __m256i data_5 = yy_loadu_256(data_ij + 5 * MAX_SB_SIZE); - const __m256i data_6 = yy_loadu_256(data_ij + 6 * MAX_SB_SIZE); - const __m256i data_7 = yy_loadu_256(data_ij + 7 * MAX_SB_SIZE); + __m256i res_a = convolve(s, coeffs_v); + __m256i res_b = convolve(s + 4, coeffs_v); - // Filter the even-indices, increasing to 32-bit precision - const __m256i src_0 = _mm256_unpacklo_epi16(data_0, data_1); - const __m256i src_2 = _mm256_unpacklo_epi16(data_2, data_3); - const __m256i src_4 = _mm256_unpacklo_epi16(data_4, data_5); - const __m256i src_6 = _mm256_unpacklo_epi16(data_6, data_7); + const __m256i res_a_round = _mm256_sra_epi32( + _mm256_add_epi32(res_a, round_const_v), round_shift_v); + const __m256i res_b_round = _mm256_sra_epi32( + _mm256_add_epi32(res_b, round_const_v), round_shift_v); - const __m256i res_0 = _mm256_madd_epi16(src_0, coeffs_01); - const __m256i res_2 = _mm256_madd_epi16(src_2, coeffs_23); - const __m256i res_4 = _mm256_madd_epi16(src_4, coeffs_45); - const __m256i res_6 = _mm256_madd_epi16(src_6, coeffs_67); + /* rounding code */ + // 16 bit conversion + const __m256i res_16bit = _mm256_packs_epi32(res_a_round, res_b_round); + // 8 bit conversion and saturation to uint8 + const __m256i res_8b = _mm256_packus_epi16(res_16bit, res_16bit); - const __m256i res_even = _mm256_add_epi32( - _mm256_add_epi32(res_0, res_2), _mm256_add_epi32(res_4, res_6)); + const __m128i res_0 = _mm256_castsi256_si128(res_8b); + const __m128i res_1 = _mm256_extracti128_si256(res_8b, 1); - // Filter the odd-indices, increasing to 32-bit precision - const __m256i src_1 = _mm256_unpackhi_epi16(data_0, data_1); - const __m256i src_3 = _mm256_unpackhi_epi16(data_2, data_3); - const __m256i src_5 = _mm256_unpackhi_epi16(data_4, data_5); - const __m256i src_7 = _mm256_unpackhi_epi16(data_6, data_7); + // Store values into the destination buffer + __m128i *const p_0 = (__m128i *)&dst[i * dst_stride + j]; + __m128i *const p_1 = (__m128i *)&dst[i * dst_stride + j + dst_stride]; - const __m256i res_1 = _mm256_madd_epi16(src_1, coeffs_01); - const __m256i res_3 = _mm256_madd_epi16(src_3, coeffs_23); - const __m256i res_5 = _mm256_madd_epi16(src_5, coeffs_45); - const __m256i res_7 = _mm256_madd_epi16(src_7, coeffs_67); + _mm_storel_epi64(p_0, res_0); + _mm_storel_epi64(p_1, res_1); - const __m256i res_odd = _mm256_add_epi32( - _mm256_add_epi32(res_1, res_3), _mm256_add_epi32(res_5, res_7)); + s[0] = s[1]; + s[1] = s[2]; + s[2] = s[3]; - // Pixels are currently in the following order: - // res_even order: [ 14 12 10 8 ] [ 6 4 2 0 ] - // res_odd order: [ 15 13 11 9 ] [ 7 5 3 1 ] - // - // Rearrange the pixels into the following order: - // res_lo order: [ 11 10 9 8 ] [ 3 2 1 0 ] - // res_hi order: [ 15 14 13 12 ] [ 7 6 5 4 ] - const __m256i res_lo = _mm256_unpacklo_epi32(res_even, res_odd); - const __m256i res_hi = _mm256_unpackhi_epi32(res_even, res_odd); + s[4] = s[5]; + s[5] = s[6]; + s[6] = s[7]; + } + if (h - i) { + s[0] = _mm256_permute2x128_si256(s[0], s[4], 0x20); + s[1] = _mm256_permute2x128_si256(s[1], s[5], 0x20); + s[2] = _mm256_permute2x128_si256(s[2], s[6], 0x20); - const __m256i res_lo_round = _mm256_srai_epi32( - _mm256_add_epi32(res_lo, round_const), conv_params->round_1); - const __m256i res_hi_round = _mm256_srai_epi32( - _mm256_add_epi32(res_hi, round_const), conv_params->round_1); + const int16_t *data = &im_block[i * im_stride]; + const __m128i s6_ = _mm_loadu_si128((__m128i *)(data + 6 * im_stride)); + const __m128i s7_ = _mm_loadu_si128((__m128i *)(data + 7 * im_stride)); - // Reduce to 16-bit precision and pack into the correct order: - // [ 15 14 13 12 11 10 9 8 ][ 7 6 5 4 3 2 1 0 ] - const __m256i res_16bit = - _mm256_packs_epi32(res_lo_round, res_hi_round); + __m128i s3 = _mm_unpacklo_epi16(s6_, s7_); + __m128i s7 = _mm_unpackhi_epi16(s6_, s7_); - // Reduce to 8-bit precision. This messes up the order: - // [ - - - - - - - - 15 14 13 12 11 10 9 8 ] - // [ - - - - - - - - 7 6 5 4 3 2 1 0 ] - const __m256i res_8bit = - _mm256_packus_epi16(res_16bit, zero_256 /* don't care value */); + s[3] = _mm256_inserti128_si256(_mm256_castsi128_si256(s3), s7, 1); + __m256i convolveres = convolve(s, coeffs_v); - // Swap the two central 32-bit values to get the order: - // [ - - - - - - - - - - - - - - - - ] - // [ 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1 0 ] - const __m256i res_8bit2 = _mm256_permute4x64_epi64(res_8bit, 0xd8); + const __m256i res_round = _mm256_sra_epi32( + _mm256_add_epi32(convolveres, round_const_v), round_shift_v); - // Store the lower 128-bit lane in the dst array - xx_storeu_128(dst + i * dst_stride + j, - _mm256_castsi256_si128(res_8bit2)); + /* rounding code */ + // 16 bit conversion + __m128i reslo = _mm256_castsi256_si128(res_round); + __m128i reshi = _mm256_extracti128_si256(res_round, 1); + const __m128i res_16bit = _mm_packus_epi32(reslo, reshi); + + // 8 bit conversion and saturation to uint8 + const __m128i res_8b = _mm_packus_epi16(res_16bit, res_16bit); + __m128i *const p_0 = (__m128i *)&dst[i * dst_stride + j]; + _mm_storel_epi64(p_0, res_8b); } } } diff --git a/third_party/aom/av1/decoder/decodeframe.c b/third_party/aom/av1/decoder/decodeframe.c index 31f14b531f..26e91d5ac8 100644 --- a/third_party/aom/av1/decoder/decodeframe.c +++ b/third_party/aom/av1/decoder/decodeframe.c @@ -64,6 +64,9 @@ #define ACCT_STR __func__ +#define AOM_MIN_THREADS_PER_TILE 1 +#define AOM_MAX_THREADS_PER_TILE 2 + // This is needed by ext_tile related unit tests. #define EXT_TILE_DEBUG 1 #define MC_TEMP_BUF_PELS \ @@ -153,13 +156,10 @@ static void inverse_transform_block(MACROBLOCKD *xd, int plane, const TX_SIZE tx_size, uint8_t *dst, int stride, int reduced_tx_set) { struct macroblockd_plane *const pd = &xd->plane[plane]; - tran_low_t *const dqcoeff = pd->dqcoeff; + tran_low_t *const dqcoeff = pd->dqcoeff_block + xd->cb_offset[plane]; eob_info *eob_data = pd->eob_data + xd->txb_offset[plane]; uint16_t scan_line = eob_data->max_scan_line; uint16_t eob = eob_data->eob; - - memcpy(dqcoeff, pd->dqcoeff_block + xd->cb_offset[plane], - (scan_line + 1) * sizeof(dqcoeff[0])); av1_inverse_transform_block(xd, dqcoeff, plane, tx_type, tx_size, dst, stride, eob, reduced_tx_set); memset(dqcoeff, 0, (scan_line + 1) * sizeof(dqcoeff[0])); @@ -268,8 +268,8 @@ static void inverse_transform_inter_block(const AV1_COMMON *const cm, int blk_h = block_size_high[bsize]; mi_to_pixel_loc(&pixel_c, &pixel_r, mi_col, mi_row, blk_col, blk_row, pd->subsampling_x, pd->subsampling_y); - mismatch_check_block_tx(dst, pd->dst.stride, cm->frame_offset, plane, pixel_c, - pixel_r, blk_w, blk_h, + mismatch_check_block_tx(dst, pd->dst.stride, cm->current_frame.order_hint, + plane, pixel_c, pixel_r, blk_w, blk_h, xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH); #endif } @@ -368,8 +368,8 @@ static void set_offsets(AV1_COMMON *const cm, MACROBLOCKD *const xd, // as they are always compared to values that are in 1/8th pel units set_mi_row_col(xd, tile, mi_row, bh, mi_col, bw, cm->mi_rows, cm->mi_cols); - av1_setup_dst_planes(xd->plane, bsize, get_frame_new_buffer(cm), mi_row, - mi_col, 0, num_planes); + av1_setup_dst_planes(xd->plane, bsize, &cm->cur_frame->buf, mi_row, mi_col, 0, + num_planes); } static void decode_mbmi_block(AV1Decoder *const pbi, MACROBLOCKD *const xd, @@ -397,9 +397,6 @@ static void decode_mbmi_block(AV1Decoder *const pbi, MACROBLOCKD *const xd, aom_internal_error(xd->error_info, AOM_CODEC_CORRUPT_FRAME, "Invalid block size."); } - - int reader_corrupted_flag = aom_reader_has_error(r); - aom_merge_corrupted_flag(&xd->corrupted, reader_corrupted_flag); } typedef struct PadBlock { @@ -699,26 +696,28 @@ static INLINE void dec_build_inter_predictors(const AV1_COMMON *cm, assert(bw < 8 || bh < 8); ConvolveParams conv_params = get_conv_params_no_round( 0, plane, xd->tmp_conv_dst, tmp_dst_stride, is_compound, xd->bd); - conv_params.use_jnt_comp_avg = 0; + conv_params.use_dist_wtd_comp_avg = 0; struct buf_2d *const dst_buf = &pd->dst; uint8_t *dst = dst_buf->buf + dst_buf->stride * y + x; ref = 0; - const RefBuffer *ref_buf = - &cm->frame_refs[this_mbmi->ref_frame[ref] - LAST_FRAME]; + const RefCntBuffer *ref_buf = + get_ref_frame_buf(cm, this_mbmi->ref_frame[ref]); + const struct scale_factors *ref_scale_factors = + get_ref_scale_factors_const(cm, this_mbmi->ref_frame[ref]); pd->pre[ref].buf0 = - (plane == 1) ? ref_buf->buf->u_buffer : ref_buf->buf->v_buffer; + (plane == 1) ? ref_buf->buf.u_buffer : ref_buf->buf.v_buffer; pd->pre[ref].buf = pd->pre[ref].buf0 + scaled_buffer_offset(pre_x, pre_y, - ref_buf->buf->uv_stride, - &ref_buf->sf); - pd->pre[ref].width = ref_buf->buf->uv_crop_width; - pd->pre[ref].height = ref_buf->buf->uv_crop_height; - pd->pre[ref].stride = ref_buf->buf->uv_stride; + ref_buf->buf.uv_stride, + ref_scale_factors); + pd->pre[ref].width = ref_buf->buf.uv_crop_width; + pd->pre[ref].height = ref_buf->buf.uv_crop_height; + pd->pre[ref].stride = ref_buf->buf.uv_stride; const struct scale_factors *const sf = - is_intrabc ? &cm->sf_identity : &ref_buf->sf; + is_intrabc ? &cm->sf_identity : ref_scale_factors; struct buf_2d *const pre_buf = is_intrabc ? dst_buf : &pd->pre[ref]; const MV mv = this_mbmi->mv[ref].as_mv; @@ -738,7 +737,7 @@ static INLINE void dec_build_inter_predictors(const AV1_COMMON *cm, &scaled_mv, &subpel_x_mv, &subpel_y_mv); pre = pre_buf->buf0 + block.y0 * pre_buf->stride + block.x0; src_stride = pre_buf->stride; - highbd = xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH; + highbd = is_cur_buf_hbd(xd); extend_mc_border(sf, pre_buf, scaled_mv, block, subpel_x_mv, subpel_y_mv, 0, is_intrabc, highbd, xd->mc_buf[ref], &pre, &src_stride); @@ -771,7 +770,7 @@ static INLINE void dec_build_inter_predictors(const AV1_COMMON *cm, int src_stride[2]; for (ref = 0; ref < 1 + is_compound; ++ref) { const struct scale_factors *const sf = - is_intrabc ? &cm->sf_identity : &xd->block_refs[ref]->sf; + is_intrabc ? &cm->sf_identity : xd->block_ref_scale_factors[ref]; struct buf_2d *const pre_buf = is_intrabc ? dst_buf : &pd->pre[ref]; const MV mv = mi->mv[ref].as_mv; PadBlock block; @@ -784,7 +783,7 @@ static INLINE void dec_build_inter_predictors(const AV1_COMMON *cm, &scaled_mv, &subpel_x_mv, &subpel_y_mv); pre[ref] = pre_buf->buf0 + block.y0 * pre_buf->stride + block.x0; src_stride[ref] = pre_buf->stride; - highbd = xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH; + highbd = is_cur_buf_hbd(xd); WarpTypesAllowed warp_types; warp_types.global_warp_allowed = is_global[ref]; @@ -792,8 +791,7 @@ static INLINE void dec_build_inter_predictors(const AV1_COMMON *cm, int do_warp = (bw >= 8 && bh >= 8 && av1_allow_warp(mi, &warp_types, &xd->global_motion[mi->ref_frame[ref]], - build_for_obmc, subpel_params[ref].xs, - subpel_params[ref].ys, NULL)); + build_for_obmc, sf, NULL)); do_warp = (do_warp && xd->cur_frame_force_integer_mv == 0); extend_mc_border(sf, pre_buf, scaled_mv, block, subpel_x_mv, subpel_y_mv, @@ -803,13 +801,13 @@ static INLINE void dec_build_inter_predictors(const AV1_COMMON *cm, ConvolveParams conv_params = get_conv_params_no_round( 0, plane, xd->tmp_conv_dst, MAX_SB_SIZE, is_compound, xd->bd); - av1_jnt_comp_weight_assign(cm, mi, 0, &conv_params.fwd_offset, - &conv_params.bck_offset, - &conv_params.use_jnt_comp_avg, is_compound); + av1_dist_wtd_comp_weight_assign( + cm, mi, 0, &conv_params.fwd_offset, &conv_params.bck_offset, + &conv_params.use_dist_wtd_comp_avg, is_compound); for (ref = 0; ref < 1 + is_compound; ++ref) { const struct scale_factors *const sf = - is_intrabc ? &cm->sf_identity : &xd->block_refs[ref]->sf; + is_intrabc ? &cm->sf_identity : xd->block_ref_scale_factors[ref]; WarpTypesAllowed warp_types; warp_types.global_warp_allowed = is_global[ref]; warp_types.local_warp_allowed = mi->motion_mode == WARPED_CAUSAL; @@ -858,7 +856,7 @@ static void dec_build_inter_predictors_for_planes(const AV1_COMMON *cm, static void dec_build_inter_predictors_sby(const AV1_COMMON *cm, MACROBLOCKD *xd, int mi_row, - int mi_col, BUFFER_SET *ctx, + int mi_col, const BUFFER_SET *ctx, BLOCK_SIZE bsize) { dec_build_inter_predictors_for_planes(cm, xd, bsize, mi_row, mi_col, 0, 0); @@ -873,7 +871,7 @@ static void dec_build_inter_predictors_sby(const AV1_COMMON *cm, static void dec_build_inter_predictors_sbuv(const AV1_COMMON *cm, MACROBLOCKD *xd, int mi_row, - int mi_col, BUFFER_SET *ctx, + int mi_col, const BUFFER_SET *ctx, BLOCK_SIZE bsize) { dec_build_inter_predictors_for_planes(cm, xd, bsize, mi_row, mi_col, 1, MAX_MB_PLANE - 1); @@ -1018,7 +1016,7 @@ static void dec_build_obmc_inter_predictors_sb(const AV1_COMMON *cm, int dst_height1[MAX_MB_PLANE] = { MAX_SB_SIZE, MAX_SB_SIZE, MAX_SB_SIZE }; int dst_height2[MAX_MB_PLANE] = { MAX_SB_SIZE, MAX_SB_SIZE, MAX_SB_SIZE }; - if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + if (is_cur_buf_hbd(xd)) { int len = sizeof(uint16_t); dst_buf1[0] = CONVERT_TO_BYTEPTR(xd->tmp_obmc_bufs[0]); dst_buf1[1] = @@ -1042,7 +1040,7 @@ static void dec_build_obmc_inter_predictors_sb(const AV1_COMMON *cm, dst_width1, dst_height1, dst_stride1); dec_build_prediction_by_left_preds(cm, xd, mi_row, mi_col, dst_buf2, dst_width2, dst_height2, dst_stride2); - av1_setup_dst_planes(xd->plane, xd->mi[0]->sb_type, get_frame_new_buffer(cm), + av1_setup_dst_planes(xd->plane, xd->mi[0]->sb_type, &cm->cur_frame->buf, mi_row, mi_col, 0, num_planes); av1_build_obmc_inter_prediction(cm, xd, mi_row, mi_col, dst_buf1, dst_stride1, dst_buf2, dst_stride2); @@ -1066,11 +1064,13 @@ static void predict_inter_block(AV1_COMMON *const cm, MACROBLOCKD *const xd, assert(frame == INTRA_FRAME); assert(ref == 0); } else { - RefBuffer *ref_buf = &cm->frame_refs[frame - LAST_FRAME]; + const RefCntBuffer *ref_buf = get_ref_frame_buf(cm, frame); + const struct scale_factors *ref_scale_factors = + get_ref_scale_factors_const(cm, frame); - xd->block_refs[ref] = ref_buf; - av1_setup_pre_planes(xd, ref, ref_buf->buf, mi_row, mi_col, &ref_buf->sf, - num_planes); + xd->block_ref_scale_factors[ref] = ref_scale_factors; + av1_setup_pre_planes(xd, ref, &ref_buf->buf, mi_row, mi_col, + ref_scale_factors, num_planes); } } @@ -1087,8 +1087,9 @@ static void predict_inter_block(AV1_COMMON *const cm, MACROBLOCKD *const xd, if (!is_chroma_reference(mi_row, mi_col, bsize, pd->subsampling_x, pd->subsampling_y)) continue; - mismatch_check_block_pre(pd->dst.buf, pd->dst.stride, cm->frame_offset, - plane, pixel_c, pixel_r, pd->width, pd->height, + mismatch_check_block_pre(pd->dst.buf, pd->dst.stride, + cm->current_frame.order_hint, plane, pixel_c, + pixel_r, pd->width, pd->height, xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH); } #endif @@ -1238,10 +1239,26 @@ static void store_bitmask_vartx(AV1_COMMON *cm, int mi_row, int mi_col, MB_MODE_INFO *mbmi); #endif +static void set_inter_tx_size(MB_MODE_INFO *mbmi, int stride_log2, + int tx_w_log2, int tx_h_log2, int min_txs, + int split_size, int txs, int blk_row, + int blk_col) { + for (int idy = 0; idy < tx_size_high_unit[split_size]; + idy += tx_size_high_unit[min_txs]) { + for (int idx = 0; idx < tx_size_wide_unit[split_size]; + idx += tx_size_wide_unit[min_txs]) { + const int index = (((blk_row + idy) >> tx_h_log2) << stride_log2) + + ((blk_col + idx) >> tx_w_log2); + mbmi->inter_tx_size[index] = txs; + } + } +} + static void read_tx_size_vartx(MACROBLOCKD *xd, MB_MODE_INFO *mbmi, TX_SIZE tx_size, int depth, #if LOOP_FILTER_BITMASK AV1_COMMON *cm, int mi_row, int mi_col, + int store_bitmask, #endif int blk_row, int blk_col, aom_reader *r) { FRAME_CONTEXT *ec_ctx = xd->tile_ctx; @@ -1251,15 +1268,17 @@ static void read_tx_size_vartx(MACROBLOCKD *xd, MB_MODE_INFO *mbmi, const int max_blocks_wide = max_block_wide(xd, bsize, 0); if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return; assert(tx_size > TX_4X4); + TX_SIZE txs = max_txsize_rect_lookup[bsize]; + for (int level = 0; level < MAX_VARTX_DEPTH - 1; ++level) + txs = sub_tx_size_map[txs]; + const int tx_w_log2 = tx_size_wide_log2[txs] - MI_SIZE_LOG2; + const int tx_h_log2 = tx_size_high_log2[txs] - MI_SIZE_LOG2; + const int bw_log2 = mi_size_wide_log2[bsize]; + const int stride_log2 = bw_log2 - tx_w_log2; if (depth == MAX_VARTX_DEPTH) { - for (int idy = 0; idy < tx_size_high_unit[tx_size]; ++idy) { - for (int idx = 0; idx < tx_size_wide_unit[tx_size]; ++idx) { - const int index = - av1_get_txb_size_index(bsize, blk_row + idy, blk_col + idx); - mbmi->inter_tx_size[index] = tx_size; - } - } + set_inter_tx_size(mbmi, stride_log2, tx_w_log2, tx_h_log2, txs, tx_size, + tx_size, blk_row, blk_col); mbmi->tx_size = tx_size; txfm_partition_update(xd->above_txfm_context + blk_col, xd->left_txfm_context + blk_row, tx_size, tx_size); @@ -1277,26 +1296,24 @@ static void read_tx_size_vartx(MACROBLOCKD *xd, MB_MODE_INFO *mbmi, const int bsh = tx_size_high_unit[sub_txs]; if (sub_txs == TX_4X4) { - for (int idy = 0; idy < tx_size_high_unit[tx_size]; ++idy) { - for (int idx = 0; idx < tx_size_wide_unit[tx_size]; ++idx) { - const int index = - av1_get_txb_size_index(bsize, blk_row + idy, blk_col + idx); - mbmi->inter_tx_size[index] = sub_txs; - } - } + set_inter_tx_size(mbmi, stride_log2, tx_w_log2, tx_h_log2, txs, tx_size, + sub_txs, blk_row, blk_col); mbmi->tx_size = sub_txs; txfm_partition_update(xd->above_txfm_context + blk_col, xd->left_txfm_context + blk_row, sub_txs, tx_size); #if LOOP_FILTER_BITMASK - store_bitmask_vartx(cm, mi_row + blk_row, mi_col + blk_col, BLOCK_8X8, - TX_4X4, mbmi); + if (store_bitmask) { + store_bitmask_vartx(cm, mi_row + blk_row, mi_col + blk_col, + txsize_to_bsize[tx_size], TX_4X4, mbmi); + } #endif return; } #if LOOP_FILTER_BITMASK - if (depth + 1 == MAX_VARTX_DEPTH) { + if (depth + 1 == MAX_VARTX_DEPTH && store_bitmask) { store_bitmask_vartx(cm, mi_row + blk_row, mi_col + blk_col, txsize_to_bsize[tx_size], sub_txs, mbmi); + store_bitmask = 0; } #endif @@ -1307,25 +1324,22 @@ static void read_tx_size_vartx(MACROBLOCKD *xd, MB_MODE_INFO *mbmi, int offsetc = blk_col + col; read_tx_size_vartx(xd, mbmi, sub_txs, depth + 1, #if LOOP_FILTER_BITMASK - cm, mi_row, mi_col, + cm, mi_row, mi_col, store_bitmask, #endif offsetr, offsetc, r); } } } else { - for (int idy = 0; idy < tx_size_high_unit[tx_size]; ++idy) { - for (int idx = 0; idx < tx_size_wide_unit[tx_size]; ++idx) { - const int index = - av1_get_txb_size_index(bsize, blk_row + idy, blk_col + idx); - mbmi->inter_tx_size[index] = tx_size; - } - } + set_inter_tx_size(mbmi, stride_log2, tx_w_log2, tx_h_log2, txs, tx_size, + tx_size, blk_row, blk_col); mbmi->tx_size = tx_size; txfm_partition_update(xd->above_txfm_context + blk_col, xd->left_txfm_context + blk_row, tx_size, tx_size); #if LOOP_FILTER_BITMASK - store_bitmask_vartx(cm, mi_row + blk_row, mi_col + blk_col, - txsize_to_bsize[tx_size], tx_size, mbmi); + if (store_bitmask) { + store_bitmask_vartx(cm, mi_row + blk_row, mi_col + blk_col, + txsize_to_bsize[tx_size], tx_size, mbmi); + } #endif } } @@ -1505,7 +1519,9 @@ static void store_bitmask_univariant_tx(AV1_COMMON *cm, int mi_row, int mi_col, } static void store_bitmask_other_info(AV1_COMMON *cm, int mi_row, int mi_col, - BLOCK_SIZE bsize, MB_MODE_INFO *mbmi) { + BLOCK_SIZE bsize, MB_MODE_INFO *mbmi, + int is_horz_coding_block_border, + int is_vert_coding_block_border) { int index; int shift; int row; @@ -1513,14 +1529,26 @@ static void store_bitmask_other_info(AV1_COMMON *cm, int mi_row, int mi_col, const int row_start = mi_row % MI_SIZE_64X64; const int col_start = mi_col % MI_SIZE_64X64; shift = get_index_shift(col_start, row_start, &index); - const uint64_t top_edge_mask = - ((uint64_t)1 << (shift + mi_size_wide[bsize])) - ((uint64_t)1 << shift); - lfm->is_horz_border.bits[index] |= top_edge_mask; - const int is_vert_border = mask_id_table_vert_border[bsize]; - const int vert_shift = block_size_high[bsize] <= 8 ? shift : col_start; - for (int i = 0; i + index < 4; ++i) { - lfm->is_vert_border.bits[i + index] |= - (left_mask_univariant_reordered[is_vert_border].bits[i] << vert_shift); + if (is_horz_coding_block_border) { + const int block_shift = shift + mi_size_wide[bsize]; + assert(block_shift <= 64); + const uint64_t right_edge_shift = + (block_shift == 64) ? 0xffffffffffffffff : ((uint64_t)1 << block_shift); + const uint64_t left_edge_shift = (block_shift == 64) + ? (((uint64_t)1 << shift) - 1) + : ((uint64_t)1 << shift); + assert(right_edge_shift > left_edge_shift); + const uint64_t top_edge_mask = right_edge_shift - left_edge_shift; + lfm->is_horz_border.bits[index] |= top_edge_mask; + } + if (is_vert_coding_block_border) { + const int is_vert_border = mask_id_table_vert_border[bsize]; + const int vert_shift = block_size_high[bsize] <= 8 ? shift : col_start; + for (int i = 0; i + index < 4; ++i) { + lfm->is_vert_border.bits[i + index] |= + (left_mask_univariant_reordered[is_vert_border].bits[i] + << vert_shift); + } } const int is_skip = mbmi->skip && is_inter_block(mbmi); if (is_skip) { @@ -1541,9 +1569,13 @@ static void store_bitmask_other_info(AV1_COMMON *cm, int mi_row, int mi_col, sizeof(uint8_t) * mi_size_wide[bsize]); memset(&lfm->lfl_y_hor[row][col_start], level_horz_y, sizeof(uint8_t) * mi_size_wide[bsize]); - memset(&lfm->lfl_u[row][col_start], level_u, + memset(&lfm->lfl_u_ver[row][col_start], level_u, sizeof(uint8_t) * mi_size_wide[bsize]); - memset(&lfm->lfl_v[row][col_start], level_v, + memset(&lfm->lfl_u_hor[row][col_start], level_u, + sizeof(uint8_t) * mi_size_wide[bsize]); + memset(&lfm->lfl_v_ver[row][col_start], level_v, + sizeof(uint8_t) * mi_size_wide[bsize]); + memset(&lfm->lfl_v_hor[row][col_start], level_v, sizeof(uint8_t) * mi_size_wide[bsize]); } } @@ -1574,7 +1606,7 @@ static void parse_decode_block(AV1Decoder *const pbi, ThreadData *const td, for (int idx = 0; idx < width; idx += bw) read_tx_size_vartx(xd, mbmi, max_tx_size, 0, #if LOOP_FILTER_BITMASK - cm, mi_row, mi_col, + cm, mi_row, mi_col, 1, #endif idy, idx, r); } else { @@ -1602,18 +1634,18 @@ static void parse_decode_block(AV1Decoder *const pbi, ThreadData *const td, const int w = mi_size_wide[bsize]; const int h = mi_size_high[bsize]; if (w <= mi_size_wide[BLOCK_64X64] && h <= mi_size_high[BLOCK_64X64]) { - store_bitmask_other_info(cm, mi_row, mi_col, bsize, mbmi); + store_bitmask_other_info(cm, mi_row, mi_col, bsize, mbmi, 1, 1); } else { for (int row = 0; row < h; row += mi_size_high[BLOCK_64X64]) { for (int col = 0; col < w; col += mi_size_wide[BLOCK_64X64]) { store_bitmask_other_info(cm, mi_row + row, mi_col + col, BLOCK_64X64, - mbmi); + mbmi, row == 0, col == 0); } } } #endif - if (cm->delta_q_present_flag) { + if (cm->delta_q_info.delta_q_present_flag) { for (int i = 0; i < MAX_SEGMENTS; i++) { const int current_qindex = av1_get_qindex(&cm->seg, i, xd->current_qindex); @@ -1633,9 +1665,6 @@ static void parse_decode_block(AV1Decoder *const pbi, ThreadData *const td, if (mbmi->skip) av1_reset_skip_context(xd, mi_row, mi_col, bsize, num_planes); decode_token_recon_block(pbi, td, mi_row, mi_col, r, bsize); - - int reader_corrupted_flag = aom_reader_has_error(r); - aom_merge_corrupted_flag(&xd->corrupted, reader_corrupted_flag); } static void set_offsets_for_pred_and_recon(AV1Decoder *const pbi, @@ -1660,8 +1689,8 @@ static void set_offsets_for_pred_and_recon(AV1Decoder *const pbi, // as they are always compared to values that are in 1/8th pel units set_mi_row_col(xd, tile, mi_row, bh, mi_col, bw, cm->mi_rows, cm->mi_cols); - av1_setup_dst_planes(xd->plane, bsize, get_frame_new_buffer(cm), mi_row, - mi_col, 0, num_planes); + av1_setup_dst_planes(xd->plane, bsize, &cm->cur_frame->buf, mi_row, mi_col, 0, + num_planes); } static void decode_block(AV1Decoder *const pbi, ThreadData *const td, @@ -1703,7 +1732,7 @@ static PARTITION_TYPE read_partition(MACROBLOCKD *xd, int mi_row, int mi_col, // TODO(slavarnway): eliminate bsize and subsize in future commits static void decode_partition(AV1Decoder *const pbi, ThreadData *const td, - int mi_row, int mi_col, aom_reader *r, + int mi_row, int mi_col, aom_reader *reader, BLOCK_SIZE bsize, int parse_decode_flag) { AV1_COMMON *const cm = &pbi->common; MACROBLOCKD *const xd = &td->xd; @@ -1736,14 +1765,14 @@ static void decode_partition(AV1Decoder *const pbi, ThreadData *const td, for (int rrow = rrow0; rrow < rrow1; ++rrow) { for (int rcol = rcol0; rcol < rcol1; ++rcol) { const int runit_idx = rcol + rrow * rstride; - loop_restoration_read_sb_coeffs(cm, xd, r, plane, runit_idx); + loop_restoration_read_sb_coeffs(cm, xd, reader, plane, runit_idx); } } } } partition = (bsize < BLOCK_8X8) ? PARTITION_NONE - : read_partition(xd, mi_row, mi_col, r, + : read_partition(xd, mi_row, mi_col, reader, has_rows, has_cols, bsize); } else { partition = get_partition(cm, mi_row, mi_col, bsize); @@ -1762,12 +1791,12 @@ static void decode_partition(AV1Decoder *const pbi, ThreadData *const td, #define DEC_BLOCK_STX_ARG #define DEC_BLOCK_EPT_ARG partition, -#define DEC_BLOCK(db_r, db_c, db_subsize) \ - block_visit[parse_decode_flag](pbi, td, DEC_BLOCK_STX_ARG(db_r), (db_c), r, \ - DEC_BLOCK_EPT_ARG(db_subsize)) -#define DEC_PARTITION(db_r, db_c, db_subsize) \ - decode_partition(pbi, td, DEC_BLOCK_STX_ARG(db_r), (db_c), r, (db_subsize), \ - parse_decode_flag) +#define DEC_BLOCK(db_r, db_c, db_subsize) \ + block_visit[parse_decode_flag](pbi, td, DEC_BLOCK_STX_ARG(db_r), (db_c), \ + reader, DEC_BLOCK_EPT_ARG(db_subsize)) +#define DEC_PARTITION(db_r, db_c, db_subsize) \ + decode_partition(pbi, td, DEC_BLOCK_STX_ARG(db_r), (db_c), reader, \ + (db_subsize), parse_decode_flag) switch (partition) { case PARTITION_NONE: DEC_BLOCK(mi_row, mi_col, subsize); break; @@ -2170,13 +2199,16 @@ static void setup_loopfilter(AV1_COMMON *cm, struct aom_read_bit_buffer *rb) { static void setup_cdef(AV1_COMMON *cm, struct aom_read_bit_buffer *rb) { const int num_planes = av1_num_planes(cm); + CdefInfo *const cdef_info = &cm->cdef_info; + if (cm->allow_intrabc) return; - cm->cdef_pri_damping = cm->cdef_sec_damping = aom_rb_read_literal(rb, 2) + 3; - cm->cdef_bits = aom_rb_read_literal(rb, 2); - cm->nb_cdef_strengths = 1 << cm->cdef_bits; - for (int i = 0; i < cm->nb_cdef_strengths; i++) { - cm->cdef_strengths[i] = aom_rb_read_literal(rb, CDEF_STRENGTH_BITS); - cm->cdef_uv_strengths[i] = + cdef_info->cdef_pri_damping = aom_rb_read_literal(rb, 2) + 3; + cdef_info->cdef_sec_damping = cdef_info->cdef_pri_damping; + cdef_info->cdef_bits = aom_rb_read_literal(rb, 2); + cdef_info->nb_cdef_strengths = 1 << cdef_info->cdef_bits; + for (int i = 0; i < cdef_info->nb_cdef_strengths; i++) { + cdef_info->cdef_strengths[i] = aom_rb_read_literal(rb, CDEF_STRENGTH_BITS); + cdef_info->cdef_uv_strengths[i] = num_planes > 1 ? aom_rb_read_literal(rb, CDEF_STRENGTH_BITS) : 0; } } @@ -2209,7 +2241,6 @@ static void setup_quantization(AV1_COMMON *const cm, cm->v_dc_delta_q = 0; cm->v_ac_delta_q = 0; } - cm->dequant_bit_depth = seq_params->bit_depth; cm->using_qmatrix = aom_rb_read_bit(rb); if (cm->using_qmatrix) { cm->qm_y = aom_rb_read_literal(rb, QM_LEVEL_BITS); @@ -2226,14 +2257,15 @@ static void setup_quantization(AV1_COMMON *const cm, } // Build y/uv dequant values based on segmentation. -static void setup_segmentation_dequant(AV1_COMMON *const cm) { +static void setup_segmentation_dequant(AV1_COMMON *const cm, + MACROBLOCKD *const xd) { const int bit_depth = cm->seq_params.bit_depth; const int using_qm = cm->using_qmatrix; // When segmentation is disabled, only the first value is used. The // remaining are don't cares. const int max_segments = cm->seg.enabled ? MAX_SEGMENTS : 1; for (int i = 0; i < max_segments; ++i) { - const int qindex = av1_get_qindex(&cm->seg, i, cm->base_qindex); + const int qindex = xd->qindex[i]; cm->y_dequant_QTX[i][0] = av1_dc_quant_QTX(qindex, cm->y_dc_delta_q, bit_depth); cm->y_dequant_QTX[i][1] = av1_ac_quant_QTX(qindex, 0, bit_depth); @@ -2245,9 +2277,7 @@ static void setup_segmentation_dequant(AV1_COMMON *const cm) { av1_dc_quant_QTX(qindex, cm->v_dc_delta_q, bit_depth); cm->v_dequant_QTX[i][1] = av1_ac_quant_QTX(qindex, cm->v_ac_delta_q, bit_depth); - const int lossless = qindex == 0 && cm->y_dc_delta_q == 0 && - cm->u_dc_delta_q == 0 && cm->u_ac_delta_q == 0 && - cm->v_dc_delta_q == 0 && cm->v_ac_delta_q == 0; + const int lossless = xd->lossless[i]; // NB: depends on base index so there is only 1 set per frame // No quant weighting when lossless or signalled not using QM int qmlevel = (lossless || using_qm == 0) ? NUM_QM_LEVELS - 1 : cm->qm_y; @@ -2344,36 +2374,27 @@ static void setup_buffer_pool(AV1_COMMON *cm) { lock_buffer_pool(pool); if (aom_realloc_frame_buffer( - get_frame_new_buffer(cm), cm->width, cm->height, - seq_params->subsampling_x, seq_params->subsampling_y, - seq_params->use_highbitdepth, AOM_BORDER_IN_PIXELS, - cm->byte_alignment, - &pool->frame_bufs[cm->new_fb_idx].raw_frame_buffer, pool->get_fb_cb, - pool->cb_priv)) { + &cm->cur_frame->buf, cm->width, cm->height, seq_params->subsampling_x, + seq_params->subsampling_y, seq_params->use_highbitdepth, + AOM_DEC_BORDER_IN_PIXELS, cm->byte_alignment, + &cm->cur_frame->raw_frame_buffer, pool->get_fb_cb, pool->cb_priv)) { unlock_buffer_pool(pool); aom_internal_error(&cm->error, AOM_CODEC_MEM_ERROR, "Failed to allocate frame buffer"); } unlock_buffer_pool(pool); - pool->frame_bufs[cm->new_fb_idx].buf.subsampling_x = - seq_params->subsampling_x; - pool->frame_bufs[cm->new_fb_idx].buf.subsampling_y = - seq_params->subsampling_y; - pool->frame_bufs[cm->new_fb_idx].buf.bit_depth = - (unsigned int)seq_params->bit_depth; - pool->frame_bufs[cm->new_fb_idx].buf.color_primaries = - seq_params->color_primaries; - pool->frame_bufs[cm->new_fb_idx].buf.transfer_characteristics = + cm->cur_frame->buf.bit_depth = (unsigned int)seq_params->bit_depth; + cm->cur_frame->buf.color_primaries = seq_params->color_primaries; + cm->cur_frame->buf.transfer_characteristics = seq_params->transfer_characteristics; - pool->frame_bufs[cm->new_fb_idx].buf.matrix_coefficients = - seq_params->matrix_coefficients; - pool->frame_bufs[cm->new_fb_idx].buf.monochrome = seq_params->monochrome; - pool->frame_bufs[cm->new_fb_idx].buf.chroma_sample_position = + cm->cur_frame->buf.matrix_coefficients = seq_params->matrix_coefficients; + cm->cur_frame->buf.monochrome = seq_params->monochrome; + cm->cur_frame->buf.chroma_sample_position = seq_params->chroma_sample_position; - pool->frame_bufs[cm->new_fb_idx].buf.color_range = seq_params->color_range; - pool->frame_bufs[cm->new_fb_idx].buf.render_width = cm->render_width; - pool->frame_bufs[cm->new_fb_idx].buf.render_height = cm->render_height; + cm->cur_frame->buf.color_range = seq_params->color_range; + cm->cur_frame->buf.render_width = cm->render_width; + cm->cur_frame->buf.render_height = cm->render_height; } static void setup_frame_size(AV1_COMMON *cm, int frame_size_override_flag, @@ -2419,17 +2440,28 @@ static void setup_frame_size_with_refs(AV1_COMMON *cm, int width, height; int found = 0; int has_valid_ref_frame = 0; - for (int i = 0; i < INTER_REFS_PER_FRAME; ++i) { + for (int i = LAST_FRAME; i <= ALTREF_FRAME; ++i) { if (aom_rb_read_bit(rb)) { - YV12_BUFFER_CONFIG *const buf = cm->frame_refs[i].buf; - width = buf->y_crop_width; - height = buf->y_crop_height; - cm->render_width = buf->render_width; - cm->render_height = buf->render_height; - setup_superres(cm, rb, &width, &height); - resize_context_buffers(cm, width, height); - found = 1; - break; + const RefCntBuffer *const ref_buf = get_ref_frame_buf(cm, i); + // This will never be NULL in a normal stream, as streams are required to + // have a shown keyframe before any inter frames, which would refresh all + // the reference buffers. However, it might be null if we're starting in + // the middle of a stream, and static analysis will error if we don't do + // a null check here. + if (ref_buf == NULL) { + aom_internal_error(&cm->error, AOM_CODEC_CORRUPT_FRAME, + "Invalid condition: invalid reference buffer"); + } else { + const YV12_BUFFER_CONFIG *const buf = &ref_buf->buf; + width = buf->y_crop_width; + height = buf->y_crop_height; + cm->render_width = buf->render_width; + cm->render_height = buf->render_height; + setup_superres(cm, rb, &width, &height); + resize_context_buffers(cm, width, height); + found = 1; + break; + } } } @@ -2450,20 +2482,20 @@ static void setup_frame_size_with_refs(AV1_COMMON *cm, // Check to make sure at least one of frames that this frame references // has valid dimensions. - for (int i = 0; i < INTER_REFS_PER_FRAME; ++i) { - RefBuffer *const ref_frame = &cm->frame_refs[i]; + for (int i = LAST_FRAME; i <= ALTREF_FRAME; ++i) { + const RefCntBuffer *const ref_frame = get_ref_frame_buf(cm, i); has_valid_ref_frame |= - valid_ref_frame_size(ref_frame->buf->y_crop_width, - ref_frame->buf->y_crop_height, width, height); + valid_ref_frame_size(ref_frame->buf.y_crop_width, + ref_frame->buf.y_crop_height, width, height); } if (!has_valid_ref_frame) aom_internal_error(&cm->error, AOM_CODEC_CORRUPT_FRAME, "Referenced frame has invalid size"); - for (int i = 0; i < INTER_REFS_PER_FRAME; ++i) { - RefBuffer *const ref_frame = &cm->frame_refs[i]; + for (int i = LAST_FRAME; i <= ALTREF_FRAME; ++i) { + const RefCntBuffer *const ref_frame = get_ref_frame_buf(cm, i); if (!valid_ref_frame_img_fmt( - ref_frame->buf->bit_depth, ref_frame->buf->subsampling_x, - ref_frame->buf->subsampling_y, seq_params->bit_depth, + ref_frame->buf.bit_depth, ref_frame->buf.subsampling_x, + ref_frame->buf.subsampling_y, seq_params->bit_depth, seq_params->subsampling_x, seq_params->subsampling_y)) aom_internal_error(&cm->error, AOM_CODEC_CORRUPT_FRAME, "Referenced frame has incompatible color format"); @@ -2546,15 +2578,18 @@ void av1_set_single_tile_decoding_mode(AV1_COMMON *const cm) { cm->single_tile_decoding = 0; if (cm->large_scale_tile) { struct loopfilter *lf = &cm->lf; + RestorationInfo *const rst_info = cm->rst_info; + const CdefInfo *const cdef_info = &cm->cdef_info; // Figure out single_tile_decoding by loopfilter_level. const int no_loopfilter = !(lf->filter_level[0] || lf->filter_level[1]); - const int no_cdef = cm->cdef_bits == 0 && cm->cdef_strengths[0] == 0 && - cm->cdef_uv_strengths[0] == 0; + const int no_cdef = cdef_info->cdef_bits == 0 && + cdef_info->cdef_strengths[0] == 0 && + cdef_info->cdef_uv_strengths[0] == 0; const int no_restoration = - cm->rst_info[0].frame_restoration_type == RESTORE_NONE && - cm->rst_info[1].frame_restoration_type == RESTORE_NONE && - cm->rst_info[2].frame_restoration_type == RESTORE_NONE; + rst_info[0].frame_restoration_type == RESTORE_NONE && + rst_info[1].frame_restoration_type == RESTORE_NONE && + rst_info[2].frame_restoration_type == RESTORE_NONE; assert(IMPLIES(cm->coded_lossless, no_loopfilter && no_cdef)); assert(IMPLIES(cm->all_lossless, no_restoration)); cm->single_tile_decoding = no_loopfilter && no_cdef && no_restoration; @@ -2694,9 +2729,10 @@ static const uint8_t *get_ls_tile_buffers( const int tile_col_size_bytes = pbi->tile_col_size_bytes; const int tile_size_bytes = pbi->tile_size_bytes; + int tile_width, tile_height; + av1_get_uniform_tile_size(cm, &tile_width, &tile_height); const int tile_copy_mode = - ((AOMMAX(cm->tile_width, cm->tile_height) << MI_SIZE_LOG2) <= 256) ? 1 - : 0; + ((AOMMAX(tile_width, tile_height) << MI_SIZE_LOG2) <= 256) ? 1 : 0; // Read tile column sizes for all columns (we need the last tile buffer) for (int c = 0; c < tile_cols; ++c) { const int is_last = c == tile_cols - 1; @@ -3184,7 +3220,7 @@ static const uint8_t *decode_tiles(AV1Decoder *pbi, const uint8_t *data, continue; td->bit_reader = &tile_data->bit_reader; - av1_zero(td->dqcoeff); + av1_zero(td->cb_buffer_base.dqcoeff); av1_tile_init(&td->xd.tile, cm, row, col); td->xd.current_qindex = cm->base_qindex; setup_bool_decoder(tile_bs_buf->data, data_end, tile_bs_buf->size, @@ -3198,7 +3234,7 @@ static const uint8_t *decode_tiles(AV1Decoder *pbi, const uint8_t *data, td->bit_reader->accounting = NULL; } #endif - av1_init_macroblockd(cm, &td->xd, td->dqcoeff); + av1_init_macroblockd(cm, &td->xd, NULL); av1_init_above_context(cm, &td->xd, row); // Initialise the tile context from the frame context @@ -3255,7 +3291,7 @@ static void tile_worker_hook_init(AV1Decoder *const pbi, int tile_col = tile_data->tile_info.tile_col; td->bit_reader = &tile_data->bit_reader; - av1_zero(td->dqcoeff); + av1_zero(td->cb_buffer_base.dqcoeff); av1_tile_init(&td->xd.tile, cm, tile_row, tile_col); td->xd.current_qindex = cm->base_qindex; setup_bool_decoder(tile_buffer->data, thread_data->data_end, @@ -3270,7 +3306,7 @@ static void tile_worker_hook_init(AV1Decoder *const pbi, td->bit_reader->accounting = NULL; } #endif - av1_init_macroblockd(cm, &td->xd, td->dqcoeff); + av1_init_macroblockd(cm, &td->xd, NULL); td->xd.error_info = &thread_data->error_info; av1_init_above_context(cm, &td->xd, tile_row); @@ -3308,10 +3344,10 @@ static int tile_worker_hook(void *arg1, void *arg2) { set_decode_func_pointers(td, 0x3); assert(cm->tile_cols > 0); - while (1) { + while (!td->xd.corrupted) { TileJobsDec *cur_job_info = get_dec_job_info(&pbi->tile_mt_info); - if (cur_job_info != NULL && !td->xd.corrupted) { + if (cur_job_info != NULL) { const TileBufferDec *const tile_buffer = cur_job_info->tile_buffer; TileDataDec *const tile_data = cur_job_info->tile_data; tile_worker_hook_init(pbi, thread_data, tile_buffer, tile_data, @@ -3328,6 +3364,30 @@ static int tile_worker_hook(void *arg1, void *arg2) { return !td->xd.corrupted; } +static INLINE int get_max_row_mt_workers_per_tile(AV1_COMMON *cm, + TileInfo tile) { + // NOTE: Currently value of max workers is calculated based + // on the parse and decode time. As per the theoretical estimate + // when percentage of parse time is equal to percentage of decode + // time, number of workers needed to parse + decode a tile can not + // exceed more than 2. + // TODO(any): Modify this value if parsing is optimized in future. + int sb_rows = av1_get_sb_rows_in_tile(cm, tile); + int max_workers = + sb_rows == 1 ? AOM_MIN_THREADS_PER_TILE : AOM_MAX_THREADS_PER_TILE; + return max_workers; +} + +// The caller must hold pbi->row_mt_mutex_ when calling this function. +// Returns 1 if either the next job is stored in *next_job_info or 1 is stored +// in *end_of_frame. +// NOTE: The caller waits on pbi->row_mt_cond_ if this function returns 0. +// The return value of this function depends on the following variables: +// - frame_row_mt_info->mi_rows_parse_done +// - frame_row_mt_info->mi_rows_decode_started +// - frame_row_mt_info->row_mt_exit +// Therefore we may need to signal or broadcast pbi->row_mt_cond_ if any of +// these variables is modified. static int get_next_job_info(AV1Decoder *const pbi, AV1DecRowMTJobInfo *next_job_info, int *end_of_frame) { @@ -3348,8 +3408,8 @@ static int get_next_job_info(AV1Decoder *const pbi, int min_threads_working = INT_MAX; int max_mis_to_decode = 0; int tile_row_idx, tile_col_idx; - int tile_row = 0; - int tile_col = 0; + int tile_row = -1; + int tile_col = -1; memset(next_job_info, 0, sizeof(*next_job_info)); @@ -3362,9 +3422,10 @@ static int get_next_job_info(AV1Decoder *const pbi, } // Decoding cannot start as bit-stream parsing is not complete. - if (frame_row_mt_info->mi_rows_parse_done - - frame_row_mt_info->mi_rows_decode_started == - 0) + assert(frame_row_mt_info->mi_rows_parse_done >= + frame_row_mt_info->mi_rows_decode_started); + if (frame_row_mt_info->mi_rows_parse_done == + frame_row_mt_info->mi_rows_decode_started) return 0; // Choose the tile to decode. @@ -3396,7 +3457,9 @@ static int get_next_job_info(AV1Decoder *const pbi, max_mis_to_decode = 0; } if (num_threads_working == min_threads_working && - num_mis_to_decode > max_mis_to_decode) { + num_mis_to_decode > max_mis_to_decode && + num_threads_working < + get_max_row_mt_workers_per_tile(cm, tile_data->tile_info)) { max_mis_to_decode = num_mis_to_decode; tile_row = tile_row_idx; tile_col = tile_col_idx; @@ -3404,6 +3467,8 @@ static int get_next_job_info(AV1Decoder *const pbi, } } } + // No job found to process + if (tile_row == -1 || tile_col == -1) return 0; tile_data = pbi->tile_data + tile_row * cm->tile_cols + tile_col; tile_info = tile_data->tile_info; @@ -3417,6 +3482,14 @@ static int get_next_job_info(AV1Decoder *const pbi, dec_row_mt_sync->num_threads_working++; dec_row_mt_sync->mi_rows_decode_started += sb_mi_size; frame_row_mt_info->mi_rows_decode_started += sb_mi_size; + assert(frame_row_mt_info->mi_rows_parse_done >= + frame_row_mt_info->mi_rows_decode_started); +#if CONFIG_MULTITHREAD + if (frame_row_mt_info->mi_rows_decode_started == + frame_row_mt_info->mi_rows_to_decode) { + pthread_cond_broadcast(pbi->row_mt_cond_); + } +#endif return 1; } @@ -3428,21 +3501,67 @@ static INLINE void signal_parse_sb_row_done(AV1Decoder *const pbi, #if CONFIG_MULTITHREAD pthread_mutex_lock(pbi->row_mt_mutex_); #endif + assert(frame_row_mt_info->mi_rows_parse_done >= + frame_row_mt_info->mi_rows_decode_started); tile_data->dec_row_mt_sync.mi_rows_parse_done += sb_mi_size; frame_row_mt_info->mi_rows_parse_done += sb_mi_size; #if CONFIG_MULTITHREAD - pthread_cond_broadcast(pbi->row_mt_cond_); + // A new decode job is available. Wake up one worker thread to handle the + // new decode job. + // NOTE: This assumes we bump mi_rows_parse_done and mi_rows_decode_started + // by the same increment (sb_mi_size). + pthread_cond_signal(pbi->row_mt_cond_); pthread_mutex_unlock(pbi->row_mt_mutex_); #endif } +// This function is very similar to decode_tile(). It would be good to figure +// out how to share code. +static void parse_tile_row_mt(AV1Decoder *pbi, ThreadData *const td, + TileDataDec *const tile_data) { + AV1_COMMON *const cm = &pbi->common; + const int sb_mi_size = mi_size_wide[cm->seq_params.sb_size]; + const int num_planes = av1_num_planes(cm); + TileInfo tile_info = tile_data->tile_info; + int tile_row = tile_info.tile_row; + + av1_zero_above_context(cm, &td->xd, tile_info.mi_col_start, + tile_info.mi_col_end, tile_row); + av1_reset_loop_filter_delta(&td->xd, num_planes); + av1_reset_loop_restoration(&td->xd, num_planes); + + for (int mi_row = tile_info.mi_row_start; mi_row < tile_info.mi_row_end; + mi_row += cm->seq_params.mib_size) { + av1_zero_left_context(&td->xd); + + for (int mi_col = tile_info.mi_col_start; mi_col < tile_info.mi_col_end; + mi_col += cm->seq_params.mib_size) { + set_cb_buffer(pbi, &td->xd, pbi->cb_buffer_base, num_planes, mi_row, + mi_col); + + // Bit-stream parsing of the superblock + decode_partition(pbi, td, mi_row, mi_col, td->bit_reader, + cm->seq_params.sb_size, 0x1); + + if (aom_reader_has_overflowed(td->bit_reader)) { + aom_merge_corrupted_flag(&td->xd.corrupted, 1); + return; + } + } + signal_parse_sb_row_done(pbi, tile_data, sb_mi_size); + } + + int corrupted = + (check_trailing_bits_after_symbol_coder(td->bit_reader)) ? 1 : 0; + aom_merge_corrupted_flag(&td->xd.corrupted, corrupted); +} + static int row_mt_worker_hook(void *arg1, void *arg2) { DecWorkerData *const thread_data = (DecWorkerData *)arg1; AV1Decoder *const pbi = (AV1Decoder *)arg2; AV1_COMMON *cm = &pbi->common; ThreadData *const td = thread_data->td; uint8_t allow_update_cdf; - const int sb_mi_size = mi_size_wide[cm->seq_params.sb_size]; AV1DecRowMTInfo *frame_row_mt_info = &pbi->frame_row_mt_info; td->xd.corrupted = 0; @@ -3464,55 +3583,54 @@ static int row_mt_worker_hook(void *arg1, void *arg2) { } thread_data->error_info.setjmp = 1; - const int num_planes = av1_num_planes(cm); allow_update_cdf = cm->large_scale_tile ? 0 : 1; allow_update_cdf = allow_update_cdf && !cm->disable_cdf_update; + set_decode_func_pointers(td, 0x1); + assert(cm->tile_cols > 0); - while (1) { + while (!td->xd.corrupted) { TileJobsDec *cur_job_info = get_dec_job_info(&pbi->tile_mt_info); - if (cur_job_info != NULL && !td->xd.corrupted) { + if (cur_job_info != NULL) { const TileBufferDec *const tile_buffer = cur_job_info->tile_buffer; TileDataDec *const tile_data = cur_job_info->tile_data; tile_worker_hook_init(pbi, thread_data, tile_buffer, tile_data, allow_update_cdf); - - set_decode_func_pointers(td, 0x1); - +#if CONFIG_MULTITHREAD + pthread_mutex_lock(pbi->row_mt_mutex_); +#endif + tile_data->dec_row_mt_sync.num_threads_working++; +#if CONFIG_MULTITHREAD + pthread_mutex_unlock(pbi->row_mt_mutex_); +#endif // decode tile - TileInfo tile_info = tile_data->tile_info; - int tile_row = tile_info.tile_row; - - av1_zero_above_context(cm, &td->xd, tile_info.mi_col_start, - tile_info.mi_col_end, tile_row); - av1_reset_loop_filter_delta(&td->xd, num_planes); - av1_reset_loop_restoration(&td->xd, num_planes); - - for (int mi_row = tile_info.mi_row_start; mi_row < tile_info.mi_row_end; - mi_row += cm->seq_params.mib_size) { - av1_zero_left_context(&td->xd); - - for (int mi_col = tile_info.mi_col_start; mi_col < tile_info.mi_col_end; - mi_col += cm->seq_params.mib_size) { - set_cb_buffer(pbi, &td->xd, pbi->cb_buffer_base, num_planes, mi_row, - mi_col); - - // Bit-stream parsing of the superblock - decode_partition(pbi, td, mi_row, mi_col, td->bit_reader, - cm->seq_params.sb_size, 0x1); - } - signal_parse_sb_row_done(pbi, tile_data, sb_mi_size); - } - - int corrupted = - (check_trailing_bits_after_symbol_coder(td->bit_reader)) ? 1 : 0; - aom_merge_corrupted_flag(&td->xd.corrupted, corrupted); + parse_tile_row_mt(pbi, td, tile_data); +#if CONFIG_MULTITHREAD + pthread_mutex_lock(pbi->row_mt_mutex_); +#endif + tile_data->dec_row_mt_sync.num_threads_working--; +#if CONFIG_MULTITHREAD + pthread_mutex_unlock(pbi->row_mt_mutex_); +#endif } else { break; } } + if (td->xd.corrupted) { + thread_data->error_info.setjmp = 0; +#if CONFIG_MULTITHREAD + pthread_mutex_lock(pbi->row_mt_mutex_); +#endif + frame_row_mt_info->row_mt_exit = 1; +#if CONFIG_MULTITHREAD + pthread_cond_broadcast(pbi->row_mt_cond_); + pthread_mutex_unlock(pbi->row_mt_mutex_); +#endif + return 0; + } + set_decode_func_pointers(td, 0x2); while (1) { @@ -3543,7 +3661,7 @@ static int row_mt_worker_hook(void *arg1, void *arg2) { TileInfo tile_info = tile_data->tile_info; av1_tile_init(&td->xd.tile, cm, tile_row, tile_col); - av1_init_macroblockd(cm, &td->xd, td->dqcoeff); + av1_init_macroblockd(cm, &td->xd, NULL); td->xd.error_info = &thread_data->error_info; decode_tile_sb_row(pbi, td, tile_info, mi_row); @@ -3570,7 +3688,7 @@ static int compare_tile_buffers(const void *a, const void *b) { static void enqueue_tile_jobs(AV1Decoder *pbi, AV1_COMMON *cm, int tile_rows_start, int tile_rows_end, int tile_cols_start, int tile_cols_end, - int startTile, int endTile) { + int start_tile, int end_tile) { AV1DecTileMT *tile_mt_info = &pbi->tile_mt_info; TileJobsDec *tile_job_queue = tile_mt_info->job_queue; tile_mt_info->jobs_enqueued = 0; @@ -3578,8 +3696,8 @@ static void enqueue_tile_jobs(AV1Decoder *pbi, AV1_COMMON *cm, for (int row = tile_rows_start; row < tile_rows_end; row++) { for (int col = tile_cols_start; col < tile_cols_end; col++) { - if (row * cm->tile_cols + col < startTile || - row * cm->tile_cols + col > endTile) + if (row * cm->tile_cols + col < start_tile || + row * cm->tile_cols + col > end_tile) continue; tile_job_queue->tile_buffer = &pbi->tile_buffers[row][col]; tile_job_queue->tile_data = pbi->tile_data + row * cm->tile_cols + col; @@ -3733,6 +3851,7 @@ static void decode_mt_init(AV1Decoder *pbi) { ++pbi->num_workers; winterface->init(worker); + worker->thread_name = "aom tile worker"; if (worker_idx < num_threads - 1 && !winterface->reset(worker)) { aom_internal_error(&cm->error, AOM_CODEC_ERROR, "Tile decoder thread creation failed"); @@ -3751,7 +3870,7 @@ static void decode_mt_init(AV1Decoder *pbi) { thread_data->error_info.setjmp = 0; } } - const int use_highbd = cm->seq_params.use_highbitdepth ? 1 : 0; + const int use_highbd = cm->seq_params.use_highbitdepth; const int buf_size = MC_TEMP_BUF_PELS << use_highbd; for (worker_idx = 0; worker_idx < pbi->max_threads - 1; ++worker_idx) { DecWorkerData *const thread_data = pbi->thread_data + worker_idx; @@ -3882,6 +4001,7 @@ static void dec_alloc_cb_buf(AV1Decoder *pbi) { av1_dec_free_cb_buf(pbi); CHECK_MEM_ERROR(cm, pbi->cb_buffer_base, aom_memalign(32, sizeof(*pbi->cb_buffer_base) * size)); + memset(pbi->cb_buffer_base, 0, sizeof(*pbi->cb_buffer_base) * size); pbi->cb_buffer_alloc_size = size; } } @@ -3969,7 +4089,8 @@ static const uint8_t *decode_tiles_row_mt(AV1Decoder *pbi, const uint8_t *data, int tile_cols_start; int tile_cols_end; int tile_count_tg; - int num_workers; + int num_workers = 0; + int max_threads; const uint8_t *raw_data_end = NULL; int max_sb_rows = 0; @@ -3985,7 +4106,7 @@ static const uint8_t *decode_tiles_row_mt(AV1Decoder *pbi, const uint8_t *data, tile_cols_end = tile_cols; } tile_count_tg = end_tile - start_tile + 1; - num_workers = pbi->max_threads; + max_threads = pbi->max_threads; // No tiles to decode. if (tile_rows_end <= tile_rows_start || tile_cols_end <= tile_cols_start || @@ -3998,7 +4119,7 @@ static const uint8_t *decode_tiles_row_mt(AV1Decoder *pbi, const uint8_t *data, assert(tile_rows <= MAX_TILE_ROWS); assert(tile_cols <= MAX_TILE_COLS); assert(tile_count_tg > 0); - assert(num_workers > 0); + assert(max_threads > 0); assert(start_tile <= end_tile); assert(start_tile >= 0 && end_tile < n_tiles); @@ -4030,8 +4151,10 @@ static const uint8_t *decode_tiles_row_mt(AV1Decoder *pbi, const uint8_t *data, max_sb_rows = AOMMAX(max_sb_rows, av1_get_sb_rows_in_tile(cm, tile_data->tile_info)); + num_workers += get_max_row_mt_workers_per_tile(cm, tile_data->tile_info); } } + num_workers = AOMMIN(num_workers, max_threads); if (pbi->allocated_row_mt_sync_rows != max_sb_rows) { for (int i = 0; i < n_tiles; ++i) { @@ -4107,7 +4230,7 @@ void av1_read_film_grain_params(AV1_COMMON *cm, } pars->random_seed = aom_rb_read_literal(rb, 16); - if (cm->frame_type == INTER_FRAME) + if (cm->current_frame.frame_type == INTER_FRAME) pars->update_parameters = aom_rb_read_bit(rb); else pars->update_parameters = 1; @@ -4116,20 +4239,19 @@ void av1_read_film_grain_params(AV1_COMMON *cm, if (!pars->update_parameters) { // inherit parameters from a previous reference frame - RefCntBuffer *const frame_bufs = cm->buffer_pool->frame_bufs; int film_grain_params_ref_idx = aom_rb_read_literal(rb, 3); - int buf_idx = cm->ref_frame_map[film_grain_params_ref_idx]; - if (buf_idx == INVALID_IDX) { + RefCntBuffer *const buf = cm->ref_frame_map[film_grain_params_ref_idx]; + if (buf == NULL) { aom_internal_error(&cm->error, AOM_CODEC_UNSUP_BITSTREAM, "Invalid Film grain reference idx"); } - if (!frame_bufs[buf_idx].film_grain_params_present) { + if (!buf->film_grain_params_present) { aom_internal_error(&cm->error, AOM_CODEC_UNSUP_BITSTREAM, "Film grain reference parameters not available"); } uint16_t random_seed = pars->random_seed; - *pars = frame_bufs[buf_idx].film_grain_params; // inherit paramaters - pars->random_seed = random_seed; // with new random seed + *pars = buf->film_grain_params; // inherit paramaters + pars->random_seed = random_seed; // with new random seed return; } @@ -4286,7 +4408,6 @@ void av1_read_color_config(struct aom_read_bit_buffer *rb, if (seq_params->color_primaries == AOM_CICP_CP_BT_709 && seq_params->transfer_characteristics == AOM_CICP_TC_SRGB && seq_params->matrix_coefficients == AOM_CICP_MC_IDENTITY) { - // It would be good to remove this dependency. seq_params->subsampling_y = seq_params->subsampling_x = 0; seq_params->color_range = 1; // assume full color-range if (!(seq_params->profile == PROFILE_1 || @@ -4431,23 +4552,23 @@ void av1_read_sequence_header(AV1_COMMON *cm, struct aom_read_bit_buffer *rb, seq_params->enable_masked_compound = 0; seq_params->enable_warped_motion = 0; seq_params->enable_dual_filter = 0; - seq_params->enable_order_hint = 0; - seq_params->enable_jnt_comp = 0; - seq_params->enable_ref_frame_mvs = 0; + seq_params->order_hint_info.enable_order_hint = 0; + seq_params->order_hint_info.enable_dist_wtd_comp = 0; + seq_params->order_hint_info.enable_ref_frame_mvs = 0; seq_params->force_screen_content_tools = 2; // SELECT_SCREEN_CONTENT_TOOLS seq_params->force_integer_mv = 2; // SELECT_INTEGER_MV - seq_params->order_hint_bits_minus_1 = -1; + seq_params->order_hint_info.order_hint_bits_minus_1 = -1; } else { seq_params->enable_interintra_compound = aom_rb_read_bit(rb); seq_params->enable_masked_compound = aom_rb_read_bit(rb); seq_params->enable_warped_motion = aom_rb_read_bit(rb); seq_params->enable_dual_filter = aom_rb_read_bit(rb); - seq_params->enable_order_hint = aom_rb_read_bit(rb); - seq_params->enable_jnt_comp = - seq_params->enable_order_hint ? aom_rb_read_bit(rb) : 0; - seq_params->enable_ref_frame_mvs = - seq_params->enable_order_hint ? aom_rb_read_bit(rb) : 0; + seq_params->order_hint_info.enable_order_hint = aom_rb_read_bit(rb); + seq_params->order_hint_info.enable_dist_wtd_comp = + seq_params->order_hint_info.enable_order_hint ? aom_rb_read_bit(rb) : 0; + seq_params->order_hint_info.enable_ref_frame_mvs = + seq_params->order_hint_info.enable_order_hint ? aom_rb_read_bit(rb) : 0; if (aom_rb_read_bit(rb)) { seq_params->force_screen_content_tools = @@ -4465,8 +4586,10 @@ void av1_read_sequence_header(AV1_COMMON *cm, struct aom_read_bit_buffer *rb, } else { seq_params->force_integer_mv = 2; // SELECT_INTEGER_MV } - seq_params->order_hint_bits_minus_1 = - seq_params->enable_order_hint ? aom_rb_read_literal(rb, 3) : -1; + seq_params->order_hint_info.order_hint_bits_minus_1 = + seq_params->order_hint_info.enable_order_hint + ? aom_rb_read_literal(rb, 3) + : -1; } seq_params->enable_superres = aom_rb_read_bit(rb); @@ -4576,7 +4699,7 @@ static void read_global_motion(AV1_COMMON *cm, struct aom_read_bit_buffer *rb) { */ /* printf("Dec Ref %d [%d/%d]: %d %d %d %d\n", - frame, cm->current_video_frame, cm->show_frame, + frame, cm->current_frame.frame_number, cm->show_frame, cm->global_motion[frame].wmmat[0], cm->global_motion[frame].wmmat[1], cm->global_motion[frame].wmmat[2], @@ -4587,96 +4710,114 @@ static void read_global_motion(AV1_COMMON *cm, struct aom_read_bit_buffer *rb) { REF_FRAMES * sizeof(WarpedMotionParams)); } -static void show_existing_frame_reset(AV1Decoder *const pbi, - int existing_frame_idx) { +// Release the references to the frame buffers in cm->ref_frame_map and reset +// all elements of cm->ref_frame_map to NULL. +static void reset_ref_frame_map(AV1_COMMON *const cm) { + BufferPool *const pool = cm->buffer_pool; + + for (int i = 0; i < REF_FRAMES; i++) { + decrease_ref_count(cm->ref_frame_map[i], pool); + cm->ref_frame_map[i] = NULL; + } +} + +// Generate next_ref_frame_map. +static void generate_next_ref_frame_map(AV1Decoder *const pbi) { AV1_COMMON *const cm = &pbi->common; BufferPool *const pool = cm->buffer_pool; - RefCntBuffer *const frame_bufs = pool->frame_bufs; - assert(cm->show_existing_frame); - - cm->frame_type = KEY_FRAME; - - pbi->refresh_frame_flags = (1 << REF_FRAMES) - 1; - - for (int i = 0; i < INTER_REFS_PER_FRAME; ++i) { - cm->frame_refs[i].idx = INVALID_IDX; - cm->frame_refs[i].buf = NULL; - } - - if (pbi->need_resync) { - memset(&cm->ref_frame_map, -1, sizeof(cm->ref_frame_map)); - pbi->need_resync = 0; - } - - cm->cur_frame->intra_only = 1; - - if (cm->seq_params.frame_id_numbers_present_flag) { - /* If bitmask is set, update reference frame id values and - mark frames as valid for reference. - Note that the displayed frame be valid for referencing - in order to have been selected. - */ - int refresh_frame_flags = pbi->refresh_frame_flags; - int display_frame_id = cm->ref_frame_id[existing_frame_idx]; - for (int i = 0; i < REF_FRAMES; i++) { - if ((refresh_frame_flags >> i) & 1) { - cm->ref_frame_id[i] = display_frame_id; - cm->valid_for_referencing[i] = 1; - } - } - } - - cm->refresh_frame_context = REFRESH_FRAME_CONTEXT_DISABLED; - - // Generate next_ref_frame_map. lock_buffer_pool(pool); + // cm->next_ref_frame_map holds references to frame buffers. After storing a + // frame buffer index in cm->next_ref_frame_map, we need to increase the + // frame buffer's ref_count. int ref_index = 0; - for (int mask = pbi->refresh_frame_flags; mask; mask >>= 1) { + for (int mask = cm->current_frame.refresh_frame_flags; mask; mask >>= 1) { if (mask & 1) { - cm->next_ref_frame_map[ref_index] = cm->new_fb_idx; - ++frame_bufs[cm->new_fb_idx].ref_count; + cm->next_ref_frame_map[ref_index] = cm->cur_frame; } else { cm->next_ref_frame_map[ref_index] = cm->ref_frame_map[ref_index]; } - // Current thread holds the reference frame. - if (cm->ref_frame_map[ref_index] >= 0) - ++frame_bufs[cm->ref_frame_map[ref_index]].ref_count; + if (cm->next_ref_frame_map[ref_index] != NULL) + ++cm->next_ref_frame_map[ref_index]->ref_count; ++ref_index; } for (; ref_index < REF_FRAMES; ++ref_index) { cm->next_ref_frame_map[ref_index] = cm->ref_frame_map[ref_index]; - - // Current thread holds the reference frame. - if (cm->ref_frame_map[ref_index] >= 0) - ++frame_bufs[cm->ref_frame_map[ref_index]].ref_count; + if (cm->next_ref_frame_map[ref_index] != NULL) + ++cm->next_ref_frame_map[ref_index]->ref_count; } unlock_buffer_pool(pool); pbi->hold_ref_buf = 1; +} + +// If the refresh_frame_flags bitmask is set, update reference frame id values +// and mark frames as valid for reference. +static void update_ref_frame_id(AV1_COMMON *const cm, int frame_id) { + assert(cm->seq_params.frame_id_numbers_present_flag); + int refresh_frame_flags = cm->current_frame.refresh_frame_flags; + for (int i = 0; i < REF_FRAMES; i++) { + if ((refresh_frame_flags >> i) & 1) { + cm->ref_frame_id[i] = frame_id; + cm->valid_for_referencing[i] = 1; + } + } +} + +static void show_existing_frame_reset(AV1Decoder *const pbi, + int existing_frame_idx) { + AV1_COMMON *const cm = &pbi->common; + + assert(cm->show_existing_frame); + + cm->current_frame.frame_type = KEY_FRAME; + + cm->current_frame.refresh_frame_flags = (1 << REF_FRAMES) - 1; + + for (int i = 0; i < INTER_REFS_PER_FRAME; ++i) { + cm->remapped_ref_idx[i] = INVALID_IDX; + } + + if (pbi->need_resync) { + reset_ref_frame_map(cm); + pbi->need_resync = 0; + } + + // Note that the displayed frame must be valid for referencing in order to + // have been selected. + if (cm->seq_params.frame_id_numbers_present_flag) { + update_ref_frame_id(cm, cm->ref_frame_id[existing_frame_idx]); + } + + cm->refresh_frame_context = REFRESH_FRAME_CONTEXT_DISABLED; + + generate_next_ref_frame_map(pbi); // Reload the adapted CDFs from when we originally coded this keyframe - *cm->fc = cm->frame_contexts[existing_frame_idx]; + *cm->fc = cm->next_ref_frame_map[existing_frame_idx]->frame_context; } static INLINE void reset_frame_buffers(AV1_COMMON *cm) { RefCntBuffer *const frame_bufs = cm->buffer_pool->frame_bufs; int i; - memset(&cm->ref_frame_map, -1, sizeof(cm->ref_frame_map)); - memset(&cm->next_ref_frame_map, -1, sizeof(cm->next_ref_frame_map)); + // We have not stored any references to frame buffers in + // cm->next_ref_frame_map, so we can directly reset it to all NULL. + for (i = 0; i < REF_FRAMES; ++i) { + cm->next_ref_frame_map[i] = NULL; + } lock_buffer_pool(cm->buffer_pool); + reset_ref_frame_map(cm); + assert(cm->cur_frame->ref_count == 1); for (i = 0; i < FRAME_BUFFERS; ++i) { - if (i != cm->new_fb_idx) { - frame_bufs[i].ref_count = 0; - cm->buffer_pool->release_fb_cb(cm->buffer_pool->cb_priv, - &frame_bufs[i].raw_frame_buffer); - } else { - assert(frame_bufs[i].ref_count == 1); + // Reset all unreferenced frame buffers. We can also reset cm->cur_frame + // because we are the sole owner of cm->cur_frame. + if (frame_bufs[i].ref_count > 0 && &frame_bufs[i] != cm->cur_frame) { + continue; } - frame_bufs[i].cur_frame_offset = 0; - av1_zero(frame_bufs[i].ref_frame_offset); + frame_bufs[i].order_hint = 0; + av1_zero(frame_bufs[i].ref_order_hints); } av1_zero_unused_internal_frame_buffers(&cm->buffer_pool->int_frame_buffers); unlock_buffer_pool(cm->buffer_pool); @@ -4688,6 +4829,7 @@ static int read_uncompressed_header(AV1Decoder *pbi, struct aom_read_bit_buffer *rb) { AV1_COMMON *const cm = &pbi->common; const SequenceHeader *const seq_params = &cm->seq_params; + CurrentFrame *const current_frame = &cm->current_frame; MACROBLOCKD *const xd = &pbi->mb; BufferPool *const pool = cm->buffer_pool; RefCntBuffer *const frame_bufs = pool->frame_bufs; @@ -4697,20 +4839,22 @@ static int read_uncompressed_header(AV1Decoder *pbi, "No sequence header"); } - cm->last_frame_type = cm->frame_type; - cm->last_intra_only = cm->intra_only; - - // NOTE: By default all coded frames to be used as a reference - cm->is_reference_frame = 1; + cm->last_frame_type = current_frame->frame_type; if (seq_params->reduced_still_picture_hdr) { cm->show_existing_frame = 0; cm->show_frame = 1; - cm->frame_type = KEY_FRAME; + current_frame->frame_type = KEY_FRAME; + if (pbi->sequence_header_changed) { + // This is the start of a new coded video sequence. + pbi->sequence_header_changed = 0; + pbi->decoding_first_frame = 1; + reset_frame_buffers(cm); + } cm->error_resilient_mode = 1; } else { cm->show_existing_frame = aom_rb_read_bit(rb); - cm->reset_decoder_state = 0; + pbi->reset_decoder_state = 0; if (cm->show_existing_frame) { if (pbi->sequence_header_changed) { @@ -4720,7 +4864,11 @@ static int read_uncompressed_header(AV1Decoder *pbi, } // Show an existing frame directly. const int existing_frame_idx = aom_rb_read_literal(rb, 3); - const int frame_to_show = cm->ref_frame_map[existing_frame_idx]; + RefCntBuffer *const frame_to_show = cm->ref_frame_map[existing_frame_idx]; + if (frame_to_show == NULL) { + aom_internal_error(&cm->error, AOM_CODEC_UNSUP_BITSTREAM, + "Buffer does not contain a decoded frame"); + } if (seq_params->decoder_model_info_present_flag && cm->timing_info.equal_picture_interval == 0) { av1_read_temporal_point_info(cm, rb); @@ -4736,44 +4884,48 @@ static int read_uncompressed_header(AV1Decoder *pbi, "Reference buffer frame ID mismatch"); } lock_buffer_pool(pool); - if (frame_to_show < 0 || frame_bufs[frame_to_show].ref_count < 1) { - unlock_buffer_pool(pool); - aom_internal_error(&cm->error, AOM_CODEC_UNSUP_BITSTREAM, - "Buffer %d does not contain a decoded frame", - frame_to_show); - } - ref_cnt_fb(frame_bufs, &cm->new_fb_idx, frame_to_show); - cm->reset_decoder_state = - frame_bufs[frame_to_show].frame_type == KEY_FRAME; + assert(frame_to_show->ref_count > 0); + // cm->cur_frame should be the buffer referenced by the return value + // of the get_free_fb() call in av1_receive_compressed_data(), and + // generate_next_ref_frame_map() has not been called, so ref_count + // should still be 1. + assert(cm->cur_frame->ref_count == 1); + // assign_frame_buffer_p() decrements ref_count directly rather than + // call decrease_ref_count(). If cm->cur_frame->raw_frame_buffer has + // already been allocated, it will not be released by + // assign_frame_buffer_p()! + assert(!cm->cur_frame->raw_frame_buffer.data); + assign_frame_buffer_p(&cm->cur_frame, frame_to_show); + pbi->reset_decoder_state = frame_to_show->frame_type == KEY_FRAME; unlock_buffer_pool(pool); cm->lf.filter_level[0] = 0; cm->lf.filter_level[1] = 0; cm->show_frame = 1; - if (!frame_bufs[frame_to_show].showable_frame) { + if (!frame_to_show->showable_frame) { aom_merge_corrupted_flag(&xd->corrupted, 1); } - if (cm->reset_decoder_state) frame_bufs[frame_to_show].showable_frame = 0; + if (pbi->reset_decoder_state) frame_to_show->showable_frame = 0; - cm->film_grain_params = frame_bufs[frame_to_show].film_grain_params; + cm->film_grain_params = frame_to_show->film_grain_params; - if (cm->reset_decoder_state) { + if (pbi->reset_decoder_state) { show_existing_frame_reset(pbi, existing_frame_idx); } else { - pbi->refresh_frame_flags = 0; + current_frame->refresh_frame_flags = 0; } return 0; } - cm->frame_type = (FRAME_TYPE)aom_rb_read_literal(rb, 2); // 2 bits + current_frame->frame_type = (FRAME_TYPE)aom_rb_read_literal(rb, 2); if (pbi->sequence_header_changed) { - if (pbi->common.frame_type == KEY_FRAME) { + if (current_frame->frame_type == KEY_FRAME) { // This is the start of a new coded video sequence. pbi->sequence_header_changed = 0; pbi->decoding_first_frame = 1; - reset_frame_buffers(&pbi->common); + reset_frame_buffers(cm); } else { aom_internal_error(&cm->error, AOM_CODEC_CORRUPT_FRAME, "Sequence header has changed without a keyframe."); @@ -4782,11 +4934,11 @@ static int read_uncompressed_header(AV1Decoder *pbi, cm->show_frame = aom_rb_read_bit(rb); if (seq_params->still_picture && - (cm->frame_type != KEY_FRAME || !cm->show_frame)) { + (current_frame->frame_type != KEY_FRAME || !cm->show_frame)) { aom_internal_error(&cm->error, AOM_CODEC_CORRUPT_FRAME, "Still pictures must be coded as shown keyframes"); } - cm->showable_frame = cm->frame_type != KEY_FRAME; + cm->showable_frame = current_frame->frame_type != KEY_FRAME; if (cm->show_frame) { if (seq_params->decoder_model_info_present_flag && cm->timing_info.equal_picture_interval == 0) @@ -4796,9 +4948,9 @@ static int read_uncompressed_header(AV1Decoder *pbi, cm->showable_frame = aom_rb_read_bit(rb); } cm->cur_frame->showable_frame = cm->showable_frame; - cm->intra_only = cm->frame_type == INTRA_ONLY_FRAME; cm->error_resilient_mode = - frame_is_sframe(cm) || (cm->frame_type == KEY_FRAME && cm->show_frame) + frame_is_sframe(cm) || + (current_frame->frame_type == KEY_FRAME && cm->show_frame) ? 1 : aom_rb_read_bit(rb); } @@ -4820,7 +4972,6 @@ static int read_uncompressed_header(AV1Decoder *pbi, cm->cur_frame_force_integer_mv = 0; } - cm->frame_refs_short_signaling = 0; int frame_size_override_flag = 0; cm->allow_intrabc = 0; cm->primary_ref_frame = PRIMARY_REF_NONE; @@ -4830,8 +4981,9 @@ static int read_uncompressed_header(AV1Decoder *pbi, int frame_id_length = seq_params->frame_id_length; int diff_len = seq_params->delta_frame_id_length; int prev_frame_id = 0; - int have_prev_frame_id = !pbi->decoding_first_frame && - !(cm->frame_type == KEY_FRAME && cm->show_frame); + int have_prev_frame_id = + !pbi->decoding_first_frame && + !(current_frame->frame_type == KEY_FRAME && cm->show_frame); if (have_prev_frame_id) { prev_frame_id = cm->current_frame_id; } @@ -4854,7 +5006,7 @@ static int read_uncompressed_header(AV1Decoder *pbi, } /* Check if some frames need to be marked as not valid for referencing */ for (int i = 0; i < REF_FRAMES; i++) { - if (cm->frame_type == KEY_FRAME && cm->show_frame) { + if (current_frame->frame_type == KEY_FRAME && cm->show_frame) { cm->valid_for_referencing[i] = 0; } else if (cm->current_frame_id - (1 << diff_len) > 0) { if (cm->ref_frame_id[i] > cm->current_frame_id || @@ -4871,9 +5023,9 @@ static int read_uncompressed_header(AV1Decoder *pbi, frame_size_override_flag = frame_is_sframe(cm) ? 1 : aom_rb_read_bit(rb); - cm->frame_offset = - aom_rb_read_literal(rb, seq_params->order_hint_bits_minus_1 + 1); - cm->current_video_frame = cm->frame_offset; + current_frame->order_hint = aom_rb_read_literal( + rb, seq_params->order_hint_info.order_hint_bits_minus_1 + 1); + current_frame->frame_number = current_frame->order_hint; if (!cm->error_resilient_mode && !frame_is_intra_only(cm)) { cm->primary_ref_frame = aom_rb_read_literal(rb, PRIMARY_REF_BITS); @@ -4905,89 +5057,84 @@ static int read_uncompressed_header(AV1Decoder *pbi, } } } - if (cm->frame_type == KEY_FRAME) { - if (!cm->show_frame) // unshown keyframe (forward keyframe) - pbi->refresh_frame_flags = aom_rb_read_literal(rb, REF_FRAMES); - else // shown keyframe - pbi->refresh_frame_flags = (1 << REF_FRAMES) - 1; + if (current_frame->frame_type == KEY_FRAME) { + if (!cm->show_frame) { // unshown keyframe (forward keyframe) + current_frame->refresh_frame_flags = aom_rb_read_literal(rb, REF_FRAMES); + } else { // shown keyframe + current_frame->refresh_frame_flags = (1 << REF_FRAMES) - 1; + } for (int i = 0; i < INTER_REFS_PER_FRAME; ++i) { - cm->frame_refs[i].idx = INVALID_IDX; - cm->frame_refs[i].buf = NULL; + cm->remapped_ref_idx[i] = INVALID_IDX; } if (pbi->need_resync) { - memset(&cm->ref_frame_map, -1, sizeof(cm->ref_frame_map)); + reset_ref_frame_map(cm); pbi->need_resync = 0; } } else { - if (cm->intra_only) { - pbi->refresh_frame_flags = aom_rb_read_literal(rb, REF_FRAMES); - if (pbi->refresh_frame_flags == 0xFF) { + if (current_frame->frame_type == INTRA_ONLY_FRAME) { + current_frame->refresh_frame_flags = aom_rb_read_literal(rb, REF_FRAMES); + if (current_frame->refresh_frame_flags == 0xFF) { aom_internal_error(&cm->error, AOM_CODEC_UNSUP_BITSTREAM, "Intra only frames cannot have refresh flags 0xFF"); } if (pbi->need_resync) { - memset(&cm->ref_frame_map, -1, sizeof(cm->ref_frame_map)); + reset_ref_frame_map(cm); pbi->need_resync = 0; } } else if (pbi->need_resync != 1) { /* Skip if need resync */ - pbi->refresh_frame_flags = + current_frame->refresh_frame_flags = frame_is_sframe(cm) ? 0xFF : aom_rb_read_literal(rb, REF_FRAMES); - if (!pbi->refresh_frame_flags) { - // NOTE: "pbi->refresh_frame_flags == 0" indicates that the coded frame - // will not be used as a reference - cm->is_reference_frame = 0; - } } } - if (!frame_is_intra_only(cm) || pbi->refresh_frame_flags != 0xFF) { + if (!frame_is_intra_only(cm) || current_frame->refresh_frame_flags != 0xFF) { // Read all ref frame order hints if error_resilient_mode == 1 - if (cm->error_resilient_mode && seq_params->enable_order_hint) { + if (cm->error_resilient_mode && + seq_params->order_hint_info.enable_order_hint) { for (int ref_idx = 0; ref_idx < REF_FRAMES; ref_idx++) { // Read order hint from bit stream - unsigned int frame_offset = - aom_rb_read_literal(rb, seq_params->order_hint_bits_minus_1 + 1); - // Get buffer index - int buf_idx = cm->ref_frame_map[ref_idx]; - assert(buf_idx < FRAME_BUFFERS); - if (buf_idx == -1 || - frame_offset != frame_bufs[buf_idx].cur_frame_offset) { - if (buf_idx >= 0) { + unsigned int order_hint = aom_rb_read_literal( + rb, seq_params->order_hint_info.order_hint_bits_minus_1 + 1); + // Get buffer + RefCntBuffer *buf = cm->ref_frame_map[ref_idx]; + if (buf == NULL || order_hint != buf->order_hint) { + if (buf != NULL) { lock_buffer_pool(pool); - decrease_ref_count(buf_idx, frame_bufs, pool); + decrease_ref_count(buf, pool); unlock_buffer_pool(pool); } // If no corresponding buffer exists, allocate a new buffer with all // pixels set to neutral grey. - buf_idx = get_free_fb(cm); + int buf_idx = get_free_fb(cm); if (buf_idx == INVALID_IDX) { aom_internal_error(&cm->error, AOM_CODEC_MEM_ERROR, "Unable to find free frame buffer"); } + buf = &frame_bufs[buf_idx]; lock_buffer_pool(pool); if (aom_realloc_frame_buffer( - &frame_bufs[buf_idx].buf, seq_params->max_frame_width, + &buf->buf, seq_params->max_frame_width, seq_params->max_frame_height, seq_params->subsampling_x, seq_params->subsampling_y, seq_params->use_highbitdepth, AOM_BORDER_IN_PIXELS, cm->byte_alignment, - &pool->frame_bufs[buf_idx].raw_frame_buffer, pool->get_fb_cb, - pool->cb_priv)) { + &buf->raw_frame_buffer, pool->get_fb_cb, pool->cb_priv)) { + decrease_ref_count(buf, pool); unlock_buffer_pool(pool); aom_internal_error(&cm->error, AOM_CODEC_MEM_ERROR, "Failed to allocate frame buffer"); } unlock_buffer_pool(pool); - set_planes_to_neutral_grey(seq_params, &frame_bufs[buf_idx].buf, 0); + set_planes_to_neutral_grey(seq_params, &buf->buf, 0); - cm->ref_frame_map[ref_idx] = buf_idx; - frame_bufs[buf_idx].cur_frame_offset = frame_offset; + cm->ref_frame_map[ref_idx] = buf; + buf->order_hint = order_hint; } } } } - if (cm->frame_type == KEY_FRAME) { + if (current_frame->frame_type == KEY_FRAME) { setup_frame_size(cm, frame_size_override_flag, rb); if (cm->allow_screen_content_tools && !av1_superres_scaled(cm)) @@ -4997,7 +5144,7 @@ static int read_uncompressed_header(AV1Decoder *pbi, } else { cm->allow_ref_frame_mvs = 0; - if (cm->intra_only) { + if (current_frame->frame_type == INTRA_ONLY_FRAME) { cm->cur_frame->film_grain_params_present = seq_params->film_grain_params_present; setup_frame_size(cm, frame_size_override_flag, rb); @@ -5005,30 +5152,30 @@ static int read_uncompressed_header(AV1Decoder *pbi, cm->allow_intrabc = aom_rb_read_bit(rb); } else if (pbi->need_resync != 1) { /* Skip if need resync */ - + int frame_refs_short_signaling = 0; // Frame refs short signaling is off when error resilient mode is on. - if (seq_params->enable_order_hint) - cm->frame_refs_short_signaling = aom_rb_read_bit(rb); + if (seq_params->order_hint_info.enable_order_hint) + frame_refs_short_signaling = aom_rb_read_bit(rb); - if (cm->frame_refs_short_signaling) { + if (frame_refs_short_signaling) { // == LAST_FRAME == const int lst_ref = aom_rb_read_literal(rb, REF_FRAMES_LOG2); - const int lst_idx = cm->ref_frame_map[lst_ref]; + const RefCntBuffer *const lst_buf = cm->ref_frame_map[lst_ref]; // == GOLDEN_FRAME == const int gld_ref = aom_rb_read_literal(rb, REF_FRAMES_LOG2); - const int gld_idx = cm->ref_frame_map[gld_ref]; + const RefCntBuffer *const gld_buf = cm->ref_frame_map[gld_ref]; // Most of the time, streams start with a keyframe. In that case, // ref_frame_map will have been filled in at that point and will not - // contain any -1's. However, streams are explicitly allowed to start + // contain any NULLs. However, streams are explicitly allowed to start // with an intra-only frame, so long as they don't then signal a // reference to a slot that hasn't been set yet. That's what we are // checking here. - if (lst_idx == -1) + if (lst_buf == NULL) aom_internal_error(&cm->error, AOM_CODEC_CORRUPT_FRAME, "Inter frame requests nonexistent reference"); - if (gld_idx == -1) + if (gld_buf == NULL) aom_internal_error(&cm->error, AOM_CODEC_CORRUPT_FRAME, "Inter frame requests nonexistent reference"); @@ -5037,9 +5184,8 @@ static int read_uncompressed_header(AV1Decoder *pbi, for (int i = 0; i < INTER_REFS_PER_FRAME; ++i) { int ref = 0; - if (!cm->frame_refs_short_signaling) { + if (!frame_refs_short_signaling) { ref = aom_rb_read_literal(rb, REF_FRAMES_LOG2); - const int idx = cm->ref_frame_map[ref]; // Most of the time, streams start with a keyframe. In that case, // ref_frame_map will have been filled in at that point and will not @@ -5047,16 +5193,12 @@ static int read_uncompressed_header(AV1Decoder *pbi, // with an intra-only frame, so long as they don't then signal a // reference to a slot that hasn't been set yet. That's what we are // checking here. - if (idx == -1) + if (cm->ref_frame_map[ref] == NULL) aom_internal_error(&cm->error, AOM_CODEC_CORRUPT_FRAME, "Inter frame requests nonexistent reference"); - - RefBuffer *const ref_frame = &cm->frame_refs[i]; - ref_frame->idx = idx; - ref_frame->buf = &frame_bufs[idx].buf; - ref_frame->map_idx = ref; + cm->remapped_ref_idx[i] = ref; } else { - ref = cm->frame_refs[i].map_idx; + ref = cm->remapped_ref_idx[i]; } cm->ref_frame_sign_bias[LAST_FRAME + i] = 0; @@ -5093,26 +5235,29 @@ static int read_uncompressed_header(AV1Decoder *pbi, cm->switchable_motion_mode = aom_rb_read_bit(rb); } - cm->prev_frame = get_prev_frame(cm); + cm->prev_frame = get_primary_ref_frame_buf(cm); if (cm->primary_ref_frame != PRIMARY_REF_NONE && - cm->frame_refs[cm->primary_ref_frame].idx < 0) { + get_primary_ref_frame_buf(cm) == NULL) { aom_internal_error(&cm->error, AOM_CODEC_CORRUPT_FRAME, "Reference frame containing this frame's initial " "frame context is unavailable."); } - if (!cm->intra_only && pbi->need_resync != 1) { + if (!(current_frame->frame_type == INTRA_ONLY_FRAME) && + pbi->need_resync != 1) { if (frame_might_allow_ref_frame_mvs(cm)) cm->allow_ref_frame_mvs = aom_rb_read_bit(rb); else cm->allow_ref_frame_mvs = 0; - for (int i = 0; i < INTER_REFS_PER_FRAME; ++i) { - RefBuffer *const ref_buf = &cm->frame_refs[i]; + for (int i = LAST_FRAME; i <= ALTREF_FRAME; ++i) { + const RefCntBuffer *const ref_buf = get_ref_frame_buf(cm, i); + struct scale_factors *const ref_scale_factors = + get_ref_scale_factors(cm, i); av1_setup_scale_factors_for_frame( - &ref_buf->sf, ref_buf->buf->y_crop_width, - ref_buf->buf->y_crop_height, cm->width, cm->height); - if ((!av1_is_valid_scale(&ref_buf->sf))) + ref_scale_factors, ref_buf->buf.y_crop_width, + ref_buf->buf.y_crop_height, cm->width, cm->height); + if ((!av1_is_valid_scale(ref_scale_factors))) aom_internal_error(&cm->error, AOM_CODEC_UNSUP_BITSTREAM, "Reference frame has invalid dimensions"); } @@ -5123,19 +5268,10 @@ static int read_uncompressed_header(AV1Decoder *pbi, av1_setup_frame_sign_bias(cm); - cm->cur_frame->intra_only = cm->frame_type == KEY_FRAME || cm->intra_only; - cm->cur_frame->frame_type = cm->frame_type; + cm->cur_frame->frame_type = current_frame->frame_type; if (seq_params->frame_id_numbers_present_flag) { - /* If bitmask is set, update reference frame id values and - mark frames as valid for reference */ - int refresh_frame_flags = pbi->refresh_frame_flags; - for (int i = 0; i < REF_FRAMES; i++) { - if ((refresh_frame_flags >> i) & 1) { - cm->ref_frame_id[i] = cm->current_frame_id; - cm->valid_for_referencing[i] = 1; - } - } + update_ref_frame_id(cm, cm->current_frame_id); } const int might_bwd_adapt = @@ -5148,18 +5284,17 @@ static int read_uncompressed_header(AV1Decoder *pbi, cm->refresh_frame_context = REFRESH_FRAME_CONTEXT_DISABLED; } - get_frame_new_buffer(cm)->bit_depth = seq_params->bit_depth; - get_frame_new_buffer(cm)->color_primaries = seq_params->color_primaries; - get_frame_new_buffer(cm)->transfer_characteristics = + cm->cur_frame->buf.bit_depth = seq_params->bit_depth; + cm->cur_frame->buf.color_primaries = seq_params->color_primaries; + cm->cur_frame->buf.transfer_characteristics = seq_params->transfer_characteristics; - get_frame_new_buffer(cm)->matrix_coefficients = - seq_params->matrix_coefficients; - get_frame_new_buffer(cm)->monochrome = seq_params->monochrome; - get_frame_new_buffer(cm)->chroma_sample_position = + cm->cur_frame->buf.matrix_coefficients = seq_params->matrix_coefficients; + cm->cur_frame->buf.monochrome = seq_params->monochrome; + cm->cur_frame->buf.chroma_sample_position = seq_params->chroma_sample_position; - get_frame_new_buffer(cm)->color_range = seq_params->color_range; - get_frame_new_buffer(cm)->render_width = cm->render_width; - get_frame_new_buffer(cm)->render_height = cm->render_height; + cm->cur_frame->buf.color_range = seq_params->color_range; + cm->cur_frame->buf.render_width = cm->render_width; + cm->cur_frame->buf.render_height = cm->render_height; if (pbi->need_resync) { aom_internal_error(&cm->error, AOM_CODEC_CORRUPT_FRAME, @@ -5167,41 +5302,17 @@ static int read_uncompressed_header(AV1Decoder *pbi, " state"); } - // Generate next_ref_frame_map. - lock_buffer_pool(pool); - int ref_index = 0; - for (int mask = pbi->refresh_frame_flags; mask; mask >>= 1) { - if (mask & 1) { - cm->next_ref_frame_map[ref_index] = cm->new_fb_idx; - ++frame_bufs[cm->new_fb_idx].ref_count; - } else { - cm->next_ref_frame_map[ref_index] = cm->ref_frame_map[ref_index]; - } - // Current thread holds the reference frame. - if (cm->ref_frame_map[ref_index] >= 0) - ++frame_bufs[cm->ref_frame_map[ref_index]].ref_count; - ++ref_index; - } - - for (; ref_index < REF_FRAMES; ++ref_index) { - cm->next_ref_frame_map[ref_index] = cm->ref_frame_map[ref_index]; - - // Current thread holds the reference frame. - if (cm->ref_frame_map[ref_index] >= 0) - ++frame_bufs[cm->ref_frame_map[ref_index]].ref_count; - } - unlock_buffer_pool(pool); - pbi->hold_ref_buf = 1; + generate_next_ref_frame_map(pbi); if (cm->allow_intrabc) { // Set parameters corresponding to no filtering. struct loopfilter *lf = &cm->lf; lf->filter_level[0] = 0; lf->filter_level[1] = 0; - cm->cdef_bits = 0; - cm->cdef_strengths[0] = 0; - cm->nb_cdef_strengths = 1; - cm->cdef_uv_strengths[0] = 0; + cm->cdef_info.cdef_bits = 0; + cm->cdef_info.cdef_strengths[0] = 0; + cm->cdef_info.nb_cdef_strengths = 1; + cm->cdef_info.cdef_uv_strengths[0] = 0; cm->rst_info[0].frame_restoration_type = RESTORE_NONE; cm->rst_info[1].frame_restoration_type = RESTORE_NONE; cm->rst_info[2].frame_restoration_type = RESTORE_NONE; @@ -5226,18 +5337,20 @@ static int read_uncompressed_header(AV1Decoder *pbi, setup_segmentation(cm, rb); - cm->delta_q_res = 1; - cm->delta_lf_res = 1; - cm->delta_lf_present_flag = 0; - cm->delta_lf_multi = 0; - cm->delta_q_present_flag = cm->base_qindex > 0 ? aom_rb_read_bit(rb) : 0; - if (cm->delta_q_present_flag) { + cm->delta_q_info.delta_q_res = 1; + cm->delta_q_info.delta_lf_res = 1; + cm->delta_q_info.delta_lf_present_flag = 0; + cm->delta_q_info.delta_lf_multi = 0; + cm->delta_q_info.delta_q_present_flag = + cm->base_qindex > 0 ? aom_rb_read_bit(rb) : 0; + if (cm->delta_q_info.delta_q_present_flag) { xd->current_qindex = cm->base_qindex; - cm->delta_q_res = 1 << aom_rb_read_literal(rb, 2); - if (!cm->allow_intrabc) cm->delta_lf_present_flag = aom_rb_read_bit(rb); - if (cm->delta_lf_present_flag) { - cm->delta_lf_res = 1 << aom_rb_read_literal(rb, 2); - cm->delta_lf_multi = aom_rb_read_bit(rb); + cm->delta_q_info.delta_q_res = 1 << aom_rb_read_literal(rb, 2); + if (!cm->allow_intrabc) + cm->delta_q_info.delta_lf_present_flag = aom_rb_read_bit(rb); + if (cm->delta_q_info.delta_lf_present_flag) { + cm->delta_q_info.delta_lf_res = 1 << aom_rb_read_literal(rb, 2); + cm->delta_q_info.delta_lf_multi = aom_rb_read_bit(rb); av1_reset_loop_filter_delta(xd, av1_num_planes(cm)); } } @@ -5245,9 +5358,7 @@ static int read_uncompressed_header(AV1Decoder *pbi, xd->cur_frame_force_integer_mv = cm->cur_frame_force_integer_mv; for (int i = 0; i < MAX_SEGMENTS; ++i) { - const int qindex = cm->seg.enabled - ? av1_get_qindex(&cm->seg, i, cm->base_qindex) - : cm->base_qindex; + const int qindex = av1_get_qindex(&cm->seg, i, cm->base_qindex); xd->lossless[i] = qindex == 0 && cm->y_dc_delta_q == 0 && cm->u_dc_delta_q == 0 && cm->u_ac_delta_q == 0 && cm->v_dc_delta_q == 0 && cm->v_ac_delta_q == 0; @@ -5255,15 +5366,15 @@ static int read_uncompressed_header(AV1Decoder *pbi, } cm->coded_lossless = is_coded_lossless(cm, xd); cm->all_lossless = cm->coded_lossless && !av1_superres_scaled(cm); - setup_segmentation_dequant(cm); + setup_segmentation_dequant(cm, xd); if (cm->coded_lossless) { cm->lf.filter_level[0] = 0; cm->lf.filter_level[1] = 0; } if (cm->coded_lossless || !seq_params->enable_cdef) { - cm->cdef_bits = 0; - cm->cdef_strengths[0] = 0; - cm->cdef_uv_strengths[0] = 0; + cm->cdef_info.cdef_bits = 0; + cm->cdef_info.cdef_strengths[0] = 0; + cm->cdef_info.cdef_uv_strengths[0] = 0; } if (cm->all_lossless || !seq_params->enable_restoration) { cm->rst_info[0].frame_restoration_type = RESTORE_NONE; @@ -5280,11 +5391,13 @@ static int read_uncompressed_header(AV1Decoder *pbi, } cm->tx_mode = read_tx_mode(cm, rb); - cm->reference_mode = read_frame_reference_mode(cm, rb); - if (cm->reference_mode != SINGLE_REFERENCE) setup_compound_reference_mode(cm); + current_frame->reference_mode = read_frame_reference_mode(cm, rb); + if (current_frame->reference_mode != SINGLE_REFERENCE) + setup_compound_reference_mode(cm); av1_setup_skip_mode_allowed(cm); - cm->skip_mode_flag = cm->is_skip_mode_allowed ? aom_rb_read_bit(rb) : 0; + current_frame->skip_mode_info.skip_mode_flag = + current_frame->skip_mode_info.skip_mode_allowed ? aom_rb_read_bit(rb) : 0; if (frame_might_allow_warped_motion(cm)) cm->allow_warped_motion = aom_rb_read_bit(rb); @@ -5335,7 +5448,7 @@ BITSTREAM_PROFILE av1_read_profile(struct aom_read_bit_buffer *rb) { return (BITSTREAM_PROFILE)profile; } -void superres_post_decode(AV1Decoder *pbi) { +static void superres_post_decode(AV1Decoder *pbi) { AV1_COMMON *const cm = &pbi->common; BufferPool *const pool = cm->buffer_pool; @@ -5357,7 +5470,8 @@ uint32_t av1_decode_frame_headers_and_setup(AV1Decoder *pbi, MACROBLOCKD *const xd = &pbi->mb; #if CONFIG_BITSTREAM_DEBUG - bitstream_queue_set_frame_read(cm->current_video_frame * 2 + cm->show_frame); + bitstream_queue_set_frame_read(cm->current_frame.frame_number * 2 + + cm->show_frame); #endif #if CONFIG_MISMATCH_DEBUG mismatch_move_frame_idx_r(); @@ -5383,7 +5497,7 @@ uint32_t av1_decode_frame_headers_and_setup(AV1Decoder *pbi, const uint32_t uncomp_hdr_size = (uint32_t)aom_rb_bytes_read(rb); // Size of the uncompressed header - YV12_BUFFER_CONFIG *new_fb = get_frame_new_buffer(cm); + YV12_BUFFER_CONFIG *new_fb = &cm->cur_frame->buf; xd->cur_buf = new_fb; if (av1_allow_intrabc(cm)) { av1_setup_scale_factors_for_frame( @@ -5394,9 +5508,9 @@ uint32_t av1_decode_frame_headers_and_setup(AV1Decoder *pbi, if (cm->show_existing_frame) { // showing a frame directly *p_data_end = data + uncomp_hdr_size; - if (cm->reset_decoder_state) { + if (pbi->reset_decoder_state) { // Use the default frame context values. - *cm->fc = cm->frame_contexts[FRAME_CONTEXT_DEFAULTS]; + *cm->fc = *cm->default_frame_context; if (!cm->fc->initialized) aom_internal_error(&cm->error, AOM_CODEC_CORRUPT_FRAME, "Uninitialized entropy context."); @@ -5406,17 +5520,15 @@ uint32_t av1_decode_frame_headers_and_setup(AV1Decoder *pbi, cm->setup_mi(cm); - cm->current_frame_seg_map = cm->cur_frame->seg_map; - av1_setup_motion_field(cm); av1_setup_block_planes(xd, cm->seq_params.subsampling_x, cm->seq_params.subsampling_y, num_planes); if (cm->primary_ref_frame == PRIMARY_REF_NONE) { // use the default frame context values - *cm->fc = cm->frame_contexts[FRAME_CONTEXT_DEFAULTS]; + *cm->fc = *cm->default_frame_context; } else { - *cm->fc = cm->frame_contexts[cm->frame_refs[cm->primary_ref_frame].idx]; + *cm->fc = get_primary_ref_frame_buf(cm)->frame_context; } if (!cm->fc->initialized) aom_internal_error(&cm->error, AOM_CODEC_CORRUPT_FRAME, @@ -5435,7 +5547,7 @@ static void setup_frame_info(AV1Decoder *pbi) { cm->rst_info[2].frame_restoration_type != RESTORE_NONE) { av1_alloc_restoration_buffers(cm); } - const int use_highbd = cm->seq_params.use_highbitdepth ? 1 : 0; + const int use_highbd = cm->seq_params.use_highbitdepth; const int buf_size = MC_TEMP_BUF_PELS << use_highbd; if (pbi->td.mc_buf_size != buf_size) { av1_free_mc_tmp_buf(&pbi->td); @@ -5455,7 +5567,6 @@ void av1_decode_tg_tiles_and_wrapup(AV1Decoder *pbi, const uint8_t *data, const int num_planes = av1_num_planes(cm); #if LOOP_FILTER_BITMASK av1_loop_filter_frame_init(cm, 0, num_planes); - av1_zero_array(cm->lf.lfm, cm->lf.lfm_num); #endif if (pbi->max_threads > 1 && !(cm->large_scale_tile && !pbi->ext_tile_debug) && @@ -5479,19 +5590,20 @@ void av1_decode_tg_tiles_and_wrapup(AV1Decoder *pbi, const uint8_t *data, if (!cm->allow_intrabc && !cm->single_tile_decoding) { if (cm->lf.filter_level[0] || cm->lf.filter_level[1]) { -#if LOOP_FILTER_BITMASK - av1_loop_filter_frame(get_frame_new_buffer(cm), cm, &pbi->mb, 1, 0, - num_planes, 0); -#else if (pbi->num_workers > 1) { - av1_loop_filter_frame_mt(get_frame_new_buffer(cm), cm, &pbi->mb, 0, - num_planes, 0, pbi->tile_workers, - pbi->num_workers, &pbi->lf_row_sync); - } else { - av1_loop_filter_frame(get_frame_new_buffer(cm), cm, &pbi->mb, 0, - num_planes, 0); - } + av1_loop_filter_frame_mt( + &cm->cur_frame->buf, cm, &pbi->mb, 0, num_planes, 0, +#if LOOP_FILTER_BITMASK + 1, #endif + pbi->tile_workers, pbi->num_workers, &pbi->lf_row_sync); + } else { + av1_loop_filter_frame(&cm->cur_frame->buf, cm, &pbi->mb, +#if LOOP_FILTER_BITMASK + 1, +#endif + 0, num_planes, 0); + } } const int do_loop_restoration = @@ -5500,20 +5612,23 @@ void av1_decode_tg_tiles_and_wrapup(AV1Decoder *pbi, const uint8_t *data, cm->rst_info[2].frame_restoration_type != RESTORE_NONE; const int do_cdef = !cm->skip_loop_filter && !cm->coded_lossless && - (cm->cdef_bits || cm->cdef_strengths[0] || cm->cdef_uv_strengths[0]); + (cm->cdef_info.cdef_bits || cm->cdef_info.cdef_strengths[0] || + cm->cdef_info.cdef_uv_strengths[0]); const int do_superres = av1_superres_scaled(cm); const int optimized_loop_restoration = !do_cdef && !do_superres; if (!optimized_loop_restoration) { if (do_loop_restoration) - av1_loop_restoration_save_boundary_lines(&pbi->cur_buf->buf, cm, 0); + av1_loop_restoration_save_boundary_lines(&pbi->common.cur_frame->buf, + cm, 0); - if (do_cdef) av1_cdef_frame(&pbi->cur_buf->buf, cm, &pbi->mb); + if (do_cdef) av1_cdef_frame(&pbi->common.cur_frame->buf, cm, &pbi->mb); superres_post_decode(pbi); if (do_loop_restoration) { - av1_loop_restoration_save_boundary_lines(&pbi->cur_buf->buf, cm, 1); + av1_loop_restoration_save_boundary_lines(&pbi->common.cur_frame->buf, + cm, 1); if (pbi->num_workers > 1) { av1_loop_restoration_filter_frame_mt( (YV12_BUFFER_CONFIG *)xd->cur_buf, cm, optimized_loop_restoration, @@ -5542,6 +5657,9 @@ void av1_decode_tg_tiles_and_wrapup(AV1Decoder *pbi, const uint8_t *data, } } } +#if LOOP_FILTER_BITMASK + av1_zero_array(cm->lf.lfm, cm->lf.lfm_num); +#endif if (!xd->corrupted) { if (cm->refresh_frame_context == REFRESH_FRAME_CONTEXT_BACKWARD) { @@ -5562,6 +5680,6 @@ void av1_decode_tg_tiles_and_wrapup(AV1Decoder *pbi, const uint8_t *data, // Non frame parallel update frame context here. if (!cm->large_scale_tile) { - cm->frame_contexts[cm->new_fb_idx] = *cm->fc; + cm->cur_frame->frame_context = *cm->fc; } } diff --git a/third_party/aom/av1/decoder/decodeframe.h b/third_party/aom/av1/decoder/decodeframe.h index ddad273f18..13b969663b 100644 --- a/third_party/aom/av1/decoder/decodeframe.h +++ b/third_party/aom/av1/decoder/decodeframe.h @@ -46,8 +46,8 @@ uint32_t av1_decode_frame_headers_and_setup(struct AV1Decoder *pbi, void av1_decode_tg_tiles_and_wrapup(struct AV1Decoder *pbi, const uint8_t *data, const uint8_t *data_end, - const uint8_t **p_data_end, int startTile, - int endTile, int initialize_flag); + const uint8_t **p_data_end, int start_tile, + int end_tile, int initialize_flag); // Implements the color_config() function in the spec. Reports errors by // calling rb->error_handler() or aom_internal_error(). diff --git a/third_party/aom/av1/decoder/decodemv.c b/third_party/aom/av1/decoder/decodemv.c index 551e4d5437..b7431f2489 100644 --- a/third_party/aom/av1/decoder/decodemv.c +++ b/third_party/aom/av1/decoder/decodemv.c @@ -41,7 +41,7 @@ static void read_cdef(AV1_COMMON *cm, aom_reader *r, MACROBLOCKD *const xd, MB_MODE_INFO *const mbmi = xd->mi[0]; if (cm->coded_lossless) return; if (cm->allow_intrabc) { - assert(cm->cdef_bits == 0); + assert(cm->cdef_info.cdef_bits == 0); return; } @@ -59,7 +59,7 @@ static void read_cdef(AV1_COMMON *cm, aom_reader *r, MACROBLOCKD *const xd, cm->mi_grid_visible[(mi_row & m) * cm->mi_stride + (mi_col & m)] ->cdef_strength = xd->cdef_preset[index] = xd->cdef_preset[index] == -1 && !mbmi->skip - ? aom_read_literal(r, cm->cdef_bits, ACCT_STR) + ? aom_read_literal(r, cm->cdef_info.cdef_bits, ACCT_STR) : xd->cdef_preset[index]; } @@ -299,7 +299,7 @@ static void set_segment_id(AV1_COMMON *cm, int mi_offset, int x_mis, int y_mis, for (int y = 0; y < y_mis; y++) for (int x = 0; x < x_mis; x++) - cm->current_frame_seg_map[mi_offset + y * cm->mi_cols + x] = segment_id; + cm->cur_frame->seg_map[mi_offset + y * cm->mi_cols + x] = segment_id; } static int read_intra_segment_id(AV1_COMMON *const cm, @@ -355,7 +355,7 @@ static int read_inter_segment_id(AV1_COMMON *const cm, MACROBLOCKD *const xd, if (!seg->enabled) return 0; // Default for disabled segmentation if (!seg->update_map) { - copy_segment_id(cm, cm->last_frame_seg_map, cm->current_frame_seg_map, + copy_segment_id(cm, cm->last_frame_seg_map, cm->cur_frame->seg_map, mi_offset, x_mis, y_mis); return get_predicted_segment_id(cm, mi_offset, x_mis, y_mis); } @@ -364,7 +364,6 @@ static int read_inter_segment_id(AV1_COMMON *const cm, MACROBLOCKD *const xd, if (preskip) { if (!seg->segid_preskip) return 0; } else { - if (seg->segid_preskip) return mbmi->segment_id; if (mbmi->skip) { if (seg->temporal_update) { mbmi->seg_id_predicted = 0; @@ -395,7 +394,7 @@ static int read_inter_segment_id(AV1_COMMON *const cm, MACROBLOCKD *const xd, static int read_skip_mode(AV1_COMMON *cm, const MACROBLOCKD *xd, int segment_id, aom_reader *r) { - if (!cm->skip_mode_flag) return 0; + if (!cm->current_frame.skip_mode_info.skip_mode_flag) return 0; if (segfeature_active(&cm->seg, segment_id, SEG_LVL_SKIP)) { return 0; @@ -612,9 +611,8 @@ void av1_read_tx_type(const AV1_COMMON *const cm, MACROBLOCKD *xd, int blk_row, return; // No need to read transform type for lossless mode(qindex==0). - const int qindex = - cm->seg.enabled ? xd->qindex[mbmi->segment_id] : cm->base_qindex; - if (qindex <= 0) return; + const int qindex = xd->qindex[mbmi->segment_id]; + if (qindex == 0) return; const int inter_block = is_inter_block(mbmi); if (get_ext_tx_types(tx_size, inter_block, cm->reduced_tx_set_used) > 1) { @@ -701,7 +699,8 @@ static void read_intrabc_info(AV1_COMMON *const cm, MACROBLOCKD *const xd, mi_col, bsize, r); if (!valid_dv) { // Intra bc motion vectors are not valid - signal corrupt frame - aom_merge_corrupted_flag(&xd->corrupted, 1); + aom_internal_error(xd->error_info, AOM_CODEC_CORRUPT_FRAME, + "Invalid intrabc dv"); } } } @@ -711,15 +710,17 @@ static void read_intrabc_info(AV1_COMMON *const cm, MACROBLOCKD *const xd, static void read_delta_q_params(AV1_COMMON *const cm, MACROBLOCKD *const xd, const int mi_row, const int mi_col, aom_reader *r) { - if (cm->delta_q_present_flag) { + DeltaQInfo *const delta_q_info = &cm->delta_q_info; + + if (delta_q_info->delta_q_present_flag) { MB_MODE_INFO *const mbmi = xd->mi[0]; - xd->current_qindex += - read_delta_qindex(cm, xd, r, mbmi, mi_col, mi_row) * cm->delta_q_res; + xd->current_qindex += read_delta_qindex(cm, xd, r, mbmi, mi_col, mi_row) * + delta_q_info->delta_q_res; /* Normative: Clamp to [1,MAXQ] to not interfere with lossless mode */ xd->current_qindex = clamp(xd->current_qindex, 1, MAXQ); FRAME_CONTEXT *const ec_ctx = xd->tile_ctx; - if (cm->delta_lf_present_flag) { - if (cm->delta_lf_multi) { + if (delta_q_info->delta_lf_present_flag) { + if (delta_q_info->delta_lf_multi) { const int frame_lf_count = av1_num_planes(cm) > 1 ? FRAME_LF_COUNT : FRAME_LF_COUNT - 2; for (int lf_id = 0; lf_id < frame_lf_count; ++lf_id) { @@ -727,7 +728,7 @@ static void read_delta_q_params(AV1_COMMON *const cm, MACROBLOCKD *const xd, xd->delta_lf[lf_id] + read_delta_lflevel(cm, r, ec_ctx->delta_lf_multi_cdf[lf_id], mbmi, mi_col, mi_row) * - cm->delta_lf_res; + delta_q_info->delta_lf_res; mbmi->delta_lf[lf_id] = xd->delta_lf[lf_id] = clamp(tmp_lvl, -MAX_LOOP_FILTER, MAX_LOOP_FILTER); } @@ -735,7 +736,7 @@ static void read_delta_q_params(AV1_COMMON *const cm, MACROBLOCKD *const xd, const int tmp_lvl = xd->delta_lf_from_base + read_delta_lflevel(cm, r, ec_ctx->delta_lf_cdf, mbmi, mi_col, mi_row) * - cm->delta_lf_res; + delta_q_info->delta_lf_res; mbmi->delta_lf_from_base = xd->delta_lf_from_base = clamp(tmp_lvl, -MAX_LOOP_FILTER, MAX_LOOP_FILTER); } @@ -882,14 +883,14 @@ static REFERENCE_MODE read_block_reference_mode(AV1_COMMON *cm, const MACROBLOCKD *xd, aom_reader *r) { if (!is_comp_ref_allowed(xd->mi[0]->sb_type)) return SINGLE_REFERENCE; - if (cm->reference_mode == REFERENCE_MODE_SELECT) { + if (cm->current_frame.reference_mode == REFERENCE_MODE_SELECT) { const int ctx = av1_get_reference_mode_context(xd); const REFERENCE_MODE mode = (REFERENCE_MODE)aom_read_symbol( r, xd->tile_ctx->comp_inter_cdf[ctx], 2, ACCT_STR); return mode; // SINGLE_REFERENCE or COMPOUND_REFERENCE } else { - assert(cm->reference_mode == SINGLE_REFERENCE); - return cm->reference_mode; + assert(cm->current_frame.reference_mode == SINGLE_REFERENCE); + return cm->current_frame.reference_mode; } } @@ -907,8 +908,8 @@ static COMP_REFERENCE_TYPE read_comp_reference_type(const MACROBLOCKD *xd, static void set_ref_frames_for_skip_mode(AV1_COMMON *const cm, MV_REFERENCE_FRAME ref_frame[2]) { - ref_frame[0] = LAST_FRAME + cm->ref_frame_idx_0; - ref_frame[1] = LAST_FRAME + cm->ref_frame_idx_1; + ref_frame[0] = LAST_FRAME + cm->current_frame.skip_mode_info.ref_frame_idx_0; + ref_frame[1] = LAST_FRAME + cm->current_frame.skip_mode_info.ref_frame_idx_1; } // Read the referncence frame @@ -1231,16 +1232,16 @@ static void dec_dump_logs(AV1_COMMON *cm, MB_MODE_INFO *const mbmi, int mi_row, } #define FRAME_TO_CHECK 11 - if (cm->current_video_frame == FRAME_TO_CHECK && cm->show_frame == 1) { + if (cm->current_frame.frame_number == FRAME_TO_CHECK && cm->show_frame == 1) { printf( "=== DECODER ===: " "Frame=%d, (mi_row,mi_col)=(%d,%d), skip_mode=%d, mode=%d, bsize=%d, " "show_frame=%d, mv[0]=(%d,%d), mv[1]=(%d,%d), ref[0]=%d, " "ref[1]=%d, motion_mode=%d, mode_ctx=%d, " "newmv_ctx=%d, zeromv_ctx=%d, refmv_ctx=%d, tx_size=%d\n", - cm->current_video_frame, mi_row, mi_col, mbmi->skip_mode, mbmi->mode, - mbmi->sb_type, cm->show_frame, mv[0].as_mv.row, mv[0].as_mv.col, - mv[1].as_mv.row, mv[1].as_mv.col, mbmi->ref_frame[0], + cm->current_frame.frame_number, mi_row, mi_col, mbmi->skip_mode, + mbmi->mode, mbmi->sb_type, cm->show_frame, mv[0].as_mv.row, + mv[0].as_mv.col, mv[1].as_mv.row, mv[1].as_mv.col, mbmi->ref_frame[0], mbmi->ref_frame[1], mbmi->motion_mode, mode_ctx, newmv_ctx, zeromv_ctx, refmv_ctx, mbmi->tx_size); } @@ -1351,16 +1352,12 @@ static void read_inter_block_mode_info(AV1Decoder *const pbi, } } - if (mbmi->skip_mode) { - assert(mbmi->mode == NEAREST_NEARESTMV); - mbmi->mv[0].as_int = nearestmv[0].as_int; - mbmi->mv[1].as_int = nearestmv[1].as_int; - } else { - int mv_corrupted_flag = - !assign_mv(cm, xd, mbmi->mode, mbmi->ref_frame, mbmi->mv, ref_mv, - nearestmv, nearmv, mi_row, mi_col, is_compound, allow_hp, r); - aom_merge_corrupted_flag(&xd->corrupted, mv_corrupted_flag); - } + if (mbmi->skip_mode) assert(mbmi->mode == NEAREST_NEARESTMV); + + int mv_corrupted_flag = + !assign_mv(cm, xd, mbmi->mode, mbmi->ref_frame, mbmi->mv, ref_mv, + nearestmv, nearmv, mi_row, mi_col, is_compound, allow_hp, r); + aom_merge_corrupted_flag(&xd->corrupted, mv_corrupted_flag); mbmi->use_wedge_interintra = 0; if (cm->seq_params.enable_interintra_compound && !mbmi->skip_mode && @@ -1391,9 +1388,7 @@ static void read_inter_block_mode_info(AV1Decoder *const pbi, for (int ref = 0; ref < 1 + has_second_ref(mbmi); ++ref) { const MV_REFERENCE_FRAME frame = mbmi->ref_frame[ref]; - RefBuffer *ref_buf = &cm->frame_refs[frame - LAST_FRAME]; - - xd->block_refs[ref] = ref_buf; + xd->block_ref_scale_factors[ref] = get_ref_scale_factors_const(cm, frame); } mbmi->motion_mode = SIMPLE_TRANSLATION; @@ -1422,7 +1417,7 @@ static void read_inter_block_mode_info(AV1Decoder *const pbi, } if (mbmi->comp_group_idx == 0) { - if (cm->seq_params.enable_jnt_comp) { + if (cm->seq_params.order_hint_info.enable_dist_wtd_comp) { const int comp_index_ctx = get_comp_index_context(cm, xd); mbmi->compound_idx = aom_read_symbol( r, ec_ctx->compound_index_cdf[comp_index_ctx], 2, ACCT_STR); @@ -1431,7 +1426,7 @@ static void read_inter_block_mode_info(AV1Decoder *const pbi, mbmi->compound_idx = 1; } } else { - assert(cm->reference_mode != SINGLE_REFERENCE && + assert(cm->current_frame.reference_mode != SINGLE_REFERENCE && is_inter_compound_mode(mbmi->mode) && mbmi->motion_mode == SIMPLE_TRANSLATION); assert(masked_compound_used); @@ -1505,7 +1500,8 @@ static void read_inter_frame_mode_info(AV1Decoder *const pbi, else mbmi->skip = read_skip(cm, xd, mbmi->segment_id, r); - mbmi->segment_id = read_inter_segment_id(cm, xd, mi_row, mi_col, 0, r); + if (!cm->seg.segid_preskip) + mbmi->segment_id = read_inter_segment_id(cm, xd, mi_row, mi_col, 0, r); read_cdef(cm, r, xd, mi_col, mi_row); diff --git a/third_party/aom/av1/decoder/decoder.c b/third_party/aom/av1/decoder/decoder.c index a5f4fd67fa..bff4b7a697 100644 --- a/third_party/aom/av1/decoder/decoder.c +++ b/third_party/aom/av1/decoder/decoder.c @@ -17,6 +17,7 @@ #include "config/aom_dsp_rtcd.h" #include "config/aom_scale_rtcd.h" +#include "aom_dsp/aom_dsp_common.h" #include "aom_mem/aom_mem.h" #include "aom_ports/system_state.h" #include "aom_ports/aom_once.h" @@ -71,12 +72,11 @@ static void dec_free_mi(AV1_COMMON *cm) { AV1Decoder *av1_decoder_create(BufferPool *const pool) { AV1Decoder *volatile const pbi = aom_memalign(32, sizeof(*pbi)); - AV1_COMMON *volatile const cm = pbi ? &pbi->common : NULL; - - if (!cm) return NULL; - + if (!pbi) return NULL; av1_zero(*pbi); + AV1_COMMON *volatile const cm = &pbi->common; + // The jmp_buf is valid only for the duration of the function that calls // setjmp(). Therefore, this function must reset the 'setjmp' field to 0 // before it returns. @@ -90,25 +90,26 @@ AV1Decoder *av1_decoder_create(BufferPool *const pool) { CHECK_MEM_ERROR(cm, cm->fc, (FRAME_CONTEXT *)aom_memalign(32, sizeof(*cm->fc))); - CHECK_MEM_ERROR(cm, cm->frame_contexts, - (FRAME_CONTEXT *)aom_memalign( - 32, FRAME_CONTEXTS * sizeof(*cm->frame_contexts))); + CHECK_MEM_ERROR( + cm, cm->default_frame_context, + (FRAME_CONTEXT *)aom_memalign(32, sizeof(*cm->default_frame_context))); memset(cm->fc, 0, sizeof(*cm->fc)); - memset(cm->frame_contexts, 0, FRAME_CONTEXTS * sizeof(*cm->frame_contexts)); + memset(cm->default_frame_context, 0, sizeof(*cm->default_frame_context)); pbi->need_resync = 1; aom_once(initialize_dec); // Initialize the references to not point to any frame buffers. - memset(&cm->ref_frame_map, -1, sizeof(cm->ref_frame_map)); - memset(&cm->next_ref_frame_map, -1, sizeof(cm->next_ref_frame_map)); + for (int i = 0; i < REF_FRAMES; i++) { + cm->ref_frame_map[i] = NULL; + cm->next_ref_frame_map[i] = NULL; + } - cm->current_video_frame = 0; + cm->current_frame.frame_number = 0; pbi->decoding_first_frame = 1; pbi->common.buffer_pool = pool; cm->seq_params.bit_depth = AOM_BITS_8; - cm->dequant_bit_depth = AOM_BITS_8; cm->alloc_mi = av1_dec_alloc_mi; cm->free_mi = dec_free_mi; @@ -126,6 +127,7 @@ AV1Decoder *av1_decoder_create(BufferPool *const pool) { cm->error.setjmp = 0; aom_get_worker_interface()->init(&pbi->lf_worker); + pbi->lf_worker.thread_name = "aom lf worker"; return pbi; } @@ -157,8 +159,7 @@ void av1_decoder_remove(AV1Decoder *pbi) { if (!pbi) return; // Free the tile list output buffer. - if (pbi->tile_list_output != NULL) aom_free(pbi->tile_list_output); - pbi->tile_list_output = NULL; + aom_free_frame_buffer(&pbi->tile_list_outbuf); aom_get_worker_interface()->end(&pbi->lf_worker); aom_free(pbi->lf_worker.data1); @@ -318,14 +319,36 @@ aom_codec_err_t av1_copy_new_frame_dec(AV1_COMMON *cm, return cm->error.error_code; } -/* If any buffer updating is signaled it should be done here. - Consumes a reference to cm->new_fb_idx. -*/ +static void release_frame_buffers(AV1Decoder *pbi) { + AV1_COMMON *const cm = &pbi->common; + BufferPool *const pool = cm->buffer_pool; + + cm->cur_frame->buf.corrupted = 1; + lock_buffer_pool(pool); + // Release all the reference buffers in cm->next_ref_frame_map if the worker + // thread is holding them. + if (pbi->hold_ref_buf) { + for (int ref_index = 0; ref_index < REF_FRAMES; ++ref_index) { + decrease_ref_count(cm->next_ref_frame_map[ref_index], pool); + cm->next_ref_frame_map[ref_index] = NULL; + } + pbi->hold_ref_buf = 0; + } + // Release current frame. + decrease_ref_count(cm->cur_frame, pool); + unlock_buffer_pool(pool); + cm->cur_frame = NULL; +} + +// If any buffer updating is signaled it should be done here. +// Consumes a reference to cm->cur_frame. +// +// This functions returns void. It reports failure by setting +// cm->error.error_code. static void swap_frame_buffers(AV1Decoder *pbi, int frame_decoded) { int ref_index = 0, mask; AV1_COMMON *const cm = &pbi->common; BufferPool *const pool = cm->buffer_pool; - RefCntBuffer *const frame_bufs = cm->buffer_pool->frame_bufs; if (frame_decoded) { lock_buffer_pool(pool); @@ -333,73 +356,78 @@ static void swap_frame_buffers(AV1Decoder *pbi, int frame_decoded) { // In ext-tile decoding, the camera frame header is only decoded once. So, // we don't release the references here. if (!pbi->camera_frame_header_ready) { - for (mask = pbi->refresh_frame_flags; mask; mask >>= 1) { - const int old_idx = cm->ref_frame_map[ref_index]; - // Current thread releases the holding of reference frame. - decrease_ref_count(old_idx, frame_bufs, pool); + // If we are not holding reference buffers in cm->next_ref_frame_map, + // assert that the following two for loops are no-ops. + assert(IMPLIES(!pbi->hold_ref_buf, + cm->current_frame.refresh_frame_flags == 0)); + assert(IMPLIES(!pbi->hold_ref_buf, + cm->show_existing_frame && !pbi->reset_decoder_state)); - // Release the reference frame holding in the reference map for the - // decoding of the next frame. - if (mask & 1) decrease_ref_count(old_idx, frame_bufs, pool); + // The following two for loops need to release the reference stored in + // cm->ref_frame_map[ref_index] before transferring the reference stored + // in cm->next_ref_frame_map[ref_index] to cm->ref_frame_map[ref_index]. + for (mask = cm->current_frame.refresh_frame_flags; mask; mask >>= 1) { + decrease_ref_count(cm->ref_frame_map[ref_index], pool); cm->ref_frame_map[ref_index] = cm->next_ref_frame_map[ref_index]; + cm->next_ref_frame_map[ref_index] = NULL; ++ref_index; } - // Current thread releases the holding of reference frame. const int check_on_show_existing_frame = - !cm->show_existing_frame || cm->reset_decoder_state; + !cm->show_existing_frame || pbi->reset_decoder_state; for (; ref_index < REF_FRAMES && check_on_show_existing_frame; ++ref_index) { - const int old_idx = cm->ref_frame_map[ref_index]; - decrease_ref_count(old_idx, frame_bufs, pool); + decrease_ref_count(cm->ref_frame_map[ref_index], pool); cm->ref_frame_map[ref_index] = cm->next_ref_frame_map[ref_index]; + cm->next_ref_frame_map[ref_index] = NULL; } } - YV12_BUFFER_CONFIG *cur_frame = get_frame_new_buffer(cm); - if (cm->show_existing_frame || cm->show_frame) { if (pbi->output_all_layers) { // Append this frame to the output queue if (pbi->num_output_frames >= MAX_NUM_SPATIAL_LAYERS) { // We can't store the new frame anywhere, so drop it and return an // error - decrease_ref_count(cm->new_fb_idx, frame_bufs, pool); + cm->cur_frame->buf.corrupted = 1; + decrease_ref_count(cm->cur_frame, pool); cm->error.error_code = AOM_CODEC_UNSUP_BITSTREAM; } else { - pbi->output_frames[pbi->num_output_frames] = cur_frame; - pbi->output_frame_index[pbi->num_output_frames] = cm->new_fb_idx; + pbi->output_frames[pbi->num_output_frames] = cm->cur_frame; pbi->num_output_frames++; } } else { // Replace any existing output frame assert(pbi->num_output_frames == 0 || pbi->num_output_frames == 1); if (pbi->num_output_frames > 0) { - decrease_ref_count((int)pbi->output_frame_index[0], frame_bufs, pool); + decrease_ref_count(pbi->output_frames[0], pool); } - pbi->output_frames[0] = cur_frame; - pbi->output_frame_index[0] = cm->new_fb_idx; + pbi->output_frames[0] = cm->cur_frame; pbi->num_output_frames = 1; } } else { - decrease_ref_count(cm->new_fb_idx, frame_bufs, pool); + decrease_ref_count(cm->cur_frame, pool); } unlock_buffer_pool(pool); } else { + // The code here assumes we are not holding reference buffers in + // cm->next_ref_frame_map. If this assertion fails, we are leaking the + // frame buffer references in cm->next_ref_frame_map. + assert(IMPLIES(!pbi->camera_frame_header_ready, !pbi->hold_ref_buf)); // Nothing was decoded, so just drop this frame buffer lock_buffer_pool(pool); - decrease_ref_count(cm->new_fb_idx, frame_bufs, pool); + decrease_ref_count(cm->cur_frame, pool); unlock_buffer_pool(pool); } + cm->cur_frame = NULL; if (!pbi->camera_frame_header_ready) { pbi->hold_ref_buf = 0; // Invalidate these references until the next frame starts. for (ref_index = 0; ref_index < INTER_REFS_PER_FRAME; ref_index++) { - cm->frame_refs[ref_index].idx = INVALID_IDX; - cm->frame_refs[ref_index].buf = NULL; + cm->remapped_ref_idx[ref_index] = INVALID_IDX; } } } @@ -407,10 +435,9 @@ static void swap_frame_buffers(AV1Decoder *pbi, int frame_decoded) { int av1_receive_compressed_data(AV1Decoder *pbi, size_t size, const uint8_t **psource) { AV1_COMMON *volatile const cm = &pbi->common; - BufferPool *volatile const pool = cm->buffer_pool; - RefCntBuffer *volatile const frame_bufs = cm->buffer_pool->frame_bufs; const uint8_t *source = *psource; cm->error.error_code = AOM_CODEC_OK; + cm->error.has_detail = 0; if (size == 0) { // This is used to signal that we are missing frames. @@ -421,29 +448,17 @@ int av1_receive_compressed_data(AV1Decoder *pbi, size_t size, // TODO(jkoleszar): Error concealment is undefined and non-normative // at this point, but if it becomes so, [0] may not always be the correct // thing to do here. - if (cm->frame_refs[0].idx > 0) { - assert(cm->frame_refs[0].buf != NULL); - cm->frame_refs[0].buf->corrupted = 1; - } + RefCntBuffer *ref_buf = get_ref_frame_buf(cm, LAST_FRAME); + if (ref_buf != NULL) ref_buf->buf.corrupted = 1; } - // Find a free buffer for the new frame, releasing the reference previously - // held. - - // Find a free frame buffer. Return error if can not find any. - cm->new_fb_idx = get_free_fb(cm); - if (cm->new_fb_idx == INVALID_IDX) { + if (assign_cur_frame_new_fb(cm) == NULL) { cm->error.error_code = AOM_CODEC_MEM_ERROR; return 1; } - // Assign a MV array to the frame buffer. - cm->cur_frame = &pool->frame_bufs[cm->new_fb_idx]; - if (!pbi->camera_frame_header_ready) pbi->hold_ref_buf = 0; - pbi->cur_buf = &frame_bufs[cm->new_fb_idx]; - // The jmp_buf is valid only for the duration of the function that calls // setjmp(). Therefore, this function must reset the 'setjmp' field to 0 // before it returns. @@ -460,35 +475,7 @@ int av1_receive_compressed_data(AV1Decoder *pbi, size_t size, winterface->sync(&pbi->tile_workers[i]); } - lock_buffer_pool(pool); - // Release all the reference buffers if worker thread is holding them. - if (pbi->hold_ref_buf == 1) { - int ref_index = 0, mask; - for (mask = pbi->refresh_frame_flags; mask; mask >>= 1) { - const int old_idx = cm->ref_frame_map[ref_index]; - // Current thread releases the holding of reference frame. - decrease_ref_count(old_idx, frame_bufs, pool); - - // Release the reference frame holding in the reference map for the - // decoding of the next frame. - if (mask & 1) decrease_ref_count(old_idx, frame_bufs, pool); - ++ref_index; - } - - // Current thread releases the holding of reference frame. - const int check_on_show_existing_frame = - !cm->show_existing_frame || cm->reset_decoder_state; - for (; ref_index < REF_FRAMES && check_on_show_existing_frame; - ++ref_index) { - const int old_idx = cm->ref_frame_map[ref_index]; - decrease_ref_count(old_idx, frame_bufs, pool); - } - pbi->hold_ref_buf = 0; - } - // Release current frame. - decrease_ref_count(cm->new_fb_idx, frame_bufs, pool); - unlock_buffer_pool(pool); - + release_frame_buffers(pbi); aom_clear_system_state(); return -1; } @@ -498,10 +485,9 @@ int av1_receive_compressed_data(AV1Decoder *pbi, size_t size, int frame_decoded = aom_decode_frame_from_obus(pbi, source, source + size, psource); - if (cm->error.error_code != AOM_CODEC_OK) { - lock_buffer_pool(pool); - decrease_ref_count(cm->new_fb_idx, frame_bufs, pool); - unlock_buffer_pool(pool); + if (frame_decoded < 0) { + assert(cm->error.error_code != AOM_CODEC_OK); + release_frame_buffers(pbi); cm->error.setjmp = 0; return 1; } @@ -515,7 +501,7 @@ int av1_receive_compressed_data(AV1Decoder *pbi, size_t size, cm->txb_count = 0; #endif - // Note: At this point, this function holds a reference to cm->new_fb_idx + // Note: At this point, this function holds a reference to cm->cur_frame // in the buffer pool. This reference is consumed by swap_frame_buffers(). swap_frame_buffers(pbi, frame_decoded); @@ -531,8 +517,6 @@ int av1_receive_compressed_data(AV1Decoder *pbi, size_t size, aom_clear_system_state(); if (!cm->show_existing_frame) { - cm->last_show_frame = cm->show_frame; - if (cm->seg.enabled) { if (cm->prev_frame && (cm->mi_rows == cm->prev_frame->mi_rows) && (cm->mi_cols == cm->prev_frame->mi_cols)) { @@ -544,10 +528,6 @@ int av1_receive_compressed_data(AV1Decoder *pbi, size_t size, } // Update progress in frame parallel decode. - cm->last_width = cm->width; - cm->last_height = cm->height; - cm->last_tile_cols = cm->tile_cols; - cm->last_tile_rows = cm->tile_rows; cm->error.setjmp = 0; return 0; @@ -556,11 +536,9 @@ int av1_receive_compressed_data(AV1Decoder *pbi, size_t size, // Get the frame at a particular index in the output queue int av1_get_raw_frame(AV1Decoder *pbi, size_t index, YV12_BUFFER_CONFIG **sd, aom_film_grain_t **grain_params) { - RefCntBuffer *const frame_bufs = pbi->common.buffer_pool->frame_bufs; - if (index >= pbi->num_output_frames) return -1; - *sd = pbi->output_frames[index]; - *grain_params = &frame_bufs[pbi->output_frame_index[index]].film_grain_params; + *sd = &pbi->output_frames[index]->buf; + *grain_params = &pbi->output_frames[index]->film_grain_params; aom_clear_system_state(); return 0; } @@ -570,6 +548,6 @@ int av1_get_raw_frame(AV1Decoder *pbi, size_t index, YV12_BUFFER_CONFIG **sd, int av1_get_frame_to_show(AV1Decoder *pbi, YV12_BUFFER_CONFIG *frame) { if (pbi->num_output_frames == 0) return -1; - *frame = *pbi->output_frames[pbi->num_output_frames - 1]; + *frame = pbi->output_frames[pbi->num_output_frames - 1]->buf; return 0; } diff --git a/third_party/aom/av1/decoder/decoder.h b/third_party/aom/av1/decoder/decoder.h index 5ca939c245..467c8a2fc1 100644 --- a/third_party/aom/av1/decoder/decoder.h +++ b/third_party/aom/av1/decoder/decoder.h @@ -50,8 +50,6 @@ typedef void (*cfl_store_inter_block_visitor_fn_t)(AV1_COMMON *const cm, typedef struct ThreadData { aom_reader *bit_reader; DECLARE_ALIGNED(32, MACROBLOCKD, xd); - /* dqcoeff are shared by all the planes. So planes must be decoded serially */ - DECLARE_ALIGNED(32, tran_low_t, dqcoeff[MAX_TX_SQUARE]); CB_BUFFER cb_buffer_base; uint8_t *mc_buf[2]; int32_t mc_buf_size; @@ -97,9 +95,26 @@ typedef struct AV1DecRowMTInfo { int tile_cols_end; int start_tile; int end_tile; - int mi_rows_parse_done; - int mi_rows_decode_started; int mi_rows_to_decode; + + // Invariant: + // mi_rows_parse_done >= mi_rows_decode_started. + // mi_rows_parse_done and mi_rows_decode_started are both initialized to 0. + // mi_rows_parse_done is incremented freely. mi_rows_decode_started may only + // be incremented to catch up with mi_rows_parse_done but is not allowed to + // surpass mi_rows_parse_done. + // + // When mi_rows_decode_started reaches mi_rows_to_decode, there are no more + // decode jobs. + + // Indicates the progress of the bit-stream parsing of superblocks. + // Initialized to 0. Incremented by sb_mi_size when parse sb row is done. + int mi_rows_parse_done; + // Indicates the progress of the decoding of superblocks. + // Initialized to 0. Incremented by sb_mi_size when decode sb row is started. + int mi_rows_decode_started; + // Boolean: Initialized to 0 (false). Set to 1 (true) on error to abort + // decoding. int row_mt_exit; } AV1DecRowMTInfo; @@ -146,13 +161,6 @@ typedef struct AV1Decoder { DECLARE_ALIGNED(32, AV1_COMMON, common); - int refresh_frame_flags; - - // TODO(hkuang): Combine this with cur_buf in macroblockd as they are - // the same. - RefCntBuffer *cur_buf; // Current decoding frame buffer. - - AVxWorker *frame_worker_owner; // frame_worker that owns this pbi. AVxWorker lf_worker; AV1LfSync lf_row_sync; AV1LrSync lr_row_sync; @@ -178,8 +186,7 @@ typedef struct AV1Decoder { // Note: The saved buffers are released at the start of the next time the // application calls aom_codec_decode(). int output_all_layers; - YV12_BUFFER_CONFIG *output_frames[MAX_NUM_SPATIAL_LAYERS]; - size_t output_frame_index[MAX_NUM_SPATIAL_LAYERS]; // Buffer pool indices + RefCntBuffer *output_frames[MAX_NUM_SPATIAL_LAYERS]; size_t num_output_frames; // How many frames are queued up so far? // In order to properly support random-access decoding, we need @@ -191,7 +198,9 @@ typedef struct AV1Decoder { int max_threads; int inv_tile_order; int need_resync; // wait for key/intra-only frame. - int hold_ref_buf; // hold the reference buffer. + int hold_ref_buf; // Boolean: whether we are holding reference buffers in + // common.next_ref_frame_map. + int reset_decoder_state; int tile_size_bytes; int tile_col_size_bytes; @@ -225,9 +234,7 @@ typedef struct AV1Decoder { unsigned int ext_tile_debug; // for ext-tile software debug & testing unsigned int row_mt; EXTERNAL_REFERENCES ext_refs; - size_t tile_list_size; - uint8_t *tile_list_output; - size_t buffer_sz; + YV12_BUFFER_CONFIG tile_list_outbuf; CB_BUFFER *cb_buffer_base; int cb_buffer_alloc_size; @@ -245,7 +252,7 @@ typedef struct AV1Decoder { // Returns 0 on success. Sets pbi->common.error.error_code to a nonzero error // code and returns a nonzero value on failure. int av1_receive_compressed_data(struct AV1Decoder *pbi, size_t size, - const uint8_t **dest); + const uint8_t **psource); // Get the frame at a particular index in the output queue int av1_get_raw_frame(AV1Decoder *pbi, size_t index, YV12_BUFFER_CONFIG **sd, @@ -266,23 +273,28 @@ aom_codec_err_t av1_copy_new_frame_dec(AV1_COMMON *cm, struct AV1Decoder *av1_decoder_create(BufferPool *const pool); void av1_decoder_remove(struct AV1Decoder *pbi); -void av1_dealloc_dec_jobs(struct AV1DecTileMTData *tile_jobs_sync); +void av1_dealloc_dec_jobs(struct AV1DecTileMTData *tile_mt_info); void av1_dec_row_mt_dealloc(AV1DecRowMTSync *dec_row_mt_sync); void av1_dec_free_cb_buf(AV1Decoder *pbi); -static INLINE void decrease_ref_count(int idx, RefCntBuffer *const frame_bufs, +static INLINE void decrease_ref_count(RefCntBuffer *const buf, BufferPool *const pool) { - if (idx >= 0) { - --frame_bufs[idx].ref_count; + if (buf != NULL) { + --buf->ref_count; + // Reference counts should never become negative. If this assertion fails, + // there is a bug in our reference count management. + assert(buf->ref_count >= 0); // A worker may only get a free framebuffer index when calling get_free_fb. - // But the private buffer is not set up until finish decoding header. - // So any error happens during decoding header, the frame_bufs will not - // have valid priv buffer. - if (frame_bufs[idx].ref_count == 0 && - frame_bufs[idx].raw_frame_buffer.priv) { - pool->release_fb_cb(pool->cb_priv, &frame_bufs[idx].raw_frame_buffer); + // But the raw frame buffer is not set up until we finish decoding header. + // So if any error happens during decoding header, frame_bufs[idx] will not + // have a valid raw frame buffer. + if (buf->ref_count == 0 && buf->raw_frame_buffer.data) { + pool->release_fb_cb(pool->cb_priv, &buf->raw_frame_buffer); + buf->raw_frame_buffer.data = NULL; + buf->raw_frame_buffer.size = 0; + buf->raw_frame_buffer.priv = NULL; } } } diff --git a/third_party/aom/av1/decoder/decodetxb.c b/third_party/aom/av1/decoder/decodetxb.c index f3ef2d55e4..223e32e990 100644 --- a/third_party/aom/av1/decoder/decodetxb.c +++ b/third_party/aom/av1/decoder/decodetxb.c @@ -136,6 +136,15 @@ uint8_t av1_read_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCKD *const xd, uint16_t *const max_scan_line = &(eob_data->max_scan_line); *max_scan_line = 0; *eob = 0; + +#if CONFIG_INSPECTION + if (plane == 0) { + const int txk_type_idx = + av1_get_txk_type_index(mbmi->sb_type, blk_row, blk_col); + mbmi->tx_skip[txk_type_idx] = all_zero; + } +#endif + if (all_zero) { *max_scan_line = 0; if (plane == 0) { @@ -146,9 +155,6 @@ uint8_t av1_read_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCKD *const xd, return 0; } - memset(levels_buf, 0, - sizeof(*levels_buf) * - ((width + TX_PAD_HOR) * (height + TX_PAD_VER) + TX_PAD_END)); if (plane == AOM_PLANE_Y) { // only y plane's tx_type is transmitted av1_read_tx_type(cm, xd, blk_row, blk_col, tx_size, r); @@ -214,23 +220,30 @@ uint8_t av1_read_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCKD *const xd, break; } - if (k_eob_offset_bits[eob_pt] > 0) { + const int eob_offset_bits = k_eob_offset_bits[eob_pt]; + if (eob_offset_bits > 0) { const int eob_ctx = eob_pt - 3; int bit = aom_read_symbol( r, ec_ctx->eob_extra_cdf[txs_ctx][plane_type][eob_ctx], 2, ACCT_STR); if (bit) { - eob_extra += (1 << (k_eob_offset_bits[eob_pt] - 1)); + eob_extra += (1 << (eob_offset_bits - 1)); } - for (int i = 1; i < k_eob_offset_bits[eob_pt]; i++) { + for (int i = 1; i < eob_offset_bits; i++) { bit = aom_read_bit(r, ACCT_STR); if (bit) { - eob_extra += (1 << (k_eob_offset_bits[eob_pt] - 1 - i)); + eob_extra += (1 << (eob_offset_bits - 1 - i)); } } } *eob = rec_eob_pos(eob_pt, eob_extra); + if (*eob > 1) { + memset(levels_buf, 0, + sizeof(*levels_buf) * + ((width + TX_PAD_HOR) * (height + TX_PAD_VER) + TX_PAD_END)); + } + { // Read the non-zero coefficient with scan index eob-1 // TODO(angiebird): Put this into a function @@ -242,12 +255,10 @@ uint8_t av1_read_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCKD *const xd, ec_ctx->coeff_base_eob_cdf[txs_ctx][plane_type][coeff_ctx]; int level = aom_read_symbol(r, cdf, nsymbs, ACCT_STR) + 1; if (level > NUM_BASE_LEVELS) { - const int br_ctx = get_br_ctx(levels, pos, bwl, tx_class); + const int br_ctx = get_br_ctx_eob(pos, bwl, tx_class); + cdf = ec_ctx->coeff_br_cdf[AOMMIN(txs_ctx, TX_32X32)][plane_type][br_ctx]; for (int idx = 0; idx < COEFF_BASE_RANGE; idx += BR_CDF_SIZE - 1) { - const int k = aom_read_symbol( - r, - ec_ctx->coeff_br_cdf[AOMMIN(txs_ctx, TX_32X32)][plane_type][br_ctx], - BR_CDF_SIZE, ACCT_STR); + const int k = aom_read_symbol(r, cdf, BR_CDF_SIZE, ACCT_STR); level += k; if (k < BR_CDF_SIZE - 1) break; } @@ -269,13 +280,6 @@ uint8_t av1_read_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCKD *const xd, } } - int16_t num_zero_coeffs = 0; - for (int c = 0; c < *eob; ++c) { - const int pos = scan[c]; - num_zero_coeffs = AOMMAX(num_zero_coeffs, pos); - } - memset(tcoeffs, 0, (num_zero_coeffs + 1) * sizeof(tcoeffs[0])); - for (int c = 0; c < *eob; ++c) { const int pos = scan[c]; uint8_t sign; diff --git a/third_party/aom/av1/decoder/dthread.h b/third_party/aom/av1/decoder/dthread.h index 1d264b07eb..c1b8719af8 100644 --- a/third_party/aom/av1/decoder/dthread.h +++ b/third_party/aom/av1/decoder/dthread.h @@ -41,40 +41,10 @@ typedef struct FrameWorkerData { void *user_priv; int worker_id; int received_frame; - - // scratch_buffer is used in frame parallel mode only. - // It is used to make a copy of the compressed data. - uint8_t *scratch_buffer; - size_t scratch_buffer_size; - -#if CONFIG_MULTITHREAD - pthread_mutex_t stats_mutex; - pthread_cond_t stats_cond; -#endif - int frame_context_ready; // Current frame's context is ready to read. int frame_decoded; // Finished decoding current frame. } FrameWorkerData; -void av1_frameworker_lock_stats(AVxWorker *const worker); -void av1_frameworker_unlock_stats(AVxWorker *const worker); -void av1_frameworker_signal_stats(AVxWorker *const worker); - -// Wait until ref_buf has been decoded to row in real pixel unit. -// Note: worker may already finish decoding ref_buf and release it in order to -// start decoding next frame. So need to check whether worker is still decoding -// ref_buf. -void av1_frameworker_wait(AVxWorker *const worker, RefCntBuffer *const ref_buf, - int row); - -// FrameWorker broadcasts its decoding progress so other workers that are -// waiting on it can resume decoding. -void av1_frameworker_broadcast(RefCntBuffer *const buf, int row); - -// Copy necessary decoding context from src worker to dst worker. -void av1_frameworker_copy_context(AVxWorker *const dst_worker, - AVxWorker *const src_worker); - #ifdef __cplusplus } // extern "C" #endif diff --git a/third_party/aom/av1/decoder/inspection.c b/third_party/aom/av1/decoder/inspection.c index e6c89298a4..a027b64e63 100644 --- a/third_party/aom/av1/decoder/inspection.c +++ b/third_party/aom/av1/decoder/inspection.c @@ -33,15 +33,18 @@ void ifd_clear(insp_frame_data *fd) { /* TODO(negge) This function may be called by more than one thread when using a multi-threaded decoder and this may cause a data race. */ -int ifd_inspect(insp_frame_data *fd, void *decoder) { +int ifd_inspect(insp_frame_data *fd, void *decoder, int skip_not_transform) { struct AV1Decoder *pbi = (struct AV1Decoder *)decoder; AV1_COMMON *const cm = &pbi->common; + if (fd->mi_rows != cm->mi_rows || fd->mi_cols != cm->mi_cols) { ifd_clear(fd); ifd_init_mi_rc(fd, cm->mi_rows, cm->mi_cols); } + fd->show_existing_frame = cm->show_existing_frame; + fd->frame_number = cm->current_frame.frame_number; fd->show_frame = cm->show_frame; - fd->frame_type = cm->frame_type; + fd->frame_type = cm->current_frame.frame_type; fd->base_qindex = cm->base_qindex; // Set width and height of the first tile until generic support can be added TileInfo tile_info; @@ -49,8 +52,8 @@ int ifd_inspect(insp_frame_data *fd, void *decoder) { av1_tile_set_col(&tile_info, cm, 0); fd->tile_mi_cols = tile_info.mi_col_end - tile_info.mi_col_start; fd->tile_mi_rows = tile_info.mi_row_end - tile_info.mi_row_start; - fd->delta_q_present_flag = cm->delta_q_present_flag; - fd->delta_q_res = cm->delta_q_res; + fd->delta_q_present_flag = cm->delta_q_info.delta_q_present_flag; + fd->delta_q_res = cm->delta_q_info.delta_q_res; #if CONFIG_ACCOUNTING fd->accounting = &pbi->accounting; #endif @@ -85,6 +88,10 @@ int ifd_inspect(insp_frame_data *fd, void *decoder) { } else { mi->uv_mode = UV_MODE_INVALID; } + + mi->motion_mode = mbmi->motion_mode; + mi->compound_type = mbmi->interinter_comp.type; + // Block Size mi->sb_type = mbmi->sb_type; // Skip Flag @@ -92,15 +99,31 @@ int ifd_inspect(insp_frame_data *fd, void *decoder) { mi->filter[0] = av1_extract_interp_filter(mbmi->interp_filters, 0); mi->filter[1] = av1_extract_interp_filter(mbmi->interp_filters, 1); mi->dual_filter_type = mi->filter[0] * 3 + mi->filter[1]; + // Transform // TODO(anyone): extract tx type info from mbmi->txk_type[]. - mi->tx_type = DCT_DCT; - mi->tx_size = mbmi->tx_size; - mi->cdef_level = - cm->cdef_strengths[mbmi->cdef_strength] / CDEF_SEC_STRENGTHS; - mi->cdef_strength = - cm->cdef_strengths[mbmi->cdef_strength] % CDEF_SEC_STRENGTHS; + const BLOCK_SIZE bsize = mbmi->sb_type; + const int c = i % mi_size_wide[bsize]; + const int r = j % mi_size_high[bsize]; + if (is_inter_block(mbmi) || is_intrabc_block(mbmi)) + mi->tx_size = mbmi->inter_tx_size[av1_get_txb_size_index(bsize, r, c)]; + else + mi->tx_size = mbmi->tx_size; + + if (skip_not_transform && mi->skip) mi->tx_size = -1; + + mi->tx_type = + (mi->skip ? 0 : mbmi->txk_type[av1_get_txk_type_index(bsize, r, c)]); + if (skip_not_transform && + (mi->skip || mbmi->tx_skip[av1_get_txk_type_index(bsize, r, c)])) + mi->tx_type = -1; + + mi->cdef_level = cm->cdef_info.cdef_strengths[mbmi->cdef_strength] / + CDEF_SEC_STRENGTHS; + mi->cdef_strength = cm->cdef_info.cdef_strengths[mbmi->cdef_strength] % + CDEF_SEC_STRENGTHS; + mi->cdef_strength += mi->cdef_strength == 3; if (mbmi->uv_mode == UV_CFL_PRED) { mi->cfl_alpha_idx = mbmi->cfl_alpha_idx; diff --git a/third_party/aom/av1/decoder/inspection.h b/third_party/aom/av1/decoder/inspection.h index 7214a9beda..ddea4a1543 100644 --- a/third_party/aom/av1/decoder/inspection.h +++ b/third_party/aom/av1/decoder/inspection.h @@ -20,7 +20,7 @@ extern "C" { #include "av1/decoder/accounting.h" #endif -#ifndef AOM_AOMDX_H_ +#ifndef AOM_AOM_AOMDX_H_ typedef void (*aom_inspect_cb)(void *decoder, void *data); #endif @@ -50,6 +50,8 @@ struct insp_mi_data { int16_t cfl_alpha_idx; int16_t cfl_alpha_sign; int16_t current_qindex; + int16_t compound_type; + int16_t motion_mode; }; typedef struct insp_frame_data insp_frame_data; @@ -59,6 +61,7 @@ struct insp_frame_data { Accounting *accounting; #endif insp_mi_data *mi_grid; + int16_t frame_number; int show_frame; int frame_type; int base_qindex; @@ -72,11 +75,12 @@ struct insp_frame_data { // TODO(negge): add per frame CDEF data int delta_q_present_flag; int delta_q_res; + int show_existing_frame; }; void ifd_init(insp_frame_data *fd, int frame_width, int frame_height); void ifd_clear(insp_frame_data *fd); -int ifd_inspect(insp_frame_data *fd, void *decoder); +int ifd_inspect(insp_frame_data *fd, void *decoder, int skip_not_transform); #ifdef __cplusplus } // extern "C" diff --git a/third_party/aom/av1/decoder/obu.c b/third_party/aom/av1/decoder/obu.c index 44ecf818e7..58f7d2ca8e 100644 --- a/third_party/aom/av1/decoder/obu.c +++ b/third_party/aom/av1/decoder/obu.c @@ -12,6 +12,7 @@ #include #include "config/aom_config.h" +#include "config/aom_scale_rtcd.h" #include "aom/aom_codec.h" #include "aom_dsp/bitreader_buffer.h" @@ -25,7 +26,7 @@ #include "av1/decoder/obu.h" // Picture prediction structures (0-12 are predefined) in scalability metadata. -typedef enum { +enum { SCALABILITY_L1T2 = 0, SCALABILITY_L1T3 = 1, SCALABILITY_L2T1 = 2, @@ -41,7 +42,7 @@ typedef enum { SCALABILITY_S2T2h = 12, SCALABILITY_S2T3h = 13, SCALABILITY_SS = 14 -} SCALABILITY_STRUCTURES; +} UENUM1BYTE(SCALABILITY_STRUCTURES); aom_codec_err_t aom_get_num_layers_from_operating_point_idc( int operating_point_idc, unsigned int *number_spatial_layers, @@ -292,6 +293,8 @@ static uint32_t read_frame_header_obu(AV1Decoder *pbi, trailing_bits_present); } +// On success, returns the tile group header size. On failure, calls +// aom_internal_error() and returns -1. static int32_t read_tile_group_header(AV1Decoder *pbi, struct aom_read_bit_buffer *rb, int *start_tile, int *end_tile, @@ -303,26 +306,33 @@ static int32_t read_tile_group_header(AV1Decoder *pbi, if (!pbi->common.large_scale_tile && num_tiles > 1) { tile_start_and_end_present_flag = aom_rb_read_bit(rb); + if (tile_start_implicit && tile_start_and_end_present_flag) { + aom_internal_error( + &cm->error, AOM_CODEC_UNSUP_BITSTREAM, + "For OBU_FRAME type obu tile_start_and_end_present_flag must be 0"); + return -1; + } } if (pbi->common.large_scale_tile || num_tiles == 1 || !tile_start_and_end_present_flag) { *start_tile = 0; *end_tile = num_tiles - 1; - return ((rb->bit_offset - saved_bit_offset + 7) >> 3); + } else { + int tile_bits = cm->log2_tile_rows + cm->log2_tile_cols; + *start_tile = aom_rb_read_literal(rb, tile_bits); + *end_tile = aom_rb_read_literal(rb, tile_bits); } - if (tile_start_implicit && tile_start_and_end_present_flag) { - aom_internal_error( - &cm->error, AOM_CODEC_UNSUP_BITSTREAM, - "For OBU_FRAME type obu tile_start_and_end_present_flag must be 0"); + if (*start_tile > *end_tile) { + aom_internal_error(&cm->error, AOM_CODEC_CORRUPT_FRAME, + "tg_end must be greater than or equal to tg_start"); return -1; } - *start_tile = - aom_rb_read_literal(rb, cm->log2_tile_rows + cm->log2_tile_cols); - *end_tile = aom_rb_read_literal(rb, cm->log2_tile_rows + cm->log2_tile_cols); return ((rb->bit_offset - saved_bit_offset + 7) >> 3); } +// On success, returns the tile group OBU size. On failure, sets +// pbi->common.error.error_code and returns 0. static uint32_t read_one_tile_group_obu( AV1Decoder *pbi, struct aom_read_bit_buffer *rb, int is_first_tg, const uint8_t *data, const uint8_t *data_end, const uint8_t **p_data_end, @@ -337,7 +347,6 @@ static uint32_t read_one_tile_group_obu( header_size = read_tile_group_header(pbi, rb, &start_tile, &end_tile, tile_start_implicit); if (header_size == -1 || byte_alignment(cm, rb)) return 0; - if (start_tile > end_tile) return header_size; data += header_size; av1_decode_tg_tiles_and_wrapup(pbi, data, data_end, p_data_end, start_tile, end_tile, is_first_tg); @@ -350,87 +359,121 @@ static uint32_t read_one_tile_group_obu( } static void alloc_tile_list_buffer(AV1Decoder *pbi) { - // TODO(yunqing): for now, copy each tile's decoded YUV data directly to the - // output buffer. This needs to be modified according to the application - // requirement. + // The resolution of the output frame is read out from the bitstream. The data + // are stored in the order of Y plane, U plane and V plane. As an example, for + // image format 4:2:0, the output frame of U plane and V plane is 1/4 of the + // output frame. AV1_COMMON *const cm = &pbi->common; - const int tile_width_in_pixels = cm->tile_width * MI_SIZE; - const int tile_height_in_pixels = cm->tile_height * MI_SIZE; - const int ssy = cm->seq_params.subsampling_y; - const int ssx = cm->seq_params.subsampling_x; - const int num_planes = av1_num_planes(cm); - const size_t yplane_tile_size = tile_height_in_pixels * tile_width_in_pixels; - const size_t uvplane_tile_size = - (num_planes > 1) - ? (tile_height_in_pixels >> ssy) * (tile_width_in_pixels >> ssx) - : 0; - const size_t tile_size = (cm->seq_params.use_highbitdepth ? 2 : 1) * - (yplane_tile_size + 2 * uvplane_tile_size); - pbi->tile_list_size = tile_size * (pbi->tile_count_minus_1 + 1); + int tile_width, tile_height; + av1_get_uniform_tile_size(cm, &tile_width, &tile_height); + const int tile_width_in_pixels = tile_width * MI_SIZE; + const int tile_height_in_pixels = tile_height * MI_SIZE; + const int output_frame_width = + (pbi->output_frame_width_in_tiles_minus_1 + 1) * tile_width_in_pixels; + const int output_frame_height = + (pbi->output_frame_height_in_tiles_minus_1 + 1) * tile_height_in_pixels; + // The output frame is used to store the decoded tile list. The decoded tile + // list has to fit into 1 output frame. + assert((pbi->tile_count_minus_1 + 1) <= + (pbi->output_frame_width_in_tiles_minus_1 + 1) * + (pbi->output_frame_height_in_tiles_minus_1 + 1)); - if (pbi->tile_list_size > pbi->buffer_sz) { - if (pbi->tile_list_output != NULL) aom_free(pbi->tile_list_output); - pbi->tile_list_output = NULL; + // Allocate the tile list output buffer. + // Note: if cm->seq_params.use_highbitdepth is 1 and cm->seq_params.bit_depth + // is 8, we could allocate less memory, namely, 8 bits/pixel. + if (aom_alloc_frame_buffer(&pbi->tile_list_outbuf, output_frame_width, + output_frame_height, cm->seq_params.subsampling_x, + cm->seq_params.subsampling_y, + (cm->seq_params.use_highbitdepth && + (cm->seq_params.bit_depth > AOM_BITS_8)), + 0, cm->byte_alignment)) + aom_internal_error(&cm->error, AOM_CODEC_MEM_ERROR, + "Failed to allocate the tile list output buffer"); +} - pbi->tile_list_output = (uint8_t *)aom_memalign(32, pbi->tile_list_size); - if (pbi->tile_list_output == NULL) - aom_internal_error(&cm->error, AOM_CODEC_MEM_ERROR, - "Failed to allocate the tile list output buffer"); - pbi->buffer_sz = pbi->tile_list_size; +static void yv12_tile_copy(const YV12_BUFFER_CONFIG *src, int hstart1, + int hend1, int vstart1, int vend1, + YV12_BUFFER_CONFIG *dst, int hstart2, int vstart2, + int plane) { + const int src_stride = (plane > 0) ? src->strides[1] : src->strides[0]; + const int dst_stride = (plane > 0) ? dst->strides[1] : dst->strides[0]; + int row, col; + + assert(src->flags & YV12_FLAG_HIGHBITDEPTH); + assert(!(dst->flags & YV12_FLAG_HIGHBITDEPTH)); + + const uint16_t *src16 = + CONVERT_TO_SHORTPTR(src->buffers[plane] + vstart1 * src_stride + hstart1); + uint8_t *dst8 = dst->buffers[plane] + vstart2 * dst_stride + hstart2; + + for (row = vstart1; row < vend1; ++row) { + for (col = 0; col < (hend1 - hstart1); ++col) *dst8++ = (uint8_t)(*src16++); + src16 += src_stride - (hend1 - hstart1); + dst8 += dst_stride - (hend1 - hstart1); } + return; } static void copy_decoded_tile_to_tile_list_buffer(AV1Decoder *pbi, - uint8_t **output) { + int tile_idx) { AV1_COMMON *const cm = &pbi->common; - const int tile_width_in_pixels = cm->tile_width * MI_SIZE; - const int tile_height_in_pixels = cm->tile_height * MI_SIZE; + int tile_width, tile_height; + av1_get_uniform_tile_size(cm, &tile_width, &tile_height); + const int tile_width_in_pixels = tile_width * MI_SIZE; + const int tile_height_in_pixels = tile_height * MI_SIZE; const int ssy = cm->seq_params.subsampling_y; const int ssx = cm->seq_params.subsampling_x; const int num_planes = av1_num_planes(cm); - // Copy decoded tile to the tile list output buffer. - YV12_BUFFER_CONFIG *cur_frame = get_frame_new_buffer(cm); - const int mi_row = pbi->dec_tile_row * cm->tile_height; - const int mi_col = pbi->dec_tile_col * cm->tile_width; - const int is_hbd = (cur_frame->flags & YV12_FLAG_HIGHBITDEPTH) ? 1 : 0; - uint8_t *bufs[MAX_MB_PLANE] = { NULL, NULL, NULL }; - int strides[MAX_MB_PLANE] = { 0, 0, 0 }; + YV12_BUFFER_CONFIG *cur_frame = &cm->cur_frame->buf; + const int tr = tile_idx / (pbi->output_frame_width_in_tiles_minus_1 + 1); + const int tc = tile_idx % (pbi->output_frame_width_in_tiles_minus_1 + 1); int plane; + // Copy decoded tile to the tile list output buffer. for (plane = 0; plane < num_planes; ++plane) { - int shift_x = plane > 0 ? ssx : 0; - int shift_y = plane > 0 ? ssy : 0; + const int shift_x = plane > 0 ? ssx : 0; + const int shift_y = plane > 0 ? ssy : 0; + const int h = tile_height_in_pixels >> shift_y; + const int w = tile_width_in_pixels >> shift_x; - bufs[plane] = cur_frame->buffers[plane]; - strides[plane] = - (plane > 0) ? cur_frame->strides[1] : cur_frame->strides[0]; + // src offset + int vstart1 = pbi->dec_tile_row * h; + int vend1 = vstart1 + h; + int hstart1 = pbi->dec_tile_col * w; + int hend1 = hstart1 + w; + // dst offset + int vstart2 = tr * h; + int hstart2 = tc * w; - bufs[plane] += mi_row * (MI_SIZE >> shift_y) * strides[plane] + - mi_col * (MI_SIZE >> shift_x); - - if (is_hbd) { - bufs[plane] = (uint8_t *)CONVERT_TO_SHORTPTR(bufs[plane]); - strides[plane] *= 2; - } - - int w, h; - w = (plane > 0 && shift_x > 0) ? ((tile_width_in_pixels + 1) >> shift_x) - : tile_width_in_pixels; - w *= (1 + is_hbd); - h = (plane > 0 && shift_y > 0) ? ((tile_height_in_pixels + 1) >> shift_y) - : tile_height_in_pixels; - int j; - - for (j = 0; j < h; ++j) { - memcpy(*output, bufs[plane], w); - bufs[plane] += strides[plane]; - *output += w; + if (cm->seq_params.use_highbitdepth && + cm->seq_params.bit_depth == AOM_BITS_8) { + yv12_tile_copy(cur_frame, hstart1, hend1, vstart1, vend1, + &pbi->tile_list_outbuf, hstart2, vstart2, plane); + } else { + switch (plane) { + case 0: + aom_yv12_partial_copy_y(cur_frame, hstart1, hend1, vstart1, vend1, + &pbi->tile_list_outbuf, hstart2, vstart2); + break; + case 1: + aom_yv12_partial_copy_u(cur_frame, hstart1, hend1, vstart1, vend1, + &pbi->tile_list_outbuf, hstart2, vstart2); + break; + case 2: + aom_yv12_partial_copy_v(cur_frame, hstart1, hend1, vstart1, vend1, + &pbi->tile_list_outbuf, hstart2, vstart2); + break; + default: assert(0); + } } } } // Only called while large_scale_tile = 1. +// +// On success, returns the tile list OBU size. On failure, sets +// pbi->common.error.error_code and returns 0. static uint32_t read_and_decode_one_tile_list(AV1Decoder *pbi, struct aom_read_bit_buffer *rb, const uint8_t *data, @@ -459,8 +502,8 @@ static uint32_t read_and_decode_one_tile_list(AV1Decoder *pbi, uint32_t tile_list_info_bytes = 4; tile_list_payload_size += tile_list_info_bytes; data += tile_list_info_bytes; - uint8_t *output = pbi->tile_list_output; + int tile_idx = 0; for (i = 0; i <= pbi->tile_count_minus_1; i++) { // Process 1 tile. // Reset the bit reader. @@ -504,7 +547,8 @@ static uint32_t read_and_decode_one_tile_list(AV1Decoder *pbi, assert(data <= data_end); // Copy the decoded tile to the tile list output buffer. - copy_decoded_tile_to_tile_list_buffer(pbi, &output); + copy_decoded_tile_to_tile_list_buffer(pbi, tile_idx); + tile_idx++; } *frame_decoding_finished = 1; @@ -613,6 +657,7 @@ static void read_metadata_timecode(const uint8_t *data, size_t sz) { } } +// Not fully implemented. Always succeeds and returns sz. static size_t read_metadata(const uint8_t *data, size_t sz) { size_t type_length; uint64_t type_value; @@ -659,7 +704,7 @@ int aom_decode_frame_from_obus(struct AV1Decoder *pbi, const uint8_t *data, if (!cm->large_scale_tile) pbi->camera_frame_header_ready = 0; // decode frame as a series of OBUs - while (!frame_decoding_finished && !cm->error.error_code) { + while (!frame_decoding_finished && cm->error.error_code == AOM_CODEC_OK) { struct aom_read_bit_buffer rb; size_t payload_size = 0; size_t decoded_payload_size = 0; @@ -769,6 +814,7 @@ int aom_decode_frame_from_obus(struct AV1Decoder *pbi, const uint8_t *data, if (obu_header.type != OBU_FRAME) break; obu_payload_offset = frame_header_size; // Byte align the reader before reading the tile group. + // byte_alignment() has set cm->error.error_code if it returns -1. if (byte_alignment(cm, &rb)) return -1; AOM_FALLTHROUGH_INTENDED; // fall through to read tile group. case OBU_TILE_GROUP: @@ -784,6 +830,7 @@ int aom_decode_frame_from_obus(struct AV1Decoder *pbi, const uint8_t *data, pbi, &rb, is_first_tg_obu_received, data + obu_payload_offset, data + payload_size, p_data_end, &frame_decoding_finished, obu_header.type == OBU_FRAME); + if (cm->error.error_code != AOM_CODEC_OK) return -1; is_first_tg_obu_received = 0; if (frame_decoding_finished) pbi->seen_frame_header = 0; break; @@ -835,5 +882,6 @@ int aom_decode_frame_from_obus(struct AV1Decoder *pbi, const uint8_t *data, data += payload_size; } + if (cm->error.error_code != AOM_CODEC_OK) return -1; return frame_decoding_finished; } diff --git a/third_party/aom/av1/decoder/obu.h b/third_party/aom/av1/decoder/obu.h index 5ab243fc90..d8ebe368e6 100644 --- a/third_party/aom/av1/decoder/obu.h +++ b/third_party/aom/av1/decoder/obu.h @@ -25,7 +25,7 @@ int aom_decode_frame_from_obus(struct AV1Decoder *pbi, const uint8_t *data, const uint8_t **p_data_end); aom_codec_err_t aom_get_num_layers_from_operating_point_idc( - int operating_point_idc, unsigned int *num_spatial_layers, - unsigned int *num_temporal_layers); + int operating_point_idc, unsigned int *number_spatial_layers, + unsigned int *number_temporal_layers); #endif // AOM_AV1_DECODER_OBU_H_ diff --git a/third_party/aom/av1/encoder/aq_complexity.c b/third_party/aom/av1/encoder/aq_complexity.c index 80f8e2e66d..16edbc69a2 100644 --- a/third_party/aom/av1/encoder/aq_complexity.c +++ b/third_party/aom/av1/encoder/aq_complexity.c @@ -93,7 +93,7 @@ void av1_setup_in_frame_q_adj(AV1_COMP *cpi) { if (segment == DEFAULT_AQ2_SEG) continue; qindex_delta = av1_compute_qdelta_by_rate( - &cpi->rc, cm->frame_type, cm->base_qindex, + &cpi->rc, cm->current_frame.frame_type, cm->base_qindex, aq_c_q_adj_factor[aq_strength][segment], cm->seq_params.bit_depth); // For AQ complexity mode, we dont allow Q0 in a segment if the base @@ -148,7 +148,7 @@ void av1_caq_select_segment(const AV1_COMP *cpi, MACROBLOCK *mb, BLOCK_SIZE bs, ? AOMMAX(exp(cpi->twopass.mb_av_energy), MIN_DEFAULT_LV_THRESH) : DEFAULT_LV_THRESH; - av1_setup_src_planes(mb, cpi->source, mi_row, mi_col, num_planes); + av1_setup_src_planes(mb, cpi->source, mi_row, mi_col, num_planes, bs); logvar = av1_log_block_var(cpi, mb, bs); segment = AQ_C_SEGMENTS - 1; // Just in case no break out below. diff --git a/third_party/aom/av1/encoder/aq_cyclicrefresh.c b/third_party/aom/av1/encoder/aq_cyclicrefresh.c index f532d48da5..8d96b23920 100644 --- a/third_party/aom/av1/encoder/aq_cyclicrefresh.c +++ b/third_party/aom/av1/encoder/aq_cyclicrefresh.c @@ -140,8 +140,8 @@ static int compute_deltaq(const AV1_COMP *cpi, int q, double rate_factor) { const CYCLIC_REFRESH *const cr = cpi->cyclic_refresh; const RATE_CONTROL *const rc = &cpi->rc; int deltaq = - av1_compute_qdelta_by_rate(rc, cpi->common.frame_type, q, rate_factor, - cpi->common.seq_params.bit_depth); + av1_compute_qdelta_by_rate(rc, cpi->common.current_frame.frame_type, q, + rate_factor, cpi->common.seq_params.bit_depth); if ((-deltaq) > cr->max_qdelta_perc * q / 100) { deltaq = -cr->max_qdelta_perc * q / 100; } @@ -166,15 +166,15 @@ int av1_cyclic_refresh_estimate_bits_at_q(const AV1_COMP *cpi, // Take segment weighted average for estimated bits. estimated_bits = (int)((1.0 - weight_segment1 - weight_segment2) * - av1_estimate_bits_at_q(cm->frame_type, cm->base_qindex, mbs, - correction_factor, + av1_estimate_bits_at_q(cm->current_frame.frame_type, + cm->base_qindex, mbs, correction_factor, cm->seq_params.bit_depth) + weight_segment1 * av1_estimate_bits_at_q( - cm->frame_type, + cm->current_frame.frame_type, cm->base_qindex + cr->qindex_delta[1], mbs, correction_factor, cm->seq_params.bit_depth) + weight_segment2 * av1_estimate_bits_at_q( - cm->frame_type, + cm->current_frame.frame_type, cm->base_qindex + cr->qindex_delta[2], mbs, correction_factor, cm->seq_params.bit_depth)); return estimated_bits; @@ -203,10 +203,11 @@ int av1_cyclic_refresh_rc_bits_per_mb(const AV1_COMP *cpi, int i, // Take segment weighted average for bits per mb. bits_per_mb = (int)((1.0 - weight_segment) * - av1_rc_bits_per_mb(cm->frame_type, i, correction_factor, + av1_rc_bits_per_mb(cm->current_frame.frame_type, i, + correction_factor, cm->seq_params.bit_depth) + - weight_segment * av1_rc_bits_per_mb(cm->frame_type, i + deltaq, - correction_factor, + weight_segment * av1_rc_bits_per_mb(cm->current_frame.frame_type, + i + deltaq, correction_factor, cm->seq_params.bit_depth)); return bits_per_mb; } @@ -496,14 +497,14 @@ void av1_cyclic_refresh_setup(AV1_COMP *const cpi) { av1_disable_segmentation(seg); return; } - if (cm->current_video_frame == 0) cr->low_content_avg = 0.0; + if (cm->current_frame.frame_number == 0) cr->low_content_avg = 0.0; // Don't apply refresh on key frame or enhancement layer frames. - if (!apply_cyclic_refresh || cm->frame_type == KEY_FRAME) { + if (!apply_cyclic_refresh || cm->current_frame.frame_type == KEY_FRAME) { // Set segmentation map to 0 and disable. unsigned char *const seg_map = cpi->segmentation_map; memset(seg_map, 0, cm->mi_rows * cm->mi_cols); av1_disable_segmentation(&cm->seg); - if (cm->frame_type == KEY_FRAME) { + if (cm->current_frame.frame_type == KEY_FRAME) { memset(cr->last_coded_q_map, MAXQ, cm->mi_rows * cm->mi_cols * sizeof(*cr->last_coded_q_map)); cr->sb_index = 0; diff --git a/third_party/aom/av1/encoder/aq_variance.c b/third_party/aom/av1/encoder/aq_variance.c index 58f906bdc0..d5729482fa 100644 --- a/third_party/aom/av1/encoder/aq_variance.c +++ b/third_party/aom/av1/encoder/aq_variance.c @@ -77,8 +77,8 @@ void av1_vaq_frame_setup(AV1_COMP *cpi) { // Set up avg segment id to be 1.0 and adjust the other segments around // it. int qindex_delta = av1_compute_qdelta_by_rate( - &cpi->rc, cm->frame_type, cm->base_qindex, rate_ratio[i] / avg_ratio, - cm->seq_params.bit_depth); + &cpi->rc, cm->current_frame.frame_type, cm->base_qindex, + rate_ratio[i] / avg_ratio, cm->seq_params.bit_depth); // We don't allow qindex 0 in a segment if the base value is not 0. // Q index 0 (lossless) implies 4x4 encoding only and in AQ mode a segment @@ -121,7 +121,7 @@ int av1_log_block_var(const AV1_COMP *cpi, MACROBLOCK *x, BLOCK_SIZE bs) { for (i = 0; i < bh; i += 4) { for (j = 0; j < bw; j += 4) { - if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + if (is_cur_buf_hbd(xd)) { var += log(1.0 + cpi->fn_ptr[BLOCK_4X4].vf( x->plane[0].src.buf + i * x->plane[0].src.stride + j, @@ -147,13 +147,13 @@ int av1_log_block_var(const AV1_COMP *cpi, MACROBLOCK *x, BLOCK_SIZE bs) { #define DEFAULT_E_MIDPOINT 10.0 -unsigned int haar_ac_energy(MACROBLOCK *x, BLOCK_SIZE bs) { +static unsigned int haar_ac_energy(MACROBLOCK *x, BLOCK_SIZE bs) { MACROBLOCKD *xd = &x->e_mbd; int stride = x->plane[0].src.stride; uint8_t *buf = x->plane[0].src.buf; const int bw = MI_SIZE * mi_size_wide[bs]; const int bh = MI_SIZE * mi_size_high[bs]; - int hbd = xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH; + const int hbd = is_cur_buf_hbd(xd); int var = 0; for (int r = 0; r < bh; r += 8) @@ -192,8 +192,8 @@ int av1_compute_deltaq_from_energy_level(const AV1_COMP *const cpi, rate_level = block_var_level; } int qindex_delta = av1_compute_qdelta_by_rate( - &cpi->rc, cm->frame_type, cm->base_qindex, deltaq_rate_ratio[rate_level], - cm->seq_params.bit_depth); + &cpi->rc, cm->current_frame.frame_type, cm->base_qindex, + deltaq_rate_ratio[rate_level], cm->seq_params.bit_depth); if ((cm->base_qindex != 0) && ((cm->base_qindex + qindex_delta) == 0)) { qindex_delta = -cm->base_qindex + 1; diff --git a/third_party/aom/av1/encoder/av1_fwd_txfm2d.c b/third_party/aom/av1/encoder/av1_fwd_txfm2d.c index f25a667cf4..d282b5f4ca 100644 --- a/third_party/aom/av1/encoder/av1_fwd_txfm2d.c +++ b/third_party/aom/av1/encoder/av1_fwd_txfm2d.c @@ -364,8 +364,6 @@ static const int8_t fadst4_range_mult2[7] = { 0, 2, 4, 3, 3, 3, 3 }; static const int8_t fadst8_range_mult2[8] = { 0, 0, 1, 3, 3, 5, 5, 5 }; static const int8_t fadst16_range_mult2[10] = { 0, 0, 1, 3, 3, 5, 5, 7, 7, 7 }; -static const int8_t max_fwd_range_mult2_col[5] = { 3, 5, 7, 9, 11 }; - static const int8_t fidtx4_range_mult2[1] = { 1 }; static const int8_t fidtx8_range_mult2[1] = { 2 }; static const int8_t fidtx16_range_mult2[1] = { 3 }; @@ -389,14 +387,12 @@ const int8_t *fwd_txfm_range_mult2_list[TXFM_TYPES] = { }; static INLINE void set_fwd_txfm_non_scale_range(TXFM_2D_FLIP_CFG *cfg) { - const int txh_idx = get_txh_idx(cfg->tx_size); av1_zero(cfg->stage_range_col); av1_zero(cfg->stage_range_row); + const int8_t *range_mult2_col = fwd_txfm_range_mult2_list[cfg->txfm_type_col]; if (cfg->txfm_type_col != TXFM_TYPE_INVALID) { int stage_num_col = cfg->stage_num_col; - const int8_t *range_mult2_col = - fwd_txfm_range_mult2_list[cfg->txfm_type_col]; for (int i = 0; i < stage_num_col; ++i) cfg->stage_range_col[i] = (range_mult2_col[i] + 1) >> 1; } @@ -405,9 +401,11 @@ static INLINE void set_fwd_txfm_non_scale_range(TXFM_2D_FLIP_CFG *cfg) { int stage_num_row = cfg->stage_num_row; const int8_t *range_mult2_row = fwd_txfm_range_mult2_list[cfg->txfm_type_row]; - for (int i = 0; i < stage_num_row; ++i) + for (int i = 0; i < stage_num_row; ++i) { cfg->stage_range_row[i] = - (max_fwd_range_mult2_col[txh_idx] + range_mult2_row[i] + 1) >> 1; + (range_mult2_col[cfg->stage_num_col - 1] + range_mult2_row[i] + 1) >> + 1; + } } } diff --git a/third_party/aom/av1/encoder/av1_multi_thread.c b/third_party/aom/av1/encoder/av1_multi_thread.c new file mode 100644 index 0000000000..1260c7af7c --- /dev/null +++ b/third_party/aom/av1/encoder/av1_multi_thread.c @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2018, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "av1/encoder/encoder.h" +#include "av1/encoder/ethread.h" +#include "av1/encoder/av1_multi_thread.h" + +void av1_row_mt_mem_alloc(AV1_COMP *cpi, int max_sb_rows) { + struct AV1Common *cm = &cpi->common; + MultiThreadHandle *multi_thread_ctxt = &cpi->multi_thread_ctxt; + int tile_row, tile_col; + const int tile_cols = cm->tile_cols; + const int tile_rows = cm->tile_rows; + + multi_thread_ctxt->allocated_tile_cols = tile_cols; + multi_thread_ctxt->allocated_tile_rows = tile_rows; + multi_thread_ctxt->allocated_sb_rows = max_sb_rows; + + // Allocate memory for row based multi-threading + for (tile_row = 0; tile_row < multi_thread_ctxt->allocated_tile_rows; + tile_row++) { + for (tile_col = 0; tile_col < multi_thread_ctxt->allocated_tile_cols; + tile_col++) { + TileDataEnc *this_tile = + &cpi->tile_data[tile_row * multi_thread_ctxt->allocated_tile_cols + + tile_col]; + av1_row_mt_sync_mem_alloc(&this_tile->row_mt_sync, cm, max_sb_rows); + if (cpi->oxcf.cdf_update_mode) + CHECK_MEM_ERROR( + cm, this_tile->row_ctx, + (FRAME_CONTEXT *)aom_memalign( + 16, + AOMMAX(1, (av1_get_sb_cols_in_tile(cm, this_tile->tile_info) - + 1)) * + sizeof(*this_tile->row_ctx))); + } + } +} + +void av1_row_mt_mem_dealloc(AV1_COMP *cpi) { + MultiThreadHandle *multi_thread_ctxt = &cpi->multi_thread_ctxt; + int tile_col; + int tile_row; + + // Free row based multi-threading sync memory + for (tile_row = 0; tile_row < multi_thread_ctxt->allocated_tile_rows; + tile_row++) { + for (tile_col = 0; tile_col < multi_thread_ctxt->allocated_tile_cols; + tile_col++) { + TileDataEnc *this_tile = + &cpi->tile_data[tile_row * multi_thread_ctxt->allocated_tile_cols + + tile_col]; + av1_row_mt_sync_mem_dealloc(&this_tile->row_mt_sync); + if (cpi->oxcf.cdf_update_mode) aom_free(this_tile->row_ctx); + } + } + multi_thread_ctxt->allocated_sb_rows = 0; + multi_thread_ctxt->allocated_tile_cols = 0; + multi_thread_ctxt->allocated_tile_rows = 0; +} diff --git a/third_party/aom/av1/encoder/av1_multi_thread.h b/third_party/aom/av1/encoder/av1_multi_thread.h new file mode 100644 index 0000000000..2a1cc7d6dd --- /dev/null +++ b/third_party/aom/av1/encoder/av1_multi_thread.h @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2018, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AV1_ENCODER_AV1_MULTI_THREAD_H +#define AV1_ENCODER_AV1_MULTI_THREAD_H + +#include "av1/encoder/encoder.h" + +void av1_row_mt_mem_alloc(AV1_COMP *cpi, int max_sb_rows); + +void av1_row_mt_mem_dealloc(AV1_COMP *cpi); + +#endif // AV1_ENCODER_AV1_MULTI_THREAD_H diff --git a/third_party/aom/av1/encoder/av1_quantize.c b/third_party/aom/av1/encoder/av1_quantize.c index a0a9260052..162724b4c3 100644 --- a/third_party/aom/av1/encoder/av1_quantize.c +++ b/third_party/aom/av1/encoder/av1_quantize.c @@ -41,47 +41,37 @@ static void quantize_fp_helper_c( const int16_t *scan, const int16_t *iscan, const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr, int log_scale) { int i, eob = -1; + const int rounding[2] = { ROUND_POWER_OF_TWO(round_ptr[0], log_scale), + ROUND_POWER_OF_TWO(round_ptr[1], log_scale) }; // TODO(jingning) Decide the need of these arguments after the // quantization process is completed. (void)zbin_ptr; (void)quant_shift_ptr; + (void)iscan; memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); if (qm_ptr == NULL && iqm_ptr == NULL) { - const int rounding0 = ROUND_POWER_OF_TWO(round_ptr[0], log_scale); - { // rc == 0 - const int coeff = coeff_ptr[0]; + for (i = 0; i < n_coeffs; i++) { + const int rc = scan[i]; + const int32_t thresh = (int32_t)(dequant_ptr[rc != 0]); + const int coeff = coeff_ptr[rc]; const int coeff_sign = (coeff >> 31); int64_t abs_coeff = (coeff ^ coeff_sign) - coeff_sign; - if ((abs_coeff << (1 + log_scale)) >= (int32_t)(dequant_ptr[0])) { - abs_coeff = clamp64(abs_coeff + rounding0, INT16_MIN, INT16_MAX); - const int tmp32 = (int)((abs_coeff * quant_ptr[0]) >> (16 - log_scale)); + int tmp32 = 0; + if ((abs_coeff << (1 + log_scale)) >= thresh) { + abs_coeff = + clamp64(abs_coeff + rounding[rc != 0], INT16_MIN, INT16_MAX); + tmp32 = (int)((abs_coeff * quant_ptr[rc != 0]) >> (16 - log_scale)); if (tmp32) { - qcoeff_ptr[0] = (tmp32 ^ coeff_sign) - coeff_sign; - const tran_low_t abs_dqcoeff = (tmp32 * dequant_ptr[0]) >> log_scale; - dqcoeff_ptr[0] = (abs_dqcoeff ^ coeff_sign) - coeff_sign; - eob = 0; - } - } - } - const int rounding1 = ROUND_POWER_OF_TWO(round_ptr[1], log_scale); - const int32_t thresh1 = (int32_t)(dequant_ptr[1]); - for (i = 1; i < n_coeffs; i++) { - const int coeff = coeff_ptr[i]; - const int coeff_sign = (coeff >> 31); - int64_t abs_coeff = (coeff ^ coeff_sign) - coeff_sign; - if ((abs_coeff << (1 + log_scale)) >= thresh1) { - abs_coeff = clamp64(abs_coeff + rounding1, INT16_MIN, INT16_MAX); - const int tmp32 = (int)((abs_coeff * quant_ptr[1]) >> (16 - log_scale)); - if (tmp32) { - qcoeff_ptr[i] = (tmp32 ^ coeff_sign) - coeff_sign; - const tran_low_t abs_dqcoeff = (tmp32 * dequant_ptr[1]) >> log_scale; - dqcoeff_ptr[i] = (abs_dqcoeff ^ coeff_sign) - coeff_sign; - eob = AOMMAX(iscan[i], eob); + qcoeff_ptr[rc] = (tmp32 ^ coeff_sign) - coeff_sign; + const tran_low_t abs_dqcoeff = + (tmp32 * dequant_ptr[rc != 0]) >> log_scale; + dqcoeff_ptr[rc] = (abs_dqcoeff ^ coeff_sign) - coeff_sign; } } + if (tmp32) eob = i; } } else { // Quantization pass: All coefficients with index >= zero_flag are @@ -99,7 +89,7 @@ static void quantize_fp_helper_c( int tmp32 = 0; if (abs_coeff * wt >= (dequant_ptr[rc != 0] << (AOM_QM_BITS - (1 + log_scale)))) { - abs_coeff += ROUND_POWER_OF_TWO(round_ptr[rc != 0], log_scale); + abs_coeff += rounding[rc != 0]; abs_coeff = clamp64(abs_coeff, INT16_MIN, INT16_MAX); tmp32 = (int)((abs_coeff * wt * quant_ptr[rc != 0]) >> (16 - log_scale + AOM_QM_BITS)); @@ -275,32 +265,65 @@ void av1_quantize_b_facade(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const SCAN_ORDER *sc, const QUANT_PARAM *qparam) { const qm_val_t *qm_ptr = qparam->qmatrix; const qm_val_t *iqm_ptr = qparam->iqmatrix; - if (qm_ptr != NULL && iqm_ptr != NULL) { - quantize_b_helper_c(coeff_ptr, n_coeffs, p->zbin_QTX, p->round_QTX, - p->quant_QTX, p->quant_shift_QTX, qcoeff_ptr, - dqcoeff_ptr, p->dequant_QTX, eob_ptr, sc->scan, - sc->iscan, qm_ptr, iqm_ptr, qparam->log_scale); + if (qparam->use_quant_b_adapt) { + // TODO(sarahparker) These quantize_b optimizations need SIMD + // implementations + if (qm_ptr != NULL && iqm_ptr != NULL) { + quantize_b_adaptive_helper_c( + coeff_ptr, n_coeffs, p->zbin_QTX, p->round_QTX, p->quant_QTX, + p->quant_shift_QTX, qcoeff_ptr, dqcoeff_ptr, p->dequant_QTX, eob_ptr, + sc->scan, sc->iscan, qm_ptr, iqm_ptr, qparam->log_scale); + } else { + switch (qparam->log_scale) { + case 0: + aom_quantize_b_adaptive_c( + coeff_ptr, n_coeffs, p->zbin_QTX, p->round_QTX, p->quant_QTX, + p->quant_shift_QTX, qcoeff_ptr, dqcoeff_ptr, p->dequant_QTX, + eob_ptr, sc->scan, sc->iscan); + break; + case 1: + aom_quantize_b_32x32_adaptive_c( + coeff_ptr, n_coeffs, p->zbin_QTX, p->round_QTX, p->quant_QTX, + p->quant_shift_QTX, qcoeff_ptr, dqcoeff_ptr, p->dequant_QTX, + eob_ptr, sc->scan, sc->iscan); + break; + case 2: + aom_quantize_b_64x64_adaptive_c( + coeff_ptr, n_coeffs, p->zbin_QTX, p->round_QTX, p->quant_QTX, + p->quant_shift_QTX, qcoeff_ptr, dqcoeff_ptr, p->dequant_QTX, + eob_ptr, sc->scan, sc->iscan); + break; + default: assert(0); + } + } } else { - switch (qparam->log_scale) { - case 0: - aom_quantize_b(coeff_ptr, n_coeffs, p->zbin_QTX, p->round_QTX, - p->quant_QTX, p->quant_shift_QTX, qcoeff_ptr, - dqcoeff_ptr, p->dequant_QTX, eob_ptr, sc->scan, - sc->iscan); - break; - case 1: - aom_quantize_b_32x32(coeff_ptr, n_coeffs, p->zbin_QTX, p->round_QTX, - p->quant_QTX, p->quant_shift_QTX, qcoeff_ptr, - dqcoeff_ptr, p->dequant_QTX, eob_ptr, sc->scan, - sc->iscan); - break; - case 2: - aom_quantize_b_64x64(coeff_ptr, n_coeffs, p->zbin_QTX, p->round_QTX, - p->quant_QTX, p->quant_shift_QTX, qcoeff_ptr, - dqcoeff_ptr, p->dequant_QTX, eob_ptr, sc->scan, - sc->iscan); - break; - default: assert(0); + if (qm_ptr != NULL && iqm_ptr != NULL) { + quantize_b_helper_c(coeff_ptr, n_coeffs, p->zbin_QTX, p->round_QTX, + p->quant_QTX, p->quant_shift_QTX, qcoeff_ptr, + dqcoeff_ptr, p->dequant_QTX, eob_ptr, sc->scan, + sc->iscan, qm_ptr, iqm_ptr, qparam->log_scale); + } else { + switch (qparam->log_scale) { + case 0: + aom_quantize_b(coeff_ptr, n_coeffs, p->zbin_QTX, p->round_QTX, + p->quant_QTX, p->quant_shift_QTX, qcoeff_ptr, + dqcoeff_ptr, p->dequant_QTX, eob_ptr, sc->scan, + sc->iscan); + break; + case 1: + aom_quantize_b_32x32(coeff_ptr, n_coeffs, p->zbin_QTX, p->round_QTX, + p->quant_QTX, p->quant_shift_QTX, qcoeff_ptr, + dqcoeff_ptr, p->dequant_QTX, eob_ptr, sc->scan, + sc->iscan); + break; + case 2: + aom_quantize_b_64x64(coeff_ptr, n_coeffs, p->zbin_QTX, p->round_QTX, + p->quant_QTX, p->quant_shift_QTX, qcoeff_ptr, + dqcoeff_ptr, p->dequant_QTX, eob_ptr, sc->scan, + sc->iscan); + break; + default: assert(0); + } } } } @@ -391,41 +414,81 @@ void av1_highbd_quantize_b_facade(const tran_low_t *coeff_ptr, const QUANT_PARAM *qparam) { const qm_val_t *qm_ptr = qparam->qmatrix; const qm_val_t *iqm_ptr = qparam->iqmatrix; - if (qm_ptr != NULL && iqm_ptr != NULL) { - highbd_quantize_b_helper_c(coeff_ptr, n_coeffs, p->zbin_QTX, p->round_QTX, - p->quant_QTX, p->quant_shift_QTX, qcoeff_ptr, - dqcoeff_ptr, p->dequant_QTX, eob_ptr, sc->scan, - sc->iscan, qm_ptr, iqm_ptr, qparam->log_scale); + if (qparam->use_quant_b_adapt) { + if (qm_ptr != NULL && iqm_ptr != NULL) { + highbd_quantize_b_adaptive_helper_c( + coeff_ptr, n_coeffs, p->zbin_QTX, p->round_QTX, p->quant_QTX, + p->quant_shift_QTX, qcoeff_ptr, dqcoeff_ptr, p->dequant_QTX, eob_ptr, + sc->scan, sc->iscan, qm_ptr, iqm_ptr, qparam->log_scale); + } else { + switch (qparam->log_scale) { + case 0: + if (LIKELY(n_coeffs >= 8)) { + aom_highbd_quantize_b_adaptive_c( + coeff_ptr, n_coeffs, p->zbin_QTX, p->round_QTX, p->quant_QTX, + p->quant_shift_QTX, qcoeff_ptr, dqcoeff_ptr, p->dequant_QTX, + eob_ptr, sc->scan, sc->iscan); + } else { + // TODO(luoyi): Need SIMD (e.g. sse2) for smaller block size + // quantization + aom_highbd_quantize_b_adaptive_c( + coeff_ptr, n_coeffs, p->zbin_QTX, p->round_QTX, p->quant_QTX, + p->quant_shift_QTX, qcoeff_ptr, dqcoeff_ptr, p->dequant_QTX, + eob_ptr, sc->scan, sc->iscan); + } + break; + case 1: + aom_highbd_quantize_b_32x32_adaptive_c( + coeff_ptr, n_coeffs, p->zbin_QTX, p->round_QTX, p->quant_QTX, + p->quant_shift_QTX, qcoeff_ptr, dqcoeff_ptr, p->dequant_QTX, + eob_ptr, sc->scan, sc->iscan); + break; + case 2: + aom_highbd_quantize_b_64x64_adaptive_c( + coeff_ptr, n_coeffs, p->zbin_QTX, p->round_QTX, p->quant_QTX, + p->quant_shift_QTX, qcoeff_ptr, dqcoeff_ptr, p->dequant_QTX, + eob_ptr, sc->scan, sc->iscan); + break; + default: assert(0); + } + } } else { - switch (qparam->log_scale) { - case 0: - if (LIKELY(n_coeffs >= 8)) { - aom_highbd_quantize_b(coeff_ptr, n_coeffs, p->zbin_QTX, p->round_QTX, - p->quant_QTX, p->quant_shift_QTX, qcoeff_ptr, - dqcoeff_ptr, p->dequant_QTX, eob_ptr, sc->scan, - sc->iscan); - } else { - // TODO(luoyi): Need SIMD (e.g. sse2) for smaller block size - // quantization - aom_highbd_quantize_b_c(coeff_ptr, n_coeffs, p->zbin_QTX, + if (qm_ptr != NULL && iqm_ptr != NULL) { + highbd_quantize_b_helper_c(coeff_ptr, n_coeffs, p->zbin_QTX, p->round_QTX, + p->quant_QTX, p->quant_shift_QTX, qcoeff_ptr, + dqcoeff_ptr, p->dequant_QTX, eob_ptr, sc->scan, + sc->iscan, qm_ptr, iqm_ptr, qparam->log_scale); + } else { + switch (qparam->log_scale) { + case 0: + if (LIKELY(n_coeffs >= 8)) { + aom_highbd_quantize_b(coeff_ptr, n_coeffs, p->zbin_QTX, p->round_QTX, p->quant_QTX, p->quant_shift_QTX, qcoeff_ptr, dqcoeff_ptr, p->dequant_QTX, eob_ptr, sc->scan, sc->iscan); - } - break; - case 1: - aom_highbd_quantize_b_32x32( - coeff_ptr, n_coeffs, p->zbin_QTX, p->round_QTX, p->quant_QTX, - p->quant_shift_QTX, qcoeff_ptr, dqcoeff_ptr, p->dequant_QTX, - eob_ptr, sc->scan, sc->iscan); - break; - case 2: - aom_highbd_quantize_b_64x64( - coeff_ptr, n_coeffs, p->zbin_QTX, p->round_QTX, p->quant_QTX, - p->quant_shift_QTX, qcoeff_ptr, dqcoeff_ptr, p->dequant_QTX, - eob_ptr, sc->scan, sc->iscan); - break; - default: assert(0); + } else { + // TODO(luoyi): Need SIMD (e.g. sse2) for smaller block size + // quantization + aom_highbd_quantize_b_c( + coeff_ptr, n_coeffs, p->zbin_QTX, p->round_QTX, p->quant_QTX, + p->quant_shift_QTX, qcoeff_ptr, dqcoeff_ptr, p->dequant_QTX, + eob_ptr, sc->scan, sc->iscan); + } + break; + case 1: + aom_highbd_quantize_b_32x32( + coeff_ptr, n_coeffs, p->zbin_QTX, p->round_QTX, p->quant_QTX, + p->quant_shift_QTX, qcoeff_ptr, dqcoeff_ptr, p->dequant_QTX, + eob_ptr, sc->scan, sc->iscan); + break; + case 2: + aom_highbd_quantize_b_64x64( + coeff_ptr, n_coeffs, p->zbin_QTX, p->round_QTX, p->quant_QTX, + p->quant_shift_QTX, qcoeff_ptr, dqcoeff_ptr, p->dequant_QTX, + eob_ptr, sc->scan, sc->iscan); + break; + default: assert(0); + } } } } @@ -697,7 +760,7 @@ void av1_frame_init_quantizer(AV1_COMP *cpi) { void av1_set_quantizer(AV1_COMMON *cm, int q) { // quantizer has to be reinitialized with av1_init_quantizer() if any // delta_q changes. - cm->base_qindex = AOMMAX(cm->delta_q_present_flag, q); + cm->base_qindex = AOMMAX(cm->delta_q_info.delta_q_present_flag, q); cm->y_dc_delta_q = 0; cm->u_dc_delta_q = 0; cm->u_ac_delta_q = 0; diff --git a/third_party/aom/av1/encoder/av1_quantize.h b/third_party/aom/av1/encoder/av1_quantize.h index 35af9a67ae..c709234956 100644 --- a/third_party/aom/av1/encoder/av1_quantize.h +++ b/third_party/aom/av1/encoder/av1_quantize.h @@ -27,6 +27,7 @@ typedef struct QUANT_PARAM { TX_SIZE tx_size; const qm_val_t *qmatrix; const qm_val_t *iqmatrix; + int use_quant_b_adapt; } QUANT_PARAM; typedef void (*AV1_QUANT_FACADE)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, @@ -67,7 +68,7 @@ typedef struct { // The Dequants structure is used only for internal quantizer setup in // av1_quantize.c. -// Fields are sufffixed according to whether or not they're expressed in +// Fields are suffixed according to whether or not they're expressed in // the same coefficient shift/precision as TX or a fixed Q3 format. typedef struct { DECLARE_ALIGNED(16, int16_t, diff --git a/third_party/aom/av1/encoder/bitstream.c b/third_party/aom/av1/encoder/bitstream.c index 2c4acdb021..3140551e9c 100644 --- a/third_party/aom/av1/encoder/bitstream.c +++ b/third_party/aom/av1/encoder/bitstream.c @@ -145,7 +145,7 @@ static void write_inter_compound_mode(MACROBLOCKD *xd, aom_writer *w, static void write_tx_size_vartx(MACROBLOCKD *xd, const MB_MODE_INFO *mbmi, TX_SIZE tx_size, int depth, int blk_row, int blk_col, aom_writer *w) { - FRAME_CONTEXT *ec_ctx = xd->tile_ctx; + FRAME_CONTEXT *const ec_ctx = xd->tile_ctx; const int max_blocks_high = max_block_high(xd, mbmi->sb_type, 0); const int max_blocks_wide = max_block_wide(xd, mbmi->sb_type, 0); @@ -229,7 +229,7 @@ static int write_skip(const AV1_COMMON *cm, const MACROBLOCKD *xd, static int write_skip_mode(const AV1_COMMON *cm, const MACROBLOCKD *xd, int segment_id, const MB_MODE_INFO *mi, aom_writer *w) { - if (!cm->skip_mode_flag) return 0; + if (!cm->current_frame.skip_mode_info.skip_mode_flag) return 0; if (segfeature_active(&cm->seg, segment_id, SEG_LVL_SKIP)) { return 0; } @@ -315,7 +315,7 @@ static void write_delta_lflevel(const AV1_COMMON *cm, const MACROBLOCKD *xd, int smallval = abs < DELTA_LF_SMALL ? 1 : 0; FRAME_CONTEXT *ec_ctx = xd->tile_ctx; - if (cm->delta_lf_multi) { + if (cm->delta_q_info.delta_lf_multi) { assert(lf_id >= 0 && lf_id < (av1_num_planes(cm) > 1 ? FRAME_LF_COUNT : FRAME_LF_COUNT - 2)); aom_write_symbol(w, AOMMIN(abs, DELTA_LF_SMALL), @@ -460,7 +460,7 @@ static void write_segment_id(AV1_COMP *cpi, const MB_MODE_INFO *const mbmi, // changing from lossless to lossy. assert(is_inter_block(mbmi) || !cpi->has_lossless_segment); - set_spatial_segment_id(cm, cm->current_frame_seg_map, mbmi->sb_type, mi_row, + set_spatial_segment_id(cm, cm->cur_frame->seg_map, mbmi->sb_type, mi_row, mi_col, pred); set_spatial_segment_id(cm, cpi->segmentation_map, mbmi->sb_type, mi_row, mi_col, pred); @@ -473,7 +473,7 @@ static void write_segment_id(AV1_COMP *cpi, const MB_MODE_INFO *const mbmi, av1_neg_interleave(mbmi->segment_id, pred, seg->last_active_segid + 1); aom_cdf_prob *pred_cdf = segp->spatial_pred_seg_cdf[cdf_num]; aom_write_symbol(w, coded_id, pred_cdf, MAX_SEGMENTS); - set_spatial_segment_id(cm, cm->current_frame_seg_map, mbmi->sb_type, mi_row, + set_spatial_segment_id(cm, cm->cur_frame->seg_map, mbmi->sb_type, mi_row, mi_col, mbmi->segment_id); } @@ -500,11 +500,12 @@ static void write_ref_frames(const AV1_COMMON *cm, const MACROBLOCKD *xd, } else { // does the feature use compound prediction or not // (if not specified at the frame/segment level) - if (cm->reference_mode == REFERENCE_MODE_SELECT) { + if (cm->current_frame.reference_mode == REFERENCE_MODE_SELECT) { if (is_comp_ref_allowed(mbmi->sb_type)) aom_write_symbol(w, is_compound, av1_get_reference_mode_cdf(xd), 2); } else { - assert((!is_compound) == (cm->reference_mode == SINGLE_REFERENCE)); + assert((!is_compound) == + (cm->current_frame.reference_mode == SINGLE_REFERENCE)); } if (is_compound) { @@ -866,14 +867,7 @@ static void write_cfl_alphas(FRAME_CONTEXT *const ec_ctx, int idx, static void write_cdef(AV1_COMMON *cm, MACROBLOCKD *const xd, aom_writer *w, int skip, int mi_col, int mi_row) { - if (cm->coded_lossless || cm->allow_intrabc) { - // Initialize to indicate no CDEF for safety. - cm->cdef_bits = 0; - cm->cdef_strengths[0] = 0; - cm->nb_cdef_strengths = 1; - cm->cdef_uv_strengths[0] = 0; - return; - } + if (cm->coded_lossless || cm->allow_intrabc) return; const int m = ~((1 << (6 - MI_SIZE_LOG2)) - 1); const MB_MODE_INFO *mbmi = @@ -891,7 +885,7 @@ static void write_cdef(AV1_COMMON *cm, MACROBLOCKD *const xd, aom_writer *w, ? !!(mi_col & mask) + 2 * !!(mi_row & mask) : 0; if (xd->cdef_preset[index] == -1 && !skip) { - aom_write_literal(w, mbmi->cdef_strength, cm->cdef_bits); + aom_write_literal(w, mbmi->cdef_strength, cm->cdef_info.cdef_bits); xd->cdef_preset[index] = mbmi->cdef_strength; } } @@ -902,7 +896,7 @@ static void write_inter_segment_id(AV1_COMP *cpi, aom_writer *w, int mi_row, int mi_col, int skip, int preskip) { MACROBLOCKD *const xd = &cpi->td.mb.e_mbd; - const MB_MODE_INFO *const mbmi = xd->mi[0]; + MB_MODE_INFO *const mbmi = xd->mi[0]; AV1_COMMON *const cm = &cpi->common; if (seg->update_map) { @@ -912,7 +906,7 @@ static void write_inter_segment_id(AV1_COMP *cpi, aom_writer *w, if (seg->segid_preskip) return; if (skip) { write_segment_id(cpi, mbmi, w, seg, segp, mi_row, mi_col, 1); - if (seg->temporal_update) ((MB_MODE_INFO *)mbmi)->seg_id_predicted = 0; + if (seg->temporal_update) mbmi->seg_id_predicted = 0; return; } } @@ -924,7 +918,7 @@ static void write_inter_segment_id(AV1_COMP *cpi, aom_writer *w, write_segment_id(cpi, mbmi, w, seg, segp, mi_row, mi_col, 0); } if (pred_flag) { - set_spatial_segment_id(cm, cm->current_frame_seg_map, mbmi->sb_type, + set_spatial_segment_id(cm, cm->cur_frame->seg_map, mbmi->sb_type, mi_row, mi_col, mbmi->segment_id); } } else { @@ -938,7 +932,9 @@ static void write_inter_segment_id(AV1_COMP *cpi, aom_writer *w, static void write_delta_q_params(AV1_COMP *cpi, const int mi_row, const int mi_col, int skip, aom_writer *w) { AV1_COMMON *const cm = &cpi->common; - if (cm->delta_q_present_flag) { + const DeltaQInfo *const delta_q_info = &cm->delta_q_info; + + if (delta_q_info->delta_q_present_flag) { MACROBLOCK *const x = &cpi->td.mb; MACROBLOCKD *const xd = &x->e_mbd; const MB_MODE_INFO *const mbmi = xd->mi[0]; @@ -951,24 +947,25 @@ static void write_delta_q_params(AV1_COMP *cpi, const int mi_row, super_block_upper_left) { assert(mbmi->current_qindex > 0); const int reduced_delta_qindex = - (mbmi->current_qindex - xd->current_qindex) / cm->delta_q_res; + (mbmi->current_qindex - xd->current_qindex) / + delta_q_info->delta_q_res; write_delta_qindex(xd, reduced_delta_qindex, w); xd->current_qindex = mbmi->current_qindex; - if (cm->delta_lf_present_flag) { - if (cm->delta_lf_multi) { + if (delta_q_info->delta_lf_present_flag) { + if (delta_q_info->delta_lf_multi) { const int frame_lf_count = av1_num_planes(cm) > 1 ? FRAME_LF_COUNT : FRAME_LF_COUNT - 2; for (int lf_id = 0; lf_id < frame_lf_count; ++lf_id) { int reduced_delta_lflevel = (mbmi->delta_lf[lf_id] - xd->delta_lf[lf_id]) / - cm->delta_lf_res; + delta_q_info->delta_lf_res; write_delta_lflevel(cm, xd, lf_id, reduced_delta_lflevel, w); xd->delta_lf[lf_id] = mbmi->delta_lf[lf_id]; } } else { int reduced_delta_lflevel = (mbmi->delta_lf_from_base - xd->delta_lf_from_base) / - cm->delta_lf_res; + delta_q_info->delta_lf_res; write_delta_lflevel(cm, xd, -1, reduced_delta_lflevel, w); xd->delta_lf_from_base = mbmi->delta_lf_from_base; } @@ -1105,7 +1102,7 @@ static void pack_inter_mode_mvs(AV1_COMP *cpi, const int mi_row, av1_encode_mv(cpi, w, &mbmi->mv[0].as_mv, &ref_mv.as_mv, nmvc, allow_hp); } - if (cpi->common.reference_mode != COMPOUND_REFERENCE && + if (cpi->common.current_frame.reference_mode != COMPOUND_REFERENCE && cpi->common.seq_params.enable_interintra_compound && is_interintra_allowed(mbmi)) { const int interintra = mbmi->ref_frame[1] == INTRA_FRAME; @@ -1130,7 +1127,7 @@ static void pack_inter_mode_mvs(AV1_COMP *cpi, const int mi_row, if (mbmi->ref_frame[1] != INTRA_FRAME) write_motion_mode(cm, xd, mbmi, w); // First write idx to indicate current compound inter prediction mode group - // Group A (0): jnt_comp, compound_average + // Group A (0): dist_wtd_comp, compound_average // Group B (1): interintra, compound_diffwtd, wedge if (has_second_ref(mbmi)) { const int masked_compound_used = is_any_masked_compound_used(bsize) && @@ -1148,7 +1145,7 @@ static void pack_inter_mode_mvs(AV1_COMP *cpi, const int mi_row, if (mbmi->compound_idx) assert(mbmi->interinter_comp.type == COMPOUND_AVERAGE); - if (cm->seq_params.enable_jnt_comp) { + if (cm->seq_params.order_hint_info.enable_dist_wtd_comp) { const int comp_index_ctx = get_comp_index_context(cm, xd); aom_write_symbol(w, mbmi->compound_idx, ec_ctx->compound_index_cdf[comp_index_ctx], 2); @@ -1156,7 +1153,7 @@ static void pack_inter_mode_mvs(AV1_COMP *cpi, const int mi_row, assert(mbmi->compound_idx == 1); } } else { - assert(cpi->common.reference_mode != SINGLE_REFERENCE && + assert(cpi->common.current_frame.reference_mode != SINGLE_REFERENCE && is_inter_compound_mode(mbmi->mode) && mbmi->motion_mode == SIMPLE_TRANSLATION); assert(masked_compound_used); @@ -1181,7 +1178,6 @@ static void pack_inter_mode_mvs(AV1_COMP *cpi, const int mi_row, } } } - write_mb_interp_filter(cpi, xd, w); } } @@ -1233,13 +1229,14 @@ static void write_mb_modes_kf(AV1_COMP *cpi, MACROBLOCKD *xd, } #if CONFIG_RD_DEBUG -static void dump_mode_info(MODE_INFO *mi) { +static void dump_mode_info(MB_MODE_INFO *mi) { printf("\nmi->mi_row == %d\n", mi->mi_row); printf("&& mi->mi_col == %d\n", mi->mi_col); printf("&& mi->sb_type == %d\n", mi->sb_type); printf("&& mi->tx_size == %d\n", mi->tx_size); printf("&& mi->mode == %d\n", mi->mode); } + static int rd_token_stats_mismatch(RD_STATS *rd_stats, TOKEN_STATS *token_stats, int plane) { if (rd_stats->txb_coeff_cost[plane] != token_stats->cost) { @@ -1270,27 +1267,26 @@ static int rd_token_stats_mismatch(RD_STATS *rd_stats, TOKEN_STATS *token_stats, #if ENC_MISMATCH_DEBUG static void enc_dump_logs(AV1_COMP *cpi, int mi_row, int mi_col) { AV1_COMMON *const cm = &cpi->common; - MACROBLOCKD *const xd = &cpi->td.mb.e_mbd; - xd->mi = cm->mi_grid_visible + (mi_row * cm->mi_stride + mi_col); - const MB_MODE_INFO *const *mbmi = xd->mi[0]; + const MB_MODE_INFO *const *mbmi = + *(cm->mi_grid_visible + (mi_row * cm->mi_stride + mi_col)); + const MB_MODE_INFO_EXT *const *mbmi_ext = + cpi->mbmi_ext_base + (mi_row * cm->mi_cols + mi_col); if (is_inter_block(mbmi)) { #define FRAME_TO_CHECK 11 - if (cm->current_video_frame == FRAME_TO_CHECK && cm->show_frame == 1) { + if (cm->current_frame.frame_number == FRAME_TO_CHECK && + cm->show_frame == 1) { const BLOCK_SIZE bsize = mbmi->sb_type; - int_mv mv[2]; - int is_comp_ref = has_second_ref(mbmi); - int ref; + int_mv mv[2] = { 0 }; + const int is_comp_ref = has_second_ref(mbmi); - for (ref = 0; ref < 1 + is_comp_ref; ++ref) + for (int ref = 0; ref < 1 + is_comp_ref; ++ref) mv[ref].as_mv = mbmi->mv[ref].as_mv; if (!is_comp_ref) { mv[1].as_int = 0; } - MACROBLOCK *const x = &cpi->td.mb; - const MB_MODE_INFO_EXT *const mbmi_ext = x->mbmi_ext; const int16_t mode_ctx = is_comp_ref ? mbmi_ext->compound_mode_context[mbmi->ref_frame[0]] : av1_mode_context_analyzer(mbmi_ext->mode_context, @@ -1312,8 +1308,8 @@ static void enc_dump_logs(AV1_COMP *cpi, int mi_row, int mi_col) { "show_frame=%d, mv[0]=(%d,%d), mv[1]=(%d,%d), ref[0]=%d, " "ref[1]=%d, motion_mode=%d, mode_ctx=%d, " "newmv_ctx=%d, zeromv_ctx=%d, refmv_ctx=%d, tx_size=%d\n", - cm->current_video_frame, mi_row, mi_col, mbmi->skip_mode, mbmi->mode, - bsize, cm->show_frame, mv[0].as_mv.row, mv[0].as_mv.col, + cm->current_frame.frame_number, mi_row, mi_col, mbmi->skip_mode, + mbmi->mode, bsize, cm->show_frame, mv[0].as_mv.row, mv[0].as_mv.col, mv[1].as_mv.row, mv[1].as_mv.col, mbmi->ref_frame[0], mbmi->ref_frame[1], mbmi->motion_mode, mode_ctx, newmv_ctx, zeromv_ctx, refmv_ctx, mbmi->tx_size); @@ -1474,14 +1470,16 @@ static void write_tokens_b(AV1_COMP *cpi, const TileInfo *const tile, row, col, &block[plane], plane); } } + } #if CONFIG_RD_DEBUG + for (plane = 0; plane < num_planes && is_inter_block(mbmi); ++plane) { if (mbmi->sb_type >= BLOCK_8X8 && rd_token_stats_mismatch(&mbmi->rd_stats, &token_stats, plane)) { - dump_mode_info(m); + dump_mode_info(mbmi); assert(0); } -#endif // CONFIG_RD_DEBUG } +#endif // CONFIG_RD_DEBUG } } } @@ -1685,9 +1683,9 @@ static void write_modes(AV1_COMP *const cpi, const TileInfo *const tile, av1_zero_above_context(cm, xd, mi_col_start, mi_col_end, tile->tile_row); av1_init_above_context(cm, xd, tile->tile_row); - if (cpi->common.delta_q_present_flag) { + if (cpi->common.delta_q_info.delta_q_present_flag) { xd->current_qindex = cpi->common.base_qindex; - if (cpi->common.delta_lf_present_flag) { + if (cpi->common.delta_q_info.delta_lf_present_flag) { av1_reset_loop_filter_delta(xd, av1_num_planes(cm)); } } @@ -1870,8 +1868,8 @@ static void loop_restoration_write_sb_coeffs(const AV1_COMMON *const cm, assert(!cm->all_lossless); const int wiener_win = (plane > 0) ? WIENER_WIN_CHROMA : WIENER_WIN; - WienerInfo *wiener_info = xd->wiener_info + plane; - SgrprojInfo *sgrproj_info = xd->sgrproj_info + plane; + WienerInfo *ref_wiener_info = &xd->wiener_info[plane]; + SgrprojInfo *ref_sgrproj_info = &xd->sgrproj_info[plane]; RestorationType unit_rtype = rui->restoration_type; if (frame_rtype == RESTORE_SWITCHABLE) { @@ -1882,10 +1880,10 @@ static void loop_restoration_write_sb_coeffs(const AV1_COMMON *const cm, #endif switch (unit_rtype) { case RESTORE_WIENER: - write_wiener_filter(wiener_win, &rui->wiener_info, wiener_info, w); + write_wiener_filter(wiener_win, &rui->wiener_info, ref_wiener_info, w); break; case RESTORE_SGRPROJ: - write_sgrproj_filter(&rui->sgrproj_info, sgrproj_info, w); + write_sgrproj_filter(&rui->sgrproj_info, ref_sgrproj_info, w); break; default: assert(unit_rtype == RESTORE_NONE); break; } @@ -1896,7 +1894,7 @@ static void loop_restoration_write_sb_coeffs(const AV1_COMMON *const cm, ++counts->wiener_restore[unit_rtype != RESTORE_NONE]; #endif if (unit_rtype != RESTORE_NONE) { - write_wiener_filter(wiener_win, &rui->wiener_info, wiener_info, w); + write_wiener_filter(wiener_win, &rui->wiener_info, ref_wiener_info, w); } } else if (frame_rtype == RESTORE_SGRPROJ) { aom_write_symbol(w, unit_rtype != RESTORE_NONE, @@ -1905,7 +1903,7 @@ static void loop_restoration_write_sb_coeffs(const AV1_COMMON *const cm, ++counts->sgrproj_restore[unit_rtype != RESTORE_NONE]; #endif if (unit_rtype != RESTORE_NONE) { - write_sgrproj_filter(&rui->sgrproj_info, sgrproj_info, w); + write_sgrproj_filter(&rui->sgrproj_info, ref_sgrproj_info, w); } } } @@ -1936,15 +1934,12 @@ static void encode_loopfilter(AV1_COMMON *cm, struct aom_write_bit_buffer *wb) { aom_wb_write_bit(wb, lf->mode_ref_delta_update); if (lf->mode_ref_delta_update) { - const int prime_idx = cm->primary_ref_frame; - const int buf_idx = - prime_idx == PRIMARY_REF_NONE ? -1 : cm->frame_refs[prime_idx].idx; + const RefCntBuffer *buf = get_primary_ref_frame_buf(cm); int8_t last_ref_deltas[REF_FRAMES]; - if (prime_idx == PRIMARY_REF_NONE || buf_idx < 0) { + if (buf == NULL) { av1_set_default_ref_deltas(last_ref_deltas); } else { - memcpy(last_ref_deltas, cm->buffer_pool->frame_bufs[buf_idx].ref_deltas, - REF_FRAMES); + memcpy(last_ref_deltas, buf->ref_deltas, REF_FRAMES); } for (i = 0; i < REF_FRAMES; i++) { const int delta = lf->ref_deltas[i]; @@ -1954,12 +1949,10 @@ static void encode_loopfilter(AV1_COMMON *cm, struct aom_write_bit_buffer *wb) { } int8_t last_mode_deltas[MAX_MODE_LF_DELTAS]; - if (prime_idx == PRIMARY_REF_NONE || buf_idx < 0) { + if (buf == NULL) { av1_set_default_mode_deltas(last_mode_deltas); } else { - memcpy(last_mode_deltas, - cm->buffer_pool->frame_bufs[buf_idx].mode_deltas, - MAX_MODE_LF_DELTAS); + memcpy(last_mode_deltas, buf->mode_deltas, MAX_MODE_LF_DELTAS); } for (i = 0; i < MAX_MODE_LF_DELTAS; i++) { const int delta = lf->mode_deltas[i]; @@ -1977,13 +1970,15 @@ static void encode_cdef(const AV1_COMMON *cm, struct aom_write_bit_buffer *wb) { if (cm->allow_intrabc) return; const int num_planes = av1_num_planes(cm); int i; - aom_wb_write_literal(wb, cm->cdef_pri_damping - 3, 2); - assert(cm->cdef_pri_damping == cm->cdef_sec_damping); - aom_wb_write_literal(wb, cm->cdef_bits, 2); - for (i = 0; i < cm->nb_cdef_strengths; i++) { - aom_wb_write_literal(wb, cm->cdef_strengths[i], CDEF_STRENGTH_BITS); + aom_wb_write_literal(wb, cm->cdef_info.cdef_pri_damping - 3, 2); + assert(cm->cdef_info.cdef_pri_damping == cm->cdef_info.cdef_sec_damping); + aom_wb_write_literal(wb, cm->cdef_info.cdef_bits, 2); + for (i = 0; i < cm->cdef_info.nb_cdef_strengths; i++) { + aom_wb_write_literal(wb, cm->cdef_info.cdef_strengths[i], + CDEF_STRENGTH_BITS); if (num_planes > 1) - aom_wb_write_literal(wb, cm->cdef_uv_strengths[i], CDEF_STRENGTH_BITS); + aom_wb_write_literal(wb, cm->cdef_info.cdef_uv_strengths[i], + CDEF_STRENGTH_BITS); } } @@ -2070,15 +2065,6 @@ static void encode_segmentation(AV1_COMMON *cm, MACROBLOCKD *xd, } } -static void write_tx_mode(AV1_COMMON *cm, TX_MODE *mode, - struct aom_write_bit_buffer *wb) { - if (cm->coded_lossless) { - *mode = ONLY_4X4; - return; - } - aom_wb_write_bit(wb, *mode == TX_MODE_SELECT); -} - static void write_frame_interp_filter(InterpFilter filter, struct aom_write_bit_buffer *wb) { aom_wb_write_bit(wb, filter == SWITCHABLE); @@ -2086,29 +2072,6 @@ static void write_frame_interp_filter(InterpFilter filter, aom_wb_write_literal(wb, filter, LOG_SWITCHABLE_FILTERS); } -static void fix_interp_filter(AV1_COMMON *cm, FRAME_COUNTS *counts) { - if (cm->interp_filter == SWITCHABLE) { - // Check to see if only one of the filters is actually used - int count[SWITCHABLE_FILTERS]; - int i, j, c = 0; - for (i = 0; i < SWITCHABLE_FILTERS; ++i) { - count[i] = 0; - for (j = 0; j < SWITCHABLE_FILTER_CONTEXTS; ++j) - count[i] += counts->switchable_interp[j][i]; - c += (count[i] > 0); - } - if (c == 1) { - // Only one filter is used. So set the filter at frame level - for (i = 0; i < SWITCHABLE_FILTERS; ++i) { - if (count[i]) { - if (i == EIGHTTAP_REGULAR) cm->interp_filter = i; - break; - } - } - } - } -} - // Same function as write_uniform but writing to uncompresses header wb static void wb_write_uniform(struct aom_write_bit_buffer *wb, int n, int v) { const int l = get_unsigned_bits(n); @@ -2206,60 +2169,12 @@ static void write_ext_tile_info(const AV1_COMMON *const cm, } } -static int get_refresh_mask(AV1_COMP *cpi) { - if ((cpi->common.frame_type == KEY_FRAME && cpi->common.show_frame) || - frame_is_sframe(&cpi->common)) - return 0xFF; - - int refresh_mask = 0; - - // NOTE(zoeliu): When LAST_FRAME is to get refreshed, the decoder will be - // notified to get LAST3_FRAME refreshed and then the virtual indexes for all - // the 3 LAST reference frames will be updated accordingly, i.e.: - // (1) The original virtual index for LAST3_FRAME will become the new virtual - // index for LAST_FRAME; and - // (2) The original virtual indexes for LAST_FRAME and LAST2_FRAME will be - // shifted and become the new virtual indexes for LAST2_FRAME and - // LAST3_FRAME. - refresh_mask |= - (cpi->refresh_last_frame << cpi->ref_fb_idx[LAST_REF_FRAMES - 1]); -#if USE_SYMM_MULTI_LAYER - refresh_mask |= - (cpi->new_bwdref_update_rule == 1) - ? (cpi->refresh_bwd_ref_frame << cpi->ref_fb_idx[EXTREF_FRAME - 1]) - : (cpi->refresh_bwd_ref_frame << cpi->ref_fb_idx[BWDREF_FRAME - 1]); -#else - refresh_mask |= - (cpi->refresh_bwd_ref_frame << cpi->ref_fb_idx[BWDREF_FRAME - 1]); -#endif - refresh_mask |= - (cpi->refresh_alt2_ref_frame << cpi->ref_fb_idx[ALTREF2_FRAME - 1]); - - if (av1_preserve_existing_gf(cpi)) { - // We have decided to preserve the previously existing golden frame as our - // new ARF frame. However, in the short term we leave it in the GF slot and, - // if we're updating the GF with the current decoded frame, we save it - // instead to the ARF slot. - // Later, in the function av1_encoder.c:av1_update_reference_frames() we - // will swap gld_fb_idx and alt_fb_idx to achieve our objective. We do it - // there so that it can be done outside of the recode loop. - // Note: This is highly specific to the use of ARF as a forward reference, - // and this needs to be generalized as other uses are implemented - // (like RTC/temporal scalability). - - if (cpi->preserve_arf_as_gld) { - return refresh_mask; - } else { - return refresh_mask | - (cpi->refresh_golden_frame << cpi->ref_fb_idx[ALTREF_FRAME - 1]); - } - } else { - const int arf_idx = cpi->ref_fb_idx[ALTREF_FRAME - 1]; - return refresh_mask | - (cpi->refresh_golden_frame << cpi->ref_fb_idx[GOLDEN_FRAME - 1]) | - (cpi->refresh_alt_ref_frame << arf_idx); - } -} +// Stores the location and size of a tile's data in the bitstream. Used for +// later identifying identical tiles +typedef struct TileBufferEnc { + uint8_t *data; + size_t size; +} TileBufferEnc; static INLINE int find_identical_tile( const int tile_row, const int tile_col, @@ -2280,18 +2195,18 @@ static INLINE int find_identical_tile( int col_offset = candidate_offset[0].col; int row = tile_row - row_offset; int col = tile_col - col_offset; - uint8_t tile_hdr; const uint8_t *tile_data; TileBufferEnc *candidate; if (row < 0 || col < 0) continue; - tile_hdr = *(tile_buffers[row][col].data); + const uint32_t tile_hdr = mem_get_le32(tile_buffers[row][col].data); - // Read out tcm bit - if ((tile_hdr >> 7) == 1) { - // The candidate is a copy tile itself - row_offset += tile_hdr & 0x7f; + // Read out tile-copy-mode bit: + if ((tile_hdr >> 31) == 1) { + // The candidate is a copy tile itself: the offset is stored in bits + // 30 through 24 inclusive. + row_offset += (tile_hdr >> 24) & 0x7f; row = tile_row - row_offset; } @@ -2361,14 +2276,13 @@ static void write_frame_size(const AV1_COMMON *cm, int frame_size_override, write_render_size(cm, wb); } -static void write_frame_size_with_refs(AV1_COMP *cpi, +static void write_frame_size_with_refs(const AV1_COMMON *const cm, struct aom_write_bit_buffer *wb) { - AV1_COMMON *const cm = &cpi->common; int found = 0; MV_REFERENCE_FRAME ref_frame; for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) { - YV12_BUFFER_CONFIG *cfg = get_ref_frame_buffer(cpi, ref_frame); + const YV12_BUFFER_CONFIG *cfg = get_ref_frame_yv12_buf(cm, ref_frame); if (cfg != NULL) { found = cm->superres_upscaled_width == cfg->y_crop_width && @@ -2431,9 +2345,7 @@ static void write_color_config(const SequenceHeader *const seq_params, } if (seq_params->color_primaries == AOM_CICP_CP_BT_709 && seq_params->transfer_characteristics == AOM_CICP_TC_SRGB && - seq_params->matrix_coefficients == - AOM_CICP_MC_IDENTITY) { // it would be better to remove this - // dependency too + seq_params->matrix_coefficients == AOM_CICP_MC_IDENTITY) { assert(seq_params->subsampling_x == 0 && seq_params->subsampling_y == 0); assert(seq_params->profile == PROFILE_1 || (seq_params->profile == PROFILE_2 && @@ -2532,34 +2444,27 @@ static void write_tu_pts_info(AV1_COMMON *const cm, cm->buffer_model.frame_presentation_time_length); } -static void write_film_grain_params(AV1_COMP *cpi, +static void write_film_grain_params(const AV1_COMP *const cpi, struct aom_write_bit_buffer *wb) { - AV1_COMMON *const cm = &cpi->common; - aom_film_grain_t *pars = &cm->film_grain_params; - - cm->cur_frame->film_grain_params = *pars; + const AV1_COMMON *const cm = &cpi->common; + const aom_film_grain_t *const pars = &cm->cur_frame->film_grain_params; aom_wb_write_bit(wb, pars->apply_grain); if (!pars->apply_grain) return; aom_wb_write_literal(wb, pars->random_seed, 16); - pars->random_seed += 3381; // Changing random seed for film grain - if (!pars->random_seed) // Random seed should not be zero - pars->random_seed += 7391; - if (cm->frame_type == INTER_FRAME) + if (cm->current_frame.frame_type == INTER_FRAME) aom_wb_write_bit(wb, pars->update_parameters); - else - pars->update_parameters = 1; + if (!pars->update_parameters) { - RefCntBuffer *const frame_bufs = cm->buffer_pool->frame_bufs; - int ref_frame, ref_idx, buf_idx; + int ref_frame, ref_idx; for (ref_frame = LAST_FRAME; ref_frame < REF_FRAMES; ref_frame++) { - ref_idx = get_ref_frame_map_idx(cpi, ref_frame); + ref_idx = get_ref_frame_map_idx(cm, ref_frame); assert(ref_idx != INVALID_IDX); - buf_idx = cm->ref_frame_map[ref_idx]; - if (frame_bufs[buf_idx].film_grain_params_present && - memcmp(pars, &frame_bufs[buf_idx].film_grain_params, sizeof(*pars))) { + const RefCntBuffer *const buf = cm->ref_frame_map[ref_idx]; + if (buf->film_grain_params_present && + av1_check_grain_params_equiv(pars, &buf->film_grain_params)) { break; } } @@ -2575,16 +2480,16 @@ static void write_film_grain_params(AV1_COMP *cpi, aom_wb_write_literal(wb, pars->scaling_points_y[i][1], 8); } - if (!cm->seq_params.monochrome) + if (!cm->seq_params.monochrome) { aom_wb_write_bit(wb, pars->chroma_scaling_from_luma); - else - pars->chroma_scaling_from_luma = 0; // for monochrome override to 0 + } else { + assert(!pars->chroma_scaling_from_luma); + } if (cm->seq_params.monochrome || pars->chroma_scaling_from_luma || ((cm->seq_params.subsampling_x == 1) && (cm->seq_params.subsampling_y == 1) && (pars->num_y_points == 0))) { - pars->num_cb_points = 0; - pars->num_cr_points = 0; + assert(pars->num_cb_points == 0 && pars->num_cr_points == 0); } else { aom_wb_write_literal(wb, pars->num_cb_points, 4); // max 10 for (int i = 0; i < pars->num_cb_points; i++) { @@ -2644,7 +2549,7 @@ static void write_film_grain_params(AV1_COMP *cpi, aom_wb_write_bit(wb, pars->clip_to_restricted_range); } -static void write_sb_size(SequenceHeader *seq_params, +static void write_sb_size(const SequenceHeader *const seq_params, struct aom_write_bit_buffer *wb) { (void)seq_params; (void)wb; @@ -2655,43 +2560,16 @@ static void write_sb_size(SequenceHeader *seq_params, aom_wb_write_bit(wb, seq_params->sb_size == BLOCK_128X128 ? 1 : 0); } -static void write_sequence_header(AV1_COMP *cpi, +static void write_sequence_header(const SequenceHeader *const seq_params, struct aom_write_bit_buffer *wb) { - AV1_COMMON *const cm = &cpi->common; - SequenceHeader *seq_params = &cm->seq_params; + aom_wb_write_literal(wb, seq_params->num_bits_width - 1, 4); + aom_wb_write_literal(wb, seq_params->num_bits_height - 1, 4); + aom_wb_write_literal(wb, seq_params->max_frame_width - 1, + seq_params->num_bits_width); + aom_wb_write_literal(wb, seq_params->max_frame_height - 1, + seq_params->num_bits_height); - int max_frame_width = cpi->oxcf.forced_max_frame_width - ? cpi->oxcf.forced_max_frame_width - : cpi->oxcf.width; - int max_frame_height = cpi->oxcf.forced_max_frame_height - ? cpi->oxcf.forced_max_frame_height - : cpi->oxcf.height; - // max((int)ceil(log2(max_frame_width)), 1) - const int num_bits_width = - (max_frame_width > 1) ? get_msb(max_frame_width - 1) + 1 : 1; - // max((int)ceil(log2(max_frame_height)), 1) - const int num_bits_height = - (max_frame_height > 1) ? get_msb(max_frame_height - 1) + 1 : 1; - assert(num_bits_width <= 16); - assert(num_bits_height <= 16); - - seq_params->num_bits_width = num_bits_width; - seq_params->num_bits_height = num_bits_height; - seq_params->max_frame_width = max_frame_width; - seq_params->max_frame_height = max_frame_height; - - aom_wb_write_literal(wb, num_bits_width - 1, 4); - aom_wb_write_literal(wb, num_bits_height - 1, 4); - aom_wb_write_literal(wb, max_frame_width - 1, num_bits_width); - aom_wb_write_literal(wb, max_frame_height - 1, num_bits_height); - - /* Placeholder for actually writing to the bitstream */ if (!seq_params->reduced_still_picture_hdr) { - seq_params->frame_id_numbers_present_flag = - cm->large_scale_tile ? 0 : cm->error_resilient_mode; - seq_params->frame_id_length = FRAME_ID_LENGTH; - seq_params->delta_frame_id_length = DELTA_FRAME_ID_LENGTH; - aom_wb_write_bit(wb, seq_params->frame_id_numbers_present_flag); if (seq_params->frame_id_numbers_present_flag) { // We must always have delta_frame_id_length < frame_id_length, @@ -2716,11 +2594,11 @@ static void write_sequence_header(AV1_COMP *cpi, aom_wb_write_bit(wb, seq_params->enable_warped_motion); aom_wb_write_bit(wb, seq_params->enable_dual_filter); - aom_wb_write_bit(wb, seq_params->enable_order_hint); + aom_wb_write_bit(wb, seq_params->order_hint_info.enable_order_hint); - if (seq_params->enable_order_hint) { - aom_wb_write_bit(wb, seq_params->enable_jnt_comp); - aom_wb_write_bit(wb, seq_params->enable_ref_frame_mvs); + if (seq_params->order_hint_info.enable_order_hint) { + aom_wb_write_bit(wb, seq_params->order_hint_info.enable_dist_wtd_comp); + aom_wb_write_bit(wb, seq_params->order_hint_info.enable_ref_frame_mvs); } if (seq_params->force_screen_content_tools == 2) { aom_wb_write_bit(wb, 1); @@ -2738,8 +2616,9 @@ static void write_sequence_header(AV1_COMP *cpi, } else { assert(seq_params->force_integer_mv == 2); } - if (seq_params->enable_order_hint) - aom_wb_write_literal(wb, seq_params->order_hint_bits_minus_1, 3); + if (seq_params->order_hint_info.enable_order_hint) + aom_wb_write_literal( + wb, seq_params->order_hint_info.order_hint_bits_minus_1, 3); } aom_wb_write_bit(wb, seq_params->enable_superres); @@ -2815,7 +2694,7 @@ static void write_global_motion(AV1_COMP *cpi, // does not work currently and causes mismatches when resize is on. // Fix it before turning the optimization back on. /* - YV12_BUFFER_CONFIG *ref_buf = get_ref_frame_buffer(cpi, frame); + YV12_BUFFER_CONFIG *ref_buf = get_ref_frame_yv12_buf(cpi, frame); if (cpi->source->y_crop_width == ref_buf->y_crop_width && cpi->source->y_crop_height == ref_buf->y_crop_height) { write_global_motion_params(&cm->global_motion[frame], @@ -2828,7 +2707,7 @@ static void write_global_motion(AV1_COMP *cpi, */ /* printf("Frame %d/%d: Enc Ref %d: %d %d %d %d\n", - cm->current_video_frame, cm->show_frame, frame, + cm->current_frame.frame_number, cm->show_frame, frame, cm->global_motion[frame].wmmat[0], cm->global_motion[frame].wmmat[1], cm->global_motion[frame].wmmat[2], cm->global_motion[frame].wmmat[3]); @@ -2836,23 +2715,22 @@ static void write_global_motion(AV1_COMP *cpi, } } -static void check_frame_refs_short_signaling(AV1_COMP *const cpi) { - AV1_COMMON *const cm = &cpi->common; - if (!cm->frame_refs_short_signaling) return; - +static int check_frame_refs_short_signaling(AV1_COMMON *const cm) { // Check whether all references are distinct frames. - int buf_markers[FRAME_BUFFERS] = { 0 }; - for (int ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) { - const int buf_idx = get_ref_frame_buf_idx(cpi, ref_frame); - if (buf_idx != INVALID_IDX) { - assert(buf_idx >= 0 && buf_idx < FRAME_BUFFERS); - buf_markers[buf_idx] = 1; - } - } - + const RefCntBuffer *seen_bufs[FRAME_BUFFERS] = { NULL }; int num_refs = 0; - for (int buf_idx = 0; buf_idx < FRAME_BUFFERS; ++buf_idx) { - num_refs += buf_markers[buf_idx]; + for (int ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) { + const RefCntBuffer *const buf = get_ref_frame_buf(cm, ref_frame); + if (buf != NULL) { + int seen = 0; + for (int i = 0; i < num_refs; i++) { + if (seen_bufs[i] == buf) { + seen = 1; + break; + } + } + if (!seen) seen_bufs[num_refs++] = buf; + } } // We only turn on frame_refs_short_signaling when all references are @@ -2860,20 +2738,22 @@ static void check_frame_refs_short_signaling(AV1_COMP *const cpi) { if (num_refs < INTER_REFS_PER_FRAME) { // It indicates that there exist more than one reference frame pointing to // the same reference buffer, i.e. two or more references are duplicate. - cm->frame_refs_short_signaling = 0; - return; + return 0; } // Check whether the encoder side ref frame choices are aligned with that to // be derived at the decoder side. - RefBuffer frame_refs_copy[INTER_REFS_PER_FRAME]; + int remapped_ref_idx_copy[REF_FRAMES]; + struct scale_factors ref_scale_factors_copy[REF_FRAMES]; // Backup the frame refs info - memcpy(frame_refs_copy, cm->frame_refs, - INTER_REFS_PER_FRAME * sizeof(RefBuffer)); + memcpy(remapped_ref_idx_copy, cm->remapped_ref_idx, + REF_FRAMES * sizeof(*remapped_ref_idx_copy)); + memcpy(ref_scale_factors_copy, cm->ref_scale_factors, + REF_FRAMES * sizeof(*ref_scale_factors_copy)); - const int lst_map_idx = get_ref_frame_map_idx(cpi, LAST_FRAME); - const int gld_map_idx = get_ref_frame_map_idx(cpi, GOLDEN_FRAME); + const int lst_map_idx = get_ref_frame_map_idx(cm, LAST_FRAME); + const int gld_map_idx = get_ref_frame_map_idx(cm, GOLDEN_FRAME); // Set up the frame refs mapping indexes according to the // frame_refs_short_signaling policy. @@ -2881,32 +2761,39 @@ static void check_frame_refs_short_signaling(AV1_COMP *const cpi) { // We only turn on frame_refs_short_signaling when the encoder side decision // on ref frames is identical to that at the decoder side. + int frame_refs_short_signaling = 1; for (int ref_idx = 0; ref_idx < INTER_REFS_PER_FRAME; ++ref_idx) { // Compare the buffer index between two reference frames indexed // respectively by the encoder and the decoder side decisions. - if (cm->frame_refs[ref_idx].idx != frame_refs_copy[ref_idx].idx) { - cm->frame_refs_short_signaling = 0; + RefCntBuffer *ref_frame_buf_copy = NULL; + if (remapped_ref_idx_copy[ref_idx] != INVALID_IDX) + ref_frame_buf_copy = cm->ref_frame_map[remapped_ref_idx_copy[ref_idx]]; + if (get_ref_frame_buf(cm, LAST_FRAME + ref_idx) != ref_frame_buf_copy) { + frame_refs_short_signaling = 0; break; } } #if 0 // For debug - printf("\nFrame=%d: \n", cm->current_video_frame); - printf("***frame_refs_short_signaling=%d\n", cm->frame_refs_short_signaling); + printf("\nFrame=%d: \n", cm->current_frame.frame_number); + printf("***frame_refs_short_signaling=%d\n", frame_refs_short_signaling); for (int ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) { - printf("enc_ref(map_idx=%d, buf_idx=%d)=%d, vs. " - "dec_ref(map_idx=%d, buf_idx=%d)=%d\n", - get_ref_frame_map_idx(cpi, ref_frame), - get_ref_frame_buf_idx(cpi, ref_frame), ref_frame, - cm->frame_refs[ref_frame - LAST_FRAME].map_idx, - cm->frame_refs[ref_frame - LAST_FRAME].idx, ref_frame); + printf("enc_ref(map_idx=%d)=%d, vs. " + "dec_ref(map_idx=%d)=%d\n", + get_ref_frame_map_idx(cm, ref_frame), ref_frame, + cm->remapped_ref_idx[ref_frame - LAST_FRAME], + ref_frame); } #endif // 0 // Restore the frame refs info if frame_refs_short_signaling is off. - if (!cm->frame_refs_short_signaling) - memcpy(cm->frame_refs, frame_refs_copy, - INTER_REFS_PER_FRAME * sizeof(RefBuffer)); + if (!frame_refs_short_signaling) { + memcpy(cm->remapped_ref_idx, remapped_ref_idx_copy, + REF_FRAMES * sizeof(*remapped_ref_idx_copy)); + memcpy(cm->ref_scale_factors, ref_scale_factors_copy, + REF_FRAMES * sizeof(*ref_scale_factors_copy)); + } + return frame_refs_short_signaling; } // New function based on HLS R18 @@ -2916,28 +2803,15 @@ static void write_uncompressed_header_obu(AV1_COMP *cpi, AV1_COMMON *const cm = &cpi->common; const SequenceHeader *const seq_params = &cm->seq_params; MACROBLOCKD *const xd = &cpi->td.mb.e_mbd; - - // NOTE: By default all coded frames to be used as a reference - cm->is_reference_frame = 1; - cm->frame_type = cm->intra_only ? INTRA_ONLY_FRAME : cm->frame_type; + CurrentFrame *const current_frame = &cm->current_frame; if (seq_params->still_picture) { assert(cm->show_existing_frame == 0); assert(cm->show_frame == 1); - assert(cm->frame_type == KEY_FRAME); + assert(current_frame->frame_type == KEY_FRAME); } if (!seq_params->reduced_still_picture_hdr) { if (encode_show_existing_frame(cm)) { - RefCntBuffer *const frame_bufs = cm->buffer_pool->frame_bufs; - const int frame_to_show = cm->ref_frame_map[cpi->existing_fb_idx_to_show]; - - if (frame_to_show < 0 || frame_bufs[frame_to_show].ref_count < 1) { - aom_internal_error(&cm->error, AOM_CODEC_UNSUP_BITSTREAM, - "Buffer %d does not contain a reconstructed frame", - frame_to_show); - } - ref_cnt_fb(frame_bufs, &cm->new_fb_idx, frame_to_show); - aom_wb_write_bit(wb, 1); // show_existing_frame aom_wb_write_literal(wb, cpi->existing_fb_idx_to_show, 3); @@ -2950,20 +2824,12 @@ static void write_uncompressed_header_obu(AV1_COMP *cpi, int display_frame_id = cm->ref_frame_id[cpi->existing_fb_idx_to_show]; aom_wb_write_literal(wb, display_frame_id, frame_id_len); } - - if (cm->reset_decoder_state && - frame_bufs[frame_to_show].frame_type != KEY_FRAME) { - aom_internal_error( - &cm->error, AOM_CODEC_UNSUP_BITSTREAM, - "show_existing_frame to reset state on KEY_FRAME only"); - } - return; } else { aom_wb_write_bit(wb, 0); // show_existing_frame } - aom_wb_write_literal(wb, cm->frame_type, 2); + aom_wb_write_literal(wb, current_frame->frame_type, 2); aom_wb_write_bit(wb, cm->show_frame); if (cm->show_frame) { @@ -2975,7 +2841,7 @@ static void write_uncompressed_header_obu(AV1_COMP *cpi, } if (frame_is_sframe(cm)) { assert(cm->error_resilient_mode); - } else if (!(cm->frame_type == KEY_FRAME && cm->show_frame)) { + } else if (!(current_frame->frame_type == KEY_FRAME && cm->show_frame)) { aom_wb_write_bit(wb, cm->error_resilient_mode); } } @@ -2998,9 +2864,7 @@ static void write_uncompressed_header_obu(AV1_COMP *cpi, assert(cm->cur_frame_force_integer_mv == 0); } - cm->invalid_delta_frame_id_minus_1 = 0; int frame_size_override_flag = 0; - cm->frame_refs_short_signaling = 0; if (seq_params->reduced_still_picture_hdr) { assert(cm->width == seq_params->max_frame_width && @@ -3023,9 +2887,10 @@ static void write_uncompressed_header_obu(AV1_COMP *cpi, cm->height != seq_params->max_frame_height); if (!frame_is_sframe(cm)) aom_wb_write_bit(wb, frame_size_override_flag); - if (seq_params->enable_order_hint) - aom_wb_write_literal(wb, cm->frame_offset, - seq_params->order_hint_bits_minus_1 + 1); + if (seq_params->order_hint_info.enable_order_hint) + aom_wb_write_literal( + wb, current_frame->order_hint, + seq_params->order_hint_info.order_hint_bits_minus_1 + 1); if (!cm->error_resilient_mode && !frame_is_intra_only(cm)) { aom_wb_write_literal(wb, cm->primary_ref_frame, PRIMARY_REF_BITS); @@ -3058,121 +2923,76 @@ static void write_uncompressed_header_obu(AV1_COMP *cpi, } } } - cpi->refresh_frame_mask = get_refresh_mask(cpi); - if (cm->frame_type == KEY_FRAME) { - if (!cm->show_frame) { // unshown keyframe (forward keyframe) - aom_wb_write_literal(wb, cpi->refresh_frame_mask, REF_FRAMES); - } else { - assert(cpi->refresh_frame_mask == 0xFF); - } - } else { - if (cm->frame_type == INTRA_ONLY_FRAME) { - assert(cpi->refresh_frame_mask != 0xFF); - int updated_fb = -1; - for (int i = 0; i < REF_FRAMES; i++) { - // If more than one frame is refreshed, it doesn't matter which one - // we pick, so pick the first. - if (cpi->refresh_frame_mask & (1 << i)) { - updated_fb = i; - break; - } - } - assert(updated_fb >= 0); - cm->fb_of_context_type[cm->frame_context_idx] = updated_fb; - aom_wb_write_literal(wb, cpi->refresh_frame_mask, REF_FRAMES); - } else if (cm->frame_type == INTER_FRAME || frame_is_sframe(cm)) { - if (cm->frame_type == INTER_FRAME) { - aom_wb_write_literal(wb, cpi->refresh_frame_mask, REF_FRAMES); - } else { - assert(frame_is_sframe(cm) && cpi->refresh_frame_mask == 0xFF); - } - int updated_fb = -1; - for (int i = 0; i < REF_FRAMES; i++) { - // If more than one frame is refreshed, it doesn't matter which one - // we pick, so pick the first. - if (cpi->refresh_frame_mask & (1 << i)) { - updated_fb = i; - break; - } - } - // large scale tile sometimes won't refresh any fbs - if (updated_fb >= 0) { - cm->fb_of_context_type[cm->frame_context_idx] = updated_fb; - } - if (!cpi->refresh_frame_mask) { - // NOTE: "cpi->refresh_frame_mask == 0" indicates that the coded frame - // will not be used as a reference - cm->is_reference_frame = 0; - } - } - } + // Shown keyframes and switch-frames automatically refreshes all reference + // frames. For all other frame types, we need to write refresh_frame_flags. + if ((current_frame->frame_type == KEY_FRAME && !cm->show_frame) || + current_frame->frame_type == INTER_FRAME || + current_frame->frame_type == INTRA_ONLY_FRAME) + aom_wb_write_literal(wb, current_frame->refresh_frame_flags, REF_FRAMES); - if (!frame_is_intra_only(cm) || cpi->refresh_frame_mask != 0xFF) { + if (!frame_is_intra_only(cm) || current_frame->refresh_frame_flags != 0xff) { // Write all ref frame order hints if error_resilient_mode == 1 - if (cm->error_resilient_mode && seq_params->enable_order_hint) { - RefCntBuffer *const frame_bufs = cm->buffer_pool->frame_bufs; + if (cm->error_resilient_mode && + seq_params->order_hint_info.enable_order_hint) { for (int ref_idx = 0; ref_idx < REF_FRAMES; ref_idx++) { - // Get buffer index - const int buf_idx = cm->ref_frame_map[ref_idx]; - assert(buf_idx >= 0 && buf_idx < FRAME_BUFFERS); - - // Write order hint to bit stream - aom_wb_write_literal(wb, frame_bufs[buf_idx].cur_frame_offset, - seq_params->order_hint_bits_minus_1 + 1); + aom_wb_write_literal( + wb, cm->ref_frame_map[ref_idx]->order_hint, + seq_params->order_hint_info.order_hint_bits_minus_1 + 1); } } } - if (cm->frame_type == KEY_FRAME) { + if (current_frame->frame_type == KEY_FRAME) { write_frame_size(cm, frame_size_override_flag, wb); assert(!av1_superres_scaled(cm) || !cm->allow_intrabc); if (cm->allow_screen_content_tools && !av1_superres_scaled(cm)) aom_wb_write_bit(wb, cm->allow_intrabc); - // all eight fbs are refreshed, pick one that will live long enough - cm->fb_of_context_type[REGULAR_FRAME] = 0; } else { - if (cm->frame_type == INTRA_ONLY_FRAME) { + if (current_frame->frame_type == INTRA_ONLY_FRAME) { write_frame_size(cm, frame_size_override_flag, wb); assert(!av1_superres_scaled(cm) || !cm->allow_intrabc); if (cm->allow_screen_content_tools && !av1_superres_scaled(cm)) aom_wb_write_bit(wb, cm->allow_intrabc); - } else if (cm->frame_type == INTER_FRAME || frame_is_sframe(cm)) { + } else if (current_frame->frame_type == INTER_FRAME || + frame_is_sframe(cm)) { MV_REFERENCE_FRAME ref_frame; // NOTE: Error resilient mode turns off frame_refs_short_signaling // automatically. + int frame_refs_short_signaling = 0; #define FRAME_REFS_SHORT_SIGNALING 0 #if FRAME_REFS_SHORT_SIGNALING - cm->frame_refs_short_signaling = seq_params->enable_order_hint; + frame_refs_short_signaling = + seq_params->order_hint_info.enable_order_hint; #endif // FRAME_REFS_SHORT_SIGNALING - if (cm->frame_refs_short_signaling) { + if (frame_refs_short_signaling) { // NOTE(zoeliu@google.com): // An example solution for encoder-side implementation on frame refs // short signaling, which is only turned on when the encoder side // decision on ref frames is identical to that at the decoder side. - check_frame_refs_short_signaling(cpi); + frame_refs_short_signaling = check_frame_refs_short_signaling(cm); } - if (seq_params->enable_order_hint) - aom_wb_write_bit(wb, cm->frame_refs_short_signaling); + if (seq_params->order_hint_info.enable_order_hint) + aom_wb_write_bit(wb, frame_refs_short_signaling); - if (cm->frame_refs_short_signaling) { - const int lst_ref = get_ref_frame_map_idx(cpi, LAST_FRAME); + if (frame_refs_short_signaling) { + const int lst_ref = get_ref_frame_map_idx(cm, LAST_FRAME); aom_wb_write_literal(wb, lst_ref, REF_FRAMES_LOG2); - const int gld_ref = get_ref_frame_map_idx(cpi, GOLDEN_FRAME); + const int gld_ref = get_ref_frame_map_idx(cm, GOLDEN_FRAME); aom_wb_write_literal(wb, gld_ref, REF_FRAMES_LOG2); } for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) { - assert(get_ref_frame_map_idx(cpi, ref_frame) != INVALID_IDX); - if (!cm->frame_refs_short_signaling) - aom_wb_write_literal(wb, get_ref_frame_map_idx(cpi, ref_frame), + assert(get_ref_frame_map_idx(cm, ref_frame) != INVALID_IDX); + if (!frame_refs_short_signaling) + aom_wb_write_literal(wb, get_ref_frame_map_idx(cm, ref_frame), REF_FRAMES_LOG2); if (seq_params->frame_id_numbers_present_flag) { - int i = get_ref_frame_map_idx(cpi, ref_frame); + int i = get_ref_frame_map_idx(cm, ref_frame); int frame_id_len = seq_params->frame_id_length; int diff_len = seq_params->delta_frame_id_length; int delta_frame_id_minus_1 = @@ -3181,24 +3001,22 @@ static void write_uncompressed_header_obu(AV1_COMP *cpi, (1 << frame_id_len)) - 1; if (delta_frame_id_minus_1 < 0 || - delta_frame_id_minus_1 >= (1 << diff_len)) - cm->invalid_delta_frame_id_minus_1 = 1; + delta_frame_id_minus_1 >= (1 << diff_len)) { + aom_internal_error(&cpi->common.error, AOM_CODEC_ERROR, + "Invalid delta_frame_id_minus_1"); + } aom_wb_write_literal(wb, delta_frame_id_minus_1, diff_len); } } if (!cm->error_resilient_mode && frame_size_override_flag) { - write_frame_size_with_refs(cpi, wb); + write_frame_size_with_refs(cm, wb); } else { write_frame_size(cm, frame_size_override_flag, wb); } - if (cm->cur_frame_force_integer_mv) { - cm->allow_high_precision_mv = 0; - } else { + if (!cm->cur_frame_force_integer_mv) aom_wb_write_bit(wb, cm->allow_high_precision_mv); - } - fix_interp_filter(cm, cpi->td.counts); write_frame_interp_filter(cm->interp_filter, wb); aom_wb_write_bit(wb, cm->switchable_motion_mode); if (frame_might_allow_ref_frame_mvs(cm)) { @@ -3212,7 +3030,7 @@ static void write_uncompressed_header_obu(AV1_COMP *cpi, const int might_bwd_adapt = !(seq_params->reduced_still_picture_hdr) && !(cm->disable_cdf_update); if (cm->large_scale_tile) - cm->refresh_frame_context = REFRESH_FRAME_CONTEXT_DISABLED; + assert(cm->refresh_frame_context == REFRESH_FRAME_CONTEXT_DISABLED); if (might_bwd_adapt) { aom_wb_write_bit( @@ -3223,19 +3041,20 @@ static void write_uncompressed_header_obu(AV1_COMP *cpi, encode_quantization(cm, wb); encode_segmentation(cm, xd, wb); - if (cm->delta_q_present_flag) assert(cm->base_qindex > 0); + const DeltaQInfo *const delta_q_info = &cm->delta_q_info; + if (delta_q_info->delta_q_present_flag) assert(cm->base_qindex > 0); if (cm->base_qindex > 0) { - aom_wb_write_bit(wb, cm->delta_q_present_flag); - if (cm->delta_q_present_flag) { - aom_wb_write_literal(wb, get_msb(cm->delta_q_res), 2); + aom_wb_write_bit(wb, delta_q_info->delta_q_present_flag); + if (delta_q_info->delta_q_present_flag) { + aom_wb_write_literal(wb, get_msb(delta_q_info->delta_q_res), 2); xd->current_qindex = cm->base_qindex; if (cm->allow_intrabc) - assert(cm->delta_lf_present_flag == 0); + assert(delta_q_info->delta_lf_present_flag == 0); else - aom_wb_write_bit(wb, cm->delta_lf_present_flag); - if (cm->delta_lf_present_flag) { - aom_wb_write_literal(wb, get_msb(cm->delta_lf_res), 2); - aom_wb_write_bit(wb, cm->delta_lf_multi); + aom_wb_write_bit(wb, delta_q_info->delta_lf_present_flag); + if (delta_q_info->delta_lf_present_flag) { + aom_wb_write_literal(wb, get_msb(delta_q_info->delta_lf_res), 2); + aom_wb_write_bit(wb, delta_q_info->delta_lf_multi); av1_reset_loop_filter_delta(xd, av1_num_planes(cm)); } } @@ -3251,15 +3070,21 @@ static void write_uncompressed_header_obu(AV1_COMP *cpi, encode_restoration_mode(cm, wb); } - write_tx_mode(cm, &cm->tx_mode, wb); + // Write TX mode + if (cm->coded_lossless) + assert(cm->tx_mode == ONLY_4X4); + else + aom_wb_write_bit(wb, cm->tx_mode == TX_MODE_SELECT); - if (cpi->allow_comp_inter_inter) { - const int use_hybrid_pred = cm->reference_mode == REFERENCE_MODE_SELECT; + if (!frame_is_intra_only(cm)) { + const int use_hybrid_pred = + current_frame->reference_mode == REFERENCE_MODE_SELECT; aom_wb_write_bit(wb, use_hybrid_pred); } - if (cm->is_skip_mode_allowed) aom_wb_write_bit(wb, cm->skip_mode_flag); + if (current_frame->skip_mode_info.skip_mode_allowed) + aom_wb_write_bit(wb, current_frame->skip_mode_info.skip_mode_flag); if (frame_might_allow_warped_motion(cm)) aom_wb_write_bit(wb, cm->allow_warped_motion); @@ -3271,19 +3096,9 @@ static void write_uncompressed_header_obu(AV1_COMP *cpi, if (!frame_is_intra_only(cm)) write_global_motion(cpi, wb); if (seq_params->film_grain_params_present && - (cm->show_frame || cm->showable_frame)) { - int flip_back_update_parameters_flag = 0; - if (cm->frame_type != INTER_FRAME && - cm->film_grain_params.update_parameters == 0) { - cm->film_grain_params.update_parameters = 1; - flip_back_update_parameters_flag = 1; - } + (cm->show_frame || cm->showable_frame)) write_film_grain_params(cpi, wb); - if (flip_back_update_parameters_flag) - cm->film_grain_params.update_parameters = 0; - } - if (cm->large_scale_tile) write_ext_tile_info(cm, saved_wb, wb); } @@ -3538,7 +3353,7 @@ uint32_t write_sequence_header_obu(AV1_COMP *cpi, uint8_t *const dst) { } } } - write_sequence_header(cpi, &wb); + write_sequence_header(&cm->seq_params, &wb); write_color_config(&cm->seq_params, &wb); @@ -3560,8 +3375,8 @@ static uint32_t write_frame_header_obu(AV1_COMP *cpi, return aom_wb_bytes_written(&wb); } -static uint32_t write_tile_group_header(uint8_t *const dst, int startTile, - int endTile, int tiles_log2, +static uint32_t write_tile_group_header(uint8_t *const dst, int start_tile, + int end_tile, int tiles_log2, int tile_start_and_end_present_flag) { struct aom_write_bit_buffer wb = { dst, 0 }; uint32_t size = 0; @@ -3571,8 +3386,8 @@ static uint32_t write_tile_group_header(uint8_t *const dst, int startTile, aom_wb_write_bit(&wb, tile_start_and_end_present_flag); if (tile_start_and_end_present_flag) { - aom_wb_write_literal(&wb, startTile, tiles_log2); - aom_wb_write_literal(&wb, endTile, tiles_log2); + aom_wb_write_literal(&wb, start_tile, tiles_log2); + aom_wb_write_literal(&wb, end_tile, tiles_log2); } size = aom_wb_bytes_written(&wb); @@ -3588,11 +3403,13 @@ typedef struct { static uint32_t write_tiles_in_tg_obus(AV1_COMP *const cpi, uint8_t *const dst, struct aom_write_bit_buffer *saved_wb, uint8_t obu_extension_header, - const FrameHeaderInfo *fh_info) { + const FrameHeaderInfo *fh_info, + int *const largest_tile_id) { AV1_COMMON *const cm = &cpi->common; aom_writer mode_bc; int tile_row, tile_col; - TileBufferEnc(*const tile_buffers)[MAX_TILE_COLS] = cpi->tile_buffers; + // Store the location and size of each tile's data in the bitstream: + TileBufferEnc tile_buffers[MAX_TILE_ROWS][MAX_TILE_COLS]; uint32_t total_size = 0; const int tile_cols = cm->tile_cols; const int tile_rows = cm->tile_rows; @@ -3613,7 +3430,7 @@ static uint32_t write_tiles_in_tg_obus(AV1_COMP *const cpi, uint8_t *const dst, const int have_tiles = tile_cols * tile_rows > 1; int first_tg = 1; - cm->largest_tile_id = 0; + *largest_tile_id = 0; if (cm->large_scale_tile) { // For large_scale_tile case, we always have only one tile group, so it can @@ -3631,9 +3448,9 @@ static uint32_t write_tiles_in_tg_obus(AV1_COMP *const cpi, uint8_t *const dst, #if EXT_TILE_DEBUG { char fn[20] = "./fh"; - fn[4] = cm->current_video_frame / 100 + '0'; - fn[5] = (cm->current_video_frame % 100) / 10 + '0'; - fn[6] = (cm->current_video_frame % 10) + '0'; + fn[4] = cm->current_frame.frame_number / 100 + '0'; + fn[5] = (cm->current_frame.frame_number % 100) / 10 + '0'; + fn[6] = (cm->current_frame.frame_number % 10) + '0'; fn[7] = '\0'; av1_print_uncompressed_frame_header(data - frame_header_size, frame_header_size, fn); @@ -3666,8 +3483,6 @@ static uint32_t write_tiles_in_tg_obus(AV1_COMP *const cpi, uint8_t *const dst, // Is CONFIG_EXT_TILE = 1, every tile in the row has a header, // even for the last one, unless no tiling is used at all. total_size += data_offset; - // Initialise tile context from the frame context - this_tile->tctx = *cm->fc; cpi->td.mb.e_mbd.tile_ctx = &this_tile->tctx; mode_bc.allow_update_cdf = !cm->large_scale_tile; mode_bc.allow_update_cdf = @@ -3681,7 +3496,7 @@ static uint32_t write_tiles_in_tg_obus(AV1_COMP *const cpi, uint8_t *const dst, // Record the maximum tile size we see, so we can compact headers later. if (tile_size > max_tile_size) { max_tile_size = tile_size; - cm->largest_tile_id = tile_cols * tile_row + tile_col; + *largest_tile_id = tile_cols * tile_row + tile_col; } if (have_tiles) { @@ -3695,10 +3510,13 @@ static uint32_t write_tiles_in_tg_obus(AV1_COMP *const cpi, uint8_t *const dst, // If tile_copy_mode = 1, check if this tile is a copy tile. // Very low chances to have copy tiles on the key frames, so don't // search on key frames to reduce unnecessary search. - if (cm->frame_type != KEY_FRAME && tile_copy_mode) { + if (cm->current_frame.frame_type != KEY_FRAME && tile_copy_mode) { const int identical_tile_offset = find_identical_tile(tile_row, tile_col, tile_buffers); + // Indicate a copy-tile by setting the most significant bit. + // The row-offset to copy from is stored in the highest byte. + // remux_tiles will move these around later if (identical_tile_offset > 0) { tile_size = 0; tile_header = identical_tile_offset | 0x80; @@ -3804,8 +3622,6 @@ static uint32_t write_tiles_in_tg_obus(AV1_COMP *const cpi, uint8_t *const dst, // The last tile of the tile group does not have a header. if (!is_last_tile_in_tg) total_size += 4; - // Initialise tile context from the frame context - this_tile->tctx = *cm->fc; cpi->td.mb.e_mbd.tile_ctx = &this_tile->tctx; mode_bc.allow_update_cdf = 1; mode_bc.allow_update_cdf = @@ -3822,7 +3638,7 @@ static uint32_t write_tiles_in_tg_obus(AV1_COMP *const cpi, uint8_t *const dst, curr_tg_data_size += (tile_size + (is_last_tile_in_tg ? 0 : 4)); buf->size = tile_size; if (tile_size > max_tile_size) { - cm->largest_tile_id = tile_cols * tile_row + tile_col; + *largest_tile_id = tile_cols * tile_row + tile_col; max_tile_size = tile_size; } @@ -3857,7 +3673,7 @@ static uint32_t write_tiles_in_tg_obus(AV1_COMP *const cpi, uint8_t *const dst, // Force context update tile to be the first tile in error // resiliant mode as the duplicate frame headers will have // context_update_tile_id set to 0 - cm->largest_tile_id = 0; + *largest_tile_id = 0; // Rewrite the OBU header to change the OBU type to Redundant Frame // Header. @@ -3880,7 +3696,7 @@ static uint32_t write_tiles_in_tg_obus(AV1_COMP *const cpi, uint8_t *const dst, // Fill in context_update_tile_id indicating the tile to use for the // cdf update. The encoder currently sets it to the largest tile // (but is up to the encoder) - aom_wb_overwrite_literal(saved_wb, cm->largest_tile_id, + aom_wb_overwrite_literal(saved_wb, *largest_tile_id, cm->log2_tile_cols + cm->log2_tile_rows); // If more than one tile group. tile_size_bytes takes the default value 4 // and does not need to be set. For a single tile group it is set in the @@ -3926,7 +3742,8 @@ static uint32_t write_tiles_in_tg_obus(AV1_COMP *const cpi, uint8_t *const dst, return total_size; } -int av1_pack_bitstream(AV1_COMP *const cpi, uint8_t *dst, size_t *size) { +int av1_pack_bitstream(AV1_COMP *const cpi, uint8_t *dst, size_t *size, + int *const largest_tile_id) { uint8_t *data = dst; uint32_t data_size; AV1_COMMON *const cm = &cpi->common; @@ -3943,7 +3760,7 @@ int av1_pack_bitstream(AV1_COMP *const cpi, uint8_t *dst, size_t *size) { // The TD is now written outside the frame encode loop // write sequence header obu if KEY_FRAME, preceded by 4-byte size - if (cm->frame_type == KEY_FRAME && cm->show_frame) { + if (cm->current_frame.frame_type == KEY_FRAME && cm->show_frame) { obu_header_size = write_obu_header(OBU_SEQUENCE_HEADER, 0, data); obu_payload_size = write_sequence_header_obu(cpi, data + obu_header_size); @@ -3990,8 +3807,8 @@ int av1_pack_bitstream(AV1_COMP *const cpi, uint8_t *dst, size_t *size) { } else { // Each tile group obu will be preceded by 4-byte size of the tile group // obu - data_size = write_tiles_in_tg_obus(cpi, data, &saved_wb, - obu_extension_header, &fh_info); + data_size = write_tiles_in_tg_obus( + cpi, data, &saved_wb, obu_extension_header, &fh_info, largest_tile_id); } data += data_size; *size = data - dst; diff --git a/third_party/aom/av1/encoder/bitstream.h b/third_party/aom/av1/encoder/bitstream.h index 465ccaed57..a567086e95 100644 --- a/third_party/aom/av1/encoder/bitstream.h +++ b/third_party/aom/av1/encoder/bitstream.h @@ -33,9 +33,10 @@ uint32_t write_obu_header(OBU_TYPE obu_type, int obu_extension, int write_uleb_obu_size(uint32_t obu_header_size, uint32_t obu_payload_size, uint8_t *dest); -int av1_pack_bitstream(AV1_COMP *const cpi, uint8_t *dest, size_t *size); +int av1_pack_bitstream(AV1_COMP *const cpi, uint8_t *dst, size_t *size, + int *const largest_tile_id); -static INLINE int av1_preserve_existing_gf(AV1_COMP *cpi) { +static INLINE int av1_preserve_existing_gf(const AV1_COMP *const cpi) { // Do not swap gf and arf indices for internal overlay frames return cpi->rc.is_src_frame_alt_ref && !cpi->rc.is_src_frame_ext_arf; } diff --git a/third_party/aom/av1/encoder/block.h b/third_party/aom/av1/encoder/block.h index 0bc5dea825..87761e9065 100644 --- a/third_party/aom/av1/encoder/block.h +++ b/third_party/aom/av1/encoder/block.h @@ -54,10 +54,10 @@ typedef struct macroblock_plane { typedef struct { int txb_skip_cost[TXB_SKIP_CONTEXTS][2]; int base_eob_cost[SIG_COEF_CONTEXTS_EOB][3]; - int base_cost[SIG_COEF_CONTEXTS][4]; + int base_cost[SIG_COEF_CONTEXTS][8]; int eob_extra_cost[EOB_COEF_CONTEXTS][2]; int dc_sign_cost[DC_SIGN_CONTEXTS][2]; - int lps_cost[LEVEL_CONTEXTS][COEFF_BASE_RANGE + 1]; + int lps_cost[LEVEL_CONTEXTS][COEFF_BASE_RANGE + 1 + COEFF_BASE_RANGE + 1]; } LV_MAP_COEFF_COST; typedef struct { @@ -74,16 +74,16 @@ typedef struct { } CB_COEFF_BUFFER; typedef struct { - int16_t mode_context[MODE_CTX_REF_FRAMES]; // TODO(angiebird): Reduce the buffer size according to sb_type tran_low_t *tcoeff[MAX_MB_PLANE]; uint16_t *eobs[MAX_MB_PLANE]; uint8_t *txb_skip_ctx[MAX_MB_PLANE]; int *dc_sign_ctx[MAX_MB_PLANE]; - uint8_t ref_mv_count[MODE_CTX_REF_FRAMES]; CANDIDATE_MV ref_mv_stack[MODE_CTX_REF_FRAMES][MAX_REF_MV_STACK_SIZE]; int_mv global_mvs[REF_FRAMES]; int16_t compound_mode_context[MODE_CTX_REF_FRAMES]; + int16_t mode_context[MODE_CTX_REF_FRAMES]; + uint8_t ref_mv_count[MODE_CTX_REF_FRAMES]; } MB_MODE_INFO_EXT; typedef struct { @@ -140,6 +140,20 @@ typedef struct tx_size_rd_info_node { struct tx_size_rd_info_node *children[4]; } TXB_RD_INFO_NODE; +// Simple translation rd state for prune_comp_search_by_single_result +typedef struct { + RD_STATS rd_stats; + RD_STATS rd_stats_y; + RD_STATS rd_stats_uv; + uint8_t blk_skip[MAX_MIB_SIZE * MAX_MIB_SIZE]; + uint8_t skip; + uint8_t disable_skip; + uint8_t early_skipped; +} SimpleRDState; + +// 4: NEAREST, NEW, NEAR, GLOBAL +#define SINGLE_REF_MODES ((REF_FRAMES - 1) * 4) + // Region size for mode decision sampling in the first pass of partition // search(two_pass_partition_search speed feature), in units of mi size(4). // Used by the mode_pruning_based_on_two_pass_partition_search speed feature. @@ -173,6 +187,19 @@ typedef struct { COMPOUND_TYPE comp_type; } INTERPOLATION_FILTER_STATS; +#define MAX_COMP_RD_STATS 64 +typedef struct { + int32_t rate[COMPOUND_TYPES]; + int64_t dist[COMPOUND_TYPES]; + int_mv mv[2]; + MV_REFERENCE_FRAME ref_frames[2]; + PREDICTION_MODE mode; + InterpFilters filter; + int ref_mv_idx; + int is_global[2]; +} COMP_RD_STATS; + +struct inter_modes_info; typedef struct macroblock MACROBLOCK; struct macroblock { struct macroblock_plane plane[MAX_MB_PLANE]; @@ -194,6 +221,9 @@ struct macroblock { INTERPOLATION_FILTER_STATS interp_filter_stats[2][MAX_INTERP_FILTER_STATS]; int interp_filter_stats_idx[2]; + // prune_comp_search_by_single_result (3:MAX_REF_MV_SERCH) + SimpleRDState simple_rd_state[SINGLE_REF_MODES][3]; + // Activate constrained coding block partition search range. int use_cb_search_range; @@ -224,12 +254,16 @@ struct macroblock { // for sub-8x8 blocks. int sadperbit4; int rdmult; + int cb_rdmult; int mb_energy; int sb_energy_level; int *m_search_count_ptr; int *ex_search_count_ptr; unsigned int txb_split_count; +#if CONFIG_SPEED_STATS + unsigned int tx_search_count; +#endif // CONFIG_SPEED_STATS // These are set to their default values at the beginning, and then adjusted // further in the encoding process. @@ -241,12 +275,10 @@ struct macroblock { unsigned int pred_sse[REF_FRAMES]; int pred_mv_sad[REF_FRAMES]; - int *nmvjointcost; int nmv_vec_cost[MV_JOINTS]; int *nmvcost[2]; int *nmvcost_hp[2]; int **mv_cost_stack; - int **mvcost; int32_t *wsrc_buf; int32_t *mask_buf; @@ -258,6 +290,16 @@ struct macroblock { CONV_BUF_TYPE *tmp_conv_dst; uint8_t *tmp_obmc_bufs[2]; + FRAME_CONTEXT *row_ctx; + // This context will be used to update color_map_cdf pointer which would be + // used during pack bitstream. For single thread and tile-multithreading case + // this ponter will be same as xd->tile_ctx, but for the case of row-mt: + // xd->tile_ctx will point to a temporary context while tile_pb_ctx will point + // to the accurate tile context. + FRAME_CONTEXT *tile_pb_ctx; + + struct inter_modes_info *inter_modes_info; + // buffer for hash value calculation of a block // used only in av1_get_block_hash_value() // [first hash/second hash] @@ -351,6 +393,9 @@ struct macroblock { // Store the second best motion vector during full-pixel motion search int_mv second_best_mv; + // Store the fractional best motion vector during sub/Qpel-pixel motion search + int_mv fractional_best_mv[3]; + // use default transform and skip transform type search for intra modes int use_default_intra_tx_type; // use default transform and skip transform type search for inter modes @@ -367,6 +412,14 @@ struct macroblock { int tx_split_prune_flag; // Flag to skip tx split RD search. int recalc_luma_mc_data; // Flag to indicate recalculation of MC data during // interpolation filter search + // The likelihood of an edge existing in the block (using partial Canny edge + // detection). For reference, 556 is the value returned for a solid + // vertical black/white edge. + uint16_t edge_strength; + + // [Saved stat index] + COMP_RD_STATS comp_rd_stats[MAX_COMP_RD_STATS]; + int comp_rd_stats_idx; }; static INLINE int is_rect_tx_allowed_bsize(BLOCK_SIZE bsize) { diff --git a/third_party/aom/av1/encoder/context_tree.c b/third_party/aom/av1/encoder/context_tree.c index 57f59f304b..40df6c1dae 100644 --- a/third_party/aom/av1/encoder/context_tree.c +++ b/third_party/aom/av1/encoder/context_tree.c @@ -16,8 +16,15 @@ static const BLOCK_SIZE square[MAX_SB_SIZE_LOG2 - 1] = { BLOCK_4X4, BLOCK_8X8, BLOCK_16X16, BLOCK_32X32, BLOCK_64X64, BLOCK_128X128, }; +typedef struct { + tran_low_t *coeff_buf[MAX_MB_PLANE]; + tran_low_t *qcoeff_buf[MAX_MB_PLANE]; + tran_low_t *dqcoeff_buf[MAX_MB_PLANE]; +} PC_TREE_SHARED_BUFFERS; + static void alloc_mode_context(AV1_COMMON *cm, int num_pix, - PICK_MODE_CONTEXT *ctx) { + PICK_MODE_CONTEXT *ctx, + PC_TREE_SHARED_BUFFERS *shared_bufs) { const int num_planes = av1_num_planes(cm); int i; const int num_blk = num_pix / 16; @@ -25,12 +32,9 @@ static void alloc_mode_context(AV1_COMMON *cm, int num_pix, CHECK_MEM_ERROR(cm, ctx->blk_skip, aom_calloc(num_blk, sizeof(uint8_t))); for (i = 0; i < num_planes; ++i) { - CHECK_MEM_ERROR(cm, ctx->coeff[i], - aom_memalign(32, num_pix * sizeof(*ctx->coeff[i]))); - CHECK_MEM_ERROR(cm, ctx->qcoeff[i], - aom_memalign(32, num_pix * sizeof(*ctx->qcoeff[i]))); - CHECK_MEM_ERROR(cm, ctx->dqcoeff[i], - aom_memalign(32, num_pix * sizeof(*ctx->dqcoeff[i]))); + ctx->coeff[i] = shared_bufs->coeff_buf[i]; + ctx->qcoeff[i] = shared_bufs->qcoeff_buf[i]; + ctx->dqcoeff[i] = shared_bufs->dqcoeff_buf[i]; CHECK_MEM_ERROR(cm, ctx->eobs[i], aom_memalign(32, num_blk * sizeof(*ctx->eobs[i]))); CHECK_MEM_ERROR( @@ -52,11 +56,8 @@ static void free_mode_context(PICK_MODE_CONTEXT *ctx, const int num_planes) { aom_free(ctx->blk_skip); ctx->blk_skip = 0; for (i = 0; i < num_planes; ++i) { - aom_free(ctx->coeff[i]); ctx->coeff[i] = 0; - aom_free(ctx->qcoeff[i]); ctx->qcoeff[i] = 0; - aom_free(ctx->dqcoeff[i]); ctx->dqcoeff[i] = 0; aom_free(ctx->eobs[i]); ctx->eobs[i] = 0; @@ -71,36 +72,37 @@ static void free_mode_context(PICK_MODE_CONTEXT *ctx, const int num_planes) { } static void alloc_tree_contexts(AV1_COMMON *cm, PC_TREE *tree, int num_pix, - int is_leaf) { - alloc_mode_context(cm, num_pix, &tree->none); + int is_leaf, + PC_TREE_SHARED_BUFFERS *shared_bufs) { + alloc_mode_context(cm, num_pix, &tree->none, shared_bufs); if (is_leaf) return; - alloc_mode_context(cm, num_pix / 2, &tree->horizontal[0]); - alloc_mode_context(cm, num_pix / 2, &tree->vertical[0]); + alloc_mode_context(cm, num_pix / 2, &tree->horizontal[0], shared_bufs); + alloc_mode_context(cm, num_pix / 2, &tree->vertical[0], shared_bufs); - alloc_mode_context(cm, num_pix / 2, &tree->horizontal[1]); - alloc_mode_context(cm, num_pix / 2, &tree->vertical[1]); + alloc_mode_context(cm, num_pix / 2, &tree->horizontal[1], shared_bufs); + alloc_mode_context(cm, num_pix / 2, &tree->vertical[1], shared_bufs); - alloc_mode_context(cm, num_pix / 4, &tree->horizontala[0]); - alloc_mode_context(cm, num_pix / 4, &tree->horizontala[1]); - alloc_mode_context(cm, num_pix / 2, &tree->horizontala[2]); + alloc_mode_context(cm, num_pix / 4, &tree->horizontala[0], shared_bufs); + alloc_mode_context(cm, num_pix / 4, &tree->horizontala[1], shared_bufs); + alloc_mode_context(cm, num_pix / 2, &tree->horizontala[2], shared_bufs); - alloc_mode_context(cm, num_pix / 2, &tree->horizontalb[0]); - alloc_mode_context(cm, num_pix / 4, &tree->horizontalb[1]); - alloc_mode_context(cm, num_pix / 4, &tree->horizontalb[2]); + alloc_mode_context(cm, num_pix / 2, &tree->horizontalb[0], shared_bufs); + alloc_mode_context(cm, num_pix / 4, &tree->horizontalb[1], shared_bufs); + alloc_mode_context(cm, num_pix / 4, &tree->horizontalb[2], shared_bufs); - alloc_mode_context(cm, num_pix / 4, &tree->verticala[0]); - alloc_mode_context(cm, num_pix / 4, &tree->verticala[1]); - alloc_mode_context(cm, num_pix / 2, &tree->verticala[2]); + alloc_mode_context(cm, num_pix / 4, &tree->verticala[0], shared_bufs); + alloc_mode_context(cm, num_pix / 4, &tree->verticala[1], shared_bufs); + alloc_mode_context(cm, num_pix / 2, &tree->verticala[2], shared_bufs); - alloc_mode_context(cm, num_pix / 2, &tree->verticalb[0]); - alloc_mode_context(cm, num_pix / 4, &tree->verticalb[1]); - alloc_mode_context(cm, num_pix / 4, &tree->verticalb[2]); + alloc_mode_context(cm, num_pix / 2, &tree->verticalb[0], shared_bufs); + alloc_mode_context(cm, num_pix / 4, &tree->verticalb[1], shared_bufs); + alloc_mode_context(cm, num_pix / 4, &tree->verticalb[2], shared_bufs); for (int i = 0; i < 4; ++i) { - alloc_mode_context(cm, num_pix / 4, &tree->horizontal4[i]); - alloc_mode_context(cm, num_pix / 4, &tree->vertical4[i]); + alloc_mode_context(cm, num_pix / 4, &tree->horizontal4[i], shared_bufs); + alloc_mode_context(cm, num_pix / 4, &tree->vertical4[i], shared_bufs); } } @@ -135,6 +137,7 @@ void av1_setup_pc_tree(AV1_COMMON *cm, ThreadData *td) { const int tree_nodes = tree_nodes_inc + 256 + 64 + 16 + 4 + 1; int pc_tree_index = 0; PC_TREE *this_pc; + PC_TREE_SHARED_BUFFERS shared_bufs; int square_index = 1; int nodes; @@ -143,11 +146,24 @@ void av1_setup_pc_tree(AV1_COMMON *cm, ThreadData *td) { aom_calloc(tree_nodes, sizeof(*td->pc_tree))); this_pc = &td->pc_tree[0]; + for (i = 0; i < 3; i++) { + const int max_num_pix = MAX_SB_SIZE * MAX_SB_SIZE; + CHECK_MEM_ERROR(cm, td->tree_coeff_buf[i], + aom_memalign(32, max_num_pix * sizeof(tran_low_t))); + CHECK_MEM_ERROR(cm, td->tree_qcoeff_buf[i], + aom_memalign(32, max_num_pix * sizeof(tran_low_t))); + CHECK_MEM_ERROR(cm, td->tree_dqcoeff_buf[i], + aom_memalign(32, max_num_pix * sizeof(tran_low_t))); + shared_bufs.coeff_buf[i] = td->tree_coeff_buf[i]; + shared_bufs.qcoeff_buf[i] = td->tree_qcoeff_buf[i]; + shared_bufs.dqcoeff_buf[i] = td->tree_dqcoeff_buf[i]; + } + // Sets up all the leaf nodes in the tree. for (pc_tree_index = 0; pc_tree_index < leaf_nodes; ++pc_tree_index) { PC_TREE *const tree = &td->pc_tree[pc_tree_index]; tree->block_size = square[0]; - alloc_tree_contexts(cm, tree, 16, 1); + alloc_tree_contexts(cm, tree, 16, 1, &shared_bufs); } // Each node has 4 leaf nodes, fill each block_size level of the tree @@ -155,7 +171,7 @@ void av1_setup_pc_tree(AV1_COMMON *cm, ThreadData *td) { for (nodes = leaf_nodes >> 2; nodes > 0; nodes >>= 2) { for (i = 0; i < nodes; ++i) { PC_TREE *const tree = &td->pc_tree[pc_tree_index]; - alloc_tree_contexts(cm, tree, 16 << (2 * square_index), 0); + alloc_tree_contexts(cm, tree, 16 << (2 * square_index), 0, &shared_bufs); tree->block_size = square[square_index]; for (j = 0; j < 4; j++) tree->split[j] = this_pc++; ++pc_tree_index; @@ -181,6 +197,14 @@ void av1_free_pc_tree(ThreadData *td, const int num_planes) { for (int i = 0; i < tree_nodes; ++i) { free_tree_contexts(&td->pc_tree[i], num_planes); } + for (int i = 0; i < 3; ++i) { + aom_free(td->tree_coeff_buf[i]); + aom_free(td->tree_qcoeff_buf[i]); + aom_free(td->tree_dqcoeff_buf[i]); + td->tree_coeff_buf[i] = NULL; + td->tree_qcoeff_buf[i] = NULL; + td->tree_dqcoeff_buf[i] = NULL; + } aom_free(td->pc_tree); td->pc_tree = NULL; } diff --git a/third_party/aom/av1/encoder/context_tree.h b/third_party/aom/av1/encoder/context_tree.h index 4efc349852..66f24f958e 100644 --- a/third_party/aom/av1/encoder/context_tree.h +++ b/third_party/aom/av1/encoder/context_tree.h @@ -23,7 +23,7 @@ struct AV1_COMP; struct AV1Common; struct ThreadData; -typedef enum { +enum { // Search all the partition types in this plane. SEARCH_FULL_PLANE = 0, // Only search none_partition coding block. @@ -32,12 +32,14 @@ typedef enum { SEARCH_SAME_PLANE = 2, // Skip search partition on this plane. Go split directly. SPLIT_PLANE = 3, -} CB_TREE_SEARCH; +} UENUM1BYTE(CB_TREE_SEARCH); // Structure to hold snapshot of coding context during the mode picking process typedef struct { MB_MODE_INFO mic; MB_MODE_INFO_EXT mbmi_ext; + int64_t dist; + int64_t rdcost; uint8_t *color_index_map[2]; uint8_t *blk_skip; @@ -62,30 +64,42 @@ typedef struct { // TODO(jingning) Use RD_COST struct here instead. This involves a boarder // scope of refactoring. int rate; - int64_t dist; - int64_t rdcost; + int rd_mode_is_ready; // Flag to indicate whether rd pick mode decision has // been made. + int mode_selected; +#if CONFIG_ONE_PASS_SVM + // Features for one pass svm early term + int seg_feat; +#endif // motion vector cache for adaptive motion search control in partition // search loop MV pred_mv[REF_FRAMES]; InterpFilter pred_interp_filter; PARTITION_TYPE partition; + + // Reference and prediction mode cache for ref/mode speedup + // TODO(zoeliu@gmail.com): The values of ref_selected and mode_selected will + // be explored for further encoder speedup, to differentiate this approach for + // setting skip_ref_frame_mask from others. For instance, it is possible that + // the underlying square block(s) share the same SIMPLE_TRANSLATION motion + // mode as well as the mode of GLOBALMV, more ref/mode combos could be + // skipped. + MV_REFERENCE_FRAME ref_selected[2]; } PICK_MODE_CONTEXT; typedef struct { + int64_t rdcost; + int64_t sub_block_rdcost[4]; int valid; int split; - int skip; - int64_t rdcost; int sub_block_split[4]; int sub_block_skip[4]; - int64_t sub_block_rdcost[4]; + int skip; } PC_TREE_STATS; typedef struct PC_TREE { - int index; PARTITION_TYPE partitioning; BLOCK_SIZE block_size; PICK_MODE_CONTEXT none; @@ -97,9 +111,11 @@ typedef struct PC_TREE { PICK_MODE_CONTEXT verticalb[3]; PICK_MODE_CONTEXT horizontal4[4]; PICK_MODE_CONTEXT vertical4[4]; - CB_TREE_SEARCH cb_search_range; struct PC_TREE *split[4]; PC_TREE_STATS pc_tree_stats; + CB_TREE_SEARCH cb_search_range; + int index; + MV mv_ref_fulls[REF_FRAMES]; } PC_TREE; void av1_setup_pc_tree(struct AV1Common *cm, struct ThreadData *td); diff --git a/third_party/aom/av1/encoder/cost.h b/third_party/aom/av1/encoder/cost.h index af5b098370..be0241a820 100644 --- a/third_party/aom/av1/encoder/cost.h +++ b/third_party/aom/av1/encoder/cost.h @@ -30,6 +30,10 @@ extern const uint16_t av1_prob_cost[128]; // Calculate the cost of a symbol with probability p15 / 2^15 static INLINE int av1_cost_symbol(aom_cdf_prob p15) { + // p15 can be out of range [1, CDF_PROB_TOP - 1]. Clamping it, so that the + // following cost calculation works correctly. Otherwise, if p15 = + // CDF_PROB_TOP, shift would be -1, and "p15 << shift" would be wrong. + p15 = (aom_cdf_prob)clamp(p15, 1, CDF_PROB_TOP - 1); assert(0 < p15 && p15 < CDF_PROB_TOP); const int shift = CDF_PROB_BITS - 1 - get_msb(p15); const int prob = get_prob(p15 << shift, CDF_PROB_TOP); diff --git a/third_party/aom/av1/encoder/encode_strategy.c b/third_party/aom/av1/encoder/encode_strategy.c new file mode 100644 index 0000000000..cc933487d0 --- /dev/null +++ b/third_party/aom/av1/encoder/encode_strategy.c @@ -0,0 +1,918 @@ +/* + * Copyright (c) 2019, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "config/aom_config.h" +#include "config/aom_scale_rtcd.h" + +#include "aom/aom_codec.h" +#include "aom/aom_encoder.h" + +#include "aom_ports/system_state.h" + +#if CONFIG_MISMATCH_DEBUG +#include "aom_util/debug_util.h" +#endif // CONFIG_MISMATCH_DEBUG + +#include "av1/common/onyxc_int.h" + +#include "av1/encoder/encoder.h" +#include "av1/encoder/encode_strategy.h" +#include "av1/encoder/firstpass.h" +#include "av1/encoder/temporal_filter.h" +#include "av1/encoder/tpl_model.h" + +// Define the reference buffers that will be updated post encode. +void av1_configure_buffer_updates(AV1_COMP *cpi, const FRAME_UPDATE_TYPE type) { + // NOTE(weitinglin): Should we define another function to take care of + // cpi->rc.is_$Source_Type to make this function as it is in the comment? + + // show_existing_frame is a flag left set from the end of encoding the + // previous frame. Alongside it, is_src_frame_alt_ref may also be left + // set so shouldn't be cleared in this case. + if (!cpi->common.show_existing_frame) cpi->rc.is_src_frame_alt_ref = 0; + + cpi->rc.is_bwd_ref_frame = 0; + cpi->rc.is_last_bipred_frame = 0; + cpi->rc.is_bipred_frame = 0; + cpi->rc.is_src_frame_ext_arf = 0; + + switch (type) { + case KF_UPDATE: + cpi->refresh_last_frame = 1; + cpi->refresh_golden_frame = 1; + cpi->refresh_bwd_ref_frame = 1; + cpi->refresh_alt2_ref_frame = 1; + cpi->refresh_alt_ref_frame = 1; + break; + + case LF_UPDATE: + cpi->refresh_last_frame = 1; + cpi->refresh_golden_frame = 0; + cpi->refresh_bwd_ref_frame = 0; + cpi->refresh_alt2_ref_frame = 0; + cpi->refresh_alt_ref_frame = 0; + break; + + case GF_UPDATE: + // TODO(zoeliu): To further investigate whether 'refresh_last_frame' is + // needed. + cpi->refresh_last_frame = 1; + cpi->refresh_golden_frame = 1; + cpi->refresh_bwd_ref_frame = 0; + cpi->refresh_alt2_ref_frame = 0; + cpi->refresh_alt_ref_frame = 0; + break; + + case OVERLAY_UPDATE: + cpi->refresh_last_frame = 0; + cpi->refresh_golden_frame = 1; + cpi->refresh_bwd_ref_frame = 0; + cpi->refresh_alt2_ref_frame = 0; + cpi->refresh_alt_ref_frame = 0; + + cpi->rc.is_src_frame_alt_ref = 1; + break; + + case ARF_UPDATE: + cpi->refresh_last_frame = 0; + cpi->refresh_golden_frame = 0; + // NOTE: BWDREF does not get updated along with ALTREF_FRAME. + cpi->refresh_bwd_ref_frame = 0; + cpi->refresh_alt2_ref_frame = 0; + cpi->refresh_alt_ref_frame = 1; + break; + + case BRF_UPDATE: + cpi->refresh_last_frame = 0; + cpi->refresh_golden_frame = 0; + cpi->refresh_bwd_ref_frame = 1; + cpi->refresh_alt2_ref_frame = 0; + cpi->refresh_alt_ref_frame = 0; + + cpi->rc.is_bwd_ref_frame = 1; + break; + + case LAST_BIPRED_UPDATE: + cpi->refresh_last_frame = 1; + cpi->refresh_golden_frame = 0; + cpi->refresh_bwd_ref_frame = 0; + cpi->refresh_alt2_ref_frame = 0; + cpi->refresh_alt_ref_frame = 0; + + cpi->rc.is_last_bipred_frame = 1; + break; + + case BIPRED_UPDATE: + cpi->refresh_last_frame = 1; + cpi->refresh_golden_frame = 0; + cpi->refresh_bwd_ref_frame = 0; + cpi->refresh_alt2_ref_frame = 0; + cpi->refresh_alt_ref_frame = 0; + + cpi->rc.is_bipred_frame = 1; + break; + + case INTNL_OVERLAY_UPDATE: + cpi->refresh_last_frame = 1; + cpi->refresh_golden_frame = 0; + cpi->refresh_bwd_ref_frame = 0; + cpi->refresh_alt2_ref_frame = 0; + cpi->refresh_alt_ref_frame = 0; + + cpi->rc.is_src_frame_alt_ref = 1; + cpi->rc.is_src_frame_ext_arf = 1; + break; + + case INTNL_ARF_UPDATE: + cpi->refresh_last_frame = 0; + cpi->refresh_golden_frame = 0; + if (cpi->new_bwdref_update_rule == 1 && cpi->oxcf.pass == 2) { + cpi->refresh_bwd_ref_frame = 1; + cpi->refresh_alt2_ref_frame = 0; + } else { + cpi->refresh_bwd_ref_frame = 0; + cpi->refresh_alt2_ref_frame = 1; + } + cpi->refresh_alt_ref_frame = 0; + break; + + default: assert(0); break; + } +} + +static void set_additional_frame_flags(const AV1_COMMON *const cm, + unsigned int *const frame_flags) { + if (frame_is_intra_only(cm)) *frame_flags |= FRAMEFLAGS_INTRAONLY; + if (frame_is_sframe(cm)) *frame_flags |= FRAMEFLAGS_SWITCH; + if (cm->error_resilient_mode) *frame_flags |= FRAMEFLAGS_ERROR_RESILIENT; +} + +static INLINE void update_keyframe_counters(AV1_COMP *cpi) { + // TODO(zoeliu): To investigate whether we should treat BWDREF_FRAME + // differently here for rc->avg_frame_bandwidth. + if (cpi->common.show_frame || cpi->rc.is_bwd_ref_frame) { + if (!cpi->common.show_existing_frame || cpi->rc.is_src_frame_alt_ref || + cpi->common.current_frame.frame_type == KEY_FRAME) { + // If this is a show_existing_frame with a source other than altref, + // or if it is not a displayed forward keyframe, the keyframe update + // counters were incremented when it was originally encoded. + cpi->rc.frames_since_key++; + cpi->rc.frames_to_key--; + } + } +} + +static INLINE int is_frame_droppable(const AV1_COMP *const cpi) { + return !(cpi->refresh_alt_ref_frame || cpi->refresh_alt2_ref_frame || + cpi->refresh_bwd_ref_frame || cpi->refresh_golden_frame || + cpi->refresh_last_frame); +} + +static INLINE void update_frames_till_gf_update(AV1_COMP *cpi) { + // TODO(weitinglin): Updating this counter for is_frame_droppable + // is a work-around to handle the condition when a frame is drop. + // We should fix the cpi->common.show_frame flag + // instead of checking the other condition to update the counter properly. + if (cpi->common.show_frame || is_frame_droppable(cpi)) { + // Decrement count down till next gf + if (cpi->rc.frames_till_gf_update_due > 0) + cpi->rc.frames_till_gf_update_due--; + } +} + +static INLINE void update_twopass_gf_group_index(AV1_COMP *cpi) { + // Increment the gf group index ready for the next frame. If this is + // a show_existing_frame with a source other than altref, or if it is not + // a displayed forward keyframe, the index was incremented when it was + // originally encoded. + if (!cpi->common.show_existing_frame || cpi->rc.is_src_frame_alt_ref || + cpi->common.current_frame.frame_type == KEY_FRAME) { + ++cpi->twopass.gf_group.index; + } +} + +static void update_rc_counts(AV1_COMP *cpi) { + update_keyframe_counters(cpi); + update_frames_till_gf_update(cpi); + if (cpi->oxcf.pass == 2) update_twopass_gf_group_index(cpi); +} + +static void check_show_existing_frame(AV1_COMP *cpi) { + const GF_GROUP *const gf_group = &cpi->twopass.gf_group; + AV1_COMMON *const cm = &cpi->common; + const FRAME_UPDATE_TYPE next_frame_update_type = + gf_group->update_type[gf_group->index]; + const int which_arf = (cpi->new_bwdref_update_rule == 1) + ? gf_group->arf_update_idx[gf_group->index] > 0 + : gf_group->arf_update_idx[gf_group->index]; + + if (cm->show_existing_frame == 1) { + cm->show_existing_frame = 0; + } else if (cpi->rc.is_last_bipred_frame) { + // NOTE: When new structure is used, every bwdref will have one overlay + // frame. Therefore, there is no need to find out which frame to + // show in advance. + if (cpi->new_bwdref_update_rule == 0) { + // NOTE: If the current frame is a last bi-predictive frame, it is + // needed next to show the BWDREF_FRAME, which is pointed by + // the last_fb_idxes[0] after reference frame buffer update + cpi->rc.is_last_bipred_frame = 0; + cm->show_existing_frame = 1; + cpi->existing_fb_idx_to_show = cm->remapped_ref_idx[0]; + } + } else if (cpi->is_arf_filter_off[which_arf] && + (next_frame_update_type == OVERLAY_UPDATE || + next_frame_update_type == INTNL_OVERLAY_UPDATE)) { + const int bwdref_to_show = + (cpi->new_bwdref_update_rule == 1) ? BWDREF_FRAME : ALTREF2_FRAME; + // Other parameters related to OVERLAY_UPDATE will be taken care of + // in av1_rc_get_second_pass_params(cpi) + cm->show_existing_frame = 1; + cpi->rc.is_src_frame_alt_ref = 1; + cpi->existing_fb_idx_to_show = + (next_frame_update_type == OVERLAY_UPDATE) + ? get_ref_frame_map_idx(cm, ALTREF_FRAME) + : get_ref_frame_map_idx(cm, bwdref_to_show); + if (cpi->new_bwdref_update_rule == 0) { + cpi->is_arf_filter_off[which_arf] = 0; + } + } + cpi->rc.is_src_frame_ext_arf = 0; +} + +static void set_ext_overrides(AV1_COMP *const cpi, + EncodeFrameParams *const frame_params) { + // Overrides the defaults with the externally supplied values with + // av1_update_reference() and av1_update_entropy() calls + // Note: The overrides are valid only for the next frame passed + // to av1_encode_lowlevel() + + AV1_COMMON *const cm = &cpi->common; + + if (cpi->ext_use_s_frame) { + frame_params->frame_type = S_FRAME; + } + + if (cpi->ext_refresh_frame_context_pending) { + cm->refresh_frame_context = cpi->ext_refresh_frame_context; + cpi->ext_refresh_frame_context_pending = 0; + } + if (cpi->ext_refresh_frame_flags_pending) { + cpi->refresh_last_frame = cpi->ext_refresh_last_frame; + cpi->refresh_golden_frame = cpi->ext_refresh_golden_frame; + cpi->refresh_alt_ref_frame = cpi->ext_refresh_alt_ref_frame; + cpi->refresh_bwd_ref_frame = cpi->ext_refresh_bwd_ref_frame; + cpi->refresh_alt2_ref_frame = cpi->ext_refresh_alt2_ref_frame; + cpi->ext_refresh_frame_flags_pending = 0; + } + cm->allow_ref_frame_mvs = cpi->ext_use_ref_frame_mvs; + + frame_params->error_resilient_mode = cpi->ext_use_error_resilient; + // A keyframe is already error resilient and keyframes with + // error_resilient_mode interferes with the use of show_existing_frame + // when forward reference keyframes are enabled. + frame_params->error_resilient_mode &= frame_params->frame_type != KEY_FRAME; + // For bitstream conformance, s-frames must be error-resilient + frame_params->error_resilient_mode |= frame_params->frame_type == S_FRAME; +} + +static int get_ref_frame_flags(const AV1_COMP *const cpi) { + const AV1_COMMON *const cm = &cpi->common; + + const RefCntBuffer *last_buf = get_ref_frame_buf(cm, LAST_FRAME); + const RefCntBuffer *last2_buf = get_ref_frame_buf(cm, LAST2_FRAME); + const RefCntBuffer *last3_buf = get_ref_frame_buf(cm, LAST3_FRAME); + const RefCntBuffer *golden_buf = get_ref_frame_buf(cm, GOLDEN_FRAME); + const RefCntBuffer *bwd_buf = get_ref_frame_buf(cm, BWDREF_FRAME); + const RefCntBuffer *alt2_buf = get_ref_frame_buf(cm, ALTREF2_FRAME); + const RefCntBuffer *alt_buf = get_ref_frame_buf(cm, ALTREF_FRAME); + + // No.1 Priority: LAST_FRAME + const int last2_is_last = (last2_buf == last_buf); + const int last3_is_last = (last3_buf == last_buf); + const int gld_is_last = (golden_buf == last_buf); + const int bwd_is_last = (bwd_buf == last_buf); + const int alt2_is_last = (alt2_buf == last_buf); + const int alt_is_last = (alt_buf == last_buf); + + // No.2 Priority: ALTREF_FRAME + const int last2_is_alt = (last2_buf == alt_buf); + const int last3_is_alt = (last3_buf == alt_buf); + const int gld_is_alt = (golden_buf == alt_buf); + const int bwd_is_alt = (bwd_buf == alt_buf); + const int alt2_is_alt = (alt2_buf == alt_buf); + + // No.3 Priority: LAST2_FRAME + const int last3_is_last2 = (last3_buf == last2_buf); + const int gld_is_last2 = (golden_buf == last2_buf); + const int bwd_is_last2 = (bwd_buf == last2_buf); + const int alt2_is_last2 = (alt2_buf == last2_buf); + + // No.4 Priority: LAST3_FRAME + const int gld_is_last3 = (golden_buf == last3_buf); + const int bwd_is_last3 = (bwd_buf == last3_buf); + const int alt2_is_last3 = (alt2_buf == last3_buf); + + // No.5 Priority: GOLDEN_FRAME + const int bwd_is_gld = (bwd_buf == golden_buf); + const int alt2_is_gld = (alt2_buf == golden_buf); + + // No.6 Priority: BWDREF_FRAME + const int alt2_is_bwd = (alt2_buf == bwd_buf); + + // No.7 Priority: ALTREF2_FRAME + + // cpi->ext_ref_frame_flags allows certain reference types to be disabled + // by the external interface. These are set by av1_apply_encoding_flags(). + // Start with what the external interface allows, then suppress any reference + // types which we have found to be duplicates. + + int flags = cpi->ext_ref_frame_flags; + + if (cpi->rc.frames_till_gf_update_due == INT_MAX) flags &= ~AOM_GOLD_FLAG; + + if (alt_is_last) flags &= ~AOM_ALT_FLAG; + + if (last2_is_last || last2_is_alt) flags &= ~AOM_LAST2_FLAG; + + if (last3_is_last || last3_is_alt || last3_is_last2) flags &= ~AOM_LAST3_FLAG; + + if (gld_is_last || gld_is_alt || gld_is_last2 || gld_is_last3) + flags &= ~AOM_GOLD_FLAG; + + if ((bwd_is_last || bwd_is_alt || bwd_is_last2 || bwd_is_last3 || bwd_is_gld)) + flags &= ~AOM_BWD_FLAG; + + if ((alt2_is_last || alt2_is_alt || alt2_is_last2 || alt2_is_last3 || + alt2_is_gld || alt2_is_bwd)) + flags &= ~AOM_ALT2_FLAG; + + return flags; +} + +static int get_current_frame_ref_type( + const AV1_COMP *const cpi, const EncodeFrameParams *const frame_params) { + const GF_GROUP *const gf_group = &cpi->twopass.gf_group; + // We choose the reference "type" of this frame from the flags which indicate + // which reference frames will be refreshed by it. More than one of these + // flags may be set, so the order here implies an order of precedence. + // This is just used to choose the primary_ref_frame (as the most recent + // reference buffer of the same reference-type as the current frame) + + const int intra_only = frame_params->frame_type == KEY_FRAME || + frame_params->frame_type == INTRA_ONLY_FRAME; + if (intra_only || frame_params->error_resilient_mode || + cpi->ext_use_primary_ref_none) + return REGULAR_FRAME; + else if (gf_group->update_type[gf_group->index] == INTNL_ARF_UPDATE) + return EXT_ARF_FRAME; + else if (cpi->refresh_alt_ref_frame) + return ARF_FRAME; + else if (cpi->rc.is_src_frame_alt_ref) + return OVERLAY_FRAME; + else if (cpi->refresh_golden_frame) + return GLD_FRAME; + else if (cpi->refresh_bwd_ref_frame) + return BRF_FRAME; + else + return REGULAR_FRAME; +} + +static int choose_primary_ref_frame( + const AV1_COMP *const cpi, const EncodeFrameParams *const frame_params) { + const AV1_COMMON *const cm = &cpi->common; + + const int intra_only = frame_params->frame_type == KEY_FRAME || + frame_params->frame_type == INTRA_ONLY_FRAME; + if (intra_only || frame_params->error_resilient_mode || + cpi->ext_use_primary_ref_none) { + return PRIMARY_REF_NONE; + } + + // Find the most recent reference frame with the same reference type as the + // current frame + const FRAME_CONTEXT_INDEX current_ref_type = + get_current_frame_ref_type(cpi, frame_params); + int wanted_fb = cpi->fb_of_context_type[current_ref_type]; + + int primary_ref_frame = PRIMARY_REF_NONE; + for (int ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ref_frame++) { + if (get_ref_frame_map_idx(cm, ref_frame) == wanted_fb) { + primary_ref_frame = ref_frame - LAST_FRAME; + } + } + return primary_ref_frame; +} + +static void update_fb_of_context_type( + const AV1_COMP *const cpi, const EncodeFrameParams *const frame_params, + int *const fb_of_context_type) { + const AV1_COMMON *const cm = &cpi->common; + + if (frame_is_intra_only(cm) || cm->error_resilient_mode || + cpi->ext_use_primary_ref_none) { + for (int i = 0; i < REF_FRAMES; i++) { + fb_of_context_type[i] = -1; + } + fb_of_context_type[REGULAR_FRAME] = + cm->show_frame ? get_ref_frame_map_idx(cm, GOLDEN_FRAME) + : get_ref_frame_map_idx(cm, ALTREF_FRAME); + } + + if (!encode_show_existing_frame(cm)) { + // Refresh fb_of_context_type[]: see encoder.h for explanation + // Note that we want the value of refresh_frame_flags for the frame that + // just happened. If we call get_refresh_frame_flags now we will get a + // different answer, because update_reference_frames() has happened. + if (cm->current_frame.frame_type == KEY_FRAME) { + // All ref frames are refreshed, pick one that will live long enough + fb_of_context_type[REGULAR_FRAME] = 0; + } else { + // If more than one frame is refreshed, it doesn't matter which one we + // pick so pick the first. LST sometimes doesn't refresh any: this is ok + const int current_frame_ref_type = + get_current_frame_ref_type(cpi, frame_params); + for (int i = 0; i < REF_FRAMES; i++) { + if (cm->current_frame.refresh_frame_flags & (1 << i)) { + fb_of_context_type[current_frame_ref_type] = i; + break; + } + } + } + } +} + +static int get_order_offset(const AV1_COMP *const cpi, + const EncodeFrameParams *const frame_params) { + // shown frame by definition has order offset 0 + // show_existing_frame ignores order_offset and simply takes the order_hint + // from the reference frame being shown. + if (frame_params->show_frame || cpi->common.show_existing_frame) return 0; + + const GF_GROUP *const gf_group = &cpi->twopass.gf_group; + const int arf_offset = + AOMMIN((MAX_GF_INTERVAL - 1), gf_group->arf_src_offset[gf_group->index]); + const int brf_offset = gf_group->brf_src_offset[gf_group->index]; + return AOMMIN((MAX_GF_INTERVAL - 1), arf_offset + brf_offset); +} + +static void adjust_frame_rate(AV1_COMP *cpi, + const struct lookahead_entry *source) { + int64_t this_duration; + int step = 0; + + // Clear down mmx registers + aom_clear_system_state(); + + if (source->ts_start == cpi->first_time_stamp_ever) { + this_duration = source->ts_end - source->ts_start; + step = 1; + } else { + int64_t last_duration = + cpi->last_end_time_stamp_seen - cpi->last_time_stamp_seen; + + this_duration = source->ts_end - cpi->last_end_time_stamp_seen; + + // do a step update if the duration changes by 10% + if (last_duration) + step = (int)((this_duration - last_duration) * 10 / last_duration); + } + + if (this_duration) { + if (step) { + av1_new_framerate(cpi, 10000000.0 / this_duration); + } else { + // Average this frame's rate into the last second's average + // frame rate. If we haven't seen 1 second yet, then average + // over the whole interval seen. + const double interval = AOMMIN( + (double)(source->ts_end - cpi->first_time_stamp_ever), 10000000.0); + double avg_duration = 10000000.0 / cpi->framerate; + avg_duration *= (interval - avg_duration + this_duration); + avg_duration /= interval; + + av1_new_framerate(cpi, 10000000.0 / avg_duration); + } + } + cpi->last_time_stamp_seen = source->ts_start; + cpi->last_end_time_stamp_seen = source->ts_end; +} + +static void check_src_altref(AV1_COMP *cpi, + const struct lookahead_entry *source) { + RATE_CONTROL *const rc = &cpi->rc; + + // If pass == 2, the parameters set here will be reset in + // av1_rc_get_second_pass_params() + + if (cpi->oxcf.pass == 2) { + const GF_GROUP *const gf_group = &cpi->twopass.gf_group; + rc->is_src_frame_alt_ref = + (gf_group->update_type[gf_group->index] == INTNL_OVERLAY_UPDATE) || + (gf_group->update_type[gf_group->index] == OVERLAY_UPDATE); + rc->is_src_frame_ext_arf = + gf_group->update_type[gf_group->index] == INTNL_OVERLAY_UPDATE; + } else { + rc->is_src_frame_alt_ref = + cpi->alt_ref_source && (source == cpi->alt_ref_source); + } + + if (rc->is_src_frame_alt_ref) { + // Current frame is an ARF overlay frame. + cpi->alt_ref_source = NULL; + + if (rc->is_src_frame_ext_arf && !cpi->common.show_existing_frame) { + // For INTNL_OVERLAY, when show_existing_frame == 0, they do need to + // refresh the LAST_FRAME, i.e. LAST3 gets retired, LAST2 becomes LAST3, + // LAST becomes LAST2, and INTNL_OVERLAY becomes LAST. + cpi->refresh_last_frame = 1; + } else { + // Don't refresh the last buffer for an ARF overlay frame. It will + // become the GF so preserve last as an alternative prediction option. + cpi->refresh_last_frame = 0; + } + } +} + +// Returns 0 if this is not an alt ref else the offset of the source frame +// used as the arf midpoint. +static int get_arf_src_index(AV1_COMP *cpi) { + RATE_CONTROL *const rc = &cpi->rc; + int arf_src_index = 0; + if (is_altref_enabled(cpi)) { + if (cpi->oxcf.pass == 2) { + const GF_GROUP *const gf_group = &cpi->twopass.gf_group; + if (gf_group->update_type[gf_group->index] == ARF_UPDATE) { + arf_src_index = gf_group->arf_src_offset[gf_group->index]; + } + } else if (rc->source_alt_ref_pending) { + arf_src_index = rc->frames_till_gf_update_due; + } + } + return arf_src_index; +} + +static int get_brf_src_index(AV1_COMP *cpi) { + int brf_src_index = 0; + const GF_GROUP *const gf_group = &cpi->twopass.gf_group; + + // TODO(zoeliu): We need to add the check on the -bwd_ref command line setup + // flag. + if (gf_group->bidir_pred_enabled[gf_group->index]) { + if (cpi->oxcf.pass == 2) { + if (gf_group->update_type[gf_group->index] == BRF_UPDATE) + brf_src_index = gf_group->brf_src_offset[gf_group->index]; + } else { + // TODO(zoeliu): To re-visit the setup for this scenario + brf_src_index = cpi->rc.bipred_group_interval - 1; + } + } + + return brf_src_index; +} + +// Returns 0 if this is not an alt ref else the offset of the source frame +// used as the arf midpoint. +static int get_arf2_src_index(AV1_COMP *cpi) { + int arf2_src_index = 0; + if (is_altref_enabled(cpi) && cpi->num_extra_arfs) { + if (cpi->oxcf.pass == 2) { + const GF_GROUP *const gf_group = &cpi->twopass.gf_group; + if (gf_group->update_type[gf_group->index] == INTNL_ARF_UPDATE) { + arf2_src_index = gf_group->arf_src_offset[gf_group->index]; + } + } + } + return arf2_src_index; +} + +// Called if this frame is an ARF or ARF2. Also handles forward-keyframes +// For an ARF set arf2=0, for ARF2 set arf2=1 +// temporal_filtered is set to 1 if we temporally filter the ARF frame, so that +// the correct post-filter buffer can be used. +static struct lookahead_entry *setup_arf_or_arf2( + AV1_COMP *const cpi, const int arf_src_index, const int arf2, + int *temporal_filtered, EncodeFrameParams *const frame_params) { + AV1_COMMON *const cm = &cpi->common; + RATE_CONTROL *const rc = &cpi->rc; + const AV1EncoderConfig *const oxcf = &cpi->oxcf; + + assert(arf_src_index <= rc->frames_to_key); + *temporal_filtered = 0; + + struct lookahead_entry *source = + av1_lookahead_peek(cpi->lookahead, arf_src_index); + + if (source != NULL) { + cm->showable_frame = 1; + cpi->alt_ref_source = source; + + // When arf_src_index == rc->frames_to_key, it indicates a fwd_kf + if (!arf2 && arf_src_index == rc->frames_to_key) { + // Skip temporal filtering and mark as intra_only if we have a fwd_kf + const GF_GROUP *const gf_group = &cpi->twopass.gf_group; + int which_arf = gf_group->arf_update_idx[gf_group->index]; + cpi->is_arf_filter_off[which_arf] = 1; + cpi->no_show_kf = 1; + } else { + if (oxcf->arnr_max_frames > 0) { + // Produce the filtered ARF frame. + av1_temporal_filter(cpi, arf_src_index); + aom_extend_frame_borders(&cpi->alt_ref_buffer, av1_num_planes(cm)); + *temporal_filtered = 1; + } + } + frame_params->show_frame = 0; + + if (oxcf->pass < 2) { + // In second pass, the buffer updates configure will be set + // in the function av1_rc_get_second_pass_params + av1_configure_buffer_updates(cpi, arf2 ? INTNL_ARF_UPDATE : ARF_UPDATE); + } + } + rc->source_alt_ref_pending = 0; + return source; +} + +// Determine whether there is a forced keyframe pending in the lookahead buffer +static int is_forced_keyframe_pending(struct lookahead_ctx *lookahead, + const int up_to_index) { + for (int i = 0; i <= up_to_index; i++) { + const struct lookahead_entry *e = av1_lookahead_peek(lookahead, i); + if (e == NULL) { + // We have reached the end of the lookahead buffer and not early-returned + // so there isn't a forced key-frame pending. + return 0; + } else if (e->flags == AOM_EFLAG_FORCE_KF) { + return 1; + } else { + continue; + } + } + return 0; // Never reached +} + +// Check if we should encode an ARF, ARF2 or BRF. If not, try a LAST +// Do some setup associated with the chosen source +// Return the frame source, or NULL if we couldn't find one +struct lookahead_entry *choose_frame_source( + AV1_COMP *const cpi, int *const temporal_filtered, int *const flush, + struct lookahead_entry **last_source, + EncodeFrameParams *const frame_params) { + AV1_COMMON *const cm = &cpi->common; + struct lookahead_entry *source = NULL; + *temporal_filtered = 0; + + // Should we encode an alt-ref frame. + int arf_src_index = get_arf_src_index(cpi); + if (arf_src_index && + is_forced_keyframe_pending(cpi->lookahead, arf_src_index)) { + arf_src_index = 0; + *flush = 1; + } + + if (arf_src_index) { + source = setup_arf_or_arf2(cpi, arf_src_index, 0, temporal_filtered, + frame_params); + } + + // Should we encode an arf2 frame (mutually exclusive to ARF) + arf_src_index = get_arf2_src_index(cpi); + if (arf_src_index && + is_forced_keyframe_pending(cpi->lookahead, arf_src_index)) { + arf_src_index = 0; + *flush = 1; + } + + if (arf_src_index) { + source = setup_arf_or_arf2(cpi, arf_src_index, 1, temporal_filtered, + frame_params); + } + + cpi->rc.is_bwd_ref_frame = 0; + int brf_src_index = get_brf_src_index(cpi); + if (brf_src_index) { + assert(brf_src_index <= cpi->rc.frames_to_key); + if ((source = av1_lookahead_peek(cpi->lookahead, brf_src_index)) != NULL) { + cm->showable_frame = 1; + frame_params->show_frame = 0; + + if (cpi->oxcf.pass < 2) { + // In second pass, the buffer updates configure will be set + // in the function av1_rc_get_second_pass_params + av1_configure_buffer_updates(cpi, BRF_UPDATE); + } + } + } + + if (!source) { + // Get last frame source. + if (cm->current_frame.frame_number > 0) { + *last_source = av1_lookahead_peek(cpi->lookahead, -1); + } + // Read in the source frame. + source = av1_lookahead_pop(cpi->lookahead, *flush); + + if (source != NULL) { + frame_params->show_frame = 1; + + // Check to see if the frame should be encoded as an arf overlay. + check_src_altref(cpi, source); + } + } + return source; +} + +int av1_encode_strategy(AV1_COMP *const cpi, size_t *const size, + uint8_t *const dest, unsigned int *frame_flags, + int64_t *const time_stamp, int64_t *const time_end, + const aom_rational_t *const timebase, int flush) { + const AV1EncoderConfig *const oxcf = &cpi->oxcf; + AV1_COMMON *const cm = &cpi->common; + + EncodeFrameInput frame_input; + EncodeFrameParams frame_params; + EncodeFrameResults frame_results; + memset(&frame_input, 0, sizeof(frame_input)); + memset(&frame_params, 0, sizeof(frame_params)); + memset(&frame_results, 0, sizeof(frame_results)); + + int temporal_filtered = 0; + struct lookahead_entry *source = NULL; + struct lookahead_entry *last_source = NULL; + if (cm->show_existing_frame) { + source = av1_lookahead_pop(cpi->lookahead, flush); + } else { + source = choose_frame_source(cpi, &temporal_filtered, &flush, &last_source, + &frame_params); + } + + if (source == NULL) { // If no source was found, we can't encode a frame. + if (flush && oxcf->pass == 1 && !cpi->twopass.first_pass_done) { + av1_end_first_pass(cpi); /* get last stats packet */ + cpi->twopass.first_pass_done = 1; + } + return -1; + } + + frame_input.source = temporal_filtered ? &cpi->alt_ref_buffer : &source->img; + frame_input.last_source = last_source != NULL ? &last_source->img : NULL; + frame_input.ts_duration = source->ts_end - source->ts_start; + + *time_stamp = source->ts_start; + *time_end = source->ts_end; + if (source->ts_start < cpi->first_time_stamp_ever) { + cpi->first_time_stamp_ever = source->ts_start; + cpi->last_end_time_stamp_seen = source->ts_start; + } + + av1_apply_encoding_flags(cpi, source->flags); + if (!cm->show_existing_frame) + *frame_flags = (source->flags & AOM_EFLAG_FORCE_KF) ? FRAMEFLAGS_KEY : 0; + cpi->frame_flags = *frame_flags; + + if (frame_params.show_frame || + (cm->show_existing_frame && cpi->rc.is_src_frame_alt_ref)) { + // Shown frames and arf-overlay frames need frame-rate considering + adjust_frame_rate(cpi, source); + } + + if (cm->show_existing_frame) { + // show_existing_frame implies this frame is shown! + frame_params.show_frame = 1; + } else { + // Retain the RF_LEVEL for the current newly coded frame. + cm->cur_frame->frame_rf_level = + cpi->twopass.gf_group.rf_level[cpi->twopass.gf_group.index]; + + if (cpi->film_grain_table) { + cm->seq_params.film_grain_params_present = aom_film_grain_table_lookup( + cpi->film_grain_table, *time_stamp, *time_end, 0 /* =erase */, + &cm->film_grain_params); + } + cm->cur_frame->film_grain_params_present = + cm->seq_params.film_grain_params_present; + + // only one operating point supported now + const int64_t pts64 = ticks_to_timebase_units(timebase, *time_stamp); + if (pts64 < 0 || pts64 > UINT32_MAX) return AOM_CODEC_ERROR; + cpi->common.frame_presentation_time = (uint32_t)pts64; + } + + if (oxcf->pass == 2 && + (!cm->show_existing_frame || cpi->rc.is_src_frame_alt_ref)) { + // GF_GROUP needs updating for arf overlays as well as non-show-existing + av1_rc_get_second_pass_params(cpi, &frame_params); + } + if (cm->show_existing_frame && frame_params.frame_type != KEY_FRAME) { + // Force show-existing frames to be INTER, except forward keyframes + frame_params.frame_type = INTER_FRAME; + } + + if (!cm->show_existing_frame) { + cm->using_qmatrix = cpi->oxcf.using_qm; + cm->min_qmlevel = cpi->oxcf.qm_minlevel; + cm->max_qmlevel = cpi->oxcf.qm_maxlevel; + if (cpi->twopass.gf_group.index == 1 && cpi->oxcf.enable_tpl_model) { + av1_set_frame_size(cpi, cm->width, cm->height); + av1_tpl_setup_stats(cpi, &frame_input); + } + } + + frame_params.frame_flags = frame_flags; + + // TODO(david.turner@argondesign.com): Move all the encode strategy + // (largely near av1_get_compressed_data) in here + + // TODO(david.turner@argondesign.com): Change all the encode strategy to + // modify frame_params instead of cm or cpi. + + // Per-frame encode speed. In theory this can vary, but things may have been + // written assuming speed-level will not change within a sequence, so this + // parameter should be used with caution. + frame_params.speed = oxcf->speed; + + // Work out some encoding parameters specific to the pass: + if (oxcf->pass == 0) { + if (cpi->oxcf.rc_mode == AOM_CBR) { + av1_rc_get_one_pass_cbr_params(cpi, &frame_params); + } else { + av1_rc_get_one_pass_vbr_params(cpi, &frame_params); + } + } else if (oxcf->pass == 1) { + av1_setup_frame_size(cpi); + cpi->td.mb.e_mbd.lossless[0] = is_lossless_requested(&cpi->oxcf); + if (!cpi->refresh_alt_ref_frame && (cm->current_frame.frame_number == 0 || + (cpi->frame_flags & FRAMEFLAGS_KEY))) { + frame_params.frame_type = KEY_FRAME; + } else { + frame_params.frame_type = INTER_FRAME; + } + } else if (oxcf->pass == 2) { +#if CONFIG_MISMATCH_DEBUG + mismatch_move_frame_idx_w(); +#endif +#if TXCOEFF_COST_TIMER + cm->txcoeff_cost_timer = 0; + cm->txcoeff_cost_count = 0; +#endif + } + + if (oxcf->pass == 0 || oxcf->pass == 2) { + // Apply external override flags + set_ext_overrides(cpi, &frame_params); + + // Work out which reference frame slots may be used. + frame_params.ref_frame_flags = get_ref_frame_flags(cpi); + } + + if (oxcf->pass == 0 || oxcf->pass == 2) { + frame_params.primary_ref_frame = + choose_primary_ref_frame(cpi, &frame_params); + frame_params.order_offset = get_order_offset(cpi, &frame_params); + } + + if (av1_encode(cpi, dest, &frame_input, &frame_params, &frame_results) != + AOM_CODEC_OK) { + return AOM_CODEC_ERROR; + } + + if (oxcf->pass == 2) { +#if TXCOEFF_COST_TIMER + cm->cum_txcoeff_cost_timer += cm->txcoeff_cost_timer; + fprintf(stderr, + "\ntxb coeff cost block number: %ld, frame time: %ld, cum time %ld " + "in us\n", + cm->txcoeff_cost_count, cm->txcoeff_cost_timer, + cm->cum_txcoeff_cost_timer); +#endif + av1_twopass_postencode_update(cpi); + } + + if (oxcf->pass == 0 || oxcf->pass == 2) { + update_fb_of_context_type(cpi, &frame_params, cpi->fb_of_context_type); + set_additional_frame_flags(cm, frame_params.frame_flags); + update_rc_counts(cpi); + check_show_existing_frame(cpi); // Is next frame a show_existing frame? + } + + // Unpack frame_results: + *size = frame_results.size; + + // Leave a signal for a higher level caller about if this frame is droppable + if (*size > 0) { + cpi->droppable = is_frame_droppable(cpi); + } + + return AOM_CODEC_OK; +} diff --git a/third_party/aom/av1/encoder/encode_strategy.h b/third_party/aom/av1/encoder/encode_strategy.h new file mode 100644 index 0000000000..1671473b76 --- /dev/null +++ b/third_party/aom/av1/encoder/encode_strategy.h @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2019, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_AV1_ENCODER_ENCODE_STRATEGY_H_ +#define AOM_AV1_ENCODER_ENCODE_STRATEGY_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +#include "aom/aom_encoder.h" + +#include "av1/encoder/encoder.h" +#include "av1/encoder/firstpass.h" + +// This function will implement high-level encode strategy, choosing frame type, +// frame placement, etc. It populates an EncodeFrameParams struct with the +// results of these decisions and then calls av1_encode() +int av1_encode_strategy(AV1_COMP *const cpi, size_t *const size, + uint8_t *const dest, unsigned int *frame_flags, + int64_t *const time_stamp, int64_t *const time_end, + const aom_rational_t *const timebase, int flush); + +// Set individual buffer update flags based on frame reference type +void av1_configure_buffer_updates(AV1_COMP *const cpi, + const FRAME_UPDATE_TYPE type); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AOM_AV1_ENCODER_ENCODE_STRATEGY_H_ diff --git a/third_party/aom/av1/encoder/encodeframe.c b/third_party/aom/av1/encoder/encodeframe.c index cb226c59e2..ea8249300c 100644 --- a/third_party/aom/av1/encoder/encodeframe.c +++ b/third_party/aom/av1/encoder/encodeframe.c @@ -11,6 +11,7 @@ #include #include +#include #include #include "config/aom_config.h" @@ -64,6 +65,10 @@ static void encode_superblock(const AV1_COMP *const cpi, TileDataEnc *tile_data, ThreadData *td, TOKENEXTRA **t, RUN_TYPE dry_run, int mi_row, int mi_col, BLOCK_SIZE bsize, int *rate); +static int ml_predict_breakout(const AV1_COMP *const cpi, BLOCK_SIZE bsize, + const MACROBLOCK *const x, + const RD_STATS *const rd_stats, + unsigned int pb_source_variance); // This is used as a reference when computing the source variance for the // purposes of activity masking. @@ -143,6 +148,15 @@ static const uint8_t num_16x16_blocks_high_lookup[BLOCK_SIZES_ALL] = { }; #endif // CONFIG_FP_MB_STATS +static const uint8_t ref_frame_flag_list[REF_FRAMES] = { 0, + AOM_LAST_FLAG, + AOM_LAST2_FLAG, + AOM_LAST3_FLAG, + AOM_GOLD_FLAG, + AOM_BWD_FLAG, + AOM_ALT2_FLAG, + AOM_ALT_FLAG }; + unsigned int av1_get_sby_perpixel_variance(const AV1_COMP *cpi, const struct buf_2d *ref, BLOCK_SIZE bs) { @@ -183,7 +197,8 @@ static unsigned int get_sby_perpixel_diff_variance(const AV1_COMP *const cpi, BLOCK_SIZE bs) { unsigned int sse, var; uint8_t *last_y; - const YV12_BUFFER_CONFIG *last = get_ref_frame_buffer(cpi, LAST_FRAME); + const YV12_BUFFER_CONFIG *last = + get_ref_frame_yv12_buf(&cpi->common, LAST_FRAME); assert(last != NULL); last_y = @@ -236,8 +251,8 @@ static void set_offsets_without_segment_id(const AV1_COMP *const cpi, xd->left_txfm_context_buffer + (mi_row & MAX_MIB_MASK); // Set up destination pointers. - av1_setup_dst_planes(xd->plane, bsize, get_frame_new_buffer(cm), mi_row, - mi_col, 0, num_planes); + av1_setup_dst_planes(xd->plane, bsize, &cm->cur_frame->buf, mi_row, mi_col, 0, + num_planes); // Set up limit values for MV components. // Mv beyond the range do not produce new/different prediction block. @@ -255,32 +270,31 @@ static void set_offsets_without_segment_id(const AV1_COMP *const cpi, cm->mi_cols); // Set up source buffers. - av1_setup_src_planes(x, cpi->source, mi_row, mi_col, num_planes); + av1_setup_src_planes(x, cpi->source, mi_row, mi_col, num_planes, bsize); // R/D setup. x->rdmult = cpi->rd.RDMULT; // required by av1_append_sub8x8_mvs_for_idx() and av1_find_best_ref_mvs() xd->tile = *tile; + + xd->cfl.mi_row = mi_row; + xd->cfl.mi_col = mi_col; } static void set_offsets(const AV1_COMP *const cpi, const TileInfo *const tile, MACROBLOCK *const x, int mi_row, int mi_col, BLOCK_SIZE bsize) { const AV1_COMMON *const cm = &cpi->common; + const struct segmentation *const seg = &cm->seg; MACROBLOCKD *const xd = &x->e_mbd; MB_MODE_INFO *mbmi; - const struct segmentation *const seg = &cm->seg; set_offsets_without_segment_id(cpi, tile, x, mi_row, mi_col, bsize); - mbmi = xd->mi[0]; - xd->cfl.mi_row = mi_row; - xd->cfl.mi_col = mi_col; - - mbmi->segment_id = 0; - // Setup segment ID. + mbmi = xd->mi[0]; + mbmi->segment_id = 0; if (seg->enabled) { if (seg->enabled && !cpi->vaq_refresh) { const uint8_t *const map = @@ -292,15 +306,6 @@ static void set_offsets(const AV1_COMP *const cpi, const TileInfo *const tile, } } -static void reset_intmv_filter_type(MB_MODE_INFO *mbmi) { - InterpFilter filters[2]; - - for (int dir = 0; dir < 2; ++dir) { - filters[dir] = av1_extract_interp_filter(mbmi->interp_filters, dir); - } - mbmi->interp_filters = av1_make_interp_filters(filters[0], filters[1]); -} - static void update_filter_type_count(uint8_t allow_update_cdf, FRAME_COUNTS *counts, const MACROBLOCKD *xd, @@ -375,8 +380,6 @@ static void update_state(const AV1_COMP *const cpi, *mi_addr = *mi; *x->mbmi_ext = ctx->mbmi_ext; - reset_intmv_filter_type(mi_addr); - memcpy(x->blk_skip, ctx->blk_skip, sizeof(x->blk_skip[0]) * ctx->num_4x4_blk); x->skip = ctx->skip; @@ -475,7 +478,8 @@ static void update_state(const AV1_COMP *const cpi, } void av1_setup_src_planes(MACROBLOCK *x, const YV12_BUFFER_CONFIG *src, - int mi_row, int mi_col, const int num_planes) { + int mi_row, int mi_col, const int num_planes, + BLOCK_SIZE bsize) { // Set current frame pointer. x->e_mbd.cur_buf = src; @@ -483,11 +487,10 @@ void av1_setup_src_planes(MACROBLOCK *x, const YV12_BUFFER_CONFIG *src, // the static analysis warnings. for (int i = 0; i < AOMMIN(num_planes, MAX_MB_PLANE); i++) { const int is_uv = i > 0; - setup_pred_plane(&x->plane[i].src, x->e_mbd.mi[0]->sb_type, src->buffers[i], - src->crop_widths[is_uv], src->crop_heights[is_uv], - src->strides[is_uv], mi_row, mi_col, NULL, - x->e_mbd.plane[i].subsampling_x, - x->e_mbd.plane[i].subsampling_y); + setup_pred_plane( + &x->plane[i].src, bsize, src->buffers[i], src->crop_widths[is_uv], + src->crop_heights[is_uv], src->strides[is_uv], mi_row, mi_col, NULL, + x->e_mbd.plane[i].subsampling_x, x->e_mbd.plane[i].subsampling_y); } } @@ -507,6 +510,19 @@ static int set_deltaq_rdmult(const AV1_COMP *const cpi, MACROBLOCKD *const xd) { cpi, cm->base_qindex + xd->delta_qindex + cm->y_dc_delta_q); } +static uint16_t edge_strength(const struct buf_2d *ref, const BLOCK_SIZE bsize, + const bool high_bd, const int bd) { + const int width = block_size_wide[bsize]; + const int height = block_size_high[bsize]; + // Implementation requires width to be a multiple of 8. It also requires + // height to be a multiple of 4, but this is always the case. + assert(height % 4 == 0); + if (width % 8 != 0) { + return 0; + } + return av1_edge_exists(ref->buf, ref->stride, width, height, high_bd, bd); +} + static void rd_pick_sb_modes(AV1_COMP *const cpi, TileDataEnc *tile_data, MACROBLOCK *const x, int mi_row, int mi_col, RD_STATS *rd_cost, PARTITION_TYPE partition, @@ -584,14 +600,21 @@ static void rd_pick_sb_modes(AV1_COMP *const cpi, TileDataEnc *tile_data, return; } - if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + if (is_cur_buf_hbd(xd)) { x->source_variance = av1_high_get_sby_perpixel_variance( cpi, &x->plane[0].src, bsize, xd->bd); } else { x->source_variance = av1_get_sby_perpixel_variance(cpi, &x->plane[0].src, bsize); } - + // If the threshold for disabling wedge search is zero, it means the feature + // should not be used. Use a value that will always succeed in the check. + if (cpi->sf.disable_wedge_search_edge_thresh == 0) { + x->edge_strength = UINT16_MAX; + } else { + x->edge_strength = + edge_strength(&x->plane[0].src, bsize, is_cur_buf_hbd(xd), xd->bd); + } // Save rdmult before it might be changed, so it can be restored later. orig_rdmult = x->rdmult; @@ -609,6 +632,8 @@ static void rd_pick_sb_modes(AV1_COMP *const cpi, TileDataEnc *tile_data, // If segment is boosted, use rdmult for that segment. if (cyclic_refresh_segment_id_boosted(mbmi->segment_id)) x->rdmult = av1_cyclic_refresh_get_rdmult(cpi->cyclic_refresh); + } else if (cpi->oxcf.enable_tpl_model) { + x->rdmult = x->cb_rdmult; } if (deltaq_mode > 0) x->rdmult = set_deltaq_rdmult(cpi, xd); @@ -622,17 +647,23 @@ static void rd_pick_sb_modes(AV1_COMP *const cpi, TileDataEnc *tile_data, if (segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) { av1_rd_pick_inter_mode_sb_seg_skip(cpi, tile_data, x, mi_row, mi_col, rd_cost, bsize, ctx, best_rd); +#if CONFIG_ONE_PASS_SVM + ctx->seg_feat = 1; +#endif } else { av1_rd_pick_inter_mode_sb(cpi, tile_data, x, mi_row, mi_col, rd_cost, bsize, ctx, best_rd); +#if CONFIG_ONE_PASS_SVM + ctx->seg_feat = 0; +#endif } } // Examine the resulting rate and for AQ mode 2 make a segment choice. if ((rd_cost->rate != INT_MAX) && (aq_mode == COMPLEXITY_AQ) && (bsize >= BLOCK_16X16) && - (cm->frame_type == KEY_FRAME || cpi->refresh_alt_ref_frame || - cpi->refresh_alt2_ref_frame || + (cm->current_frame.frame_type == KEY_FRAME || + cpi->refresh_alt_ref_frame || cpi->refresh_alt2_ref_frame || (cpi->refresh_golden_frame && !cpi->rc.is_src_frame_alt_ref))) { av1_caq_select_segment(cpi, x, bsize, mi_row, mi_col, rd_cost->rate); } @@ -865,6 +896,7 @@ static void update_stats(const AV1_COMMON *const cm, TileDataEnc *tile_data, MACROBLOCKD *const xd = &x->e_mbd; const MB_MODE_INFO *const mbmi = xd->mi[0]; const MB_MODE_INFO_EXT *const mbmi_ext = x->mbmi_ext; + const CurrentFrame *const current_frame = &cm->current_frame; const BLOCK_SIZE bsize = mbmi->sb_type; FRAME_CONTEXT *fc = xd->tile_ctx; const uint8_t allow_update_cdf = tile_data->allow_update_cdf; @@ -877,7 +909,8 @@ static void update_stats(const AV1_COMMON *const cm, TileDataEnc *tile_data, const int seg_ref_active = segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_REF_FRAME); - if (cm->skip_mode_flag && !seg_ref_active && is_comp_ref_allowed(bsize)) { + if (current_frame->skip_mode_info.skip_mode_flag && !seg_ref_active && + is_comp_ref_allowed(bsize)) { const int skip_mode_ctx = av1_get_skip_mode_context(xd); #if CONFIG_ENTROPY_STATS td->counts->skip_mode[skip_mode_ctx][mbmi->skip_mode]++; @@ -896,12 +929,13 @@ static void update_stats(const AV1_COMMON *const cm, TileDataEnc *tile_data, } } - if (cm->delta_q_present_flag && + const DeltaQInfo *const delta_q_info = &cm->delta_q_info; + if (delta_q_info->delta_q_present_flag && (bsize != cm->seq_params.sb_size || !mbmi->skip) && super_block_upper_left) { #if CONFIG_ENTROPY_STATS const int dq = - (mbmi->current_qindex - xd->current_qindex) / cm->delta_q_res; + (mbmi->current_qindex - xd->current_qindex) / delta_q_info->delta_q_res; const int absdq = abs(dq); for (int i = 0; i < AOMMIN(absdq, DELTA_Q_SMALL); ++i) { td->counts->delta_q[i][1]++; @@ -909,14 +943,14 @@ static void update_stats(const AV1_COMMON *const cm, TileDataEnc *tile_data, if (absdq < DELTA_Q_SMALL) td->counts->delta_q[absdq][0]++; #endif xd->current_qindex = mbmi->current_qindex; - if (cm->delta_lf_present_flag) { - if (cm->delta_lf_multi) { + if (delta_q_info->delta_lf_present_flag) { + if (delta_q_info->delta_lf_multi) { const int frame_lf_count = av1_num_planes(cm) > 1 ? FRAME_LF_COUNT : FRAME_LF_COUNT - 2; for (int lf_id = 0; lf_id < frame_lf_count; ++lf_id) { #if CONFIG_ENTROPY_STATS - const int delta_lf = - (mbmi->delta_lf[lf_id] - xd->delta_lf[lf_id]) / cm->delta_lf_res; + const int delta_lf = (mbmi->delta_lf[lf_id] - xd->delta_lf[lf_id]) / + delta_q_info->delta_lf_res; const int abs_delta_lf = abs(delta_lf); for (int i = 0; i < AOMMIN(abs_delta_lf, DELTA_LF_SMALL); ++i) { td->counts->delta_lf_multi[lf_id][i][1]++; @@ -930,7 +964,7 @@ static void update_stats(const AV1_COMMON *const cm, TileDataEnc *tile_data, #if CONFIG_ENTROPY_STATS const int delta_lf = (mbmi->delta_lf_from_base - xd->delta_lf_from_base) / - cm->delta_lf_res; + delta_q_info->delta_lf_res; const int abs_delta_lf = abs(delta_lf); for (int i = 0; i < AOMMIN(abs_delta_lf, DELTA_LF_SMALL); ++i) { td->counts->delta_lf[i][1]++; @@ -964,7 +998,7 @@ static void update_stats(const AV1_COMMON *const cm, TileDataEnc *tile_data, if (mbmi->skip_mode) { rdc->skip_mode_used_flag = 1; - if (cm->reference_mode == REFERENCE_MODE_SELECT) { + if (current_frame->reference_mode == REFERENCE_MODE_SELECT) { assert(has_second_ref(mbmi)); rdc->compound_ref_used_flag = 1; } @@ -991,7 +1025,7 @@ static void update_stats(const AV1_COMMON *const cm, TileDataEnc *tile_data, av1_collect_neighbors_ref_counts(xd); - if (cm->reference_mode == REFERENCE_MODE_SELECT) { + if (current_frame->reference_mode == REFERENCE_MODE_SELECT) { if (has_second_ref(mbmi)) // This flag is also updated for 4x4 blocks rdc->compound_ref_used_flag = 1; @@ -1219,7 +1253,7 @@ static void update_stats(const AV1_COMMON *const cm, TileDataEnc *tile_data, } if (has_second_ref(mbmi)) { - assert(cm->reference_mode != SINGLE_REFERENCE && + assert(current_frame->reference_mode != SINGLE_REFERENCE && is_inter_compound_mode(mbmi->mode) && mbmi->motion_mode == SIMPLE_TRANSLATION); @@ -1427,18 +1461,20 @@ static void encode_b(const AV1_COMP *const cpi, TileDataEnc *tile_data, MB_MODE_INFO *mbmi = xd->mi[0]; mbmi->partition = partition; update_state(cpi, tile_data, td, ctx, mi_row, mi_col, bsize, dry_run); + if (cpi->oxcf.enable_tpl_model && cpi->oxcf.aq_mode == NO_AQ && + cpi->oxcf.deltaq_mode == 0) { + x->rdmult = x->cb_rdmult; + } if (!dry_run) av1_set_coeff_buffer(cpi, x, mi_row, mi_col); encode_superblock(cpi, tile_data, td, tp, dry_run, mi_row, mi_col, bsize, rate); - if (dry_run == 0) - x->cb_offset += block_size_wide[bsize] * block_size_high[bsize]; - if (!dry_run) { + x->cb_offset += block_size_wide[bsize] * block_size_high[bsize]; if (bsize == cpi->common.seq_params.sb_size && mbmi->skip == 1 && - cpi->common.delta_lf_present_flag) { + cpi->common.delta_q_info.delta_lf_present_flag) { const int frame_lf_count = av1_num_planes(&cpi->common) > 1 ? FRAME_LF_COUNT : FRAME_LF_COUNT - 2; @@ -2057,6 +2093,102 @@ static int active_edge_sb(const AV1_COMP *cpi, int mi_row, int mi_col) { active_v_edge(cpi, mi_col, cpi->common.seq_params.mib_size); } +// Performs a motion search in SIMPLE_TRANSLATION mode using reference frame +// ref. Note that this sets the offset of mbmi, so we will need to reset it +// after calling this function. +static void simple_motion_search(AV1_COMP *const cpi, MACROBLOCK *x, int mi_row, + int mi_col, BLOCK_SIZE bsize, int ref, + MV ref_mv_full, int num_planes, + int use_subpixel) { + assert(num_planes == 1 && + "Currently simple_motion_search only supports luma plane"); + AV1_COMMON *const cm = &cpi->common; + MACROBLOCKD *xd = &x->e_mbd; + + set_offsets(cpi, &xd->tile, x, mi_row, mi_col, bsize); + + MB_MODE_INFO *mbmi = xd->mi[0]; + mbmi->sb_type = bsize; + mbmi->ref_frame[0] = ref; + mbmi->ref_frame[1] = NONE_FRAME; + mbmi->motion_mode = SIMPLE_TRANSLATION; + + const YV12_BUFFER_CONFIG *yv12 = get_ref_frame_yv12_buf(cm, ref); + const YV12_BUFFER_CONFIG *scaled_ref_frame = + av1_get_scaled_ref_frame(cpi, ref); + struct buf_2d backup_yv12; + // ref_mv is used to code the motion vector. ref_mv_full is the initial point. + // ref_mv is in units of 1/8 pel whereas ref_mv_full is in units of pel. + MV ref_mv = { 0, 0 }; + const int step_param = cpi->mv_step_param; + const MvLimits tmp_mv_limits = x->mv_limits; + const SEARCH_METHODS search_methods = NSTEP; + const int do_mesh_search = 0; + const int sadpb = x->sadperbit16; + int cost_list[5]; + const int ref_idx = 0; + int var; + + if (scaled_ref_frame) { + backup_yv12 = xd->plane[AOM_PLANE_Y].pre[ref_idx]; + av1_setup_pre_planes(xd, ref_idx, scaled_ref_frame, mi_row, mi_col, NULL, + num_planes); + } else { + av1_setup_pre_planes(xd, ref_idx, yv12, mi_row, mi_col, + get_ref_scale_factors(cm, ref), num_planes); + } + + // This overwrites the mv_limits so we will need to restore it later. + av1_set_mv_search_range(&x->mv_limits, &ref_mv); + var = av1_full_pixel_search(cpi, x, bsize, &ref_mv_full, step_param, + search_methods, do_mesh_search, sadpb, + cond_cost_list(cpi, cost_list), &ref_mv, INT_MAX, + 1, mi_col * MI_SIZE, mi_row * MI_SIZE, 0); + // Restore + x->mv_limits = tmp_mv_limits; + + const int use_subpel_search = + var < INT_MAX && !cpi->common.cur_frame_force_integer_mv && use_subpixel; + if (use_subpel_search) { + int not_used = 0; + if (cpi->sf.use_accurate_subpel_search) { + const int pw = block_size_wide[bsize]; + const int ph = block_size_high[bsize]; + cpi->find_fractional_mv_step( + x, cm, mi_row, mi_col, &ref_mv, cm->allow_high_precision_mv, + x->errorperbit, &cpi->fn_ptr[bsize], cpi->sf.mv.subpel_force_stop, + cpi->sf.mv.subpel_iters_per_step, cond_cost_list(cpi, cost_list), + x->nmv_vec_cost, x->mv_cost_stack, ¬_used, &x->pred_sse[ref], NULL, + NULL, 0, 0, pw, ph, cpi->sf.use_accurate_subpel_search, 1); + } else { + cpi->find_fractional_mv_step( + x, cm, mi_row, mi_col, &ref_mv, cm->allow_high_precision_mv, + x->errorperbit, &cpi->fn_ptr[bsize], cpi->sf.mv.subpel_force_stop, + cpi->sf.mv.subpel_iters_per_step, cond_cost_list(cpi, cost_list), + x->nmv_vec_cost, x->mv_cost_stack, ¬_used, &x->pred_sse[ref], NULL, + NULL, 0, 0, 0, 0, 0, 1); + } + } else { + // Manually convert from units of pixel to 1/8-pixels if we are not doing + // subpel search + x->best_mv.as_mv.row *= 8; + x->best_mv.as_mv.col *= 8; + } + + mbmi->mv[0].as_mv = x->best_mv.as_mv; + + // Get a copy of the prediction output + set_ref_ptrs(cm, xd, mbmi->ref_frame[0], mbmi->ref_frame[1]); + av1_enc_build_inter_predictor(cm, xd, mi_row, mi_col, NULL, bsize, + AOM_PLANE_Y, AOM_PLANE_Y); + + aom_clear_system_state(); + + if (scaled_ref_frame) { + xd->plane[AOM_PLANE_Y].pre[ref_idx] = backup_yv12; + } +} + // Look at neighboring blocks and set a min and max partition size based on // what they chose. static void rd_auto_partition_range(AV1_COMP *cpi, const TileInfo *const tile, @@ -2074,7 +2206,8 @@ static void rd_auto_partition_range(AV1_COMP *cpi, const TileInfo *const tile, BLOCK_SIZE max_size = BLOCK_LARGEST; // Trap case where we do not have a prediction. - if (left_in_image || above_in_image || cm->frame_type != KEY_FRAME) { + if (left_in_image || above_in_image || + cm->current_frame.frame_type != KEY_FRAME) { // Default "min to max" and "max to min" min_size = BLOCK_LARGEST; max_size = BLOCK_4X4; @@ -2082,7 +2215,7 @@ static void rd_auto_partition_range(AV1_COMP *cpi, const TileInfo *const tile, // NOTE: each call to get_sb_partition_size_range() uses the previous // passed in values for min and max as a starting point. // Find the min and max partition used in previous frame at this location - if (cm->frame_type != KEY_FRAME) { + if (cm->current_frame.frame_type != KEY_FRAME) { MB_MODE_INFO **prev_mi = &cm->prev_mi_grid_visible[mi_row * xd->mi_stride + mi_col]; get_sb_partition_size_range(cm, xd, prev_mi, &min_size, &max_size); @@ -2210,14 +2343,14 @@ const int complexity_16x16_blocks_threshold[BLOCK_SIZES] = { 8, 8, 10 }; -typedef enum { +enum { MV_ZERO = 0, MV_LEFT = 1, MV_UP = 2, MV_RIGHT = 3, MV_DOWN = 4, MV_INVALID -} MOTION_DIRECTION; +} UENUM1BYTE(MOTION_DIRECTION); static INLINE MOTION_DIRECTION get_motion_direction_fp(uint8_t fp_byte) { if (fp_byte & FPMB_MOTION_ZERO_MASK) { @@ -2346,6 +2479,17 @@ static void reset_partition(PC_TREE *pc_tree, BLOCK_SIZE bsize) { pc_tree->cb_search_range = SEARCH_FULL_PLANE; pc_tree->none.skip = 0; + pc_tree->pc_tree_stats.valid = 0; + pc_tree->pc_tree_stats.split = 0; + pc_tree->pc_tree_stats.skip = 0; + pc_tree->pc_tree_stats.rdcost = INT64_MAX; + + for (int i = 0; i < 4; i++) { + pc_tree->pc_tree_stats.sub_block_split[i] = 0; + pc_tree->pc_tree_stats.sub_block_skip[i] = 0; + pc_tree->pc_tree_stats.sub_block_rdcost[i] = INT64_MAX; + } + if (bsize >= BLOCK_8X8) { BLOCK_SIZE subsize = get_partition_subsize(bsize, PARTITION_SPLIT); for (int idx = 0; idx < 4; ++idx) @@ -2467,9 +2611,9 @@ static void rd_pick_sqr_partition(AV1_COMP *const cpi, ThreadData *td, ? partition_cost[PARTITION_NONE] : 0; } - int64_t partition_rd_cost = RDCOST(x->rdmult, pt_cost, 0); - int64_t best_remain_rdcost = best_rdc.rdcost == INT64_MAX - ? INT64_MAX + const int64_t partition_rd_cost = RDCOST(x->rdmult, pt_cost, 0); + const int64_t best_remain_rdcost = + best_rdc.rdcost == INT64_MAX ? INT64_MAX : (best_rdc.rdcost - partition_rd_cost); rd_pick_sb_modes(cpi, tile_data, x, mi_row, mi_col, &this_rdc, PARTITION_NONE, bsize, ctx_none, best_remain_rdcost); @@ -2499,15 +2643,30 @@ static void rd_pick_sqr_partition(AV1_COMP *const cpi, ThreadData *td, pc_tree->cb_search_range = SEARCH_FULL_PLANE; - // If all y, u, v transform blocks in this partition are skippable, and - // the dist & rate are within the thresholds, the partition search is - // terminated for current branch of the partition search tree. - // The dist & rate thresholds are set to 0 at speed 0 to disable the - // early termination at that speed. - if (!x->e_mbd.lossless[xd->mi[0]->segment_id] && - (ctx_none->skippable && best_rdc.dist < dist_breakout_thr && - best_rdc.rate < rate_breakout_thr)) { - do_square_split = 0; + if (!x->e_mbd.lossless[xd->mi[0]->segment_id] && ctx_none->skippable) { + const int use_ml_based_breakout = + bsize <= cpi->sf.use_square_partition_only_threshold && + bsize > BLOCK_4X4 && xd->bd == 8; + + // TODO(anyone): Currently this is using the same model and threshold + // values as in rd_pick_partition. Retraining the model and tuning the + // threshold values might be helpful to improve the speed. + if (use_ml_based_breakout) { + if (ml_predict_breakout(cpi, bsize, x, &this_rdc, + x->source_variance)) { + do_square_split = 0; + } + } + + // If all y, u, v transform blocks in this partition are skippable, + // and the dist & rate are within the thresholds, the partition search + // is terminated for current branch of the partition search tree. The + // dist & rate thresholds are set to 0 at speed 0 to disable the early + // termination at that speed. + if (best_rdc.dist < dist_breakout_thr && + best_rdc.rate < rate_breakout_thr) { + do_square_split = 0; + } } } } @@ -2527,6 +2686,9 @@ static void rd_pick_sqr_partition(AV1_COMP *const cpi, ThreadData *td, subsize = get_partition_subsize(bsize, PARTITION_SPLIT); int idx; + sum_rdc.rate = partition_cost[PARTITION_SPLIT]; + sum_rdc.rdcost = RDCOST(x->rdmult, sum_rdc.rate, 0); + for (idx = 0; idx < 4 && sum_rdc.rdcost < temp_best_rdcost; ++idx) { const int x_idx = (idx & 1) * mi_step; const int y_idx = (idx >> 1) * mi_step; @@ -2538,12 +2700,12 @@ static void rd_pick_sqr_partition(AV1_COMP *const cpi, ThreadData *td, pc_tree->split[idx]->index = idx; int64_t *p_split_rd = &split_rd[idx]; - // TODO(Cherma) : Account for partition cost while passing best rd to - // rd_pick_sqr_partition() - rd_pick_sqr_partition(cpi, td, tile_data, tp, mi_row + y_idx, - mi_col + x_idx, subsize, &this_rdc, - temp_best_rdcost - sum_rdc.rdcost, - pc_tree->split[idx], p_split_rd); + const int64_t best_remain_rdcost = + (temp_best_rdcost == INT64_MAX) ? INT64_MAX + : (temp_best_rdcost - sum_rdc.rdcost); + rd_pick_sqr_partition( + cpi, td, tile_data, tp, mi_row + y_idx, mi_col + x_idx, subsize, + &this_rdc, best_remain_rdcost, pc_tree->split[idx], p_split_rd); pc_tree->pc_tree_stats.sub_block_rdcost[idx] = this_rdc.rdcost; pc_tree->pc_tree_stats.sub_block_skip[idx] = @@ -2561,7 +2723,6 @@ static void rd_pick_sqr_partition(AV1_COMP *const cpi, ThreadData *td, reached_last_index = (idx == 4); if (reached_last_index && sum_rdc.rdcost < best_rdc.rdcost) { - sum_rdc.rate += partition_cost[PARTITION_SPLIT]; sum_rdc.rdcost = RDCOST(x->rdmult, sum_rdc.rate, sum_rdc.dist); if (sum_rdc.rdcost < best_rdc.rdcost) { @@ -2626,77 +2787,6 @@ static void rd_pick_sqr_partition(AV1_COMP *const cpi, ThreadData *td, } } -#define FEATURE_SIZE 19 -static const float two_pass_split_partition_weights_128[FEATURE_SIZE + 1] = { - 2.683936f, -0.193620f, -4.106470f, -0.141320f, -0.282289f, - 0.125296f, -1.134961f, 0.862757f, -0.418799f, -0.637666f, - 0.016232f, 0.345013f, 0.018823f, -0.393394f, -1.130700f, - 0.695357f, 0.112569f, -0.341975f, -0.513882f, 5.7488966f, -}; - -static const float two_pass_split_partition_weights_64[FEATURE_SIZE + 1] = { - 2.990993f, 0.423273f, -0.926544f, 0.454646f, -0.292698f, - -1.311632f, -0.284432f, 0.717141f, -0.419257f, -0.574760f, - -0.674444f, 0.669047f, -0.374255f, 0.380624f, -0.804036f, - 0.264021f, 0.004163f, 1.896802f, 0.924287f, 0.13490619f, -}; - -static const float two_pass_split_partition_weights_32[FEATURE_SIZE + 1] = { - 2.795181f, -0.136943f, -0.924842f, 0.405330f, -0.463505f, - -0.584076f, -0.831472f, 0.382985f, -0.597544f, -0.138915f, - -1.354350f, 0.466035f, -0.553961f, 0.213202f, -1.166429f, - 0.010776f, -0.096236f, 2.335084f, 1.699857f, -0.58178353f, -}; - -static const float two_pass_split_partition_weights_16[FEATURE_SIZE + 1] = { - 1.987888f, -0.431100f, -1.687703f, 0.262602f, -0.425298f, - -0.463870f, -1.493457f, 0.470917f, -0.528457f, -0.087700f, - -1.815092f, 0.152883f, -0.337908f, 0.093679f, -1.548267f, - -0.042387f, -0.000861f, 2.556746f, 1.619192f, 0.03643292f, -}; - -static const float two_pass_split_partition_weights_8[FEATURE_SIZE + 1] = { - 2.188344f, -0.817528f, -2.119219f, 0.000000f, -0.348167f, - -0.658074f, -1.960362f, 0.000000f, -0.403080f, 0.282699f, - -2.061088f, 0.000000f, -0.431919f, -0.127960f, -1.099550f, - 0.000000f, 0.121622f, 2.017455f, 2.058228f, -0.15475988f, -}; - -static const float two_pass_none_partition_weights_128[FEATURE_SIZE + 1] = { - -1.006689f, 0.777908f, 4.461072f, -0.395782f, -0.014610f, - -0.853863f, 0.729997f, -0.420477f, 0.282429f, -1.194595f, - 3.181220f, -0.511416f, 0.117084f, -1.149348f, 1.507990f, - -0.477212f, 0.202963f, -1.469581f, 0.624461f, -0.89081228f, -}; - -static const float two_pass_none_partition_weights_64[FEATURE_SIZE + 1] = { - -1.241117f, 0.844878f, 5.638803f, -0.489780f, -0.108796f, - -4.576821f, 1.540624f, -0.477519f, 0.227791f, -1.443968f, - 1.586911f, -0.505125f, 0.140764f, -0.464194f, 1.466658f, - -0.641166f, 0.195412f, 1.427905f, 2.080007f, -1.98272777f, -}; - -static const float two_pass_none_partition_weights_32[FEATURE_SIZE + 1] = { - -2.130825f, 0.476023f, 5.907343f, -0.516002f, -0.097471f, - -2.662754f, 0.614858f, -0.576728f, 0.085261f, -0.031901f, - 0.727842f, -0.600034f, 0.079326f, 0.324328f, 0.504502f, - -0.547105f, -0.037670f, 0.304995f, 0.369018f, -2.66299987f, -}; - -static const float two_pass_none_partition_weights_16[FEATURE_SIZE + 1] = { - -1.626410f, 0.872047f, 5.414965f, -0.554781f, -0.084514f, - -3.020550f, 0.467632f, -0.382280f, 0.199568f, 0.426220f, - 0.829426f, -0.467100f, 0.153098f, 0.662994f, 0.327545f, - -0.560106f, -0.141610f, 0.403372f, 0.523991f, -3.02891231f, -}; - -static const float two_pass_none_partition_weights_8[FEATURE_SIZE + 1] = { - -1.463349f, 0.375376f, 4.751430f, 0.000000f, -0.184451f, - -1.655447f, 0.443214f, 0.000000f, 0.127961f, 0.152435f, - 0.083288f, 0.000000f, 0.143105f, 0.438012f, 0.073238f, - 0.000000f, -0.278137f, 0.186134f, 0.073737f, -1.6494962f, -}; - // split_score indicates confidence of picking split partition; // none_score indicates confidence of picking none partition; static int ml_prune_2pass_split_partition(const PC_TREE_STATS *pc_tree_stats, @@ -2819,7 +2909,7 @@ static void ml_prune_rect_partition(const AV1_COMP *const cpi, // Variance ratios const MACROBLOCKD *const xd = &x->e_mbd; int whole_block_variance; - if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + if (is_cur_buf_hbd(xd)) { whole_block_variance = av1_high_get_sby_perpixel_variance( cpi, &x->plane[0].src, bsize, xd->bd); } else { @@ -2837,7 +2927,7 @@ static void ml_prune_rect_partition(const AV1_COMP *const cpi, const int x_idx = (i & 1) * bw / 2; const int y_idx = (i >> 1) * bw / 2; buf.buf = x->plane[0].src.buf + x_idx + y_idx * buf.stride; - if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + if (is_cur_buf_hbd(xd)) { split_variance[i] = av1_high_get_sby_perpixel_variance(cpi, &buf, subsize, xd->bd); } else { @@ -2851,6 +2941,7 @@ static void ml_prune_rect_partition(const AV1_COMP *const cpi, // 2. Do the prediction and prune 0-2 partitions based on their probabilities float raw_scores[3] = { 0.0f }; av1_nn_predict(features, nn_config, raw_scores); + aom_clear_system_state(); float probs[3] = { 0.0f }; av1_nn_softmax(raw_scores, probs, 3); @@ -2918,6 +3009,7 @@ static void ml_prune_ab_partition(BLOCK_SIZE bsize, int part_ctx, int var_ctx, // Calculate scores using the NN model. float score[16] = { 0.0f }; av1_nn_predict(features, nn_config, score); + aom_clear_system_state(); int int_score[16]; int max_score = -1000; for (int i = 0; i < 16; ++i) { @@ -3008,7 +3100,7 @@ static void ml_prune_4_partition(const AV1_COMP *const cpi, MACROBLOCK *const x, BLOCK_SIZE horz_4_bs = get_partition_subsize(bsize, PARTITION_HORZ_4); BLOCK_SIZE vert_4_bs = get_partition_subsize(bsize, PARTITION_VERT_4); av1_setup_src_planes(x, cpi->source, mi_row, mi_col, - av1_num_planes(&cpi->common)); + av1_num_planes(&cpi->common), bsize); const int src_stride = x->plane[0].src.stride; const uint8_t *src = x->plane[0].src.buf; const MACROBLOCKD *const xd = &x->e_mbd; @@ -3017,7 +3109,7 @@ static void ml_prune_4_partition(const AV1_COMP *const cpi, MACROBLOCK *const x, src + i * block_size_high[horz_4_bs] * src_stride; const uint8_t *vert_src = src + i * block_size_wide[vert_4_bs]; unsigned int horz_var, vert_var, sse; - if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + if (is_cur_buf_hbd(xd)) { switch (xd->bd) { case 10: horz_var = cpi->fn_ptr[horz_4_bs].vf( @@ -3084,6 +3176,7 @@ static void ml_prune_4_partition(const AV1_COMP *const cpi, MACROBLOCK *const x, // Calculate scores using the NN model. float score[LABELS] = { 0.0f }; av1_nn_predict(features, nn_config, score); + aom_clear_system_state(); int int_score[LABELS]; int max_score = -1000; for (int i = 0; i < LABELS; ++i) { @@ -3168,18 +3261,495 @@ static int ml_predict_breakout(const AV1_COMP *const cpi, BLOCK_SIZE bsize, // Calculate score using the NN model. float score = 0.0f; av1_nn_predict(features, nn_config, &score); + aom_clear_system_state(); // Make decision. return (int)(score * 100) >= thresh; } #undef FEATURES -// TODO(jingning,jimbankoski,rbultje): properly skip partition types that are +#if CONFIG_ONE_PASS_SVM +#define FEATURES 24 +static void ml_op_svm_early_term(const AV1_COMP *const cpi, + const MACROBLOCK *const x, + const MACROBLOCKD *const xd, + const PICK_MODE_CONTEXT *ctx_none, + const RD_STATS *none_rdc, int pb_source_var, + BLOCK_SIZE bsize, float *const score) { + const float *ml_weights = NULL, *ml_mean = NULL, *ml_std = NULL; + if (bsize == BLOCK_128X128) { + ml_weights = av1_op_svm_early_term_weights_128; + ml_mean = av1_op_svm_early_term_mean_128; + ml_std = av1_op_svm_early_term_std_128; + } else if (bsize == BLOCK_64X64) { + ml_weights = av1_op_svm_early_term_weights_64; + ml_mean = av1_op_svm_early_term_mean_64; + ml_std = av1_op_svm_early_term_std_64; + } else if (bsize == BLOCK_32X32) { + ml_weights = av1_op_svm_early_term_weights_32; + ml_mean = av1_op_svm_early_term_mean_32; + ml_std = av1_op_svm_early_term_std_32; + } else if (bsize == BLOCK_16X16) { + ml_weights = av1_op_svm_early_term_weights_16; + ml_mean = av1_op_svm_early_term_mean_16; + ml_std = av1_op_svm_early_term_std_16; + } else { + assert(bsize == BLOCK_128X128 || bsize == BLOCK_64X64 || + bsize == BLOCK_32X32 || bsize == BLOCK_8X8); + } + if (ml_weights != NULL) { + // Compute some features + + float features[FEATURES] = { 0 }; + int f_idx = 0; + int r_idx = 0; + + // None features + // Get none stats + features[f_idx++] = none_rdc->rate; + features[f_idx++] = none_rdc->dist; + features[f_idx++] = none_rdc->rdcost; + features[f_idx++] = ctx_none->skip; + + // EOBS + features[f_idx++] = none_rdc->eob; + int scaled_eob = none_rdc->eob * 32 * 32; + features[f_idx++] = (1.0f + none_rdc->eob_0) / (4.0f + scaled_eob); + features[f_idx++] = (1.0f + none_rdc->eob_1) / (4.0f + scaled_eob); + features[f_idx++] = (1.0f + none_rdc->eob_2) / (4.0f + scaled_eob); + features[f_idx++] = (1.0f + none_rdc->eob_3) / (4.0f + scaled_eob); + + // Y_RD + features[f_idx++] = none_rdc->rd; + int64_t scaled_rd = none_rdc->rd * 32 * 32; + features[f_idx++] = (1.0f + none_rdc->rd_0) / (4.0f + scaled_rd); + features[f_idx++] = (1.0f + none_rdc->rd_1) / (4.0f + scaled_rd); + features[f_idx++] = (1.0f + none_rdc->rd_2) / (4.0f + scaled_rd); + features[f_idx++] = (1.0f + none_rdc->rd_3) / (4.0f + scaled_rd); + + // Q_SQUARED + features[f_idx++] = + (x->plane[0].dequant_QTX[0]) * (x->plane[0].dequant_QTX[0]); + + // SIZE + // Get size of surrounding blocks + int above_size = 18, left_size = 18; + const MB_MODE_INFO *above_block = xd->above_mbmi; + const MB_MODE_INFO *left_block = xd->left_mbmi; + + if (above_block) { + above_size = above_block->sb_type; + } + if (left_block) { + left_size = left_block->sb_type; + } + + features[f_idx++] = left_size; + features[f_idx++] = left_size != 18; + + features[f_idx++] = above_size; + features[f_idx++] = above_size != 18; + + // Variance + // Get variance + int var = pb_source_var, var_reg[4] = { 0 }; + const int bw = block_size_wide[bsize]; + const int bh = block_size_high[bsize]; + const BLOCK_SIZE split_size = get_partition_subsize(bsize, PARTITION_SPLIT); + struct buf_2d buf; + buf.stride = x->plane[0].src.stride; + for (int i = 0; i < 4; ++i) { + const int x_idx = (i & 1) * bw / 2; + const int y_idx = (i >> 1) * bh / 2; + buf.buf = x->plane[0].src.buf + x_idx + y_idx * buf.stride; + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + var_reg[i] = + av1_high_get_sby_perpixel_variance(cpi, &buf, split_size, xd->bd); + } else { + var_reg[i] = av1_get_sby_perpixel_variance(cpi, &buf, split_size); + } + } + + features[f_idx++] = var; + for (r_idx = 0; r_idx < 4; r_idx++) { + features[f_idx] = (var_reg[r_idx] + 1.0f) / (var + 4.0f); + f_idx++; + } + + assert(f_idx == FEATURES); + + // Calculate the score + *score = 0.0f; + for (f_idx = 0; f_idx < FEATURES; f_idx++) { + *score += ml_weights[f_idx] * (features[f_idx] - ml_mean[f_idx]) / + ml_std[f_idx]; + } + // Dont forget the bias + *score += ml_weights[FEATURES]; + } +} +#undef FEATURES +#endif + +// Performs a full_pixel_motion_search with a single reference frame and extract +// the variance of residues. Here features is assumed to be a length 6 array. +// After this function is called, we will store the following in to features: +// features[0] = log(1 + dc_q**2/256) +// features[1] = log(1 + variance_of_residue) +// for i in [2, 3, 4, 5]: +// features[i] = log(1 + variance_of_residue_in_block[i]/variance_of_residue) +static void get_res_var_features(AV1_COMP *const cpi, MACROBLOCK *x, int mi_row, + int mi_col, BLOCK_SIZE bsize, + float *features) { + // TODO(chiyotsai@google.com): The data this model trained on did not also use + // SIMPLE_TRANSLATION to build the inter_predictor. Retraining and tuning the + // model with the correct data should give better performance. + assert(mi_size_wide[bsize] == mi_size_high[bsize]); + + MACROBLOCKD *xd = &x->e_mbd; + + // Perform a single motion search in Y_PLANE to make a prediction + const MV_REFERENCE_FRAME ref = + cpi->rc.is_src_frame_alt_ref ? ALTREF_FRAME : LAST_FRAME; + const int use_subpixel = 0; + const int num_planes = 1; + + const MV ref_mv_full = { .row = 0, .col = 0 }; + simple_motion_search(cpi, x, mi_row, mi_col, bsize, ref, ref_mv_full, + num_planes, use_subpixel); + aom_clear_system_state(); + + // Start getting the features + int f_idx = 0; + + // Q_INDEX + const int dc_q = av1_dc_quant_QTX(x->qindex, 0, xd->bd) >> (xd->bd - 8); + features[f_idx++] = logf(1.0f + (float)(dc_q * dc_q) / 256.0f); + + // VARIANCE + const uint8_t *src = x->plane[0].src.buf; + const int src_stride = x->plane[0].src.stride; + const uint8_t *dst = xd->plane[0].dst.buf; + const int dst_stride = xd->plane[0].dst.stride; + unsigned int sse = 0; + + // Whole block + const unsigned int var = + cpi->fn_ptr[bsize].vf(src, src_stride, dst, dst_stride, &sse); + features[f_idx++] = logf(1.0f + (float)var); + + // Regional + const int bw = block_size_wide[bsize]; + const int bh = block_size_high[bsize]; + const BLOCK_SIZE subsize = get_partition_subsize(bsize, PARTITION_SPLIT); + int r_idx = 0; + for (r_idx = 0; r_idx < 4; r_idx++) { + const int x_idx = (r_idx & 1) * bw / 2; + const int y_idx = (r_idx >> 1) * bh / 2; + const int src_offset = y_idx * src_stride + x_idx; + const int dst_offset = y_idx * dst_stride + x_idx; + const unsigned int sub_var = cpi->fn_ptr[subsize].vf( + src + src_offset, src_stride, dst + dst_offset, dst_stride, &sse); + const float var_ratio = (1.0f + (float)sub_var) / (4.0f + (float)var); + features[f_idx++] = var_ratio; + } +} + +static void full_pixel_motion_search_based_split( + AV1_COMP *const cpi, MACROBLOCK *x, int mi_row, int mi_col, + BLOCK_SIZE bsize, int *partition_none_allowed, int *partition_horz_allowed, + int *partition_vert_allowed, int *do_rectangular_split) { + const NN_CONFIG *nn_config = NULL; + float split_only_thresh = 0.0f; + if (bsize == BLOCK_128X128) { + nn_config = &full_pixel_motion_search_based_split_nn_config_128; + split_only_thresh = full_pixel_motion_search_based_split_thresh_128; + } else if (bsize == BLOCK_64X64) { + nn_config = &full_pixel_motion_search_based_split_nn_config_64; + split_only_thresh = full_pixel_motion_search_based_split_thresh_64; + } else if (bsize == BLOCK_32X32) { + nn_config = &full_pixel_motion_search_based_split_nn_config_32; + split_only_thresh = full_pixel_motion_search_based_split_thresh_32; + } else if (bsize == BLOCK_16X16) { + nn_config = &full_pixel_motion_search_based_split_nn_config_16; + split_only_thresh = full_pixel_motion_search_based_split_thresh_16; + } else if (bsize == BLOCK_8X8) { + // Disable BLOCK_8X8 for now +#if !CONFIG_DISABLE_FULL_PIXEL_SPLIT_8X8 + nn_config = &full_pixel_motion_search_based_split_nn_config_8; + split_only_thresh = full_pixel_motion_search_based_split_thresh_8; +#endif + } else { + assert(0 && "Unexpected block size in full_pixel_motion_based_split"); + } + if (nn_config) { + float features[6] = { 0 }; + float score = 0; + get_res_var_features(cpi, x, mi_row, mi_col, bsize, features); + av1_nn_predict(features, nn_config, &score); + + if (score > split_only_thresh) { + *partition_none_allowed = 0; + *partition_horz_allowed = 0; + *partition_vert_allowed = 0; + *do_rectangular_split = 0; + } + } +} + +// Given a list of ref frames in refs, performs simple_motion_search on each of +// the refs and returns the ref with the smallest sse. Returns -1 if none of the +// ref in the list is available. Also stores the best sse and var in best_sse, +// best_var, respectively. If save_mv_code is -1, don't update mv_ref_fulls in +// pc_tree. If save_mv_code is between 0 and 3, update mv_ref_fulls under +// pc_tree->split[i]. If save_mv_code is 4, update mv_ref_fulls under pc_tree. +static int simple_motion_search_get_best_ref( + AV1_COMP *const cpi, MACROBLOCK *x, PC_TREE *pc_tree, int mi_row, + int mi_col, BLOCK_SIZE bsize, const int *const refs, int num_refs, + int use_subpixel, int save_mv_code, unsigned int *best_sse, + unsigned int *best_var) { + // TODO(chiyotsai@google.com): The calculation of variance currently uses + // bsize, so we might take area outside of the image into account. We need to + // modify the SIMD functions to fix this later. + const AV1_COMMON *const cm = &cpi->common; + int best_ref = -1; + + if (mi_col >= cm->mi_cols || mi_row >= cm->mi_rows) { + // If the whole block is outside of the image, set the var and sse to 0. + *best_var = 0; + *best_sse = 0; + + return best_ref; + } + + // Otherwise do loop through the reference frames and find the one with the + // minimum SSE + const MACROBLOCKD *xd = &x->e_mbd; + const MV *mv_ref_fulls = pc_tree->mv_ref_fulls; + + const int num_planes = 1; + + *best_sse = INT_MAX; + + for (int ref_idx = 0; ref_idx < num_refs; ref_idx++) { + const int ref = refs[ref_idx]; + + if (cpi->ref_frame_flags & ref_frame_flag_list[ref]) { + unsigned int curr_sse = 0, curr_var = 0; + simple_motion_search(cpi, x, mi_row, mi_col, bsize, ref, + mv_ref_fulls[ref], num_planes, use_subpixel); + curr_var = cpi->fn_ptr[bsize].vf( + x->plane[0].src.buf, x->plane[0].src.stride, xd->plane[0].dst.buf, + xd->plane[0].dst.stride, &curr_sse); + if (curr_sse < *best_sse) { + *best_sse = curr_sse; + *best_var = curr_var; + best_ref = ref; + } + + if (save_mv_code == 4) { + pc_tree->mv_ref_fulls[ref].row = x->best_mv.as_mv.row / 8; + pc_tree->mv_ref_fulls[ref].col = x->best_mv.as_mv.col / 8; + } else if (save_mv_code >= 0 && save_mv_code < 4) { + // Propagate the new motion vectors to a lower level + pc_tree->split[save_mv_code]->mv_ref_fulls[ref].row = + x->best_mv.as_mv.row / 8; + pc_tree->split[save_mv_code]->mv_ref_fulls[ref].col = + x->best_mv.as_mv.col / 8; + } + } + } + + return best_ref; +} + +// Performs fullpixel simple_motion_search with LAST_FRAME and ALTREF_FRAME on +// each subblocks and extract the variance and sse of residues. Then store the +// var and sse from each partition subblock to features. The DC qindex is also +// stored in features. +// Here features is assumed to be a length 19 array. +// After this function is called, we will store the following to features: +// features[0:17] = var and sse from subblocks +// features[18] = DC q_index +#define NUM_FEATURES 19 +static void simple_motion_search_prune_part_features( + AV1_COMP *const cpi, MACROBLOCK *x, PC_TREE *pc_tree, int mi_row, + int mi_col, BLOCK_SIZE bsize, float *features) { + // TODO(chiyotsai@google.com): Cache the result of the motion search from the + // larger bbsize. + const int w_mi = mi_size_wide[bsize]; + const int h_mi = mi_size_high[bsize]; + int f_idx = 0; + assert(mi_size_wide[bsize] == mi_size_high[bsize]); + assert(cpi->ref_frame_flags & ref_frame_flag_list[LAST_FRAME] || + cpi->ref_frame_flags & ref_frame_flag_list[ALTREF_FRAME]); + + // Setting up motion search + const int ref_list[] = { LAST_FRAME, ALTREF_FRAME }; + const int num_refs = 2; + const int use_subpixel = 0; + + unsigned int none_sse = 0, none_var = 0; + unsigned int int_features[NUM_FEATURES - 1]; + + // Doing whole block first to update the mv + simple_motion_search_get_best_ref(cpi, x, pc_tree, mi_row, mi_col, bsize, + ref_list, num_refs, use_subpixel, 4, + &none_sse, &none_var); + + // Split subblocks + BLOCK_SIZE subsize = get_partition_subsize(bsize, PARTITION_SPLIT); + int r_idx = 0; + for (r_idx = 0; r_idx < 4; r_idx++) { + const int sub_mi_col = mi_col + (r_idx & 1) * w_mi / 2; + const int sub_mi_row = mi_row + (r_idx >> 1) * h_mi / 2; + + simple_motion_search_get_best_ref( + cpi, x, pc_tree, sub_mi_row, sub_mi_col, subsize, ref_list, num_refs, + use_subpixel, r_idx, &int_features[f_idx + 1], &int_features[f_idx]); + f_idx += 2; + } + + // Horz subblocks + subsize = get_partition_subsize(bsize, PARTITION_HORZ); + for (r_idx = 0; r_idx < 2; r_idx++) { + const int sub_mi_col = mi_col + 0; + const int sub_mi_row = mi_row + r_idx * h_mi / 2; + + simple_motion_search_get_best_ref( + cpi, x, pc_tree, sub_mi_row, sub_mi_col, subsize, ref_list, num_refs, + use_subpixel, -1, &int_features[f_idx + 1], &int_features[f_idx]); + + f_idx += 2; + } + + // Vert subblock + subsize = get_partition_subsize(bsize, PARTITION_VERT); + for (r_idx = 0; r_idx < 2; r_idx++) { + const int sub_mi_col = mi_col + r_idx * w_mi / 2; + const int sub_mi_row = mi_row + 0; + + simple_motion_search_get_best_ref( + cpi, x, pc_tree, sub_mi_row, sub_mi_col, subsize, ref_list, num_refs, + use_subpixel, -1, &int_features[f_idx + 1], &int_features[f_idx]); + + f_idx += 2; + } + + // Whole block + int_features[f_idx++] = none_var; + int_features[f_idx++] = none_sse; + + aom_clear_system_state(); + for (int idx = 0; idx < f_idx; idx++) { + features[idx] = logf(1.0f + (float)int_features[idx]); + } + + const MACROBLOCKD *xd = &x->e_mbd; + set_offsets(cpi, &xd->tile, x, mi_row, mi_col, bsize); + + // Q_INDEX + const int dc_q = av1_dc_quant_QTX(x->qindex, 0, xd->bd) >> (xd->bd - 8); + features[f_idx++] = logf(1.0f + (float)(dc_q * dc_q) / 256.0f); + + assert(f_idx == NUM_FEATURES); +} + +#define MAX_NUM_CLASSES 10 +static void simple_motion_search_prune_part( + AV1_COMP *const cpi, MACROBLOCK *x, PC_TREE *pc_tree, int mi_row, + int mi_col, BLOCK_SIZE bsize, int *partition_none_allowed, + int *partition_horz_allowed, int *partition_vert_allowed, + int *do_square_split, int *do_rectangular_split, int *prune_horz, + int *prune_vert) { + const AV1_COMMON *const cm = &cpi->common; + // Get model parameters + const NN_CONFIG *nn_config = NULL; + const float *prune_thresh = NULL, *only_thresh = NULL; + const float *ml_mean = NULL, *ml_std = NULL; + + if (bsize == BLOCK_128X128) { + nn_config = &simple_motion_search_prune_part_nn_config_128; + ml_mean = simple_motion_search_prune_part_mean_128; + ml_std = simple_motion_search_prune_part_std_128; + prune_thresh = simple_motion_search_prune_part_prune_thresh_128; + only_thresh = simple_motion_search_prune_part_only_thresh_128; + } else if (bsize == BLOCK_64X64) { + nn_config = &simple_motion_search_prune_part_nn_config_64; + ml_mean = simple_motion_search_prune_part_mean_64; + ml_std = simple_motion_search_prune_part_std_64; + prune_thresh = simple_motion_search_prune_part_prune_thresh_64; + only_thresh = simple_motion_search_prune_part_only_thresh_64; + } else if (bsize == BLOCK_32X32) { + nn_config = &simple_motion_search_prune_part_nn_config_32; + ml_mean = simple_motion_search_prune_part_mean_32; + ml_std = simple_motion_search_prune_part_std_32; + prune_thresh = simple_motion_search_prune_part_prune_thresh_32; + only_thresh = simple_motion_search_prune_part_only_thresh_32; + } else if (bsize == BLOCK_16X16) { + nn_config = &simple_motion_search_prune_part_nn_config_16; + ml_mean = simple_motion_search_prune_part_mean_16; + ml_std = simple_motion_search_prune_part_std_16; + prune_thresh = simple_motion_search_prune_part_prune_thresh_16; + only_thresh = simple_motion_search_prune_part_only_thresh_16; + } else if (bsize == BLOCK_8X8) { + nn_config = &simple_motion_search_prune_part_nn_config_8; + ml_mean = simple_motion_search_prune_part_mean_8; + ml_std = simple_motion_search_prune_part_std_8; + prune_thresh = simple_motion_search_prune_part_prune_thresh_8; + only_thresh = simple_motion_search_prune_part_only_thresh_8; + } else { + assert(0 && "Unexpected block size in simple_motion_prune_part"); + } + + // If there is no valid threshold, return immediately. + if (!nn_config || (prune_thresh[PARTITION_HORZ] == 0.0f && + prune_thresh[PARTITION_VERT] == 0.0f)) { + return; + } + + // Get features + float features[NUM_FEATURES] = { 0.0f }; + simple_motion_search_prune_part_features(cpi, x, pc_tree, mi_row, mi_col, + bsize, features); + for (int f_idx = 0; f_idx < NUM_FEATURES; f_idx++) { + features[f_idx] = (features[f_idx] - ml_mean[f_idx]) / ml_std[f_idx]; + } + + // Get probabilities + float scores[MAX_NUM_CLASSES] = { 0.0f }, probs[MAX_NUM_CLASSES] = { 0.0f }; + const int num_classes = + (bsize == BLOCK_128X128 || bsize == BLOCK_8X8) ? 4 : 10; + + av1_nn_predict(features, nn_config, scores); + aom_clear_system_state(); + + av1_nn_softmax(scores, probs, num_classes); + + // Determine if we should prune rectangular partitions. + if (cpi->sf.simple_motion_search_prune_rect && !frame_is_intra_only(cm) && + (*partition_horz_allowed || *partition_vert_allowed) && + bsize >= BLOCK_8X8 && !av1_superres_scaled(cm)) { + *prune_horz = probs[PARTITION_HORZ] <= prune_thresh[PARTITION_HORZ]; + *prune_vert = probs[PARTITION_VERT] <= prune_thresh[PARTITION_VERT]; + } + + // Silence compiler warnings + (void)only_thresh; + (void)partition_none_allowed; + (void)do_square_split; + (void)do_rectangular_split; +} +#undef MAX_NUM_CLASSES +#undef NUM_FEATURES + +// TODO(jinging,jimbankoski,rbultje): properly skip partition types that are // unlikely to be selected depending on previous rate-distortion optimization // results, for encoding speed-up. static void rd_pick_partition(AV1_COMP *const cpi, ThreadData *td, TileDataEnc *tile_data, TOKENEXTRA **tp, int mi_row, int mi_col, BLOCK_SIZE bsize, + BLOCK_SIZE max_sq_part, BLOCK_SIZE min_sq_part, RD_STATS *rd_cost, int64_t best_rd, PC_TREE *pc_tree, int64_t *none_rd) { const AV1_COMMON *const cm = &cpi->common; @@ -3202,17 +3772,29 @@ static void rd_pick_partition(AV1_COMP *const cpi, ThreadData *td, const int *partition_cost = pl >= 0 ? x->partition_cost[pl] : x->partition_cost[0]; - int do_rectangular_split = 1; + int do_rectangular_split = cpi->oxcf.enable_rect_partitions; int64_t cur_none_rd = 0; int64_t split_rd[4] = { 0, 0, 0, 0 }; int64_t horz_rd[2] = { 0, 0 }; int64_t vert_rd[2] = { 0, 0 }; + int prune_horz = 0; + int prune_vert = 0; int split_ctx_is_ready[2] = { 0, 0 }; int horz_ctx_is_ready = 0; int vert_ctx_is_ready = 0; BLOCK_SIZE bsize2 = get_partition_subsize(bsize, PARTITION_SPLIT); + // Max and min square partition levels are defined as the partition nodes that + // the recursive function rd_pick_partition() can reach. To implement this: + // only PARTITION_SPLIT is NOT allowed if the current node equals min_sq_part, + // only PARTITION_SPLIT is allowed if the current node exceeds max_sq_part. + assert(block_size_wide[min_sq_part] == block_size_high[min_sq_part]); + assert(block_size_wide[max_sq_part] == block_size_high[max_sq_part]); + assert(min_sq_part <= max_sq_part); + int is_eq_min_sq_part = bsize == min_sq_part; + int is_gt_max_sq_part = bsize > max_sq_part; + if (best_rd < 0) { pc_tree->none.rdcost = INT64_MAX; pc_tree->none.skip = 0; @@ -3238,11 +3820,20 @@ static void rd_pick_partition(AV1_COMP *const cpi, ThreadData *td, #endif int partition_none_allowed = has_rows && has_cols; - int partition_horz_allowed = has_cols && yss <= xss && bsize_at_least_8x8; - int partition_vert_allowed = has_rows && xss <= yss && bsize_at_least_8x8; + int partition_horz_allowed = has_cols && yss <= xss && bsize_at_least_8x8 && + cpi->oxcf.enable_rect_partitions; + int partition_vert_allowed = has_rows && xss <= yss && bsize_at_least_8x8 && + cpi->oxcf.enable_rect_partitions; (void)*tp_orig; +#if CONFIG_COLLECT_PARTITION_STATS + PartitionStats *part_stats = &cpi->partition_stats; + const int bsize_idx = av1_get_bsize_idx_for_part_stats(bsize); + int *partition_decisions = part_stats->partition_decisions[bsize_idx]; + int *partition_attempts = part_stats->partition_attempts[bsize_idx]; +#endif + // Override partition costs at the edges of the frame in the same // way as in read_partition (see decodeframe.c) if (!(has_rows && has_cols)) { @@ -3292,7 +3883,7 @@ static void rd_pick_partition(AV1_COMP *const cpi, ThreadData *td, if (cpi->sf.cb_partition_search && bsize == BLOCK_16X16) { const int cb_partition_search_ctrl = ((pc_tree->index == 0 || pc_tree->index == 3) + - get_chessboard_index(cm->current_video_frame)) & + get_chessboard_index(cm->current_frame.frame_number)) & 0x1; if (cb_partition_search_ctrl && bsize > min_size && bsize < max_size) @@ -3362,8 +3953,10 @@ static void rd_pick_partition(AV1_COMP *const cpi, ThreadData *td, partition_horz_allowed == 0 && partition_vert_allowed == 0) { do_square_split = bsize_at_least_8x8; partition_none_allowed = has_rows && has_cols; - partition_horz_allowed = has_cols && yss <= xss && bsize_at_least_8x8; - partition_vert_allowed = has_rows && xss <= yss && bsize_at_least_8x8; + partition_horz_allowed = has_cols && yss <= xss && bsize_at_least_8x8 && + cpi->oxcf.enable_rect_partitions; + partition_vert_allowed = has_rows && xss <= yss && bsize_at_least_8x8 && + cpi->oxcf.enable_rect_partitions; } } @@ -3435,11 +4028,41 @@ static void rd_pick_partition(AV1_COMP *const cpi, ThreadData *td, 0, }; + MB_MODE_INFO *split_mbmi[4] = { 0 }; + + // Use simple_motion_search to prune partitions. This must be done prior to + // PARTITION_SPLIT to propagate the initial mvs to a smaller blocksize. + const int try_split_only = + cpi->sf.simple_motion_search_split_only && bsize >= BLOCK_8X8 && + do_square_split && mi_row + mi_size_high[bsize] <= cm->mi_rows && + mi_col + mi_size_wide[bsize] <= cm->mi_cols && !frame_is_intra_only(cm) && + !av1_superres_scaled(cm); + + if (try_split_only) { + full_pixel_motion_search_based_split( + cpi, x, mi_row, mi_col, bsize, &partition_none_allowed, + &partition_horz_allowed, &partition_vert_allowed, + &do_rectangular_split); + } + + const int try_prune_rect = + cpi->sf.simple_motion_search_prune_rect && !frame_is_intra_only(cm) && + (partition_horz_allowed || partition_vert_allowed) && bsize >= BLOCK_8X8; + + if (try_prune_rect) { + simple_motion_search_prune_part( + cpi, x, pc_tree, mi_row, mi_col, bsize, &partition_none_allowed, + &partition_horz_allowed, &partition_vert_allowed, &do_square_split, + &do_rectangular_split, &prune_horz, &prune_vert); + } + BEGIN_PARTITION_SEARCH: if (x->must_find_valid_partition) { partition_none_allowed = has_rows && has_cols; - partition_horz_allowed = has_cols && yss <= xss && bsize_at_least_8x8; - partition_vert_allowed = has_rows && xss <= yss && bsize_at_least_8x8; + partition_horz_allowed = has_cols && yss <= xss && bsize_at_least_8x8 && + cpi->oxcf.enable_rect_partitions; + partition_vert_allowed = has_rows && xss <= yss && bsize_at_least_8x8 && + cpi->oxcf.enable_rect_partitions; } // Partition block source pixel variance. @@ -3455,17 +4078,23 @@ BEGIN_PARTITION_SEARCH: #endif // PARTITION_NONE - if (partition_none_allowed) { + if (is_eq_min_sq_part) partition_none_allowed = 1; + if (partition_none_allowed && !is_gt_max_sq_part) { int pt_cost = 0; if (bsize_at_least_8x8) { pt_cost = partition_cost[PARTITION_NONE] < INT_MAX ? partition_cost[PARTITION_NONE] : 0; } - int64_t partition_rd_cost = RDCOST(x->rdmult, pt_cost, 0); - int64_t best_remain_rdcost = (best_rdc.rdcost == INT64_MAX) - ? INT64_MAX - : (best_rdc.rdcost - partition_rd_cost); + const int64_t partition_rd_cost = RDCOST(x->rdmult, pt_cost, 0); + const int64_t best_remain_rdcost = + (best_rdc.rdcost == INT64_MAX) ? INT64_MAX + : (best_rdc.rdcost - partition_rd_cost); +#if CONFIG_COLLECT_PARTITION_STATS + if (!frame_is_intra_only(cm) && best_remain_rdcost >= 0) { + partition_attempts[PARTITION_NONE] += 1; + } +#endif rd_pick_sb_modes(cpi, tile_data, x, mi_row, mi_col, &this_rdc, PARTITION_NONE, bsize, ctx_none, best_remain_rdcost); pb_source_variance = x->source_variance; @@ -3496,6 +4125,25 @@ BEGIN_PARTITION_SEARCH: best_rdc = this_rdc; if (bsize_at_least_8x8) pc_tree->partitioning = PARTITION_NONE; +#if CONFIG_ONE_PASS_SVM + // Use ML if the block size is square and >= 16X16 + if (bsize >= BLOCK_16X16 && !frame_is_intra_only(cm) && + this_rdc.rate < INT_MAX && this_rdc.rate >= 0 && + !ctx_none->seg_feat) { + // Model Prediction + float score = 0.0f; + ml_op_svm_early_term(cpi, x, xd, ctx_none, &this_rdc, + pb_source_variance, bsize, &score); + + // Decide if we want to terminate early + if (score >= 0) { + do_square_split = 0; + do_rectangular_split = 0; + partition_horz_allowed = 0; + partition_vert_allowed = 0; + } + } +#endif if ((do_square_split || do_rectangular_split) && !x->e_mbd.lossless[xd->mi[0]->segment_id] && ctx_none->skippable) { const int use_ml_based_breakout = @@ -3576,13 +4224,19 @@ BEGIN_PARTITION_SEARCH: if (cpi->sf.adaptive_motion_search) store_pred_mv(x, ctx_none); // PARTITION_SPLIT - if (do_square_split) { + if (is_eq_min_sq_part) do_square_split = 0; + if (do_square_split || is_gt_max_sq_part) { av1_init_rd_stats(&sum_rdc); subsize = get_partition_subsize(bsize, PARTITION_SPLIT); sum_rdc.rate = partition_cost[PARTITION_SPLIT]; sum_rdc.rdcost = RDCOST(x->rdmult, sum_rdc.rate, 0); int idx; +#if CONFIG_COLLECT_PARTITION_STATS + if (!frame_is_intra_only(cm) && best_rdc.rdcost - sum_rdc.rdcost >= 0) { + partition_attempts[PARTITION_SPLIT] += 1; + } +#endif for (idx = 0; idx < 4 && sum_rdc.rdcost < best_rdc.rdcost; ++idx) { const int x_idx = (idx & 1) * mi_step; const int y_idx = (idx >> 1) * mi_step; @@ -3594,14 +4248,14 @@ BEGIN_PARTITION_SEARCH: pc_tree->split[idx]->index = idx; int64_t *p_split_rd = &split_rd[idx]; - int64_t best_remain_rdcost = best_rdc.rdcost == INT64_MAX - ? INT64_MAX + const int64_t best_remain_rdcost = + best_rdc.rdcost == INT64_MAX ? INT64_MAX : (best_rdc.rdcost - sum_rdc.rdcost); if (cpi->sf.prune_ref_frame_for_rect_partitions) pc_tree->split[idx]->none.rate = INT_MAX; rd_pick_partition(cpi, td, tile_data, tp, mi_row + y_idx, mi_col + x_idx, - subsize, &this_rdc, best_remain_rdcost, - pc_tree->split[idx], p_split_rd); + subsize, max_sq_part, min_sq_part, &this_rdc, + best_remain_rdcost, pc_tree->split[idx], p_split_rd); if (this_rdc.rate == INT_MAX) { sum_rdc.rdcost = INT64_MAX; @@ -3615,6 +4269,10 @@ BEGIN_PARTITION_SEARCH: const int ref_type = av1_ref_frame_type(pc_tree->split[idx]->none.mic.ref_frame); ref_frames_used[idx] |= (1 << ref_type); + + if (cpi->sf.prune_ref_mode_for_partitions) { + split_mbmi[idx] = &pc_tree->split[idx]->none.mic; + } } if (idx <= 1 && (bsize <= BLOCK_8X8 || pc_tree->split[idx]->partitioning == PARTITION_NONE)) { @@ -3662,18 +4320,93 @@ BEGIN_PARTITION_SEARCH: if (used_frames) pc_tree->vertical[1].skip_ref_frame_mask = ~used_frames; } - int prune_horz = 0; - int prune_vert = 0; + for (int i = 0; i < 2; ++i) { + pc_tree->horizontal[i].ref_selected[0] = + pc_tree->horizontal[i].ref_selected[1] = NONE_FRAME; + pc_tree->horizontal[i].mode_selected = -1; + pc_tree->vertical[i].ref_selected[0] = + pc_tree->vertical[i].ref_selected[1] = NONE_FRAME; + pc_tree->vertical[i].mode_selected = -1; + } + + if (cpi->sf.prune_ref_mode_for_partitions) { + // horizontal partition + for (int idx = 0; idx < 4; idx += 2) { + const int horz_idx = idx / 2; + if (split_mbmi[idx] && split_mbmi[idx + 1] && + split_mbmi[idx]->ref_frame[0] > INTRA_FRAME) { + if (!has_second_ref(split_mbmi[idx])) { + // Single ref + if (split_mbmi[idx]->ref_frame[0] == + split_mbmi[idx + 1]->ref_frame[0] && + !has_second_ref(split_mbmi[idx + 1])) { + const int ref_type = av1_ref_frame_type(split_mbmi[idx]->ref_frame); + // Overwrite skip_ref_frame_mask for the current block + const int used_frames = (1 << ref_type); + pc_tree->horizontal[horz_idx].skip_ref_frame_mask = ~used_frames; + pc_tree->horizontal[horz_idx].ref_selected[0] = + split_mbmi[idx]->ref_frame[0]; +#if 0 + // TODO(zoeliu@gmail.com): To consider the scenario of obmc + if (split_mbmi[idx]->motion_mode == + split_mbmi[idx + 1]->motion_mode && + split_mbmi[idx]->motion_mode == SIMPLE_TRANSLATION && + split_mbmi[idx]->use_wedge_interintra == 0) { + pc_tree->horizontal[horz_idx].mode_selected = SIMPLE_TRANSLATION; + } +#endif // 0 + } + } else { + // TODO(zoeliu@gmail.com): To handle comp ref + } + } + } + // vertical partition + for (int idx = 0; idx < 2; ++idx) { + const int vert_idx = idx; + if (split_mbmi[idx] && split_mbmi[idx + 2] && + split_mbmi[idx]->ref_frame[0] > INTRA_FRAME) { + if (!has_second_ref(split_mbmi[idx])) { + // Single ref + if (split_mbmi[idx]->ref_frame[0] == + split_mbmi[idx + 2]->ref_frame[0] && + !has_second_ref(split_mbmi[idx + 2])) { + const int ref_type = av1_ref_frame_type(split_mbmi[idx]->ref_frame); + // Overwrite skip_ref_frame_mask for the current block + const int used_frames = (1 << ref_type); + pc_tree->vertical[vert_idx].skip_ref_frame_mask = ~used_frames; + pc_tree->vertical[vert_idx].ref_selected[0] = + split_mbmi[idx]->ref_frame[0]; +#if 0 + // TODO(zoeliu@gmail.com): To consider the scenario of obmc + if (split_mbmi[idx]->motion_mode == + split_mbmi[idx + 2]->motion_mode && + split_mbmi[idx]->motion_mode == SIMPLE_TRANSLATION && + split_mbmi[idx]->use_wedge_interintra == 0) { + pc_tree->vertical[vert_idx].mode_selected = SIMPLE_TRANSLATION; + } +#endif // 0 + } + } else { + // TODO(zoeliu@gmail.com): To handle comp ref + } + } + } + } + if (cpi->sf.ml_prune_rect_partition && !frame_is_intra_only(cm) && - (partition_horz_allowed || partition_vert_allowed)) { - av1_setup_src_planes(x, cpi->source, mi_row, mi_col, num_planes); + (partition_horz_allowed || partition_vert_allowed) && + !(prune_horz || prune_vert)) { + av1_setup_src_planes(x, cpi->source, mi_row, mi_col, num_planes, bsize); ml_prune_rect_partition(cpi, x, bsize, best_rdc.rdcost, cur_none_rd, split_rd, &prune_horz, &prune_vert); } // PARTITION_HORZ + assert(IMPLIES(!cpi->oxcf.enable_rect_partitions, !partition_horz_allowed)); if (partition_horz_allowed && !prune_horz && - (do_rectangular_split || active_h_edge(cpi, mi_row, mi_step))) { + (do_rectangular_split || active_h_edge(cpi, mi_row, mi_step)) && + !is_gt_max_sq_part) { av1_init_rd_stats(&sum_rdc); subsize = get_partition_subsize(bsize, PARTITION_HORZ); if (cpi->sf.adaptive_motion_search) load_pred_mv(x, ctx_none); @@ -3682,11 +4415,16 @@ BEGIN_PARTITION_SEARCH: pc_tree->horizontal[0].pred_interp_filter = av1_extract_interp_filter(ctx_none->mic.interp_filters, 0); } - int64_t best_remain_rdcost = best_rdc.rdcost == INT64_MAX - ? INT64_MAX - : (best_rdc.rdcost - sum_rdc.rdcost); sum_rdc.rate = partition_cost[PARTITION_HORZ]; sum_rdc.rdcost = RDCOST(x->rdmult, sum_rdc.rate, 0); + const int64_t best_remain_rdcost = best_rdc.rdcost == INT64_MAX + ? INT64_MAX + : (best_rdc.rdcost - sum_rdc.rdcost); +#if CONFIG_COLLECT_PARTITION_STATS + if (!frame_is_intra_only(cm) && best_remain_rdcost >= 0) { + partition_attempts[PARTITION_HORZ] += 1; + } +#endif rd_pick_sb_modes(cpi, tile_data, x, mi_row, mi_col, &this_rdc, PARTITION_HORZ, subsize, &pc_tree->horizontal[0], best_remain_rdcost); @@ -3745,8 +4483,10 @@ BEGIN_PARTITION_SEARCH: } // PARTITION_VERT + assert(IMPLIES(!cpi->oxcf.enable_rect_partitions, !partition_vert_allowed)); if (partition_vert_allowed && !prune_vert && - (do_rectangular_split || active_v_edge(cpi, mi_col, mi_step))) { + (do_rectangular_split || active_v_edge(cpi, mi_col, mi_step)) && + !is_gt_max_sq_part) { av1_init_rd_stats(&sum_rdc); subsize = get_partition_subsize(bsize, PARTITION_VERT); @@ -3759,9 +4499,14 @@ BEGIN_PARTITION_SEARCH: } sum_rdc.rate = partition_cost[PARTITION_VERT]; sum_rdc.rdcost = RDCOST(x->rdmult, sum_rdc.rate, 0); - int64_t best_remain_rdcost = best_rdc.rdcost == INT64_MAX - ? INT64_MAX - : (best_rdc.rdcost - sum_rdc.rdcost); + const int64_t best_remain_rdcost = best_rdc.rdcost == INT64_MAX + ? INT64_MAX + : (best_rdc.rdcost - sum_rdc.rdcost); +#if CONFIG_COLLECT_PARTITION_STATS + if (!frame_is_intra_only(cm) && best_remain_rdcost >= 0) { + partition_attempts[PARTITION_VERT] += 1; + } +#endif rd_pick_sb_modes(cpi, tile_data, x, mi_row, mi_col, &this_rdc, PARTITION_VERT, subsize, &pc_tree->vertical[0], best_remain_rdcost); @@ -3774,8 +4519,7 @@ BEGIN_PARTITION_SEARCH: sum_rdc.rdcost += this_rdc.rdcost; } vert_rd[0] = this_rdc.rdcost; - const int64_t vert_max_rdcost = best_rdc.rdcost; - if (sum_rdc.rdcost < vert_max_rdcost && has_cols) { + if (sum_rdc.rdcost < best_rdc.rdcost && has_cols) { const MB_MODE_INFO *const mbmi = &pc_tree->vertical[0].mic; const PALETTE_MODE_INFO *const pmi = &mbmi->palette_mode_info; // Neither palette mode nor cfl predicted @@ -3820,8 +4564,8 @@ BEGIN_PARTITION_SEARCH: } if (pb_source_variance == UINT_MAX) { - av1_setup_src_planes(x, cpi->source, mi_row, mi_col, num_planes); - if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + av1_setup_src_planes(x, cpi->source, mi_row, mi_col, num_planes, bsize); + if (is_cur_buf_hbd(xd)) { pb_source_variance = av1_high_get_sby_perpixel_variance( cpi, &x->plane[0].src, bsize, xd->bd); } else { @@ -3830,6 +4574,8 @@ BEGIN_PARTITION_SEARCH: } } + assert(IMPLIES(!cpi->oxcf.enable_rect_partitions, !do_rectangular_split)); + const int ext_partition_allowed = do_rectangular_split && bsize > BLOCK_8X8 && partition_none_allowed; @@ -3912,9 +4658,9 @@ BEGIN_PARTITION_SEARCH: if (cpi->sf.ml_prune_ab_partition && ext_partition_allowed && partition_horz_allowed && partition_vert_allowed) { - // TODO(huisu@google.com): x->source_variance may not be the current block's - // variance. The correct one to use is pb_source_variance. - // Need to re-train the model to fix it. + // TODO(huisu@google.com): x->source_variance may not be the current + // block's variance. The correct one to use is pb_source_variance. Need to + // re-train the model to fix it. ml_prune_ab_partition(bsize, pc_tree->partitioning, get_unsigned_bits(x->source_variance), best_rdc.rdcost, horz_rd, vert_rd, split_rd, @@ -3923,7 +4669,7 @@ BEGIN_PARTITION_SEARCH: } // PARTITION_HORZ_A - if (partition_horz_allowed && horza_partition_allowed) { + if (partition_horz_allowed && horza_partition_allowed && !is_gt_max_sq_part) { subsize = get_partition_subsize(bsize, PARTITION_HORZ_A); pc_tree->horizontala[0].rd_mode_is_ready = 0; pc_tree->horizontala[1].rd_mode_is_ready = 0; @@ -3939,9 +4685,11 @@ BEGIN_PARTITION_SEARCH: pc_tree->horizontala[1].rd_mode_is_ready = 1; } } - pc_tree->horizontala[0].skip_ref_frame_mask = 0; - pc_tree->horizontala[1].skip_ref_frame_mask = 0; - pc_tree->horizontala[2].skip_ref_frame_mask = 0; + for (int i = 0; i < 3; ++i) { + pc_tree->horizontala[i].skip_ref_frame_mask = 0; + pc_tree->horizontala[i].ref_selected[0] = + pc_tree->horizontala[i].ref_selected[1] = NONE_FRAME; + } if (cpi->sf.prune_ref_frame_for_rect_partitions) { int used_frames; used_frames = ref_frames_used[0]; @@ -3954,6 +4702,42 @@ BEGIN_PARTITION_SEARCH: if (used_frames) pc_tree->horizontala[2].skip_ref_frame_mask = ~used_frames; } + if (cpi->sf.prune_ref_mode_for_partitions) { + // Overwrite skip_ref_frame_mask for the current block + if (split_mbmi[0] && split_mbmi[0]->ref_frame[0] > INTRA_FRAME && + !has_second_ref(split_mbmi[0])) { // single ref + const int used_frames = 1 << (int)split_mbmi[0]->ref_frame[0]; + pc_tree->horizontala[0].skip_ref_frame_mask = ~used_frames; + pc_tree->horizontala[0].ref_selected[0] = split_mbmi[0]->ref_frame[0]; + } + if (split_mbmi[1] && split_mbmi[1]->ref_frame[0] > INTRA_FRAME && + !has_second_ref(split_mbmi[1])) { // single ref + const int used_frames = 1 << (int)split_mbmi[1]->ref_frame[0]; + pc_tree->horizontala[1].skip_ref_frame_mask = ~used_frames; + pc_tree->horizontala[1].ref_selected[0] = split_mbmi[1]->ref_frame[0]; + } + if (split_mbmi[2] && split_mbmi[3] && + split_mbmi[2]->ref_frame[0] > INTRA_FRAME && + split_mbmi[2]->ref_frame[0] == split_mbmi[3]->ref_frame[0] && + !has_second_ref(split_mbmi[2]) && + !has_second_ref(split_mbmi[3])) { // single ref + const int used_frames = 1 << (int)split_mbmi[2]->ref_frame[0]; + pc_tree->horizontala[2].skip_ref_frame_mask = ~used_frames; + pc_tree->horizontala[2].ref_selected[0] = split_mbmi[2]->ref_frame[0]; + } + } +#if CONFIG_COLLECT_PARTITION_STATS + { + RD_STATS tmp_sum_rdc; + av1_init_rd_stats(&tmp_sum_rdc); + tmp_sum_rdc.rate = x->partition_cost[pl][PARTITION_HORZ_A]; + tmp_sum_rdc.rdcost = RDCOST(x->rdmult, tmp_sum_rdc.rate, 0); + if (!frame_is_intra_only(cm) && + best_rdc.rdcost - tmp_sum_rdc.rdcost >= 0) { + partition_attempts[PARTITION_HORZ_A] += 1; + } + } +#endif rd_test_partition3(cpi, td, tile_data, tp, pc_tree, &best_rdc, pc_tree->horizontala, ctx_none, mi_row, mi_col, bsize, PARTITION_HORZ_A, mi_row, mi_col, bsize2, mi_row, @@ -3962,7 +4746,7 @@ BEGIN_PARTITION_SEARCH: restore_context(x, &x_ctx, mi_row, mi_col, bsize, num_planes); } // PARTITION_HORZ_B - if (partition_horz_allowed && horzb_partition_allowed) { + if (partition_horz_allowed && horzb_partition_allowed && !is_gt_max_sq_part) { subsize = get_partition_subsize(bsize, PARTITION_HORZ_B); pc_tree->horizontalb[0].rd_mode_is_ready = 0; pc_tree->horizontalb[1].rd_mode_is_ready = 0; @@ -3972,9 +4756,11 @@ BEGIN_PARTITION_SEARCH: pc_tree->horizontalb[0].mic.partition = PARTITION_HORZ_B; pc_tree->horizontalb[0].rd_mode_is_ready = 1; } - pc_tree->horizontalb[0].skip_ref_frame_mask = 0; - pc_tree->horizontalb[1].skip_ref_frame_mask = 0; - pc_tree->horizontalb[2].skip_ref_frame_mask = 0; + for (int i = 0; i < 3; ++i) { + pc_tree->horizontalb[i].skip_ref_frame_mask = 0; + pc_tree->horizontalb[i].ref_selected[0] = + pc_tree->horizontalb[i].ref_selected[1] = NONE_FRAME; + } if (cpi->sf.prune_ref_frame_for_rect_partitions) { int used_frames; used_frames = ref_frames_used[0] | ref_frames_used[1]; @@ -3987,6 +4773,42 @@ BEGIN_PARTITION_SEARCH: if (used_frames) pc_tree->horizontalb[2].skip_ref_frame_mask = ~used_frames; } + if (cpi->sf.prune_ref_mode_for_partitions) { + // Overwrite skip_ref_frame_mask for the current block + if (split_mbmi[0] && split_mbmi[1] && + split_mbmi[0]->ref_frame[0] > INTRA_FRAME && + split_mbmi[0]->ref_frame[0] == split_mbmi[1]->ref_frame[0] && + !has_second_ref(split_mbmi[0]) && + !has_second_ref(split_mbmi[1])) { // single ref + const int used_frames = 1 << (int)split_mbmi[0]->ref_frame[0]; + pc_tree->horizontalb[0].skip_ref_frame_mask = ~used_frames; + pc_tree->horizontalb[0].ref_selected[0] = split_mbmi[0]->ref_frame[0]; + } + if (split_mbmi[2] && split_mbmi[2]->ref_frame[0] > INTRA_FRAME && + !has_second_ref(split_mbmi[2])) { // single ref + const int used_frames = 1 << (int)split_mbmi[2]->ref_frame[0]; + pc_tree->horizontalb[1].skip_ref_frame_mask = ~used_frames; + pc_tree->horizontalb[1].ref_selected[0] = split_mbmi[2]->ref_frame[0]; + } + if (split_mbmi[3] && split_mbmi[3]->ref_frame[0] > INTRA_FRAME && + !has_second_ref(split_mbmi[3])) { // single ref + const int used_frames = 1 << (int)split_mbmi[3]->ref_frame[0]; + pc_tree->horizontalb[2].skip_ref_frame_mask = ~used_frames; + pc_tree->horizontalb[2].ref_selected[0] = split_mbmi[3]->ref_frame[0]; + } + } +#if CONFIG_COLLECT_PARTITION_STATS + { + RD_STATS tmp_sum_rdc; + av1_init_rd_stats(&tmp_sum_rdc); + tmp_sum_rdc.rate = x->partition_cost[pl][PARTITION_HORZ_B]; + tmp_sum_rdc.rdcost = RDCOST(x->rdmult, tmp_sum_rdc.rate, 0); + if (!frame_is_intra_only(cm) && + best_rdc.rdcost - tmp_sum_rdc.rdcost >= 0) { + partition_attempts[PARTITION_HORZ_B] += 1; + } + } +#endif rd_test_partition3(cpi, td, tile_data, tp, pc_tree, &best_rdc, pc_tree->horizontalb, ctx_none, mi_row, mi_col, bsize, PARTITION_HORZ_B, mi_row, mi_col, subsize, @@ -3996,7 +4818,7 @@ BEGIN_PARTITION_SEARCH: } // PARTITION_VERT_A - if (partition_vert_allowed && verta_partition_allowed) { + if (partition_vert_allowed && verta_partition_allowed && !is_gt_max_sq_part) { subsize = get_partition_subsize(bsize, PARTITION_VERT_A); pc_tree->verticala[0].rd_mode_is_ready = 0; pc_tree->verticala[1].rd_mode_is_ready = 0; @@ -4006,9 +4828,11 @@ BEGIN_PARTITION_SEARCH: pc_tree->verticala[0].mic.partition = PARTITION_VERT_A; pc_tree->verticala[0].rd_mode_is_ready = 1; } - pc_tree->verticala[0].skip_ref_frame_mask = 0; - pc_tree->verticala[1].skip_ref_frame_mask = 0; - pc_tree->verticala[2].skip_ref_frame_mask = 0; + for (int i = 0; i < 3; ++i) { + pc_tree->verticala[i].skip_ref_frame_mask = 0; + pc_tree->verticala[i].ref_selected[0] = + pc_tree->verticala[i].ref_selected[1] = NONE_FRAME; + } if (cpi->sf.prune_ref_frame_for_rect_partitions) { int used_frames; used_frames = ref_frames_used[0]; @@ -4018,6 +4842,42 @@ BEGIN_PARTITION_SEARCH: used_frames = ref_frames_used[1] | ref_frames_used[3]; if (used_frames) pc_tree->verticala[2].skip_ref_frame_mask = ~used_frames; } + if (cpi->sf.prune_ref_mode_for_partitions) { + // Overwrite skip_ref_frame_mask for the current block + if (split_mbmi[0] && split_mbmi[0]->ref_frame[0] > INTRA_FRAME && + !has_second_ref(split_mbmi[0])) { // single ref + const int used_frames = 1 << (int)split_mbmi[0]->ref_frame[0]; + pc_tree->verticala[0].skip_ref_frame_mask = ~used_frames; + pc_tree->verticala[0].ref_selected[0] = split_mbmi[0]->ref_frame[0]; + } + if (split_mbmi[2] && split_mbmi[2]->ref_frame[0] > INTRA_FRAME && + !has_second_ref(split_mbmi[2])) { // single ref + const int used_frames = 1 << (int)split_mbmi[2]->ref_frame[0]; + pc_tree->verticala[1].skip_ref_frame_mask = ~used_frames; + pc_tree->verticala[1].ref_selected[0] = split_mbmi[2]->ref_frame[0]; + } + if (split_mbmi[1] && split_mbmi[3] && + split_mbmi[1]->ref_frame[0] > INTRA_FRAME && + split_mbmi[1]->ref_frame[0] == split_mbmi[3]->ref_frame[0] && + !has_second_ref(split_mbmi[1]) && + !has_second_ref(split_mbmi[3])) { // single ref + const int used_frames = 1 << (int)split_mbmi[1]->ref_frame[0]; + pc_tree->verticala[2].skip_ref_frame_mask = ~used_frames; + pc_tree->verticala[2].ref_selected[0] = split_mbmi[1]->ref_frame[0]; + } + } +#if CONFIG_COLLECT_PARTITION_STATS + { + RD_STATS tmp_sum_rdc; + av1_init_rd_stats(&tmp_sum_rdc); + tmp_sum_rdc.rate = x->partition_cost[pl][PARTITION_VERT_A]; + tmp_sum_rdc.rdcost = RDCOST(x->rdmult, tmp_sum_rdc.rate, 0); + if (!frame_is_intra_only(cm) && + best_rdc.rdcost - tmp_sum_rdc.rdcost >= 0) { + partition_attempts[PARTITION_VERT_A] += 1; + } + } +#endif rd_test_partition3(cpi, td, tile_data, tp, pc_tree, &best_rdc, pc_tree->verticala, ctx_none, mi_row, mi_col, bsize, PARTITION_VERT_A, mi_row, mi_col, bsize2, @@ -4026,7 +4886,7 @@ BEGIN_PARTITION_SEARCH: restore_context(x, &x_ctx, mi_row, mi_col, bsize, num_planes); } // PARTITION_VERT_B - if (partition_vert_allowed && vertb_partition_allowed) { + if (partition_vert_allowed && vertb_partition_allowed && !is_gt_max_sq_part) { subsize = get_partition_subsize(bsize, PARTITION_VERT_B); pc_tree->verticalb[0].rd_mode_is_ready = 0; pc_tree->verticalb[1].rd_mode_is_ready = 0; @@ -4036,9 +4896,11 @@ BEGIN_PARTITION_SEARCH: pc_tree->verticalb[0].mic.partition = PARTITION_VERT_B; pc_tree->verticalb[0].rd_mode_is_ready = 1; } - pc_tree->verticalb[0].skip_ref_frame_mask = 0; - pc_tree->verticalb[1].skip_ref_frame_mask = 0; - pc_tree->verticalb[2].skip_ref_frame_mask = 0; + for (int i = 0; i < 3; ++i) { + pc_tree->verticalb[i].skip_ref_frame_mask = 0; + pc_tree->verticalb[i].ref_selected[0] = + pc_tree->verticalb[i].ref_selected[1] = NONE_FRAME; + } if (cpi->sf.prune_ref_frame_for_rect_partitions) { int used_frames; used_frames = ref_frames_used[0] | ref_frames_used[2]; @@ -4048,6 +4910,42 @@ BEGIN_PARTITION_SEARCH: used_frames = ref_frames_used[3]; if (used_frames) pc_tree->verticalb[2].skip_ref_frame_mask = ~used_frames; } + if (cpi->sf.prune_ref_mode_for_partitions) { + // Overwrite skip_ref_frame_mask for the current block + if (split_mbmi[0] && split_mbmi[2] && + split_mbmi[0]->ref_frame[0] > INTRA_FRAME && + split_mbmi[0]->ref_frame[0] == split_mbmi[2]->ref_frame[0] && + !has_second_ref(split_mbmi[0]) && + !has_second_ref(split_mbmi[2])) { // single ref + const int used_frames = 1 << (int)split_mbmi[0]->ref_frame[0]; + pc_tree->verticalb[0].skip_ref_frame_mask = ~used_frames; + pc_tree->verticalb[0].ref_selected[0] = split_mbmi[0]->ref_frame[0]; + } + if (split_mbmi[1] && split_mbmi[1]->ref_frame[0] > INTRA_FRAME && + !has_second_ref(split_mbmi[1])) { // single ref + const int used_frames = 1 << (int)split_mbmi[1]->ref_frame[0]; + pc_tree->verticalb[1].skip_ref_frame_mask = ~used_frames; + pc_tree->verticalb[1].ref_selected[0] = split_mbmi[1]->ref_frame[0]; + } + if (split_mbmi[3] && split_mbmi[3]->ref_frame[0] > INTRA_FRAME && + !has_second_ref(split_mbmi[3])) { // single ref + const int used_frames = 1 << (int)split_mbmi[3]->ref_frame[0]; + pc_tree->verticalb[2].skip_ref_frame_mask = ~used_frames; + pc_tree->verticalb[2].ref_selected[0] = split_mbmi[3]->ref_frame[0]; + } + } +#if CONFIG_COLLECT_PARTITION_STATS + { + RD_STATS tmp_sum_rdc; + av1_init_rd_stats(&tmp_sum_rdc); + tmp_sum_rdc.rate = x->partition_cost[pl][PARTITION_VERT_B]; + tmp_sum_rdc.rdcost = RDCOST(x->rdmult, tmp_sum_rdc.rate, 0); + if (!frame_is_intra_only(cm) && + best_rdc.rdcost - tmp_sum_rdc.rdcost >= 0) { + partition_attempts[PARTITION_VERT_B] += 1; + } + } +#endif rd_test_partition3(cpi, td, tile_data, tp, pc_tree, &best_rdc, pc_tree->verticalb, ctx_none, mi_row, mi_col, bsize, PARTITION_VERT_B, mi_row, mi_col, subsize, mi_row, @@ -4058,8 +4956,8 @@ BEGIN_PARTITION_SEARCH: // partition4_allowed is 1 if we can use a PARTITION_HORZ_4 or // PARTITION_VERT_4 for this block. This is almost the same as - // ext_partition_allowed, except that we don't allow 128x32 or 32x128 blocks, - // so we require that bsize is not BLOCK_128X128. + // ext_partition_allowed, except that we don't allow 128x32 or 32x128 + // blocks, so we require that bsize is not BLOCK_128X128. const int partition4_allowed = ext_partition_allowed && bsize != BLOCK_128X128; int partition_horz4_allowed = partition4_allowed && partition_horz_allowed; @@ -4094,8 +4992,10 @@ BEGIN_PARTITION_SEARCH: #endif // PARTITION_HORZ_4 + assert(IMPLIES(!cpi->oxcf.enable_rect_partitions, !partition_horz4_allowed)); if (partition_horz4_allowed && has_rows && - (do_rectangular_split || active_h_edge(cpi, mi_row, mi_step))) { + (do_rectangular_split || active_h_edge(cpi, mi_row, mi_step)) && + !is_gt_max_sq_part) { av1_init_rd_stats(&sum_rdc); const int quarter_step = mi_size_high[bsize] / 4; PICK_MODE_CONTEXT *ctx_prev = ctx_none; @@ -4104,6 +5004,11 @@ BEGIN_PARTITION_SEARCH: sum_rdc.rate = partition_cost[PARTITION_HORZ_4]; sum_rdc.rdcost = RDCOST(x->rdmult, sum_rdc.rate, 0); +#if CONFIG_COLLECT_PARTITION_STATS + if (!frame_is_intra_only(cm) && best_rdc.rdcost - sum_rdc.rdcost >= 0) { + partition_attempts[PARTITION_HORZ_4] += 1; + } +#endif for (int i = 0; i < 4; ++i) { const int this_mi_row = mi_row + i * quarter_step; @@ -4138,8 +5043,10 @@ BEGIN_PARTITION_SEARCH: } // PARTITION_VERT_4 + assert(IMPLIES(!cpi->oxcf.enable_rect_partitions, !partition_vert4_allowed)); if (partition_vert4_allowed && has_cols && - (do_rectangular_split || active_v_edge(cpi, mi_row, mi_step))) { + (do_rectangular_split || active_v_edge(cpi, mi_row, mi_step)) && + !is_gt_max_sq_part) { av1_init_rd_stats(&sum_rdc); const int quarter_step = mi_size_wide[bsize] / 4; PICK_MODE_CONTEXT *ctx_prev = ctx_none; @@ -4148,6 +5055,11 @@ BEGIN_PARTITION_SEARCH: sum_rdc.rate = partition_cost[PARTITION_VERT_4]; sum_rdc.rdcost = RDCOST(x->rdmult, sum_rdc.rate, 0); +#if CONFIG_COLLECT_PARTITION_STATS + if (!frame_is_intra_only(cm) && best_rdc.rdcost - sum_rdc.rdcost >= 0) { + partition_attempts[PARTITION_VERT_4] += 1; + } +#endif for (int i = 0; i < 4; ++i) { const int this_mi_col = mi_col + i * quarter_step; @@ -4185,6 +5097,11 @@ BEGIN_PARTITION_SEARCH: // Did not find a valid partition, go back and search again, with less // constraint on which partition types to search. x->must_find_valid_partition = 1; +#if CONFIG_COLLECT_PARTITION_STATS + if (!frame_is_intra_only(cm)) { + part_stats->partition_redo += 1; + } +#endif goto BEGIN_PARTITION_SEARCH; } @@ -4195,6 +5112,13 @@ BEGIN_PARTITION_SEARCH: (void)best_rd; *rd_cost = best_rdc; +#if CONFIG_COLLECT_PARTITION_STATS + if (!frame_is_intra_only(cm) && best_rdc.rate < INT_MAX && + best_rdc.dist < INT64_MAX) { + partition_decisions[pc_tree->partitioning] += 1; + } +#endif + if (best_rdc.rate < INT_MAX && best_rdc.dist < INT64_MAX && pc_tree->index != 3) { if (bsize == cm->seq_params.sb_size) { @@ -4225,19 +5149,380 @@ static void init_first_partition_pass_stats_tables( } } -// clear pc_tree_stats -static INLINE void clear_pc_tree_stats(PC_TREE *pt) { - if (pt == NULL) return; - pt->pc_tree_stats.valid = 0; - for (int i = 0; i < 4; ++i) { - clear_pc_tree_stats(pt->split[i]); - } -} - // Minimum number of samples to trigger the // mode_pruning_based_on_two_pass_partition_search feature. #define FIRST_PARTITION_PASS_MIN_SAMPLES 16 +static int get_rdmult_delta(AV1_COMP *cpi, BLOCK_SIZE bsize, int mi_row, + int mi_col, int orig_rdmult) { + TplDepFrame *tpl_frame = &cpi->tpl_stats[cpi->twopass.gf_group.index]; + TplDepStats *tpl_stats = tpl_frame->tpl_stats_ptr; + int tpl_stride = tpl_frame->stride; + int64_t intra_cost = 0; + int64_t mc_dep_cost = 0; + int mi_wide = mi_size_wide[bsize]; + int mi_high = mi_size_high[bsize]; + int row, col; + + int dr = 0; + int count = 0; + double r0, rk, beta; + + if (tpl_frame->is_valid == 0) return orig_rdmult; + + if (cpi->common.show_frame) return orig_rdmult; + + if (cpi->twopass.gf_group.index >= MAX_LAG_BUFFERS) return orig_rdmult; + + for (row = mi_row; row < mi_row + mi_high; ++row) { + for (col = mi_col; col < mi_col + mi_wide; ++col) { + TplDepStats *this_stats = &tpl_stats[row * tpl_stride + col]; + + if (row >= cpi->common.mi_rows || col >= cpi->common.mi_cols) continue; + + intra_cost += this_stats->intra_cost; + mc_dep_cost += this_stats->mc_dep_cost; + + ++count; + } + } + + aom_clear_system_state(); + + r0 = cpi->rd.r0; + rk = (double)intra_cost / mc_dep_cost; + beta = r0 / rk; + dr = av1_get_adaptive_rdmult(cpi, beta); + + dr = AOMMIN(dr, orig_rdmult * 3 / 2); + dr = AOMMAX(dr, orig_rdmult * 1 / 2); + + dr = AOMMAX(1, dr); + + return dr; +} + +static void setup_delta_q(AV1_COMP *const cpi, MACROBLOCK *const x, + const TileInfo *const tile_info, int mi_row, + int mi_col, int num_planes) { + AV1_COMMON *const cm = &cpi->common; + MACROBLOCKD *const xd = &x->e_mbd; + const DeltaQInfo *const delta_q_info = &cm->delta_q_info; + const BLOCK_SIZE sb_size = cm->seq_params.sb_size; + const int mib_size = cm->seq_params.mib_size; + + // Delta-q modulation based on variance + av1_setup_src_planes(x, cpi->source, mi_row, mi_col, num_planes, sb_size); + + int offset_qindex; + if (DELTAQ_MODULATION == 1) { + const int block_wavelet_energy_level = + av1_block_wavelet_energy_level(cpi, x, sb_size); + x->sb_energy_level = block_wavelet_energy_level; + offset_qindex = + av1_compute_deltaq_from_energy_level(cpi, block_wavelet_energy_level); + } else { + const int block_var_level = av1_log_block_var(cpi, x, sb_size); + x->sb_energy_level = block_var_level; + offset_qindex = av1_compute_deltaq_from_energy_level(cpi, block_var_level); + } + const int qmask = ~(delta_q_info->delta_q_res - 1); + int current_qindex = + clamp(cm->base_qindex + offset_qindex, delta_q_info->delta_q_res, + 256 - delta_q_info->delta_q_res); + current_qindex = + ((current_qindex - cm->base_qindex + delta_q_info->delta_q_res / 2) & + qmask) + + cm->base_qindex; + assert(current_qindex > 0); + + xd->delta_qindex = current_qindex - cm->base_qindex; + set_offsets(cpi, tile_info, x, mi_row, mi_col, sb_size); + xd->mi[0]->current_qindex = current_qindex; + av1_init_plane_quantizers(cpi, x, xd->mi[0]->segment_id); + if (cpi->oxcf.deltaq_mode == DELTA_Q_LF) { + const int lfmask = ~(delta_q_info->delta_lf_res - 1); + const int delta_lf_from_base = + ((offset_qindex / 2 + delta_q_info->delta_lf_res / 2) & lfmask); + + // pre-set the delta lf for loop filter. Note that this value is set + // before mi is assigned for each block in current superblock + for (int j = 0; j < AOMMIN(mib_size, cm->mi_rows - mi_row); j++) { + for (int k = 0; k < AOMMIN(mib_size, cm->mi_cols - mi_col); k++) { + cm->mi[(mi_row + j) * cm->mi_stride + (mi_col + k)].delta_lf_from_base = + clamp(delta_lf_from_base, -MAX_LOOP_FILTER, MAX_LOOP_FILTER); + const int frame_lf_count = + av1_num_planes(cm) > 1 ? FRAME_LF_COUNT : FRAME_LF_COUNT - 2; + for (int lf_id = 0; lf_id < frame_lf_count; ++lf_id) { + cm->mi[(mi_row + j) * cm->mi_stride + (mi_col + k)].delta_lf[lf_id] = + clamp(delta_lf_from_base, -MAX_LOOP_FILTER, MAX_LOOP_FILTER); + } + } + } + } +} + +// First pass of partition search only considers square partition block sizes. +// The results will be used in the second partition search pass to prune +// unlikely partition candidates. +static void first_partition_search_pass(AV1_COMP *cpi, ThreadData *td, + TileDataEnc *tile_data, int mi_row, + int mi_col, TOKENEXTRA **tp) { + MACROBLOCK *const x = &td->mb; + x->cb_partition_scan = 1; + + const SPEED_FEATURES *const sf = &cpi->sf; + // Reset the stats tables. + if (sf->mode_pruning_based_on_two_pass_partition_search) + av1_zero(x->first_partition_pass_stats); + + AV1_COMMON *const cm = &cpi->common; + const BLOCK_SIZE sb_size = cm->seq_params.sb_size; + const int mib_size_log2 = cm->seq_params.mib_size_log2; + PC_TREE *const pc_root = td->pc_root[mib_size_log2 - MIN_MIB_SIZE_LOG2]; + RD_STATS dummy_rdc; + rd_pick_sqr_partition(cpi, td, tile_data, tp, mi_row, mi_col, sb_size, + &dummy_rdc, INT64_MAX, pc_root, NULL); + x->cb_partition_scan = 0; + + x->source_variance = UINT_MAX; + if (sf->adaptive_pred_interp_filter) { + const int leaf_nodes = 256; + for (int i = 0; i < leaf_nodes; ++i) { + td->pc_tree[i].vertical[0].pred_interp_filter = SWITCHABLE; + td->pc_tree[i].vertical[1].pred_interp_filter = SWITCHABLE; + td->pc_tree[i].horizontal[0].pred_interp_filter = SWITCHABLE; + td->pc_tree[i].horizontal[1].pred_interp_filter = SWITCHABLE; + } + } + + x->mb_rd_record.num = x->mb_rd_record.index_start = 0; + av1_zero(x->txb_rd_record_8X8); + av1_zero(x->txb_rd_record_16X16); + av1_zero(x->txb_rd_record_32X32); + av1_zero(x->txb_rd_record_64X64); + av1_zero(x->txb_rd_record_intra); + av1_zero(x->pred_mv); + pc_root->index = 0; + + for (int idy = 0; idy < mi_size_high[sb_size]; ++idy) { + for (int idx = 0; idx < mi_size_wide[sb_size]; ++idx) { + const int offset = cm->mi_stride * (mi_row + idy) + (mi_col + idx); + cm->mi_grid_visible[offset] = 0; + } + } + + x->use_cb_search_range = 1; + + if (sf->mode_pruning_based_on_two_pass_partition_search) { + for (int i = 0; i < FIRST_PARTITION_PASS_STATS_TABLES; ++i) { + FIRST_PARTITION_PASS_STATS *const stat = + &x->first_partition_pass_stats[i]; + if (stat->sample_counts < FIRST_PARTITION_PASS_MIN_SAMPLES) { + // If there are not enough samples collected, make all available. + memset(stat->ref0_counts, 0xff, sizeof(stat->ref0_counts)); + memset(stat->ref1_counts, 0xff, sizeof(stat->ref1_counts)); + } else if (sf->selective_ref_frame < 3) { + // ALTREF2_FRAME and BWDREF_FRAME may be skipped during the + // initial partition scan, so we don't eliminate them. + stat->ref0_counts[ALTREF2_FRAME] = 0xff; + stat->ref1_counts[ALTREF2_FRAME] = 0xff; + stat->ref0_counts[BWDREF_FRAME] = 0xff; + stat->ref1_counts[BWDREF_FRAME] = 0xff; + } + } + } +} + +static void init_simple_motion_search_mvs(PC_TREE *pc_tree) { + for (int idx = 0; idx < REF_FRAMES; idx++) { + pc_tree->mv_ref_fulls[idx].row = 0; + pc_tree->mv_ref_fulls[idx].col = 0; + } + if (pc_tree->block_size >= BLOCK_8X8) { + init_simple_motion_search_mvs(pc_tree->split[0]); + init_simple_motion_search_mvs(pc_tree->split[1]); + init_simple_motion_search_mvs(pc_tree->split[2]); + init_simple_motion_search_mvs(pc_tree->split[3]); + } +} + +#define AVG_CDF_WEIGHT_LEFT 3 +#define AVG_CDF_WEIGHT_TOP_RIGHT 1 + +static void avg_cdf_symbol(aom_cdf_prob *cdf_ptr_left, aom_cdf_prob *cdf_ptr_tr, + int num_cdfs, int cdf_stride, int nsymbs, + int wt_left, int wt_tr) { + for (int i = 0; i < num_cdfs; i++) { + for (int j = 0; j <= nsymbs; j++) { + cdf_ptr_left[i * cdf_stride + j] = + (aom_cdf_prob)(((int)cdf_ptr_left[i * cdf_stride + j] * wt_left + + (int)cdf_ptr_tr[i * cdf_stride + j] * wt_tr + + ((wt_left + wt_tr) / 2)) / + (wt_left + wt_tr)); + assert(cdf_ptr_left[i * cdf_stride + j] >= 0 && + cdf_ptr_left[i * cdf_stride + j] < CDF_PROB_TOP); + } + } +} + +#define AVERAGE_CDF(cname_left, cname_tr, nsymbs) \ + AVG_CDF_STRIDE(cname_left, cname_tr, nsymbs, CDF_SIZE(nsymbs)) + +#define AVG_CDF_STRIDE(cname_left, cname_tr, nsymbs, cdf_stride) \ + do { \ + aom_cdf_prob *cdf_ptr_left = (aom_cdf_prob *)cname_left; \ + aom_cdf_prob *cdf_ptr_tr = (aom_cdf_prob *)cname_tr; \ + int array_size = (int)sizeof(cname_left) / sizeof(aom_cdf_prob); \ + int num_cdfs = array_size / cdf_stride; \ + avg_cdf_symbol(cdf_ptr_left, cdf_ptr_tr, num_cdfs, cdf_stride, nsymbs, \ + wt_left, wt_tr); \ + } while (0) + +static void avg_nmv(nmv_context *nmv_left, nmv_context *nmv_tr, int wt_left, + int wt_tr) { + AVERAGE_CDF(nmv_left->joints_cdf, nmv_tr->joints_cdf, 4); + for (int i = 0; i < 2; i++) { + AVERAGE_CDF(nmv_left->comps[i].classes_cdf, nmv_tr->comps[i].classes_cdf, + MV_CLASSES); + AVERAGE_CDF(nmv_left->comps[i].class0_fp_cdf, + nmv_tr->comps[i].class0_fp_cdf, MV_FP_SIZE); + AVERAGE_CDF(nmv_left->comps[i].fp_cdf, nmv_tr->comps[i].fp_cdf, MV_FP_SIZE); + AVERAGE_CDF(nmv_left->comps[i].sign_cdf, nmv_tr->comps[i].sign_cdf, 2); + AVERAGE_CDF(nmv_left->comps[i].class0_hp_cdf, + nmv_tr->comps[i].class0_hp_cdf, 2); + AVERAGE_CDF(nmv_left->comps[i].hp_cdf, nmv_tr->comps[i].hp_cdf, 2); + AVERAGE_CDF(nmv_left->comps[i].class0_cdf, nmv_tr->comps[i].class0_cdf, + CLASS0_SIZE); + AVERAGE_CDF(nmv_left->comps[i].bits_cdf, nmv_tr->comps[i].bits_cdf, 2); + } +} + +// In case of row-based multi-threading of encoder, since we always +// keep a top - right sync, we can average the top - right SB's CDFs and +// the left SB's CDFs and use the same for current SB's encoding to +// improve the performance. This function facilitates the averaging +// of CDF and used only when row-mt is enabled in encoder. +static void avg_cdf_symbols(FRAME_CONTEXT *ctx_left, FRAME_CONTEXT *ctx_tr, + int wt_left, int wt_tr) { + AVERAGE_CDF(ctx_left->txb_skip_cdf, ctx_tr->txb_skip_cdf, 2); + AVERAGE_CDF(ctx_left->eob_extra_cdf, ctx_tr->eob_extra_cdf, 2); + AVERAGE_CDF(ctx_left->dc_sign_cdf, ctx_tr->dc_sign_cdf, 2); + AVERAGE_CDF(ctx_left->eob_flag_cdf16, ctx_tr->eob_flag_cdf16, 5); + AVERAGE_CDF(ctx_left->eob_flag_cdf32, ctx_tr->eob_flag_cdf32, 6); + AVERAGE_CDF(ctx_left->eob_flag_cdf64, ctx_tr->eob_flag_cdf64, 7); + AVERAGE_CDF(ctx_left->eob_flag_cdf128, ctx_tr->eob_flag_cdf128, 8); + AVERAGE_CDF(ctx_left->eob_flag_cdf256, ctx_tr->eob_flag_cdf256, 9); + AVERAGE_CDF(ctx_left->eob_flag_cdf512, ctx_tr->eob_flag_cdf512, 10); + AVERAGE_CDF(ctx_left->eob_flag_cdf1024, ctx_tr->eob_flag_cdf1024, 11); + AVERAGE_CDF(ctx_left->coeff_base_eob_cdf, ctx_tr->coeff_base_eob_cdf, 3); + AVERAGE_CDF(ctx_left->coeff_base_cdf, ctx_tr->coeff_base_cdf, 4); + AVERAGE_CDF(ctx_left->coeff_br_cdf, ctx_tr->coeff_br_cdf, BR_CDF_SIZE); + AVERAGE_CDF(ctx_left->newmv_cdf, ctx_tr->newmv_cdf, 2); + AVERAGE_CDF(ctx_left->zeromv_cdf, ctx_tr->zeromv_cdf, 2); + AVERAGE_CDF(ctx_left->refmv_cdf, ctx_tr->refmv_cdf, 2); + AVERAGE_CDF(ctx_left->drl_cdf, ctx_tr->drl_cdf, 2); + AVERAGE_CDF(ctx_left->inter_compound_mode_cdf, + ctx_tr->inter_compound_mode_cdf, INTER_COMPOUND_MODES); + AVERAGE_CDF(ctx_left->compound_type_cdf, ctx_tr->compound_type_cdf, + COMPOUND_TYPES - 1); + AVERAGE_CDF(ctx_left->wedge_idx_cdf, ctx_tr->wedge_idx_cdf, 16); + AVERAGE_CDF(ctx_left->interintra_cdf, ctx_tr->interintra_cdf, 2); + AVERAGE_CDF(ctx_left->wedge_interintra_cdf, ctx_tr->wedge_interintra_cdf, 2); + AVERAGE_CDF(ctx_left->interintra_mode_cdf, ctx_tr->interintra_mode_cdf, + INTERINTRA_MODES); + AVERAGE_CDF(ctx_left->motion_mode_cdf, ctx_tr->motion_mode_cdf, MOTION_MODES); + AVERAGE_CDF(ctx_left->obmc_cdf, ctx_tr->obmc_cdf, 2); + AVERAGE_CDF(ctx_left->palette_y_size_cdf, ctx_tr->palette_y_size_cdf, + PALETTE_SIZES); + AVERAGE_CDF(ctx_left->palette_uv_size_cdf, ctx_tr->palette_uv_size_cdf, + PALETTE_SIZES); + for (int j = 0; j < PALETTE_SIZES; j++) { + int nsymbs = j + PALETTE_MIN_SIZE; + AVG_CDF_STRIDE(ctx_left->palette_y_color_index_cdf[j], + ctx_tr->palette_y_color_index_cdf[j], nsymbs, + CDF_SIZE(PALETTE_COLORS)); + AVG_CDF_STRIDE(ctx_left->palette_uv_color_index_cdf[j], + ctx_tr->palette_uv_color_index_cdf[j], nsymbs, + CDF_SIZE(PALETTE_COLORS)); + } + AVERAGE_CDF(ctx_left->palette_y_mode_cdf, ctx_tr->palette_y_mode_cdf, 2); + AVERAGE_CDF(ctx_left->palette_uv_mode_cdf, ctx_tr->palette_uv_mode_cdf, 2); + AVERAGE_CDF(ctx_left->comp_inter_cdf, ctx_tr->comp_inter_cdf, 2); + AVERAGE_CDF(ctx_left->single_ref_cdf, ctx_tr->single_ref_cdf, 2); + AVERAGE_CDF(ctx_left->comp_ref_type_cdf, ctx_tr->comp_ref_type_cdf, 2); + AVERAGE_CDF(ctx_left->uni_comp_ref_cdf, ctx_tr->uni_comp_ref_cdf, 2); + AVERAGE_CDF(ctx_left->comp_ref_cdf, ctx_tr->comp_ref_cdf, 2); + AVERAGE_CDF(ctx_left->comp_bwdref_cdf, ctx_tr->comp_bwdref_cdf, 2); + AVERAGE_CDF(ctx_left->txfm_partition_cdf, ctx_tr->txfm_partition_cdf, 2); + AVERAGE_CDF(ctx_left->compound_index_cdf, ctx_tr->compound_index_cdf, 2); + AVERAGE_CDF(ctx_left->comp_group_idx_cdf, ctx_tr->comp_group_idx_cdf, 2); + AVERAGE_CDF(ctx_left->skip_mode_cdfs, ctx_tr->skip_mode_cdfs, 2); + AVERAGE_CDF(ctx_left->skip_cdfs, ctx_tr->skip_cdfs, 2); + AVERAGE_CDF(ctx_left->intra_inter_cdf, ctx_tr->intra_inter_cdf, 2); + avg_nmv(&ctx_left->nmvc, &ctx_tr->nmvc, wt_left, wt_tr); + avg_nmv(&ctx_left->ndvc, &ctx_tr->ndvc, wt_left, wt_tr); + AVERAGE_CDF(ctx_left->intrabc_cdf, ctx_tr->intrabc_cdf, 2); + AVERAGE_CDF(ctx_left->seg.tree_cdf, ctx_tr->seg.tree_cdf, MAX_SEGMENTS); + AVERAGE_CDF(ctx_left->seg.pred_cdf, ctx_tr->seg.pred_cdf, 2); + AVERAGE_CDF(ctx_left->seg.spatial_pred_seg_cdf, + ctx_tr->seg.spatial_pred_seg_cdf, MAX_SEGMENTS); + AVERAGE_CDF(ctx_left->filter_intra_cdfs, ctx_tr->filter_intra_cdfs, 2); + AVERAGE_CDF(ctx_left->filter_intra_mode_cdf, ctx_tr->filter_intra_mode_cdf, + FILTER_INTRA_MODES); + AVERAGE_CDF(ctx_left->switchable_restore_cdf, ctx_tr->switchable_restore_cdf, + RESTORE_SWITCHABLE_TYPES); + AVERAGE_CDF(ctx_left->wiener_restore_cdf, ctx_tr->wiener_restore_cdf, 2); + AVERAGE_CDF(ctx_left->sgrproj_restore_cdf, ctx_tr->sgrproj_restore_cdf, 2); + AVERAGE_CDF(ctx_left->y_mode_cdf, ctx_tr->y_mode_cdf, INTRA_MODES); + AVG_CDF_STRIDE(ctx_left->uv_mode_cdf[0], ctx_tr->uv_mode_cdf[0], + UV_INTRA_MODES - 1, CDF_SIZE(UV_INTRA_MODES)); + AVERAGE_CDF(ctx_left->uv_mode_cdf[1], ctx_tr->uv_mode_cdf[1], UV_INTRA_MODES); + for (int i = 0; i < PARTITION_CONTEXTS; i++) { + if (i < 4) { + AVG_CDF_STRIDE(ctx_left->partition_cdf[i], ctx_tr->partition_cdf[i], 4, + CDF_SIZE(10)); + } else if (i < 16) { + AVERAGE_CDF(ctx_left->partition_cdf[i], ctx_tr->partition_cdf[i], 10); + } else { + AVG_CDF_STRIDE(ctx_left->partition_cdf[i], ctx_tr->partition_cdf[i], 8, + CDF_SIZE(10)); + } + } + AVERAGE_CDF(ctx_left->switchable_interp_cdf, ctx_tr->switchable_interp_cdf, + SWITCHABLE_FILTERS); + AVERAGE_CDF(ctx_left->kf_y_cdf, ctx_tr->kf_y_cdf, INTRA_MODES); + AVERAGE_CDF(ctx_left->angle_delta_cdf, ctx_tr->angle_delta_cdf, + 2 * MAX_ANGLE_DELTA + 1); + AVG_CDF_STRIDE(ctx_left->tx_size_cdf[0], ctx_tr->tx_size_cdf[0], MAX_TX_DEPTH, + CDF_SIZE(MAX_TX_DEPTH + 1)); + AVERAGE_CDF(ctx_left->tx_size_cdf[1], ctx_tr->tx_size_cdf[1], + MAX_TX_DEPTH + 1); + AVERAGE_CDF(ctx_left->tx_size_cdf[2], ctx_tr->tx_size_cdf[2], + MAX_TX_DEPTH + 1); + AVERAGE_CDF(ctx_left->tx_size_cdf[3], ctx_tr->tx_size_cdf[3], + MAX_TX_DEPTH + 1); + AVERAGE_CDF(ctx_left->delta_q_cdf, ctx_tr->delta_q_cdf, DELTA_Q_PROBS + 1); + AVERAGE_CDF(ctx_left->delta_lf_cdf, ctx_tr->delta_lf_cdf, DELTA_LF_PROBS + 1); + for (int i = 0; i < FRAME_LF_COUNT; i++) { + AVERAGE_CDF(ctx_left->delta_lf_multi_cdf[i], ctx_tr->delta_lf_multi_cdf[i], + DELTA_LF_PROBS + 1); + } + AVG_CDF_STRIDE(ctx_left->intra_ext_tx_cdf[1], ctx_tr->intra_ext_tx_cdf[1], 7, + CDF_SIZE(TX_TYPES)); + AVG_CDF_STRIDE(ctx_left->intra_ext_tx_cdf[2], ctx_tr->intra_ext_tx_cdf[2], 5, + CDF_SIZE(TX_TYPES)); + AVG_CDF_STRIDE(ctx_left->inter_ext_tx_cdf[1], ctx_tr->inter_ext_tx_cdf[1], 16, + CDF_SIZE(TX_TYPES)); + AVG_CDF_STRIDE(ctx_left->inter_ext_tx_cdf[2], ctx_tr->inter_ext_tx_cdf[2], 12, + CDF_SIZE(TX_TYPES)); + AVG_CDF_STRIDE(ctx_left->inter_ext_tx_cdf[3], ctx_tr->inter_ext_tx_cdf[3], 2, + CDF_SIZE(TX_TYPES)); + AVERAGE_CDF(ctx_left->cfl_sign_cdf, ctx_tr->cfl_sign_cdf, CFL_JOINT_SIGNS); + AVERAGE_CDF(ctx_left->cfl_alpha_cdf, ctx_tr->cfl_alpha_cdf, + CFL_ALPHABET_SIZE); +} + static void encode_rd_sb_row(AV1_COMP *cpi, ThreadData *td, TileDataEnc *tile_data, int mi_row, TOKENEXTRA **tp) { @@ -4246,25 +5531,47 @@ static void encode_rd_sb_row(AV1_COMP *cpi, ThreadData *td, const TileInfo *const tile_info = &tile_data->tile_info; MACROBLOCK *const x = &td->mb; MACROBLOCKD *const xd = &x->e_mbd; - SPEED_FEATURES *const sf = &cpi->sf; + const SPEED_FEATURES *const sf = &cpi->sf; const int leaf_nodes = 256; + const int sb_cols_in_tile = av1_get_sb_cols_in_tile(cm, tile_data->tile_info); + const BLOCK_SIZE sb_size = cm->seq_params.sb_size; + const int mib_size = cm->seq_params.mib_size; + const int mib_size_log2 = cm->seq_params.mib_size_log2; + const int sb_row = (mi_row - tile_info->mi_row_start) >> mib_size_log2; // Initialize the left context for the new SB row av1_zero_left_context(xd); // Reset delta for every tile if (mi_row == tile_info->mi_row_start) { - if (cm->delta_q_present_flag) xd->current_qindex = cm->base_qindex; - if (cm->delta_lf_present_flag) { + if (cm->delta_q_info.delta_q_present_flag) + xd->current_qindex = cm->base_qindex; + if (cm->delta_q_info.delta_lf_present_flag) { av1_reset_loop_filter_delta(xd, av1_num_planes(cm)); } } - PC_TREE *const pc_root = - td->pc_root[cm->seq_params.mib_size_log2 - MIN_MIB_SIZE_LOG2]; // Code each SB in the row - for (int mi_col = tile_info->mi_col_start; mi_col < tile_info->mi_col_end; - mi_col += cm->seq_params.mib_size) { + for (int mi_col = tile_info->mi_col_start, sb_col_in_tile = 0; + mi_col < tile_info->mi_col_end; mi_col += mib_size, sb_col_in_tile++) { + (*(cpi->row_mt_sync_read_ptr))(&tile_data->row_mt_sync, sb_row, + sb_col_in_tile); + if (tile_data->allow_update_cdf && (cpi->row_mt == 1) && + (tile_info->mi_row_start != mi_row)) { + if ((tile_info->mi_col_start == mi_col)) { + // restore frame context of 1st column sb + memcpy(xd->tile_ctx, x->row_ctx, sizeof(*xd->tile_ctx)); + } else { + int wt_left = AVG_CDF_WEIGHT_LEFT; + int wt_tr = AVG_CDF_WEIGHT_TOP_RIGHT; + if (tile_info->mi_col_end > (mi_col + mib_size)) + avg_cdf_symbols(xd->tile_ctx, x->row_ctx + sb_col_in_tile, wt_left, + wt_tr); + else + avg_cdf_symbols(xd->tile_ctx, x->row_ctx + sb_col_in_tile - 1, + wt_left, wt_tr); + } + } av1_fill_coeff_costs(&td->mb, xd->tile_ctx, num_planes); av1_fill_mode_rates(cm, x, xd->tile_ctx); @@ -4286,76 +5593,27 @@ static void encode_rd_sb_row(AV1_COMP *cpi, ThreadData *td, av1_zero(x->txb_rd_record_intra); av1_zero(x->pred_mv); + PC_TREE *const pc_root = td->pc_root[mib_size_log2 - MIN_MIB_SIZE_LOG2]; pc_root->index = 0; + if (sf->simple_motion_search_prune_rect && !frame_is_intra_only(cm)) { + init_simple_motion_search_mvs(pc_root); + } + const struct segmentation *const seg = &cm->seg; int seg_skip = 0; if (seg->enabled) { const uint8_t *const map = seg->update_map ? cpi->segmentation_map : cm->last_frame_seg_map; const int segment_id = - map ? get_segment_id(cm, map, cm->seq_params.sb_size, mi_row, mi_col) - : 0; + map ? get_segment_id(cm, map, sb_size, mi_row, mi_col) : 0; seg_skip = segfeature_active(seg, segment_id, SEG_LVL_SKIP); } xd->cur_frame_force_integer_mv = cm->cur_frame_force_integer_mv; x->sb_energy_level = 0; - if (cm->delta_q_present_flag) { - // Delta-q modulation based on variance - av1_setup_src_planes(x, cpi->source, mi_row, mi_col, num_planes); - - int offset_qindex; - if (DELTAQ_MODULATION == 1) { - const int block_wavelet_energy_level = - av1_block_wavelet_energy_level(cpi, x, cm->seq_params.sb_size); - x->sb_energy_level = block_wavelet_energy_level; - offset_qindex = av1_compute_deltaq_from_energy_level( - cpi, block_wavelet_energy_level); - } else { - const int block_var_level = - av1_log_block_var(cpi, x, cm->seq_params.sb_size); - x->sb_energy_level = block_var_level; - offset_qindex = - av1_compute_deltaq_from_energy_level(cpi, block_var_level); - } - const int qmask = ~(cm->delta_q_res - 1); - int current_qindex = clamp(cm->base_qindex + offset_qindex, - cm->delta_q_res, 256 - cm->delta_q_res); - current_qindex = - ((current_qindex - cm->base_qindex + cm->delta_q_res / 2) & qmask) + - cm->base_qindex; - assert(current_qindex > 0); - - xd->delta_qindex = current_qindex - cm->base_qindex; - set_offsets(cpi, tile_info, x, mi_row, mi_col, cm->seq_params.sb_size); - xd->mi[0]->current_qindex = current_qindex; - av1_init_plane_quantizers(cpi, x, xd->mi[0]->segment_id); - if (cpi->oxcf.deltaq_mode == DELTA_Q_LF) { - const int lfmask = ~(cm->delta_lf_res - 1); - const int delta_lf_from_base = - ((offset_qindex / 2 + cm->delta_lf_res / 2) & lfmask); - - // pre-set the delta lf for loop filter. Note that this value is set - // before mi is assigned for each block in current superblock - for (int j = 0; - j < AOMMIN(cm->seq_params.mib_size, cm->mi_rows - mi_row); j++) { - for (int k = 0; - k < AOMMIN(cm->seq_params.mib_size, cm->mi_cols - mi_col); k++) { - cm->mi[(mi_row + j) * cm->mi_stride + (mi_col + k)] - .delta_lf_from_base = - clamp(delta_lf_from_base, -MAX_LOOP_FILTER, MAX_LOOP_FILTER); - const int frame_lf_count = - av1_num_planes(cm) > 1 ? FRAME_LF_COUNT : FRAME_LF_COUNT - 2; - for (int lf_id = 0; lf_id < frame_lf_count; ++lf_id) { - cm->mi[(mi_row + j) * cm->mi_stride + (mi_col + k)] - .delta_lf[lf_id] = - clamp(delta_lf_from_base, -MAX_LOOP_FILTER, MAX_LOOP_FILTER); - } - } - } - } - } + if (cm->delta_q_info.delta_q_present_flag) + setup_delta_q(cpi, x, tile_info, mi_row, mi_col, num_planes); int dummy_rate; int64_t dummy_dist; @@ -4364,107 +5622,68 @@ static void encode_rd_sb_row(AV1_COMP *cpi, ThreadData *td, MB_MODE_INFO **mi = cm->mi_grid_visible + idx_str; x->source_variance = UINT_MAX; if (sf->partition_search_type == FIXED_PARTITION || seg_skip) { - set_offsets(cpi, tile_info, x, mi_row, mi_col, cm->seq_params.sb_size); - const BLOCK_SIZE bsize = - seg_skip ? cm->seq_params.sb_size : sf->always_this_block_size; + set_offsets(cpi, tile_info, x, mi_row, mi_col, sb_size); + const BLOCK_SIZE bsize = seg_skip ? sb_size : sf->always_this_block_size; set_fixed_partitioning(cpi, tile_info, mi, mi_row, mi_col, bsize); - rd_use_partition(cpi, td, tile_data, mi, tp, mi_row, mi_col, - cm->seq_params.sb_size, &dummy_rate, &dummy_dist, 1, - pc_root); + rd_use_partition(cpi, td, tile_data, mi, tp, mi_row, mi_col, sb_size, + &dummy_rate, &dummy_dist, 1, pc_root); } else if (cpi->partition_search_skippable_frame) { - set_offsets(cpi, tile_info, x, mi_row, mi_col, cm->seq_params.sb_size); + set_offsets(cpi, tile_info, x, mi_row, mi_col, sb_size); const BLOCK_SIZE bsize = get_rd_var_based_fixed_partition(cpi, x, mi_row, mi_col); set_fixed_partitioning(cpi, tile_info, mi, mi_row, mi_col, bsize); - rd_use_partition(cpi, td, tile_data, mi, tp, mi_row, mi_col, - cm->seq_params.sb_size, &dummy_rate, &dummy_dist, 1, - pc_root); + rd_use_partition(cpi, td, tile_data, mi, tp, mi_row, mi_col, sb_size, + &dummy_rate, &dummy_dist, 1, pc_root); } else { + const int orig_rdmult = cpi->rd.RDMULT; + x->cb_rdmult = orig_rdmult; + if (cpi->twopass.gf_group.index > 0 && cpi->oxcf.enable_tpl_model && + cpi->oxcf.aq_mode == NO_AQ && cpi->oxcf.deltaq_mode == 0) { + const int dr = + get_rdmult_delta(cpi, BLOCK_128X128, mi_row, mi_col, orig_rdmult); + + x->cb_rdmult = dr; + x->rdmult = x->cb_rdmult; + } + // If required set upper and lower partition size limits if (sf->auto_min_max_partition_size) { - set_offsets(cpi, tile_info, x, mi_row, mi_col, cm->seq_params.sb_size); + set_offsets(cpi, tile_info, x, mi_row, mi_col, sb_size); rd_auto_partition_range(cpi, tile_info, xd, mi_row, mi_col, &x->min_partition_size, &x->max_partition_size); } - reset_partition(pc_root, cm->seq_params.sb_size); + reset_partition(pc_root, sb_size); x->use_cb_search_range = 0; init_first_partition_pass_stats_tables(x->first_partition_pass_stats); // Do the first pass if we need two pass partition search if (cpi->sf.two_pass_partition_search && cpi->sf.use_square_partition_only_threshold > BLOCK_4X4 && - mi_row + mi_size_high[cm->seq_params.sb_size] < cm->mi_rows && - mi_col + mi_size_wide[cm->seq_params.sb_size] < cm->mi_cols && - cm->frame_type != KEY_FRAME) { - x->cb_partition_scan = 1; - // Reset the stats tables. - if (sf->mode_pruning_based_on_two_pass_partition_search) - av1_zero(x->first_partition_pass_stats); - clear_pc_tree_stats(pc_root); - rd_pick_sqr_partition(cpi, td, tile_data, tp, mi_row, mi_col, - cm->seq_params.sb_size, &dummy_rdc, INT64_MAX, - pc_root, NULL); - x->cb_partition_scan = 0; - - x->source_variance = UINT_MAX; - if (sf->adaptive_pred_interp_filter) { - for (int i = 0; i < leaf_nodes; ++i) { - td->pc_tree[i].vertical[0].pred_interp_filter = SWITCHABLE; - td->pc_tree[i].vertical[1].pred_interp_filter = SWITCHABLE; - td->pc_tree[i].horizontal[0].pred_interp_filter = SWITCHABLE; - td->pc_tree[i].horizontal[1].pred_interp_filter = SWITCHABLE; - } - } - - x->mb_rd_record.num = x->mb_rd_record.index_start = 0; - av1_zero(x->txb_rd_record_8X8); - av1_zero(x->txb_rd_record_16X16); - av1_zero(x->txb_rd_record_32X32); - av1_zero(x->txb_rd_record_64X64); - av1_zero(x->txb_rd_record_intra); - av1_zero(x->pred_mv); - pc_root->index = 0; - - for (int idy = 0; idy < mi_size_high[cm->seq_params.sb_size]; ++idy) { - for (int idx = 0; idx < mi_size_wide[cm->seq_params.sb_size]; ++idx) { - const int offset = cm->mi_stride * (mi_row + idy) + (mi_col + idx); - cm->mi_grid_visible[offset] = 0; - } - } - - x->use_cb_search_range = 1; - - if (sf->mode_pruning_based_on_two_pass_partition_search) { - for (int i = 0; i < FIRST_PARTITION_PASS_STATS_TABLES; ++i) { - FIRST_PARTITION_PASS_STATS *const stat = - &x->first_partition_pass_stats[i]; - if (stat->sample_counts < FIRST_PARTITION_PASS_MIN_SAMPLES) { - // If there are not enough samples collected, make all available. - memset(stat->ref0_counts, 0xff, sizeof(stat->ref0_counts)); - memset(stat->ref1_counts, 0xff, sizeof(stat->ref1_counts)); - } else if (sf->selective_ref_frame < 2) { - // ALTREF2_FRAME and BWDREF_FRAME may be skipped during the - // initial partition scan, so we don't eliminate them. - stat->ref0_counts[ALTREF2_FRAME] = 0xff; - stat->ref1_counts[ALTREF2_FRAME] = 0xff; - stat->ref0_counts[BWDREF_FRAME] = 0xff; - stat->ref1_counts[BWDREF_FRAME] = 0xff; - } - } - } + mi_row + mi_size_high[sb_size] < cm->mi_rows && + mi_col + mi_size_wide[sb_size] < cm->mi_cols && + cm->current_frame.frame_type != KEY_FRAME) { + first_partition_search_pass(cpi, td, tile_data, mi_row, mi_col, tp); } - rd_pick_partition(cpi, td, tile_data, tp, mi_row, mi_col, - cm->seq_params.sb_size, &dummy_rdc, INT64_MAX, pc_root, + rd_pick_partition(cpi, td, tile_data, tp, mi_row, mi_col, sb_size, + sb_size, BLOCK_4X4, &dummy_rdc, INT64_MAX, pc_root, NULL); } -#if CONFIG_COLLECT_INTER_MODE_RD_STATS // TODO(angiebird): Let inter_mode_rd_model_estimation support multi-tile. - if (cpi->sf.inter_mode_rd_model_estimation && cm->tile_cols == 1 && + if (cpi->sf.inter_mode_rd_model_estimation == 1 && cm->tile_cols == 1 && cm->tile_rows == 1) { av1_inter_mode_data_fit(tile_data, x->rdmult); } -#endif + if (tile_data->allow_update_cdf && (cpi->row_mt == 1) && + (tile_info->mi_row_end > (mi_row + mib_size))) { + if (sb_cols_in_tile == 1) + memcpy(x->row_ctx, xd->tile_ctx, sizeof(*xd->tile_ctx)); + else if (sb_col_in_tile >= 1) + memcpy(x->row_ctx + sb_col_in_tile - 1, xd->tile_ctx, + sizeof(*xd->tile_ctx)); + } + (*(cpi->row_mt_sync_write_ptr))(&tile_data->row_mt_sync, sb_row, + sb_col_in_tile, sb_cols_in_tile); } } @@ -4475,7 +5694,8 @@ static void init_encode_frame_mb_context(AV1_COMP *cpi) { MACROBLOCKD *const xd = &x->e_mbd; // Copy data over into macro block data structures. - av1_setup_src_planes(x, cpi->source, 0, 0, num_planes); + av1_setup_src_planes(x, cpi->source, 0, 0, num_planes, + cm->seq_params.sb_size); av1_setup_block_planes(xd, cm->seq_params.subsampling_x, cm->seq_params.subsampling_y, num_planes); @@ -4527,7 +5747,6 @@ void av1_alloc_tile_data(AV1_COMP *cpi) { for (i = 0; i < BLOCK_SIZES_ALL; ++i) { for (j = 0; j < MAX_MODES; ++j) { tile_data->thresh_freq_fact[i][j] = 32; - tile_data->mode_map[i][j] = j; } } } @@ -4561,6 +5780,7 @@ void av1_init_tile_data(AV1_COMP *cpi) { tile_data->allow_update_cdf = !cm->large_scale_tile; tile_data->allow_update_cdf = tile_data->allow_update_cdf && !cm->disable_cdf_update; + tile_data->tctx = *cm->fc; } } } @@ -4573,14 +5793,12 @@ void av1_encode_sb_row(AV1_COMP *cpi, ThreadData *td, int tile_row, TileDataEnc *this_tile = &cpi->tile_data[tile_row * tile_cols + tile_col]; const TileInfo *const tile_info = &this_tile->tile_info; TOKENEXTRA *tok = NULL; - int sb_row_in_tile; - int tile_mb_cols = (tile_info->mi_col_end - tile_info->mi_col_start + 2) >> 2; - - int num_mb_rows_in_sb = - ((1 << (cm->seq_params.mib_size_log2 + MI_SIZE_LOG2)) + 8) >> 4; - - sb_row_in_tile = + const int sb_row_in_tile = (mi_row - tile_info->mi_row_start) >> cm->seq_params.mib_size_log2; + const int tile_mb_cols = + (tile_info->mi_col_end - tile_info->mi_col_start + 2) >> 2; + const int num_mb_rows_in_sb = + ((1 << (cm->seq_params.mib_size_log2 + MI_SIZE_LOG2)) + 8) >> 4; get_start_tok(cpi, tile_row, tile_col, mi_row, &tok, cm->seq_params.mib_size_log2 + MI_SIZE_LOG2, num_planes); @@ -4611,9 +5829,7 @@ void av1_encode_tile(AV1_COMP *cpi, ThreadData *td, int tile_row, const TileInfo *const tile_info = &this_tile->tile_info; int mi_row; -#if CONFIG_COLLECT_INTER_MODE_RD_STATS av1_inter_mode_data_init(this_tile); -#endif av1_zero_above_context(cm, &td->mb.e_mbd, tile_info->mi_col_start, tile_info->mi_col_end, tile_row); @@ -4624,15 +5840,11 @@ void av1_encode_tile(AV1_COMP *cpi, ThreadData *td, int tile_row, this_tile->ex_search_count = 0; // Exhaustive mesh search hits. td->mb.m_search_count_ptr = &this_tile->m_search_count; td->mb.ex_search_count_ptr = &this_tile->ex_search_count; - this_tile->tctx = *cm->fc; - td->mb.e_mbd.tile_ctx = &this_tile->tctx; cfl_init(&td->mb.e_mbd.cfl, &cm->seq_params); av1_crc32c_calculator_init(&td->mb.mb_rd_record.crc_calculator); - td->intrabc_used_this_tile = 0; - for (mi_row = tile_info->mi_row_start; mi_row < tile_info->mi_row_end; mi_row += cm->seq_params.mib_size) { av1_encode_sb_row(cpi, td, tile_row, tile_col, mi_row); @@ -4652,8 +5864,13 @@ static void encode_tiles(AV1_COMP *cpi) { for (tile_row = 0; tile_row < tile_rows; ++tile_row) { for (tile_col = 0; tile_col < tile_cols; ++tile_col) { + TileDataEnc *const this_tile = + &cpi->tile_data[tile_row * cm->tile_cols + tile_col]; + cpi->td.intrabc_used = 0; + cpi->td.mb.e_mbd.tile_ctx = &this_tile->tctx; + cpi->td.mb.tile_pb_ctx = &this_tile->tctx; av1_encode_tile(cpi, &cpi->td, tile_row, tile_col); - cpi->intrabc_used |= cpi->td.intrabc_used_this_tile; + cpi->intrabc_used |= cpi->td.intrabc_used; } } } @@ -4661,8 +5878,9 @@ static void encode_tiles(AV1_COMP *cpi) { #if CONFIG_FP_MB_STATS static int input_fpmb_stats(FIRSTPASS_MB_STATS *firstpass_mb_stats, AV1_COMMON *cm, uint8_t **this_frame_mb_stats) { - uint8_t *mb_stats_in = firstpass_mb_stats->mb_stats_start + - cm->current_video_frame * cm->MBs * sizeof(uint8_t); + uint8_t *mb_stats_in = + firstpass_mb_stats->mb_stats_start + + cm->current_frame.frame_number * cm->MBs * sizeof(uint8_t); if (mb_stats_in > firstpass_mb_stats->mb_stats_end) return EOF; @@ -4728,47 +5946,17 @@ static int do_gm_search_logic(SPEED_FEATURES *const sf, int num_refs_using_gm, (void)frame; switch (sf->gm_search_type) { case GM_FULL_SEARCH: return 1; - case GM_REDUCED_REF_SEARCH: + case GM_REDUCED_REF_SEARCH_SKIP_L2_L3: return !(frame == LAST2_FRAME || frame == LAST3_FRAME); + case GM_REDUCED_REF_SEARCH_SKIP_L2_L3_ARF2: + return !(frame == LAST2_FRAME || frame == LAST3_FRAME || + (frame == ALTREF2_FRAME)); case GM_DISABLE_SEARCH: return 0; default: assert(0); } return 1; } -// Estimate if the source frame is screen content, based on the portion of -// blocks that have no more than 4 (experimentally selected) luma colors. -static int is_screen_content(const uint8_t *src, int use_hbd, int bd, - int stride, int width, int height) { - assert(src != NULL); - int counts = 0; - const int blk_w = 16; - const int blk_h = 16; - const int limit = 4; - for (int r = 0; r + blk_h <= height; r += blk_h) { - for (int c = 0; c + blk_w <= width; c += blk_w) { - int count_buf[1 << 12]; // Maximum (1 << 12) color levels. - const int n_colors = - use_hbd ? av1_count_colors_highbd(src + r * stride + c, stride, blk_w, - blk_h, bd, count_buf) - : av1_count_colors(src + r * stride + c, stride, blk_w, blk_h, - count_buf); - if (n_colors > 1 && n_colors <= limit) counts++; - } - } - // The threshold is 10%. - return counts * blk_h * blk_w * 10 > width * height; -} - -static const uint8_t ref_frame_flag_list[REF_FRAMES] = { 0, - AOM_LAST_FLAG, - AOM_LAST2_FLAG, - AOM_LAST3_FLAG, - AOM_GOLD_FLAG, - AOM_BWD_FLAG, - AOM_ALT2_FLAG, - AOM_ALT_FLAG }; - // Enforce the number of references for each arbitrary frame limited to // (INTER_REFS_PER_FRAME - 1) static void enforce_max_ref_frames(AV1_COMP *cpi) { @@ -4788,40 +5976,41 @@ static void enforce_max_ref_frames(AV1_COMP *cpi) { // the same quality level, remove the earliest reference frame. if (total_valid_refs == INTER_REFS_PER_FRAME) { - unsigned int min_ref_offset = UINT_MAX; - unsigned int second_min_ref_offset = UINT_MAX; + unsigned int min_ref_order_hint = UINT_MAX; + unsigned int second_min_ref_order_hint = UINT_MAX; MV_REFERENCE_FRAME earliest_ref_frames[2] = { LAST3_FRAME, LAST2_FRAME }; - int earliest_buf_idxes[2] = { 0 }; + const RefCntBuffer *earliest_bufs[2] = { NULL }; // Locate the earliest two reference frames except GOLDEN/ALTREF. for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) { // Retain GOLDEN/ALTERF if (ref_frame == GOLDEN_FRAME || ref_frame == ALTREF_FRAME) continue; - const int buf_idx = cm->frame_refs[ref_frame - LAST_FRAME].idx; - if (buf_idx >= 0) { - const unsigned int ref_offset = - cm->buffer_pool->frame_bufs[buf_idx].cur_frame_offset; + const RefCntBuffer *const buf = get_ref_frame_buf(cm, ref_frame); + if (buf != NULL) { + const unsigned int ref_order_hint = buf->order_hint; - if (min_ref_offset == UINT_MAX) { - min_ref_offset = ref_offset; + if (min_ref_order_hint == UINT_MAX) { + min_ref_order_hint = ref_order_hint; earliest_ref_frames[0] = ref_frame; - earliest_buf_idxes[0] = buf_idx; + earliest_bufs[0] = buf; } else { - if (get_relative_dist(cm, ref_offset, min_ref_offset) < 0) { - second_min_ref_offset = min_ref_offset; + if (get_relative_dist(&cm->seq_params.order_hint_info, ref_order_hint, + min_ref_order_hint) < 0) { + second_min_ref_order_hint = min_ref_order_hint; earliest_ref_frames[1] = earliest_ref_frames[0]; - earliest_buf_idxes[1] = earliest_buf_idxes[0]; + earliest_bufs[1] = earliest_bufs[0]; - min_ref_offset = ref_offset; + min_ref_order_hint = ref_order_hint; earliest_ref_frames[0] = ref_frame; - earliest_buf_idxes[0] = buf_idx; - } else if (second_min_ref_offset == UINT_MAX || - get_relative_dist(cm, ref_offset, second_min_ref_offset) < - 0) { - second_min_ref_offset = ref_offset; + earliest_bufs[0] = buf; + } else if (second_min_ref_order_hint == UINT_MAX || + get_relative_dist(&cm->seq_params.order_hint_info, + ref_order_hint, + second_min_ref_order_hint) < 0) { + second_min_ref_order_hint = ref_order_hint; earliest_ref_frames[1] = ref_frame; - earliest_buf_idxes[1] = buf_idx; + earliest_bufs[1] = buf; } } } @@ -4830,7 +6019,7 @@ static void enforce_max_ref_frames(AV1_COMP *cpi) { RATE_FACTOR_LEVEL ref_rf_level[2]; double ref_rf_deltas[2]; for (int i = 0; i < 2; ++i) { - ref_rf_level[i] = cpi->frame_rf_level[earliest_buf_idxes[i]]; + ref_rf_level[i] = earliest_bufs[i]->frame_rf_level; ref_rf_deltas[i] = rate_factor_deltas[ref_rf_level[i]]; } (void)ref_rf_level; @@ -4865,13 +6054,13 @@ static INLINE int av1_refs_are_one_sided(const AV1_COMMON *cm) { assert(!frame_is_intra_only(cm)); int one_sided_refs = 1; - for (int ref = 0; ref < INTER_REFS_PER_FRAME; ++ref) { - const int buf_idx = cm->frame_refs[ref].idx; - if (buf_idx == INVALID_IDX) continue; + for (int ref = LAST_FRAME; ref <= ALTREF_FRAME; ++ref) { + const RefCntBuffer *const buf = get_ref_frame_buf(cm, ref); + if (buf == NULL) continue; - const int ref_offset = - cm->buffer_pool->frame_bufs[buf_idx].cur_frame_offset; - if (get_relative_dist(cm, ref_offset, (int)cm->frame_offset) > 0) { + const int ref_order_hint = buf->order_hint; + if (get_relative_dist(&cm->seq_params.order_hint_info, ref_order_hint, + (int)cm->current_frame.order_hint) > 0) { one_sided_refs = 0; // bwd reference break; } @@ -4880,31 +6069,36 @@ static INLINE int av1_refs_are_one_sided(const AV1_COMMON *cm) { } static INLINE void get_skip_mode_ref_offsets(const AV1_COMMON *cm, - int ref_offset[2]) { - ref_offset[0] = ref_offset[1] = 0; - if (!cm->is_skip_mode_allowed) return; + int ref_order_hint[2]) { + const SkipModeInfo *const skip_mode_info = &cm->current_frame.skip_mode_info; + ref_order_hint[0] = ref_order_hint[1] = 0; + if (!skip_mode_info->skip_mode_allowed) return; - const int buf_idx_0 = cm->frame_refs[cm->ref_frame_idx_0].idx; - const int buf_idx_1 = cm->frame_refs[cm->ref_frame_idx_1].idx; - assert(buf_idx_0 != INVALID_IDX && buf_idx_1 != INVALID_IDX); + const RefCntBuffer *const buf_0 = + get_ref_frame_buf(cm, LAST_FRAME + skip_mode_info->ref_frame_idx_0); + const RefCntBuffer *const buf_1 = + get_ref_frame_buf(cm, LAST_FRAME + skip_mode_info->ref_frame_idx_1); + assert(buf_0 != NULL && buf_1 != NULL); - ref_offset[0] = cm->buffer_pool->frame_bufs[buf_idx_0].cur_frame_offset; - ref_offset[1] = cm->buffer_pool->frame_bufs[buf_idx_1].cur_frame_offset; + ref_order_hint[0] = buf_0->order_hint; + ref_order_hint[1] = buf_1->order_hint; } static int check_skip_mode_enabled(AV1_COMP *const cpi) { AV1_COMMON *const cm = &cpi->common; av1_setup_skip_mode_allowed(cm); - if (!cm->is_skip_mode_allowed) return 0; + if (!cm->current_frame.skip_mode_info.skip_mode_allowed) return 0; // Turn off skip mode if the temporal distances of the reference pair to the // current frame are different by more than 1 frame. - const int cur_offset = (int)cm->frame_offset; + const int cur_offset = (int)cm->current_frame.order_hint; int ref_offset[2]; get_skip_mode_ref_offsets(cm, ref_offset); - const int cur_to_ref0 = get_relative_dist(cm, cur_offset, ref_offset[0]); - const int cur_to_ref1 = abs(get_relative_dist(cm, cur_offset, ref_offset[1])); + const int cur_to_ref0 = get_relative_dist(&cm->seq_params.order_hint_info, + cur_offset, ref_offset[0]); + const int cur_to_ref1 = abs(get_relative_dist(&cm->seq_params.order_hint_info, + cur_offset, ref_offset[1])); if (abs(cur_to_ref0 - cur_to_ref1) > 1) return 0; // High Latency: Turn off skip mode if all refs are fwd. @@ -4918,8 +6112,10 @@ static int check_skip_mode_enabled(AV1_COMP *const cpi) { AOM_BWD_FLAG, AOM_ALT2_FLAG, AOM_ALT_FLAG }; - const int ref_frame[2] = { cm->ref_frame_idx_0 + LAST_FRAME, - cm->ref_frame_idx_1 + LAST_FRAME }; + const int ref_frame[2] = { + cm->current_frame.skip_mode_info.ref_frame_idx_0 + LAST_FRAME, + cm->current_frame.skip_mode_info.ref_frame_idx_1 + LAST_FRAME + }; if (!(cpi->ref_frame_flags & flag_list[ref_frame[0]]) || !(cpi->ref_frame_flags & flag_list[ref_frame[1]])) return 0; @@ -4933,8 +6129,9 @@ static INLINE int skip_gm_frame(AV1_COMMON *const cm, int ref_frame) { if ((ref_frame == LAST3_FRAME || ref_frame == LAST2_FRAME) && cm->global_motion[GOLDEN_FRAME].wmtype != IDENTITY) { return get_relative_dist( - cm, cm->cur_frame->ref_frame_offset[ref_frame - LAST_FRAME], - cm->cur_frame->ref_frame_offset[GOLDEN_FRAME - LAST_FRAME]) <= 0; + &cm->seq_params.order_hint_info, + cm->cur_frame->ref_order_hints[ref_frame - LAST_FRAME], + cm->cur_frame->ref_order_hints[GOLDEN_FRAME - LAST_FRAME]) <= 0; } return 0; } @@ -4968,20 +6165,6 @@ static void encode_frame_internal(AV1_COMP *cpi) { av1_zero(*td->counts); av1_zero(rdc->comp_pred_diff); - if (frame_is_intra_only(cm)) { - if (cm->seq_params.force_screen_content_tools == 2) { - cm->allow_screen_content_tools = - cpi->oxcf.content == AOM_CONTENT_SCREEN || - is_screen_content(cpi->source->y_buffer, - cpi->source->flags & YV12_FLAG_HIGHBITDEPTH, xd->bd, - cpi->source->y_stride, cpi->source->y_width, - cpi->source->y_height); - } else { - cm->allow_screen_content_tools = - cm->seq_params.force_screen_content_tools; - } - } - // Allow intrabc when screen content tools are enabled. cm->allow_intrabc = cm->allow_screen_content_tools; // Reset the flag. @@ -5084,15 +6267,42 @@ static void encode_frame_internal(AV1_COMP *cpi) { cm->tx_mode = select_tx_mode(cpi); // Fix delta q resolution for the moment - cm->delta_q_res = DEFAULT_DELTA_Q_RES; + cm->delta_q_info.delta_q_res = DEFAULT_DELTA_Q_RES; // Set delta_q_present_flag before it is used for the first time - cm->delta_lf_res = DEFAULT_DELTA_LF_RES; - cm->delta_q_present_flag = cpi->oxcf.deltaq_mode != NO_DELTA_Q; - cm->delta_lf_present_flag = cpi->oxcf.deltaq_mode == DELTA_Q_LF; - cm->delta_lf_multi = DEFAULT_DELTA_LF_MULTI; - // update delta_q_present_flag and delta_lf_present_flag based on base_qindex - cm->delta_q_present_flag &= cm->base_qindex > 0; - cm->delta_lf_present_flag &= cm->base_qindex > 0; + cm->delta_q_info.delta_lf_res = DEFAULT_DELTA_LF_RES; + cm->delta_q_info.delta_q_present_flag = cpi->oxcf.deltaq_mode != NO_DELTA_Q; + cm->delta_q_info.delta_lf_present_flag = cpi->oxcf.deltaq_mode == DELTA_Q_LF; + cm->delta_q_info.delta_lf_multi = DEFAULT_DELTA_LF_MULTI; + // update delta_q_present_flag and delta_lf_present_flag based on + // base_qindex + cm->delta_q_info.delta_q_present_flag &= cm->base_qindex > 0; + cm->delta_q_info.delta_lf_present_flag &= cm->base_qindex > 0; + + if (cpi->twopass.gf_group.index && + cpi->twopass.gf_group.index < MAX_LAG_BUFFERS && + cpi->oxcf.enable_tpl_model) { + TplDepFrame *tpl_frame = &cpi->tpl_stats[cpi->twopass.gf_group.index]; + TplDepStats *tpl_stats = tpl_frame->tpl_stats_ptr; + + int tpl_stride = tpl_frame->stride; + int64_t intra_cost_base = 0; + int64_t mc_dep_cost_base = 0; + int row, col; + + for (row = 0; row < cm->mi_rows; ++row) { + for (col = 0; col < cm->mi_cols; ++col) { + TplDepStats *this_stats = &tpl_stats[row * tpl_stride + col]; + intra_cost_base += this_stats->intra_cost; + mc_dep_cost_base += this_stats->mc_dep_cost; + } + } + + aom_clear_system_state(); + + if (tpl_frame->is_valid) + cpi->rd.r0 = + (double)intra_cost_base / (intra_cost_base + mc_dep_cost_base); + } av1_frame_init_quantizer(cpi); @@ -5104,7 +6314,6 @@ static void encode_frame_internal(AV1_COMP *cpi) { cm->last_frame_seg_map = cm->prev_frame->seg_map; else cm->last_frame_seg_map = NULL; - cm->current_frame_seg_map = cm->cur_frame->seg_map; if (cm->allow_intrabc || cm->coded_lossless) { av1_set_default_ref_deltas(cm->lf.ref_deltas); av1_set_default_mode_deltas(cm->lf.mode_deltas); @@ -5120,27 +6329,31 @@ static void encode_frame_internal(AV1_COMP *cpi) { cm->prev_mi = cm->allow_ref_frame_mvs ? cm->prev_mip : NULL; x->txb_split_count = 0; +#if CONFIG_SPEED_STATS + x->tx_search_count = 0; +#endif // CONFIG_SPEED_STATS av1_zero(rdc->global_motion_used); av1_zero(cpi->gmparams_cost); -#if !CONFIG_GLOBAL_MOTION_SEARCH - cpi->global_motion_search_done = 1; -#endif // !CONFIG_GLOBAL_MOTION_SEARCH - if (cpi->common.frame_type == INTER_FRAME && cpi->source && - !cpi->global_motion_search_done) { + if (cpi->common.current_frame.frame_type == INTER_FRAME && cpi->source && + cpi->oxcf.enable_global_motion && !cpi->global_motion_search_done) { YV12_BUFFER_CONFIG *ref_buf[REF_FRAMES]; int frame; double params_by_motion[RANSAC_NUM_MOTIONS * (MAX_PARAMDIM - 1)]; const double *params_this_motion; int inliers_by_motion[RANSAC_NUM_MOTIONS]; WarpedMotionParams tmp_wm_params; + // clang-format off static const double kIdentityParams[MAX_PARAMDIM - 1] = { 0.0, 0.0, 1.0, 0.0, 0.0, 1.0, 0.0, 0.0 }; + // clang-format on int num_refs_using_gm = 0; for (frame = ALTREF_FRAME; frame >= LAST_FRAME; --frame) { - ref_buf[frame] = get_ref_frame_buffer(cpi, frame); + ref_buf[frame] = NULL; + RefCntBuffer *buf = get_ref_frame_buf(cm, frame); + if (buf != NULL) ref_buf[frame] = &buf->buf; int pframe; cm->global_motion[frame] = default_warp_params; const WarpedMotionParams *ref_params = @@ -5159,11 +6372,10 @@ static void encode_frame_internal(AV1_COMP *cpi) { do_gm_search_logic(&cpi->sf, num_refs_using_gm, frame) && !(cpi->sf.selective_ref_gm && skip_gm_frame(cm, frame))) { TransformationType model; - const int64_t ref_frame_error = - av1_frame_error(xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH, xd->bd, - ref_buf[frame]->y_buffer, ref_buf[frame]->y_stride, - cpi->source->y_buffer, cpi->source->y_width, - cpi->source->y_height, cpi->source->y_stride); + const int64_t ref_frame_error = av1_frame_error( + is_cur_buf_hbd(xd), xd->bd, ref_buf[frame]->y_buffer, + ref_buf[frame]->y_stride, cpi->source->y_buffer, + cpi->source->y_width, cpi->source->y_height, cpi->source->y_stride); if (ref_frame_error == 0) continue; @@ -5176,30 +6388,30 @@ static void encode_frame_internal(AV1_COMP *cpi) { (MAX_PARAMDIM - 1) * sizeof(*params_by_motion)); } - compute_global_motion_feature_based( - model, cpi->source, ref_buf[frame], - cpi->common.seq_params.bit_depth, inliers_by_motion, - params_by_motion, RANSAC_NUM_MOTIONS); + av1_compute_global_motion(model, cpi->source, ref_buf[frame], + cpi->common.seq_params.bit_depth, + inliers_by_motion, params_by_motion, + RANSAC_NUM_MOTIONS); for (i = 0; i < RANSAC_NUM_MOTIONS; ++i) { if (inliers_by_motion[i] == 0) continue; params_this_motion = params_by_motion + (MAX_PARAMDIM - 1) * i; - convert_model_to_params(params_this_motion, &tmp_wm_params); + av1_convert_model_to_params(params_this_motion, &tmp_wm_params); if (tmp_wm_params.wmtype != IDENTITY) { - const int64_t warp_error = refine_integerized_param( - &tmp_wm_params, tmp_wm_params.wmtype, - xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH, xd->bd, - ref_buf[frame]->y_buffer, ref_buf[frame]->y_width, + const int64_t warp_error = av1_refine_integerized_param( + &tmp_wm_params, tmp_wm_params.wmtype, is_cur_buf_hbd(xd), + xd->bd, ref_buf[frame]->y_buffer, ref_buf[frame]->y_width, ref_buf[frame]->y_height, ref_buf[frame]->y_stride, cpi->source->y_buffer, cpi->source->y_width, cpi->source->y_height, cpi->source->y_stride, 5, best_warp_error); if (warp_error < best_warp_error) { best_warp_error = warp_error; - // Save the wm_params modified by refine_integerized_param() - // rather than motion index to avoid rerunning refine() below. + // Save the wm_params modified by + // av1_refine_integerized_param() rather than motion index to + // avoid rerunning refine() below. memcpy(&(cm->global_motion[frame]), &tmp_wm_params, sizeof(WarpedMotionParams)); } @@ -5222,7 +6434,7 @@ static void encode_frame_internal(AV1_COMP *cpi) { // If the best error advantage found doesn't meet the threshold for // this motion type, revert to IDENTITY. - if (!is_enough_erroradvantage( + if (!av1_is_enough_erroradvantage( (double)best_warp_error / ref_frame_error, gm_get_params_cost(&cm->global_motion[frame], ref_params, cm->allow_high_precision_mv), @@ -5259,12 +6471,10 @@ static void encode_frame_internal(AV1_COMP *cpi) { cpi->all_one_sided_refs = frame_is_intra_only(cm) ? 0 : av1_refs_are_one_sided(cm); - cm->skip_mode_flag = check_skip_mode_enabled(cpi); + cm->current_frame.skip_mode_info.skip_mode_flag = + check_skip_mode_enabled(cpi); { - struct aom_usec_timer emr_timer; - aom_usec_timer_start(&emr_timer); - #if CONFIG_FP_MB_STATS if (cpi->use_fp_mb_stats) { input_fpmb_stats(&cpi->twopass.firstpass_mb_stats, cm, @@ -5272,41 +6482,34 @@ static void encode_frame_internal(AV1_COMP *cpi) { } #endif - if (cpi->row_mt && (cpi->oxcf.max_threads > 1)) - av1_encode_tiles_mt(cpi); - else if (AOMMIN(cpi->oxcf.max_threads, cm->tile_cols * cm->tile_rows) > 1) - av1_encode_tiles_mt(cpi); - else - encode_tiles(cpi); - - aom_usec_timer_mark(&emr_timer); - cpi->time_encode_sb_row += aom_usec_timer_elapsed(&emr_timer); + cpi->row_mt_sync_read_ptr = av1_row_mt_sync_read_dummy; + cpi->row_mt_sync_write_ptr = av1_row_mt_sync_write_dummy; + cpi->row_mt = 0; + if (cpi->oxcf.row_mt && (cpi->oxcf.max_threads > 1)) { + cpi->row_mt = 1; + cpi->row_mt_sync_read_ptr = av1_row_mt_sync_read; + cpi->row_mt_sync_write_ptr = av1_row_mt_sync_write; + av1_encode_tiles_row_mt(cpi); + } else { + if (AOMMIN(cpi->oxcf.max_threads, cm->tile_cols * cm->tile_rows) > 1) + av1_encode_tiles_mt(cpi); + else + encode_tiles(cpi); + } } // If intrabc is allowed but never selected, reset the allow_intrabc flag. if (cm->allow_intrabc && !cpi->intrabc_used) cm->allow_intrabc = 0; - if (cm->allow_intrabc) cm->delta_lf_present_flag = 0; + if (cm->allow_intrabc) cm->delta_q_info.delta_lf_present_flag = 0; } void av1_encode_frame(AV1_COMP *cpi) { AV1_COMMON *const cm = &cpi->common; + CurrentFrame *const current_frame = &cm->current_frame; const int num_planes = av1_num_planes(cm); // Indicates whether or not to use a default reduced set for ext-tx // rather than the potential full set of 16 transforms - cm->reduced_tx_set_used = 0; - - if (cm->show_frame == 0) { - int arf_offset = AOMMIN( - (MAX_GF_INTERVAL - 1), - cpi->twopass.gf_group.arf_src_offset[cpi->twopass.gf_group.index]); - int brf_offset = - cpi->twopass.gf_group.brf_src_offset[cpi->twopass.gf_group.index]; - arf_offset = AOMMIN((MAX_GF_INTERVAL - 1), arf_offset + brf_offset); - cm->frame_offset = cm->current_video_frame + arf_offset; - } else { - cm->frame_offset = cm->current_video_frame; - } - cm->frame_offset %= (1 << (cm->seq_params.order_hint_bits_minus_1 + 1)); + cm->reduced_tx_set_used = cpi->oxcf.reduced_tx_type_set; // Make sure segment_id is no larger than last_active_segid. if (cm->seg.enabled && cm->seg.update_map) { @@ -5323,7 +6526,9 @@ void av1_encode_frame(AV1_COMP *cpi) { } av1_setup_frame_buf_refs(cm); - if (cpi->sf.selective_ref_frame >= 2) enforce_max_ref_frames(cpi); + if (cpi->sf.selective_ref_frame >= 3 && cpi->oxcf.max_reference_frames == 7) { + enforce_max_ref_frames(cpi); + } av1_setup_frame_sign_bias(cm); #if CONFIG_MISMATCH_DEBUG @@ -5332,8 +6537,6 @@ void av1_encode_frame(AV1_COMP *cpi) { (void)num_planes; #endif - cpi->allow_comp_inter_inter = !frame_is_intra_only(cm); - if (cpi->sf.frame_parameter_update) { int i; RD_OPT *const rd_opt = &cpi->rd; @@ -5355,10 +6558,10 @@ void av1_encode_frame(AV1_COMP *cpi) { /* prediction (compound, single or hybrid) mode selection */ // NOTE: "is_alt_ref" is true only for OVERLAY/INTNL_OVERLAY frames - if (is_alt_ref || !cpi->allow_comp_inter_inter) - cm->reference_mode = SINGLE_REFERENCE; + if (is_alt_ref || frame_is_intra_only(cm)) + current_frame->reference_mode = SINGLE_REFERENCE; else - cm->reference_mode = REFERENCE_MODE_SELECT; + current_frame->reference_mode = REFERENCE_MODE_SELECT; cm->interp_filter = SWITCHABLE; if (cm->large_scale_tile) cm->interp_filter = EIGHTTAP_REGULAR; @@ -5373,22 +6576,25 @@ void av1_encode_frame(AV1_COMP *cpi) { for (i = 0; i < REFERENCE_MODES; ++i) mode_thrs[i] = (mode_thrs[i] + rdc->comp_pred_diff[i] / cm->MBs) / 2; - if (cm->reference_mode == REFERENCE_MODE_SELECT) { + if (current_frame->reference_mode == REFERENCE_MODE_SELECT) { // Use a flag that includes 4x4 blocks if (rdc->compound_ref_used_flag == 0) { - cm->reference_mode = SINGLE_REFERENCE; + current_frame->reference_mode = SINGLE_REFERENCE; #if CONFIG_ENTROPY_STATS av1_zero(cpi->td.counts->comp_inter); #endif // CONFIG_ENTROPY_STATS } } - // Re-check on the skip mode status as reference mode may have been changed. - if (frame_is_intra_only(cm) || cm->reference_mode == SINGLE_REFERENCE) { - cm->is_skip_mode_allowed = 0; - cm->skip_mode_flag = 0; + // Re-check on the skip mode status as reference mode may have been + // changed. + SkipModeInfo *const skip_mode_info = ¤t_frame->skip_mode_info; + if (frame_is_intra_only(cm) || + current_frame->reference_mode == SINGLE_REFERENCE) { + skip_mode_info->skip_mode_allowed = 0; + skip_mode_info->skip_mode_flag = 0; } - if (cm->skip_mode_flag && rdc->skip_mode_used_flag == 0) - cm->skip_mode_flag = 0; + if (skip_mode_info->skip_mode_flag && rdc->skip_mode_used_flag == 0) + skip_mode_info->skip_mode_flag = 0; if (!cm->large_scale_tile) { if (cm->tx_mode == TX_MODE_SELECT && cpi->td.mb.txb_split_count == 0) @@ -5625,15 +6831,19 @@ static void encode_superblock(const AV1_COMP *const cpi, TileDataEnc *tile_data, set_ref_ptrs(cm, xd, mbmi->ref_frame[0], mbmi->ref_frame[1]); for (ref = 0; ref < 1 + is_compound; ++ref) { - YV12_BUFFER_CONFIG *cfg = get_ref_frame_buffer(cpi, mbmi->ref_frame[ref]); + const YV12_BUFFER_CONFIG *cfg = + get_ref_frame_yv12_buf(cm, mbmi->ref_frame[ref]); assert(IMPLIES(!is_intrabc_block(mbmi), cfg)); av1_setup_pre_planes(xd, ref, cfg, mi_row, mi_col, - &xd->block_refs[ref]->sf, num_planes); + xd->block_ref_scale_factors[ref], num_planes); } - av1_build_inter_predictors_sb(cm, xd, mi_row, mi_col, NULL, bsize); - if (mbmi->motion_mode == OBMC_CAUSAL) + av1_enc_build_inter_predictor(cm, xd, mi_row, mi_col, NULL, bsize, 0, + av1_num_planes(cm) - 1); + if (mbmi->motion_mode == OBMC_CAUSAL) { + assert(cpi->oxcf.enable_obmc == 1); av1_build_obmc_inter_predictors_sb(cm, xd, mi_row, mi_col); + } #if CONFIG_MISMATCH_DEBUG if (dry_run == OUTPUT_ENABLED) { @@ -5645,9 +6855,9 @@ static void encode_superblock(const AV1_COMP *const cpi, TileDataEnc *tile_data, if (!is_chroma_reference(mi_row, mi_col, bsize, pd->subsampling_x, pd->subsampling_y)) continue; - mismatch_record_block_pre(pd->dst.buf, pd->dst.stride, cm->frame_offset, - plane, pixel_c, pixel_r, pd->width, - pd->height, + mismatch_record_block_pre(pd->dst.buf, pd->dst.stride, + cm->current_frame.order_hint, plane, pixel_c, + pixel_r, pd->width, pd->height, xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH); } } @@ -5661,8 +6871,7 @@ static void encode_superblock(const AV1_COMP *const cpi, TileDataEnc *tile_data, } if (!dry_run) { - if (av1_allow_intrabc(cm) && is_intrabc_block(mbmi)) - td->intrabc_used_this_tile = 1; + if (av1_allow_intrabc(cm) && is_intrabc_block(mbmi)) td->intrabc_used = 1; if (cm->tx_mode == TX_MODE_SELECT && !xd->lossless[mbmi->segment_id] && mbmi->sb_type > BLOCK_4X4 && !(is_inter && (mbmi->skip || seg_skip))) { if (is_inter) { diff --git a/third_party/aom/av1/encoder/encodeframe.h b/third_party/aom/av1/encoder/encodeframe.h index e8cf9b4685..3b1730d323 100644 --- a/third_party/aom/av1/encoder/encodeframe.h +++ b/third_party/aom/av1/encoder/encodeframe.h @@ -29,7 +29,7 @@ struct ThreadData; void av1_setup_src_planes(struct macroblock *x, const struct yv12_buffer_config *src, int mi_row, - int mi_col, const int num_planes); + int mi_col, const int num_planes, BLOCK_SIZE bsize); void av1_encode_frame(struct AV1_COMP *cpi); diff --git a/third_party/aom/av1/encoder/encodemb.c b/third_party/aom/av1/encoder/encodemb.c index ad12577e6e..6edb0f88cd 100644 --- a/third_party/aom/av1/encoder/encodemb.c +++ b/third_party/aom/av1/encoder/encodemb.c @@ -43,7 +43,7 @@ static void subtract_block(const MACROBLOCKD *xd, int rows, int cols, const uint8_t *src8, ptrdiff_t src_stride, const uint8_t *pred8, ptrdiff_t pred_stride) { if (check_subtract_block_size(rows, cols)) { - if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + if (is_cur_buf_hbd(xd)) { aom_highbd_subtract_block_c(rows, cols, diff, diff_stride, src8, src_stride, pred8, pred_stride, xd->bd); return; @@ -54,7 +54,7 @@ static void subtract_block(const MACROBLOCKD *xd, int rows, int cols, return; } - if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + if (is_cur_buf_hbd(xd)) { aom_highbd_subtract_block(rows, cols, diff, diff_stride, src8, src_stride, pred8, pred_stride, xd->bd); return; @@ -116,11 +116,11 @@ int av1_optimize_b(const struct AV1_COMP *cpi, MACROBLOCK *mb, int plane, rate_cost, cpi->oxcf.sharpness); } -typedef enum QUANT_FUNC { +enum { QUANT_FUNC_LOWBD = 0, QUANT_FUNC_HIGHBD = 1, QUANT_FUNC_TYPES = 2 -} QUANT_FUNC; +} UENUM1BYTE(QUANT_FUNC); static AV1_QUANT_FACADE quant_func_list[AV1_XFORM_QUANT_TYPES][QUANT_FUNC_TYPES] = { @@ -163,6 +163,7 @@ void av1_xform_quant(const AV1_COMMON *cm, MACROBLOCK *x, int plane, int block, qparam.tx_size = tx_size; qparam.qmatrix = qmatrix; qparam.iqmatrix = iqmatrix; + qparam.use_quant_b_adapt = cm->use_quant_b_adapt; TxfmParam txfm_param; txfm_param.tx_type = tx_type; txfm_param.tx_size = tx_size; @@ -171,7 +172,7 @@ void av1_xform_quant(const AV1_COMMON *cm, MACROBLOCK *x, int plane, int block, txfm_param.tx_size, is_inter_block(mbmi), cm->reduced_tx_set_used); txfm_param.bd = xd->bd; - txfm_param.is_hbd = get_bitdepth_data_path_index(xd); + txfm_param.is_hbd = is_cur_buf_hbd(xd); av1_fwd_txfm(src_diff, coeff, diff_stride, &txfm_param); @@ -226,7 +227,7 @@ static void encode_block(int plane, int block, int blk_row, int blk_col, if (!is_blk_skip(x, plane, blk_row * bw + blk_col) && !mbmi->skip_mode) { TX_TYPE tx_type = av1_get_tx_type(pd->plane_type, xd, blk_row, blk_col, tx_size, cm->reduced_tx_set_used); - if (args->enable_optimize_b) { + if (args->enable_optimize_b != NO_TRELLIS_OPT) { av1_xform_quant(cm, x, plane, block, blk_row, blk_col, plane_bsize, tx_size, tx_type, AV1_XFORM_QUANT_FP); TXB_CTX txb_ctx; @@ -256,11 +257,11 @@ static void encode_block(int plane, int block, int blk_row, int blk_col, } if (p->eobs[block] == 0 && plane == 0) { - // TODO(debargha, jingning): Temporarily disable txk_type check for eob=0 - // case. It is possible that certain collision in hash index would cause - // the assertion failure. To further optimize the rate-distortion - // performance, we need to re-visit this part and enable this assert - // again. + // TODO(debargha, jingning): Temporarily disable txk_type check for eob=0 + // case. It is possible that certain collision in hash index would cause + // the assertion failure. To further optimize the rate-distortion + // performance, we need to re-visit this part and enable this assert + // again. #if 0 if (args->cpi->oxcf.aq_mode == NO_AQ && args->cpi->oxcf.deltaq_mode == NO_DELTA_Q) { @@ -285,8 +286,8 @@ static void encode_block(int plane, int block, int blk_row, int blk_col, int blk_h = block_size_high[bsize]; mi_to_pixel_loc(&pixel_c, &pixel_r, mi_col, mi_row, blk_col, blk_row, pd->subsampling_x, pd->subsampling_y); - mismatch_record_block_tx(dst, pd->dst.stride, cm->frame_offset, plane, - pixel_c, pixel_r, blk_w, blk_h, + mismatch_record_block_tx(dst, pd->dst.stride, cm->current_frame.order_hint, + plane, pixel_c, pixel_r, blk_w, blk_h, xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH); } #endif @@ -431,7 +432,7 @@ static void encode_block_pass1(int plane, int block, int blk_row, int blk_col, if (p->eobs[block] > 0) { txfm_param.bd = xd->bd; - txfm_param.is_hbd = get_bitdepth_data_path_index(xd); + txfm_param.is_hbd = is_cur_buf_hbd(xd); txfm_param.tx_type = DCT_DCT; txfm_param.tx_size = tx_size; txfm_param.eob = p->eobs[block]; @@ -578,7 +579,7 @@ void av1_encode_block_intra(int plane, int block, int blk_row, int blk_col, const ENTROPY_CONTEXT *a = &args->ta[blk_col]; const ENTROPY_CONTEXT *l = &args->tl[blk_row]; - if (args->enable_optimize_b) { + if (args->enable_optimize_b != NO_TRELLIS_OPT) { av1_xform_quant(cm, x, plane, block, blk_row, blk_col, plane_bsize, tx_size, tx_type, AV1_XFORM_QUANT_FP); TXB_CTX txb_ctx; @@ -598,11 +599,11 @@ void av1_encode_block_intra(int plane, int block, int blk_row, int blk_col, } if (*eob == 0 && plane == 0) { - // TODO(jingning): Temporarily disable txk_type check for eob=0 case. - // It is possible that certain collision in hash index would cause - // the assertion failure. To further optimize the rate-distortion - // performance, we need to re-visit this part and enable this assert - // again. + // TODO(jingning): Temporarily disable txk_type check for eob=0 case. + // It is possible that certain collision in hash index would cause + // the assertion failure. To further optimize the rate-distortion + // performance, we need to re-visit this part and enable this assert + // again. #if 0 if (args->cpi->oxcf.aq_mode == NO_AQ && args->cpi->oxcf.deltaq_mode == NO_DELTA_Q) { diff --git a/third_party/aom/av1/encoder/encodemb.h b/third_party/aom/av1/encoder/encodemb.h index 39080de599..d4394cf0be 100644 --- a/third_party/aom/av1/encoder/encodemb.h +++ b/third_party/aom/av1/encoder/encodemb.h @@ -37,13 +37,13 @@ struct encode_b_args { int8_t enable_optimize_b; }; -typedef enum AV1_XFORM_QUANT { +enum { AV1_XFORM_QUANT_FP = 0, AV1_XFORM_QUANT_B = 1, AV1_XFORM_QUANT_DC = 2, AV1_XFORM_QUANT_SKIP_QUANT, AV1_XFORM_QUANT_TYPES, -} AV1_XFORM_QUANT; +} UENUM1BYTE(AV1_XFORM_QUANT); void av1_encode_sb(const struct AV1_COMP *cpi, MACROBLOCK *x, BLOCK_SIZE bsize, int mi_row, int mi_col, RUN_TYPE dry_run); diff --git a/third_party/aom/av1/encoder/encoder.c b/third_party/aom/av1/encoder/encoder.c index a2da2df899..9428a92740 100644 --- a/third_party/aom/av1/encoder/encoder.c +++ b/third_party/aom/av1/encoder/encoder.c @@ -33,9 +33,9 @@ #include "aom_ports/mem.h" #include "aom_ports/system_state.h" #include "aom_scale/aom_scale.h" -#if CONFIG_BITSTREAM_DEBUG || CONFIG_MISMATCH_DEBUG +#if CONFIG_BITSTREAM_DEBUG #include "aom_util/debug_util.h" -#endif // CONFIG_BITSTREAM_DEBUG || CONFIG_MISMATCH_DEBUG +#endif // CONFIG_BITSTREAM_DEBUG #include "av1/common/alloccommon.h" #include "av1/common/cdef.h" @@ -46,6 +46,7 @@ #include "av1/common/resize.h" #include "av1/common/tile_common.h" +#include "av1/encoder/av1_multi_thread.h" #include "av1/encoder/aq_complexity.h" #include "av1/encoder/aq_cyclicrefresh.h" #include "av1/encoder/aq_variance.h" @@ -53,6 +54,7 @@ #include "av1/encoder/context_tree.h" #include "av1/encoder/encodeframe.h" #include "av1/encoder/encodemv.h" +#include "av1/encoder/encode_strategy.h" #include "av1/encoder/encoder.h" #include "av1/encoder/encodetxb.h" #include "av1/encoder/ethread.h" @@ -65,15 +67,13 @@ #include "av1/encoder/random.h" #include "av1/encoder/ratectrl.h" #include "av1/encoder/rd.h" +#include "av1/encoder/rdopt.h" #include "av1/encoder/segmentation.h" #include "av1/encoder/speed_features.h" -#include "av1/encoder/temporal_filter.h" +#include "av1/encoder/reconinter_enc.h" #define DEFAULT_EXPLICIT_ORDER_HINT_BITS 7 -// av1 uses 10,000,000 ticks/second as time stamp -#define TICKS_PER_SEC 10000000LL - #if CONFIG_ENTROPY_STATS FRAME_COUNTS aggregate_fc; #endif // CONFIG_ENTROPY_STATS @@ -97,6 +97,30 @@ FILE *yuv_rec_file; #define FILE_NAME_LEN 100 #endif +// Estimate if the source frame is screen content, based on the portion of +// blocks that have no more than 4 (experimentally selected) luma colors. +static int is_screen_content(const uint8_t *src, int use_hbd, int bd, + int stride, int width, int height) { + assert(src != NULL); + int counts = 0; + const int blk_w = 16; + const int blk_h = 16; + const int limit = 4; + for (int r = 0; r + blk_h <= height; r += blk_h) { + for (int c = 0; c + blk_w <= width; c += blk_w) { + int count_buf[1 << 12]; // Maximum (1 << 12) color levels. + const int n_colors = + use_hbd ? av1_count_colors_highbd(src + r * stride + c, stride, blk_w, + blk_h, bd, count_buf) + : av1_count_colors(src + r * stride + c, stride, blk_w, blk_h, + count_buf); + if (n_colors > 1 && n_colors <= limit) counts++; + } + } + // The threshold is 10%. + return counts * blk_h * blk_w * 10 > width * height; +} + static INLINE void Scale2Ratio(AOM_SCALING mode, int *hr, int *hs) { switch (mode) { case NORMAL: @@ -238,6 +262,66 @@ int av1_get_active_map(AV1_COMP *cpi, unsigned char *new_map_16x16, int rows, } } +// Compute the horizontal frequency components' energy in a frame +// by calculuating the 16x4 Horizontal DCT. This is to be used to +// decide the superresolution parameters. +void analyze_hor_freq(const AV1_COMP *cpi, double *energy) { + uint64_t freq_energy[16] = { 0 }; + const YV12_BUFFER_CONFIG *buf = cpi->source; + const int bd = cpi->td.mb.e_mbd.bd; + const int width = buf->y_crop_width; + const int height = buf->y_crop_height; + DECLARE_ALIGNED(16, int32_t, coeff[16 * 4]); + int n = 0; + memset(freq_energy, 0, sizeof(freq_energy)); + if (buf->flags & YV12_FLAG_HIGHBITDEPTH) { + const int16_t *src16 = (const int16_t *)CONVERT_TO_SHORTPTR(buf->y_buffer); + for (int i = 0; i < height - 4; i += 4) { + for (int j = 0; j < width - 16; j += 16) { + av1_fwd_txfm2d_16x4(src16 + i * buf->y_stride + j, coeff, buf->y_stride, + H_DCT, bd); + for (int k = 1; k < 16; ++k) { + const uint64_t this_energy = + ((int64_t)coeff[k] * coeff[k]) + + ((int64_t)coeff[k + 16] * coeff[k + 16]) + + ((int64_t)coeff[k + 32] * coeff[k + 32]) + + ((int64_t)coeff[k + 48] * coeff[k + 48]); + freq_energy[k] += ROUND_POWER_OF_TWO(this_energy, 2 + 2 * (bd - 8)); + } + n++; + } + } + } else { + assert(bd == 8); + DECLARE_ALIGNED(16, int16_t, src16[16 * 4]); + for (int i = 0; i < height - 4; i += 4) { + for (int j = 0; j < width - 16; j += 16) { + for (int ii = 0; ii < 4; ++ii) + for (int jj = 0; jj < 16; ++jj) + src16[ii * 16 + jj] = + buf->y_buffer[(i + ii) * buf->y_stride + (j + jj)]; + av1_fwd_txfm2d_16x4(src16, coeff, 16, H_DCT, bd); + for (int k = 1; k < 16; ++k) { + const uint64_t this_energy = + ((int64_t)coeff[k] * coeff[k]) + + ((int64_t)coeff[k + 16] * coeff[k + 16]) + + ((int64_t)coeff[k + 32] * coeff[k + 32]) + + ((int64_t)coeff[k + 48] * coeff[k + 48]); + freq_energy[k] += ROUND_POWER_OF_TWO(this_energy, 2); + } + n++; + } + } + } + if (n) { + for (int k = 1; k < 16; ++k) energy[k] = (double)freq_energy[k] / n; + // Convert to cumulative energy + for (int k = 14; k > 0; --k) energy[k] += energy[k + 1]; + } else { + for (int k = 1; k < 16; ++k) energy[k] = 1e+20; + } +} + static void set_high_precision_mv(AV1_COMP *cpi, int allow_high_precision_mv, int cur_frame_force_integer_mv) { MACROBLOCK *const mb = &cpi->td.mb; @@ -270,6 +354,9 @@ static BLOCK_SIZE select_sb_size(const AV1_COMP *const cpi) { // When superres / resize is on, 'cm->width / height' can change between // calls, so we don't apply this heuristic there. Also, this heuristic gives // compression gain for speed >= 2 only. + // Things break if superblock size changes per-frame which is why this + // heuristic is set based on configured speed rather than actual + // speed-features (which may change per-frame in future) if (cpi->oxcf.superres_mode == SUPERRES_NONE && cpi->oxcf.resize_mode == RESIZE_NONE && cpi->oxcf.speed >= 2) { return (cm->width >= 480 && cm->height >= 360) ? BLOCK_128X128 @@ -287,64 +374,34 @@ static void setup_frame(AV1_COMP *cpi) { // other inter-frames the encoder currently uses only two contexts; // context 1 for ALTREF frames and context 0 for the others. - cm->primary_ref_frame = PRIMARY_REF_NONE; if (frame_is_intra_only(cm) || cm->error_resilient_mode || - cm->force_primary_ref_none) { + cpi->ext_use_primary_ref_none) { av1_setup_past_independence(cm); - for (int i = 0; i < REF_FRAMES; i++) { - cm->fb_of_context_type[i] = -1; - } - cm->fb_of_context_type[REGULAR_FRAME] = - cm->show_frame ? get_ref_frame_map_idx(cpi, GOLDEN_FRAME) - : get_ref_frame_map_idx(cpi, ALTREF_FRAME); - cm->frame_context_idx = REGULAR_FRAME; - } else { - const GF_GROUP *gf_group = &cpi->twopass.gf_group; - if (gf_group->update_type[gf_group->index] == INTNL_ARF_UPDATE) - cm->frame_context_idx = EXT_ARF_FRAME; - else if (cpi->refresh_alt_ref_frame) - cm->frame_context_idx = ARF_FRAME; - else if (cpi->rc.is_src_frame_alt_ref) - cm->frame_context_idx = OVERLAY_FRAME; - else if (cpi->refresh_golden_frame) - cm->frame_context_idx = GLD_FRAME; - else if (cpi->refresh_bwd_ref_frame) - cm->frame_context_idx = BRF_FRAME; - else - cm->frame_context_idx = REGULAR_FRAME; - int wanted_fb = cm->fb_of_context_type[cm->frame_context_idx]; - for (int ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ref_frame++) { - int fb = get_ref_frame_map_idx(cpi, ref_frame); - if (fb == wanted_fb) { - cm->primary_ref_frame = ref_frame - LAST_FRAME; - } - } } - if (cm->frame_type == KEY_FRAME && cm->show_frame) { + if (cm->current_frame.frame_type == KEY_FRAME && cm->show_frame) { cpi->refresh_golden_frame = 1; cpi->refresh_alt_ref_frame = 1; av1_zero(cpi->interp_filter_selected); set_sb_size(&cm->seq_params, select_sb_size(cpi)); - set_use_reference_buffer(cm, 0); } else if (frame_is_sframe(cm)) { cpi->refresh_golden_frame = 1; cpi->refresh_alt_ref_frame = 1; av1_zero(cpi->interp_filter_selected); set_sb_size(&cm->seq_params, select_sb_size(cpi)); } else { - if (cm->primary_ref_frame == PRIMARY_REF_NONE || - cm->frame_refs[cm->primary_ref_frame].idx < 0) { + const RefCntBuffer *const primary_ref_buf = get_primary_ref_frame_buf(cm); + if (primary_ref_buf == NULL) { av1_setup_past_independence(cm); cm->seg.update_map = 1; cm->seg.update_data = 1; } else { - *cm->fc = cm->frame_contexts[cm->frame_refs[cm->primary_ref_frame].idx]; + *cm->fc = primary_ref_buf->frame_context; } av1_zero(cpi->interp_filter_selected[0]); } - cm->prev_frame = get_prev_frame(cm); + cm->prev_frame = get_primary_ref_frame_buf(cm); cpi->vaq_refresh = 0; } @@ -438,6 +495,20 @@ static void alloc_context_buffers_ext(AV1_COMP *cpi) { aom_calloc(mi_size, sizeof(*cpi->mbmi_ext_base))); } +static void reset_film_grain_chroma_params(aom_film_grain_t *pars) { + pars->num_cr_points = 0; + pars->cr_mult = 0; + pars->cr_luma_mult = 0; + memset(pars->scaling_points_cr, 0, sizeof(pars->scaling_points_cr)); + memset(pars->ar_coeffs_cr, 0, sizeof(pars->ar_coeffs_cr)); + pars->num_cb_points = 0; + pars->cb_mult = 0; + pars->cb_luma_mult = 0; + pars->chroma_scaling_from_luma = 0; + memset(pars->scaling_points_cb, 0, sizeof(pars->scaling_points_cb)); + memset(pars->ar_coeffs_cb, 0, sizeof(pars->ar_coeffs_cb)); +} + static void update_film_grain_parameters(struct AV1_COMP *cpi, const AV1EncoderConfig *oxcf) { AV1_COMMON *const cm = &cpi->common; @@ -451,11 +522,12 @@ static void update_film_grain_parameters(struct AV1_COMP *cpi, if (oxcf->film_grain_test_vector) { cm->seq_params.film_grain_params_present = 1; - if (cm->frame_type == KEY_FRAME) { + if (cm->current_frame.frame_type == KEY_FRAME) { memcpy(&cm->film_grain_params, film_grain_test_vectors + oxcf->film_grain_test_vector - 1, sizeof(cm->film_grain_params)); - + if (oxcf->monochrome) + reset_film_grain_chroma_params(&cm->film_grain_params); cm->film_grain_params.bit_depth = cm->seq_params.bit_depth; if (cm->seq_params.color_range == AOM_CR_FULL_RANGE) { cm->film_grain_params.clip_to_restricted_range = 0; @@ -501,6 +573,9 @@ static void dealloc_compressor_data(AV1_COMP *cpi) { aom_free(cpi->td.mb.wsrc_buf); cpi->td.mb.wsrc_buf = NULL; + aom_free(cpi->td.mb.inter_modes_info); + cpi->td.mb.inter_modes_info = NULL; + for (int i = 0; i < 2; i++) for (int j = 0; j < 2; j++) { aom_free(cpi->td.mb.hash_value_buffer[i][j]); @@ -588,7 +663,7 @@ static void configure_static_seg_features(AV1_COMP *cpi) { int qi_delta; // Disable and clear down for KF - if (cm->frame_type == KEY_FRAME) { + if (cm->current_frame.frame_type == KEY_FRAME) { // Clear down the global segmentation map memset(cpi->segmentation_map, 0, cm->mi_rows * cm->mi_cols); seg->update_map = 0; @@ -716,7 +791,7 @@ static void configure_static_seg_features(AV1_COMP *cpi) { static void update_reference_segmentation_map(AV1_COMP *cpi) { AV1_COMMON *const cm = &cpi->common; MB_MODE_INFO **mi_4x4_ptr = cm->mi_grid_visible; - uint8_t *cache_ptr = cm->current_frame_seg_map; + uint8_t *cache_ptr = cm->cur_frame->seg_map; int row, col; for (row = 0; row < cm->mi_rows; row++) { @@ -735,10 +810,10 @@ static void alloc_raw_frame_buffers(AV1_COMP *cpi) { const AV1EncoderConfig *oxcf = &cpi->oxcf; if (!cpi->lookahead) - cpi->lookahead = - av1_lookahead_init(oxcf->width, oxcf->height, seq_params->subsampling_x, - seq_params->subsampling_y, - seq_params->use_highbitdepth, oxcf->lag_in_frames); + cpi->lookahead = av1_lookahead_init( + oxcf->width, oxcf->height, seq_params->subsampling_x, + seq_params->subsampling_y, seq_params->use_highbitdepth, + oxcf->lag_in_frames, oxcf->border_in_pixels); if (!cpi->lookahead) aom_internal_error(&cm->error, AOM_CODEC_MEM_ERROR, "Failed to allocate lag buffers"); @@ -747,7 +822,7 @@ static void alloc_raw_frame_buffers(AV1_COMP *cpi) { if (aom_realloc_frame_buffer( &cpi->alt_ref_buffer, oxcf->width, oxcf->height, seq_params->subsampling_x, seq_params->subsampling_y, - seq_params->use_highbitdepth, AOM_BORDER_IN_PIXELS, + seq_params->use_highbitdepth, oxcf->border_in_pixels, cm->byte_alignment, NULL, NULL, NULL)) aom_internal_error(&cm->error, AOM_CODEC_MEM_ERROR, "Failed to allocate altref buffer"); @@ -759,7 +834,7 @@ static void alloc_util_frame_buffers(AV1_COMP *cpi) { if (aom_realloc_frame_buffer( &cpi->last_frame_uf, cm->width, cm->height, seq_params->subsampling_x, seq_params->subsampling_y, seq_params->use_highbitdepth, - AOM_BORDER_IN_PIXELS, cm->byte_alignment, NULL, NULL, NULL)) + cpi->oxcf.border_in_pixels, cm->byte_alignment, NULL, NULL, NULL)) aom_internal_error(&cm->error, AOM_CODEC_MEM_ERROR, "Failed to allocate last frame buffer"); @@ -767,21 +842,21 @@ static void alloc_util_frame_buffers(AV1_COMP *cpi) { &cpi->trial_frame_rst, cm->superres_upscaled_width, cm->superres_upscaled_height, seq_params->subsampling_x, seq_params->subsampling_y, seq_params->use_highbitdepth, - AOM_BORDER_IN_PIXELS, cm->byte_alignment, NULL, NULL, NULL)) + AOM_RESTORATION_FRAME_BORDER, cm->byte_alignment, NULL, NULL, NULL)) aom_internal_error(&cm->error, AOM_CODEC_MEM_ERROR, "Failed to allocate trial restored frame buffer"); if (aom_realloc_frame_buffer( &cpi->scaled_source, cm->width, cm->height, seq_params->subsampling_x, seq_params->subsampling_y, seq_params->use_highbitdepth, - AOM_BORDER_IN_PIXELS, cm->byte_alignment, NULL, NULL, NULL)) + cpi->oxcf.border_in_pixels, cm->byte_alignment, NULL, NULL, NULL)) aom_internal_error(&cm->error, AOM_CODEC_MEM_ERROR, "Failed to allocate scaled source buffer"); if (aom_realloc_frame_buffer( &cpi->scaled_last_source, cm->width, cm->height, seq_params->subsampling_x, seq_params->subsampling_y, - seq_params->use_highbitdepth, AOM_BORDER_IN_PIXELS, + seq_params->use_highbitdepth, cpi->oxcf.border_in_pixels, cm->byte_alignment, NULL, NULL, NULL)) aom_internal_error(&cm->error, AOM_CODEC_MEM_ERROR, "Failed to allocate scaled last source buffer"); @@ -885,10 +960,9 @@ static void update_frame_size(AV1_COMP *cpi) { static void init_buffer_indices(AV1_COMP *cpi) { int fb_idx; for (fb_idx = 0; fb_idx < REF_FRAMES; ++fb_idx) - cpi->ref_fb_idx[fb_idx] = fb_idx; + cpi->common.remapped_ref_idx[fb_idx] = fb_idx; cpi->rate_index = 0; cpi->rate_size = 0; - cpi->cur_poc = -1; } static INLINE int does_level_match(int width, int height, double fps, @@ -999,30 +1073,50 @@ static void init_seq_coding_tools(SequenceHeader *seq, AV1_COMMON *cm, seq->reduced_still_picture_hdr &= !oxcf->full_still_picture_hdr; seq->force_screen_content_tools = 2; seq->force_integer_mv = 2; - seq->enable_order_hint = oxcf->enable_order_hint; - seq->frame_id_numbers_present_flag = oxcf->large_scale_tile; + seq->order_hint_info.enable_order_hint = oxcf->enable_order_hint; + seq->frame_id_numbers_present_flag = + !(seq->still_picture && seq->reduced_still_picture_hdr) && + !oxcf->large_scale_tile && oxcf->error_resilient_mode; if (seq->still_picture && seq->reduced_still_picture_hdr) { - seq->enable_order_hint = 0; - seq->frame_id_numbers_present_flag = 0; + seq->order_hint_info.enable_order_hint = 0; seq->force_screen_content_tools = 2; seq->force_integer_mv = 2; } - seq->order_hint_bits_minus_1 = - seq->enable_order_hint ? DEFAULT_EXPLICIT_ORDER_HINT_BITS - 1 : -1; + seq->order_hint_info.order_hint_bits_minus_1 = + seq->order_hint_info.enable_order_hint + ? DEFAULT_EXPLICIT_ORDER_HINT_BITS - 1 + : -1; + + seq->max_frame_width = + oxcf->forced_max_frame_width ? oxcf->forced_max_frame_width : oxcf->width; + seq->max_frame_height = oxcf->forced_max_frame_height + ? oxcf->forced_max_frame_height + : oxcf->height; + seq->num_bits_width = + (seq->max_frame_width > 1) ? get_msb(seq->max_frame_width - 1) + 1 : 1; + seq->num_bits_height = + (seq->max_frame_height > 1) ? get_msb(seq->max_frame_height - 1) + 1 : 1; + assert(seq->num_bits_width <= 16); + assert(seq->num_bits_height <= 16); + + seq->frame_id_length = FRAME_ID_LENGTH; + seq->delta_frame_id_length = DELTA_FRAME_ID_LENGTH; seq->enable_dual_filter = oxcf->enable_dual_filter; - seq->enable_jnt_comp = oxcf->enable_jnt_comp; - seq->enable_jnt_comp &= seq->enable_order_hint; - seq->enable_ref_frame_mvs = oxcf->enable_ref_frame_mvs; - seq->enable_ref_frame_mvs &= seq->enable_order_hint; + seq->order_hint_info.enable_dist_wtd_comp = oxcf->enable_dist_wtd_comp; + seq->order_hint_info.enable_dist_wtd_comp &= + seq->order_hint_info.enable_order_hint; + seq->order_hint_info.enable_ref_frame_mvs = oxcf->enable_ref_frame_mvs; + seq->order_hint_info.enable_ref_frame_mvs &= + seq->order_hint_info.enable_order_hint; seq->enable_superres = oxcf->enable_superres; seq->enable_cdef = oxcf->enable_cdef; seq->enable_restoration = oxcf->enable_restoration; seq->enable_warped_motion = oxcf->enable_warped_motion; - seq->enable_interintra_compound = 1; - seq->enable_masked_compound = 1; - seq->enable_intra_edge_filter = 1; - seq->enable_filter_intra = 1; + seq->enable_interintra_compound = oxcf->enable_interintra_comp; + seq->enable_masked_compound = oxcf->enable_masked_comp; + seq->enable_intra_edge_filter = oxcf->enable_intra_edge_filter; + seq->enable_filter_intra = oxcf->enable_filter_intra; set_bitstream_level_tier(seq, cm, oxcf); @@ -1219,14 +1313,14 @@ static void set_rc_buffer_sizes(RATE_CONTROL *rc, static unsigned int fnname##_bits8( \ const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, \ int ref_stride, const uint8_t *second_pred, \ - const JNT_COMP_PARAMS *jcp_param) { \ + const DIST_WTD_COMP_PARAMS *jcp_param) { \ return fnname(src_ptr, source_stride, ref_ptr, ref_stride, second_pred, \ jcp_param); \ } \ static unsigned int fnname##_bits10( \ const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, \ int ref_stride, const uint8_t *second_pred, \ - const JNT_COMP_PARAMS *jcp_param) { \ + const DIST_WTD_COMP_PARAMS *jcp_param) { \ return fnname(src_ptr, source_stride, ref_ptr, ref_stride, second_pred, \ jcp_param) >> \ 2; \ @@ -1234,7 +1328,7 @@ static void set_rc_buffer_sizes(RATE_CONTROL *rc, static unsigned int fnname##_bits12( \ const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, \ int ref_stride, const uint8_t *second_pred, \ - const JNT_COMP_PARAMS *jcp_param) { \ + const DIST_WTD_COMP_PARAMS *jcp_param) { \ return fnname(src_ptr, source_stride, ref_ptr, ref_stride, second_pred, \ jcp_param) >> \ 4; \ @@ -1308,28 +1402,28 @@ MAKE_BFP_SAD_WRAPPER(aom_highbd_sad64x16) MAKE_BFP_SADAVG_WRAPPER(aom_highbd_sad64x16_avg) MAKE_BFP_SAD4D_WRAPPER(aom_highbd_sad64x16x4d) -MAKE_BFP_JSADAVG_WRAPPER(aom_highbd_jnt_sad128x128_avg) -MAKE_BFP_JSADAVG_WRAPPER(aom_highbd_jnt_sad128x64_avg) -MAKE_BFP_JSADAVG_WRAPPER(aom_highbd_jnt_sad64x128_avg) -MAKE_BFP_JSADAVG_WRAPPER(aom_highbd_jnt_sad32x16_avg) -MAKE_BFP_JSADAVG_WRAPPER(aom_highbd_jnt_sad16x32_avg) -MAKE_BFP_JSADAVG_WRAPPER(aom_highbd_jnt_sad64x32_avg) -MAKE_BFP_JSADAVG_WRAPPER(aom_highbd_jnt_sad32x64_avg) -MAKE_BFP_JSADAVG_WRAPPER(aom_highbd_jnt_sad32x32_avg) -MAKE_BFP_JSADAVG_WRAPPER(aom_highbd_jnt_sad64x64_avg) -MAKE_BFP_JSADAVG_WRAPPER(aom_highbd_jnt_sad16x16_avg) -MAKE_BFP_JSADAVG_WRAPPER(aom_highbd_jnt_sad16x8_avg) -MAKE_BFP_JSADAVG_WRAPPER(aom_highbd_jnt_sad8x16_avg) -MAKE_BFP_JSADAVG_WRAPPER(aom_highbd_jnt_sad8x8_avg) -MAKE_BFP_JSADAVG_WRAPPER(aom_highbd_jnt_sad8x4_avg) -MAKE_BFP_JSADAVG_WRAPPER(aom_highbd_jnt_sad4x8_avg) -MAKE_BFP_JSADAVG_WRAPPER(aom_highbd_jnt_sad4x4_avg) -MAKE_BFP_JSADAVG_WRAPPER(aom_highbd_jnt_sad4x16_avg) -MAKE_BFP_JSADAVG_WRAPPER(aom_highbd_jnt_sad16x4_avg) -MAKE_BFP_JSADAVG_WRAPPER(aom_highbd_jnt_sad8x32_avg) -MAKE_BFP_JSADAVG_WRAPPER(aom_highbd_jnt_sad32x8_avg) -MAKE_BFP_JSADAVG_WRAPPER(aom_highbd_jnt_sad16x64_avg) -MAKE_BFP_JSADAVG_WRAPPER(aom_highbd_jnt_sad64x16_avg) +MAKE_BFP_JSADAVG_WRAPPER(aom_highbd_dist_wtd_sad128x128_avg) +MAKE_BFP_JSADAVG_WRAPPER(aom_highbd_dist_wtd_sad128x64_avg) +MAKE_BFP_JSADAVG_WRAPPER(aom_highbd_dist_wtd_sad64x128_avg) +MAKE_BFP_JSADAVG_WRAPPER(aom_highbd_dist_wtd_sad32x16_avg) +MAKE_BFP_JSADAVG_WRAPPER(aom_highbd_dist_wtd_sad16x32_avg) +MAKE_BFP_JSADAVG_WRAPPER(aom_highbd_dist_wtd_sad64x32_avg) +MAKE_BFP_JSADAVG_WRAPPER(aom_highbd_dist_wtd_sad32x64_avg) +MAKE_BFP_JSADAVG_WRAPPER(aom_highbd_dist_wtd_sad32x32_avg) +MAKE_BFP_JSADAVG_WRAPPER(aom_highbd_dist_wtd_sad64x64_avg) +MAKE_BFP_JSADAVG_WRAPPER(aom_highbd_dist_wtd_sad16x16_avg) +MAKE_BFP_JSADAVG_WRAPPER(aom_highbd_dist_wtd_sad16x8_avg) +MAKE_BFP_JSADAVG_WRAPPER(aom_highbd_dist_wtd_sad8x16_avg) +MAKE_BFP_JSADAVG_WRAPPER(aom_highbd_dist_wtd_sad8x8_avg) +MAKE_BFP_JSADAVG_WRAPPER(aom_highbd_dist_wtd_sad8x4_avg) +MAKE_BFP_JSADAVG_WRAPPER(aom_highbd_dist_wtd_sad4x8_avg) +MAKE_BFP_JSADAVG_WRAPPER(aom_highbd_dist_wtd_sad4x4_avg) +MAKE_BFP_JSADAVG_WRAPPER(aom_highbd_dist_wtd_sad4x16_avg) +MAKE_BFP_JSADAVG_WRAPPER(aom_highbd_dist_wtd_sad16x4_avg) +MAKE_BFP_JSADAVG_WRAPPER(aom_highbd_dist_wtd_sad8x32_avg) +MAKE_BFP_JSADAVG_WRAPPER(aom_highbd_dist_wtd_sad32x8_avg) +MAKE_BFP_JSADAVG_WRAPPER(aom_highbd_dist_wtd_sad16x64_avg) +MAKE_BFP_JSADAVG_WRAPPER(aom_highbd_dist_wtd_sad64x16_avg) #define HIGHBD_MBFP(BT, MCSDF, MCSVF) \ cpi->fn_ptr[BT].msdf = MCSDF; \ @@ -1438,166 +1532,167 @@ static void highbd_set_var_fns(AV1_COMP *const cpi) { aom_highbd_8_sub_pixel_variance64x16, aom_highbd_8_sub_pixel_avg_variance64x16, aom_highbd_sad64x16x4d_bits8, - aom_highbd_jnt_sad64x16_avg_bits8, - aom_highbd_8_jnt_sub_pixel_avg_variance64x16) + aom_highbd_dist_wtd_sad64x16_avg_bits8, + aom_highbd_8_dist_wtd_sub_pixel_avg_variance64x16) HIGHBD_BFP(BLOCK_16X64, aom_highbd_sad16x64_bits8, aom_highbd_sad16x64_avg_bits8, aom_highbd_8_variance16x64, aom_highbd_8_sub_pixel_variance16x64, aom_highbd_8_sub_pixel_avg_variance16x64, aom_highbd_sad16x64x4d_bits8, - aom_highbd_jnt_sad16x64_avg_bits8, - aom_highbd_8_jnt_sub_pixel_avg_variance16x64) + aom_highbd_dist_wtd_sad16x64_avg_bits8, + aom_highbd_8_dist_wtd_sub_pixel_avg_variance16x64) HIGHBD_BFP( BLOCK_32X8, aom_highbd_sad32x8_bits8, aom_highbd_sad32x8_avg_bits8, aom_highbd_8_variance32x8, aom_highbd_8_sub_pixel_variance32x8, aom_highbd_8_sub_pixel_avg_variance32x8, - aom_highbd_sad32x8x4d_bits8, aom_highbd_jnt_sad32x8_avg_bits8, - aom_highbd_8_jnt_sub_pixel_avg_variance32x8) + aom_highbd_sad32x8x4d_bits8, aom_highbd_dist_wtd_sad32x8_avg_bits8, + aom_highbd_8_dist_wtd_sub_pixel_avg_variance32x8) HIGHBD_BFP( BLOCK_8X32, aom_highbd_sad8x32_bits8, aom_highbd_sad8x32_avg_bits8, aom_highbd_8_variance8x32, aom_highbd_8_sub_pixel_variance8x32, aom_highbd_8_sub_pixel_avg_variance8x32, - aom_highbd_sad8x32x4d_bits8, aom_highbd_jnt_sad8x32_avg_bits8, - aom_highbd_8_jnt_sub_pixel_avg_variance8x32) + aom_highbd_sad8x32x4d_bits8, aom_highbd_dist_wtd_sad8x32_avg_bits8, + aom_highbd_8_dist_wtd_sub_pixel_avg_variance8x32) HIGHBD_BFP( BLOCK_16X4, aom_highbd_sad16x4_bits8, aom_highbd_sad16x4_avg_bits8, aom_highbd_8_variance16x4, aom_highbd_8_sub_pixel_variance16x4, aom_highbd_8_sub_pixel_avg_variance16x4, - aom_highbd_sad16x4x4d_bits8, aom_highbd_jnt_sad16x4_avg_bits8, - aom_highbd_8_jnt_sub_pixel_avg_variance16x4) + aom_highbd_sad16x4x4d_bits8, aom_highbd_dist_wtd_sad16x4_avg_bits8, + aom_highbd_8_dist_wtd_sub_pixel_avg_variance16x4) HIGHBD_BFP( BLOCK_4X16, aom_highbd_sad4x16_bits8, aom_highbd_sad4x16_avg_bits8, aom_highbd_8_variance4x16, aom_highbd_8_sub_pixel_variance4x16, aom_highbd_8_sub_pixel_avg_variance4x16, - aom_highbd_sad4x16x4d_bits8, aom_highbd_jnt_sad4x16_avg_bits8, - aom_highbd_8_jnt_sub_pixel_avg_variance4x16) + aom_highbd_sad4x16x4d_bits8, aom_highbd_dist_wtd_sad4x16_avg_bits8, + aom_highbd_8_dist_wtd_sub_pixel_avg_variance4x16) HIGHBD_BFP(BLOCK_32X16, aom_highbd_sad32x16_bits8, aom_highbd_sad32x16_avg_bits8, aom_highbd_8_variance32x16, aom_highbd_8_sub_pixel_variance32x16, aom_highbd_8_sub_pixel_avg_variance32x16, aom_highbd_sad32x16x4d_bits8, - aom_highbd_jnt_sad32x16_avg_bits8, - aom_highbd_8_jnt_sub_pixel_avg_variance32x16) + aom_highbd_dist_wtd_sad32x16_avg_bits8, + aom_highbd_8_dist_wtd_sub_pixel_avg_variance32x16) HIGHBD_BFP(BLOCK_16X32, aom_highbd_sad16x32_bits8, aom_highbd_sad16x32_avg_bits8, aom_highbd_8_variance16x32, aom_highbd_8_sub_pixel_variance16x32, aom_highbd_8_sub_pixel_avg_variance16x32, aom_highbd_sad16x32x4d_bits8, - aom_highbd_jnt_sad16x32_avg_bits8, - aom_highbd_8_jnt_sub_pixel_avg_variance16x32) + aom_highbd_dist_wtd_sad16x32_avg_bits8, + aom_highbd_8_dist_wtd_sub_pixel_avg_variance16x32) HIGHBD_BFP(BLOCK_64X32, aom_highbd_sad64x32_bits8, aom_highbd_sad64x32_avg_bits8, aom_highbd_8_variance64x32, aom_highbd_8_sub_pixel_variance64x32, aom_highbd_8_sub_pixel_avg_variance64x32, aom_highbd_sad64x32x4d_bits8, - aom_highbd_jnt_sad64x32_avg_bits8, - aom_highbd_8_jnt_sub_pixel_avg_variance64x32) + aom_highbd_dist_wtd_sad64x32_avg_bits8, + aom_highbd_8_dist_wtd_sub_pixel_avg_variance64x32) HIGHBD_BFP(BLOCK_32X64, aom_highbd_sad32x64_bits8, aom_highbd_sad32x64_avg_bits8, aom_highbd_8_variance32x64, aom_highbd_8_sub_pixel_variance32x64, aom_highbd_8_sub_pixel_avg_variance32x64, aom_highbd_sad32x64x4d_bits8, - aom_highbd_jnt_sad32x64_avg_bits8, - aom_highbd_8_jnt_sub_pixel_avg_variance32x64) + aom_highbd_dist_wtd_sad32x64_avg_bits8, + aom_highbd_8_dist_wtd_sub_pixel_avg_variance32x64) HIGHBD_BFP(BLOCK_32X32, aom_highbd_sad32x32_bits8, aom_highbd_sad32x32_avg_bits8, aom_highbd_8_variance32x32, aom_highbd_8_sub_pixel_variance32x32, aom_highbd_8_sub_pixel_avg_variance32x32, aom_highbd_sad32x32x4d_bits8, - aom_highbd_jnt_sad32x32_avg_bits8, - aom_highbd_8_jnt_sub_pixel_avg_variance32x32) + aom_highbd_dist_wtd_sad32x32_avg_bits8, + aom_highbd_8_dist_wtd_sub_pixel_avg_variance32x32) HIGHBD_BFP(BLOCK_64X64, aom_highbd_sad64x64_bits8, aom_highbd_sad64x64_avg_bits8, aom_highbd_8_variance64x64, aom_highbd_8_sub_pixel_variance64x64, aom_highbd_8_sub_pixel_avg_variance64x64, aom_highbd_sad64x64x4d_bits8, - aom_highbd_jnt_sad64x64_avg_bits8, - aom_highbd_8_jnt_sub_pixel_avg_variance64x64) + aom_highbd_dist_wtd_sad64x64_avg_bits8, + aom_highbd_8_dist_wtd_sub_pixel_avg_variance64x64) HIGHBD_BFP(BLOCK_16X16, aom_highbd_sad16x16_bits8, aom_highbd_sad16x16_avg_bits8, aom_highbd_8_variance16x16, aom_highbd_8_sub_pixel_variance16x16, aom_highbd_8_sub_pixel_avg_variance16x16, aom_highbd_sad16x16x4d_bits8, - aom_highbd_jnt_sad16x16_avg_bits8, - aom_highbd_8_jnt_sub_pixel_avg_variance16x16) + aom_highbd_dist_wtd_sad16x16_avg_bits8, + aom_highbd_8_dist_wtd_sub_pixel_avg_variance16x16) HIGHBD_BFP( BLOCK_16X8, aom_highbd_sad16x8_bits8, aom_highbd_sad16x8_avg_bits8, aom_highbd_8_variance16x8, aom_highbd_8_sub_pixel_variance16x8, aom_highbd_8_sub_pixel_avg_variance16x8, - aom_highbd_sad16x8x4d_bits8, aom_highbd_jnt_sad16x8_avg_bits8, - aom_highbd_8_jnt_sub_pixel_avg_variance16x8) + aom_highbd_sad16x8x4d_bits8, aom_highbd_dist_wtd_sad16x8_avg_bits8, + aom_highbd_8_dist_wtd_sub_pixel_avg_variance16x8) HIGHBD_BFP( BLOCK_8X16, aom_highbd_sad8x16_bits8, aom_highbd_sad8x16_avg_bits8, aom_highbd_8_variance8x16, aom_highbd_8_sub_pixel_variance8x16, aom_highbd_8_sub_pixel_avg_variance8x16, - aom_highbd_sad8x16x4d_bits8, aom_highbd_jnt_sad8x16_avg_bits8, - aom_highbd_8_jnt_sub_pixel_avg_variance8x16) - - HIGHBD_BFP(BLOCK_8X8, aom_highbd_sad8x8_bits8, - aom_highbd_sad8x8_avg_bits8, aom_highbd_8_variance8x8, - aom_highbd_8_sub_pixel_variance8x8, - aom_highbd_8_sub_pixel_avg_variance8x8, - aom_highbd_sad8x8x4d_bits8, aom_highbd_jnt_sad8x8_avg_bits8, - aom_highbd_8_jnt_sub_pixel_avg_variance8x8) - - HIGHBD_BFP(BLOCK_8X4, aom_highbd_sad8x4_bits8, - aom_highbd_sad8x4_avg_bits8, aom_highbd_8_variance8x4, - aom_highbd_8_sub_pixel_variance8x4, - aom_highbd_8_sub_pixel_avg_variance8x4, - aom_highbd_sad8x4x4d_bits8, aom_highbd_jnt_sad8x4_avg_bits8, - aom_highbd_8_jnt_sub_pixel_avg_variance8x4) - - HIGHBD_BFP(BLOCK_4X8, aom_highbd_sad4x8_bits8, - aom_highbd_sad4x8_avg_bits8, aom_highbd_8_variance4x8, - aom_highbd_8_sub_pixel_variance4x8, - aom_highbd_8_sub_pixel_avg_variance4x8, - aom_highbd_sad4x8x4d_bits8, aom_highbd_jnt_sad4x8_avg_bits8, - aom_highbd_8_jnt_sub_pixel_avg_variance4x8) - - HIGHBD_BFP(BLOCK_4X4, aom_highbd_sad4x4_bits8, - aom_highbd_sad4x4_avg_bits8, aom_highbd_8_variance4x4, - aom_highbd_8_sub_pixel_variance4x4, - aom_highbd_8_sub_pixel_avg_variance4x4, - aom_highbd_sad4x4x4d_bits8, aom_highbd_jnt_sad4x4_avg_bits8, - aom_highbd_8_jnt_sub_pixel_avg_variance4x4) + aom_highbd_sad8x16x4d_bits8, aom_highbd_dist_wtd_sad8x16_avg_bits8, + aom_highbd_8_dist_wtd_sub_pixel_avg_variance8x16) HIGHBD_BFP( - BLOCK_128X128, aom_highbd_sad128x128_bits8, - aom_highbd_sad128x128_avg_bits8, aom_highbd_8_variance128x128, - aom_highbd_8_sub_pixel_variance128x128, - aom_highbd_8_sub_pixel_avg_variance128x128, - aom_highbd_sad128x128x4d_bits8, aom_highbd_jnt_sad128x128_avg_bits8, - aom_highbd_8_jnt_sub_pixel_avg_variance128x128) + BLOCK_8X8, aom_highbd_sad8x8_bits8, aom_highbd_sad8x8_avg_bits8, + aom_highbd_8_variance8x8, aom_highbd_8_sub_pixel_variance8x8, + aom_highbd_8_sub_pixel_avg_variance8x8, aom_highbd_sad8x8x4d_bits8, + aom_highbd_dist_wtd_sad8x8_avg_bits8, + aom_highbd_8_dist_wtd_sub_pixel_avg_variance8x8) + + HIGHBD_BFP( + BLOCK_8X4, aom_highbd_sad8x4_bits8, aom_highbd_sad8x4_avg_bits8, + aom_highbd_8_variance8x4, aom_highbd_8_sub_pixel_variance8x4, + aom_highbd_8_sub_pixel_avg_variance8x4, aom_highbd_sad8x4x4d_bits8, + aom_highbd_dist_wtd_sad8x4_avg_bits8, + aom_highbd_8_dist_wtd_sub_pixel_avg_variance8x4) + + HIGHBD_BFP( + BLOCK_4X8, aom_highbd_sad4x8_bits8, aom_highbd_sad4x8_avg_bits8, + aom_highbd_8_variance4x8, aom_highbd_8_sub_pixel_variance4x8, + aom_highbd_8_sub_pixel_avg_variance4x8, aom_highbd_sad4x8x4d_bits8, + aom_highbd_dist_wtd_sad4x8_avg_bits8, + aom_highbd_8_dist_wtd_sub_pixel_avg_variance4x8) + + HIGHBD_BFP( + BLOCK_4X4, aom_highbd_sad4x4_bits8, aom_highbd_sad4x4_avg_bits8, + aom_highbd_8_variance4x4, aom_highbd_8_sub_pixel_variance4x4, + aom_highbd_8_sub_pixel_avg_variance4x4, aom_highbd_sad4x4x4d_bits8, + aom_highbd_dist_wtd_sad4x4_avg_bits8, + aom_highbd_8_dist_wtd_sub_pixel_avg_variance4x4) + + HIGHBD_BFP(BLOCK_128X128, aom_highbd_sad128x128_bits8, + aom_highbd_sad128x128_avg_bits8, + aom_highbd_8_variance128x128, + aom_highbd_8_sub_pixel_variance128x128, + aom_highbd_8_sub_pixel_avg_variance128x128, + aom_highbd_sad128x128x4d_bits8, + aom_highbd_dist_wtd_sad128x128_avg_bits8, + aom_highbd_8_dist_wtd_sub_pixel_avg_variance128x128) HIGHBD_BFP(BLOCK_128X64, aom_highbd_sad128x64_bits8, aom_highbd_sad128x64_avg_bits8, aom_highbd_8_variance128x64, aom_highbd_8_sub_pixel_variance128x64, aom_highbd_8_sub_pixel_avg_variance128x64, aom_highbd_sad128x64x4d_bits8, - aom_highbd_jnt_sad128x64_avg_bits8, - aom_highbd_8_jnt_sub_pixel_avg_variance128x64) + aom_highbd_dist_wtd_sad128x64_avg_bits8, + aom_highbd_8_dist_wtd_sub_pixel_avg_variance128x64) HIGHBD_BFP(BLOCK_64X128, aom_highbd_sad64x128_bits8, aom_highbd_sad64x128_avg_bits8, aom_highbd_8_variance64x128, aom_highbd_8_sub_pixel_variance64x128, aom_highbd_8_sub_pixel_avg_variance64x128, aom_highbd_sad64x128x4d_bits8, - aom_highbd_jnt_sad64x128_avg_bits8, - aom_highbd_8_jnt_sub_pixel_avg_variance64x128) + aom_highbd_dist_wtd_sad64x128_avg_bits8, + aom_highbd_8_dist_wtd_sub_pixel_avg_variance64x128) HIGHBD_MBFP(BLOCK_128X128, aom_highbd_masked_sad128x128_bits8, aom_highbd_8_masked_sub_pixel_variance128x128) @@ -1717,148 +1812,148 @@ static void highbd_set_var_fns(AV1_COMP *const cpi) { aom_highbd_10_sub_pixel_variance64x16, aom_highbd_10_sub_pixel_avg_variance64x16, aom_highbd_sad64x16x4d_bits10, - aom_highbd_jnt_sad64x16_avg_bits10, - aom_highbd_10_jnt_sub_pixel_avg_variance64x16); + aom_highbd_dist_wtd_sad64x16_avg_bits10, + aom_highbd_10_dist_wtd_sub_pixel_avg_variance64x16); HIGHBD_BFP(BLOCK_16X64, aom_highbd_sad16x64_bits10, aom_highbd_sad16x64_avg_bits10, aom_highbd_10_variance16x64, aom_highbd_10_sub_pixel_variance16x64, aom_highbd_10_sub_pixel_avg_variance16x64, aom_highbd_sad16x64x4d_bits10, - aom_highbd_jnt_sad16x64_avg_bits10, - aom_highbd_10_jnt_sub_pixel_avg_variance16x64); + aom_highbd_dist_wtd_sad16x64_avg_bits10, + aom_highbd_10_dist_wtd_sub_pixel_avg_variance16x64); HIGHBD_BFP(BLOCK_32X8, aom_highbd_sad32x8_bits10, aom_highbd_sad32x8_avg_bits10, aom_highbd_10_variance32x8, aom_highbd_10_sub_pixel_variance32x8, aom_highbd_10_sub_pixel_avg_variance32x8, aom_highbd_sad32x8x4d_bits10, - aom_highbd_jnt_sad32x8_avg_bits10, - aom_highbd_10_jnt_sub_pixel_avg_variance32x8); + aom_highbd_dist_wtd_sad32x8_avg_bits10, + aom_highbd_10_dist_wtd_sub_pixel_avg_variance32x8); HIGHBD_BFP(BLOCK_8X32, aom_highbd_sad8x32_bits10, aom_highbd_sad8x32_avg_bits10, aom_highbd_10_variance8x32, aom_highbd_10_sub_pixel_variance8x32, aom_highbd_10_sub_pixel_avg_variance8x32, aom_highbd_sad8x32x4d_bits10, - aom_highbd_jnt_sad8x32_avg_bits10, - aom_highbd_10_jnt_sub_pixel_avg_variance8x32); + aom_highbd_dist_wtd_sad8x32_avg_bits10, + aom_highbd_10_dist_wtd_sub_pixel_avg_variance8x32); HIGHBD_BFP(BLOCK_16X4, aom_highbd_sad16x4_bits10, aom_highbd_sad16x4_avg_bits10, aom_highbd_10_variance16x4, aom_highbd_10_sub_pixel_variance16x4, aom_highbd_10_sub_pixel_avg_variance16x4, aom_highbd_sad16x4x4d_bits10, - aom_highbd_jnt_sad16x4_avg_bits10, - aom_highbd_10_jnt_sub_pixel_avg_variance16x4); + aom_highbd_dist_wtd_sad16x4_avg_bits10, + aom_highbd_10_dist_wtd_sub_pixel_avg_variance16x4); HIGHBD_BFP(BLOCK_4X16, aom_highbd_sad4x16_bits10, aom_highbd_sad4x16_avg_bits10, aom_highbd_10_variance4x16, aom_highbd_10_sub_pixel_variance4x16, aom_highbd_10_sub_pixel_avg_variance4x16, aom_highbd_sad4x16x4d_bits10, - aom_highbd_jnt_sad4x16_avg_bits10, - aom_highbd_10_jnt_sub_pixel_avg_variance4x16); + aom_highbd_dist_wtd_sad4x16_avg_bits10, + aom_highbd_10_dist_wtd_sub_pixel_avg_variance4x16); HIGHBD_BFP(BLOCK_32X16, aom_highbd_sad32x16_bits10, aom_highbd_sad32x16_avg_bits10, aom_highbd_10_variance32x16, aom_highbd_10_sub_pixel_variance32x16, aom_highbd_10_sub_pixel_avg_variance32x16, aom_highbd_sad32x16x4d_bits10, - aom_highbd_jnt_sad32x16_avg_bits10, - aom_highbd_10_jnt_sub_pixel_avg_variance32x16); + aom_highbd_dist_wtd_sad32x16_avg_bits10, + aom_highbd_10_dist_wtd_sub_pixel_avg_variance32x16); HIGHBD_BFP(BLOCK_16X32, aom_highbd_sad16x32_bits10, aom_highbd_sad16x32_avg_bits10, aom_highbd_10_variance16x32, aom_highbd_10_sub_pixel_variance16x32, aom_highbd_10_sub_pixel_avg_variance16x32, aom_highbd_sad16x32x4d_bits10, - aom_highbd_jnt_sad16x32_avg_bits10, - aom_highbd_10_jnt_sub_pixel_avg_variance16x32); + aom_highbd_dist_wtd_sad16x32_avg_bits10, + aom_highbd_10_dist_wtd_sub_pixel_avg_variance16x32); HIGHBD_BFP(BLOCK_64X32, aom_highbd_sad64x32_bits10, aom_highbd_sad64x32_avg_bits10, aom_highbd_10_variance64x32, aom_highbd_10_sub_pixel_variance64x32, aom_highbd_10_sub_pixel_avg_variance64x32, aom_highbd_sad64x32x4d_bits10, - aom_highbd_jnt_sad64x32_avg_bits10, - aom_highbd_10_jnt_sub_pixel_avg_variance64x32); + aom_highbd_dist_wtd_sad64x32_avg_bits10, + aom_highbd_10_dist_wtd_sub_pixel_avg_variance64x32); HIGHBD_BFP(BLOCK_32X64, aom_highbd_sad32x64_bits10, aom_highbd_sad32x64_avg_bits10, aom_highbd_10_variance32x64, aom_highbd_10_sub_pixel_variance32x64, aom_highbd_10_sub_pixel_avg_variance32x64, aom_highbd_sad32x64x4d_bits10, - aom_highbd_jnt_sad32x64_avg_bits10, - aom_highbd_10_jnt_sub_pixel_avg_variance32x64); + aom_highbd_dist_wtd_sad32x64_avg_bits10, + aom_highbd_10_dist_wtd_sub_pixel_avg_variance32x64); HIGHBD_BFP(BLOCK_32X32, aom_highbd_sad32x32_bits10, aom_highbd_sad32x32_avg_bits10, aom_highbd_10_variance32x32, aom_highbd_10_sub_pixel_variance32x32, aom_highbd_10_sub_pixel_avg_variance32x32, aom_highbd_sad32x32x4d_bits10, - aom_highbd_jnt_sad32x32_avg_bits10, - aom_highbd_10_jnt_sub_pixel_avg_variance32x32); + aom_highbd_dist_wtd_sad32x32_avg_bits10, + aom_highbd_10_dist_wtd_sub_pixel_avg_variance32x32); HIGHBD_BFP(BLOCK_64X64, aom_highbd_sad64x64_bits10, aom_highbd_sad64x64_avg_bits10, aom_highbd_10_variance64x64, aom_highbd_10_sub_pixel_variance64x64, aom_highbd_10_sub_pixel_avg_variance64x64, aom_highbd_sad64x64x4d_bits10, - aom_highbd_jnt_sad64x64_avg_bits10, - aom_highbd_10_jnt_sub_pixel_avg_variance64x64); + aom_highbd_dist_wtd_sad64x64_avg_bits10, + aom_highbd_10_dist_wtd_sub_pixel_avg_variance64x64); HIGHBD_BFP(BLOCK_16X16, aom_highbd_sad16x16_bits10, aom_highbd_sad16x16_avg_bits10, aom_highbd_10_variance16x16, aom_highbd_10_sub_pixel_variance16x16, aom_highbd_10_sub_pixel_avg_variance16x16, aom_highbd_sad16x16x4d_bits10, - aom_highbd_jnt_sad16x16_avg_bits10, - aom_highbd_10_jnt_sub_pixel_avg_variance16x16); + aom_highbd_dist_wtd_sad16x16_avg_bits10, + aom_highbd_10_dist_wtd_sub_pixel_avg_variance16x16); HIGHBD_BFP(BLOCK_16X8, aom_highbd_sad16x8_bits10, aom_highbd_sad16x8_avg_bits10, aom_highbd_10_variance16x8, aom_highbd_10_sub_pixel_variance16x8, aom_highbd_10_sub_pixel_avg_variance16x8, aom_highbd_sad16x8x4d_bits10, - aom_highbd_jnt_sad16x8_avg_bits10, - aom_highbd_10_jnt_sub_pixel_avg_variance16x8); + aom_highbd_dist_wtd_sad16x8_avg_bits10, + aom_highbd_10_dist_wtd_sub_pixel_avg_variance16x8); HIGHBD_BFP(BLOCK_8X16, aom_highbd_sad8x16_bits10, aom_highbd_sad8x16_avg_bits10, aom_highbd_10_variance8x16, aom_highbd_10_sub_pixel_variance8x16, aom_highbd_10_sub_pixel_avg_variance8x16, aom_highbd_sad8x16x4d_bits10, - aom_highbd_jnt_sad8x16_avg_bits10, - aom_highbd_10_jnt_sub_pixel_avg_variance8x16); + aom_highbd_dist_wtd_sad8x16_avg_bits10, + aom_highbd_10_dist_wtd_sub_pixel_avg_variance8x16); HIGHBD_BFP( BLOCK_8X8, aom_highbd_sad8x8_bits10, aom_highbd_sad8x8_avg_bits10, aom_highbd_10_variance8x8, aom_highbd_10_sub_pixel_variance8x8, aom_highbd_10_sub_pixel_avg_variance8x8, - aom_highbd_sad8x8x4d_bits10, aom_highbd_jnt_sad8x8_avg_bits10, - aom_highbd_10_jnt_sub_pixel_avg_variance8x8); + aom_highbd_sad8x8x4d_bits10, aom_highbd_dist_wtd_sad8x8_avg_bits10, + aom_highbd_10_dist_wtd_sub_pixel_avg_variance8x8); HIGHBD_BFP( BLOCK_8X4, aom_highbd_sad8x4_bits10, aom_highbd_sad8x4_avg_bits10, aom_highbd_10_variance8x4, aom_highbd_10_sub_pixel_variance8x4, aom_highbd_10_sub_pixel_avg_variance8x4, - aom_highbd_sad8x4x4d_bits10, aom_highbd_jnt_sad8x4_avg_bits10, - aom_highbd_10_jnt_sub_pixel_avg_variance8x4); + aom_highbd_sad8x4x4d_bits10, aom_highbd_dist_wtd_sad8x4_avg_bits10, + aom_highbd_10_dist_wtd_sub_pixel_avg_variance8x4); HIGHBD_BFP( BLOCK_4X8, aom_highbd_sad4x8_bits10, aom_highbd_sad4x8_avg_bits10, aom_highbd_10_variance4x8, aom_highbd_10_sub_pixel_variance4x8, aom_highbd_10_sub_pixel_avg_variance4x8, - aom_highbd_sad4x8x4d_bits10, aom_highbd_jnt_sad4x8_avg_bits10, - aom_highbd_10_jnt_sub_pixel_avg_variance4x8); + aom_highbd_sad4x8x4d_bits10, aom_highbd_dist_wtd_sad4x8_avg_bits10, + aom_highbd_10_dist_wtd_sub_pixel_avg_variance4x8); HIGHBD_BFP( BLOCK_4X4, aom_highbd_sad4x4_bits10, aom_highbd_sad4x4_avg_bits10, aom_highbd_10_variance4x4, aom_highbd_10_sub_pixel_variance4x4, aom_highbd_10_sub_pixel_avg_variance4x4, - aom_highbd_sad4x4x4d_bits10, aom_highbd_jnt_sad4x4_avg_bits10, - aom_highbd_10_jnt_sub_pixel_avg_variance4x4); + aom_highbd_sad4x4x4d_bits10, aom_highbd_dist_wtd_sad4x4_avg_bits10, + aom_highbd_10_dist_wtd_sub_pixel_avg_variance4x4); HIGHBD_BFP(BLOCK_128X128, aom_highbd_sad128x128_bits10, aom_highbd_sad128x128_avg_bits10, @@ -1866,24 +1961,26 @@ static void highbd_set_var_fns(AV1_COMP *const cpi) { aom_highbd_10_sub_pixel_variance128x128, aom_highbd_10_sub_pixel_avg_variance128x128, aom_highbd_sad128x128x4d_bits10, - aom_highbd_jnt_sad128x128_avg_bits10, - aom_highbd_10_jnt_sub_pixel_avg_variance128x128); + aom_highbd_dist_wtd_sad128x128_avg_bits10, + aom_highbd_10_dist_wtd_sub_pixel_avg_variance128x128); - HIGHBD_BFP( - BLOCK_128X64, aom_highbd_sad128x64_bits10, - aom_highbd_sad128x64_avg_bits10, aom_highbd_10_variance128x64, - aom_highbd_10_sub_pixel_variance128x64, - aom_highbd_10_sub_pixel_avg_variance128x64, - aom_highbd_sad128x64x4d_bits10, aom_highbd_jnt_sad128x64_avg_bits10, - aom_highbd_10_jnt_sub_pixel_avg_variance128x64); + HIGHBD_BFP(BLOCK_128X64, aom_highbd_sad128x64_bits10, + aom_highbd_sad128x64_avg_bits10, + aom_highbd_10_variance128x64, + aom_highbd_10_sub_pixel_variance128x64, + aom_highbd_10_sub_pixel_avg_variance128x64, + aom_highbd_sad128x64x4d_bits10, + aom_highbd_dist_wtd_sad128x64_avg_bits10, + aom_highbd_10_dist_wtd_sub_pixel_avg_variance128x64); - HIGHBD_BFP( - BLOCK_64X128, aom_highbd_sad64x128_bits10, - aom_highbd_sad64x128_avg_bits10, aom_highbd_10_variance64x128, - aom_highbd_10_sub_pixel_variance64x128, - aom_highbd_10_sub_pixel_avg_variance64x128, - aom_highbd_sad64x128x4d_bits10, aom_highbd_jnt_sad64x128_avg_bits10, - aom_highbd_10_jnt_sub_pixel_avg_variance64x128); + HIGHBD_BFP(BLOCK_64X128, aom_highbd_sad64x128_bits10, + aom_highbd_sad64x128_avg_bits10, + aom_highbd_10_variance64x128, + aom_highbd_10_sub_pixel_variance64x128, + aom_highbd_10_sub_pixel_avg_variance64x128, + aom_highbd_sad64x128x4d_bits10, + aom_highbd_dist_wtd_sad64x128_avg_bits10, + aom_highbd_10_dist_wtd_sub_pixel_avg_variance64x128); HIGHBD_MBFP(BLOCK_128X128, aom_highbd_masked_sad128x128_bits10, aom_highbd_10_masked_sub_pixel_variance128x128) @@ -2009,148 +2106,148 @@ static void highbd_set_var_fns(AV1_COMP *const cpi) { aom_highbd_12_sub_pixel_variance64x16, aom_highbd_12_sub_pixel_avg_variance64x16, aom_highbd_sad64x16x4d_bits12, - aom_highbd_jnt_sad64x16_avg_bits12, - aom_highbd_12_jnt_sub_pixel_avg_variance64x16); + aom_highbd_dist_wtd_sad64x16_avg_bits12, + aom_highbd_12_dist_wtd_sub_pixel_avg_variance64x16); HIGHBD_BFP(BLOCK_16X64, aom_highbd_sad16x64_bits12, aom_highbd_sad16x64_avg_bits12, aom_highbd_12_variance16x64, aom_highbd_12_sub_pixel_variance16x64, aom_highbd_12_sub_pixel_avg_variance16x64, aom_highbd_sad16x64x4d_bits12, - aom_highbd_jnt_sad16x64_avg_bits12, - aom_highbd_12_jnt_sub_pixel_avg_variance16x64); + aom_highbd_dist_wtd_sad16x64_avg_bits12, + aom_highbd_12_dist_wtd_sub_pixel_avg_variance16x64); HIGHBD_BFP(BLOCK_32X8, aom_highbd_sad32x8_bits12, aom_highbd_sad32x8_avg_bits12, aom_highbd_12_variance32x8, aom_highbd_12_sub_pixel_variance32x8, aom_highbd_12_sub_pixel_avg_variance32x8, aom_highbd_sad32x8x4d_bits12, - aom_highbd_jnt_sad32x8_avg_bits12, - aom_highbd_12_jnt_sub_pixel_avg_variance32x8); + aom_highbd_dist_wtd_sad32x8_avg_bits12, + aom_highbd_12_dist_wtd_sub_pixel_avg_variance32x8); HIGHBD_BFP(BLOCK_8X32, aom_highbd_sad8x32_bits12, aom_highbd_sad8x32_avg_bits12, aom_highbd_12_variance8x32, aom_highbd_12_sub_pixel_variance8x32, aom_highbd_12_sub_pixel_avg_variance8x32, aom_highbd_sad8x32x4d_bits12, - aom_highbd_jnt_sad8x32_avg_bits12, - aom_highbd_12_jnt_sub_pixel_avg_variance8x32); + aom_highbd_dist_wtd_sad8x32_avg_bits12, + aom_highbd_12_dist_wtd_sub_pixel_avg_variance8x32); HIGHBD_BFP(BLOCK_16X4, aom_highbd_sad16x4_bits12, aom_highbd_sad16x4_avg_bits12, aom_highbd_12_variance16x4, aom_highbd_12_sub_pixel_variance16x4, aom_highbd_12_sub_pixel_avg_variance16x4, aom_highbd_sad16x4x4d_bits12, - aom_highbd_jnt_sad16x4_avg_bits12, - aom_highbd_12_jnt_sub_pixel_avg_variance16x4); + aom_highbd_dist_wtd_sad16x4_avg_bits12, + aom_highbd_12_dist_wtd_sub_pixel_avg_variance16x4); HIGHBD_BFP(BLOCK_4X16, aom_highbd_sad4x16_bits12, aom_highbd_sad4x16_avg_bits12, aom_highbd_12_variance4x16, aom_highbd_12_sub_pixel_variance4x16, aom_highbd_12_sub_pixel_avg_variance4x16, aom_highbd_sad4x16x4d_bits12, - aom_highbd_jnt_sad4x16_avg_bits12, - aom_highbd_12_jnt_sub_pixel_avg_variance4x16); + aom_highbd_dist_wtd_sad4x16_avg_bits12, + aom_highbd_12_dist_wtd_sub_pixel_avg_variance4x16); HIGHBD_BFP(BLOCK_32X16, aom_highbd_sad32x16_bits12, aom_highbd_sad32x16_avg_bits12, aom_highbd_12_variance32x16, aom_highbd_12_sub_pixel_variance32x16, aom_highbd_12_sub_pixel_avg_variance32x16, aom_highbd_sad32x16x4d_bits12, - aom_highbd_jnt_sad32x16_avg_bits12, - aom_highbd_12_jnt_sub_pixel_avg_variance32x16); + aom_highbd_dist_wtd_sad32x16_avg_bits12, + aom_highbd_12_dist_wtd_sub_pixel_avg_variance32x16); HIGHBD_BFP(BLOCK_16X32, aom_highbd_sad16x32_bits12, aom_highbd_sad16x32_avg_bits12, aom_highbd_12_variance16x32, aom_highbd_12_sub_pixel_variance16x32, aom_highbd_12_sub_pixel_avg_variance16x32, aom_highbd_sad16x32x4d_bits12, - aom_highbd_jnt_sad16x32_avg_bits12, - aom_highbd_12_jnt_sub_pixel_avg_variance16x32); + aom_highbd_dist_wtd_sad16x32_avg_bits12, + aom_highbd_12_dist_wtd_sub_pixel_avg_variance16x32); HIGHBD_BFP(BLOCK_64X32, aom_highbd_sad64x32_bits12, aom_highbd_sad64x32_avg_bits12, aom_highbd_12_variance64x32, aom_highbd_12_sub_pixel_variance64x32, aom_highbd_12_sub_pixel_avg_variance64x32, aom_highbd_sad64x32x4d_bits12, - aom_highbd_jnt_sad64x32_avg_bits12, - aom_highbd_12_jnt_sub_pixel_avg_variance64x32); + aom_highbd_dist_wtd_sad64x32_avg_bits12, + aom_highbd_12_dist_wtd_sub_pixel_avg_variance64x32); HIGHBD_BFP(BLOCK_32X64, aom_highbd_sad32x64_bits12, aom_highbd_sad32x64_avg_bits12, aom_highbd_12_variance32x64, aom_highbd_12_sub_pixel_variance32x64, aom_highbd_12_sub_pixel_avg_variance32x64, aom_highbd_sad32x64x4d_bits12, - aom_highbd_jnt_sad32x64_avg_bits12, - aom_highbd_12_jnt_sub_pixel_avg_variance32x64); + aom_highbd_dist_wtd_sad32x64_avg_bits12, + aom_highbd_12_dist_wtd_sub_pixel_avg_variance32x64); HIGHBD_BFP(BLOCK_32X32, aom_highbd_sad32x32_bits12, aom_highbd_sad32x32_avg_bits12, aom_highbd_12_variance32x32, aom_highbd_12_sub_pixel_variance32x32, aom_highbd_12_sub_pixel_avg_variance32x32, aom_highbd_sad32x32x4d_bits12, - aom_highbd_jnt_sad32x32_avg_bits12, - aom_highbd_12_jnt_sub_pixel_avg_variance32x32); + aom_highbd_dist_wtd_sad32x32_avg_bits12, + aom_highbd_12_dist_wtd_sub_pixel_avg_variance32x32); HIGHBD_BFP(BLOCK_64X64, aom_highbd_sad64x64_bits12, aom_highbd_sad64x64_avg_bits12, aom_highbd_12_variance64x64, aom_highbd_12_sub_pixel_variance64x64, aom_highbd_12_sub_pixel_avg_variance64x64, aom_highbd_sad64x64x4d_bits12, - aom_highbd_jnt_sad64x64_avg_bits12, - aom_highbd_12_jnt_sub_pixel_avg_variance64x64); + aom_highbd_dist_wtd_sad64x64_avg_bits12, + aom_highbd_12_dist_wtd_sub_pixel_avg_variance64x64); HIGHBD_BFP(BLOCK_16X16, aom_highbd_sad16x16_bits12, aom_highbd_sad16x16_avg_bits12, aom_highbd_12_variance16x16, aom_highbd_12_sub_pixel_variance16x16, aom_highbd_12_sub_pixel_avg_variance16x16, aom_highbd_sad16x16x4d_bits12, - aom_highbd_jnt_sad16x16_avg_bits12, - aom_highbd_12_jnt_sub_pixel_avg_variance16x16); + aom_highbd_dist_wtd_sad16x16_avg_bits12, + aom_highbd_12_dist_wtd_sub_pixel_avg_variance16x16); HIGHBD_BFP(BLOCK_16X8, aom_highbd_sad16x8_bits12, aom_highbd_sad16x8_avg_bits12, aom_highbd_12_variance16x8, aom_highbd_12_sub_pixel_variance16x8, aom_highbd_12_sub_pixel_avg_variance16x8, aom_highbd_sad16x8x4d_bits12, - aom_highbd_jnt_sad16x8_avg_bits12, - aom_highbd_12_jnt_sub_pixel_avg_variance16x8); + aom_highbd_dist_wtd_sad16x8_avg_bits12, + aom_highbd_12_dist_wtd_sub_pixel_avg_variance16x8); HIGHBD_BFP(BLOCK_8X16, aom_highbd_sad8x16_bits12, aom_highbd_sad8x16_avg_bits12, aom_highbd_12_variance8x16, aom_highbd_12_sub_pixel_variance8x16, aom_highbd_12_sub_pixel_avg_variance8x16, aom_highbd_sad8x16x4d_bits12, - aom_highbd_jnt_sad8x16_avg_bits12, - aom_highbd_12_jnt_sub_pixel_avg_variance8x16); + aom_highbd_dist_wtd_sad8x16_avg_bits12, + aom_highbd_12_dist_wtd_sub_pixel_avg_variance8x16); HIGHBD_BFP( BLOCK_8X8, aom_highbd_sad8x8_bits12, aom_highbd_sad8x8_avg_bits12, aom_highbd_12_variance8x8, aom_highbd_12_sub_pixel_variance8x8, aom_highbd_12_sub_pixel_avg_variance8x8, - aom_highbd_sad8x8x4d_bits12, aom_highbd_jnt_sad8x8_avg_bits12, - aom_highbd_12_jnt_sub_pixel_avg_variance8x8); + aom_highbd_sad8x8x4d_bits12, aom_highbd_dist_wtd_sad8x8_avg_bits12, + aom_highbd_12_dist_wtd_sub_pixel_avg_variance8x8); HIGHBD_BFP( BLOCK_8X4, aom_highbd_sad8x4_bits12, aom_highbd_sad8x4_avg_bits12, aom_highbd_12_variance8x4, aom_highbd_12_sub_pixel_variance8x4, aom_highbd_12_sub_pixel_avg_variance8x4, - aom_highbd_sad8x4x4d_bits12, aom_highbd_jnt_sad8x4_avg_bits12, - aom_highbd_12_jnt_sub_pixel_avg_variance8x4); + aom_highbd_sad8x4x4d_bits12, aom_highbd_dist_wtd_sad8x4_avg_bits12, + aom_highbd_12_dist_wtd_sub_pixel_avg_variance8x4); HIGHBD_BFP( BLOCK_4X8, aom_highbd_sad4x8_bits12, aom_highbd_sad4x8_avg_bits12, aom_highbd_12_variance4x8, aom_highbd_12_sub_pixel_variance4x8, aom_highbd_12_sub_pixel_avg_variance4x8, - aom_highbd_sad4x8x4d_bits12, aom_highbd_jnt_sad4x8_avg_bits12, - aom_highbd_12_jnt_sub_pixel_avg_variance4x8); + aom_highbd_sad4x8x4d_bits12, aom_highbd_dist_wtd_sad4x8_avg_bits12, + aom_highbd_12_dist_wtd_sub_pixel_avg_variance4x8); HIGHBD_BFP( BLOCK_4X4, aom_highbd_sad4x4_bits12, aom_highbd_sad4x4_avg_bits12, aom_highbd_12_variance4x4, aom_highbd_12_sub_pixel_variance4x4, aom_highbd_12_sub_pixel_avg_variance4x4, - aom_highbd_sad4x4x4d_bits12, aom_highbd_jnt_sad4x4_avg_bits12, - aom_highbd_12_jnt_sub_pixel_avg_variance4x4); + aom_highbd_sad4x4x4d_bits12, aom_highbd_dist_wtd_sad4x4_avg_bits12, + aom_highbd_12_dist_wtd_sub_pixel_avg_variance4x4); HIGHBD_BFP(BLOCK_128X128, aom_highbd_sad128x128_bits12, aom_highbd_sad128x128_avg_bits12, @@ -2158,24 +2255,26 @@ static void highbd_set_var_fns(AV1_COMP *const cpi) { aom_highbd_12_sub_pixel_variance128x128, aom_highbd_12_sub_pixel_avg_variance128x128, aom_highbd_sad128x128x4d_bits12, - aom_highbd_jnt_sad128x128_avg_bits12, - aom_highbd_12_jnt_sub_pixel_avg_variance128x128); + aom_highbd_dist_wtd_sad128x128_avg_bits12, + aom_highbd_12_dist_wtd_sub_pixel_avg_variance128x128); - HIGHBD_BFP( - BLOCK_128X64, aom_highbd_sad128x64_bits12, - aom_highbd_sad128x64_avg_bits12, aom_highbd_12_variance128x64, - aom_highbd_12_sub_pixel_variance128x64, - aom_highbd_12_sub_pixel_avg_variance128x64, - aom_highbd_sad128x64x4d_bits12, aom_highbd_jnt_sad128x64_avg_bits12, - aom_highbd_12_jnt_sub_pixel_avg_variance128x64); + HIGHBD_BFP(BLOCK_128X64, aom_highbd_sad128x64_bits12, + aom_highbd_sad128x64_avg_bits12, + aom_highbd_12_variance128x64, + aom_highbd_12_sub_pixel_variance128x64, + aom_highbd_12_sub_pixel_avg_variance128x64, + aom_highbd_sad128x64x4d_bits12, + aom_highbd_dist_wtd_sad128x64_avg_bits12, + aom_highbd_12_dist_wtd_sub_pixel_avg_variance128x64); - HIGHBD_BFP( - BLOCK_64X128, aom_highbd_sad64x128_bits12, - aom_highbd_sad64x128_avg_bits12, aom_highbd_12_variance64x128, - aom_highbd_12_sub_pixel_variance64x128, - aom_highbd_12_sub_pixel_avg_variance64x128, - aom_highbd_sad64x128x4d_bits12, aom_highbd_jnt_sad64x128_avg_bits12, - aom_highbd_12_jnt_sub_pixel_avg_variance64x128); + HIGHBD_BFP(BLOCK_64X128, aom_highbd_sad64x128_bits12, + aom_highbd_sad64x128_avg_bits12, + aom_highbd_12_variance64x128, + aom_highbd_12_sub_pixel_variance64x128, + aom_highbd_12_sub_pixel_avg_variance64x128, + aom_highbd_sad64x128x4d_bits12, + aom_highbd_dist_wtd_sad64x128_avg_bits12, + aom_highbd_12_dist_wtd_sub_pixel_avg_variance64x128); HIGHBD_MBFP(BLOCK_128X128, aom_highbd_masked_sad128x128_bits12, aom_highbd_12_masked_sub_pixel_variance128x128) @@ -2369,7 +2468,6 @@ void av1_change_config(struct AV1_COMP *cpi, const AV1EncoderConfig *oxcf) { cpi->oxcf = *oxcf; cpi->common.options = oxcf->cfg; - cpi->row_mt = oxcf->row_mt; x->e_mbd.bd = (int)seq_params->bit_depth; x->e_mbd.global_motion = cm->global_motion; @@ -2404,7 +2502,7 @@ void av1_change_config(struct AV1_COMP *cpi, const AV1EncoderConfig *oxcf) { for (int i = 0; i < 2; ++i) { if (x->tmp_obmc_bufs[i] == NULL) { CHECK_MEM_ERROR(cm, x->tmp_obmc_bufs[i], - aom_memalign(16, 2 * MAX_MB_PLANE * MAX_SB_SQUARE * + aom_memalign(32, 2 * MAX_MB_PLANE * MAX_SB_SQUARE * sizeof(*x->tmp_obmc_bufs[i]))); x->e_mbd.tmp_obmc_bufs[i] = x->tmp_obmc_bufs[i]; } @@ -2507,11 +2605,11 @@ AV1_COMP *av1_create_compressor(AV1EncoderConfig *oxcf, CHECK_MEM_ERROR(cm, cm->fc, (FRAME_CONTEXT *)aom_memalign(32, sizeof(*cm->fc))); - CHECK_MEM_ERROR(cm, cm->frame_contexts, - (FRAME_CONTEXT *)aom_memalign( - 32, FRAME_CONTEXTS * sizeof(*cm->frame_contexts))); + CHECK_MEM_ERROR( + cm, cm->default_frame_context, + (FRAME_CONTEXT *)aom_memalign(32, sizeof(*cm->default_frame_context))); memset(cm->fc, 0, sizeof(*cm->fc)); - memset(cm->frame_contexts, 0, FRAME_CONTEXTS * sizeof(*cm->frame_contexts)); + memset(cm->default_frame_context, 0, sizeof(*cm->default_frame_context)); cpi->resize_state = 0; cpi->resize_avg_qp = 0; @@ -2522,12 +2620,12 @@ AV1_COMP *av1_create_compressor(AV1EncoderConfig *oxcf, init_config(cpi, oxcf); av1_rc_init(&cpi->oxcf, oxcf->pass, &cpi->rc); - cm->current_video_frame = 0; + cm->current_frame.frame_number = 0; + cm->current_frame_id = -1; cpi->seq_params_locked = 0; cpi->partition_search_skippable_frame = 0; cpi->tile_data = NULL; - cpi->last_show_frame_buf_idx = INVALID_IDX; - + cpi->last_show_frame_buf = NULL; realloc_segmentation_maps(cpi); memset(cpi->nmv_costs, 0, sizeof(cpi->nmv_costs)); @@ -2563,6 +2661,9 @@ AV1_COMP *av1_create_compressor(AV1EncoderConfig *oxcf, cpi->count = 0; cpi->bytes = 0; +#if CONFIG_SPEED_STATS + cpi->tx_search_count = 0; +#endif // CONFIG_SPEED_STATS if (cpi->b_calculate_psnr) { cpi->total_sq_error = 0; @@ -2644,6 +2745,10 @@ AV1_COMP *av1_create_compressor(AV1EncoderConfig *oxcf, (int32_t *)aom_memalign( 16, MAX_SB_SQUARE * sizeof(*cpi->td.mb.wsrc_buf))); + CHECK_MEM_ERROR( + cm, cpi->td.mb.inter_modes_info, + (InterModesInfo *)aom_malloc(sizeof(*cpi->td.mb.inter_modes_info))); + for (int x = 0; x < 2; x++) for (int y = 0; y < 2; y++) CHECK_MEM_ERROR( @@ -2657,8 +2762,27 @@ AV1_COMP *av1_create_compressor(AV1EncoderConfig *oxcf, (int32_t *)aom_memalign( 16, MAX_SB_SQUARE * sizeof(*cpi->td.mb.mask_buf))); - av1_set_speed_features_framesize_independent(cpi); - av1_set_speed_features_framesize_dependent(cpi); + av1_set_speed_features_framesize_independent(cpi, oxcf->speed); + av1_set_speed_features_framesize_dependent(cpi, oxcf->speed); + + for (int frame = 0; frame < MAX_LAG_BUFFERS; ++frame) { + int mi_cols = ALIGN_POWER_OF_TWO(cm->mi_cols, MAX_MIB_SIZE_LOG2); + int mi_rows = ALIGN_POWER_OF_TWO(cm->mi_rows, MAX_MIB_SIZE_LOG2); + + CHECK_MEM_ERROR(cm, cpi->tpl_stats[frame].tpl_stats_ptr, + aom_calloc(mi_rows * mi_cols, + sizeof(*cpi->tpl_stats[frame].tpl_stats_ptr))); + cpi->tpl_stats[frame].is_valid = 0; + cpi->tpl_stats[frame].width = mi_cols; + cpi->tpl_stats[frame].height = mi_rows; + cpi->tpl_stats[frame].stride = mi_cols; + cpi->tpl_stats[frame].mi_rows = cm->mi_rows; + cpi->tpl_stats[frame].mi_cols = cm->mi_cols; + } + +#if CONFIG_COLLECT_PARTITION_STATS + av1_zero(cpi->partition_stats); +#endif #define BFP(BT, SDF, SDAF, VF, SVF, SVAF, SDX4DF, JSDAF, JSVAF) \ cpi->fn_ptr[BT].sdf = SDF; \ @@ -2672,103 +2796,109 @@ AV1_COMP *av1_create_compressor(AV1EncoderConfig *oxcf, BFP(BLOCK_4X16, aom_sad4x16, aom_sad4x16_avg, aom_variance4x16, aom_sub_pixel_variance4x16, aom_sub_pixel_avg_variance4x16, - aom_sad4x16x4d, aom_jnt_sad4x16_avg, aom_jnt_sub_pixel_avg_variance4x16) + aom_sad4x16x4d, aom_dist_wtd_sad4x16_avg, + aom_dist_wtd_sub_pixel_avg_variance4x16) BFP(BLOCK_16X4, aom_sad16x4, aom_sad16x4_avg, aom_variance16x4, aom_sub_pixel_variance16x4, aom_sub_pixel_avg_variance16x4, - aom_sad16x4x4d, aom_jnt_sad16x4_avg, aom_jnt_sub_pixel_avg_variance16x4) + aom_sad16x4x4d, aom_dist_wtd_sad16x4_avg, + aom_dist_wtd_sub_pixel_avg_variance16x4) BFP(BLOCK_8X32, aom_sad8x32, aom_sad8x32_avg, aom_variance8x32, aom_sub_pixel_variance8x32, aom_sub_pixel_avg_variance8x32, - aom_sad8x32x4d, aom_jnt_sad8x32_avg, aom_jnt_sub_pixel_avg_variance8x32) + aom_sad8x32x4d, aom_dist_wtd_sad8x32_avg, + aom_dist_wtd_sub_pixel_avg_variance8x32) BFP(BLOCK_32X8, aom_sad32x8, aom_sad32x8_avg, aom_variance32x8, aom_sub_pixel_variance32x8, aom_sub_pixel_avg_variance32x8, - aom_sad32x8x4d, aom_jnt_sad32x8_avg, aom_jnt_sub_pixel_avg_variance32x8) + aom_sad32x8x4d, aom_dist_wtd_sad32x8_avg, + aom_dist_wtd_sub_pixel_avg_variance32x8) BFP(BLOCK_16X64, aom_sad16x64, aom_sad16x64_avg, aom_variance16x64, aom_sub_pixel_variance16x64, aom_sub_pixel_avg_variance16x64, - aom_sad16x64x4d, aom_jnt_sad16x64_avg, - aom_jnt_sub_pixel_avg_variance16x64) + aom_sad16x64x4d, aom_dist_wtd_sad16x64_avg, + aom_dist_wtd_sub_pixel_avg_variance16x64) BFP(BLOCK_64X16, aom_sad64x16, aom_sad64x16_avg, aom_variance64x16, aom_sub_pixel_variance64x16, aom_sub_pixel_avg_variance64x16, - aom_sad64x16x4d, aom_jnt_sad64x16_avg, - aom_jnt_sub_pixel_avg_variance64x16) + aom_sad64x16x4d, aom_dist_wtd_sad64x16_avg, + aom_dist_wtd_sub_pixel_avg_variance64x16) BFP(BLOCK_128X128, aom_sad128x128, aom_sad128x128_avg, aom_variance128x128, aom_sub_pixel_variance128x128, aom_sub_pixel_avg_variance128x128, - aom_sad128x128x4d, aom_jnt_sad128x128_avg, - aom_jnt_sub_pixel_avg_variance128x128) + aom_sad128x128x4d, aom_dist_wtd_sad128x128_avg, + aom_dist_wtd_sub_pixel_avg_variance128x128) BFP(BLOCK_128X64, aom_sad128x64, aom_sad128x64_avg, aom_variance128x64, aom_sub_pixel_variance128x64, aom_sub_pixel_avg_variance128x64, - aom_sad128x64x4d, aom_jnt_sad128x64_avg, - aom_jnt_sub_pixel_avg_variance128x64) + aom_sad128x64x4d, aom_dist_wtd_sad128x64_avg, + aom_dist_wtd_sub_pixel_avg_variance128x64) BFP(BLOCK_64X128, aom_sad64x128, aom_sad64x128_avg, aom_variance64x128, aom_sub_pixel_variance64x128, aom_sub_pixel_avg_variance64x128, - aom_sad64x128x4d, aom_jnt_sad64x128_avg, - aom_jnt_sub_pixel_avg_variance64x128) + aom_sad64x128x4d, aom_dist_wtd_sad64x128_avg, + aom_dist_wtd_sub_pixel_avg_variance64x128) BFP(BLOCK_32X16, aom_sad32x16, aom_sad32x16_avg, aom_variance32x16, aom_sub_pixel_variance32x16, aom_sub_pixel_avg_variance32x16, - aom_sad32x16x4d, aom_jnt_sad32x16_avg, - aom_jnt_sub_pixel_avg_variance32x16) + aom_sad32x16x4d, aom_dist_wtd_sad32x16_avg, + aom_dist_wtd_sub_pixel_avg_variance32x16) BFP(BLOCK_16X32, aom_sad16x32, aom_sad16x32_avg, aom_variance16x32, aom_sub_pixel_variance16x32, aom_sub_pixel_avg_variance16x32, - aom_sad16x32x4d, aom_jnt_sad16x32_avg, - aom_jnt_sub_pixel_avg_variance16x32) + aom_sad16x32x4d, aom_dist_wtd_sad16x32_avg, + aom_dist_wtd_sub_pixel_avg_variance16x32) BFP(BLOCK_64X32, aom_sad64x32, aom_sad64x32_avg, aom_variance64x32, aom_sub_pixel_variance64x32, aom_sub_pixel_avg_variance64x32, - aom_sad64x32x4d, aom_jnt_sad64x32_avg, - aom_jnt_sub_pixel_avg_variance64x32) + aom_sad64x32x4d, aom_dist_wtd_sad64x32_avg, + aom_dist_wtd_sub_pixel_avg_variance64x32) BFP(BLOCK_32X64, aom_sad32x64, aom_sad32x64_avg, aom_variance32x64, aom_sub_pixel_variance32x64, aom_sub_pixel_avg_variance32x64, - aom_sad32x64x4d, aom_jnt_sad32x64_avg, - aom_jnt_sub_pixel_avg_variance32x64) + aom_sad32x64x4d, aom_dist_wtd_sad32x64_avg, + aom_dist_wtd_sub_pixel_avg_variance32x64) BFP(BLOCK_32X32, aom_sad32x32, aom_sad32x32_avg, aom_variance32x32, aom_sub_pixel_variance32x32, aom_sub_pixel_avg_variance32x32, - aom_sad32x32x4d, aom_jnt_sad32x32_avg, - aom_jnt_sub_pixel_avg_variance32x32) + aom_sad32x32x4d, aom_dist_wtd_sad32x32_avg, + aom_dist_wtd_sub_pixel_avg_variance32x32) BFP(BLOCK_64X64, aom_sad64x64, aom_sad64x64_avg, aom_variance64x64, aom_sub_pixel_variance64x64, aom_sub_pixel_avg_variance64x64, - aom_sad64x64x4d, aom_jnt_sad64x64_avg, - aom_jnt_sub_pixel_avg_variance64x64) + aom_sad64x64x4d, aom_dist_wtd_sad64x64_avg, + aom_dist_wtd_sub_pixel_avg_variance64x64) BFP(BLOCK_16X16, aom_sad16x16, aom_sad16x16_avg, aom_variance16x16, aom_sub_pixel_variance16x16, aom_sub_pixel_avg_variance16x16, - aom_sad16x16x4d, aom_jnt_sad16x16_avg, - aom_jnt_sub_pixel_avg_variance16x16) + aom_sad16x16x4d, aom_dist_wtd_sad16x16_avg, + aom_dist_wtd_sub_pixel_avg_variance16x16) BFP(BLOCK_16X8, aom_sad16x8, aom_sad16x8_avg, aom_variance16x8, aom_sub_pixel_variance16x8, aom_sub_pixel_avg_variance16x8, - aom_sad16x8x4d, aom_jnt_sad16x8_avg, aom_jnt_sub_pixel_avg_variance16x8) + aom_sad16x8x4d, aom_dist_wtd_sad16x8_avg, + aom_dist_wtd_sub_pixel_avg_variance16x8) BFP(BLOCK_8X16, aom_sad8x16, aom_sad8x16_avg, aom_variance8x16, aom_sub_pixel_variance8x16, aom_sub_pixel_avg_variance8x16, - aom_sad8x16x4d, aom_jnt_sad8x16_avg, aom_jnt_sub_pixel_avg_variance8x16) + aom_sad8x16x4d, aom_dist_wtd_sad8x16_avg, + aom_dist_wtd_sub_pixel_avg_variance8x16) BFP(BLOCK_8X8, aom_sad8x8, aom_sad8x8_avg, aom_variance8x8, aom_sub_pixel_variance8x8, aom_sub_pixel_avg_variance8x8, aom_sad8x8x4d, - aom_jnt_sad8x8_avg, aom_jnt_sub_pixel_avg_variance8x8) + aom_dist_wtd_sad8x8_avg, aom_dist_wtd_sub_pixel_avg_variance8x8) BFP(BLOCK_8X4, aom_sad8x4, aom_sad8x4_avg, aom_variance8x4, aom_sub_pixel_variance8x4, aom_sub_pixel_avg_variance8x4, aom_sad8x4x4d, - aom_jnt_sad8x4_avg, aom_jnt_sub_pixel_avg_variance8x4) + aom_dist_wtd_sad8x4_avg, aom_dist_wtd_sub_pixel_avg_variance8x4) BFP(BLOCK_4X8, aom_sad4x8, aom_sad4x8_avg, aom_variance4x8, aom_sub_pixel_variance4x8, aom_sub_pixel_avg_variance4x8, aom_sad4x8x4d, - aom_jnt_sad4x8_avg, aom_jnt_sub_pixel_avg_variance4x8) + aom_dist_wtd_sad4x8_avg, aom_dist_wtd_sub_pixel_avg_variance4x8) BFP(BLOCK_4X4, aom_sad4x4, aom_sad4x4_avg, aom_variance4x4, aom_sub_pixel_variance4x4, aom_sub_pixel_avg_variance4x4, aom_sad4x4x4d, - aom_jnt_sad4x4_avg, aom_jnt_sub_pixel_avg_variance4x4) + aom_dist_wtd_sad4x4_avg, aom_dist_wtd_sub_pixel_avg_variance4x4) #define OBFP(BT, OSDF, OVF, OSVF) \ cpi->fn_ptr[BT].osdf = OSDF; \ @@ -2892,7 +3022,7 @@ void av1_remove_compressor(AV1_COMP *cpi) { cm = &cpi->common; const int num_planes = av1_num_planes(cm); - if (cm->current_video_frame > 0) { + if (cm->current_frame.frame_number > 0) { #if CONFIG_ENTROPY_STATS if (cpi->oxcf.pass != 1) { fprintf(stderr, "Writing counts.stt\n"); @@ -2966,9 +3096,25 @@ void av1_remove_compressor(AV1_COMP *cpi) { fclose(f); } #endif // CONFIG_INTERNAL_STATS +#if CONFIG_SPEED_STATS + if (cpi->oxcf.pass != 1) { + fprintf(stdout, "tx_search_count = %d\n", cpi->tx_search_count); + } +#endif // CONFIG_SPEED_STATS + +#if CONFIG_COLLECT_PARTITION_STATS + if (cpi->oxcf.pass != 1) { + av1_print_partition_stats(&cpi->partition_stats); + } +#endif } - for (t = 0; t < cpi->num_workers; ++t) { + for (int frame = 0; frame < MAX_LAG_BUFFERS; ++frame) { + aom_free(cpi->tpl_stats[frame].tpl_stats_ptr); + cpi->tpl_stats[frame].is_valid = 0; + } + + for (t = cpi->num_workers - 1; t >= 0; --t) { AVxWorker *const worker = &cpi->workers[t]; EncWorkerData *const thread_data = &cpi->tile_thr_data[t]; @@ -2976,7 +3122,8 @@ void av1_remove_compressor(AV1_COMP *cpi) { aom_get_worker_interface()->end(worker); // Deallocate allocated thread data. - if (t < cpi->num_workers - 1) { + if (cpi->row_mt == 1) aom_free(thread_data->td->tctx); + if (t > 0) { aom_free(thread_data->td->palette_buffer); aom_free(thread_data->td->tmp_conv_dst); for (int j = 0; j < 2; ++j) { @@ -2985,6 +3132,8 @@ void av1_remove_compressor(AV1_COMP *cpi) { aom_free(thread_data->td->above_pred_buf); aom_free(thread_data->td->left_pred_buf); aom_free(thread_data->td->wsrc_buf); + + aom_free(thread_data->td->inter_modes_info); for (int x = 0; x < 2; x++) { for (int y = 0; y < 2; y++) { aom_free(thread_data->td->hash_value_buffer[x][y]); @@ -2997,6 +3146,15 @@ void av1_remove_compressor(AV1_COMP *cpi) { aom_free(thread_data->td); } } +#if CONFIG_MULTITHREAD + if (cpi->row_mt == 1) { + if (cpi->row_mt_mutex_ != NULL) { + pthread_mutex_destroy(cpi->row_mt_mutex_); + aom_free(cpi->row_mt_mutex_); + } + } +#endif + av1_row_mt_mem_dealloc(cpi); aom_free(cpi->tile_thr_data); aom_free(cpi->workers); @@ -3043,7 +3201,7 @@ static void generate_psnr_packet(AV1_COMP *cpi) { struct aom_codec_cx_pkt pkt; int i; PSNR_STATS psnr; - aom_calc_highbd_psnr(cpi->source, cpi->common.frame_to_show, &psnr, + aom_calc_highbd_psnr(cpi->source, &cpi->common.cur_frame->buf, &psnr, cpi->td.mb.e_mbd.bd, cpi->oxcf.input_bit_depth); for (i = 0; i < 4; ++i) { @@ -3062,15 +3220,6 @@ int av1_use_as_reference(AV1_COMP *cpi, int ref_frame_flags) { return 0; } -void av1_update_reference(AV1_COMP *cpi, int ref_frame_upd_flags) { - cpi->ext_refresh_last_frame = (ref_frame_upd_flags & AOM_LAST_FLAG) != 0; - cpi->ext_refresh_golden_frame = (ref_frame_upd_flags & AOM_GOLD_FLAG) != 0; - cpi->ext_refresh_alt_ref_frame = (ref_frame_upd_flags & AOM_ALT_FLAG) != 0; - cpi->ext_refresh_bwd_ref_frame = (ref_frame_upd_flags & AOM_BWD_FLAG) != 0; - cpi->ext_refresh_alt2_ref_frame = (ref_frame_upd_flags & AOM_ALT2_FLAG) != 0; - cpi->ext_refresh_frame_flags_pending = 1; -} - int av1_copy_reference_enc(AV1_COMP *cpi, int idx, YV12_BUFFER_CONFIG *sd) { AV1_COMMON *const cm = &cpi->common; const int num_planes = av1_num_planes(cm); @@ -3133,61 +3282,6 @@ void aom_write_yuv_frame_420(YV12_BUFFER_CONFIG *s, FILE *f) { } #endif -static void check_show_existing_frame(AV1_COMP *cpi) { - const GF_GROUP *const gf_group = &cpi->twopass.gf_group; - AV1_COMMON *const cm = &cpi->common; - const FRAME_UPDATE_TYPE next_frame_update_type = - gf_group->update_type[gf_group->index]; -#if USE_SYMM_MULTI_LAYER - const int which_arf = (cpi->new_bwdref_update_rule == 1) - ? gf_group->arf_update_idx[gf_group->index] > 0 - : gf_group->arf_update_idx[gf_group->index]; -#else - const int which_arf = gf_group->arf_update_idx[gf_group->index]; -#endif - - if (cm->show_existing_frame == 1) { - cm->show_existing_frame = 0; - } else if (cpi->rc.is_last_bipred_frame) { -#if USE_SYMM_MULTI_LAYER - // NOTE: When new structure is used, every bwdref will have one overlay - // frame. Therefore, there is no need to find out which frame to - // show in advance. - if (cpi->new_bwdref_update_rule == 0) { -#endif - // NOTE: If the current frame is a last bi-predictive frame, it is - // needed next to show the BWDREF_FRAME, which is pointed by - // the last_fb_idxes[0] after reference frame buffer update - cpi->rc.is_last_bipred_frame = 0; - cm->show_existing_frame = 1; - cpi->existing_fb_idx_to_show = cpi->ref_fb_idx[0]; -#if USE_SYMM_MULTI_LAYER - } -#endif - } else if (cpi->is_arf_filter_off[which_arf] && - (next_frame_update_type == OVERLAY_UPDATE || - next_frame_update_type == INTNL_OVERLAY_UPDATE)) { -#if USE_SYMM_MULTI_LAYER - const int bwdref_to_show = - (cpi->new_bwdref_update_rule == 1) ? BWDREF_FRAME : ALTREF2_FRAME; -#else - const int bwdref_to_show = ALTREF2_FRAME; -#endif - // Other parameters related to OVERLAY_UPDATE will be taken care of - // in av1_rc_get_second_pass_params(cpi) - cm->show_existing_frame = 1; - cpi->rc.is_src_frame_alt_ref = 1; - cpi->existing_fb_idx_to_show = (next_frame_update_type == OVERLAY_UPDATE) - ? cpi->ref_fb_idx[ALTREF_FRAME - 1] - : cpi->ref_fb_idx[bwdref_to_show - 1]; -#if USE_SYMM_MULTI_LAYER - if (cpi->new_bwdref_update_rule == 0) -#endif - cpi->is_arf_filter_off[which_arf] = 0; - } - cpi->rc.is_src_frame_ext_arf = 0; -} - #ifdef OUTPUT_YUV_REC void aom_write_one_yuv_frame(AV1_COMMON *cm, YV12_BUFFER_CONFIG *s) { uint8_t *src = s->y_buffer; @@ -3342,8 +3436,8 @@ static void dump_ref_frame_images(AV1_COMP *cpi) { for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) { char file_name[256] = ""; snprintf(file_name, sizeof(file_name), "/tmp/enc_F%d_ref_%d.yuv", - cm->current_video_frame, ref_frame); - dump_one_image(cm, get_ref_frame_buffer(cpi, ref_frame), file_name); + cm->current_frame.frame_number, ref_frame); + dump_one_image(cm, get_ref_frame_yv12_buf(cpi, ref_frame), file_name); } } #endif // DUMP_REF_FRAME_IMAGES == 1 @@ -3354,38 +3448,36 @@ static void dump_ref_frame_images(AV1_COMP *cpi) { // when the LAST_FRAME is updated. static INLINE void shift_last_ref_frames(AV1_COMP *cpi) { // TODO(isbs): shift the scaled indices as well - int ref_frame; - for (ref_frame = LAST_REF_FRAMES - 1; ref_frame > 0; --ref_frame) { - cpi->ref_fb_idx[ref_frame] = cpi->ref_fb_idx[ref_frame - 1]; + for (int ref_frame = LAST3_FRAME; ref_frame > LAST_FRAME; --ref_frame) { + const int ref_idx = ref_frame - LAST_FRAME; + cpi->common.remapped_ref_idx[ref_idx] = + cpi->common.remapped_ref_idx[ref_idx - 1]; - // [0] is allocated to the current coded frame. The statistics for the - // reference frames start at [LAST_FRAME], i.e. [1]. if (!cpi->rc.is_src_frame_alt_ref) { - memcpy(cpi->interp_filter_selected[ref_frame + LAST_FRAME], - cpi->interp_filter_selected[ref_frame - 1 + LAST_FRAME], - sizeof(cpi->interp_filter_selected[ref_frame - 1 + LAST_FRAME])); + memcpy(cpi->interp_filter_selected[ref_frame], + cpi->interp_filter_selected[ref_frame - 1], + sizeof(cpi->interp_filter_selected[ref_frame - 1])); } } } -#if USE_SYMM_MULTI_LAYER // This function is used to shift the virtual indices of bwd reference // frames as follows: // BWD_REF -> ALT2_REF -> EXT_REF // to clear a space to store the closest bwdref static INLINE void rshift_bwd_ref_frames(AV1_COMP *cpi) { // TODO(isbs): shift the scaled indices as well - static const int ordered_bwd[3] = { BWDREF_FRAME - 1, ALTREF2_FRAME - 1, - EXTREF_FRAME - 1 }; + static const int ordered_bwd[3] = { BWDREF_FRAME, ALTREF2_FRAME, + EXTREF_FRAME }; for (int i = 2; i > 0; --i) { // [0] is allocated to the current coded frame, i.e. bwdref - memcpy( - cpi->interp_filter_selected[ordered_bwd[i] + LAST_FRAME], - cpi->interp_filter_selected[ordered_bwd[i - 1] + LAST_FRAME], - sizeof(cpi->interp_filter_selected[ordered_bwd[i - 1] + LAST_FRAME])); + memcpy(cpi->interp_filter_selected[ordered_bwd[i]], + cpi->interp_filter_selected[ordered_bwd[i - 1]], + sizeof(cpi->interp_filter_selected[ordered_bwd[i - 1]])); - cpi->ref_fb_idx[ordered_bwd[i]] = cpi->ref_fb_idx[ordered_bwd[i - 1]]; + cpi->common.remapped_ref_idx[ordered_bwd[i] - LAST_FRAME] = + cpi->common.remapped_ref_idx[ordered_bwd[i - 1] - LAST_FRAME]; } } @@ -3395,33 +3487,37 @@ static INLINE void rshift_bwd_ref_frames(AV1_COMP *cpi) { // to update the bwd reference frame for coding the next frame. static INLINE void lshift_bwd_ref_frames(AV1_COMP *cpi) { // TODO(isbs): shift the scaled indices as well - static const int ordered_bwd[3] = { BWDREF_FRAME - 1, ALTREF2_FRAME - 1, - EXTREF_FRAME - 1 }; + static const int ordered_bwd[3] = { BWDREF_FRAME, ALTREF2_FRAME, + EXTREF_FRAME }; for (int i = 0; i < 2; ++i) { // [0] is allocated to the current coded frame, i.e. bwdref - memcpy( - cpi->interp_filter_selected[ordered_bwd[i] + LAST_FRAME], - cpi->interp_filter_selected[ordered_bwd[i + 1] + LAST_FRAME], - sizeof(cpi->interp_filter_selected[ordered_bwd[i + 1] + LAST_FRAME])); + memcpy(cpi->interp_filter_selected[ordered_bwd[i]], + cpi->interp_filter_selected[ordered_bwd[i + 1]], + sizeof(cpi->interp_filter_selected[ordered_bwd[i + 1]])); - cpi->ref_fb_idx[ordered_bwd[i]] = cpi->ref_fb_idx[ordered_bwd[i + 1]]; + cpi->common.remapped_ref_idx[ordered_bwd[i] - LAST_FRAME] = + cpi->common.remapped_ref_idx[ordered_bwd[i + 1] - LAST_FRAME]; } } -#endif // USE_SYMM_MULTI_LAYER static void update_reference_frames(AV1_COMP *cpi) { AV1_COMMON *const cm = &cpi->common; // NOTE: Save the new show frame buffer index for --test-code=warn, i.e., // for the purpose to verify no mismatch between encoder and decoder. - if (cm->show_frame) cpi->last_show_frame_buf_idx = cm->new_fb_idx; + if (cm->show_frame) cpi->last_show_frame_buf = cm->cur_frame; // In the case of show_existing frame, we will not send fresh flag // to decoder. Any change in the reference frame buffer can be done by // switching the virtual indices. if (cm->show_existing_frame) { - cpi->refresh_last_frame = 0; + // If we are not indicating to the decoder that this frame is + // a show_existing_frame, which occurs in error_resilient mode, + // we still want to refresh the LAST_FRAME when the current frame + // was the source of an ext_arf. + cpi->refresh_last_frame = + !encode_show_existing_frame(cm) && cpi->rc.is_src_frame_ext_arf; cpi->refresh_golden_frame = 0; cpi->refresh_bwd_ref_frame = 0; cpi->refresh_alt2_ref_frame = 0; @@ -3432,18 +3528,16 @@ static void update_reference_frames(AV1_COMP *cpi) { cpi->rc.is_bipred_frame = 0; } - BufferPool *const pool = cm->buffer_pool; - // At this point the new frame has been encoded. // If any buffer copy / swapping is signaled it should be done here. // Only update all of the reference buffers if a KEY_FRAME is also a // show_frame. This ensures a fwd keyframe does not update all of the buffers - if ((cm->frame_type == KEY_FRAME && cm->show_frame) || frame_is_sframe(cm)) { + if ((cm->current_frame.frame_type == KEY_FRAME && cm->show_frame) || + frame_is_sframe(cm)) { for (int ref_frame = 0; ref_frame < REF_FRAMES; ++ref_frame) { - ref_cnt_fb(pool->frame_bufs, - &cm->ref_frame_map[cpi->ref_fb_idx[ref_frame]], - cm->new_fb_idx); + assign_frame_buffer_p(&cm->ref_frame_map[cm->remapped_ref_idx[ref_frame]], + cm->cur_frame); } return; } @@ -3463,56 +3557,50 @@ static void update_reference_frames(AV1_COMP *cpi) { // reference instead of replacing it with overlay. if (!cpi->preserve_arf_as_gld) { - ref_cnt_fb(pool->frame_bufs, - &cm->ref_frame_map[cpi->ref_fb_idx[ALTREF_FRAME - 1]], - cm->new_fb_idx); + assign_frame_buffer_p( + &cm->ref_frame_map[get_ref_frame_map_idx(cm, ALTREF_FRAME)], + cm->cur_frame); } - tmp = cpi->ref_fb_idx[ALTREF_FRAME - 1]; - cpi->ref_fb_idx[ALTREF_FRAME - 1] = cpi->ref_fb_idx[GOLDEN_FRAME - 1]; - cpi->ref_fb_idx[GOLDEN_FRAME - 1] = tmp; + tmp = get_ref_frame_map_idx(cm, ALTREF_FRAME); + cm->remapped_ref_idx[ALTREF_FRAME - LAST_FRAME] = + get_ref_frame_map_idx(cm, GOLDEN_FRAME); + cm->remapped_ref_idx[GOLDEN_FRAME - LAST_FRAME] = tmp; // TODO(zoeliu): Do we need to copy cpi->interp_filter_selected[0] over to // cpi->interp_filter_selected[GOLDEN_FRAME]? - } else if (cpi->rc.is_src_frame_ext_arf && cm->show_existing_frame) { + } else if (cpi->rc.is_src_frame_ext_arf && encode_show_existing_frame(cm)) { #if CONFIG_DEBUG const GF_GROUP *const gf_group = &cpi->twopass.gf_group; assert(gf_group->update_type[gf_group->index] == INTNL_OVERLAY_UPDATE); -#endif -#if USE_SYMM_MULTI_LAYER +#endif // CONFIG_DEBUG const int bwdref_to_show = (cpi->new_bwdref_update_rule == 1) ? BWDREF_FRAME : ALTREF2_FRAME; -#else - const int bwdref_to_show = ALTREF2_FRAME; -#endif // Deal with the special case for showing existing internal ALTREF_FRAME // Refresh the LAST_FRAME with the ALTREF_FRAME and retire the LAST3_FRAME // by updating the virtual indices. - const int tmp = cpi->ref_fb_idx[LAST_REF_FRAMES - 1]; + const int last3_remapped_idx = get_ref_frame_map_idx(cm, LAST3_FRAME); shift_last_ref_frames(cpi); - cpi->ref_fb_idx[LAST_FRAME - 1] = cpi->ref_fb_idx[bwdref_to_show - 1]; + cm->remapped_ref_idx[LAST_FRAME - LAST_FRAME] = + get_ref_frame_map_idx(cm, bwdref_to_show); memcpy(cpi->interp_filter_selected[LAST_FRAME], cpi->interp_filter_selected[bwdref_to_show], sizeof(cpi->interp_filter_selected[bwdref_to_show])); -#if USE_SYMM_MULTI_LAYER if (cpi->new_bwdref_update_rule == 1) { lshift_bwd_ref_frames(cpi); // pass outdated forward reference frame (previous LAST3) to the // spared space - cpi->ref_fb_idx[EXTREF_FRAME - 1] = tmp; + cm->remapped_ref_idx[EXTREF_FRAME - LAST_FRAME] = last3_remapped_idx; } else { -#endif - cpi->ref_fb_idx[bwdref_to_show - 1] = tmp; -#if USE_SYMM_MULTI_LAYER + cm->remapped_ref_idx[bwdref_to_show - LAST_FRAME] = last3_remapped_idx; } -#endif } else { /* For non key/golden frames */ // === ALTREF_FRAME === if (cpi->refresh_alt_ref_frame) { - int arf_idx = cpi->ref_fb_idx[ALTREF_FRAME - 1]; - ref_cnt_fb(pool->frame_bufs, &cm->ref_frame_map[arf_idx], cm->new_fb_idx); + int arf_idx = get_ref_frame_map_idx(cm, ALTREF_FRAME); + assign_frame_buffer_p(&cm->ref_frame_map[arf_idx], cm->cur_frame); memcpy(cpi->interp_filter_selected[ALTREF_FRAME], cpi->interp_filter_selected[0], @@ -3521,9 +3609,9 @@ static void update_reference_frames(AV1_COMP *cpi) { // === GOLDEN_FRAME === if (cpi->refresh_golden_frame) { - ref_cnt_fb(pool->frame_bufs, - &cm->ref_frame_map[cpi->ref_fb_idx[GOLDEN_FRAME - 1]], - cm->new_fb_idx); + assign_frame_buffer_p( + &cm->ref_frame_map[get_ref_frame_map_idx(cm, GOLDEN_FRAME)], + cm->cur_frame); memcpy(cpi->interp_filter_selected[GOLDEN_FRAME], cpi->interp_filter_selected[0], @@ -3532,25 +3620,21 @@ static void update_reference_frames(AV1_COMP *cpi) { // === BWDREF_FRAME === if (cpi->refresh_bwd_ref_frame) { -#if USE_SYMM_MULTI_LAYER if (cpi->new_bwdref_update_rule) { // We shift the backward reference frame as follows: // BWDREF -> ALTREF2 -> EXTREF // and assign the newly coded frame to BWDREF so that it always // keeps the nearest future frame - int tmp = cpi->ref_fb_idx[EXTREF_FRAME - 1]; - ref_cnt_fb(pool->frame_bufs, &cm->ref_frame_map[tmp], cm->new_fb_idx); + const int tmp = get_ref_frame_map_idx(cm, EXTREF_FRAME); + assign_frame_buffer_p(&cm->ref_frame_map[tmp], cm->cur_frame); rshift_bwd_ref_frames(cpi); - cpi->ref_fb_idx[BWDREF_FRAME - 1] = tmp; + cm->remapped_ref_idx[BWDREF_FRAME - LAST_FRAME] = tmp; } else { -#endif // USE_SYMM_MULTI_LAYER - ref_cnt_fb(pool->frame_bufs, - &cm->ref_frame_map[cpi->ref_fb_idx[BWDREF_FRAME - 1]], - cm->new_fb_idx); -#if USE_SYMM_MULTI_LAYER + assign_frame_buffer_p( + &cm->ref_frame_map[get_ref_frame_map_idx(cm, BWDREF_FRAME)], + cm->cur_frame); } -#endif memcpy(cpi->interp_filter_selected[BWDREF_FRAME], cpi->interp_filter_selected[0], sizeof(cpi->interp_filter_selected[0])); @@ -3558,9 +3642,9 @@ static void update_reference_frames(AV1_COMP *cpi) { // === ALTREF2_FRAME === if (cpi->refresh_alt2_ref_frame) { - ref_cnt_fb(pool->frame_bufs, - &cm->ref_frame_map[cpi->ref_fb_idx[ALTREF2_FRAME - 1]], - cm->new_fb_idx); + assign_frame_buffer_p( + &cm->ref_frame_map[get_ref_frame_map_idx(cm, ALTREF2_FRAME)], + cm->cur_frame); memcpy(cpi->interp_filter_selected[ALTREF2_FRAME], cpi->interp_filter_selected[0], @@ -3571,15 +3655,15 @@ static void update_reference_frames(AV1_COMP *cpi) { if (cpi->refresh_last_frame) { // NOTE(zoeliu): We have two layers of mapping (1) from the per-frame // reference to the reference frame buffer virtual index; and then (2) from - // the virtual index to the reference frame buffer physical index: + // the virtual index to the reference frame buffer (RefCntBuffer): // - // LAST_FRAME, ..., LAST3_FRAME, ..., ALTREF_FRAME - // | | | - // v v v - // ref_fb_idx[0], ..., ref_fb_idx[2], ..., ref_fb_idx[ALTREF_FRAME-1] - // | | | - // v v v - // ref_frame_map[], ..., ref_frame_map[], ..., ref_frame_map[] + // LAST_FRAME, ..., EXTREF_FRAME + // | | + // v v + // remapped_ref_idx[LAST_FRAME - 1], ..., remapped_ref_idx[EXTREF_FRAME - 1] + // | | + // v v + // ref_frame_map[], ..., ref_frame_map[] // // When refresh_last_frame is set, it is intended to retire LAST3_FRAME, // have the other 2 LAST reference frames shifted as follows: @@ -3590,27 +3674,25 @@ static void update_reference_frames(AV1_COMP *cpi) { // // (a) To change ref_frame_map[] and have the virtual index of LAST3_FRAME // to point to the newly coded frame, i.e. - // ref_frame_map[lst_fb_idexes[2]] => new_fb_idx; + // ref_frame_map[lst_fb_idexes[2]] => cur_frame; // // (b) To change the 1st layer mapping to have LAST_FRAME mapped to the // original virtual index of LAST3_FRAME and have the other mappings // shifted as follows: - // LAST_FRAME, LAST2_FRAME, LAST3_FRAME - // | | | - // v v v - // ref_fb_idx[2], ref_fb_idx[0], ref_fb_idx[1] - int tmp; + // LAST_FRAME, LAST2_FRAME, LAST3_FRAME + // | | | + // v v v + // remapped_ref_idx[2], remapped_ref_idx[0], remapped_ref_idx[1] + assign_frame_buffer_p( + &cm->ref_frame_map[get_ref_frame_map_idx(cm, LAST3_FRAME)], + cm->cur_frame); - ref_cnt_fb(pool->frame_bufs, - &cm->ref_frame_map[cpi->ref_fb_idx[LAST_REF_FRAMES - 1]], - cm->new_fb_idx); - - tmp = cpi->ref_fb_idx[LAST_REF_FRAMES - 1]; + int last3_remapped_idx = get_ref_frame_map_idx(cm, LAST3_FRAME); shift_last_ref_frames(cpi); - cpi->ref_fb_idx[0] = tmp; + cm->remapped_ref_idx[LAST_FRAME - LAST_FRAME] = last3_remapped_idx; - assert(cm->show_existing_frame == 0); + assert(!encode_show_existing_frame(cm)); memcpy(cpi->interp_filter_selected[LAST_FRAME], cpi->interp_filter_selected[0], sizeof(cpi->interp_filter_selected[0])); @@ -3618,11 +3700,7 @@ static void update_reference_frames(AV1_COMP *cpi) { // If the new structure is used, we will always have overlay frames coupled // with bwdref frames. Therefore, we won't have to perform this update // in advance (we do this update when the overlay frame shows up). -#if USE_SYMM_MULTI_LAYER if (cpi->new_bwdref_update_rule == 0 && cpi->rc.is_last_bipred_frame) { -#else - if (cpi->rc.is_last_bipred_frame) { -#endif // Refresh the LAST_FRAME with the BWDREF_FRAME and retire the // LAST3_FRAME by updating the virtual indices. // @@ -3631,11 +3709,12 @@ static void update_reference_frames(AV1_COMP *cpi) { // virtual index reshuffling for BWDREF, the encoder always // specifies a LAST_BIPRED right before BWDREF and completes the // reshuffling job accordingly. - tmp = cpi->ref_fb_idx[LAST_REF_FRAMES - 1]; + last3_remapped_idx = get_ref_frame_map_idx(cm, LAST3_FRAME); shift_last_ref_frames(cpi); - cpi->ref_fb_idx[0] = cpi->ref_fb_idx[BWDREF_FRAME - 1]; - cpi->ref_fb_idx[BWDREF_FRAME - 1] = tmp; + cm->remapped_ref_idx[LAST_FRAME - LAST_FRAME] = + get_ref_frame_map_idx(cm, BWDREF_FRAME); + cm->remapped_ref_idx[BWDREF_FRAME - LAST_FRAME] = last3_remapped_idx; memcpy(cpi->interp_filter_selected[LAST_FRAME], cpi->interp_filter_selected[BWDREF_FRAME], @@ -3649,14 +3728,6 @@ static void update_reference_frames(AV1_COMP *cpi) { #endif // DUMP_REF_FRAME_IMAGES } -static INLINE void alloc_frame_mvs(AV1_COMMON *const cm, int buffer_idx) { - assert(buffer_idx != INVALID_IDX); - RefCntBuffer *const new_fb_ptr = &cm->buffer_pool->frame_bufs[buffer_idx]; - ensure_mv_buffer(new_fb_ptr, cm); - new_fb_ptr->width = cm->width; - new_fb_ptr->height = cm->height; -} - static void scale_references(AV1_COMP *cpi) { AV1_COMMON *cm = &cpi->common; const int num_planes = av1_num_planes(cm); @@ -3671,62 +3742,65 @@ static void scale_references(AV1_COMP *cpi) { if (cpi->ref_frame_flags & ref_mask[ref_frame - 1]) { BufferPool *const pool = cm->buffer_pool; const YV12_BUFFER_CONFIG *const ref = - get_ref_frame_buffer(cpi, ref_frame); + get_ref_frame_yv12_buf(cm, ref_frame); if (ref == NULL) { - cpi->scaled_ref_idx[ref_frame - 1] = INVALID_IDX; + cpi->scaled_ref_buf[ref_frame - 1] = NULL; continue; } if (ref->y_crop_width != cm->width || ref->y_crop_height != cm->height) { - RefCntBuffer *new_fb_ptr = NULL; int force_scaling = 0; - int new_fb = cpi->scaled_ref_idx[ref_frame - 1]; - if (new_fb == INVALID_IDX) { - new_fb = get_free_fb(cm); + RefCntBuffer *new_fb = cpi->scaled_ref_buf[ref_frame - 1]; + if (new_fb == NULL) { + const int new_fb_idx = get_free_fb(cm); + if (new_fb_idx == INVALID_IDX) { + aom_internal_error(&cm->error, AOM_CODEC_MEM_ERROR, + "Unable to find free frame buffer"); + } force_scaling = 1; + new_fb = &pool->frame_bufs[new_fb_idx]; } - if (new_fb == INVALID_IDX) return; - new_fb_ptr = &pool->frame_bufs[new_fb]; - if (force_scaling || new_fb_ptr->buf.y_crop_width != cm->width || - new_fb_ptr->buf.y_crop_height != cm->height) { + + if (force_scaling || new_fb->buf.y_crop_width != cm->width || + new_fb->buf.y_crop_height != cm->height) { if (aom_realloc_frame_buffer( - &new_fb_ptr->buf, cm->width, cm->height, + &new_fb->buf, cm->width, cm->height, cm->seq_params.subsampling_x, cm->seq_params.subsampling_y, - cm->seq_params.use_highbitdepth, AOM_BORDER_IN_PIXELS, - cm->byte_alignment, NULL, NULL, NULL)) + cm->seq_params.use_highbitdepth, cpi->oxcf.border_in_pixels, + cm->byte_alignment, NULL, NULL, NULL)) { + if (force_scaling) { + // Release the reference acquired in the get_free_fb() call above. + --new_fb->ref_count; + } aom_internal_error(&cm->error, AOM_CODEC_MEM_ERROR, "Failed to allocate frame buffer"); + } av1_resize_and_extend_frame( - ref, &new_fb_ptr->buf, (int)cm->seq_params.bit_depth, num_planes); - cpi->scaled_ref_idx[ref_frame - 1] = new_fb; + ref, &new_fb->buf, (int)cm->seq_params.bit_depth, num_planes); + cpi->scaled_ref_buf[ref_frame - 1] = new_fb; alloc_frame_mvs(cm, new_fb); } } else { - const int buf_idx = get_ref_frame_buf_idx(cpi, ref_frame); - RefCntBuffer *const buf = &pool->frame_bufs[buf_idx]; + RefCntBuffer *buf = get_ref_frame_buf(cm, ref_frame); buf->buf.y_crop_width = ref->y_crop_width; buf->buf.y_crop_height = ref->y_crop_height; - cpi->scaled_ref_idx[ref_frame - 1] = buf_idx; + cpi->scaled_ref_buf[ref_frame - 1] = buf; ++buf->ref_count; } } else { - if (cpi->oxcf.pass != 0) cpi->scaled_ref_idx[ref_frame - 1] = INVALID_IDX; + if (cpi->oxcf.pass != 0) cpi->scaled_ref_buf[ref_frame - 1] = NULL; } } } static void release_scaled_references(AV1_COMP *cpi) { - AV1_COMMON *cm = &cpi->common; - int i; // TODO(isbs): only refresh the necessary frames, rather than all of them - for (i = 0; i < REF_FRAMES; ++i) { - const int idx = cpi->scaled_ref_idx[i]; - RefCntBuffer *const buf = - idx != INVALID_IDX ? &cm->buffer_pool->frame_bufs[idx] : NULL; + for (int i = 0; i < INTER_REFS_PER_FRAME; ++i) { + RefCntBuffer *const buf = cpi->scaled_ref_buf[i]; if (buf != NULL) { --buf->ref_count; - cpi->scaled_ref_idx[i] = INVALID_IDX; + cpi->scaled_ref_buf[i] = NULL; } } } @@ -3758,15 +3832,30 @@ static void set_mv_search_params(AV1_COMP *cpi) { static void set_size_independent_vars(AV1_COMP *cpi) { int i; + AV1_COMMON *cm = &cpi->common; for (i = LAST_FRAME; i <= ALTREF_FRAME; ++i) { - cpi->common.global_motion[i] = default_warp_params; + cm->global_motion[i] = default_warp_params; } cpi->global_motion_search_done = 0; - av1_set_speed_features_framesize_independent(cpi); + av1_set_speed_features_framesize_independent(cpi, cpi->speed); av1_set_rd_speed_thresholds(cpi); - av1_set_rd_speed_thresholds_sub8x8(cpi); - cpi->common.interp_filter = SWITCHABLE; - cpi->common.switchable_motion_mode = 1; + cm->interp_filter = SWITCHABLE; + cm->switchable_motion_mode = 1; + + if (frame_is_intra_only(cm)) { + if (cm->seq_params.force_screen_content_tools == 2) { + cm->allow_screen_content_tools = + cpi->oxcf.content == AOM_CONTENT_SCREEN || + is_screen_content(cpi->source->y_buffer, + cpi->source->flags & YV12_FLAG_HIGHBITDEPTH, + cm->seq_params.bit_depth, cpi->source->y_stride, + cpi->source->y_width, cpi->source->y_height); + } else { + cm->allow_screen_content_tools = + cm->seq_params.force_screen_content_tools; + } + } + cpi->is_screen_content_type = (cm->allow_screen_content_tools != 0); } static void set_size_dependent_vars(AV1_COMP *cpi, int *q, int *bottom_index, @@ -3775,7 +3864,7 @@ static void set_size_dependent_vars(AV1_COMP *cpi, int *q, int *bottom_index, const AV1EncoderConfig *const oxcf = &cpi->oxcf; // Setup variables that depend on the dimensions of the frame. - av1_set_speed_features_framesize_dependent(cpi); + av1_set_speed_features_framesize_dependent(cpi, cpi->speed); // Decide q and q bounds. *q = av1_rc_pick_q_and_bounds(cpi, cm->width, cm->height, bottom_index, @@ -3829,9 +3918,11 @@ static void init_ref_frame_bufs(AV1_COMP *cpi) { AV1_COMMON *const cm = &cpi->common; int i; BufferPool *const pool = cm->buffer_pool; - cm->new_fb_idx = INVALID_IDX; + cm->cur_frame = NULL; for (i = 0; i < REF_FRAMES; ++i) { - cm->ref_frame_map[i] = INVALID_IDX; + cm->ref_frame_map[i] = NULL; + } + for (i = 0; i < FRAME_BUFFERS; ++i) { pool->frame_bufs[i].ref_count = 0; } if (cm->seq_params.force_screen_content_tools) { @@ -3891,7 +3982,7 @@ static int set_size_literal(AV1_COMP *cpi, int width, int height) { return 0; } -static void set_frame_size(AV1_COMP *cpi, int width, int height) { +void av1_set_frame_size(AV1_COMP *cpi, int width, int height) { AV1_COMMON *const cm = &cpi->common; const SequenceHeader *const seq_params = &cm->seq_params; const int num_planes = av1_num_planes(cm); @@ -3910,7 +4001,7 @@ static void set_frame_size(AV1_COMP *cpi, int width, int height) { av1_set_target_rate(cpi, cm->width, cm->height); } - alloc_frame_mvs(cm, cm->new_fb_idx); + alloc_frame_mvs(cm, cm->cur_frame); // Allocate above context buffers if (cm->num_allocated_above_context_planes < av1_num_planes(cm) || @@ -3924,10 +4015,9 @@ static void set_frame_size(AV1_COMP *cpi, int width, int height) { // Reset the frame pointers to the current frame size. if (aom_realloc_frame_buffer( - get_frame_new_buffer(cm), cm->width, cm->height, - seq_params->subsampling_x, seq_params->subsampling_y, - seq_params->use_highbitdepth, AOM_BORDER_IN_PIXELS, - cm->byte_alignment, NULL, NULL, NULL)) + &cm->cur_frame->buf, cm->width, cm->height, seq_params->subsampling_x, + seq_params->subsampling_y, seq_params->use_highbitdepth, + cpi->oxcf.border_in_pixels, cm->byte_alignment, NULL, NULL, NULL)) aom_internal_error(&cm->error, AOM_CODEC_MEM_ERROR, "Failed to allocate frame buffer"); @@ -3940,25 +4030,17 @@ static void set_frame_size(AV1_COMP *cpi, int width, int height) { cm->rst_info[i].frame_restoration_type = RESTORE_NONE; av1_alloc_restoration_buffers(cm); - alloc_util_frame_buffers(cpi); // TODO(afergs): Remove? Gets called anyways. + alloc_util_frame_buffers(cpi); init_motion_estimation(cpi); for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) { - RefBuffer *const ref_buf = &cm->frame_refs[ref_frame - LAST_FRAME]; - const int buf_idx = get_ref_frame_buf_idx(cpi, ref_frame); - - ref_buf->idx = buf_idx; - - if (buf_idx != INVALID_IDX) { - YV12_BUFFER_CONFIG *const buf = &cm->buffer_pool->frame_bufs[buf_idx].buf; - ref_buf->buf = buf; - av1_setup_scale_factors_for_frame(&ref_buf->sf, buf->y_crop_width, - buf->y_crop_height, cm->width, + RefCntBuffer *const buf = get_ref_frame_buf(cm, ref_frame); + if (buf != NULL) { + struct scale_factors *sf = get_ref_scale_factors(cm, ref_frame); + av1_setup_scale_factors_for_frame(sf, buf->buf.y_crop_width, + buf->buf.y_crop_height, cm->width, cm->height); - if (av1_is_scaled(&ref_buf->sf)) - aom_extend_frame_borders(buf, num_planes); - } else { - ref_buf->buf = NULL; + if (av1_is_scaled(sf)) aom_extend_frame_borders(&buf->buf, num_planes); } } @@ -3979,7 +4061,7 @@ static uint8_t calculate_next_resize_scale(const AV1_COMP *cpi) { switch (oxcf->resize_mode) { case RESIZE_NONE: new_denom = SCALE_NUMERATOR; break; case RESIZE_FIXED: - if (cpi->common.frame_type == KEY_FRAME) + if (cpi->common.current_frame.frame_type == KEY_FRAME) new_denom = oxcf->resize_kf_scale_denominator; else new_denom = oxcf->resize_scale_denominator; @@ -3990,6 +4072,35 @@ static uint8_t calculate_next_resize_scale(const AV1_COMP *cpi) { return new_denom; } +#define ENERGY_BY_Q2_THRESH 0.01 +#define ENERGY_BY_AC_THRESH 0.2 + +static uint8_t get_superres_denom_from_qindex_energy(int qindex, double *energy, + double threshq, + double threshp) { + const double q = av1_convert_qindex_to_q(qindex, AOM_BITS_8); + const double tq = threshq * q * q; + const double tp = threshp * energy[1]; + const double thresh = AOMMIN(tq, tp); + int k; + for (k = 16; k > 8; --k) { + if (energy[k - 1] > thresh) break; + } + return 3 * SCALE_NUMERATOR - k; +} + +static uint8_t get_superres_denom_for_qindex(const AV1_COMP *cpi, int qindex) { + double energy[16]; + analyze_hor_freq(cpi, energy); + /* + printf("\nenergy = ["); + for (int k = 1; k < 16; ++k) printf("%f, ", energy[k]); + printf("]\n"); + */ + return get_superres_denom_from_qindex_energy( + qindex, energy, ENERGY_BY_Q2_THRESH, ENERGY_BY_AC_THRESH); +} + static uint8_t calculate_next_superres_scale(AV1_COMP *cpi) { // Choose an arbitrary random number static unsigned int seed = 34567; @@ -4007,38 +4118,28 @@ static uint8_t calculate_next_superres_scale(AV1_COMP *cpi) { switch (oxcf->superres_mode) { case SUPERRES_NONE: new_denom = SCALE_NUMERATOR; break; case SUPERRES_FIXED: - if (cpi->common.frame_type == KEY_FRAME) + if (cpi->common.current_frame.frame_type == KEY_FRAME) new_denom = oxcf->superres_kf_scale_denominator; else new_denom = oxcf->superres_scale_denominator; break; case SUPERRES_RANDOM: new_denom = lcg_rand16(&seed) % 9 + 8; break; case SUPERRES_QTHRESH: { - const GF_GROUP *const gf_group = &cpi->twopass.gf_group; - const RATE_FACTOR_LEVEL rf_level = gf_group->rf_level[gf_group->index]; - const double rate_factor_delta = rate_factor_deltas[rf_level]; - const int qthresh = (rate_factor_delta <= 1.0) - ? oxcf->superres_qthresh - : oxcf->superres_kf_qthresh; - av1_set_target_rate(cpi, cpi->oxcf.width, cpi->oxcf.height); + // Do not use superres when screen content tools are used. + if (cpi->common.allow_screen_content_tools) break; + if (oxcf->rc_mode == AOM_VBR || oxcf->rc_mode == AOM_CQ) + av1_set_target_rate(cpi, cpi->oxcf.width, cpi->oxcf.height); int bottom_index, top_index; const int q = av1_rc_pick_q_and_bounds( cpi, cpi->oxcf.width, cpi->oxcf.height, &bottom_index, &top_index); - if (q < qthresh) { + + const int qthresh = (frame_is_intra_only(&cpi->common)) + ? oxcf->superres_kf_qthresh + : oxcf->superres_qthresh; + if (q <= qthresh) { new_denom = SCALE_NUMERATOR; } else { - const uint8_t min_denom = SCALE_NUMERATOR + 1; - const uint8_t denom_step = (MAXQ - qthresh + 1) >> 3; - - if (q == qthresh) { - new_denom = min_denom; - } else if (denom_step == 0) { - new_denom = SCALE_NUMERATOR << 1; - } else { - const uint8_t additional_denom = (q - qthresh) / denom_step; - new_denom = - AOMMIN(min_denom + additional_denom, SCALE_NUMERATOR << 1); - } + new_denom = get_superres_denom_for_qindex(cpi, q); } break; } @@ -4115,7 +4216,7 @@ static int validate_size_scales(RESIZE_MODE resize_mode, } // Calculates resize and superres params for next frame -size_params_type av1_calculate_next_size_params(AV1_COMP *cpi) { +static size_params_type calculate_next_size_params(AV1_COMP *cpi) { const AV1EncoderConfig *oxcf = &cpi->oxcf; size_params_type rsz = { oxcf->width, oxcf->height, SCALE_NUMERATOR }; int resize_denom; @@ -4138,7 +4239,8 @@ size_params_type av1_calculate_next_size_params(AV1_COMP *cpi) { return rsz; } -static void setup_frame_size_from_params(AV1_COMP *cpi, size_params_type *rsz) { +static void setup_frame_size_from_params(AV1_COMP *cpi, + const size_params_type *rsz) { int encode_width = rsz->resize_width; int encode_height = rsz->resize_height; @@ -4148,11 +4250,13 @@ static void setup_frame_size_from_params(AV1_COMP *cpi, size_params_type *rsz) { cm->superres_scale_denominator = rsz->superres_denom; av1_calculate_scaled_superres_size(&encode_width, &encode_height, rsz->superres_denom); - set_frame_size(cpi, encode_width, encode_height); + av1_set_frame_size(cpi, encode_width, encode_height); } -static void setup_frame_size(AV1_COMP *cpi) { - size_params_type rsz = av1_calculate_next_size_params(cpi); +void av1_setup_frame_size(AV1_COMP *cpi) { + // Reset superres params from previous frame. + cpi->common.superres_scale_denominator = SCALE_NUMERATOR; + const size_params_type rsz = calculate_next_size_params(cpi); setup_frame_size_from_params(cpi, &rsz); } @@ -4227,33 +4331,36 @@ static void loopfilter_frame(AV1_COMP *cpi, AV1_COMMON *cm) { } if (lf->filter_level[0] || lf->filter_level[1]) { -#if LOOP_FILTER_BITMASK - av1_loop_filter_frame(cm->frame_to_show, cm, xd, 0, 0, num_planes, 0); -#else if (cpi->num_workers > 1) - av1_loop_filter_frame_mt(cm->frame_to_show, cm, xd, 0, num_planes, 0, + av1_loop_filter_frame_mt(&cm->cur_frame->buf, cm, xd, 0, num_planes, 0, +#if LOOP_FILTER_BITMASK + 0, +#endif cpi->workers, cpi->num_workers, &cpi->lf_row_sync); else - av1_loop_filter_frame(cm->frame_to_show, cm, xd, 0, num_planes, 0); + av1_loop_filter_frame(&cm->cur_frame->buf, cm, xd, +#if LOOP_FILTER_BITMASK + 0, #endif + 0, num_planes, 0); } if (!no_restoration) - av1_loop_restoration_save_boundary_lines(cm->frame_to_show, cm, 0); + av1_loop_restoration_save_boundary_lines(&cm->cur_frame->buf, cm, 0); if (no_cdef) { - cm->cdef_bits = 0; - cm->cdef_strengths[0] = 0; - cm->nb_cdef_strengths = 1; - cm->cdef_uv_strengths[0] = 0; + cm->cdef_info.cdef_bits = 0; + cm->cdef_info.cdef_strengths[0] = 0; + cm->cdef_info.nb_cdef_strengths = 1; + cm->cdef_info.cdef_uv_strengths[0] = 0; } else { // Find CDEF parameters - av1_cdef_search(cm->frame_to_show, cpi->source, cm, xd, + av1_cdef_search(&cm->cur_frame->buf, cpi->source, cm, xd, cpi->sf.fast_cdef_search); // Apply the filter - av1_cdef_frame(cm->frame_to_show, cm, xd); + av1_cdef_frame(&cm->cur_frame->buf, cm, xd); } superres_post_encode(cpi); @@ -4263,170 +4370,365 @@ static void loopfilter_frame(AV1_COMP *cpi, AV1_COMMON *cm) { cm->rst_info[1].frame_restoration_type = RESTORE_NONE; cm->rst_info[2].frame_restoration_type = RESTORE_NONE; } else { - av1_loop_restoration_save_boundary_lines(cm->frame_to_show, cm, 1); + av1_loop_restoration_save_boundary_lines(&cm->cur_frame->buf, cm, 1); av1_pick_filter_restoration(cpi->source, cpi); if (cm->rst_info[0].frame_restoration_type != RESTORE_NONE || cm->rst_info[1].frame_restoration_type != RESTORE_NONE || cm->rst_info[2].frame_restoration_type != RESTORE_NONE) { if (cpi->num_workers > 1) - av1_loop_restoration_filter_frame_mt(cm->frame_to_show, cm, 0, + av1_loop_restoration_filter_frame_mt(&cm->cur_frame->buf, cm, 0, cpi->workers, cpi->num_workers, &cpi->lr_row_sync, &cpi->lr_ctxt); else - av1_loop_restoration_filter_frame(cm->frame_to_show, cm, 0, + av1_loop_restoration_filter_frame(&cm->cur_frame->buf, cm, 0, &cpi->lr_ctxt); } } } -static int encode_without_recode_loop(AV1_COMP *cpi) { - AV1_COMMON *const cm = &cpi->common; - int q = 0, bottom_index = 0, top_index = 0; // Dummy variables. +static int get_refresh_frame_flags(const AV1_COMP *const cpi) { + const AV1_COMMON *const cm = &cpi->common; - aom_clear_system_state(); + // Switch frames and shown key-frames overwrite all reference slots + if ((cm->current_frame.frame_type == KEY_FRAME && cm->show_frame) || + frame_is_sframe(cm)) + return 0xFF; - set_size_independent_vars(cpi); + int refresh_mask = 0; - setup_frame_size(cpi); + // NOTE(zoeliu): When LAST_FRAME is to get refreshed, the decoder will be + // notified to get LAST3_FRAME refreshed and then the virtual indexes for all + // the 3 LAST reference frames will be updated accordingly, i.e.: + // (1) The original virtual index for LAST3_FRAME will become the new virtual + // index for LAST_FRAME; and + // (2) The original virtual indexes for LAST_FRAME and LAST2_FRAME will be + // shifted and become the new virtual indexes for LAST2_FRAME and + // LAST3_FRAME. + refresh_mask |= + (cpi->refresh_last_frame << get_ref_frame_map_idx(cm, LAST3_FRAME)); - assert(cm->width == cpi->scaled_source.y_crop_width); - assert(cm->height == cpi->scaled_source.y_crop_height); + const int bwd_ref_frame = + (cpi->new_bwdref_update_rule == 1) ? EXTREF_FRAME : BWDREF_FRAME; + refresh_mask |= + (cpi->refresh_bwd_ref_frame << get_ref_frame_map_idx(cm, bwd_ref_frame)); - set_size_dependent_vars(cpi, &q, &bottom_index, &top_index); + refresh_mask |= + (cpi->refresh_alt2_ref_frame << get_ref_frame_map_idx(cm, ALTREF2_FRAME)); - cpi->source = - av1_scale_if_required(cm, cpi->unscaled_source, &cpi->scaled_source); - if (cpi->unscaled_last_source != NULL) - cpi->last_source = av1_scale_if_required(cm, cpi->unscaled_last_source, - &cpi->scaled_last_source); - cpi->source->buf_8bit_valid = 0; - if (frame_is_intra_only(cm) == 0) { - scale_references(cpi); - } + if (av1_preserve_existing_gf(cpi)) { + // We have decided to preserve the previously existing golden frame as our + // new ARF frame. However, in the short term we leave it in the GF slot and, + // if we're updating the GF with the current decoded frame, we save it + // instead to the ARF slot. + // Later, in the function av1_encoder.c:av1_update_reference_frames() we + // will swap gld_fb_idx and alt_fb_idx to achieve our objective. We do it + // there so that it can be done outside of the recode loop. + // Note: This is highly specific to the use of ARF as a forward reference, + // and this needs to be generalized as other uses are implemented + // (like RTC/temporal scalability). - av1_set_quantizer(cm, q); - setup_frame(cpi); - suppress_active_map(cpi); - - // Variance adaptive and in frame q adjustment experiments are mutually - // exclusive. - if (cpi->oxcf.aq_mode == VARIANCE_AQ) { - av1_vaq_frame_setup(cpi); - } else if (cpi->oxcf.aq_mode == COMPLEXITY_AQ) { - av1_setup_in_frame_q_adj(cpi); - } else if (cpi->oxcf.aq_mode == CYCLIC_REFRESH_AQ) { - av1_cyclic_refresh_setup(cpi); - } - apply_active_map(cpi); - if (cm->seg.enabled) { - if (!cm->seg.update_data && cm->prev_frame) { - segfeatures_copy(&cm->seg, &cm->prev_frame->seg); - } else { - calculate_segdata(&cm->seg); + if (!cpi->preserve_arf_as_gld) { + refresh_mask |= (cpi->refresh_golden_frame + << get_ref_frame_map_idx(cm, ALTREF_FRAME)); } } else { - memset(&cm->seg, 0, sizeof(cm->seg)); + refresh_mask |= + (cpi->refresh_golden_frame << get_ref_frame_map_idx(cm, GOLDEN_FRAME)); + refresh_mask |= + (cpi->refresh_alt_ref_frame << get_ref_frame_map_idx(cm, ALTREF_FRAME)); } - segfeatures_copy(&cm->cur_frame->seg, &cm->seg); + return refresh_mask; +} - // transform / motion compensation build reconstruction frame - av1_encode_frame(cpi); +static void fix_interp_filter(InterpFilter *const interp_filter, + const FRAME_COUNTS *const counts) { + if (*interp_filter == SWITCHABLE) { + // Check to see if only one of the filters is actually used + int count[SWITCHABLE_FILTERS] = { 0 }; + int num_filters_used = 0; + for (int i = 0; i < SWITCHABLE_FILTERS; ++i) { + for (int j = 0; j < SWITCHABLE_FILTER_CONTEXTS; ++j) + count[i] += counts->switchable_interp[j][i]; + num_filters_used += (count[i] > 0); + } + if (num_filters_used == 1) { + // Only one filter is used. So set the filter at frame level + for (int i = 0; i < SWITCHABLE_FILTERS; ++i) { + if (count[i]) { + if (i == EIGHTTAP_REGULAR) *interp_filter = i; + break; + } + } + } + } +} - // Update some stats from cyclic refresh, and check if we should not update - // golden reference, for 1 pass CBR. - if (cpi->oxcf.aq_mode == CYCLIC_REFRESH_AQ && cm->frame_type != KEY_FRAME && - (cpi->oxcf.pass == 0 && cpi->oxcf.rc_mode == AOM_CBR)) - av1_cyclic_refresh_check_golden_update(cpi); +static void finalize_encoded_frame(AV1_COMP *const cpi) { + AV1_COMMON *const cm = &cpi->common; + CurrentFrame *const current_frame = &cm->current_frame; - // Update the skip mb flag probabilities based on the distribution - // seen in the last encoder iteration. - // update_base_skip_probs(cpi); - aom_clear_system_state(); - return AOM_CODEC_OK; + // This bitfield indicates which reference frame slots will be overwritten by + // the current frame + current_frame->refresh_frame_flags = get_refresh_frame_flags(cpi); + + if (!cm->seq_params.reduced_still_picture_hdr && + encode_show_existing_frame(cm)) { + RefCntBuffer *const frame_to_show = + cm->ref_frame_map[cpi->existing_fb_idx_to_show]; + + if (frame_to_show == NULL) { + aom_internal_error(&cm->error, AOM_CODEC_UNSUP_BITSTREAM, + "Buffer does not contain a reconstructed frame"); + } + assert(frame_to_show->ref_count > 0); + assign_frame_buffer_p(&cm->cur_frame, frame_to_show); + } + + if (!encode_show_existing_frame(cm) && + cm->seq_params.film_grain_params_present && + (cm->show_frame || cm->showable_frame)) { + // Copy the current frame's film grain params to the its corresponding + // RefCntBuffer slot. + cm->cur_frame->film_grain_params = cm->film_grain_params; + + // We must update the parameters if this is not an INTER_FRAME + if (current_frame->frame_type != INTER_FRAME) + cm->cur_frame->film_grain_params.update_parameters = 1; + + // Iterate the random seed for the next frame. + cm->film_grain_params.random_seed += 3381; + if (cm->film_grain_params.random_seed == 0) + cm->film_grain_params.random_seed = 7391; + } + + // Initialise all tiles' contexts from the global frame context + for (int tile_col = 0; tile_col < cm->tile_cols; tile_col++) { + for (int tile_row = 0; tile_row < cm->tile_rows; tile_row++) { + const int tile_idx = tile_row * cm->tile_cols + tile_col; + cpi->tile_data[tile_idx].tctx = *cm->fc; + } + } + + fix_interp_filter(&cm->interp_filter, cpi->td.counts); +} + +// Called after encode_with_recode_loop() has just encoded a frame and packed +// its bitstream. This function works out whether we under- or over-shot +// our bitrate target and adjusts q as appropriate. Also decides whether +// or not we should do another recode loop, indicated by *loop +static void recode_loop_update_q(AV1_COMP *const cpi, int *const loop, + int *const q, int *const q_low, + int *const q_high, const int top_index, + const int bottom_index, + int *const undershoot_seen, + int *const overshoot_seen, + const int loop_at_this_size) { + AV1_COMMON *const cm = &cpi->common; + RATE_CONTROL *const rc = &cpi->rc; + + int frame_over_shoot_limit = 0, frame_under_shoot_limit = 0; + av1_rc_compute_frame_size_bounds(cpi, rc->this_frame_target, + &frame_under_shoot_limit, + &frame_over_shoot_limit); + if (frame_over_shoot_limit == 0) frame_over_shoot_limit = 1; + + if ((cm->current_frame.frame_type == KEY_FRAME) && + rc->this_key_frame_forced && + (rc->projected_frame_size < rc->max_frame_bandwidth)) { + int last_q = *q; + int64_t kf_err; + + int64_t high_err_target = cpi->ambient_err; + int64_t low_err_target = cpi->ambient_err >> 1; + + if (cm->seq_params.use_highbitdepth) { + kf_err = aom_highbd_get_y_sse(cpi->source, &cm->cur_frame->buf); + } else { + kf_err = aom_get_y_sse(cpi->source, &cm->cur_frame->buf); + } + // Prevent possible divide by zero error below for perfect KF + kf_err += !kf_err; + + // The key frame is not good enough or we can afford + // to make it better without undue risk of popping. + if ((kf_err > high_err_target && + rc->projected_frame_size <= frame_over_shoot_limit) || + (kf_err > low_err_target && + rc->projected_frame_size <= frame_under_shoot_limit)) { + // Lower q_high + *q_high = *q > *q_low ? *q - 1 : *q_low; + + // Adjust Q + *q = (int)((*q * high_err_target) / kf_err); + *q = AOMMIN(*q, (*q_high + *q_low) >> 1); + } else if (kf_err < low_err_target && + rc->projected_frame_size >= frame_under_shoot_limit) { + // The key frame is much better than the previous frame + // Raise q_low + *q_low = *q < *q_high ? *q + 1 : *q_high; + + // Adjust Q + *q = (int)((*q * low_err_target) / kf_err); + *q = AOMMIN(*q, (*q_high + *q_low + 1) >> 1); + } + + // Clamp Q to upper and lower limits: + *q = clamp(*q, *q_low, *q_high); + + *loop = *q != last_q; + } else if (recode_loop_test(cpi, frame_over_shoot_limit, + frame_under_shoot_limit, *q, + AOMMAX(*q_high, top_index), bottom_index)) { + // Is the projected frame size out of range and are we allowed + // to attempt to recode. + int last_q = *q; + int retries = 0; + + // Frame size out of permitted range: + // Update correction factor & compute new Q to try... + // Frame is too large + if (rc->projected_frame_size > rc->this_frame_target) { + // Special case if the projected size is > the max allowed. + if (rc->projected_frame_size >= rc->max_frame_bandwidth) + *q_high = rc->worst_quality; + + // Raise Qlow as to at least the current value + *q_low = *q < *q_high ? *q + 1 : *q_high; + + if (*undershoot_seen || loop_at_this_size > 1) { + // Update rate_correction_factor unless + av1_rc_update_rate_correction_factors(cpi, cm->width, cm->height); + + *q = (*q_high + *q_low + 1) / 2; + } else { + // Update rate_correction_factor unless + av1_rc_update_rate_correction_factors(cpi, cm->width, cm->height); + + *q = av1_rc_regulate_q(cpi, rc->this_frame_target, bottom_index, + AOMMAX(*q_high, top_index), cm->width, + cm->height); + + while (*q < *q_low && retries < 10) { + av1_rc_update_rate_correction_factors(cpi, cm->width, cm->height); + *q = av1_rc_regulate_q(cpi, rc->this_frame_target, bottom_index, + AOMMAX(*q_high, top_index), cm->width, + cm->height); + retries++; + } + } + + *overshoot_seen = 1; + } else { + // Frame is too small + *q_high = *q > *q_low ? *q - 1 : *q_low; + + if (*overshoot_seen || loop_at_this_size > 1) { + av1_rc_update_rate_correction_factors(cpi, cm->width, cm->height); + *q = (*q_high + *q_low) / 2; + } else { + av1_rc_update_rate_correction_factors(cpi, cm->width, cm->height); + *q = av1_rc_regulate_q(cpi, rc->this_frame_target, bottom_index, + top_index, cm->width, cm->height); + // Special case reset for qlow for constrained quality. + // This should only trigger where there is very substantial + // undershoot on a frame and the auto cq level is above + // the user passsed in value. + if (cpi->oxcf.rc_mode == AOM_CQ && *q < *q_low) { + *q_low = *q; + } + + while (*q > *q_high && retries < 10) { + av1_rc_update_rate_correction_factors(cpi, cm->width, cm->height); + *q = av1_rc_regulate_q(cpi, rc->this_frame_target, bottom_index, + top_index, cm->width, cm->height); + retries++; + } + } + + *undershoot_seen = 1; + } + + // Clamp Q to upper and lower limits: + *q = clamp(*q, *q_low, *q_high); + + *loop = (*q != last_q); + } else { + *loop = 0; + } } static int encode_with_recode_loop(AV1_COMP *cpi, size_t *size, uint8_t *dest) { AV1_COMMON *const cm = &cpi->common; RATE_CONTROL *const rc = &cpi->rc; - int bottom_index, top_index; - int loop_count = 0; - int loop_at_this_size = 0; - int loop = 0; - int overshoot_seen = 0; - int undershoot_seen = 0; - int frame_over_shoot_limit; - int frame_under_shoot_limit; - int q = 0, q_low = 0, q_high = 0; + const int allow_recode = cpi->sf.recode_loop != DISALLOW_RECODE; set_size_independent_vars(cpi); cpi->source->buf_8bit_valid = 0; - aom_clear_system_state(); - setup_frame_size(cpi); - set_size_dependent_vars(cpi, &q, &bottom_index, &top_index); + av1_setup_frame_size(cpi); + int top_index = 0, bottom_index = 0; + int q = 0, q_low = 0, q_high = 0; + set_size_dependent_vars(cpi, &q, &bottom_index, &top_index); + q_low = bottom_index; + q_high = top_index; + + // Loop variables + int loop_count = 0; + int loop_at_this_size = 0; + int loop = 0; + int overshoot_seen = 0; + int undershoot_seen = 0; do { aom_clear_system_state(); - if (loop_count == 0) { - // TODO(agrange) Scale cpi->max_mv_magnitude if frame-size has changed. - set_mv_search_params(cpi); - - // Reset the loop state for new frame size. - overshoot_seen = 0; - undershoot_seen = 0; - - q_low = bottom_index; - q_high = top_index; - - loop_at_this_size = 0; - - // Decide frame size bounds first time through. - av1_rc_compute_frame_size_bounds(cpi, rc->this_frame_target, - &frame_under_shoot_limit, - &frame_over_shoot_limit); - } - // if frame was scaled calculate global_motion_search again if already // done - if (loop_count > 0 && cpi->source && cpi->global_motion_search_done) + if (loop_count > 0 && cpi->source && cpi->global_motion_search_done) { if (cpi->source->y_crop_width != cm->width || - cpi->source->y_crop_height != cm->height) + cpi->source->y_crop_height != cm->height) { cpi->global_motion_search_done = 0; + } + } cpi->source = av1_scale_if_required(cm, cpi->unscaled_source, &cpi->scaled_source); - if (cpi->unscaled_last_source != NULL) + if (cpi->unscaled_last_source != NULL) { cpi->last_source = av1_scale_if_required(cm, cpi->unscaled_last_source, &cpi->scaled_last_source); + } - if (frame_is_intra_only(cm) == 0) { + if (!frame_is_intra_only(cm)) { if (loop_count > 0) { release_scaled_references(cpi); } scale_references(cpi); } av1_set_quantizer(cm, q); - // printf("Frame %d/%d: q = %d, frame_type = %d\n", cm->current_video_frame, - // cm->show_frame, q, cm->frame_type); + av1_init_quantizer(cpi); + // printf("Frame %d/%d: q = %d, frame_type = %d superres_denom = %d\n", + // cm->current_frame.frame_number, cm->show_frame, q, + // cm->current_frame.frame_type, cm->superres_scale_denominator); - if (loop_count == 0) setup_frame(cpi); - - // Base q-index may have changed, so we need to assign proper default coef - // probs before every iteration. - if (cm->primary_ref_frame == PRIMARY_REF_NONE || - cm->frame_refs[cm->primary_ref_frame].idx < 0) { + if (loop_count == 0) { + setup_frame(cpi); + } else if (get_primary_ref_frame_buf(cm) == NULL) { + // Base q-index may have changed, so we need to assign proper default coef + // probs before every iteration. av1_default_coef_probs(cm); av1_setup_frame_contexts(cm); } - // Variance adaptive and in frame q adjustment experiments are mutually - // exclusive. if (cpi->oxcf.aq_mode == VARIANCE_AQ) { av1_vaq_frame_setup(cpi); } else if (cpi->oxcf.aq_mode == COMPLEXITY_AQ) { av1_setup_in_frame_q_adj(cpi); + } else if (cpi->oxcf.aq_mode == CYCLIC_REFRESH_AQ && !allow_recode) { + suppress_active_map(cpi); + av1_cyclic_refresh_setup(cpi); + apply_active_map(cpi); } + if (cm->seg.enabled) { if (!cm->seg.update_data && cm->prev_frame) { segfeatures_copy(&cm->seg, &cm->prev_frame->seg); @@ -4438,13 +4740,18 @@ static int encode_with_recode_loop(AV1_COMP *cpi, size_t *size, uint8_t *dest) { } segfeatures_copy(&cm->cur_frame->seg, &cm->seg); + if (allow_recode) save_coding_context(cpi); + // transform / motion compensation build reconstruction frame - save_coding_context(cpi); av1_encode_frame(cpi); - // Update the skip mb flag probabilities based on the distribution - // seen in the last encoder iteration. - // update_base_skip_probs(cpi); + // Update some stats from cyclic refresh, and check if we should not update + // golden reference, for 1 pass CBR. + if (cpi->oxcf.aq_mode == CYCLIC_REFRESH_AQ && + cm->current_frame.frame_type != KEY_FRAME && + (cpi->oxcf.pass == 0 && cpi->oxcf.rc_mode == AOM_CBR)) { + av1_cyclic_refresh_check_golden_update(cpi); + } aom_clear_system_state(); @@ -4454,140 +4761,20 @@ static int encode_with_recode_loop(AV1_COMP *cpi, size_t *size, uint8_t *dest) { if (cpi->sf.recode_loop >= ALLOW_RECODE_KFARFGF) { restore_coding_context(cpi); - if (av1_pack_bitstream(cpi, dest, size) != AOM_CODEC_OK) + finalize_encoded_frame(cpi); + int largest_tile_id = 0; // Output from bitstream: unused here + if (av1_pack_bitstream(cpi, dest, size, &largest_tile_id) != AOM_CODEC_OK) return AOM_CODEC_ERROR; rc->projected_frame_size = (int)(*size) << 3; restore_coding_context(cpi); - - if (frame_over_shoot_limit == 0) frame_over_shoot_limit = 1; } - if (cpi->oxcf.rc_mode == AOM_Q) { - loop = 0; - } else { - if ((cm->frame_type == KEY_FRAME) && rc->this_key_frame_forced && - (rc->projected_frame_size < rc->max_frame_bandwidth)) { - int last_q = q; - int64_t kf_err; - - int64_t high_err_target = cpi->ambient_err; - int64_t low_err_target = cpi->ambient_err >> 1; - - if (cm->seq_params.use_highbitdepth) { - kf_err = aom_highbd_get_y_sse(cpi->source, get_frame_new_buffer(cm)); - } else { - kf_err = aom_get_y_sse(cpi->source, get_frame_new_buffer(cm)); - } - // Prevent possible divide by zero error below for perfect KF - kf_err += !kf_err; - - // The key frame is not good enough or we can afford - // to make it better without undue risk of popping. - if ((kf_err > high_err_target && - rc->projected_frame_size <= frame_over_shoot_limit) || - (kf_err > low_err_target && - rc->projected_frame_size <= frame_under_shoot_limit)) { - // Lower q_high - q_high = q > q_low ? q - 1 : q_low; - - // Adjust Q - q = (int)((q * high_err_target) / kf_err); - q = AOMMIN(q, (q_high + q_low) >> 1); - } else if (kf_err < low_err_target && - rc->projected_frame_size >= frame_under_shoot_limit) { - // The key frame is much better than the previous frame - // Raise q_low - q_low = q < q_high ? q + 1 : q_high; - - // Adjust Q - q = (int)((q * low_err_target) / kf_err); - q = AOMMIN(q, (q_high + q_low + 1) >> 1); - } - - // Clamp Q to upper and lower limits: - q = clamp(q, q_low, q_high); - - loop = q != last_q; - } else if (recode_loop_test(cpi, frame_over_shoot_limit, - frame_under_shoot_limit, q, - AOMMAX(q_high, top_index), bottom_index)) { - // Is the projected frame size out of range and are we allowed - // to attempt to recode. - int last_q = q; - int retries = 0; - - // Frame size out of permitted range: - // Update correction factor & compute new Q to try... - // Frame is too large - if (rc->projected_frame_size > rc->this_frame_target) { - // Special case if the projected size is > the max allowed. - if (rc->projected_frame_size >= rc->max_frame_bandwidth) - q_high = rc->worst_quality; - - // Raise Qlow as to at least the current value - q_low = q < q_high ? q + 1 : q_high; - - if (undershoot_seen || loop_at_this_size > 1) { - // Update rate_correction_factor unless - av1_rc_update_rate_correction_factors(cpi, cm->width, cm->height); - - q = (q_high + q_low + 1) / 2; - } else { - // Update rate_correction_factor unless - av1_rc_update_rate_correction_factors(cpi, cm->width, cm->height); - - q = av1_rc_regulate_q(cpi, rc->this_frame_target, bottom_index, - AOMMAX(q_high, top_index), cm->width, - cm->height); - - while (q < q_low && retries < 10) { - av1_rc_update_rate_correction_factors(cpi, cm->width, cm->height); - q = av1_rc_regulate_q(cpi, rc->this_frame_target, bottom_index, - AOMMAX(q_high, top_index), cm->width, - cm->height); - retries++; - } - } - - overshoot_seen = 1; - } else { - // Frame is too small - q_high = q > q_low ? q - 1 : q_low; - - if (overshoot_seen || loop_at_this_size > 1) { - av1_rc_update_rate_correction_factors(cpi, cm->width, cm->height); - q = (q_high + q_low) / 2; - } else { - av1_rc_update_rate_correction_factors(cpi, cm->width, cm->height); - q = av1_rc_regulate_q(cpi, rc->this_frame_target, bottom_index, - top_index, cm->width, cm->height); - // Special case reset for qlow for constrained quality. - // This should only trigger where there is very substantial - // undershoot on a frame and the auto cq level is above - // the user passsed in value. - if (cpi->oxcf.rc_mode == AOM_CQ && q < q_low) { - q_low = q; - } - - while (q > q_high && retries < 10) { - av1_rc_update_rate_correction_factors(cpi, cm->width, cm->height); - q = av1_rc_regulate_q(cpi, rc->this_frame_target, bottom_index, - top_index, cm->width, cm->height); - retries++; - } - } - - undershoot_seen = 1; - } - - // Clamp Q to upper and lower limits: - q = clamp(q, q_low, q_high); - - loop = (q != last_q); - } else { - loop = 0; - } + if (allow_recode && cpi->oxcf.rc_mode != AOM_Q) { + // Update q and decide whether to do a recode loop + recode_loop_update_q(cpi, &loop, &q, &q_low, &q_high, top_index, + bottom_index, &undershoot_seen, &overshoot_seen, + loop_at_this_size); } // Special case for overlay frame. @@ -4595,8 +4782,9 @@ static int encode_with_recode_loop(AV1_COMP *cpi, size_t *size, uint8_t *dest) { rc->projected_frame_size < rc->max_frame_bandwidth) loop = 0; - if (!cpi->sf.gm_disable_recode) { - if (recode_loop_test_global_motion(cpi)) loop = 1; + if (allow_recode && !cpi->sf.gm_disable_recode && + recode_loop_test_global_motion(cpi)) { + loop = 1; } if (loop) { @@ -4612,128 +4800,17 @@ static int encode_with_recode_loop(AV1_COMP *cpi, size_t *size, uint8_t *dest) { return AOM_CODEC_OK; } -static int get_ref_frame_flags(const AV1_COMP *cpi) { - const int *const map = cpi->common.ref_frame_map; - - // No.1 Priority: LAST_FRAME - const int last2_is_last = map[cpi->ref_fb_idx[1]] == map[cpi->ref_fb_idx[0]]; - const int last3_is_last = map[cpi->ref_fb_idx[2]] == map[cpi->ref_fb_idx[0]]; - const int gld_is_last = - map[cpi->ref_fb_idx[GOLDEN_FRAME - 1]] == map[cpi->ref_fb_idx[0]]; - const int bwd_is_last = - map[cpi->ref_fb_idx[BWDREF_FRAME - 1]] == map[cpi->ref_fb_idx[0]]; - const int alt2_is_last = - map[cpi->ref_fb_idx[ALTREF2_FRAME - 1]] == map[cpi->ref_fb_idx[0]]; - const int alt_is_last = - map[cpi->ref_fb_idx[ALTREF_FRAME - 1]] == map[cpi->ref_fb_idx[0]]; - - // No.2 Priority: ALTREF_FRAME - const int last2_is_alt = - map[cpi->ref_fb_idx[1]] == map[cpi->ref_fb_idx[ALTREF_FRAME - 1]]; - const int last3_is_alt = - map[cpi->ref_fb_idx[2]] == map[cpi->ref_fb_idx[ALTREF_FRAME - 1]]; - const int gld_is_alt = map[cpi->ref_fb_idx[GOLDEN_FRAME - 1]] == - map[cpi->ref_fb_idx[ALTREF_FRAME - 1]]; - const int bwd_is_alt = map[cpi->ref_fb_idx[BWDREF_FRAME - 1]] == - map[cpi->ref_fb_idx[ALTREF_FRAME - 1]]; - const int alt2_is_alt = map[cpi->ref_fb_idx[ALTREF2_FRAME - 1]] == - map[cpi->ref_fb_idx[ALTREF_FRAME - 1]]; - - // No.3 Priority: LAST2_FRAME - const int last3_is_last2 = map[cpi->ref_fb_idx[2]] == map[cpi->ref_fb_idx[1]]; - const int gld_is_last2 = - map[cpi->ref_fb_idx[GOLDEN_FRAME - 1]] == map[cpi->ref_fb_idx[1]]; - const int bwd_is_last2 = - map[cpi->ref_fb_idx[BWDREF_FRAME - 1]] == map[cpi->ref_fb_idx[1]]; - const int alt2_is_last2 = - map[cpi->ref_fb_idx[ALTREF2_FRAME - 1]] == map[cpi->ref_fb_idx[1]]; - - // No.4 Priority: LAST3_FRAME - const int gld_is_last3 = - map[cpi->ref_fb_idx[GOLDEN_FRAME - 1]] == map[cpi->ref_fb_idx[2]]; - const int bwd_is_last3 = - map[cpi->ref_fb_idx[BWDREF_FRAME - 1]] == map[cpi->ref_fb_idx[2]]; - const int alt2_is_last3 = - map[cpi->ref_fb_idx[ALTREF2_FRAME - 1]] == map[cpi->ref_fb_idx[2]]; - - // No.5 Priority: GOLDEN_FRAME - const int bwd_is_gld = map[cpi->ref_fb_idx[BWDREF_FRAME - 1]] == - map[cpi->ref_fb_idx[GOLDEN_FRAME - 1]]; - const int alt2_is_gld = map[cpi->ref_fb_idx[ALTREF2_FRAME - 1]] == - map[cpi->ref_fb_idx[GOLDEN_FRAME - 1]]; - - // No.6 Priority: BWDREF_FRAME - const int alt2_is_bwd = map[cpi->ref_fb_idx[ALTREF2_FRAME - 1]] == - map[cpi->ref_fb_idx[BWDREF_FRAME - 1]]; - - // No.7 Priority: ALTREF2_FRAME - - // After av1_apply_encoding_flags() is called, cpi->ref_frame_flags might be - // adjusted according to external encoder flags. - int flags = cpi->ext_ref_frame_flags; - - if (cpi->rc.frames_till_gf_update_due == INT_MAX) flags &= ~AOM_GOLD_FLAG; - - if (alt_is_last) flags &= ~AOM_ALT_FLAG; - - if (last2_is_last || last2_is_alt) flags &= ~AOM_LAST2_FLAG; - - if (last3_is_last || last3_is_alt || last3_is_last2) flags &= ~AOM_LAST3_FLAG; - - if (gld_is_last || gld_is_alt || gld_is_last2 || gld_is_last3) - flags &= ~AOM_GOLD_FLAG; - - if ((bwd_is_last || bwd_is_alt || bwd_is_last2 || bwd_is_last3 || - bwd_is_gld) && - (flags & AOM_BWD_FLAG)) - flags &= ~AOM_BWD_FLAG; - - if ((alt2_is_last || alt2_is_alt || alt2_is_last2 || alt2_is_last3 || - alt2_is_gld || alt2_is_bwd) && - (flags & AOM_ALT2_FLAG)) - flags &= ~AOM_ALT2_FLAG; - - return flags; -} - -static void set_ext_overrides(AV1_COMP *cpi) { - // Overrides the defaults with the externally supplied values with - // av1_update_reference() and av1_update_entropy() calls - // Note: The overrides are valid only for the next frame passed - // to encode_frame_to_data_rate() function - if (cpi->ext_use_s_frame) cpi->common.frame_type = S_FRAME; - cpi->common.force_primary_ref_none = cpi->ext_use_primary_ref_none; - - if (cpi->ext_refresh_frame_context_pending) { - cpi->common.refresh_frame_context = cpi->ext_refresh_frame_context; - cpi->ext_refresh_frame_context_pending = 0; - } - if (cpi->ext_refresh_frame_flags_pending) { - cpi->refresh_last_frame = cpi->ext_refresh_last_frame; - cpi->refresh_golden_frame = cpi->ext_refresh_golden_frame; - cpi->refresh_alt_ref_frame = cpi->ext_refresh_alt_ref_frame; - cpi->refresh_bwd_ref_frame = cpi->ext_refresh_bwd_ref_frame; - cpi->refresh_alt2_ref_frame = cpi->ext_refresh_alt2_ref_frame; - cpi->ext_refresh_frame_flags_pending = 0; - } - cpi->common.allow_ref_frame_mvs = cpi->ext_use_ref_frame_mvs; - // A keyframe is already error resilient and keyframes with - // error_resilient_mode interferes with the use of show_existing_frame - // when forward reference keyframes are enabled. - cpi->common.error_resilient_mode = - cpi->ext_use_error_resilient && cpi->common.frame_type != KEY_FRAME; -} - #define DUMP_RECON_FRAMES 0 #if DUMP_RECON_FRAMES == 1 // NOTE(zoeliu): For debug - Output the filtered reconstructed video. static void dump_filtered_recon_frames(AV1_COMP *cpi) { AV1_COMMON *const cm = &cpi->common; - const YV12_BUFFER_CONFIG *recon_buf = cm->frame_to_show; + const CurrentFrame *const current_frame = &cm->current_frame; + const YV12_BUFFER_CONFIG *recon_buf = &cm->cur_frame->buf; if (recon_buf == NULL) { - printf("Frame %d is not ready.\n", cm->current_video_frame); + printf("Frame %d is not ready.\n", current_frame->frame_number); return; } @@ -4749,25 +4826,21 @@ static void dump_filtered_recon_frames(AV1_COMP *cpi) { "\n***Frame=%d (frame_offset=%d, show_frame=%d, " "show_existing_frame=%d) " "[LAST LAST2 LAST3 GOLDEN BWD ALT2 ALT]=[", - cm->current_video_frame, cm->frame_offset, cm->show_frame, + current_frame->frame_number, current_frame->order_hint, cm->show_frame, cm->show_existing_frame); for (int ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) { - const int buf_idx = cm->frame_refs[ref_frame - LAST_FRAME].idx; - const int ref_offset = - (buf_idx >= 0) - ? (int)cm->buffer_pool->frame_bufs[buf_idx].cur_frame_offset - : -1; - printf( - " %d(%c-%d-%4.2f)", ref_offset, - (cpi->ref_frame_flags & flag_list[ref_frame]) ? 'Y' : 'N', - (buf_idx >= 0) ? (int)cpi->frame_rf_level[buf_idx] : -1, - (buf_idx >= 0) ? rate_factor_deltas[cpi->frame_rf_level[buf_idx]] : -1); + const RefCntBuffer *const buf = get_ref_frame_buf(cm, ref_frame); + const int ref_offset = buf != NULL ? (int)buf->order_hint : -1; + printf(" %d(%c-%d-%4.2f)", ref_offset, + (cpi->ref_frame_flags & flag_list[ref_frame]) ? 'Y' : 'N', + buf ? (int)buf->frame_rf_level : -1, + buf ? rate_factor_deltas[buf->frame_rf_level] : -1); } printf(" ]\n"); if (!cm->show_frame) { printf("Frame %d is a no show frame, so no image dump.\n", - cm->current_video_frame); + current_frame->frame_number); return; } @@ -4775,7 +4848,7 @@ static void dump_filtered_recon_frames(AV1_COMP *cpi) { char file_name[256] = "/tmp/enc_filtered_recon.yuv"; FILE *f_recon = NULL; - if (cm->current_video_frame == 0) { + if (current_frame->frame_number == 0) { if ((f_recon = fopen(file_name, "wb")) == NULL) { printf("Unable to open file %s to write.\n", file_name); return; @@ -4791,9 +4864,9 @@ static void dump_filtered_recon_frames(AV1_COMP *cpi) { "show_frame=%d, show_existing_frame=%d, source_alt_ref_active=%d, " "refresh_alt_ref_frame=%d, rf_level=%d, " "y_stride=%4d, uv_stride=%4d, cm->width=%4d, cm->height=%4d\n\n", - cm->current_video_frame, cpi->twopass.gf_group.index, + current_frame->frame_number, cpi->twopass.gf_group.index, cpi->twopass.gf_group.update_type[cpi->twopass.gf_group.index], - cm->frame_offset, cm->show_frame, cm->show_existing_frame, + current_frame->order_hint, cm->show_frame, cm->show_existing_frame, cpi->rc.source_alt_ref_active, cpi->refresh_alt_ref_frame, cpi->twopass.gf_group.rf_level[cpi->twopass.gf_group.index], recon_buf->y_stride, recon_buf->uv_stride, cm->width, cm->height); @@ -4801,13 +4874,7 @@ static void dump_filtered_recon_frames(AV1_COMP *cpi) { int ref_frame; printf("get_ref_frame_map_idx: ["); for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) - printf(" %d", get_ref_frame_map_idx(cpi, ref_frame)); - printf(" ]\n"); - printf("cm->new_fb_idx = %d\n", cm->new_fb_idx); - printf("cm->ref_frame_map = ["); - for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) { - printf(" %d", cm->ref_frame_map[ref_frame - LAST_FRAME]); - } + printf(" %d", get_ref_frame_map_idx(cm, ref_frame)); printf(" ]\n"); #endif // 0 @@ -4831,811 +4898,36 @@ static void dump_filtered_recon_frames(AV1_COMP *cpi) { } #endif // DUMP_RECON_FRAMES -static INLINE int is_frame_droppable(AV1_COMP *cpi) { - return !(cpi->refresh_alt_ref_frame || cpi->refresh_alt2_ref_frame || - cpi->refresh_bwd_ref_frame || cpi->refresh_golden_frame || - cpi->refresh_last_frame); +static int setup_interp_filter_search_mask(AV1_COMP *cpi) { + InterpFilters ifilter; + int ref_total[REF_FRAMES] = { 0 }; + MV_REFERENCE_FRAME ref; + int mask = 0; + int arf_idx = ALTREF_FRAME; + if (cpi->common.last_frame_type == KEY_FRAME || cpi->refresh_alt_ref_frame) + return mask; + for (ref = LAST_FRAME; ref <= ALTREF_FRAME; ++ref) + for (ifilter = EIGHTTAP_REGULAR; ifilter <= MULTITAP_SHARP; ++ifilter) + ref_total[ref] += cpi->interp_filter_selected[ref][ifilter]; + + for (ifilter = EIGHTTAP_REGULAR; ifilter <= MULTITAP_SHARP; ++ifilter) { + if ((ref_total[LAST_FRAME] && + cpi->interp_filter_selected[LAST_FRAME][ifilter] * 30 <= + ref_total[LAST_FRAME]) && + (((cpi->interp_filter_selected[LAST2_FRAME][ifilter] * 20) + + (cpi->interp_filter_selected[LAST3_FRAME][ifilter] * 20) + + (cpi->interp_filter_selected[GOLDEN_FRAME][ifilter] * 20) + + (cpi->interp_filter_selected[BWDREF_FRAME][ifilter] * 10) + + (cpi->interp_filter_selected[ALTREF2_FRAME][ifilter] * 10) + + (cpi->interp_filter_selected[arf_idx][ifilter] * 10)) < + (ref_total[LAST2_FRAME] + ref_total[LAST3_FRAME] + + ref_total[GOLDEN_FRAME] + ref_total[BWDREF_FRAME] + + ref_total[ALTREF2_FRAME] + ref_total[ALTREF_FRAME]))) + mask |= 1 << ifilter; + } + return mask; } -static int encode_frame_to_data_rate(AV1_COMP *cpi, size_t *size, uint8_t *dest, - int skip_adapt, - unsigned int *frame_flags) { - AV1_COMMON *const cm = &cpi->common; - SequenceHeader *const seq_params = &cm->seq_params; - const AV1EncoderConfig *const oxcf = &cpi->oxcf; - struct segmentation *const seg = &cm->seg; - - set_ext_overrides(cpi); - aom_clear_system_state(); - - // frame type has been decided outside of this function call - cm->cur_frame->intra_only = frame_is_intra_only(cm); - cm->cur_frame->frame_type = cm->frame_type; - - // S_FRAMEs are always error resilient - cm->error_resilient_mode |= frame_is_sframe(cm); - - cm->large_scale_tile = cpi->oxcf.large_scale_tile; - cm->single_tile_decoding = cpi->oxcf.single_tile_decoding; - if (cm->large_scale_tile) seq_params->frame_id_numbers_present_flag = 0; - - cm->allow_ref_frame_mvs &= frame_might_allow_ref_frame_mvs(cm); - // cm->allow_ref_frame_mvs needs to be written into the frame header while - // cm->large_scale_tile is 1, therefore, "cm->large_scale_tile=1" case is - // separated from frame_might_allow_ref_frame_mvs(). - cm->allow_ref_frame_mvs &= !cm->large_scale_tile; - - cm->allow_warped_motion = - cpi->oxcf.allow_warped_motion && frame_might_allow_warped_motion(cm); - - // Reset the frame packet stamp index. - if (cm->frame_type == KEY_FRAME && cm->show_frame) - cm->current_video_frame = 0; - - // NOTE: - // (1) Move the setup of the ref_frame_flags upfront as it would be - // determined by the current frame properties; - // (2) The setup of the ref_frame_flags applies to both - // show_existing_frame's - // and the other cases. - if (cm->current_video_frame > 0) - cpi->ref_frame_flags = get_ref_frame_flags(cpi); - - if (encode_show_existing_frame(cm)) { - // NOTE(zoeliu): In BIDIR_PRED, the existing frame to show is the current - // BWDREF_FRAME in the reference frame buffer. - if (cm->frame_type == KEY_FRAME) { - cm->reset_decoder_state = 1; - } else { - cm->frame_type = INTER_FRAME; - } - cm->show_frame = 1; - cpi->frame_flags = *frame_flags; - - restore_coding_context(cpi); - - // Build the bitstream - if (av1_pack_bitstream(cpi, dest, size) != AOM_CODEC_OK) - return AOM_CODEC_ERROR; - - cpi->seq_params_locked = 1; - - // Set up frame to show to get ready for stats collection. - cm->frame_to_show = get_frame_new_buffer(cm); - - // Update current frame offset. - cm->frame_offset = - cm->buffer_pool->frame_bufs[cm->new_fb_idx].cur_frame_offset; - -#if DUMP_RECON_FRAMES == 1 - // NOTE(zoeliu): For debug - Output the filtered reconstructed video. - dump_filtered_recon_frames(cpi); -#endif // DUMP_RECON_FRAMES - - // Update the LAST_FRAME in the reference frame buffer. - // NOTE: - // (1) For BWDREF_FRAME as the show_existing_frame, the reference frame - // update has been done previously when handling the LAST_BIPRED_FRAME - // right before BWDREF_FRAME (in the display order); - // (2) For INTNL_OVERLAY as the show_existing_frame, the reference frame - // update will be done when the following is called, which will - // exchange - // the virtual indexes between LAST_FRAME and ALTREF2_FRAME, so that - // LAST3 will get retired, LAST2 becomes LAST3, LAST becomes LAST2, - // and - // ALTREF2_FRAME will serve as the new LAST_FRAME. - update_reference_frames(cpi); - - // Update frame flags - cpi->frame_flags &= ~FRAMEFLAGS_GOLDEN; - cpi->frame_flags &= ~FRAMEFLAGS_BWDREF; - cpi->frame_flags &= ~FRAMEFLAGS_ALTREF; - - *frame_flags = cpi->frame_flags & ~FRAMEFLAGS_KEY; - - // Update the frame type - cm->last_frame_type = cm->frame_type; - - // Since we allocate a spot for the OVERLAY frame in the gf group, we need - // to do post-encoding update accordingly. - if (cpi->rc.is_src_frame_alt_ref) { - av1_set_target_rate(cpi, cm->width, cm->height); - av1_rc_postencode_update(cpi, *size); - } - - ++cm->current_video_frame; - - return AOM_CODEC_OK; - } - - // Set default state for segment based loop filter update flags. - cm->lf.mode_ref_delta_update = 0; - - // Set various flags etc to special state if it is a key frame. - if (frame_is_intra_only(cm) || frame_is_sframe(cm)) { - // Reset the loop filter deltas and segmentation map. - av1_reset_segment_features(cm); - - // If segmentation is enabled force a map update for key frames. - if (seg->enabled) { - seg->update_map = 1; - seg->update_data = 1; - } - - // The alternate reference frame cannot be active for a key frame. - cpi->rc.source_alt_ref_active = 0; - } - if (cpi->oxcf.mtu == 0) { - cm->num_tg = cpi->oxcf.num_tile_groups; - } else { - // Use a default value for the purposes of weighting costs in probability - // updates - cm->num_tg = DEFAULT_MAX_NUM_TG; - } - - // For 1 pass CBR, check if we are dropping this frame. - // Never drop on key frame. - if (oxcf->pass == 0 && oxcf->rc_mode == AOM_CBR && - cm->frame_type != KEY_FRAME) { - if (av1_rc_drop_frame(cpi)) { - av1_rc_postencode_update_drop_frame(cpi); - return AOM_CODEC_OK; - } - } - - aom_clear_system_state(); - -#if CONFIG_INTERNAL_STATS - memset(cpi->mode_chosen_counts, 0, - MAX_MODES * sizeof(*cpi->mode_chosen_counts)); -#endif - - if (seq_params->frame_id_numbers_present_flag) { - /* Non-normative definition of current_frame_id ("frame counter" with - * wraparound) */ - const int frame_id_length = FRAME_ID_LENGTH; - if (cm->current_frame_id == -1) { - int lsb, msb; - /* quasi-random initialization of current_frame_id for a key frame */ - if (cpi->source->flags & YV12_FLAG_HIGHBITDEPTH) { - lsb = CONVERT_TO_SHORTPTR(cpi->source->y_buffer)[0] & 0xff; - msb = CONVERT_TO_SHORTPTR(cpi->source->y_buffer)[1] & 0xff; - } else { - lsb = cpi->source->y_buffer[0] & 0xff; - msb = cpi->source->y_buffer[1] & 0xff; - } - cm->current_frame_id = ((msb << 8) + lsb) % (1 << frame_id_length); - - // S_frame is meant for stitching different streams of different - // resolutions together, so current_frame_id must be the - // same across different streams of the same content current_frame_id - // should be the same and not random. 0x37 is a chosen number as start - // point - if (cpi->oxcf.sframe_enabled) cm->current_frame_id = 0x37; - } else { - cm->current_frame_id = - (cm->current_frame_id + 1 + (1 << frame_id_length)) % - (1 << frame_id_length); - } - } - - switch (cpi->oxcf.cdf_update_mode) { - case 0: // No CDF update for any frames(4~6% compression loss). - cm->disable_cdf_update = 1; - break; - case 1: // Enable CDF update for all frames. - cm->disable_cdf_update = 0; - break; - case 2: - // Strategically determine at which frames to do CDF update. - // Currently only enable CDF update for all-intra and no-show frames(1.5% - // compression loss). - // TODO(huisu@google.com): design schemes for various trade-offs between - // compression quality and decoding speed. - cm->disable_cdf_update = - (frame_is_intra_only(cm) || !cm->show_frame) ? 0 : 1; - break; - } - cm->timing_info_present &= !seq_params->reduced_still_picture_hdr; - - if (cpi->sf.recode_loop == DISALLOW_RECODE) { - if (encode_without_recode_loop(cpi) != AOM_CODEC_OK) return AOM_CODEC_ERROR; - } else { - if (encode_with_recode_loop(cpi, size, dest) != AOM_CODEC_OK) - return AOM_CODEC_ERROR; - } - - cm->last_tile_cols = cm->tile_cols; - cm->last_tile_rows = cm->tile_rows; - -#ifdef OUTPUT_YUV_SKINMAP - if (cpi->common.current_video_frame > 1) { - av1_compute_skin_map(cpi, yuv_skinmap_file); - } -#endif // OUTPUT_YUV_SKINMAP - - // Special case code to reduce pulsing when key frames are forced at a - // fixed interval. Note the reconstruction error if it is the frame before - // the force key frame - if (cpi->rc.next_key_frame_forced && cpi->rc.frames_to_key == 1) { - if (seq_params->use_highbitdepth) { - cpi->ambient_err = - aom_highbd_get_y_sse(cpi->source, get_frame_new_buffer(cm)); - } else { - cpi->ambient_err = aom_get_y_sse(cpi->source, get_frame_new_buffer(cm)); - } - } - - // If the encoder forced a KEY_FRAME decision or if frame is an S_FRAME - if ((cm->frame_type == KEY_FRAME && cm->show_frame) || frame_is_sframe(cm)) { - cpi->refresh_last_frame = 1; - } - - cm->frame_to_show = get_frame_new_buffer(cm); - cm->frame_to_show->color_primaries = seq_params->color_primaries; - cm->frame_to_show->transfer_characteristics = - seq_params->transfer_characteristics; - cm->frame_to_show->matrix_coefficients = seq_params->matrix_coefficients; - cm->frame_to_show->monochrome = seq_params->monochrome; - cm->frame_to_show->chroma_sample_position = - seq_params->chroma_sample_position; - cm->frame_to_show->color_range = seq_params->color_range; - cm->frame_to_show->render_width = cm->render_width; - cm->frame_to_show->render_height = cm->render_height; - - // TODO(zoeliu): For non-ref frames, loop filtering may need to be turned - // off. - - // Pick the loop filter level for the frame. - if (!cm->allow_intrabc) { - loopfilter_frame(cpi, cm); - } else { - cm->lf.filter_level[0] = 0; - cm->lf.filter_level[1] = 0; - cm->cdef_bits = 0; - cm->cdef_strengths[0] = 0; - cm->nb_cdef_strengths = 1; - cm->cdef_uv_strengths[0] = 0; - cm->rst_info[0].frame_restoration_type = RESTORE_NONE; - cm->rst_info[1].frame_restoration_type = RESTORE_NONE; - cm->rst_info[2].frame_restoration_type = RESTORE_NONE; - } - - // TODO(debargha): Fix mv search range on encoder side - // aom_extend_frame_inner_borders(cm->frame_to_show, av1_num_planes(cm)); - aom_extend_frame_borders(cm->frame_to_show, av1_num_planes(cm)); - -#ifdef OUTPUT_YUV_REC - aom_write_one_yuv_frame(cm, cm->frame_to_show); -#endif - - // Build the bitstream - if (av1_pack_bitstream(cpi, dest, size) != AOM_CODEC_OK) - return AOM_CODEC_ERROR; - - cpi->seq_params_locked = 1; - - if (skip_adapt) return AOM_CODEC_OK; - - if (seq_params->frame_id_numbers_present_flag) { - int i; - // Update reference frame id values based on the value of refresh_frame_mask - for (i = 0; i < REF_FRAMES; i++) { - if ((cpi->refresh_frame_mask >> i) & 1) { - cm->ref_frame_id[i] = cm->current_frame_id; - } - } - } - -#if DUMP_RECON_FRAMES == 1 - // NOTE(zoeliu): For debug - Output the filtered reconstructed video. - dump_filtered_recon_frames(cpi); -#endif // DUMP_RECON_FRAMES - - if (cm->seg.enabled) { - if (cm->seg.update_map) { - update_reference_segmentation_map(cpi); - } else if (cm->last_frame_seg_map) { - memcpy(cm->current_frame_seg_map, cm->last_frame_seg_map, - cm->mi_cols * cm->mi_rows * sizeof(uint8_t)); - } - } - - if (frame_is_intra_only(cm) == 0) { - release_scaled_references(cpi); - } - - update_reference_frames(cpi); - -#if CONFIG_ENTROPY_STATS - av1_accumulate_frame_counts(&aggregate_fc, &cpi->counts); -#endif // CONFIG_ENTROPY_STATS - - if (cm->refresh_frame_context == REFRESH_FRAME_CONTEXT_BACKWARD) { - *cm->fc = cpi->tile_data[cm->largest_tile_id].tctx; - av1_reset_cdf_symbol_counters(cm->fc); - } - - if (cpi->refresh_golden_frame == 1) - cpi->frame_flags |= FRAMEFLAGS_GOLDEN; - else - cpi->frame_flags &= ~FRAMEFLAGS_GOLDEN; - - if (cpi->refresh_alt_ref_frame == 1) - cpi->frame_flags |= FRAMEFLAGS_ALTREF; - else - cpi->frame_flags &= ~FRAMEFLAGS_ALTREF; - - if (cpi->refresh_bwd_ref_frame == 1) - cpi->frame_flags |= FRAMEFLAGS_BWDREF; - else - cpi->frame_flags &= ~FRAMEFLAGS_BWDREF; - - cm->last_frame_type = cm->frame_type; - - av1_rc_postencode_update(cpi, *size); - - if (cm->frame_type == KEY_FRAME) { - // Tell the caller that the frame was coded as a key frame - *frame_flags = cpi->frame_flags | FRAMEFLAGS_KEY; - } else { - *frame_flags = cpi->frame_flags & ~FRAMEFLAGS_KEY; - } - - // Clear the one shot update flags for segmentation map and mode/ref loop - // filter deltas. - cm->seg.update_map = 0; - cm->seg.update_data = 0; - cm->lf.mode_ref_delta_update = 0; - - // A droppable frame might not be shown but it always - // takes a space in the gf group. Therefore, even when - // it is not shown, we still need update the count down. - - if (cm->show_frame) { - // TODO(zoeliu): We may only swamp mi and prev_mi for those frames that - // are - // being used as reference. - swap_mi_and_prev_mi(cm); - // Don't increment frame counters if this was an altref buffer - // update not a real frame - - ++cm->current_video_frame; - } - - // NOTE: Shall not refer to any frame not used as reference. - if (cm->is_reference_frame) { - // keep track of the last coded dimensions - cm->last_width = cm->width; - cm->last_height = cm->height; - - // reset to normal state now that we are done. - cm->last_show_frame = cm->show_frame; - } - - return AOM_CODEC_OK; -} - -static INLINE void update_keyframe_counters(AV1_COMP *cpi) { - // TODO(zoeliu): To investigate whether we should treat BWDREF_FRAME - // differently here for rc->avg_frame_bandwidth. - if (cpi->common.show_frame || cpi->rc.is_bwd_ref_frame) { - if (!cpi->common.show_existing_frame || cpi->rc.is_src_frame_alt_ref || - cpi->common.frame_type == KEY_FRAME) { - // If this is a show_existing_frame with a source other than altref, - // or if it is not a displayed forward keyframe, the keyframe update - // counters were incremented when it was originally encoded. - cpi->rc.frames_since_key++; - cpi->rc.frames_to_key--; - } - } -} - -static INLINE void update_frames_till_gf_update(AV1_COMP *cpi) { - // TODO(weitinglin): Updating this counter for is_frame_droppable - // is a work-around to handle the condition when a frame is drop. - // We should fix the cpi->common.show_frame flag - // instead of checking the other condition to update the counter properly. - if (cpi->common.show_frame || is_frame_droppable(cpi)) { - // Decrement count down till next gf - if (cpi->rc.frames_till_gf_update_due > 0) - cpi->rc.frames_till_gf_update_due--; - } -} - -static INLINE void update_twopass_gf_group_index(AV1_COMP *cpi) { - // Increment the gf group index ready for the next frame. If this is - // a show_existing_frame with a source other than altref, or if it is not - // a displayed forward keyframe, the index was incremented when it was - // originally encoded. - if (!cpi->common.show_existing_frame || cpi->rc.is_src_frame_alt_ref || - cpi->common.frame_type == KEY_FRAME) { - ++cpi->twopass.gf_group.index; - } -} - -static void update_rc_counts(AV1_COMP *cpi) { - update_keyframe_counters(cpi); - update_frames_till_gf_update(cpi); - if (cpi->oxcf.pass == 2) update_twopass_gf_group_index(cpi); -} - -static int Pass0Encode(AV1_COMP *cpi, size_t *size, uint8_t *dest, - int skip_adapt, unsigned int *frame_flags) { - if (cpi->oxcf.rc_mode == AOM_CBR) { - av1_rc_get_one_pass_cbr_params(cpi); - } else { - av1_rc_get_one_pass_vbr_params(cpi); - } - if (encode_frame_to_data_rate(cpi, size, dest, skip_adapt, frame_flags) != - AOM_CODEC_OK) { - return AOM_CODEC_ERROR; - } - update_rc_counts(cpi); - check_show_existing_frame(cpi); - return AOM_CODEC_OK; -} - -static int Pass2Encode(AV1_COMP *cpi, size_t *size, uint8_t *dest, - unsigned int *frame_flags) { -#if CONFIG_MISMATCH_DEBUG - mismatch_move_frame_idx_w(); -#endif -#if TXCOEFF_COST_TIMER - AV1_COMMON *cm = &cpi->common; - cm->txcoeff_cost_timer = 0; - cm->txcoeff_cost_count = 0; -#endif - - if (encode_frame_to_data_rate(cpi, size, dest, 0, frame_flags) != - AOM_CODEC_OK) { - return AOM_CODEC_ERROR; - } - -#if TXCOEFF_COST_TIMER - cm->cum_txcoeff_cost_timer += cm->txcoeff_cost_timer; - fprintf(stderr, - "\ntxb coeff cost block number: %ld, frame time: %ld, cum time %ld " - "in us\n", - cm->txcoeff_cost_count, cm->txcoeff_cost_timer, - cm->cum_txcoeff_cost_timer); -#endif - - av1_twopass_postencode_update(cpi); - update_rc_counts(cpi); - check_show_existing_frame(cpi); - return AOM_CODEC_OK; -} - -#if CONFIG_DENOISE -static int apply_denoise_2d(AV1_COMP *cpi, YV12_BUFFER_CONFIG *sd, - int block_size, float noise_level, - int64_t time_stamp, int64_t end_time) { - AV1_COMMON *const cm = &cpi->common; - if (!cpi->denoise_and_model) { - cpi->denoise_and_model = aom_denoise_and_model_alloc( - cm->seq_params.bit_depth, block_size, noise_level); - if (!cpi->denoise_and_model) { - aom_internal_error(&cm->error, AOM_CODEC_MEM_ERROR, - "Error allocating denoise and model"); - return -1; - } - } - if (!cpi->film_grain_table) { - cpi->film_grain_table = aom_malloc(sizeof(*cpi->film_grain_table)); - if (!cpi->film_grain_table) { - aom_internal_error(&cm->error, AOM_CODEC_MEM_ERROR, - "Error allocating grain table"); - return -1; - } - memset(cpi->film_grain_table, 0, sizeof(*cpi->film_grain_table)); - } - if (aom_denoise_and_model_run(cpi->denoise_and_model, sd, - &cm->film_grain_params)) { - if (cm->film_grain_params.apply_grain) { - aom_film_grain_table_append(cpi->film_grain_table, time_stamp, end_time, - &cm->film_grain_params); - } - } - return 0; -} -#endif - -int av1_receive_raw_frame(AV1_COMP *cpi, aom_enc_frame_flags_t frame_flags, - YV12_BUFFER_CONFIG *sd, int64_t time_stamp, - int64_t end_time) { - AV1_COMMON *const cm = &cpi->common; - const SequenceHeader *const seq_params = &cm->seq_params; - struct aom_usec_timer timer; - int res = 0; - const int subsampling_x = sd->subsampling_x; - const int subsampling_y = sd->subsampling_y; - const int use_highbitdepth = (sd->flags & YV12_FLAG_HIGHBITDEPTH) != 0; - - check_initial_width(cpi, use_highbitdepth, subsampling_x, subsampling_y); - - aom_usec_timer_start(&timer); - -#if CONFIG_DENOISE - if (cpi->oxcf.noise_level > 0) - if (apply_denoise_2d(cpi, sd, cpi->oxcf.noise_block_size, - cpi->oxcf.noise_level, time_stamp, end_time) < 0) - res = -1; -#endif // CONFIG_DENOISE - - if (av1_lookahead_push(cpi->lookahead, sd, time_stamp, end_time, - use_highbitdepth, frame_flags)) - res = -1; - aom_usec_timer_mark(&timer); - cpi->time_receive_data += aom_usec_timer_elapsed(&timer); - - if ((seq_params->profile == PROFILE_0) && !seq_params->monochrome && - (subsampling_x != 1 || subsampling_y != 1)) { - aom_internal_error(&cm->error, AOM_CODEC_INVALID_PARAM, - "Non-4:2:0 color format requires profile 1 or 2"); - res = -1; - } - if ((seq_params->profile == PROFILE_1) && - !(subsampling_x == 0 && subsampling_y == 0)) { - aom_internal_error(&cm->error, AOM_CODEC_INVALID_PARAM, - "Profile 1 requires 4:4:4 color format"); - res = -1; - } - if ((seq_params->profile == PROFILE_2) && - (seq_params->bit_depth <= AOM_BITS_10) && - !(subsampling_x == 1 && subsampling_y == 0)) { - aom_internal_error(&cm->error, AOM_CODEC_INVALID_PARAM, - "Profile 2 bit-depth < 10 requires 4:2:2 color format"); - res = -1; - } - - return res; -} - -static int frame_is_reference(const AV1_COMP *cpi) { - const AV1_COMMON *cm = &cpi->common; - - return cm->frame_type == KEY_FRAME || cpi->refresh_last_frame || - cpi->refresh_golden_frame || cpi->refresh_bwd_ref_frame || - cpi->refresh_alt2_ref_frame || cpi->refresh_alt_ref_frame || - !cm->error_resilient_mode || cm->lf.mode_ref_delta_update || - cm->seg.update_map || cm->seg.update_data; -} - -static void adjust_frame_rate(AV1_COMP *cpi, - const struct lookahead_entry *source) { - int64_t this_duration; - int step = 0; - - if (source->ts_start == cpi->first_time_stamp_ever) { - this_duration = source->ts_end - source->ts_start; - step = 1; - } else { - int64_t last_duration = - cpi->last_end_time_stamp_seen - cpi->last_time_stamp_seen; - - this_duration = source->ts_end - cpi->last_end_time_stamp_seen; - - // do a step update if the duration changes by 10% - if (last_duration) - step = (int)((this_duration - last_duration) * 10 / last_duration); - } - - if (this_duration) { - if (step) { - av1_new_framerate(cpi, 10000000.0 / this_duration); - } else { - // Average this frame's rate into the last second's average - // frame rate. If we haven't seen 1 second yet, then average - // over the whole interval seen. - const double interval = AOMMIN( - (double)(source->ts_end - cpi->first_time_stamp_ever), 10000000.0); - double avg_duration = 10000000.0 / cpi->framerate; - avg_duration *= (interval - avg_duration + this_duration); - avg_duration /= interval; - - av1_new_framerate(cpi, 10000000.0 / avg_duration); - } - } - cpi->last_time_stamp_seen = source->ts_start; - cpi->last_end_time_stamp_seen = source->ts_end; -} - -// Returns 0 if this is not an alt ref else the offset of the source frame -// used as the arf midpoint. -static int get_arf_src_index(AV1_COMP *cpi) { - RATE_CONTROL *const rc = &cpi->rc; - int arf_src_index = 0; - if (is_altref_enabled(cpi)) { - if (cpi->oxcf.pass == 2) { - const GF_GROUP *const gf_group = &cpi->twopass.gf_group; - if (gf_group->update_type[gf_group->index] == ARF_UPDATE) { - arf_src_index = gf_group->arf_src_offset[gf_group->index]; - } - } else if (rc->source_alt_ref_pending) { - arf_src_index = rc->frames_till_gf_update_due; - } - } - return arf_src_index; -} - -static int get_brf_src_index(AV1_COMP *cpi) { - int brf_src_index = 0; - const GF_GROUP *const gf_group = &cpi->twopass.gf_group; - - // TODO(zoeliu): We need to add the check on the -bwd_ref command line setup - // flag. - if (gf_group->bidir_pred_enabled[gf_group->index]) { - if (cpi->oxcf.pass == 2) { - if (gf_group->update_type[gf_group->index] == BRF_UPDATE) - brf_src_index = gf_group->brf_src_offset[gf_group->index]; - } else { - // TODO(zoeliu): To re-visit the setup for this scenario - brf_src_index = cpi->rc.bipred_group_interval - 1; - } - } - - return brf_src_index; -} - -// Returns 0 if this is not an alt ref else the offset of the source frame -// used as the arf midpoint. -static int get_arf2_src_index(AV1_COMP *cpi) { - int arf2_src_index = 0; - if (is_altref_enabled(cpi) && cpi->num_extra_arfs) { - if (cpi->oxcf.pass == 2) { - const GF_GROUP *const gf_group = &cpi->twopass.gf_group; - if (gf_group->update_type[gf_group->index] == INTNL_ARF_UPDATE) { - arf2_src_index = gf_group->arf_src_offset[gf_group->index]; - } - } - } - return arf2_src_index; -} - -static void check_src_altref(AV1_COMP *cpi, - const struct lookahead_entry *source) { - RATE_CONTROL *const rc = &cpi->rc; - - // If pass == 2, the parameters set here will be reset in - // av1_rc_get_second_pass_params() - - if (cpi->oxcf.pass == 2) { - const GF_GROUP *const gf_group = &cpi->twopass.gf_group; - rc->is_src_frame_alt_ref = - (gf_group->update_type[gf_group->index] == INTNL_OVERLAY_UPDATE) || - (gf_group->update_type[gf_group->index] == OVERLAY_UPDATE); - rc->is_src_frame_ext_arf = - gf_group->update_type[gf_group->index] == INTNL_OVERLAY_UPDATE; - } else { - rc->is_src_frame_alt_ref = - cpi->alt_ref_source && (source == cpi->alt_ref_source); - } - - if (rc->is_src_frame_alt_ref) { - // Current frame is an ARF overlay frame. - cpi->alt_ref_source = NULL; - - if (rc->is_src_frame_ext_arf && !cpi->common.show_existing_frame) { - // For INTNL_OVERLAY, when show_existing_frame == 0, they do need to - // refresh the LAST_FRAME, i.e. LAST3 gets retired, LAST2 becomes LAST3, - // LAST becomes LAST2, and INTNL_OVERLAY becomes LAST. - cpi->refresh_last_frame = 1; - } else { - // Don't refresh the last buffer for an ARF overlay frame. It will - // become the GF so preserve last as an alternative prediction option. - cpi->refresh_last_frame = 0; - } - } -} - -#if CONFIG_INTERNAL_STATS -extern double av1_get_blockiness(const unsigned char *img1, int img1_pitch, - const unsigned char *img2, int img2_pitch, - int width, int height); - -static void adjust_image_stat(double y, double u, double v, double all, - ImageStat *s) { - s->stat[STAT_Y] += y; - s->stat[STAT_U] += u; - s->stat[STAT_V] += v; - s->stat[STAT_ALL] += all; - s->worst = AOMMIN(s->worst, all); -} - -static void compute_internal_stats(AV1_COMP *cpi, int frame_bytes) { - AV1_COMMON *const cm = &cpi->common; - double samples = 0.0; - uint32_t in_bit_depth = 8; - uint32_t bit_depth = 8; - -#if CONFIG_INTER_STATS_ONLY - if (cm->frame_type == KEY_FRAME) return; // skip key frame -#endif - cpi->bytes += frame_bytes; - - if (cm->seq_params.use_highbitdepth) { - in_bit_depth = cpi->oxcf.input_bit_depth; - bit_depth = cm->seq_params.bit_depth; - } - if (cm->show_frame) { - const YV12_BUFFER_CONFIG *orig = cpi->source; - const YV12_BUFFER_CONFIG *recon = cpi->common.frame_to_show; - double y, u, v, frame_all; - - cpi->count++; - if (cpi->b_calculate_psnr) { - PSNR_STATS psnr; - double frame_ssim2 = 0.0, weight = 0.0; - aom_clear_system_state(); - // TODO(yaowu): unify these two versions into one. - aom_calc_highbd_psnr(orig, recon, &psnr, bit_depth, in_bit_depth); - - adjust_image_stat(psnr.psnr[1], psnr.psnr[2], psnr.psnr[3], psnr.psnr[0], - &cpi->psnr); - cpi->total_sq_error += psnr.sse[0]; - cpi->total_samples += psnr.samples[0]; - samples = psnr.samples[0]; - // TODO(yaowu): unify these two versions into one. - if (cm->seq_params.use_highbitdepth) - frame_ssim2 = - aom_highbd_calc_ssim(orig, recon, &weight, bit_depth, in_bit_depth); - else - frame_ssim2 = aom_calc_ssim(orig, recon, &weight); - - cpi->worst_ssim = AOMMIN(cpi->worst_ssim, frame_ssim2); - cpi->summed_quality += frame_ssim2 * weight; - cpi->summed_weights += weight; - -#if 0 - { - FILE *f = fopen("q_used.stt", "a"); - double y2 = psnr.psnr[1]; - double u2 = psnr.psnr[2]; - double v2 = psnr.psnr[3]; - double frame_psnr2 = psnr.psnr[0]; - fprintf(f, "%5d : Y%f7.3:U%f7.3:V%f7.3:F%f7.3:S%7.3f\n", - cm->current_video_frame, y2, u2, v2, - frame_psnr2, frame_ssim2); - fclose(f); - } -#endif - } - if (cpi->b_calculate_blockiness) { - if (!cm->seq_params.use_highbitdepth) { - const double frame_blockiness = - av1_get_blockiness(orig->y_buffer, orig->y_stride, recon->y_buffer, - recon->y_stride, orig->y_width, orig->y_height); - cpi->worst_blockiness = AOMMAX(cpi->worst_blockiness, frame_blockiness); - cpi->total_blockiness += frame_blockiness; - } - - if (cpi->b_calculate_consistency) { - if (!cm->seq_params.use_highbitdepth) { - const double this_inconsistency = aom_get_ssim_metrics( - orig->y_buffer, orig->y_stride, recon->y_buffer, recon->y_stride, - orig->y_width, orig->y_height, cpi->ssim_vars, &cpi->metrics, 1); - - const double peak = (double)((1 << in_bit_depth) - 1); - const double consistency = - aom_sse_to_psnr(samples, peak, cpi->total_inconsistency); - if (consistency > 0.0) - cpi->worst_consistency = - AOMMIN(cpi->worst_consistency, consistency); - cpi->total_inconsistency += this_inconsistency; - } - } - } - - frame_all = - aom_calc_fastssim(orig, recon, &y, &u, &v, bit_depth, in_bit_depth); - adjust_image_stat(y, u, v, frame_all, &cpi->fastssim); - frame_all = aom_psnrhvs(orig, recon, &y, &u, &v, bit_depth, in_bit_depth); - adjust_image_stat(y, u, v, frame_all, &cpi->psnrhvs); - } -} -#endif // CONFIG_INTERNAL_STATS - static int is_integer_mv(AV1_COMP *cpi, const YV12_BUFFER_CONFIG *cur_picture, const YV12_BUFFER_CONFIG *last_picture, hash_table *last_hash_table) { @@ -5769,31 +5061,661 @@ static int is_integer_mv(AV1_COMP *cpi, const YV12_BUFFER_CONFIG *cur_picture, return 0; } +static int encode_frame_to_data_rate(AV1_COMP *cpi, size_t *size, uint8_t *dest, + unsigned int *frame_flags) { + AV1_COMMON *const cm = &cpi->common; + SequenceHeader *const seq_params = &cm->seq_params; + CurrentFrame *const current_frame = &cm->current_frame; + const AV1EncoderConfig *const oxcf = &cpi->oxcf; + struct segmentation *const seg = &cm->seg; + + // frame type has been decided outside of this function call + cm->cur_frame->frame_type = current_frame->frame_type; + + cm->large_scale_tile = cpi->oxcf.large_scale_tile; + cm->single_tile_decoding = cpi->oxcf.single_tile_decoding; + + cm->allow_ref_frame_mvs &= frame_might_allow_ref_frame_mvs(cm); + // cm->allow_ref_frame_mvs needs to be written into the frame header while + // cm->large_scale_tile is 1, therefore, "cm->large_scale_tile=1" case is + // separated from frame_might_allow_ref_frame_mvs(). + cm->allow_ref_frame_mvs &= !cm->large_scale_tile; + + cm->allow_warped_motion = + cpi->oxcf.allow_warped_motion && frame_might_allow_warped_motion(cm); + + cm->last_frame_type = current_frame->frame_type; + if (cpi->oxcf.pass == 2 && cpi->sf.adaptive_interp_filter_search) + cpi->sf.interp_filter_search_mask = setup_interp_filter_search_mask(cpi); + + if (encode_show_existing_frame(cm)) { + // NOTE(zoeliu): In BIDIR_PRED, the existing frame to show is the current + // BWDREF_FRAME in the reference frame buffer. + cpi->frame_flags = *frame_flags; + + restore_coding_context(cpi); + + finalize_encoded_frame(cpi); + // Build the bitstream + int largest_tile_id = 0; // Output from bitstream: unused here + if (av1_pack_bitstream(cpi, dest, size, &largest_tile_id) != AOM_CODEC_OK) + return AOM_CODEC_ERROR; + + if (seq_params->frame_id_numbers_present_flag && + current_frame->frame_type == KEY_FRAME) { + // Displaying a forward key-frame, so reset the ref buffer IDs + int display_frame_id = cm->ref_frame_id[cpi->existing_fb_idx_to_show]; + for (int i = 0; i < REF_FRAMES; i++) + cm->ref_frame_id[i] = display_frame_id; + } + + cpi->seq_params_locked = 1; + +#if DUMP_RECON_FRAMES == 1 + // NOTE(zoeliu): For debug - Output the filtered reconstructed video. + dump_filtered_recon_frames(cpi); +#endif // DUMP_RECON_FRAMES + + // Update the LAST_FRAME in the reference frame buffer. + // NOTE: + // (1) For BWDREF_FRAME as the show_existing_frame, the reference frame + // update has been done previously when handling the LAST_BIPRED_FRAME + // right before BWDREF_FRAME (in the display order); + // (2) For INTNL_OVERLAY as the show_existing_frame, the reference frame + // update will be done when the following is called, which will + // exchange + // the virtual indexes between LAST_FRAME and ALTREF2_FRAME, so that + // LAST3 will get retired, LAST2 becomes LAST3, LAST becomes LAST2, + // and + // ALTREF2_FRAME will serve as the new LAST_FRAME. + update_reference_frames(cpi); + + // Update frame flags + cpi->frame_flags &= ~FRAMEFLAGS_GOLDEN; + cpi->frame_flags &= ~FRAMEFLAGS_BWDREF; + cpi->frame_flags &= ~FRAMEFLAGS_ALTREF; + + *frame_flags = cpi->frame_flags & ~FRAMEFLAGS_KEY; + + // Since we allocate a spot for the OVERLAY frame in the gf group, we need + // to do post-encoding update accordingly. + if (cpi->rc.is_src_frame_alt_ref) { + av1_set_target_rate(cpi, cm->width, cm->height); + av1_rc_postencode_update(cpi, *size); + } + + ++current_frame->frame_number; + + return AOM_CODEC_OK; + } + + // Work out whether to force_integer_mv this frame + if (oxcf->pass != 1 && cpi->common.allow_screen_content_tools && + !frame_is_intra_only(cm)) { + if (cpi->common.seq_params.force_integer_mv == 2) { + // Adaptive mode: see what previous frame encoded did + if (cpi->unscaled_last_source != NULL) { + cm->cur_frame_force_integer_mv = + is_integer_mv(cpi, cpi->source, cpi->unscaled_last_source, + cpi->previous_hash_table); + } else { + cpi->common.cur_frame_force_integer_mv = 0; + } + } else { + cpi->common.cur_frame_force_integer_mv = + cpi->common.seq_params.force_integer_mv; + } + } else { + cpi->common.cur_frame_force_integer_mv = 0; + } + + // Set default state for segment based loop filter update flags. + cm->lf.mode_ref_delta_update = 0; + + // Set various flags etc to special state if it is a key frame. + if (frame_is_intra_only(cm) || frame_is_sframe(cm)) { + // Reset the loop filter deltas and segmentation map. + av1_reset_segment_features(cm); + + // If segmentation is enabled force a map update for key frames. + if (seg->enabled) { + seg->update_map = 1; + seg->update_data = 1; + } + + // The alternate reference frame cannot be active for a key frame. + cpi->rc.source_alt_ref_active = 0; + } + if (cpi->oxcf.mtu == 0) { + cm->num_tg = cpi->oxcf.num_tile_groups; + } else { + // Use a default value for the purposes of weighting costs in probability + // updates + cm->num_tg = DEFAULT_MAX_NUM_TG; + } + + // For 1 pass CBR, check if we are dropping this frame. + // Never drop on key frame. + if (oxcf->pass == 0 && oxcf->rc_mode == AOM_CBR && + current_frame->frame_type != KEY_FRAME) { + if (av1_rc_drop_frame(cpi)) { + av1_rc_postencode_update_drop_frame(cpi); + release_scaled_references(cpi); + return AOM_CODEC_OK; + } + } + + aom_clear_system_state(); + +#if CONFIG_INTERNAL_STATS + memset(cpi->mode_chosen_counts, 0, + MAX_MODES * sizeof(*cpi->mode_chosen_counts)); +#endif + + if (seq_params->frame_id_numbers_present_flag) { + /* Non-normative definition of current_frame_id ("frame counter" with + * wraparound) */ + if (cm->current_frame_id == -1) { + int lsb, msb; + /* quasi-random initialization of current_frame_id for a key frame */ + if (cpi->source->flags & YV12_FLAG_HIGHBITDEPTH) { + lsb = CONVERT_TO_SHORTPTR(cpi->source->y_buffer)[0] & 0xff; + msb = CONVERT_TO_SHORTPTR(cpi->source->y_buffer)[1] & 0xff; + } else { + lsb = cpi->source->y_buffer[0] & 0xff; + msb = cpi->source->y_buffer[1] & 0xff; + } + cm->current_frame_id = + ((msb << 8) + lsb) % (1 << seq_params->frame_id_length); + + // S_frame is meant for stitching different streams of different + // resolutions together, so current_frame_id must be the + // same across different streams of the same content current_frame_id + // should be the same and not random. 0x37 is a chosen number as start + // point + if (cpi->oxcf.sframe_enabled) cm->current_frame_id = 0x37; + } else { + cm->current_frame_id = + (cm->current_frame_id + 1 + (1 << seq_params->frame_id_length)) % + (1 << seq_params->frame_id_length); + } + } + + switch (cpi->oxcf.cdf_update_mode) { + case 0: // No CDF update for any frames(4~6% compression loss). + cm->disable_cdf_update = 1; + break; + case 1: // Enable CDF update for all frames. + cm->disable_cdf_update = 0; + break; + case 2: + // Strategically determine at which frames to do CDF update. + // Currently only enable CDF update for all-intra and no-show frames(1.5% + // compression loss). + // TODO(huisu@google.com): design schemes for various trade-offs between + // compression quality and decoding speed. + cm->disable_cdf_update = + (frame_is_intra_only(cm) || !cm->show_frame) ? 0 : 1; + break; + } + cm->timing_info_present &= !seq_params->reduced_still_picture_hdr; + + if (encode_with_recode_loop(cpi, size, dest) != AOM_CODEC_OK) + return AOM_CODEC_ERROR; + +#ifdef OUTPUT_YUV_SKINMAP + if (cpi->common.current_frame.frame_number > 1) { + av1_compute_skin_map(cpi, yuv_skinmap_file); + } +#endif // OUTPUT_YUV_SKINMAP + + // Special case code to reduce pulsing when key frames are forced at a + // fixed interval. Note the reconstruction error if it is the frame before + // the force key frame + if (cpi->rc.next_key_frame_forced && cpi->rc.frames_to_key == 1) { + if (seq_params->use_highbitdepth) { + cpi->ambient_err = aom_highbd_get_y_sse(cpi->source, &cm->cur_frame->buf); + } else { + cpi->ambient_err = aom_get_y_sse(cpi->source, &cm->cur_frame->buf); + } + } + + // If the encoder forced a KEY_FRAME decision or if frame is an S_FRAME + if ((current_frame->frame_type == KEY_FRAME && cm->show_frame) || + frame_is_sframe(cm)) { + cpi->refresh_last_frame = 1; + } + + cm->cur_frame->buf.color_primaries = seq_params->color_primaries; + cm->cur_frame->buf.transfer_characteristics = + seq_params->transfer_characteristics; + cm->cur_frame->buf.matrix_coefficients = seq_params->matrix_coefficients; + cm->cur_frame->buf.monochrome = seq_params->monochrome; + cm->cur_frame->buf.chroma_sample_position = + seq_params->chroma_sample_position; + cm->cur_frame->buf.color_range = seq_params->color_range; + cm->cur_frame->buf.render_width = cm->render_width; + cm->cur_frame->buf.render_height = cm->render_height; + + // TODO(zoeliu): For non-ref frames, loop filtering may need to be turned + // off. + + // Pick the loop filter level for the frame. + if (!cm->allow_intrabc) { + loopfilter_frame(cpi, cm); + } else { + cm->lf.filter_level[0] = 0; + cm->lf.filter_level[1] = 0; + cm->cdef_info.cdef_bits = 0; + cm->cdef_info.cdef_strengths[0] = 0; + cm->cdef_info.nb_cdef_strengths = 1; + cm->cdef_info.cdef_uv_strengths[0] = 0; + cm->rst_info[0].frame_restoration_type = RESTORE_NONE; + cm->rst_info[1].frame_restoration_type = RESTORE_NONE; + cm->rst_info[2].frame_restoration_type = RESTORE_NONE; + } + + // TODO(debargha): Fix mv search range on encoder side + // aom_extend_frame_inner_borders(&cm->cur_frame->buf, av1_num_planes(cm)); + aom_extend_frame_borders(&cm->cur_frame->buf, av1_num_planes(cm)); + +#ifdef OUTPUT_YUV_REC + aom_write_one_yuv_frame(cm, &cm->cur_frame->buf); +#endif + + finalize_encoded_frame(cpi); + // Build the bitstream + int largest_tile_id = 0; // Output from pack_bitstream + if (av1_pack_bitstream(cpi, dest, size, &largest_tile_id) != AOM_CODEC_OK) + return AOM_CODEC_ERROR; + + cpi->seq_params_locked = 1; + + // Update reference frame ids for reference frames this frame will overwrite + if (seq_params->frame_id_numbers_present_flag) { + for (int i = 0; i < REF_FRAMES; i++) { + if ((current_frame->refresh_frame_flags >> i) & 1) { + cm->ref_frame_id[i] = cm->current_frame_id; + } + } + } + +#if DUMP_RECON_FRAMES == 1 + // NOTE(zoeliu): For debug - Output the filtered reconstructed video. + dump_filtered_recon_frames(cpi); +#endif // DUMP_RECON_FRAMES + + if (cm->seg.enabled) { + if (cm->seg.update_map) { + update_reference_segmentation_map(cpi); + } else if (cm->last_frame_seg_map) { + memcpy(cm->cur_frame->seg_map, cm->last_frame_seg_map, + cm->mi_cols * cm->mi_rows * sizeof(uint8_t)); + } + } + + if (frame_is_intra_only(cm) == 0) { + release_scaled_references(cpi); + } + + update_reference_frames(cpi); + +#if CONFIG_ENTROPY_STATS + av1_accumulate_frame_counts(&aggregate_fc, &cpi->counts); +#endif // CONFIG_ENTROPY_STATS + + if (cm->refresh_frame_context == REFRESH_FRAME_CONTEXT_BACKWARD) { + *cm->fc = cpi->tile_data[largest_tile_id].tctx; + av1_reset_cdf_symbol_counters(cm->fc); + } + if (!cm->large_scale_tile) { + cm->cur_frame->frame_context = *cm->fc; + } +#define EXT_TILE_DEBUG 0 +#if EXT_TILE_DEBUG + if (cm->large_scale_tile && oxcf->pass == 2) { + char fn[20] = "./fc"; + fn[4] = current_frame->frame_number / 100 + '0'; + fn[5] = (current_frame->frame_number % 100) / 10 + '0'; + fn[6] = (current_frame->frame_number % 10) + '0'; + fn[7] = '\0'; + av1_print_frame_contexts(cm->fc, fn); + } +#endif // EXT_TILE_DEBUG +#undef EXT_TILE_DEBUG + + if (cpi->refresh_golden_frame == 1) + cpi->frame_flags |= FRAMEFLAGS_GOLDEN; + else + cpi->frame_flags &= ~FRAMEFLAGS_GOLDEN; + + if (cpi->refresh_alt_ref_frame == 1) + cpi->frame_flags |= FRAMEFLAGS_ALTREF; + else + cpi->frame_flags &= ~FRAMEFLAGS_ALTREF; + + if (cpi->refresh_bwd_ref_frame == 1) + cpi->frame_flags |= FRAMEFLAGS_BWDREF; + else + cpi->frame_flags &= ~FRAMEFLAGS_BWDREF; + cm->last_frame_type = current_frame->frame_type; + + av1_rc_postencode_update(cpi, *size); + + if (current_frame->frame_type == KEY_FRAME) { + // Tell the caller that the frame was coded as a key frame + *frame_flags = cpi->frame_flags | FRAMEFLAGS_KEY; + } else { + *frame_flags = cpi->frame_flags & ~FRAMEFLAGS_KEY; + } + + // Store encoded frame's hash table for is_integer_mv() next time + if (oxcf->pass != 1 && cpi->common.allow_screen_content_tools) { + cpi->previous_hash_table = &cm->cur_frame->hash_table; + } + + // Clear the one shot update flags for segmentation map and mode/ref loop + // filter deltas. + cm->seg.update_map = 0; + cm->seg.update_data = 0; + cm->lf.mode_ref_delta_update = 0; + + // A droppable frame might not be shown but it always + // takes a space in the gf group. Therefore, even when + // it is not shown, we still need update the count down. + + if (cm->show_frame) { + // TODO(zoeliu): We may only swamp mi and prev_mi for those frames that + // are + // being used as reference. + swap_mi_and_prev_mi(cm); + // Don't increment frame counters if this was an altref buffer + // update not a real frame + + ++current_frame->frame_number; + } + + return AOM_CODEC_OK; +} + +int av1_encode(AV1_COMP *const cpi, uint8_t *const dest, + const EncodeFrameInput *const frame_input, + const EncodeFrameParams *const frame_params, + EncodeFrameResults *const frame_results) { + AV1_COMMON *const cm = &cpi->common; + CurrentFrame *const current_frame = &cm->current_frame; + + cpi->unscaled_source = frame_input->source; + cpi->source = frame_input->source; + cpi->unscaled_last_source = frame_input->last_source; + + cm->error_resilient_mode = frame_params->error_resilient_mode; + cm->primary_ref_frame = frame_params->primary_ref_frame; + cm->current_frame.frame_type = frame_params->frame_type; + cm->show_frame = frame_params->show_frame; + cpi->ref_frame_flags = frame_params->ref_frame_flags; + cpi->speed = frame_params->speed; + + if (current_frame->frame_type == KEY_FRAME && cm->show_frame) + current_frame->frame_number = 0; + + if (cm->show_existing_frame) { + current_frame->order_hint = cm->cur_frame->order_hint; + } else { + current_frame->order_hint = + current_frame->frame_number + frame_params->order_offset; + current_frame->order_hint %= + (1 << (cm->seq_params.order_hint_info.order_hint_bits_minus_1 + 1)); + } + + if (cpi->oxcf.pass == 1) { + av1_first_pass(cpi, frame_input->ts_duration); + } else if (cpi->oxcf.pass == 0 || cpi->oxcf.pass == 2) { + if (encode_frame_to_data_rate(cpi, &frame_results->size, dest, + frame_params->frame_flags) != AOM_CODEC_OK) { + return AOM_CODEC_ERROR; + } + } else { + return AOM_CODEC_ERROR; + } + + return AOM_CODEC_OK; +} + +#if CONFIG_DENOISE +static int apply_denoise_2d(AV1_COMP *cpi, YV12_BUFFER_CONFIG *sd, + int block_size, float noise_level, + int64_t time_stamp, int64_t end_time) { + AV1_COMMON *const cm = &cpi->common; + if (!cpi->denoise_and_model) { + cpi->denoise_and_model = aom_denoise_and_model_alloc( + cm->seq_params.bit_depth, block_size, noise_level); + if (!cpi->denoise_and_model) { + aom_internal_error(&cm->error, AOM_CODEC_MEM_ERROR, + "Error allocating denoise and model"); + return -1; + } + } + if (!cpi->film_grain_table) { + cpi->film_grain_table = aom_malloc(sizeof(*cpi->film_grain_table)); + if (!cpi->film_grain_table) { + aom_internal_error(&cm->error, AOM_CODEC_MEM_ERROR, + "Error allocating grain table"); + return -1; + } + memset(cpi->film_grain_table, 0, sizeof(*cpi->film_grain_table)); + } + if (aom_denoise_and_model_run(cpi->denoise_and_model, sd, + &cm->film_grain_params)) { + if (cm->film_grain_params.apply_grain) { + aom_film_grain_table_append(cpi->film_grain_table, time_stamp, end_time, + &cm->film_grain_params); + } + } + return 0; +} +#endif + +int av1_receive_raw_frame(AV1_COMP *cpi, aom_enc_frame_flags_t frame_flags, + YV12_BUFFER_CONFIG *sd, int64_t time_stamp, + int64_t end_time) { + AV1_COMMON *const cm = &cpi->common; + const SequenceHeader *const seq_params = &cm->seq_params; + struct aom_usec_timer timer; + int res = 0; + const int subsampling_x = sd->subsampling_x; + const int subsampling_y = sd->subsampling_y; + const int use_highbitdepth = (sd->flags & YV12_FLAG_HIGHBITDEPTH) != 0; + + check_initial_width(cpi, use_highbitdepth, subsampling_x, subsampling_y); + + aom_usec_timer_start(&timer); + +#if CONFIG_DENOISE + if (cpi->oxcf.noise_level > 0) + if (apply_denoise_2d(cpi, sd, cpi->oxcf.noise_block_size, + cpi->oxcf.noise_level, time_stamp, end_time) < 0) + res = -1; +#endif // CONFIG_DENOISE + + if (av1_lookahead_push(cpi->lookahead, sd, time_stamp, end_time, + use_highbitdepth, frame_flags)) + res = -1; + aom_usec_timer_mark(&timer); + cpi->time_receive_data += aom_usec_timer_elapsed(&timer); + + if ((seq_params->profile == PROFILE_0) && !seq_params->monochrome && + (subsampling_x != 1 || subsampling_y != 1)) { + aom_internal_error(&cm->error, AOM_CODEC_INVALID_PARAM, + "Non-4:2:0 color format requires profile 1 or 2"); + res = -1; + } + if ((seq_params->profile == PROFILE_1) && + !(subsampling_x == 0 && subsampling_y == 0)) { + aom_internal_error(&cm->error, AOM_CODEC_INVALID_PARAM, + "Profile 1 requires 4:4:4 color format"); + res = -1; + } + if ((seq_params->profile == PROFILE_2) && + (seq_params->bit_depth <= AOM_BITS_10) && + !(subsampling_x == 1 && subsampling_y == 0)) { + aom_internal_error(&cm->error, AOM_CODEC_INVALID_PARAM, + "Profile 2 bit-depth < 10 requires 4:2:2 color format"); + res = -1; + } + + return res; +} + +#if CONFIG_INTERNAL_STATS +extern double av1_get_blockiness(const unsigned char *img1, int img1_pitch, + const unsigned char *img2, int img2_pitch, + int width, int height); + +static void adjust_image_stat(double y, double u, double v, double all, + ImageStat *s) { + s->stat[STAT_Y] += y; + s->stat[STAT_U] += u; + s->stat[STAT_V] += v; + s->stat[STAT_ALL] += all; + s->worst = AOMMIN(s->worst, all); +} + +static void compute_internal_stats(AV1_COMP *cpi, int frame_bytes) { + AV1_COMMON *const cm = &cpi->common; + double samples = 0.0; + uint32_t in_bit_depth = 8; + uint32_t bit_depth = 8; + +#if CONFIG_INTER_STATS_ONLY + if (cm->current_frame.frame_type == KEY_FRAME) return; // skip key frame +#endif + cpi->bytes += frame_bytes; + + if (cm->seq_params.use_highbitdepth) { + in_bit_depth = cpi->oxcf.input_bit_depth; + bit_depth = cm->seq_params.bit_depth; + } + if (cm->show_frame) { + const YV12_BUFFER_CONFIG *orig = cpi->source; + const YV12_BUFFER_CONFIG *recon = &cpi->common.cur_frame->buf; + double y, u, v, frame_all; + + cpi->count++; + if (cpi->b_calculate_psnr) { + PSNR_STATS psnr; + double frame_ssim2 = 0.0, weight = 0.0; + aom_clear_system_state(); + // TODO(yaowu): unify these two versions into one. + aom_calc_highbd_psnr(orig, recon, &psnr, bit_depth, in_bit_depth); + + adjust_image_stat(psnr.psnr[1], psnr.psnr[2], psnr.psnr[3], psnr.psnr[0], + &cpi->psnr); + cpi->total_sq_error += psnr.sse[0]; + cpi->total_samples += psnr.samples[0]; + samples = psnr.samples[0]; + // TODO(yaowu): unify these two versions into one. + if (cm->seq_params.use_highbitdepth) + frame_ssim2 = + aom_highbd_calc_ssim(orig, recon, &weight, bit_depth, in_bit_depth); + else + frame_ssim2 = aom_calc_ssim(orig, recon, &weight); + + cpi->worst_ssim = AOMMIN(cpi->worst_ssim, frame_ssim2); + cpi->summed_quality += frame_ssim2 * weight; + cpi->summed_weights += weight; + +#if 0 + { + FILE *f = fopen("q_used.stt", "a"); + double y2 = psnr.psnr[1]; + double u2 = psnr.psnr[2]; + double v2 = psnr.psnr[3]; + double frame_psnr2 = psnr.psnr[0]; + fprintf(f, "%5d : Y%f7.3:U%f7.3:V%f7.3:F%f7.3:S%7.3f\n", + cm->current_frame.frame_number, y2, u2, v2, + frame_psnr2, frame_ssim2); + fclose(f); + } +#endif + } + if (cpi->b_calculate_blockiness) { + if (!cm->seq_params.use_highbitdepth) { + const double frame_blockiness = + av1_get_blockiness(orig->y_buffer, orig->y_stride, recon->y_buffer, + recon->y_stride, orig->y_width, orig->y_height); + cpi->worst_blockiness = AOMMAX(cpi->worst_blockiness, frame_blockiness); + cpi->total_blockiness += frame_blockiness; + } + + if (cpi->b_calculate_consistency) { + if (!cm->seq_params.use_highbitdepth) { + const double this_inconsistency = aom_get_ssim_metrics( + orig->y_buffer, orig->y_stride, recon->y_buffer, recon->y_stride, + orig->y_width, orig->y_height, cpi->ssim_vars, &cpi->metrics, 1); + + const double peak = (double)((1 << in_bit_depth) - 1); + const double consistency = + aom_sse_to_psnr(samples, peak, cpi->total_inconsistency); + if (consistency > 0.0) + cpi->worst_consistency = + AOMMIN(cpi->worst_consistency, consistency); + cpi->total_inconsistency += this_inconsistency; + } + } + } + + frame_all = + aom_calc_fastssim(orig, recon, &y, &u, &v, bit_depth, in_bit_depth); + adjust_image_stat(y, u, v, frame_all, &cpi->fastssim); + frame_all = aom_psnrhvs(orig, recon, &y, &u, &v, bit_depth, in_bit_depth); + adjust_image_stat(y, u, v, frame_all, &cpi->psnrhvs); + } +} +#endif // CONFIG_INTERNAL_STATS + +// Don't allow a show_existing_frame to coincide with an error resilient or +// S-Frame. An exception can be made in the case of a keyframe, since it does +// not depend on any previous frames. +static int allow_show_existing(const AV1_COMP *const cpi) { + if (cpi->common.current_frame.frame_number == 0) return 0; + + const struct lookahead_entry *lookahead_src = + av1_lookahead_peek(cpi->lookahead, 0); + if (lookahead_src == NULL) return 1; + + const int is_error_resilient = + cpi->oxcf.error_resilient_mode || + (lookahead_src->flags & AOM_EFLAG_ERROR_RESILIENT); + const int is_s_frame = + cpi->oxcf.s_frame_mode || (lookahead_src->flags & AOM_EFLAG_SET_S_FRAME); + const int is_key_frame = + (cpi->rc.frames_to_key == 0) || (cpi->frame_flags & FRAMEFLAGS_KEY); + return !(is_error_resilient || is_s_frame) || is_key_frame; +} + int av1_get_compressed_data(AV1_COMP *cpi, unsigned int *frame_flags, size_t *size, uint8_t *dest, int64_t *time_stamp, int64_t *time_end, int flush, const aom_rational_t *timebase) { const AV1EncoderConfig *const oxcf = &cpi->oxcf; AV1_COMMON *const cm = &cpi->common; - const int num_planes = av1_num_planes(cm); - BufferPool *const pool = cm->buffer_pool; - RATE_CONTROL *const rc = &cpi->rc; struct aom_usec_timer cmptimer; - YV12_BUFFER_CONFIG *force_src_buffer = NULL; - struct lookahead_entry *last_source = NULL; - struct lookahead_entry *source = NULL; - int arf_src_index; - int brf_src_index; - int i; #if CONFIG_BITSTREAM_DEBUG assert(cpi->oxcf.max_threads == 0 && "bitstream debug tool does not support multithreading"); bitstream_queue_record_write(); - bitstream_queue_set_frame_write(cm->current_video_frame * 2 + cm->show_frame); + bitstream_queue_set_frame_write(current_frame->frame_number * 2 + + cm->show_frame); #endif + // Indicates whether or not to use an adaptive quantize b rather than + // the traditional version + cm->use_quant_b_adapt = cpi->oxcf.quant_b_adapt; + cm->showable_frame = 0; + *size = 0; aom_usec_timer_start(&cmptimer); set_high_precision_mv(cpi, ALTREF_HIGH_PRECISION_MV, 0); @@ -5806,381 +5728,44 @@ int av1_get_compressed_data(AV1_COMP *cpi, unsigned int *frame_flags, cm->refresh_frame_context = REFRESH_FRAME_CONTEXT_DISABLED; // default reference buffers update config - av1_configure_buffer_updates_firstpass(cpi, LF_UPDATE); + av1_configure_buffer_updates(cpi, LF_UPDATE); // Initialize fields related to forward keyframes cpi->no_show_kf = 0; - cm->reset_decoder_state = 0; - // Don't allow a show_existing_frame to coincide with an error resilient or - // S-Frame. An exception can be made in the case of a keyframe, since it - // does not depend on any previous frames. We must make this exception here - // because of the use of show_existing_frame with forward coded keyframes. - struct lookahead_entry *lookahead_src = NULL; - if (cm->current_video_frame > 0) - lookahead_src = av1_lookahead_peek(cpi->lookahead, 0); + cm->show_existing_frame &= allow_show_existing(cpi); - int use_show_existing = 1; - if (lookahead_src != NULL) { - const int is_error_resilient = - cpi->oxcf.error_resilient_mode || - (lookahead_src->flags & AOM_EFLAG_ERROR_RESILIENT); - const int is_s_frame = cpi->oxcf.s_frame_mode || - (lookahead_src->flags & AOM_EFLAG_SET_S_FRAME); - const int is_key_frame = - (rc->frames_to_key == 0) || (cpi->frame_flags & FRAMEFLAGS_KEY); - use_show_existing = !(is_error_resilient || is_s_frame) || is_key_frame; - } + if (assign_cur_frame_new_fb(cm) == NULL) return AOM_CODEC_ERROR; - if (oxcf->pass == 2 && cm->show_existing_frame && use_show_existing) { - // Manage the source buffer and flush out the source frame that has been - // coded already; Also get prepared for PSNR calculation if needed. - if ((source = av1_lookahead_pop(cpi->lookahead, flush)) == NULL) { - *size = 0; - return -1; - } - av1_apply_encoding_flags(cpi, source->flags); - cpi->source = &source->img; - // TODO(zoeliu): To track down to determine whether it's needed to adjust - // the frame rate. - *time_stamp = source->ts_start; - *time_end = source->ts_end; - - // We need to adjust frame rate for an overlay frame - if (cpi->rc.is_src_frame_alt_ref) adjust_frame_rate(cpi, source); - - // Find a free buffer for the new frame, releasing the reference - // previously - // held. - if (cm->new_fb_idx != INVALID_IDX) { - --pool->frame_bufs[cm->new_fb_idx].ref_count; - } - cm->new_fb_idx = get_free_fb(cm); - - if (cm->new_fb_idx == INVALID_IDX) return -1; - - // Clear down mmx registers - aom_clear_system_state(); - - // Start with a 0 size frame. - *size = 0; - - // We need to update the gf_group for show_existing overlay frame - if (cpi->rc.is_src_frame_alt_ref) av1_rc_get_second_pass_params(cpi); - - if (Pass2Encode(cpi, size, dest, frame_flags) != AOM_CODEC_OK) - return AOM_CODEC_ERROR; - - if (cpi->b_calculate_psnr) generate_psnr_packet(cpi); - -#if CONFIG_INTERNAL_STATS - compute_internal_stats(cpi, (int)(*size)); -#endif // CONFIG_INTERNAL_STATS - - // Clear down mmx registers - aom_clear_system_state(); - - cm->show_existing_frame = 0; - return 0; - } - - // Should we encode an arf frame. - arf_src_index = get_arf_src_index(cpi); - if (arf_src_index) { - for (i = 0; i <= arf_src_index; ++i) { - struct lookahead_entry *e = av1_lookahead_peek(cpi->lookahead, i); - // Avoid creating an alt-ref if there's a forced keyframe pending. - if (e == NULL) { - break; - } else if (e->flags == AOM_EFLAG_FORCE_KF) { - arf_src_index = 0; - flush = 1; - break; - } - } - } - - if (arf_src_index) { - assert(arf_src_index <= rc->frames_to_key); - - if ((source = av1_lookahead_peek(cpi->lookahead, arf_src_index)) != NULL) { - cm->showable_frame = 1; - cpi->alt_ref_source = source; - // When arf_src_index == rc->frames_to_key, it indicates a fwd_kf - if (arf_src_index == rc->frames_to_key) { - // Skip temporal filtering and mark as intra_only if we have a fwd_kf - const GF_GROUP *const gf_group = &cpi->twopass.gf_group; - int which_arf = gf_group->arf_update_idx[gf_group->index]; - cpi->is_arf_filter_off[which_arf] = 1; - cpi->no_show_kf = 1; - } else { - if (oxcf->arnr_max_frames > 0) { - // Produce the filtered ARF frame. - av1_temporal_filter(cpi, arf_src_index); - aom_extend_frame_borders(&cpi->alt_ref_buffer, num_planes); - force_src_buffer = &cpi->alt_ref_buffer; - } - } - cm->show_frame = 0; - cm->intra_only = 0; - - if (oxcf->pass < 2) { - // In second pass, the buffer updates configure will be set - // in the function av1_rc_get_second_pass_params - av1_configure_buffer_updates_firstpass(cpi, ARF_UPDATE); - } - } - rc->source_alt_ref_pending = 0; - } - - // Should we encode an arf2 frame. - arf_src_index = get_arf2_src_index(cpi); - if (arf_src_index) { - for (i = 0; i <= arf_src_index; ++i) { - struct lookahead_entry *e = av1_lookahead_peek(cpi->lookahead, i); - // Avoid creating an alt-ref if there's a forced keyframe pending. - if (e == NULL) { - break; - } else if (e->flags == AOM_EFLAG_FORCE_KF) { - arf_src_index = 0; - flush = 1; - break; - } - } - } - - if (arf_src_index) { - assert(arf_src_index <= rc->frames_to_key); - - if ((source = av1_lookahead_peek(cpi->lookahead, arf_src_index)) != NULL) { - cm->showable_frame = 1; - cpi->alt_ref_source = source; - - if (oxcf->arnr_max_frames > 0) { - // Produce the filtered ARF frame. - av1_temporal_filter(cpi, arf_src_index); - aom_extend_frame_borders(&cpi->alt_ref_buffer, num_planes); - force_src_buffer = &cpi->alt_ref_buffer; - } - - cm->show_frame = 0; - cm->intra_only = 0; - - if (oxcf->pass < 2) { - // In second pass, the buffer updates configure will be set - // in the function av1_rc_get_second_pass_params - av1_configure_buffer_updates_firstpass(cpi, INTNL_ARF_UPDATE); - } - } - rc->source_alt_ref_pending = 0; - } - - rc->is_bwd_ref_frame = 0; - brf_src_index = get_brf_src_index(cpi); - if (brf_src_index) { - assert(brf_src_index <= rc->frames_to_key); - if ((source = av1_lookahead_peek(cpi->lookahead, brf_src_index)) != NULL) { - cm->showable_frame = 1; - cm->show_frame = 0; - cm->intra_only = 0; - - if (oxcf->pass < 2) { - // In second pass, the buffer updates configure will be set - // in the function av1_rc_get_second_pass_params - av1_configure_buffer_updates_firstpass(cpi, BIPRED_UPDATE); - } - } - } - - if (!source) { - // Get last frame source. - if (cm->current_video_frame > 0) { - if ((last_source = av1_lookahead_peek(cpi->lookahead, -1)) == NULL) - return -1; - } - if (cm->current_video_frame > 0) assert(last_source != NULL); - // Read in the source frame. - source = av1_lookahead_pop(cpi->lookahead, flush); - - if (source != NULL) { - cm->show_frame = 1; - cm->intra_only = 0; - - // Check to see if the frame should be encoded as an arf overlay. - check_src_altref(cpi, source); - } - } - if (source) { - cpi->unscaled_source = cpi->source = - force_src_buffer ? force_src_buffer : &source->img; - cpi->unscaled_last_source = last_source != NULL ? &last_source->img : NULL; - - *time_stamp = source->ts_start; - *time_end = source->ts_end; - av1_apply_encoding_flags(cpi, source->flags); - *frame_flags = (source->flags & AOM_EFLAG_FORCE_KF) ? FRAMEFLAGS_KEY : 0; - - } else { - *size = 0; - if (flush && oxcf->pass == 1 && !cpi->twopass.first_pass_done) { - av1_end_first_pass(cpi); /* get last stats packet */ - cpi->twopass.first_pass_done = 1; - } + const int result = av1_encode_strategy(cpi, size, dest, frame_flags, + time_stamp, time_end, timebase, flush); + if (result != AOM_CODEC_OK && result != -1) { + return AOM_CODEC_ERROR; + } else if (result == -1) { + // Returning -1 indicates no frame encoded; more input is required return -1; } - if (source->ts_start < cpi->first_time_stamp_ever) { - cpi->first_time_stamp_ever = source->ts_start; - cpi->last_end_time_stamp_seen = source->ts_start; - } - - // Clear down mmx registers - aom_clear_system_state(); - - // adjust frame rates based on timestamps given - if (cm->show_frame) adjust_frame_rate(cpi, source); - - // Find a free buffer for the new frame, releasing the reference previously - // held. - if (cm->new_fb_idx != INVALID_IDX) { - --pool->frame_bufs[cm->new_fb_idx].ref_count; - } - cm->new_fb_idx = get_free_fb(cm); - - if (cm->new_fb_idx == INVALID_IDX) return -1; - - // Retain the RF_LEVEL for the current newly coded frame. - cpi->frame_rf_level[cm->new_fb_idx] = - cpi->twopass.gf_group.rf_level[cpi->twopass.gf_group.index]; - - cm->cur_frame = &pool->frame_bufs[cm->new_fb_idx]; - cm->cur_frame->buf.buf_8bit_valid = 0; - - if (cpi->film_grain_table) { - cm->seq_params.film_grain_params_present = aom_film_grain_table_lookup( - cpi->film_grain_table, *time_stamp, *time_end, 0 /* =erase */, - &cm->film_grain_params); - } - cm->cur_frame->film_grain_params_present = - cm->seq_params.film_grain_params_present; - - // only one operating point supported now - const int64_t pts64 = ticks_to_timebase_units(timebase, *time_stamp); - if (pts64 < 0 || pts64 > UINT32_MAX) return AOM_CODEC_ERROR; - cpi->common.frame_presentation_time = (uint32_t)pts64; - - // Start with a 0 size frame. - *size = 0; - - cpi->frame_flags = *frame_flags; - - if (oxcf->pass == 2) { - av1_rc_get_second_pass_params(cpi); - } else if (oxcf->pass == 1) { - setup_frame_size(cpi); - } - - if (cpi->oxcf.pass != 0 || frame_is_intra_only(cm) == 1) { - for (i = 0; i < REF_FRAMES; ++i) cpi->scaled_ref_idx[i] = INVALID_IDX; - } - - cm->using_qmatrix = cpi->oxcf.using_qm; - cm->min_qmlevel = cpi->oxcf.qm_minlevel; - cm->max_qmlevel = cpi->oxcf.qm_maxlevel; - - if (cm->seq_params.frame_id_numbers_present_flag) { - if (*time_stamp == 0) { - cpi->common.current_frame_id = -1; - } - } - - cpi->cur_poc++; - if (oxcf->pass != 1 && cpi->common.allow_screen_content_tools && - !frame_is_intra_only(cm)) { - if (cpi->common.seq_params.force_integer_mv == 2) { - struct lookahead_entry *previous_entry = - av1_lookahead_peek(cpi->lookahead, cpi->previous_index); - if (!previous_entry) - cpi->common.cur_frame_force_integer_mv = 0; - else - cpi->common.cur_frame_force_integer_mv = is_integer_mv( - cpi, cpi->source, &previous_entry->img, cpi->previous_hash_table); - } else { - cpi->common.cur_frame_force_integer_mv = - cpi->common.seq_params.force_integer_mv; - } - } else { - cpi->common.cur_frame_force_integer_mv = 0; - } - - if (oxcf->pass == 1) { - cpi->td.mb.e_mbd.lossless[0] = is_lossless_requested(oxcf); - av1_first_pass(cpi, source); - } else if (oxcf->pass == 2) { - if (Pass2Encode(cpi, size, dest, frame_flags) != AOM_CODEC_OK) - return AOM_CODEC_ERROR; - } else { - // One pass encode - if (Pass0Encode(cpi, size, dest, 0, frame_flags) != AOM_CODEC_OK) - return AOM_CODEC_ERROR; - } - if (oxcf->pass != 1 && cpi->common.allow_screen_content_tools) { - cpi->previous_hash_table = &cm->cur_frame->hash_table; - { - int l; - for (l = -MAX_PRE_FRAMES; l < cpi->lookahead->max_sz; l++) { - if ((cpi->lookahead->buf + l) == source) { - cpi->previous_index = l; - break; - } - } - - if (l == cpi->lookahead->max_sz) { - aom_internal_error(&cm->error, AOM_CODEC_MEM_ERROR, - "Failed to find last frame original buffer"); - } - } - } - - if (!cm->large_scale_tile) { - cm->frame_contexts[cm->new_fb_idx] = *cm->fc; - } - -#define EXT_TILE_DEBUG 0 -#if EXT_TILE_DEBUG - if (cm->large_scale_tile && oxcf->pass == 2) { - char fn[20] = "./fc"; - fn[4] = cm->current_video_frame / 100 + '0'; - fn[5] = (cm->current_video_frame % 100) / 10 + '0'; - fn[6] = (cm->current_video_frame % 10) + '0'; - fn[7] = '\0'; - av1_print_frame_contexts(cm->fc, fn); - } -#endif // EXT_TILE_DEBUG -#undef EXT_TILE_DEBUG - - cm->showable_frame = !cm->show_frame && cm->showable_frame; - - // No frame encoded, or frame was dropped, release scaled references. - if ((*size == 0) && (frame_is_intra_only(cm) == 0)) { - release_scaled_references(cpi); - } - - if (*size > 0) { - cpi->droppable = !frame_is_reference(cpi); - } - aom_usec_timer_mark(&cmptimer); cpi->time_compress_data += aom_usec_timer_elapsed(&cmptimer); - if (cpi->b_calculate_psnr && oxcf->pass != 1 && cm->show_frame) - generate_psnr_packet(cpi); + if (cpi->b_calculate_psnr) { + if (cm->show_existing_frame || (oxcf->pass != 1 && cm->show_frame)) { + generate_psnr_packet(cpi); + } + } #if CONFIG_INTERNAL_STATS if (oxcf->pass != 1) { compute_internal_stats(cpi, (int)(*size)); } #endif // CONFIG_INTERNAL_STATS +#if CONFIG_SPEED_STATS + if (cpi->oxcf.pass != 1 && !cm->show_existing_frame) { + cpi->tx_search_count += cpi->td.mb.tx_search_count; + cpi->td.mb.tx_search_count = 0; + } +#endif // CONFIG_SPEED_STATS aom_clear_system_state(); @@ -6193,8 +5778,8 @@ int av1_get_preview_raw_frame(AV1_COMP *cpi, YV12_BUFFER_CONFIG *dest) { return -1; } else { int ret; - if (cm->frame_to_show) { - *dest = *cm->frame_to_show; + if (cm->cur_frame != NULL) { + *dest = cm->cur_frame->buf; dest->y_width = cm->width; dest->y_height = cm->height; dest->uv_width = cm->width >> cm->seq_params.subsampling_x; @@ -6209,10 +5794,9 @@ int av1_get_preview_raw_frame(AV1_COMP *cpi, YV12_BUFFER_CONFIG *dest) { } int av1_get_last_show_frame(AV1_COMP *cpi, YV12_BUFFER_CONFIG *frame) { - if (cpi->last_show_frame_buf_idx == INVALID_IDX) return -1; + if (cpi->last_show_frame_buf == NULL) return -1; - *frame = - cpi->common.buffer_pool->frame_bufs[cpi->last_show_frame_buf_idx].buf; + *frame = cpi->last_show_frame_buf->buf; return 0; } @@ -6364,7 +5948,12 @@ void av1_apply_encoding_flags(AV1_COMP *cpi, aom_enc_frame_flags_t flags) { upd ^= AOM_ALT2_FLAG; } - av1_update_reference(cpi, upd); + cpi->ext_refresh_last_frame = (upd & AOM_LAST_FLAG) != 0; + cpi->ext_refresh_golden_frame = (upd & AOM_GOLD_FLAG) != 0; + cpi->ext_refresh_alt_ref_frame = (upd & AOM_ALT_FLAG) != 0; + cpi->ext_refresh_bwd_ref_frame = (upd & AOM_BWD_FLAG) != 0; + cpi->ext_refresh_alt2_ref_frame = (upd & AOM_ALT2_FLAG) != 0; + cpi->ext_refresh_frame_flags_pending = 1; } cpi->ext_use_ref_frame_mvs = cpi->oxcf.allow_ref_frame_mvs & @@ -6380,15 +5969,6 @@ void av1_apply_encoding_flags(AV1_COMP *cpi, aom_enc_frame_flags_t flags) { } } -int64_t timebase_units_to_ticks(const aom_rational_t *timebase, int64_t n) { - return n * TICKS_PER_SEC * timebase->num / timebase->den; -} - -int64_t ticks_to_timebase_units(const aom_rational_t *timebase, int64_t n) { - const int64_t round = TICKS_PER_SEC * timebase->num / 2 - 1; - return (n * timebase->den + round) / timebase->num / TICKS_PER_SEC; -} - aom_fixed_buf_t *av1_get_global_headers(AV1_COMP *cpi) { if (!cpi) return NULL; diff --git a/third_party/aom/av1/encoder/encoder.h b/third_party/aom/av1/encoder/encoder.h index ee7fc46379..dec531fa88 100644 --- a/third_party/aom/av1/encoder/encoder.h +++ b/third_party/aom/av1/encoder/encoder.h @@ -59,7 +59,7 @@ typedef struct { FRAME_CONTEXT fc; } CODING_CONTEXT; -typedef enum { +enum { // regular inter frame REGULAR_FRAME = 0, // alternate reference frame @@ -73,51 +73,54 @@ typedef enum { // extra alternate reference frame EXT_ARF_FRAME = 5, FRAME_CONTEXT_INDEXES -} FRAME_CONTEXT_INDEX; +} UENUM1BYTE(FRAME_CONTEXT_INDEX); -typedef enum { +enum { NORMAL = 0, FOURFIVE = 1, THREEFIVE = 2, ONETWO = 3 -} AOM_SCALING; +} UENUM1BYTE(AOM_SCALING); -typedef enum { +enum { // Good Quality Fast Encoding. The encoder balances quality with the amount of // time it takes to encode the output. Speed setting controls how fast. GOOD -} MODE; +} UENUM1BYTE(MODE); -typedef enum { +enum { FRAMEFLAGS_KEY = 1 << 0, FRAMEFLAGS_GOLDEN = 1 << 1, FRAMEFLAGS_BWDREF = 1 << 2, // TODO(zoeliu): To determine whether a frame flag is needed for ALTREF2_FRAME FRAMEFLAGS_ALTREF = 1 << 3, -} FRAMETYPE_FLAGS; + FRAMEFLAGS_INTRAONLY = 1 << 4, + FRAMEFLAGS_SWITCH = 1 << 5, + FRAMEFLAGS_ERROR_RESILIENT = 1 << 6, +} UENUM1BYTE(FRAMETYPE_FLAGS); -typedef enum { +enum { NO_AQ = 0, VARIANCE_AQ = 1, COMPLEXITY_AQ = 2, CYCLIC_REFRESH_AQ = 3, AQ_MODE_COUNT // This should always be the last member of the enum -} AQ_MODE; -typedef enum { +} UENUM1BYTE(AQ_MODE); +enum { NO_DELTA_Q = 0, DELTA_Q_ONLY = 1, DELTA_Q_LF = 2, DELTAQ_MODE_COUNT // This should always be the last member of the enum -} DELTAQ_MODE; +} UENUM1BYTE(DELTAQ_MODE); -typedef enum { +enum { RESIZE_NONE = 0, // No frame resizing allowed. RESIZE_FIXED = 1, // All frames are coded at the specified scale. RESIZE_RANDOM = 2, // All frames are coded at a random scale. RESIZE_MODES -} RESIZE_MODE; +} UENUM1BYTE(RESIZE_MODE); -typedef enum { +enum { SUPERRES_NONE = 0, // No frame superres allowed SUPERRES_FIXED = 1, // All frames are coded at the specified scale, // and super-resolved. @@ -126,7 +129,31 @@ typedef enum { SUPERRES_QTHRESH = 3, // Superres scale for a frame is determined based on // q_index SUPERRES_MODES -} SUPERRES_MODE; +} UENUM1BYTE(SUPERRES_MODE); + +typedef struct TplDepStats { + int64_t intra_cost; + int64_t inter_cost; + int64_t mc_flow; + int64_t mc_dep_cost; + int64_t mc_ref_cost; + + int ref_frame_index; + int_mv mv; +} TplDepStats; + +typedef struct TplDepFrame { + uint8_t is_valid; + TplDepStats *tpl_stats_ptr; + int stride; + int width; + int height; + int mi_rows; + int mi_cols; + int base_qindex; +} TplDepFrame; + +#define TPL_DEP_COST_SCALE_LOG2 4 typedef struct AV1EncoderConfig { BITSTREAM_PROFILE profile; @@ -188,6 +215,7 @@ typedef struct AV1EncoderConfig { DELTAQ_MODE deltaq_mode; int enable_cdef; int enable_restoration; + int enable_obmc; int disable_trellis_quant; int using_qm; int qm_y; @@ -247,6 +275,7 @@ typedef struct AV1EncoderConfig { int min_gf_interval; int max_gf_interval; + int gf_max_pyr_height; int row_mt; int tile_columns; @@ -256,10 +285,11 @@ typedef struct AV1EncoderConfig { int tile_widths[MAX_TILE_COLS]; int tile_heights[MAX_TILE_ROWS]; + int enable_tpl_model; + int max_threads; aom_fixed_buf_t two_pass_stats_in; - struct aom_codec_pkt_list *output_pkt_list; #if CONFIG_FP_MB_STATS aom_fixed_buf_t firstpass_mb_stats_in; @@ -275,15 +305,12 @@ typedef struct AV1EncoderConfig { int color_range; int render_width; int render_height; - aom_timing_info_type_t timing_info_type; int timing_info_present; aom_timing_info_t timing_info; int decoder_model_info_present_flag; int display_model_info_present_flag; int buffer_removal_time_present; aom_dec_model_info_t buffer_model; - aom_dec_model_op_parameters_t op_params[MAX_NUM_OPERATING_POINTS + 1]; - aom_op_timing_info_t op_frame_timing[MAX_NUM_OPERATING_POINTS + 1]; int film_grain_test_vector; const char *film_grain_table_filename; @@ -291,18 +318,36 @@ typedef struct AV1EncoderConfig { aom_superblock_size_t superblock_size; unsigned int large_scale_tile; unsigned int single_tile_decoding; - int monochrome; + uint8_t monochrome; unsigned int full_still_picture_hdr; int enable_dual_filter; unsigned int motion_vector_unit_test; const cfg_options_t *cfg; + int enable_rect_partitions; + int enable_intra_edge_filter; + int enable_tx64; int enable_order_hint; - int enable_jnt_comp; + int enable_dist_wtd_comp; int enable_ref_frame_mvs; + unsigned int max_reference_frames; unsigned int allow_ref_frame_mvs; + int enable_masked_comp; + int enable_interintra_comp; + int enable_smooth_interintra; + int enable_diff_wtd_comp; + int enable_interinter_wedge; + int enable_interintra_wedge; + int enable_global_motion; int enable_warped_motion; int allow_warped_motion; + int enable_filter_intra; + int enable_smooth_intra; + int enable_paeth_intra; + int enable_cfl_intra; int enable_superres; + int enable_palette; + int enable_intrabc; + int enable_angle_delta; unsigned int save_as_annexb; #if CONFIG_DENOISE @@ -312,6 +357,11 @@ typedef struct AV1EncoderConfig { unsigned int chroma_subsampling_x; unsigned int chroma_subsampling_y; + int reduced_tx_type_set; + int use_intra_dct_only; + int use_inter_dct_only; + int quant_b_adapt; + int border_in_pixels; } AV1EncoderConfig; static INLINE int is_lossless_requested(const AV1EncoderConfig *cfg) { @@ -404,7 +454,6 @@ typedef struct FRAME_COUNTS { [SWITCHABLE_FILTERS]; } FRAME_COUNTS; -#if CONFIG_COLLECT_INTER_MODE_RD_STATS #define INTER_MODE_RD_DATA_OVERALL_SIZE 6400 typedef struct { @@ -439,22 +488,37 @@ typedef struct inter_modes_info { int64_t est_rd_arr[MAX_INTER_MODES]; RdIdxPair rd_idx_pair_arr[MAX_INTER_MODES]; } InterModesInfo; + +// Encoder row synchronization +typedef struct AV1RowMTSyncData { +#if CONFIG_MULTITHREAD + pthread_mutex_t *mutex_; + pthread_cond_t *cond_; #endif + // Allocate memory to store the sb/mb block index in each row. + int *cur_col; + int sync_range; + int rows; +} AV1RowMTSync; + +typedef struct AV1RowMTInfo { + int current_mi_row; + int num_threads_working; +} AV1RowMTInfo; // TODO(jingning) All spatially adaptive variables should go to TileDataEnc. typedef struct TileDataEnc { TileInfo tile_info; int thresh_freq_fact[BLOCK_SIZES_ALL][MAX_MODES]; - int mode_map[BLOCK_SIZES_ALL][MAX_MODES]; int m_search_count; int ex_search_count; CFL_CTX cfl; DECLARE_ALIGNED(16, FRAME_CONTEXT, tctx); + FRAME_CONTEXT *row_ctx; uint8_t allow_update_cdf; -#if CONFIG_COLLECT_INTER_MODE_RD_STATS InterModeRdModel inter_mode_rd_models[BLOCK_SIZES_ALL]; - InterModesInfo inter_modes_info; -#endif + AV1RowMTSync row_mt_sync; + AV1RowMTInfo row_mt_info; } TileDataEnc; typedef struct { @@ -463,6 +527,13 @@ typedef struct { unsigned int count; } TOKENLIST; +typedef struct MultiThreadHandle { + int allocated_tile_rows; + int allocated_tile_cols; + int allocated_sb_rows; + int thread_id_to_tile_id[MAX_NUM_THREADS]; // Mapping of threads to tiles +} MultiThreadHandle; + typedef struct RD_COUNTS { int64_t comp_pred_diff[REFERENCE_MODES]; // Stores number of 4x4 blocks using global motion per reference frame. @@ -477,6 +548,10 @@ typedef struct ThreadData { FRAME_COUNTS *counts; PC_TREE *pc_tree; PC_TREE *pc_root[MAX_MIB_SIZE_LOG2 - MIN_MIB_SIZE_LOG2 + 1]; + tran_low_t *tree_coeff_buf[MAX_MB_PLANE]; + tran_low_t *tree_qcoeff_buf[MAX_MB_PLANE]; + tran_low_t *tree_dqcoeff_buf[MAX_MB_PLANE]; + InterModesInfo *inter_modes_info; uint32_t *hash_value_buffer[2][2]; int32_t *wsrc_buf; int32_t *mask_buf; @@ -485,7 +560,8 @@ typedef struct ThreadData { PALETTE_BUFFER *palette_buffer; CONV_BUF_TYPE *tmp_conv_dst; uint8_t *tmp_obmc_bufs[2]; - int intrabc_used_this_tile; + int intrabc_used; + FRAME_CONTEXT *tctx; } ThreadData; struct EncWorkerData; @@ -498,13 +574,13 @@ typedef struct ActiveMap { #if CONFIG_INTERNAL_STATS // types of stats -typedef enum { +enum { STAT_Y, STAT_U, STAT_V, STAT_ALL, NUM_STAT_TYPES // This should always be the last member of the enum -} StatType; +} UENUM1BYTE(StatType); typedef struct IMAGE_STAT { double stat[NUM_STAT_TYPES]; @@ -517,10 +593,14 @@ typedef struct { YV12_BUFFER_CONFIG buf; } EncRefCntBuffer; -typedef struct TileBufferEnc { - uint8_t *data; - size_t size; -} TileBufferEnc; +#if CONFIG_COLLECT_PARTITION_STATS +typedef struct PartitionStats { + int partition_decisions[6][EXT_PARTITION_TYPES]; + int partition_attempts[6][EXT_PARTITION_TYPES]; + + int partition_redo; +} PartitionStats; +#endif typedef struct AV1_COMP { QUANTS quants; @@ -545,6 +625,9 @@ typedef struct AV1_COMP { YV12_BUFFER_CONFIG *unscaled_last_source; YV12_BUFFER_CONFIG scaled_last_source; + TplDepFrame tpl_stats[MAX_LAG_BUFFERS]; + YV12_BUFFER_CONFIG *tpl_recon_frames[INTER_REFS_PER_FRAME + 1]; + // For a still frame, this flag is set to 1 to skip partition search. int partition_search_skippable_frame; double csm_rate_array[32]; @@ -553,23 +636,41 @@ typedef struct AV1_COMP { int rate_index; hash_table *previous_hash_table; int previous_index; - int cur_poc; // DebugInfo unsigned int row_mt; - int scaled_ref_idx[REF_FRAMES]; - int ref_fb_idx[REF_FRAMES]; - int refresh_fb_idx; // ref frame buffer index to refresh + RefCntBuffer *scaled_ref_buf[INTER_REFS_PER_FRAME]; - int last_show_frame_buf_idx; // last show frame buffer index + RefCntBuffer *last_show_frame_buf; // last show frame buffer + // refresh_*_frame are boolean flags. If 'refresh_xyz_frame' is true, then + // after the current frame is encoded, the XYZ reference frame gets refreshed + // (updated) to be the current frame. + // + // Special case: 'refresh_last_frame' specifies that: + // - LAST_FRAME reference should be updated to be the current frame (as usual) + // - Also, LAST2_FRAME and LAST3_FRAME references are implicitly updated to be + // the two past reference frames just before LAST_FRAME that are available. + // + // Note: Usually at most one of these refresh flags is true at a time. + // But a key-frame is special, for which all the flags are true at once. int refresh_last_frame; int refresh_golden_frame; int refresh_bwd_ref_frame; int refresh_alt2_ref_frame; int refresh_alt_ref_frame; -#if USE_SYMM_MULTI_LAYER + + // For each type of reference frame, this contains the index of a reference + // frame buffer for a reference frame of the same type. We use this to + // choose our primary reference frame (which is the most recent reference + // frame of the same type as the current frame). + int fb_of_context_type[REF_FRAMES]; + + // When true, a new rule for backward (future) reference frames is in effect: + // - BWDREF_FRAME is always the closest future frame available + // - ALTREF2_FRAME is always the 2nd closest future frame available + // - 'refresh_bwd_ref_frame' flag is used for updating both the BWDREF_FRAME + // and ALTREF2_FRAME. ('refresh_alt2_ref_frame' flag is irrelevant). int new_bwdref_update_rule; -#endif int ext_refresh_frame_flags_pending; int ext_refresh_last_frame; @@ -608,9 +709,10 @@ typedef struct AV1_COMP { RATE_CONTROL rc; double framerate; - // NOTE(zoeliu): Any inter frame allows maximum of REF_FRAMES inter - // references; Plus the currently coded frame itself, it is needed to allocate - // sufficient space to the size of the maximum possible number of frames. + // Relevant for an inter frame. + // - Index '0' corresponds to the values for the currently coded frame. + // - Indices LAST_FRAME ... EXTREF_FRAMES are used to store values for all the + // possible inter reference frames. int interp_filter_selected[REF_FRAMES + 1][SWITCHABLE]; struct aom_codec_pkt_list *output_pkt_list; @@ -620,14 +722,15 @@ typedef struct AV1_COMP { int static_mb_pct; // % forced skip mbs by segmentation int ref_frame_flags; int ext_ref_frame_flags; - RATE_FACTOR_LEVEL frame_rf_level[FRAME_BUFFERS]; + // speed is passed as a per-frame parameter into the encoder + int speed; + // sf contains fine-grained config set internally based on speed SPEED_FEATURES sf; unsigned int max_mv_magnitude; int mv_step_param; - int allow_comp_inter_inter; int all_one_sided_refs; uint8_t *segmentation_map; @@ -641,7 +744,6 @@ typedef struct AV1_COMP { uint64_t time_receive_data; uint64_t time_compress_data; uint64_t time_pick_lpf; - uint64_t time_encode_sb_row; #if CONFIG_FP_MB_STATS int use_fp_mb_stats; @@ -680,6 +782,9 @@ typedef struct AV1_COMP { Metrics metrics; #endif int b_calculate_psnr; +#if CONFIG_SPEED_STATS + unsigned int tx_search_count; +#endif // CONFIG_SPEED_STATS int droppable; @@ -705,15 +810,11 @@ typedef struct AV1_COMP { int allocated_tiles; // Keep track of memory allocated for tiles. TOKENEXTRA *tile_tok[MAX_TILE_ROWS][MAX_TILE_COLS]; - unsigned int tok_count[MAX_TILE_ROWS][MAX_TILE_COLS]; TOKENLIST *tplist[MAX_TILE_ROWS][MAX_TILE_COLS]; - TileBufferEnc tile_buffers[MAX_TILE_ROWS][MAX_TILE_COLS]; - int resize_state; int resize_avg_qp; int resize_buffer_underflow; - int resize_count; // Sequence parameters have been transmitted already and locked // or not. Once locked av1_change_config cannot change the seq @@ -727,14 +828,12 @@ typedef struct AV1_COMP { int num_workers; AVxWorker *workers; struct EncWorkerData *tile_thr_data; - int refresh_frame_mask; int existing_fb_idx_to_show; int is_arf_filter_off[MAX_EXT_ARFS + 1]; int num_extra_arfs; int arf_pos_in_gf[MAX_EXT_ARFS + 1]; int arf_pos_for_ovrly[MAX_EXT_ARFS + 1]; int global_motion_search_done; - tran_low_t *tcoeff_buf[MAX_MB_PLANE]; int extra_arf_allowed; // A flag to indicate if intrabc is ever used in current frame. int intrabc_used; @@ -743,10 +842,16 @@ typedef struct AV1_COMP { int dv_joint_cost[MV_JOINTS]; int has_lossless_segment; - // For frame refs short signaling: - // A mapping of each reference frame from its encoder side value to the - // decoder side value obtained following the short signaling procedure. - int ref_conv[REF_FRAMES]; + // Factors to control gating of compound type selection based on best + // approximate rd so far + int max_comp_type_rd_threshold_mul; + int max_comp_type_rd_threshold_div; + + unsigned int tx_domain_dist_threshold; + + // Factor to control R-D optimization of coeffs based on block + // mse. + unsigned int coeff_opt_dist_threshold; AV1LfSync lf_row_sync; AV1LrSync lr_row_sync; @@ -760,8 +865,50 @@ typedef struct AV1_COMP { // Set as 1 for monochrome and 3 for other color formats int default_interp_skip_flags; int preserve_arf_as_gld; + MultiThreadHandle multi_thread_ctxt; + void (*row_mt_sync_read_ptr)(AV1RowMTSync *const, int, int); + void (*row_mt_sync_write_ptr)(AV1RowMTSync *const, int, int, const int); +#if CONFIG_MULTITHREAD + pthread_mutex_t *row_mt_mutex_; +#endif + // Set if screen content is set or relevant tools are enabled + int is_screen_content_type; +#if CONFIG_COLLECT_PARTITION_STATS + PartitionStats partition_stats; +#endif } AV1_COMP; +typedef struct { + YV12_BUFFER_CONFIG *source; + YV12_BUFFER_CONFIG *last_source; + int64_t ts_duration; +} EncodeFrameInput; + +// EncodeFrameParams contains per-frame encoding parameters decided upon by +// av1_encode_strategy() and passed down to av1_encode() +struct EncodeFrameParams { + int error_resilient_mode; + FRAME_TYPE frame_type; + int primary_ref_frame; + int order_offset; + int show_frame; + + // This is a bitmask of which reference slots can be used in this frame + int ref_frame_flags; + + // Speed level to use for this frame: Bigger number means faster. + int speed; + + unsigned int *frame_flags; +}; +typedef struct EncodeFrameParams EncodeFrameParams; + +// EncodeFrameResults contains information about the result of encoding a +// single frame +typedef struct { + size_t size; // Size of resulting bitstream +} EncodeFrameResults; + // Must not be called more than once. void av1_initialize_enc(void); @@ -782,6 +929,11 @@ int av1_get_compressed_data(AV1_COMP *cpi, unsigned int *frame_flags, int64_t *time_end, int flush, const aom_rational_t *timebase); +int av1_encode(AV1_COMP *const cpi, uint8_t *const dest, + const EncodeFrameInput *const frame_input, + const EncodeFrameParams *const frame_params, + EncodeFrameResults *const frame_results); + int av1_get_preview_raw_frame(AV1_COMP *cpi, YV12_BUFFER_CONFIG *dest); int av1_get_last_show_frame(AV1_COMP *cpi, YV12_BUFFER_CONFIG *frame); @@ -792,12 +944,12 @@ aom_codec_err_t av1_copy_new_frame_enc(AV1_COMMON *cm, int av1_use_as_reference(AV1_COMP *cpi, int ref_frame_flags); -void av1_update_reference(AV1_COMP *cpi, int ref_frame_flags); - int av1_copy_reference_enc(AV1_COMP *cpi, int idx, YV12_BUFFER_CONFIG *sd); int av1_set_reference_enc(AV1_COMP *cpi, int idx, YV12_BUFFER_CONFIG *sd); +void av1_set_frame_size(AV1_COMP *cpi, int width, int height); + int av1_update_entropy(AV1_COMP *cpi, int update); int av1_set_active_map(AV1_COMP *cpi, unsigned char *map, int rows, int cols); @@ -811,59 +963,62 @@ int av1_get_quantizer(struct AV1_COMP *cpi); int av1_convert_sect5obus_to_annexb(uint8_t *buffer, size_t *input_size); -int64_t timebase_units_to_ticks(const aom_rational_t *timebase, int64_t n); -int64_t ticks_to_timebase_units(const aom_rational_t *timebase, int64_t n); +// av1 uses 10,000,000 ticks/second as time stamp +#define TICKS_PER_SEC 10000000LL + +static INLINE int64_t timebase_units_to_ticks(const aom_rational_t *timebase, + int64_t n) { + return n * TICKS_PER_SEC * timebase->num / timebase->den; +} + +static INLINE int64_t ticks_to_timebase_units(const aom_rational_t *timebase, + int64_t n) { + const int64_t round = TICKS_PER_SEC * timebase->num / 2 - 1; + return (n * timebase->den + round) / timebase->num / TICKS_PER_SEC; +} static INLINE int frame_is_kf_gf_arf(const AV1_COMP *cpi) { return frame_is_intra_only(&cpi->common) || cpi->refresh_alt_ref_frame || (cpi->refresh_golden_frame && !cpi->rc.is_src_frame_alt_ref); } -static INLINE int get_ref_frame_map_idx(const AV1_COMP *cpi, - MV_REFERENCE_FRAME ref_frame) { - return (ref_frame >= 1) ? cpi->ref_fb_idx[ref_frame - 1] : INVALID_IDX; -} - -static INLINE int get_ref_frame_buf_idx(const AV1_COMP *cpi, - MV_REFERENCE_FRAME ref_frame) { - const AV1_COMMON *const cm = &cpi->common; - const int map_idx = get_ref_frame_map_idx(cpi, ref_frame); - return (map_idx != INVALID_IDX) ? cm->ref_frame_map[map_idx] : INVALID_IDX; -} - // TODO(huisu@google.com, youzhou@microsoft.com): enable hash-me for HBD. static INLINE int av1_use_hash_me(const AV1_COMMON *const cm) { return cm->allow_screen_content_tools; } static INLINE hash_table *av1_get_ref_frame_hash_map( - const AV1_COMP *cpi, MV_REFERENCE_FRAME ref_frame) { - const AV1_COMMON *const cm = &cpi->common; - const int buf_idx = get_ref_frame_buf_idx(cpi, ref_frame); - return buf_idx != INVALID_IDX - ? &cm->buffer_pool->frame_bufs[buf_idx].hash_table - : NULL; + const AV1_COMMON *cm, MV_REFERENCE_FRAME ref_frame) { + const int map_idx = get_ref_frame_map_idx(cm, ref_frame); + RefCntBuffer *buf = + (map_idx != INVALID_IDX) ? cm->ref_frame_map[map_idx] : NULL; + return buf ? &buf->hash_table : NULL; } -static INLINE YV12_BUFFER_CONFIG *get_ref_frame_buffer( - const AV1_COMP *cpi, MV_REFERENCE_FRAME ref_frame) { - const AV1_COMMON *const cm = &cpi->common; - const int buf_idx = get_ref_frame_buf_idx(cpi, ref_frame); - return buf_idx != INVALID_IDX ? &cm->buffer_pool->frame_bufs[buf_idx].buf - : NULL; +static INLINE const YV12_BUFFER_CONFIG *get_ref_frame_yv12_buf( + const AV1_COMMON *const cm, MV_REFERENCE_FRAME ref_frame) { + const RefCntBuffer *const buf = get_ref_frame_buf(cm, ref_frame); + return buf != NULL ? &buf->buf : NULL; } -static INLINE int enc_is_ref_frame_buf(AV1_COMP *cpi, RefCntBuffer *frame_buf) { +static INLINE int enc_is_ref_frame_buf(const AV1_COMMON *const cm, + const RefCntBuffer *const frame_buf) { MV_REFERENCE_FRAME ref_frame; - AV1_COMMON *const cm = &cpi->common; for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) { - const int buf_idx = get_ref_frame_buf_idx(cpi, ref_frame); - if (buf_idx == INVALID_IDX) continue; - if (frame_buf == &cm->buffer_pool->frame_bufs[buf_idx]) break; + const RefCntBuffer *const buf = get_ref_frame_buf(cm, ref_frame); + if (buf == NULL) continue; + if (frame_buf == buf) break; } return (ref_frame <= ALTREF_FRAME); } +static INLINE void alloc_frame_mvs(AV1_COMMON *const cm, RefCntBuffer *buf) { + assert(buf != NULL); + ensure_mv_buffer(buf, cm); + buf->width = cm->width; + buf->height = cm->height; +} + // Token buffer is only used for palette tokens. static INLINE unsigned int get_token_alloc(int mb_rows, int mb_cols, int sb_size_log2, @@ -919,10 +1074,10 @@ static INLINE int is_altref_enabled(const AV1_COMP *const cpi) { static INLINE void set_ref_ptrs(const AV1_COMMON *cm, MACROBLOCKD *xd, MV_REFERENCE_FRAME ref0, MV_REFERENCE_FRAME ref1) { - xd->block_refs[0] = - &cm->frame_refs[ref0 >= LAST_FRAME ? ref0 - LAST_FRAME : 0]; - xd->block_refs[1] = - &cm->frame_refs[ref1 >= LAST_FRAME ? ref1 - LAST_FRAME : 0]; + xd->block_ref_scale_factors[0] = + get_ref_scale_factors_const(cm, ref0 >= LAST_FRAME ? ref0 : 1); + xd->block_ref_scale_factors[1] = + get_ref_scale_factors_const(cm, ref1 >= LAST_FRAME ? ref1 : 1); } static INLINE int get_chessboard_index(int frame_index) { @@ -935,20 +1090,10 @@ static INLINE int *cond_cost_list(const struct AV1_COMP *cpi, int *cost_list) { void av1_new_framerate(AV1_COMP *cpi, double framerate); +void av1_setup_frame_size(AV1_COMP *cpi); + #define LAYER_IDS_TO_IDX(sl, tl, num_tl) ((sl) * (num_tl) + (tl)) -// Update up-sampled reference frame index. -static INLINE void uref_cnt_fb(EncRefCntBuffer *ubufs, int *uidx, - int new_uidx) { - const int ref_index = *uidx; - - if (ref_index >= 0 && ubufs[ref_index].ref_count > 0) - ubufs[ref_index].ref_count--; - - *uidx = new_uidx; - ubufs[new_uidx].ref_count++; -} - // Returns 1 if a frame is scaled and 0 otherwise. static INLINE int av1_resize_scaled(const AV1_COMMON *cm) { return !(cm->superres_upscaled_width == cm->render_width && @@ -963,8 +1108,8 @@ static INLINE int av1_frame_scaled(const AV1_COMMON *cm) { // frame. An exception can be made for a forward keyframe since it has no // previous dependencies. static INLINE int encode_show_existing_frame(const AV1_COMMON *cm) { - return cm->show_existing_frame && - (!cm->error_resilient_mode || cm->frame_type == KEY_FRAME); + return cm->show_existing_frame && (!cm->error_resilient_mode || + cm->current_frame.frame_type == KEY_FRAME); } // Returns a Sequence Header OBU stored in an aom_fixed_buf_t, or NULL upon @@ -978,6 +1123,52 @@ static INLINE int encode_show_existing_frame(const AV1_COMMON *cm) { // field. aom_fixed_buf_t *av1_get_global_headers(AV1_COMP *cpi); +#if CONFIG_COLLECT_PARTITION_STATS +static INLINE void av1_print_partition_stats(PartitionStats *part_stats) { + FILE *f = fopen("partition_stats.csv", "w"); + if (!f) { + return; + } + + fprintf(f, "bsize,redo,"); + for (int part = 0; part < EXT_PARTITION_TYPES; part++) { + fprintf(f, "decision_%d,", part); + } + for (int part = 0; part < EXT_PARTITION_TYPES; part++) { + fprintf(f, "attempt_%d,", part); + } + fprintf(f, "\n"); + + const int bsizes[6] = { 128, 64, 32, 16, 8, 4 }; + + for (int bsize_idx = 0; bsize_idx < 6; bsize_idx++) { + fprintf(f, "%d,%d,", bsizes[bsize_idx], part_stats->partition_redo); + for (int part = 0; part < EXT_PARTITION_TYPES; part++) { + fprintf(f, "%d,", part_stats->partition_decisions[bsize_idx][part]); + } + for (int part = 0; part < EXT_PARTITION_TYPES; part++) { + fprintf(f, "%d,", part_stats->partition_attempts[bsize_idx][part]); + } + fprintf(f, "\n"); + } + fclose(f); +} + +static INLINE int av1_get_bsize_idx_for_part_stats(BLOCK_SIZE bsize) { + assert(bsize == BLOCK_128X128 || bsize == BLOCK_64X64 || + bsize == BLOCK_32X32 || bsize == BLOCK_16X16 || bsize == BLOCK_8X8); + switch (bsize) { + case BLOCK_128X128: return 0; + case BLOCK_64X64: return 1; + case BLOCK_32X32: return 2; + case BLOCK_16X16: return 3; + case BLOCK_8X8: return 4; + case BLOCK_4X4: return 5; + default: assert(0 && "Invalid bsize for partition_stats."); return -1; + } +} +#endif + #ifdef __cplusplus } // extern "C" #endif diff --git a/third_party/aom/av1/encoder/encodetxb.c b/third_party/aom/av1/encoder/encodetxb.c index 5a31d93d73..0f1a2666c4 100644 --- a/third_party/aom/av1/encoder/encodetxb.c +++ b/third_party/aom/av1/encoder/encodetxb.c @@ -284,20 +284,16 @@ static INLINE int get_sign_bit_cost(tran_low_t qc, int coeff_idx, return av1_cost_literal(1); } -static INLINE int get_br_cost(tran_low_t abs_qc, int ctx, - const int *coeff_lps) { - const tran_low_t min_level = 1 + NUM_BASE_LEVELS; - const tran_low_t max_level = 1 + NUM_BASE_LEVELS + COEFF_BASE_RANGE; - (void)ctx; - if (abs_qc >= min_level) { - if (abs_qc >= max_level) { - return coeff_lps[COEFF_BASE_RANGE]; // COEFF_BASE_RANGE * cost0; - } else { - return coeff_lps[(abs_qc - min_level)]; // * cost0 + cost1; - } - } - return 0; -} +static const int golomb_bits_cost[32] = { + 0, 512, 512 * 3, 512 * 3, 512 * 5, 512 * 5, 512 * 5, 512 * 5, + 512 * 7, 512 * 7, 512 * 7, 512 * 7, 512 * 7, 512 * 7, 512 * 7, 512 * 7, + 512 * 9, 512 * 9, 512 * 9, 512 * 9, 512 * 9, 512 * 9, 512 * 9, 512 * 9, + 512 * 9, 512 * 9, 512 * 9, 512 * 9, 512 * 9, 512 * 9, 512 * 9, 512 * 9 +}; +static const int golomb_cost_diff[32] = { + 0, 512, 512 * 2, 0, 512 * 2, 0, 0, 0, 512 * 2, 0, 0, 0, 0, 0, 0, 0, + 512 * 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +}; static INLINE int get_golomb_cost(int abs_qc) { if (abs_qc >= 1 + NUM_BASE_LEVELS + COEFF_BASE_RANGE) { @@ -308,6 +304,32 @@ static INLINE int get_golomb_cost(int abs_qc) { return 0; } +static INLINE int get_br_cost_with_diff(tran_low_t level, const int *coeff_lps, + int *diff) { + const int base_range = AOMMIN(level - 1 - NUM_BASE_LEVELS, COEFF_BASE_RANGE); + int golomb_bits = 0; + if (level <= COEFF_BASE_RANGE + 1 + NUM_BASE_LEVELS) + *diff += coeff_lps[base_range + COEFF_BASE_RANGE + 1]; + + if (level >= COEFF_BASE_RANGE + 1 + NUM_BASE_LEVELS) { + int r = level - COEFF_BASE_RANGE - NUM_BASE_LEVELS; + if (r < 32) { + golomb_bits = golomb_bits_cost[r]; + *diff += golomb_cost_diff[r]; + } else { + golomb_bits = get_golomb_cost(level); + *diff += (r & (r - 1)) == 0 ? 1024 : 0; + } + } + + return coeff_lps[base_range] + golomb_bits; +} + +static INLINE int get_br_cost(tran_low_t level, const int *coeff_lps) { + const int base_range = AOMMIN(level - 1 - NUM_BASE_LEVELS, COEFF_BASE_RANGE); + return coeff_lps[base_range] + get_golomb_cost(level); +} + static int get_coeff_cost(const tran_low_t qc, const int scan_idx, const int is_eob, const TxbInfo *const txb_info, const LV_MAP_COEFF_COST *const txb_costs, @@ -331,8 +353,7 @@ static int get_coeff_cost(const tran_low_t qc, const int scan_idx, if (abs_qc > NUM_BASE_LEVELS) { const int ctx = get_br_ctx(txb_info->levels, pos, txb_info->bwl, tx_class); - cost += get_br_cost(abs_qc, ctx, txb_costs->lps_cost[ctx]); - cost += get_golomb_cost(abs_qc); + cost += get_br_cost(abs_qc, txb_costs->lps_cost[ctx]); } } return cost; @@ -464,8 +485,6 @@ void av1_txb_init_levels_c(const tran_low_t *const coeff, const int width, const int stride = width + TX_PAD_HOR; uint8_t *ls = levels; - memset(levels - TX_PAD_TOP * stride, 0, - sizeof(*levels) * TX_PAD_TOP * stride); memset(levels + stride * height, 0, sizeof(*levels) * (TX_PAD_BOTTOM * stride + TX_PAD_END)); @@ -554,14 +573,15 @@ void av1_write_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCKD *xd, break; } - if (k_eob_offset_bits[eob_pt] > 0) { + const int eob_offset_bits = k_eob_offset_bits[eob_pt]; + if (eob_offset_bits > 0) { const int eob_ctx = eob_pt - 3; - int eob_shift = k_eob_offset_bits[eob_pt] - 1; + int eob_shift = eob_offset_bits - 1; int bit = (eob_extra & (1 << eob_shift)) ? 1 : 0; aom_write_symbol(w, bit, ec_ctx->eob_extra_cdf[txs_ctx][plane_type][eob_ctx], 2); - for (int i = 1; i < k_eob_offset_bits[eob_pt]; i++) { - eob_shift = k_eob_offset_bits[eob_pt] - 1 - i; + for (int i = 1; i < eob_offset_bits; i++) { + eob_shift = eob_offset_bits - 1 - i; bit = (eob_extra & (1 << eob_shift)) ? 1 : 0; aom_write_bit(w, bit); } @@ -588,12 +608,11 @@ void av1_write_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCKD *xd, // level is above 1. const int base_range = level - 1 - NUM_BASE_LEVELS; const int br_ctx = get_br_ctx(levels, pos, bwl, tx_class); + aom_cdf_prob *cdf = + ec_ctx->coeff_br_cdf[AOMMIN(txs_ctx, TX_32X32)][plane_type][br_ctx]; for (int idx = 0; idx < COEFF_BASE_RANGE; idx += BR_CDF_SIZE - 1) { const int k = AOMMIN(base_range - idx, BR_CDF_SIZE - 1); - aom_write_symbol( - w, k, - ec_ctx->coeff_br_cdf[AOMMIN(txs_ctx, TX_32X32)][plane_type][br_ctx], - BR_CDF_SIZE); + aom_write_symbol(w, k, cdf, BR_CDF_SIZE); if (k < BR_CDF_SIZE - 1) break; } } @@ -745,7 +764,8 @@ static AOM_FORCE_INLINE int warehouse_efficients_txb( av1_get_nz_map_contexts(levels, scan, eob, tx_size, tx_class, coeff_contexts); - const int(*lps_cost)[COEFF_BASE_RANGE + 1] = coeff_costs->lps_cost; + const int(*lps_cost)[COEFF_BASE_RANGE + 1 + COEFF_BASE_RANGE + 1] = + coeff_costs->lps_cost; int c = eob - 1; { const int pos = scan[c]; @@ -758,11 +778,8 @@ static AOM_FORCE_INLINE int warehouse_efficients_txb( if (v) { // sign bit cost if (level > NUM_BASE_LEVELS) { - const int ctx = get_br_ctx(levels, pos, bwl, tx_class); - const int base_range = - AOMMIN(level - 1 - NUM_BASE_LEVELS, COEFF_BASE_RANGE); - cost += lps_cost[ctx][base_range]; - cost += get_golomb_cost(level); + const int ctx = get_br_ctx_eob(pos, bwl, tx_class); + cost += get_br_cost(level, lps_cost[ctx]); } if (c) { cost += av1_cost_literal(1); @@ -774,7 +791,7 @@ static AOM_FORCE_INLINE int warehouse_efficients_txb( } } } - const int(*base_cost)[4] = coeff_costs->base_cost; + const int(*base_cost)[8] = coeff_costs->base_cost; for (c = eob - 2; c >= 1; --c) { const int pos = scan[c]; const int coeff_ctx = coeff_contexts[pos]; @@ -786,10 +803,7 @@ static AOM_FORCE_INLINE int warehouse_efficients_txb( cost += av1_cost_literal(1); if (level > NUM_BASE_LEVELS) { const int ctx = get_br_ctx(levels, pos, bwl, tx_class); - const int base_range = - AOMMIN(level - 1 - NUM_BASE_LEVELS, COEFF_BASE_RANGE); - cost += lps_cost[ctx][base_range]; - cost += get_golomb_cost(level); + cost += get_br_cost(level, lps_cost[ctx]); } } cost += cost0; @@ -809,10 +823,7 @@ static AOM_FORCE_INLINE int warehouse_efficients_txb( cost += coeff_costs->dc_sign_cost[dc_sign_ctx][sign01]; if (level > NUM_BASE_LEVELS) { const int ctx = get_br_ctx(levels, pos, bwl, tx_class); - const int base_range = - AOMMIN(level - 1 - NUM_BASE_LEVELS, COEFF_BASE_RANGE); - cost += lps_cost[ctx][base_range]; - cost += get_golomb_cost(level); + cost += get_br_cost(level, lps_cost[ctx]); } } } @@ -1014,13 +1025,7 @@ static int optimize_txb(TxbInfo *txb_info, const LV_MAP_COEFF_COST *txb_costs, return update; } -// These numbers are empirically obtained. -static const int plane_rd_mult[REF_TYPES][PLANE_TYPES] = { - { 17, 13 }, - { 16, 10 }, -}; - -void hbt_init() { +static void hbt_init() { hbt_hash_table = aom_malloc(sizeof(OptTxbQcoeff) * HBT_TABLE_SIZE * HBT_ARRAY_LENGTH); memset(hbt_hash_table, 0, @@ -1032,11 +1037,11 @@ void hbt_init() { void hbt_destroy() { aom_free(hbt_hash_table); } -int hbt_hash_miss(uint32_t hbt_ctx_hash, uint32_t hbt_qc_hash, - TxbInfo *txb_info, const LV_MAP_COEFF_COST *txb_costs, - const LV_MAP_EOB_COST *txb_eob_costs, - const struct macroblock_plane *p, int block, int fast_mode, - int *rate_cost) { +static int hbt_hash_miss(uint32_t hbt_ctx_hash, uint32_t hbt_qc_hash, + TxbInfo *txb_info, const LV_MAP_COEFF_COST *txb_costs, + const LV_MAP_EOB_COST *txb_eob_costs, + const struct macroblock_plane *p, int block, + int fast_mode, int *rate_cost) { (void)fast_mode; const int16_t *scan = txb_info->scan_order->scan; int prev_eob = txb_info->eob; @@ -1098,9 +1103,9 @@ int hbt_hash_miss(uint32_t hbt_ctx_hash, uint32_t hbt_qc_hash, return txb_info->eob; } -int hbt_hash_hit(uint32_t hbt_table_index, int hbt_array_index, - TxbInfo *txb_info, const struct macroblock_plane *p, int block, - int *rate_cost) { +static int hbt_hash_hit(uint32_t hbt_table_index, int hbt_array_index, + TxbInfo *txb_info, const struct macroblock_plane *p, + int block, int *rate_cost) { const int16_t *scan = txb_info->scan_order->scan; int new_eob = 0; int update = 0; @@ -1140,11 +1145,12 @@ int hbt_hash_hit(uint32_t hbt_table_index, int hbt_array_index, return txb_info->eob; } -int hbt_search_match(uint32_t hbt_ctx_hash, uint32_t hbt_qc_hash, - TxbInfo *txb_info, const LV_MAP_COEFF_COST *txb_costs, - const LV_MAP_EOB_COST *txb_eob_costs, - const struct macroblock_plane *p, int block, int fast_mode, - int *rate_cost) { +static int hbt_search_match(uint32_t hbt_ctx_hash, uint32_t hbt_qc_hash, + TxbInfo *txb_info, + const LV_MAP_COEFF_COST *txb_costs, + const LV_MAP_EOB_COST *txb_eob_costs, + const struct macroblock_plane *p, int block, + int fast_mode, int *rate_cost) { // Check for qcoeff match int hbt_array_index = hbt_qc_hash % HBT_ARRAY_LENGTH; int hbt_table_index = hbt_ctx_hash % HBT_TABLE_SIZE; @@ -1163,10 +1169,11 @@ int hbt_search_match(uint32_t hbt_ctx_hash, uint32_t hbt_qc_hash, } } -int hbt_create_hashes(TxbInfo *txb_info, const LV_MAP_COEFF_COST *txb_costs, - const LV_MAP_EOB_COST *txb_eob_costs, - const struct macroblock_plane *p, int block, - int fast_mode, int *rate_cost) { +static int hbt_create_hashes(TxbInfo *txb_info, + const LV_MAP_COEFF_COST *txb_costs, + const LV_MAP_EOB_COST *txb_eob_costs, + const struct macroblock_plane *p, int block, + int fast_mode, int *rate_cost) { // Initialize hash table if needed. if (hbt_needs_init) { hbt_init(); @@ -1288,20 +1295,47 @@ int hbt_create_hashes(TxbInfo *txb_info, const LV_MAP_COEFF_COST *txb_costs, txb_eob_costs, p, block, fast_mode, rate_cost); } -static AOM_FORCE_INLINE int get_coeff_cost_simple( +static AOM_FORCE_INLINE int get_two_coeff_cost_simple( int ci, tran_low_t abs_qc, int coeff_ctx, const LV_MAP_COEFF_COST *txb_costs, int bwl, TX_CLASS tx_class, - const uint8_t *levels) { + const uint8_t *levels, int *cost_low) { // this simple version assumes the coeff's scan_idx is not DC (scan_idx != 0) // and not the last (scan_idx != eob - 1) assert(ci > 0); int cost = txb_costs->base_cost[coeff_ctx][AOMMIN(abs_qc, 3)]; + int diff = 0; + if (abs_qc <= 3) diff = txb_costs->base_cost[coeff_ctx][abs_qc + 4]; if (abs_qc) { cost += av1_cost_literal(1); if (abs_qc > NUM_BASE_LEVELS) { const int br_ctx = get_br_ctx(levels, ci, bwl, tx_class); - cost += get_br_cost(abs_qc, br_ctx, txb_costs->lps_cost[br_ctx]); - cost += get_golomb_cost(abs_qc); + int brcost_diff = 0; + cost += get_br_cost_with_diff(abs_qc, txb_costs->lps_cost[br_ctx], + &brcost_diff); + diff += brcost_diff; + } + } + *cost_low = cost - diff; + + return cost; +} + +static INLINE int get_coeff_cost_eob(int ci, tran_low_t abs_qc, int sign, + int coeff_ctx, int dc_sign_ctx, + const LV_MAP_COEFF_COST *txb_costs, + int bwl, TX_CLASS tx_class) { + int cost = 0; + cost += txb_costs->base_eob_cost[coeff_ctx][AOMMIN(abs_qc, 3) - 1]; + if (abs_qc != 0) { + if (ci == 0) { + cost += txb_costs->dc_sign_cost[dc_sign_ctx][sign]; + } else { + cost += av1_cost_literal(1); + } + if (abs_qc > NUM_BASE_LEVELS) { + int br_ctx; + br_ctx = get_br_ctx_eob(ci, bwl, tx_class); + cost += get_br_cost(abs_qc, txb_costs->lps_cost[br_ctx]); } } return cost; @@ -1326,9 +1360,12 @@ static INLINE int get_coeff_cost_general(int is_last, int ci, tran_low_t abs_qc, cost += av1_cost_literal(1); } if (abs_qc > NUM_BASE_LEVELS) { - const int br_ctx = get_br_ctx(levels, ci, bwl, tx_class); - cost += get_br_cost(abs_qc, br_ctx, txb_costs->lps_cost[br_ctx]); - cost += get_golomb_cost(abs_qc); + int br_ctx; + if (is_last) + br_ctx = get_br_ctx_eob(ci, bwl, tx_class); + else + br_ctx = get_br_ctx(levels, ci, bwl, tx_class); + cost += get_br_cost(abs_qc, txb_costs->lps_cost[br_ctx]); } } return cost; @@ -1372,13 +1409,23 @@ static INLINE void update_coeff_general( const int64_t rd = RDCOST(rdmult, rate, dist); tran_low_t qc_low, dqc_low; - get_qc_dqc_low(abs_qc, sign, dqv, shift, &qc_low, &dqc_low); - const tran_low_t abs_qc_low = abs_qc - 1; - const int64_t dist_low = get_coeff_dist(tqc, dqc_low, shift); - const int rate_low = - get_coeff_cost_general(is_last, ci, abs_qc_low, sign, coeff_ctx, - dc_sign_ctx, txb_costs, bwl, tx_class, levels); - const int64_t rd_low = RDCOST(rdmult, rate_low, dist_low); + tran_low_t abs_qc_low; + int64_t dist_low, rd_low; + int rate_low; + if (abs_qc == 1) { + abs_qc_low = qc_low = dqc_low = 0; + dist_low = dist0; + rate_low = txb_costs->base_cost[coeff_ctx][0]; + } else { + get_qc_dqc_low(abs_qc, sign, dqv, shift, &qc_low, &dqc_low); + abs_qc_low = abs_qc - 1; + dist_low = get_coeff_dist(tqc, dqc_low, shift); + rate_low = + get_coeff_cost_general(is_last, ci, abs_qc_low, sign, coeff_ctx, + dc_sign_ctx, txb_costs, bwl, tx_class, levels); + } + + rd_low = RDCOST(rdmult, rate_low, dist_low); if (rd_low < rd) { qcoeff[ci] = qc_low; dqcoeff[ci] = dqc_low; @@ -1412,28 +1459,28 @@ static AOM_FORCE_INLINE void update_coeff_simple( *accu_rate += txb_costs->base_cost[coeff_ctx][0]; } else { const tran_low_t abs_qc = abs(qc); - const tran_low_t tqc = tcoeff[ci]; - const tran_low_t dqc = dqcoeff[ci]; - const int rate = get_coeff_cost_simple(ci, abs_qc, coeff_ctx, txb_costs, - bwl, tx_class, levels); - if (abs(dqc) < abs(tqc)) { + const tran_low_t abs_tqc = abs(tcoeff[ci]); + const tran_low_t abs_dqc = abs(dqcoeff[ci]); + int rate_low = 0; + const int rate = get_two_coeff_cost_simple( + ci, abs_qc, coeff_ctx, txb_costs, bwl, tx_class, levels, &rate_low); + if (abs_dqc < abs_tqc) { *accu_rate += rate; return; } - const int64_t dist = get_coeff_dist(tqc, dqc, shift); + + const int64_t dist = get_coeff_dist(abs_tqc, abs_dqc, shift); const int64_t rd = RDCOST(rdmult, rate, dist); - const int sign = (qc < 0) ? 1 : 0; - tran_low_t qc_low, dqc_low; - get_qc_dqc_low(abs_qc, sign, dqv, shift, &qc_low, &dqc_low); const tran_low_t abs_qc_low = abs_qc - 1; - const int64_t dist_low = get_coeff_dist(tqc, dqc_low, shift); - const int rate_low = get_coeff_cost_simple( - ci, abs_qc_low, coeff_ctx, txb_costs, bwl, tx_class, levels); + const tran_low_t abs_dqc_low = (abs_qc_low * dqv) >> shift; + const int64_t dist_low = get_coeff_dist(abs_tqc, abs_dqc_low, shift); const int64_t rd_low = RDCOST(rdmult, rate_low, dist_low); + if (rd_low < rd) { - qcoeff[ci] = qc_low; - dqcoeff[ci] = dqc_low; + const int sign = (qc < 0) ? 1 : 0; + qcoeff[ci] = (-sign ^ abs_qc_low) + sign; + dqcoeff[ci] = (-sign ^ abs_dqc_low) + sign; levels[get_padded_idx(ci, bwl)] = AOMMIN(abs_qc_low, INT8_MAX); *accu_rate += rate_low; } else { @@ -1471,40 +1518,42 @@ static AOM_FORCE_INLINE void update_coeff_eob( int64_t rd = RDCOST(rdmult, *accu_rate + rate, *accu_dist + dist); tran_low_t qc_low, dqc_low; - get_qc_dqc_low(abs_qc, sign, dqv, shift, &qc_low, &dqc_low); - const tran_low_t abs_qc_low = abs_qc - 1; - const int64_t dist_low = get_coeff_dist(tqc, dqc_low, shift) - dist0; - const int rate_low = - get_coeff_cost_general(0, ci, abs_qc_low, sign, coeff_ctx, dc_sign_ctx, - txb_costs, bwl, tx_class, levels); - const int64_t rd_low = - RDCOST(rdmult, *accu_rate + rate_low, *accu_dist + dist_low); + tran_low_t abs_qc_low; + int64_t dist_low, rd_low; + int rate_low; + if (abs_qc == 1) { + abs_qc_low = 0; + dqc_low = qc_low = 0; + dist_low = 0; + rate_low = txb_costs->base_cost[coeff_ctx][0]; + rd_low = RDCOST(rdmult, *accu_rate + rate_low, *accu_dist); + } else { + get_qc_dqc_low(abs_qc, sign, dqv, shift, &qc_low, &dqc_low); + abs_qc_low = abs_qc - 1; + dist_low = get_coeff_dist(tqc, dqc_low, shift) - dist0; + rate_low = + get_coeff_cost_general(0, ci, abs_qc_low, sign, coeff_ctx, + dc_sign_ctx, txb_costs, bwl, tx_class, levels); + rd_low = RDCOST(rdmult, *accu_rate + rate_low, *accu_dist + dist_low); + } int lower_level_new_eob = 0; const int new_eob = si + 1; - uint8_t tmp_levels[3]; - for (int ni = 0; ni < *nz_num; ++ni) { - const int last_ci = nz_ci[ni]; - tmp_levels[ni] = levels[get_padded_idx(last_ci, bwl)]; - levels[get_padded_idx(last_ci, bwl)] = 0; - } - - const int coeff_ctx_new_eob = get_lower_levels_ctx_general( - 1, si, bwl, height, levels, ci, tx_size, tx_class); + const int coeff_ctx_new_eob = get_lower_levels_ctx_eob(bwl, height, si); const int new_eob_cost = get_eob_cost(new_eob, txb_eob_costs, txb_costs, tx_class); int rate_coeff_eob = - new_eob_cost + get_coeff_cost_general(1, ci, abs_qc, sign, - coeff_ctx_new_eob, dc_sign_ctx, - txb_costs, bwl, tx_class, levels); + new_eob_cost + get_coeff_cost_eob(ci, abs_qc, sign, coeff_ctx_new_eob, + dc_sign_ctx, txb_costs, bwl, + tx_class); int64_t dist_new_eob = dist; int64_t rd_new_eob = RDCOST(rdmult, rate_coeff_eob, dist_new_eob); if (abs_qc_low > 0) { const int rate_coeff_eob_low = - new_eob_cost + - get_coeff_cost_general(1, ci, abs_qc_low, sign, coeff_ctx_new_eob, - dc_sign_ctx, txb_costs, bwl, tx_class, levels); + new_eob_cost + get_coeff_cost_eob(ci, abs_qc_low, sign, + coeff_ctx_new_eob, dc_sign_ctx, + txb_costs, bwl, tx_class); const int64_t dist_new_eob_low = dist_low; const int64_t rd_new_eob_low = RDCOST(rdmult, rate_coeff_eob_low, dist_new_eob_low); @@ -1526,7 +1575,7 @@ static AOM_FORCE_INLINE void update_coeff_eob( if (sharpness == 0 && rd_new_eob < rd) { for (int ni = 0; ni < *nz_num; ++ni) { int last_ci = nz_ci[ni]; - // levels[get_padded_idx(last_ci, bwl)] = 0; + levels[get_padded_idx(last_ci, bwl)] = 0; qcoeff[last_ci] = 0; dqcoeff[last_ci] = 0; } @@ -1536,10 +1585,6 @@ static AOM_FORCE_INLINE void update_coeff_eob( *accu_dist = dist_new_eob; lower_level = lower_level_new_eob; } else { - for (int ni = 0; ni < *nz_num; ++ni) { - const int last_ci = nz_ci[ni]; - levels[get_padded_idx(last_ci, bwl)] = tmp_levels[ni]; - } *accu_rate += rate; *accu_dist += dist; } @@ -1619,14 +1664,14 @@ int av1_optimize_txb_new(const struct AV1_COMP *cpi, MACROBLOCK *x, int plane, uint8_t levels_buf[TX_PAD_2D]; uint8_t *const levels = set_levels(levels_buf, width); + int eob = p->eobs[block]; - av1_txb_init_levels(qcoeff, width, height, levels); + if (eob > 1) av1_txb_init_levels(qcoeff, width, height, levels); // TODO(angirbird): check iqmatrix const int non_skip_cost = txb_costs->txb_skip_cost[txb_ctx->txb_skip_ctx][0]; const int skip_cost = txb_costs->txb_skip_cost[txb_ctx->txb_skip_ctx][1]; - int eob = p->eobs[block]; const int eob_cost = get_eob_cost(eob, txb_eob_costs, txb_costs, tx_class); int accu_rate = eob_cost; int64_t accu_dist = 0; @@ -1646,11 +1691,10 @@ int av1_optimize_txb_new(const struct AV1_COMP *cpi, MACROBLOCK *x, int plane, --si; } else { assert(abs_qc == 1); - const int coeff_ctx = get_lower_levels_ctx_general( - 1, si, bwl, height, levels, ci, tx_size, tx_class); - accu_rate += get_coeff_cost_general(1, ci, abs_qc, sign, coeff_ctx, - txb_ctx->dc_sign_ctx, txb_costs, bwl, - tx_class, levels); + const int coeff_ctx = get_lower_levels_ctx_eob(bwl, height, si); + accu_rate += + get_coeff_cost_eob(ci, abs_qc, sign, coeff_ctx, txb_ctx->dc_sign_ctx, + txb_costs, bwl, tx_class); const tran_low_t tqc = tcoeff[ci]; const tran_low_t dqc = dqcoeff[ci]; const int64_t dist = get_coeff_dist(tqc, dqc, shift); @@ -1767,10 +1811,9 @@ int av1_optimize_txb(const struct AV1_COMP *cpi, MACROBLOCK *x, int plane, assert(width == (1 << bwl)); const int tx_type_cost = get_tx_type_cost(cm, x, xd, plane, tx_size, tx_type); TxbInfo txb_info = { - qcoeff, levels, dqcoeff, tcoeff, dequant, shift, - tx_size, txs_ctx, tx_type, bwl, width, height, - eob, seg_eob, scan_order, txb_ctx, rdmult, &cm->coeff_ctx_table, - iqmatrix, tx_type_cost, + qcoeff, levels, dqcoeff, tcoeff, dequant, shift, tx_size, + txs_ctx, tx_type, bwl, width, height, eob, seg_eob, + scan_order, txb_ctx, rdmult, iqmatrix, tx_type_cost, }; // Hash based trellis (hbt) speed feature: avoid expensive optimize_txb calls diff --git a/third_party/aom/av1/encoder/encodetxb.h b/third_party/aom/av1/encoder/encodetxb.h index 40ae343b0d..bfc49dea8c 100644 --- a/third_party/aom/av1/encoder/encodetxb.h +++ b/third_party/aom/av1/encoder/encodetxb.h @@ -42,7 +42,6 @@ typedef struct TxbInfo { const SCAN_ORDER *scan_order; TXB_CTX *txb_ctx; int64_t rdmult; - const LV_MAP_CTX_TABLE *coeff_ctx_table; const qm_val_t *iqmatrix; int tx_type_cost; } TxbInfo; @@ -80,6 +79,13 @@ int av1_optimize_txb_new(const struct AV1_COMP *cpi, MACROBLOCK *x, int plane, int block, TX_SIZE tx_size, TX_TYPE tx_type, const TXB_CTX *const txb_ctx, int *rate_cost, int sharpness); + +// These numbers are empirically obtained. +static const int plane_rd_mult[REF_TYPES][PLANE_TYPES] = { + { 17, 13 }, + { 16, 10 }, +}; + #ifdef __cplusplus } #endif diff --git a/third_party/aom/av1/encoder/ethread.c b/third_party/aom/av1/encoder/ethread.c index e8ac30bb52..77fb811b14 100644 --- a/third_party/aom/av1/encoder/ethread.c +++ b/third_party/aom/av1/encoder/ethread.c @@ -9,9 +9,11 @@ * PATENTS file, you can obtain it at www.aomedia.org/license/patent. */ +#include "av1/encoder/av1_multi_thread.h" #include "av1/encoder/encodeframe.h" #include "av1/encoder/encoder.h" #include "av1/encoder/ethread.h" +#include "av1/encoder/rdopt.h" #include "aom_dsp/aom_dsp_common.h" static void accumulate_rd_opt(ThreadData *td, ThreadData *td_t) { @@ -27,6 +29,318 @@ static void accumulate_rd_opt(ThreadData *td, ThreadData *td_t) { td->rd_counts.skip_mode_used_flag |= td_t->rd_counts.skip_mode_used_flag; } +static void update_delta_lf_for_row_mt(AV1_COMP *cpi) { + AV1_COMMON *cm = &cpi->common; + MACROBLOCKD *xd = &cpi->td.mb.e_mbd; + const int mib_size = cm->seq_params.mib_size; + const int frame_lf_count = + av1_num_planes(cm) > 1 ? FRAME_LF_COUNT : FRAME_LF_COUNT - 2; + for (int row = 0; row < cm->tile_rows; row++) { + for (int col = 0; col < cm->tile_cols; col++) { + TileDataEnc *tile_data = &cpi->tile_data[row * cm->tile_cols + col]; + const TileInfo *const tile_info = &tile_data->tile_info; + for (int mi_row = tile_info->mi_row_start; mi_row < tile_info->mi_row_end; + mi_row += mib_size) { + if (mi_row == tile_info->mi_row_start) + av1_reset_loop_filter_delta(xd, av1_num_planes(cm)); + for (int mi_col = tile_info->mi_col_start; + mi_col < tile_info->mi_col_end; mi_col += mib_size) { + const int idx_str = cm->mi_stride * mi_row + mi_col; + MB_MODE_INFO **mi = cm->mi_grid_visible + idx_str; + MB_MODE_INFO *mbmi = mi[0]; + if (mbmi->skip == 1 && (mbmi->sb_type == cm->seq_params.sb_size)) { + for (int lf_id = 0; lf_id < frame_lf_count; ++lf_id) + mbmi->delta_lf[lf_id] = xd->delta_lf[lf_id]; + mbmi->delta_lf_from_base = xd->delta_lf_from_base; + } else { + if (cm->delta_q_info.delta_lf_multi) { + for (int lf_id = 0; lf_id < frame_lf_count; ++lf_id) + xd->delta_lf[lf_id] = mbmi->delta_lf[lf_id]; + } else { + xd->delta_lf_from_base = mbmi->delta_lf_from_base; + } + } + } + } + } + } +} + +void av1_row_mt_sync_read_dummy(struct AV1RowMTSyncData *const row_mt_sync, + int r, int c) { + (void)row_mt_sync; + (void)r; + (void)c; + return; +} + +void av1_row_mt_sync_write_dummy(struct AV1RowMTSyncData *const row_mt_sync, + int r, int c, const int cols) { + (void)row_mt_sync; + (void)r; + (void)c; + (void)cols; + return; +} + +void av1_row_mt_sync_read(AV1RowMTSync *const row_mt_sync, int r, int c) { +#if CONFIG_MULTITHREAD + const int nsync = row_mt_sync->sync_range; + + if (r) { + pthread_mutex_t *const mutex = &row_mt_sync->mutex_[r - 1]; + pthread_mutex_lock(mutex); + + while (c > row_mt_sync->cur_col[r - 1] - nsync) { + pthread_cond_wait(&row_mt_sync->cond_[r - 1], mutex); + } + pthread_mutex_unlock(mutex); + } +#else + (void)row_mt_sync; + (void)r; + (void)c; +#endif // CONFIG_MULTITHREAD +} + +void av1_row_mt_sync_write(AV1RowMTSync *const row_mt_sync, int r, int c, + const int cols) { +#if CONFIG_MULTITHREAD + const int nsync = row_mt_sync->sync_range; + int cur; + // Only signal when there are enough encoded blocks for next row to run. + int sig = 1; + + if (c < cols - 1) { + cur = c; + if (c % nsync) sig = 0; + } else { + cur = cols + nsync; + } + + if (sig) { + pthread_mutex_lock(&row_mt_sync->mutex_[r]); + + row_mt_sync->cur_col[r] = cur; + + pthread_cond_signal(&row_mt_sync->cond_[r]); + pthread_mutex_unlock(&row_mt_sync->mutex_[r]); + } +#else + (void)row_mt_sync; + (void)r; + (void)c; + (void)cols; +#endif // CONFIG_MULTITHREAD +} + +// Allocate memory for row synchronization +void av1_row_mt_sync_mem_alloc(AV1RowMTSync *row_mt_sync, AV1_COMMON *cm, + int rows) { + row_mt_sync->rows = rows; +#if CONFIG_MULTITHREAD + { + int i; + + CHECK_MEM_ERROR(cm, row_mt_sync->mutex_, + aom_malloc(sizeof(*row_mt_sync->mutex_) * rows)); + if (row_mt_sync->mutex_) { + for (i = 0; i < rows; ++i) { + pthread_mutex_init(&row_mt_sync->mutex_[i], NULL); + } + } + + CHECK_MEM_ERROR(cm, row_mt_sync->cond_, + aom_malloc(sizeof(*row_mt_sync->cond_) * rows)); + if (row_mt_sync->cond_) { + for (i = 0; i < rows; ++i) { + pthread_cond_init(&row_mt_sync->cond_[i], NULL); + } + } + } +#endif // CONFIG_MULTITHREAD + + CHECK_MEM_ERROR(cm, row_mt_sync->cur_col, + aom_malloc(sizeof(*row_mt_sync->cur_col) * rows)); + + // Set up nsync. + row_mt_sync->sync_range = 1; +} + +// Deallocate row based multi-threading synchronization related mutex and data +void av1_row_mt_sync_mem_dealloc(AV1RowMTSync *row_mt_sync) { + if (row_mt_sync != NULL) { +#if CONFIG_MULTITHREAD + int i; + + if (row_mt_sync->mutex_ != NULL) { + for (i = 0; i < row_mt_sync->rows; ++i) { + pthread_mutex_destroy(&row_mt_sync->mutex_[i]); + } + aom_free(row_mt_sync->mutex_); + } + if (row_mt_sync->cond_ != NULL) { + for (i = 0; i < row_mt_sync->rows; ++i) { + pthread_cond_destroy(&row_mt_sync->cond_[i]); + } + aom_free(row_mt_sync->cond_); + } +#endif // CONFIG_MULTITHREAD + aom_free(row_mt_sync->cur_col); + // clear the structure as the source of this call may be dynamic change + // in tiles in which case this call will be followed by an _alloc() + // which may fail. + av1_zero(*row_mt_sync); + } +} + +static void assign_tile_to_thread(MultiThreadHandle *multi_thread_ctxt, + int num_tiles, int num_workers) { + int tile_id = 0; + int i; + + for (i = 0; i < num_workers; i++) { + multi_thread_ctxt->thread_id_to_tile_id[i] = tile_id++; + if (tile_id == num_tiles) tile_id = 0; + } +} + +static int get_next_job(AV1_COMP *const cpi, int *current_mi_row, + int cur_tile_id) { + AV1_COMMON *const cm = &cpi->common; + TileDataEnc *const this_tile = &cpi->tile_data[cur_tile_id]; + AV1RowMTInfo *row_mt_info = &this_tile->row_mt_info; + + if (row_mt_info->current_mi_row < this_tile->tile_info.mi_row_end) { + *current_mi_row = row_mt_info->current_mi_row; + row_mt_info->num_threads_working++; + row_mt_info->current_mi_row += cm->seq_params.mib_size; + return 1; + } + return 0; +} + +static void switch_tile_and_get_next_job(AV1_COMP *const cpi, int *cur_tile_id, + int *current_mi_row, + int *end_of_frame) { + AV1_COMMON *const cm = &cpi->common; + const int tile_cols = cm->tile_cols; + const int tile_rows = cm->tile_rows; + + int tile_id = -1; // Stores the tile ID with minimum proc done + int max_mis_to_encode = 0; + int min_num_threads_working = INT_MAX; + + for (int tile_row = 0; tile_row < tile_rows; tile_row++) { + for (int tile_col = 0; tile_col < tile_cols; tile_col++) { + int tile_index = tile_row * tile_cols + tile_col; + TileDataEnc *this_tile = &cpi->tile_data[tile_index]; + AV1RowMTInfo *row_mt_info = &this_tile->row_mt_info; + int num_mis_to_encode = + this_tile->tile_info.mi_row_end - row_mt_info->current_mi_row; + + // Tile to be processed by this thread is selected on the basis of + // availability of jobs: + // 1) If jobs are available, tile to be processed is chosen on the + // basis of minimum number of threads working for that tile. If two or + // more tiles have same number of threads working for them, then the tile + // with maximum number of jobs available will be chosen. + // 2) If no jobs are available, then end_of_frame is reached. + if (num_mis_to_encode > 0) { + int num_threads_working = row_mt_info->num_threads_working; + if (num_threads_working < min_num_threads_working) { + min_num_threads_working = num_threads_working; + max_mis_to_encode = 0; + } + if (num_threads_working == min_num_threads_working && + num_mis_to_encode > max_mis_to_encode) { + tile_id = tile_index; + max_mis_to_encode = num_mis_to_encode; + } + } + } + } + if (tile_id == -1) { + *end_of_frame = 1; + } else { + // Update the cur ID to the next tile ID that will be processed, + // which will be the least processed tile + *cur_tile_id = tile_id; + get_next_job(cpi, current_mi_row, *cur_tile_id); + } +} + +static int enc_row_mt_worker_hook(void *arg1, void *unused) { + EncWorkerData *const thread_data = (EncWorkerData *)arg1; + AV1_COMP *const cpi = thread_data->cpi; + AV1_COMMON *const cm = &cpi->common; + + MultiThreadHandle *multi_thread_ctxt = &cpi->multi_thread_ctxt; + int thread_id = thread_data->thread_id; + int cur_tile_id = multi_thread_ctxt->thread_id_to_tile_id[thread_id]; + (void)unused; + + assert(cur_tile_id != -1); + + int end_of_frame = 0; + while (1) { + int current_mi_row = -1; +#if CONFIG_MULTITHREAD + pthread_mutex_lock(cpi->row_mt_mutex_); +#endif + if (!get_next_job(cpi, ¤t_mi_row, cur_tile_id)) { + // No jobs are available for the current tile. Query for the status of + // other tiles and get the next job if available + switch_tile_and_get_next_job(cpi, &cur_tile_id, ¤t_mi_row, + &end_of_frame); + } +#if CONFIG_MULTITHREAD + pthread_mutex_unlock(cpi->row_mt_mutex_); +#endif + if (end_of_frame == 1) break; + + TileDataEnc *const this_tile = &cpi->tile_data[cur_tile_id]; + int tile_row = this_tile->tile_info.tile_row; + int tile_col = this_tile->tile_info.tile_col; + + assert(current_mi_row != -1 && + current_mi_row <= this_tile->tile_info.mi_row_end); + + ThreadData *td = thread_data->td; + + td->mb.e_mbd.tile_ctx = td->tctx; + td->mb.tile_pb_ctx = &this_tile->tctx; + if (this_tile->allow_update_cdf) { + td->mb.row_ctx = this_tile->row_ctx; + if (current_mi_row == this_tile->tile_info.mi_row_start) + memcpy(td->mb.e_mbd.tile_ctx, &this_tile->tctx, sizeof(FRAME_CONTEXT)); + } else { + memcpy(td->mb.e_mbd.tile_ctx, &this_tile->tctx, sizeof(FRAME_CONTEXT)); + } + + av1_init_above_context(cm, &td->mb.e_mbd, tile_row); + + // Disable exhaustive search speed features for row based multi-threading of + // encoder. + td->mb.m_search_count_ptr = NULL; + td->mb.ex_search_count_ptr = NULL; + + cfl_init(&td->mb.e_mbd.cfl, &cm->seq_params); + av1_crc32c_calculator_init(&td->mb.mb_rd_record.crc_calculator); + + av1_encode_sb_row(cpi, td, tile_row, tile_col, current_mi_row); +#if CONFIG_MULTITHREAD + pthread_mutex_lock(cpi->row_mt_mutex_); +#endif + this_tile->row_mt_info.num_threads_working--; +#if CONFIG_MULTITHREAD + pthread_mutex_unlock(cpi->row_mt_mutex_); +#endif + } + + return 1; +} + static int enc_worker_hook(void *arg1, void *unused) { EncWorkerData *const thread_data = (EncWorkerData *)arg1; AV1_COMP *const cpi = thread_data->cpi; @@ -42,6 +356,10 @@ static int enc_worker_hook(void *arg1, void *unused) { int tile_row = t / tile_cols; int tile_col = t % tile_cols; + TileDataEnc *const this_tile = + &cpi->tile_data[tile_row * cm->tile_cols + tile_col]; + thread_data->td->mb.e_mbd.tile_ctx = &this_tile->tctx; + thread_data->td->mb.tile_pb_ctx = &this_tile->tctx; av1_encode_tile(cpi, thread_data->td, tile_row, tile_col); } @@ -58,16 +376,28 @@ static void create_enc_workers(AV1_COMP *cpi, int num_workers) { CHECK_MEM_ERROR(cm, cpi->tile_thr_data, aom_calloc(num_workers, sizeof(*cpi->tile_thr_data))); - for (int i = 0; i < num_workers; i++) { +#if CONFIG_MULTITHREAD + if (cpi->row_mt == 1) { + if (cpi->row_mt_mutex_ == NULL) { + CHECK_MEM_ERROR(cm, cpi->row_mt_mutex_, + aom_malloc(sizeof(*(cpi->row_mt_mutex_)))); + if (cpi->row_mt_mutex_) pthread_mutex_init(cpi->row_mt_mutex_, NULL); + } + } +#endif + + for (int i = num_workers - 1; i >= 0; i--) { AVxWorker *const worker = &cpi->workers[i]; EncWorkerData *const thread_data = &cpi->tile_thr_data[i]; ++cpi->num_workers; winterface->init(worker); + worker->thread_name = "aom enc worker"; thread_data->cpi = cpi; + thread_data->thread_id = i; - if (i < num_workers - 1) { + if (i > 0) { // Allocate thread data. CHECK_MEM_ERROR(cm, thread_data->td, aom_memalign(32, sizeof(*thread_data->td))); @@ -91,6 +421,10 @@ static void create_enc_workers(AV1_COMP *cpi, int num_workers) { (int32_t *)aom_memalign( 16, MAX_SB_SQUARE * sizeof(*thread_data->td->wsrc_buf))); + CHECK_MEM_ERROR(cm, thread_data->td->inter_modes_info, + (InterModesInfo *)aom_malloc( + sizeof(*thread_data->td->inter_modes_info))); + for (int x = 0; x < 2; x++) for (int y = 0; y < 2; y++) CHECK_MEM_ERROR( @@ -119,7 +453,7 @@ static void create_enc_workers(AV1_COMP *cpi, int num_workers) { for (int j = 0; j < 2; ++j) { CHECK_MEM_ERROR( cm, thread_data->td->tmp_obmc_bufs[j], - aom_memalign(16, 2 * MAX_MB_PLANE * MAX_SB_SQUARE * + aom_memalign(32, 2 * MAX_MB_PLANE * MAX_SB_SQUARE * sizeof(*thread_data->td->tmp_obmc_bufs[j]))); } @@ -131,6 +465,10 @@ static void create_enc_workers(AV1_COMP *cpi, int num_workers) { // Main thread acts as a worker and uses the thread data in cpi. thread_data->td = &cpi->td; } + if (cpi->row_mt == 1) + CHECK_MEM_ERROR( + cm, thread_data->td->tctx, + (FRAME_CONTEXT *)aom_memalign(16, sizeof(*thread_data->td->tctx))); winterface->sync(worker); } } @@ -138,14 +476,14 @@ static void create_enc_workers(AV1_COMP *cpi, int num_workers) { static void launch_enc_workers(AV1_COMP *cpi, int num_workers) { const AVxWorkerInterface *const winterface = aom_get_worker_interface(); // Encode a frame - for (int i = 0; i < num_workers; i++) { + for (int i = num_workers - 1; i >= 0; i--) { AVxWorker *const worker = &cpi->workers[i]; EncWorkerData *const thread_data = (EncWorkerData *)worker->data1; // Set the starting tile for each thread. thread_data->start = i; - if (i == cpi->num_workers - 1) + if (i == 0) winterface->execute(worker); else winterface->launch(worker); @@ -154,31 +492,39 @@ static void launch_enc_workers(AV1_COMP *cpi, int num_workers) { static void sync_enc_workers(AV1_COMP *cpi, int num_workers) { const AVxWorkerInterface *const winterface = aom_get_worker_interface(); + int had_error = 0; // Encoding ends. - for (int i = 0; i < num_workers; i++) { + for (int i = num_workers - 1; i >= 0; i--) { AVxWorker *const worker = &cpi->workers[i]; - winterface->sync(worker); + had_error |= !winterface->sync(worker); } + + if (had_error) + aom_internal_error(&cpi->common.error, AOM_CODEC_ERROR, + "Failed to encode tile data"); } static void accumulate_counters_enc_workers(AV1_COMP *cpi, int num_workers) { - for (int i = 0; i < num_workers; i++) { + for (int i = num_workers - 1; i >= 0; i--) { AVxWorker *const worker = &cpi->workers[i]; EncWorkerData *const thread_data = (EncWorkerData *)worker->data1; - cpi->intrabc_used |= thread_data->td->intrabc_used_this_tile; + cpi->intrabc_used |= thread_data->td->intrabc_used; // Accumulate counters. - if (i < cpi->num_workers - 1) { + if (i > 0) { av1_accumulate_frame_counts(&cpi->counts, thread_data->td->counts); accumulate_rd_opt(&cpi->td, thread_data->td); cpi->td.mb.txb_split_count += thread_data->td->mb.txb_split_count; +#if CONFIG_SPEED_STATS + cpi->td.mb.tx_search_count += thread_data->td->mb.tx_search_count; +#endif // CONFIG_SPEED_STATS } } } static void prepare_enc_workers(AV1_COMP *cpi, AVxWorkerHook hook, int num_workers) { - for (int i = 0; i < num_workers; i++) { + for (int i = num_workers - 1; i >= 0; i--) { AVxWorker *const worker = &cpi->workers[i]; EncWorkerData *const thread_data = &cpi->tile_thr_data[i]; @@ -186,6 +532,8 @@ static void prepare_enc_workers(AV1_COMP *cpi, AVxWorkerHook hook, worker->data1 = thread_data; worker->data2 = NULL; + thread_data->td->intrabc_used = 0; + // Before encoding a frame, copy the thread data from cpi. if (thread_data->td != &cpi->td) { thread_data->td->mb = cpi->td.mb; @@ -193,6 +541,8 @@ static void prepare_enc_workers(AV1_COMP *cpi, AVxWorkerHook hook, thread_data->td->mb.above_pred_buf = thread_data->td->above_pred_buf; thread_data->td->mb.left_pred_buf = thread_data->td->left_pred_buf; thread_data->td->mb.wsrc_buf = thread_data->td->wsrc_buf; + + thread_data->td->mb.inter_modes_info = thread_data->td->inter_modes_info; for (int x = 0; x < 2; x++) { for (int y = 0; y < 2; y++) { memcpy(thread_data->td->hash_value_buffer[x][y], @@ -209,7 +559,7 @@ static void prepare_enc_workers(AV1_COMP *cpi, AVxWorkerHook hook, memcpy(thread_data->td->counts, &cpi->counts, sizeof(cpi->counts)); } - if (i < num_workers - 1) { + if (i > 0) { thread_data->td->mb.palette_buffer = thread_data->td->palette_buffer; thread_data->td->mb.tmp_conv_dst = thread_data->td->tmp_conv_dst; for (int j = 0; j < 2; ++j) { @@ -259,3 +609,74 @@ void av1_accumulate_frame_counts(FRAME_COUNTS *acc_counts, for (unsigned int i = 0; i < n_counts; i++) acc[i] += cnt[i]; } + +void av1_encode_tiles_row_mt(AV1_COMP *cpi) { + AV1_COMMON *const cm = &cpi->common; + const int tile_cols = cm->tile_cols; + const int tile_rows = cm->tile_rows; + MultiThreadHandle *multi_thread_ctxt = &cpi->multi_thread_ctxt; + int num_workers = 0; + int total_num_sb_rows = 0; + int max_sb_rows = 0; + + if (cpi->tile_data == NULL || cpi->allocated_tiles < tile_cols * tile_rows) { + av1_row_mt_mem_dealloc(cpi); + av1_alloc_tile_data(cpi); + } + + av1_init_tile_data(cpi); + + for (int row = 0; row < tile_rows; row++) { + for (int col = 0; col < tile_cols; col++) { + TileDataEnc *tile_data = &cpi->tile_data[row * cm->tile_cols + col]; + int num_sb_rows_in_tile = + av1_get_sb_rows_in_tile(cm, tile_data->tile_info); + total_num_sb_rows += num_sb_rows_in_tile; + max_sb_rows = AOMMAX(max_sb_rows, num_sb_rows_in_tile); + } + } + num_workers = AOMMIN(cpi->oxcf.max_threads, total_num_sb_rows); + + if (multi_thread_ctxt->allocated_tile_cols != tile_cols || + multi_thread_ctxt->allocated_tile_rows != tile_rows || + multi_thread_ctxt->allocated_sb_rows != max_sb_rows) { + av1_row_mt_mem_dealloc(cpi); + av1_row_mt_mem_alloc(cpi, max_sb_rows); + } + + memset(multi_thread_ctxt->thread_id_to_tile_id, -1, + sizeof(*multi_thread_ctxt->thread_id_to_tile_id) * MAX_NUM_THREADS); + + for (int tile_row = 0; tile_row < tile_rows; tile_row++) { + for (int tile_col = 0; tile_col < tile_cols; tile_col++) { + int tile_id = tile_row * tile_cols + tile_col; + TileDataEnc *this_tile = &cpi->tile_data[tile_id]; + + // Initialize cur_col to -1 for all rows. + memset(this_tile->row_mt_sync.cur_col, -1, + sizeof(*this_tile->row_mt_sync.cur_col) * max_sb_rows); + this_tile->row_mt_info.current_mi_row = this_tile->tile_info.mi_row_start; + this_tile->row_mt_info.num_threads_working = 0; + + av1_inter_mode_data_init(this_tile); + av1_zero_above_context(cm, &cpi->td.mb.e_mbd, + this_tile->tile_info.mi_col_start, + this_tile->tile_info.mi_col_end, tile_row); + this_tile->m_search_count = 0; // Count of motion search hits. + this_tile->ex_search_count = 0; // Exhaustive mesh search hits. + } + } + + // Only run once to create threads and allocate thread data. + if (cpi->num_workers == 0) { + create_enc_workers(cpi, num_workers); + } else { + num_workers = AOMMIN(num_workers, cpi->num_workers); + } + assign_tile_to_thread(multi_thread_ctxt, tile_cols * tile_rows, num_workers); + prepare_enc_workers(cpi, enc_row_mt_worker_hook, num_workers); + launch_enc_workers(cpi, num_workers); + sync_enc_workers(cpi, num_workers); + if (cm->delta_q_info.delta_lf_present_flag) update_delta_lf_for_row_mt(cpi); + accumulate_counters_enc_workers(cpi, num_workers); +} diff --git a/third_party/aom/av1/encoder/ethread.h b/third_party/aom/av1/encoder/ethread.h index 5de4b48038..1830759504 100644 --- a/third_party/aom/av1/encoder/ethread.h +++ b/third_party/aom/av1/encoder/ethread.h @@ -18,14 +18,31 @@ extern "C" { struct AV1_COMP; struct ThreadData; +struct AV1RowMTSyncData; typedef struct EncWorkerData { struct AV1_COMP *cpi; struct ThreadData *td; int start; + int thread_id; } EncWorkerData; +void av1_row_mt_sync_read(AV1RowMTSync *const row_mt_sync, int r, int c); +void av1_row_mt_sync_write(AV1RowMTSync *const row_mt_sync, int r, int c, + const int cols); + +void av1_row_mt_sync_read_dummy(struct AV1RowMTSyncData *const row_mt_sync, + int r, int c); +void av1_row_mt_sync_write_dummy(struct AV1RowMTSyncData *const row_mt_sync, + int r, int c, const int cols); + +void av1_row_mt_sync_mem_dealloc(AV1RowMTSync *row_mt_sync); +// Allocate memory for row based multi-threading synchronization. +void av1_row_mt_sync_mem_alloc(AV1RowMTSync *row_mt_sync, struct AV1Common *cm, + int rows); + void av1_encode_tiles_mt(struct AV1_COMP *cpi); +void av1_encode_tiles_row_mt(struct AV1_COMP *cpi); void av1_accumulate_frame_counts(struct FRAME_COUNTS *acc_counts, const struct FRAME_COUNTS *counts); diff --git a/third_party/aom/av1/encoder/firstpass.c b/third_party/aom/av1/encoder/firstpass.c index 69dd20c526..8b9513b645 100644 --- a/third_party/aom/av1/encoder/firstpass.c +++ b/third_party/aom/av1/encoder/firstpass.c @@ -36,6 +36,7 @@ #include "av1/encoder/encodemb.h" #include "av1/encoder/encodemv.h" #include "av1/encoder/encoder.h" +#include "av1/encoder/encode_strategy.h" #include "av1/encoder/extend.h" #include "av1/encoder/firstpass.h" #include "av1/encoder/mcomp.h" @@ -58,7 +59,8 @@ #define KF_MAX_FRAME_BOOST 128.0 #define MIN_ARF_GF_BOOST 240 #define MIN_DECAY_FACTOR 0.01 -#define MIN_KF_BOOST 300 +#define MIN_KF_BOOST 300 // Minimum boost for non-static KF interval +#define MIN_STATIC_KF_BOOST 5400 // Minimum boost for static KF interval #define NEW_MV_MODE_PENALTY 32 #define DARK_THRESH 64 #define DEFAULT_GRP_WEIGHT 1.0 @@ -379,7 +381,7 @@ static void first_pass_motion_search(AV1_COMP *cpi, MACROBLOCK *x, // Override the default variance function to use MSE. v_fn_ptr.vf = get_block_variance_fn(bsize); - if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + if (is_cur_buf_hbd(xd)) { v_fn_ptr.vf = highbd_get_block_variance_fn(bsize, xd->bd); } @@ -448,18 +450,6 @@ static int find_fp_qindex(aom_bit_depth_t bit_depth) { return i; } -static void set_first_pass_params(AV1_COMP *cpi) { - AV1_COMMON *const cm = &cpi->common; - if (!cpi->refresh_alt_ref_frame && - (cm->current_video_frame == 0 || (cpi->frame_flags & FRAMEFLAGS_KEY))) { - cm->frame_type = KEY_FRAME; - } else { - cm->frame_type = INTER_FRAME; - } - // Do not use periodic key frames. - cpi->rc.frames_to_key = INT_MAX; -} - static double raw_motion_error_stdev(int *raw_motion_err_list, int raw_motion_err_counts) { int64_t sum_raw_err = 0; @@ -485,10 +475,11 @@ static double raw_motion_error_stdev(int *raw_motion_err_list, #define UL_INTRA_THRESH 50 #define INVALID_ROW -1 -void av1_first_pass(AV1_COMP *cpi, const struct lookahead_entry *source) { +void av1_first_pass(AV1_COMP *cpi, const int64_t ts_duration) { int mb_row, mb_col; MACROBLOCK *const x = &cpi->td.mb; AV1_COMMON *const cm = &cpi->common; + CurrentFrame *const current_frame = &cm->current_frame; const SequenceHeader *const seq_params = &cm->seq_params; const int num_planes = av1_num_planes(cm); MACROBLOCKD *const xd = &x->e_mbd; @@ -521,13 +512,13 @@ void av1_first_pass(AV1_COMP *cpi, const struct lookahead_entry *source) { TWO_PASS *twopass = &cpi->twopass; int recon_y_stride, recon_uv_stride, uv_mb_height; - YV12_BUFFER_CONFIG *const lst_yv12 = get_ref_frame_buffer(cpi, LAST_FRAME); - YV12_BUFFER_CONFIG *gld_yv12 = get_ref_frame_buffer(cpi, GOLDEN_FRAME); - YV12_BUFFER_CONFIG *const new_yv12 = get_frame_new_buffer(cm); + const YV12_BUFFER_CONFIG *const lst_yv12 = + get_ref_frame_yv12_buf(cm, LAST_FRAME); + const YV12_BUFFER_CONFIG *gld_yv12 = get_ref_frame_yv12_buf(cm, GOLDEN_FRAME); + YV12_BUFFER_CONFIG *const new_yv12 = &cm->cur_frame->buf; const YV12_BUFFER_CONFIG *first_ref_buf = lst_yv12; double intra_factor; double brightness_factor; - BufferPool *const pool = cm->buffer_pool; const int qindex = find_fp_qindex(seq_params->bit_depth); const int mb_scale = mi_size_wide[BLOCK_16X16]; @@ -556,13 +547,16 @@ void av1_first_pass(AV1_COMP *cpi, const struct lookahead_entry *source) { brightness_factor = 0.0; neutral_count = 0.0; - set_first_pass_params(cpi); + // Do not use periodic key frames. + cpi->rc.frames_to_key = INT_MAX; + av1_set_quantizer(cm, qindex); av1_setup_block_planes(&x->e_mbd, seq_params->subsampling_x, seq_params->subsampling_y, num_planes); - av1_setup_src_planes(x, cpi->source, 0, 0, num_planes); + av1_setup_src_planes(x, cpi->source, 0, 0, num_planes, + x->e_mbd.mi[0]->sb_type); av1_setup_dst_planes(xd->plane, seq_params->sb_size, new_yv12, 0, 0, 0, num_planes); @@ -586,7 +580,6 @@ void av1_first_pass(AV1_COMP *cpi, const struct lookahead_entry *source) { } av1_init_mv_probs(cm); - av1_init_lv_map(cm); av1_initialize_rd_consts(cpi); // Tiling is ignored in the first pass. @@ -699,14 +692,15 @@ void av1_first_pass(AV1_COMP *cpi, const struct lookahead_entry *source) { // Accumulate the intra error. intra_error += (int64_t)this_error; - int stride = x->plane[0].src.stride; + const int hbd = is_cur_buf_hbd(xd); + const int stride = x->plane[0].src.stride; uint8_t *buf = x->plane[0].src.buf; - for (int r8 = 0; r8 < 2; ++r8) + for (int r8 = 0; r8 < 2; ++r8) { for (int c8 = 0; c8 < 2; ++c8) { - int hbd = xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH; frame_avg_wavelet_energy += av1_haar_ac_sad_8x8_uint8_input( buf + c8 * 8 + r8 * 8 * stride, stride, hbd); } + } #if CONFIG_FP_MB_STATS if (cpi->use_fp_mb_stats) { @@ -728,7 +722,7 @@ void av1_first_pass(AV1_COMP *cpi, const struct lookahead_entry *source) { struct buf_2d unscaled_last_source_buf_2d; xd->plane[0].pre[0].buf = first_ref_buf->y_buffer + recon_yoffset; - if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + if (is_cur_buf_hbd(xd)) { motion_error = highbd_get_prediction_error( bsize, &x->plane[0].src, &xd->plane[0].pre[0], xd->bd); } else { @@ -743,7 +737,7 @@ void av1_first_pass(AV1_COMP *cpi, const struct lookahead_entry *source) { cpi->unscaled_last_source->y_buffer + recon_yoffset; unscaled_last_source_buf_2d.stride = cpi->unscaled_last_source->y_stride; - if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + if (is_cur_buf_hbd(xd)) { raw_motion_error = highbd_get_prediction_error( bsize, &x->plane[0].src, &unscaled_last_source_buf_2d, xd->bd); } else { @@ -770,12 +764,12 @@ void av1_first_pass(AV1_COMP *cpi, const struct lookahead_entry *source) { } // Search in an older reference frame. - if ((cm->current_video_frame > 1) && gld_yv12 != NULL) { + if ((current_frame->frame_number > 1) && gld_yv12 != NULL) { // Assume 0,0 motion with no mv overhead. int gf_motion_error; xd->plane[0].pre[0].buf = gld_yv12->y_buffer + recon_yoffset; - if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + if (is_cur_buf_hbd(xd)) { gf_motion_error = highbd_get_prediction_error( bsize, &x->plane[0].src, &xd->plane[0].pre[0], xd->bd); } else { @@ -852,8 +846,9 @@ void av1_first_pass(AV1_COMP *cpi, const struct lookahead_entry *source) { xd->mi[0]->tx_size = TX_4X4; xd->mi[0]->ref_frame[0] = LAST_FRAME; xd->mi[0]->ref_frame[1] = NONE_FRAME; - av1_build_inter_predictors_sby(cm, xd, mb_row * mb_scale, - mb_col * mb_scale, NULL, bsize); + av1_enc_build_inter_predictor(cm, xd, mb_row * mb_scale, + mb_col * mb_scale, NULL, bsize, + AOM_PLANE_Y, AOM_PLANE_Y); av1_encode_sby_pass1(cm, x, bsize); sum_mvr += mv.row; sum_mvr_abs += abs(mv.row); @@ -995,7 +990,7 @@ void av1_first_pass(AV1_COMP *cpi, const struct lookahead_entry *source) { brightness_factor = brightness_factor / (double)num_mbs; fps.weight = intra_factor * brightness_factor; - fps.frame = cm->current_video_frame; + fps.frame = current_frame->frame_number; fps.coded_error = (double)(coded_error >> 8) + min_err; fps.sr_coded_error = (double)(sr_coded_error >> 8) + min_err; fps.intra_error = (double)(intra_error >> 8) + min_err; @@ -1036,7 +1031,7 @@ void av1_first_pass(AV1_COMP *cpi, const struct lookahead_entry *source) { // TODO(paulwilkins): Handle the case when duration is set to 0, or // something less than the full time between subsequent values of // cpi->source_time_stamp. - fps.duration = (double)(source->ts_end - source->ts_start); + fps.duration = (double)ts_duration; // Don't want to do output stats with a stack variable! twopass->this_frame_stats = fps; @@ -1054,14 +1049,14 @@ void av1_first_pass(AV1_COMP *cpi, const struct lookahead_entry *source) { // Copy the previous Last Frame back into gf and and arf buffers if // the prediction is good enough... but also don't allow it to lag too far. if ((twopass->sr_update_lag > 3) || - ((cm->current_video_frame > 0) && + ((current_frame->frame_number > 0) && (twopass->this_frame_stats.pcnt_inter > 0.20) && ((twopass->this_frame_stats.intra_error / DOUBLE_DIVIDE_CHECK(twopass->this_frame_stats.coded_error)) > 2.0))) { if (gld_yv12 != NULL) { - ref_cnt_fb(pool->frame_bufs, - &cm->ref_frame_map[cpi->ref_fb_idx[GOLDEN_FRAME - 1]], - cm->ref_frame_map[cpi->ref_fb_idx[LAST_FRAME - 1]]); + assign_frame_buffer_p( + &cm->ref_frame_map[get_ref_frame_map_idx(cm, GOLDEN_FRAME)], + cm->ref_frame_map[get_ref_frame_map_idx(cm, LAST_FRAME)]); } twopass->sr_update_lag = 1; } else { @@ -1071,17 +1066,16 @@ void av1_first_pass(AV1_COMP *cpi, const struct lookahead_entry *source) { aom_extend_frame_borders(new_yv12, num_planes); // The frame we just compressed now becomes the last frame. - ref_cnt_fb(pool->frame_bufs, - &cm->ref_frame_map[cpi->ref_fb_idx[LAST_FRAME - 1]], - cm->new_fb_idx); + assign_frame_buffer_p( + &cm->ref_frame_map[get_ref_frame_map_idx(cm, LAST_FRAME)], cm->cur_frame); // Special case for the first frame. Copy into the GF buffer as a second // reference. - if (cm->current_video_frame == 0 && - cpi->ref_fb_idx[GOLDEN_FRAME - 1] != INVALID_IDX) { - ref_cnt_fb(pool->frame_bufs, - &cm->ref_frame_map[cpi->ref_fb_idx[GOLDEN_FRAME - 1]], - cm->ref_frame_map[cpi->ref_fb_idx[LAST_FRAME - 1]]); + if (current_frame->frame_number == 0 && + get_ref_frame_map_idx(cm, GOLDEN_FRAME) != INVALID_IDX) { + assign_frame_buffer_p( + &cm->ref_frame_map[get_ref_frame_map_idx(cm, GOLDEN_FRAME)], + cm->ref_frame_map[get_ref_frame_map_idx(cm, LAST_FRAME)]); } // Use this to see what the first pass reconstruction looks like. @@ -1089,9 +1083,9 @@ void av1_first_pass(AV1_COMP *cpi, const struct lookahead_entry *source) { char filename[512]; FILE *recon_file; snprintf(filename, sizeof(filename), "enc%04d.yuv", - (int)cm->current_video_frame); + (int)current_frame->frame_number); - if (cm->current_video_frame == 0) + if (current_frame->frame_number == 0) recon_file = fopen(filename, "wb"); else recon_file = fopen(filename, "ab"); @@ -1100,7 +1094,7 @@ void av1_first_pass(AV1_COMP *cpi, const struct lookahead_entry *source) { fclose(recon_file); } - ++cm->current_video_frame; + ++current_frame->frame_number; } static double calc_correction_factor(double err_per_mb, double err_divisor, @@ -1565,7 +1559,6 @@ static int calculate_boost_bits(int frame_count, int boost, 0); } -#if USE_SYMM_MULTI_LAYER // #define CHCEK_GF_PARAMETER #ifdef CHCEK_GF_PARAMETER void check_frame_params(GF_GROUP *const gf_group, int gf_interval, @@ -1587,7 +1580,7 @@ void check_frame_params(GF_GROUP *const gf_group, int gf_interval, } fprintf(fid, "number of nodes in each level: \n"); - for (int i = 0; i < MAX_PYRAMID_LVL; ++i) { + for (int i = 0; i < gf_group->pyramid_height; ++i) { fprintf(fid, "lvl %d: %d ", i, gf_group->pyramid_lvl_nodes[i]); } fprintf(fid, "\n"); @@ -1652,7 +1645,7 @@ static void set_multi_layer_params(GF_GROUP *const gf_group, int l, int r, } static INLINE unsigned char get_pyramid_height(int pyramid_width) { - assert(pyramid_width <= 16 && pyramid_width >= 4 && + assert(pyramid_width <= MAX_GF_INTERVAL && pyramid_width >= MIN_GF_INTERVAL && "invalid gf interval for pyramid structure"); return pyramid_width > 12 ? 4 : (pyramid_width > 6 ? 3 : 2); @@ -1689,14 +1682,46 @@ static int construct_multi_layer_gf_structure(GF_GROUP *const gf_group, return frame_index; } -void define_customized_gf_group_structure(AV1_COMP *cpi) { +// Given the maximum allowed height of the pyramid structure, return the maximum +// GF length supported by the same. +static INLINE int get_max_gf_length(int max_pyr_height) { + // We allow a frame to have at most two left/right descendants before changing + // them into to a subtree, i.e., we allow the following structure: + /* OUT_OF_ORDER_FRAME + / / \ \ + (two left children) F F F F (two right children) */ + // For example, the max gf size supported by 4 layer structure is: + // 1 (KEY/OVERLAY) + 1 + 2 + 4 + 16 (two children on both side of their + // parent) + switch (max_pyr_height) { + case 2: return 6; // = 1 (KEY/OVERLAY) + 1 + 4 + case 3: return 12; // = 1 (KEY/OVERLAY) + 1 + 2 + 8 + case 4: return 24; // = 1 (KEY/OVERLAY) + 1 + 2 + 4 + 16 + case 1: + return MAX_GF_INTERVAL; // Special case: uses the old pyramid structure. + default: assert(0 && "Invalid max_pyr_height"); return -1; + } +} + +// Given the maximum allowed height of the pyramid structure, return the fixed +// GF length to be used. +int av1_rc_get_fixed_gf_length(int max_pyr_height) { + const int max_gf_length_allowed = get_max_gf_length(max_pyr_height); + return AOMMIN(max_gf_length_allowed, MAX_GF_INTERVAL); +} + +static void define_customized_gf_group_structure( + AV1_COMP *cpi, const EncodeFrameParams *const frame_params) { RATE_CONTROL *const rc = &cpi->rc; TWO_PASS *const twopass = &cpi->twopass; GF_GROUP *const gf_group = &twopass->gf_group; - const int key_frame = cpi->common.frame_type == KEY_FRAME; + const int key_frame = frame_params->frame_type == KEY_FRAME; - assert(rc->baseline_gf_interval >= 4 && - rc->baseline_gf_interval <= MAX_PYRAMID_SIZE); + assert(rc->baseline_gf_interval >= MIN_GF_INTERVAL && + rc->baseline_gf_interval <= + get_max_gf_length(cpi->oxcf.gf_max_pyr_height)); + assert(cpi->oxcf.gf_max_pyr_height >= MIN_PYRAMID_LVL && + cpi->oxcf.gf_max_pyr_height <= MAX_PYRAMID_LVL); const int gf_update_frames = construct_multi_layer_gf_structure(gf_group, rc->baseline_gf_interval); @@ -1760,166 +1785,31 @@ void define_customized_gf_group_structure(AV1_COMP *cpi) { // It is an example of how to define a GF stucture manually. The function will // result in exactly the same GF group structure as // define_customized_gf_group_structure() when rc->baseline_gf_interval == 4 -#if USE_MANUAL_GF4_STRUCT -#define GF_INTERVAL_4 4 -static const unsigned char gf4_multi_layer_params[][GF_FRAME_PARAMS] = { - { - // gf_group->index == 0 (Frame 0) - // It can also be KEY frame. Will assign the proper value - // in define_gf_group_structure - OVERLAY_UPDATE, // update_type (default value) - 0, // arf_src_offset - 0, // arf_pos_in_gf - 0 // arf_update_idx - }, - { - // gf_group->index == 1 (Frame 4) - ARF_UPDATE, // update_type - GF_INTERVAL_4 - 1, // arf_src_offset - 0, // arf_pos_in_gf - 0 // arf_update_idx - }, - { - // gf_group->index == 2 (Frame 2) - INTNL_ARF_UPDATE, // update_type - (GF_INTERVAL_4 >> 1) - 1, // arf_src_offset - 0, // arf_pos_in_gf - 0 // arf_update_idx - }, - { - // gf_group->index == 3 (Frame 1) - LAST_BIPRED_UPDATE, // update_type - 0, // arf_src_offset - 0, // arf_pos_in_gf - 0 // arf_update_idx - }, - { - // gf_group->index == 4 (Frame 2 - OVERLAY) - INTNL_OVERLAY_UPDATE, // update_type - 0, // arf_src_offset - 2, // arf_pos_in_gf - 0 // arf_update_idx - }, - { - // gf_group->index == 5 (Frame 3) - LF_UPDATE, // update_type - 0, // arf_src_offset - 0, // arf_pos_in_gf - 1 // arf_update_idx - } -}; - -static int define_gf_group_structure_4(AV1_COMP *cpi) { - RATE_CONTROL *const rc = &cpi->rc; - TWO_PASS *const twopass = &cpi->twopass; - GF_GROUP *const gf_group = &twopass->gf_group; - const int key_frame = cpi->common.frame_type == KEY_FRAME; - - assert(rc->baseline_gf_interval == GF_INTERVAL_4); - - const int gf_update_frames = rc->baseline_gf_interval + 2; - int frame_index; - - for (frame_index = 0; frame_index < gf_update_frames; ++frame_index) { - int param_idx = 0; - - gf_group->bidir_pred_enabled[frame_index] = 0; - - if (frame_index == 0) { - // gf_group->arf_src_offset[frame_index] = 0; - gf_group->brf_src_offset[frame_index] = 0; - gf_group->bidir_pred_enabled[frame_index] = 0; - - // For key frames the frame target rate is already set and it - // is also the golden frame. - if (key_frame) continue; - - gf_group->update_type[frame_index] = - gf4_multi_layer_params[frame_index][param_idx++]; - - if (rc->source_alt_ref_active) { - gf_group->update_type[frame_index] = OVERLAY_UPDATE; - } else { - gf_group->update_type[frame_index] = GF_UPDATE; - } - param_idx++; - } else { - gf_group->update_type[frame_index] = - gf4_multi_layer_params[frame_index][param_idx++]; - } - - // setup other parameters - gf_group->rf_level[frame_index] = - update_type_2_rf_level(gf_group->update_type[frame_index]); - - // == arf_src_offset == - gf_group->arf_src_offset[frame_index] = - gf4_multi_layer_params[frame_index][param_idx++]; - - // == arf_pos_in_gf == - gf_group->arf_pos_in_gf[frame_index] = - gf4_multi_layer_params[frame_index][param_idx++]; - - // == arf_update_idx == - gf_group->brf_src_offset[frame_index] = - gf4_multi_layer_params[frame_index][param_idx]; - } - - // NOTE: We need to configure the frame at the end of the sequence + 1 that - // will be the start frame for the next group. Otherwise prior to the - // call to av1_rc_get_second_pass_params() the data will be undefined. - gf_group->arf_update_idx[frame_index] = 0; - gf_group->arf_ref_idx[frame_index] = 0; - - if (rc->source_alt_ref_pending) { - gf_group->update_type[frame_index] = OVERLAY_UPDATE; - gf_group->rf_level[frame_index] = INTER_NORMAL; - - } else { - gf_group->update_type[frame_index] = GF_UPDATE; - gf_group->rf_level[frame_index] = GF_ARF_STD; - } - - gf_group->bidir_pred_enabled[frame_index] = 0; - gf_group->brf_src_offset[frame_index] = 0; - - // This value is only used for INTNL_OVERLAY_UPDATE - gf_group->arf_pos_in_gf[frame_index] = 0; - - return gf_update_frames; -} -#endif // USE_MANUAL_GF4_STRUCT -#endif // USE_SYMM_MULTI_LAYER - -static void define_gf_group_structure(AV1_COMP *cpi) { +static void define_gf_group_structure( + AV1_COMP *cpi, const EncodeFrameParams *const frame_params) { RATE_CONTROL *const rc = &cpi->rc; -#if USE_SYMM_MULTI_LAYER + const int max_pyr_height = cpi->oxcf.gf_max_pyr_height; const int valid_customized_gf_length = - rc->baseline_gf_interval >= 4 && - rc->baseline_gf_interval <= MAX_PYRAMID_SIZE; + max_pyr_height >= MIN_PYRAMID_LVL && max_pyr_height <= MAX_PYRAMID_LVL && + rc->baseline_gf_interval >= MIN_GF_INTERVAL && + rc->baseline_gf_interval <= get_max_gf_length(max_pyr_height); // used the new structure only if extra_arf is allowed if (valid_customized_gf_length && rc->source_alt_ref_pending && cpi->extra_arf_allowed > 0) { -#if USE_MANUAL_GF4_STRUCT - if (rc->baseline_gf_interval == 4) - define_gf_group_structure_4(cpi); - else -#endif - define_customized_gf_group_structure(cpi); + define_customized_gf_group_structure(cpi, frame_params); cpi->new_bwdref_update_rule = 1; return; } else { cpi->new_bwdref_update_rule = 0; } -#endif TWO_PASS *const twopass = &cpi->twopass; GF_GROUP *const gf_group = &twopass->gf_group; int i; int frame_index = 0; - const int key_frame = cpi->common.frame_type == KEY_FRAME; + const int key_frame = frame_params->frame_type == KEY_FRAME; // The use of bi-predictive frames are only enabled when following 3 // conditions are met: @@ -2015,7 +1905,10 @@ static void define_gf_group_structure(AV1_COMP *cpi) { accumulative_subgroup_interval += subgroup_interval[cpi->num_extra_arfs]; } - for (i = 0; i < rc->baseline_gf_interval - rc->source_alt_ref_pending; ++i) { + const int normal_frames = + rc->baseline_gf_interval - (key_frame || rc->source_alt_ref_pending); + + for (i = 0; i < normal_frames; ++i) { gf_group->arf_update_idx[frame_index] = which_arf; gf_group->arf_ref_idx[frame_index] = which_arf; @@ -2127,41 +2020,28 @@ static void define_gf_group_structure(AV1_COMP *cpi) { gf_group->brf_src_offset[frame_index] = 0; } -#if USE_SYMM_MULTI_LAYER -#define LEAF_REDUCTION_FACTOR 0.75f -#define LVL_3_BOOST_FACTOR 0.8f -#define LVL_2_BOOST_FACTOR 0.3f - -static float_t lvl_budget_factor[MAX_PYRAMID_LVL - 1][MAX_PYRAMID_LVL - 1] = { - { 1, 0, 0 }, - { LVL_3_BOOST_FACTOR, 0, 0 }, // Leaking budget works better - { LVL_3_BOOST_FACTOR, (1 - LVL_3_BOOST_FACTOR) * LVL_2_BOOST_FACTOR, - (1 - LVL_3_BOOST_FACTOR) * (1 - LVL_2_BOOST_FACTOR) } +#define LEAF_REDUCTION_FACTOR 0.75 +static double lvl_budget_factor[MAX_PYRAMID_LVL - 1][MAX_PYRAMID_LVL - 1] = { + { 1.0, 0.0, 0.0 }, { 0.6, 0.4, 0 }, { 0.45, 0.35, 0.20 } }; -#endif // USE_SYMM_MULTI_LAYER -static void allocate_gf_group_bits(AV1_COMP *cpi, int64_t gf_group_bits, - double group_error, int gf_arf_bits) { +static void allocate_gf_group_bits( + AV1_COMP *cpi, int64_t gf_group_bits, double group_error, int gf_arf_bits, + const EncodeFrameParams *const frame_params) { RATE_CONTROL *const rc = &cpi->rc; const AV1EncoderConfig *const oxcf = &cpi->oxcf; TWO_PASS *const twopass = &cpi->twopass; GF_GROUP *const gf_group = &twopass->gf_group; - FIRSTPASS_STATS frame_stats; int i; int frame_index = 0; - int target_frame_size; - int key_frame; + const int key_frame = frame_params->frame_type == KEY_FRAME; const int max_bits = frame_max_bits(&cpi->rc, &cpi->oxcf); int64_t total_group_bits = gf_group_bits; - double modified_err = 0.0; - double err_fraction; int ext_arf_boost[MAX_EXT_ARFS]; - define_gf_group_structure(cpi); + define_gf_group_structure(cpi, frame_params); av1_zero_array(ext_arf_boost, MAX_EXT_ARFS); - key_frame = cpi->common.frame_type == KEY_FRAME; - // For key frames the frame target rate is already set and it // is also the golden frame. // === [frame_index == 0] === @@ -2172,6 +2052,7 @@ static void allocate_gf_group_bits(AV1_COMP *cpi, int64_t gf_group_bits, gf_group->bit_allocation[frame_index] = gf_arf_bits; // Step over the golden frame / overlay frame + FIRSTPASS_STATS frame_stats; if (EOF == input_stats(twopass, &frame_stats)) return; } @@ -2196,21 +2077,26 @@ static void allocate_gf_group_bits(AV1_COMP *cpi, int64_t gf_group_bits, } } + // Save. + const int tmp_frame_index = frame_index; + int budget_reduced_from_leaf_level = 0; + // Allocate bits to the other frames in the group. - for (i = 0; i < rc->baseline_gf_interval - rc->source_alt_ref_pending; ++i) { + const int normal_frames = + rc->baseline_gf_interval - (key_frame || rc->source_alt_ref_pending); + + for (i = 0; i < normal_frames; ++i) { + FIRSTPASS_STATS frame_stats; if (EOF == input_stats(twopass, &frame_stats)) break; - modified_err = calculate_modified_err(cpi, twopass, oxcf, &frame_stats); - - if (group_error > 0) - err_fraction = modified_err / DOUBLE_DIVIDE_CHECK(group_error); - else - err_fraction = 0.0; - - target_frame_size = (int)((double)total_group_bits * err_fraction); - - target_frame_size = - clamp(target_frame_size, 0, AOMMIN(max_bits, (int)total_group_bits)); + const double modified_err = + calculate_modified_err(cpi, twopass, oxcf, &frame_stats); + const double err_fraction = + (group_error > 0) ? modified_err / DOUBLE_DIVIDE_CHECK(group_error) + : 0.0; + const int target_frame_size = + clamp((int)((double)total_group_bits * err_fraction), 0, + AOMMIN(max_bits, (int)total_group_bits)); if (gf_group->update_type[frame_index] == BRF_UPDATE) { // Boost up the allocated bits on BWDREF_FRAME @@ -2224,40 +2110,26 @@ static void allocate_gf_group_bits(AV1_COMP *cpi, int64_t gf_group_bits, // TODO(zoeliu): To investigate whether the allocated bits on // BIPRED_UPDATE frames need to be further adjusted. gf_group->bit_allocation[frame_index] = target_frame_size; -#if USE_SYMM_MULTI_LAYER } else if (cpi->new_bwdref_update_rule && gf_group->update_type[frame_index] == INTNL_OVERLAY_UPDATE) { assert(gf_group->pyramid_height <= MAX_PYRAMID_LVL && - gf_group->pyramid_height >= 0 && "non-valid height for a pyramid structure"); - int arf_pos = gf_group->arf_pos_in_gf[frame_index]; + const int arf_pos = gf_group->arf_pos_in_gf[frame_index]; gf_group->bit_allocation[frame_index] = 0; gf_group->bit_allocation[arf_pos] = target_frame_size; -#if MULTI_LVL_BOOST_VBR_CQ - const int pyr_h = gf_group->pyramid_height - 2; - const int this_lvl = gf_group->pyramid_level[arf_pos]; - const int dist2top = gf_group->pyramid_height - 1 - this_lvl; - - const float_t budget = - LEAF_REDUCTION_FACTOR * gf_group->pyramid_lvl_nodes[0]; - const float_t lvl_boost = budget * lvl_budget_factor[pyr_h][dist2top] / - gf_group->pyramid_lvl_nodes[this_lvl]; - - gf_group->bit_allocation[arf_pos] += (int)(target_frame_size * lvl_boost); -#endif // MULTI_LVL_BOOST_VBR_CQ -#endif // USE_SYMM_MULTI_LAYER + // Note: Boost, if needed, is added in the next loop. } else { assert(gf_group->update_type[frame_index] == LF_UPDATE || gf_group->update_type[frame_index] == INTNL_OVERLAY_UPDATE); gf_group->bit_allocation[frame_index] = target_frame_size; -#if MULTI_LVL_BOOST_VBR_CQ if (cpi->new_bwdref_update_rule) { - gf_group->bit_allocation[frame_index] -= + const int this_budget_reduction = (int)(target_frame_size * LEAF_REDUCTION_FACTOR); + gf_group->bit_allocation[frame_index] -= this_budget_reduction; + budget_reduced_from_leaf_level += this_budget_reduction; } -#endif // MULTI_LVL_BOOST_VBR_CQ } ++frame_index; @@ -2269,11 +2141,37 @@ static void allocate_gf_group_bits(AV1_COMP *cpi, int64_t gf_group_bits, } } -#if USE_SYMM_MULTI_LAYER + if (budget_reduced_from_leaf_level > 0) { + // Restore. + frame_index = tmp_frame_index; + + // Re-distribute this extra budget to overlay frames in the group. + for (i = 0; i < normal_frames; ++i) { + if (cpi->new_bwdref_update_rule && + gf_group->update_type[frame_index] == INTNL_OVERLAY_UPDATE) { + assert(gf_group->pyramid_height <= MAX_PYRAMID_LVL && + "non-valid height for a pyramid structure"); + const int arf_pos = gf_group->arf_pos_in_gf[frame_index]; + const int this_lvl = gf_group->pyramid_level[arf_pos]; + const int dist2top = gf_group->pyramid_height - 1 - this_lvl; + const double lvl_boost_factor = + lvl_budget_factor[gf_group->pyramid_height - 2][dist2top]; + const int extra_size = + (int)(budget_reduced_from_leaf_level * lvl_boost_factor / + gf_group->pyramid_lvl_nodes[this_lvl]); + gf_group->bit_allocation[arf_pos] += extra_size; + } + ++frame_index; + + // Skip all the extra-ARF's. + if (cpi->num_extra_arfs) { + while (gf_group->update_type[frame_index] == INTNL_ARF_UPDATE) + ++frame_index; + } + } + } + if (cpi->new_bwdref_update_rule == 0 && rc->source_alt_ref_pending) { -#else - if (rc->source_alt_ref_pending) { -#endif if (cpi->num_extra_arfs) { // NOTE: For bit allocation, move the allocated bits associated with // INTNL_OVERLAY_UPDATE to the corresponding INTNL_ARF_UPDATE. @@ -2295,8 +2193,17 @@ static void allocate_gf_group_bits(AV1_COMP *cpi, int64_t gf_group_bits, } } +// Returns true if KF group and GF group both are almost completely static. +static INLINE int is_almost_static(double gf_zero_motion, int kf_zero_motion) { + return (gf_zero_motion >= 0.995) && + (kf_zero_motion >= STATIC_KF_GROUP_THRESH); +} + +#define ARF_ABS_ZOOM_THRESH 4.4 + // Analyse and define a gf/arf group. -static void define_gf_group(AV1_COMP *cpi, FIRSTPASS_STATS *this_frame) { +static void define_gf_group(AV1_COMP *cpi, FIRSTPASS_STATS *this_frame, + const EncodeFrameParams *const frame_params) { AV1_COMMON *const cm = &cpi->common; RATE_CONTROL *const rc = &cpi->rc; AV1EncoderConfig *const oxcf = &cpi->oxcf; @@ -2306,12 +2213,7 @@ static void define_gf_group(AV1_COMP *cpi, FIRSTPASS_STATS *this_frame) { int i; double boost_score = 0.0; -#if !CONFIG_FIX_GF_LENGTH - double old_boost_score = 0.0; - double mv_ratio_accumulator_thresh; - int active_max_gf_interval; int active_min_gf_interval; -#endif double gf_group_err = 0.0; #if GROUP_ADAPTIVE_MAXQ double gf_group_raw_error = 0.0; @@ -2340,14 +2242,15 @@ static void define_gf_group(AV1_COMP *cpi, FIRSTPASS_STATS *this_frame) { int64_t gf_group_bits; double gf_group_error_left; int gf_arf_bits; - const int is_key_frame = frame_is_intra_only(cm); - const int arf_active_or_kf = is_key_frame || rc->source_alt_ref_active; + const int is_intra_only = frame_params->frame_type == KEY_FRAME || + frame_params->frame_type == INTRA_ONLY_FRAME; + const int arf_active_or_kf = is_intra_only || rc->source_alt_ref_active; cpi->extra_arf_allowed = 1; // Reset the GF group data structures unless this is a key // frame in which case it will already have been done. - if (is_key_frame == 0) { + if (!is_intra_only) { av1_zero(twopass->gf_group); } @@ -2371,35 +2274,13 @@ static void define_gf_group(AV1_COMP *cpi, FIRSTPASS_STATS *this_frame) { gf_group_skip_pct -= this_frame->intra_skip_pct; gf_group_inactive_zone_rows -= this_frame->inactive_zone_rows; } -#if !CONFIG_FIX_GF_LENGTH // Motion breakout threshold for loop below depends on image size. - mv_ratio_accumulator_thresh = + const double mv_ratio_accumulator_thresh = (cpi->initial_height + cpi->initial_width) / 4.0; - // Set a maximum and minimum interval for the GF group. - // If the image appears almost completely static we can extend beyond this. - { - int int_max_q = (int)(av1_convert_qindex_to_q( - twopass->active_worst_quality, cpi->common.seq_params.bit_depth)); - int int_lbq = (int)(av1_convert_qindex_to_q( - rc->last_boosted_qindex, cpi->common.seq_params.bit_depth)); - active_min_gf_interval = rc->min_gf_interval + AOMMIN(2, int_max_q / 200); - if (active_min_gf_interval > rc->max_gf_interval) - active_min_gf_interval = rc->max_gf_interval; + // TODO(urvang): Try logic to vary min and max interval based on q. + active_min_gf_interval = rc->min_gf_interval; - // The value chosen depends on the active Q range. At low Q we have - // bits to spare and are better with a smaller interval and smaller boost. - // At high Q when there are few bits to spare we are better with a longer - // interval to spread the cost of the GF. - active_max_gf_interval = 12 + AOMMIN(4, (int_lbq / 6)); - - // We have: active_min_gf_interval <= rc->max_gf_interval - if (active_max_gf_interval < active_min_gf_interval) - active_max_gf_interval = active_min_gf_interval; - else if (active_max_gf_interval > rc->max_gf_interval) - active_max_gf_interval = rc->max_gf_interval; - } -#endif // !CONFIG_FIX_GF_LENGTH double avg_sr_coded_error = 0; double avg_raw_err_stdev = 0; int non_zero_stdev_count = 0; @@ -2442,8 +2323,10 @@ static void define_gf_group(AV1_COMP *cpi, FIRSTPASS_STATS *this_frame) { decay_accumulator = decay_accumulator * loop_decay_rate; // Monitor for static sections. - zero_motion_accumulator = AOMMIN( - zero_motion_accumulator, get_zero_motion_factor(cpi, &next_frame)); + if ((rc->frames_since_key + i - 1) > 1) { + zero_motion_accumulator = AOMMIN( + zero_motion_accumulator, get_zero_motion_factor(cpi, &next_frame)); + } // Break clause to detect very still sections after motion. For example, // a static image after a fade or other transition. @@ -2458,34 +2341,23 @@ static void define_gf_group(AV1_COMP *cpi, FIRSTPASS_STATS *this_frame) { boost_score += decay_accumulator * calc_frame_boost(cpi, &next_frame, this_frame_mv_in_out, GF_MAX_BOOST); -#if CONFIG_FIX_GF_LENGTH - if (i == (FIXED_GF_LENGTH + 1)) break; -#else - // Skip breaking condition for CONFIG_FIX_GF_LENGTH - // Break out conditions. - if ( - // Break at active_max_gf_interval unless almost totally static. - (i >= (active_max_gf_interval + arf_active_or_kf) && - zero_motion_accumulator < 0.995) || - ( - // Don't break out with a very short interval. - (i >= active_min_gf_interval + arf_active_or_kf) && - (!flash_detected) && - ((mv_ratio_accumulator > mv_ratio_accumulator_thresh) || - (abs_mv_in_out_accumulator > 3.0) || - (mv_in_out_accumulator < -2.0) || - ((boost_score - old_boost_score) < BOOST_BREAKOUT)))) { - // If GF group interval is < 12, we force it to be 8. Otherwise, - // if it is >= 12, we keep it as is. - // NOTE: 'i' is 1 more than the GF group interval candidate that is being - // checked. - if (i == (8 + 1) || i >= (12 + 1)) { - boost_score = old_boost_score; - break; - } + // If almost totally static, we will not use the the fixed GF length later, + // so we can continue for more frames. + if (i >= (av1_rc_get_fixed_gf_length(oxcf->gf_max_pyr_height) + 1) && + !is_almost_static(zero_motion_accumulator, + twopass->kf_zeromotion_pct)) { + break; + } + + // Some conditions to breakout after min interval. + if (i >= active_min_gf_interval && + // If possible don't break very close to a kf + (rc->frames_to_key - i >= rc->min_gf_interval) && (i & 0x01) && + !flash_detected && + (mv_ratio_accumulator > mv_ratio_accumulator_thresh || + abs_mv_in_out_accumulator > ARF_ABS_ZOOM_THRESH)) { + break; } - old_boost_score = boost_score; -#endif // CONFIG_FIX_GF_LENGTH *this_frame = next_frame; } twopass->gf_zeromotion_pct = (int)(zero_motion_accumulator * 1000.0); @@ -2512,23 +2384,23 @@ static void define_gf_group(AV1_COMP *cpi, FIRSTPASS_STATS *this_frame) { if (disable_bwd_extarf) cpi->extra_arf_allowed = 0; + const int use_alt_ref = + !is_almost_static(zero_motion_accumulator, twopass->kf_zeromotion_pct) && + allow_alt_ref && (i < cpi->oxcf.lag_in_frames) && + (i >= rc->min_gf_interval); + #define REDUCE_GF_LENGTH_THRESH 4 #define REDUCE_GF_LENGTH_TO_KEY_THRESH 9 #define REDUCE_GF_LENGTH_BY 1 int alt_offset = 0; -#if REDUCE_LAST_GF_LENGTH - // TODO(weitinglin): The length reduction stretagy is tweaking using AOM_Q - // mode, and hurting the performance of VBR mode. We need to investigate how - // to adjust GF length for other modes. + // The length reduction strategy is tweaked using AOM_Q mode, and doesn't work + // for VBR mode. + // Also, we don't have do adjustment for lossless mode. + const int allow_gf_length_reduction = + (cpi->oxcf.rc_mode == AOM_Q || cpi->extra_arf_allowed == 0) && + !is_lossless_requested(&cpi->oxcf); - int allow_gf_length_reduction = - cpi->oxcf.rc_mode == AOM_Q || cpi->extra_arf_allowed == 0; - - // We are going to have an alt ref, but we don't have do adjustment for - // lossless mode - if (allow_alt_ref && allow_gf_length_reduction && - (i < cpi->oxcf.lag_in_frames) && (i >= rc->min_gf_interval) && - !is_lossless_requested(&cpi->oxcf)) { + if (allow_gf_length_reduction && use_alt_ref) { // adjust length of this gf group if one of the following condition met // 1: only one overlay frame left and this gf is too long // 2: next gf group is too short to have arf compared to the current gf @@ -2552,11 +2424,9 @@ static void define_gf_group(AV1_COMP *cpi, FIRSTPASS_STATS *this_frame) { i -= roll_back; } } -#endif // Should we use the alternate reference frame. - if (allow_alt_ref && (i < cpi->oxcf.lag_in_frames) && - (i >= rc->min_gf_interval)) { + if (use_alt_ref) { // Calculate the boost for alt ref. rc->gfu_boost = calc_arf_boost(cpi, alt_offset, (i - 1), (i - 1), &f_boost, &b_boost); @@ -2582,7 +2452,7 @@ static void define_gf_group(AV1_COMP *cpi, FIRSTPASS_STATS *this_frame) { AOMMAX(MIN_FWD_KF_INTERVAL, rc->min_gf_interval)) && (rc->frames_to_key != i)) { // if possible, merge the last two gf groups - if (rc->frames_to_key <= MAX_PYRAMID_SIZE) { + if (rc->frames_to_key <= get_max_gf_length(oxcf->gf_max_pyr_height)) { rc->baseline_gf_interval = rc->frames_to_key; // if merging the last two gf groups creates a group that is too long, // split them and force the last gf group to be the MIN_FWD_KF_INTERVAL @@ -2590,14 +2460,12 @@ static void define_gf_group(AV1_COMP *cpi, FIRSTPASS_STATS *this_frame) { rc->baseline_gf_interval = rc->frames_to_key - MIN_FWD_KF_INTERVAL; } } else { - rc->baseline_gf_interval = - i - (is_key_frame || rc->source_alt_ref_pending); + rc->baseline_gf_interval = i - rc->source_alt_ref_pending; } } else { - rc->baseline_gf_interval = i - (is_key_frame || rc->source_alt_ref_pending); + rc->baseline_gf_interval = i - rc->source_alt_ref_pending; } -#if REDUCE_LAST_ALT_BOOST #define LAST_ALR_BOOST_FACTOR 0.2f rc->arf_boost_factor = 1.0; if (rc->source_alt_ref_pending && !is_lossless_requested(&cpi->oxcf)) { @@ -2607,29 +2475,24 @@ static void define_gf_group(AV1_COMP *cpi, FIRSTPASS_STATS *this_frame) { rc->arf_boost_factor = LAST_ALR_BOOST_FACTOR; } } -#endif if (!cpi->extra_arf_allowed) { cpi->num_extra_arfs = 0; } else { -#if USE_SYMM_MULTI_LAYER - if (rc->baseline_gf_interval == 4 && rc->source_alt_ref_pending) + // Calculate 'num_extra_arfs' (internal alt-refs) that we are allowed. + // Note: When new pyramid structure is used through + // 'define_customized_gf_group_structure()' function, this value is + // overridden. + if (rc->baseline_gf_interval == MIN_GF_INTERVAL && + rc->source_alt_ref_pending) { cpi->num_extra_arfs = 1; - else - cpi->num_extra_arfs = get_number_of_extra_arfs( - rc->baseline_gf_interval, rc->source_alt_ref_pending); -#else - // Compute how many extra alt_refs we can have - cpi->num_extra_arfs = get_number_of_extra_arfs(rc->baseline_gf_interval, - rc->source_alt_ref_pending); -#endif // USE_SYMM_MULTI_LAYER + } else { + cpi->num_extra_arfs = get_number_of_extra_arfs(rc->baseline_gf_interval, + rc->source_alt_ref_pending, + oxcf->gf_max_pyr_height); + } } -#if !USE_SYMM_MULTI_LAYER - // Currently at maximum two extra ARFs' are allowed - assert(cpi->num_extra_arfs <= MAX_EXT_ARFS); -#endif - rc->frames_till_gf_update_due = rc->baseline_gf_interval; rc->bipred_group_interval = BFG_INTERVAL; @@ -2691,20 +2554,21 @@ static void define_gf_group(AV1_COMP *cpi, FIRSTPASS_STATS *this_frame) { // also a key frame in which case it has already been accounted for. if (rc->source_alt_ref_pending) { gf_group_error_left = gf_group_err - mod_frame_err; - } else if (is_key_frame == 0) { + } else if (!is_intra_only) { gf_group_error_left = gf_group_err - gf_first_frame_err; } else { gf_group_error_left = gf_group_err; } // Allocate bits to each of the frames in the GF group. - allocate_gf_group_bits(cpi, gf_group_bits, gf_group_error_left, gf_arf_bits); + allocate_gf_group_bits(cpi, gf_group_bits, gf_group_error_left, gf_arf_bits, + frame_params); // Reset the file position. reset_fpf_position(twopass, start_pos); // Calculate a section intra ratio used in setting max loop filter. - if (cpi->common.frame_type != KEY_FRAME) { + if (frame_params->frame_type != KEY_FRAME) { twopass->section_intra_rating = calculate_section_intra_ratio( start_pos, twopass->stats_in_end, rc->baseline_gf_interval); } @@ -2843,7 +2707,7 @@ static void find_next_key_frame(AV1_COMP *cpi, FIRSTPASS_STATS *this_frame) { av1_zero(next_frame); - cpi->common.frame_type = KEY_FRAME; + rc->frames_since_key = 0; // Reset the GF group data structures. av1_zero(*gf_group); @@ -2988,8 +2852,13 @@ static void find_next_key_frame(AV1_COMP *cpi, FIRSTPASS_STATS *this_frame) { if (EOF == input_stats(twopass, &next_frame)) break; // Monitor for static sections. - zero_motion_accumulator = AOMMIN(zero_motion_accumulator, - get_zero_motion_factor(cpi, &next_frame)); + // For the first frame in kf group, the second ref indicator is invalid. + if (i > 0) { + zero_motion_accumulator = AOMMIN( + zero_motion_accumulator, get_zero_motion_factor(cpi, &next_frame)); + } else { + zero_motion_accumulator = next_frame.pcnt_inter - next_frame.pcnt_motion; + } // Not all frames in the group are necessarily used in calculating boost. if ((i <= rc->max_gf_interval) || @@ -3021,10 +2890,18 @@ static void find_next_key_frame(AV1_COMP *cpi, FIRSTPASS_STATS *this_frame) { twopass->section_intra_rating = calculate_section_intra_ratio( start_position, twopass->stats_in_end, rc->frames_to_key); - // Apply various clamps for min and max boost rc->kf_boost = (int)(av_decay_accumulator * boost_score); - rc->kf_boost = AOMMAX(rc->kf_boost, (rc->frames_to_key * 3)); - rc->kf_boost = AOMMAX(rc->kf_boost, MIN_KF_BOOST); + + // Special case for static / slide show content but don't apply + // if the kf group is very short. + if ((zero_motion_accumulator > STATIC_KF_GROUP_FLOAT_THRESH) && + (rc->frames_to_key > 8)) { + rc->kf_boost = AOMMAX(rc->kf_boost, MIN_STATIC_KF_BOOST); + } else { + // Apply various clamps for min and max boost + rc->kf_boost = AOMMAX(rc->kf_boost, (rc->frames_to_key * 3)); + rc->kf_boost = AOMMAX(rc->kf_boost, MIN_KF_BOOST); + } // Work out how many bits to allocate for the key frame itself. kf_bits = calculate_boost_bits((rc->frames_to_key - 1), rc->kf_boost, @@ -3058,172 +2935,6 @@ static void find_next_key_frame(AV1_COMP *cpi, FIRSTPASS_STATS *this_frame) { twopass->modified_error_left -= kf_group_err; } -// Define the reference buffers that will be updated post encode. -static void configure_buffer_updates(AV1_COMP *cpi) { - TWO_PASS *const twopass = &cpi->twopass; - - // NOTE(weitinglin): Should we define another function to take care of - // cpi->rc.is_$Source_Type to make this function as it is in the comment? - - cpi->rc.is_src_frame_alt_ref = 0; - cpi->rc.is_bwd_ref_frame = 0; - cpi->rc.is_last_bipred_frame = 0; - cpi->rc.is_bipred_frame = 0; - cpi->rc.is_src_frame_ext_arf = 0; - - switch (twopass->gf_group.update_type[twopass->gf_group.index]) { - case KF_UPDATE: - cpi->refresh_last_frame = 1; - cpi->refresh_golden_frame = 1; - cpi->refresh_bwd_ref_frame = 1; - cpi->refresh_alt2_ref_frame = 1; - cpi->refresh_alt_ref_frame = 1; - break; - - case LF_UPDATE: - cpi->refresh_last_frame = 1; - cpi->refresh_golden_frame = 0; - cpi->refresh_bwd_ref_frame = 0; - cpi->refresh_alt2_ref_frame = 0; - cpi->refresh_alt_ref_frame = 0; - break; - - case GF_UPDATE: - // TODO(zoeliu): To further investigate whether 'refresh_last_frame' is - // needed. - cpi->refresh_last_frame = 1; - cpi->refresh_golden_frame = 1; - cpi->refresh_bwd_ref_frame = 0; - cpi->refresh_alt2_ref_frame = 0; - cpi->refresh_alt_ref_frame = 0; - break; - - case OVERLAY_UPDATE: - cpi->refresh_last_frame = 0; - cpi->refresh_golden_frame = 1; - cpi->refresh_bwd_ref_frame = 0; - cpi->refresh_alt2_ref_frame = 0; - cpi->refresh_alt_ref_frame = 0; - - cpi->rc.is_src_frame_alt_ref = 1; - break; - - case ARF_UPDATE: - cpi->refresh_last_frame = 0; - cpi->refresh_golden_frame = 0; - // NOTE: BWDREF does not get updated along with ALTREF_FRAME. - cpi->refresh_bwd_ref_frame = 0; - cpi->refresh_alt2_ref_frame = 0; - cpi->refresh_alt_ref_frame = 1; - break; - - case BRF_UPDATE: - cpi->refresh_last_frame = 0; - cpi->refresh_golden_frame = 0; - cpi->refresh_bwd_ref_frame = 1; - cpi->refresh_alt2_ref_frame = 0; - cpi->refresh_alt_ref_frame = 0; - - cpi->rc.is_bwd_ref_frame = 1; - break; - - case LAST_BIPRED_UPDATE: - cpi->refresh_last_frame = 1; - cpi->refresh_golden_frame = 0; - cpi->refresh_bwd_ref_frame = 0; - cpi->refresh_alt2_ref_frame = 0; - cpi->refresh_alt_ref_frame = 0; - - cpi->rc.is_last_bipred_frame = 1; - break; - - case BIPRED_UPDATE: - cpi->refresh_last_frame = 1; - cpi->refresh_golden_frame = 0; - cpi->refresh_bwd_ref_frame = 0; - cpi->refresh_alt2_ref_frame = 0; - cpi->refresh_alt_ref_frame = 0; - - cpi->rc.is_bipred_frame = 1; - break; - - case INTNL_OVERLAY_UPDATE: - cpi->refresh_last_frame = 1; - cpi->refresh_golden_frame = 0; - cpi->refresh_bwd_ref_frame = 0; - cpi->refresh_alt2_ref_frame = 0; - cpi->refresh_alt_ref_frame = 0; - - cpi->rc.is_src_frame_alt_ref = 1; - cpi->rc.is_src_frame_ext_arf = 1; - break; - - case INTNL_ARF_UPDATE: - cpi->refresh_last_frame = 0; - cpi->refresh_golden_frame = 0; -#if USE_SYMM_MULTI_LAYER - if (cpi->new_bwdref_update_rule == 1) { - cpi->refresh_bwd_ref_frame = 1; - cpi->refresh_alt2_ref_frame = 0; - } else { -#endif - cpi->refresh_bwd_ref_frame = 0; - cpi->refresh_alt2_ref_frame = 1; -#if USE_SYMM_MULTI_LAYER - } -#endif - cpi->refresh_alt_ref_frame = 0; - break; - - default: assert(0); break; - } -} - -void av1_configure_buffer_updates_firstpass(AV1_COMP *cpi, - FRAME_UPDATE_TYPE update_type) { - RATE_CONTROL *rc = &cpi->rc; - - cpi->refresh_last_frame = 1; - cpi->refresh_golden_frame = 0; - cpi->refresh_bwd_ref_frame = 0; - cpi->refresh_alt2_ref_frame = 0; - cpi->refresh_alt_ref_frame = 0; - - rc->is_bwd_ref_frame = 0; - - switch (update_type) { - case ARF_UPDATE: - cpi->refresh_alt_ref_frame = 1; - cpi->refresh_last_frame = 0; - cpi->refresh_golden_frame = 0; - cpi->refresh_bwd_ref_frame = 0; - cpi->refresh_alt2_ref_frame = 0; - - rc->is_src_frame_alt_ref = 0; - break; - case INTNL_ARF_UPDATE: - cpi->refresh_alt2_ref_frame = 1; - cpi->refresh_last_frame = 0; - cpi->refresh_golden_frame = 0; - cpi->refresh_bwd_ref_frame = 0; - cpi->refresh_alt_ref_frame = 0; - rc->is_src_frame_alt_ref = 0; - rc->is_src_frame_ext_arf = 0; - - break; - case BIPRED_UPDATE: - cpi->refresh_bwd_ref_frame = 1; - cpi->refresh_last_frame = 0; - cpi->refresh_golden_frame = 0; - cpi->refresh_alt2_ref_frame = 0; - cpi->refresh_alt_ref_frame = 0; - - rc->is_bwd_ref_frame = 1; - break; - default: break; - } -} - static int is_skippable_frame(const AV1_COMP *cpi) { // If the current frame does not have non-zero motion vector detected in the // first pass, and so do its previous and forward frames, then this frame @@ -3243,8 +2954,10 @@ static int is_skippable_frame(const AV1_COMP *cpi) { twopass->stats_in->pcnt_inter - twopass->stats_in->pcnt_motion == 1); } -void av1_rc_get_second_pass_params(AV1_COMP *cpi) { +void av1_rc_get_second_pass_params(AV1_COMP *cpi, + EncodeFrameParams *const frame_params) { AV1_COMMON *const cm = &cpi->common; + CurrentFrame *const current_frame = &cm->current_frame; RATE_CONTROL *const rc = &cpi->rc; TWO_PASS *const twopass = &cpi->twopass; GF_GROUP *const gf_group = &twopass->gf_group; @@ -3253,7 +2966,7 @@ void av1_rc_get_second_pass_params(AV1_COMP *cpi) { int target_rate; - frames_left = (int)(twopass->total_stats.count - cm->current_video_frame); + frames_left = (int)(twopass->total_stats.count - current_frame->frame_number); if (!twopass->stats_in) return; @@ -3261,16 +2974,16 @@ void av1_rc_get_second_pass_params(AV1_COMP *cpi) { // advance the input pointer as we already have what we need. if (gf_group->update_type[gf_group->index] == ARF_UPDATE || gf_group->update_type[gf_group->index] == INTNL_ARF_UPDATE) { - configure_buffer_updates(cpi); + av1_configure_buffer_updates(cpi, gf_group->update_type[gf_group->index]); target_rate = gf_group->bit_allocation[gf_group->index]; target_rate = av1_rc_clamp_pframe_target_size(cpi, target_rate); rc->base_frame_target = target_rate; if (cpi->no_show_kf) { assert(gf_group->update_type[gf_group->index] == ARF_UPDATE); - cm->frame_type = KEY_FRAME; + frame_params->frame_type = KEY_FRAME; } else { - cm->frame_type = INTER_FRAME; + frame_params->frame_type = INTER_FRAME; } // Do the firstpass stats indicate that this frame is skippable for the @@ -3286,7 +2999,7 @@ void av1_rc_get_second_pass_params(AV1_COMP *cpi) { if (cpi->oxcf.rc_mode == AOM_Q) { twopass->active_worst_quality = cpi->oxcf.cq_level; - } else if (cm->current_video_frame == 0) { + } else if (current_frame->frame_number == 0) { // Special case code for first frame. const int section_target_bandwidth = (int)(twopass->bits_left / frames_left); @@ -3325,16 +3038,17 @@ void av1_rc_get_second_pass_params(AV1_COMP *cpi) { if (rc->frames_to_key == 0 || (cpi->frame_flags & FRAMEFLAGS_KEY)) { FIRSTPASS_STATS this_frame_copy; this_frame_copy = this_frame; + frame_params->frame_type = KEY_FRAME; // Define next KF group and assign bits to it. find_next_key_frame(cpi, &this_frame); this_frame = this_frame_copy; } else { - cm->frame_type = INTER_FRAME; + frame_params->frame_type = INTER_FRAME; } // Define a new GF/ARF group. (Should always enter here for key frames). if (rc->frames_till_gf_update_due == 0) { - define_gf_group(cpi, &this_frame); + define_gf_group(cpi, &this_frame, frame_params); rc->frames_till_gf_update_due = rc->baseline_gf_interval; @@ -3343,7 +3057,7 @@ void av1_rc_get_second_pass_params(AV1_COMP *cpi) { FILE *fpfile; fpfile = fopen("arf.stt", "a"); ++arf_count; - fprintf(fpfile, "%10d %10d %10d %10d %10d\n", cm->current_video_frame, + fprintf(fpfile, "%10d %10d %10d %10d %10d\n", current_frame->frame_number, rc->frames_till_gf_update_due, rc->kf_boost, arf_count, rc->gfu_boost); @@ -3352,7 +3066,7 @@ void av1_rc_get_second_pass_params(AV1_COMP *cpi) { #endif } - configure_buffer_updates(cpi); + av1_configure_buffer_updates(cpi, gf_group->update_type[gf_group->index]); // Do the firstpass stats indicate that this frame is skippable for the // partition search? @@ -3362,7 +3076,7 @@ void av1_rc_get_second_pass_params(AV1_COMP *cpi) { target_rate = gf_group->bit_allocation[gf_group->index]; - if (cpi->common.frame_type == KEY_FRAME) + if (frame_params->frame_type == KEY_FRAME) target_rate = av1_rc_clamp_iframe_target_size(cpi, target_rate); else target_rate = av1_rc_clamp_pframe_target_size(cpi, target_rate); @@ -3409,7 +3123,7 @@ void av1_twopass_postencode_update(AV1_COMP *cpi) { rc->rate_error_estimate = 0; } - if (cpi->common.frame_type != KEY_FRAME) { + if (cpi->common.current_frame.frame_type != KEY_FRAME) { twopass->kf_group_bits -= bits_used; twopass->last_kfgroup_zeromotion_pct = twopass->kf_zeromotion_pct; } diff --git a/third_party/aom/av1/encoder/firstpass.h b/third_party/aom/av1/encoder/firstpass.h index 4b7325ae21..be1101a4b0 100644 --- a/third_party/aom/av1/encoder/firstpass.h +++ b/third_party/aom/av1/encoder/firstpass.h @@ -86,7 +86,7 @@ typedef struct { double raw_error_stdev; } FIRSTPASS_STATS; -typedef enum { +enum { KF_UPDATE = 0, LF_UPDATE = 1, GF_UPDATE = 2, @@ -98,34 +98,29 @@ typedef enum { INTNL_OVERLAY_UPDATE = 8, // Internal Overlay Frame INTNL_ARF_UPDATE = 9, // Internal Altref Frame (candidate for ALTREF2) FRAME_UPDATE_TYPES = 10 -} FRAME_UPDATE_TYPE; +} UENUM1BYTE(FRAME_UPDATE_TYPE); #define FC_ANIMATION_THRESH 0.15 -typedef enum { +enum { FC_NORMAL = 0, FC_GRAPHICS_ANIMATION = 1, FRAME_CONTENT_TYPES = 2 -} FRAME_CONTENT_TYPE; +} UENUM1BYTE(FRAME_CONTENT_TYPE); typedef struct { unsigned char index; - RATE_FACTOR_LEVEL rf_level[(MAX_LAG_BUFFERS * 2) + 1]; - FRAME_UPDATE_TYPE update_type[(MAX_LAG_BUFFERS * 2) + 1]; - unsigned char arf_src_offset[(MAX_LAG_BUFFERS * 2) + 1]; - unsigned char arf_update_idx[(MAX_LAG_BUFFERS * 2) + 1]; - unsigned char arf_ref_idx[(MAX_LAG_BUFFERS * 2) + 1]; -#if USE_SYMM_MULTI_LAYER - unsigned char arf_pos_in_gf[(MAX_LAG_BUFFERS * 2) + 1]; - unsigned char pyramid_level[(MAX_LAG_BUFFERS * 2) + 1]; + RATE_FACTOR_LEVEL rf_level[MAX_STATIC_GF_GROUP_LENGTH + 1]; + FRAME_UPDATE_TYPE update_type[MAX_STATIC_GF_GROUP_LENGTH + 1]; + unsigned char arf_src_offset[MAX_STATIC_GF_GROUP_LENGTH + 1]; + unsigned char arf_update_idx[MAX_STATIC_GF_GROUP_LENGTH + 1]; + unsigned char arf_ref_idx[MAX_STATIC_GF_GROUP_LENGTH + 1]; + unsigned char arf_pos_in_gf[MAX_STATIC_GF_GROUP_LENGTH + 1]; + unsigned char pyramid_level[MAX_STATIC_GF_GROUP_LENGTH + 1]; unsigned char pyramid_height; unsigned char pyramid_lvl_nodes[MAX_PYRAMID_LVL]; -#endif - unsigned char brf_src_offset[(MAX_LAG_BUFFERS * 2) + 1]; - unsigned char bidir_pred_enabled[(MAX_LAG_BUFFERS * 2) + 1]; - unsigned char ref_fb_idx_map[(MAX_LAG_BUFFERS * 2) + 1][REF_FRAMES]; - unsigned char refresh_idx[(MAX_LAG_BUFFERS * 2) + 1]; - unsigned char refresh_flag[(MAX_LAG_BUFFERS * 2) + 1]; - int bit_allocation[(MAX_LAG_BUFFERS * 2) + 1]; + unsigned char brf_src_offset[MAX_STATIC_GF_GROUP_LENGTH + 1]; + unsigned char bidir_pred_enabled[MAX_STATIC_GF_GROUP_LENGTH + 1]; + int bit_allocation[MAX_STATIC_GF_GROUP_LENGTH + 1]; } GF_GROUP; typedef struct { @@ -176,29 +171,37 @@ typedef struct { } TWO_PASS; struct AV1_COMP; +struct EncodeFrameParams; void av1_init_first_pass(struct AV1_COMP *cpi); void av1_rc_get_first_pass_params(struct AV1_COMP *cpi); -void av1_first_pass(struct AV1_COMP *cpi, const struct lookahead_entry *source); +void av1_first_pass(struct AV1_COMP *cpi, const int64_t ts_duration); void av1_end_first_pass(struct AV1_COMP *cpi); void av1_init_second_pass(struct AV1_COMP *cpi); -void av1_rc_get_second_pass_params(struct AV1_COMP *cpi); -void av1_configure_buffer_updates_firstpass(struct AV1_COMP *cpi, - FRAME_UPDATE_TYPE update_type); +void av1_rc_get_second_pass_params( + struct AV1_COMP *cpi, struct EncodeFrameParams *const frame_params); // Post encode update of the rate control parameters for 2-pass void av1_twopass_postencode_update(struct AV1_COMP *cpi); -static INLINE int get_number_of_extra_arfs(int interval, int arf_pending) { - if (arf_pending && MAX_EXT_ARFS > 0) - return interval >= MIN_EXT_ARF_INTERVAL * (MAX_EXT_ARFS + 1) - ? MAX_EXT_ARFS - : interval >= MIN_EXT_ARF_INTERVAL * MAX_EXT_ARFS - ? MAX_EXT_ARFS - 1 - : 0; - else - return 0; +static INLINE int get_number_of_extra_arfs(int interval, int arf_pending, + int max_pyr_height) { + // Max extra (internal) alt-refs allowed based on interval. + int extra_arfs_from_interval = 0; + if (arf_pending && MAX_EXT_ARFS > 0) { + extra_arfs_from_interval = + (interval >= MIN_EXT_ARF_INTERVAL * (MAX_EXT_ARFS + 1)) + ? MAX_EXT_ARFS + : (interval >= MIN_EXT_ARF_INTERVAL * MAX_EXT_ARFS) + ? MAX_EXT_ARFS - 1 + : 0; + } + // Max extra (internal) alt-refs allowed based on max pyramid height. + assert(max_pyr_height >= 1); + const int ext_arfs_from_max_height = max_pyr_height - 1; + // Finally, min of the two above is our actual max allowance. + return AOMMIN(extra_arfs_from_interval, ext_arfs_from_max_height); } #ifdef __cplusplus diff --git a/third_party/aom/av1/encoder/global_motion.c b/third_party/aom/av1/encoder/global_motion.c index e9f8b0bb47..142e44d804 100644 --- a/third_party/aom/av1/encoder/global_motion.c +++ b/third_party/aom/av1/encoder/global_motion.c @@ -15,8 +15,12 @@ #include #include +#include "config/aom_dsp_rtcd.h" + #include "av1/encoder/global_motion.h" +#include "av1/common/convolve.h" +#include "av1/common/resize.h" #include "av1/common/warped_motion.h" #include "av1/encoder/segmentation.h" @@ -28,15 +32,45 @@ #define MIN_INLIER_PROB 0.1 #define MIN_TRANS_THRESH (1 * GM_TRANS_DECODE_FACTOR) +#define USE_GM_FEATURE_BASED 1 // Border over which to compute the global motion #define ERRORADV_BORDER 0 +// Number of pyramid levels in disflow computation +#define N_LEVELS 2 +// Size of square patches in the disflow dense grid +#define PATCH_SIZE 8 +// Center point of square patch +#define PATCH_CENTER ((PATCH_SIZE + 1) >> 1) +// Step size between patches, lower value means greater patch overlap +#define PATCH_STEP 1 +// Minimum size of border padding for disflow +#define MIN_PAD 7 +// Warp error convergence threshold for disflow +#define DISFLOW_ERROR_TR 0.01 +// Max number of iterations if warp convergence is not found +#define DISFLOW_MAX_ITR 10 + +// Struct for an image pyramid +typedef struct { + int n_levels; + int pad_size; + int has_gradient; + int widths[N_LEVELS]; + int heights[N_LEVELS]; + int strides[N_LEVELS]; + int level_loc[N_LEVELS]; + unsigned char *level_buffer; + double *level_dx_buffer; + double *level_dy_buffer; +} ImagePyramid; + static const double erroradv_tr[] = { 0.65, 0.60, 0.55 }; static const double erroradv_prod_tr[] = { 20000, 18000, 16000 }; -int is_enough_erroradvantage(double best_erroradvantage, int params_cost, - int erroradv_type) { +int av1_is_enough_erroradvantage(double best_erroradvantage, int params_cost, + int erroradv_type) { assert(erroradv_type < GM_ERRORADV_TR_TYPES); return best_erroradvantage < erroradv_tr[erroradv_type] && best_erroradvantage * params_cost < erroradv_prod_tr[erroradv_type]; @@ -75,9 +109,10 @@ static void convert_to_params(const double *params, int32_t *model) { } } -void convert_model_to_params(const double *params, WarpedMotionParams *model) { +void av1_convert_model_to_params(const double *params, + WarpedMotionParams *model) { convert_to_params(params, model->wmmat); - model->wmtype = get_gmtype(model); + model->wmtype = get_wmtype(model); model->invalid = 0; } @@ -131,12 +166,13 @@ static void force_wmtype(WarpedMotionParams *wm, TransformationType wmtype) { wm->wmtype = wmtype; } -int64_t refine_integerized_param(WarpedMotionParams *wm, - TransformationType wmtype, int use_hbd, int bd, - uint8_t *ref, int r_width, int r_height, - int r_stride, uint8_t *dst, int d_width, - int d_height, int d_stride, int n_refinements, - int64_t best_frame_error) { +int64_t av1_refine_integerized_param(WarpedMotionParams *wm, + TransformationType wmtype, int use_hbd, + int bd, uint8_t *ref, int r_width, + int r_height, int r_stride, uint8_t *dst, + int d_width, int d_height, int d_stride, + int n_refinements, + int64_t best_frame_error) { static const int max_trans_model_params[TRANS_TYPES] = { 0, 2, 4, 6 }; const int border = ERRORADV_BORDER; int i = 0, p; @@ -209,7 +245,7 @@ int64_t refine_integerized_param(WarpedMotionParams *wm, } } force_wmtype(wm, wmtype); - wm->wmtype = get_gmtype(wm); + wm->wmtype = get_wmtype(wm); return best_error; } @@ -240,12 +276,11 @@ static unsigned char *downconvert_frame(YV12_BUFFER_CONFIG *frm, return buf_8bit; } -int compute_global_motion_feature_based(TransformationType type, - YV12_BUFFER_CONFIG *frm, - YV12_BUFFER_CONFIG *ref, int bit_depth, - int *num_inliers_by_motion, - double *params_by_motion, - int num_motions) { +#if USE_GM_FEATURE_BASED +static int compute_global_motion_feature_based( + TransformationType type, YV12_BUFFER_CONFIG *frm, YV12_BUFFER_CONFIG *ref, + int bit_depth, int *num_inliers_by_motion, double *params_by_motion, + int num_motions) { int i; int num_frm_corners, num_ref_corners; int num_correspondences; @@ -296,3 +331,557 @@ int compute_global_motion_feature_based(TransformationType type, } return 0; } +#else +static INLINE RansacFuncDouble +get_ransac_double_prec_type(TransformationType type) { + switch (type) { + case AFFINE: return ransac_affine_double_prec; + case ROTZOOM: return ransac_rotzoom_double_prec; + case TRANSLATION: return ransac_translation_double_prec; + default: assert(0); return NULL; + } +} + +// Don't use points around the frame border since they are less reliable +static INLINE int valid_point(int x, int y, int width, int height) { + return (x > (PATCH_SIZE + PATCH_CENTER)) && + (x < (width - PATCH_SIZE - PATCH_CENTER)) && + (y > (PATCH_SIZE + PATCH_CENTER)) && + (y < (height - PATCH_SIZE - PATCH_CENTER)); +} + +static int determine_disflow_correspondence(int *frm_corners, + int num_frm_corners, double *flow_u, + double *flow_v, int width, + int height, int stride, + double *correspondences) { + int num_correspondences = 0; + int x, y; + for (int i = 0; i < num_frm_corners; ++i) { + x = frm_corners[2 * i]; + y = frm_corners[2 * i + 1]; + if (valid_point(x, y, width, height)) { + correspondences[4 * num_correspondences] = x; + correspondences[4 * num_correspondences + 1] = y; + correspondences[4 * num_correspondences + 2] = x + flow_u[y * stride + x]; + correspondences[4 * num_correspondences + 3] = y + flow_v[y * stride + x]; + num_correspondences++; + } + } + return num_correspondences; +} + +double getCubicValue(double p[4], double x) { + return p[1] + 0.5 * x * + (p[2] - p[0] + + x * (2.0 * p[0] - 5.0 * p[1] + 4.0 * p[2] - p[3] + + x * (3.0 * (p[1] - p[2]) + p[3] - p[0]))); +} + +void get_subcolumn(unsigned char *ref, double col[4], int stride, int x, + int y_start) { + int i; + for (i = 0; i < 4; ++i) { + col[i] = ref[(i + y_start) * stride + x]; + } +} + +double bicubic(unsigned char *ref, double x, double y, int stride) { + double arr[4]; + int k; + int i = (int)x; + int j = (int)y; + for (k = 0; k < 4; ++k) { + double arr_temp[4]; + get_subcolumn(ref, arr_temp, stride, i + k - 1, j - 1); + arr[k] = getCubicValue(arr_temp, y - j); + } + return getCubicValue(arr, x - i); +} + +// Interpolate a warped block using bicubic interpolation when possible +unsigned char interpolate(unsigned char *ref, double x, double y, int width, + int height, int stride) { + if (x < 0 && y < 0) + return ref[0]; + else if (x < 0 && y > height - 1) + return ref[(height - 1) * stride]; + else if (x > width - 1 && y < 0) + return ref[width - 1]; + else if (x > width - 1 && y > height - 1) + return ref[(height - 1) * stride + (width - 1)]; + else if (x < 0) { + int v; + int i = (int)y; + double a = y - i; + if (y > 1 && y < height - 2) { + double arr[4]; + get_subcolumn(ref, arr, stride, 0, i - 1); + return clamp((int)(getCubicValue(arr, a) + 0.5), 0, 255); + } + v = (int)(ref[i * stride] * (1 - a) + ref[(i + 1) * stride] * a + 0.5); + return clamp(v, 0, 255); + } else if (y < 0) { + int v; + int j = (int)x; + double b = x - j; + if (x > 1 && x < width - 2) { + double arr[4] = { ref[j - 1], ref[j], ref[j + 1], ref[j + 2] }; + return clamp((int)(getCubicValue(arr, b) + 0.5), 0, 255); + } + v = (int)(ref[j] * (1 - b) + ref[j + 1] * b + 0.5); + return clamp(v, 0, 255); + } else if (x > width - 1) { + int v; + int i = (int)y; + double a = y - i; + if (y > 1 && y < height - 2) { + double arr[4]; + get_subcolumn(ref, arr, stride, width - 1, i - 1); + return clamp((int)(getCubicValue(arr, a) + 0.5), 0, 255); + } + v = (int)(ref[i * stride + width - 1] * (1 - a) + + ref[(i + 1) * stride + width - 1] * a + 0.5); + return clamp(v, 0, 255); + } else if (y > height - 1) { + int v; + int j = (int)x; + double b = x - j; + if (x > 1 && x < width - 2) { + int row = (height - 1) * stride; + double arr[4] = { ref[row + j - 1], ref[row + j], ref[row + j + 1], + ref[row + j + 2] }; + return clamp((int)(getCubicValue(arr, b) + 0.5), 0, 255); + } + v = (int)(ref[(height - 1) * stride + j] * (1 - b) + + ref[(height - 1) * stride + j + 1] * b + 0.5); + return clamp(v, 0, 255); + } else if (x > 1 && y > 1 && x < width - 2 && y < height - 2) { + return clamp((int)(bicubic(ref, x, y, stride) + 0.5), 0, 255); + } else { + int i = (int)y; + int j = (int)x; + double a = y - i; + double b = x - j; + int v = (int)(ref[i * stride + j] * (1 - a) * (1 - b) + + ref[i * stride + j + 1] * (1 - a) * b + + ref[(i + 1) * stride + j] * a * (1 - b) + + ref[(i + 1) * stride + j + 1] * a * b); + return clamp(v, 0, 255); + } +} + +// Warps a block using flow vector [u, v] and computes the mse +double compute_warp_and_error(unsigned char *ref, unsigned char *frm, int width, + int height, int stride, int x, int y, double u, + double v, int16_t *dt) { + int i, j; + unsigned char warped; + double x_w, y_w; + double mse = 0; + int16_t err = 0; + for (i = y; i < y + PATCH_SIZE; ++i) + for (j = x; j < x + PATCH_SIZE; ++j) { + x_w = (double)j + u; + y_w = (double)i + v; + warped = interpolate(ref, x_w, y_w, width, height, stride); + err = warped - frm[j + i * stride]; + mse += err * err; + dt[(i - y) * PATCH_SIZE + (j - x)] = err; + } + + mse /= (PATCH_SIZE * PATCH_SIZE); + return mse; +} + +// Computes the components of the system of equations used to solve for +// a flow vector. This includes: +// 1.) The hessian matrix for optical flow. This matrix is in the +// form of: +// +// M = |sum(dx * dx) sum(dx * dy)| +// |sum(dx * dy) sum(dy * dy)| +// +// 2.) b = |sum(dx * dt)| +// |sum(dy * dt)| +// Where the sums are computed over a square window of PATCH_SIZE. +static INLINE void compute_flow_system(const double *dx, int dx_stride, + const double *dy, int dy_stride, + const int16_t *dt, int dt_stride, + double *M, double *b) { + for (int i = 0; i < PATCH_SIZE; i++) { + for (int j = 0; j < PATCH_SIZE; j++) { + M[0] += dx[i * dx_stride + j] * dx[i * dx_stride + j]; + M[1] += dx[i * dx_stride + j] * dy[i * dy_stride + j]; + M[3] += dy[i * dy_stride + j] * dy[i * dy_stride + j]; + + b[0] += dx[i * dx_stride + j] * dt[i * dt_stride + j]; + b[1] += dy[i * dy_stride + j] * dt[i * dt_stride + j]; + } + } + + M[2] = M[1]; +} + +// Solves a general Mx = b where M is a 2x2 matrix and b is a 2x1 matrix +static INLINE void solve_2x2_system(const double *M, const double *b, + double *output_vec) { + double M_0 = M[0]; + double M_3 = M[3]; + double det = (M_0 * M_3) - (M[1] * M[2]); + if (det < 1e-5) { + // Handle singular matrix + // TODO(sarahparker) compare results using pseudo inverse instead + M_0 += 1e-10; + M_3 += 1e-10; + det = (M_0 * M_3) - (M[1] * M[2]); + } + const double det_inv = 1 / det; + const double mult_b0 = det_inv * b[0]; + const double mult_b1 = det_inv * b[1]; + output_vec[0] = M_3 * mult_b0 - M[1] * mult_b1; + output_vec[1] = -M[2] * mult_b0 + M_0 * mult_b1; +} + +static INLINE void image_difference(const uint8_t *src, int src_stride, + const uint8_t *ref, int ref_stride, + int16_t *dst, int dst_stride, int height, + int width) { + const int block_unit = 8; + // Take difference in 8x8 blocks to make use of optimized diff function + for (int i = 0; i < height; i += block_unit) { + for (int j = 0; j < width; j += block_unit) { + aom_subtract_block(block_unit, block_unit, dst + i * dst_stride + j, + dst_stride, src + i * src_stride + j, src_stride, + ref + i * ref_stride + j, ref_stride); + } + } +} + +// Compute an image gradient using a sobel filter. +// If dir == 1, compute the x gradient. If dir == 0, compute y. This function +// assumes the images have been padded so that they can be processed in units +// of 8. +static INLINE void sobel_xy_image_gradient(const uint8_t *src, int src_stride, + double *dst, int dst_stride, + int height, int width, int dir) { + double norm = 1.0; + // TODO(sarahparker) experiment with doing this over larger block sizes + const int block_unit = 8; + // Filter in 8x8 blocks to eventually make use of optimized convolve function + for (int i = 0; i < height; i += block_unit) { + for (int j = 0; j < width; j += block_unit) { + av1_convolve_2d_sobel_y_c(src + i * src_stride + j, src_stride, + dst + i * dst_stride + j, dst_stride, + block_unit, block_unit, dir, norm); + } + } +} + +static ImagePyramid *alloc_pyramid(int width, int height, int pad_size, + int compute_gradient) { + ImagePyramid *pyr = aom_malloc(sizeof(*pyr)); + pyr->has_gradient = compute_gradient; + // 2 * width * height is the upper bound for a buffer that fits + // all pyramid levels + padding for each level + const int buffer_size = sizeof(*pyr->level_buffer) * 2 * width * height + + (width + 2 * pad_size) * 2 * pad_size * N_LEVELS; + pyr->level_buffer = aom_malloc(buffer_size); + memset(pyr->level_buffer, 0, buffer_size); + + if (compute_gradient) { + const int gradient_size = + sizeof(*pyr->level_dx_buffer) * 2 * width * height + + (width + 2 * pad_size) * 2 * pad_size * N_LEVELS; + pyr->level_dx_buffer = aom_malloc(gradient_size); + pyr->level_dy_buffer = aom_malloc(gradient_size); + memset(pyr->level_dx_buffer, 0, gradient_size); + memset(pyr->level_dy_buffer, 0, gradient_size); + } + return pyr; +} + +static void free_pyramid(ImagePyramid *pyr) { + aom_free(pyr->level_buffer); + if (pyr->has_gradient) { + aom_free(pyr->level_dx_buffer); + aom_free(pyr->level_dy_buffer); + } + aom_free(pyr); +} + +static INLINE void update_level_dims(ImagePyramid *frm_pyr, int level) { + frm_pyr->widths[level] = frm_pyr->widths[level - 1] >> 1; + frm_pyr->heights[level] = frm_pyr->heights[level - 1] >> 1; + frm_pyr->strides[level] = frm_pyr->widths[level] + 2 * frm_pyr->pad_size; + // Point the beginning of the next level buffer to the correct location inside + // the padded border + frm_pyr->level_loc[level] = + frm_pyr->level_loc[level - 1] + + frm_pyr->strides[level - 1] * + (2 * frm_pyr->pad_size + frm_pyr->heights[level - 1]); +} + +// Compute coarse to fine pyramids for a frame +static void compute_flow_pyramids(unsigned char *frm, const int frm_width, + const int frm_height, const int frm_stride, + int n_levels, int pad_size, int compute_grad, + ImagePyramid *frm_pyr) { + int cur_width, cur_height, cur_stride, cur_loc; + assert((frm_width >> n_levels) > 0); + assert((frm_height >> n_levels) > 0); + + // Initialize first level + frm_pyr->n_levels = n_levels; + frm_pyr->pad_size = pad_size; + frm_pyr->widths[0] = frm_width; + frm_pyr->heights[0] = frm_height; + frm_pyr->strides[0] = frm_width + 2 * frm_pyr->pad_size; + // Point the beginning of the level buffer to the location inside + // the padded border + frm_pyr->level_loc[0] = + frm_pyr->strides[0] * frm_pyr->pad_size + frm_pyr->pad_size; + // This essentially copies the original buffer into the pyramid buffer + // without the original padding + av1_resize_plane(frm, frm_height, frm_width, frm_stride, + frm_pyr->level_buffer + frm_pyr->level_loc[0], + frm_pyr->heights[0], frm_pyr->widths[0], + frm_pyr->strides[0]); + + if (compute_grad) { + cur_width = frm_pyr->widths[0]; + cur_height = frm_pyr->heights[0]; + cur_stride = frm_pyr->strides[0]; + cur_loc = frm_pyr->level_loc[0]; + assert(frm_pyr->has_gradient && frm_pyr->level_dx_buffer != NULL && + frm_pyr->level_dy_buffer != NULL); + // Computation x gradient + sobel_xy_image_gradient(frm_pyr->level_buffer + cur_loc, cur_stride, + frm_pyr->level_dx_buffer + cur_loc, cur_stride, + cur_height, cur_width, 1); + + // Computation y gradient + sobel_xy_image_gradient(frm_pyr->level_buffer + cur_loc, cur_stride, + frm_pyr->level_dy_buffer + cur_loc, cur_stride, + cur_height, cur_width, 0); + } + + // Start at the finest level and resize down to the coarsest level + for (int level = 1; level < n_levels; ++level) { + update_level_dims(frm_pyr, level); + cur_width = frm_pyr->widths[level]; + cur_height = frm_pyr->heights[level]; + cur_stride = frm_pyr->strides[level]; + cur_loc = frm_pyr->level_loc[level]; + + av1_resize_plane(frm_pyr->level_buffer + frm_pyr->level_loc[level - 1], + frm_pyr->heights[level - 1], frm_pyr->widths[level - 1], + frm_pyr->strides[level - 1], + frm_pyr->level_buffer + cur_loc, cur_height, cur_width, + cur_stride); + + if (compute_grad) { + assert(frm_pyr->has_gradient && frm_pyr->level_dx_buffer != NULL && + frm_pyr->level_dy_buffer != NULL); + // Computation x gradient + sobel_xy_image_gradient(frm_pyr->level_buffer + cur_loc, cur_stride, + frm_pyr->level_dx_buffer + cur_loc, cur_stride, + cur_height, cur_width, 1); + + // Computation y gradient + sobel_xy_image_gradient(frm_pyr->level_buffer + cur_loc, cur_stride, + frm_pyr->level_dy_buffer + cur_loc, cur_stride, + cur_height, cur_width, 0); + } + } +} + +static INLINE void compute_flow_at_point(unsigned char *frm, unsigned char *ref, + double *dx, double *dy, int x, int y, + int width, int height, int stride, + double *u, double *v) { + double M[4] = { 0 }; + double b[2] = { 0 }; + double tmp_output_vec[2] = { 0 }; + double error = 0; + int16_t dt[PATCH_SIZE * PATCH_SIZE]; + double o_u = *u; + double o_v = *v; + + for (int itr = 0; itr < DISFLOW_MAX_ITR; itr++) { + error = compute_warp_and_error(ref, frm, width, height, stride, x, y, *u, + *v, dt); + if (error <= DISFLOW_ERROR_TR) break; + compute_flow_system(dx, stride, dy, stride, dt, PATCH_SIZE, M, b); + solve_2x2_system(M, b, tmp_output_vec); + *u += tmp_output_vec[0]; + *v += tmp_output_vec[1]; + } + if (fabs(*u - o_u) > PATCH_SIZE || fabs(*v - o_u) > PATCH_SIZE) { + *u = o_u; + *v = o_v; + } +} + +// make sure flow_u and flow_v start at 0 +static void compute_flow_field(ImagePyramid *frm_pyr, ImagePyramid *ref_pyr, + double *flow_u, double *flow_v) { + int cur_width, cur_height, cur_stride, cur_loc, patch_loc, patch_center; + double *u_upscale = + aom_malloc(frm_pyr->strides[0] * frm_pyr->heights[0] * sizeof(*flow_u)); + double *v_upscale = + aom_malloc(frm_pyr->strides[0] * frm_pyr->heights[0] * sizeof(*flow_v)); + + assert(frm_pyr->n_levels == ref_pyr->n_levels); + + // Compute flow field from coarsest to finest level of the pyramid + for (int level = frm_pyr->n_levels - 1; level >= 0; --level) { + cur_width = frm_pyr->widths[level]; + cur_height = frm_pyr->heights[level]; + cur_stride = frm_pyr->strides[level]; + cur_loc = frm_pyr->level_loc[level]; + + for (int i = PATCH_SIZE; i < cur_height - PATCH_SIZE; i += PATCH_STEP) { + for (int j = PATCH_SIZE; j < cur_width - PATCH_SIZE; j += PATCH_STEP) { + patch_loc = i * cur_stride + j; + patch_center = patch_loc + PATCH_CENTER * cur_stride + PATCH_CENTER; + compute_flow_at_point(frm_pyr->level_buffer + cur_loc, + ref_pyr->level_buffer + cur_loc, + frm_pyr->level_dx_buffer + cur_loc + patch_loc, + frm_pyr->level_dy_buffer + cur_loc + patch_loc, j, + i, cur_width, cur_height, cur_stride, + flow_u + patch_center, flow_v + patch_center); + } + } + // TODO(sarahparker) Replace this with upscale function in resize.c + if (level > 0) { + int h_upscale = frm_pyr->heights[level - 1]; + int w_upscale = frm_pyr->widths[level - 1]; + int s_upscale = frm_pyr->strides[level - 1]; + for (int i = 0; i < h_upscale; ++i) { + for (int j = 0; j < w_upscale; ++j) { + u_upscale[j + i * s_upscale] = + flow_u[(int)(j >> 1) + (int)(i >> 1) * cur_stride]; + v_upscale[j + i * s_upscale] = + flow_v[(int)(j >> 1) + (int)(i >> 1) * cur_stride]; + } + } + memcpy(flow_u, u_upscale, + frm_pyr->strides[0] * frm_pyr->heights[0] * sizeof(*flow_u)); + memcpy(flow_v, v_upscale, + frm_pyr->strides[0] * frm_pyr->heights[0] * sizeof(*flow_v)); + } + } + aom_free(u_upscale); + aom_free(v_upscale); +} + +static int compute_global_motion_disflow_based( + TransformationType type, YV12_BUFFER_CONFIG *frm, YV12_BUFFER_CONFIG *ref, + int bit_depth, int *num_inliers_by_motion, double *params_by_motion, + int num_motions) { + unsigned char *frm_buffer = frm->y_buffer; + unsigned char *ref_buffer = ref->y_buffer; + const int frm_width = frm->y_width; + const int frm_height = frm->y_height; + const int ref_width = ref->y_width; + const int ref_height = ref->y_height; + const int pad_size = AOMMAX(PATCH_SIZE, MIN_PAD); + int num_frm_corners; + int num_correspondences; + double *correspondences; + int frm_corners[2 * MAX_CORNERS]; + RansacFuncDouble ransac = get_ransac_double_prec_type(type); + assert(frm_width == ref_width); + assert(frm_height == ref_height); + + // Ensure the number of pyramid levels will work with the frame resolution + const int msb = + frm_width < frm_height ? get_msb(frm_width) : get_msb(frm_height); + const int n_levels = AOMMIN(msb, N_LEVELS); + + if (frm->flags & YV12_FLAG_HIGHBITDEPTH) { + // The frame buffer is 16-bit, so we need to convert to 8 bits for the + // following code. We cache the result until the frame is released. + frm_buffer = downconvert_frame(frm, bit_depth); + } + if (ref->flags & YV12_FLAG_HIGHBITDEPTH) { + ref_buffer = downconvert_frame(ref, bit_depth); + } + + // TODO(sarahparker) We will want to do the source pyramid computation + // outside of this function so it doesn't get recomputed for every + // reference. We also don't need to compute every pyramid level for the + // reference in advance, since lower levels can be overwritten once their + // flow field is computed and upscaled. I'll add these optimizations + // once the full implementation is working. + // Allocate frm image pyramids + int compute_gradient = 1; + ImagePyramid *frm_pyr = + alloc_pyramid(frm_width, frm_height, pad_size, compute_gradient); + compute_flow_pyramids(frm_buffer, frm_width, frm_height, frm->y_stride, + n_levels, pad_size, compute_gradient, frm_pyr); + // Allocate ref image pyramids + compute_gradient = 0; + ImagePyramid *ref_pyr = + alloc_pyramid(ref_width, ref_height, pad_size, compute_gradient); + compute_flow_pyramids(ref_buffer, ref_width, ref_height, ref->y_stride, + n_levels, pad_size, compute_gradient, ref_pyr); + + double *flow_u = + aom_malloc(frm_pyr->strides[0] * frm_pyr->heights[0] * sizeof(*flow_u)); + double *flow_v = + aom_malloc(frm_pyr->strides[0] * frm_pyr->heights[0] * sizeof(*flow_v)); + + memset(flow_u, 0, + frm_pyr->strides[0] * frm_pyr->heights[0] * sizeof(*flow_u)); + memset(flow_v, 0, + frm_pyr->strides[0] * frm_pyr->heights[0] * sizeof(*flow_v)); + + compute_flow_field(frm_pyr, ref_pyr, flow_u, flow_v); + + // compute interest points in images using FAST features + num_frm_corners = fast_corner_detect(frm_buffer, frm_width, frm_height, + frm->y_stride, frm_corners, MAX_CORNERS); + // find correspondences between the two images using the flow field + correspondences = aom_malloc(num_frm_corners * 4 * sizeof(*correspondences)); + num_correspondences = determine_disflow_correspondence( + frm_corners, num_frm_corners, flow_u, flow_v, frm_width, frm_height, + frm_pyr->strides[0], correspondences); + ransac(correspondences, num_correspondences, num_inliers_by_motion, + params_by_motion, num_motions); + + free_pyramid(frm_pyr); + free_pyramid(ref_pyr); + aom_free(correspondences); + aom_free(flow_u); + aom_free(flow_v); + // Set num_inliers = 0 for motions with too few inliers so they are ignored. + for (int i = 0; i < num_motions; ++i) { + if (num_inliers_by_motion[i] < MIN_INLIER_PROB * num_correspondences) { + num_inliers_by_motion[i] = 0; + } + } + + // Return true if any one of the motions has inliers. + for (int i = 0; i < num_motions; ++i) { + if (num_inliers_by_motion[i] > 0) return 1; + } + return 0; +} +#endif + +int av1_compute_global_motion(TransformationType type, YV12_BUFFER_CONFIG *frm, + YV12_BUFFER_CONFIG *ref, int bit_depth, + int *num_inliers_by_motion, + double *params_by_motion, int num_motions) { +#if USE_GM_FEATURE_BASED + return compute_global_motion_feature_based(type, frm, ref, bit_depth, + num_inliers_by_motion, + params_by_motion, num_motions); +#else + return compute_global_motion_disflow_based(type, frm, ref, bit_depth, + num_inliers_by_motion, + params_by_motion, num_motions); +#endif +} diff --git a/third_party/aom/av1/encoder/global_motion.h b/third_party/aom/av1/encoder/global_motion.h index c7c016c430..42cf221697 100644 --- a/third_party/aom/av1/encoder/global_motion.h +++ b/third_party/aom/av1/encoder/global_motion.h @@ -22,20 +22,22 @@ extern "C" { #define RANSAC_NUM_MOTIONS 1 -void convert_model_to_params(const double *params, WarpedMotionParams *model); +void av1_convert_model_to_params(const double *params, + WarpedMotionParams *model); -int is_enough_erroradvantage(double best_erroradvantage, int params_cost, - int erroradv_type); +int av1_is_enough_erroradvantage(double best_erroradvantage, int params_cost, + int erroradv_type); // Returns the av1_warp_error between "dst" and the result of applying the // motion params that result from fine-tuning "wm" to "ref". Note that "wm" is // modified in place. -int64_t refine_integerized_param(WarpedMotionParams *wm, - TransformationType wmtype, int use_hbd, int bd, - uint8_t *ref, int r_width, int r_height, - int r_stride, uint8_t *dst, int d_width, - int d_height, int d_stride, int n_refinements, - int64_t best_frame_error); +int64_t av1_refine_integerized_param(WarpedMotionParams *wm, + TransformationType wmtype, int use_hbd, + int bd, uint8_t *ref, int r_width, + int r_height, int r_stride, uint8_t *dst, + int d_width, int d_height, int d_stride, + int n_refinements, + int64_t best_frame_error); /* Computes "num_motions" candidate global motion parameters between two frames. @@ -52,12 +54,10 @@ int64_t refine_integerized_param(WarpedMotionParams *wm, number of inlier feature points for each motion. Params for which the num_inliers entry is 0 should be ignored by the caller. */ -int compute_global_motion_feature_based(TransformationType type, - YV12_BUFFER_CONFIG *frm, - YV12_BUFFER_CONFIG *ref, int bit_depth, - int *num_inliers_by_motion, - double *params_by_motion, - int num_motions); +int av1_compute_global_motion(TransformationType type, YV12_BUFFER_CONFIG *frm, + YV12_BUFFER_CONFIG *ref, int bit_depth, + int *num_inliers_by_motion, + double *params_by_motion, int num_motions); #ifdef __cplusplus } // extern "C" #endif diff --git a/third_party/aom/av1/encoder/hash_motion.c b/third_party/aom/av1/encoder/hash_motion.c index e85a516e85..4c872f15aa 100644 --- a/third_party/aom/av1/encoder/hash_motion.c +++ b/third_party/aom/av1/encoder/hash_motion.c @@ -147,7 +147,8 @@ static void hash_table_add_to_table(hash_table *p_hash_table, } } -int32_t av1_hash_table_count(hash_table *p_hash_table, uint32_t hash_value) { +int32_t av1_hash_table_count(const hash_table *p_hash_table, + uint32_t hash_value) { if (p_hash_table->p_lookup_table[hash_value] == NULL) { return 0; } else { diff --git a/third_party/aom/av1/encoder/hash_motion.h b/third_party/aom/av1/encoder/hash_motion.h index df3ec32158..ed9bb6e52d 100644 --- a/third_party/aom/av1/encoder/hash_motion.h +++ b/third_party/aom/av1/encoder/hash_motion.h @@ -37,7 +37,8 @@ typedef struct _hash_table { void av1_hash_table_init(hash_table *p_hash_table, struct macroblock *x); void av1_hash_table_destroy(hash_table *p_hash_table); void av1_hash_table_create(hash_table *p_hash_table); -int32_t av1_hash_table_count(hash_table *p_hash_table, uint32_t hash_value); +int32_t av1_hash_table_count(const hash_table *p_hash_table, + uint32_t hash_value); Iterator av1_hash_get_first_iterator(hash_table *p_hash_table, uint32_t hash_value); int32_t av1_has_exact_match(hash_table *p_hash_table, uint32_t hash_value1, diff --git a/third_party/aom/av1/encoder/hybrid_fwd_txfm.c b/third_party/aom/av1/encoder/hybrid_fwd_txfm.c index 67898fd184..06990857a0 100644 --- a/third_party/aom/av1/encoder/hybrid_fwd_txfm.c +++ b/third_party/aom/av1/encoder/hybrid_fwd_txfm.c @@ -87,35 +87,21 @@ static void highbd_fwd_txfm_4x4(const int16_t *src_diff, tran_low_t *coeff, av1_highbd_fwht4x4(src_diff, coeff, diff_stride); return; } - switch (tx_type) { - // use the c version for anything including identity for now - case V_DCT: - case H_DCT: - case V_ADST: - case H_ADST: - case V_FLIPADST: - case H_FLIPADST: - case IDTX: - av1_fwd_txfm2d_4x4_c(src_diff, dst_coeff, diff_stride, tx_type, bd); - break; - default: - av1_fwd_txfm2d_4x4(src_diff, dst_coeff, diff_stride, tx_type, bd); - break; - } + av1_fwd_txfm2d_4x4(src_diff, dst_coeff, diff_stride, tx_type, bd); } static void highbd_fwd_txfm_4x8(const int16_t *src_diff, tran_low_t *coeff, int diff_stride, TxfmParam *txfm_param) { int32_t *dst_coeff = (int32_t *)coeff; - av1_fwd_txfm2d_4x8_c(src_diff, dst_coeff, diff_stride, txfm_param->tx_type, - txfm_param->bd); + av1_fwd_txfm2d_4x8(src_diff, dst_coeff, diff_stride, txfm_param->tx_type, + txfm_param->bd); } static void highbd_fwd_txfm_8x4(const int16_t *src_diff, tran_low_t *coeff, int diff_stride, TxfmParam *txfm_param) { int32_t *dst_coeff = (int32_t *)coeff; - av1_fwd_txfm2d_8x4_c(src_diff, dst_coeff, diff_stride, txfm_param->tx_type, - txfm_param->bd); + av1_fwd_txfm2d_8x4(src_diff, dst_coeff, diff_stride, txfm_param->tx_type, + txfm_param->bd); } static void highbd_fwd_txfm_8x16(const int16_t *src_diff, tran_low_t *coeff, @@ -123,21 +109,7 @@ static void highbd_fwd_txfm_8x16(const int16_t *src_diff, tran_low_t *coeff, int32_t *dst_coeff = (int32_t *)coeff; const TX_TYPE tx_type = txfm_param->tx_type; const int bd = txfm_param->bd; - switch (tx_type) { - // use the c version for anything including identity for now - case V_DCT: - case H_DCT: - case V_ADST: - case H_ADST: - case V_FLIPADST: - case H_FLIPADST: - case IDTX: - av1_fwd_txfm2d_8x16_c(src_diff, dst_coeff, diff_stride, tx_type, bd); - break; - default: - av1_fwd_txfm2d_8x16(src_diff, dst_coeff, diff_stride, tx_type, bd); - break; - } + av1_fwd_txfm2d_8x16(src_diff, dst_coeff, diff_stride, tx_type, bd); } static void highbd_fwd_txfm_16x8(const int16_t *src_diff, tran_low_t *coeff, @@ -145,63 +117,49 @@ static void highbd_fwd_txfm_16x8(const int16_t *src_diff, tran_low_t *coeff, int32_t *dst_coeff = (int32_t *)coeff; const TX_TYPE tx_type = txfm_param->tx_type; const int bd = txfm_param->bd; - switch (tx_type) { - // use the c version for anything including identity for now - case V_DCT: - case H_DCT: - case V_ADST: - case H_ADST: - case V_FLIPADST: - case H_FLIPADST: - case IDTX: - av1_fwd_txfm2d_16x8_c(src_diff, dst_coeff, diff_stride, tx_type, bd); - break; - default: - av1_fwd_txfm2d_16x8(src_diff, dst_coeff, diff_stride, tx_type, bd); - break; - } + av1_fwd_txfm2d_16x8(src_diff, dst_coeff, diff_stride, tx_type, bd); } static void highbd_fwd_txfm_16x32(const int16_t *src_diff, tran_low_t *coeff, int diff_stride, TxfmParam *txfm_param) { int32_t *dst_coeff = (int32_t *)coeff; - av1_fwd_txfm2d_16x32_c(src_diff, dst_coeff, diff_stride, txfm_param->tx_type, - txfm_param->bd); + av1_fwd_txfm2d_16x32(src_diff, dst_coeff, diff_stride, txfm_param->tx_type, + txfm_param->bd); } static void highbd_fwd_txfm_32x16(const int16_t *src_diff, tran_low_t *coeff, int diff_stride, TxfmParam *txfm_param) { int32_t *dst_coeff = (int32_t *)coeff; - av1_fwd_txfm2d_32x16_c(src_diff, dst_coeff, diff_stride, txfm_param->tx_type, - txfm_param->bd); + av1_fwd_txfm2d_32x16(src_diff, dst_coeff, diff_stride, txfm_param->tx_type, + txfm_param->bd); } static void highbd_fwd_txfm_16x4(const int16_t *src_diff, tran_low_t *coeff, int diff_stride, TxfmParam *txfm_param) { int32_t *dst_coeff = (int32_t *)coeff; - av1_fwd_txfm2d_16x4_c(src_diff, dst_coeff, diff_stride, txfm_param->tx_type, - txfm_param->bd); + av1_fwd_txfm2d_16x4(src_diff, dst_coeff, diff_stride, txfm_param->tx_type, + txfm_param->bd); } static void highbd_fwd_txfm_4x16(const int16_t *src_diff, tran_low_t *coeff, int diff_stride, TxfmParam *txfm_param) { int32_t *dst_coeff = (int32_t *)coeff; - av1_fwd_txfm2d_4x16_c(src_diff, dst_coeff, diff_stride, txfm_param->tx_type, - txfm_param->bd); + av1_fwd_txfm2d_4x16(src_diff, dst_coeff, diff_stride, txfm_param->tx_type, + txfm_param->bd); } static void highbd_fwd_txfm_32x8(const int16_t *src_diff, tran_low_t *coeff, int diff_stride, TxfmParam *txfm_param) { int32_t *dst_coeff = (int32_t *)coeff; - av1_fwd_txfm2d_32x8_c(src_diff, dst_coeff, diff_stride, txfm_param->tx_type, - txfm_param->bd); + av1_fwd_txfm2d_32x8(src_diff, dst_coeff, diff_stride, txfm_param->tx_type, + txfm_param->bd); } static void highbd_fwd_txfm_8x32(const int16_t *src_diff, tran_low_t *coeff, int diff_stride, TxfmParam *txfm_param) { int32_t *dst_coeff = (int32_t *)coeff; - av1_fwd_txfm2d_8x32_c(src_diff, dst_coeff, diff_stride, txfm_param->tx_type, - txfm_param->bd); + av1_fwd_txfm2d_8x32(src_diff, dst_coeff, diff_stride, txfm_param->tx_type, + txfm_param->bd); } static void highbd_fwd_txfm_8x8(const int16_t *src_diff, tran_low_t *coeff, @@ -209,21 +167,7 @@ static void highbd_fwd_txfm_8x8(const int16_t *src_diff, tran_low_t *coeff, int32_t *dst_coeff = (int32_t *)coeff; const TX_TYPE tx_type = txfm_param->tx_type; const int bd = txfm_param->bd; - switch (tx_type) { - // use the c version for anything including identity for now - case V_DCT: - case H_DCT: - case V_ADST: - case H_ADST: - case V_FLIPADST: - case H_FLIPADST: - case IDTX: - av1_fwd_txfm2d_8x8_c(src_diff, dst_coeff, diff_stride, tx_type, bd); - break; - default: - av1_fwd_txfm2d_8x8(src_diff, dst_coeff, diff_stride, tx_type, bd); - break; - } + av1_fwd_txfm2d_8x8(src_diff, dst_coeff, diff_stride, tx_type, bd); } static void highbd_fwd_txfm_16x16(const int16_t *src_diff, tran_low_t *coeff, @@ -231,21 +175,7 @@ static void highbd_fwd_txfm_16x16(const int16_t *src_diff, tran_low_t *coeff, int32_t *dst_coeff = (int32_t *)coeff; const TX_TYPE tx_type = txfm_param->tx_type; const int bd = txfm_param->bd; - switch (tx_type) { - // use the c version for anything including identity for now - case V_DCT: - case H_DCT: - case V_ADST: - case H_ADST: - case V_FLIPADST: - case H_FLIPADST: - case IDTX: - av1_fwd_txfm2d_16x16_c(src_diff, dst_coeff, diff_stride, tx_type, bd); - break; - default: - av1_fwd_txfm2d_16x16(src_diff, dst_coeff, diff_stride, tx_type, bd); - break; - } + av1_fwd_txfm2d_16x16(src_diff, dst_coeff, diff_stride, tx_type, bd); } static void highbd_fwd_txfm_32x32(const int16_t *src_diff, tran_low_t *coeff, @@ -253,21 +183,7 @@ static void highbd_fwd_txfm_32x32(const int16_t *src_diff, tran_low_t *coeff, int32_t *dst_coeff = (int32_t *)coeff; const TX_TYPE tx_type = txfm_param->tx_type; const int bd = txfm_param->bd; - switch (tx_type) { - // use the c version for anything including identity for now - case V_DCT: - case H_DCT: - case V_ADST: - case H_ADST: - case V_FLIPADST: - case H_FLIPADST: - case IDTX: - av1_fwd_txfm2d_32x32_c(src_diff, dst_coeff, diff_stride, tx_type, bd); - break; - default: - av1_fwd_txfm2d_32x32(src_diff, dst_coeff, diff_stride, tx_type, bd); - break; - } + av1_fwd_txfm2d_32x32(src_diff, dst_coeff, diff_stride, tx_type, bd); } static void highbd_fwd_txfm_32x64(const int16_t *src_diff, tran_low_t *coeff, @@ -275,7 +191,8 @@ static void highbd_fwd_txfm_32x64(const int16_t *src_diff, tran_low_t *coeff, assert(txfm_param->tx_type == DCT_DCT); int32_t *dst_coeff = (int32_t *)coeff; const int bd = txfm_param->bd; - av1_fwd_txfm2d_32x64_c(src_diff, dst_coeff, diff_stride, DCT_DCT, bd); + av1_fwd_txfm2d_32x64(src_diff, dst_coeff, diff_stride, txfm_param->tx_type, + bd); } static void highbd_fwd_txfm_64x32(const int16_t *src_diff, tran_low_t *coeff, @@ -283,7 +200,8 @@ static void highbd_fwd_txfm_64x32(const int16_t *src_diff, tran_low_t *coeff, assert(txfm_param->tx_type == DCT_DCT); int32_t *dst_coeff = (int32_t *)coeff; const int bd = txfm_param->bd; - av1_fwd_txfm2d_64x32_c(src_diff, dst_coeff, diff_stride, DCT_DCT, bd); + av1_fwd_txfm2d_64x32(src_diff, dst_coeff, diff_stride, txfm_param->tx_type, + bd); } static void highbd_fwd_txfm_16x64(const int16_t *src_diff, tran_low_t *coeff, @@ -291,7 +209,7 @@ static void highbd_fwd_txfm_16x64(const int16_t *src_diff, tran_low_t *coeff, assert(txfm_param->tx_type == DCT_DCT); int32_t *dst_coeff = (int32_t *)coeff; const int bd = txfm_param->bd; - av1_fwd_txfm2d_16x64_c(src_diff, dst_coeff, diff_stride, DCT_DCT, bd); + av1_fwd_txfm2d_16x64(src_diff, dst_coeff, diff_stride, DCT_DCT, bd); } static void highbd_fwd_txfm_64x16(const int16_t *src_diff, tran_low_t *coeff, @@ -299,7 +217,7 @@ static void highbd_fwd_txfm_64x16(const int16_t *src_diff, tran_low_t *coeff, assert(txfm_param->tx_type == DCT_DCT); int32_t *dst_coeff = (int32_t *)coeff; const int bd = txfm_param->bd; - av1_fwd_txfm2d_64x16_c(src_diff, dst_coeff, diff_stride, DCT_DCT, bd); + av1_fwd_txfm2d_64x16(src_diff, dst_coeff, diff_stride, DCT_DCT, bd); } static void highbd_fwd_txfm_64x64(const int16_t *src_diff, tran_low_t *coeff, diff --git a/third_party/aom/av1/encoder/lookahead.c b/third_party/aom/av1/encoder/lookahead.c index 1bf8ecbacb..fe3bc20512 100644 --- a/third_party/aom/av1/encoder/lookahead.c +++ b/third_party/aom/av1/encoder/lookahead.c @@ -43,7 +43,8 @@ void av1_lookahead_destroy(struct lookahead_ctx *ctx) { struct lookahead_ctx *av1_lookahead_init( unsigned int width, unsigned int height, unsigned int subsampling_x, - unsigned int subsampling_y, int use_highbitdepth, unsigned int depth) { + unsigned int subsampling_y, int use_highbitdepth, unsigned int depth, + const int border_in_pixels) { struct lookahead_ctx *ctx = NULL; // Clamp the lookahead queue depth @@ -63,7 +64,7 @@ struct lookahead_ctx *av1_lookahead_init( for (i = 0; i < depth; i++) if (aom_alloc_frame_buffer(&ctx->buf[i].img, width, height, subsampling_x, subsampling_y, use_highbitdepth, - AOM_BORDER_IN_PIXELS, legacy_byte_alignment)) + border_in_pixels, legacy_byte_alignment)) goto bail; } return ctx; diff --git a/third_party/aom/av1/encoder/lookahead.h b/third_party/aom/av1/encoder/lookahead.h index e55224cf75..e5d9702443 100644 --- a/third_party/aom/av1/encoder/lookahead.h +++ b/third_party/aom/av1/encoder/lookahead.h @@ -46,7 +46,8 @@ struct lookahead_ctx { */ struct lookahead_ctx *av1_lookahead_init( unsigned int width, unsigned int height, unsigned int subsampling_x, - unsigned int subsampling_y, int use_highbitdepth, unsigned int depth); + unsigned int subsampling_y, int use_highbitdepth, unsigned int depth, + const int border_in_pixels); /**\brief Destroys the lookahead stage */ diff --git a/third_party/aom/av1/encoder/mbgraph.c b/third_party/aom/av1/encoder/mbgraph.c index 1a35ff77c7..0cb628682f 100644 --- a/third_party/aom/av1/encoder/mbgraph.c +++ b/third_party/aom/av1/encoder/mbgraph.c @@ -60,7 +60,7 @@ static unsigned int do_16x16_motion_iteration(AV1_COMP *cpi, const MV *ref_mv, x, &cpi->common, mb_row, mb_col, ref_mv, cpi->common.allow_high_precision_mv, x->errorperbit, &v_fn_ptr, 0, mv_sf->subpel_iters_per_step, cond_cost_list(cpi, cost_list), NULL, - NULL, &distortion, &sse, NULL, NULL, 0, 0, 0, 0, 0); + NULL, &distortion, &sse, NULL, NULL, 0, 0, 0, 0, 0, 1); } if (has_second_ref(xd->mi[0])) @@ -71,8 +71,8 @@ static unsigned int do_16x16_motion_iteration(AV1_COMP *cpi, const MV *ref_mv, xd->mi[0]->mv[0] = x->best_mv; xd->mi[0]->ref_frame[1] = NONE_FRAME; - av1_build_inter_predictors_sby(&cpi->common, xd, mb_row, mb_col, NULL, - BLOCK_16X16); + av1_enc_build_inter_predictor(&cpi->common, xd, mb_row, mb_col, NULL, + BLOCK_16X16, AOM_PLANE_Y, AOM_PLANE_Y); /* restore UMV window */ x->mv_limits = tmp_mv_limits; @@ -179,8 +179,8 @@ static void update_mbgraph_mb_stats(AV1_COMP *cpi, MBGRAPH_MB_STATS *stats, x->plane[0].src.buf = buf->y_buffer + mb_y_offset; x->plane[0].src.stride = buf->y_stride; - xd->plane[0].dst.buf = get_frame_new_buffer(cm)->y_buffer + mb_y_offset; - xd->plane[0].dst.stride = get_frame_new_buffer(cm)->y_stride; + xd->plane[0].dst.buf = cm->cur_frame->buf.y_buffer + mb_y_offset; + xd->plane[0].dst.stride = cm->cur_frame->buf.y_stride; // do intra 16x16 prediction intra_error = find_best_16x16_intra(cpi, &stats->ref[INTRA_FRAME].m.mode); @@ -364,7 +364,7 @@ static void separate_arf_mbs(AV1_COMP *cpi) { void av1_update_mbgraph_stats(AV1_COMP *cpi) { AV1_COMMON *const cm = &cpi->common; int i, n_frames = av1_lookahead_depth(cpi->lookahead); - YV12_BUFFER_CONFIG *golden_ref = get_ref_frame_buffer(cpi, GOLDEN_FRAME); + YV12_BUFFER_CONFIG *golden_ref = &get_ref_frame_buf(cm, GOLDEN_FRAME)->buf; assert(golden_ref != NULL); diff --git a/third_party/aom/av1/encoder/mcomp.c b/third_party/aom/av1/encoder/mcomp.c index 8f6de9b532..1bab45c7f1 100644 --- a/third_party/aom/av1/encoder/mcomp.c +++ b/third_party/aom/av1/encoder/mcomp.c @@ -112,7 +112,7 @@ static int mvsad_err_cost(const MACROBLOCK *x, const MV *mv, const MV *ref, int sad_per_bit) { const MV diff = { (mv->row - ref->row) * 8, (mv->col - ref->col) * 8 }; return ROUND_POWER_OF_TWO( - (unsigned)mv_cost(&diff, x->nmvjointcost, x->mvcost) * sad_per_bit, + (unsigned)mv_cost(&diff, x->nmv_vec_cost, x->mv_cost_stack) * sad_per_bit, AV1_PROB_COST_SHIFT); } @@ -179,36 +179,31 @@ static INLINE const uint8_t *pre(const uint8_t *buf, int stride, int r, int c) { } /* checks if (r, c) has better score than previous best */ -#define CHECK_BETTER(v, r, c) \ - if (c >= minc && c <= maxc && r >= minr && r <= maxr) { \ - MV this_mv = { r, c }; \ - v = mv_err_cost(&this_mv, ref_mv, mvjcost, mvcost, error_per_bit); \ - if (second_pred == NULL) { \ - thismse = vfp->svf(pre(y, y_stride, r, c), y_stride, sp(c), sp(r), \ - src_address, src_stride, &sse); \ - } else if (mask) { \ - thismse = vfp->msvf(pre(y, y_stride, r, c), y_stride, sp(c), sp(r), \ - src_address, src_stride, second_pred, mask, \ - mask_stride, invert_mask, &sse); \ - } else { \ - if (xd->jcp_param.use_jnt_comp_avg) \ - thismse = vfp->jsvaf(pre(y, y_stride, r, c), y_stride, sp(c), sp(r), \ - src_address, src_stride, &sse, second_pred, \ - &xd->jcp_param); \ - else \ - thismse = vfp->svaf(pre(y, y_stride, r, c), y_stride, sp(c), sp(r), \ - src_address, src_stride, &sse, second_pred); \ - } \ - v += thismse; \ - if (v < besterr) { \ - besterr = v; \ - br = r; \ - bc = c; \ - *distortion = thismse; \ - *sse1 = sse; \ - } \ - } else { \ - v = INT_MAX; \ +#define CHECK_BETTER(v, r, c) \ + if (c >= minc && c <= maxc && r >= minr && r <= maxr) { \ + MV this_mv = { r, c }; \ + v = mv_err_cost(&this_mv, ref_mv, mvjcost, mvcost, error_per_bit); \ + if (second_pred == NULL) { \ + thismse = vfp->svf(pre(y, y_stride, r, c), y_stride, sp(c), sp(r), \ + src_address, src_stride, &sse); \ + } else if (mask) { \ + thismse = vfp->msvf(pre(y, y_stride, r, c), y_stride, sp(c), sp(r), \ + src_address, src_stride, second_pred, mask, \ + mask_stride, invert_mask, &sse); \ + } else { \ + thismse = vfp->svaf(pre(y, y_stride, r, c), y_stride, sp(c), sp(r), \ + src_address, src_stride, &sse, second_pred); \ + } \ + v += thismse; \ + if (v < besterr) { \ + besterr = v; \ + br = r; \ + bc = c; \ + *distortion = thismse; \ + *sse1 = sse; \ + } \ + } else { \ + v = INT_MAX; \ } #define CHECK_BETTER0(v, r, c) CHECK_BETTER(v, r, c) @@ -341,19 +336,15 @@ static unsigned int setup_center_error( int *mvcost[2], unsigned int *sse1, int *distortion) { unsigned int besterr; if (second_pred != NULL) { - if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + if (is_cur_buf_hbd(xd)) { DECLARE_ALIGNED(16, uint16_t, comp_pred16[MAX_SB_SQUARE]); uint8_t *comp_pred = CONVERT_TO_BYTEPTR(comp_pred16); if (mask) { aom_highbd_comp_mask_pred(comp_pred, second_pred, w, h, y + offset, y_stride, mask, mask_stride, invert_mask); } else { - if (xd->jcp_param.use_jnt_comp_avg) - aom_highbd_jnt_comp_avg_pred(comp_pred, second_pred, w, h, y + offset, - y_stride, &xd->jcp_param); - else - aom_highbd_comp_avg_pred(comp_pred, second_pred, w, h, y + offset, - y_stride); + aom_highbd_comp_avg_pred(comp_pred, second_pred, w, h, y + offset, + y_stride); } besterr = vfp->vf(comp_pred, w, src, src_stride, sse1); } else { @@ -362,11 +353,7 @@ static unsigned int setup_center_error( aom_comp_mask_pred(comp_pred, second_pred, w, h, y + offset, y_stride, mask, mask_stride, invert_mask); } else { - if (xd->jcp_param.use_jnt_comp_avg) - aom_jnt_comp_avg_pred(comp_pred, second_pred, w, h, y + offset, - y_stride, &xd->jcp_param); - else - aom_comp_avg_pred(comp_pred, second_pred, w, h, y + offset, y_stride); + aom_comp_avg_pred(comp_pred, second_pred, w, h, y + offset, y_stride); } besterr = vfp->vf(comp_pred, w, src, src_stride, sse1); } @@ -409,7 +396,7 @@ int av1_find_best_sub_pixel_tree_pruned_evenmore( int *cost_list, int *mvjcost, int *mvcost[2], int *distortion, unsigned int *sse1, const uint8_t *second_pred, const uint8_t *mask, int mask_stride, int invert_mask, int w, int h, - int use_accurate_subpel_search) { + int use_accurate_subpel_search, const int do_reset_fractional_mv) { SETUP_SUBPEL_SEARCH; besterr = setup_center_error(xd, bestmv, ref_mv, error_per_bit, vfp, src_address, src_stride, y, y_stride, @@ -426,6 +413,7 @@ int av1_find_best_sub_pixel_tree_pruned_evenmore( (void)cm; (void)mi_row; (void)mi_col; + (void)do_reset_fractional_mv; if (cost_list && cost_list[0] != INT_MAX && cost_list[1] != INT_MAX && cost_list[2] != INT_MAX && cost_list[3] != INT_MAX && @@ -481,12 +469,13 @@ int av1_find_best_sub_pixel_tree_pruned_more( int *cost_list, int *mvjcost, int *mvcost[2], int *distortion, unsigned int *sse1, const uint8_t *second_pred, const uint8_t *mask, int mask_stride, int invert_mask, int w, int h, - int use_accurate_subpel_search) { + int use_accurate_subpel_search, const int do_reset_fractional_mv) { SETUP_SUBPEL_SEARCH; (void)use_accurate_subpel_search; (void)cm; (void)mi_row; (void)mi_col; + (void)do_reset_fractional_mv; besterr = setup_center_error(xd, bestmv, ref_mv, error_per_bit, vfp, src_address, src_stride, y, y_stride, @@ -549,12 +538,13 @@ int av1_find_best_sub_pixel_tree_pruned( int *cost_list, int *mvjcost, int *mvcost[2], int *distortion, unsigned int *sse1, const uint8_t *second_pred, const uint8_t *mask, int mask_stride, int invert_mask, int w, int h, - int use_accurate_subpel_search) { + int use_accurate_subpel_search, const int do_reset_fractional_mv) { SETUP_SUBPEL_SEARCH; (void)use_accurate_subpel_search; (void)cm; (void)mi_row; (void)mi_col; + (void)do_reset_fractional_mv; besterr = setup_center_error(xd, bestmv, ref_mv, error_per_bit, vfp, src_address, src_stride, y, y_stride, @@ -651,7 +641,7 @@ static int upsampled_pref_error(MACROBLOCKD *xd, const AV1_COMMON *const cm, int mask_stride, int invert_mask, int w, int h, unsigned int *sse, int subpel_search) { unsigned int besterr; - if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + if (is_cur_buf_hbd(xd)) { DECLARE_ALIGNED(16, uint16_t, pred16[MAX_SB_SQUARE]); uint8_t *pred8 = CONVERT_TO_BYTEPTR(pred16); if (second_pred != NULL) { @@ -661,14 +651,9 @@ static int upsampled_pref_error(MACROBLOCKD *xd, const AV1_COMMON *const cm, subpel_y_q3, y, y_stride, mask, mask_stride, invert_mask, xd->bd, subpel_search); } else { - if (xd->jcp_param.use_jnt_comp_avg) - aom_highbd_jnt_comp_avg_upsampled_pred( - xd, cm, mi_row, mi_col, mv, pred8, second_pred, w, h, subpel_x_q3, - subpel_y_q3, y, y_stride, xd->bd, &xd->jcp_param, subpel_search); - else - aom_highbd_comp_avg_upsampled_pred( - xd, cm, mi_row, mi_col, mv, pred8, second_pred, w, h, subpel_x_q3, - subpel_y_q3, y, y_stride, xd->bd, subpel_search); + aom_highbd_comp_avg_upsampled_pred( + xd, cm, mi_row, mi_col, mv, pred8, second_pred, w, h, subpel_x_q3, + subpel_y_q3, y, y_stride, xd->bd, subpel_search); } } else { aom_highbd_upsampled_pred(xd, cm, mi_row, mi_col, mv, pred8, w, h, @@ -685,14 +670,9 @@ static int upsampled_pref_error(MACROBLOCKD *xd, const AV1_COMMON *const cm, subpel_y_q3, y, y_stride, mask, mask_stride, invert_mask, subpel_search); } else { - if (xd->jcp_param.use_jnt_comp_avg) - aom_jnt_comp_avg_upsampled_pred( - xd, cm, mi_row, mi_col, mv, pred, second_pred, w, h, subpel_x_q3, - subpel_y_q3, y, y_stride, &xd->jcp_param, subpel_search); - else - aom_comp_avg_upsampled_pred(xd, cm, mi_row, mi_col, mv, pred, - second_pred, w, h, subpel_x_q3, - subpel_y_q3, y, y_stride, subpel_search); + aom_comp_avg_upsampled_pred(xd, cm, mi_row, mi_col, mv, pred, + second_pred, w, h, subpel_x_q3, subpel_y_q3, + y, y_stride, subpel_search); } } else { aom_upsampled_pred(xd, cm, mi_row, mi_col, mv, pred, w, h, subpel_x_q3, @@ -723,7 +703,7 @@ static unsigned int upsampled_setup_center_error( // when use_accurate_subpel_search == 0 static INLINE unsigned int estimate_upsampled_pref_error( - MACROBLOCKD *xd, const aom_variance_fn_ptr_t *vfp, const uint8_t *const src, + const aom_variance_fn_ptr_t *vfp, const uint8_t *const src, const int src_stride, const uint8_t *const pre, int y_stride, int subpel_x_q3, int subpel_y_q3, const uint8_t *second_pred, const uint8_t *mask, int mask_stride, int invert_mask, unsigned int *sse) { @@ -734,12 +714,8 @@ static INLINE unsigned int estimate_upsampled_pref_error( return vfp->msvf(pre, y_stride, subpel_x_q3, subpel_y_q3, src, src_stride, second_pred, mask, mask_stride, invert_mask, sse); } else { - if (xd->jcp_param.use_jnt_comp_avg) - return vfp->jsvaf(pre, y_stride, subpel_x_q3, subpel_y_q3, src, - src_stride, sse, second_pred, &xd->jcp_param); - else - return vfp->svaf(pre, y_stride, subpel_x_q3, subpel_y_q3, src, src_stride, - sse, second_pred); + return vfp->svaf(pre, y_stride, subpel_x_q3, subpel_y_q3, src, src_stride, + sse, second_pred); } } @@ -750,7 +726,7 @@ int av1_find_best_sub_pixel_tree( int *cost_list, int *mvjcost, int *mvcost[2], int *distortion, unsigned int *sse1, const uint8_t *second_pred, const uint8_t *mask, int mask_stride, int invert_mask, int w, int h, - int use_accurate_subpel_search) { + int use_accurate_subpel_search, const int do_reset_fractional_mv) { const uint8_t *const src_address = x->plane[0].src.buf; const int src_stride = x->plane[0].src.stride; MACROBLOCKD *xd = &x->e_mbd; @@ -796,7 +772,16 @@ int av1_find_best_sub_pixel_tree( (void)cost_list; // to silence compiler warning + if (do_reset_fractional_mv) { + av1_set_fractional_mv(x->fractional_best_mv); + } + for (iter = 0; iter < round; ++iter) { + if ((x->fractional_best_mv[iter].as_mv.row == br) && + (x->fractional_best_mv[iter].as_mv.col == bc)) + return INT_MAX; + x->fractional_best_mv[iter].as_mv.row = br; + x->fractional_best_mv[iter].as_mv.col = bc; // Check vertical and horizontal sub-pixel positions. for (idx = 0; idx < 4; ++idx) { tr = br + search_step[idx].row; @@ -812,9 +797,9 @@ int av1_find_best_sub_pixel_tree( use_accurate_subpel_search); } else { thismse = estimate_upsampled_pref_error( - xd, vfp, src_address, src_stride, pre(y, y_stride, tr, tc), - y_stride, sp(tc), sp(tr), second_pred, mask, mask_stride, - invert_mask, &sse); + vfp, src_address, src_stride, pre(y, y_stride, tr, tc), y_stride, + sp(tc), sp(tr), second_pred, mask, mask_stride, invert_mask, + &sse); } cost_array[idx] = thismse + mv_err_cost(&this_mv, ref_mv, mvjcost, @@ -848,9 +833,8 @@ int av1_find_best_sub_pixel_tree( use_accurate_subpel_search); } else { thismse = estimate_upsampled_pref_error( - xd, vfp, src_address, src_stride, pre(y, y_stride, tr, tc), - y_stride, sp(tc), sp(tr), second_pred, mask, mask_stride, - invert_mask, &sse); + vfp, src_address, src_stride, pre(y, y_stride, tr, tc), y_stride, + sp(tc), sp(tr), second_pred, mask, mask_stride, invert_mask, &sse); } cost_array[4] = thismse + mv_err_cost(&this_mv, ref_mv, mvjcost, mvcost, @@ -915,9 +899,10 @@ unsigned int av1_compute_motion_cost(const AV1_COMP *cpi, MACROBLOCK *const x, unsigned int mse; unsigned int sse; - av1_build_inter_predictors_sby(cm, xd, mi_row, mi_col, NULL, bsize); + av1_enc_build_inter_predictor(cm, xd, mi_row, mi_col, NULL, bsize, + AOM_PLANE_Y, AOM_PLANE_Y); mse = vfp->vf(dst, dst_stride, src, src_stride, &sse); - mse += mv_err_cost(this_mv, &ref_mv.as_mv, x->nmvjointcost, x->mvcost, + mse += mv_err_cost(this_mv, &ref_mv.as_mv, x->nmv_vec_cost, x->mv_cost_stack, x->errorperbit); return mse; } @@ -1057,8 +1042,8 @@ static INLINE void calc_int_cost_list(const MACROBLOCK *x, cost_list[i + 1] = fn_ptr->vf(what->buf, what->stride, get_buf_from_mv(in_what, &neighbor_mv), in_what->stride, &sse) + - mv_err_cost(&neighbor_mv, &fcenter_mv, x->nmvjointcost, - x->mvcost, x->errorperbit); + mv_err_cost(&neighbor_mv, &fcenter_mv, x->nmv_vec_cost, + x->mv_cost_stack, x->errorperbit); } } else { for (i = 0; i < 4; i++) { @@ -1070,8 +1055,8 @@ static INLINE void calc_int_cost_list(const MACROBLOCK *x, fn_ptr->vf(what->buf, what->stride, get_buf_from_mv(in_what, &neighbor_mv), in_what->stride, &sse) + - mv_err_cost(&neighbor_mv, &fcenter_mv, x->nmvjointcost, x->mvcost, - x->errorperbit); + mv_err_cost(&neighbor_mv, &fcenter_mv, x->nmv_vec_cost, + x->mv_cost_stack, x->errorperbit); } } } @@ -1393,8 +1378,8 @@ int av1_get_mvpred_var(const MACROBLOCK *x, const MV *best_mv, return vfp->vf(what->buf, what->stride, get_buf_from_mv(in_what, best_mv), in_what->stride, &unused) + - (use_mvcost ? mv_err_cost(&mv, center_mv, x->nmvjointcost, x->mvcost, - x->errorperbit) + (use_mvcost ? mv_err_cost(&mv, center_mv, x->nmv_vec_cost, + x->mv_cost_stack, x->errorperbit) : 0); } @@ -1407,19 +1392,11 @@ int av1_get_mvpred_av_var(const MACROBLOCK *x, const MV *best_mv, const MV mv = { best_mv->row * 8, best_mv->col * 8 }; unsigned int unused; - if (xd->jcp_param.use_jnt_comp_avg) - return vfp->jsvaf(get_buf_from_mv(in_what, best_mv), in_what->stride, 0, 0, - what->buf, what->stride, &unused, second_pred, - &xd->jcp_param) + - (use_mvcost ? mv_err_cost(&mv, center_mv, x->nmvjointcost, x->mvcost, - x->errorperbit) - : 0); - else - return vfp->svaf(get_buf_from_mv(in_what, best_mv), in_what->stride, 0, 0, - what->buf, what->stride, &unused, second_pred) + - (use_mvcost ? mv_err_cost(&mv, center_mv, x->nmvjointcost, x->mvcost, - x->errorperbit) - : 0); + return vfp->svaf(get_buf_from_mv(in_what, best_mv), in_what->stride, 0, 0, + what->buf, what->stride, &unused, second_pred) + + (use_mvcost ? mv_err_cost(&mv, center_mv, x->nmv_vec_cost, + x->mv_cost_stack, x->errorperbit) + : 0); } int av1_get_mvpred_mask_var(const MACROBLOCK *x, const MV *best_mv, @@ -1436,8 +1413,8 @@ int av1_get_mvpred_mask_var(const MACROBLOCK *x, const MV *best_mv, return vfp->msvf(what->buf, what->stride, 0, 0, get_buf_from_mv(in_what, best_mv), in_what->stride, second_pred, mask, mask_stride, invert_mask, &unused) + - (use_mvcost ? mv_err_cost(&mv, center_mv, x->nmvjointcost, x->mvcost, - x->errorperbit) + (use_mvcost ? mv_err_cost(&mv, center_mv, x->nmv_vec_cost, + x->mv_cost_stack, x->errorperbit) : 0); } @@ -1897,7 +1874,7 @@ static int full_pixel_exhaustive(const AV1_COMP *const cpi, MACROBLOCK *x, int baseline_interval_divisor; // Keep track of number of exhaustive calls (this frame in this thread). - ++(*x->ex_search_count_ptr); + if (x->ex_search_count_ptr != NULL) ++(*x->ex_search_count_ptr); // Trap illegal values for interval and range for this function. if ((range < MIN_RANGE) || (range > MAX_RANGE) || (interval < MIN_INTERVAL) || @@ -2051,16 +2028,10 @@ int av1_refining_search_8p_c(MACROBLOCK *x, int error_per_bit, int search_range, second_pred, mask, mask_stride, invert_mask) + mvsad_err_cost(x, best_mv, &fcenter_mv, error_per_bit); } else { - if (xd->jcp_param.use_jnt_comp_avg) - best_sad = fn_ptr->jsdaf(what->buf, what->stride, - get_buf_from_mv(in_what, best_mv), - in_what->stride, second_pred, &xd->jcp_param) + - mvsad_err_cost(x, best_mv, &fcenter_mv, error_per_bit); - else - best_sad = fn_ptr->sdaf(what->buf, what->stride, - get_buf_from_mv(in_what, best_mv), - in_what->stride, second_pred) + - mvsad_err_cost(x, best_mv, &fcenter_mv, error_per_bit); + best_sad = + fn_ptr->sdaf(what->buf, what->stride, get_buf_from_mv(in_what, best_mv), + in_what->stride, second_pred) + + mvsad_err_cost(x, best_mv, &fcenter_mv, error_per_bit); } do_refine_search_grid[grid_coord] = 1; @@ -2084,14 +2055,9 @@ int av1_refining_search_8p_c(MACROBLOCK *x, int error_per_bit, int search_range, get_buf_from_mv(in_what, &mv), in_what->stride, second_pred, mask, mask_stride, invert_mask); } else { - if (xd->jcp_param.use_jnt_comp_avg) - sad = fn_ptr->jsdaf(what->buf, what->stride, - get_buf_from_mv(in_what, &mv), in_what->stride, - second_pred, &xd->jcp_param); - else - sad = fn_ptr->sdaf(what->buf, what->stride, - get_buf_from_mv(in_what, &mv), in_what->stride, - second_pred); + sad = fn_ptr->sdaf(what->buf, what->stride, + get_buf_from_mv(in_what, &mv), in_what->stride, + second_pred); } if (sad < best_sad) { sad += mvsad_err_cost(x, &mv, &fcenter_mv, error_per_bit); @@ -2117,13 +2083,16 @@ int av1_refining_search_8p_c(MACROBLOCK *x, int error_per_bit, int search_range, #define MIN_EX_SEARCH_LIMIT 128 static int is_exhaustive_allowed(const AV1_COMP *const cpi, MACROBLOCK *x) { const SPEED_FEATURES *const sf = &cpi->sf; - const int max_ex = - AOMMAX(MIN_EX_SEARCH_LIMIT, - (*x->m_search_count_ptr * sf->max_exaustive_pct) / 100); - - return sf->allow_exhaustive_searches && - (sf->exhaustive_searches_thresh < INT_MAX) && - (*x->ex_search_count_ptr <= max_ex) && !cpi->rc.is_src_frame_alt_ref; + int is_allowed = sf->allow_exhaustive_searches && + (sf->exhaustive_searches_thresh < INT_MAX) && + !cpi->rc.is_src_frame_alt_ref; + if (x->m_search_count_ptr != NULL && x->ex_search_count_ptr != NULL) { + const int max_ex = + AOMMAX(MIN_EX_SEARCH_LIMIT, + (*x->m_search_count_ptr * sf->max_exaustive_pct) / 100); + is_allowed = *x->ex_search_count_ptr <= max_ex && is_allowed; + } + return is_allowed; } int av1_full_pixel_search(const AV1_COMP *cpi, MACROBLOCK *x, BLOCK_SIZE bsize, @@ -2144,7 +2113,7 @@ int av1_full_pixel_search(const AV1_COMP *cpi, MACROBLOCK *x, BLOCK_SIZE bsize, } // Keep track of number of searches (this frame in this thread). - ++(*x->m_search_count_ptr); + if (x->m_search_count_ptr != NULL) ++(*x->m_search_count_ptr); switch (method) { case FAST_DIAMOND: @@ -2241,13 +2210,12 @@ int av1_full_pixel_search(const AV1_COMP *cpi, MACROBLOCK *x, BLOCK_SIZE bsize, // for the hashMap hash_table *ref_frame_hash = - intra - ? &cpi->common.cur_frame->hash_table - : av1_get_ref_frame_hash_map(cpi, x->e_mbd.mi[0]->ref_frame[0]); + intra ? &cpi->common.cur_frame->hash_table + : av1_get_ref_frame_hash_map(&cpi->common, + x->e_mbd.mi[0]->ref_frame[0]); - av1_get_block_hash_value( - what, what_stride, block_width, &hash_value1, &hash_value2, - x->e_mbd.cur_buf->flags & YV12_FLAG_HIGHBITDEPTH, x); + av1_get_block_hash_value(what, what_stride, block_width, &hash_value1, + &hash_value2, is_cur_buf_hbd(&x->e_mbd), x); const int count = av1_hash_table_count(ref_frame_hash, hash_value1); // for intra, at lest one matching can be found, itself. @@ -2333,8 +2301,9 @@ int av1_full_pixel_search(const AV1_COMP *cpi, MACROBLOCK *x, BLOCK_SIZE bsize, mask, vfp, z, pre(y, y_stride, r, c), \ y_stride, sp(c), sp(r), w, h, &sse, \ use_accurate_subpel_search); \ - if ((v = MVC(r, c) + thismse) < besterr) { \ - besterr = v; \ + v = mv_err_cost(&this_mv, ref_mv, mvjcost, mvcost, error_per_bit); \ + if ((v + thismse) < besterr) { \ + besterr = v + thismse; \ br = r; \ bc = c; \ *distortion = thismse; \ @@ -2365,7 +2334,7 @@ static int upsampled_obmc_pref_error( unsigned int besterr; DECLARE_ALIGNED(16, uint8_t, pred[2 * MAX_SB_SQUARE]); - if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + if (is_cur_buf_hbd(xd)) { uint8_t *pred8 = CONVERT_TO_BYTEPTR(pred); aom_highbd_upsampled_pred(xd, cm, mi_row, mi_col, mv, pred8, w, h, subpel_x_q3, subpel_y_q3, y, y_stride, xd->bd, @@ -2568,16 +2537,16 @@ static int get_obmc_mvpred_var(const MACROBLOCK *x, const int32_t *wsrc, return vfp->ovf(get_buf_from_mv(in_what, best_mv), in_what->stride, wsrc, mask, &unused) + - (use_mvcost ? mv_err_cost(&mv, center_mv, x->nmvjointcost, x->mvcost, - x->errorperbit) + (use_mvcost ? mv_err_cost(&mv, center_mv, x->nmv_vec_cost, + x->mv_cost_stack, x->errorperbit) : 0); } -int obmc_refining_search_sad(const MACROBLOCK *x, const int32_t *wsrc, - const int32_t *mask, MV *ref_mv, int error_per_bit, - int search_range, - const aom_variance_fn_ptr_t *fn_ptr, - const MV *center_mv, int is_second) { +static int obmc_refining_search_sad(const MACROBLOCK *x, const int32_t *wsrc, + const int32_t *mask, MV *ref_mv, + int error_per_bit, int search_range, + const aom_variance_fn_ptr_t *fn_ptr, + const MV *center_mv, int is_second) { const MV neighbors[4] = { { -1, 0 }, { 0, -1 }, { 0, 1 }, { 1, 0 } }; const MACROBLOCKD *const xd = &x->e_mbd; const struct buf_2d *const in_what = &xd->plane[0].pre[is_second]; @@ -2616,12 +2585,13 @@ int obmc_refining_search_sad(const MACROBLOCK *x, const int32_t *wsrc, return best_sad; } -int obmc_diamond_search_sad(const MACROBLOCK *x, const search_site_config *cfg, - const int32_t *wsrc, const int32_t *mask, - MV *ref_mv, MV *best_mv, int search_param, - int sad_per_bit, int *num00, - const aom_variance_fn_ptr_t *fn_ptr, - const MV *center_mv, int is_second) { +static int obmc_diamond_search_sad(const MACROBLOCK *x, + const search_site_config *cfg, + const int32_t *wsrc, const int32_t *mask, + MV *ref_mv, MV *best_mv, int search_param, + int sad_per_bit, int *num00, + const aom_variance_fn_ptr_t *fn_ptr, + const MV *center_mv, int is_second) { const MACROBLOCKD *const xd = &x->e_mbd; const struct buf_2d *const in_what = &xd->plane[0].pre[is_second]; // search_param determines the length of the initial step and hence the number @@ -2824,16 +2794,14 @@ int av1_obmc_full_pixel_search(const AV1_COMP *cpi, MACROBLOCK *x, MV *mvp_full, (void)thismse; \ (void)cost_list; // Return the maximum MV. -int av1_return_max_sub_pixel_mv(MACROBLOCK *x, const AV1_COMMON *const cm, - int mi_row, int mi_col, const MV *ref_mv, - int allow_hp, int error_per_bit, - const aom_variance_fn_ptr_t *vfp, - int forced_stop, int iters_per_step, - int *cost_list, int *mvjcost, int *mvcost[2], - int *distortion, unsigned int *sse1, - const uint8_t *second_pred, const uint8_t *mask, - int mask_stride, int invert_mask, int w, int h, - int use_accurate_subpel_search) { +int av1_return_max_sub_pixel_mv( + MACROBLOCK *x, const AV1_COMMON *const cm, int mi_row, int mi_col, + const MV *ref_mv, int allow_hp, int error_per_bit, + const aom_variance_fn_ptr_t *vfp, int forced_stop, int iters_per_step, + int *cost_list, int *mvjcost, int *mvcost[2], int *distortion, + unsigned int *sse1, const uint8_t *second_pred, const uint8_t *mask, + int mask_stride, int invert_mask, int w, int h, + int use_accurate_subpel_search, const int do_reset_fractional_mv) { COMMON_MV_TEST; (void)mask; (void)mask_stride; @@ -2844,6 +2812,7 @@ int av1_return_max_sub_pixel_mv(MACROBLOCK *x, const AV1_COMMON *const cm, (void)cm; (void)mi_row; (void)mi_col; + (void)do_reset_fractional_mv; bestmv->row = maxr; bestmv->col = maxc; @@ -2854,16 +2823,14 @@ int av1_return_max_sub_pixel_mv(MACROBLOCK *x, const AV1_COMMON *const cm, return besterr; } // Return the minimum MV. -int av1_return_min_sub_pixel_mv(MACROBLOCK *x, const AV1_COMMON *const cm, - int mi_row, int mi_col, const MV *ref_mv, - int allow_hp, int error_per_bit, - const aom_variance_fn_ptr_t *vfp, - int forced_stop, int iters_per_step, - int *cost_list, int *mvjcost, int *mvcost[2], - int *distortion, unsigned int *sse1, - const uint8_t *second_pred, const uint8_t *mask, - int mask_stride, int invert_mask, int w, int h, - int use_accurate_subpel_search) { +int av1_return_min_sub_pixel_mv( + MACROBLOCK *x, const AV1_COMMON *const cm, int mi_row, int mi_col, + const MV *ref_mv, int allow_hp, int error_per_bit, + const aom_variance_fn_ptr_t *vfp, int forced_stop, int iters_per_step, + int *cost_list, int *mvjcost, int *mvcost[2], int *distortion, + unsigned int *sse1, const uint8_t *second_pred, const uint8_t *mask, + int mask_stride, int invert_mask, int w, int h, + int use_accurate_subpel_search, const int do_reset_fractional_mv) { COMMON_MV_TEST; (void)maxr; (void)maxc; @@ -2874,6 +2841,7 @@ int av1_return_min_sub_pixel_mv(MACROBLOCK *x, const AV1_COMMON *const cm, (void)cm; (void)mi_row; (void)mi_col; + (void)do_reset_fractional_mv; bestmv->row = minr; bestmv->col = minc; diff --git a/third_party/aom/av1/encoder/mcomp.h b/third_party/aom/av1/encoder/mcomp.h index a975218b09..3f8b3b1fd3 100644 --- a/third_party/aom/av1/encoder/mcomp.h +++ b/third_party/aom/av1/encoder/mcomp.h @@ -103,7 +103,7 @@ typedef int(fractional_mv_step_fp)( int iters_per_step, int *cost_list, int *mvjcost, int *mvcost[2], int *distortion, unsigned int *sse1, const uint8_t *second_pred, const uint8_t *mask, int mask_stride, int invert_mask, int w, int h, - int use_accurate_subpel_search); + int use_accurate_subpel_search, const int do_reset_fractional_mv); extern fractional_mv_step_fp av1_find_best_sub_pixel_tree; extern fractional_mv_step_fp av1_find_best_sub_pixel_tree_pruned; @@ -154,6 +154,12 @@ unsigned int av1_refine_warped_mv(const struct AV1_COMP *cpi, int mi_row, int mi_col, int *pts0, int *pts_inref0, int total_samples); +static INLINE void av1_set_fractional_mv(int_mv *fractional_best_mv) { + for (int z = 0; z < 3; z++) { + fractional_best_mv[z].as_int = INVALID_MV; + } +} + #ifdef __cplusplus } // extern "C" #endif diff --git a/third_party/aom/av1/encoder/mips/msa/temporal_filter_msa.c b/third_party/aom/av1/encoder/mips/msa/temporal_filter_msa.c index 531ae090a3..effa75b83f 100644 --- a/third_party/aom/av1/encoder/mips/msa/temporal_filter_msa.c +++ b/third_party/aom/av1/encoder/mips/msa/temporal_filter_msa.c @@ -267,6 +267,7 @@ static void temporal_filter_apply_16size_msa(uint8_t *frm1_ptr, uint32_t stride, } } +// TODO(yunqing) The following optimization is not used since c code changes. void av1_temporal_filter_apply_msa(uint8_t *frame1_ptr, uint32_t stride, uint8_t *frame2_ptr, uint32_t blk_w, uint32_t blk_h, int32_t strength, diff --git a/third_party/aom/av1/encoder/ml.c b/third_party/aom/av1/encoder/ml.c index d21def43a8..579900afd3 100644 --- a/third_party/aom/av1/encoder/ml.c +++ b/third_party/aom/av1/encoder/ml.c @@ -15,31 +15,31 @@ #include "aom_dsp/aom_dsp_common.h" #include "av1/encoder/ml.h" -void av1_nn_predict(const float *features, const NN_CONFIG *nn_config, - float *output) { +// Calculate prediction based on the given input features and neural net config. +// Assume there are no more than NN_MAX_NODES_PER_LAYER nodes in each hidden +// layer. +void av1_nn_predict_c(const float *input_nodes, + const NN_CONFIG *const nn_config, float *const output) { int num_input_nodes = nn_config->num_inputs; int buf_index = 0; float buf[2][NN_MAX_NODES_PER_LAYER]; - const float *input_nodes = features; // Propagate hidden layers. const int num_layers = nn_config->num_hidden_layers; assert(num_layers <= NN_MAX_HIDDEN_LAYERS); for (int layer = 0; layer < num_layers; ++layer) { - const float *weights = nn_config->weights[layer]; - const float *bias = nn_config->bias[layer]; + const float *layer_weights = nn_config->weights[layer]; + const float *layer_bias = nn_config->bias[layer]; float *output_nodes = buf[buf_index]; const int num_output_nodes = nn_config->num_hidden_nodes[layer]; assert(num_output_nodes < NN_MAX_NODES_PER_LAYER); for (int node = 0; node < num_output_nodes; ++node) { - float val = 0.0f; + float val = layer_bias[node]; for (int i = 0; i < num_input_nodes; ++i) - val += weights[i] * input_nodes[i]; - val += bias[node]; + val += layer_weights[node * num_input_nodes + i] * input_nodes[i]; // ReLU as activation function. val = val > 0.0f ? val : 0.0f; // Could use AOMMAX(). output_nodes[node] = val; - weights += num_input_nodes; } num_input_nodes = num_output_nodes; input_nodes = output_nodes; @@ -47,14 +47,13 @@ void av1_nn_predict(const float *features, const NN_CONFIG *nn_config, } // Final output layer. - const float *weights = nn_config->weights[num_layers]; + const float *layer_weights = nn_config->weights[num_layers]; + const float *layer_bias = nn_config->bias[num_layers]; for (int node = 0; node < nn_config->num_outputs; ++node) { - const float *bias = nn_config->bias[num_layers]; - float val = 0.0f; + float val = layer_bias[node]; for (int i = 0; i < num_input_nodes; ++i) - val += weights[i] * input_nodes[i]; - output[node] = val + bias[node]; - weights += num_input_nodes; + val += layer_weights[node * num_input_nodes + i] * input_nodes[i]; + output[node] = val; } } @@ -66,7 +65,9 @@ void av1_nn_softmax(const float *input, float *output, int n) { for (int i = 1; i < n; i++) max_inp = AOMMAX(max_inp, input[i]); float sum_out = 0.0f; for (int i = 0; i < n; i++) { - output[i] = (float)exp(input[i] - max_inp); + // Clamp to range [-10.0, 0.0] to prevent FE_UNDERFLOW errors. + const float normalized_input = AOMMAX(input[i] - max_inp, -10.0f); + output[i] = (float)exp(normalized_input); sum_out += output[i]; } for (int i = 0; i < n; i++) output[i] /= sum_out; diff --git a/third_party/aom/av1/encoder/ml.h b/third_party/aom/av1/encoder/ml.h index cb8ef2871b..7f2750b31d 100644 --- a/third_party/aom/av1/encoder/ml.h +++ b/third_party/aom/av1/encoder/ml.h @@ -16,10 +16,12 @@ extern "C" { #endif +#include "config/av1_rtcd.h" + #define NN_MAX_HIDDEN_LAYERS 10 #define NN_MAX_NODES_PER_LAYER 128 -typedef struct { +struct NN_CONFIG { int num_inputs; // Number of input nodes, i.e. features. int num_outputs; // Number of output nodes. int num_hidden_layers; // Number of hidden layers, maximum 10. @@ -29,13 +31,8 @@ typedef struct { const float *weights[NN_MAX_HIDDEN_LAYERS + 1]; // Bias parameters, indexed by layer. const float *bias[NN_MAX_HIDDEN_LAYERS + 1]; -} NN_CONFIG; - -// Calculate prediction based on the given input features and neural net config. -// Assume there are no more than NN_MAX_NODES_PER_LAYER nodes in each hidden -// layer. -void av1_nn_predict(const float *features, const NN_CONFIG *nn_config, - float *output); +}; +// Typedef from struct NN_CONFIG to NN_CONFIG is in rtcd_defs // Applies the softmax normalization function to the input // to get a valid probability distribution in the output: diff --git a/third_party/aom/av1/encoder/partition_model_weights.h b/third_party/aom/av1/encoder/partition_model_weights.h index 437ea43f9e..2e7b2c62af 100644 --- a/third_party/aom/av1/encoder/partition_model_weights.h +++ b/third_party/aom/av1/encoder/partition_model_weights.h @@ -2441,6 +2441,1420 @@ static const NN_CONFIG av1_rect_partition_nnconfig_128 = { #undef NUM_NODES #undef LABEL_SIZE +#if CONFIG_ONE_PASS_SVM +#define FEATURE_SIZE 24 +static const float av1_op_svm_early_term_weights_128[FEATURE_SIZE + 1] = { + -4.5893036051f, 6.9065208136f, -9.1579514692f, 0.1353151366f, + -1.0271889653f, -0.0020988254f, -0.0094355949f, 0.0040209656f, + 0.0073014747f, 0.7939705382f, 0.0254545714f, 0.0557559708f, + -0.0339662064f, -0.0496818300f, 0.3053600283f, 0.3699486845f, + 0.0848271391f, 0.4091075988f, 0.1196729398f, -0.0038137193f, + -0.0773495909f, -0.0651630642f, -0.0123704995f, -0.0036697401f, + -4.1930227095f, +}; + +static const float av1_op_svm_early_term_weights_64[FEATURE_SIZE + 1] = { + -2.7600454480f, 5.6822046712f, -6.7576830133f, 0.1326457117f, + -1.0541818372f, 0.0107782654f, 0.0050469147f, -0.0021362631f, + -0.0135151040f, -0.1020115005f, -0.0283409957f, -0.0176311233f, + 0.0250648204f, 0.0196228570f, 0.5441528594f, 0.2767320141f, + 0.1261231351f, 0.2998476408f, 0.1336215695f, -0.1107823946f, + -0.0697279598f, -0.0577520545f, -0.0558441075f, -0.0699750617f, + -2.6995991503f, +}; + +static const float av1_op_svm_early_term_weights_32[FEATURE_SIZE + 1] = { + -0.8950734172f, 1.3559565008f, -2.6733642653f, 0.2661361319f, + -0.0314731140f, 0.0044943456f, 0.0006438044f, -0.0029066686f, + -0.0021903213f, 0.5845049496f, -0.0003629350f, 0.0006982840f, + 0.0014157386f, -0.0017427528f, 0.7078456733f, 0.1600998068f, + 0.0933852747f, 0.2822125876f, 0.1923826165f, -0.0905903459f, + -0.0564717590f, -0.0591007486f, -0.0692268554f, -0.0677411981f, + -0.7101853206f, +}; + +static const float av1_op_svm_early_term_weights_16[FEATURE_SIZE + 1] = { + -0.1719124013f, -0.3192305362f, -1.1714597182f, 0.4437770294f, + -0.0042344643f, 0.0000027764f, 0.0018827450f, -0.0015555613f, + -0.0003250050f, 0.9413693294f, 0.0076188418f, -0.0067870352f, + 0.0006329246f, -0.0013059613f, 0.8596697254f, 0.0635558018f, + 0.0447224598f, 0.0915706321f, 0.0741662273f, -0.0269096547f, + -0.0244610614f, -0.0281113318f, -0.0326108845f, -0.0350908892f, + -0.0307521675f, +}; + +static const float av1_op_svm_early_term_mean_128[FEATURE_SIZE] = { + 940540.3259649610f, 3988285.5905584921f, 575475302.3545289040f, + 0.5775348803f, 866.9828469502f, 0.2503762393f, + 0.2501466215f, 0.2513213770f, 0.2481557622f, + 521994448.3219169378f, 0.2666920631f, 0.2535864361f, + 0.2481589186f, 0.2315625823f, 100519.1049708007f, + 12.1299754840f, 0.8279971004f, 12.6664603305f, + 0.7313258998f, 935.8233056680f, 0.7436563032f, + 0.7710055018f, 0.7376516970f, 0.6859818720f, +}; + +static const float av1_op_svm_early_term_mean_64[FEATURE_SIZE] = { + 420419.7529613562f, 839754.4414347620f, 129360420.5256031156f, + 0.6525652037f, 548.8972009954f, 0.2506918565f, + 0.2488349076f, 0.2501724146f, 0.2503008213f, + 113132974.7944754064f, 0.2479344278f, 0.2471446791f, + 0.2524478512f, 0.2524730419f, 91147.9854189453f, + 10.9642508460f, 0.8936554428f, 11.3877865621f, + 0.8307555282f, 752.7787491956f, 0.7243363939f, + 0.7198362119f, 0.7329432336f, 0.7245090283f, +}; + +static const float av1_op_svm_early_term_mean_32[FEATURE_SIZE] = { + 105111.0236438536f, 184296.0939716828f, 29117017.6751756854f, + 0.6402298612f, 140.2223339218f, 0.2495860872f, + 0.2496407600f, 0.2506238629f, 0.2501492900f, + 24480304.9390618578f, 0.2494442027f, 0.2496080963f, + 0.2504881563f, 0.2504595447f, 60297.6762059058f, + 9.4279752138f, 0.9287901132f, 9.6516813792f, + 0.9009173677f, 591.5406335030f, 0.6944486917f, + 0.6983941982f, 0.6927236901f, 0.6921613649f, +}; + +static const float av1_op_svm_early_term_mean_16[FEATURE_SIZE] = { + 34080.7994802934f, 44108.1176228864f, 7494288.4946180154f, 0.6240636218f, + 36.4539515827f, 0.2490867417f, 0.2499231014f, 0.2505361492f, + 0.2504540077f, 5913397.2957480755f, 0.2487482536f, 0.2495500728f, + 0.2503693302f, 0.2513323434f, 36574.9686737814f, 7.4345592768f, + 0.9592429205f, 7.6001764585f, 0.9459867777f, 490.4635033056f, + 0.6626215237f, 0.6580791886f, 0.6655481064f, 0.6589010119f, +}; + +static const float av1_op_svm_early_term_std_128[FEATURE_SIZE] = { + 2054266.2732957317f, 7550554.6241466375f, 1078688147.1656334400f, + 0.4939517611f, 1414.3139592985f, 0.1504634077f, + 0.1515907199f, 0.1590329744f, 0.1515653324f, + 1006422867.8989596367f, 0.1168668155f, 0.1195725959f, + 0.1195825693f, 0.1123065533f, 195261.0940245980f, + 4.5876675121f, 0.3773829648f, 4.8017339769f, + 0.4432700397f, 973.7532938848f, 0.4790027843f, + 0.5056275222f, 0.5262278749f, 0.4685586148f, +}; + +static const float av1_op_svm_early_term_std_64[FEATURE_SIZE] = { + 1093636.0522712648f, 1749863.5221569177f, 255168612.8025657237f, + 0.4761552884f, 1084.7927994662f, 0.1099344646f, + 0.1100619440f, 0.1090853225f, 0.1115303745f, + 232084513.1365262568f, 0.0759732385f, 0.0762942913f, + 0.0785624106f, 0.0779284747f, 185687.9441778057f, + 4.4371901245f, 0.3082781088f, 4.6670562831f, + 0.3749677061f, 854.3212307408f, 0.4920531348f, + 0.5073919158f, 0.5054698298f, 0.4904895620f, +}; + +static const float av1_op_svm_early_term_std_32[FEATURE_SIZE] = { + 238229.7484988807f, 400136.8703966461f, 60267828.4581554681f, + 0.4799328974f, 268.9377064297f, 0.1122938575f, + 0.1126479260f, 0.1137018559f, 0.1126389337f, + 52174139.1477040648f, 0.0715628767f, 0.0720997035f, + 0.0728961434f, 0.0732065300f, 147785.0049793872f, + 4.2092341484f, 0.2571751131f, 4.3893075417f, + 0.2987729310f, 769.0253148602f, 0.5027558039f, + 0.4982811444f, 0.5092312751f, 0.4991214994f, +}; + +static const float av1_op_svm_early_term_std_16[FEATURE_SIZE] = { + 64177.9527087587f, 103729.9987511119f, 16632490.8146969266f, + 0.4843637247f, 65.8114470725f, 0.0884226846f, + 0.0912638659f, 0.0914771167f, 0.0916078800f, + 13364581.3877149168f, 0.0677468925f, 0.0689631274f, + 0.0689915367f, 0.0702648469f, 111397.2620676765f, + 3.7858187888f, 0.1977269328f, 3.9420183951f, + 0.2260437881f, 717.5336868275f, 0.5017939514f, + 0.5066633533f, 0.5086806985f, 0.5085585987f, +}; + +#undef FEATURE_SIZE +#endif // CONFIG_ONE_PASS_SVM + +// Below are the models used for full_pixel_motion_search_based_split +static const float full_pixel_motion_search_based_split_thresh_128 = 2.0f; +static const float full_pixel_motion_search_based_split_thresh_64 = 2.0f; +static const float full_pixel_motion_search_based_split_thresh_32 = 2.0f; +static const float full_pixel_motion_search_based_split_thresh_16 = 2.0f; +static const float full_pixel_motion_search_based_split_thresh_8 = 2.0f; + +// BLOCK_128X128 +#define NUM_HIDDEN_LAYERS_128 1 +#define NUM_FEATURES_128 6 +#define NUM_LAYER_0_UNITS_128 16 +#define NUM_LOGITS_128 1 + +static const float full_pixel_motion_search_based_split_layer_0_kernel_128[] = { + -0.807346f, 0.242298f, 12.9862f, -1.19161f, 5.21734f, -1.1363f, + -2.39127f, 0.930915f, -2.44285f, -2.42966f, 5.73476f, 0.0506879f, + -0.234878f, -0.317875f, 0.361322f, 0.431648f, -0.39105f, -0.110225f, + -2.46236f, 0.979713f, -10.5596f, -7.76653f, -3.06518f, 2.42554f, + 0.0492961f, -0.467176f, 0.130746f, 0.494527f, -0.0336645f, 0.501755f, + 0.176486f, -0.869541f, 7.77757f, 6.81303f, 6.00771f, 7.35696f, + 0.150731f, -0.307017f, -0.437639f, -0.082924f, 0.379107f, 0.452278f, + -0.0143836f, -0.183691f, -0.604698f, -9.2681f, -2.06087f, 11.0256f, + 0.0487599f, -0.249168f, -0.180407f, 0.304772f, 0.218642f, -0.406073f, + -0.0289919f, -0.794381f, 5.45092f, 5.38374f, 3.25745f, 5.32903f, + 1.12718f, -0.0215478f, 2.78552f, 4.8951f, -0.959671f, 0.694264f, + -0.0611219f, -0.331937f, 0.258252f, -0.495331f, -0.285923f, 0.294713f, + -0.119947f, 0.0753204f, 10.2021f, -5.82147f, -12.0137f, 3.0365f, + 0.366697f, 0.142683f, -3.29731f, -5.76651f, -5.62578f, 10.9462f, + -0.325459f, 0.092602f, -0.868027f, -0.691768f, -0.292017f, -0.00841203f, + 0.702545f, -0.612227f, -7.68881f, 9.52225f, -1.18581f, -2.56762f +}; + +static const float full_pixel_motion_search_based_split_logits_kernel_128[] = { + 0.364895f, 0.577553f, 0.115758f, -0.999496f, 0.124885f, 3.23193f, + -0.00386642f, 0.970794f, 0.136637f, -4.28052f, -1.49234f, 0.370436f, + 0.576981f, -0.469656f, -0.124071f, 1.07669f +}; + +static const float full_pixel_motion_search_based_split_layer_0_bias_128[] = { + 1.32916f, 0.817212f, 0.0f, -0.921066f, 0.0f, 3.57649f, + -0.0204517f, 2.97286f, 0.0f, 5.49957f, -8.14518f, 0.0f, + 1.30826f, -0.349536f, -0.638933f, 5.4496f +}; + +static const float full_pixel_motion_search_based_split_logits_bias_128[] = { + 0.683442f +}; + +static const NN_CONFIG full_pixel_motion_search_based_split_nn_config_128 = { + NUM_FEATURES_128, + NUM_LOGITS_128, + NUM_HIDDEN_LAYERS_128, + { + NUM_LAYER_0_UNITS_128, + }, + { + full_pixel_motion_search_based_split_layer_0_kernel_128, + full_pixel_motion_search_based_split_logits_kernel_128, + }, + { + full_pixel_motion_search_based_split_layer_0_bias_128, + full_pixel_motion_search_based_split_logits_bias_128, + }, +}; + +#undef NUM_HIDDEN_LAYERS_128 +#undef NUM_FEATURES_128 +#undef NUM_LAYER_0_UNITS_128 +#undef NUM_LOGITS_128 + +// BLOCK_64X64 +#define NUM_HIDDEN_LAYERS_64 1 +#define NUM_FEATURES_64 6 +#define NUM_LAYER_0_UNITS_64 16 +#define NUM_LOGITS_64 1 + +static const float full_pixel_motion_search_based_split_layer_0_kernel_64[] = { + 0.0345945f, -0.394064f, 0.0919978f, 0.270358f, -0.384502f, -0.504608f, + -0.25759f, 0.155981f, 2.62567f, -10.7204f, -0.709802f, 8.15948f, + 0.589866f, -0.445645f, -1.68232f, 10.0061f, -3.17671f, 4.87259f, + -0.448886f, -0.205568f, -0.462388f, 0.385001f, -0.451687f, 0.49602f, + -0.256708f, 0.803322f, 3.25594f, 0.38541f, -1.83867f, -2.15132f, + 0.936059f, -0.203056f, -5.92959f, -6.24554f, -6.68631f, -6.85977f, + -0.0407565f, -0.258902f, 0.195053f, -0.366515f, 0.339543f, -0.433017f, + -2.67026f, 0.385457f, 1.86683f, 1.9501f, 0.0381398f, 1.086f, + -0.153729f, 0.173772f, -42.9029f, -36.8934f, -2.892f, -0.0540691f, + 0.77469f, -0.380145f, 2.2689f, -9.53332f, 1.15712f, 2.86601f, + -0.437036f, 0.247132f, -8.51058f, -3.62972f, -8.99449f, -0.638738f, + 0.0609263f, -0.0614603f, 5.42307f, 5.35926f, 5.27437f, 5.26599f, + -0.0729677f, 0.0306104f, -7.77867f, 5.03598f, -8.17832f, 5.85461f, + -0.253269f, 0.164582f, -4.49713f, 3.83265f, 9.04851f, -2.85668f, + 1.22618f, 0.166904f, -1.51975f, -4.01576f, -1.44374f, -2.22147f, + -0.217072f, -0.0984913f, -0.265515f, 0.360021f, 0.0779512f, 0.361516f +}; + +static const float full_pixel_motion_search_based_split_logits_kernel_64[] = { + 0.470821f, 0.474747f, -0.571292f, 0.403221f, 0.628966f, -0.617029f, + 0.501105f, 0.499962f, -1.5451f, -0.473518f, -0.730568f, -5.55817f, + 0.776761f, 0.42569f, 0.311925f, 0.469968f +}; + +static const float full_pixel_motion_search_based_split_layer_0_bias_64[] = { + -0.134085f, 0.0758715f, 1.10419f, 0.0f, -5.75737f, 1.65494f, + 0.0f, 3.44047f, 0.394852f, 3.43858f, 3.65871f, -4.84987f, + 1.21207f, -1.7705f, -5.46469f, -0.0889634f +}; + +static const float full_pixel_motion_search_based_split_logits_bias_64[] = { + -0.479491f +}; + +static const NN_CONFIG full_pixel_motion_search_based_split_nn_config_64 = { + NUM_FEATURES_64, + NUM_LOGITS_64, + NUM_HIDDEN_LAYERS_64, + { + NUM_LAYER_0_UNITS_64, + }, + { + full_pixel_motion_search_based_split_layer_0_kernel_64, + full_pixel_motion_search_based_split_logits_kernel_64, + }, + { + full_pixel_motion_search_based_split_layer_0_bias_64, + full_pixel_motion_search_based_split_logits_bias_64, + }, +}; + +#undef NUM_HIDDEN_LAYERS_64 +#undef NUM_FEATURES_64 +#undef NUM_LAYER_0_UNITS_64 +#undef NUM_LOGITS_64 + +// BLOCK_32X32 +#define NUM_HIDDEN_LAYERS_32 1 +#define NUM_FEATURES_32 6 +#define NUM_LAYER_0_UNITS_32 16 +#define NUM_LOGITS_32 1 + +static const float full_pixel_motion_search_based_split_layer_0_kernel_32[] = { + -1.61796f, 0.0585128f, 1.57904f, 1.52703f, 0.367779f, 0.220434f, + 1.66652f, -1.77782f, 6.41118f, 4.16976f, 4.97299f, 4.84111f, + -0.0956536f, -0.163284f, -0.143662f, 0.129329f, 0.449659f, -0.528844f, + -1.00067f, 1.17203f, -4.26777f, -4.78521f, 8.45658f, -3.49498f, + -1.78386f, 0.111488f, 4.176f, 6.31911f, -10.5369f, 6.26983f, + -1.32233f, 1.22999f, -4.1666f, -10.0359f, -4.14779f, -10.4695f, + 1.83011f, -0.333152f, -9.87986f, -8.11992f, -8.2775f, -7.79918f, + -0.101404f, 0.00401393f, 8.89046f, -7.32186f, -6.59597f, 9.66257f, + -1.1492f, 1.23067f, -3.6341f, 6.59275f, -3.2373f, -3.42564f, + 0.371736f, -0.140902f, -2.75715f, 5.92487f, -7.9185f, 9.13743f, + -3.52698f, -0.191044f, 5.96691f, 6.26327f, 4.36378f, 5.69354f, + -0.608845f, -0.191236f, -0.482191f, -0.180474f, -3.8838f, -3.92934f, + -1.03191f, 0.994568f, 7.95516f, -4.0035f, -2.86266f, -4.96105f, + 1.75022f, 0.125058f, -1.52159f, -3.59304f, -2.82634f, -2.49556f, + -2.05557f, -0.222577f, 3.7608f, 5.50475f, 2.7046f, 5.25952f, + -1.91327f, -0.0356497f, 1.47611f, 1.27499f, -1.76108f, -0.578954f +}; + +static const float full_pixel_motion_search_based_split_logits_kernel_32[] = { + -0.220382f, -0.693902f, 0.424827f, 0.379952f, -0.413791f, -0.326785f, + -0.455086f, 0.242402f, 0.307986f, 0.175746f, 0.498901f, -0.628053f, + 0.285447f, 0.230052f, 0.415151f, -0.842946f +}; + +static const float full_pixel_motion_search_based_split_layer_0_bias_32[] = { + -1.80751f, 6.40356f, -0.0512058f, -4.59163f, -0.369933f, -0.195755f, + -0.16648f, -0.599755f, -5.35975f, -1.21349f, 2.48414f, 1.07096f, + -3.66684f, -6.17761f, 4.2159f, -1.05286f +}; + +static const float full_pixel_motion_search_based_split_logits_bias_32[] = { + -2.58676f +}; + +static const NN_CONFIG full_pixel_motion_search_based_split_nn_config_32 = { + NUM_FEATURES_32, + NUM_LOGITS_32, + NUM_HIDDEN_LAYERS_32, + { + NUM_LAYER_0_UNITS_32, + }, + { + full_pixel_motion_search_based_split_layer_0_kernel_32, + full_pixel_motion_search_based_split_logits_kernel_32, + }, + { + full_pixel_motion_search_based_split_layer_0_bias_32, + full_pixel_motion_search_based_split_logits_bias_32, + }, +}; + +#undef NUM_HIDDEN_LAYERS_32 +#undef NUM_FEATURES_32 +#undef NUM_LAYER_0_UNITS_32 +#undef NUM_LOGITS_32 + +// BLOCK_16X16 +#define NUM_HIDDEN_LAYERS_16 1 +#define NUM_FEATURES_16 6 +#define NUM_LAYER_0_UNITS_16 16 +#define NUM_LOGITS_16 1 + +static const float full_pixel_motion_search_based_split_layer_0_kernel_16[] = { + -0.611497f, -0.0422086f, -0.555957f, -0.632451f, -0.144179f, -0.152722f, + -0.330265f, -0.419866f, 0.287343f, 0.385295f, -0.424486f, 0.424281f, + 2.27442f, -2.47933f, 5.24731f, 4.33827f, 4.73215f, 3.41909f, + 1.16058f, -0.364505f, 0.12207f, -0.287749f, 0.0509783f, -0.0200119f, + 1.52907f, -1.1905f, -2.56978f, -3.00186f, -3.56084f, -3.89276f, + 0.00365657f, 1.57125f, -4.421f, -2.48803f, -2.51531f, -4.28646f, + 2.52248f, -1.03377f, -1.09607f, -1.44633f, -1.58736f, -1.25927f, + -1.45841f, -0.566619f, -0.246166f, -0.182289f, -0.238156f, 0.177991f, + 0.0112509f, -0.17677f, -0.485877f, 0.0812852f, 0.104975f, 0.222793f, + -0.372858f, -0.48624f, -0.00870389f, -0.385019f, 0.405842f, 0.288523f, + 0.167374f, -0.204208f, -8.74148f, -8.59267f, -8.42492f, -8.3778f, + -5.57063f, -0.406818f, -0.873199f, -0.896224f, -0.701479f, -0.985736f, + -0.625956f, -0.0446202f, -0.509987f, -0.321804f, -0.470759f, -0.248556f, + -0.369436f, -0.160828f, 0.0591148f, 0.405218f, 0.142584f, -0.130106f, + 0.125321f, 0.0888179f, 7.34822f, -6.71488f, -7.06592f, 6.33224f, + 0.0333619f, -0.377782f, 0.160767f, -0.128169f, -0.484818f, -0.311973f +}; + +static const float full_pixel_motion_search_based_split_logits_kernel_16[] = { + -0.132207f, 0.15176f, -0.680086f, 0.605921f, -0.43294f, 0.485811f, + -0.306286f, 0.551368f, 0.413904f, 0.548748f, -0.437391f, 0.560778f, + -0.00685266f, -0.558657f, 0.122127f, 0.260165f +}; + +static const float full_pixel_motion_search_based_split_layer_0_bias_16[] = { + -0.200928f, -0.074132f, 8.69963f, -9.00807f, 9.08983f, -6.83586f, + -3.89329f, 10.4881f, -0.0670618f, 0.0f, 9.21614f, 8.41773f, + -0.145851f, 0.0f, -1.43038f, -0.0460311f +}; + +static const float full_pixel_motion_search_based_split_logits_bias_16[] = { + -4.19885f +}; + +static const NN_CONFIG full_pixel_motion_search_based_split_nn_config_16 = { + NUM_FEATURES_16, + NUM_LOGITS_16, + NUM_HIDDEN_LAYERS_16, + { + NUM_LAYER_0_UNITS_16, + }, + { + full_pixel_motion_search_based_split_layer_0_kernel_16, + full_pixel_motion_search_based_split_logits_kernel_16, + }, + { + full_pixel_motion_search_based_split_layer_0_bias_16, + full_pixel_motion_search_based_split_logits_bias_16, + }, +}; + +#undef NUM_HIDDEN_LAYERS_16 +#undef NUM_FEATURES_16 +#undef NUM_LAYER_0_UNITS_16 +#undef NUM_LOGITS_16 + +#if !CONFIG_DISABLE_FULL_PIXEL_SPLIT_8X8 +// BLOCK_8X8 +#define NUM_HIDDEN_LAYERS_8 1 +#define NUM_FEATURES_8 6 +#define NUM_LAYER_0_UNITS_8 16 +#define NUM_LOGITS_8 1 + +static const float full_pixel_motion_search_based_split_layer_0_kernel_8[] = { + 0.0370236f, -0.580211f, 2.0134f, 1.69637f, 2.43181f, -0.521648f, + -0.00375187f, 0.122712f, -4.74411f, 7.36187f, 5.42574f, -5.53557f, + 0.0993344f, -0.358843f, 0.0765453f, -0.615987f, -0.754633f, -0.175846f, + 0.714976f, 0.492862f, 0.346604f, -1.23922f, -2.67031f, 2.12749f, + 1.71511f, -1.4239f, 2.09396f, 2.42478f, 2.40151f, 2.90487f, + 0.540813f, -0.0954257f, -4.57571f, -4.88078f, -4.62386f, -5.75167f, + 1.35351f, -1.08114f, 1.43744f, 1.44333f, 0.608153f, 0.193742f, + -0.405512f, -0.155164f, 0.0771456f, -0.473182f, -0.057984f, 0.140435f, + 0.743021f, -0.418589f, -0.377622f, -0.531411f, -0.668025f, -0.826607f, + 1.37834f, -1.07753f, 0.870466f, 0.516756f, 0.708689f, 0.286795f, + -3.97895f, -0.338629f, 2.79427f, 1.80561f, 1.46275f, 1.50438f, + 0.0232533f, -0.43174f, -0.348251f, 0.0863006f, 0.0321103f, 0.129674f, + -1.12024f, -0.0990596f, -0.283472f, -0.238713f, -0.239175f, -0.40816f, + -0.00106566f, 0.0972736f, 5.19284f, -3.70862f, 6.39657f, -5.27588f, + -2.08003f, 0.38825f, 2.38771f, -1.27501f, -2.45619f, 3.07324f, + 0.616966f, -0.451472f, -0.319365f, 0.00807278f, -0.303261f, -0.351679f +}; + +static const float full_pixel_motion_search_based_split_logits_kernel_8[] = { + -0.625847f, 0.381323f, 0.342475f, 0.526161f, -0.665965f, -0.515317f, + -0.406218f, 0.568007f, 0.479397f, -0.426116f, 0.615638f, 0.338572f, + 0.185583f, 0.308031f, 0.260748f, 0.531619f +}; + +static const float full_pixel_motion_search_based_split_layer_0_bias_8[] = { + 4.73775f, -1.12658f, -0.258038f, -6.06696f, 1.79131f, 2.49609f, + 4.28388f, 0.0f, -4.63598f, 3.06034f, 5.31994f, -0.152142f, + 0.514738f, -1.30098f, 3.00296f, -3.83481f +}; + +static const float full_pixel_motion_search_based_split_logits_bias_8[] = { + -3.44508f +}; + +static const NN_CONFIG full_pixel_motion_search_based_split_nn_config_8 = { + NUM_FEATURES_8, + NUM_LOGITS_8, + NUM_HIDDEN_LAYERS_8, + { + NUM_LAYER_0_UNITS_8, + }, + { + full_pixel_motion_search_based_split_layer_0_kernel_8, + full_pixel_motion_search_based_split_logits_kernel_8, + }, + { + full_pixel_motion_search_based_split_layer_0_bias_8, + full_pixel_motion_search_based_split_logits_bias_8, + }, +}; + +#endif + +// Model based on simple_motion_search + +// Thresholds for doing a single type of partition +// TODO(chiyotsai@google.com): Set the thresholds for PARTITION_SPLIT. +static const float simple_motion_search_prune_part_only_thresh_128[10] = { + 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f +}; +static const float simple_motion_search_prune_part_only_thresh_64[10] = { + 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f +}; +static const float simple_motion_search_prune_part_only_thresh_32[10] = { + 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f +}; +static const float simple_motion_search_prune_part_only_thresh_16[10] = { + 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f +}; +static const float simple_motion_search_prune_part_only_thresh_8[10] = { + 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f +}; + +// Thresholds for pruning a partition type +// TODO(chiyotsai@google.com): Retune the thresholds for rectangular partition. +static const float simple_motion_search_prune_part_prune_thresh_128[10] = { + 0.0f, 0.0110f, 0.0110f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f +}; +static const float simple_motion_search_prune_part_prune_thresh_64[10] = { + 0.0f, 0.0110f, 0.0110f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f +}; +static const float simple_motion_search_prune_part_prune_thresh_32[10] = { + 0.0f, 0.0110f, 0.0110f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f +}; +static const float simple_motion_search_prune_part_prune_thresh_16[10] = { + 0.0f, 0.0110f, 0.0110f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f +}; +static const float simple_motion_search_prune_part_prune_thresh_8[10] = { + 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f +}; + +// Mean and std +static const float simple_motion_search_prune_part_mean_128[19] = { + 11.561261f, 11.626691f, 10.820066f, 10.880652f, 11.193911f, + 11.269741f, 10.509796f, 10.584754f, 12.659430f, 12.724323f, + 12.254200f, 12.326454f, 12.632568f, 12.694887f, 11.814313f, + 11.872458f, 13.690346f, 13.753730f, 5.165726f, +}; +static const float simple_motion_search_prune_part_std_128[19] = { + 2.434728f, 2.429037f, 3.701918f, 3.709713f, 3.288111f, 3.286657f, 4.174337f, + 4.185339f, 2.531916f, 2.540137f, 3.488878f, 3.496263f, 2.459336f, 2.460739f, + 3.919058f, 3.930766f, 2.511685f, 2.524923f, 2.257989f, +}; +static const float simple_motion_search_prune_part_mean_64[19] = { + 10.073195f, 10.158249f, 9.627341f, 9.708094f, 9.002025f, + 9.076430f, 8.597153f, 8.668316f, 11.021699f, 11.097507f, + 9.852887f, 9.919513f, 11.007151f, 11.085718f, 10.525035f, + 10.599575f, 11.936800f, 12.007580f, 4.860905f, +}; +static const float simple_motion_search_prune_part_std_64[19] = { + 2.272113f, 2.256767f, 3.023892f, 3.027094f, 3.794186f, 3.808349f, 4.143885f, + 4.164291f, 2.262401f, 2.249636f, 4.039632f, 4.053496f, 2.234799f, 2.219867f, + 3.136350f, 3.139975f, 2.199973f, 2.187191f, 2.185985f, +}; +static const float simple_motion_search_prune_part_mean_32[19] = { + 8.499073f, 8.600074f, 8.508158f, 8.611221f, 8.498978f, 8.599821f, 8.506126f, + 8.609448f, 9.481358f, 9.569901f, 9.481697f, 9.569918f, 9.469805f, 9.557789f, + 9.483047f, 9.572816f, 10.416696f, 10.496502f, 4.413157f, +}; +static const float simple_motion_search_prune_part_std_32[19] = { + 2.129560f, 2.117602f, 2.115962f, 2.104043f, 2.136564f, 2.125870f, 2.125247f, + 2.114250f, 2.146261f, 2.134374f, 2.160289f, 2.149360f, 2.116179f, 2.103744f, + 2.099453f, 2.087138f, 2.115011f, 2.101419f, 2.024279f, +}; +static const float simple_motion_search_prune_part_mean_16[19] = { + 6.990048f, 7.116367f, 6.883459f, 7.007796f, 6.993151f, 7.119445f, 6.886147f, + 7.010345f, 7.994230f, 8.099907f, 7.998478f, 8.104088f, 7.988468f, 8.092865f, + 7.868373f, 7.971079f, 8.946122f, 9.037547f, 3.976610f, +}; +static const float simple_motion_search_prune_part_std_16[19] = { + 1.963746f, 1.954729f, 2.127820f, 2.126451f, 1.963392f, 1.955016f, 2.126736f, + 2.126004f, 1.991209f, 1.981070f, 1.992683f, 1.983143f, 1.986913f, 1.977408f, + 2.200119f, 2.197786f, 1.997332f, 1.986091f, 1.829684f, +}; +static const float simple_motion_search_prune_part_mean_8[19] = { + 5.362649f, 5.561670f, 5.362880f, 5.564151f, 5.365307f, 5.565139f, 5.365830f, + 5.567452f, 6.453082f, 6.598073f, 6.458814f, 6.604363f, 6.454421f, 6.599441f, + 6.458431f, 6.604804f, 7.469143f, 7.585015f, 3.625800f, +}; +static const float simple_motion_search_prune_part_std_8[19] = { + 1.846273f, 1.835266f, 1.846272f, 1.834365f, 1.836967f, 1.826428f, 1.837247f, + 1.825985f, 1.886601f, 1.874138f, 1.874622f, 1.862656f, 1.877218f, 1.865398f, + 1.874494f, 1.862057f, 1.896421f, 1.883210f, 1.670200f, +}; + +// BLOCK_128X128 +#define NUM_HIDDEN_LAYERS_128 1 +#define NUM_FEATURES_128 19 +#define NUM_LAYER_0_UNITS_128 24 +#define NUM_LOGITS_128 4 + +static const float simple_motion_search_prune_part_logits_kernel_128[] = { + -0.12815f, -1.2107f, 0.525183f, 0.379155f, -2.04966f, -0.212818f, + -0.0609484f, -0.119338f, 1.07054f, -0.0304129f, 0.563817f, -0.0204729f, + 0.757013f, 0.464791f, 0.191185f, 0.183903f, -2.71738f, 1.41455f, + -0.449089f, -0.0344047f, 0.349133f, -0.945292f, 0.63729f, -0.973444f, + -0.203447f, -19.7875f, -12.7536f, -0.260806f, 0.934733f, -3.17648f, + -1.11806f, -4.63145f, -16.0947f, -24.203f, -2.94649f, -8.34109f, + -0.720804f, -5.16623f, -0.443715f, 0.510014f, -4.2846f, -11.2272f, + 0.187633f, -10.0189f, -0.915574f, -0.235434f, -2.57912f, 0.151642f, + -0.180428f, -0.110439f, 0.349015f, -1.83906f, -3.38461f, -0.309185f, + -9.3706f, -5.54824f, 0.45334f, -8.48801f, 0.392325f, -4.19948f, + -0.535814f, -4.92133f, 0.0159878f, 0.275078f, -2.65358f, 1.32524f, + -0.761795f, -14.782f, -4.15338f, -0.177581f, -0.158376f, -0.0832793f, + 0.226712f, -0.349409f, 0.0529463f, -0.412873f, -0.241807f, 0.19054f, + 0.79237f, 0.218937f, -1.60466f, 0.845653f, -0.287058f, 0.766271f, + -0.455829f, 0.00261991f, 0.506805f, -0.0593004f, 2.25348f, -2.29282f, + 0.0491916f, 0.38261f, -0.281848f, -0.17802f, -0.472124f, 0.204326f +}; + +static const float simple_motion_search_prune_part_layer_0_kernel_128[] = { + 0.485797f, -0.326149f, 0.238042f, -1.41255f, -0.00899515f, + -0.715324f, 0.623049f, 0.502229f, 0.54064f, -0.0667834f, + 0.0591777f, -0.435869f, 0.221466f, -0.222184f, 0.696493f, + -0.514433f, -0.771882f, -0.080914f, -1.58162f, -0.214871f, + -0.0141856f, -0.566004f, -1.01071f, -0.0400122f, -0.212581f, + 0.144009f, 0.156065f, 0.419812f, 0.351579f, -0.0363798f, + 0.272286f, 0.177348f, 0.643484f, -0.686943f, -1.5021f, + -0.283203f, -0.317958f, 0.834615f, 1.18195f, 0.704402f, + 0.723567f, 0.0765064f, 0.0998969f, 0.358114f, -0.471958f, + -0.33568f, -0.0455057f, 0.0659417f, 0.717502f, 1.40203f, + -1.78214f, -1.00743f, -0.217992f, -0.244456f, -0.681988f, + 0.199185f, 1.27797f, 0.744351f, 0.853728f, 0.976386f, + 1.5212f, 0.246886f, 0.458125f, -0.278508f, -0.601164f, + -0.874409f, -0.0882617f, -0.93093f, -0.0279819f, 0.289613f, + -0.795015f, 1.2224f, 0.464495f, -0.996107f, -1.05131f, + 0.780663f, 1.70779f, -0.122481f, 2.64976f, 0.284897f, + -0.129159f, -0.0472128f, 0.190105f, 0.549306f, -0.829239f, + -2.87392f, 0.0735607f, -1.00692f, -0.0171315f, 0.217386f, + 0.715055f, 1.2976f, -0.387801f, -0.301424f, -0.265159f, + 0.381381f, -0.637641f, 0.424966f, 0.317794f, 1.99832f, + 0.604094f, 0.444838f, 0.354898f, -0.2603f, -0.625206f, + 1.37372f, 0.474128f, 1.47492f, -0.318267f, -0.32122f, + -1.29434f, 0.426303f, -0.604944f, -2.33438f, -1.33476f, + -2.13254f, 2.28472f, 0.844804f, 0.366394f, 0.148218f, + -0.0996413f, -0.665787f, 1.30951f, 0.447868f, -0.160892f, + 0.336539f, -0.461251f, -0.940939f, 1.9886f, 0.774578f, + -0.0274317f, -0.288034f, -0.188082f, 0.296018f, -0.171846f, + 0.245583f, 0.39913f, -0.0544446f, 0.823534f, 0.215969f, + -0.393239f, -0.208693f, -0.336752f, 0.258191f, 1.03315f, + -0.0356378f, -0.35999f, 0.48529f, 1.09123f, -0.243654f, + 0.131594f, -2.5832f, 1.26485f, -0.700618f, 0.466223f, + -0.525984f, 2.19872f, 0.685663f, 1.40228f, 1.10528f, + 0.0451285f, -0.39981f, -0.644025f, -1.31137f, 0.836056f, + -0.884238f, 0.394441f, -0.16017f, -1.45246f, -1.24252f, + 0.195623f, 0.716485f, -0.0522381f, 0.271321f, 0.560211f, + 0.685537f, -0.117774f, -0.227596f, 0.346762f, -0.402137f, + 0.00528979f, 0.756388f, 0.181091f, -0.513235f, -0.354307f, + 1.04355f, -0.114563f, 0.457349f, -0.525175f, -1.306f, + -1.57685f, -1.69919f, 0.393799f, 1.35822f, -0.496799f, + 0.102504f, -0.0950497f, 0.518029f, 0.211844f, 0.385925f, + 0.366614f, 0.52377f, -0.22419f, -0.209355f, 0.343361f, + 0.0535484f, 0.320409f, 0.467987f, -0.00542184f, -1.02011f, + 0.488822f, -0.248647f, 0.113298f, -0.665206f, 0.374419f, + 0.731859f, 1.20004f, -0.000319085f, -0.000491633f, -0.260543f, + 0.804927f, -0.410925f, 0.960751f, 0.281493f, 0.607288f, + -0.681745f, -0.0925707f, -2.06876f, -0.52926f, 0.864695f, + -0.712922f, 0.706643f, -0.651177f, -0.231887f, -0.318515f, + 0.0635488f, -0.183294f, -0.589271f, 0.288583f, 0.38702f, + 0.582912f, 0.225519f, 1.23027f, 0.108471f, 1.3328f, + 0.689896f, -2.21988f, -0.337789f, -0.762238f, 1.79149f, + 0.840519f, 0.825442f, 0.460811f, 0.434437f, 2.13268f, + -0.341805f, -0.139746f, -1.66773f, -0.583511f, -0.238547f, + 0.156871f, 0.719404f, 1.05882f, -0.173445f, -0.867368f, + 0.170469f, 1.57755f, 1.36095f, -0.418987f, -0.396689f, + 0.289496f, -0.292606f, 0.688943f, 0.176553f, 0.853828f, + 1.19608f, -0.557087f, -0.58118f, 1.12805f, 0.232023f, + -0.167864f, -0.222265f, -0.719284f, -0.802184f, -2.85383f, + -1.01816f, -1.344f, -0.154875f, 0.308999f, -0.62822f, + 0.389045f, -0.498046f, -0.0646747f, 0.056343f, 0.188288f, + 0.493475f, 0.241337f, 0.122423f, -0.506522f, 0.0492923f, + 0.0642163f, -0.508636f, 0.0328815f, 0.78567f, 0.224673f, + 0.627972f, 0.759423f, 1.16089f, 1.28274f, 1.19646f, + 1.63401f, 1.56657f, -0.131442f, -0.423053f, -0.599536f, + -1.20933f, 0.0438751f, -0.538444f, -0.749136f, -0.769951f, + -1.06683f, -1.66474f, 0.136208f, 1.35745f, -0.960854f, + 2.32612f, 0.349384f, 1.04506f, -0.888194f, 1.28114f, + 0.579831f, 0.53927f, -1.07214f, 0.456708f, -0.884062f, + 0.789767f, -0.776746f, -0.0143946f, -0.930974f, -0.632568f, + -1.487f, 0.0284456f, -0.865849f, -1.51263f, 0.117335f, + -0.671304f, -0.367558f, -0.650224f, 1.09562f, 0.246097f, + 0.461775f, -0.722606f, 0.0886829f, -0.490978f, 1.92718f, + 0.864959f, 0.0705385f, -0.543902f, 1.4704f, -0.643494f, + 0.194486f, 0.890657f, 0.901154f, -0.458931f, 0.209679f, + 0.303869f, -0.167078f, 1.60197f, 1.05005f, -0.499959f, + 0.930484f, -0.555236f, -0.778482f, -0.772935f, -0.590988f, + -0.122397f, -0.346605f, 0.100747f, -0.237732f, -0.180927f, + 0.136491f, -0.230076f, 1.52049f, 1.70212f, 0.415758f, + -0.811552f, 0.570249f, 1.73167f, 0.808393f, -1.28608f, + 0.518307f, 0.601205f, 0.258751f, -0.498631f, -0.790802f, + -0.465072f, -0.507542f, 0.312956f, -0.485421f, -0.0525576f, + 0.24136f, -0.201595f, -0.721587f, 0.190955f, -0.123108f, + -0.263243f, -0.36142f, -0.105093f, 0.0651603f, 0.240065f, + 0.241104f, -0.238277f, -1.11714f, 1.50723f, 0.443715f, + 1.83051f, 1.29719f, -0.0367089f, 1.57883f, 1.96011f, + 0.0373268f, 0.480469f, 0.497726f, 1.47078f, -0.359254f, + 0.120578f, -0.651964f, -0.537215f, -1.20527f, -0.374681f, + -0.443443f, -0.39421f, 0.688228f, 0.0989643f, -1.28785f, + -1.24392f, -0.165236f, 1.1812f, -0.76329f, 1.44085f, + -0.540544f, 0.0958765f, -0.31224f, 0.471211f, -0.997358f, + 1.16174f, -0.948466f, 0.660749f, -0.574517f, 1.42706f, + -0.477758f, 0.509914f, -0.930534f, 0.722869f, -0.955477f, + -0.383073f +}; + +static const float simple_motion_search_prune_part_logits_bias_128[] = { + 0.814703f, -1.18485f, -1.49788f, -0.0054509f +}; + +static const float simple_motion_search_prune_part_layer_0_bias_128[] = { + -0.283185f, -2.5867f, -2.07531f, -2.31513f, -1.85869f, -0.310685f, + -0.802324f, 0.056964f, -1.43881f, 2.84016f, -1.15494f, -2.30652f, + 0.015622f, 0.303776f, 0.774322f, -1.25979f, -2.37144f, -1.35687f, + 0.743607f, 0.768569f, -1.09908f, -0.470101f, -0.615497f, 1.04214f +}; + +static const NN_CONFIG simple_motion_search_prune_part_nn_config_128 = { + NUM_FEATURES_128, + NUM_LOGITS_128, + NUM_HIDDEN_LAYERS_128, + { + NUM_LAYER_0_UNITS_128, + }, + { + simple_motion_search_prune_part_layer_0_kernel_128, + simple_motion_search_prune_part_logits_kernel_128, + }, + { + simple_motion_search_prune_part_layer_0_bias_128, + simple_motion_search_prune_part_logits_bias_128, + }, +}; + +#undef NUM_HIDDEN_LAYERS_128 +#undef NUM_FEATURES_128 +#undef NUM_LAYER_0_UNITS_128 +#undef NUM_LOGITS_128 + +// BLOCK_64X64 +#define NUM_HIDDEN_LAYERS_64 1 +#define NUM_FEATURES_64 19 +#define NUM_LAYER_0_UNITS_64 24 +#define NUM_LOGITS_64 10 + +static const float simple_motion_search_prune_part_logits_kernel_64[] = { + -2.71432f, 0.853994f, 0.740156f, -0.30982f, -0.448512f, 0.0984177f, + -0.664821f, -0.150188f, 0.104798f, 1.27746f, 0.217043f, -0.553914f, + -0.985639f, 1.99904f, -0.994877f, 0.0697683f, 0.64856f, -0.412276f, + -0.740292f, 0.93851f, 0.647461f, 0.874707f, -1.06881f, 0.704589f, + 0.641148f, -0.23997f, -0.0955951f, 0.249869f, -0.240726f, 0.420737f, + -0.558423f, -0.863005f, 0.981276f, -0.520796f, 0.206416f, -0.0512005f, + -0.76665f, -0.341038f, -0.727661f, 0.348269f, -0.0786858f, -0.370214f, + -0.292226f, -0.106107f, 0.262638f, 1.0065f, 0.558744f, -0.0492801f, + -1.22877f, 0.225909f, 0.485238f, -0.009513f, -0.299224f, -0.405938f, + -0.379131f, 0.0190173f, -0.66387f, 0.846164f, 0.298606f, -0.811507f, + -0.0853147f, -0.146277f, 0.0860954f, 0.151524f, 0.239613f, -0.0141985f, + 0.684708f, -0.0732401f, -0.709585f, -1.86211f, -0.622393f, -0.503456f, + 0.482376f, -0.10075f, -0.449184f, 0.999368f, 0.442391f, -0.60335f, + 0.631041f, 0.377315f, -0.141438f, -0.395021f, -0.750439f, 0.79456f, + 0.29555f, -1.51727f, 0.964882f, -0.256708f, -0.182677f, 0.727967f, + 0.411337f, -0.252901f, -0.546165f, -0.620733f, 0.948828f, 0.187498f, + -6.20879f, 0.1671f, -0.131667f, -0.0719152f, -0.101839f, -0.326827f, + 0.00418824f, -0.819253f, -0.168854f, -0.588437f, -0.192699f, 0.126879f, + -0.0287044f, -0.686708f, -0.445826f, -0.0993483f, -0.682881f, 0.0685043f, + -0.472371f, -0.399788f, 0.170985f, -0.164692f, -0.380181f, -0.268889f, + -3.16193f, -0.965943f, -0.178737f, -0.517274f, -0.102274f, -0.359742f, + 0.0117419f, -0.0690246f, 0.206539f, -1.29449f, 0.0976349f, -0.188444f, + 0.160939f, -0.387867f, -0.34513f, -0.221073f, -0.326615f, 0.0302811f, + 0.245349f, -0.381586f, 0.0414228f, -0.414076f, 0.0232891f, -0.118837f, + -2.81659f, -0.288328f, -0.0328096f, -0.61535f, -0.23593f, -1.16019f, + -0.0619081f, 0.0646133f, -0.313036f, -0.390373f, -0.0481636f, 0.0450287f, + 0.196415f, -0.717419f, -0.0707908f, -0.2078f, -0.459761f, 0.230835f, + -0.0703036f, -0.112402f, -0.170908f, 0.0485359f, -0.237293f, -0.989058f, + -2.05295f, 0.134268f, -0.173105f, -0.493955f, -0.28408f, -0.91465f, + -0.00206022f, -0.297255f, 0.161636f, -0.52414f, -0.324677f, -0.321039f, + 0.188412f, -0.333399f, -0.325067f, -0.024484f, 0.128572f, 0.20541f, + -0.0743476f, 0.0762302f, -0.0370672f, -1.53757f, -0.199054f, -0.318512f, + -3.41239f, -0.50942f, 0.100088f, 0.395587f, 0.00633016f, 0.851201f, + -0.0700995f, -0.685436f, 0.0817234f, -0.632139f, -0.41931f, 0.15613f, + -0.201166f, -0.0721472f, -0.239791f, 0.656797f, -0.501886f, -0.160969f, + -0.561536f, -0.491154f, -0.213447f, 0.21926f, -0.542261f, 0.594975f, + -1.75303f, 0.284615f, -0.0128245f, -0.0316972f, -0.300138f, -0.980628f, + -0.332859f, 0.19298f, -0.248332f, 0.445278f, -0.470188f, -0.483686f, + 0.361407f, -0.363551f, 0.175335f, -0.401025f, 0.494573f, 0.366983f, + 0.302351f, -0.368503f, -0.443071f, -0.562775f, -0.375816f, -0.657459f +}; + +static const float simple_motion_search_prune_part_layer_0_kernel_64[] = { + 0.615272f, -0.151592f, 0.346172f, -0.0656032f, 0.212875f, + 0.0798838f, -0.146093f, -0.230484f, -0.620322f, 0.0277852f, + -0.400931f, -0.777044f, -0.0871401f, -0.248128f, 0.471518f, + -0.34174f, -0.149884f, -0.0763207f, 0.900372f, 0.233323f, + -0.788699f, 0.28221f, 0.327214f, -0.244877f, -0.414568f, + 0.0472988f, 0.168193f, 0.64126f, 0.56686f, -0.318429f, + 0.530612f, -0.750709f, 0.0400333f, -0.023951f, -0.0218727f, + -0.0449836f, -0.0764245f, -0.070088f, -0.0427561f, -0.0873884f, + -0.216423f, 0.622787f, 0.0297492f, -0.233673f, 0.510639f, + 0.481654f, -0.234274f, 0.347734f, -0.480302f, 0.832741f, + 0.228223f, 0.247019f, -0.32416f, 0.352315f, -1.1491f, + 0.00806435f, -0.439654f, 0.151723f, -0.0719021f, 0.463499f, + 0.00134313f, 0.0553619f, -0.582163f, -0.215118f, -0.291413f, + 0.889738f, 0.284575f, 0.00450409f, -0.896799f, 0.0303598f, + 0.187935f, -0.160993f, 0.089854f, 0.0450689f, -0.548899f, + -0.10556f, 0.29384f, -0.154158f, 0.423591f, 0.322065f, + 0.244633f, -0.320866f, 0.739022f, -0.500495f, 0.595557f, + -0.62001f, 0.259688f, 0.114985f, -0.13888f, -0.0737616f, + 0.801863f, -0.0606944f, 0.238712f, -0.0636556f, 0.04317f, + 0.359843f, -0.273265f, 0.782092f, 0.341857f, 0.884373f, + 0.378388f, 0.632049f, 0.633597f, -0.32103f, -1.1014f, + -0.852525f, -0.867999f, 0.228625f, 0.444601f, 0.246595f, + 0.386288f, -0.341772f, -0.335849f, -0.482717f, 0.00928882f, + -0.262618f, 0.497003f, 0.579316f, 0.371681f, -0.440324f, + 0.115786f, -0.112295f, 0.07564f, -0.326419f, -0.195152f, + 0.315485f, -0.368989f, 0.281852f, 0.418865f, 0.807047f, + 0.293759f, -0.129749f, -1.26096f, -0.0526466f, -0.618876f, + 0.330563f, 0.26479f, 0.364751f, 0.68361f, 0.500237f, + -0.564011f, 0.158277f, -0.214105f, 0.50558f, 0.69422f, + -0.172224f, -0.280568f, 0.17241f, -0.500658f, -0.00418558f, + -0.420356f, 0.0453594f, 0.00471851f, -0.273979f, 0.355633f, + 0.515393f, 0.0699701f, -0.413576f, 0.487255f, -0.388845f, + -0.192441f, -0.353274f, -0.1773f, 0.256746f, -0.380197f, + 0.023061f, 0.431125f, 1.0368f, -0.0915799f, -0.268898f, + 0.0408016f, 0.593356f, 0.308486f, 0.508452f, -0.0623963f, + 0.307066f, 0.180916f, 0.326197f, 0.246641f, 0.270665f, + -0.145655f, -0.341277f, 0.0218018f, -0.326939f, -0.203528f, + -0.25496f, -0.772783f, -0.306114f, -0.79778f, 0.690016f, + -0.183109f, -0.076211f, -0.474873f, -0.0219729f, -0.152253f, + -0.0315833f, -0.134908f, -0.111871f, 0.0409385f, -0.282586f, + -0.101171f, 0.351942f, 0.30023f, 0.369243f, 0.0619639f, + -0.170842f, -0.235501f, -0.615927f, 0.883574f, 0.150949f, + -0.338489f, -0.213431f, -0.6075f, -0.107739f, -0.251963f, + 0.143906f, 0.254205f, -0.260084f, 0.224897f, 0.609902f, + 0.5606f, 0.795273f, 0.117204f, 0.345592f, 0.243912f, + 0.0626369f, -0.166448f, -0.886364f, -0.280833f, -0.0891074f, + 0.0372486f, -0.138729f, 0.120189f, 0.345857f, 0.176341f, + 0.481862f, -0.137464f, -0.208001f, 0.621677f, -0.121462f, + 0.00445475f, 0.159351f, 0.149918f, 0.198679f, -0.489814f, + -0.30931f, -0.39161f, 0.268615f, 0.323864f, 0.770061f, + 0.27962f, 0.536173f, 0.122727f, 0.571712f, 0.48351f, + -0.13558f, -0.426532f, -0.100426f, -0.209695f, 0.470483f, + -0.424789f, 0.0973791f, -0.296483f, -0.302057f, -0.918591f, + 0.16328f, 0.0335916f, 0.415762f, -0.235873f, -0.00934998f, + -0.0967718f, 0.616418f, 0.0931619f, -0.447975f, 0.156388f, + -0.40139f, 0.382807f, -0.578455f, 1.01437f, 0.37469f, + -0.986411f, -0.827175f, 0.0344976f, -0.00244486f, -0.142117f, + -0.344406f, -0.0124787f, -0.226427f, -0.301835f, -0.705097f, + 0.214542f, -0.0346878f, -0.255193f, 0.552685f, -0.104125f, + 0.77408f, 0.00984395f, 0.0476041f, -0.244548f, -0.35082f, + -0.487322f, 0.700091f, -0.138812f, -0.0110718f, -0.0400376f, + 0.254488f, -0.0867923f, -0.215463f, 1.03355f, 0.214947f, + 0.385536f, 0.140597f, -0.234973f, -0.0611124f, 1.0974f, + 0.463592f, -0.393788f, -0.428132f, -0.14664f, -0.156533f, + -0.113426f, -0.111081f, -0.0228256f, 0.354909f, -0.0232978f, + 0.177394f, 0.0503901f, -0.220314f, 0.170207f, 0.236719f, + 0.681525f, -0.245544f, -0.119278f, -0.258491f, -0.285309f, + 0.186384f, -0.119177f, -0.279398f, -0.097341f, -0.324634f, + -0.296797f, -1.3325f, 0.147225f, 0.0158408f, -0.0365257f, + -0.166737f, -0.159859f, -0.0765398f, 0.0973026f, 0.294529f, + -0.16425f, 0.241506f, 0.561487f, 0.799885f, -0.565667f, + -0.254387f, -0.360831f, -0.138432f, 0.406999f, 0.463259f, + -0.497314f, -0.00549591f, -0.0313973f, -0.386042f, -0.364537f, + -0.079987f, 0.148503f, 0.0532473f, -0.0196838f, 0.232687f, + 0.212998f, 0.2142f, 0.409001f, -0.104871f, -0.217793f, + 0.0161162f, -0.371663f, 0.887427f, 0.420187f, -0.649711f, + -0.398999f, 0.448972f, -0.116108f, 0.0444676f, -0.501289f, + -0.0410878f, 0.00447241f, -0.402102f, -0.0926268f, -0.254362f, + -0.395526f, 0.165273f, 0.339347f, 0.91932f, -0.280113f, + 0.320112f, -0.102728f, -0.131682f, -0.566621f, 0.386611f, + 0.146975f, 0.333248f, -0.179748f, 0.655328f, 0.222658f, + 0.349512f, -0.135588f, -0.22506f, -0.353001f, -0.400244f, + -0.815125f, 0.72407f, 0.281178f, -0.109426f, -0.0977383f, + -0.679406f, -0.401318f, 0.308941f, -0.594461f, 0.482289f, + -0.168467f, 0.76529f, -0.155056f, 0.0874018f, 0.266124f, + -0.0808725f, 0.078792f, -0.33271f, -1.0637f, -0.172125f, + 0.427025f, 0.332865f, 0.625523f, 0.264166f, -0.843882f, + -0.275777f, -0.657372f, 0.172814f, 0.0490389f, -0.561928f, + 0.0998399f, 0.106047f, 0.39061f, 0.422161f, 0.46369f, + -0.364271f, -0.326968f, -0.527243f, -0.618535f, -0.287804f, + -0.211365f, 0.349277f, 0.456535f, -0.0815191f, -0.156835f, + 0.57273f +}; + +static const float simple_motion_search_prune_part_logits_bias_64[] = { + 0.272852f, -0.260088f, 0.11881f, 0.146764f, 0.00289019f, + -0.559007f, -0.203668f, -0.268267f, -0.218921f, -0.260717f +}; + +static const float simple_motion_search_prune_part_layer_0_bias_64[] = { + -2.12804f, 0.445147f, -0.0210471f, 0.533662f, 0.662023f, 0.020519f, + -0.0452071f, -0.343995f, 0.112379f, -0.16776f, 0.112004f, 0.932147f, + -1.22944f, -0.822248f, 0.370083f, -0.0951491f, 0.0631258f, -0.875801f, + 0.996276f, 0.116103f, 0.925914f, 0.054293f, -1.02697f, -0.359516f +}; + +static const NN_CONFIG simple_motion_search_prune_part_nn_config_64 = { + NUM_FEATURES_64, + NUM_LOGITS_64, + NUM_HIDDEN_LAYERS_64, + { + NUM_LAYER_0_UNITS_64, + }, + { + simple_motion_search_prune_part_layer_0_kernel_64, + simple_motion_search_prune_part_logits_kernel_64, + }, + { + simple_motion_search_prune_part_layer_0_bias_64, + simple_motion_search_prune_part_logits_bias_64, + }, +}; + +#undef NUM_HIDDEN_LAYERS_64 +#undef NUM_FEATURES_64 +#undef NUM_LAYER_0_UNITS_64 +#undef NUM_LOGITS_64 + +// BLOCK_32X32 +#define NUM_HIDDEN_LAYERS_32 1 +#define NUM_FEATURES_32 19 +#define NUM_LAYER_0_UNITS_32 24 +#define NUM_LOGITS_32 10 + +static const float simple_motion_search_prune_part_logits_kernel_32[] = { + -1.22965f, 0.49978f, -0.376271f, 0.0615445f, -0.212491f, 0.58427f, + 0.142683f, 2.62142f, -0.459751f, -0.570143f, 0.72257f, -0.64127f, + 0.266919f, 0.3235f, 0.64405f, -0.0398269f, -1.82822f, -0.366573f, + 0.753387f, -0.274241f, -0.484971f, -0.503907f, -0.911176f, -0.455827f, + -0.706345f, 0.0687611f, -0.362788f, -0.369416f, -0.530408f, 0.0812615f, + -0.863012f, -0.613333f, -0.268833f, 0.0982181f, 0.19374f, -0.147078f, + -0.0200551f, 0.363817f, 0.179229f, 0.429671f, 0.905826f, -0.441191f, + 0.48874f, 0.539192f, -0.418992f, -0.468762f, -0.482158f, -0.770778f, + -0.743739f, 0.0175955f, -0.40662f, 0.0276107f, 0.0110658f, -0.119298f, + 0.698777f, -0.187602f, -0.300483f, -0.147972f, 0.324042f, -0.0265165f, + -0.375496f, 0.245762f, -0.111271f, -0.829879f, -0.434606f, -0.367099f, + 0.113229f, -0.755101f, -0.0623393f, -0.13764f, -0.205864f, -0.332027f, + 0.966638f, 0.477078f, 0.262501f, -0.110232f, 0.141242f, -0.699602f, + -0.14943f, -2.36868f, 0.556236f, 0.345883f, -0.466765f, 0.279446f, + -0.56809f, -0.356351f, -0.503855f, -0.607959f, -0.0328514f, 0.313352f, + -0.644447f, 0.221808f, 0.254575f, 0.476802f, 1.40433f, 0.317169f, + 0.109761f, -0.0105197f, -0.22896f, -0.389234f, 0.423948f, -0.272114f, + -1.02704f, -1.50706f, 0.0563541f, -0.0371609f, -0.306187f, 0.0570354f, + -0.367998f, 0.0949103f, -0.254703f, -0.0853516f, 1.56471f, -0.594372f, + -0.23582f, 0.267108f, 0.015297f, -0.318317f, 0.287599f, -0.687022f, + 0.360804f, -0.177637f, -0.118144f, -0.418225f, -0.985658f, -0.391334f, + -0.772539f, -0.44002f, 0.0722112f, -0.0825626f, 0.0466844f, 0.0525818f, + -0.788047f, 0.120401f, -0.785688f, -0.346177f, 0.240363f, 0.0248826f, + -0.0815654f, 0.39317f, -0.122372f, 0.199969f, 0.202777f, -0.1979f, + 0.313558f, -0.180202f, 0.191204f, -0.313598f, 0.155464f, -0.600106f, + 0.0230461f, -0.718126f, 0.266874f, -0.313614f, 0.0352148f, 0.131058f, + -0.221112f, 0.115419f, -1.41928f, -0.909453f, 0.341747f, -0.25318f, + -0.312476f, -0.288919f, -0.0135884f, -0.278496f, 0.381449f, -0.0725586f, + 0.254479f, -0.0658647f, -0.694657f, -0.0653643f, -0.0233378f, -0.484902f, + -0.104919f, -1.52446f, 0.0140919f, -0.142396f, -0.11836f, 0.113235f, + -1.21564f, 0.0618471f, 0.0421833f, -0.883143f, 0.288008f, -0.278322f, + -0.185151f, -0.616223f, 0.334549f, 0.370898f, 0.24646f, -0.027439f, + 0.366905f, -0.40628f, 0.0260464f, -0.506151f, 0.0191214f, -0.104617f, + -0.848722f, -1.66357f, -0.194607f, 0.0346545f, -0.412502f, 0.200471f, + -0.285588f, 0.0309609f, -0.150546f, 0.569096f, 0.210382f, 0.14322f, + 0.908182f, 1.23553f, -0.432935f, -0.0963189f, -0.610778f, -0.532583f, + -0.302028f, -0.600906f, -0.0777476f, 0.0230625f, 0.375802f, -0.466292f, + 1.01104f, -1.16706f, 0.0121855f, 0.0100479f, -0.147181f, 0.255058f, + -0.285898f, -0.167804f, -0.240171f, -1.15287f, -1.51181f, 0.0859809f, + 0.0811324f, -0.829305f, 0.272793f, 0.407484f, 0.324943f, 0.230197f +}; + +static const float simple_motion_search_prune_part_layer_0_kernel_32[] = { + -0.0711848f, 0.187893f, -0.0532872f, -0.0403218f, 0.132223f, + 0.0150309f, 0.132917f, -0.00883543f, -0.0614934f, 0.207838f, + -0.102772f, 0.127833f, -0.332145f, 0.00237219f, -0.338402f, + 0.123288f, -0.248341f, -0.0182967f, -1.16283f, -0.166611f, + -0.220962f, -0.268939f, -0.207455f, -0.214904f, -0.0639896f, + -0.0729676f, -0.000552725f, 0.217902f, 0.350537f, -0.0217492f, + 0.136239f, 0.0641448f, 0.515814f, 0.525507f, 0.240539f, + 0.109375f, 0.322722f, -1.20791f, 0.323287f, 0.0231209f, + -0.194629f, 0.143504f, 0.193943f, -0.15059f, -0.0156431f, + -0.210354f, 0.13426f, -0.379795f, 0.0585494f, 0.0947257f, + 0.948153f, 0.492681f, -0.977907f, -0.30367f, 0.216922f, + -0.366185f, 0.00734769f, 0.528627f, 0.334327f, 0.0222445f, + 0.400606f, -0.0313966f, -0.047664f, -0.512993f, -0.433836f, + 0.340769f, -0.131347f, -0.518235f, -0.220933f, -0.379338f, + -0.116978f, 0.143766f, 0.456281f, -0.0921022f, -0.0109533f, + -0.107467f, 0.335672f, -0.313228f, 0.175586f, 0.108769f, + 0.129646f, -0.053502f, 0.272574f, 0.220172f, 1.51046f, + 0.255032f, -0.162058f, -0.770721f, 0.238623f, -0.65159f, + -0.0354393f, -0.577323f, 0.0954972f, -0.661719f, 0.227893f, + -0.0584787f, 0.234849f, -0.16515f, -0.247194f, 0.12563f, + 0.39043f, -0.178219f, -0.0629665f, 0.395761f, 0.24224f, + -0.336032f, 0.552893f, -0.300073f, -0.235499f, -0.33557f, + 0.0398952f, -0.853526f, -0.550247f, 1.13965f, 0.197325f, + -0.0750203f, 0.145096f, 0.128642f, 0.113606f, 0.0972738f, + 0.219874f, 0.252284f, 0.833398f, 0.398332f, 0.802753f, + 0.650573f, -0.725374f, -0.548798f, -0.668723f, -0.705524f, + -0.929613f, -0.138145f, 0.161306f, 0.578878f, 0.0605056f, + 0.720406f, 0.163571f, 0.707962f, -0.0777347f, 0.52085f, + 0.279096f, 0.180012f, -0.459535f, 0.219678f, -0.273058f, + -0.030387f, -0.262664f, 0.367791f, -0.899145f, -1.07191f, + -1.15497f, 0.460442f, 0.020009f, -0.329794f, 0.846777f, + 0.287855f, 0.603544f, 0.645424f, 0.00753003f, -0.0666031f, + -0.141114f, -0.149588f, -0.431897f, -0.591811f, 0.134026f, + 0.0933469f, -0.130808f, -0.0335611f, -0.488446f, -0.275566f, + -0.937488f, 0.245793f, -0.548915f, 0.0108641f, -0.216292f, + 0.00680863f, -0.113137f, -0.137064f, 0.083321f, 0.689742f, + -0.213193f, 0.172408f, -0.123794f, 0.383401f, -0.496176f, + 0.427836f, -0.644447f, 0.967995f, 0.181371f, 0.301512f, + -0.376098f, -0.0571203f, 0.0012891f, 0.467542f, -0.400791f, + -0.204521f, 0.253847f, 0.107368f, -0.434741f, -0.389665f, + 0.30953f, 0.053949f, -0.0415711f, 0.0432576f, -0.319656f, + -0.228939f, -0.0928546f, 0.164408f, 1.26655f, 0.991928f, + -0.0596439f, 0.0786943f, 0.001457f, 0.282831f, 0.0975484f, + 0.13724f, 0.0891527f, -0.0471925f, 0.255467f, -0.469073f, + -0.0759757f, 0.331197f, 0.139434f, -0.0118866f, 0.359463f, + -0.774947f, -0.474647f, -1.36563f, 0.24901f, 0.776025f, + 0.150621f, 0.192407f, 0.752499f, 0.0435322f, -0.0334263f, + 0.470741f, -0.781907f, -0.149471f, -0.649484f, -0.866079f, + 0.536967f, 0.541611f, -0.00237101f, -0.119688f, -0.594591f, + -0.967503f, 0.545025f, -0.335037f, -0.172105f, -0.532514f, + -0.442211f, -0.0864824f, 0.038418f, -0.337492f, -0.412653f, + 0.0778177f, 0.0824839f, -0.232478f, 0.553077f, 0.179073f, + 0.222848f, -0.261019f, 0.854277f, -0.643185f, 0.476021f, + -0.0174448f, 0.621669f, -0.527978f, 0.45279f, -0.286967f, + 0.328291f, 0.0627781f, 0.495794f, 0.0156424f, -0.170515f, + -0.435815f, -0.264061f, -0.964473f, 0.259361f, -0.297514f, + 0.667044f, 0.719631f, -0.812974f, -0.350331f, 0.594795f, + -0.106071f, -0.409225f, 0.283481f, 0.5735f, -0.231666f, + -0.270059f, 0.687725f, 0.349693f, 0.0637911f, 0.208738f, + 0.299087f, -0.249188f, -0.426365f, -1.01906f, 0.0919539f, + -0.198362f, 0.256663f, -0.302723f, 0.457283f, 0.433322f, + 0.242223f, 0.740515f, 0.247248f, 0.436274f, 0.588906f, + 0.852493f, 0.385838f, 0.243232f, -0.501191f, 0.212121f, + 0.297253f, -0.664029f, -0.470196f, 0.0175681f, -1.01849f, + -0.619865f, -1.26982f, -0.088134f, -0.29233f, -0.196112f, + -0.141886f, 0.0602684f, 0.469327f, 0.0227135f, 0.275863f, + 0.161156f, -0.864189f, -0.72576f, 0.62214f, 0.804845f, + 0.183776f, -0.155777f, -0.0826676f, -0.23302f, 0.0845181f, + -0.126365f, 0.0925764f, 0.889037f, -0.208894f, 0.441273f, + 0.476529f, 0.510144f, 0.321464f, 0.764386f, 0.198938f, + -0.669809f, -0.13214f, -0.426596f, -0.547804f, -0.0953238f, + 0.164997f, 0.275626f, -0.0469242f, -0.923263f, -1.05855f, + -0.487197f, 0.261922f, -0.108634f, 0.409852f, 0.142348f, + 0.153656f, -0.119714f, 0.490507f, -0.26725f, -0.0651784f, + -0.962803f, -0.596098f, -0.686329f, 0.66723f, 0.290971f, + 0.692586f, 0.56926f, -0.0494205f, -0.498585f, -0.1832f, + -0.571507f, -0.438308f, 0.607958f, 0.985154f, 0.131991f, + 0.0469712f, 0.415769f, -0.518799f, -0.353338f, -0.219197f, + -0.159141f, 0.085484f, -0.405673f, -0.222816f, 0.813276f, + 0.323344f, -0.235178f, -0.40799f, -0.083177f, 0.514834f, + 0.177916f, 0.0197744f, -0.523574f, -0.445548f, -0.21085f, + 0.760991f, 0.170654f, -0.00312635f, -0.175464f, 0.665364f, + -0.713881f, -0.701069f, -0.628863f, 1.03196f, -0.0414137f, + -0.0163186f, 0.16796f, -0.0263498f, 0.484873f, 0.153922f, + 0.0593931f, 0.171615f, 0.330269f, -0.0728948f, 0.42332f, + 0.294273f, -0.0185935f, -0.609248f, -0.305841f, -0.293534f, + 0.0766015f, 0.130506f, 0.16716f, 0.0168997f, -0.429074f, + -0.744995f, -0.972633f, 0.50621f, -0.275645f, 0.402252f, + -0.0190824f, 0.637343f, -0.410791f, 0.546844f, -0.555088f, + 0.491182f, -0.0369695f, 0.974657f, -0.142378f, -0.148588f, + -0.217919f, 0.0146815f, -0.419912f, 0.111356f, -0.640505f, + -0.352691f +}; + +static const float simple_motion_search_prune_part_logits_bias_32[] = { + 0.0868818f, 0.370904f, 0.126136f, 0.57768f, -0.316245f, + -0.526958f, -0.451775f, -0.595569f, -0.805166f, -0.592605f +}; + +static const float simple_motion_search_prune_part_layer_0_bias_32[] = { + -1.85942f, 0.727049f, -0.0421259f, 0.759194f, 0.241057f, 0.0752973f, + 0.321934f, -0.593116f, -0.873202f, -0.238779f, 0.851339f, -0.536633f, + -0.18552f, -0.0941714f, 0.0153855f, -1.34736f, -0.870722f, 0.0973251f, + 0.491483f, 0.650638f, -0.205986f, -0.0499654f, -1.00592f, 0.65992f +}; + +static const NN_CONFIG simple_motion_search_prune_part_nn_config_32 = { + NUM_FEATURES_32, + NUM_LOGITS_32, + NUM_HIDDEN_LAYERS_32, + { + NUM_LAYER_0_UNITS_32, + }, + { + simple_motion_search_prune_part_layer_0_kernel_32, + simple_motion_search_prune_part_logits_kernel_32, + }, + { + simple_motion_search_prune_part_layer_0_bias_32, + simple_motion_search_prune_part_logits_bias_32, + }, +}; + +#undef NUM_HIDDEN_LAYERS_32 +#undef NUM_FEATURES_32 +#undef NUM_LAYER_0_UNITS_32 +#undef NUM_LOGITS_32 + +// BLOCK_16X16 +#define NUM_HIDDEN_LAYERS_16 1 +#define NUM_FEATURES_16 19 +#define NUM_LAYER_0_UNITS_16 16 +#define NUM_LOGITS_16 10 + +static const float simple_motion_search_prune_part_logits_kernel_16[] = { + 0.274123f, -0.551439f, 0.578938f, -0.213001f, -0.310021f, + 0.0301311f, -0.377388f, -3.16339f, 0.434616f, 0.264216f, + 0.320385f, -1.41657f, 1.05919f, -0.396075f, -0.855458f, + 0.418105f, -0.0769093f, -0.54773f, -0.215041f, -0.13885f, + -0.209062f, -0.526914f, 0.211082f, -2.34741f, -0.000587922f, + -0.431294f, 0.757511f, -0.184199f, -0.623552f, -0.221362f, + -0.628871f, 0.646393f, -0.192193f, 0.0166809f, -0.061691f, + -0.214283f, -0.35264f, 0.361452f, -0.0160958f, -0.0475499f, + 0.196822f, 0.44674f, -0.408064f, -0.559148f, -0.67489f, + -0.202201f, -0.358012f, -0.341735f, -0.325682f, 0.220407f, + -0.642416f, 0.350069f, 0.430505f, -0.0997745f, -0.300435f, + 1.25364f, -0.869717f, -0.0728051f, -0.172433f, 0.385328f, + -2.13737f, 0.673629f, 0.580102f, -0.0745359f, -0.442542f, + -0.208393f, -0.44303f, 0.318855f, 0.0557663f, 0.268257f, + -0.0941132f, -4.52876f, -0.638897f, -0.907808f, 0.28328f, + 0.619328f, -1.64249f, 0.0241431f, -0.0880383f, 0.299287f, + -0.248316f, 0.039253f, -0.544645f, -0.141389f, 0.0185283f, + -1.01973f, -0.269349f, -4.88803f, -0.632892f, 0.181151f, + 0.193291f, 0.592713f, -1.71466f, 0.398923f, 0.00355734f, + 0.212158f, -0.949188f, 0.386679f, -0.243317f, -0.17458f, + 0.0576348f, 0.096034f, -0.282317f, -4.13556f, -0.516689f, + 0.169427f, -0.541422f, 0.529281f, -1.45161f, -0.0417835f, + 0.0174299f, -0.26404f, -0.105773f, 0.102701f, -0.65039f, + 0.308121f, -0.185395f, 0.273611f, -0.408464f, -6.09866f, + -0.512249f, 0.0668662f, -0.547454f, 0.62565f, -1.63952f, + 0.510116f, 0.127461f, -0.506263f, 0.113016f, -0.258605f, + -0.308066f, 0.0957733f, 0.059395f, -0.915338f, 0.499407f, + -10.3199f, -0.32224f, -0.677307f, 1.13218f, -0.0888689f, + -0.108029f, -0.156533f, -0.391431f, 0.576607f, -0.0833897f, + 0.536752f, -0.0832957f, -0.0957492f, -0.135149f, 0.423881f, + 0.317691f, -7.29755f, 0.104775f, 0.534134f, -0.797288f, + -0.386335f, -0.20785f, -0.0507494f, -0.338127f, -0.868801f +}; + +static const float simple_motion_search_prune_part_layer_0_kernel_16[] = { + -0.244782f, 0.00425059f, -0.337534f, -0.423125f, -0.349549f, + -0.00261207f, -0.0845866f, -0.259278f, -0.375872f, 0.00052185f, + -0.427483f, 0.175625f, -0.357737f, 0.129946f, 0.97588f, + 1.19281f, -0.51292f, -0.100629f, 1.04044f, 0.197375f, + -0.210977f, -0.20334f, 0.173408f, 0.431999f, 0.083979f, + -0.334416f, -0.171408f, 0.160035f, -0.0464933f, 0.164574f, + -0.162781f, 1.01648f, 0.624844f, -1.20372f, -0.698742f, + 0.335747f, -0.362217f, -0.0817772f, -0.0366522f, -0.249627f, + -0.3135f, 0.256151f, -0.257869f, 0.0816391f, -0.256592f, + 0.071219f, -0.295956f, 0.287213f, -0.375107f, -0.149275f, + 0.0449292f, 0.49489f, -0.958944f, 0.386166f, -0.76147f, + -0.223786f, 1.91139f, -0.0866129f, -0.516556f, 0.871813f, + 0.389086f, 0.3205f, 0.605997f, -0.531384f, -0.693121f, + 0.728219f, 0.223733f, -0.58534f, -0.410249f, 0.0271334f, + -0.43904f, 0.325998f, -0.123988f, 0.103988f, -0.2227f, + 0.0779071f, 0.100073f, -0.924956f, -0.00472742f, -0.21932f, + -0.220187f, -0.719278f, 0.099118f, -0.0891471f, 0.743678f, + -0.173308f, 0.588225f, -0.31524f, 1.1415f, 0.408762f, + -0.213088f, -0.291621f, 0.655754f, 0.531725f, -0.013178f, + -0.100753f, -0.0775942f, -0.181806f, -0.441297f, -0.00589338f, + 0.14217f, -0.0592671f, 0.10907f, 1.47619f, 0.813817f, + 0.1996f, 0.178263f, -0.569958f, -0.374858f, -0.743922f, + -0.269343f, -0.118752f, 0.0797114f, 0.027829f, -0.133402f, + -0.164884f, -0.42919f, 0.0682661f, -0.206438f, 0.0184534f, + -0.141316f, -0.138817f, 0.0305671f, 0.295578f, -0.236413f, + 0.172918f, -0.35082f, -0.0986917f, -0.650484f, 0.13655f, + 0.212195f, 0.549501f, 1.64941f, 0.152883f, 0.0117464f, + -0.200688f, -0.0412075f, -0.0296324f, 0.122624f, 0.1871f, + -0.515693f, 0.20351f, 0.333617f, 0.254752f, 0.293802f, + -0.428904f, -0.0312053f, -0.879481f, -0.516353f, -0.417837f, + -0.230442f, 0.229367f, 0.615339f, 0.327347f, 0.288747f, + 0.596166f, 0.497054f, 0.536953f, 0.462426f, 0.517022f, + -0.287359f, -0.162824f, -0.249174f, -0.364887f, -0.538065f, + -0.506783f, -0.163213f, -0.617007f, -0.70433f, -0.759151f, + -0.235183f, -0.196496f, 0.0925943f, -0.0786896f, 0.135073f, + -0.106243f, -0.190647f, -0.311848f, -0.428009f, 0.467477f, + 0.259139f, 1.4948f, 0.986061f, -0.534924f, -0.193525f, + -0.66125f, -0.693799f, -0.489594f, 0.238264f, 0.249978f, + -0.365372f, -0.197998f, -0.664471f, -0.0312145f, -0.297293f, + 0.0278792f, -0.877716f, 0.0473106f, 0.913044f, 0.815569f, + 0.789972f, 0.641374f, -0.122848f, -0.353137f, -0.381102f, + -0.960138f, -0.166101f, 0.230364f, 0.534173f, 0.398882f, + 0.0599076f, 0.00176035f, 0.291062f, 0.021087f, 0.374562f, + 0.402516f, 0.144006f, -0.149598f, 0.065063f, -0.0774656f, + -0.18804f, -0.21123f, 0.0119723f, -0.244286f, 0.109912f, + -0.98136f, -0.583136f, -1.3897f, 0.33196f, 0.319226f, + 0.490015f, 0.316438f, 0.565049f, 0.163159f, 0.523771f, + 0.325115f, -0.0515304f, -0.0827355f, 0.112763f, -0.517608f, + -0.161398f, -0.124513f, -0.0384277f, -0.49491f, -1.53384f, + -1.3086f, 1.20601f, 0.399591f, 0.233774f, 0.0154149f, + -0.431196f, 0.187116f, 0.033398f, 0.655468f, -0.390814f, + 0.418144f, -0.572053f, 0.936493f, -0.116933f, -0.20803f, + -0.908834f, 1.2524f, -0.285888f, 0.227437f, -0.317727f, + -0.6331f, 0.159656f, 0.244652f, 0.169768f, 0.379279f, + 0.37089f, 0.118264f, 0.150946f, 0.536513f, -0.261398f, + -0.15329f, -0.413112f, -0.243792f, -0.215096f, 0.288741f, + -0.193763f, -0.130652f, -0.328159f, -0.719996f, -1.66213f, + 0.0815627f, 0.129643f, -0.0974464f, -0.0660102f, 0.349602f, + 0.120356f, -0.144144f, -0.13641f, -0.154818f, -0.468552f, + -0.442585f, -0.312206f, 0.879662f, 0.846103f, 0.0485413f, + 0.18676f, -0.477959f, -0.112404f, -0.366477f +}; + +static const float simple_motion_search_prune_part_logits_bias_16[] = { + 0.791815f, 0.888054f, 0.406071f, -0.217564f, -0.504713f, + -0.514339f, -0.200842f, -1.09239f, -1.78297f, -2.41754f +}; + +static const float simple_motion_search_prune_part_layer_0_bias_16[] = { + 0.89469f, 0.10871f, 0.664404f, -0.00639695f, -0.745647f, 0.910444f, + 0.168436f, -4.05854f, 1.00981f, 0.743352f, -0.73947f, -2.5654f, + -0.654753f, 0.602252f, -0.973004f, 1.1113f +}; + +static const NN_CONFIG simple_motion_search_prune_part_nn_config_16 = { + NUM_FEATURES_16, + NUM_LOGITS_16, + NUM_HIDDEN_LAYERS_16, + { + NUM_LAYER_0_UNITS_16, + }, + { + simple_motion_search_prune_part_layer_0_kernel_16, + simple_motion_search_prune_part_logits_kernel_16, + }, + { + simple_motion_search_prune_part_layer_0_bias_16, + simple_motion_search_prune_part_logits_bias_16, + }, +}; + +#undef NUM_HIDDEN_LAYERS_16 +#undef NUM_FEATURES_16 +#undef NUM_LAYER_0_UNITS_16 +#undef NUM_LOGITS_16 + +// BLOCK_8X8 +#define NUM_HIDDEN_LAYERS_8 1 +#define NUM_FEATURES_8 19 +#define NUM_LAYER_0_UNITS_8 24 +#define NUM_LOGITS_8 4 + +static const float simple_motion_search_prune_part_logits_kernel_8[] = { + -0.806717f, 0.0244927f, -0.0280725f, 0.210426f, 0.0820434f, -0.0222324f, + -0.0728984f, -0.424785f, -0.0568035f, -0.0744017f, 0.204282f, 0.43611f, + 0.319456f, 0.119903f, -0.333224f, -0.932499f, 0.105907f, 0.0905287f, + -0.612538f, 0.0978191f, -0.0858663f, 0.0468925f, -0.312509f, 0.0527469f, + 0.308466f, -0.139768f, -0.468023f, -0.069522f, 0.248949f, 0.0408582f, + 0.199935f, -0.448705f, 0.242315f, 0.00472276f, -0.468445f, -0.591226f, + 0.107975f, 0.261228f, -0.10724f, 0.104639f, -0.211234f, -0.304658f, + -0.261689f, -0.259529f, 0.0331369f, -0.468468f, -0.0555605f, -0.0495414f, + 0.247612f, -0.112518f, -0.393122f, 0.302172f, -0.252151f, 0.076605f, + -0.34155f, -0.182731f, 0.154079f, -0.464538f, -0.396327f, -0.654858f, + 0.0921198f, 0.202867f, -0.0712573f, 0.0349611f, 0.243043f, -0.177452f, + -0.205536f, -0.268024f, 0.0358315f, -0.560347f, -0.204207f, -0.0890628f, + -1.013f, -0.461748f, -0.429801f, -0.100352f, -0.34369f, 0.115318f, + -0.556285f, -0.0751575f, 0.183418f, -0.285223f, 0.197177f, -0.825216f, + -0.125709f, -0.100452f, 0.180481f, 0.764462f, 0.136609f, -0.660743f, + 0.212887f, -0.410408f, -0.330839f, -0.714646f, -0.115863f, -0.389509f +}; + +static const float simple_motion_search_prune_part_layer_0_kernel_8[] = { + 0.0561017f, 0.530692f, 0.129047f, 0.296138f, 0.407593f, + 0.151159f, 0.180472f, 0.298995f, -0.0265362f, -0.380313f, + -0.136716f, -0.0644648f, 0.00808564f, 0.0875876f, -0.0719683f, + -0.0996071f, -1.49722f, -1.31176f, 1.54169f, -0.50717f, + -0.0186767f, -0.525924f, 0.0402954f, -0.553913f, 0.139015f, + 0.171752f, -0.376497f, 0.485084f, 0.182424f, -0.0236465f, + 0.380048f, -0.185493f, 0.554435f, 0.241565f, -0.225917f, + -1.08637f, -0.305121f, 1.3603f, 0.0806333f, -0.0697466f, + 0.216192f, 0.0493694f, 0.0620039f, -0.0042489f, 0.0707283f, + 0.0998331f, -0.0252022f, 0.342077f, -0.200664f, 0.335261f, + -0.100772f, 0.212881f, -0.374258f, 0.310442f, -1.95981f, + -1.16615f, 1.88243f, 0.503536f, -0.0919037f, 0.392323f, + 0.33149f, 0.563121f, -0.0898854f, 0.130267f, 0.404959f, + 1.06767f, 0.75134f, 0.810919f, 0.865237f, -1.08001f, + -0.949809f, -1.24941f, -0.95081f, -0.539575f, -0.355324f, + -0.110592f, -0.311277f, 0.023022f, -0.039287f, 0.491298f, + -0.248853f, -0.294741f, 0.288811f, 0.470638f, 0.330094f, + 0.146603f, 0.0958486f, 0.308326f, -0.87134f, -1.06074f, + -0.315428f, 0.717577f, -0.0174599f, -0.491904f, 0.977014f, + 0.922693f, 0.0265707f, -0.743468f, -0.273647f, -0.430625f, + -0.473789f, 0.119983f, 0.476441f, 0.639803f, -0.0753475f, + -0.0746323f, -0.0422539f, 0.013454f, -0.0174412f, -0.253015f, + -0.279585f, 0.11562f, -0.396919f, 0.306388f, 0.430346f, + 0.275945f, -0.196508f, 0.00180146f, 0.396652f, 0.324022f, + -0.142132f, 0.1768f, -0.0746152f, 0.314189f, -0.031924f, + 0.254196f, -0.253449f, 0.1537f, -0.98635f, -0.850926f, + 0.0762147f, -0.652507f, 1.16528f, -0.122595f, -0.00561333f, + 0.424954f, -0.00258127f, -0.421957f, -0.0714059f, 0.782031f, + -0.0129133f, -0.00544328f, -0.193527f, -0.0936037f, -0.00490983f, + -0.852396f, -0.649662f, 0.820325f, 0.484966f, 0.265405f, + -0.33032f, 0.104917f, -0.364478f, 0.26858f, 0.068847f, + -0.0751046f, 0.238931f, -0.151554f, 0.0296385f, -0.14869f, + -0.173547f, -0.397285f, 0.0561029f, -0.0681254f, 0.222631f, + -0.00800993f, 0.0686761f, 0.196754f, 0.249358f, 0.348125f, + -2.22757f, 0.186675f, -0.0426998f, 0.123301f, -0.0196412f, + 0.104882f, 0.138482f, 0.340663f, 0.0684671f, -1.22841f, + -1.08539f, -1.1055f, -0.732678f, 0.766626f, 0.727742f, + 1.26906f, 0.812594f, -0.446312f, -0.166752f, 0.396628f, + 0.00141587f, 0.0137288f, -0.0920456f, 0.123027f, -0.118517f, + -0.0412321f, -0.124952f, 0.00875945f, 0.0149883f, -0.0218859f, + -0.233299f, -0.0812722f, 0.269917f, 0.0246593f, 0.0915655f, + -0.019122f, 0.706718f, -0.172051f, -1.85551f, 0.0578997f, + 0.100269f, 0.131412f, -0.0952751f, 0.181419f, -0.0106305f, + 0.106179f, -0.142344f, 0.0916649f, -0.41298f, 0.113846f, + -0.322459f, 0.0415754f, -0.2205f, -0.224231f, 0.147907f, + -0.135093f, -0.398757f, 1.93381f, 0.20456f, -0.205381f, + -0.263999f, 0.364463f, -0.424052f, -0.0166924f, -0.319968f, + 0.0439144f, -0.692733f, 0.678982f, -0.445194f, -0.527451f, + 0.383248f, 0.330927f, -0.0367725f, 0.572847f, -0.620804f, + -0.286471f, 1.39367f, 0.470732f, 0.0751676f, 0.637958f, + 0.574865f, 0.411091f, 0.263806f, 0.555064f, 0.047788f, + 0.201792f, -0.0821822f, -0.0226181f, -0.176679f, -0.612642f, + -0.992254f, -0.292099f, -0.562201f, -0.567948f, -0.767681f, + 1.33645f, 0.0932564f, -0.421675f, 0.198262f, -0.401979f, + 0.244609f, -0.261484f, 0.0304534f, -0.200004f, 0.00951232f, + -0.085364f, 0.555152f, -0.243778f, 0.371369f, 0.152997f, + 0.154443f, -0.281229f, 0.967176f, 1.0212f, 0.276169f, + 0.0962747f, 0.449737f, 0.300859f, 0.376659f, 0.165669f, + 0.332263f, 0.238315f, 0.376668f, -0.139238f, -0.0694765f, + -0.256137f, -0.262146f, -0.246265f, 0.0261978f, -0.118975f, + -0.207184f, -0.614041f, -0.866987f, -1.75604f, -0.04917f, + 0.0794673f, -0.355224f, 0.0035685f, -0.340216f, 0.398567f, + -0.314826f, -0.12464f, 0.693428f, 0.701697f, 0.511653f, + 0.861303f, -0.64449f, -1.06257f, -0.773944f, -0.838084f, + 0.802429f, -0.24573f, 0.720913f, -0.31002f, 0.615694f, + 0.287994f, 0.18495f, 0.0188525f, 0.374338f, -0.12243f, + 0.613532f, 0.132456f, 0.177285f, -0.187238f, 0.0872563f, + -0.425068f, 0.154711f, -0.24283f, -0.193267f, -1.14008f, + -1.17759f, 0.517845f, 0.458782f, -0.161137f, 0.142772f, + 0.2811f, 0.171871f, 0.276129f, 0.018275f, 0.390464f, + -0.156414f, 0.108248f, -0.0909085f, -0.11346f, 0.0815138f, + 0.0729377f, -0.0769677f, 0.0110251f, -0.46098f, -0.421538f, + -2.18003f, 0.6639f, 0.541278f, 0.704337f, 0.480695f, + 0.727474f, 0.473937f, 0.770863f, 0.41774f, -0.112943f, + -0.107412f, 0.083639f, -0.351127f, -0.149221f, 0.0705735f, + -0.0908294f, -0.0789572f, -1.62533f, -2.14051f, -0.188871f, + 0.796224f, 0.00144302f, 0.467569f, -0.113915f, 0.660595f, + 0.101025f, 0.020895f, 0.604505f, -0.26598f, -0.149282f, + -0.548818f, -0.140463f, 0.13669f, -0.0955724f, -0.694851f, + -0.13438f, -0.384164f, -0.600513f, 1.48183f, 0.16481f, + 0.340894f, -0.190813f, 0.487458f, 0.0769957f, 0.192901f, + 0.039483f, 0.213366f, -0.508968f, -0.494034f, -0.987906f, + -0.188502f, -0.907188f, 0.0109838f, -0.275008f, -0.331017f, + -0.447202f, 0.0521897f, 0.837215f, 0.344802f, 0.216526f, + 0.456105f, 0.139706f, -0.373955f, 0.0232853f, -0.365632f, + -0.0586727f, 0.828841f, 0.737974f, -0.933899f, -0.0620807f, + -0.262087f, -0.0551274f, -0.374378f, 0.118385f, -0.0375933f, + -0.39549f, -0.0844191f, 0.274343f, 0.21814f, 0.52751f, + 0.449546f, 0.075518f, 0.369501f, 0.345248f, 0.403972f, + -0.465168f, -0.551238f, -0.354305f, -0.233872f, 0.664137f, + -0.214848f, -0.0363864f, 0.446154f, -0.500073f, -1.07883f, + -1.07772f +}; + +static const float simple_motion_search_prune_part_logits_bias_8[] = { + 1.09425f, -0.345875f, -0.56098f, -1.84859f +}; + +static const float simple_motion_search_prune_part_layer_0_bias_8[] = { + -1.75332f, 0.419404f, -0.0423004f, 0.345601f, -2.40097f, -0.574857f, + -2.53366f, -0.207096f, -1.08866f, 0.801763f, -2.72428f, -2.39884f, + 1.04072f, -0.782749f, -1.47003f, -2.98207f, 0.8829f, 1.81741f, + -2.02369f, 0.241717f, -0.178749f, -3.00588f, -0.232007f, 0.21717f +}; + +static const NN_CONFIG simple_motion_search_prune_part_nn_config_8 = { + NUM_FEATURES_8, + NUM_LOGITS_8, + NUM_HIDDEN_LAYERS_8, + { + NUM_LAYER_0_UNITS_8, + }, + { + simple_motion_search_prune_part_layer_0_kernel_8, + simple_motion_search_prune_part_logits_kernel_8, + }, + { + simple_motion_search_prune_part_layer_0_bias_8, + simple_motion_search_prune_part_logits_bias_8, + }, +}; + +#undef NUM_HIDDEN_LAYERS_8 +#undef NUM_FEATURES_8 +#undef NUM_LAYER_0_UNITS_8 +#undef NUM_LOGITS_8 + +#define FEATURE_SIZE 19 +static const float two_pass_split_partition_weights_128[FEATURE_SIZE + 1] = { + 2.683936f, -0.193620f, -4.106470f, -0.141320f, -0.282289f, + 0.125296f, -1.134961f, 0.862757f, -0.418799f, -0.637666f, + 0.016232f, 0.345013f, 0.018823f, -0.393394f, -1.130700f, + 0.695357f, 0.112569f, -0.341975f, -0.513882f, 5.7488966f, +}; + +static const float two_pass_split_partition_weights_64[FEATURE_SIZE + 1] = { + 2.990993f, 0.423273f, -0.926544f, 0.454646f, -0.292698f, + -1.311632f, -0.284432f, 0.717141f, -0.419257f, -0.574760f, + -0.674444f, 0.669047f, -0.374255f, 0.380624f, -0.804036f, + 0.264021f, 0.004163f, 1.896802f, 0.924287f, 0.13490619f, +}; + +static const float two_pass_split_partition_weights_32[FEATURE_SIZE + 1] = { + 2.795181f, -0.136943f, -0.924842f, 0.405330f, -0.463505f, + -0.584076f, -0.831472f, 0.382985f, -0.597544f, -0.138915f, + -1.354350f, 0.466035f, -0.553961f, 0.213202f, -1.166429f, + 0.010776f, -0.096236f, 2.335084f, 1.699857f, -0.58178353f, +}; + +static const float two_pass_split_partition_weights_16[FEATURE_SIZE + 1] = { + 1.987888f, -0.431100f, -1.687703f, 0.262602f, -0.425298f, + -0.463870f, -1.493457f, 0.470917f, -0.528457f, -0.087700f, + -1.815092f, 0.152883f, -0.337908f, 0.093679f, -1.548267f, + -0.042387f, -0.000861f, 2.556746f, 1.619192f, 0.03643292f, +}; + +static const float two_pass_split_partition_weights_8[FEATURE_SIZE + 1] = { + 2.188344f, -0.817528f, -2.119219f, 0.000000f, -0.348167f, + -0.658074f, -1.960362f, 0.000000f, -0.403080f, 0.282699f, + -2.061088f, 0.000000f, -0.431919f, -0.127960f, -1.099550f, + 0.000000f, 0.121622f, 2.017455f, 2.058228f, -0.15475988f, +}; + +static const float two_pass_none_partition_weights_128[FEATURE_SIZE + 1] = { + -1.006689f, 0.777908f, 4.461072f, -0.395782f, -0.014610f, + -0.853863f, 0.729997f, -0.420477f, 0.282429f, -1.194595f, + 3.181220f, -0.511416f, 0.117084f, -1.149348f, 1.507990f, + -0.477212f, 0.202963f, -1.469581f, 0.624461f, -0.89081228f, +}; + +static const float two_pass_none_partition_weights_64[FEATURE_SIZE + 1] = { + -1.241117f, 0.844878f, 5.638803f, -0.489780f, -0.108796f, + -4.576821f, 1.540624f, -0.477519f, 0.227791f, -1.443968f, + 1.586911f, -0.505125f, 0.140764f, -0.464194f, 1.466658f, + -0.641166f, 0.195412f, 1.427905f, 2.080007f, -1.98272777f, +}; + +static const float two_pass_none_partition_weights_32[FEATURE_SIZE + 1] = { + -2.130825f, 0.476023f, 5.907343f, -0.516002f, -0.097471f, + -2.662754f, 0.614858f, -0.576728f, 0.085261f, -0.031901f, + 0.727842f, -0.600034f, 0.079326f, 0.324328f, 0.504502f, + -0.547105f, -0.037670f, 0.304995f, 0.369018f, -2.66299987f, +}; + +static const float two_pass_none_partition_weights_16[FEATURE_SIZE + 1] = { + -1.626410f, 0.872047f, 5.414965f, -0.554781f, -0.084514f, + -3.020550f, 0.467632f, -0.382280f, 0.199568f, 0.426220f, + 0.829426f, -0.467100f, 0.153098f, 0.662994f, 0.327545f, + -0.560106f, -0.141610f, 0.403372f, 0.523991f, -3.02891231f, +}; + +static const float two_pass_none_partition_weights_8[FEATURE_SIZE + 1] = { + -1.463349f, 0.375376f, 4.751430f, 0.000000f, -0.184451f, + -1.655447f, 0.443214f, 0.000000f, 0.127961f, 0.152435f, + 0.083288f, 0.000000f, 0.143105f, 0.438012f, 0.073238f, + 0.000000f, -0.278137f, 0.186134f, 0.073737f, -1.6494962f, +}; + #ifdef __cplusplus } // extern "C" #endif diff --git a/third_party/aom/av1/encoder/pickcdef.c b/third_party/aom/av1/encoder/pickcdef.c index 6d154a7d22..fb07056b84 100644 --- a/third_party/aom/av1/encoder/pickcdef.c +++ b/third_party/aom/av1/encoder/pickcdef.c @@ -281,6 +281,7 @@ uint64_t compute_cdef_dist(uint16_t *dst, int dstride, uint16_t *src, void av1_cdef_search(YV12_BUFFER_CONFIG *frame, const YV12_BUFFER_CONFIG *ref, AV1_COMMON *cm, MACROBLOCKD *xd, int fast) { + CdefInfo *const cdef_info = &cm->cdef_info; int r, c; int fbr, fbc; uint16_t *src[3]; @@ -475,23 +476,23 @@ void av1_cdef_search(YV12_BUFFER_CONFIG *frame, const YV12_BUFFER_CONFIG *ref, best_tot_mse = tot_mse; nb_strength_bits = i; for (j = 0; j < 1 << nb_strength_bits; j++) { - cm->cdef_strengths[j] = best_lev0[j]; - cm->cdef_uv_strengths[j] = best_lev1[j]; + cdef_info->cdef_strengths[j] = best_lev0[j]; + cdef_info->cdef_uv_strengths[j] = best_lev1[j]; } } } nb_strengths = 1 << nb_strength_bits; - cm->cdef_bits = nb_strength_bits; - cm->nb_cdef_strengths = nb_strengths; + cdef_info->cdef_bits = nb_strength_bits; + cdef_info->nb_cdef_strengths = nb_strengths; for (i = 0; i < sb_count; i++) { int gi; int best_gi; uint64_t best_mse = (uint64_t)1 << 63; best_gi = 0; - for (gi = 0; gi < cm->nb_cdef_strengths; gi++) { - uint64_t curr = mse[0][i][cm->cdef_strengths[gi]]; - if (num_planes >= 3) curr += mse[1][i][cm->cdef_uv_strengths[gi]]; + for (gi = 0; gi < cdef_info->nb_cdef_strengths; gi++) { + uint64_t curr = mse[0][i][cdef_info->cdef_strengths[gi]]; + if (num_planes >= 3) curr += mse[1][i][cdef_info->cdef_uv_strengths[gi]]; if (curr < best_mse) { best_gi = gi; best_mse = curr; @@ -503,18 +504,18 @@ void av1_cdef_search(YV12_BUFFER_CONFIG *frame, const YV12_BUFFER_CONFIG *ref, if (fast) { for (int j = 0; j < nb_strengths; j++) { - cm->cdef_strengths[j] = - priconv[cm->cdef_strengths[j] / CDEF_SEC_STRENGTHS] * + cdef_info->cdef_strengths[j] = + priconv[cm->cdef_info.cdef_strengths[j] / CDEF_SEC_STRENGTHS] * CDEF_SEC_STRENGTHS + - (cm->cdef_strengths[j] % CDEF_SEC_STRENGTHS); - cm->cdef_uv_strengths[j] = - priconv[cm->cdef_uv_strengths[j] / CDEF_SEC_STRENGTHS] * + (cdef_info->cdef_strengths[j] % CDEF_SEC_STRENGTHS); + cdef_info->cdef_uv_strengths[j] = + priconv[cdef_info->cdef_uv_strengths[j] / CDEF_SEC_STRENGTHS] * CDEF_SEC_STRENGTHS + - (cm->cdef_uv_strengths[j] % CDEF_SEC_STRENGTHS); + (cdef_info->cdef_uv_strengths[j] % CDEF_SEC_STRENGTHS); } } - cm->cdef_pri_damping = pri_damping; - cm->cdef_sec_damping = sec_damping; + cdef_info->cdef_pri_damping = pri_damping; + cdef_info->cdef_sec_damping = sec_damping; aom_free(mse[0]); aom_free(mse[1]); for (pli = 0; pli < num_planes; pli++) { diff --git a/third_party/aom/av1/encoder/picklpf.c b/third_party/aom/av1/encoder/picklpf.c index c5508e25c4..8aeec0c08a 100644 --- a/third_party/aom/av1/encoder/picklpf.c +++ b/third_party/aom/av1/encoder/picklpf.c @@ -67,26 +67,27 @@ static int64_t try_filter_frame(const YV12_BUFFER_CONFIG *sd, case 2: cm->lf.filter_level_v = filter_level[0]; break; } - // TODO(any): please enable multi-thread and remove the flag when loop - // filter mask is compatible with multi-thread. -#if LOOP_FILTER_BITMASK - av1_loop_filter_frame(cm->frame_to_show, cm, &cpi->td.mb.e_mbd, 0, plane, - plane + 1, partial_frame); -#else + // TODO(any): please enable multi-thread and remove the flag when loop + // filter mask is compatible with multi-thread. if (cpi->num_workers > 1) - av1_loop_filter_frame_mt(cm->frame_to_show, cm, &cpi->td.mb.e_mbd, plane, - plane + 1, partial_frame, cpi->workers, - cpi->num_workers, &cpi->lf_row_sync); - else - av1_loop_filter_frame(cm->frame_to_show, cm, &cpi->td.mb.e_mbd, plane, - plane + 1, partial_frame); + av1_loop_filter_frame_mt(&cm->cur_frame->buf, cm, &cpi->td.mb.e_mbd, plane, + plane + 1, partial_frame, +#if LOOP_FILTER_BITMASK + 0, #endif + cpi->workers, cpi->num_workers, &cpi->lf_row_sync); + else + av1_loop_filter_frame(&cm->cur_frame->buf, cm, &cpi->td.mb.e_mbd, +#if LOOP_FILTER_BITMASK + 0, +#endif + plane, plane + 1, partial_frame); - filt_err = aom_get_sse_plane(sd, cm->frame_to_show, plane, + filt_err = aom_get_sse_plane(sd, &cm->cur_frame->buf, plane, cm->seq_params.use_highbitdepth); // Re-instate the unfiltered frame - yv12_copy_plane(&cpi->last_frame_uf, cm->frame_to_show, plane); + yv12_copy_plane(&cpi->last_frame_uf, &cm->cur_frame->buf, plane); return filt_err; } @@ -107,7 +108,17 @@ static int search_filter_level(const YV12_BUFFER_CONFIG *sd, AV1_COMP *cpi, // range. int lvl; switch (plane) { - case 0: lvl = last_frame_filter_level[dir]; break; + case 0: + switch (dir) { + case 2: + lvl = (last_frame_filter_level[0] + last_frame_filter_level[1] + 1) >> + 1; + break; + case 0: + case 1: lvl = last_frame_filter_level[dir]; break; + default: assert(dir >= 0 && dir <= 2); return 0; + } + break; case 1: lvl = last_frame_filter_level[2]; break; case 2: lvl = last_frame_filter_level[3]; break; default: assert(plane >= 0 && plane <= 2); return 0; @@ -119,7 +130,7 @@ static int search_filter_level(const YV12_BUFFER_CONFIG *sd, AV1_COMP *cpi, // Set each entry to -1 memset(ss_err, 0xFF, sizeof(ss_err)); - yv12_copy_plane(cm->frame_to_show, &cpi->last_frame_uf, plane); + yv12_copy_plane(&cm->cur_frame->buf, &cpi->last_frame_uf, plane); best_err = try_filter_frame(sd, cpi, filt_mid, partial_frame, plane, dir); filt_best = filt_mid; ss_err[filt_mid] = best_err; @@ -212,7 +223,7 @@ void av1_pick_filter_level(const YV12_BUFFER_CONFIG *sd, AV1_COMP *cpi, int filt_guess; switch (cm->seq_params.bit_depth) { case AOM_BITS_8: - filt_guess = (cm->frame_type == KEY_FRAME) + filt_guess = (cm->current_frame.frame_type == KEY_FRAME) ? ROUND_POWER_OF_TWO(q * 17563 - 421574, 18) : ROUND_POWER_OF_TWO(q * 6017 + 650707, 18); break; @@ -228,7 +239,8 @@ void av1_pick_filter_level(const YV12_BUFFER_CONFIG *sd, AV1_COMP *cpi, "or AOM_BITS_12"); return; } - if (cm->seq_params.bit_depth != AOM_BITS_8 && cm->frame_type == KEY_FRAME) + if (cm->seq_params.bit_depth != AOM_BITS_8 && + cm->current_frame.frame_type == KEY_FRAME) filt_guess -= 4; // TODO(chengchen): retrain the model for Y, U, V filter levels lf->filter_level[0] = clamp(filt_guess, min_filter_level, max_filter_level); diff --git a/third_party/aom/av1/encoder/pickrst.c b/third_party/aom/av1/encoder/pickrst.c index e7804f6b44..39db3d2ea8 100644 --- a/third_party/aom/av1/encoder/pickrst.c +++ b/third_party/aom/av1/encoder/pickrst.c @@ -43,6 +43,9 @@ static const RestorationType force_restore_type = RESTORE_TYPES; // Penalty factor for use of dual sgr #define DUAL_SGR_PENALTY_MULT 0.01 +// Working precision for Wiener filter coefficients +#define WIENER_TAP_SCALE_FACTOR ((int64_t)1 << 16) + const int frame_level_restore_bits[RESTORE_TYPES] = { 2, 2, 2, 2 }; typedef int64_t (*sse_extractor_type)(const YV12_BUFFER_CONFIG *a, @@ -113,15 +116,11 @@ typedef struct { AV1PixelRect tile_rect; } RestSearchCtxt; -static void rsc_on_tile(int tile_row, int tile_col, void *priv) { - (void)tile_col; - +static void rsc_on_tile(void *priv) { RestSearchCtxt *rsc = (RestSearchCtxt *)priv; set_default_sgrproj(&rsc->sgrproj); set_default_wiener(&rsc->wiener); - - rsc->tile_stripe0 = - (tile_row == 0) ? 0 : rsc->cm->rst_end_stripe[tile_row - 1]; + rsc->tile_stripe0 = 0; } static void reset_rsc(RestSearchCtxt *rsc) { @@ -141,7 +140,7 @@ static void init_rsc(const YV12_BUFFER_CONFIG *src, const AV1_COMMON *cm, rsc->rusi = rusi; rsc->sf = sf; - const YV12_BUFFER_CONFIG *dgd = cm->frame_to_show; + const YV12_BUFFER_CONFIG *dgd = &cm->cur_frame->buf; const int is_uv = plane != AOM_PLANE_Y; rsc->plane_width = src->crop_widths[is_uv]; rsc->plane_height = src->crop_heights[is_uv]; @@ -166,7 +165,7 @@ static int64_t try_restoration_unit(const RestSearchCtxt *rsc, const int bit_depth = cm->seq_params.bit_depth; const int highbd = cm->seq_params.use_highbitdepth; - const YV12_BUFFER_CONFIG *fts = cm->frame_to_show; + const YV12_BUFFER_CONFIG *fts = &cm->cur_frame->buf; // TODO(yunqing): For now, only use optimized LR filter in decoder. Can be // also used in encoder. const int optimized_lr = 0; @@ -201,7 +200,7 @@ int64_t av1_lowbd_pixel_proj_error_c(const uint8_t *src8, int width, int height, v += xq[0] * (flt0[j] - u) + xq[1] * (flt1[j] - u); const int32_t e = ROUND_POWER_OF_TWO(v, SGRPROJ_RST_BITS + SGRPROJ_PRJ_BITS) - src[j]; - err += e * e; + err += ((int64_t)e * e); } dat += dat_stride; src += src_stride; @@ -217,7 +216,7 @@ int64_t av1_lowbd_pixel_proj_error_c(const uint8_t *src8, int width, int height, v += xq[0] * (flt0[j] - u); const int32_t e = ROUND_POWER_OF_TWO(v, SGRPROJ_RST_BITS + SGRPROJ_PRJ_BITS) - src[j]; - err += e * e; + err += ((int64_t)e * e); } dat += dat_stride; src += src_stride; @@ -232,7 +231,7 @@ int64_t av1_lowbd_pixel_proj_error_c(const uint8_t *src8, int width, int height, v += xq[1] * (flt1[j] - u); const int32_t e = ROUND_POWER_OF_TWO(v, SGRPROJ_RST_BITS + SGRPROJ_PRJ_BITS) - src[j]; - err += e * e; + err += ((int64_t)e * e); } dat += dat_stride; src += src_stride; @@ -242,7 +241,7 @@ int64_t av1_lowbd_pixel_proj_error_c(const uint8_t *src8, int width, int height, for (i = 0; i < height; ++i) { for (j = 0; j < width; ++j) { const int32_t e = (int32_t)(dat[j]) - src[j]; - err += e * e; + err += ((int64_t)e * e); } dat += dat_stride; src += src_stride; @@ -252,88 +251,97 @@ int64_t av1_lowbd_pixel_proj_error_c(const uint8_t *src8, int width, int height, return err; } +int64_t av1_highbd_pixel_proj_error_c(const uint8_t *src8, int width, + int height, int src_stride, + const uint8_t *dat8, int dat_stride, + int32_t *flt0, int flt0_stride, + int32_t *flt1, int flt1_stride, int xq[2], + const sgr_params_type *params) { + const uint16_t *src = CONVERT_TO_SHORTPTR(src8); + const uint16_t *dat = CONVERT_TO_SHORTPTR(dat8); + int i, j; + int64_t err = 0; + const int32_t half = 1 << (SGRPROJ_RST_BITS + SGRPROJ_PRJ_BITS - 1); + if (params->r[0] > 0 && params->r[1] > 0) { + int xq0 = xq[0]; + int xq1 = xq[1]; + for (i = 0; i < height; ++i) { + for (j = 0; j < width; ++j) { + const int32_t d = dat[j]; + const int32_t s = src[j]; + const int32_t u = (int32_t)(d << SGRPROJ_RST_BITS); + int32_t v0 = flt0[j] - u; + int32_t v1 = flt1[j] - u; + int32_t v = half; + v += xq0 * v0; + v += xq1 * v1; + const int32_t e = (v >> (SGRPROJ_RST_BITS + SGRPROJ_PRJ_BITS)) + d - s; + err += ((int64_t)e * e); + } + dat += dat_stride; + flt0 += flt0_stride; + flt1 += flt1_stride; + src += src_stride; + } + } else if (params->r[0] > 0 || params->r[1] > 0) { + int exq; + int32_t *flt; + int flt_stride; + if (params->r[0] > 0) { + exq = xq[0]; + flt = flt0; + flt_stride = flt0_stride; + } else { + exq = xq[1]; + flt = flt1; + flt_stride = flt1_stride; + } + for (i = 0; i < height; ++i) { + for (j = 0; j < width; ++j) { + const int32_t d = dat[j]; + const int32_t s = src[j]; + const int32_t u = (int32_t)(d << SGRPROJ_RST_BITS); + int32_t v = half; + v += exq * (flt[j] - u); + const int32_t e = (v >> (SGRPROJ_RST_BITS + SGRPROJ_PRJ_BITS)) + d - s; + err += ((int64_t)e * e); + } + dat += dat_stride; + flt += flt_stride; + src += src_stride; + } + } else { + for (i = 0; i < height; ++i) { + for (j = 0; j < width; ++j) { + const int32_t d = dat[j]; + const int32_t s = src[j]; + const int32_t e = d - s; + err += ((int64_t)e * e); + } + dat += dat_stride; + src += src_stride; + } + } + return err; +} + static int64_t get_pixel_proj_error(const uint8_t *src8, int width, int height, int src_stride, const uint8_t *dat8, int dat_stride, int use_highbitdepth, int32_t *flt0, int flt0_stride, int32_t *flt1, int flt1_stride, int *xqd, const sgr_params_type *params) { - int i, j; - int64_t err = 0; int xq[2]; decode_xq(xqd, xq, params); if (!use_highbitdepth) { - err = av1_lowbd_pixel_proj_error(src8, width, height, src_stride, dat8, - dat_stride, flt0, flt0_stride, flt1, - flt1_stride, xq, params); + return av1_lowbd_pixel_proj_error(src8, width, height, src_stride, dat8, + dat_stride, flt0, flt0_stride, flt1, + flt1_stride, xq, params); } else { - const uint16_t *src = CONVERT_TO_SHORTPTR(src8); - const uint16_t *dat = CONVERT_TO_SHORTPTR(dat8); - const int32_t half = 1 << (SGRPROJ_RST_BITS + SGRPROJ_PRJ_BITS - 1); - if (params->r[0] > 0 && params->r[1] > 0) { - int xq0 = xq[0]; - int xq1 = xq[1]; - for (i = 0; i < height; ++i) { - for (j = 0; j < width; ++j) { - const int32_t d = dat[j]; - const int32_t s = src[j]; - const int32_t u = (int32_t)(d << SGRPROJ_RST_BITS); - int32_t v0 = flt0[j] - u; - int32_t v1 = flt1[j] - u; - int32_t v = half; - v += xq0 * v0; - v += xq1 * v1; - const int32_t e = - (v >> (SGRPROJ_RST_BITS + SGRPROJ_PRJ_BITS)) + d - s; - err += e * e; - } - dat += dat_stride; - flt0 += flt0_stride; - flt1 += flt1_stride; - src += src_stride; - } - } else if (params->r[0] > 0 || params->r[1] > 0) { - int exq; - int32_t *flt; - int flt_stride; - if (params->r[0] > 0) { - exq = xq[0]; - flt = flt0; - flt_stride = flt0_stride; - } else { - exq = xq[1]; - flt = flt1; - flt_stride = flt1_stride; - } - for (i = 0; i < height; ++i) { - for (j = 0; j < width; ++j) { - const int32_t d = dat[j]; - const int32_t s = src[j]; - const int32_t u = (int32_t)(d << SGRPROJ_RST_BITS); - int32_t v = half; - v += exq * (flt[j] - u); - const int32_t e = - (v >> (SGRPROJ_RST_BITS + SGRPROJ_PRJ_BITS)) + d - s; - err += e * e; - } - dat += dat_stride; - flt += flt_stride; - src += src_stride; - } - } else { - for (i = 0; i < height; ++i) { - for (j = 0; j < width; ++j) { - const int32_t d = dat[j]; - const int32_t s = src[j]; - const int32_t e = d - s; - err += e * e; - } - dat += dat_stride; - src += src_stride; - } - } + return av1_highbd_pixel_proj_error(src8, width, height, src_stride, dat8, + dat_stride, flt0, flt0_stride, flt1, + flt1_stride, xq, params); } - return err; } #define USE_SGRPROJ_REFINEMENT_SEARCH 1 @@ -398,6 +406,13 @@ static int64_t finer_search_pixel_proj_error( return err; } +static int64_t signed_rounded_divide(int64_t dividend, int64_t divisor) { + if (dividend < 0) + return (dividend - divisor / 2) / divisor; + else + return (dividend + divisor / 2) / divisor; +} + static void get_proj_subspace(const uint8_t *src8, int width, int height, int src_stride, const uint8_t *dat8, int dat_stride, int use_highbitdepth, @@ -405,34 +420,32 @@ static void get_proj_subspace(const uint8_t *src8, int width, int height, int flt1_stride, int *xq, const sgr_params_type *params) { int i, j; - double H[2][2] = { { 0, 0 }, { 0, 0 } }; - double C[2] = { 0, 0 }; - double Det; - double x[2]; + int64_t H[2][2] = { { 0, 0 }, { 0, 0 } }; + int64_t C[2] = { 0, 0 }; const int size = width * height; - aom_clear_system_state(); - - // Default + // Default values to be returned if the problem becomes ill-posed xq[0] = 0; xq[1] = 0; + if (!use_highbitdepth) { const uint8_t *src = src8; const uint8_t *dat = dat8; for (i = 0; i < height; ++i) { for (j = 0; j < width; ++j) { - const double u = (double)(dat[i * dat_stride + j] << SGRPROJ_RST_BITS); - const double s = - (double)(src[i * src_stride + j] << SGRPROJ_RST_BITS) - u; - const double f1 = - (params->r[0] > 0) ? (double)flt0[i * flt0_stride + j] - u : 0; - const double f2 = - (params->r[1] > 0) ? (double)flt1[i * flt1_stride + j] - u : 0; - H[0][0] += f1 * f1; - H[1][1] += f2 * f2; - H[0][1] += f1 * f2; - C[0] += f1 * s; - C[1] += f2 * s; + const int32_t u = + (int32_t)(dat[i * dat_stride + j] << SGRPROJ_RST_BITS); + const int32_t s = + (int32_t)(src[i * src_stride + j] << SGRPROJ_RST_BITS) - u; + const int32_t f1 = + (params->r[0] > 0) ? (int32_t)flt0[i * flt0_stride + j] - u : 0; + const int32_t f2 = + (params->r[1] > 0) ? (int32_t)flt1[i * flt1_stride + j] - u : 0; + H[0][0] += (int64_t)f1 * f1; + H[1][1] += (int64_t)f2 * f2; + H[0][1] += (int64_t)f1 * f2; + C[0] += (int64_t)f1 * s; + C[1] += (int64_t)f2 * s; } } } else { @@ -440,18 +453,19 @@ static void get_proj_subspace(const uint8_t *src8, int width, int height, const uint16_t *dat = CONVERT_TO_SHORTPTR(dat8); for (i = 0; i < height; ++i) { for (j = 0; j < width; ++j) { - const double u = (double)(dat[i * dat_stride + j] << SGRPROJ_RST_BITS); - const double s = - (double)(src[i * src_stride + j] << SGRPROJ_RST_BITS) - u; - const double f1 = - (params->r[0] > 0) ? (double)flt0[i * flt0_stride + j] - u : 0; - const double f2 = - (params->r[1] > 0) ? (double)flt1[i * flt1_stride + j] - u : 0; - H[0][0] += f1 * f1; - H[1][1] += f2 * f2; - H[0][1] += f1 * f2; - C[0] += f1 * s; - C[1] += f2 * s; + const int32_t u = + (int32_t)(dat[i * dat_stride + j] << SGRPROJ_RST_BITS); + const int32_t s = + (int32_t)(src[i * src_stride + j] << SGRPROJ_RST_BITS) - u; + const int32_t f1 = + (params->r[0] > 0) ? (int32_t)flt0[i * flt0_stride + j] - u : 0; + const int32_t f2 = + (params->r[1] > 0) ? (int32_t)flt1[i * flt1_stride + j] - u : 0; + H[0][0] += (int64_t)f1 * f1; + H[1][1] += (int64_t)f2 * f2; + H[0][1] += (int64_t)f1 * f2; + C[0] += (int64_t)f1 * s; + C[1] += (int64_t)f2 * s; } } } @@ -464,35 +478,39 @@ static void get_proj_subspace(const uint8_t *src8, int width, int height, if (params->r[0] == 0) { // H matrix is now only the scalar H[1][1] // C vector is now only the scalar C[1] - Det = H[1][1]; - if (Det < 1e-8) return; // ill-posed, return default values - x[0] = 0; - x[1] = C[1] / Det; - + const int64_t Det = H[1][1]; + if (Det == 0) return; // ill-posed, return default values xq[0] = 0; - xq[1] = (int)rint(x[1] * (1 << SGRPROJ_PRJ_BITS)); + xq[1] = (int)signed_rounded_divide(C[1] * (1 << SGRPROJ_PRJ_BITS), Det); } else if (params->r[1] == 0) { // H matrix is now only the scalar H[0][0] // C vector is now only the scalar C[0] - Det = H[0][0]; - if (Det < 1e-8) return; // ill-posed, return default values - x[0] = C[0] / Det; - x[1] = 0; - - xq[0] = (int)rint(x[0] * (1 << SGRPROJ_PRJ_BITS)); + const int64_t Det = H[0][0]; + if (Det == 0) return; // ill-posed, return default values + xq[0] = (int)signed_rounded_divide(C[0] * (1 << SGRPROJ_PRJ_BITS), Det); xq[1] = 0; } else { - Det = (H[0][0] * H[1][1] - H[0][1] * H[1][0]); - if (Det < 1e-8) return; // ill-posed, return default values - x[0] = (H[1][1] * C[0] - H[0][1] * C[1]) / Det; - x[1] = (H[0][0] * C[1] - H[1][0] * C[0]) / Det; + const int64_t Det = H[0][0] * H[1][1] - H[0][1] * H[1][0]; + if (Det == 0) return; // ill-posed, return default values - xq[0] = (int)rint(x[0] * (1 << SGRPROJ_PRJ_BITS)); - xq[1] = (int)rint(x[1] * (1 << SGRPROJ_PRJ_BITS)); + // If scaling up dividend would overflow, instead scale down the divisor + const int64_t div1 = H[1][1] * C[0] - H[0][1] * C[1]; + if ((div1 > 0 && INT64_MAX / (1 << SGRPROJ_PRJ_BITS) < div1) || + (div1 < 0 && INT64_MIN / (1 << SGRPROJ_PRJ_BITS) > div1)) + xq[0] = (int)signed_rounded_divide(div1, Det / (1 << SGRPROJ_PRJ_BITS)); + else + xq[0] = (int)signed_rounded_divide(div1 * (1 << SGRPROJ_PRJ_BITS), Det); + + const int64_t div2 = H[0][0] * C[1] - H[1][0] * C[0]; + if ((div2 > 0 && INT64_MAX / (1 << SGRPROJ_PRJ_BITS) < div2) || + (div2 < 0 && INT64_MIN / (1 << SGRPROJ_PRJ_BITS) > div2)) + xq[1] = (int)signed_rounded_divide(div2, Det / (1 << SGRPROJ_PRJ_BITS)); + else + xq[1] = (int)signed_rounded_divide(div2 * (1 << SGRPROJ_PRJ_BITS), Det); } } -void encode_xq(int *xq, int *xqd, const sgr_params_type *params) { +static void encode_xq(int *xq, int *xqd, const sgr_params_type *params) { if (params->r[0] == 0) { xqd[0] = 0; xqd[1] = clamp((1 << SGRPROJ_PRJ_BITS) - xq[1], SGRPROJ_PRJ_MIN1, @@ -651,34 +669,34 @@ static void search_sgrproj(const RestorationTileLimits *limits, void av1_compute_stats_c(int wiener_win, const uint8_t *dgd, const uint8_t *src, int h_start, int h_end, int v_start, int v_end, - int dgd_stride, int src_stride, double *M, double *H) { + int dgd_stride, int src_stride, int64_t *M, + int64_t *H) { int i, j, k, l; - double Y[WIENER_WIN2]; + int16_t Y[WIENER_WIN2]; const int wiener_win2 = wiener_win * wiener_win; const int wiener_halfwin = (wiener_win >> 1); - const double avg = - find_average(dgd, h_start, h_end, v_start, v_end, dgd_stride); + uint8_t avg = find_average(dgd, h_start, h_end, v_start, v_end, dgd_stride); memset(M, 0, sizeof(*M) * wiener_win2); memset(H, 0, sizeof(*H) * wiener_win2 * wiener_win2); for (i = v_start; i < v_end; i++) { for (j = h_start; j < h_end; j++) { - const double X = (double)src[i * src_stride + j] - avg; + const int16_t X = (int16_t)src[i * src_stride + j] - (int16_t)avg; int idx = 0; for (k = -wiener_halfwin; k <= wiener_halfwin; k++) { for (l = -wiener_halfwin; l <= wiener_halfwin; l++) { - Y[idx] = (double)dgd[(i + l) * dgd_stride + (j + k)] - avg; + Y[idx] = (int16_t)dgd[(i + l) * dgd_stride + (j + k)] - (int16_t)avg; idx++; } } assert(idx == wiener_win2); for (k = 0; k < wiener_win2; ++k) { - M[k] += Y[k] * X; + M[k] += (int32_t)Y[k] * X; for (l = k; l < wiener_win2; ++l) { // H is a symmetric matrix, so we only need to fill out the upper // triangle here. We can copy it down to the lower triangle outside // the (i, j) loops. - H[k * wiener_win2 + l] += Y[k] * Y[l]; + H[k * wiener_win2 + l] += (int32_t)Y[k] * Y[l]; } } } @@ -690,60 +708,55 @@ void av1_compute_stats_c(int wiener_win, const uint8_t *dgd, const uint8_t *src, } } -static double find_average_highbd(const uint16_t *src, int h_start, int h_end, - int v_start, int v_end, int stride) { - uint64_t sum = 0; - double avg = 0; - int i, j; - aom_clear_system_state(); - for (i = v_start; i < v_end; i++) - for (j = h_start; j < h_end; j++) sum += src[i * stride + j]; - avg = (double)sum / ((v_end - v_start) * (h_end - h_start)); - return avg; -} - -static AOM_FORCE_INLINE void compute_stats_highbd( - int wiener_win, const uint8_t *dgd8, const uint8_t *src8, int h_start, - int h_end, int v_start, int v_end, int dgd_stride, int src_stride, - double *M, double *H) { +void av1_compute_stats_highbd_c(int wiener_win, const uint8_t *dgd8, + const uint8_t *src8, int h_start, int h_end, + int v_start, int v_end, int dgd_stride, + int src_stride, int64_t *M, int64_t *H, + aom_bit_depth_t bit_depth) { int i, j, k, l; - double Y[WIENER_WIN2]; + int32_t Y[WIENER_WIN2]; const int wiener_win2 = wiener_win * wiener_win; const int wiener_halfwin = (wiener_win >> 1); const uint16_t *src = CONVERT_TO_SHORTPTR(src8); const uint16_t *dgd = CONVERT_TO_SHORTPTR(dgd8); - const double avg = + uint16_t avg = find_average_highbd(dgd, h_start, h_end, v_start, v_end, dgd_stride); + uint8_t bit_depth_divider = 1; + if (bit_depth == AOM_BITS_12) + bit_depth_divider = 16; + else if (bit_depth == AOM_BITS_10) + bit_depth_divider = 4; + memset(M, 0, sizeof(*M) * wiener_win2); memset(H, 0, sizeof(*H) * wiener_win2 * wiener_win2); for (i = v_start; i < v_end; i++) { for (j = h_start; j < h_end; j++) { - const double X = (double)src[i * src_stride + j] - avg; + const int32_t X = (int32_t)src[i * src_stride + j] - (int32_t)avg; int idx = 0; for (k = -wiener_halfwin; k <= wiener_halfwin; k++) { for (l = -wiener_halfwin; l <= wiener_halfwin; l++) { - Y[idx] = (double)dgd[(i + l) * dgd_stride + (j + k)] - avg; + Y[idx] = (int32_t)dgd[(i + l) * dgd_stride + (j + k)] - (int32_t)avg; idx++; } } assert(idx == wiener_win2); for (k = 0; k < wiener_win2; ++k) { - double Yk = Y[k]; - M[k] += Yk * X; - double *H2 = &H[k * wiener_win2]; - H2[k] += Yk * Yk; - for (l = k + 1; l < wiener_win2; ++l) { + M[k] += (int64_t)Y[k] * X; + for (l = k; l < wiener_win2; ++l) { // H is a symmetric matrix, so we only need to fill out the upper // triangle here. We can copy it down to the lower triangle outside // the (i, j) loops. - H2[l] += Yk * Y[l]; + H[k * wiener_win2 + l] += (int64_t)Y[k] * Y[l]; } } } } for (k = 0; k < wiener_win2; ++k) { + M[k] /= bit_depth_divider; + H[k * wiener_win2 + k] /= bit_depth_divider; for (l = k + 1; l < wiener_win2; ++l) { + H[k * wiener_win2 + l] /= bit_depth_divider; H[l * wiener_win2 + k] = H[k * wiener_win2 + l]; } } @@ -754,12 +767,56 @@ static INLINE int wrap_index(int i, int wiener_win) { return (i >= wiener_halfwin1 ? wiener_win - 1 - i : i); } +// Solve linear equations to find Wiener filter tap values +// Taps are output scaled by WIENER_FILT_STEP +static int linsolve_wiener(int n, int64_t *A, int stride, int64_t *b, + int32_t *x) { + for (int k = 0; k < n - 1; k++) { + // Partial pivoting: bring the row with the largest pivot to the top + for (int i = n - 1; i > k; i--) { + // If row i has a better (bigger) pivot than row (i-1), swap them + if (llabs(A[(i - 1) * stride + k]) < llabs(A[i * stride + k])) { + for (int j = 0; j < n; j++) { + const int64_t c = A[i * stride + j]; + A[i * stride + j] = A[(i - 1) * stride + j]; + A[(i - 1) * stride + j] = c; + } + const int64_t c = b[i]; + b[i] = b[i - 1]; + b[i - 1] = c; + } + } + // Forward elimination (convert A to row-echelon form) + for (int i = k; i < n - 1; i++) { + if (A[k * stride + k] == 0) return 0; + const int64_t c = A[(i + 1) * stride + k]; + const int64_t cd = A[k * stride + k]; + for (int j = 0; j < n; j++) { + A[(i + 1) * stride + j] -= c / 256 * A[k * stride + j] / cd * 256; + } + b[i + 1] -= c * b[k] / cd; + } + } + // Back-substitution + for (int i = n - 1; i >= 0; i--) { + if (A[i * stride + i] == 0) return 0; + int64_t c = 0; + for (int j = i + 1; j <= n - 1; j++) { + c += A[i * stride + j] * x[j] / WIENER_TAP_SCALE_FACTOR; + } + // Store filter taps x in scaled form. + x[i] = (int32_t)(WIENER_TAP_SCALE_FACTOR * (b[i] - c) / A[i * stride + i]); + } + + return 1; +} + // Fix vector b, update vector a -static void update_a_sep_sym(int wiener_win, double **Mc, double **Hc, - double *a, double *b) { +static void update_a_sep_sym(int wiener_win, int64_t **Mc, int64_t **Hc, + int32_t *a, int32_t *b) { int i, j; - double S[WIENER_WIN]; - double A[WIENER_HALFWIN1], B[WIENER_HALFWIN1 * WIENER_HALFWIN1]; + int32_t S[WIENER_WIN]; + int64_t A[WIENER_HALFWIN1], B[WIENER_HALFWIN1 * WIENER_HALFWIN1]; const int wiener_win2 = wiener_win * wiener_win; const int wiener_halfwin1 = (wiener_win >> 1) + 1; memset(A, 0, sizeof(A)); @@ -767,36 +824,41 @@ static void update_a_sep_sym(int wiener_win, double **Mc, double **Hc, for (i = 0; i < wiener_win; i++) { for (j = 0; j < wiener_win; ++j) { const int jj = wrap_index(j, wiener_win); - A[jj] += Mc[i][j] * b[i]; + A[jj] += Mc[i][j] * b[i] / WIENER_TAP_SCALE_FACTOR; } } for (i = 0; i < wiener_win; i++) { for (j = 0; j < wiener_win; j++) { int k, l; - for (k = 0; k < wiener_win; ++k) + for (k = 0; k < wiener_win; ++k) { for (l = 0; l < wiener_win; ++l) { const int kk = wrap_index(k, wiener_win); const int ll = wrap_index(l, wiener_win); B[ll * wiener_halfwin1 + kk] += - Hc[j * wiener_win + i][k * wiener_win2 + l] * b[i] * b[j]; + Hc[j * wiener_win + i][k * wiener_win2 + l] * b[i] / + WIENER_TAP_SCALE_FACTOR * b[j] / WIENER_TAP_SCALE_FACTOR; } + } } } // Normalization enforcement in the system of equations itself - for (i = 0; i < wiener_halfwin1 - 1; ++i) + for (i = 0; i < wiener_halfwin1 - 1; ++i) { A[i] -= A[wiener_halfwin1 - 1] * 2 + B[i * wiener_halfwin1 + wiener_halfwin1 - 1] - 2 * B[(wiener_halfwin1 - 1) * wiener_halfwin1 + (wiener_halfwin1 - 1)]; - for (i = 0; i < wiener_halfwin1 - 1; ++i) - for (j = 0; j < wiener_halfwin1 - 1; ++j) + } + for (i = 0; i < wiener_halfwin1 - 1; ++i) { + for (j = 0; j < wiener_halfwin1 - 1; ++j) { B[i * wiener_halfwin1 + j] -= 2 * (B[i * wiener_halfwin1 + (wiener_halfwin1 - 1)] + B[(wiener_halfwin1 - 1) * wiener_halfwin1 + j] - 2 * B[(wiener_halfwin1 - 1) * wiener_halfwin1 + (wiener_halfwin1 - 1)]); - if (linsolve(wiener_halfwin1 - 1, B, wiener_halfwin1, A, S)) { - S[wiener_halfwin1 - 1] = 1.0; + } + } + if (linsolve_wiener(wiener_halfwin1 - 1, B, wiener_halfwin1, A, S)) { + S[wiener_halfwin1 - 1] = WIENER_TAP_SCALE_FACTOR; for (i = wiener_halfwin1; i < wiener_win; ++i) { S[i] = S[wiener_win - 1 - i]; S[wiener_halfwin1 - 1] -= 2 * S[i]; @@ -806,18 +868,20 @@ static void update_a_sep_sym(int wiener_win, double **Mc, double **Hc, } // Fix vector a, update vector b -static void update_b_sep_sym(int wiener_win, double **Mc, double **Hc, - double *a, double *b) { +static void update_b_sep_sym(int wiener_win, int64_t **Mc, int64_t **Hc, + int32_t *a, int32_t *b) { int i, j; - double S[WIENER_WIN]; - double A[WIENER_HALFWIN1], B[WIENER_HALFWIN1 * WIENER_HALFWIN1]; + int32_t S[WIENER_WIN]; + int64_t A[WIENER_HALFWIN1], B[WIENER_HALFWIN1 * WIENER_HALFWIN1]; const int wiener_win2 = wiener_win * wiener_win; const int wiener_halfwin1 = (wiener_win >> 1) + 1; memset(A, 0, sizeof(A)); memset(B, 0, sizeof(B)); for (i = 0; i < wiener_win; i++) { const int ii = wrap_index(i, wiener_win); - for (j = 0; j < wiener_win; j++) A[ii] += Mc[i][j] * a[j]; + for (j = 0; j < wiener_win; j++) { + A[ii] += Mc[i][j] * a[j] / WIENER_TAP_SCALE_FACTOR; + } } for (i = 0; i < wiener_win; i++) { @@ -825,27 +889,33 @@ static void update_b_sep_sym(int wiener_win, double **Mc, double **Hc, const int ii = wrap_index(i, wiener_win); const int jj = wrap_index(j, wiener_win); int k, l; - for (k = 0; k < wiener_win; ++k) - for (l = 0; l < wiener_win; ++l) + for (k = 0; k < wiener_win; ++k) { + for (l = 0; l < wiener_win; ++l) { B[jj * wiener_halfwin1 + ii] += - Hc[i * wiener_win + j][k * wiener_win2 + l] * a[k] * a[l]; + Hc[i * wiener_win + j][k * wiener_win2 + l] * a[k] / + WIENER_TAP_SCALE_FACTOR * a[l] / WIENER_TAP_SCALE_FACTOR; + } + } } } // Normalization enforcement in the system of equations itself - for (i = 0; i < wiener_halfwin1 - 1; ++i) + for (i = 0; i < wiener_halfwin1 - 1; ++i) { A[i] -= A[wiener_halfwin1 - 1] * 2 + B[i * wiener_halfwin1 + wiener_halfwin1 - 1] - 2 * B[(wiener_halfwin1 - 1) * wiener_halfwin1 + (wiener_halfwin1 - 1)]; - for (i = 0; i < wiener_halfwin1 - 1; ++i) - for (j = 0; j < wiener_halfwin1 - 1; ++j) + } + for (i = 0; i < wiener_halfwin1 - 1; ++i) { + for (j = 0; j < wiener_halfwin1 - 1; ++j) { B[i * wiener_halfwin1 + j] -= 2 * (B[i * wiener_halfwin1 + (wiener_halfwin1 - 1)] + B[(wiener_halfwin1 - 1) * wiener_halfwin1 + j] - 2 * B[(wiener_halfwin1 - 1) * wiener_halfwin1 + (wiener_halfwin1 - 1)]); - if (linsolve(wiener_halfwin1 - 1, B, wiener_halfwin1, A, S)) { - S[wiener_halfwin1 - 1] = 1.0; + } + } + if (linsolve_wiener(wiener_halfwin1 - 1, B, wiener_halfwin1, A, S)) { + S[wiener_halfwin1 - 1] = WIENER_TAP_SCALE_FACTOR; for (i = wiener_halfwin1; i < wiener_win; ++i) { S[i] = S[wiener_win - 1 - i]; S[wiener_halfwin1 - 1] -= 2 * S[i]; @@ -854,20 +924,21 @@ static void update_b_sep_sym(int wiener_win, double **Mc, double **Hc, } } -static int wiener_decompose_sep_sym(int wiener_win, double *M, double *H, - double *a, double *b) { - static const int init_filt[WIENER_WIN] = { +static int wiener_decompose_sep_sym(int wiener_win, int64_t *M, int64_t *H, + int32_t *a, int32_t *b) { + static const int32_t init_filt[WIENER_WIN] = { WIENER_FILT_TAP0_MIDV, WIENER_FILT_TAP1_MIDV, WIENER_FILT_TAP2_MIDV, WIENER_FILT_TAP3_MIDV, WIENER_FILT_TAP2_MIDV, WIENER_FILT_TAP1_MIDV, WIENER_FILT_TAP0_MIDV, }; - double *Hc[WIENER_WIN2]; - double *Mc[WIENER_WIN]; + int64_t *Hc[WIENER_WIN2]; + int64_t *Mc[WIENER_WIN]; int i, j, iter; const int plane_off = (WIENER_WIN - wiener_win) >> 1; const int wiener_win2 = wiener_win * wiener_win; for (i = 0; i < wiener_win; i++) { - a[i] = b[i] = (double)init_filt[i + plane_off] / WIENER_FILT_STEP; + a[i] = b[i] = + WIENER_TAP_SCALE_FACTOR / WIENER_FILT_STEP * init_filt[i + plane_off]; } for (i = 0; i < wiener_win; i++) { Mc[i] = M + i * wiener_win; @@ -889,23 +960,23 @@ static int wiener_decompose_sep_sym(int wiener_win, double *M, double *H, // Computes the function x'*H*x - x'*M for the learned 2D filter x, and compares // against identity filters; Final score is defined as the difference between // the function values -static double compute_score(int wiener_win, double *M, double *H, - InterpKernel vfilt, InterpKernel hfilt) { - double ab[WIENER_WIN * WIENER_WIN]; +static int64_t compute_score(int wiener_win, int64_t *M, int64_t *H, + InterpKernel vfilt, InterpKernel hfilt) { + int32_t ab[WIENER_WIN * WIENER_WIN]; + int16_t a[WIENER_WIN], b[WIENER_WIN]; + int64_t P = 0, Q = 0; + int64_t iP = 0, iQ = 0; + int64_t Score, iScore; int i, k, l; - double P = 0, Q = 0; - double iP = 0, iQ = 0; - double Score, iScore; - double a[WIENER_WIN], b[WIENER_WIN]; const int plane_off = (WIENER_WIN - wiener_win) >> 1; const int wiener_win2 = wiener_win * wiener_win; aom_clear_system_state(); - a[WIENER_HALFWIN] = b[WIENER_HALFWIN] = 1.0; + a[WIENER_HALFWIN] = b[WIENER_HALFWIN] = WIENER_FILT_STEP; for (i = 0; i < WIENER_HALFWIN; ++i) { - a[i] = a[WIENER_WIN - i - 1] = (double)vfilt[i] / WIENER_FILT_STEP; - b[i] = b[WIENER_WIN - i - 1] = (double)hfilt[i] / WIENER_FILT_STEP; + a[i] = a[WIENER_WIN - i - 1] = vfilt[i]; + b[i] = b[WIENER_WIN - i - 1] = hfilt[i]; a[WIENER_HALFWIN] -= 2 * a[i]; b[WIENER_HALFWIN] -= 2 * b[i]; } @@ -915,9 +986,11 @@ static double compute_score(int wiener_win, double *M, double *H, ab[k * wiener_win + l] = a[l + plane_off] * b[k + plane_off]; } for (k = 0; k < wiener_win2; ++k) { - P += ab[k] * M[k]; - for (l = 0; l < wiener_win2; ++l) - Q += ab[k] * H[k * wiener_win2 + l] * ab[l]; + P += ab[k] * M[k] / WIENER_FILT_STEP / WIENER_FILT_STEP; + for (l = 0; l < wiener_win2; ++l) { + Q += ab[k] * H[k * wiener_win2 + l] * ab[l] / WIENER_FILT_STEP / + WIENER_FILT_STEP / WIENER_FILT_STEP / WIENER_FILT_STEP; + } } Score = Q - 2 * P; @@ -928,11 +1001,19 @@ static double compute_score(int wiener_win, double *M, double *H, return Score - iScore; } -static void quantize_sym_filter(int wiener_win, double *f, InterpKernel fi) { +static void finalize_sym_filter(int wiener_win, int32_t *f, InterpKernel fi) { int i; const int wiener_halfwin = (wiener_win >> 1); + for (i = 0; i < wiener_halfwin; ++i) { - fi[i] = RINT(f[i] * WIENER_FILT_STEP); + const int64_t dividend = f[i] * WIENER_FILT_STEP; + const int64_t divisor = WIENER_TAP_SCALE_FACTOR; + // Perform this division with proper rounding rather than truncation + if (dividend < 0) { + fi[i] = (int16_t)((dividend - (divisor / 2)) / divisor); + } else { + fi[i] = (int16_t)((dividend + (divisor / 2)) / divisor); + } } // Specialize for 7-tap filter if (wiener_win == WIENER_WIN) { @@ -1110,15 +1191,16 @@ static void search_wiener(const RestorationTileLimits *limits, const int wiener_win = (rsc->plane == AOM_PLANE_Y) ? WIENER_WIN : WIENER_WIN_CHROMA; - double M[WIENER_WIN2]; - double H[WIENER_WIN2 * WIENER_WIN2]; - double vfilterd[WIENER_WIN], hfilterd[WIENER_WIN]; + int64_t M[WIENER_WIN2]; + int64_t H[WIENER_WIN2 * WIENER_WIN2]; + int32_t vfilter[WIENER_WIN], hfilter[WIENER_WIN]; const AV1_COMMON *const cm = rsc->cm; if (cm->seq_params.use_highbitdepth) { - compute_stats_highbd(wiener_win, rsc->dgd_buffer, rsc->src_buffer, - limits->h_start, limits->h_end, limits->v_start, - limits->v_end, rsc->dgd_stride, rsc->src_stride, M, H); + av1_compute_stats_highbd(wiener_win, rsc->dgd_buffer, rsc->src_buffer, + limits->h_start, limits->h_end, limits->v_start, + limits->v_end, rsc->dgd_stride, rsc->src_stride, M, + H, cm->seq_params.bit_depth); } else { av1_compute_stats(wiener_win, rsc->dgd_buffer, rsc->src_buffer, limits->h_start, limits->h_end, limits->v_start, @@ -1128,7 +1210,7 @@ static void search_wiener(const RestorationTileLimits *limits, const MACROBLOCK *const x = rsc->x; const int64_t bits_none = x->wiener_restore_cost[0]; - if (!wiener_decompose_sep_sym(wiener_win, M, H, vfilterd, hfilterd)) { + if (!wiener_decompose_sep_sym(wiener_win, M, H, vfilter, hfilter)) { rsc->bits += bits_none; rsc->sse += rusi->sse[RESTORE_NONE]; rusi->best_rtype[RESTORE_WIENER - 1] = RESTORE_NONE; @@ -1139,8 +1221,8 @@ static void search_wiener(const RestorationTileLimits *limits, RestorationUnitInfo rui; memset(&rui, 0, sizeof(rui)); rui.restoration_type = RESTORE_WIENER; - quantize_sym_filter(wiener_win, vfilterd, rui.wiener_info.vfilter); - quantize_sym_filter(wiener_win, hfilterd, rui.wiener_info.hfilter); + finalize_sym_filter(wiener_win, vfilter, rui.wiener_info.vfilter); + finalize_sym_filter(wiener_win, hfilter, rui.wiener_info.hfilter); // Filter score computes the value of the function x'*A*x - x'*b for the // learned filter and compares it against identity filer. If there is no @@ -1199,7 +1281,7 @@ static void search_norestore(const RestorationTileLimits *limits, const int highbd = rsc->cm->seq_params.use_highbitdepth; rusi->sse[RESTORE_NONE] = sse_restoration_unit( - limits, rsc->src, rsc->cm->frame_to_show, rsc->plane, highbd); + limits, rsc->src, &rsc->cm->cur_frame->buf, rsc->plane, highbd); rsc->sse += rusi->sse[RESTORE_NONE]; } @@ -1283,7 +1365,8 @@ static double search_rest_type(RestSearchCtxt *rsc, RestorationType rtype) { }; reset_rsc(rsc); - rsc_on_tile(LR_TILE_ROW, LR_TILE_COL, rsc); + rsc_on_tile(rsc); + av1_foreach_rest_unit_in_plane(rsc->cm, rsc->plane, funs[rtype], rsc, &rsc->tile_rect, rsc->cm->rst_tmpbuf, NULL); return RDCOST_DBL(rsc->x->rdmult, rsc->bits >> 4, rsc->sse); diff --git a/third_party/aom/av1/encoder/pickrst.h b/third_party/aom/av1/encoder/pickrst.h index 3fec0c34b3..f34359c1fc 100644 --- a/third_party/aom/av1/encoder/pickrst.h +++ b/third_party/aom/av1/encoder/pickrst.h @@ -25,16 +25,34 @@ static const uint8_t g_shuffle_stats_data[16] = { 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, }; -static INLINE double find_average(const uint8_t *src, int h_start, int h_end, - int v_start, int v_end, int stride) { +static const uint8_t g_shuffle_stats_highbd_data[32] = { + 0, 1, 2, 3, 2, 3, 4, 5, 4, 5, 6, 7, 6, 7, 8, 9, + 0, 1, 2, 3, 2, 3, 4, 5, 4, 5, 6, 7, 6, 7, 8, 9, +}; + +static INLINE uint8_t find_average(const uint8_t *src, int h_start, int h_end, + int v_start, int v_end, int stride) { uint64_t sum = 0; - double avg = 0; - int i, j; - aom_clear_system_state(); - for (i = v_start; i < v_end; i++) - for (j = h_start; j < h_end; j++) sum += src[i * stride + j]; - avg = (double)sum / ((v_end - v_start) * (h_end - h_start)); - return avg; + for (int i = v_start; i < v_end; i++) { + for (int j = h_start; j < h_end; j++) { + sum += src[i * stride + j]; + } + } + uint64_t avg = sum / ((v_end - v_start) * (h_end - h_start)); + return (uint8_t)avg; +} + +static INLINE uint16_t find_average_highbd(const uint16_t *src, int h_start, + int h_end, int v_start, int v_end, + int stride) { + uint64_t sum = 0; + for (int i = v_start; i < v_end; i++) { + for (int j = h_start; j < h_end; j++) { + sum += src[i * stride + j]; + } + } + uint64_t avg = sum / ((v_end - v_start) * (h_end - h_start)); + return (uint16_t)avg; } void av1_pick_filter_restoration(const YV12_BUFFER_CONFIG *sd, AV1_COMP *cpi); diff --git a/third_party/aom/av1/encoder/ransac.c b/third_party/aom/av1/encoder/ransac.c index 781f528ebf..6a8854c58c 100644 --- a/third_party/aom/av1/encoder/ransac.c +++ b/third_party/aom/av1/encoder/ransac.c @@ -23,7 +23,7 @@ #define MAX_DEGENERATE_ITER 10 #define MINPTS_MULTIPLIER 5 -#define INLIER_THRESHOLD 1.0 +#define INLIER_THRESHOLD 1.25 #define MIN_TRIALS 20 //////////////////////////////////////////////////////////////////////////////// @@ -376,10 +376,7 @@ static int ransac(const int *matched_points, int npoints, IsDegenerateFunc is_degenerate, FindTransformationFunc find_transformation, ProjectPointsDoubleFunc projectpoints) { - static const double PROBABILITY_REQUIRED = 0.9; - static const double EPS = 1e-12; - - int N = 10000, trial_count = 0; + int trial_count = 0; int i = 0; int ret_val = 0; @@ -444,7 +441,7 @@ static int ransac(const int *matched_points, int npoints, *(cnp2++) = *(matched_points++); } - while (N > trial_count) { + while (MIN_TRIALS > trial_count) { double sum_distance = 0.0; double sum_distance_squared = 0.0; @@ -491,8 +488,7 @@ static int ransac(const int *matched_points, int npoints, if (current_motion.num_inliers >= worst_kept_motion->num_inliers && current_motion.num_inliers > 1) { - int temp; - double fracinliers, pNoOutliers, mean_distance, dtemp; + double mean_distance; mean_distance = sum_distance / ((double)current_motion.num_inliers); current_motion.variance = sum_distance_squared / ((double)current_motion.num_inliers - 1.0) - @@ -506,21 +502,183 @@ static int ransac(const int *matched_points, int npoints, worst_kept_motion->variance = current_motion.variance; memcpy(worst_kept_motion->inlier_indices, current_motion.inlier_indices, sizeof(*current_motion.inlier_indices) * npoints); - assert(npoints > 0); - fracinliers = (double)current_motion.num_inliers / (double)npoints; - pNoOutliers = 1 - pow(fracinliers, minpts); - pNoOutliers = fmax(EPS, pNoOutliers); - pNoOutliers = fmin(1 - EPS, pNoOutliers); - dtemp = log(1.0 - PROBABILITY_REQUIRED) / log(pNoOutliers); - temp = (dtemp > (double)INT32_MAX) - ? INT32_MAX - : dtemp < (double)INT32_MIN ? INT32_MIN : (int)dtemp; - - if (temp > 0 && temp < N) { - N = AOMMAX(temp, MIN_TRIALS); + // Determine the new worst kept motion and its num_inliers and variance. + for (i = 0; i < num_desired_motions; ++i) { + if (is_better_motion(worst_kept_motion, &motions[i])) { + worst_kept_motion = &motions[i]; + } } + } + } + trial_count++; + } + // Sort the motions, best first. + qsort(motions, num_desired_motions, sizeof(RANSAC_MOTION), compare_motions); + + // Recompute the motions using only the inliers. + for (i = 0; i < num_desired_motions; ++i) { + if (motions[i].num_inliers >= minpts) { + copy_points_at_indices(points1, corners1, motions[i].inlier_indices, + motions[i].num_inliers); + copy_points_at_indices(points2, corners2, motions[i].inlier_indices, + motions[i].num_inliers); + + find_transformation(motions[i].num_inliers, points1, points2, + params_by_motion + (MAX_PARAMDIM - 1) * i); + } + num_inliers_by_motion[i] = motions[i].num_inliers; + } + +finish_ransac: + aom_free(points1); + aom_free(points2); + aom_free(corners1); + aom_free(corners2); + aom_free(image1_coord); + aom_free(current_motion.inlier_indices); + for (i = 0; i < num_desired_motions; ++i) { + aom_free(motions[i].inlier_indices); + } + aom_free(motions); + + return ret_val; +} + +static int ransac_double_prec(const double *matched_points, int npoints, + int *num_inliers_by_motion, + double *params_by_motion, int num_desired_motions, + const int minpts, IsDegenerateFunc is_degenerate, + FindTransformationFunc find_transformation, + ProjectPointsDoubleFunc projectpoints) { + int trial_count = 0; + int i = 0; + int ret_val = 0; + + unsigned int seed = (unsigned int)npoints; + + int indices[MAX_MINPTS] = { 0 }; + + double *points1, *points2; + double *corners1, *corners2; + double *image1_coord; + + // Store information for the num_desired_motions best transformations found + // and the worst motion among them, as well as the motion currently under + // consideration. + RANSAC_MOTION *motions, *worst_kept_motion = NULL; + RANSAC_MOTION current_motion; + + // Store the parameters and the indices of the inlier points for the motion + // currently under consideration. + double params_this_motion[MAX_PARAMDIM]; + + double *cnp1, *cnp2; + + for (i = 0; i < num_desired_motions; ++i) { + num_inliers_by_motion[i] = 0; + } + if (npoints < minpts * MINPTS_MULTIPLIER || npoints == 0) { + return 1; + } + + points1 = (double *)aom_malloc(sizeof(*points1) * npoints * 2); + points2 = (double *)aom_malloc(sizeof(*points2) * npoints * 2); + corners1 = (double *)aom_malloc(sizeof(*corners1) * npoints * 2); + corners2 = (double *)aom_malloc(sizeof(*corners2) * npoints * 2); + image1_coord = (double *)aom_malloc(sizeof(*image1_coord) * npoints * 2); + + motions = + (RANSAC_MOTION *)aom_malloc(sizeof(RANSAC_MOTION) * num_desired_motions); + for (i = 0; i < num_desired_motions; ++i) { + motions[i].inlier_indices = + (int *)aom_malloc(sizeof(*motions->inlier_indices) * npoints); + clear_motion(motions + i, npoints); + } + current_motion.inlier_indices = + (int *)aom_malloc(sizeof(*current_motion.inlier_indices) * npoints); + clear_motion(¤t_motion, npoints); + + worst_kept_motion = motions; + + if (!(points1 && points2 && corners1 && corners2 && image1_coord && motions && + current_motion.inlier_indices)) { + ret_val = 1; + goto finish_ransac; + } + + cnp1 = corners1; + cnp2 = corners2; + for (i = 0; i < npoints; ++i) { + *(cnp1++) = *(matched_points++); + *(cnp1++) = *(matched_points++); + *(cnp2++) = *(matched_points++); + *(cnp2++) = *(matched_points++); + } + + while (MIN_TRIALS > trial_count) { + double sum_distance = 0.0; + double sum_distance_squared = 0.0; + + clear_motion(¤t_motion, npoints); + + int degenerate = 1; + int num_degenerate_iter = 0; + + while (degenerate) { + num_degenerate_iter++; + if (!get_rand_indices(npoints, minpts, indices, &seed)) { + ret_val = 1; + goto finish_ransac; + } + + copy_points_at_indices(points1, corners1, indices, minpts); + copy_points_at_indices(points2, corners2, indices, minpts); + + degenerate = is_degenerate(points1); + if (num_degenerate_iter > MAX_DEGENERATE_ITER) { + ret_val = 1; + goto finish_ransac; + } + } + + if (find_transformation(minpts, points1, points2, params_this_motion)) { + trial_count++; + continue; + } + + projectpoints(params_this_motion, corners1, image1_coord, npoints, 2, 2); + + for (i = 0; i < npoints; ++i) { + double dx = image1_coord[i * 2] - corners2[i * 2]; + double dy = image1_coord[i * 2 + 1] - corners2[i * 2 + 1]; + double distance = sqrt(dx * dx + dy * dy); + + if (distance < INLIER_THRESHOLD) { + current_motion.inlier_indices[current_motion.num_inliers++] = i; + sum_distance += distance; + sum_distance_squared += distance * distance; + } + } + + if (current_motion.num_inliers >= worst_kept_motion->num_inliers && + current_motion.num_inliers > 1) { + double mean_distance; + mean_distance = sum_distance / ((double)current_motion.num_inliers); + current_motion.variance = + sum_distance_squared / ((double)current_motion.num_inliers - 1.0) - + mean_distance * mean_distance * ((double)current_motion.num_inliers) / + ((double)current_motion.num_inliers - 1.0); + if (is_better_motion(¤t_motion, worst_kept_motion)) { + // This motion is better than the worst currently kept motion. Remember + // the inlier points and variance. The parameters for each kept motion + // will be recomputed later using only the inliers. + worst_kept_motion->num_inliers = current_motion.num_inliers; + worst_kept_motion->variance = current_motion.variance; + memcpy(worst_kept_motion->inlier_indices, current_motion.inlier_indices, + sizeof(*current_motion.inlier_indices) * npoints); + assert(npoints > 0); // Determine the new worst kept motion and its num_inliers and variance. for (i = 0; i < num_desired_motions; ++i) { if (is_better_motion(worst_kept_motion, &motions[i])) { @@ -601,3 +759,33 @@ int ransac_affine(int *matched_points, int npoints, int *num_inliers_by_motion, params_by_motion, num_desired_motions, 3, is_degenerate_affine, find_affine, project_points_double_affine); } + +int ransac_translation_double_prec(double *matched_points, int npoints, + int *num_inliers_by_motion, + double *params_by_motion, + int num_desired_motions) { + return ransac_double_prec(matched_points, npoints, num_inliers_by_motion, + params_by_motion, num_desired_motions, 3, + is_degenerate_translation, find_translation, + project_points_double_translation); +} + +int ransac_rotzoom_double_prec(double *matched_points, int npoints, + int *num_inliers_by_motion, + double *params_by_motion, + int num_desired_motions) { + return ransac_double_prec(matched_points, npoints, num_inliers_by_motion, + params_by_motion, num_desired_motions, 3, + is_degenerate_affine, find_rotzoom, + project_points_double_rotzoom); +} + +int ransac_affine_double_prec(double *matched_points, int npoints, + int *num_inliers_by_motion, + double *params_by_motion, + int num_desired_motions) { + return ransac_double_prec(matched_points, npoints, num_inliers_by_motion, + params_by_motion, num_desired_motions, 3, + is_degenerate_affine, find_affine, + project_points_double_affine); +} diff --git a/third_party/aom/av1/encoder/ransac.h b/third_party/aom/av1/encoder/ransac.h index c429f2ce5e..b754bacdf9 100644 --- a/third_party/aom/av1/encoder/ransac.h +++ b/third_party/aom/av1/encoder/ransac.h @@ -22,6 +22,9 @@ typedef int (*RansacFunc)(int *matched_points, int npoints, int *num_inliers_by_motion, double *params_by_motion, int num_motions); +typedef int (*RansacFuncDouble)(double *matched_points, int npoints, + int *num_inliers_by_motion, + double *params_by_motion, int num_motions); /* Each of these functions fits a motion model from a set of corresponding points in 2 frames using RANSAC. */ @@ -32,4 +35,16 @@ int ransac_rotzoom(int *matched_points, int npoints, int *num_inliers_by_motion, int ransac_translation(int *matched_points, int npoints, int *num_inliers_by_motion, double *params_by_motion, int num_motions); +int ransac_translation_double_prec(double *matched_points, int npoints, + int *num_inliers_by_motion, + double *params_by_motion, + int num_desired_motions); +int ransac_rotzoom_double_prec(double *matched_points, int npoints, + int *num_inliers_by_motion, + double *params_by_motion, + int num_desired_motions); +int ransac_affine_double_prec(double *matched_points, int npoints, + int *num_inliers_by_motion, + double *params_by_motion, + int num_desired_motions); #endif // AOM_AV1_ENCODER_RANSAC_H_ diff --git a/third_party/aom/av1/encoder/ratectrl.c b/third_party/aom/av1/encoder/ratectrl.c index 2597fb9908..6a7637154d 100644 --- a/third_party/aom/av1/encoder/ratectrl.c +++ b/third_party/aom/av1/encoder/ratectrl.c @@ -29,6 +29,7 @@ #include "av1/common/seg_common.h" #include "av1/encoder/encodemv.h" +#include "av1/encoder/encode_strategy.h" #include "av1/encoder/random.h" #include "av1/encoder/ratectrl.h" @@ -250,12 +251,10 @@ int av1_rc_get_default_min_gf_interval(int width, int height, // 4K60: 12 } -int av1_rc_get_default_max_gf_interval(double framerate, int min_gf_interval) { +int av1_rc_get_default_max_gf_interval(double framerate, int min_gf_interval, + int max_pyr_height) { int interval = AOMMIN(MAX_GF_INTERVAL, (int)(framerate * 0.75)); - interval += (interval & 0x01); // Round to even value -#if CONFIG_FIX_GF_LENGTH - interval = AOMMAX(FIXED_GF_LENGTH, interval); -#endif + interval = AOMMAX(av1_rc_get_fixed_gf_length(max_pyr_height), interval); return AOMMAX(interval, min_gf_interval); } @@ -312,7 +311,7 @@ void av1_rc_init(const AV1EncoderConfig *oxcf, int pass, RATE_CONTROL *rc) { oxcf->width, oxcf->height, oxcf->init_framerate); if (rc->max_gf_interval == 0) rc->max_gf_interval = av1_rc_get_default_max_gf_interval( - oxcf->init_framerate, rc->min_gf_interval); + oxcf->init_framerate, rc->min_gf_interval, oxcf->gf_max_pyr_height); rc->baseline_gf_interval = (rc->min_gf_interval + rc->max_gf_interval) / 2; } @@ -357,7 +356,7 @@ static double get_rate_correction_factor(const AV1_COMP *cpi, int width, const RATE_CONTROL *const rc = &cpi->rc; double rcf; - if (cpi->common.frame_type == KEY_FRAME) { + if (cpi->common.current_frame.frame_type == KEY_FRAME) { rcf = rc->rate_correction_factors[KF_STD]; } else if (cpi->oxcf.pass == 2) { RATE_FACTOR_LEVEL rf_lvl = @@ -384,7 +383,7 @@ static void set_rate_correction_factor(AV1_COMP *cpi, double factor, int width, factor = fclamp(factor, MIN_BPB_FACTOR, MAX_BPB_FACTOR); - if (cpi->common.frame_type == KEY_FRAME) { + if (cpi->common.current_frame.frame_type == KEY_FRAME) { rc->rate_correction_factors[KF_STD] = factor; } else if (cpi->oxcf.pass == 2) { RATE_FACTOR_LEVEL rf_lvl = @@ -425,8 +424,8 @@ void av1_rc_update_rate_correction_factors(AV1_COMP *cpi, int width, av1_cyclic_refresh_estimate_bits_at_q(cpi, rate_correction_factor); } else { projected_size_based_on_q = av1_estimate_bits_at_q( - cpi->common.frame_type, cm->base_qindex, MBs, rate_correction_factor, - cm->seq_params.bit_depth); + cpi->common.current_frame.frame_type, cm->base_qindex, MBs, + rate_correction_factor, cm->seq_params.bit_depth); } // Work out a size correction factor. if (projected_size_based_on_q > FRAME_OVERHEAD_BITS) @@ -497,8 +496,9 @@ int av1_rc_regulate_q(const AV1_COMP *cpi, int target_bits_per_frame, bits_per_mb_at_this_q = (int)av1_cyclic_refresh_rc_bits_per_mb(cpi, i, correction_factor); } else { - bits_per_mb_at_this_q = (int)av1_rc_bits_per_mb( - cm->frame_type, i, correction_factor, cm->seq_params.bit_depth); + bits_per_mb_at_this_q = + (int)av1_rc_bits_per_mb(cm->current_frame.frame_type, i, + correction_factor, cm->seq_params.bit_depth); } if (bits_per_mb_at_this_q <= target_bits_per_mb) { @@ -559,20 +559,18 @@ static int get_gf_active_quality(const RATE_CONTROL *const rc, int q, arfgf_low_motion_minq, arfgf_high_motion_minq); } -#if REDUCE_LAST_ALT_BOOST static int get_gf_high_motion_quality(int q, aom_bit_depth_t bit_depth) { int *arfgf_high_motion_minq; ASSIGN_MINQ_TABLE(bit_depth, arfgf_high_motion_minq); return arfgf_high_motion_minq[q]; } -#endif static int calc_active_worst_quality_one_pass_vbr(const AV1_COMP *cpi) { const RATE_CONTROL *const rc = &cpi->rc; - const unsigned int curr_frame = cpi->common.current_video_frame; + const unsigned int curr_frame = cpi->common.current_frame.frame_number; int active_worst_quality; - if (cpi->common.frame_type == KEY_FRAME) { + if (cpi->common.current_frame.frame_type == KEY_FRAME) { active_worst_quality = curr_frame == 0 ? rc->worst_quality : rc->last_q[KEY_FRAME] * 2; } else { @@ -604,13 +602,13 @@ static int calc_active_worst_quality_one_pass_cbr(const AV1_COMP *cpi) { int adjustment = 0; int active_worst_quality; int ambient_qp; - if (cm->frame_type == KEY_FRAME) return rc->worst_quality; + if (cm->current_frame.frame_type == KEY_FRAME) return rc->worst_quality; // For ambient_qp we use minimum of avg_frame_qindex[KEY_FRAME/INTER_FRAME] // for the first few frames following key frame. These are both initialized // to worst_quality and updated with (3/4, 1/4) average in postencode_update. // So for first few frames following key, the qp of that key frame is weighted // into the active_worst_quality setting. - ambient_qp = (cm->current_video_frame < 5) + ambient_qp = (cm->current_frame.frame_number < 5) ? AOMMIN(rc->avg_frame_qindex[INTER_FRAME], rc->avg_frame_qindex[KEY_FRAME]) : rc->avg_frame_qindex[INTER_FRAME]; @@ -650,6 +648,7 @@ static int rc_pick_q_and_bounds_one_pass_cbr(const AV1_COMP *cpi, int width, int *top_index) { const AV1_COMMON *const cm = &cpi->common; const RATE_CONTROL *const rc = &cpi->rc; + const CurrentFrame *const current_frame = &cm->current_frame; int active_best_quality; int active_worst_quality = calc_active_worst_quality_one_pass_cbr(cpi); int q; @@ -668,7 +667,7 @@ static int rc_pick_q_and_bounds_one_pass_cbr(const AV1_COMP *cpi, int width, int delta_qindex = av1_compute_qdelta(rc, last_boosted_q, (last_boosted_q * 0.75), bit_depth); active_best_quality = AOMMAX(qindex + delta_qindex, rc->best_quality); - } else if (cm->current_video_frame > 0) { + } else if (current_frame->frame_number > 0) { // not first frame of one pass and kf_boost is set double q_adj_factor = 1.0; double q_val; @@ -701,7 +700,7 @@ static int rc_pick_q_and_bounds_one_pass_cbr(const AV1_COMP *cpi, int width, active_best_quality = get_gf_active_quality(rc, q, bit_depth); } else { // Use the lower of active_worst_quality and recent/average Q. - if (cm->current_video_frame > 1) { + if (current_frame->frame_number > 1) { if (rc->avg_frame_qindex[INTER_FRAME] < active_worst_quality) active_best_quality = rtc_minq[rc->avg_frame_qindex[INTER_FRAME]]; else @@ -724,18 +723,18 @@ static int rc_pick_q_and_bounds_one_pass_cbr(const AV1_COMP *cpi, int width, *bottom_index = active_best_quality; // Limit Q range for the adaptive loop. - if (cm->frame_type == KEY_FRAME && !rc->this_key_frame_forced && - !(cm->current_video_frame == 0)) { + if (current_frame->frame_type == KEY_FRAME && !rc->this_key_frame_forced && + !(current_frame->frame_number == 0)) { int qdelta = 0; aom_clear_system_state(); - qdelta = av1_compute_qdelta_by_rate(&cpi->rc, cm->frame_type, + qdelta = av1_compute_qdelta_by_rate(&cpi->rc, current_frame->frame_type, active_worst_quality, 2.0, bit_depth); *top_index = active_worst_quality + qdelta; *top_index = AOMMAX(*top_index, *bottom_index); } // Special case code to try and match quality with forced key frames - if (cm->frame_type == KEY_FRAME && rc->this_key_frame_forced) { + if (current_frame->frame_type == KEY_FRAME && rc->this_key_frame_forced) { q = rc->last_boosted_qindex; } else { q = av1_rc_regulate_q(cpi, rc->this_frame_target, active_best_quality, @@ -756,10 +755,28 @@ static int rc_pick_q_and_bounds_one_pass_cbr(const AV1_COMP *cpi, int width, return q; } +static int gf_group_pyramid_level(const AV1_COMP *cpi) { + const GF_GROUP *gf_group = &cpi->twopass.gf_group; + int this_height = gf_group->pyramid_level[gf_group->index]; + return this_height; +} + static int get_active_cq_level(const RATE_CONTROL *rc, - const AV1EncoderConfig *const oxcf) { + const AV1EncoderConfig *const oxcf, + int intra_only, int superres_denom) { static const double cq_adjust_threshold = 0.1; int active_cq_level = oxcf->cq_level; + (void)intra_only; + if (oxcf->rc_mode == AOM_CQ || oxcf->rc_mode == AOM_Q) { + // printf("Superres %d %d %d = %d\n", superres_denom, intra_only, + // rc->frames_to_key, !(intra_only && rc->frames_to_key <= 1)); + if (oxcf->superres_mode == SUPERRES_QTHRESH && + superres_denom != SCALE_NUMERATOR && + !(intra_only && rc->frames_to_key <= 1)) { + active_cq_level = + AOMMAX(active_cq_level - ((superres_denom - SCALE_NUMERATOR) * 4), 0); + } + } if (oxcf->rc_mode == AOM_CQ && rc->total_target_bits > 0) { const double x = (double)rc->total_actual_bits / rc->total_target_bits; if (x < cq_adjust_threshold) { @@ -774,8 +791,10 @@ static int rc_pick_q_and_bounds_one_pass_vbr(const AV1_COMP *cpi, int width, int *top_index) { const AV1_COMMON *const cm = &cpi->common; const RATE_CONTROL *const rc = &cpi->rc; + const CurrentFrame *const current_frame = &cm->current_frame; const AV1EncoderConfig *const oxcf = &cpi->oxcf; - const int cq_level = get_active_cq_level(rc, oxcf); + const int cq_level = get_active_cq_level(rc, oxcf, frame_is_intra_only(cm), + cm->superres_scale_denominator); int active_best_quality; int active_worst_quality = calc_active_worst_quality_one_pass_vbr(cpi); int q; @@ -849,12 +868,12 @@ static int rc_pick_q_and_bounds_one_pass_vbr(const AV1_COMP *cpi, int width, 0.70, 1.0, 0.85, 1.0 }; const int delta_qindex = av1_compute_qdelta( rc, q_val, - q_val * delta_rate[cm->current_video_frame % FIXED_GF_INTERVAL], + q_val * delta_rate[current_frame->frame_number % FIXED_GF_INTERVAL], bit_depth); active_best_quality = AOMMAX(qindex + delta_qindex, rc->best_quality); } else { // Use the lower of active_worst_quality and recent/average Q. - active_best_quality = (cm->current_video_frame > 1) + active_best_quality = (current_frame->frame_number > 1) ? inter_minq[rc->avg_frame_qindex[INTER_FRAME]] : inter_minq[rc->avg_frame_qindex[KEY_FRAME]]; // For the constrained quality mode we don't want @@ -878,14 +897,15 @@ static int rc_pick_q_and_bounds_one_pass_vbr(const AV1_COMP *cpi, int width, { int qdelta = 0; aom_clear_system_state(); - if (cm->frame_type == KEY_FRAME && !rc->this_key_frame_forced && - !(cm->current_video_frame == 0)) { - qdelta = av1_compute_qdelta_by_rate(&cpi->rc, cm->frame_type, + if (current_frame->frame_type == KEY_FRAME && !rc->this_key_frame_forced && + !(current_frame->frame_number == 0)) { + qdelta = av1_compute_qdelta_by_rate(&cpi->rc, current_frame->frame_type, active_worst_quality, 2.0, bit_depth); } else if (!rc->is_src_frame_alt_ref && (cpi->refresh_golden_frame || cpi->refresh_alt_ref_frame)) { - qdelta = av1_compute_qdelta_by_rate( - &cpi->rc, cm->frame_type, active_worst_quality, 1.75, bit_depth); + qdelta = + av1_compute_qdelta_by_rate(&cpi->rc, current_frame->frame_type, + active_worst_quality, 1.75, bit_depth); } *top_index = active_worst_quality + qdelta; *top_index = AOMMAX(*top_index, *bottom_index); @@ -894,7 +914,8 @@ static int rc_pick_q_and_bounds_one_pass_vbr(const AV1_COMP *cpi, int width, if (oxcf->rc_mode == AOM_Q) { q = active_best_quality; // Special case code to try and match quality with forced key frames - } else if ((cm->frame_type == KEY_FRAME) && rc->this_key_frame_forced) { + } else if ((current_frame->frame_type == KEY_FRAME) && + rc->this_key_frame_forced) { q = rc->last_boosted_qindex; } else { q = av1_rc_regulate_q(cpi, rc->this_frame_target, active_best_quality, @@ -934,7 +955,8 @@ static int rc_pick_q_and_bounds_two_pass(const AV1_COMP *cpi, int width, const RATE_CONTROL *const rc = &cpi->rc; const AV1EncoderConfig *const oxcf = &cpi->oxcf; const GF_GROUP *gf_group = &cpi->twopass.gf_group; - const int cq_level = get_active_cq_level(rc, oxcf); + const int cq_level = get_active_cq_level(rc, oxcf, frame_is_intra_only(cm), + cm->superres_scale_denominator); int active_best_quality; int active_worst_quality = cpi->twopass.active_worst_quality; int q; @@ -942,18 +964,20 @@ static int rc_pick_q_and_bounds_two_pass(const AV1_COMP *cpi, int width, const int bit_depth = cm->seq_params.bit_depth; ASSIGN_MINQ_TABLE(bit_depth, inter_minq); -#if CUSTOMIZED_GF const int is_intrl_arf_boost = gf_group->update_type[gf_group->index] == INTNL_ARF_UPDATE; -#else - const int is_intrl_arf_boost = cpi->refresh_alt2_ref_frame; -#endif // CUSTOMIZED_GF if (frame_is_intra_only(cm)) { - // Handle the special case for key frames forced when we have reached - // the maximum key frame interval. Here force the Q to a range - // based on the ambient Q to reduce the risk of popping. - if (rc->this_key_frame_forced) { + if (rc->frames_to_key == 1 && oxcf->rc_mode == AOM_Q) { + // If the next frame is also a key frame or the current frame is the + // only frame in the sequence in AOM_Q mode, just use the cq_level + // as q. + active_best_quality = cq_level; + active_worst_quality = cq_level; + } else if (rc->this_key_frame_forced) { + // Handle the special case for key frames forced when we have reached + // the maximum key frame interval. Here force the Q to a range + // based on the ambient Q to reduce the risk of popping. double last_boosted_q; int delta_qindex; int qindex; @@ -967,10 +991,13 @@ static int rc_pick_q_and_bounds_two_pass(const AV1_COMP *cpi, int width, active_worst_quality = AOMMIN(qindex + delta_qindex, active_worst_quality); } else { + // Increase the boost if the forced keyframe is a forward reference. + // These numbers were derived empirically. + const double boost_factor = cpi->oxcf.fwd_kf_enabled ? 0.25 : 0.50; qindex = rc->last_boosted_qindex; last_boosted_q = av1_convert_qindex_to_q(qindex, bit_depth); - delta_qindex = av1_compute_qdelta(rc, last_boosted_q, - last_boosted_q * 0.5, bit_depth); + delta_qindex = av1_compute_qdelta( + rc, last_boosted_q, last_boosted_q * boost_factor, bit_depth); active_best_quality = AOMMAX(qindex + delta_qindex, rc->best_quality); } } else { @@ -982,6 +1009,10 @@ static int rc_pick_q_and_bounds_two_pass(const AV1_COMP *cpi, int width, active_best_quality = get_kf_active_quality(rc, active_worst_quality, bit_depth); + if (cpi->twopass.kf_zeromotion_pct >= STATIC_KF_GROUP_THRESH) { + active_best_quality /= 3; + } + // Allow somewhat lower kf minq with small image formats. if ((width * height) <= (352 * 288)) { q_adj_factor -= 0.25; @@ -1011,33 +1042,30 @@ static int rc_pick_q_and_bounds_two_pass(const AV1_COMP *cpi, int width, // For constrained quality dont allow Q less than the cq level if (oxcf->rc_mode == AOM_CQ) { if (q < cq_level) q = cq_level; -#if USE_SYMM_MULTI_LAYER && MULTI_LVL_BOOST_VBR_CQ + + active_best_quality = get_gf_active_quality(rc, q, bit_depth); + + // Constrained quality use slightly lower active best. + active_best_quality = active_best_quality * 15 / 16; + if (gf_group->update_type[gf_group->index] == ARF_UPDATE || (is_intrl_arf_boost && !cpi->new_bwdref_update_rule)) { -#endif // USE_SYMM_MULTI_LAYER && MULTI_LVL_BOOST_VBR_CQ - active_best_quality = get_gf_active_quality(rc, q, bit_depth); - - // Constrained quality use slightly lower active best. - active_best_quality = active_best_quality * 15 / 16; -#if REDUCE_LAST_ALT_BOOST if (gf_group->update_type[gf_group->index] == ARF_UPDATE) { const int min_boost = get_gf_high_motion_quality(q, bit_depth); const int boost = min_boost - active_best_quality; active_best_quality = min_boost - (int)(boost * rc->arf_boost_factor); } -#endif *arf_q = active_best_quality; -#if USE_SYMM_MULTI_LAYER && MULTI_LVL_BOOST_VBR_CQ - } else { + } else if (cpi->new_bwdref_update_rule && is_intrl_arf_boost) { + assert(rc->arf_q >= 0); // Ensure it is set to a valid value. active_best_quality = rc->arf_q; - int this_height = gf_group->pyramid_level[gf_group->index]; + int this_height = gf_group_pyramid_level(cpi); while (this_height < gf_group->pyramid_height) { active_best_quality = (active_best_quality + cq_level + 1) / 2; ++this_height; } } -#endif // USE_SYMM_MULTI_LAYER && MULTI_LVL_BOOST_VBR_CQ } else if (oxcf->rc_mode == AOM_Q) { if (!cpi->refresh_alt_ref_frame && !is_intrl_arf_boost) { active_best_quality = cq_level; @@ -1045,50 +1073,41 @@ static int rc_pick_q_and_bounds_two_pass(const AV1_COMP *cpi, int width, if (gf_group->update_type[gf_group->index] == ARF_UPDATE) { active_best_quality = get_gf_active_quality(rc, q, bit_depth); *arf_q = active_best_quality; -#if REDUCE_LAST_ALT_BOOST const int min_boost = get_gf_high_motion_quality(q, bit_depth); const int boost = min_boost - active_best_quality; active_best_quality = min_boost - (int)(boost * rc->arf_boost_factor); -#endif } else { + assert(rc->arf_q >= 0); // Ensure it is set to a valid value. active_best_quality = rc->arf_q; } -#if USE_SYMM_MULTI_LAYER if (cpi->new_bwdref_update_rule && is_intrl_arf_boost) { - int this_height = gf_group->pyramid_level[gf_group->index]; + int this_height = gf_group_pyramid_level(cpi); while (this_height < gf_group->pyramid_height) { active_best_quality = (active_best_quality + cq_level + 1) / 2; ++this_height; } } else { -#endif // Modify best quality for second level arfs. For mode AOM_Q this // becomes the baseline frame q. if (gf_group->rf_level[gf_group->index] == GF_ARF_LOW) active_best_quality = (active_best_quality + cq_level + 1) / 2; -#if USE_SYMM_MULTI_LAYER } -#endif } } else { active_best_quality = get_gf_active_quality(rc, q, bit_depth); -#if REDUCE_LAST_ALT_BOOST const int min_boost = get_gf_high_motion_quality(q, bit_depth); const int boost = min_boost - active_best_quality; active_best_quality = min_boost - (int)(boost * rc->arf_boost_factor); -#endif -#if USE_SYMM_MULTI_LAYER if (cpi->new_bwdref_update_rule && is_intrl_arf_boost) { - int this_height = gf_group->pyramid_level[gf_group->index]; + int this_height = gf_group_pyramid_level(cpi); while (this_height < gf_group->pyramid_height) { active_best_quality = (active_best_quality + active_worst_quality + 1) / 2; ++this_height; } } -#endif } } else { if (oxcf->rc_mode == AOM_Q) { @@ -1135,7 +1154,7 @@ static int rc_pick_q_and_bounds_two_pass(const AV1_COMP *cpi, int width, // Modify active_best_quality for downscaled normal frames. if (av1_frame_scaled(cm) && !frame_is_kf_gf_arf(cpi)) { int qdelta = av1_compute_qdelta_by_rate( - rc, cm->frame_type, active_best_quality, 2.0, bit_depth); + rc, cm->current_frame.frame_type, active_best_quality, 2.0, bit_depth); active_best_quality = AOMMAX(active_best_quality + qdelta, rc->best_quality); } @@ -1145,7 +1164,9 @@ static int rc_pick_q_and_bounds_two_pass(const AV1_COMP *cpi, int width, active_worst_quality = clamp(active_worst_quality, active_best_quality, rc->worst_quality); - if (oxcf->rc_mode == AOM_Q) { + if (oxcf->rc_mode == AOM_Q || + (frame_is_intra_only(cm) && !rc->this_key_frame_forced && + cpi->twopass.kf_zeromotion_pct >= STATIC_KF_GROUP_THRESH)) { q = active_best_quality; // Special case code to try and match quality with forced key frames. } else if (frame_is_intra_only(cm) && rc->this_key_frame_forced) { @@ -1193,7 +1214,7 @@ int av1_rc_pick_q_and_bounds(AV1_COMP *cpi, int width, int height, assert(cpi->oxcf.pass == 2 && "invalid encode pass"); GF_GROUP *gf_group = &cpi->twopass.gf_group; - int arf_q = 0; + int arf_q = -1; // Initialize to invalid value, for sanity check later. q = rc_pick_q_and_bounds_two_pass(cpi, width, height, bottom_index, top_index, &arf_q); @@ -1253,16 +1274,12 @@ static void update_alt_ref_frame_stats(AV1_COMP *cpi) { static void update_golden_frame_stats(AV1_COMP *cpi) { RATE_CONTROL *const rc = &cpi->rc; -#if CUSTOMIZED_GF const TWO_PASS *const twopass = &cpi->twopass; const GF_GROUP *const gf_group = &twopass->gf_group; const int is_intrnl_arf = cpi->oxcf.pass == 2 ? gf_group->update_type[gf_group->index] == INTNL_ARF_UPDATE : cpi->refresh_alt2_ref_frame; -#else - const int is_intnl_arf = cpi->refresh_alt2_ref_frame; -#endif // Update the Golden frame usage counts. // NOTE(weitinglin): If we use show_existing_frame for an OVERLAY frame, @@ -1288,19 +1305,42 @@ static void update_golden_frame_stats(AV1_COMP *cpi) { } } +void av1_estimate_qp_gop(AV1_COMP *cpi) { + AV1_COMMON *const cm = &cpi->common; + int gop_length = cpi->rc.baseline_gf_interval; + int bottom_index, top_index; + int idx; + const int gf_index = cpi->twopass.gf_group.index; + + for (idx = 1; idx <= gop_length + 1 && idx < MAX_LAG_BUFFERS; ++idx) { + TplDepFrame *tpl_frame = &cpi->tpl_stats[idx]; + int target_rate = cpi->twopass.gf_group.bit_allocation[idx]; + int arf_q = 0; + + cpi->twopass.gf_group.index = idx; + rc_set_frame_target(cpi, target_rate, cm->width, cm->height); + av1_configure_buffer_updates( + cpi, cpi->twopass.gf_group.update_type[cpi->twopass.gf_group.index]); + tpl_frame->base_qindex = rc_pick_q_and_bounds_two_pass( + cpi, cm->width, cm->height, &bottom_index, &top_index, &arf_q); + tpl_frame->base_qindex = AOMMAX(tpl_frame->base_qindex, 1); + } + // Reset the actual index and frame update + cpi->twopass.gf_group.index = gf_index; + av1_configure_buffer_updates( + cpi, cpi->twopass.gf_group.update_type[cpi->twopass.gf_group.index]); +} + void av1_rc_postencode_update(AV1_COMP *cpi, uint64_t bytes_used) { const AV1_COMMON *const cm = &cpi->common; + const CurrentFrame *const current_frame = &cm->current_frame; RATE_CONTROL *const rc = &cpi->rc; -#if CUSTOMIZED_GF const TWO_PASS *const twopass = &cpi->twopass; const GF_GROUP *const gf_group = &twopass->gf_group; const int is_intrnl_arf = cpi->oxcf.pass == 2 ? gf_group->update_type[gf_group->index] == INTNL_ARF_UPDATE : cpi->refresh_alt2_ref_frame; -#else - const int is_intrnl_arf = cpi->refresh_alt2_ref_frame; -#endif const int qindex = cm->base_qindex; @@ -1315,7 +1355,7 @@ void av1_rc_postencode_update(AV1_COMP *cpi, uint64_t bytes_used) { av1_rc_update_rate_correction_factors(cpi, cm->width, cm->height); // Keep a record of last Q and ambient average Q. - if (cm->frame_type == KEY_FRAME) { + if (current_frame->frame_type == KEY_FRAME) { rc->last_q[KEY_FRAME] = qindex; rc->avg_frame_qindex[KEY_FRAME] = ROUND_POWER_OF_TWO(3 * rc->avg_frame_qindex[KEY_FRAME] + qindex, 2); @@ -1341,13 +1381,14 @@ void av1_rc_postencode_update(AV1_COMP *cpi, uint64_t bytes_used) { // If all mbs in this group are skipped only update if the Q value is // better than that already stored. // This is used to help set quality in forced key frames to reduce popping - if ((qindex < rc->last_boosted_qindex) || (cm->frame_type == KEY_FRAME) || + if ((qindex < rc->last_boosted_qindex) || + (current_frame->frame_type == KEY_FRAME) || (!rc->constrained_gf_group && (cpi->refresh_alt_ref_frame || is_intrnl_arf || (cpi->refresh_golden_frame && !rc->is_src_frame_alt_ref)))) { rc->last_boosted_qindex = qindex; } - if (cm->frame_type == KEY_FRAME) rc->last_kf_qindex = qindex; + if (current_frame->frame_type == KEY_FRAME) rc->last_kf_qindex = qindex; update_buffer_level(cpi, rc->projected_frame_size); @@ -1357,7 +1398,7 @@ void av1_rc_postencode_update(AV1_COMP *cpi, uint64_t bytes_used) { rc->this_frame_target = (int)(rc->this_frame_target / resize_rate_factor(cpi, cm->width, cm->height)); - if (cm->frame_type != KEY_FRAME) { + if (current_frame->frame_type != KEY_FRAME) { rc->rolling_target_bits = ROUND_POWER_OF_TWO( rc->rolling_target_bits * 3 + rc->this_frame_target, 2); rc->rolling_actual_bits = ROUND_POWER_OF_TWO( @@ -1378,15 +1419,15 @@ void av1_rc_postencode_update(AV1_COMP *cpi, uint64_t bytes_used) { rc->total_target_vs_actual = rc->total_actual_bits - rc->total_target_bits; if (is_altref_enabled(cpi) && cpi->refresh_alt_ref_frame && - (cm->frame_type != KEY_FRAME)) + (current_frame->frame_type != KEY_FRAME)) // Update the alternate reference frame stats as appropriate. update_alt_ref_frame_stats(cpi); else // Update the Golden frame stats as appropriate. update_golden_frame_stats(cpi); - if (cm->frame_type == KEY_FRAME) rc->frames_since_key = 0; - // if (cm->current_video_frame == 1 && cm->show_frame) + if (current_frame->frame_type == KEY_FRAME) rc->frames_since_key = 0; + // if (current_frame->frame_number == 1 && cm->show_frame) /* rc->this_frame_target = (int)(rc->this_frame_target / resize_rate_factor(cpi, cm->width, @@ -1431,9 +1472,11 @@ static int calc_iframe_target_size_one_pass_vbr(const AV1_COMP *const cpi) { return av1_rc_clamp_iframe_target_size(cpi, target); } -void av1_rc_get_one_pass_vbr_params(AV1_COMP *cpi) { +void av1_rc_get_one_pass_vbr_params(AV1_COMP *cpi, + EncodeFrameParams *const frame_params) { AV1_COMMON *const cm = &cpi->common; RATE_CONTROL *const rc = &cpi->rc; + CurrentFrame *const current_frame = &cm->current_frame; int target; int altref_enabled = is_altref_enabled(cpi); int sframe_dist = cpi->oxcf.sframe_dist; @@ -1441,44 +1484,44 @@ void av1_rc_get_one_pass_vbr_params(AV1_COMP *cpi) { int sframe_enabled = cpi->oxcf.sframe_enabled; // TODO(yaowu): replace the "auto_key && 0" below with proper decision logic. if (!cpi->refresh_alt_ref_frame && - (cm->current_video_frame == 0 || (cpi->frame_flags & FRAMEFLAGS_KEY) || - rc->frames_to_key == 0 || (cpi->oxcf.auto_key && 0))) { - cm->frame_type = KEY_FRAME; + (current_frame->frame_number == 0 || + (cpi->frame_flags & FRAMEFLAGS_KEY) || rc->frames_to_key == 0 || + (cpi->oxcf.auto_key && 0))) { + frame_params->frame_type = KEY_FRAME; rc->this_key_frame_forced = - cm->current_video_frame != 0 && rc->frames_to_key == 0; + current_frame->frame_number != 0 && rc->frames_to_key == 0; rc->frames_to_key = cpi->oxcf.key_freq; rc->kf_boost = DEFAULT_KF_BOOST; rc->source_alt_ref_active = 0; } else { - cm->frame_type = INTER_FRAME; + frame_params->frame_type = INTER_FRAME; if (sframe_enabled) { if (altref_enabled) { if (sframe_mode == 1) { // sframe_mode == 1: insert sframe if it matches altref frame. - if (cm->current_video_frame % sframe_dist == 0 && - cm->frame_type != KEY_FRAME && cm->current_video_frame != 0 && - cpi->refresh_alt_ref_frame) { - cm->frame_type = S_FRAME; + if (current_frame->frame_number % sframe_dist == 0 && + current_frame->frame_number != 0 && cpi->refresh_alt_ref_frame) { + frame_params->frame_type = S_FRAME; } } else { // sframe_mode != 1: if sframe will be inserted at the next available // altref frame - if (cm->current_video_frame % sframe_dist == 0 && - cm->frame_type != KEY_FRAME && cm->current_video_frame != 0) { + if (current_frame->frame_number % sframe_dist == 0 && + current_frame->frame_number != 0) { rc->sframe_due = 1; } if (rc->sframe_due && cpi->refresh_alt_ref_frame) { - cm->frame_type = S_FRAME; + frame_params->frame_type = S_FRAME; rc->sframe_due = 0; } } } else { - if (cm->current_video_frame % sframe_dist == 0 && - cm->frame_type != KEY_FRAME && cm->current_video_frame != 0) { - cm->frame_type = S_FRAME; + if (current_frame->frame_number % sframe_dist == 0 && + current_frame->frame_number != 0) { + frame_params->frame_type = S_FRAME; } } } @@ -1501,7 +1544,7 @@ void av1_rc_get_one_pass_vbr_params(AV1_COMP *cpi) { if (cpi->oxcf.aq_mode == CYCLIC_REFRESH_AQ) av1_cyclic_refresh_update_parameters(cpi); - if (cm->frame_type == KEY_FRAME) + if (frame_params->frame_type == KEY_FRAME) target = calc_iframe_target_size_one_pass_vbr(cpi); else target = calc_pframe_target_size_one_pass_vbr(cpi); @@ -1550,7 +1593,7 @@ static int calc_pframe_target_size_one_pass_cbr(const AV1_COMP *cpi) { static int calc_iframe_target_size_one_pass_cbr(const AV1_COMP *cpi) { const RATE_CONTROL *rc = &cpi->rc; int target; - if (cpi->common.current_video_frame == 0) { + if (cpi->common.current_frame.frame_number == 0) { target = ((rc->starting_buffer_level / 2) > INT_MAX) ? INT_MAX : (int)(rc->starting_buffer_level / 2); @@ -1567,21 +1610,24 @@ static int calc_iframe_target_size_one_pass_cbr(const AV1_COMP *cpi) { return av1_rc_clamp_iframe_target_size(cpi, target); } -void av1_rc_get_one_pass_cbr_params(AV1_COMP *cpi) { +void av1_rc_get_one_pass_cbr_params(AV1_COMP *cpi, + EncodeFrameParams *const frame_params) { AV1_COMMON *const cm = &cpi->common; RATE_CONTROL *const rc = &cpi->rc; + CurrentFrame *const current_frame = &cm->current_frame; int target; // TODO(yaowu): replace the "auto_key && 0" below with proper decision logic. - if ((cm->current_video_frame == 0 || (cpi->frame_flags & FRAMEFLAGS_KEY) || - rc->frames_to_key == 0 || (cpi->oxcf.auto_key && 0))) { - cm->frame_type = KEY_FRAME; + if ((current_frame->frame_number == 0 || + (cpi->frame_flags & FRAMEFLAGS_KEY) || rc->frames_to_key == 0 || + (cpi->oxcf.auto_key && 0))) { + frame_params->frame_type = KEY_FRAME; rc->this_key_frame_forced = - cm->current_video_frame != 0 && rc->frames_to_key == 0; + current_frame->frame_number != 0 && rc->frames_to_key == 0; rc->frames_to_key = cpi->oxcf.key_freq; rc->kf_boost = DEFAULT_KF_BOOST; rc->source_alt_ref_active = 0; } else { - cm->frame_type = INTER_FRAME; + frame_params->frame_type = INTER_FRAME; } if (rc->frames_till_gf_update_due == 0) { if (cpi->oxcf.aq_mode == CYCLIC_REFRESH_AQ) @@ -1602,7 +1648,7 @@ void av1_rc_get_one_pass_cbr_params(AV1_COMP *cpi) { if (cpi->oxcf.aq_mode == CYCLIC_REFRESH_AQ) av1_cyclic_refresh_update_parameters(cpi); - if (cm->frame_type == KEY_FRAME) + if (frame_params->frame_type == KEY_FRAME) target = calc_iframe_target_size_one_pass_cbr(cpi); else target = calc_pframe_target_size_one_pass_cbr(cpi); @@ -1674,15 +1720,10 @@ void av1_rc_set_gf_interval_range(const AV1_COMP *const cpi, oxcf->width, oxcf->height, cpi->framerate); if (rc->max_gf_interval == 0) rc->max_gf_interval = av1_rc_get_default_max_gf_interval( - cpi->framerate, rc->min_gf_interval); + cpi->framerate, rc->min_gf_interval, oxcf->gf_max_pyr_height); - // Extended interval for genuinely static scenes - rc->static_scene_max_gf_interval = MAX_LAG_BUFFERS * 2; - - if (is_altref_enabled(cpi)) { - if (rc->static_scene_max_gf_interval > oxcf->lag_in_frames - 1) - rc->static_scene_max_gf_interval = oxcf->lag_in_frames - 1; - } + // Extended max interval for genuinely static scenes like slide shows. + rc->static_scene_max_gf_interval = MAX_STATIC_GF_GROUP_LENGTH; if (rc->max_gf_interval > rc->static_scene_max_gf_interval) rc->max_gf_interval = rc->static_scene_max_gf_interval; @@ -1733,7 +1774,7 @@ static void vbr_rate_correction(AV1_COMP *cpi, int *this_frame_target) { // This number is used to damp the per frame rate correction. // Range 0 - 1.0 if (cpi->twopass.total_stats.count != 0.) { - position_factor = sqrt((double)cpi->common.current_video_frame / + position_factor = sqrt((double)cpi->common.current_frame.frame_number / cpi->twopass.total_stats.count); } max_delta = (int)(position_factor * diff --git a/third_party/aom/av1/encoder/ratectrl.h b/third_party/aom/av1/encoder/ratectrl.h index 198ecab975..36c738c8cc 100644 --- a/third_party/aom/av1/encoder/ratectrl.h +++ b/third_party/aom/av1/encoder/ratectrl.h @@ -16,6 +16,7 @@ #include "aom/aom_integer.h" #include "av1/common/blockd.h" +#include "av1/common/onyxc_int.h" #ifdef __cplusplus extern "C" { @@ -24,48 +25,24 @@ extern "C" { // Bits Per MB at different Q (Multiplied by 512) #define BPER_MB_NORMBITS 9 -#define CUSTOMIZED_GF 1 +// Threshold used to define if a KF group is static (e.g. a slide show). +// Essentially, this means that no frame in the group has more than 1% of MBs +// that are not marked as coded with 0,0 motion in the first pass. +#define STATIC_KF_GROUP_THRESH 99 +#define STATIC_KF_GROUP_FLOAT_THRESH 0.99 -#if CONFIG_FIX_GF_LENGTH -#define FIXED_GF_LENGTH 16 +// The maximum duration of a GF group that is static (e.g. a slide show). +#define MAX_STATIC_GF_GROUP_LENGTH 250 + +// Minimum and maximum height for the new pyramid structure. +// (Old structure supports height = 1, but does NOT support height = 4). +#define MIN_PYRAMID_LVL 2 #define MAX_PYRAMID_LVL 4 -// We allow a frame to have at most two left/right descendants before changing -// them into to a subtree, i.e., we allow the following structure: -/* OUT_OF_ORDER_FRAME - / / \ \ -(two left children) F F F F (two right children) */ -// Therefore the max gf size supported by 4 layer structure is -// 1 (KEY/OVERLAY) + 1 + 2 + 4 + 16 (two children on both side of their parent) -#define MAX_PYRAMID_SIZE 24 -#define USE_SYMM_MULTI_LAYER 1 -#define REDUCE_LAST_ALT_BOOST 1 -#define REDUCE_LAST_GF_LENGTH 1 -#define MULTI_LVL_BOOST_VBR_CQ 1 -#else -#define USE_SYMM_MULTI_LAYER 0 -#define REDUCE_LAST_ALT_BOOST 0 -#define REDUCE_LAST_GF_LENGTH 0 -#define MULTI_LVL_BOOST_VBR_CQ 0 -#endif - -#if USE_SYMM_MULTI_LAYER -#define USE_MANUAL_GF4_STRUCT 0 -#endif #define MIN_GF_INTERVAL 4 #define MAX_GF_INTERVAL 16 #define FIXED_GF_INTERVAL 8 // Used in some testing modes only -typedef enum { - INTER_NORMAL = 0, - INTER_LOW = 1, - INTER_HIGH = 2, - GF_ARF_LOW = 3, - GF_ARF_STD = 4, - KF_STD = 5, - RATE_FACTOR_LEVELS = 6 -} RATE_FACTOR_LEVEL; - static const double rate_factor_deltas[RATE_FACTOR_LEVELS] = { 1.00, // INTER_NORMAL 0.80, // INTER_LOW @@ -93,7 +70,6 @@ typedef struct { int last_kf_qindex; // Q index of the last key frame coded. int gfu_boost; - int last_boost; int kf_boost; double rate_correction_factors[RATE_FACTOR_LEVELS]; @@ -195,7 +171,10 @@ int av1_rc_get_default_min_gf_interval(int width, int height, double framerate); // Note av1_rc_get_default_max_gf_interval() requires the min_gf_interval to // be passed in to ensure that the max_gf_interval returned is at least as bis // as that. -int av1_rc_get_default_max_gf_interval(double framerate, int min_frame_rate); +int av1_rc_get_default_max_gf_interval(double framerate, int min_frame_rate, + int max_pyr_height); + +int av1_rc_get_fixed_gf_length(int max_pyr_height); // Generally at the high level, the following flow is expected // to be enforced for rate control: @@ -220,8 +199,11 @@ int av1_rc_get_default_max_gf_interval(double framerate, int min_frame_rate); // Functions to set parameters for encoding before the actual // encode_frame_to_data_rate() function. -void av1_rc_get_one_pass_vbr_params(struct AV1_COMP *cpi); -void av1_rc_get_one_pass_cbr_params(struct AV1_COMP *cpi); +struct EncodeFrameParams; +void av1_rc_get_one_pass_vbr_params( + struct AV1_COMP *cpi, struct EncodeFrameParams *const frame_params); +void av1_rc_get_one_pass_cbr_params( + struct AV1_COMP *cpi, struct EncodeFrameParams *const frame_params); // Post encode update of the rate control parameters based // on bytes used @@ -262,9 +244,6 @@ int av1_rc_clamp_iframe_target_size(const struct AV1_COMP *const cpi, int target); int av1_rc_clamp_pframe_target_size(const struct AV1_COMP *const cpi, int target); -// Utility to set frame_target into the RATE_CONTROL structure -// This function is called only from the av1_rc_get_..._params() functions. -void av1_rc_set_frame_target(struct AV1_COMP *cpi, int target); // Computes a q delta (in "q index" terms) to get from a starting q value // to a target q value @@ -288,6 +267,8 @@ void av1_set_target_rate(struct AV1_COMP *cpi, int width, int height); int av1_resize_one_pass_cbr(struct AV1_COMP *cpi); +void av1_estimate_qp_gop(struct AV1_COMP *cpi); + #ifdef __cplusplus } // extern "C" #endif diff --git a/third_party/aom/av1/encoder/rd.c b/third_party/aom/av1/encoder/rd.c index b87d89e50b..f71ec8033b 100644 --- a/third_party/aom/av1/encoder/rd.c +++ b/third_party/aom/av1/encoder/rd.c @@ -91,7 +91,7 @@ void av1_fill_mode_rates(AV1_COMMON *const cm, MACROBLOCK *x, for (i = 0; i < PARTITION_CONTEXTS; ++i) av1_cost_tokens_from_cdf(x->partition_cost[i], fc->partition_cdf[i], NULL); - if (cm->skip_mode_flag) { + if (cm->current_frame.skip_mode_info.skip_mode_flag) { for (i = 0; i < SKIP_CONTEXTS; ++i) { av1_cost_tokens_from_cdf(x->skip_mode_cost[i], fc->skip_mode_cdfs[i], NULL); @@ -353,19 +353,54 @@ static const int rd_frame_type_factor[FRAME_UPDATE_TYPES] = { 128 // INTNL_ARF_UPDATE }; -int av1_compute_rd_mult(const AV1_COMP *cpi, int qindex) { - const int64_t q = - av1_dc_quant_Q3(qindex, 0, cpi->common.seq_params.bit_depth); - int64_t rdmult = 0; +int av1_compute_rd_mult_based_on_qindex(const AV1_COMP *cpi, int qindex) { + const int q = av1_dc_quant_Q3(qindex, 0, cpi->common.seq_params.bit_depth); + int rdmult = q * q; + rdmult = rdmult * 3 + (rdmult * 2 / 3); switch (cpi->common.seq_params.bit_depth) { - case AOM_BITS_8: rdmult = 88 * q * q / 24; break; - case AOM_BITS_10: rdmult = ROUND_POWER_OF_TWO(88 * q * q / 24, 4); break; - case AOM_BITS_12: rdmult = ROUND_POWER_OF_TWO(88 * q * q / 24, 8); break; + case AOM_BITS_8: break; + case AOM_BITS_10: rdmult = ROUND_POWER_OF_TWO(rdmult, 4); break; + case AOM_BITS_12: rdmult = ROUND_POWER_OF_TWO(rdmult, 8); break; default: assert(0 && "bit_depth should be AOM_BITS_8, AOM_BITS_10 or AOM_BITS_12"); return -1; } - if (cpi->oxcf.pass == 2 && (cpi->common.frame_type != KEY_FRAME)) { + return rdmult > 0 ? rdmult : 1; +} + +int av1_compute_rd_mult(const AV1_COMP *cpi, int qindex) { + int64_t rdmult = av1_compute_rd_mult_based_on_qindex(cpi, qindex); + if (cpi->oxcf.pass == 2 && + (cpi->common.current_frame.frame_type != KEY_FRAME)) { + const GF_GROUP *const gf_group = &cpi->twopass.gf_group; + const FRAME_UPDATE_TYPE frame_type = gf_group->update_type[gf_group->index]; + const int boost_index = AOMMIN(15, (cpi->rc.gfu_boost / 100)); + + rdmult = (rdmult * rd_frame_type_factor[frame_type]) >> 7; + rdmult += ((rdmult * rd_boost_factor[boost_index]) >> 7); + } + return (int)rdmult; +} + +int av1_get_adaptive_rdmult(const AV1_COMP *cpi, double beta) { + const AV1_COMMON *cm = &cpi->common; + int64_t q = + av1_dc_quant_Q3(cm->base_qindex, 0, cpi->common.seq_params.bit_depth); + int64_t rdmult = 0; + + switch (cpi->common.seq_params.bit_depth) { + case AOM_BITS_8: rdmult = (int)((88 * q * q / beta) / 24); break; + case AOM_BITS_10: + rdmult = ROUND_POWER_OF_TWO((int)((88 * q * q / beta) / 24), 4); + break; + default: + assert(cpi->common.seq_params.bit_depth == AOM_BITS_12); + rdmult = ROUND_POWER_OF_TWO((int)((88 * q * q / beta) / 24), 8); + break; + } + + if (cpi->oxcf.pass == 2 && + (cpi->common.current_frame.frame_type != KEY_FRAME)) { const GF_GROUP *const gf_group = &cpi->twopass.gf_group; const FRAME_UPDATE_TYPE frame_type = gf_group->update_type[gf_group->index]; const int boost_index = AOMMIN(15, (cpi->rc.gfu_boost / 100)); @@ -434,13 +469,6 @@ static void set_block_thresholds(const AV1_COMMON *cm, RD_OPT *rd) { } } -void av1_set_mvcost(MACROBLOCK *x, int ref, int ref_mv_idx) { - (void)ref; - (void)ref_mv_idx; - x->mvcost = x->mv_cost_stack; - x->nmvjointcost = x->nmv_vec_cost; -} - void av1_fill_coeff_costs(MACROBLOCK *x, FRAME_CONTEXT *fc, const int num_planes) { const int nplanes = AOMMIN(num_planes, PLANE_TYPES); @@ -480,6 +508,17 @@ void av1_fill_coeff_costs(MACROBLOCK *x, FRAME_CONTEXT *fc, av1_cost_tokens_from_cdf(pcost->base_cost[ctx], fc->coeff_base_cdf[tx_size][plane][ctx], NULL); + for (int ctx = 0; ctx < SIG_COEF_CONTEXTS; ++ctx) { + pcost->base_cost[ctx][4] = 0; + pcost->base_cost[ctx][5] = pcost->base_cost[ctx][1] + + av1_cost_literal(1) - + pcost->base_cost[ctx][0]; + pcost->base_cost[ctx][6] = + pcost->base_cost[ctx][2] - pcost->base_cost[ctx][1]; + pcost->base_cost[ctx][7] = + pcost->base_cost[ctx][3] - pcost->base_cost[ctx][2]; + } + for (int ctx = 0; ctx < EOB_COEF_CONTEXTS; ++ctx) av1_cost_tokens_from_cdf(pcost->eob_extra_cost[ctx], fc->eob_extra_cdf[tx_size][plane][ctx], NULL); @@ -510,10 +549,30 @@ void av1_fill_coeff_costs(MACROBLOCK *x, FRAME_CONTEXT *fc, // printf("%5d ", pcost->lps_cost[ctx][i]); // printf("\n"); } + for (int ctx = 0; ctx < LEVEL_CONTEXTS; ++ctx) { + pcost->lps_cost[ctx][0 + COEFF_BASE_RANGE + 1] = + pcost->lps_cost[ctx][0]; + for (int i = 1; i <= COEFF_BASE_RANGE; ++i) { + pcost->lps_cost[ctx][i + COEFF_BASE_RANGE + 1] = + pcost->lps_cost[ctx][i] - pcost->lps_cost[ctx][i - 1]; + } + } } } } +void av1_initialize_cost_tables(const AV1_COMMON *const cm, MACROBLOCK *x) { + if (cm->cur_frame_force_integer_mv) { + av1_build_nmv_cost_table(x->nmv_vec_cost, x->nmvcost, &cm->fc->nmvc, + MV_SUBPEL_NONE); + } else { + av1_build_nmv_cost_table( + x->nmv_vec_cost, + cm->allow_high_precision_mv ? x->nmvcost_hp : x->nmvcost, &cm->fc->nmvc, + cm->allow_high_precision_mv); + } +} + void av1_initialize_rd_consts(AV1_COMP *cpi) { AV1_COMMON *const cm = &cpi->common; MACROBLOCK *const x = &cpi->td.mb; @@ -527,18 +586,7 @@ void av1_initialize_rd_consts(AV1_COMP *cpi) { set_block_thresholds(cm, rd); - if (cm->cur_frame_force_integer_mv) { - av1_build_nmv_cost_table(x->nmv_vec_cost, x->nmvcost, &cm->fc->nmvc, - MV_SUBPEL_NONE); - } else { - av1_build_nmv_cost_table( - x->nmv_vec_cost, - cm->allow_high_precision_mv ? x->nmvcost_hp : x->nmvcost, &cm->fc->nmvc, - cm->allow_high_precision_mv); - } - - x->mvcost = x->mv_cost_stack; - x->nmvjointcost = x->nmv_vec_cost; + av1_initialize_cost_tables(cm, x); if (frame_is_intra_only(cm) && cm->allow_screen_content_tools && cpi->oxcf.pass != 1) { @@ -665,385 +713,601 @@ static double interp_bicubic(const double *p, int p_stride, double x, return interp_cubic(q, y); } -static const double interp_rgrid_surf[65 * 18] = { - 0.104019, 0.245714, 0.293686, 0.358635, 0.382167, 0.412446, - 0.419955, 0.421388, 0.426672, 0.427990, 0.428531, 0.456868, - 0.569880, 0.638822, 1.016319, 2.143453, 3.565229, 4.720880, - 0.124618, 0.294211, 0.352023, 0.429991, 0.458206, 0.494510, - 0.503513, 0.505232, 0.511566, 0.513234, 0.519365, 0.570225, - 0.697373, 0.840624, 1.462198, 3.289054, 6.256517, 6.852788, - 0.118630, 0.269669, 0.346620, 0.430999, 0.459385, 0.495783, - 0.504808, 0.506532, 0.512884, 0.514988, 0.543437, 0.662772, - 0.795876, 1.313596, 2.403841, 4.163098, 7.440589, 8.616275, - 0.093329, 0.168205, 0.321320, 0.430607, 0.459385, 0.495783, - 0.504813, 0.506548, 0.512975, 0.520662, 0.571659, 0.701841, - 1.010727, 2.138851, 3.460626, 6.317955, 10.098127, 14.418553, - 0.087021, 0.142905, 0.315011, 0.430509, 0.459385, 0.495787, - 0.505075, 0.507599, 0.513584, 0.543182, 0.669941, 0.825620, - 1.362800, 2.572187, 4.205047, 7.498399, 12.303118, 16.641735, - 0.086923, 0.142513, 0.314913, 0.430508, 0.459385, 0.495803, - 0.506126, 0.511816, 0.514810, 0.549705, 0.725350, 1.127334, - 2.168597, 3.463686, 6.318605, 10.162284, 18.556041, 19.847042, - 0.086923, 0.142513, 0.314913, 0.430506, 0.459376, 0.495805, - 0.506388, 0.512954, 0.520772, 0.580215, 0.810474, 1.391548, - 2.579442, 4.205160, 7.498399, 12.381597, 21.703618, 24.015457, - 0.086923, 0.142513, 0.314911, 0.430353, 0.458765, 0.495652, - 0.506391, 0.513406, 0.544098, 0.702950, 1.121860, 2.168961, - 3.463798, 6.318607, 10.162284, 18.685361, 28.188192, 37.638872, - 0.086923, 0.142513, 0.314901, 0.429742, 0.456313, 0.495045, - 0.506484, 0.519195, 0.580104, 0.810126, 1.391462, 2.579441, - 4.205160, 7.498399, 12.381597, 21.848607, 33.367199, 42.623190, - 0.086923, 0.142513, 0.314899, 0.429589, 0.455706, 0.495155, - 0.507882, 0.542426, 0.702360, 1.119921, 2.168478, 3.463791, - 6.318607, 10.162284, 18.685361, 28.345760, 47.802028, 49.163533, - 0.086924, 0.142548, 0.315086, 0.429842, 0.455870, 0.496336, - 0.512412, 0.556953, 0.773373, 1.266396, 2.548277, 4.204676, - 7.498399, 12.381597, 21.848607, 33.548250, 54.301011, 56.262859, - 0.087067, 0.144957, 0.327436, 0.446616, 0.466362, 0.505706, - 0.522077, 0.610747, 0.972543, 1.666916, 3.338812, 6.316669, - 10.162284, 18.685361, 28.345760, 48.065311, 66.145302, 78.396020, - 0.094295, 0.164235, 0.393722, 0.534219, 0.530922, 0.579308, - 0.603889, 0.760870, 1.229961, 2.423214, 4.173513, 7.497916, - 12.381597, 21.848607, 33.548250, 54.589585, 74.875848, 86.468182, - 0.124096, 0.213005, 0.497188, 0.665176, 0.685973, 0.800200, - 0.911394, 1.077971, 1.677290, 3.332129, 6.314960, 10.162257, - 18.685361, 28.345760, 48.065311, 66.453506, 98.275189, 96.862588, - 0.140999, 0.270140, 0.658212, 0.867661, 0.970183, 1.149516, - 1.480599, 1.664833, 2.421893, 3.857981, 7.418830, 12.380371, - 21.848607, 33.548250, 54.589585, 75.188867, 106.657971, 99.762997, - 0.178353, 0.398001, 0.988462, 1.241473, 1.340967, 1.713568, - 2.335030, 2.701432, 3.348532, 5.077158, 9.829903, 18.676528, - 28.345700, 48.065311, 66.453506, 98.588283, 117.057193, 101.130722, - 0.281079, 0.548300, 1.395825, 1.780770, 2.000508, 2.702964, - 3.638454, 4.573843, 5.051641, 7.079129, 11.293332, 21.594861, - 33.544335, 54.589585, 75.188867, 106.971065, 119.957601, 101.466632, - 0.476762, 0.842189, 2.019678, 2.723895, 3.188467, 4.011610, - 5.545111, 7.508984, 8.176339, 9.774504, 14.720782, 27.334416, - 48.049609, 66.453506, 98.588283, 117.370357, 121.329855, 101.509242, - 0.993999, 1.520111, 3.013605, 4.203530, 4.982992, 6.074944, - 8.583581, 11.818375, 14.192544, 14.937517, 21.258160, 33.305953, - 54.585735, 75.188867, 106.971135, 120.279824, 121.976055, 102.690130, - 1.776487, 2.613655, 4.356487, 6.161726, 7.622196, 9.464193, - 13.077233, 18.051656, 23.221051, 24.080068, 30.085038, 48.345269, - 66.457698, 98.588353, 117.379415, 121.976128, 124.356210, 107.713202, - 3.191085, 4.495201, 5.686033, 8.365566, 11.275339, 14.706437, - 20.300969, 28.152237, 35.688355, 39.341382, 41.030743, 55.752262, - 75.211764, 106.980285, 120.608403, 124.680746, 130.222528, 112.260098, - 6.136611, 7.305215, 7.272532, 10.646713, 15.630815, 22.383168, - 31.349131, 42.419822, 52.301680, 58.983454, 58.915405, 69.161305, - 98.992460, 117.713855, 124.344836, 130.623638, 138.442401, 127.846670, - 11.707980, 13.490761, 11.640845, 14.176132, 22.131124, 33.776462, - 47.365711, 61.603834, 75.281056, 83.463985, 85.510533, 86.026513, - 108.787480, 123.031136, 130.607284, 138.954406, 160.867784, 158.958882, - 27.062874, 32.195139, 24.147297, 22.114632, 35.580506, 52.551674, - 71.652956, 88.606776, 102.107193, 110.703186, 114.398733, 111.118539, - 121.503578, 132.455924, 139.490806, 161.412674, 193.563210, 172.203945, - 35.625692, 47.953028, 42.639820, 42.276254, 58.815664, 84.977282, - 110.656412, 126.168446, 134.658126, 140.604482, 144.006012, 141.702382, - 140.125323, 153.122630, 164.748041, 194.156197, 206.854650, 174.013079, - 49.516447, 65.335381, 71.738306, 81.872819, 98.400740, 136.840488, - 163.775802, 169.440078, 172.747876, 171.222919, 171.679604, 172.173550, - 168.200129, 187.617133, 199.683394, 207.768200, 210.062520, 175.478356, - 60.341673, 92.487135, 119.907299, 136.068010, 144.778950, 189.443534, - 220.120077, 219.641635, 214.616503, 205.894657, 198.453924, 200.013069, - 195.938103, 206.118661, 210.447375, 212.061379, 216.078218, 181.162805, - 78.422159, 112.242899, 158.416312, 181.404320, 193.188690, 229.296967, - 270.461799, 275.168977, 256.511701, 244.706786, 231.344608, 226.065087, - 222.248618, 218.662324, 217.966722, 218.248574, 218.818588, 182.740573, - 88.713664, 123.594164, 172.928179, 213.781414, 245.800351, 252.063414, - 313.283141, 331.703831, 305.866639, 285.177142, 269.759635, 251.988739, - 245.998388, 232.688076, 230.588702, 230.882657, 230.319053, 192.120741, - 102.540561, 152.905927, 189.137131, 241.806756, 273.868497, 284.258017, - 339.689853, 373.561104, 362.657463, 326.291984, 311.922687, 290.460189, - 276.774381, 273.012072, 277.751792, 279.123748, 278.820447, 233.813798, - 132.983118, 176.307242, 197.415684, 243.307787, 280.893995, 332.922370, - 340.329043, 404.530166, 419.475405, 375.775209, 351.300889, 340.042759, - 315.683832, 306.123530, 306.359319, 306.733063, 307.609556, 261.647847, - 149.579109, 185.925581, 207.937033, 245.159084, 301.890957, 350.040480, - 352.250771, 418.742329, 458.112686, 430.125208, 386.460441, 380.346839, - 354.679150, 337.305620, 334.504124, 335.889932, 341.060725, 286.898578, - 153.576812, 202.105624, 219.366967, 248.524506, 314.255692, 350.607526, - 390.567688, 408.629209, 488.000213, 480.563823, 432.461799, 410.412624, - 398.607371, 400.188740, 402.780916, 408.853470, 430.449735, 363.777088, - 161.353129, 214.848904, 231.549852, 258.536466, 313.163177, 368.140577, - 412.136393, 413.409032, 499.838438, 519.571063, 485.833867, 444.562715, - 435.738129, 442.358549, 450.166531, 453.208524, 458.424358, 385.823139, - 175.109034, 227.608058, 250.069563, 286.101747, 312.256740, 378.421485, - 413.344147, 435.058646, 476.960941, 542.448886, 530.189154, 495.408402, - 475.326752, 465.017144, 464.694045, 465.144689, 466.905382, 398.669138, - 184.750180, 240.766694, 283.240772, 305.480150, 322.409001, 374.526162, - 427.141326, 452.840323, 472.604139, 545.366105, 567.676694, 541.666203, - 509.591873, 492.044219, 492.778569, 493.765684, 493.235693, 413.684325, - 194.728357, 254.928927, 289.991157, 300.193195, 324.194589, 371.563147, - 439.226438, 468.295088, 495.654854, 533.506353, 587.476353, 578.298989, - 548.041942, 527.393885, 538.965146, 545.070442, 544.295454, 454.012211, - 205.195287, 283.135677, 297.921431, 319.295927, 355.621830, 392.466463, - 446.696167, 485.053519, 516.426615, 532.264584, 588.481600, 615.906737, - 589.319634, 555.754316, 558.389367, 569.094521, 569.779764, 475.384946, - 218.552054, 298.511016, 319.188338, 351.781666, 372.789510, 412.827434, - 464.569387, 506.270203, 533.049810, 553.347364, 580.644599, 632.759854, - 622.235843, 569.960552, 580.799340, 586.553714, 579.488366, 491.826482, - 244.803348, 299.790203, 324.187975, 363.280782, 403.710443, 441.724083, - 492.732682, 534.722691, 552.193622, 575.112647, 586.097705, 635.224970, - 644.642944, 606.017786, 640.321218, 642.316989, 616.397020, 548.300111, - 256.957358, 318.638991, 355.063346, 389.889307, 433.607315, 468.209001, - 515.178157, 573.556591, 578.113115, 587.246475, 601.762801, 638.454644, - 656.574853, 641.184609, 676.908189, 684.198162, 678.387412, 574.805864, - 251.211502, 323.448532, 364.227424, 411.792704, 462.226488, 503.572288, - 549.299249, 599.124071, 601.227977, 597.118176, 613.247552, 633.278532, - 658.074755, 664.930719, 685.731531, 693.632845, 693.076350, 578.326477, - 267.695377, 354.273736, 389.976833, 438.518178, 493.332686, 544.343027, - 588.895829, 620.206193, 628.327410, 606.067827, 620.998532, 657.985256, - 683.936059, 691.345257, 693.894723, 695.175306, 693.618786, 578.517148, - 274.290725, 363.465288, 411.808596, 463.369805, 515.310226, 581.009306, - 613.070738, 636.638714, 647.333929, 629.867603, 644.646319, 687.796202, - 702.859596, 713.495479, 704.068069, 704.991807, 704.188594, 587.283658, - 302.538449, 389.174737, 438.518422, 493.398902, 547.662399, 601.981814, - 624.773046, 641.629484, 644.699451, 645.848784, 668.033340, 703.643523, - 707.422408, 717.329600, 726.298973, 744.127507, 745.365167, 617.954068, - 310.328188, 410.984766, 463.369805, 515.315010, 581.309832, 613.787792, - 634.988538, 654.145284, 662.632978, 668.413496, 706.494057, 750.545471, - 730.724808, 730.002100, 743.625262, 750.801609, 745.308457, 606.505800, - 329.948756, 437.600191, 493.398902, 547.661910, 601.917884, 622.557745, - 633.244395, 644.055898, 648.224221, 665.062911, 763.555733, 812.391078, - 769.063582, 744.865168, 727.579796, 724.950408, 722.179707, 598.564510, - 350.848328, 462.437458, 515.315010, 581.309823, 613.779123, 634.465309, - 652.056257, 662.179143, 671.466297, 726.881256, 819.824030, 880.232789, - 810.371672, 754.246481, 725.053473, 724.253390, 723.503395, 603.394909, - 373.704088, 492.408266, 547.661910, 601.917884, 622.557620, 633.236320, - 644.023513, 648.232514, 666.381639, 785.498283, 929.441612, 999.772800, - 890.339033, 775.852504, 731.840181, 726.905100, 725.251844, 604.899901, - 394.473422, 514.261306, 581.309823, 613.779123, 634.465309, 652.056257, - 662.179143, 671.466557, 727.134512, 835.764144, 981.747089, 1018.462934, - 939.686967, 811.276731, 739.398459, 727.365647, 725.285425, 604.923525, - 419.976505, 546.538939, 601.917884, 622.557620, 633.236320, 644.023513, - 648.232514, 666.381639, 785.545191, 932.841398, 1036.609617, 1026.945092, - 963.822765, 840.827315, 755.532423, 730.241865, 725.366847, 604.924155, - 437.281359, 580.116337, 613.779123, 634.465309, 652.056257, 662.179143, - 671.466557, 727.134512, 835.764859, 981.996194, 1031.896881, 1002.544732, - 881.157178, 828.151494, 799.340975, 751.314325, 728.316587, 605.005504, - 464.713920, 600.649281, 622.557620, 633.236320, 644.023513, 648.232514, - 666.381639, 785.545191, 932.841398, 1036.735329, 1035.037004, 995.478339, - 858.093733, 823.471976, 819.881754, 798.749289, 749.440463, 607.955244, - 495.880237, 612.473139, 634.465309, 652.056257, 662.179143, 671.466557, - 727.134512, 835.764859, 981.996194, 1032.339788, 1031.105117, 995.303259, - 857.733663, 823.435877, 822.822791, 819.873050, 796.882480, 629.038445, - 510.391280, 621.158273, 633.236320, 644.023513, 648.232514, 666.381639, - 785.545191, 932.841398, 1036.735329, 1035.566013, 1029.599350, 994.926093, - 857.645648, 823.435143, 822.904139, 822.822791, 817.965681, 673.856962, - 514.588176, 632.947715, 652.056257, 662.179143, 671.466557, 727.134512, - 835.764859, 981.996194, 1032.339788, 1031.547475, 1023.835377, 972.158629, - 851.968626, 823.347128, 822.904770, 822.904139, 820.752301, 684.418900, - 520.013294, 631.668183, 644.023513, 648.232514, 666.381639, 785.545191, - 932.841398, 1036.735329, 1035.567378, 1029.776746, 1001.044108, 880.853721, - 829.201546, 822.994150, 822.904770, 822.904770, 820.792975, 684.582020, - 531.253628, 650.479606, 662.179143, 671.466557, 727.134512, 835.764859, - 981.996194, 1032.339788, 1031.636855, 1029.601779, 995.366703, 858.086641, - 823.524524, 822.906135, 822.904770, 822.904770, 820.792975, 684.582020, - 528.531744, 642.424501, 648.232514, 666.381639, 785.545191, 932.841398, - 1036.735329, 1035.567378, 1030.219103, 1029.576226, 995.278687, 857.733663, - 823.436508, 822.904770, 822.904770, 822.904770, 820.792975, 684.582020, - 545.401164, 660.550678, 671.508859, 727.304161, 835.807162, 981.996850, - 1032.339788, 1031.636855, 1030.130788, 1029.487827, 994.925709, 857.645648, - 823.435143, 822.904770, 822.904770, 822.904770, 820.792975, 684.582020, - 537.684760, 646.650947, 669.110131, 796.487512, 935.569890, 1036.777631, - 1035.567378, 1030.219103, 1030.018584, 1023.810805, 972.158629, 851.968626, - 823.347128, 822.904770, 822.904770, 822.904770, 820.792975, 684.582020, - 552.408370, 670.001885, 738.246482, 879.690154, 992.939171, 1032.509436, - 1031.636855, 1030.132153, 1029.665223, 1001.043724, 880.853721, 829.201546, - 822.994150, 822.904770, 822.904770, 822.904770, 820.792975, 684.582020, - 539.835902, 667.496388, 799.216004, 946.512211, 1039.506123, 1035.609680, - 1030.219103, 1030.107964, 1029.577207, 995.366703, 858.086641, 823.524524, - 822.906135, 822.904770, 822.904770, 822.904770, 820.792975, 684.582020, - 558.362529, 734.277451, 877.197218, 990.478243, 1029.908393, 1028.993978, - 1027.488620, 1027.464048, 1026.933674, 992.724534, 855.532488, 821.323349, - 820.792975, 820.792975, 820.792975, 820.792975, 818.686600, 682.825198, - 453.127195, 649.075095, 780.278390, 867.165890, 862.469711, 857.067460, - 856.956321, 856.955937, 856.513579, 827.981461, 713.556496, 685.024378, - 684.582020, 684.582020, 684.582020, 684.582020, 682.825198, 569.510056, +static const uint8_t bsize_model_cat_lookup[BLOCK_SIZES_ALL] = { + 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 3, 3, 3, 0, 0, 1, 1, 2, 2 }; -static const double interp_dgrid_surf[65 * 18] = { - 10.650434, 12.204694, 12.040917, 11.843008, 11.845578, 12.051535, 12.103583, - 12.136780, 12.266709, 12.299107, 12.299673, 12.303120, 12.316337, 12.293431, - 12.092165, 11.602421, 11.141559, 8.864495, 12.770003, 14.634889, 14.437149, - 14.199413, 14.202487, 14.449423, 14.511827, 14.551629, 14.707410, 14.746265, - 14.747610, 14.753705, 14.762194, 14.699395, 14.390525, 13.690970, 12.874168, - 10.367121, 12.832328, 14.790730, 14.503765, 14.236403, 14.239028, 14.486600, - 14.549164, 14.589069, 14.745250, 14.784258, 14.788320, 14.801930, 14.762798, - 14.499088, 14.021544, 13.469684, 12.661560, 10.108384, 12.950520, 15.264726, - 14.621957, 14.238236, 14.239028, 14.486601, 14.549264, 14.589469, 14.745361, - 14.784949, 14.791572, 14.798652, 14.660251, 14.119394, 13.651131, 12.935657, - 12.176082, 9.228999, 12.979992, 15.382918, 14.651428, 14.238693, 14.239028, - 14.486701, 14.555710, 14.615321, 14.751849, 14.787700, 14.797104, 14.743189, - 14.475057, 13.944406, 13.450468, 12.687876, 11.824993, 8.906683, 12.980449, - 15.384750, 14.651885, 14.238700, 14.239028, 14.487102, 14.581562, 14.718998, - 14.777721, 14.788445, 14.778661, 14.582790, 14.099785, 13.649637, 12.935359, - 12.201859, 10.891931, 8.482221, 12.980449, 15.384750, 14.651886, 14.238801, - 14.239434, 14.487303, 14.588010, 14.744860, 14.784773, 14.786094, 14.735647, - 14.455704, 13.939591, 13.450393, 12.687876, 11.849334, 10.476658, 8.043672, - 12.980449, 15.384750, 14.651987, 14.245320, 14.265579, 14.493824, 14.588211, - 14.745312, 14.787263, 14.775934, 14.582036, 14.099475, 13.649563, 12.935358, - 12.201859, 10.911285, 9.730570, 6.696921, 12.980449, 15.384750, 14.652393, - 14.271466, 14.370434, 14.520069, 14.589027, 14.746028, 14.785482, 14.735605, - 14.455693, 13.939590, 13.450393, 12.687876, 11.849334, 10.494514, 9.195398, - 6.215460, 12.980449, 15.384750, 14.652494, 14.277985, 14.396679, 14.533035, - 14.615021, 14.754825, 14.775610, 14.582796, 14.099664, 13.649565, 12.935358, - 12.201859, 10.911285, 9.747361, 7.779960, 5.617541, 12.980448, 15.384731, - 14.652415, 14.278078, 14.397578, 14.559053, 14.718657, 14.776398, 14.747044, - 14.504690, 13.951810, 13.450583, 12.687876, 11.849334, 10.494514, 9.210817, - 7.210003, 5.164575, 12.980446, 15.383448, 14.647073, 14.277541, 14.403813, - 14.569546, 14.744956, 14.765103, 14.629073, 14.296161, 13.698573, 12.936118, - 12.201859, 10.911285, 9.747361, 7.790897, 6.322998, 3.931551, 12.981550, - 15.376916, 14.615597, 14.274820, 14.437479, 14.575942, 14.707492, 14.734111, - 14.515975, 14.000806, 13.462803, 12.688066, 11.849334, 10.494514, 9.210817, - 7.219566, 5.781392, 3.486081, 12.991899, 15.376201, 14.579444, 14.296898, - 14.473361, 14.522910, 14.491600, 14.543267, 14.288580, 13.700311, 12.936579, - 12.201867, 10.911285, 9.747361, 7.790897, 6.331506, 4.480348, 2.923138, - 13.019848, 15.383477, 14.582260, 14.385262, 14.452673, 14.436019, 14.238174, - 14.255993, 13.977481, 13.532342, 12.705591, 11.849605, 10.494514, 9.210817, - 7.219566, 5.789642, 4.018194, 2.766222, 13.028558, 15.315782, 14.439141, - 14.326286, 14.452429, 14.311731, 14.033235, 13.922587, 13.665868, 13.207897, - 12.274375, 10.912967, 9.747371, 7.790897, 6.331506, 4.488594, 3.454993, - 2.692682, 12.992752, 15.321471, 14.409573, 14.236340, 14.322969, 14.049072, - 13.764823, 13.479242, 13.250105, 12.759133, 12.019174, 10.532951, 9.211409, - 7.219566, 5.789642, 4.026440, 3.298077, 2.674624, 12.945493, 15.276596, - 14.315745, 14.026198, 14.085774, 13.844563, 13.447576, 12.964935, 12.735525, - 12.288592, 11.511693, 9.900227, 7.793270, 6.331506, 4.488594, 3.463236, - 3.224318, 2.672433, 12.757570, 15.056661, 14.095011, 13.722362, 13.812624, - 13.608480, 13.021206, 12.367627, 11.937931, 11.581049, 10.599552, 9.247860, - 7.220151, 5.789642, 4.026437, 3.305882, 3.191260, 2.615317, 12.581293, - 14.824658, 13.909074, 13.496158, 13.491402, 13.221550, 12.514140, 11.677229, - 10.936895, 10.619912, 9.634779, 7.763570, 6.331082, 4.488590, 3.462798, - 3.216460, 3.076315, 2.373499, 12.283499, 14.455760, 13.890593, 13.427587, - 13.183783, 12.763833, 11.861006, 10.740618, 9.820756, 9.354945, 8.669862, - 7.123268, 5.787860, 4.025994, 3.290000, 3.084410, 2.810905, 2.222916, - 12.010893, 14.300919, 13.986624, 13.484026, 13.025385, 12.224281, 11.064265, - 9.631040, 8.594396, 8.003736, 7.561587, 6.274418, 4.466637, 3.446574, - 3.102467, 2.816989, 2.598688, 1.951541, 11.581477, 13.831132, 13.632027, - 13.380414, 12.807880, 11.665651, 10.218236, 8.562237, 7.222614, 6.611808, - 6.261676, 5.402793, 3.938544, 3.174375, 2.818166, 2.602758, 2.213911, - 1.434763, 11.050735, 12.893449, 12.363152, 12.712829, 12.012961, 10.887854, - 9.109699, 7.421701, 5.965603, 5.272129, 4.991435, 4.423000, 3.369988, - 2.800371, 2.593901, 2.217431, 1.670917, 1.215265, 10.641194, 11.766277, - 10.777082, 10.972917, 10.689298, 9.701545, 7.719947, 6.145654, 4.872442, - 4.099600, 3.880934, 3.514159, 2.786474, 2.368963, 2.162376, 1.673670, - 1.450770, 1.185424, 10.071964, 11.107701, 9.172361, 8.551313, 8.412080, - 7.641397, 6.174246, 4.853916, 3.904549, 3.246810, 2.959903, 2.785066, - 2.240001, 1.793166, 1.585520, 1.449824, 1.405368, 1.168856, 9.213182, - 9.173278, 7.219231, 6.242951, 5.626013, 5.768007, 4.908666, 3.809589, - 3.115109, 2.617899, 2.274793, 2.172960, 1.838597, 1.505915, 1.414333, - 1.392666, 1.338173, 1.105611, 7.365015, 7.471370, 5.622346, 4.520127, - 3.936272, 4.208822, 3.623024, 2.977794, 2.450003, 2.097261, 1.824090, - 1.643270, 1.473525, 1.351388, 1.327504, 1.323865, 1.307894, 1.088234, - 6.198210, 6.580712, 4.682511, 3.416952, 2.941929, 2.766637, 2.650686, - 2.315439, 1.925838, 1.659784, 1.464419, 1.252806, 1.162722, 1.197518, - 1.199875, 1.197365, 1.194040, 0.995797, 5.402507, 5.055466, 3.728724, - 2.624359, 2.165810, 1.943189, 1.918190, 1.738078, 1.516328, 1.290520, - 1.155793, 1.015962, 0.881900, 0.807203, 0.754242, 0.743378, 0.740288, - 0.614158, 3.937867, 3.862507, 2.884664, 2.088147, 1.648496, 1.473584, - 1.340123, 1.291769, 1.165381, 1.000224, 0.893316, 0.821333, 0.691363, - 0.610501, 0.586766, 0.583762, 0.577840, 0.468733, 3.104660, 3.181078, - 2.420208, 1.747442, 1.297956, 1.109835, 0.970385, 0.943229, 0.876923, - 0.777584, 0.678183, 0.628623, 0.553745, 0.523430, 0.519490, 0.514394, - 0.492259, 0.403172, 2.593833, 2.533720, 2.010452, 1.480944, 1.060302, - 0.846383, 0.738703, 0.673144, 0.658010, 0.592449, 0.518236, 0.470335, - 0.425088, 0.393168, 0.378116, 0.355846, 0.275469, 0.213128, 2.176988, - 2.089575, 1.671284, 1.225008, 0.895382, 0.672008, 0.566241, 0.496746, - 0.488005, 0.449874, 0.400899, 0.354002, 0.318150, 0.281533, 0.238545, - 0.224159, 0.202399, 0.160681, 1.874679, 1.769165, 1.430124, 1.068727, - 0.780272, 0.557801, 0.441643, 0.377256, 0.352957, 0.338452, 0.304965, - 0.273172, 0.240052, 0.208724, 0.193431, 0.190845, 0.185025, 0.138166, - 1.590226, 1.502830, 1.193127, 0.917885, 0.670432, 0.474546, 0.355420, - 0.292305, 0.259035, 0.249937, 0.232079, 0.208943, 0.181936, 0.160038, - 0.152257, 0.151235, 0.149583, 0.120747, 1.331730, 1.255907, 1.012871, - 0.778422, 0.578977, 0.412432, 0.293155, 0.231824, 0.197187, 0.183921, - 0.174876, 0.157252, 0.140263, 0.127050, 0.110244, 0.105041, 0.104323, - 0.086944, 1.153994, 1.118771, 0.822355, 0.612321, 0.478249, 0.348222, - 0.247408, 0.186141, 0.152714, 0.135445, 0.129810, 0.119994, 0.115619, - 0.131626, 0.095612, 0.079343, 0.077502, 0.064550, 0.946317, 0.925894, - 0.677969, 0.499906, 0.397101, 0.297931, 0.214467, 0.152333, 0.120731, - 0.102686, 0.095062, 0.090361, 0.122319, 0.240194, 0.112687, 0.070690, - 0.070461, 0.054194, 0.824155, 0.787241, 0.581856, 0.419228, 0.313167, - 0.245582, 0.183500, 0.128101, 0.096577, 0.080267, 0.071022, 0.066851, - 0.085754, 0.154163, 0.075884, 0.052401, 0.054270, 0.026656, 0.716310, - 0.671378, 0.489580, 0.349569, 0.256155, 0.206343, 0.157853, 0.111950, - 0.079271, 0.062518, 0.053441, 0.049660, 0.051400, 0.063778, 0.039993, - 0.029133, 0.023382, 0.013725, 0.614125, 0.579096, 0.417126, 0.299465, - 0.217849, 0.165515, 0.129040, 0.093127, 0.065612, 0.049543, 0.041429, - 0.036850, 0.034416, 0.033989, 0.024216, 0.017377, 0.014833, 0.011987, - 0.520407, 0.487239, 0.349473, 0.251741, 0.184897, 0.135813, 0.107098, - 0.073607, 0.053938, 0.040531, 0.032931, 0.028876, 0.025759, 0.022168, - 0.016739, 0.014638, 0.014333, 0.011947, 0.449954, 0.415124, 0.299452, - 0.216942, 0.158874, 0.115334, 0.088821, 0.060105, 0.042610, 0.032566, - 0.026903, 0.023123, 0.019913, 0.016835, 0.014306, 0.013625, 0.013535, - 0.011284, 0.377618, 0.347773, 0.251741, 0.184839, 0.132857, 0.095439, - 0.070462, 0.052244, 0.036078, 0.026025, 0.021518, 0.018487, 0.015361, - 0.012905, 0.011470, 0.010569, 0.010283, 0.008297, 0.319953, 0.297976, - 0.216942, 0.158842, 0.113280, 0.080426, 0.057367, 0.041987, 0.030135, - 0.022295, 0.017901, 0.015121, 0.012224, 0.010035, 0.009353, 0.009108, - 0.008695, 0.006139, 0.267864, 0.250502, 0.184839, 0.132851, 0.095039, - 0.068220, 0.049135, 0.035315, 0.025144, 0.018237, 0.013857, 0.012094, - 0.009715, 0.007743, 0.006937, 0.006446, 0.006243, 0.004929, 0.230449, - 0.215895, 0.158842, 0.113280, 0.080417, 0.057174, 0.041304, 0.029959, - 0.021866, 0.015673, 0.012133, 0.010083, 0.007801, 0.006053, 0.005401, - 0.003834, 0.003429, 0.002851, 0.193984, 0.183963, 0.132851, 0.095039, - 0.068220, 0.049133, 0.035305, 0.025140, 0.018150, 0.013175, 0.010422, - 0.008491, 0.006397, 0.004567, 0.003494, 0.002933, 0.002825, 0.002355, - 0.167298, 0.158088, 0.113280, 0.080417, 0.057174, 0.041304, 0.029959, - 0.021866, 0.015669, 0.011955, 0.009257, 0.007051, 0.005543, 0.003905, - 0.002984, 0.002825, 0.002814, 0.002347, 0.143228, 0.132220, 0.095039, - 0.068220, 0.049133, 0.035305, 0.025140, 0.018150, 0.013174, 0.010394, - 0.008403, 0.006661, 0.005378, 0.003545, 0.002876, 0.002818, 0.002814, - 0.002347, 0.122934, 0.112735, 0.080417, 0.057174, 0.041304, 0.029959, - 0.021866, 0.015669, 0.011955, 0.009258, 0.007182, 0.006012, 0.003762, - 0.002866, 0.002739, 0.002788, 0.002810, 0.002347, 0.101934, 0.094569, - 0.068220, 0.049133, 0.035305, 0.025140, 0.018150, 0.013174, 0.010394, - 0.008405, 0.006797, 0.005845, 0.003333, 0.002703, 0.002695, 0.002723, - 0.002781, 0.002343, 0.086702, 0.080014, 0.057174, 0.041304, 0.029959, - 0.021866, 0.015669, 0.011955, 0.009258, 0.007190, 0.006533, 0.005839, - 0.003326, 0.002700, 0.002690, 0.002694, 0.002716, 0.002314, 0.073040, - 0.067886, 0.049133, 0.035305, 0.025140, 0.018150, 0.013174, 0.010394, - 0.008405, 0.006807, 0.006468, 0.005831, 0.003325, 0.002700, 0.002690, - 0.002690, 0.002687, 0.002253, 0.061685, 0.056890, 0.041304, 0.029959, - 0.021866, 0.015669, 0.011955, 0.009258, 0.007190, 0.006542, 0.006360, - 0.005416, 0.003221, 0.002698, 0.002690, 0.002690, 0.002683, 0.002238, - 0.052465, 0.048894, 0.035305, 0.025140, 0.018150, 0.013174, 0.010394, - 0.008405, 0.006807, 0.006472, 0.005943, 0.003748, 0.002805, 0.002692, - 0.002690, 0.002690, 0.002683, 0.002238, 0.043838, 0.041101, 0.029959, - 0.021866, 0.015669, 0.011955, 0.009258, 0.007190, 0.006543, 0.006465, - 0.005839, 0.003333, 0.002702, 0.002690, 0.002690, 0.002690, 0.002683, - 0.002238, 0.037824, 0.035133, 0.025140, 0.018150, 0.013174, 0.010394, - 0.008405, 0.006807, 0.006480, 0.006464, 0.005838, 0.003326, 0.002700, - 0.002690, 0.002690, 0.002690, 0.002683, 0.002238, 0.031865, 0.029815, - 0.021866, 0.015668, 0.011955, 0.009258, 0.007190, 0.006543, 0.006475, - 0.006462, 0.005831, 0.003325, 0.002700, 0.002690, 0.002690, 0.002690, - 0.002683, 0.002238, 0.027150, 0.025016, 0.018128, 0.013083, 0.010371, - 0.008405, 0.006807, 0.006480, 0.006472, 0.006359, 0.005416, 0.003221, - 0.002698, 0.002690, 0.002690, 0.002690, 0.002683, 0.002238, 0.023094, - 0.021760, 0.015577, 0.011590, 0.009167, 0.007188, 0.006543, 0.006475, - 0.006466, 0.005943, 0.003748, 0.002805, 0.002692, 0.002690, 0.002690, - 0.002690, 0.002683, 0.002238, 0.019269, 0.018038, 0.013060, 0.010280, - 0.008382, 0.006806, 0.006480, 0.006474, 0.006464, 0.005839, 0.003333, - 0.002702, 0.002690, 0.002690, 0.002690, 0.002690, 0.002683, 0.002238, - 0.016874, 0.015472, 0.011566, 0.009148, 0.007171, 0.006527, 0.006458, - 0.006457, 0.006447, 0.005823, 0.003318, 0.002693, 0.002683, 0.002683, - 0.002683, 0.002683, 0.002676, 0.002232, 0.011968, 0.011056, 0.008762, - 0.007219, 0.005717, 0.005391, 0.005386, 0.005386, 0.005377, 0.004856, - 0.002767, 0.002246, 0.002238, 0.002238, 0.002238, 0.002238, 0.002232, - 0.001862, +static int sse_norm_model_cat_lookup(double sse_norm) { + return (sse_norm > 16.0); +} + +static const double interp_rgrid_surf[4][33 * 18] = { + { + 29.726102, 30.738006, 25.294088, 25.736759, 41.255961, + 41.937828, 41.937901, 42.110487, 46.038554, 46.593284, + 55.290144, 55.672257, 55.672289, 55.672284, 55.641633, + 54.251766, 38.404522, 36.186584, 30.977201, 32.031695, + 26.358675, 26.819956, 42.992269, 43.702888, 43.702966, + 43.882830, 47.992288, 48.920311, 57.633255, 58.015369, + 58.015396, 57.984745, 56.594876, 40.717227, 39.327227, + 37.709563, 30.977308, 32.041872, 26.587891, 26.801416, + 42.339251, 43.674340, 43.710377, 44.048333, 48.365747, + 57.251292, 57.999483, 58.015595, 57.984977, 56.595076, + 40.717426, 39.327559, 39.313319, 38.085580, 30.977328, + 32.271000, 31.802769, 26.377378, 27.473270, 43.021187, + 43.875643, 47.811034, 48.547182, 57.617338, 58.015563, + 58.001989, 56.982277, 40.734438, 39.327560, 39.313487, + 40.065111, 46.657660, 30.977329, 32.281067, 32.031899, + 26.358835, 26.820108, 42.992490, 43.882906, 47.992438, + 48.920477, 57.633450, 58.001989, 57.369478, 49.547092, + 39.714761, 39.313520, 40.065295, 48.834535, 51.525122, + 30.977329, 32.281072, 32.077511, 27.396520, 26.865699, + 42.992474, 43.882906, 48.358483, 57.251290, 57.985906, + 57.369511, 49.564105, 40.101962, 39.330532, 40.065297, + 48.835069, 57.817665, 145.394655, 30.977369, 32.281197, + 33.076074, 50.151877, 28.497451, 42.792931, 43.512994, + 48.377350, 57.604551, 57.386489, 49.951306, 40.118976, + 39.330567, 40.065298, 48.835069, 57.818203, 151.867755, + 153.625733, 31.447893, 32.750906, 32.266221, 32.302627, + 42.268290, 38.854272, 35.574619, 49.427510, 56.878041, + 50.330896, 48.931678, 39.717770, 40.065331, 48.835069, + 57.818191, 151.733328, 156.892368, 150.600520, 42.125514, + 43.448292, 33.901056, 49.284440, 47.710751, 52.761179, + 46.446220, 71.330907, 47.520016, 49.126258, 40.678046, + 40.107037, 48.835077, 57.818191, 151.733068, 153.820869, + 83.953859, 77.615621, 42.078145, 44.460837, 53.413056, + 53.860750, 51.935802, 60.688133, 54.022549, 59.058236, + 61.487243, 49.511225, 53.558701, 50.619860, 57.871196, + 151.733230, 153.821045, 83.819613, 77.759808, 76.318016, + 47.334708, 55.115021, 63.088351, 65.025437, 66.447005, + 66.685816, 72.840860, 70.204772, 75.247159, 75.769300, + 62.167236, 83.948064, 149.878947, 155.679346, 86.031186, + 83.012511, 86.265353, 124.454706, 64.496779, 78.702984, + 76.659370, 83.931191, 87.076287, 91.454837, 94.323380, + 104.708654, 109.932924, 118.596035, 126.154595, 114.409068, + 89.213630, 131.569970, 134.824613, 205.866155, 239.628810, + 233.070258, 72.308997, 94.104934, 101.404120, 121.159064, + 139.921349, 156.802796, 164.908506, 170.895200, 184.099064, + 198.250167, 207.100063, 207.922921, 207.455763, 210.915553, + 241.757520, 246.227265, 247.706317, 237.758893, 138.726610, + 135.459290, 198.494473, 240.094885, 275.356590, 295.461068, + 304.435273, 303.110796, 296.273053, 317.439936, 319.323966, + 331.200883, 330.246858, 256.379818, 247.994095, 247.767952, + 247.767210, 237.760453, 243.406633, 292.096487, 381.480937, + 421.494757, 458.357268, 483.808573, 487.800297, 481.212962, + 472.226349, 472.996935, 480.732289, 475.694812, 418.477476, + 412.710307, 268.338558, 261.989349, 261.987913, 251.406810, + 469.068582, 568.835648, 608.315967, 642.717253, 653.879813, + 681.008793, 688.659203, 677.706251, 668.348079, 652.995530, + 636.214467, 632.005555, 550.175654, 522.412011, 588.754096, + 591.721361, 591.719586, 567.821360, 667.228892, 787.133995, + 830.015474, 848.841400, 863.280871, 882.475988, 896.321308, + 889.205975, 878.059084, 871.467549, 860.073920, 814.054753, + 726.521001, 747.583633, 751.922326, 752.065525, 752.062970, + 721.688835, 795.516527, 916.651129, 1045.662316, 1064.160855, + 1070.930169, 1060.177543, 1080.056283, 1096.204211, 1095.337870, + 1098.401980, 1087.336698, 1022.507849, 916.586633, 938.424837, + 942.750167, 945.352203, 945.457385, 907.272493, 943.653980, + 1021.248655, 1129.544563, 1267.035138, 1281.163505, 1235.783117, + 1258.747831, 1285.425350, 1293.324247, 1314.802791, 1320.786990, + 1253.910457, 1138.645930, 1091.248129, 1161.397763, 1220.617170, + 1223.081462, 1173.684183, 1077.194216, 1216.741493, 1278.148407, + 1383.489545, 1497.622061, 1505.771495, 1422.262568, 1462.870558, + 1491.544736, 1494.922359, 1533.495237, 1509.573542, 1356.235452, + 1322.878872, 1324.869188, 1327.897500, 1337.708123, 1284.090364, + 1177.946473, 1335.104679, 1414.612819, 1508.772208, 1656.204257, + 1636.726625, 1693.515385, 1658.176544, 1626.718453, 1699.881842, + 1721.127481, 1715.540667, 1610.027798, 1528.475428, 1525.090509, + 1534.987574, 1760.238672, 1698.643936, 1285.821939, 1417.714456, + 1527.788112, 1660.663143, 1648.660011, 1838.036905, 1838.669356, + 1803.685279, 1895.404145, 1878.456820, 1907.761437, 1891.806178, + 1798.474576, 1772.695413, 1777.922135, 1782.961020, 1891.441373, + 1819.624352, 1363.652278, 1525.975687, 1664.733346, 1725.645678, + 1842.181220, 1845.935374, 1847.645643, 2084.649919, 2156.327082, + 2070.132265, 2093.920913, 2066.856209, 1935.233655, 1757.519179, + 1894.998543, 1904.433642, 1909.668939, 1832.757449, 1466.924321, + 1622.799826, 1723.952077, 1846.075992, 1857.802996, 1849.640165, + 2127.098845, 2370.422248, 2375.277013, 2471.317955, 2282.670767, + 2242.894713, 2110.737592, 1924.405669, 2022.847422, 2092.614827, + 2113.100659, 2028.502155, 1561.024406, 1719.325091, 1845.872940, + 1857.825889, 1850.155079, 2127.184115, 2372.221523, 2397.560183, + 2793.802030, 2525.955468, 2512.515727, 2465.117634, 2335.991284, + 2104.381407, 2124.161214, 2128.328755, 2129.221941, 2043.260174, + 1651.007418, 1782.604815, 1855.046326, 1850.154844, 2127.184127, + 2372.221529, 2397.560385, 2794.384764, 2544.781302, 2651.443272, + 2739.000803, 2702.474497, 2522.919941, 2276.563499, 2274.919319, + 2275.308573, 2275.300889, 2183.406597, 1713.105119, 1852.260885, + 1850.032717, 2127.168115, 2371.857312, 2397.544382, 2794.384763, + 2544.781370, 2651.673334, 2746.255820, 2763.494638, 2872.508140, + 2846.367495, 2305.137739, 2281.712444, 2281.725436, 2281.717621, + 2189.564169, 1780.286046, 1850.026376, 2127.168073, 2371.493095, + 2389.254845, 2794.020546, 2544.781339, 2651.673334, 2746.255845, + 2763.564597, 2874.688737, 2884.399707, 2860.263639, 2306.077266, + 2281.728045, 2281.725981, 2281.718165, 2189.564691, 1775.067091, + 2127.161715, 2371.493094, 2389.238842, 2793.656329, 2544.765337, + 2651.673332, 2746.255845, 2763.564597, 2874.688743, 2884.401957, + 2884.614099, 2860.264919, 2306.079244, 2281.728130, 2281.725981, + 2281.718165, 2189.564691, 2052.948444, 2371.486799, 2389.238842, + 2793.656328, 2544.765306, 2651.673331, 2746.255845, 2763.564597, + 2874.688743, 2884.401957, 2884.616164, 2884.614218, 2861.248329, + 2328.461556, 2282.711540, 2281.726065, 2281.718165, 2189.564691, + 2285.514142, 2389.231550, 2793.656328, 2544.765306, 2651.673331, + 2746.255845, 2763.564597, 2874.688743, 2884.401957, 2884.616164, + 2884.616200, 2884.616117, 2883.630641, 2837.880625, 2305.093852, + 2281.727963, 2281.718165, 2189.564691, 2293.011609, 2793.639080, + 2544.755886, 2651.663879, 2746.245982, 2763.554715, 2874.678861, + 2884.392075, 2884.606282, 2884.606318, 2884.606318, 2884.606318, + 2884.604170, 2860.253139, 2306.069362, 2281.720230, 2281.710349, + 2189.557190, 2600.476153, 2437.607663, 2540.606072, 2634.813199, + 2647.265343, 2758.176106, 2767.889302, 2768.103509, 2768.103545, + 2768.103545, 2768.103545, 2768.103545, 2768.101564, 2744.735758, + 2212.932478, 2189.566673, 2189.557190, 2101.125891, + }, + { + 8.370360, 8.509764, 8.942497, 8.962541, 8.993650, + 9.401615, 9.414854, 9.308823, 9.304164, 9.304163, + 9.304164, 9.304609, 9.324803, 9.555049, 9.565277, + 9.571194, 9.647370, 9.451428, 8.722648, 8.867100, + 9.300225, 9.352116, 9.672213, 9.809754, 9.794381, + 9.699874, 9.695753, 9.695753, 9.696199, 9.716392, + 9.947083, 9.967407, 9.973767, 10.050124, 10.454503, + 14.384681, 8.722677, 8.848489, 8.875968, 9.334087, + 9.685450, 9.793645, 9.413799, 9.683184, 9.695785, + 9.696231, 9.716425, 9.947116, 9.967440, 9.973802, + 10.050158, 10.454591, 15.023561, 15.153937, 8.723107, + 8.857467, 8.857756, 9.333268, 9.685451, 9.792911, + 9.397076, 9.682449, 9.695786, 9.706329, 9.946672, + 9.967440, 9.973802, 10.050158, 10.454591, 15.023647, + 16.163497, 23.616388, 8.732904, 9.080445, 8.867552, + 9.333269, 9.685451, 9.792910, 9.397075, 9.682449, + 9.696231, 9.716869, 9.957343, 9.973801, 10.050158, + 10.454591, 15.023647, 16.163589, 24.700350, 25.680478, + 8.733342, 9.090251, 8.868006, 9.333704, 9.685477, + 9.792913, 9.397086, 9.682895, 9.716424, 9.947246, + 9.973357, 10.050158, 10.454591, 15.023647, 16.163589, + 24.700554, 28.087109, 55.858717, 8.827122, 9.186580, + 8.932093, 9.387586, 9.780396, 9.827339, 9.522726, + 9.715771, 9.936594, 9.972878, 10.050158, 10.454591, + 15.023647, 16.163589, 24.700554, 28.087318, 58.323463, + 58.504090, 11.020305, 11.418729, 10.432068, 10.462985, + 11.980204, 10.705178, 12.392570, 10.364495, 9.829351, + 10.102947, 10.457228, 15.023647, 16.163589, 24.700554, + 28.087319, 58.323704, 60.984846, 58.926665, 12.554530, + 12.511580, 11.828923, 12.308025, 13.298760, 13.044522, + 12.945689, 13.556029, 12.673484, 12.067338, 15.194674, + 16.168241, 24.700556, 28.087321, 58.339708, 61.349323, + 61.787659, 67.232638, 15.008518, 15.485683, 15.872869, + 16.107583, 15.835820, 16.877645, 17.678106, 18.822352, + 20.208378, 19.665073, 19.090588, 25.203257, 28.104179, + 58.339741, 61.713565, 70.078019, 70.458033, 67.961299, + 17.761908, 35.462897, 20.615440, 19.929266, 20.679593, + 22.692816, 25.941528, 29.215386, 34.956494, 38.908872, + 35.866805, 37.824252, 59.112479, 61.729892, 70.173287, + 72.702736, 72.797652, 69.857555, 20.506234, 39.710259, + 24.054829, 25.703729, 31.621917, 38.509103, 44.229053, + 56.544042, 67.468427, 79.581678, 80.144809, 78.556111, + 74.399792, 73.852202, 76.027762, 117.654284, 119.482115, + 114.656650, 25.548773, 48.434418, 42.617322, 59.026359, + 82.885084, 103.416136, 110.596012, 118.893312, 139.741780, + 150.511338, 161.901080, 167.647532, 162.429576, 152.184833, + 152.445359, 168.398896, 169.099625, 162.269514, 82.673806, + 70.110218, 132.312464, 183.332020, 215.910819, 237.790158, + 239.207817, 229.469490, 230.174506, 250.717349, 253.902836, + 250.549622, 207.721430, 175.337781, 174.022082, 174.844559, + 179.095719, 164.620978, 109.303621, 158.273825, 282.689646, + 346.940666, 386.914531, 410.989383, 397.460378, 370.331321, + 364.107004, 366.410030, 380.333542, 363.501310, 294.254304, + 262.048447, 260.790777, 265.975674, 362.177173, 183.336338, + 404.520239, 473.542827, 507.606249, 529.861510, 559.718099, + 588.632991, 577.691379, 533.799138, 522.780262, 528.340923, + 525.905829, 507.879853, 415.994095, 425.708669, 426.932731, + 441.248848, 446.721366, 435.160437, 604.741350, 709.039378, + 780.416591, 732.829427, 733.032326, 766.737728, 758.413295, + 720.856125, 705.746779, 717.211527, 703.753940, 661.340782, + 552.231698, 523.727433, 522.707356, 523.327476, 523.380946, + 502.845082, 745.431184, 827.490462, 962.723712, 998.756854, + 911.671845, 907.248783, 925.966430, 901.879365, 900.337680, + 919.439892, 914.334537, 842.878262, 704.639674, 702.883177, + 703.067185, 703.067254, 703.064850, 674.669690, 894.169844, + 1025.903412, 1134.439771, 1219.193918, 1158.526825, 1079.781195, + 1077.325043, 1076.106019, 1075.934290, 1117.511500, 1135.469693, + 1063.699731, 901.900714, 920.553711, 921.681648, 921.681860, + 921.679125, 884.454592, 994.327035, 1175.512444, 1231.834682, + 1307.830164, 1407.421778, 1326.963114, 1253.371428, 1250.787362, + 1269.653825, 1282.857814, 1343.429863, 1307.501250, 1114.048857, + 983.551449, 978.228429, 979.601009, 984.578211, 945.021162, + 1135.950879, 1278.106415, 1310.418579, 1419.457570, 1488.462247, + 1521.009066, 1542.754625, 1434.182316, 1454.788272, 1471.133321, + 1506.092266, 1521.256433, 1327.843116, 1309.574134, 1310.302024, + 1341.725430, 1459.021892, 1404.992306, 1232.172010, 1349.471091, + 1428.291402, 1487.206606, 1535.398237, 1746.731849, 1716.656287, + 1635.940921, 1639.303784, 1673.047286, 1693.682270, 1671.177816, + 1517.793233, 1479.700025, 1478.369392, 1483.694229, 1578.610159, + 1518.853600, 1297.751330, 1428.849626, 1485.708645, 1493.412575, + 1741.940596, 1728.475751, 1735.779511, 1858.578642, 1918.863689, + 1863.724239, 1891.004802, 1842.815568, 1675.715043, 1588.071911, + 1589.814363, 1592.306606, 1596.345158, 1532.039001, 1373.297412, + 1461.343441, 1492.261194, 1736.797392, 1653.192795, 1733.476694, + 1888.263820, 2082.789610, 1991.468137, 1883.805569, 2116.637406, + 2071.956368, 1871.698996, 1644.776129, 1756.448731, 1809.269770, + 1811.354640, 1738.198277, 1403.551810, 1491.185946, 1736.750346, + 1653.047461, 1730.172539, 1888.230584, 2085.624280, 1993.438043, + 1786.471183, 2337.838134, 2448.464613, 2336.935884, 2097.581279, + 1808.183187, 1828.142934, 1821.170551, 1820.795051, 1747.257197, + 1427.896429, 1736.558511, 1653.047441, 1730.172527, 1888.230448, + 2087.322730, 2032.094869, 1787.954651, 2341.066895, 2641.531977, + 2539.340235, 2507.302262, 2309.116599, 2030.685235, 2078.550644, + 1842.720183, 1832.216691, 1758.216718, 1580.198506, 1648.792285, + 1730.172167, 1888.230448, 2087.322736, 2032.169494, 1789.653110, + 2341.141530, 2641.873005, 2555.239313, 2709.221434, 2715.218691, + 2682.937371, 2124.893499, 2102.967545, 2092.601766, 2092.133383, + 2007.636776, 1578.033231, 1729.979789, 1888.230432, 2087.322736, + 2032.169494, 1789.653117, 2341.141675, 2641.873012, 2555.239371, + 2709.566764, 2723.749115, 2724.064349, 2699.003101, 2128.626010, + 2103.563660, 2103.560569, 2103.553324, 2018.595528, 1663.710423, + 1888.224796, 2087.322736, 2032.169494, 1789.653117, 2341.141675, + 2641.873012, 2555.239371, 2709.566764, 2723.749145, 2724.067198, + 2724.065212, 2700.016608, 2151.662616, 2104.575854, 2103.561585, + 2103.554293, 2018.596458, 1818.473656, 2087.316820, 2032.169494, + 1789.653117, 2341.141675, 2641.873012, 2555.239371, 2709.566764, + 2723.749145, 2724.067198, 2724.067252, 2724.067166, 2723.052897, + 2675.966135, 2127.612143, 2103.563538, 2103.554293, 2018.596458, + 2011.142209, 2032.163033, 1789.653117, 2341.141675, 2641.873012, + 2555.239371, 2709.566764, 2723.749145, 2724.067198, 2724.067252, + 2724.067252, 2724.067252, 2724.065041, 2699.002510, 2128.626241, + 2103.563710, 2103.554293, 2018.596458, 1947.414707, 1789.636821, + 2341.132961, 2641.863707, 2555.230039, 2709.557432, 2723.739813, + 2724.057867, 2724.057920, 2724.057920, 2724.057920, 2724.057920, + 2724.055881, 2700.007276, 2151.653370, 2104.568562, 2103.547173, + 2018.589543, 1614.950117, 2233.740132, 2539.133260, 2445.528333, + 2599.538876, 2613.721229, 2614.039283, 2614.039337, 2614.039337, + 2614.039337, 2614.039337, 2614.039337, 2614.039251, 2613.025067, + 2566.950536, 2041.634786, 2018.591496, 1937.063243, + }, + { + 2.147514, 2.237898, 2.237803, 2.237151, 2.275320, + 2.279324, 2.332179, 2.334501, 2.322470, 2.048664, + 2.036634, 2.036633, 2.043731, 2.205267, 2.213955, + 2.286115, 3.114672, 3.150236, 2.237900, 2.332085, + 2.329658, 2.278309, 2.368754, 2.375255, 2.430335, + 2.432248, 2.408693, 2.134381, 2.122351, 2.122650, + 2.136546, 2.299971, 2.379228, 3.209393, 3.546162, + 6.294600, 2.271800, 2.333582, 2.329564, 2.275984, + 2.368873, 2.380129, 2.430556, 2.420732, 2.146419, + 2.122864, 2.122657, 2.136254, 2.293180, 2.378937, + 3.209400, 3.546195, 6.591552, 7.028860, 3.043184, + 2.367477, 2.329567, 2.275984, 2.373739, 2.490860, + 2.435421, 2.420225, 2.134895, 2.122658, 2.136254, + 2.293180, 2.378936, 3.209400, 3.546195, 6.591617, + 7.795421, 17.743823, 3.078566, 2.402859, 2.331056, + 2.275985, 2.373953, 2.495725, 2.435129, 2.408702, + 2.134688, 2.136255, 2.293180, 2.378936, 3.209400, + 3.546195, 6.591617, 7.795492, 18.578087, 19.754344, + 3.112459, 3.174471, 2.370323, 2.279235, 2.375398, + 2.494530, 2.423939, 2.146738, 2.136763, 2.293180, + 2.378939, 3.209400, 3.546195, 6.591617, 7.795492, + 18.578249, 21.659892, 44.204400, 3.089303, 3.189642, + 2.509317, 2.372448, 2.432176, 2.489915, 2.461894, + 2.170933, 2.301935, 2.380378, 3.234280, 3.547288, + 6.591617, 7.795492, 18.578249, 21.660057, 46.157016, + 46.346972, 2.545327, 2.662725, 2.893299, 2.840660, + 3.006570, 3.048065, 3.224317, 2.907751, 2.627415, + 3.268959, 4.113541, 6.616496, 7.795494, 18.578249, + 21.660057, 46.157182, 48.299592, 46.394222, 2.942757, + 3.155386, 3.688593, 3.481851, 3.715109, 4.145502, + 4.681446, 5.080585, 4.600294, 4.278096, 6.726591, + 7.800142, 18.578250, 21.660057, 46.157182, 48.299758, + 48.346843, 46.394230, 3.793838, 5.473983, 4.608110, + 4.665834, 4.593326, 5.621563, 7.102711, 9.025169, + 10.122234, 10.159723, 10.023976, 18.857332, 21.668316, + 46.157261, 48.299839, 48.347091, 48.346956, 46.394491, + 4.271176, 6.466779, 5.484235, 6.027476, 7.301091, + 9.352794, 12.715546, 16.192475, 22.560070, 27.958500, + 27.052873, 26.766614, 46.406220, 49.221059, 49.307058, + 49.315280, 49.587875, 49.469333, 4.312908, 7.475387, + 7.833860, 10.223714, 16.775457, 24.234924, 32.175390, + 45.584094, 54.342692, 64.941124, 66.654416, 62.626466, + 52.266425, 72.572219, 73.485826, 73.672773, 79.880600, + 119.538616, 9.063034, 10.103318, 19.539238, 36.472083, + 62.703355, 82.709235, 91.988082, 100.285999, 115.687111, + 124.329614, 125.634719, 122.315209, 112.379054, 130.322056, + 131.130875, 131.139174, 131.411947, 127.988734, 19.796341, + 25.882925, 93.253237, 153.548489, 189.401787, 213.347763, + 206.988813, 189.758327, 189.286642, 204.628607, 202.531100, + 201.472079, 186.352604, 214.273591, 215.531911, 215.769339, + 221.170008, 212.465343, 169.309448, 175.824536, 224.720662, + 303.438505, 343.063029, 370.535703, 343.624360, 300.549614, + 289.296503, 298.082336, 310.854450, 294.466802, 270.163254, + 249.832086, 248.983097, 254.384429, 377.318513, 367.262701, + 353.359003, 463.883442, 447.419222, 453.002989, 489.006379, + 528.921013, 500.655371, 439.860852, 423.703334, 423.144668, + 431.190760, 422.628418, 346.619232, 383.306337, 385.067653, + 385.305123, 390.705188, 375.153218, 539.087109, 664.575466, + 708.656277, 662.482491, 618.723324, 674.195070, 656.531281, + 596.084694, 576.463478, 584.628279, 577.147684, 562.515800, + 453.416064, 428.149599, 427.247359, 427.247313, 427.246307, + 409.990807, 687.292802, 777.058828, 905.030245, 912.381642, + 788.119446, 769.561998, 795.174180, 761.722895, 738.637994, + 753.718308, 761.364923, 712.753474, 603.465105, 591.662313, + 594.079856, 594.200005, 594.197958, 570.199152, 779.060173, + 951.796204, 1069.175314, 1170.755086, 1067.610429, 949.009502, + 934.793193, 914.719926, 899.055222, 929.548018, 948.443491, + 883.741844, 754.604267, 719.256051, 780.076337, 782.811403, + 782.547500, 745.230787, 922.165691, 1108.252645, 1185.274121, + 1237.276156, 1266.012843, 1226.812751, 1102.465568, 1085.286013, + 1075.433883, 1091.747391, 1130.914031, 1079.716598, 927.463157, + 942.209973, 945.879069, 946.263934, 946.332394, 778.375752, + 1069.516638, 1181.232895, 1244.763295, 1280.392765, 1419.980489, + 1443.722749, 1374.005679, 1247.118477, 1252.786860, 1265.078694, + 1280.267151, 1273.804486, 1085.050290, 1037.883124, 1036.171983, + 1042.457032, 1185.241611, 1137.692305, 1136.326843, 1246.301355, + 1321.499026, 1422.581245, 1459.231744, 1574.624247, 1593.556714, + 1554.290382, 1474.539686, 1480.100956, 1464.679108, 1412.507057, + 1251.278472, 1197.245685, 1195.175062, 1201.168704, 1337.571298, + 1289.300549, 1198.629451, 1322.569335, 1424.892277, 1470.063106, + 1576.317080, 1629.881049, 1759.007438, 1685.455960, 1823.183401, + 1696.763728, 1720.897219, 1582.701816, 1409.816846, 1352.692221, + 1350.761108, 1356.950874, 1362.935844, 1308.131482, 1273.245632, + 1447.544983, 1471.079493, 1576.789596, 1629.891022, 1759.113129, + 1704.167080, 2077.760859, 1944.393655, 2008.282739, 1951.682502, + 1905.896173, 1653.621076, 1524.210363, 1526.775296, 1662.009621, + 1667.942643, 1600.578688, 1394.223453, 1472.070445, 1576.833336, + 1629.890970, 1757.978972, 1678.353822, 2077.085726, 1963.548056, + 2214.617152, 2129.636312, 2170.262437, 2204.652056, 1892.295487, + 1652.520236, 1673.779173, 1681.078895, 1681.334159, 1613.428869, + 1413.634564, 1576.837954, 1629.890974, 1757.978960, 1678.303800, + 2075.951574, 1963.498706, 2214.984962, 2140.104142, 2224.748512, + 2358.489855, 2338.333779, 2147.326062, 1959.161343, 1704.142189, + 1693.264072, 1693.257371, 1624.870507, 1522.455341, 1630.102341, + 1757.978970, 1678.205343, 2073.710713, 1963.613482, 2219.840292, + 2140.317509, 2224.830630, 2367.413225, 2514.706448, 2519.715697, + 2490.578281, 1997.672226, 1965.092163, 1964.608831, 1964.602061, + 1885.256194, 1566.312385, 1757.983049, 1678.205154, 2071.469838, + 1912.611809, 2222.454760, 2250.824415, 2229.685973, 2367.413669, + 2515.007972, 2527.205354, 2527.466626, 2505.215096, 1998.786468, + 1976.532889, 1976.530885, 1976.524114, 1896.696743, 1692.302152, + 1678.199584, 2071.469829, 1912.513351, 2220.213903, 2250.939287, + 2234.541308, 2367.626998, 2515.007990, 2527.205379, 2527.469148, + 2527.467382, 2506.115004, 2019.240121, 1977.432526, 1976.531972, + 1976.525125, 1896.697713, 1606.783592, 2071.463772, 1912.513351, + 2220.213894, 2250.939097, 2234.541317, 2367.627410, 2515.008008, + 2527.205379, 2527.469148, 2527.469193, 2527.469116, 2526.568562, + 2484.760968, 1997.886085, 1976.533707, 1976.525125, 1896.697713, + 2004.589287, 1912.507679, 2220.213894, 2250.939097, 2234.541317, + 2367.627410, 2515.008008, 2527.205379, 2527.469148, 2527.469193, + 2527.469193, 2527.469193, 2527.467229, 2505.214602, 1998.786486, + 1976.533860, 1976.525125, 1896.697713, 1827.779426, 2220.199076, + 2250.930988, 2234.532682, 2367.618752, 2514.999350, 2527.196721, + 2527.460490, 2527.460534, 2527.460534, 2527.460534, 2527.460534, + 2527.458723, 2506.106346, 2019.231539, 1977.425679, 1976.518431, + 1896.691216, 2056.541124, 2160.944440, 2138.934044, 2265.803052, + 2412.920916, 2425.118264, 2425.382033, 2425.382078, 2425.382078, + 2425.382078, 2425.382078, 2425.382078, 2425.382002, 2424.481524, + 2383.572672, 1917.153083, 1896.692951, 1820.088116, + }, + { + 0.254129, 0.265575, 0.267769, 0.316933, 0.320396, + 0.350337, 0.358368, 0.358663, 0.358663, 0.358675, + 0.359199, 0.365181, 0.366036, 0.392895, 0.701286, + 0.812014, 1.926722, 2.011743, 0.247749, 0.276002, + 0.279039, 0.330191, 0.332015, 0.365000, 0.373450, + 0.373758, 0.373759, 0.374032, 0.380276, 0.381405, + 0.408276, 0.717257, 0.841415, 1.958868, 2.298681, + 4.856667, 0.246998, 0.275970, 0.279040, 0.328326, + 0.289563, 0.363136, 0.373451, 0.373760, 0.373771, + 0.374308, 0.381144, 0.408277, 0.717258, 0.841416, + 1.958870, 2.298703, 5.088461, 5.484975, 0.246965, + 0.275218, 0.279007, 0.328244, 0.287698, 0.363054, + 0.373451, 0.373771, 0.374296, 0.380881, 0.408266, + 0.717258, 0.841416, 1.958870, 2.298703, 5.088514, + 6.102534, 14.287379, 0.246213, 0.258111, 0.278255, + 0.328244, 0.287698, 0.363054, 0.373452, 0.374034, + 0.380870, 0.408644, 0.725849, 0.841794, 1.958870, + 2.298703, 5.088514, 6.102591, 14.959455, 15.913957, + 0.246180, 0.257402, 0.279186, 0.328526, 0.290138, + 0.364877, 0.376133, 0.375014, 0.408382, 0.734439, + 1.037308, 1.967461, 2.298704, 5.088514, 6.102591, + 14.959584, 17.441072, 35.429097, 0.250019, 0.262303, + 0.303891, 0.340011, 0.350106, 0.413735, 0.446832, + 0.410301, 0.734390, 1.038841, 2.002354, 2.300237, + 5.088514, 6.102591, 14.959584, 17.441204, 36.992539, + 37.110962, 0.339750, 0.354696, 0.373261, 0.461008, + 0.462495, 0.598987, 0.695421, 0.760530, 1.068705, + 2.029958, 2.898902, 5.114817, 6.102593, 14.959584, + 17.441205, 36.992675, 38.648954, 36.568097, 0.400795, + 0.421342, 0.508307, 0.588014, 0.677109, 0.972744, + 1.364551, 2.005727, 2.968255, 3.091537, 5.221096, + 6.106993, 14.959585, 17.441202, 36.995441, 38.712050, + 38.170575, 37.947398, 0.489106, 0.499732, 0.682198, + 0.797985, 0.928841, 1.529889, 3.407135, 6.047262, + 9.020716, 9.566838, 8.225222, 15.250390, 17.449015, + 36.967227, 38.773771, 39.603706, 39.612863, 38.072105, + 0.782539, 0.400752, 1.185871, 1.379942, 2.131362, + 4.169824, 9.556239, 12.949414, 21.418888, 24.451152, + 23.409241, 22.939144, 37.166430, 38.131684, 39.591435, + 39.985395, 39.999803, 38.384307, 1.616514, 0.576268, + 3.109295, 3.930083, 8.248113, 15.543679, 28.108995, + 37.139521, 46.111565, 52.306471, 54.245700, 51.814241, + 40.882242, 41.810659, 42.231519, 48.976557, 49.261637, + 47.271630, 5.094715, 1.500966, 10.566421, 19.002838, + 38.783971, 59.976285, 70.642488, 79.940911, 88.863960, + 91.805239, 95.624856, 92.381504, 94.587340, 93.842585, + 93.403266, 93.670318, 93.432487, 89.648397, 0.664794, + 7.849196, 61.154449, 96.443290, 137.609372, 162.556446, + 148.037064, 138.292519, 136.715840, 145.772994, 149.246615, + 151.612049, 131.470885, 109.201663, 101.316335, 101.904288, + 101.932270, 97.814994, 6.925953, 50.062367, 155.094765, + 216.414869, 252.438993, 271.718839, 232.337598, 204.289605, + 199.194140, 205.331210, 215.677297, 214.590331, 188.109722, + 172.081271, 228.048207, 250.892300, 251.787562, 241.618501, + 134.674271, 296.106246, 298.718911, 312.356451, 323.413855, + 342.631007, 307.749299, 283.067516, 265.987870, 265.460375, + 276.156298, 267.602576, 244.127989, 255.341106, 257.373708, + 258.333723, 258.372207, 247.937137, 342.349647, 510.603738, + 581.005531, 478.386434, 373.143790, 378.600404, 377.561939, + 356.141074, 338.631477, 331.842551, 340.566852, 327.567230, + 297.260507, 287.191935, 263.781497, 262.767950, 262.766967, + 252.154399, 475.742336, 629.567678, 578.444958, 564.378757, + 438.113186, 396.420987, 414.959234, 431.302969, 421.606208, + 403.786618, 405.430259, 389.856615, 363.613524, 366.303795, + 365.460005, 365.415394, 365.414130, 350.655776, 624.537781, + 642.232055, 604.750655, 630.663539, 588.111844, 475.625654, + 468.814426, 477.664040, 494.124835, 483.676981, 477.873259, + 452.815795, 423.548326, 431.959314, 432.336669, 431.259561, + 431.158392, 412.599201, 599.136090, 596.383540, 649.766837, + 714.641904, 647.024168, 546.886099, 544.047374, 542.897551, + 558.506982, 555.978025, 571.819113, 532.351774, 510.677525, + 495.125872, 493.402635, 468.981876, 468.871284, 423.950331, + 572.225475, 705.639818, 1024.735591, 667.143308, 595.280675, + 762.720257, 552.209148, 613.853858, 630.029121, 653.039002, + 643.569732, 623.030443, 573.888949, 571.441956, 571.380435, + 572.467453, 621.621663, 597.446835, 681.815611, 1037.097688, + 905.438670, 599.972561, 762.219180, 517.929098, 688.053947, + 744.181205, 679.524546, 682.569374, 713.780286, 707.985178, + 646.983619, 636.062414, 635.699032, 635.794133, 637.956277, + 612.281811, 1009.018046, 905.871087, 604.028246, 629.564692, + 511.991374, 689.885845, 836.027416, 766.534552, 881.860194, + 735.849782, 783.725360, 809.653496, 750.636979, 811.765460, + 814.639525, 816.244480, 818.253804, 785.288396, 867.862571, + 604.231326, 629.308626, 506.143862, 689.628928, 836.193031, + 774.742632, 1007.244245, 1293.627165, 897.984304, 931.611215, + 973.236387, 945.068518, 947.276414, 950.892263, 987.500595, + 1033.288662, 993.423200, 674.214249, 634.009759, 506.144239, + 689.628432, 836.193010, 774.742663, 1007.434892, 1306.991055, + 1109.422389, 1067.304686, 1126.861498, 1124.180515, 989.502672, + 997.732672, 1042.608506, 1046.158446, 1048.166963, 1005.915872, + 633.151925, 507.355996, 689.628570, 836.193810, 774.742698, + 1007.434892, 1306.991101, 1109.775470, 1079.300073, 1239.065289, + 1628.963638, 1322.149598, 1152.072478, 1133.642807, 1135.074392, + 1135.159950, 1135.156239, 1089.309834, 1002.301724, 712.557708, + 836.610527, 784.181750, 1007.849615, 1306.991136, 1109.775470, + 1079.300117, 1239.231971, 1644.540785, 1609.385289, 1575.587962, + 1175.113940, 1139.762609, 1138.978460, 1138.978401, 1138.974499, + 1092.973876, 693.249751, 850.650934, 794.193638, 1222.681498, + 1316.430188, 1109.776271, 1079.300117, 1239.231971, 1644.540839, + 1609.866572, 1604.515037, 1603.618487, 1568.302641, 1157.017568, + 1138.980254, 1138.978724, 1138.974823, 1092.974187, 834.755585, + 1090.883669, 1236.131974, 1325.870346, 1110.190994, 1079.300152, + 1239.231971, 1644.540839, 1609.866572, 1604.515142, 1604.380213, + 1604.378532, 1585.580675, 1157.776710, 1138.980319, 1138.978724, + 1138.974823, 1092.974187, 1056.722338, 1249.164781, 1326.443132, + 1110.191843, 1079.300187, 1239.231971, 1644.540839, 1609.866572, + 1604.515142, 1604.380213, 1604.380190, 1604.378596, 1585.582205, + 1157.778240, 1138.980383, 1138.978724, 1138.974823, 1092.974187, + 1204.957557, 1326.440488, 1110.191892, 1079.300187, 1239.231971, + 1644.540839, 1609.866572, 1604.515142, 1604.380213, 1604.380190, + 1604.380190, 1604.378661, 1586.341349, 1175.056279, 1139.739527, + 1138.978789, 1138.974823, 1092.974187, 1275.872102, 1110.183396, + 1079.294485, 1239.226464, 1644.535343, 1609.861076, 1604.509646, + 1604.374717, 1604.374695, 1604.374695, 1604.374695, 1604.374630, + 1603.613892, 1568.297204, 1157.013600, 1138.976353, 1138.970921, + 1092.970443, 1014.576161, 1029.697894, 1172.002606, 1579.603765, + 1545.063972, 1539.712554, 1539.577625, 1539.577603, 1539.577603, + 1539.577603, 1539.577603, 1539.577603, 1539.576008, 1521.537360, + 1111.012900, 1092.975717, 1092.970443, 1048.827820, + }, }; -void av1_model_rd_surffit(double xm, double yl, double *rate_f, - double *dist_f) { +static const double interp_dgrid_surf[33 * 18] = { + 15.491252, 15.496413, 15.496379, 15.394447, 15.394431, 15.446941, 15.480480, + 15.480491, 15.480491, 15.480491, 15.480573, 15.603112, 15.603153, 15.602880, + 15.196621, 15.195983, 14.447585, 14.442076, 15.599420, 15.501644, 15.501575, + 15.399610, 15.399669, 15.452121, 15.485671, 15.485682, 15.485682, 15.485724, + 15.608303, 15.608385, 15.608113, 15.201854, 15.201079, 14.452681, 14.451350, + 13.210484, 15.599455, 15.501644, 15.501575, 15.399686, 15.626744, 15.452197, + 15.485671, 15.485682, 15.485682, 15.485765, 15.608344, 15.608113, 15.201854, + 15.201079, 14.452681, 14.451350, 13.214913, 13.208705, 15.599455, 15.501679, + 15.501575, 15.399686, 15.626820, 15.452197, 15.485671, 15.485682, 15.485765, + 15.608303, 15.608113, 15.201854, 15.201079, 14.452681, 14.451350, 13.214913, + 13.212184, 10.377543, 15.599489, 15.604652, 15.501610, 15.399686, 15.626820, + 15.452197, 15.485671, 15.485724, 15.608303, 15.608113, 15.201794, 15.201079, + 14.452681, 14.451350, 13.214913, 13.212184, 10.381022, 10.373758, 15.599489, + 15.604686, 15.501645, 15.399727, 15.626807, 15.452213, 15.485673, 15.485765, + 15.608072, 15.201734, 15.022022, 14.452621, 14.451350, 13.214913, 13.212184, + 10.381022, 10.375351, 4.752366, 15.599553, 15.604785, 15.606745, 15.523109, + 15.588440, 15.498592, 15.490939, 15.608113, 15.201734, 15.022022, 14.452398, + 14.451350, 13.214913, 13.212184, 10.381022, 10.375351, 4.753958, 4.748594, + 15.788491, 15.793763, 15.727178, 15.593404, 15.576185, 15.541155, 15.528638, + 15.443234, 15.022046, 14.452234, 13.963908, 13.214750, 13.212184, 10.381022, + 10.375351, 4.753958, 4.750187, 4.748513, 15.783667, 15.788604, 15.715040, + 15.574608, 15.604897, 15.345511, 15.322323, 15.003579, 14.280362, 13.963611, + 13.214382, 13.212184, 10.381022, 10.375351, 4.753958, 4.750106, 4.750025, + 4.508804, 15.747709, 14.765834, 15.706678, 15.619092, 15.637946, 15.437393, + 14.705235, 13.886464, 13.092309, 12.499506, 12.601594, 10.380302, 10.375350, + 4.753958, 4.750026, 4.510253, 4.510173, 4.508580, 15.641901, 14.968111, + 15.663074, 15.583828, 15.420627, 14.983936, 13.912901, 13.425841, 11.601603, + 10.600757, 10.110971, 8.839811, 4.753363, 4.749963, 4.510190, 4.322919, + 4.322857, 4.321407, 15.545263, 14.859883, 15.429167, 15.185696, 14.527376, + 13.650042, 12.244185, 10.755706, 9.356137, 8.229212, 8.166627, 7.324414, + 4.511146, 4.322709, 4.322566, 4.322503, 4.322503, 4.321054, 15.151031, + 14.579126, 14.592456, 13.733213, 11.980508, 10.685709, 9.143695, 7.466555, + 6.434770, 5.659204, 5.592407, 5.356211, 4.597869, 3.456439, 3.456056, + 3.456176, 3.456176, 3.455018, 13.478598, 13.969965, 10.921203, 9.274388, + 7.530261, 6.614303, 5.617812, 4.558886, 4.008929, 3.739688, 3.597707, + 3.510418, 3.041638, 3.096611, 3.096750, 3.455008, 3.455038, 3.453880, + 11.982351, 11.102072, 6.384662, 5.366178, 4.365376, 3.782955, 3.285890, + 2.758904, 2.439067, 2.338593, 2.257332, 2.190385, 2.088145, 1.867744, + 1.867445, 1.196878, 0.929332, 0.928931, 6.882587, 5.102224, 3.928701, + 3.021205, 2.614349, 2.162491, 1.870802, 1.628127, 1.448370, 1.352404, + 1.349332, 1.277236, 1.128399, 0.928668, 0.928601, 0.928485, 0.928395, + 0.928084, 4.066496, 3.233175, 2.413117, 1.830193, 1.473572, 1.273461, + 1.065156, 0.929824, 0.844729, 0.777037, 0.754300, 0.759737, 0.658871, + 0.603118, 0.603209, 0.928096, 0.928205, 0.927893, 2.747433, 2.039019, + 1.400016, 1.123698, 0.918772, 0.744824, 0.627536, 0.528263, 0.480634, + 0.446093, 0.423444, 0.401394, 0.394353, 0.360914, 0.360903, 0.361012, + 0.361012, 0.360891, 1.709038, 1.258373, 0.985529, 0.673582, 0.559577, + 0.456411, 0.372560, 0.312622, 0.268425, 0.251698, 0.238132, 0.222477, + 0.209643, 0.165264, 0.165254, 0.165254, 0.165254, 0.165199, 1.088160, + 0.806677, 0.673453, 0.490758, 0.350173, 0.277724, 0.229342, 0.190245, + 0.158717, 0.138414, 0.132405, 0.123800, 0.120550, 0.128355, 0.142808, + 0.142812, 0.142800, 0.142752, 0.806227, 0.588621, 0.490714, 0.350136, + 0.236497, 0.188583, 0.143991, 0.115039, 0.097317, 0.081406, 0.071159, + 0.067980, 0.062769, 0.066575, 0.066581, 0.066569, 0.030318, 0.030291, + 0.588339, 0.456572, 0.302189, 0.236473, 0.188564, 0.126491, 0.101248, + 0.075242, 0.059591, 0.051942, 0.042824, 0.037390, 0.034523, 0.030277, + 0.030275, 0.030275, 0.030258, 0.014677, 0.456374, 0.302172, 0.236448, + 0.164278, 0.126481, 0.094667, 0.068489, 0.053835, 0.038513, 0.031335, + 0.028947, 0.022753, 0.019347, 0.014676, 0.014675, 0.014674, 0.014674, + 0.014664, 0.302014, 0.221625, 0.164273, 0.126473, 0.094667, 0.068483, + 0.053831, 0.034298, 0.027445, 0.022220, 0.016315, 0.015692, 0.012949, + 0.009448, 0.009442, 0.007914, 0.007913, 0.007910, 0.221524, 0.164268, + 0.126473, 0.094667, 0.068480, 0.044800, 0.034295, 0.027442, 0.019666, + 0.014389, 0.011041, 0.011550, 0.009117, 0.021326, 0.007915, 0.007911, + 0.007911, 0.007908, 0.164193, 0.126473, 0.094667, 0.068480, 0.044800, + 0.034291, 0.027442, 0.019665, 0.014388, 0.008515, 0.008320, 0.009117, + 0.007357, 0.004348, 0.004343, 0.007908, 0.007910, 0.007907, 0.125623, + 0.094667, 0.068480, 0.044800, 0.034289, 0.023364, 0.019663, 0.011887, + 0.008514, 0.008318, 0.007208, 0.007208, 0.007206, 0.004342, 0.004341, + 0.004342, 0.004342, 0.004341, 0.094624, 0.068480, 0.044800, 0.034289, + 0.023364, 0.019662, 0.011887, 0.008514, 0.008318, 0.007208, 0.007207, + 0.007207, 0.007206, 0.004342, 0.004341, 0.004341, 0.004341, 0.004340, + 0.068449, 0.044800, 0.034289, 0.023364, 0.019662, 0.011887, 0.008514, + 0.008318, 0.007208, 0.007207, 0.007207, 0.007207, 0.007206, 0.004343, + 0.004341, 0.004341, 0.004341, 0.004340, 0.044777, 0.034289, 0.023364, + 0.019662, 0.011887, 0.008514, 0.008318, 0.007208, 0.007207, 0.007207, + 0.007207, 0.007207, 0.007207, 0.007205, 0.004342, 0.004341, 0.004341, + 0.004340, 0.034274, 0.023364, 0.019662, 0.011887, 0.008514, 0.008318, + 0.007208, 0.007207, 0.007207, 0.007207, 0.007207, 0.007207, 0.007207, + 0.007206, 0.004342, 0.004341, 0.004341, 0.004340, 0.023353, 0.019662, + 0.011887, 0.008514, 0.008318, 0.007208, 0.007207, 0.007207, 0.007207, + 0.007207, 0.007207, 0.007207, 0.007207, 0.007206, 0.004343, 0.004341, + 0.004341, 0.004340, 0.019650, 0.011884, 0.008511, 0.008316, 0.007205, + 0.007205, 0.007205, 0.007205, 0.007205, 0.007205, 0.007205, 0.007205, + 0.007205, 0.007205, 0.007203, 0.004341, 0.004340, 0.004338, +}; + +void av1_model_rd_surffit(BLOCK_SIZE bsize, double sse_norm, double xm, + double yl, double *rate_f, double *dist_f) { + (void)sse_norm; const double x_start = -0.5; const double x_end = 16.5; - const double x_step = 1; + const double x_step = 1.0; const double y_start = -15.5; const double y_end = 16.5; - const double y_step = 0.5; + const double y_step = 1.0; const double epsilon = 1e-6; const int stride = (int)rint((x_end - x_start) / x_step) + 1; + const int rcat = bsize_model_cat_lookup[bsize]; (void)y_end; xm = AOMMAX(xm, x_start + x_step + epsilon); @@ -1061,44 +1325,112 @@ void av1_model_rd_surffit(double xm, double yl, double *rate_f, const double yo = y - yi; const double xo = x - xi; - const double *prate = &interp_rgrid_surf[(yi - 1) * stride + (xi - 1)]; + const double *prate = &interp_rgrid_surf[rcat][(yi - 1) * stride + (xi - 1)]; const double *pdist = &interp_dgrid_surf[(yi - 1) * stride + (xi - 1)]; *rate_f = interp_bicubic(prate, stride, xo, yo); *dist_f = interp_bicubic(pdist, stride, xo, yo); } -static const double interp_rgrid_curv[65] = { - 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, - 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, - 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 4.759876, - 8.132086, 13.651828, 21.908271, 33.522054, 48.782376, 71.530983, - 106.728649, 151.942795, 199.893011, 242.850965, 283.933923, 322.154203, - 360.684608, 394.801656, 426.879017, 460.234313, 484.103987, 508.261495, - 536.486763, 558.196737, 586.285894, 614.764511, 634.166333, 647.706472, - 658.211478, 681.360407, 701.052141, 727.007310, 768.663973, 804.407660, - 884.627751, 1065.658131, 1238.875214, 1440.185176, 1678.377931, 1962.243390, - 2300.571467, 2702.152072, 3175.775119, 3730.230519, 4374.308184, 5116.798028, - 5966.489961, 6932.173897, 8022.639747, 9246.677424, 10613.076839, +static const double interp_rgrid_curv[4][65] = { + { + 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, + 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, + 0.000000, 23.801499, 28.387688, 33.388795, 42.298282, + 41.525408, 51.597692, 49.566271, 54.632979, 60.321507, + 67.730678, 75.766165, 85.324032, 96.600012, 120.839562, + 173.917577, 255.974908, 354.107573, 458.063476, 562.345966, + 668.568424, 772.072881, 878.598490, 982.202274, 1082.708946, + 1188.037853, 1287.702240, 1395.588773, 1490.825830, 1584.231230, + 1691.386090, 1766.822555, 1869.630904, 1926.743565, 2002.949495, + 2047.431137, 2138.486068, 2154.743767, 2209.242472, 2277.593051, + 2290.996432, 2307.452938, 2343.567091, 2397.654644, 2469.425868, + 2558.591037, 2664.860422, 2787.944296, 2927.552932, 3083.396602, + 3255.185579, 3442.630134, 3645.440541, 3863.327072, 4096.000000, + }, + { + 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, + 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, + 0.000000, 8.998436, 9.439592, 9.731837, 10.865931, + 11.561347, 12.578139, 14.205101, 16.770584, 19.094853, + 21.330863, 23.298907, 26.901921, 34.501017, 57.891733, + 112.234763, 194.853189, 288.302032, 380.499422, 472.625309, + 560.226809, 647.928463, 734.155122, 817.489721, 906.265783, + 999.260562, 1094.489206, 1197.062998, 1293.296825, 1378.926484, + 1472.760990, 1552.663779, 1635.196884, 1692.451951, 1759.741063, + 1822.162720, 1916.515921, 1966.686071, 2031.647506, 2033.700134, + 2087.847688, 2161.688858, 2242.536028, 2334.023491, 2436.337802, + 2549.665519, 2674.193198, 2810.107395, 2957.594666, 3116.841567, + 3288.034655, 3471.360486, 3667.005616, 3875.156602, 4096.000000, + }, + { + 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, + 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, + 0.000000, 2.377584, 2.557185, 2.732445, 2.851114, + 3.281800, 3.765589, 4.342578, 5.145582, 5.611038, + 6.642238, 7.945977, 11.800522, 17.346624, 37.501413, + 87.216800, 165.860942, 253.865564, 332.039345, 408.518863, + 478.120452, 547.268590, 616.067676, 680.022540, 753.863541, + 834.529973, 919.489191, 1008.264989, 1092.230318, 1173.971886, + 1249.514122, 1330.510941, 1399.523249, 1466.923387, 1530.533471, + 1586.515722, 1695.197774, 1746.648696, 1837.136959, 1909.075485, + 1975.074651, 2060.159200, 2155.335095, 2259.762505, 2373.710437, + 2497.447898, 2631.243895, 2775.367434, 2930.087523, 3095.673170, + 3272.393380, 3460.517161, 3660.313520, 3872.051464, 4096.000000, + }, + { + 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, + 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, + 0.000000, 0.296997, 0.342545, 0.403097, 0.472889, + 0.614483, 0.842937, 1.050824, 1.326663, 1.717750, + 2.530591, 3.582302, 6.995373, 9.973335, 24.042464, + 56.598240, 113.680735, 180.018689, 231.050567, 266.101082, + 294.957934, 323.326511, 349.434429, 380.443211, 408.171987, + 441.214916, 475.716772, 512.900000, 551.186939, 592.364455, + 624.527378, 661.940693, 679.185473, 724.800679, 764.781792, + 873.050019, 950.299001, 939.292954, 1052.406153, 1033.893184, + 1112.182406, 1219.174326, 1337.296681, 1471.648357, 1622.492809, + 1790.093491, 1974.713858, 2176.617364, 2396.067465, 2633.327614, + 2888.661266, 3162.331876, 3454.602899, 3765.737789, 4096.000000, + }, }; -static const double interp_dgrid_curv[65] = { - 14.604855, 14.604855, 14.604855, 14.604855, 14.604855, 14.604855, 14.604855, - 14.604855, 14.604855, 14.604855, 14.604855, 14.604855, 14.555776, 14.533692, - 14.439920, 14.257791, 13.977230, 13.623229, 13.064884, 12.355411, 11.560773, - 10.728960, 9.861975, 8.643612, 6.916021, 5.154769, 3.734940, 2.680051, - 1.925506, 1.408410, 1.042223, 0.767641, 0.565392, 0.420116, 0.310427, - 0.231711, 0.172999, 0.128293, 0.094992, 0.072171, 0.052972, 0.039354, - 0.029555, 0.022857, 0.016832, 0.013297, 0.000000, 0.000000, 0.000000, - 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, - 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, - 0.000000, 0.000000, +static const double interp_dgrid_curv[2][65] = { + { + 16.000000, 15.962891, 15.925174, 15.886888, 15.848074, 15.808770, + 15.769015, 15.728850, 15.688313, 15.647445, 15.606284, 15.564870, + 15.525918, 15.483820, 15.373330, 15.126844, 14.637442, 14.184387, + 13.560070, 12.880717, 12.165995, 11.378144, 10.438769, 9.130790, + 7.487633, 5.688649, 4.267515, 3.196300, 2.434201, 1.834064, + 1.369920, 1.035921, 0.775279, 0.574895, 0.427232, 0.314123, + 0.233236, 0.171440, 0.128188, 0.092762, 0.067569, 0.049324, + 0.036330, 0.027008, 0.019853, 0.015539, 0.011093, 0.008733, + 0.007624, 0.008105, 0.005427, 0.004065, 0.003427, 0.002848, + 0.002328, 0.001865, 0.001457, 0.001103, 0.000801, 0.000550, + 0.000348, 0.000193, 0.000085, 0.000021, 0.000000, + }, + { + 16.000000, 15.996116, 15.984769, 15.966413, 15.941505, 15.910501, + 15.873856, 15.832026, 15.785466, 15.734633, 15.679981, 15.621967, + 15.560961, 15.460157, 15.288367, 15.052462, 14.466922, 13.921212, + 13.073692, 12.222005, 11.237799, 9.985848, 8.898823, 7.423519, + 5.995325, 4.773152, 3.744032, 2.938217, 2.294526, 1.762412, + 1.327145, 1.020728, 0.765535, 0.570548, 0.425833, 0.313825, + 0.232959, 0.171324, 0.128174, 0.092750, 0.067558, 0.049319, + 0.036330, 0.027008, 0.019853, 0.015539, 0.011093, 0.008733, + 0.007624, 0.008105, 0.005427, 0.004065, 0.003427, 0.002848, + 0.002328, 0.001865, 0.001457, 0.001103, 0.000801, 0.000550, + 0.000348, 0.000193, 0.000085, 0.000021, -0.000000, + }, }; -void av1_model_rd_curvfit(double xqr, double *rate_f, double *distbysse_f) { +void av1_model_rd_curvfit(BLOCK_SIZE bsize, double sse_norm, double xqr, + double *rate_f, double *distbysse_f) { const double x_start = -15.5; const double x_end = 16.5; const double x_step = 0.5; const double epsilon = 1e-6; + const int rcat = bsize_model_cat_lookup[bsize]; + const int dcat = sse_norm_model_cat_lookup(sse_norm); (void)x_end; xqr = AOMMAX(xqr, x_start + x_step + epsilon); @@ -1109,9 +1441,9 @@ void av1_model_rd_curvfit(double xqr, double *rate_f, double *distbysse_f) { assert(xi > 0); - const double *prate = &interp_rgrid_curv[(xi - 1)]; - const double *pdist = &interp_dgrid_curv[(xi - 1)]; + const double *prate = &interp_rgrid_curv[rcat][(xi - 1)]; *rate_f = interp_cubic(prate, xo); + const double *pdist = &interp_dgrid_curv[dcat][(xi - 1)]; *distbysse_f = interp_cubic(pdist, xo); } @@ -1228,12 +1560,12 @@ int16_t *av1_raster_block_offset_int16(BLOCK_SIZE plane_bsize, int raster_block, YV12_BUFFER_CONFIG *av1_get_scaled_ref_frame(const AV1_COMP *cpi, int ref_frame) { - const AV1_COMMON *const cm = &cpi->common; - const int scaled_idx = cpi->scaled_ref_idx[ref_frame - 1]; - const int ref_idx = get_ref_frame_buf_idx(cpi, ref_frame); - return (scaled_idx != ref_idx && scaled_idx != INVALID_IDX) - ? &cm->buffer_pool->frame_bufs[scaled_idx].buf - : NULL; + assert(ref_frame >= LAST_FRAME && ref_frame <= ALTREF_FRAME); + RefCntBuffer *const scaled_buf = cpi->scaled_ref_buf[ref_frame - 1]; + const RefCntBuffer *const ref_buf = + get_ref_frame_buf(&cpi->common, ref_frame); + return (scaled_buf != ref_buf && scaled_buf != NULL) ? &scaled_buf->buf + : NULL; } int av1_get_switchable_rate(const AV1_COMMON *const cm, MACROBLOCK *x, @@ -1274,7 +1606,7 @@ void av1_set_rd_speed_thresholds(AV1_COMP *cpi) { } else { rd->thresh_mult[THR_NEARESTMV] = 0; rd->thresh_mult[THR_NEARESTL2] = 0; - rd->thresh_mult[THR_NEARESTL3] = 0; + rd->thresh_mult[THR_NEARESTL3] = 100; rd->thresh_mult[THR_NEARESTB] = 0; rd->thresh_mult[THR_NEARESTA2] = 0; rd->thresh_mult[THR_NEARESTA] = 0; @@ -1285,7 +1617,7 @@ void av1_set_rd_speed_thresholds(AV1_COMP *cpi) { rd->thresh_mult[THR_NEWL2] += 1000; rd->thresh_mult[THR_NEWL3] += 1000; rd->thresh_mult[THR_NEWB] += 1000; - rd->thresh_mult[THR_NEWA2] = 1000; + rd->thresh_mult[THR_NEWA2] = 1100; rd->thresh_mult[THR_NEWA] += 1000; rd->thresh_mult[THR_NEWG] += 1000; @@ -1297,18 +1629,18 @@ void av1_set_rd_speed_thresholds(AV1_COMP *cpi) { rd->thresh_mult[THR_NEARA] += 1000; rd->thresh_mult[THR_NEARG] += 1000; - rd->thresh_mult[THR_GLOBALMV] += 2000; + rd->thresh_mult[THR_GLOBALMV] += 2200; rd->thresh_mult[THR_GLOBALL2] += 2000; rd->thresh_mult[THR_GLOBALL3] += 2000; - rd->thresh_mult[THR_GLOBALB] += 2000; + rd->thresh_mult[THR_GLOBALB] += 2400; rd->thresh_mult[THR_GLOBALA2] = 2000; rd->thresh_mult[THR_GLOBALG] += 2000; - rd->thresh_mult[THR_GLOBALA] += 2000; + rd->thresh_mult[THR_GLOBALA] += 2400; - rd->thresh_mult[THR_COMP_NEAREST_NEARESTLA] += 1000; + rd->thresh_mult[THR_COMP_NEAREST_NEARESTLA] += 1100; rd->thresh_mult[THR_COMP_NEAREST_NEARESTL2A] += 1000; - rd->thresh_mult[THR_COMP_NEAREST_NEARESTL3A] += 1000; - rd->thresh_mult[THR_COMP_NEAREST_NEARESTGA] += 1000; + rd->thresh_mult[THR_COMP_NEAREST_NEARESTL3A] += 800; + rd->thresh_mult[THR_COMP_NEAREST_NEARESTGA] += 900; rd->thresh_mult[THR_COMP_NEAREST_NEARESTLB] += 1000; rd->thresh_mult[THR_COMP_NEAREST_NEARESTL2B] += 1000; rd->thresh_mult[THR_COMP_NEAREST_NEARESTL3B] += 1000; @@ -1326,17 +1658,17 @@ void av1_set_rd_speed_thresholds(AV1_COMP *cpi) { rd->thresh_mult[THR_COMP_NEAR_NEARLA] += 1200; rd->thresh_mult[THR_COMP_NEAREST_NEWLA] += 1500; rd->thresh_mult[THR_COMP_NEW_NEARESTLA] += 1500; - rd->thresh_mult[THR_COMP_NEAR_NEWLA] += 1700; - rd->thresh_mult[THR_COMP_NEW_NEARLA] += 1700; - rd->thresh_mult[THR_COMP_NEW_NEWLA] += 2000; - rd->thresh_mult[THR_COMP_GLOBAL_GLOBALLA] += 2500; + rd->thresh_mult[THR_COMP_NEAR_NEWLA] += 1530; + rd->thresh_mult[THR_COMP_NEW_NEARLA] += 1870; + rd->thresh_mult[THR_COMP_NEW_NEWLA] += 2400; + rd->thresh_mult[THR_COMP_GLOBAL_GLOBALLA] += 2750; rd->thresh_mult[THR_COMP_NEAR_NEARL2A] += 1200; rd->thresh_mult[THR_COMP_NEAREST_NEWL2A] += 1500; rd->thresh_mult[THR_COMP_NEW_NEARESTL2A] += 1500; - rd->thresh_mult[THR_COMP_NEAR_NEWL2A] += 1700; + rd->thresh_mult[THR_COMP_NEAR_NEWL2A] += 1870; rd->thresh_mult[THR_COMP_NEW_NEARL2A] += 1700; - rd->thresh_mult[THR_COMP_NEW_NEWL2A] += 2000; + rd->thresh_mult[THR_COMP_NEW_NEWL2A] += 1800; rd->thresh_mult[THR_COMP_GLOBAL_GLOBALL2A] += 2500; rd->thresh_mult[THR_COMP_NEAR_NEARL3A] += 1200; @@ -1345,23 +1677,23 @@ void av1_set_rd_speed_thresholds(AV1_COMP *cpi) { rd->thresh_mult[THR_COMP_NEAR_NEWL3A] += 1700; rd->thresh_mult[THR_COMP_NEW_NEARL3A] += 1700; rd->thresh_mult[THR_COMP_NEW_NEWL3A] += 2000; - rd->thresh_mult[THR_COMP_GLOBAL_GLOBALL3A] += 2500; + rd->thresh_mult[THR_COMP_GLOBAL_GLOBALL3A] += 3000; - rd->thresh_mult[THR_COMP_NEAR_NEARGA] += 1200; + rd->thresh_mult[THR_COMP_NEAR_NEARGA] += 1320; rd->thresh_mult[THR_COMP_NEAREST_NEWGA] += 1500; rd->thresh_mult[THR_COMP_NEW_NEARESTGA] += 1500; - rd->thresh_mult[THR_COMP_NEAR_NEWGA] += 1700; + rd->thresh_mult[THR_COMP_NEAR_NEWGA] += 2040; rd->thresh_mult[THR_COMP_NEW_NEARGA] += 1700; rd->thresh_mult[THR_COMP_NEW_NEWGA] += 2000; - rd->thresh_mult[THR_COMP_GLOBAL_GLOBALGA] += 2500; + rd->thresh_mult[THR_COMP_GLOBAL_GLOBALGA] += 2250; rd->thresh_mult[THR_COMP_NEAR_NEARLB] += 1200; rd->thresh_mult[THR_COMP_NEAREST_NEWLB] += 1500; rd->thresh_mult[THR_COMP_NEW_NEARESTLB] += 1500; - rd->thresh_mult[THR_COMP_NEAR_NEWLB] += 1700; + rd->thresh_mult[THR_COMP_NEAR_NEWLB] += 1360; rd->thresh_mult[THR_COMP_NEW_NEARLB] += 1700; - rd->thresh_mult[THR_COMP_NEW_NEWLB] += 2000; - rd->thresh_mult[THR_COMP_GLOBAL_GLOBALLB] += 2500; + rd->thresh_mult[THR_COMP_NEW_NEWLB] += 2400; + rd->thresh_mult[THR_COMP_GLOBAL_GLOBALLB] += 2250; rd->thresh_mult[THR_COMP_NEAR_NEARL2B] += 1200; rd->thresh_mult[THR_COMP_NEAREST_NEWL2B] += 1500; @@ -1374,7 +1706,7 @@ void av1_set_rd_speed_thresholds(AV1_COMP *cpi) { rd->thresh_mult[THR_COMP_NEAR_NEARL3B] += 1200; rd->thresh_mult[THR_COMP_NEAREST_NEWL3B] += 1500; rd->thresh_mult[THR_COMP_NEW_NEARESTL3B] += 1500; - rd->thresh_mult[THR_COMP_NEAR_NEWL3B] += 1700; + rd->thresh_mult[THR_COMP_NEAR_NEWL3B] += 1870; rd->thresh_mult[THR_COMP_NEW_NEARL3B] += 1700; rd->thresh_mult[THR_COMP_NEW_NEWL3B] += 2000; rd->thresh_mult[THR_COMP_GLOBAL_GLOBALL3B] += 2500; @@ -1388,7 +1720,7 @@ void av1_set_rd_speed_thresholds(AV1_COMP *cpi) { rd->thresh_mult[THR_COMP_GLOBAL_GLOBALGB] += 2500; rd->thresh_mult[THR_COMP_NEAR_NEARLA2] += 1200; - rd->thresh_mult[THR_COMP_NEAREST_NEWLA2] += 1500; + rd->thresh_mult[THR_COMP_NEAREST_NEWLA2] += 1800; rd->thresh_mult[THR_COMP_NEW_NEARESTLA2] += 1500; rd->thresh_mult[THR_COMP_NEAR_NEWLA2] += 1700; rd->thresh_mult[THR_COMP_NEW_NEARLA2] += 1700; @@ -1403,7 +1735,7 @@ void av1_set_rd_speed_thresholds(AV1_COMP *cpi) { rd->thresh_mult[THR_COMP_NEW_NEWL2A2] += 2000; rd->thresh_mult[THR_COMP_GLOBAL_GLOBALL2A2] += 2500; - rd->thresh_mult[THR_COMP_NEAR_NEARL3A2] += 1200; + rd->thresh_mult[THR_COMP_NEAR_NEARL3A2] += 1440; rd->thresh_mult[THR_COMP_NEAREST_NEWL3A2] += 1500; rd->thresh_mult[THR_COMP_NEW_NEARESTL3A2] += 1500; rd->thresh_mult[THR_COMP_NEAR_NEWL3A2] += 1700; @@ -1417,29 +1749,29 @@ void av1_set_rd_speed_thresholds(AV1_COMP *cpi) { rd->thresh_mult[THR_COMP_NEAR_NEWGA2] += 1700; rd->thresh_mult[THR_COMP_NEW_NEARGA2] += 1700; rd->thresh_mult[THR_COMP_NEW_NEWGA2] += 2000; - rd->thresh_mult[THR_COMP_GLOBAL_GLOBALGA2] += 2500; + rd->thresh_mult[THR_COMP_GLOBAL_GLOBALGA2] += 2750; rd->thresh_mult[THR_COMP_NEAR_NEARLL2] += 1600; rd->thresh_mult[THR_COMP_NEAREST_NEWLL2] += 2000; rd->thresh_mult[THR_COMP_NEW_NEARESTLL2] += 2000; - rd->thresh_mult[THR_COMP_NEAR_NEWLL2] += 2200; + rd->thresh_mult[THR_COMP_NEAR_NEWLL2] += 2640; rd->thresh_mult[THR_COMP_NEW_NEARLL2] += 2200; rd->thresh_mult[THR_COMP_NEW_NEWLL2] += 2400; rd->thresh_mult[THR_COMP_GLOBAL_GLOBALLL2] += 3200; rd->thresh_mult[THR_COMP_NEAR_NEARLL3] += 1600; rd->thresh_mult[THR_COMP_NEAREST_NEWLL3] += 2000; - rd->thresh_mult[THR_COMP_NEW_NEARESTLL3] += 2000; + rd->thresh_mult[THR_COMP_NEW_NEARESTLL3] += 1800; rd->thresh_mult[THR_COMP_NEAR_NEWLL3] += 2200; rd->thresh_mult[THR_COMP_NEW_NEARLL3] += 2200; rd->thresh_mult[THR_COMP_NEW_NEWLL3] += 2400; rd->thresh_mult[THR_COMP_GLOBAL_GLOBALLL3] += 3200; - rd->thresh_mult[THR_COMP_NEAR_NEARLG] += 1600; - rd->thresh_mult[THR_COMP_NEAREST_NEWLG] += 2000; + rd->thresh_mult[THR_COMP_NEAR_NEARLG] += 1760; + rd->thresh_mult[THR_COMP_NEAREST_NEWLG] += 2400; rd->thresh_mult[THR_COMP_NEW_NEARESTLG] += 2000; - rd->thresh_mult[THR_COMP_NEAR_NEWLG] += 2200; - rd->thresh_mult[THR_COMP_NEW_NEARLG] += 2200; + rd->thresh_mult[THR_COMP_NEAR_NEWLG] += 1760; + rd->thresh_mult[THR_COMP_NEW_NEARLG] += 2640; rd->thresh_mult[THR_COMP_NEW_NEWLG] += 2400; rd->thresh_mult[THR_COMP_GLOBAL_GLOBALLG] += 3200; @@ -1447,34 +1779,25 @@ void av1_set_rd_speed_thresholds(AV1_COMP *cpi) { rd->thresh_mult[THR_COMP_NEAREST_NEWBA] += 2000; rd->thresh_mult[THR_COMP_NEW_NEARESTBA] += 2000; rd->thresh_mult[THR_COMP_NEAR_NEWBA] += 2200; - rd->thresh_mult[THR_COMP_NEW_NEARBA] += 2200; - rd->thresh_mult[THR_COMP_NEW_NEWBA] += 2400; + rd->thresh_mult[THR_COMP_NEW_NEARBA] += 1980; + rd->thresh_mult[THR_COMP_NEW_NEWBA] += 2640; rd->thresh_mult[THR_COMP_GLOBAL_GLOBALBA] += 3200; rd->thresh_mult[THR_DC] += 1000; rd->thresh_mult[THR_PAETH] += 1000; - rd->thresh_mult[THR_SMOOTH] += 2000; + rd->thresh_mult[THR_SMOOTH] += 2200; rd->thresh_mult[THR_SMOOTH_V] += 2000; rd->thresh_mult[THR_SMOOTH_H] += 2000; rd->thresh_mult[THR_H_PRED] += 2000; - rd->thresh_mult[THR_V_PRED] += 2000; + rd->thresh_mult[THR_V_PRED] += 1800; rd->thresh_mult[THR_D135_PRED] += 2500; - rd->thresh_mult[THR_D203_PRED] += 2500; + rd->thresh_mult[THR_D203_PRED] += 2000; rd->thresh_mult[THR_D157_PRED] += 2500; - rd->thresh_mult[THR_D67_PRED] += 2500; + rd->thresh_mult[THR_D67_PRED] += 2000; rd->thresh_mult[THR_D113_PRED] += 2500; rd->thresh_mult[THR_D45_PRED] += 2500; } -void av1_set_rd_speed_thresholds_sub8x8(AV1_COMP *cpi) { - static const int thresh_mult[MAX_REFS] = { 2500, 2500, 2500, 2500, 2500, - 2500, 2500, 4500, 4500, 4500, - 4500, 4500, 4500, 4500, 4500, - 4500, 4500, 4500, 4500, 2500 }; - RD_OPT *const rd = &cpi->rd; - memcpy(rd->thresh_mult_sub8x8, thresh_mult, sizeof(thresh_mult)); -} - void av1_update_rd_thresh_fact(const AV1_COMMON *const cm, int (*factor_buf)[MAX_MODES], int rd_thresh, int bsize, int best_mode_index) { diff --git a/third_party/aom/av1/encoder/rd.h b/third_party/aom/av1/encoder/rd.h index 755b61df50..350eeb6d5f 100644 --- a/third_party/aom/av1/encoder/rd.h +++ b/third_party/aom/av1/encoder/rd.h @@ -48,7 +48,7 @@ extern "C" { // This enumerator type needs to be kept aligned with the mode order in // const MODE_DEFINITION av1_mode_order[MAX_MODES] used in the rd code. -typedef enum { +enum { THR_NEARESTMV, THR_NEARESTL2, THR_NEARESTL3, @@ -246,9 +246,9 @@ typedef enum { MAX_SINGLE_REF_MODES = LAST_SINGLE_REF_MODES + 1, LAST_COMP_REF_MODES = THR_COMP_GLOBAL_GLOBALBA, MAX_COMP_REF_MODES = LAST_COMP_REF_MODES + 1 -} THR_MODES; +} UENUM1BYTE(THR_MODES); -typedef enum { +enum { THR_LAST, THR_LAST2, THR_LAST3, @@ -275,7 +275,7 @@ typedef enum { THR_INTRA, MAX_REFS -} THR_MODES_SUB8X8; +} UENUM1BYTE(THR_MODES_SUB8X8); typedef struct RD_OPT { // Thresh_mult is used to set a threshold for the rd score. A higher value @@ -283,13 +283,14 @@ typedef struct RD_OPT { // is used in combination with the current block size, and thresh_freq_fact // to pick a threshold. int thresh_mult[MAX_MODES]; - int thresh_mult_sub8x8[MAX_REFS]; int threshes[MAX_SEGMENTS][BLOCK_SIZES_ALL][MAX_MODES]; int64_t prediction_type_threshes[REF_FRAMES][REFERENCE_MODES]; int RDMULT; + + double r0; } RD_OPT; static INLINE void av1_init_rd_stats(RD_STATS *rd_stats) { @@ -317,6 +318,25 @@ static INLINE void av1_init_rd_stats(RD_STATS *rd_stats) { } } #endif +#if CONFIG_ONE_PASS_SVM + rd_stats->eob = 0; + rd_stats->eob_0 = 0; + rd_stats->eob_1 = 0; + rd_stats->eob_2 = 0; + rd_stats->eob_3 = 0; + + rd_stats->rd = 0; + rd_stats->rd_0 = 0; + rd_stats->rd_1 = 0; + rd_stats->rd_2 = 0; + rd_stats->rd_3 = 0; + + rd_stats->y_sse = 0; + rd_stats->sse_0 = 0; + rd_stats->sse_1 = 0; + rd_stats->sse_2 = 0; + rd_stats->sse_3 = 0; +#endif } static INLINE void av1_invalid_rd_stats(RD_STATS *rd_stats) { @@ -344,6 +364,30 @@ static INLINE void av1_invalid_rd_stats(RD_STATS *rd_stats) { } } #endif +#if CONFIG_ONE_PASS_SVM + // TODO(chiyotsai@google.com): Change invalid values to INT_MAX and + // INT64_MAX. Currently there are some code paths where rd_stats's properties + // are set directly without calling av1_init_rd_stats, so changing it now will + // break this speed feature. Need to hunt down all places where rd_stats is + // used without initialized. + rd_stats->eob = 0; + rd_stats->eob_0 = 0; + rd_stats->eob_1 = 0; + rd_stats->eob_2 = 0; + rd_stats->eob_3 = 0; + + rd_stats->rd = 0; + rd_stats->rd_0 = 0; + rd_stats->rd_1 = 0; + rd_stats->rd_2 = 0; + rd_stats->rd_3 = 0; + + rd_stats->y_sse = 0; + rd_stats->sse_0 = 0; + rd_stats->sse_1 = 0; + rd_stats->sse_2 = 0; + rd_stats->sse_3 = 0; +#endif } static INLINE void av1_merge_rd_stats(RD_STATS *rd_stats_dst, @@ -377,26 +421,245 @@ static INLINE void av1_merge_rd_stats(RD_STATS *rd_stats_dst, } } #endif +#if CONFIG_ONE_PASS_SVM + rd_stats_dst->eob += rd_stats_src->eob; + rd_stats_dst->eob_0 += rd_stats_src->eob_0; + rd_stats_dst->eob_1 += rd_stats_src->eob_1; + rd_stats_dst->eob_2 += rd_stats_src->eob_2; + rd_stats_dst->eob_3 += rd_stats_src->eob_3; + + rd_stats_dst->rd += rd_stats_src->rd; + rd_stats_dst->rd_0 += rd_stats_src->rd_0; + rd_stats_dst->rd_1 += rd_stats_src->rd_1; + rd_stats_dst->rd_2 += rd_stats_src->rd_2; + rd_stats_dst->rd_3 += rd_stats_src->rd_3; + + rd_stats_dst->y_sse += rd_stats_src->y_sse; + rd_stats_dst->sse_0 += rd_stats_src->sse_0; + rd_stats_dst->sse_1 += rd_stats_src->sse_1; + rd_stats_dst->sse_2 += rd_stats_src->sse_2; + rd_stats_dst->sse_3 += rd_stats_src->sse_3; +#endif } +#if CONFIG_ONE_PASS_SVM +static INLINE void av1_add_reg_stat(RD_STATS *rd_stats, int eob, int64_t rd, + int64_t sse, int blk_row, int blk_col, + BLOCK_SIZE bsize, BLOCK_SIZE crop_bsize) { + // NOTE: Currently the calculation of regional features works by assuming + // bsize is square so that each transform block of size crop_bsize either + // 1. locates completely within a quadrant or + // 2. is exactly half of bsize or + // 3. is the entire prediction block + // Size of TX block and SB + const int block_width_mi = mi_size_wide[bsize]; + const int block_height_mi = mi_size_high[bsize]; + const int crop_width_mi = mi_size_wide[crop_bsize]; + const int crop_height_mi = mi_size_high[crop_bsize]; + + // Increment the eob proportionally to how much the tx_block overlaps with + // each quadrant. We will scale it by MAX_MIB_SIZE * MAX_MIB_SIZE to avoid + // being truncated. + const int max_scaling_factor = MAX_MIB_SIZE * MAX_MIB_SIZE; + + // Update the stats + rd_stats->eob = eob; + rd_stats->rd = rd; + rd_stats->y_sse = sse; + + if (crop_width_mi <= block_width_mi / 2 && + crop_height_mi <= block_width_mi / 2) { + // The transform block lies completely in a quadrant. + const int scaling_factor = max_scaling_factor; + const int r_eob = eob * scaling_factor, r_rd = rd * scaling_factor, + r_sse = sse * scaling_factor; + + if (blk_row < block_height_mi / 2 && blk_col < block_width_mi / 2) { + rd_stats->eob_0 = r_eob; + rd_stats->rd_0 = r_rd; + rd_stats->sse_0 = r_sse; + } else if (blk_row < block_height_mi / 2 && blk_col >= block_width_mi / 2) { + rd_stats->eob_1 = r_eob; + rd_stats->rd_1 = r_rd; + rd_stats->sse_1 = r_sse; + } else if (blk_row >= block_height_mi / 2 && blk_col < block_width_mi / 2) { + rd_stats->eob_2 = r_eob; + rd_stats->rd_2 = r_rd; + rd_stats->sse_2 = r_sse; + } else { + rd_stats->eob_3 = r_eob; + rd_stats->rd_3 = r_rd; + rd_stats->sse_3 = r_sse; + } + } else if (crop_height_mi == block_height_mi && + crop_width_mi == block_width_mi) { + // The transform block is the whole prediction block + const int scaling_factor = max_scaling_factor; + const int r_eob = eob * scaling_factor, r_rd = rd * scaling_factor, + r_sse = sse * scaling_factor; + + rd_stats->eob_0 = r_eob; + rd_stats->rd_0 = r_rd; + rd_stats->sse_0 = r_sse; + + rd_stats->eob_1 = r_eob; + rd_stats->rd_1 = r_rd; + rd_stats->sse_1 = r_sse; + + rd_stats->eob_2 = r_eob; + rd_stats->rd_2 = r_rd; + rd_stats->sse_2 = r_sse; + + rd_stats->eob_3 = r_eob; + rd_stats->rd_3 = r_rd; + rd_stats->sse_3 = r_sse; + } else if (crop_height_mi == block_height_mi) { + // The tranform block is a vertical block + const int scaling_factor = max_scaling_factor / 2; + const int r_eob = eob * scaling_factor, r_rd = rd * scaling_factor, + r_sse = sse * scaling_factor; + + if (blk_col < block_width_mi / 2) { + rd_stats->eob_0 = r_eob; + rd_stats->rd_0 = r_rd; + rd_stats->sse_0 = r_sse; + + rd_stats->eob_2 = r_eob; + rd_stats->rd_2 = r_rd; + rd_stats->sse_2 = r_sse; + } else { + rd_stats->eob_1 = r_eob; + rd_stats->rd_1 = r_rd; + rd_stats->sse_1 = r_sse; + + rd_stats->eob_3 = r_eob; + rd_stats->rd_3 = r_rd; + rd_stats->sse_3 = r_sse; + } + } else if (crop_width_mi == block_width_mi) { + // The tranform block is a horizontal block half the size of predition block + const int scaling_factor = max_scaling_factor / 2; + const int r_eob = eob * scaling_factor, r_rd = rd * scaling_factor, + r_sse = sse * scaling_factor; + + if (blk_row < block_height_mi / 2) { + rd_stats->eob_0 = r_eob; + rd_stats->rd_0 = r_rd; + rd_stats->sse_0 = r_sse; + + rd_stats->eob_1 = r_eob; + rd_stats->rd_1 = r_rd; + rd_stats->sse_1 = r_sse; + } else { + rd_stats->eob_2 = r_eob; + rd_stats->rd_2 = r_rd; + rd_stats->sse_2 = r_sse; + + rd_stats->eob_3 = r_eob; + rd_stats->rd_3 = r_rd; + rd_stats->sse_3 = r_sse; + } + } else { + assert(0 && "Unexpected transform size"); + } +} + +static INLINE void av1_reg_stat_skipmode_update(RD_STATS *rd_stats, + int rdmult) { + // Update the stats + rd_stats->eob = 0; + rd_stats->eob_0 = 0; + rd_stats->eob_1 = 0; + rd_stats->eob_2 = 0; + rd_stats->eob_3 = 0; + + rd_stats->rd = RDCOST(rdmult, 0, rd_stats->sse); + rd_stats->rd_0 = RDCOST(rdmult, 0, rd_stats->sse_0); + rd_stats->rd_1 = RDCOST(rdmult, 0, rd_stats->sse_1); + rd_stats->rd_2 = RDCOST(rdmult, 0, rd_stats->sse_2); + rd_stats->rd_3 = RDCOST(rdmult, 0, rd_stats->sse_3); +} + +static INLINE void av1_copy_reg_stat(RD_STATS *rd_stats_dst, + RD_STATS *rd_stats_src) { + rd_stats_dst->eob = rd_stats_src->eob; + rd_stats_dst->eob_0 = rd_stats_src->eob_0; + rd_stats_dst->eob_1 = rd_stats_src->eob_1; + rd_stats_dst->eob_2 = rd_stats_src->eob_2; + rd_stats_dst->eob_3 = rd_stats_src->eob_3; + + rd_stats_dst->rd = rd_stats_src->rd; + rd_stats_dst->rd_0 = rd_stats_src->rd_0; + rd_stats_dst->rd_1 = rd_stats_src->rd_1; + rd_stats_dst->rd_2 = rd_stats_src->rd_2; + rd_stats_dst->rd_3 = rd_stats_src->rd_3; + + rd_stats_dst->y_sse = rd_stats_src->y_sse; + rd_stats_dst->sse_0 = rd_stats_src->sse_0; + rd_stats_dst->sse_1 = rd_stats_src->sse_1; + rd_stats_dst->sse_2 = rd_stats_src->sse_2; + rd_stats_dst->sse_3 = rd_stats_src->sse_3; +} + +static INLINE void av1_unpack_reg_stat(RD_STATS *rd_stats, int *eob, int *eob_0, + int *eob_1, int *eob_2, int *eob_3, + int64_t *rd, int64_t *rd_0, + int64_t *rd_1, int64_t *rd_2, + int64_t *rd_3) { + *rd = rd_stats->rd; + *rd_0 = rd_stats->rd_0; + *rd_1 = rd_stats->rd_1; + *rd_2 = rd_stats->rd_2; + *rd_3 = rd_stats->rd_3; + + *eob = rd_stats->eob; + *eob_0 = rd_stats->eob_0; + *eob_1 = rd_stats->eob_1; + *eob_2 = rd_stats->eob_2; + *eob_3 = rd_stats->eob_3; +} + +static INLINE void av1_set_reg_stat(RD_STATS *rd_stats, int eob, int eob_0, + int eob_1, int eob_2, int eob_3, int64_t rd, + int64_t rd_0, int64_t rd_1, int64_t rd_2, + int64_t rd_3) { + rd_stats->rd = rd; + rd_stats->rd_0 = rd_0; + rd_stats->rd_1 = rd_1; + rd_stats->rd_2 = rd_2; + rd_stats->rd_3 = rd_3; + + rd_stats->eob = eob; + rd_stats->eob_0 = eob_0; + rd_stats->eob_1 = eob_1; + rd_stats->eob_2 = eob_2; + rd_stats->eob_3 = eob_3; +} +#endif + struct TileInfo; struct TileDataEnc; struct AV1_COMP; struct macroblock; +int av1_compute_rd_mult_based_on_qindex(const struct AV1_COMP *cpi, int qindex); + int av1_compute_rd_mult(const struct AV1_COMP *cpi, int qindex); void av1_initialize_rd_consts(struct AV1_COMP *cpi); +void av1_initialize_cost_tables(const AV1_COMMON *const cm, MACROBLOCK *x); + void av1_initialize_me_consts(const struct AV1_COMP *cpi, MACROBLOCK *x, int qindex); void av1_model_rd_from_var_lapndz(int64_t var, unsigned int n, unsigned int qstep, int *rate, int64_t *dist); -void av1_model_rd_curvfit(double xqr, double *rate_f, double *distbysse_f); -void av1_model_rd_surffit(double xm, double yl, double *rate_f, - double *distbysse_f); +void av1_model_rd_curvfit(BLOCK_SIZE bsize, double sse_norm, double xqr, + double *rate_f, double *distbysse_f); +void av1_model_rd_surffit(BLOCK_SIZE bsize, double sse_norm, double xm, + double yl, double *rate_f, double *distbysse_f); int av1_get_switchable_rate(const AV1_COMMON *const cm, MACROBLOCK *x, const MACROBLOCKD *xd); @@ -421,8 +684,6 @@ void av1_get_entropy_contexts(BLOCK_SIZE bsize, void av1_set_rd_speed_thresholds(struct AV1_COMP *cpi); -void av1_set_rd_speed_thresholds_sub8x8(struct AV1_COMP *cpi); - void av1_update_rd_thresh_fact(const AV1_COMMON *const cm, int (*fact)[MAX_MODES], int rd_thresh, int bsize, int best_mode_index); @@ -457,6 +718,8 @@ void av1_fill_mode_rates(AV1_COMMON *const cm, MACROBLOCK *x, void av1_fill_coeff_costs(MACROBLOCK *x, FRAME_CONTEXT *fc, const int num_planes); +int av1_get_adaptive_rdmult(const struct AV1_COMP *cpi, double beta); + #ifdef __cplusplus } // extern "C" #endif diff --git a/third_party/aom/av1/encoder/rdopt.c b/third_party/aom/av1/encoder/rdopt.c index c2d15534f0..0be76e4cb2 100644 --- a/third_party/aom/av1/encoder/rdopt.c +++ b/third_party/aom/av1/encoder/rdopt.c @@ -11,6 +11,7 @@ #include #include +#include #include "config/aom_dsp_rtcd.h" #include "config/av1_rtcd.h" @@ -30,6 +31,7 @@ #include "av1/common/idct.h" #include "av1/common/mvref_common.h" #include "av1/common/obmc.h" +#include "av1/common/onyxc_int.h" #include "av1/common/pred_common.h" #include "av1/common/quant_common.h" #include "av1/common/reconinter.h" @@ -59,6 +61,13 @@ #include "av1/encoder/tokenize.h" #include "av1/encoder/tx_prune_model_weights.h" +// Set this macro as 1 to collect data about tx size selection. +#define COLLECT_TX_SIZE_DATA 0 + +#if COLLECT_TX_SIZE_DATA +static const char av1_tx_size_data_output_file[] = "tx_size_data.txt"; +#endif + typedef void (*model_rd_for_sb_type)( const AV1_COMP *const cpi, BLOCK_SIZE bsize, MACROBLOCK *x, MACROBLOCKD *xd, int plane_from, int plane_to, int mi_row, int mi_col, int *out_rate_sum, @@ -116,14 +125,14 @@ static void model_rd_with_surffit(const AV1_COMP *const cpi, int64_t sse, int num_samples, int *rate, int64_t *dist); -typedef enum { +enum { MODELRD_LEGACY, MODELRD_CURVFIT, MODELRD_SUFFIT, MODELRD_DNN, MODELRD_FULLRDY, MODELRD_TYPES -} ModelRdType; +} UENUM1BYTE(ModelRdType); static model_rd_for_sb_type model_rd_sb_fn[MODELRD_TYPES] = { model_rd_for_sb, model_rd_for_sb_with_curvfit, model_rd_for_sb_with_surffit, @@ -141,11 +150,12 @@ static model_rd_from_sse_type model_rd_sse_fn[MODELRD_TYPES] = { // 3: DNN regression model // 4: Full rd model #define MODELRD_TYPE_INTERP_FILTER 1 -#define MODELRD_TYPE_TX_SEARCH_PRUNE 2 +#define MODELRD_TYPE_TX_SEARCH_PRUNE 1 #define MODELRD_TYPE_MASKED_COMPOUND 1 #define MODELRD_TYPE_INTERINTRA 1 #define MODELRD_TYPE_INTRA 1 -#define MODELRD_TYPE_JNT_COMPOUND 1 +#define MODELRD_TYPE_DIST_WTD_COMPOUND 1 +#define MODELRD_TYPE_MOTION_MODE_RD 1 #define DUAL_FILTER_SET_SIZE (SWITCHABLE_FILTERS * SWITCHABLE_FILTERS) static const InterpFilters filter_sets[DUAL_FILTER_SET_SIZE] = { @@ -158,8 +168,6 @@ static const InterpFilters filter_sets[DUAL_FILTER_SET_SIZE] = { ((1 << ALTREF_FRAME) | (1 << ALTREF2_FRAME) | (1 << BWDREF_FRAME) | \ (1 << GOLDEN_FRAME) | (1 << LAST2_FRAME) | 0x01) -#define ANGLE_SKIP_THRESH 10 - static const double ADST_FLIP_SVM[8] = { /* vertical */ -6.6623, -2.8062, -3.2531, 3.1671, @@ -176,22 +184,12 @@ typedef struct { MV_REFERENCE_FRAME ref_frame[2]; } REF_DEFINITION; -typedef enum { +enum { FTXS_NONE = 0, FTXS_DCT_AND_1D_DCT_ONLY = 1 << 0, FTXS_DISABLE_TRELLIS_OPT = 1 << 1, FTXS_USE_TRANSFORM_DOMAIN = 1 << 2 -} FAST_TX_SEARCH_MODE; - -static void select_tx_type_yrd(const AV1_COMP *cpi, MACROBLOCK *x, - RD_STATS *rd_stats, BLOCK_SIZE bsize, int mi_row, - int mi_col, int64_t ref_best_rd); - -static int inter_block_uvrd(const AV1_COMP *cpi, MACROBLOCK *x, - RD_STATS *rd_stats, BLOCK_SIZE bsize, - int64_t non_skip_ref_best_rd, - int64_t skip_ref_best_rd, - FAST_TX_SEARCH_MODE ftxs_mode); +} UENUM1BYTE(FAST_TX_SEARCH_MODE); struct rdcost_block_args { const AV1_COMP *cpi; @@ -406,32 +404,36 @@ static const MODE_DEFINITION av1_mode_order[MAX_MODES] = { }; static const int16_t intra_to_mode_idx[INTRA_MODE_NUM] = { - 7, // DC_PRED, - 134, // V_PRED, - 133, // H_PRED, - 140, // D45_PRED, - 135, // D135_PRED, - 139, // D113_PRED, - 137, // D157_PRED, - 136, // D203_PRED, - 138, // D67_PRED, - 46, // SMOOTH_PRED, - 47, // SMOOTH_V_PRED, - 48, // SMOOTH_H_PRED, - 45, // PAETH_PRED, + THR_DC, // DC_PRED, + THR_V_PRED, // V_PRED, + THR_H_PRED, // H_PRED, + THR_D45_PRED, // D45_PRED, + THR_D135_PRED, // D135_PRED, + THR_D113_PRED, // D113_PRED, + THR_D157_PRED, // D157_PRED, + THR_D203_PRED, // D203_PRED, + THR_D67_PRED, // D67_PRED, + THR_SMOOTH, // SMOOTH_PRED, + THR_SMOOTH_V, // SMOOTH_V_PRED, + THR_SMOOTH_H, // SMOOTH_H_PRED, + THR_PAETH, // PAETH_PRED, }; /* clang-format off */ static const int16_t single_inter_to_mode_idx[SINGLE_INTER_MODE_NUM] [REF_FRAMES] = { // NEARESTMV, - { -1, 0, 1, 2, 6, 3, 4, 5, }, + { -1, THR_NEARESTMV, THR_NEARESTL2, THR_NEARESTL3, + THR_NEARESTG, THR_NEARESTB, THR_NEARESTA2, THR_NEARESTA, }, // NEARMV, - { -1, 15, 16, 17, 21, 18, 19, 20, }, + { -1, THR_NEARMV, THR_NEARL2, THR_NEARL3, + THR_NEARG, THR_NEARB, THR_NEARA2, THR_NEARA, }, // GLOBALMV, - { -1, 22, 23, 24, 27, 25, 26, 28, }, + { -1, THR_GLOBALMV, THR_GLOBALL2, THR_GLOBALL3, + THR_GLOBALG, THR_GLOBALB, THR_GLOBALA2, THR_GLOBALA, }, // NEWMV, - { -1, 8, 9, 10, 14, 11, 12, 13, }, + { -1, THR_NEWMV, THR_NEWL2, THR_NEWL3, + THR_NEWG, THR_NEWB, THR_NEWA2, THR_NEWA, }, }; /* clang-format on */ @@ -440,91 +442,211 @@ static const int16_t comp_inter_to_mode_idx[COMP_INTER_MODE_NUM][REF_FRAMES] [REF_FRAMES] = { // NEAREST_NEARESTMV, { - { -1, -1, -1, -1, -1, -1, -1, -1, }, - { -1, -1, 41, 42, 43, 33, 37, 29, }, - { -1, -1, -1, -1, -1, 34, 38, 30, }, - { -1, -1, -1, -1, -1, 35, 39, 31, }, - { -1, -1, -1, -1, -1, 36, 40, 32, }, - { -1, -1, -1, -1, -1, -1, -1, 44, }, - { -1, -1, -1, -1, -1, -1, -1, -1, }, - { -1, -1, -1, -1, -1, -1, -1, -1, }, + { -1, -1, -1, -1, -1, -1, -1, -1, }, + { -1, -1, + THR_COMP_NEAREST_NEARESTLL2, THR_COMP_NEAREST_NEARESTLL3, + THR_COMP_NEAREST_NEARESTLG, THR_COMP_NEAREST_NEARESTLB, + THR_COMP_NEAREST_NEARESTLA2, THR_COMP_NEAREST_NEARESTLA, }, + { -1, -1, + -1, -1, + -1, THR_COMP_NEAREST_NEARESTL2B, + THR_COMP_NEAREST_NEARESTL2A2, THR_COMP_NEAREST_NEARESTL2A, }, + { -1, -1, + -1, -1, + -1, THR_COMP_NEAREST_NEARESTL3B, + THR_COMP_NEAREST_NEARESTL3A2, THR_COMP_NEAREST_NEARESTL3A, }, + { -1, -1, + -1, -1, + -1, THR_COMP_NEAREST_NEARESTGB, + THR_COMP_NEAREST_NEARESTGA2, THR_COMP_NEAREST_NEARESTGA, }, + { -1, -1, + -1, -1, + -1, -1, + -1, THR_COMP_NEAREST_NEARESTBA, }, + { -1, -1, -1, -1, -1, -1, -1, -1, }, + { -1, -1, -1, -1, -1, -1, -1, -1, }, }, // NEAR_NEARMV, { - { -1, -1, -1, -1, -1, -1, -1, -1, }, - { -1, -1, 141, 148, 155, 77, 105, 49, }, - { -1, -1, -1, -1, -1, 84, 112, 56, }, - { -1, -1, -1, -1, -1, 91, 119, 63, }, - { -1, -1, -1, -1, -1, 98, 126, 70, }, - { -1, -1, -1, -1, -1, -1, -1, 162, }, - { -1, -1, -1, -1, -1, -1, -1, -1, }, - { -1, -1, -1, -1, -1, -1, -1, -1, }, + { -1, -1, -1, -1, -1, -1, -1, -1, }, + { -1, -1, + THR_COMP_NEAR_NEARLL2, THR_COMP_NEAR_NEARLL3, + THR_COMP_NEAR_NEARLG, THR_COMP_NEAR_NEARLB, + THR_COMP_NEAR_NEARLA2, THR_COMP_NEAR_NEARLA, }, + { -1, -1, + -1, -1, + -1, THR_COMP_NEAR_NEARL2B, + THR_COMP_NEAR_NEARL2A2, THR_COMP_NEAR_NEARL2A, }, + { -1, -1, + -1, -1, + -1, THR_COMP_NEAR_NEARL3B, + THR_COMP_NEAR_NEARL3A2, THR_COMP_NEAR_NEARL3A, }, + { -1, -1, + -1, -1, + -1, THR_COMP_NEAR_NEARGB, + THR_COMP_NEAR_NEARGA2, THR_COMP_NEAR_NEARGA, }, + { -1, -1, + -1, -1, + -1, -1, + -1, THR_COMP_NEAR_NEARBA, }, + { -1, -1, -1, -1, -1, -1, -1, -1, }, + { -1, -1, -1, -1, -1, -1, -1, -1, }, }, // NEAREST_NEWMV, { - { -1, -1, -1, -1, -1, -1, -1, -1, }, - { -1, -1, 143, 150, 157, 79, 107, 51, }, - { -1, -1, -1, -1, -1, 86, 114, 58, }, - { -1, -1, -1, -1, -1, 93, 121, 65, }, - { -1, -1, -1, -1, -1, 100, 128, 72, }, - { -1, -1, -1, -1, -1, -1, -1, 164, }, - { -1, -1, -1, -1, -1, -1, -1, -1, }, - { -1, -1, -1, -1, -1, -1, -1, -1, }, + { -1, -1, -1, -1, -1, -1, -1, -1, }, + { -1, -1, + THR_COMP_NEAREST_NEWLL2, THR_COMP_NEAREST_NEWLL3, + THR_COMP_NEAREST_NEWLG, THR_COMP_NEAREST_NEWLB, + THR_COMP_NEAREST_NEWLA2, THR_COMP_NEAREST_NEWLA, }, + { -1, -1, + -1, -1, + -1, THR_COMP_NEAREST_NEWL2B, + THR_COMP_NEAREST_NEWL2A2, THR_COMP_NEAREST_NEWL2A, }, + { -1, -1, + -1, -1, + -1, THR_COMP_NEAREST_NEWL3B, + THR_COMP_NEAREST_NEWL3A2, THR_COMP_NEAREST_NEWL3A, }, + { -1, -1, + -1, -1, + -1, THR_COMP_NEAREST_NEWGB, + THR_COMP_NEAREST_NEWGA2, THR_COMP_NEAREST_NEWGA, }, + { -1, -1, + -1, -1, + -1, -1, + -1, THR_COMP_NEAREST_NEWBA, }, + { -1, -1, -1, -1, -1, -1, -1, -1, }, + { -1, -1, -1, -1, -1, -1, -1, -1, }, }, // NEW_NEARESTMV, { - { -1, -1, -1, -1, -1, -1, -1, -1, }, - { -1, -1, 142, 149, 156, 78, 106, 50, }, - { -1, -1, -1, -1, -1, 85, 113, 57, }, - { -1, -1, -1, -1, -1, 92, 120, 64, }, - { -1, -1, -1, -1, -1, 99, 127, 71, }, - { -1, -1, -1, -1, -1, -1, -1, 163, }, - { -1, -1, -1, -1, -1, -1, -1, -1, }, - { -1, -1, -1, -1, -1, -1, -1, -1, }, + { -1, -1, -1, -1, -1, -1, -1, -1, }, + { -1, -1, + THR_COMP_NEW_NEARESTLL2, THR_COMP_NEW_NEARESTLL3, + THR_COMP_NEW_NEARESTLG, THR_COMP_NEW_NEARESTLB, + THR_COMP_NEW_NEARESTLA2, THR_COMP_NEW_NEARESTLA, }, + { -1, -1, + -1, -1, + -1, THR_COMP_NEW_NEARESTL2B, + THR_COMP_NEW_NEARESTL2A2, THR_COMP_NEW_NEARESTL2A, }, + { -1, -1, + -1, -1, + -1, THR_COMP_NEW_NEARESTL3B, + THR_COMP_NEW_NEARESTL3A2, THR_COMP_NEW_NEARESTL3A, }, + { -1, -1, + -1, -1, + -1, THR_COMP_NEW_NEARESTGB, + THR_COMP_NEW_NEARESTGA2, THR_COMP_NEW_NEARESTGA, }, + { -1, -1, + -1, -1, + -1, -1, + -1, THR_COMP_NEW_NEARESTBA, }, + { -1, -1, -1, -1, -1, -1, -1, -1, }, + { -1, -1, -1, -1, -1, -1, -1, -1, }, }, // NEAR_NEWMV, { - { -1, -1, -1, -1, -1, -1, -1, -1, }, - { -1, -1, 145, 152, 159, 81, 109, 53, }, - { -1, -1, -1, -1, -1, 88, 116, 60, }, - { -1, -1, -1, -1, -1, 95, 123, 67, }, - { -1, -1, -1, -1, -1, 102, 130, 74, }, - { -1, -1, -1, -1, -1, -1, -1, 166, }, - { -1, -1, -1, -1, -1, -1, -1, -1, }, - { -1, -1, -1, -1, -1, -1, -1, -1, }, + { -1, -1, -1, -1, -1, -1, -1, -1, }, + { -1, -1, + THR_COMP_NEAR_NEWLL2, THR_COMP_NEAR_NEWLL3, + THR_COMP_NEAR_NEWLG, THR_COMP_NEAR_NEWLB, + THR_COMP_NEAR_NEWLA2, THR_COMP_NEAR_NEWLA, }, + { -1, -1, + -1, -1, + -1, THR_COMP_NEAR_NEWL2B, + THR_COMP_NEAR_NEWL2A2, THR_COMP_NEAR_NEWL2A, }, + { -1, -1, + -1, -1, + -1, THR_COMP_NEAR_NEWL3B, + THR_COMP_NEAR_NEWL3A2, THR_COMP_NEAR_NEWL3A, }, + { -1, -1, + -1, -1, + -1, THR_COMP_NEAR_NEWGB, + THR_COMP_NEAR_NEWGA2, THR_COMP_NEAR_NEWGA, }, + { -1, -1, + -1, -1, + -1, -1, + -1, THR_COMP_NEAR_NEWBA, }, + { -1, -1, -1, -1, -1, -1, -1, -1, }, + { -1, -1, -1, -1, -1, -1, -1, -1, }, }, // NEW_NEARMV, { - { -1, -1, -1, -1, -1, -1, -1, -1, }, - { -1, -1, 144, 151, 158, 80, 108, 52, }, - { -1, -1, -1, -1, -1, 87, 115, 59, }, - { -1, -1, -1, -1, -1, 94, 122, 66, }, - { -1, -1, -1, -1, -1, 101, 129, 73, }, - { -1, -1, -1, -1, -1, -1, -1, 165, }, - { -1, -1, -1, -1, -1, -1, -1, -1, }, - { -1, -1, -1, -1, -1, -1, -1, -1, }, + { -1, -1, -1, -1, -1, -1, -1, -1, }, + { -1, -1, + THR_COMP_NEW_NEARLL2, THR_COMP_NEW_NEARLL3, + THR_COMP_NEW_NEARLG, THR_COMP_NEW_NEARLB, + THR_COMP_NEW_NEARLA2, THR_COMP_NEW_NEARLA, }, + { -1, -1, + -1, -1, + -1, THR_COMP_NEW_NEARL2B, + THR_COMP_NEW_NEARL2A2, THR_COMP_NEW_NEARL2A, }, + { -1, -1, + -1, -1, + -1, THR_COMP_NEW_NEARL3B, + THR_COMP_NEW_NEARL3A2, THR_COMP_NEW_NEARL3A, }, + { -1, -1, + -1, -1, + -1, THR_COMP_NEW_NEARGB, + THR_COMP_NEW_NEARGA2, THR_COMP_NEW_NEARGA, }, + { -1, -1, + -1, -1, + -1, -1, + -1, THR_COMP_NEW_NEARBA, }, + { -1, -1, -1, -1, -1, -1, -1, -1, }, + { -1, -1, -1, -1, -1, -1, -1, -1, }, }, // GLOBAL_GLOBALMV, { - { -1, -1, -1, -1, -1, -1, -1, -1, }, - { -1, -1, 147, 154, 161, 83, 111, 55, }, - { -1, -1, -1, -1, -1, 90, 118, 62, }, - { -1, -1, -1, -1, -1, 97, 125, 69, }, - { -1, -1, -1, -1, -1, 104, 132, 76, }, - { -1, -1, -1, -1, -1, -1, -1, 168, }, - { -1, -1, -1, -1, -1, -1, -1, -1, }, - { -1, -1, -1, -1, -1, -1, -1, -1, }, + { -1, -1, -1, -1, -1, -1, -1, -1, }, + { -1, -1, + THR_COMP_GLOBAL_GLOBALLL2, THR_COMP_GLOBAL_GLOBALLL3, + THR_COMP_GLOBAL_GLOBALLG, THR_COMP_GLOBAL_GLOBALLB, + THR_COMP_GLOBAL_GLOBALLA2, THR_COMP_GLOBAL_GLOBALLA, }, + { -1, -1, + -1, -1, + -1, THR_COMP_GLOBAL_GLOBALL2B, + THR_COMP_GLOBAL_GLOBALL2A2, THR_COMP_GLOBAL_GLOBALL2A, }, + { -1, -1, + -1, -1, + -1, THR_COMP_GLOBAL_GLOBALL3B, + THR_COMP_GLOBAL_GLOBALL3A2, THR_COMP_GLOBAL_GLOBALL3A, }, + { -1, -1, + -1, -1, + -1, THR_COMP_GLOBAL_GLOBALGB, + THR_COMP_GLOBAL_GLOBALGA2, THR_COMP_GLOBAL_GLOBALGA, }, + { -1, -1, + -1, -1, + -1, -1, + -1, THR_COMP_GLOBAL_GLOBALBA, }, + { -1, -1, -1, -1, -1, -1, -1, -1, }, + { -1, -1, -1, -1, -1, -1, -1, -1, }, }, // NEW_NEWMV, { - { -1, -1, -1, -1, -1, -1, -1, -1, }, - { -1, -1, 146, 153, 160, 82, 110, 54, }, - { -1, -1, -1, -1, -1, 89, 117, 61, }, - { -1, -1, -1, -1, -1, 96, 124, 68, }, - { -1, -1, -1, -1, -1, 103, 131, 75, }, - { -1, -1, -1, -1, -1, -1, -1, 167, }, - { -1, -1, -1, -1, -1, -1, -1, -1, }, - { -1, -1, -1, -1, -1, -1, -1, -1, }, + { -1, -1, -1, -1, -1, -1, -1, -1, }, + { -1, -1, + THR_COMP_NEW_NEWLL2, THR_COMP_NEW_NEWLL3, + THR_COMP_NEW_NEWLG, THR_COMP_NEW_NEWLB, + THR_COMP_NEW_NEWLA2, THR_COMP_NEW_NEWLA, }, + { -1, -1, + -1, -1, + -1, THR_COMP_NEW_NEWL2B, + THR_COMP_NEW_NEWL2A2, THR_COMP_NEW_NEWL2A, }, + { -1, -1, + -1, -1, + -1, THR_COMP_NEW_NEWL3B, + THR_COMP_NEW_NEWL3A2, THR_COMP_NEW_NEWL3A, }, + { -1, -1, + -1, -1, + -1, THR_COMP_NEW_NEWGB, + THR_COMP_NEW_NEWGA2, THR_COMP_NEW_NEWGA, }, + { -1, -1, + -1, -1, + -1, -1, + -1, THR_COMP_NEW_NEWBA, }, + { -1, -1, -1, -1, -1, -1, -1, -1, }, + { -1, -1, -1, -1, -1, -1, -1, -1, }, }, }; /* clang-format on */ @@ -618,12 +740,12 @@ typedef struct InterModeSearchState { MV_REFERENCE_FRAME single_rd_order[2][SINGLE_INTER_MODE_NUM][FWD_REFS]; } InterModeSearchState; -#if CONFIG_COLLECT_INTER_MODE_RD_STATS -int inter_mode_data_block_idx(BLOCK_SIZE bsize) { - if (bsize == BLOCK_8X8) return 1; - if (bsize == BLOCK_16X16) return 2; - if (bsize == BLOCK_32X32) return 3; - return -1; +static int inter_mode_data_block_idx(BLOCK_SIZE bsize) { + if (bsize == BLOCK_4X4 || bsize == BLOCK_4X8 || bsize == BLOCK_8X4 || + bsize == BLOCK_4X16 || bsize == BLOCK_16X4) { + return -1; + } + return 1; } void av1_inter_mode_data_init(TileDataEnc *tile_data) { @@ -639,37 +761,41 @@ void av1_inter_mode_data_init(TileDataEnc *tile_data) { } } -static int get_est_rate_dist(TileDataEnc *tile_data, BLOCK_SIZE bsize, +static int get_est_rate_dist(const TileDataEnc *tile_data, BLOCK_SIZE bsize, int64_t sse, int *est_residue_cost, int64_t *est_dist) { aom_clear_system_state(); const InterModeRdModel *md = &tile_data->inter_mode_rd_models[bsize]; if (md->ready) { - const double est_ld = md->a * sse + md->b; if (sse < md->dist_mean) { *est_residue_cost = 0; *est_dist = sse; } else { - *est_residue_cost = (int)round((sse - md->dist_mean) / est_ld); *est_dist = (int64_t)round(md->dist_mean); + const double est_ld = md->a * sse + md->b; + // Clamp estimated rate cost by INT_MAX / 2. + // TODO(angiebird@google.com): find better solution than clamping. + if (fabs(est_ld) < 1e-2) { + *est_residue_cost = INT_MAX / 2; + } else { + double est_residue_cost_dbl = ((sse - md->dist_mean) / est_ld); + if (est_residue_cost_dbl < 0) { + *est_residue_cost = 0; + } else { + *est_residue_cost = + (int)AOMMIN((int64_t)round(est_residue_cost_dbl), INT_MAX / 2); + } + } + if (*est_residue_cost <= 0) { + *est_residue_cost = 0; + *est_dist = sse; + } } return 1; } return 0; } -static int64_t get_est_rd(TileDataEnc *tile_data, BLOCK_SIZE bsize, int rdmult, - int64_t sse, int curr_cost) { - int est_residue_cost; - int64_t est_dist; - if (get_est_rate_dist(tile_data, bsize, sse, &est_residue_cost, &est_dist)) { - int rate = est_residue_cost + curr_cost; - int64_t est_rd = RDCOST(rdmult, rate, est_dist); - return est_rd; - } - return 0; -} - void av1_inter_mode_data_fit(TileDataEnc *tile_data, int rdmult) { aom_clear_system_state(); for (int bsize = 0; bsize < BLOCK_SIZES_ALL; ++bsize) { @@ -734,7 +860,7 @@ static void inter_mode_data_push(TileDataEnc *tile_data, BLOCK_SIZE bsize, rd_model->dist_sum += dist; rd_model->ld_sum += ld; rd_model->sse_sum += sse; - rd_model->sse_sse_sum += sse * sse; + rd_model->sse_sse_sum += (double)sse * (double)sse; rd_model->sse_ld_sum += sse * ld; } } @@ -773,7 +899,6 @@ static void inter_modes_info_sort(const InterModesInfo *inter_modes_info, qsort(rd_idx_pair_arr, inter_modes_info->num, sizeof(rd_idx_pair_arr[0]), compare_rd_idx_pair); } -#endif // CONFIG_COLLECT_INTER_MODE_RD_STATS static INLINE int write_uniform_cost(int n, int v) { const int l = get_unsigned_bits(n); @@ -830,7 +955,7 @@ static unsigned pixel_dist_visible_only( } const MACROBLOCKD *xd = &x->e_mbd; - if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + if (is_cur_buf_hbd(xd)) { uint64_t sse64 = aom_highbd_sse_odd_size(src, src_stride, dst, dst_stride, visible_cols, visible_rows); return (unsigned int)ROUND_POWER_OF_TWO(sse64, (xd->bd - 8) * 2); @@ -1086,7 +1211,7 @@ int64_t av1_dist_8x8(const AV1_COMP *const cpi, const MACROBLOCK *x, if (x->tune_metric == AOM_TUNE_CDEF_DIST || x->tune_metric == AOM_TUNE_DAALA_DIST) { - if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + if (is_cur_buf_hbd(xd)) { for (j = 0; j < bsh; j++) for (i = 0; i < bsw; i++) orig[j * bsw + i] = CONVERT_TO_SHORTPTR(src)[j * src_stride + i]; @@ -1150,8 +1275,7 @@ int64_t av1_dist_8x8(const AV1_COMP *const cpi, const MACROBLOCK *x, bsw, coeff_shift); } } - if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) - d = ((uint64_t)d) >> 2 * coeff_shift; + if (is_cur_buf_hbd(xd)) d = ((uint64_t)d) >> 2 * coeff_shift; } else { // Otherwise, MSE by default d = pixel_dist_visible_only(cpi, x, src, src_stride, dst, dst_stride, @@ -1179,7 +1303,7 @@ static int64_t dist_8x8_diff(const MACROBLOCK *x, const uint8_t *src, if (x->tune_metric == AOM_TUNE_CDEF_DIST || x->tune_metric == AOM_TUNE_DAALA_DIST) { - if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + if (is_cur_buf_hbd(xd)) { for (j = 0; j < bsh; j++) for (i = 0; i < bsw; i++) orig[j * bsw + i] = CONVERT_TO_SHORTPTR(src)[j * src_stride + i]; @@ -1372,54 +1496,10 @@ static int adst_vs_flipadst(const AV1_COMP *cpi, BLOCK_SIZE bsize, return prune_bitmask; } -static void get_horver_correlation(const int16_t *diff, int stride, int w, - int h, double *hcorr, double *vcorr) { - // Returns hor/ver correlation coefficient - const int num = (h - 1) * (w - 1); - double num_r; - int i, j; - int64_t xy_sum = 0, xz_sum = 0; - int64_t x_sum = 0, y_sum = 0, z_sum = 0; - int64_t x2_sum = 0, y2_sum = 0, z2_sum = 0; - double x_var_n, y_var_n, z_var_n, xy_var_n, xz_var_n; - *hcorr = *vcorr = 1; - - assert(num > 0); - num_r = 1.0 / num; - for (i = 1; i < h; ++i) { - for (j = 1; j < w; ++j) { - const int16_t x = diff[i * stride + j]; - const int16_t y = diff[i * stride + j - 1]; - const int16_t z = diff[(i - 1) * stride + j]; - xy_sum += x * y; - xz_sum += x * z; - x_sum += x; - y_sum += y; - z_sum += z; - x2_sum += x * x; - y2_sum += y * y; - z2_sum += z * z; - } - } - x_var_n = x2_sum - (x_sum * x_sum) * num_r; - y_var_n = y2_sum - (y_sum * y_sum) * num_r; - z_var_n = z2_sum - (z_sum * z_sum) * num_r; - xy_var_n = xy_sum - (x_sum * y_sum) * num_r; - xz_var_n = xz_sum - (x_sum * z_sum) * num_r; - if (x_var_n > 0 && y_var_n > 0) { - *hcorr = xy_var_n / sqrt(x_var_n * y_var_n); - *hcorr = *hcorr < 0 ? 0 : *hcorr; - } - if (x_var_n > 0 && z_var_n > 0) { - *vcorr = xz_var_n / sqrt(x_var_n * z_var_n); - *vcorr = *vcorr < 0 ? 0 : *vcorr; - } -} - static int dct_vs_idtx(const int16_t *diff, int stride, int w, int h) { - double hcorr, vcorr; + float hcorr, vcorr; int prune_bitmask = 0; - get_horver_correlation(diff, stride, w, h, &hcorr, &vcorr); + av1_get_horver_correlation_full(diff, stride, w, h, &hcorr, &vcorr); if (vcorr > FAST_EXT_TX_CORR_MID + FAST_EXT_TX_CORR_MARGIN) prune_bitmask |= 1 << IDTX_1D; @@ -1539,69 +1619,100 @@ static void get_energy_distribution_finer(const int16_t *diff, int stride, // Similar to get_horver_correlation, but also takes into account first // row/column, when computing horizontal/vertical correlation. -static void get_horver_correlation_full(const int16_t *diff, int stride, int w, - int h, float *hcorr, float *vcorr) { - const float num_hor = (float)(h * (w - 1)); - const float num_ver = (float)((h - 1) * w); - int i, j; - +void av1_get_horver_correlation_full_c(const int16_t *diff, int stride, + int width, int height, float *hcorr, + float *vcorr) { // The following notation is used: // x - current pixel // y - left neighbor pixel // z - top neighbor pixel - int64_t xy_sum = 0, xz_sum = 0; - int64_t xhor_sum = 0, xver_sum = 0, y_sum = 0, z_sum = 0; - int64_t x2hor_sum = 0, x2ver_sum = 0, y2_sum = 0, z2_sum = 0; + int64_t x_sum = 0, x2_sum = 0, xy_sum = 0, xz_sum = 0; + int64_t x_firstrow = 0, x_finalrow = 0, x_firstcol = 0, x_finalcol = 0; + int64_t x2_firstrow = 0, x2_finalrow = 0, x2_firstcol = 0, x2_finalcol = 0; - int16_t x, y, z; - for (j = 1; j < w; ++j) { - x = diff[j]; - y = diff[j - 1]; + // First, process horizontal correlation on just the first row + x_sum += diff[0]; + x2_sum += diff[0] * diff[0]; + x_firstrow += diff[0]; + x2_firstrow += diff[0] * diff[0]; + for (int j = 1; j < width; ++j) { + const int16_t x = diff[j]; + const int16_t y = diff[j - 1]; + x_sum += x; + x_firstrow += x; + x2_sum += x * x; + x2_firstrow += x * x; xy_sum += x * y; - xhor_sum += x; - y_sum += y; - x2hor_sum += x * x; - y2_sum += y * y; } - for (i = 1; i < h; ++i) { - x = diff[i * stride]; - z = diff[(i - 1) * stride]; + + // Process vertical correlation in the first column + x_firstcol += diff[0]; + x2_firstcol += diff[0] * diff[0]; + for (int i = 1; i < height; ++i) { + const int16_t x = diff[i * stride]; + const int16_t z = diff[(i - 1) * stride]; + x_sum += x; + x_firstcol += x; + x2_sum += x * x; + x2_firstcol += x * x; xz_sum += x * z; - xver_sum += x; - z_sum += z; - x2ver_sum += x * x; - z2_sum += z * z; - for (j = 1; j < w; ++j) { - x = diff[i * stride + j]; - y = diff[i * stride + j - 1]; - z = diff[(i - 1) * stride + j]; + } + + // Now process horiz and vert correlation through the rest unit + for (int i = 1; i < height; ++i) { + for (int j = 1; j < width; ++j) { + const int16_t x = diff[i * stride + j]; + const int16_t y = diff[i * stride + j - 1]; + const int16_t z = diff[(i - 1) * stride + j]; + x_sum += x; + x2_sum += x * x; xy_sum += x * y; xz_sum += x * z; - xhor_sum += x; - xver_sum += x; - y_sum += y; - z_sum += z; - x2hor_sum += x * x; - x2ver_sum += x * x; - y2_sum += y * y; - z2_sum += z * z; } } + + for (int j = 0; j < width; ++j) { + x_finalrow += diff[(height - 1) * stride + j]; + x2_finalrow += + diff[(height - 1) * stride + j] * diff[(height - 1) * stride + j]; + } + for (int i = 0; i < height; ++i) { + x_finalcol += diff[i * stride + width - 1]; + x2_finalcol += diff[i * stride + width - 1] * diff[i * stride + width - 1]; + } + + int64_t xhor_sum = x_sum - x_finalcol; + int64_t xver_sum = x_sum - x_finalrow; + int64_t y_sum = x_sum - x_firstcol; + int64_t z_sum = x_sum - x_firstrow; + int64_t x2hor_sum = x2_sum - x2_finalcol; + int64_t x2ver_sum = x2_sum - x2_finalrow; + int64_t y2_sum = x2_sum - x2_firstcol; + int64_t z2_sum = x2_sum - x2_firstrow; + + const float num_hor = (float)(height * (width - 1)); + const float num_ver = (float)((height - 1) * width); + const float xhor_var_n = x2hor_sum - (xhor_sum * xhor_sum) / num_hor; - const float y_var_n = y2_sum - (y_sum * y_sum) / num_hor; - const float xy_var_n = xy_sum - (xhor_sum * y_sum) / num_hor; const float xver_var_n = x2ver_sum - (xver_sum * xver_sum) / num_ver; + + const float y_var_n = y2_sum - (y_sum * y_sum) / num_hor; const float z_var_n = z2_sum - (z_sum * z_sum) / num_ver; + + const float xy_var_n = xy_sum - (xhor_sum * y_sum) / num_hor; const float xz_var_n = xz_sum - (xver_sum * z_sum) / num_ver; - *hcorr = *vcorr = 1; if (xhor_var_n > 0 && y_var_n > 0) { *hcorr = xy_var_n / sqrtf(xhor_var_n * y_var_n); *hcorr = *hcorr < 0 ? 0 : *hcorr; + } else { + *hcorr = 1.0; } if (xver_var_n > 0 && z_var_n > 0) { *vcorr = xz_var_n / sqrtf(xver_var_n * z_var_n); *vcorr = *vcorr < 0 ? 0 : *vcorr; + } else { + *vcorr = 1.0; } } @@ -1609,16 +1720,19 @@ static void get_horver_correlation_full(const int16_t *diff, int stride, int w, static void score_2D_transform_pow8(float *scores_2D, float shift) { float sum = 0.0f; int i; - for (i = 0; i < 16; i++) { - float v, v2, v4; - v = AOMMAX(scores_2D[i] + shift, 0.0f); - v2 = v * v; - v4 = v2 * v2; + const float v = AOMMIN(AOMMAX(scores_2D[i] + shift, 0.0f), 100.0f); + const float v2 = v * v; + const float v4 = v2 * v2; scores_2D[i] = v4 * v4; sum += scores_2D[i]; } - for (i = 0; i < 16; i++) scores_2D[i] /= sum; + for (i = 0; i < 16; i++) { + if (scores_2D[i] < sum * 1e-4) + scores_2D[i] = 0.0f; + else + scores_2D[i] /= sum; + } } // These thresholds were calibrated to provide a certain number of TX types @@ -1714,11 +1828,12 @@ static uint16_t prune_tx_2D(MACROBLOCK *x, BLOCK_SIZE bsize, TX_SIZE tx_size, const int16_t *diff = p->src_diff + 4 * blk_row * diff_stride + 4 * blk_col; get_energy_distribution_finer(diff, diff_stride, bw, bh, hfeatures, vfeatures); - get_horver_correlation_full(diff, diff_stride, bw, bh, - &hfeatures[hfeatures_num - 1], - &vfeatures[vfeatures_num - 1]); + av1_get_horver_correlation_full(diff, diff_stride, bw, bh, + &hfeatures[hfeatures_num - 1], + &vfeatures[vfeatures_num - 1]); av1_nn_predict(hfeatures, nn_config_hor, hscores); av1_nn_predict(vfeatures, nn_config_ver, vscores); + aom_clear_system_state(); float score_2D_average = 0.0f; for (int i = 0; i < 4; i++) { @@ -1790,7 +1905,13 @@ static void prune_tx(const AV1_COMP *cpi, BLOCK_SIZE bsize, MACROBLOCK *x, x->tx_search_prune[tx_set_type] = 0; x->tx_split_prune_flag = 0; const MB_MODE_INFO *mbmi = xd->mi[0]; - if (!is_inter_block(mbmi) || cpi->sf.tx_type_search.prune_mode == NO_PRUNE || + const int is_inter = is_inter_block(mbmi); + if ((is_inter && cpi->oxcf.use_inter_dct_only) || + (!is_inter && cpi->oxcf.use_intra_dct_only)) { + x->tx_search_prune[tx_set_type] = ~(1 << DCT_DCT); + return; + } + if (!is_inter || cpi->sf.tx_type_search.prune_mode == NO_PRUNE || x->use_default_inter_tx_type || xd->lossless[mbmi->segment_id] || x->cb_partition_scan) return; @@ -1829,8 +1950,7 @@ static void model_rd_from_sse(const AV1_COMP *const cpi, (void)num_samples; const MACROBLOCKD *const xd = &x->e_mbd; const struct macroblockd_plane *const pd = &xd->plane[plane]; - const int dequant_shift = - (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) ? xd->bd - 5 : 3; + const int dequant_shift = (is_cur_buf_hbd(xd)) ? xd->bd - 5 : 3; // Fast approximate the modelling function. if (cpi->sf.simple_model_rd_from_var) { @@ -1852,7 +1972,6 @@ static void model_rd_from_sse(const AV1_COMP *const cpi, *dist <<= 4; } -#if CONFIG_COLLECT_INTER_MODE_RD_STATS static int64_t get_sse(const AV1_COMP *cpi, const MACROBLOCK *x) { const AV1_COMMON *cm = &cpi->common; const int num_planes = av1_num_planes(cm); @@ -1875,7 +1994,6 @@ static int64_t get_sse(const AV1_COMP *cpi, const MACROBLOCK *x) { total_sse <<= 4; return total_sse; } -#endif static void model_rd_for_sb(const AV1_COMP *const cpi, BLOCK_SIZE bsize, MACROBLOCK *x, MACROBLOCKD *xd, int plane_from, @@ -1909,7 +2027,7 @@ static void model_rd_for_sb(const AV1_COMP *const cpi, BLOCK_SIZE bsize, if (x->skip_chroma_rd && plane) continue; - if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + if (is_cur_buf_hbd(xd)) { sse = aom_highbd_sse(p->src.buf, p->src.stride, pd->dst.buf, pd->dst.stride, bw, bh); } else { @@ -1938,43 +2056,6 @@ static void model_rd_for_sb(const AV1_COMP *const cpi, BLOCK_SIZE bsize, *out_dist_sum = dist_sum; } -static void check_block_skip(const AV1_COMP *const cpi, BLOCK_SIZE bsize, - MACROBLOCK *x, MACROBLOCKD *xd, int plane_from, - int plane_to, int *skip_txfm_sb) { - *skip_txfm_sb = 1; - for (int plane = plane_from; plane <= plane_to; ++plane) { - struct macroblock_plane *const p = &x->plane[plane]; - struct macroblockd_plane *const pd = &xd->plane[plane]; - const BLOCK_SIZE bs = - get_plane_block_size(bsize, pd->subsampling_x, pd->subsampling_y); - unsigned int sse; - - if (x->skip_chroma_rd && plane) continue; - - // Since fast HBD variance functions scale down sse by 4 bit, we first use - // fast vf implementation to rule out blocks with non-zero scaled sse. Then, - // only if the source is HBD and the scaled sse is 0, accurate sse - // computation is applied to determine if the sse is really 0. This step is - // necessary for HBD lossless coding. - cpi->fn_ptr[bs].vf(p->src.buf, p->src.stride, pd->dst.buf, pd->dst.stride, - &sse); - if (sse) { - *skip_txfm_sb = 0; - return; - } else if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { - uint64_t sse64 = aom_highbd_sse_odd_size( - p->src.buf, p->src.stride, pd->dst.buf, pd->dst.stride, - block_size_wide[bs], block_size_high[bs]); - - if (sse64) { - *skip_txfm_sb = 0; - return; - } - } - } - return; -} - int64_t av1_block_error_c(const tran_low_t *coeff, const tran_low_t *dqcoeff, intptr_t block_size, int64_t *ssz) { int i; @@ -2076,7 +2157,8 @@ static unsigned pixel_dist(const AV1_COMP *const cpi, const MACROBLOCK *x, static INLINE int64_t pixel_diff_dist(const MACROBLOCK *x, int plane, int blk_row, int blk_col, const BLOCK_SIZE plane_bsize, - const BLOCK_SIZE tx_bsize) { + const BLOCK_SIZE tx_bsize, + unsigned int *block_mse_q8) { int visible_rows, visible_cols; const MACROBLOCKD *xd = &x->e_mbd; get_txb_dimensions(xd, plane, plane_bsize, blk_row, blk_col, tx_bsize, NULL, @@ -2099,7 +2181,11 @@ static INLINE int64_t pixel_diff_dist(const MACROBLOCK *x, int plane, } #endif diff += ((blk_row * diff_stride + blk_col) << tx_size_wide_log2[0]); - return aom_sum_squares_2d_i16(diff, diff_stride, visible_cols, visible_rows); + uint64_t sse = + aom_sum_squares_2d_i16(diff, diff_stride, visible_cols, visible_rows); + if (block_mse_q8 != NULL) + *block_mse_q8 = (unsigned int)((256 * sse) / (visible_cols * visible_rows)); + return sse; } int av1_count_colors(const uint8_t *src, int stride, int rows, int cols, @@ -2199,7 +2285,7 @@ static INLINE void dist_block_tx_domain(MACROBLOCK *x, int plane, int block, tran_low_t *const coeff = BLOCK_OFFSET(p->coeff, block); tran_low_t *const dqcoeff = BLOCK_OFFSET(pd->dqcoeff, block); - if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) + if (is_cur_buf_hbd(xd)) *out_dist = av1_highbd_block_error(coeff, dqcoeff, buffer_length, &this_sse, xd->bd); else @@ -2235,7 +2321,7 @@ static INLINE int64_t dist_block_px_domain(const AV1_COMP *cpi, MACROBLOCK *x, uint8_t *recon; DECLARE_ALIGNED(16, uint16_t, recon16[MAX_TX_SQUARE]); - if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + if (is_cur_buf_hbd(xd)) { recon = CONVERT_TO_BYTEPTR(recon16); av1_highbd_convolve_2d_copy_sr(CONVERT_TO_SHORTPTR(dst), dst_stride, CONVERT_TO_SHORTPTR(recon), MAX_TX_SIZE, bsw, @@ -2257,11 +2343,29 @@ static INLINE int64_t dist_block_px_domain(const AV1_COMP *cpi, MACROBLOCK *x, blk_row, blk_col, plane_bsize, tx_bsize); } -static double get_mean(const int16_t *diff, int stride, int w, int h) { +static double get_diff_mean(const uint8_t *src, int src_stride, + const uint8_t *dst, int dst_stride, int w, int h) { double sum = 0.0; for (int j = 0; j < h; ++j) { for (int i = 0; i < w; ++i) { - sum += diff[j * stride + i]; + const int diff = src[j * src_stride + i] - dst[j * dst_stride + i]; + sum += diff; + } + } + assert(w > 0 && h > 0); + return sum / (w * h); +} + +static double get_highbd_diff_mean(const uint8_t *src8, int src_stride, + const uint8_t *dst8, int dst_stride, int w, + int h) { + const uint16_t *src = CONVERT_TO_SHORTPTR(src8); + const uint16_t *dst = CONVERT_TO_SHORTPTR(dst8); + double sum = 0.0; + for (int j = 0; j < h; ++j) { + for (int i = 0; i < w; ++i) { + const int diff = src[j * src_stride + i] - dst[j * dst_stride + i]; + sum += diff; } } assert(w > 0 && h > 0); @@ -2350,6 +2454,17 @@ static void get_2x2_normalized_sses_and_sads( #if CONFIG_COLLECT_RD_STATS #if CONFIG_COLLECT_RD_STATS == 1 +static double get_mean(const int16_t *diff, int stride, int w, int h) { + double sum = 0.0; + for (int j = 0; j < h; ++j) { + for (int i = 0; i < w; ++i) { + sum += diff[j * stride + i]; + } + } + assert(w > 0 && h > 0); + return sum / (w * h); +} + static void PrintTransformUnitStats(const AV1_COMP *const cpi, MACROBLOCK *x, const RD_STATS *const rd_stats, int blk_row, int blk_col, BLOCK_SIZE plane_bsize, @@ -2372,10 +2487,9 @@ static void PrintTransformUnitStats(const AV1_COMP *const cpi, MACROBLOCK *x, const struct macroblockd_plane *const pd = &xd->plane[plane]; const int txw = tx_size_wide[tx_size]; const int txh = tx_size_high[tx_size]; - const int dequant_shift = - (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) ? xd->bd - 5 : 3; + const int dequant_shift = (is_cur_buf_hbd(xd)) ? xd->bd - 5 : 3; const int q_step = pd->dequant_Q3[1] >> dequant_shift; - const double num_samples = txw * txh; + const int num_samples = txw * txh; const double rate_norm = (double)rd_stats->rate / num_samples; const double dist_norm = (double)rd_stats->dist / num_samples; @@ -2428,9 +2542,9 @@ static void PrintTransformUnitStats(const AV1_COMP *const cpi, MACROBLOCK *x, fprintf(fout, " %g %g", model_rate_norm, model_dist_norm); const double mean = get_mean(src_diff, diff_stride, txw, txh); - double hor_corr, vert_corr; - get_horver_correlation(src_diff, diff_stride, txw, txh, &hor_corr, - &vert_corr); + float hor_corr, vert_corr; + av1_get_horver_correlation_full(src_diff, diff_stride, txw, txh, &hor_corr, + &vert_corr); fprintf(fout, " %g %g %g", mean, hor_corr, vert_corr); double hdist[4] = { 0 }, vdist[4] = { 0 }; @@ -2447,15 +2561,25 @@ static void PrintTransformUnitStats(const AV1_COMP *const cpi, MACROBLOCK *x, #endif // CONFIG_COLLECT_RD_STATS == 1 #if CONFIG_COLLECT_RD_STATS >= 2 -static void PrintPredictionUnitStats(const AV1_COMP *const cpi, MACROBLOCK *x, +static void PrintPredictionUnitStats(const AV1_COMP *const cpi, + const TileDataEnc *tile_data, + MACROBLOCK *x, const RD_STATS *const rd_stats, BLOCK_SIZE plane_bsize) { if (rd_stats->invalid_rate) return; if (rd_stats->rate == INT_MAX || rd_stats->dist == INT64_MAX) return; + if (cpi->sf.inter_mode_rd_model_estimation == 1 && + (tile_data == NULL || + !tile_data->inter_mode_rd_models[plane_bsize].ready)) + return; + (void)tile_data; // Generate small sample to restrict output size. static unsigned int seed = 95014; - if (lcg_rand16(&seed) % 256 > 0) return; + + if ((lcg_rand16(&seed) % (1 << (14 - num_pels_log2_lookup[plane_bsize]))) != + 1) + return; const char output_file[] = "pu_stats.txt"; FILE *fout = fopen(output_file, "a"); @@ -2470,8 +2594,7 @@ static void PrintPredictionUnitStats(const AV1_COMP *const cpi, MACROBLOCK *x, get_txb_dimensions(xd, plane, plane_bsize, 0, 0, plane_bsize, NULL, NULL, &bw, &bh); const int num_samples = bw * bh; - const int dequant_shift = - (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) ? xd->bd - 5 : 3; + const int dequant_shift = (is_cur_buf_hbd(xd)) ? xd->bd - 5 : 3; const int q_step = pd->dequant_Q3[1] >> dequant_shift; const double rate_norm = (double)rd_stats->rate / num_samples; @@ -2488,7 +2611,14 @@ static void PrintPredictionUnitStats(const AV1_COMP *const cpi, MACROBLOCK *x, const int16_t *const src_diff = p->src_diff; const int shift = (xd->bd - 8); - int64_t sse = aom_sum_squares_2d_i16(src_diff, diff_stride, bw, bh); + int64_t sse; + if (is_cur_buf_hbd(xd)) { + sse = aom_highbd_sse(p->src.buf, p->src.stride, pd->dst.buf, pd->dst.stride, + bw, bh); + } else { + sse = + aom_sse(p->src.buf, p->src.stride, pd->dst.buf, pd->dst.stride, bw, bh); + } sse = ROUND_POWER_OF_TWO(sse, shift * 2); const double sse_norm = (double)sse / num_samples; @@ -2527,10 +2657,18 @@ static void PrintPredictionUnitStats(const AV1_COMP *const cpi, MACROBLOCK *x, fprintf(fout, " %g %g %g", model_rate_norm, model_dist_norm, model_rdcost_norm); - double mean = get_mean(src_diff, diff_stride, bw, bh); + double mean; + if (is_cur_buf_hbd(xd)) { + mean = get_highbd_diff_mean(p->src.buf, p->src.stride, pd->dst.buf, + pd->dst.stride, bw, bh); + } else { + mean = get_diff_mean(p->src.buf, p->src.stride, pd->dst.buf, pd->dst.stride, + bw, bh); + } mean /= (1 << shift); - double hor_corr, vert_corr; - get_horver_correlation(src_diff, diff_stride, bw, bh, &hor_corr, &vert_corr); + float hor_corr, vert_corr; + av1_get_horver_correlation_full(src_diff, diff_stride, bw, bh, &hor_corr, + &vert_corr); fprintf(fout, " %g %g %g", mean, hor_corr, vert_corr); double hdist[4] = { 0 }, vdist[4] = { 0 }; @@ -2539,6 +2677,21 @@ static void PrintPredictionUnitStats(const AV1_COMP *const cpi, MACROBLOCK *x, fprintf(fout, " %g %g %g %g %g %g %g %g", hdist[0], hdist[1], hdist[2], hdist[3], vdist[0], vdist[1], vdist[2], vdist[3]); + if (cpi->sf.inter_mode_rd_model_estimation == 1) { + assert(tile_data->inter_mode_rd_models[plane_bsize].ready); + const int64_t overall_sse = get_sse(cpi, x); + int est_residue_cost = 0; + int64_t est_dist = 0; + get_est_rate_dist(tile_data, plane_bsize, overall_sse, &est_residue_cost, + &est_dist); + const double est_residue_cost_norm = (double)est_residue_cost / num_samples; + const double est_dist_norm = (double)est_dist / num_samples; + const double est_rdcost_norm = + (double)RDCOST(x->rdmult, est_residue_cost, est_dist) / num_samples; + fprintf(fout, " %g %g %g", est_residue_cost_norm, est_dist_norm, + est_rdcost_norm); + } + fprintf(fout, "\n"); fclose(fout); } @@ -2553,8 +2706,7 @@ static void model_rd_with_dnn(const AV1_COMP *const cpi, const struct macroblockd_plane *const pd = &xd->plane[plane]; const int log_numpels = num_pels_log2_lookup[plane_bsize]; - const int dequant_shift = - (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) ? xd->bd - 5 : 3; + const int dequant_shift = (is_cur_buf_hbd(xd)) ? xd->bd - 5 : 3; const int q_step = AOMMAX(pd->dequant_Q3[1] >> dequant_shift, 1); const struct macroblock_plane *const p = &x->plane[plane]; @@ -2591,7 +2743,12 @@ static void model_rd_with_dnn(const AV1_COMP *const cpi, get_2x2_normalized_sses_and_sads(cpi, plane_bsize, src, src_stride, dst, dst_stride, src_diff, diff_stride, sse_norm_arr, NULL); - double mean = get_mean(src_diff, bw, bw, bh); + double mean; + if (is_cur_buf_hbd(xd)) { + mean = get_highbd_diff_mean(src, src_stride, dst, dst_stride, bw, bh); + } else { + mean = get_diff_mean(src, src_stride, dst, dst_stride, bw, bh); + } if (shift) { for (int k = 0; k < 4; ++k) sse_norm_arr[k] /= (1 << (2 * shift)); mean /= (1 << shift); @@ -2604,8 +2761,9 @@ static void model_rd_with_dnn(const AV1_COMP *const cpi, const double q_sqr = (double)(q_step * q_step); const double q_sqr_by_sse_norm = q_sqr / (sse_norm + 1.0); const double mean_sqr_by_sse_norm = mean * mean / (sse_norm + 1.0); - double hor_corr, vert_corr; - get_horver_correlation(src_diff, diff_stride, bw, bh, &hor_corr, &vert_corr); + float hor_corr, vert_corr; + av1_get_horver_correlation_full(src_diff, diff_stride, bw, bh, &hor_corr, + &vert_corr); float features[NUM_FEATURES_PUSTATS]; features[0] = (float)hor_corr; @@ -2620,10 +2778,10 @@ static void model_rd_with_dnn(const AV1_COMP *const cpi, float rate_f, dist_by_sse_norm_f; av1_nn_predict(features, &av1_pustats_dist_nnconfig, &dist_by_sse_norm_f); av1_nn_predict(features, &av1_pustats_rate_nnconfig, &rate_f); + aom_clear_system_state(); const float dist_f = (float)((double)dist_by_sse_norm_f * (1.0 + sse_norm)); int rate_i = (int)(AOMMAX(0.0, rate_f * num_samples) + 0.5); int64_t dist_i = (int64_t)(AOMMAX(0.0, dist_f * num_samples) + 0.5); - aom_clear_system_state(); // Check if skip is better if (rate_i == 0) { @@ -2669,7 +2827,7 @@ static void model_rd_for_sb_with_dnn( int bw, bh; get_txb_dimensions(xd, plane, plane_bsize, 0, 0, plane_bsize, NULL, NULL, &bw, &bh); - if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + if (is_cur_buf_hbd(xd)) { sse = aom_highbd_sse(p->src.buf, p->src.stride, pd->dst.buf, pd->dst.stride, bw, bh); } else { @@ -2708,8 +2866,7 @@ static void model_rd_with_surffit(const AV1_COMP *const cpi, (void)plane_bsize; const MACROBLOCKD *const xd = &x->e_mbd; const struct macroblockd_plane *const pd = &xd->plane[plane]; - const int dequant_shift = - (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) ? xd->bd - 5 : 3; + const int dequant_shift = (is_cur_buf_hbd(xd)) ? xd->bd - 5 : 3; const int qstep = AOMMAX(pd->dequant_Q3[1] >> dequant_shift, 1); if (sse == 0) { if (rate) *rate = 0; @@ -2723,7 +2880,8 @@ static void model_rd_with_surffit(const AV1_COMP *const cpi, const double yl = log(sse_norm / qstepsqr) / log(2.0); double rate_f, dist_by_sse_norm_f; - av1_model_rd_surffit(xm, yl, &rate_f, &dist_by_sse_norm_f); + av1_model_rd_surffit(plane_bsize, sse_norm, xm, yl, &rate_f, + &dist_by_sse_norm_f); const double dist_f = dist_by_sse_norm_f * sse_norm; int rate_i = (int)(AOMMAX(0.0, rate_f * num_samples) + 0.5); @@ -2773,7 +2931,7 @@ static void model_rd_for_sb_with_surffit( const int shift = (xd->bd - 8); get_txb_dimensions(xd, plane, plane_bsize, 0, 0, plane_bsize, NULL, NULL, &bw, &bh); - if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + if (is_cur_buf_hbd(xd)) { sse = aom_highbd_sse(p->src.buf, p->src.stride, pd->dst.buf, pd->dst.stride, bw, bh); } else { @@ -2813,8 +2971,7 @@ static void model_rd_with_curvfit(const AV1_COMP *const cpi, (void)plane_bsize; const MACROBLOCKD *const xd = &x->e_mbd; const struct macroblockd_plane *const pd = &xd->plane[plane]; - const int dequant_shift = - (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) ? xd->bd - 5 : 3; + const int dequant_shift = (is_cur_buf_hbd(xd)) ? xd->bd - 5 : 3; const int qstep = AOMMAX(pd->dequant_Q3[1] >> dequant_shift, 1); if (sse == 0) { @@ -2828,7 +2985,8 @@ static void model_rd_with_curvfit(const AV1_COMP *const cpi, const double xqr = log(sse_norm / qstepsqr) / log(2.0); double rate_f, dist_by_sse_norm_f; - av1_model_rd_curvfit(xqr, &rate_f, &dist_by_sse_norm_f); + av1_model_rd_curvfit(plane_bsize, sse_norm, xqr, &rate_f, + &dist_by_sse_norm_f); const double dist_f = dist_by_sse_norm_f * sse_norm; int rate_i = (int)(AOMMAX(0.0, rate_f * num_samples) + 0.5); @@ -2879,7 +3037,7 @@ static void model_rd_for_sb_with_curvfit( get_txb_dimensions(xd, plane, plane_bsize, 0, 0, plane_bsize, NULL, NULL, &bw, &bh); - if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + if (is_cur_buf_hbd(xd)) { sse = aom_highbd_sse(p->src.buf, p->src.stride, pd->dst.buf, pd->dst.stride, bw, bh); } else { @@ -2908,71 +3066,6 @@ static void model_rd_for_sb_with_curvfit( *out_dist_sum = dist_sum; } -static void model_rd_for_sb_with_fullrdy( - const AV1_COMP *const cpi, BLOCK_SIZE bsize, MACROBLOCK *x, MACROBLOCKD *xd, - int plane_from, int plane_to, int mi_row, int mi_col, int *out_rate_sum, - int64_t *out_dist_sum, int *skip_txfm_sb, int64_t *skip_sse_sb, - int *plane_rate, int64_t *plane_sse, int64_t *plane_dist) { - const int ref = xd->mi[0]->ref_frame[0]; - - int64_t rate_sum = 0; - int64_t dist_sum = 0; - int64_t total_sse = 0; - - for (int plane = plane_from; plane <= plane_to; ++plane) { - struct macroblock_plane *const p = &x->plane[plane]; - struct macroblockd_plane *const pd = &xd->plane[plane]; - const BLOCK_SIZE plane_bsize = - get_plane_block_size(bsize, pd->subsampling_x, pd->subsampling_y); - const int bw = block_size_wide[plane_bsize]; - const int bh = block_size_high[plane_bsize]; - int64_t sse; - int rate; - int64_t dist; - - if (x->skip_chroma_rd && plane) continue; - - if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { - sse = aom_highbd_sse(p->src.buf, p->src.stride, pd->dst.buf, - pd->dst.stride, bw, bh); - } else { - sse = aom_sse(p->src.buf, p->src.stride, pd->dst.buf, pd->dst.stride, bw, - bh); - } - sse = ROUND_POWER_OF_TWO(sse, (xd->bd - 8) * 2); - - RD_STATS rd_stats; - if (plane == 0) { - select_tx_type_yrd(cpi, x, &rd_stats, bsize, mi_row, mi_col, INT64_MAX); - if (rd_stats.invalid_rate) { - rate = 0; - dist = sse << 4; - } else { - rate = rd_stats.rate; - dist = rd_stats.dist; - } - } else { - model_rd_with_curvfit(cpi, x, plane_bsize, plane, sse, bw * bh, &rate, - &dist); - } - - if (plane == 0) x->pred_sse[ref] = (unsigned int)AOMMIN(sse, UINT_MAX); - - total_sse += sse; - rate_sum += rate; - dist_sum += dist; - - if (plane_rate) plane_rate[plane] = rate; - if (plane_sse) plane_sse[plane] = sse; - if (plane_dist) plane_dist[plane] = dist; - } - - if (skip_txfm_sb) *skip_txfm_sb = total_sse == 0; - if (skip_sse_sb) *skip_sse_sb = total_sse << 4; - *out_rate_sum = (int)rate_sum; - *out_dist_sum = dist_sum; -} - static int64_t search_txk_type(const AV1_COMP *cpi, MACROBLOCK *x, int plane, int block, int blk_row, int blk_col, BLOCK_SIZE plane_bsize, TX_SIZE tx_size, @@ -2997,6 +3090,7 @@ static int64_t search_txk_type(const AV1_COMP *cpi, MACROBLOCK *x, int plane, tran_low_t *best_dqcoeff = this_dqcoeff; const int txk_type_idx = av1_get_txk_type_index(plane_bsize, blk_row, blk_col); + int perform_block_coeff_opt; av1_invalid_rd_stats(best_rd_stats); TXB_RD_INFO *intra_txb_rd_info = NULL; @@ -3047,7 +3141,8 @@ static int64_t search_txk_type(const AV1_COMP *cpi, MACROBLOCK *x, int plane, TX_TYPE txk_end = TX_TYPES - 1; if ((!is_inter && x->use_default_intra_tx_type) || (is_inter && x->use_default_inter_tx_type)) { - txk_start = txk_end = get_default_tx_type(0, xd, tx_size); + txk_start = txk_end = + get_default_tx_type(0, xd, tx_size, cpi->is_screen_content_type); } else if (x->rd_model == LOW_TXFM_RD || x->cb_partition_scan) { if (plane == 0) txk_end = DCT_DCT; } @@ -3065,7 +3160,9 @@ static int64_t search_txk_type(const AV1_COMP *cpi, MACROBLOCK *x, int plane, } const uint16_t ext_tx_used_flag = av1_ext_tx_used_flag[tx_set_type]; if (xd->lossless[mbmi->segment_id] || txsize_sqr_up_map[tx_size] > TX_32X32 || - ext_tx_used_flag == 0x0001) { + ext_tx_used_flag == 0x0001 || + (is_inter && cpi->oxcf.use_inter_dct_only) || + (!is_inter && cpi->oxcf.use_intra_dct_only)) { txk_start = txk_end = DCT_DCT; } uint16_t allowed_tx_mask = 0; // 1: allow; 0: skip. @@ -3097,8 +3194,22 @@ static int64_t search_txk_type(const AV1_COMP *cpi, MACROBLOCK *x, int plane, allowed_tx_mask = (1 << txk_start); } + const BLOCK_SIZE tx_bsize = txsize_to_bsize[tx_size]; + int64_t block_sse = 0; + unsigned int block_mse_q8 = UINT_MAX; + block_sse = pixel_diff_dist(x, plane, blk_row, blk_col, plane_bsize, tx_bsize, + &block_mse_q8); + assert(block_mse_q8 != UINT_MAX); + if (is_cur_buf_hbd(xd)) { + block_sse = ROUND_POWER_OF_TWO(block_sse, (xd->bd - 8) * 2); + block_mse_q8 = ROUND_POWER_OF_TWO(block_mse_q8, (xd->bd - 8) * 2); + } + block_sse *= 16; + // Tranform domain distortion is accurate for higher residuals. + // TODO(any): Experiment with variance and mean based thresholds int use_transform_domain_distortion = (cpi->sf.use_transform_domain_distortion > 0) && + (block_mse_q8 >= cpi->tx_domain_dist_threshold) && // Any 64-pt transforms only preserves half the coefficients. // Therefore transform domain distortion is not valid for these // transform sizes. @@ -3116,20 +3227,19 @@ static int64_t search_txk_type(const AV1_COMP *cpi, MACROBLOCK *x, int plane, const uint16_t *eobs_ptr = x->plane[plane].eobs; - const BLOCK_SIZE tx_bsize = txsize_to_bsize[tx_size]; - int64_t block_sse = - pixel_diff_dist(x, plane, blk_row, blk_col, plane_bsize, tx_bsize); - if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) - block_sse = ROUND_POWER_OF_TWO(block_sse, (xd->bd - 8) * 2); - block_sse *= 16; + // Used mse based threshold logic to take decision of R-D of optimization of + // coeffs. For snaller residuals, coeff optimization would be helpful. For + // larger residuals, R-D optimization may not be effective. + // TODO(any): Experiment with variance and mean based thresholds + perform_block_coeff_opt = (block_mse_q8 <= cpi->coeff_opt_dist_threshold); for (TX_TYPE tx_type = txk_start; tx_type <= txk_end; ++tx_type) { if (!(allowed_tx_mask & (1 << tx_type))) continue; if (plane == 0) mbmi->txk_type[txk_type_idx] = tx_type; RD_STATS this_rd_stats; av1_invalid_rd_stats(&this_rd_stats); - - if (!cpi->optimize_seg_arr[mbmi->segment_id]) { + if ((cpi->optimize_seg_arr[mbmi->segment_id] != FULL_TRELLIS_OPT) || + (!perform_block_coeff_opt)) { av1_xform_quant( cm, x, plane, block, blk_row, blk_col, plane_bsize, tx_size, tx_type, USE_B_QUANT_NO_TRELLIS ? AV1_XFORM_QUANT_B : AV1_XFORM_QUANT_FP); @@ -3148,13 +3258,6 @@ static int64_t search_txk_type(const AV1_COMP *cpi, MACROBLOCK *x, int plane, const int64_t dist_cost_estimate = RDCOST(x->rdmult, 0, AOMMIN(this_rd_stats.dist, this_rd_stats.sse)); if (dist_cost_estimate - (dist_cost_estimate >> 3) > best_rd_) continue; - - rate_cost = av1_cost_coeffs(cm, x, plane, block, tx_size, tx_type, - txb_ctx, use_fast_coef_costing); - const int64_t rd_estimate = - AOMMIN(RDCOST(x->rdmult, rate_cost, this_rd_stats.dist), - RDCOST(x->rdmult, 0, this_rd_stats.sse)); - if (rd_estimate - (rd_estimate >> 3) > best_rd_) continue; } av1_optimize_b(cpi, x, plane, block, tx_size, tx_type, txb_ctx, 1, &rate_cost); @@ -3197,6 +3300,41 @@ static int64_t search_txk_type(const AV1_COMP *cpi, MACROBLOCK *x, int plane, } #endif // CONFIG_COLLECT_RD_STATS == 1 +#if COLLECT_TX_SIZE_DATA + // Generate small sample to restrict output size. + static unsigned int seed = 21743; + if (lcg_rand16(&seed) % 200 == 0) { + FILE *fp = NULL; + + if (within_border) { + fp = fopen(av1_tx_size_data_output_file, "a"); + } + + if (fp) { + // Transform info and RD + const int txb_w = tx_size_wide[tx_size]; + const int txb_h = tx_size_high[tx_size]; + + // Residue signal. + const int diff_stride = block_size_wide[plane_bsize]; + struct macroblock_plane *const p = &x->plane[plane]; + const int16_t *src_diff = + &p->src_diff[(blk_row * diff_stride + blk_col) * 4]; + + for (int r = 0; r < txb_h; ++r) { + for (int c = 0; c < txb_w; ++c) { + fprintf(fp, "%d,", src_diff[c]); + } + src_diff += diff_stride; + } + + fprintf(fp, "%d,%d,%d,%" PRId64, txb_w, txb_h, tx_type, rd); + fprintf(fp, "\n"); + fclose(fp); + } + } +#endif // COLLECT_TX_SIZE_DATA + if (cpi->sf.adaptive_txb_search_level) { if ((best_rd - (best_rd >> cpi->sf.adaptive_txb_search_level)) > ref_best_rd) { @@ -3247,7 +3385,7 @@ RECON_INTRA: // if the last search tx_type is the best tx_type, we don't need to // do this again if (best_tx_type != last_tx_type) { - if (!cpi->optimize_seg_arr[mbmi->segment_id]) { + if (cpi->optimize_seg_arr[mbmi->segment_id] != FULL_TRELLIS_OPT) { av1_xform_quant( cm, x, plane, block, blk_row, blk_col, plane_bsize, tx_size, best_tx_type, @@ -3283,12 +3421,11 @@ static void block_rd_txfm(int plane, int block, int blk_row, int blk_col, struct rdcost_block_args *args = arg; MACROBLOCK *const x = args->x; MACROBLOCKD *const xd = &x->e_mbd; - const MB_MODE_INFO *const mbmi = xd->mi[0]; + const int is_inter = is_inter_block(xd->mi[0]); const AV1_COMP *cpi = args->cpi; ENTROPY_CONTEXT *a = args->t_above + blk_col; ENTROPY_CONTEXT *l = args->t_left + blk_row; const AV1_COMMON *cm = &cpi->common; - int64_t rd1, rd2, rd; RD_STATS this_rd_stats; av1_init_rd_stats(&this_rd_stats); @@ -3298,7 +3435,7 @@ static void block_rd_txfm(int plane, int block, int blk_row, int blk_col, return; } - if (!is_inter_block(mbmi)) { + if (!is_inter) { av1_predict_intra_block_facade(cm, xd, plane, blk_col, blk_row, tx_size); av1_subtract_txb(x, plane, plane_bsize, blk_col, blk_row, tx_size); } @@ -3309,7 +3446,7 @@ static void block_rd_txfm(int plane, int block, int blk_row, int blk_col, args->best_rd - args->this_rd, &this_rd_stats); if (plane == AOM_PLANE_Y && xd->cfl.store_y) { - assert(!is_inter_block(mbmi) || plane_bsize < BLOCK_8X8); + assert(!is_inter || plane_bsize < BLOCK_8X8); cfl_store_tx(xd, blk_row, blk_col, tx_size, plane_bsize); } @@ -3328,28 +3465,33 @@ static void block_rd_txfm(int plane, int block, int blk_row, int blk_col, else set_blk_skip(x, plane, blk_idx, 0); - rd1 = RDCOST(x->rdmult, this_rd_stats.rate, this_rd_stats.dist); - rd2 = RDCOST(x->rdmult, 0, this_rd_stats.sse); + const int64_t rd1 = RDCOST(x->rdmult, this_rd_stats.rate, this_rd_stats.dist); + const int64_t rd2 = RDCOST(x->rdmult, 0, this_rd_stats.sse); // TODO(jingning): temporarily enabled only for luma component - rd = AOMMIN(rd1, rd2); + const int64_t rd = AOMMIN(rd1, rd2); this_rd_stats.skip &= !x->plane[plane].eobs[block]; +#if CONFIG_ONE_PASS_SVM + if (plane == AOM_PLANE_Y && plane_bsize >= BLOCK_8X8) { + const int eob = x->plane[plane].eobs[block]; + av1_add_reg_stat(&this_rd_stats, eob, rd, this_rd_stats.sse, blk_row, + blk_col, plane_bsize, txsize_to_bsize[tx_size]); + } +#endif + av1_merge_rd_stats(&args->rd_stats, &this_rd_stats); args->this_rd += rd; - if (args->this_rd > args->best_rd) { - args->exit_early = 1; - return; - } + if (args->this_rd > args->best_rd) args->exit_early = 1; } static void txfm_rd_in_plane(MACROBLOCK *x, const AV1_COMP *cpi, - RD_STATS *rd_stats, int64_t ref_best_rd, int plane, - BLOCK_SIZE bsize, TX_SIZE tx_size, - int use_fast_coef_casting, + RD_STATS *rd_stats, int64_t ref_best_rd, + int64_t this_rd, int plane, BLOCK_SIZE bsize, + TX_SIZE tx_size, int use_fast_coef_casting, FAST_TX_SEARCH_MODE ftxs_mode) { MACROBLOCKD *const xd = &x->e_mbd; const struct macroblockd_plane *const pd = &xd->plane[plane]; @@ -3360,12 +3502,22 @@ static void txfm_rd_in_plane(MACROBLOCK *x, const AV1_COMP *cpi, args.best_rd = ref_best_rd; args.use_fast_coef_costing = use_fast_coef_casting; args.ftxs_mode = ftxs_mode; + args.this_rd = this_rd; av1_init_rd_stats(&args.rd_stats); + if (!cpi->oxcf.enable_tx64 && txsize_sqr_up_map[tx_size] == TX_64X64) { + av1_invalid_rd_stats(rd_stats); + return; + } + if (plane == 0) xd->mi[0]->tx_size = tx_size; av1_get_entropy_contexts(bsize, pd, args.t_above, args.t_left); + if (args.this_rd > args.best_rd) { + args.exit_early = 1; + } + av1_foreach_transformed_block_in_plane(xd, bsize, plane, block_rd_txfm, &args); @@ -3382,18 +3534,14 @@ static void txfm_rd_in_plane(MACROBLOCK *x, const AV1_COMP *cpi, static int tx_size_cost(const AV1_COMMON *const cm, const MACROBLOCK *const x, BLOCK_SIZE bsize, TX_SIZE tx_size) { - const MACROBLOCKD *const xd = &x->e_mbd; - const MB_MODE_INFO *const mbmi = xd->mi[0]; + assert(bsize == x->e_mbd.mi[0]->sb_type); + if (cm->tx_mode != TX_MODE_SELECT || !block_signals_txsize(bsize)) return 0; - if (cm->tx_mode == TX_MODE_SELECT && block_signals_txsize(mbmi->sb_type)) { - const int32_t tx_size_cat = bsize_to_tx_size_cat(bsize); - const int depth = tx_size_to_depth(tx_size, bsize); - const int tx_size_ctx = get_tx_size_context(xd); - int r_tx_size = x->tx_size_cost[tx_size_cat][tx_size_ctx][depth]; - return r_tx_size; - } else { - return 0; - } + const int32_t tx_size_cat = bsize_to_tx_size_cat(bsize); + const int depth = tx_size_to_depth(tx_size, bsize); + const MACROBLOCKD *const xd = &x->e_mbd; + const int tx_size_ctx = get_tx_size_context(xd); + return x->tx_size_cost[tx_size_cat][tx_size_ctx][depth]; } static int64_t txfm_yrd(const AV1_COMP *const cpi, MACROBLOCK *x, @@ -3418,43 +3566,79 @@ static int64_t txfm_yrd(const AV1_COMP *const cpi, MACROBLOCK *x, s0 = x->skip_cost[skip_ctx][0]; s1 = x->skip_cost[skip_ctx][1]; + int64_t skip_rd; + int64_t this_rd; + + if (is_inter) { + skip_rd = RDCOST(x->rdmult, s1, 0); + this_rd = RDCOST(x->rdmult, s0 + r_tx_size * tx_select, 0); + } else { + skip_rd = RDCOST(x->rdmult, s1 + r_tx_size * tx_select, 0); + this_rd = RDCOST(x->rdmult, s0 + r_tx_size * tx_select, 0); + } + mbmi->tx_size = tx_size; - txfm_rd_in_plane(x, cpi, rd_stats, ref_best_rd, AOM_PLANE_Y, bs, tx_size, - cpi->sf.use_fast_coef_costing, ftxs_mode); + txfm_rd_in_plane(x, cpi, rd_stats, ref_best_rd, AOMMIN(this_rd, skip_rd), + AOM_PLANE_Y, bs, tx_size, cpi->sf.use_fast_coef_costing, + ftxs_mode); if (rd_stats->rate == INT_MAX) return INT64_MAX; + // rdstats->rate should include all the rate except skip/non-skip cost as the + // same is accounted in the caller functions after rd evaluation of all + // planes. However the decisions should be done after considering the + // skip/non-skip header cost if (rd_stats->skip) { if (is_inter) { rd = RDCOST(x->rdmult, s1, rd_stats->sse); +#if CONFIG_ONE_PASS_SVM + // TODO(chiyotsai@google.com): Investigate if these updates are really + // needed. + av1_reg_stat_skipmode_update(rd_stats, x->rdmult); +#endif } else { rd = RDCOST(x->rdmult, s1 + r_tx_size * tx_select, rd_stats->sse); +#if CONFIG_ONE_PASS_SVM + av1_reg_stat_skipmode_update(rd_stats, x->rdmult); +#endif + rd_stats->rate += r_tx_size * tx_select; } } else { rd = RDCOST(x->rdmult, rd_stats->rate + s0 + r_tx_size * tx_select, rd_stats->dist); + rd_stats->rate += r_tx_size * tx_select; + } + if (is_inter && !xd->lossless[xd->mi[0]->segment_id]) { + int64_t temp_skip_rd = RDCOST(x->rdmult, s1, rd_stats->sse); + if (temp_skip_rd <= rd) { + rd = temp_skip_rd; + rd_stats->rate = 0; + rd_stats->dist = rd_stats->sse; + rd_stats->skip = 1; + } } - - if (tx_select) rd_stats->rate += r_tx_size; - - if (is_inter && !xd->lossless[xd->mi[0]->segment_id] && !(rd_stats->skip)) - rd = AOMMIN(rd, RDCOST(x->rdmult, s1, rd_stats->sse)); return rd; } static int64_t estimate_yrd_for_sb(const AV1_COMP *const cpi, BLOCK_SIZE bs, - MACROBLOCK *x, int *r, int64_t *d, int *s, - int64_t *sse, int64_t ref_best_rd) { - RD_STATS rd_stats; + MACROBLOCK *x, int64_t ref_best_rd, + RD_STATS *rd_stats) { av1_subtract_plane(x, bs, 0); x->rd_model = LOW_TXFM_RD; - int64_t rd = txfm_yrd(cpi, x, &rd_stats, ref_best_rd, bs, - max_txsize_rect_lookup[bs], FTXS_NONE); + const int64_t rd = txfm_yrd(cpi, x, rd_stats, ref_best_rd, bs, + max_txsize_rect_lookup[bs], FTXS_NONE); x->rd_model = FULL_TXFM_RD; - *r = rd_stats.rate; - *d = rd_stats.dist; - *s = rd_stats.skip; - *sse = rd_stats.sse; + if (rd != INT64_MAX) { + MACROBLOCKD *const xd = &x->e_mbd; + const int skip_ctx = av1_get_skip_context(xd); + if (rd_stats->skip) { + const int s1 = x->skip_cost[skip_ctx][1]; + rd_stats->rate = s1; + } else { + const int s0 = x->skip_cost[skip_ctx][0]; + rd_stats->rate += s0; + } + } return rd; } @@ -3469,8 +3653,18 @@ static void choose_largest_tx_size(const AV1_COMP *const cpi, MACROBLOCK *x, const TxSetType tx_set_type = av1_get_ext_tx_set_type(mbmi->tx_size, is_inter, cm->reduced_tx_set_used); prune_tx(cpi, bs, x, xd, tx_set_type); - txfm_rd_in_plane(x, cpi, rd_stats, ref_best_rd, AOM_PLANE_Y, bs, - mbmi->tx_size, cpi->sf.use_fast_coef_costing, FTXS_NONE); + const int skip_ctx = av1_get_skip_context(xd); + int s0, s1; + + s0 = x->skip_cost[skip_ctx][0]; + s1 = x->skip_cost[skip_ctx][1]; + + int64_t skip_rd = RDCOST(x->rdmult, s1, 0); + int64_t this_rd = RDCOST(x->rdmult, s0, 0); + + txfm_rd_in_plane(x, cpi, rd_stats, ref_best_rd, AOMMIN(this_rd, skip_rd), + AOM_PLANE_Y, bs, mbmi->tx_size, + cpi->sf.use_fast_coef_costing, FTXS_NONE); // Reset the pruning flags. av1_zero(x->tx_search_prune); x->tx_split_prune_flag = 0; @@ -3483,7 +3677,8 @@ static void choose_smallest_tx_size(const AV1_COMP *const cpi, MACROBLOCK *x, MB_MODE_INFO *const mbmi = xd->mi[0]; mbmi->tx_size = TX_4X4; - txfm_rd_in_plane(x, cpi, rd_stats, ref_best_rd, 0, bs, mbmi->tx_size, + // TODO(any) : Pass this_rd based on skip/non-skip cost + txfm_rd_in_plane(x, cpi, rd_stats, ref_best_rd, 0, 0, bs, mbmi->tx_size, cpi->sf.use_fast_coef_costing, FTXS_NONE); } @@ -3514,22 +3709,15 @@ static int get_search_init_depth(int mi_width, int mi_height, int is_inter, static void choose_tx_size_type_from_rd(const AV1_COMP *const cpi, MACROBLOCK *x, RD_STATS *rd_stats, int64_t ref_best_rd, BLOCK_SIZE bs) { + av1_invalid_rd_stats(rd_stats); + const AV1_COMMON *const cm = &cpi->common; MACROBLOCKD *const xd = &x->e_mbd; MB_MODE_INFO *const mbmi = xd->mi[0]; - int64_t rd = INT64_MAX; - int n; + const TX_SIZE max_rect_tx_size = max_txsize_rect_lookup[bs]; + const int tx_select = cm->tx_mode == TX_MODE_SELECT; int start_tx; int depth; - int64_t best_rd = INT64_MAX; - const TX_SIZE max_rect_tx_size = max_txsize_rect_lookup[bs]; - TX_SIZE best_tx_size = max_rect_tx_size; - TX_TYPE best_txk_type[TXK_TYPE_BUF_LEN]; - uint8_t best_blk_skip[MAX_MIB_SIZE * MAX_MIB_SIZE]; - const int n4 = bsize_to_num_blk(bs); - const int tx_select = cm->tx_mode == TX_MODE_SELECT; - - av1_invalid_rd_stats(rd_stats); if (tx_select) { start_tx = max_rect_tx_size; @@ -3543,16 +3731,24 @@ static void choose_tx_size_type_from_rd(const AV1_COMP *const cpi, prune_tx(cpi, bs, x, xd, EXT_TX_SET_ALL16); - for (n = start_tx; depth <= MAX_TX_DEPTH; depth++, n = sub_tx_size_map[n]) { + TX_TYPE best_txk_type[TXK_TYPE_BUF_LEN]; + uint8_t best_blk_skip[MAX_MIB_SIZE * MAX_MIB_SIZE]; + TX_SIZE best_tx_size = max_rect_tx_size; + int64_t best_rd = INT64_MAX; + const int n4 = bsize_to_num_blk(bs); + x->rd_model = FULL_TXFM_RD; + for (int n = start_tx; depth <= MAX_TX_DEPTH; + depth++, n = sub_tx_size_map[n]) { #if CONFIG_DIST_8X8 if (x->using_dist_8x8) { if (tx_size_wide[n] < 8 || tx_size_high[n] < 8) continue; } #endif + if (!cpi->oxcf.enable_tx64 && txsize_sqr_up_map[n] == TX_64X64) continue; + RD_STATS this_rd_stats; - if (mbmi->ref_mv_idx > 0) x->rd_model = LOW_TXFM_RD; - rd = txfm_yrd(cpi, x, &this_rd_stats, ref_best_rd, bs, n, FTXS_NONE); - x->rd_model = FULL_TXFM_RD; + const int64_t rd = + txfm_yrd(cpi, x, &this_rd_stats, ref_best_rd, bs, n, FTXS_NONE); if (rd < best_rd) { memcpy(best_txk_type, mbmi->txk_type, @@ -4196,21 +4392,13 @@ static const uint8_t mode_to_angle_bin[INTRA_MODES] = { }; /* clang-format on */ -static void angle_estimation(const uint8_t *src, int src_stride, int rows, - int cols, BLOCK_SIZE bsize, - uint8_t *directional_mode_skip_mask) { - memset(directional_mode_skip_mask, 0, - INTRA_MODES * sizeof(*directional_mode_skip_mask)); - // Check if angle_delta is used - if (!av1_use_angle_delta(bsize)) return; - uint64_t hist[DIRECTIONAL_MODES]; - memset(hist, 0, DIRECTIONAL_MODES * sizeof(hist[0])); +static void get_gradient_hist(const uint8_t *src, int src_stride, int rows, + int cols, uint64_t *hist) { src += src_stride; - int r, c, dx, dy; - for (r = 1; r < rows; ++r) { - for (c = 1; c < cols; ++c) { - dx = src[c] - src[c - 1]; - dy = src[c] - src[c - src_stride]; + for (int r = 1; r < rows; ++r) { + for (int c = 1; c < cols; ++c) { + int dx = src[c] - src[c - 1]; + int dy = src[c] - src[c - src_stride]; int index; const int temp = dx * dx + dy * dy; if (dy == 0) { @@ -4227,45 +4415,16 @@ static void angle_estimation(const uint8_t *src, int src_stride, int rows, } src += src_stride; } - - int i; - uint64_t hist_sum = 0; - for (i = 0; i < DIRECTIONAL_MODES; ++i) hist_sum += hist[i]; - for (i = 0; i < INTRA_MODES; ++i) { - if (av1_is_directional_mode(i)) { - const uint8_t angle_bin = mode_to_angle_bin[i]; - uint64_t score = 2 * hist[angle_bin]; - int weight = 2; - if (angle_bin > 0) { - score += hist[angle_bin - 1]; - ++weight; - } - if (angle_bin < DIRECTIONAL_MODES - 1) { - score += hist[angle_bin + 1]; - ++weight; - } - if (score * ANGLE_SKIP_THRESH < hist_sum * weight) - directional_mode_skip_mask[i] = 1; - } - } } -static void highbd_angle_estimation(const uint8_t *src8, int src_stride, - int rows, int cols, BLOCK_SIZE bsize, - uint8_t *directional_mode_skip_mask) { - memset(directional_mode_skip_mask, 0, - INTRA_MODES * sizeof(*directional_mode_skip_mask)); - // Check if angle_delta is used - if (!av1_use_angle_delta(bsize)) return; +static void get_highbd_gradient_hist(const uint8_t *src8, int src_stride, + int rows, int cols, uint64_t *hist) { uint16_t *src = CONVERT_TO_SHORTPTR(src8); - uint64_t hist[DIRECTIONAL_MODES]; - memset(hist, 0, DIRECTIONAL_MODES * sizeof(hist[0])); src += src_stride; - int r, c, dx, dy; - for (r = 1; r < rows; ++r) { - for (c = 1; c < cols; ++c) { - dx = src[c] - src[c - 1]; - dy = src[c] - src[c - src_stride]; + for (int r = 1; r < rows; ++r) { + for (int c = 1; c < cols; ++c) { + int dx = src[c] - src[c - 1]; + int dy = src[c] - src[c - src_stride]; int index; const int temp = dx * dx + dy * dy; if (dy == 0) { @@ -4282,6 +4441,19 @@ static void highbd_angle_estimation(const uint8_t *src8, int src_stride, } src += src_stride; } +} + +static void angle_estimation(const uint8_t *src, int src_stride, int rows, + int cols, BLOCK_SIZE bsize, int is_hbd, + uint8_t *directional_mode_skip_mask) { + // Check if angle_delta is used + if (!av1_use_angle_delta(bsize)) return; + + uint64_t hist[DIRECTIONAL_MODES] = { 0 }; + if (is_hbd) + get_highbd_gradient_hist(src, src_stride, rows, cols, hist); + else + get_gradient_hist(src, src_stride, rows, cols, hist); int i; uint64_t hist_sum = 0; @@ -4299,8 +4471,8 @@ static void highbd_angle_estimation(const uint8_t *src8, int src_stride, score += hist[angle_bin + 1]; ++weight; } - if (score * ANGLE_SKIP_THRESH < hist_sum * weight) - directional_mode_skip_mask[i] = 1; + const int thresh = 10; + if (score * thresh < hist_sum * weight) directional_mode_skip_mask[i] = 1; } } } @@ -4353,13 +4525,12 @@ static int64_t rd_pick_intra_sby_mode(const AV1_COMP *const cpi, MACROBLOCK *x, const int rows = block_size_high[bsize]; const int cols = block_size_wide[bsize]; int is_directional_mode; - uint8_t directional_mode_skip_mask[INTRA_MODES]; - const int src_stride = x->plane[0].src.stride; - const uint8_t *src = x->plane[0].src.buf; + uint8_t directional_mode_skip_mask[INTRA_MODES] = { 0 }; int beat_best_rd = 0; const int *bmode_costs; PALETTE_MODE_INFO *const pmi = &mbmi->palette_mode_info; const int try_palette = + cpi->oxcf.enable_palette && av1_allow_palette(cpi->common.allow_screen_content_tools, mbmi->sb_type); uint8_t *best_palette_color_map = try_palette ? x->palette_buffer->best_palette_color_map : NULL; @@ -4372,12 +4543,12 @@ static int64_t rd_pick_intra_sby_mode(const AV1_COMP *const cpi, MACROBLOCK *x, bmode_costs = x->y_mode_costs[above_ctx][left_ctx]; mbmi->angle_delta[PLANE_TYPE_Y] = 0; - if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) - highbd_angle_estimation(src, src_stride, rows, cols, bsize, - directional_mode_skip_mask); - else - angle_estimation(src, src_stride, rows, cols, bsize, + if (cpi->sf.intra_angle_estimation) { + const int src_stride = x->plane[0].src.stride; + const uint8_t *src = x->plane[0].src.buf; + angle_estimation(src, src_stride, rows, cols, bsize, is_cur_buf_hbd(xd), directional_mode_skip_mask); + } mbmi->filter_intra_mode_info.use_filter_intra = 0; pmi->palette_size[0] = 0; @@ -4393,6 +4564,11 @@ static int64_t rd_pick_intra_sby_mode(const AV1_COMP *const cpi, MACROBLOCK *x, int this_rate, this_rate_tokenonly, s; int64_t this_distortion, this_rd, this_model_rd; mbmi->mode = intra_rd_search_mode_order[mode_idx]; + if (!cpi->oxcf.enable_smooth_intra && + (mbmi->mode == SMOOTH_PRED || mbmi->mode == SMOOTH_H_PRED || + mbmi->mode == SMOOTH_V_PRED)) + continue; + if (!cpi->oxcf.enable_paeth_intra && mbmi->mode == PAETH_PRED) continue; mbmi->angle_delta[PLANE_TYPE_Y] = 0; this_model_rd = intra_model_yrd(cpi, x, bsize, bmode_costs[mbmi->mode], mi_row, mi_col); @@ -4402,7 +4578,8 @@ static int64_t rd_pick_intra_sby_mode(const AV1_COMP *const cpi, MACROBLOCK *x, if (this_model_rd < best_model_rd) best_model_rd = this_model_rd; is_directional_mode = av1_is_directional_mode(mbmi->mode); if (is_directional_mode && directional_mode_skip_mask[mbmi->mode]) continue; - if (is_directional_mode && av1_use_angle_delta(bsize)) { + if (is_directional_mode && av1_use_angle_delta(bsize) && + cpi->oxcf.enable_angle_delta) { this_rd_stats.rate = INT_MAX; rd_pick_intra_angle_sby(cpi, x, mi_row, mi_col, &this_rate, &this_rd_stats, bsize, bmode_costs[mbmi->mode], @@ -4497,7 +4674,7 @@ static int super_block_uvrd(const AV1_COMP *const cpi, MACROBLOCK *x, if (is_cost_valid) { for (plane = 1; plane < MAX_MB_PLANE; ++plane) { RD_STATS pn_rd_stats; - txfm_rd_in_plane(x, cpi, &pn_rd_stats, ref_best_rd, plane, bsize, + txfm_rd_in_plane(x, cpi, &pn_rd_stats, ref_best_rd, 0, plane, bsize, uv_tx_size, cpi->sf.use_fast_coef_costing, FTXS_NONE); if (pn_rd_stats.rate == INT_MAX) { is_cost_valid = 0; @@ -4520,11 +4697,12 @@ static int super_block_uvrd(const AV1_COMP *const cpi, MACROBLOCK *x, return is_cost_valid; } -static void tx_block_rd_b(const AV1_COMP *cpi, MACROBLOCK *x, TX_SIZE tx_size, - int blk_row, int blk_col, int plane, int block, - int plane_bsize, TXB_CTX *txb_ctx, RD_STATS *rd_stats, - FAST_TX_SEARCH_MODE ftxs_mode, int64_t ref_rdcost, - TXB_RD_INFO *rd_info_array) { +// Pick transform type for a transform block of tx_size. +static void tx_type_rd(const AV1_COMP *cpi, MACROBLOCK *x, TX_SIZE tx_size, + int blk_row, int blk_col, int plane, int block, + int plane_bsize, TXB_CTX *txb_ctx, RD_STATS *rd_stats, + FAST_TX_SEARCH_MODE ftxs_mode, int64_t ref_rdcost, + TXB_RD_INFO *rd_info_array) { const struct macroblock_plane *const p = &x->plane[plane]; const uint16_t cur_joint_ctx = (txb_ctx->dc_sign_ctx << 8) + txb_ctx->txb_skip_ctx; @@ -4675,6 +4853,7 @@ static int ml_predict_tx_split(MACROBLOCK *x, BLOCK_SIZE bsize, int blk_row, float score = 0.0f; av1_nn_predict(features, nn_config, &score); + aom_clear_system_state(); if (score > 8.0f) return 100; if (score < -8.0f) return 0; score = 1.0f / (1.0f + (float)exp(-score)); @@ -4716,9 +4895,9 @@ static void try_tx_block_no_split( rd_stats->zero_rate = zero_blk_rate; const int index = av1_get_txb_size_index(plane_bsize, blk_row, blk_col); mbmi->inter_tx_size[index] = tx_size; - tx_block_rd_b(cpi, x, tx_size, blk_row, blk_col, 0, block, plane_bsize, - &txb_ctx, rd_stats, ftxs_mode, ref_best_rd, - rd_info_node != NULL ? rd_info_node->rd_info_array : NULL); + tx_type_rd(cpi, x, tx_size, blk_row, blk_col, 0, block, plane_bsize, &txb_ctx, + rd_stats, ftxs_mode, ref_best_rd, + rd_info_node != NULL ? rd_info_node->rd_info_array : NULL); assert(rd_stats->rate < INT_MAX); if ((RDCOST(x->rdmult, rd_stats->rate, rd_stats->dist) >= @@ -4726,7 +4905,7 @@ static void try_tx_block_no_split( rd_stats->skip == 1) && !xd->lossless[mbmi->segment_id]) { #if CONFIG_RD_DEBUG - av1_update_txb_coeff_cost(rd_stats, plane, tx_size, blk_row, blk_col, + av1_update_txb_coeff_cost(rd_stats, 0, tx_size, blk_row, blk_col, zero_blk_rate - rd_stats->rate); #endif // CONFIG_RD_DEBUG rd_stats->rate = zero_blk_rate; @@ -4749,6 +4928,13 @@ static void try_tx_block_no_split( const int txk_type_idx = av1_get_txk_type_index(plane_bsize, blk_row, blk_col); no_split->tx_type = mbmi->txk_type[txk_type_idx]; + +#if CONFIG_ONE_PASS_SVM + if (plane_bsize >= BLOCK_8X8) { + av1_add_reg_stat(rd_stats, p->eobs[block], no_split->rd, rd_stats->sse, + blk_row, blk_col, plane_bsize, txsize_to_bsize[tx_size]); + } +#endif } static void select_tx_block(const AV1_COMP *cpi, MACROBLOCK *x, int blk_row, @@ -4756,8 +4942,8 @@ static void select_tx_block(const AV1_COMP *cpi, MACROBLOCK *x, int blk_row, BLOCK_SIZE plane_bsize, ENTROPY_CONTEXT *ta, ENTROPY_CONTEXT *tl, TXFM_CONTEXT *tx_above, TXFM_CONTEXT *tx_left, RD_STATS *rd_stats, - int64_t ref_best_rd, int *is_cost_valid, - FAST_TX_SEARCH_MODE ftxs_mode, + int64_t prev_level_rd, int64_t ref_best_rd, + int *is_cost_valid, FAST_TX_SEARCH_MODE ftxs_mode, TXB_RD_INFO_NODE *rd_info_node); static void try_tx_block_split( @@ -4767,6 +4953,7 @@ static void try_tx_block_split( int txfm_partition_ctx, int64_t no_split_rd, int64_t ref_best_rd, FAST_TX_SEARCH_MODE ftxs_mode, TXB_RD_INFO_NODE *rd_info_node, RD_STATS *split_rd_stats, int64_t *split_rd) { + assert(tx_size < TX_SIZES_ALL); MACROBLOCKD *const xd = &x->e_mbd; const int max_blocks_high = max_block_high(xd, plane_bsize, 0); const int max_blocks_wide = max_block_wide(xd, plane_bsize, 0); @@ -4774,44 +4961,37 @@ static void try_tx_block_split( const int bsw = tx_size_wide_unit[sub_txs]; const int bsh = tx_size_high_unit[sub_txs]; const int sub_step = bsw * bsh; - RD_STATS this_rd_stats; - int this_cost_valid = 1; + const int nblks = + (tx_size_high_unit[tx_size] / bsh) * (tx_size_wide_unit[tx_size] / bsw); + assert(nblks > 0); + int blk_idx = 0; int64_t tmp_rd = 0; - + *split_rd = INT64_MAX; split_rd_stats->rate = x->txfm_partition_cost[txfm_partition_ctx][1]; - assert(tx_size < TX_SIZES_ALL); - - int blk_idx = 0; for (int r = 0; r < tx_size_high_unit[tx_size]; r += bsh) { for (int c = 0; c < tx_size_wide_unit[tx_size]; c += bsw, ++blk_idx) { + assert(blk_idx < 4); const int offsetr = blk_row + r; const int offsetc = blk_col + c; if (offsetr >= max_blocks_high || offsetc >= max_blocks_wide) continue; - assert(blk_idx < 4); + + RD_STATS this_rd_stats; + int this_cost_valid = 1; select_tx_block( cpi, x, offsetr, offsetc, block, sub_txs, depth + 1, plane_bsize, ta, - tl, tx_above, tx_left, &this_rd_stats, ref_best_rd - tmp_rd, - &this_cost_valid, ftxs_mode, + tl, tx_above, tx_left, &this_rd_stats, no_split_rd / nblks, + ref_best_rd - tmp_rd, &this_cost_valid, ftxs_mode, (rd_info_node != NULL) ? rd_info_node->children[blk_idx] : NULL); - - if (!this_cost_valid) goto LOOP_EXIT; - + if (!this_cost_valid) return; av1_merge_rd_stats(split_rd_stats, &this_rd_stats); - tmp_rd = RDCOST(x->rdmult, split_rd_stats->rate, split_rd_stats->dist); - - if (no_split_rd < tmp_rd) { - this_cost_valid = 0; - goto LOOP_EXIT; - } + if (no_split_rd < tmp_rd) return; block += sub_step; } } -LOOP_EXIT : {} - - if (this_cost_valid) *split_rd = tmp_rd; + *split_rd = tmp_rd; } // Search for the best tx partition/type for a given luma block. @@ -4820,8 +5000,8 @@ static void select_tx_block(const AV1_COMP *cpi, MACROBLOCK *x, int blk_row, BLOCK_SIZE plane_bsize, ENTROPY_CONTEXT *ta, ENTROPY_CONTEXT *tl, TXFM_CONTEXT *tx_above, TXFM_CONTEXT *tx_left, RD_STATS *rd_stats, - int64_t ref_best_rd, int *is_cost_valid, - FAST_TX_SEARCH_MODE ftxs_mode, + int64_t prev_level_rd, int64_t ref_best_rd, + int *is_cost_valid, FAST_TX_SEARCH_MODE ftxs_mode, TXB_RD_INFO_NODE *rd_info_node) { assert(tx_size < TX_SIZES_ALL); av1_init_rd_stats(rd_stats); @@ -4841,7 +5021,8 @@ static void select_tx_block(const AV1_COMP *cpi, MACROBLOCK *x, int blk_row, mbmi->sb_type, tx_size); struct macroblock_plane *const p = &x->plane[0]; - const int try_no_split = 1; + const int try_no_split = + cpi->oxcf.enable_tx64 || txsize_sqr_up_map[tx_size] != TX_64X64; int try_split = tx_size > TX_4X4 && depth < MAX_VARTX_DEPTH; #if CONFIG_DIST_8X8 if (x->using_dist_8x8) @@ -4866,6 +5047,13 @@ static void select_tx_block(const AV1_COMP *cpi, MACROBLOCK *x, int blk_row, if (cpi->sf.txb_split_cap) { if (p->eobs[block] == 0) try_split = 0; } + + if (cpi->sf.adaptive_txb_search_level && + (no_split.rd - + (no_split.rd >> (2 + cpi->sf.adaptive_txb_search_level))) > + prev_level_rd) { + try_split = 0; + } } if (x->e_mbd.bd == 8 && !x->cb_partition_scan && try_split) { @@ -4913,95 +5101,12 @@ static void select_tx_block(const AV1_COMP *cpi, MACROBLOCK *x, int blk_row, } } -static void select_inter_block_yrd(const AV1_COMP *cpi, MACROBLOCK *x, - RD_STATS *rd_stats, BLOCK_SIZE bsize, - int64_t ref_best_rd, - FAST_TX_SEARCH_MODE ftxs_mode, - TXB_RD_INFO_NODE *rd_info_tree) { - MACROBLOCKD *const xd = &x->e_mbd; - int is_cost_valid = 1; - int64_t this_rd = 0, skip_rd = 0; - - if (ref_best_rd < 0) is_cost_valid = 0; - - av1_init_rd_stats(rd_stats); - - if (is_cost_valid) { - const struct macroblockd_plane *const pd = &xd->plane[0]; - const BLOCK_SIZE plane_bsize = - get_plane_block_size(bsize, pd->subsampling_x, pd->subsampling_y); - const int mi_width = mi_size_wide[plane_bsize]; - const int mi_height = mi_size_high[plane_bsize]; - const TX_SIZE max_tx_size = max_txsize_rect_lookup[plane_bsize]; - const int bh = tx_size_high_unit[max_tx_size]; - const int bw = tx_size_wide_unit[max_tx_size]; - int idx, idy; - int block = 0; - int step = tx_size_wide_unit[max_tx_size] * tx_size_high_unit[max_tx_size]; - ENTROPY_CONTEXT ctxa[MAX_MIB_SIZE]; - ENTROPY_CONTEXT ctxl[MAX_MIB_SIZE]; - TXFM_CONTEXT tx_above[MAX_MIB_SIZE]; - TXFM_CONTEXT tx_left[MAX_MIB_SIZE]; - - RD_STATS pn_rd_stats; - const int init_depth = - get_search_init_depth(mi_width, mi_height, 1, &cpi->sf); - av1_init_rd_stats(&pn_rd_stats); - - av1_get_entropy_contexts(bsize, pd, ctxa, ctxl); - memcpy(tx_above, xd->above_txfm_context, sizeof(TXFM_CONTEXT) * mi_width); - memcpy(tx_left, xd->left_txfm_context, sizeof(TXFM_CONTEXT) * mi_height); - const int skip_ctx = av1_get_skip_context(xd); - const int s0 = x->skip_cost[skip_ctx][0]; - const int s1 = x->skip_cost[skip_ctx][1]; - - skip_rd = RDCOST(x->rdmult, s1, 0); - this_rd = RDCOST(x->rdmult, s0, 0); - for (idy = 0; idy < mi_height; idy += bh) { - for (idx = 0; idx < mi_width; idx += bw) { - int64_t best_rd_sofar = (ref_best_rd - (AOMMIN(skip_rd, this_rd))); - select_tx_block(cpi, x, idy, idx, block, max_tx_size, init_depth, - plane_bsize, ctxa, ctxl, tx_above, tx_left, - &pn_rd_stats, best_rd_sofar, &is_cost_valid, ftxs_mode, - rd_info_tree); - if (!is_cost_valid || pn_rd_stats.rate == INT_MAX) { - av1_invalid_rd_stats(rd_stats); - return; - } - av1_merge_rd_stats(rd_stats, &pn_rd_stats); - skip_rd = RDCOST(x->rdmult, s1, rd_stats->sse); - this_rd = RDCOST(x->rdmult, rd_stats->rate + s0, rd_stats->dist); - block += step; - if (rd_info_tree != NULL) rd_info_tree += 1; - } - } - if (skip_rd <= this_rd) { - rd_stats->rate = 0; - rd_stats->dist = rd_stats->sse; - rd_stats->skip = 1; - } else { - rd_stats->skip = 0; - } - } - - if (!is_cost_valid) { - // reset cost value - av1_invalid_rd_stats(rd_stats); - } -} - -static int64_t select_tx_size_fix_type(const AV1_COMP *cpi, MACROBLOCK *x, +static int64_t select_tx_size_and_type(const AV1_COMP *cpi, MACROBLOCK *x, RD_STATS *rd_stats, BLOCK_SIZE bsize, int64_t ref_best_rd, TXB_RD_INFO_NODE *rd_info_tree) { - const int fast_tx_search = cpi->sf.tx_size_search_method > USE_FULL_RD; MACROBLOCKD *const xd = &x->e_mbd; - MB_MODE_INFO *const mbmi = xd->mi[0]; - const int is_inter = is_inter_block(mbmi); - const int skip_ctx = av1_get_skip_context(xd); - int s0 = x->skip_cost[skip_ctx][0]; - int s1 = x->skip_cost[skip_ctx][1]; - int64_t rd; + assert(is_inter_block(xd->mi[0])); // TODO(debargha): enable this as a speed feature where the // select_inter_block_yrd() function above will use a simplified search @@ -5009,16 +5114,74 @@ static int64_t select_tx_size_fix_type(const AV1_COMP *cpi, MACROBLOCK *x, // will use more complex search given that the transform partitions have // already been decided. + const int fast_tx_search = cpi->sf.tx_size_search_method > USE_FULL_RD; int64_t rd_thresh = ref_best_rd; if (fast_tx_search && rd_thresh < INT64_MAX) { if (INT64_MAX - rd_thresh > (rd_thresh >> 3)) rd_thresh += (rd_thresh >> 3); } assert(rd_thresh > 0); - FAST_TX_SEARCH_MODE ftxs_mode = + const FAST_TX_SEARCH_MODE ftxs_mode = fast_tx_search ? FTXS_DCT_AND_1D_DCT_ONLY : FTXS_NONE; - select_inter_block_yrd(cpi, x, rd_stats, bsize, rd_thresh, ftxs_mode, - rd_info_tree); + const struct macroblockd_plane *const pd = &xd->plane[0]; + const BLOCK_SIZE plane_bsize = + get_plane_block_size(bsize, pd->subsampling_x, pd->subsampling_y); + const int mi_width = mi_size_wide[plane_bsize]; + const int mi_height = mi_size_high[plane_bsize]; + ENTROPY_CONTEXT ctxa[MAX_MIB_SIZE]; + ENTROPY_CONTEXT ctxl[MAX_MIB_SIZE]; + TXFM_CONTEXT tx_above[MAX_MIB_SIZE]; + TXFM_CONTEXT tx_left[MAX_MIB_SIZE]; + av1_get_entropy_contexts(bsize, pd, ctxa, ctxl); + memcpy(tx_above, xd->above_txfm_context, sizeof(TXFM_CONTEXT) * mi_width); + memcpy(tx_left, xd->left_txfm_context, sizeof(TXFM_CONTEXT) * mi_height); + + const int skip_ctx = av1_get_skip_context(xd); + const int s0 = x->skip_cost[skip_ctx][0]; + const int s1 = x->skip_cost[skip_ctx][1]; + const int init_depth = + get_search_init_depth(mi_width, mi_height, 1, &cpi->sf); + const TX_SIZE max_tx_size = max_txsize_rect_lookup[plane_bsize]; + const int bh = tx_size_high_unit[max_tx_size]; + const int bw = tx_size_wide_unit[max_tx_size]; + const int step = bw * bh; + int64_t skip_rd = RDCOST(x->rdmult, s1, 0); + int64_t this_rd = RDCOST(x->rdmult, s0, 0); + int block = 0; + + av1_init_rd_stats(rd_stats); + for (int idy = 0; idy < mi_height; idy += bh) { + for (int idx = 0; idx < mi_width; idx += bw) { + const int64_t best_rd_sofar = + (rd_thresh == INT64_MAX) ? INT64_MAX + : (rd_thresh - (AOMMIN(skip_rd, this_rd))); + int is_cost_valid = 1; + RD_STATS pn_rd_stats; + select_tx_block(cpi, x, idy, idx, block, max_tx_size, init_depth, + plane_bsize, ctxa, ctxl, tx_above, tx_left, &pn_rd_stats, + INT64_MAX, best_rd_sofar, &is_cost_valid, ftxs_mode, + rd_info_tree); + if (!is_cost_valid || pn_rd_stats.rate == INT_MAX) { + av1_invalid_rd_stats(rd_stats); + return INT64_MAX; + } + av1_merge_rd_stats(rd_stats, &pn_rd_stats); + skip_rd = RDCOST(x->rdmult, s1, rd_stats->sse); + this_rd = RDCOST(x->rdmult, rd_stats->rate + s0, rd_stats->dist); + block += step; + if (rd_info_tree != NULL) rd_info_tree += 1; + } + } + + if (skip_rd <= this_rd) { + rd_stats->skip = 1; +#if CONFIG_ONE_PASS_SVM + av1_reg_stat_skipmode_update(rd_stats, x->rdmult); +#endif + } else { + rd_stats->skip = 0; + } + if (rd_stats->rate == INT_MAX) return INT64_MAX; // If fast_tx_search is true, only DCT and 1D DCT were tested in @@ -5029,13 +5192,19 @@ static int64_t select_tx_size_fix_type(const AV1_COMP *cpi, MACROBLOCK *x, return INT64_MAX; } - if (rd_stats->skip) + int64_t rd; + if (rd_stats->skip) { rd = RDCOST(x->rdmult, s1, rd_stats->sse); - else +#if CONFIG_ONE_PASS_SVM + // TODO(chiyotsai@google.com): Investigate if these updates are really + // needed. + av1_reg_stat_skipmode_update(rd_stats, x->rdmult); +#endif + } else { rd = RDCOST(x->rdmult, rd_stats->rate + s0, rd_stats->dist); - - if (is_inter && !xd->lossless[xd->mi[0]->segment_id] && !(rd_stats->skip)) - rd = AOMMIN(rd, RDCOST(x->rdmult, s1, rd_stats->sse)); + if (!xd->lossless[xd->mi[0]->segment_id]) + rd = AOMMIN(rd, RDCOST(x->rdmult, s1, rd_stats->sse)); + } return rd; } @@ -5075,8 +5244,8 @@ static void tx_block_yrd(const AV1_COMP *cpi, MACROBLOCK *x, int blk_row, .txb_skip_cost[txb_ctx.txb_skip_ctx][1]; rd_stats->zero_rate = zero_blk_rate; rd_stats->ref_rdcost = ref_best_rd; - tx_block_rd_b(cpi, x, tx_size, blk_row, blk_col, 0, block, plane_bsize, - &txb_ctx, rd_stats, ftxs_mode, ref_best_rd, NULL); + tx_type_rd(cpi, x, tx_size, blk_row, blk_col, 0, block, plane_bsize, + &txb_ctx, rd_stats, ftxs_mode, ref_best_rd, NULL); const int mi_width = block_size_wide[plane_bsize] >> tx_size_wide_log2[0]; if (RDCOST(x->rdmult, rd_stats->rate, rd_stats->dist) >= RDCOST(x->rdmult, zero_blk_rate, rd_stats->sse) || @@ -5089,9 +5258,20 @@ static void tx_block_yrd(const AV1_COMP *cpi, MACROBLOCK *x, int blk_row, x->plane[0].txb_entropy_ctx[block] = 0; update_txk_array(mbmi->txk_type, plane_bsize, blk_row, blk_col, tx_size, DCT_DCT); +#if CONFIG_ONE_PASS_SVM + av1_add_reg_stat(rd_stats, 0, RDCOST(x->rdmult, 0, rd_stats->sse), + rd_stats->sse, blk_row, blk_col, plane_bsize, + txsize_to_bsize[tx_size]); +#endif } else { rd_stats->skip = 0; set_blk_skip(x, 0, blk_row * mi_width + blk_col, 0); +#if CONFIG_ONE_PASS_SVM + av1_add_reg_stat(rd_stats, x->plane[0].eobs[block], + RDCOST(x->rdmult, rd_stats->rate, rd_stats->dist), + rd_stats->sse, blk_row, blk_col, plane_bsize, + txsize_to_bsize[tx_size]); +#endif } if (tx_size > TX_4X4 && depth < MAX_VARTX_DEPTH) rd_stats->rate += x->txfm_partition_cost[ctx][0]; @@ -5199,6 +5379,11 @@ static int inter_block_yrd(const AV1_COMP *cpi, MACROBLOCK *x, rd_stats->rate = 0; rd_stats->dist = rd_stats->sse; rd_stats->skip = 1; +#if CONFIG_ONE_PASS_SVM + // TODO(chiyotasi@google.com): Investigate if these updates are really + // needed. + av1_reg_stat_skipmode_update(rd_stats, x->rdmult); +#endif } if (this_rd > ref_best_rd) is_cost_valid = 0; @@ -5542,7 +5727,8 @@ static int predict_skip_flag(MACROBLOCK *x, BLOCK_SIZE bsize, int64_t *dist, const MACROBLOCKD *xd = &x->e_mbd; const int16_t dc_q = av1_dc_quant_QTX(x->qindex, 0, xd->bd); - *dist = pixel_diff_dist(x, 0, 0, 0, bsize, bsize); + *dist = pixel_diff_dist(x, 0, 0, 0, bsize, bsize, NULL); + const int64_t mse = *dist / bw / bh; // Normalized quantizer takes the transform upscaling factor (8 for tx size // smaller than 32) into account. @@ -5559,7 +5745,7 @@ static int predict_skip_flag(MACROBLOCK *x, BLOCK_SIZE bsize, int64_t *dist, param.tx_type = DCT_DCT; param.tx_size = max_tx_size; param.bd = xd->bd; - param.is_hbd = get_bitdepth_data_path_index(xd); + param.is_hbd = is_cur_buf_hbd(xd); param.lossless = 0; param.tx_set_type = av1_get_ext_tx_set_type( param.tx_size, is_inter_block(xd->mi[0]), reduced_tx_set); @@ -5586,6 +5772,42 @@ static int predict_skip_flag(MACROBLOCK *x, BLOCK_SIZE bsize, int64_t *dist, return 1; } +#if CONFIG_ONE_PASS_SVM +static void calc_regional_sse(MACROBLOCK *x, BLOCK_SIZE bsize, int64_t dist, + RD_STATS *rd_stats) { + // TODO(chiyotsai@google.com): Don't need regional sse's unless we are doing + // none. + const int bw = block_size_wide[bsize]; + const int bw_mi = bw >> tx_size_wide_log2[0]; + const int bh_mi = bw >> tx_size_high_log2[0]; + const BLOCK_SIZE split_size = get_partition_subsize(bsize, PARTITION_SPLIT); + int64_t dist_0, dist_1, dist_2, dist_3; + MACROBLOCKD *xd = &x->e_mbd; + dist_0 = pixel_diff_dist(x, AOM_PLANE_Y, 0, 0, bsize, split_size, NULL); + dist_1 = + pixel_diff_dist(x, AOM_PLANE_Y, 0, bw_mi / 2, bsize, split_size, NULL); + dist_2 = + pixel_diff_dist(x, AOM_PLANE_Y, bh_mi / 2, 0, bsize, split_size, NULL); + dist_3 = pixel_diff_dist(x, AOM_PLANE_Y, bh_mi / 2, bw_mi / 2, bsize, + split_size, NULL); + + if (is_cur_buf_hbd(xd)) { + dist = ROUND_POWER_OF_TWO(dist, (xd->bd - 8) * 2); + dist_0 = ROUND_POWER_OF_TWO(dist_0, (xd->bd - 8) * 2); + dist_1 = ROUND_POWER_OF_TWO(dist_1, (xd->bd - 8) * 2); + dist_2 = ROUND_POWER_OF_TWO(dist_2, (xd->bd - 8) * 2); + dist_3 = ROUND_POWER_OF_TWO(dist_3, (xd->bd - 8) * 2); + } + const int scaling_factor = MAX_MIB_SIZE * MAX_MIB_SIZE; + rd_stats->y_sse = (dist << 4); + rd_stats->sse_0 = (dist_0 << 4) * scaling_factor; + rd_stats->sse_1 = (dist_1 << 4) * scaling_factor; + rd_stats->sse_2 = (dist_2 << 4) * scaling_factor; + rd_stats->sse_3 = (dist_3 << 4) * scaling_factor; + av1_reg_stat_skipmode_update(rd_stats, x->rdmult); +} +#endif + // Used to set proper context for early termination with skip = 1. static void set_skip_flag(MACROBLOCK *x, RD_STATS *rd_stats, int bsize, int64_t dist) { @@ -5598,31 +5820,41 @@ static void set_skip_flag(MACROBLOCK *x, RD_STATS *rd_stats, int bsize, mbmi->tx_size = tx_size; for (int i = 0; i < n4; ++i) set_blk_skip(x, 0, i, 1); rd_stats->skip = 1; - rd_stats->rate = 0; - if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) - dist = ROUND_POWER_OF_TWO(dist, (xd->bd - 8) * 2); + if (is_cur_buf_hbd(xd)) dist = ROUND_POWER_OF_TWO(dist, (xd->bd - 8) * 2); rd_stats->dist = rd_stats->sse = (dist << 4); + // Though decision is to make the block as skip based on luma stats, + // it is possible that block becomes non skip after chroma rd. In addition + // intermediate non skip costs calculated by caller function will be + // incorrect, if rate is set as zero (i.e., if zero_blk_rate is not + // accounted). Hence intermediate rate is populated to code the luma tx blks + // as skip, the caller function based on final rd decision (i.e., skip vs + // non-skip) sets the final rate accordingly. Here the rate populated + // corresponds to coding all the tx blocks with zero_blk_rate (based on max tx + // size possible) in the current block. Eg: For 128*128 block, rate would be + // 4 * zero_blk_rate where zero_blk_rate corresponds to coding of one 64x64 tx + // block as 'all zeros' + ENTROPY_CONTEXT ctxa[MAX_MIB_SIZE]; + ENTROPY_CONTEXT ctxl[MAX_MIB_SIZE]; + av1_get_entropy_contexts(bsize, &xd->plane[0], ctxa, ctxl); + ENTROPY_CONTEXT *ta = ctxa; + ENTROPY_CONTEXT *tl = ctxl; + const TX_SIZE txs_ctx = get_txsize_entropy_ctx(tx_size); + TXB_CTX txb_ctx; + get_txb_ctx(bsize, tx_size, 0, ta, tl, &txb_ctx); + const int zero_blk_rate = x->coeff_costs[txs_ctx][PLANE_TYPE_Y] + .txb_skip_cost[txb_ctx.txb_skip_ctx][1]; + rd_stats->rate = zero_blk_rate * + (block_size_wide[bsize] >> tx_size_wide_log2[tx_size]) * + (block_size_high[bsize] >> tx_size_high_log2[tx_size]); } -static void select_tx_type_yrd(const AV1_COMP *cpi, MACROBLOCK *x, - RD_STATS *rd_stats, BLOCK_SIZE bsize, int mi_row, - int mi_col, int64_t ref_best_rd) { +// Search for best transform size and type for luma inter blocks. +static void pick_tx_size_type_yrd(const AV1_COMP *cpi, MACROBLOCK *x, + RD_STATS *rd_stats, BLOCK_SIZE bsize, + int mi_row, int mi_col, int64_t ref_best_rd) { const AV1_COMMON *cm = &cpi->common; MACROBLOCKD *const xd = &x->e_mbd; - MB_MODE_INFO *const mbmi = xd->mi[0]; - int64_t rd = INT64_MAX; - int64_t best_rd = INT64_MAX; - const int is_inter = is_inter_block(mbmi); - const int n4 = bsize_to_num_blk(bsize); - // Get the tx_size 1 level down - const TX_SIZE min_tx_size = sub_tx_size_map[max_txsize_rect_lookup[bsize]]; - const TxSetType tx_set_type = - av1_get_ext_tx_set_type(min_tx_size, is_inter, cm->reduced_tx_set_used); - const int within_border = - mi_row >= xd->tile.mi_row_start && - (mi_row + mi_size_high[bsize] < xd->tile.mi_row_end) && - mi_col >= xd->tile.mi_col_start && - (mi_col + mi_size_wide[bsize] < xd->tile.mi_col_end); + assert(is_inter_block(xd->mi[0])); av1_invalid_rd_stats(rd_stats); @@ -5639,8 +5871,7 @@ static void select_tx_type_yrd(const AV1_COMP *cpi, MACROBLOCK *x, // tighter. assert(cpi->sf.model_based_prune_tx_search_level >= 0 && cpi->sf.model_based_prune_tx_search_level <= 2); - static const int prune_factor_by8[] = { 2 + MODELRD_TYPE_TX_SEARCH_PRUNE, - 4 + MODELRD_TYPE_TX_SEARCH_PRUNE }; + static const int prune_factor_by8[] = { 3, 5 }; if (!model_skip && ((model_rd * prune_factor_by8[cpi->sf.model_based_prune_tx_search_level - 1]) >> @@ -5648,18 +5879,29 @@ static void select_tx_type_yrd(const AV1_COMP *cpi, MACROBLOCK *x, return; } - const uint32_t hash = get_block_residue_hash(x, bsize); - MB_RD_RECORD *mb_rd_record = &x->mb_rd_record; - - if (ref_best_rd != INT64_MAX && within_border && cpi->sf.use_mb_rd_hash) { - for (int i = 0; i < mb_rd_record->num; ++i) { - const int index = (mb_rd_record->index_start + i) % RD_RECORD_BUFFER_LEN; - // If there is a match in the tx_rd_record, fetch the RD decision and - // terminate early. - if (mb_rd_record->tx_rd_info[index].hash_value == hash) { - MB_RD_INFO *tx_rd_info = &mb_rd_record->tx_rd_info[index]; - fetch_tx_rd_info(n4, tx_rd_info, rd_stats, x); - return; + uint32_t hash = 0; + MB_RD_RECORD *mb_rd_record = NULL; + const int within_border = + mi_row >= xd->tile.mi_row_start && + (mi_row + mi_size_high[bsize] < xd->tile.mi_row_end) && + mi_col >= xd->tile.mi_col_start && + (mi_col + mi_size_wide[bsize] < xd->tile.mi_col_end); + const int is_mb_rd_hash_enabled = (within_border && cpi->sf.use_mb_rd_hash); + const int n4 = bsize_to_num_blk(bsize); + if (is_mb_rd_hash_enabled) { + hash = get_block_residue_hash(x, bsize); + mb_rd_record = &x->mb_rd_record; + if (ref_best_rd != INT64_MAX) { + for (int i = 0; i < mb_rd_record->num; ++i) { + const int index = + (mb_rd_record->index_start + i) % RD_RECORD_BUFFER_LEN; + // If there is a match in the tx_rd_record, fetch the RD decision and + // terminate early. + if (mb_rd_record->tx_rd_info[index].hash_value == hash) { + MB_RD_INFO *tx_rd_info = &mb_rd_record->tx_rd_info[index]; + fetch_tx_rd_info(n4, tx_rd_info, rd_stats, x); + return; + } } } } @@ -5667,13 +5909,23 @@ static void select_tx_type_yrd(const AV1_COMP *cpi, MACROBLOCK *x, // If we predict that skip is the optimal RD decision - set the respective // context and terminate early. int64_t dist; - if (is_inter && cpi->sf.tx_type_search.use_skip_flag_prediction && + if (cpi->sf.tx_type_search.use_skip_flag_prediction && predict_skip_flag(x, bsize, &dist, cm->reduced_tx_set_used)) { set_skip_flag(x, rd_stats, bsize, dist); +#if CONFIG_ONE_PASS_SVM + if (bsize >= BLOCK_8X8 && mi_size_wide[bsize] == mi_size_high[bsize] && + xd->mi[0]->partition == PARTITION_NONE) { + calc_regional_sse(x, bsize, dist, rd_stats); + } +#endif // Save the RD search results into tx_rd_record. - if (within_border) save_tx_rd_info(n4, hash, x, rd_stats, mb_rd_record); + if (is_mb_rd_hash_enabled) + save_tx_rd_info(n4, hash, x, rd_stats, mb_rd_record); return; } +#if CONFIG_SPEED_STATS + ++x->tx_search_count; +#endif // CONFIG_SPEED_STATS // Precompute residual hashes and find existing or add new RD records to // store and reuse rate and distortion values to speed up TX size search. @@ -5684,20 +5936,20 @@ static void select_tx_type_yrd(const AV1_COMP *cpi, MACROBLOCK *x, find_tx_size_rd_records(x, bsize, mi_row, mi_col, matched_rd_info); } + // Get the tx_size 1 level down + const TX_SIZE min_tx_size = sub_tx_size_map[max_txsize_rect_lookup[bsize]]; + const TxSetType tx_set_type = + av1_get_ext_tx_set_type(min_tx_size, 1, cm->reduced_tx_set_used); prune_tx(cpi, bsize, x, xd, tx_set_type); int found = 0; - RD_STATS this_rd_stats; av1_init_rd_stats(&this_rd_stats); + const int64_t rd = + select_tx_size_and_type(cpi, x, &this_rd_stats, bsize, ref_best_rd, + found_rd_info ? matched_rd_info : NULL); - rd = select_tx_size_fix_type(cpi, x, &this_rd_stats, bsize, ref_best_rd, - found_rd_info ? matched_rd_info : NULL); - assert(IMPLIES(this_rd_stats.skip && !this_rd_stats.invalid_rate, - this_rd_stats.rate == 0)); - - ref_best_rd = AOMMIN(rd, ref_best_rd); - if (rd < best_rd) { + if (rd < INT64_MAX) { *rd_stats = this_rd_stats; found = 1; } @@ -5713,127 +5965,76 @@ static void select_tx_type_yrd(const AV1_COMP *cpi, MACROBLOCK *x, if (!found) return; // Save the RD search results into tx_rd_record. - if (within_border && cpi->sf.use_mb_rd_hash) + if (is_mb_rd_hash_enabled) { + assert(mb_rd_record != NULL); save_tx_rd_info(n4, hash, x, rd_stats, mb_rd_record); + } } -static void tx_block_uvrd(const AV1_COMP *cpi, MACROBLOCK *x, int blk_row, - int blk_col, int plane, int block, TX_SIZE tx_size, - BLOCK_SIZE plane_bsize, ENTROPY_CONTEXT *above_ctx, - ENTROPY_CONTEXT *left_ctx, RD_STATS *rd_stats, - FAST_TX_SEARCH_MODE ftxs_mode) { - assert(plane > 0); - assert(tx_size < TX_SIZES_ALL); - MACROBLOCKD *const xd = &x->e_mbd; - MB_MODE_INFO *const mbmi = xd->mi[0]; - const int max_blocks_high = max_block_high(xd, plane_bsize, plane); - const int max_blocks_wide = max_block_wide(xd, plane_bsize, plane); - if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return; +static void model_rd_for_sb_with_fullrdy( + const AV1_COMP *const cpi, BLOCK_SIZE bsize, MACROBLOCK *x, MACROBLOCKD *xd, + int plane_from, int plane_to, int mi_row, int mi_col, int *out_rate_sum, + int64_t *out_dist_sum, int *skip_txfm_sb, int64_t *skip_sse_sb, + int *plane_rate, int64_t *plane_sse, int64_t *plane_dist) { + const int ref = xd->mi[0]->ref_frame[0]; - ENTROPY_CONTEXT *ta = above_ctx + blk_col; - ENTROPY_CONTEXT *tl = left_ctx + blk_row; - TXB_CTX txb_ctx; - get_txb_ctx(plane_bsize, tx_size, plane, ta, tl, &txb_ctx); - const TX_SIZE txs_ctx = get_txsize_entropy_ctx(tx_size); - const int zero_blk_rate = x->coeff_costs[txs_ctx][PLANE_TYPE_UV] - .txb_skip_cost[txb_ctx.txb_skip_ctx][1]; - tx_block_rd_b(cpi, x, tx_size, blk_row, blk_col, plane, block, plane_bsize, - &txb_ctx, rd_stats, ftxs_mode, INT64_MAX, NULL); + int64_t rate_sum = 0; + int64_t dist_sum = 0; + int64_t total_sse = 0; - const int mi_width = block_size_wide[plane_bsize] >> tx_size_wide_log2[0]; - const int blk_idx = blk_row * mi_width + blk_col; + for (int plane = plane_from; plane <= plane_to; ++plane) { + struct macroblock_plane *const p = &x->plane[plane]; + struct macroblockd_plane *const pd = &xd->plane[plane]; + const BLOCK_SIZE plane_bsize = + get_plane_block_size(bsize, pd->subsampling_x, pd->subsampling_y); + const int bw = block_size_wide[plane_bsize]; + const int bh = block_size_high[plane_bsize]; + int64_t sse; + int rate; + int64_t dist; - av1_set_txb_context(x, plane, block, tx_size, ta, tl); - if ((RDCOST(x->rdmult, rd_stats->rate, rd_stats->dist) >= - RDCOST(x->rdmult, zero_blk_rate, rd_stats->sse) || - rd_stats->skip == 1) && - !xd->lossless[mbmi->segment_id]) { - rd_stats->rate = zero_blk_rate; - rd_stats->dist = rd_stats->sse; - } + if (x->skip_chroma_rd && plane) continue; - // Set chroma blk_skip to 0 - set_blk_skip(x, plane, blk_idx, 0); -} - -// Return value 0: early termination triggered, no valid rd cost available; -// 1: rd cost values are valid. -static int inter_block_uvrd(const AV1_COMP *cpi, MACROBLOCK *x, - RD_STATS *rd_stats, BLOCK_SIZE bsize, - int64_t non_skip_ref_best_rd, - int64_t skip_ref_best_rd, - FAST_TX_SEARCH_MODE ftxs_mode) { - MACROBLOCKD *const xd = &x->e_mbd; - MB_MODE_INFO *const mbmi = xd->mi[0]; - int plane; - int is_cost_valid = 1; - int64_t this_rd = 0; - int64_t skip_rd = 0; - - if ((non_skip_ref_best_rd < 0) && (skip_ref_best_rd < 0)) is_cost_valid = 0; - - av1_init_rd_stats(rd_stats); - - if (x->skip_chroma_rd) { - if (!is_cost_valid) av1_invalid_rd_stats(rd_stats); - - return is_cost_valid; - } - - const BLOCK_SIZE bsizec = scale_chroma_bsize( - bsize, xd->plane[1].subsampling_x, xd->plane[1].subsampling_y); - - if (is_inter_block(mbmi) && is_cost_valid) { - for (plane = 1; plane < MAX_MB_PLANE; ++plane) - av1_subtract_plane(x, bsizec, plane); - } - - if (is_cost_valid) { - for (plane = 1; plane < MAX_MB_PLANE; ++plane) { - const struct macroblockd_plane *const pd = &xd->plane[plane]; - const BLOCK_SIZE plane_bsize = - get_plane_block_size(bsizec, pd->subsampling_x, pd->subsampling_y); - const int mi_width = block_size_wide[plane_bsize] >> tx_size_wide_log2[0]; - const int mi_height = - block_size_high[plane_bsize] >> tx_size_high_log2[0]; - const TX_SIZE max_tx_size = get_vartx_max_txsize(xd, plane_bsize, plane); - const int bh = tx_size_high_unit[max_tx_size]; - const int bw = tx_size_wide_unit[max_tx_size]; - int idx, idy; - int block = 0; - const int step = bh * bw; - ENTROPY_CONTEXT ta[MAX_MIB_SIZE]; - ENTROPY_CONTEXT tl[MAX_MIB_SIZE]; - av1_get_entropy_contexts(bsizec, pd, ta, tl); - - for (idy = 0; idy < mi_height; idy += bh) { - for (idx = 0; idx < mi_width; idx += bw) { - RD_STATS pn_rd_stats; - av1_init_rd_stats(&pn_rd_stats); - tx_block_uvrd(cpi, x, idy, idx, plane, block, max_tx_size, - plane_bsize, ta, tl, &pn_rd_stats, ftxs_mode); - if (pn_rd_stats.rate == INT_MAX) { - av1_invalid_rd_stats(rd_stats); - return 0; - } - av1_merge_rd_stats(rd_stats, &pn_rd_stats); - this_rd = RDCOST(x->rdmult, rd_stats->rate, rd_stats->dist); - skip_rd = RDCOST(x->rdmult, 0, rd_stats->sse); - if ((this_rd > non_skip_ref_best_rd) && - (skip_rd > skip_ref_best_rd)) { - av1_invalid_rd_stats(rd_stats); - return 0; - } - block += step; - } - } + if (is_cur_buf_hbd(xd)) { + sse = aom_highbd_sse(p->src.buf, p->src.stride, pd->dst.buf, + pd->dst.stride, bw, bh); + } else { + sse = aom_sse(p->src.buf, p->src.stride, pd->dst.buf, pd->dst.stride, bw, + bh); } - } else { - // reset cost value - av1_invalid_rd_stats(rd_stats); + sse = ROUND_POWER_OF_TWO(sse, (xd->bd - 8) * 2); + + RD_STATS rd_stats; + if (plane == 0) { + pick_tx_size_type_yrd(cpi, x, &rd_stats, bsize, mi_row, mi_col, + INT64_MAX); + if (rd_stats.invalid_rate) { + rate = 0; + dist = sse << 4; + } else { + rate = rd_stats.rate; + dist = rd_stats.dist; + } + } else { + model_rd_with_curvfit(cpi, x, plane_bsize, plane, sse, bw * bh, &rate, + &dist); + } + + if (plane == 0) x->pred_sse[ref] = (unsigned int)AOMMIN(sse, UINT_MAX); + + total_sse += sse; + rate_sum += rate; + dist_sum += dist; + + if (plane_rate) plane_rate[plane] = rate; + if (plane_sse) plane_sse[plane] = sse; + if (plane_dist) plane_dist[plane] = dist; } - return is_cost_valid; + if (skip_txfm_sb) *skip_txfm_sb = total_sse == 0; + if (skip_sse_sb) *skip_sse_sb = total_sse << 4; + *out_rate_sum = (int)rate_sum; + *out_dist_sum = dist_sum; } static void rd_pick_palette_intra_sbuv(const AV1_COMP *const cpi, MACROBLOCK *x, @@ -6081,7 +6282,7 @@ static int cfl_rd_pick_alpha(MACROBLOCK *const x, const AV1_COMP *const cpi, const BLOCK_SIZE bsize = mbmi->sb_type; #if CONFIG_DEBUG - assert(is_cfl_allowed(xd)); + assert(is_cfl_allowed(xd) && cpi->oxcf.enable_cfl_intra); const int ssx = xd->plane[AOM_PLANE_U].subsampling_x; const int ssy = xd->plane[AOM_PLANE_U].subsampling_y; const BLOCK_SIZE plane_bsize = get_plane_block_size(mbmi->sb_type, ssx, ssy); @@ -6117,8 +6318,8 @@ static int cfl_rd_pick_alpha(MACROBLOCK *const x, const AV1_COMP *const cpi, if (i == CFL_SIGN_NEG) { mbmi->cfl_alpha_idx = 0; mbmi->cfl_alpha_signs = joint_sign; - txfm_rd_in_plane(x, cpi, &rd_stats, best_rd, plane + 1, bsize, tx_size, - cpi->sf.use_fast_coef_costing, FTXS_NONE); + txfm_rd_in_plane(x, cpi, &rd_stats, best_rd, 0, plane + 1, bsize, + tx_size, cpi->sf.use_fast_coef_costing, FTXS_NONE); if (rd_stats.rate == INT_MAX) break; } const int alpha_rate = x->cfl_cost[joint_sign][plane][0]; @@ -6145,7 +6346,7 @@ static int cfl_rd_pick_alpha(MACROBLOCK *const x, const AV1_COMP *const cpi, if (i == 0) { mbmi->cfl_alpha_idx = (c << CFL_ALPHABET_SIZE_LOG2) + c; mbmi->cfl_alpha_signs = joint_sign; - txfm_rd_in_plane(x, cpi, &rd_stats, best_rd, plane + 1, bsize, + txfm_rd_in_plane(x, cpi, &rd_stats, best_rd, 0, plane + 1, bsize, tx_size, cpi->sf.use_fast_coef_costing, FTXS_NONE); if (rd_stats.rate == INT_MAX) break; } @@ -6223,14 +6424,15 @@ static int64_t rd_pick_intra_sbuv_mode(const AV1_COMP *const cpi, MACROBLOCK *x, mbmi->uv_mode = mode; int cfl_alpha_rate = 0; if (mode == UV_CFL_PRED) { - if (!is_cfl_allowed(xd)) continue; + if (!is_cfl_allowed(xd) || !cpi->oxcf.enable_cfl_intra) continue; assert(!is_directional_mode); const TX_SIZE uv_tx_size = av1_get_tx_size(AOM_PLANE_U, xd); cfl_alpha_rate = cfl_rd_pick_alpha(x, cpi, uv_tx_size, best_rd); if (cfl_alpha_rate == INT_MAX) continue; } mbmi->angle_delta[PLANE_TYPE_UV] = 0; - if (is_directional_mode && av1_use_angle_delta(mbmi->sb_type)) { + if (is_directional_mode && av1_use_angle_delta(mbmi->sb_type) && + cpi->oxcf.enable_angle_delta) { const int rate_overhead = x->intra_uv_mode_cost[is_cfl_allowed(xd)][mbmi->mode][mode]; if (!rd_pick_intra_angle_sbuv(cpi, x, bsize, rate_overhead, best_rd, @@ -6247,7 +6449,7 @@ static int64_t rd_pick_intra_sbuv_mode(const AV1_COMP *const cpi, MACROBLOCK *x, this_rate = tokenonly_rd_stats.rate + intra_mode_info_cost_uv(cpi, x, mbmi, bsize, mode_cost); if (mode == UV_CFL_PRED) { - assert(is_cfl_allowed(xd)); + assert(is_cfl_allowed(xd) && cpi->oxcf.enable_cfl_intra); #if CONFIG_DEBUG if (!xd->lossless[mbmi->segment_id]) assert(xd->cfl.rate == tokenonly_rd_stats.rate + mode_cost); @@ -6266,6 +6468,7 @@ static int64_t rd_pick_intra_sbuv_mode(const AV1_COMP *const cpi, MACROBLOCK *x, } const int try_palette = + cpi->oxcf.enable_palette && av1_allow_palette(cpi->common.allow_screen_content_tools, mbmi->sb_type); if (try_palette) { uint8_t *best_palette_color_map = x->palette_buffer->best_palette_color_map; @@ -6443,7 +6646,7 @@ static void joint_motion_search(const AV1_COMP *cpi, MACROBLOCK *x, const int p_row = ((mi_row * MI_SIZE) >> pd->subsampling_y) + 4 * ir; ConvolveParams conv_params = get_conv_params(0, plane, xd->bd); - conv_params.use_jnt_comp_avg = 0; + conv_params.use_dist_wtd_comp_avg = 0; WarpTypesAllowed warp_types[2]; for (ref = 0; ref < 2; ++ref) { const WarpedMotionParams *const wm = @@ -6466,8 +6669,6 @@ static void joint_motion_search(const AV1_COMP *cpi, MACROBLOCK *x, uint8_t *second_pred = get_buf_by_bd(xd, second_pred16); (void)ref_mv_sub8x8; - const int have_newmv = have_nearmv_in_inter_mode(mbmi->mode); - const int ref_mv_idx = mbmi->ref_mv_idx + (have_newmv ? 1 : 0); MV *const best_mv = &x->best_mv.as_mv; const int search_range = SEARCH_RANGE_8P; const int sadpb = x->sadperbit16; @@ -6486,7 +6687,7 @@ static void joint_motion_search(const AV1_COMP *cpi, MACROBLOCK *x, } else { int_mv cur_int_mv, init_int_mv; cur_int_mv.as_mv.col = cur_mv[id].as_mv.col >> 3; - cur_int_mv.as_mv.row = cur_mv[id].as_mv.col >> 3; + cur_int_mv.as_mv.row = cur_mv[id].as_mv.row >> 3; init_int_mv.as_mv.row = init_mv[id].as_mv.row >> 3; init_int_mv.as_mv.col = init_mv[id].as_mv.col >> 3; if (cur_int_mv.as_int == init_int_mv.as_int) { @@ -6532,9 +6733,9 @@ static void joint_motion_search(const AV1_COMP *cpi, MACROBLOCK *x, mi_row * MI_SIZE, xd, cm->allow_warped_motion); const int order_idx = id != 0; - av1_jnt_comp_weight_assign(cm, mbmi, order_idx, &xd->jcp_param.fwd_offset, - &xd->jcp_param.bck_offset, - &xd->jcp_param.use_jnt_comp_avg, 1); + av1_dist_wtd_comp_weight_assign( + cm, mbmi, order_idx, &xd->jcp_param.fwd_offset, + &xd->jcp_param.bck_offset, &xd->jcp_param.use_dist_wtd_comp_avg, 1); // Do full-pixel compound motion search on the current reference frame. if (id) xd->plane[plane].pre[0] = ref_yv12[id]; @@ -6546,8 +6747,6 @@ static void joint_motion_search(const AV1_COMP *cpi, MACROBLOCK *x, best_mv->col >>= 3; best_mv->row >>= 3; - av1_set_mvcost(x, id, ref_mv_idx); - // Small-range full-pixel motion search. bestsme = av1_refining_search_8p_c(x, sadpb, search_range, &cpi->fn_ptr[bsize], mask, mask_stride, @@ -6592,8 +6791,8 @@ static void joint_motion_search(const AV1_COMP *cpi, MACROBLOCK *x, x, cm, mi_row, mi_col, &ref_mv[id].as_mv, cpi->common.allow_high_precision_mv, x->errorperbit, &cpi->fn_ptr[bsize], 0, cpi->sf.mv.subpel_iters_per_step, NULL, - x->nmvjointcost, x->mvcost, &dis, &sse, second_pred, mask, - mask_stride, id, pw, ph, cpi->sf.use_accurate_subpel_search); + x->nmv_vec_cost, x->mv_cost_stack, &dis, &sse, second_pred, mask, + mask_stride, id, pw, ph, cpi->sf.use_accurate_subpel_search, 1); } // Restore the pointer to the first prediction buffer. @@ -6609,10 +6808,10 @@ static void joint_motion_search(const AV1_COMP *cpi, MACROBLOCK *x, *rate_mv = 0; for (ref = 0; ref < 2; ++ref) { - av1_set_mvcost(x, ref, ref_mv_idx); const int_mv curr_ref_mv = av1_get_ref_mv(x, ref); - *rate_mv += av1_mv_bit_cost(&cur_mv[ref].as_mv, &curr_ref_mv.as_mv, - x->nmvjointcost, x->mvcost, MV_COST_WEIGHT); + *rate_mv += + av1_mv_bit_cost(&cur_mv[ref].as_mv, &curr_ref_mv.as_mv, x->nmv_vec_cost, + x->mv_cost_stack, MV_COST_WEIGHT); } } @@ -6679,7 +6878,7 @@ static void estimate_ref_frame_costs( ref_costs_single[BWDREF_FRAME] += x->single_ref_cost[ctx_p6][5][0]; ref_costs_single[ALTREF2_FRAME] += x->single_ref_cost[ctx_p6][5][1]; - if (cm->reference_mode != SINGLE_REFERENCE) { + if (cm->current_frame.reference_mode != SINGLE_REFERENCE) { // Similar to single ref, determine cost of compound ref frames. // cost_compound_refs = cost_first_ref + cost_second_ref const int bwdref_comp_ctx_p = av1_get_pred_context_comp_bwdref_p(xd); @@ -6790,18 +6989,25 @@ static void setup_buffer_ref_mvs_inter( struct buf_2d yv12_mb[REF_FRAMES][MAX_MB_PLANE]) { const AV1_COMMON *cm = &cpi->common; const int num_planes = av1_num_planes(cm); - const YV12_BUFFER_CONFIG *yv12 = get_ref_frame_buffer(cpi, ref_frame); + const YV12_BUFFER_CONFIG *scaled_ref_frame = + av1_get_scaled_ref_frame(cpi, ref_frame); MACROBLOCKD *const xd = &x->e_mbd; MB_MODE_INFO *const mbmi = xd->mi[0]; - const struct scale_factors *const sf = &cm->frame_refs[ref_frame - 1].sf; MB_MODE_INFO_EXT *const mbmi_ext = x->mbmi_ext; - + const struct scale_factors *const sf = + get_ref_scale_factors_const(cm, ref_frame); + const YV12_BUFFER_CONFIG *yv12 = get_ref_frame_yv12_buf(cm, ref_frame); assert(yv12 != NULL); - // TODO(jkoleszar): Is the UV buffer ever used here? If so, need to make this - // use the UV scaling factors. - av1_setup_pred_block(xd, yv12_mb[ref_frame], yv12, mi_row, mi_col, sf, sf, - num_planes); + if (scaled_ref_frame) { + // Setup pred block based on scaled reference, because av1_mv_pred() doesn't + // support scaling. + av1_setup_pred_block(xd, yv12_mb[ref_frame], scaled_ref_frame, mi_row, + mi_col, NULL, NULL, num_planes); + } else { + av1_setup_pred_block(xd, yv12_mb[ref_frame], yv12, mi_row, mi_col, sf, sf, + num_planes); + } // Gets an initial list of candidate vectors from neighbours and orders them av1_find_mv_refs(cm, xd, mbmi, ref_frame, mbmi_ext->ref_mv_count, @@ -6809,11 +7015,18 @@ static void setup_buffer_ref_mvs_inter( mi_col, mbmi_ext->mode_context); // Further refinement that is encode side only to test the top few candidates - // in full and choose the best as the centre point for subsequent searches. + // in full and choose the best as the center point for subsequent searches. // The current implementation doesn't support scaling. - (void)block_size; - av1_mv_pred(cpi, x, yv12_mb[ref_frame][0].buf, yv12->y_stride, ref_frame, - block_size); + av1_mv_pred(cpi, x, yv12_mb[ref_frame][0].buf, yv12_mb[ref_frame][0].stride, + ref_frame, block_size); + + // Go back to unscaled reference. + if (scaled_ref_frame) { + // We had temporarily setup pred block based on scaled reference above. Go + // back to unscaled reference now, for subsequent use. + av1_setup_pred_block(xd, yv12_mb[ref_frame], yv12, mi_row, mi_col, sf, sf, + num_planes); + } } static void single_motion_search(const AV1_COMP *const cpi, MACROBLOCK *x, @@ -6849,10 +7062,6 @@ static void single_motion_search(const AV1_COMP *const cpi, MACROBLOCK *x, num_planes); } - av1_set_mvcost( - x, ref_idx, - mbmi->ref_mv_idx + (have_nearmv_in_inter_mode(mbmi->mode) ? 1 : 0)); - // Work out the size of the first step in the mv step search. // 0 here is maximum length first step. 1 is AOMMAX >> 1 etc. if (cpi->sf.mv.auto_mv_step_size && cm->show_frame) { @@ -6958,13 +7167,12 @@ static void single_motion_search(const AV1_COMP *const cpi, MACROBLOCK *x, x->second_best_mv.as_int != x->best_mv.as_int; const int pw = block_size_wide[bsize]; const int ph = block_size_high[bsize]; - best_mv_var = cpi->find_fractional_mv_step( x, cm, mi_row, mi_col, &ref_mv, cm->allow_high_precision_mv, x->errorperbit, &cpi->fn_ptr[bsize], cpi->sf.mv.subpel_force_stop, cpi->sf.mv.subpel_iters_per_step, cond_cost_list(cpi, cost_list), - x->nmvjointcost, x->mvcost, &dis, &x->pred_sse[ref], NULL, NULL, - 0, 0, pw, ph, 1); + x->nmv_vec_cost, x->mv_cost_stack, &dis, &x->pred_sse[ref], NULL, + NULL, 0, 0, pw, ph, cpi->sf.use_accurate_subpel_search, 1); if (try_second) { const int minc = @@ -6988,8 +7196,9 @@ static void single_motion_search(const AV1_COMP *const cpi, MACROBLOCK *x, x->errorperbit, &cpi->fn_ptr[bsize], cpi->sf.mv.subpel_force_stop, cpi->sf.mv.subpel_iters_per_step, - cond_cost_list(cpi, cost_list), x->nmvjointcost, x->mvcost, - &dis, &x->pred_sse[ref], NULL, NULL, 0, 0, pw, ph, 1); + cond_cost_list(cpi, cost_list), x->nmv_vec_cost, + x->mv_cost_stack, &dis, &x->pred_sse[ref], NULL, NULL, 0, 0, + pw, ph, cpi->sf.use_accurate_subpel_search, 0); if (this_var < best_mv_var) best_mv = x->best_mv.as_mv; x->best_mv.as_mv = best_mv; } @@ -6999,8 +7208,8 @@ static void single_motion_search(const AV1_COMP *const cpi, MACROBLOCK *x, x, cm, mi_row, mi_col, &ref_mv, cm->allow_high_precision_mv, x->errorperbit, &cpi->fn_ptr[bsize], cpi->sf.mv.subpel_force_stop, cpi->sf.mv.subpel_iters_per_step, cond_cost_list(cpi, cost_list), - x->nmvjointcost, x->mvcost, &dis, &x->pred_sse[ref], NULL, NULL, - 0, 0, 0, 0, 0); + x->nmv_vec_cost, x->mv_cost_stack, &dis, &x->pred_sse[ref], NULL, + NULL, 0, 0, 0, 0, 0, 1); } break; case OBMC_CAUSAL: @@ -7008,23 +7217,22 @@ static void single_motion_search(const AV1_COMP *const cpi, MACROBLOCK *x, x, cm, mi_row, mi_col, &x->best_mv.as_mv, &ref_mv, cm->allow_high_precision_mv, x->errorperbit, &cpi->fn_ptr[bsize], cpi->sf.mv.subpel_force_stop, cpi->sf.mv.subpel_iters_per_step, - x->nmvjointcost, x->mvcost, &dis, &x->pred_sse[ref], 0, + x->nmv_vec_cost, x->mv_cost_stack, &dis, &x->pred_sse[ref], 0, cpi->sf.use_accurate_subpel_search); break; default: assert(0 && "Invalid motion mode!\n"); } } - *rate_mv = av1_mv_bit_cost(&x->best_mv.as_mv, &ref_mv, x->nmvjointcost, - x->mvcost, MV_COST_WEIGHT); + *rate_mv = av1_mv_bit_cost(&x->best_mv.as_mv, &ref_mv, x->nmv_vec_cost, + x->mv_cost_stack, MV_COST_WEIGHT); if (cpi->sf.adaptive_motion_search && mbmi->motion_mode == SIMPLE_TRANSLATION) x->pred_mv[ref] = x->best_mv.as_mv; } -static INLINE void restore_dst_buf(MACROBLOCKD *xd, BUFFER_SET dst, +static INLINE void restore_dst_buf(MACROBLOCKD *xd, const BUFFER_SET dst, const int num_planes) { - int i; - for (i = 0; i < num_planes; i++) { + for (int i = 0; i < num_planes; i++) { xd->plane[i].dst.buf = dst.plane[i]; xd->plane[i].dst.stride = dst.stride[i]; } @@ -7071,9 +7279,9 @@ static void build_second_inter_pred(const AV1_COMP *cpi, MACROBLOCK *x, plane, !ref_idx, MV_PRECISION_Q3, mi_col * MI_SIZE, mi_row * MI_SIZE, xd, cm->allow_warped_motion); - av1_jnt_comp_weight_assign(cm, mbmi, 0, &xd->jcp_param.fwd_offset, - &xd->jcp_param.bck_offset, - &xd->jcp_param.use_jnt_comp_avg, 1); + av1_dist_wtd_comp_weight_assign(cm, mbmi, 0, &xd->jcp_param.fwd_offset, + &xd->jcp_param.bck_offset, + &xd->jcp_param.use_dist_wtd_comp_avg, 1); } // Search for the best mv for one component of a compound, @@ -7135,10 +7343,6 @@ static void compound_single_motion_search(const AV1_COMP *cpi, MACROBLOCK *x, best_mv->col >>= 3; best_mv->row >>= 3; - av1_set_mvcost( - x, ref_idx, - mbmi->ref_mv_idx + (have_nearmv_in_inter_mode(mbmi->mode) ? 1 : 0)); - // Small-range full-pixel motion search. bestsme = av1_refining_search_8p_c(x, sadpb, search_range, &cpi->fn_ptr[bsize], mask, mask_stride, @@ -7175,8 +7379,8 @@ static void compound_single_motion_search(const AV1_COMP *cpi, MACROBLOCK *x, x, cm, mi_row, mi_col, &ref_mv.as_mv, cpi->common.allow_high_precision_mv, x->errorperbit, &cpi->fn_ptr[bsize], 0, cpi->sf.mv.subpel_iters_per_step, NULL, - x->nmvjointcost, x->mvcost, &dis, &sse, second_pred, mask, mask_stride, - ref_idx, pw, ph, cpi->sf.use_accurate_subpel_search); + x->nmv_vec_cost, x->mv_cost_stack, &dis, &sse, second_pred, mask, + mask_stride, ref_idx, pw, ph, cpi->sf.use_accurate_subpel_search, 1); } // Restore the pointer to the first unscaled prediction buffer. @@ -7186,11 +7390,8 @@ static void compound_single_motion_search(const AV1_COMP *cpi, MACROBLOCK *x, *rate_mv = 0; - av1_set_mvcost( - x, ref_idx, - mbmi->ref_mv_idx + (have_nearmv_in_inter_mode(mbmi->mode) ? 1 : 0)); - *rate_mv += av1_mv_bit_cost(this_mv, &ref_mv.as_mv, x->nmvjointcost, - x->mvcost, MV_COST_WEIGHT); + *rate_mv += av1_mv_bit_cost(this_mv, &ref_mv.as_mv, x->nmv_vec_cost, + x->mv_cost_stack, MV_COST_WEIGHT); } // Wrapper for compound_single_motion_search, for the common case @@ -7206,7 +7407,7 @@ static void compound_single_motion_search_interinter( // Prediction buffer from second frame. DECLARE_ALIGNED(16, uint16_t, second_pred_alloc_16[MAX_SB_SQUARE]); uint8_t *second_pred; - if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) + if (is_cur_buf_hbd(xd)) second_pred = CONVERT_TO_BYTEPTR(second_pred_alloc_16); else second_pred = (uint8_t *)second_pred_alloc_16; @@ -7336,7 +7537,7 @@ static int estimate_wedge_sign(const AV1_COMP *cpi, const MACROBLOCK *x, const BLOCK_SIZE f_index = split_qtr[bsize]; assert(f_index != BLOCK_INVALID); - if (x->e_mbd.cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + if (is_cur_buf_hbd(&x->e_mbd)) { pred0 = CONVERT_TO_BYTEPTR(pred0); pred1 = CONVERT_TO_BYTEPTR(pred1); } @@ -7386,7 +7587,7 @@ static int64_t pick_wedge(const AV1_COMP *const cpi, const MACROBLOCK *const x, int wedge_types = (1 << get_wedge_bits_lookup(bsize)); const uint8_t *mask; uint64_t sse; - const int hbd = xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH; + const int hbd = is_cur_buf_hbd(xd); const int bd_round = hbd ? (xd->bd - 8) * 2 : 0; DECLARE_ALIGNED(32, int16_t, residual0[MAX_SB_SQUARE]); // src - pred0 @@ -7457,7 +7658,7 @@ static int64_t pick_wedge_fixed_sign(const AV1_COMP *const cpi, int wedge_types = (1 << get_wedge_bits_lookup(bsize)); const uint8_t *mask; uint64_t sse; - const int hbd = xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH; + const int hbd = is_cur_buf_hbd(xd); const int bd_round = hbd ? (xd->bd - 8) * 2 : 0; for (wedge_index = 0; wedge_index < wedge_types; ++wedge_index) { mask = av1_get_contiguous_soft_mask(wedge_index, wedge_sign, bsize); @@ -7523,7 +7724,7 @@ static int64_t pick_interinter_seg(const AV1_COMP *const cpi, DIFFWTD_MASK_TYPE cur_mask_type; int64_t best_rd = INT64_MAX; DIFFWTD_MASK_TYPE best_mask_type = 0; - const int hbd = xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH; + const int hbd = is_cur_buf_hbd(xd); const int bd_round = hbd ? (xd->bd - 8) * 2 : 0; DECLARE_ALIGNED(16, uint8_t, seg_mask[2 * MAX_SB_SQUARE]); uint8_t *tmp_mask[2] = { xd->seg_mask, seg_mask }; @@ -7574,7 +7775,7 @@ static int64_t pick_interintra_wedge(const AV1_COMP *const cpi, const int bh = block_size_high[bsize]; DECLARE_ALIGNED(32, int16_t, residual1[MAX_SB_SQUARE]); // src - pred1 DECLARE_ALIGNED(32, int16_t, diff10[MAX_SB_SQUARE]); // pred1 - pred0 - if (get_bitdepth_data_path_index(xd)) { + if (is_cur_buf_hbd(xd)) { aom_highbd_subtract_block(bh, bw, residual1, bw, src->buf, src->stride, CONVERT_TO_BYTEPTR(p1), bw, xd->bd); aom_highbd_subtract_block(bh, bw, diff10, bw, CONVERT_TO_BYTEPTR(p1), bw, @@ -7653,7 +7854,7 @@ static void get_inter_predictors_masked_compound( av1_build_inter_predictors_for_planes_single_buf( xd, bsize, 0, 0, mi_row, mi_col, 1, preds1, strides, can_use_previous); const struct buf_2d *const src = &x->plane[0].src; - if (get_bitdepth_data_path_index(xd)) { + if (is_cur_buf_hbd(xd)) { aom_highbd_subtract_block(bh, bw, residual1, bw, src->buf, src->stride, CONVERT_TO_BYTEPTR(*preds1), bw, xd->bd); aom_highbd_subtract_block(bh, bw, diff10, bw, CONVERT_TO_BYTEPTR(*preds1), @@ -7668,26 +7869,38 @@ static void get_inter_predictors_masked_compound( static int64_t build_and_cost_compound_type( const AV1_COMP *const cpi, MACROBLOCK *x, const int_mv *const cur_mv, const BLOCK_SIZE bsize, const PREDICTION_MODE this_mode, int *rs2, - int rate_mv, BUFFER_SET *ctx, int *out_rate_mv, uint8_t **preds0, + int rate_mv, const BUFFER_SET *ctx, int *out_rate_mv, uint8_t **preds0, uint8_t **preds1, int16_t *residual1, int16_t *diff10, int *strides, int mi_row, int mi_col, int mode_rate, int64_t ref_best_rd, - int *calc_pred_masked_compound) { + int *calc_pred_masked_compound, int32_t *comp_rate, int64_t *comp_dist) { const AV1_COMMON *const cm = &cpi->common; MACROBLOCKD *xd = &x->e_mbd; MB_MODE_INFO *const mbmi = xd->mi[0]; - int rate_sum; - int64_t dist_sum; int64_t best_rd_cur = INT64_MAX; int64_t rd = INT64_MAX; - int tmp_skip_txfm_sb; - int64_t tmp_skip_sse_sb; const COMPOUND_TYPE compound_type = mbmi->interinter_comp.type; + // TODO(any): Save pred and mask calculation as well into records. However + // this may increase memory requirements as compound segment mask needs to be + // stored in each record. if (*calc_pred_masked_compound) { get_inter_predictors_masked_compound(cpi, x, bsize, mi_row, mi_col, preds0, preds1, residual1, diff10, strides); *calc_pred_masked_compound = 0; } + if (cpi->sf.prune_wedge_pred_diff_based && compound_type == COMPOUND_WEDGE) { + unsigned int sse; + if (is_cur_buf_hbd(xd)) + (void)cpi->fn_ptr[bsize].vf(CONVERT_TO_BYTEPTR(*preds0), *strides, + CONVERT_TO_BYTEPTR(*preds1), *strides, &sse); + else + (void)cpi->fn_ptr[bsize].vf(*preds0, *strides, *preds1, *strides, &sse); + const unsigned int mse = + ROUND_POWER_OF_TWO(sse, num_pels_log2_lookup[bsize]); + // If two predictors are very similar, skip wedge compound mode search + if (mse < 8 || (!have_newmv_in_inter_mode(this_mode) && mse < 64)) + return INT64_MAX; + } best_rd_cur = pick_interinter_mask(cpi, x, bsize, *preds0, *preds1, residual1, diff10); @@ -7700,37 +7913,53 @@ static int64_t build_and_cost_compound_type( int64_t mode_rd = RDCOST(x->rdmult, *rs2 + mode_rate, 0); if (mode_rd > ref_best_rd) return INT64_MAX; - if (have_newmv_in_inter_mode(this_mode) && compound_type == COMPOUND_WEDGE) { - *out_rate_mv = interinter_compound_motion_search(cpi, x, cur_mv, bsize, - this_mode, mi_row, mi_col); - av1_build_inter_predictors_sby(cm, xd, mi_row, mi_col, ctx, bsize); - model_rd_sb_fn[MODELRD_TYPE_MASKED_COMPOUND]( - cpi, bsize, x, xd, 0, 0, mi_row, mi_col, &rate_sum, &dist_sum, - &tmp_skip_txfm_sb, &tmp_skip_sse_sb, NULL, NULL, NULL); - rd = RDCOST(x->rdmult, *rs2 + *out_rate_mv + rate_sum, dist_sum); - if (rd >= best_rd_cur) { - mbmi->mv[0].as_int = cur_mv[0].as_int; - mbmi->mv[1].as_int = cur_mv[1].as_int; + // Reuse data if matching record is found + if (comp_rate[compound_type] == INT_MAX) { + if (have_newmv_in_inter_mode(this_mode) && + compound_type == COMPOUND_WEDGE && + !cpi->sf.disable_interinter_wedge_newmv_search) { + *out_rate_mv = interinter_compound_motion_search( + cpi, x, cur_mv, bsize, this_mode, mi_row, mi_col); + av1_enc_build_inter_predictor(cm, xd, mi_row, mi_col, ctx, bsize, + AOM_PLANE_Y, AOM_PLANE_Y); + int rate_sum, tmp_skip_txfm_sb; + int64_t dist_sum, tmp_skip_sse_sb; + model_rd_sb_fn[MODELRD_TYPE_MASKED_COMPOUND]( + cpi, bsize, x, xd, 0, 0, mi_row, mi_col, &rate_sum, &dist_sum, + &tmp_skip_txfm_sb, &tmp_skip_sse_sb, NULL, NULL, NULL); + rd = RDCOST(x->rdmult, *rs2 + *out_rate_mv + rate_sum, dist_sum); + if (rd >= best_rd_cur) { + mbmi->mv[0].as_int = cur_mv[0].as_int; + mbmi->mv[1].as_int = cur_mv[1].as_int; + *out_rate_mv = rate_mv; + av1_build_wedge_inter_predictor_from_buf(xd, bsize, 0, 0, preds0, + strides, preds1, strides); + } + } else { *out_rate_mv = rate_mv; av1_build_wedge_inter_predictor_from_buf(xd, bsize, 0, 0, preds0, strides, preds1, strides); } - rd = estimate_yrd_for_sb(cpi, bsize, x, &rate_sum, &dist_sum, - &tmp_skip_txfm_sb, &tmp_skip_sse_sb, INT64_MAX); - if (rd != INT64_MAX) - rd = RDCOST(x->rdmult, *rs2 + *out_rate_mv + rate_sum, dist_sum); - best_rd_cur = rd; + RD_STATS rd_stats; + rd = estimate_yrd_for_sb(cpi, bsize, x, INT64_MAX, &rd_stats); + if (rd != INT64_MAX) { + rd = + RDCOST(x->rdmult, *rs2 + *out_rate_mv + rd_stats.rate, rd_stats.dist); + // Backup rate and distortion for future reuse + comp_rate[compound_type] = rd_stats.rate; + comp_dist[compound_type] = rd_stats.dist; + } } else { - av1_build_wedge_inter_predictor_from_buf(xd, bsize, 0, 0, preds0, strides, - preds1, strides); - rd = estimate_yrd_for_sb(cpi, bsize, x, &rate_sum, &dist_sum, - &tmp_skip_txfm_sb, &tmp_skip_sse_sb, INT64_MAX); - if (rd != INT64_MAX) - rd = RDCOST(x->rdmult, *rs2 + rate_mv + rate_sum, dist_sum); - best_rd_cur = rd; + assert(comp_dist[compound_type] != INT64_MAX); + // Rate is populated assuming no refinement is done=> not NEWMV case + assert(!have_newmv_in_inter_mode(this_mode)); + *out_rate_mv = rate_mv; + // Calculate RD cost based on stored stats + rd = RDCOST(x->rdmult, *rs2 + *out_rate_mv + comp_rate[compound_type], + comp_dist[compound_type]); } - return best_rd_cur; + return rd; } typedef struct { @@ -7753,6 +7982,8 @@ typedef struct { int64_t (*simple_rd)[MAX_REF_MV_SERCH][REF_FRAMES]; int skip_motion_mode; INTERINTRA_MODE *inter_intra_mode; + int single_ref_first_pass; + SimpleRDState *simple_rd_state; } HandleInterModeArgs; /* If the current mode shares the same mv with other modes with higher cost, @@ -7851,10 +8082,9 @@ static int64_t handle_newmv(const AV1_COMP *const cpi, MACROBLOCK *const x, *rate_mv = 0; for (i = 0; i < 2; ++i) { const int_mv ref_mv = av1_get_ref_mv(x, i); - av1_set_mvcost(x, i, mbmi->ref_mv_idx); *rate_mv += - av1_mv_bit_cost(&cur_mv[i].as_mv, &ref_mv.as_mv, x->nmvjointcost, - x->mvcost, MV_COST_WEIGHT); + av1_mv_bit_cost(&cur_mv[i].as_mv, &ref_mv.as_mv, x->nmv_vec_cost, + x->mv_cost_stack, MV_COST_WEIGHT); } } } else if (this_mode == NEAREST_NEWMV || this_mode == NEAR_NEWMV) { @@ -7863,11 +8093,10 @@ static int64_t handle_newmv(const AV1_COMP *const cpi, MACROBLOCK *const x, compound_single_motion_search_interinter( cpi, x, bsize, cur_mv, mi_row, mi_col, NULL, 0, rate_mv, 0, 1); } else { - av1_set_mvcost(x, 1, - mbmi->ref_mv_idx + (this_mode == NEAR_NEWMV ? 1 : 0)); const int_mv ref_mv = av1_get_ref_mv(x, 1); - *rate_mv = av1_mv_bit_cost(&cur_mv[1].as_mv, &ref_mv.as_mv, - x->nmvjointcost, x->mvcost, MV_COST_WEIGHT); + *rate_mv = + av1_mv_bit_cost(&cur_mv[1].as_mv, &ref_mv.as_mv, x->nmv_vec_cost, + x->mv_cost_stack, MV_COST_WEIGHT); } } else { assert(this_mode == NEW_NEARESTMV || this_mode == NEW_NEARMV); @@ -7877,10 +8106,9 @@ static int64_t handle_newmv(const AV1_COMP *const cpi, MACROBLOCK *const x, cpi, x, bsize, cur_mv, mi_row, mi_col, NULL, 0, rate_mv, 0, 0); } else { const int_mv ref_mv = av1_get_ref_mv(x, 0); - av1_set_mvcost(x, 0, - mbmi->ref_mv_idx + (this_mode == NEW_NEARMV ? 1 : 0)); - *rate_mv = av1_mv_bit_cost(&cur_mv[0].as_mv, &ref_mv.as_mv, - x->nmvjointcost, x->mvcost, MV_COST_WEIGHT); + *rate_mv = + av1_mv_bit_cost(&cur_mv[0].as_mv, &ref_mv.as_mv, x->nmv_vec_cost, + x->mv_cost_stack, MV_COST_WEIGHT); } } } else { @@ -7928,8 +8156,9 @@ static INLINE int get_switchable_rate(MACROBLOCK *const x, // calculate the rdcost of given interpolation_filter static INLINE int64_t interpolation_filter_rd( - MACROBLOCK *const x, const AV1_COMP *const cpi, BLOCK_SIZE bsize, - int mi_row, int mi_col, BUFFER_SET *const orig_dst, int64_t *const rd, + MACROBLOCK *const x, const AV1_COMP *const cpi, + const TileDataEnc *tile_data, BLOCK_SIZE bsize, int mi_row, int mi_col, + const BUFFER_SET *const orig_dst, int64_t *const rd, int *const switchable_rate, int *const skip_txfm_sb, int64_t *const skip_sse_sb, const BUFFER_SET *dst_bufs[2], int filter_idx, const int switchable_ctx[2], const int skip_pred, int *rate, @@ -7946,6 +8175,14 @@ static INLINE int64_t interpolation_filter_rd( const int tmp_rs = get_switchable_rate(x, mbmi->interp_filters, switchable_ctx); + int64_t min_rd = RDCOST(x->rdmult, tmp_rs, 0); + if (min_rd > *rd) { + mbmi->interp_filters = last_best; + return 0; + } + + (void)tile_data; + assert(skip_pred != 2); assert((skip_pred >= 0) && (skip_pred <= cpi->default_interp_skip_flags)); assert(rate[0] >= 0); @@ -7959,11 +8196,13 @@ static INLINE int64_t interpolation_filter_rd( if (skip_pred != cpi->default_interp_skip_flags) { if (skip_pred != DEFAULT_LUMA_INTERP_SKIP_FLAG) { - av1_build_inter_predictors_sby(cm, xd, mi_row, mi_col, orig_dst, bsize); + av1_enc_build_inter_predictor(cm, xd, mi_row, mi_col, orig_dst, bsize, + AOM_PLANE_Y, AOM_PLANE_Y); #if CONFIG_COLLECT_RD_STATS == 3 RD_STATS rd_stats_y; - select_tx_type_yrd(cpi, x, &rd_stats_y, bsize, mi_row, mi_col, INT64_MAX); - PrintPredictionUnitStats(cpi, x, &rd_stats_y, bsize); + pick_tx_size_type_yrd(cpi, x, &rd_stats_y, bsize, mi_row, mi_col, + INT64_MAX); + PrintPredictionUnitStats(cpi, tile_data, x, &rd_stats_y, bsize); #endif // CONFIG_COLLECT_RD_STATS == 3 model_rd_sb_fn[MODELRD_TYPE_INTERP_FILTER]( cpi, bsize, x, xd, 0, 0, mi_row, mi_col, &tmp_rate[0], &tmp_dist[0], @@ -7984,8 +8223,8 @@ static INLINE int64_t interpolation_filter_rd( mbmi->interp_filters = last_best; return 0; } - av1_build_inter_predictors_sbp(cm, xd, mi_row, mi_col, orig_dst, bsize, - plane); + av1_enc_build_inter_predictor(cm, xd, mi_row, mi_col, orig_dst, bsize, + plane, plane); model_rd_sb_fn[MODELRD_TYPE_INTERP_FILTER]( cpi, bsize, x, xd, plane, plane, mi_row, mi_col, &tmp_rate_uv, &tmp_dist_uv, &tmp_skip_sb_uv, &tmp_skip_sse_uv, NULL, NULL, NULL); @@ -8037,73 +8276,119 @@ static INLINE int64_t interpolation_filter_rd( return 0; } -// Find the best rd filter in horizontal direction -static INLINE int find_best_horiz_interp_filter_rd( - MACROBLOCK *const x, const AV1_COMP *const cpi, BLOCK_SIZE bsize, - int mi_row, int mi_col, BUFFER_SET *const orig_dst, int64_t *const rd, +static INLINE void pred_dual_interp_filter_rd( + MACROBLOCK *const x, const AV1_COMP *const cpi, + const TileDataEnc *tile_data, BLOCK_SIZE bsize, int mi_row, int mi_col, + const BUFFER_SET *const orig_dst, int64_t *const rd, int *const switchable_rate, int *const skip_txfm_sb, int64_t *const skip_sse_sb, const BUFFER_SET *dst_bufs[2], - const int switchable_ctx[2], const int skip_hor, int *rate, int64_t *dist, - int best_dual_mode) { - int i; - const int bw = block_size_wide[bsize]; - assert(best_dual_mode == 0); - if ((bw <= 4) && (skip_hor != cpi->default_interp_skip_flags)) { - int skip_pred = cpi->default_interp_skip_flags; - // Process the filters in reverse order to enable reusing rate and - // distortion (calcuated during EIGHTTAP_REGULAR) for MULTITAP_SHARP - for (i = (SWITCHABLE_FILTERS - 1); i >= 1; --i) { - if (interpolation_filter_rd(x, cpi, bsize, mi_row, mi_col, orig_dst, rd, - switchable_rate, skip_txfm_sb, skip_sse_sb, - dst_bufs, i, switchable_ctx, skip_pred, rate, - dist)) { - best_dual_mode = i; + InterpFilters filter_idx, const int switchable_ctx[2], const int skip_pred, + int *rate, int64_t *dist, InterpFilters af_horiz, InterpFilters af_vert, + InterpFilters lf_horiz, InterpFilters lf_vert) { + if ((af_horiz == lf_horiz) && (af_horiz != SWITCHABLE)) { + if (((af_vert == lf_vert) && (af_vert != SWITCHABLE))) { + filter_idx = af_horiz + (af_vert * SWITCHABLE_FILTERS); + if (filter_idx) { + interpolation_filter_rd(x, cpi, tile_data, bsize, mi_row, mi_col, + orig_dst, rd, switchable_rate, skip_txfm_sb, + skip_sse_sb, dst_bufs, filter_idx, + switchable_ctx, skip_pred, rate, dist); + } + } else { + for (filter_idx = af_horiz; filter_idx < (DUAL_FILTER_SET_SIZE); + filter_idx += SWITCHABLE_FILTERS) { + if (filter_idx) { + interpolation_filter_rd(x, cpi, tile_data, bsize, mi_row, mi_col, + orig_dst, rd, switchable_rate, skip_txfm_sb, + skip_sse_sb, dst_bufs, filter_idx, + switchable_ctx, skip_pred, rate, dist); + } } - skip_pred = skip_hor; } - } else { - for (i = 1; i < SWITCHABLE_FILTERS; ++i) { - if (interpolation_filter_rd(x, cpi, bsize, mi_row, mi_col, orig_dst, rd, - switchable_rate, skip_txfm_sb, skip_sse_sb, - dst_bufs, i, switchable_ctx, skip_hor, rate, - dist)) { - best_dual_mode = i; + } else if ((af_vert == lf_vert) && (af_vert != SWITCHABLE)) { + for (filter_idx = (af_vert * SWITCHABLE_FILTERS); + filter_idx <= ((af_vert * SWITCHABLE_FILTERS) + 2); filter_idx += 1) { + if (filter_idx) { + interpolation_filter_rd(x, cpi, tile_data, bsize, mi_row, mi_col, + orig_dst, rd, switchable_rate, skip_txfm_sb, + skip_sse_sb, dst_bufs, filter_idx, + switchable_ctx, skip_pred, rate, dist); } } } - return best_dual_mode; } -// Find the best rd filter in vertical direction -static INLINE void find_best_vert_interp_filter_rd( - MACROBLOCK *const x, const AV1_COMP *const cpi, BLOCK_SIZE bsize, - int mi_row, int mi_col, BUFFER_SET *const orig_dst, int64_t *const rd, +// Find the best interp filter if dual_interp_filter = 0 +static INLINE void find_best_non_dual_interp_filter( + MACROBLOCK *const x, const AV1_COMP *const cpi, + const TileDataEnc *tile_data, BLOCK_SIZE bsize, int mi_row, int mi_col, + const BUFFER_SET *const orig_dst, int64_t *const rd, int *const switchable_rate, int *const skip_txfm_sb, int64_t *const skip_sse_sb, const BUFFER_SET *dst_bufs[2], - const int switchable_ctx[2], const int skip_ver, int *rate, int64_t *dist, - int best_dual_mode, int filter_set_size) { - int i; - const int bh = block_size_high[bsize]; - if ((bh <= 4) && (skip_ver != cpi->default_interp_skip_flags)) { + const int switchable_ctx[2], const int skip_ver, const int skip_hor, + int *rate, int64_t *dist, int filter_set_size) { + int16_t i; + + // Regular filter evaluation should have been done and hence the same should + // be the winner + assert(x->e_mbd.mi[0]->interp_filters == filter_sets[0]); + assert(filter_set_size == DUAL_FILTER_SET_SIZE); + + // Reuse regular filter's modeled rd data for sharp filter for following + // cases + // 1) When bsize is 4x4 + // 2) When block width is 4 (i.e. 4x8/4x16 blocks) and MV in vertical + // direction is full-pel + // 3) When block height is 4 (i.e. 8x4/16x4 blocks) and MV in horizontal + // direction is full-pel + // TODO(any): Optimize cases 2 and 3 further if luma MV in relavant direction + // alone is full-pel + + if ((bsize == BLOCK_4X4) || + (block_size_wide[bsize] == 4 && + skip_ver == cpi->default_interp_skip_flags) || + (block_size_high[bsize] == 4 && + skip_hor == cpi->default_interp_skip_flags)) { int skip_pred = cpi->default_interp_skip_flags; - // Process the filters in reverse order to enable reusing rate and - // distortion (calcuated during EIGHTTAP_REGULAR) for MULTITAP_SHARP - assert(filter_set_size == DUAL_FILTER_SET_SIZE); - for (i = (filter_set_size - SWITCHABLE_FILTERS + best_dual_mode); - i >= (best_dual_mode + SWITCHABLE_FILTERS); i -= SWITCHABLE_FILTERS) { - interpolation_filter_rd(x, cpi, bsize, mi_row, mi_col, orig_dst, rd, - switchable_rate, skip_txfm_sb, skip_sse_sb, - dst_bufs, i, switchable_ctx, skip_pred, rate, - dist); - skip_pred = skip_ver; + for (i = filter_set_size - 1; i > 0; i -= (SWITCHABLE_FILTERS + 1)) { + // This assert tells that (filter_x == filter_y) for non-dual filter case + assert((filter_sets[i] & 0xffff) == (filter_sets[i] >> 16)); + if (cpi->sf.adaptive_interp_filter_search && + (cpi->sf.interp_filter_search_mask & (1 << (i >> 2)))) { + continue; + } + interpolation_filter_rd(x, cpi, tile_data, bsize, mi_row, mi_col, + orig_dst, rd, switchable_rate, skip_txfm_sb, + skip_sse_sb, dst_bufs, i, switchable_ctx, + skip_pred, rate, dist); + skip_pred = (skip_hor & skip_ver); } } else { - for (i = best_dual_mode + SWITCHABLE_FILTERS; i < filter_set_size; - i += SWITCHABLE_FILTERS) { - interpolation_filter_rd(x, cpi, bsize, mi_row, mi_col, orig_dst, rd, - switchable_rate, skip_txfm_sb, skip_sse_sb, - dst_bufs, i, switchable_ctx, skip_ver, rate, - dist); + int skip_pred = (skip_hor & skip_ver); + for (i = (SWITCHABLE_FILTERS + 1); i < filter_set_size; + i += (SWITCHABLE_FILTERS + 1)) { + // This assert tells that (filter_x == filter_y) for non-dual filter case + assert((filter_sets[i] & 0xffff) == (filter_sets[i] >> 16)); + if (cpi->sf.adaptive_interp_filter_search && + (cpi->sf.interp_filter_search_mask & (1 << (i >> 2)))) { + continue; + } + interpolation_filter_rd(x, cpi, tile_data, bsize, mi_row, mi_col, + orig_dst, rd, switchable_rate, skip_txfm_sb, + skip_sse_sb, dst_bufs, i, switchable_ctx, + skip_pred, rate, dist); + // In first iteration, smooth filter is evaluated. If smooth filter + // (which is less sharper) is the winner among regular and smooth filters, + // sharp filter evaluation is skipped + // TODO(any): Refine this gating based on modelled rd only (i.e., by not + // accounting switchable filter rate) + if (cpi->sf.skip_sharp_interp_filter_search && + skip_pred != cpi->default_interp_skip_flags) { + MACROBLOCKD *const xd = &x->e_mbd; + MB_MODE_INFO *const mbmi = xd->mi[0]; + if (mbmi->interp_filters == filter_sets[(SWITCHABLE_FILTERS + 1)]) + break; + } } } } @@ -8121,6 +8406,45 @@ static INLINE int is_interp_filter_match(const INTERPOLATION_FILTER_STATS *st, return 1; } +// Checks if characteristics of search match +static INLINE int is_comp_rd_match(const MACROBLOCK *const x, + const COMP_RD_STATS *st, + const MB_MODE_INFO *const mi, + int32_t *comp_rate, int64_t *comp_dist) { + // TODO(ranjit): Ensure that compound type search use regular filter always + // and check if following check can be removed + // Check if interp filter matches with previous case + if (st->filter != mi->interp_filters) return 0; + + const MACROBLOCKD *const xd = &x->e_mbd; + // Match MV and reference indices + for (int i = 0; i < 2; ++i) { + if ((st->ref_frames[i] != mi->ref_frame[i]) || + (st->mv[i].as_int != mi->mv[i].as_int)) { + return 0; + } + const WarpedMotionParams *const wm = &xd->global_motion[mi->ref_frame[i]]; + if (is_global_mv_block(mi, wm->wmtype) != st->is_global[i]) return 0; + } + + // Store the stats for compound average + comp_rate[0] = st->rate[0]; + comp_dist[0] = st->dist[0]; + + // For compound wedge/segment, reuse data only if NEWMV is not present in + // either of the directions + if (!have_newmv_in_inter_mode(mi->mode) && + !have_newmv_in_inter_mode(st->mode)) { + memcpy(&comp_rate[1], &st->rate[1], sizeof(comp_rate[1]) * 2); + memcpy(&comp_dist[1], &st->dist[1], sizeof(comp_dist[1]) * 2); + } + + // TODO(ranjit) : Check if compound wedge/segment can reuse data for NEWMV + // case as well based on speed feature + + return 1; +} + static INLINE int find_interp_filter_in_stats(MACROBLOCK *x, MB_MODE_INFO *const mbmi) { const int comp_idx = mbmi->compound_idx; @@ -8134,6 +8458,19 @@ static INLINE int find_interp_filter_in_stats(MACROBLOCK *x, } return -1; // no match result found } +// Checks if similar compound type search case is accounted earlier +// If found, returns relevant rd data +static INLINE int find_comp_rd_in_stats(const MACROBLOCK *x, + const MB_MODE_INFO *const mbmi, + int32_t *comp_rate, + int64_t *comp_dist) { + for (int j = 0; j < x->comp_rd_stats_idx; ++j) { + if (is_comp_rd_match(x, &x->comp_rd_stats[j], mbmi, comp_rate, comp_dist)) { + return 1; + } + } + return 0; // no match result found +} static INLINE void save_interp_filter_search_stat(MACROBLOCK *x, MB_MODE_INFO *const mbmi) { @@ -8150,11 +8487,37 @@ static INLINE void save_interp_filter_search_stat(MACROBLOCK *x, } } +static INLINE void save_comp_rd_search_stat(MACROBLOCK *x, + const MB_MODE_INFO *const mbmi, + const int32_t *comp_rate, + const int64_t *comp_dist, + const int_mv *cur_mv) { + const int offset = x->comp_rd_stats_idx; + if (offset < MAX_COMP_RD_STATS) { + COMP_RD_STATS *const rd_stats = x->comp_rd_stats + offset; + memcpy(rd_stats->rate, comp_rate, sizeof(rd_stats->rate)); + memcpy(rd_stats->dist, comp_dist, sizeof(rd_stats->dist)); + memcpy(rd_stats->mv, cur_mv, sizeof(rd_stats->mv)); + memcpy(rd_stats->ref_frames, mbmi->ref_frame, sizeof(rd_stats->ref_frames)); + rd_stats->mode = mbmi->mode; + rd_stats->filter = mbmi->interp_filters; + rd_stats->ref_mv_idx = mbmi->ref_mv_idx; + const MACROBLOCKD *const xd = &x->e_mbd; + for (int i = 0; i < 2; ++i) { + const WarpedMotionParams *const wm = + &xd->global_motion[mbmi->ref_frame[i]]; + rd_stats->is_global[i] = is_global_mv_block(mbmi, wm->wmtype); + } + ++x->comp_rd_stats_idx; + } +} + static int64_t interpolation_filter_search( - MACROBLOCK *const x, const AV1_COMP *const cpi, BLOCK_SIZE bsize, - int mi_row, int mi_col, const BUFFER_SET *const tmp_dst, - BUFFER_SET *const orig_dst, InterpFilter (*const single_filter)[REF_FRAMES], - int64_t *const rd, int *const switchable_rate, int *const skip_txfm_sb, + MACROBLOCK *const x, const AV1_COMP *const cpi, + const TileDataEnc *tile_data, BLOCK_SIZE bsize, int mi_row, int mi_col, + const BUFFER_SET *const tmp_dst, const BUFFER_SET *const orig_dst, + InterpFilter (*const single_filter)[REF_FRAMES], int64_t *const rd, + int *const switchable_rate, int *const skip_txfm_sb, int64_t *const skip_sse_sb, const int skip_build_pred, HandleInterModeArgs *args, int64_t ref_best_rd) { const AV1_COMMON *cm = &cpi->common; @@ -8186,13 +8549,15 @@ static int64_t interpolation_filter_search( switchable_ctx[1] = av1_get_pred_context_switchable_interp(xd, 1); *switchable_rate = get_switchable_rate(x, mbmi->interp_filters, switchable_ctx); - if (!skip_build_pred) - av1_build_inter_predictors_sb(cm, xd, mi_row, mi_col, orig_dst, bsize); + if (!skip_build_pred) { + av1_enc_build_inter_predictor(cm, xd, mi_row, mi_col, orig_dst, bsize, 0, + av1_num_planes(cm) - 1); + } #if CONFIG_COLLECT_RD_STATS == 3 RD_STATS rd_stats_y; - select_tx_type_yrd(cpi, x, &rd_stats_y, bsize, mi_row, mi_col, INT64_MAX); - PrintPredictionUnitStats(cpi, x, &rd_stats_y, bsize); + pick_tx_size_type_yrd(cpi, x, &rd_stats_y, bsize, mi_row, mi_col, INT64_MAX); + PrintPredictionUnitStats(cpi, tile_data, x, &rd_stats_y, bsize); #endif // CONFIG_COLLECT_RD_STATS == 3 model_rd_sb_fn[MODELRD_TYPE_INTERP_FILTER]( cpi, bsize, x, xd, 0, 0, mi_row, mi_col, &tmp_rate[0], &tmp_dist[0], @@ -8248,8 +8613,8 @@ static int64_t interpolation_filter_search( const int is_compound = has_second_ref(mbmi); assert(is_intrabc_block(mbmi) == 0); for (int j = 0; j < 1 + is_compound; ++j) { - const RefBuffer *ref_buf = &cm->frame_refs[mbmi->ref_frame[j] - LAST_FRAME]; - const struct scale_factors *const sf = &ref_buf->sf; + const struct scale_factors *const sf = + get_ref_scale_factors_const(cm, mbmi->ref_frame[j]); // TODO(any): Refine skip flag calculation considering scaling if (av1_is_scaled(sf)) { skip_hor = 0; @@ -8296,30 +8661,75 @@ static int64_t interpolation_filter_search( // default to (R,R): EIGHTTAP_REGULARxEIGHTTAP_REGULAR int best_dual_mode = 0; // Find best of {R}x{R,Sm,Sh} - // EIGHTTAP_REGULAR mode is calculated beforehand - best_dual_mode = find_best_horiz_interp_filter_rd( - x, cpi, bsize, mi_row, mi_col, orig_dst, rd, switchable_rate, - best_skip_txfm_sb, best_skip_sse_sb, dst_bufs, switchable_ctx, skip_hor, - tmp_rate, tmp_dist, best_dual_mode); - - // From best of horizontal EIGHTTAP_REGULAR modes, check vertical modes - find_best_vert_interp_filter_rd( - x, cpi, bsize, mi_row, mi_col, orig_dst, rd, switchable_rate, + const int bw = block_size_wide[bsize]; + const int bh = block_size_high[bsize]; + int skip_pred; + int bsl, pred_filter_search; + InterpFilters af_horiz = SWITCHABLE, af_vert = SWITCHABLE, + lf_horiz = SWITCHABLE, lf_vert = SWITCHABLE, filter_idx = 0; + const MB_MODE_INFO *const above_mbmi = xd->above_mbmi; + const MB_MODE_INFO *const left_mbmi = xd->left_mbmi; + bsl = mi_size_wide_log2[bsize]; + pred_filter_search = + cpi->sf.cb_pred_filter_search + ? (((mi_row + mi_col) >> bsl) + + get_chessboard_index(cm->current_frame.frame_number)) & + 0x1 + : 0; + if (above_mbmi && is_inter_block(above_mbmi)) { + af_horiz = av1_extract_interp_filter(above_mbmi->interp_filters, 1); + af_vert = av1_extract_interp_filter(above_mbmi->interp_filters, 0); + } + if (left_mbmi && is_inter_block(left_mbmi)) { + lf_horiz = av1_extract_interp_filter(left_mbmi->interp_filters, 1); + lf_vert = av1_extract_interp_filter(left_mbmi->interp_filters, 0); + } + pred_filter_search &= !have_newmv_in_inter_mode(mbmi->mode); + pred_filter_search &= + ((af_horiz == lf_horiz) && (af_horiz != SWITCHABLE)) || + ((af_vert == lf_vert) && (af_vert != SWITCHABLE)); + if (pred_filter_search) { + pred_dual_interp_filter_rd( + x, cpi, tile_data, bsize, mi_row, mi_col, orig_dst, rd, + switchable_rate, best_skip_txfm_sb, best_skip_sse_sb, dst_bufs, + filter_idx, switchable_ctx, (skip_hor & skip_ver), tmp_rate, tmp_dist, + af_horiz, af_vert, lf_horiz, lf_vert); + } else { + skip_pred = bw <= 4 ? cpi->default_interp_skip_flags : skip_hor; + for (i = (SWITCHABLE_FILTERS - 1); i >= 1; --i) { + if (interpolation_filter_rd( + x, cpi, tile_data, bsize, mi_row, mi_col, orig_dst, rd, + switchable_rate, best_skip_txfm_sb, best_skip_sse_sb, dst_bufs, + i, switchable_ctx, skip_pred, tmp_rate, tmp_dist)) { + best_dual_mode = i; + } + skip_pred = skip_hor; + } + // From best of horizontal EIGHTTAP_REGULAR modes, check vertical modes + skip_pred = bh <= 4 ? cpi->default_interp_skip_flags : skip_ver; + assert(filter_set_size == DUAL_FILTER_SET_SIZE); + for (i = (best_dual_mode + (SWITCHABLE_FILTERS * 2)); + i >= (best_dual_mode + SWITCHABLE_FILTERS); + i -= SWITCHABLE_FILTERS) { + interpolation_filter_rd( + x, cpi, tile_data, bsize, mi_row, mi_col, orig_dst, rd, + switchable_rate, best_skip_txfm_sb, best_skip_sse_sb, dst_bufs, i, + switchable_ctx, skip_pred, tmp_rate, tmp_dist); + skip_pred = skip_ver; + } + } + } else if (cm->seq_params.enable_dual_filter == 0) { + find_best_non_dual_interp_filter( + x, cpi, tile_data, bsize, mi_row, mi_col, orig_dst, rd, switchable_rate, best_skip_txfm_sb, best_skip_sse_sb, dst_bufs, switchable_ctx, skip_ver, - tmp_rate, tmp_dist, best_dual_mode, filter_set_size); + skip_hor, tmp_rate, tmp_dist, filter_set_size); } else { // EIGHTTAP_REGULAR mode is calculated beforehand for (i = 1; i < filter_set_size; ++i) { - if (cm->seq_params.enable_dual_filter == 0) { - const int16_t filter_y = filter_sets[i] & 0xffff; - const int16_t filter_x = filter_sets[i] >> 16; - if (filter_x != filter_y) continue; - } - interpolation_filter_rd(x, cpi, bsize, mi_row, mi_col, orig_dst, rd, - switchable_rate, best_skip_txfm_sb, - best_skip_sse_sb, dst_bufs, i, switchable_ctx, 0, - tmp_rate, tmp_dist); - assert(x->recalc_luma_mc_data == 0); + interpolation_filter_rd(x, cpi, tile_data, bsize, mi_row, mi_col, + orig_dst, rd, switchable_rate, best_skip_txfm_sb, + best_skip_sse_sb, dst_bufs, i, switchable_ctx, + (skip_hor & skip_ver), tmp_rate, tmp_dist); } } swap_dst_buf(xd, dst_bufs, num_planes); @@ -8329,7 +8739,8 @@ static int64_t interpolation_filter_search( // in either of the directions Condition below is necessary, but not // sufficient assert((skip_hor == 1) || (skip_ver == 1)); - av1_build_inter_predictors_sby(cm, xd, mi_row, mi_col, orig_dst, bsize); + av1_enc_build_inter_predictor(cm, xd, mi_row, mi_col, orig_dst, bsize, + AOM_PLANE_Y, AOM_PLANE_Y); } *skip_txfm_sb = best_skip_txfm_sb[1]; *skip_sse_sb = best_skip_sse_sb[1]; @@ -8343,173 +8754,172 @@ static int64_t interpolation_filter_search( return 0; } -static int txfm_search(const AV1_COMP *cpi, MACROBLOCK *x, BLOCK_SIZE bsize, - int mi_row, int mi_col, RD_STATS *rd_stats, - RD_STATS *rd_stats_y, RD_STATS *rd_stats_uv, - int mode_rate, int64_t ref_best_rd) { +static int txfm_search(const AV1_COMP *cpi, const TileDataEnc *tile_data, + MACROBLOCK *x, BLOCK_SIZE bsize, int mi_row, int mi_col, + RD_STATS *rd_stats, RD_STATS *rd_stats_y, + RD_STATS *rd_stats_uv, int mode_rate, + int64_t ref_best_rd) { /* * This function combines y and uv planes' transform search processes - * together, when the prediction is generated. It first does subtration to + * together, when the prediction is generated. It first does subtraction to * obtain the prediction error. Then it calls - * select_tx_type_yrd/super_block_yrd and inter_block_uvrd sequentially and - * handles the early terminations happen in those functions. At the end, it + * pick_tx_size_type_yrd/super_block_yrd and super_block_uvrd sequentially and + * handles the early terminations happening in those functions. At the end, it * computes the rd_stats/_y/_uv accordingly. */ const AV1_COMMON *cm = &cpi->common; MACROBLOCKD *const xd = &x->e_mbd; MB_MODE_INFO *const mbmi = xd->mi[0]; - int skip_txfm_sb = 0; - const int num_planes = av1_num_planes(cm); const int ref_frame_1 = mbmi->ref_frame[1]; const int64_t mode_rd = RDCOST(x->rdmult, mode_rate, 0); const int64_t rd_thresh = ref_best_rd == INT64_MAX ? INT64_MAX : ref_best_rd - mode_rd; const int skip_ctx = av1_get_skip_context(xd); + const int skip_flag_cost[2] = { x->skip_cost[skip_ctx][0], + x->skip_cost[skip_ctx][1] }; const int64_t min_header_rate = - mode_rate + AOMMIN(x->skip_cost[skip_ctx][0], x->skip_cost[skip_ctx][1]); + mode_rate + AOMMIN(skip_flag_cost[0], skip_flag_cost[1]); // Account for minimum skip and non_skip rd. // Eventually either one of them will be added to mode_rate const int64_t min_header_rd_possible = RDCOST(x->rdmult, min_header_rate, 0); + (void)tile_data; if (min_header_rd_possible > ref_best_rd) { av1_invalid_rd_stats(rd_stats_y); - av1_invalid_rd_stats(rd_stats); return 0; } av1_init_rd_stats(rd_stats); av1_init_rd_stats(rd_stats_y); - av1_init_rd_stats(rd_stats_uv); rd_stats->rate = mode_rate; - if (!cpi->common.all_lossless) - check_block_skip(cpi, bsize, x, xd, 0, num_planes - 1, &skip_txfm_sb); - if (!skip_txfm_sb) { - int64_t non_skip_rdcosty = INT64_MAX; - int64_t skip_rdcosty = INT64_MAX; - int64_t min_rdcosty = INT64_MAX; - int is_cost_valid_uv = 0; - - // cost and distortion - av1_subtract_plane(x, bsize, 0); - if (cm->tx_mode == TX_MODE_SELECT && !xd->lossless[mbmi->segment_id]) { - // Motion mode - select_tx_type_yrd(cpi, x, rd_stats_y, bsize, mi_row, mi_col, rd_thresh); + // cost and distortion + av1_subtract_plane(x, bsize, 0); + if (cm->tx_mode == TX_MODE_SELECT && !xd->lossless[mbmi->segment_id]) { + pick_tx_size_type_yrd(cpi, x, rd_stats_y, bsize, mi_row, mi_col, rd_thresh); #if CONFIG_COLLECT_RD_STATS == 2 - PrintPredictionUnitStats(cpi, x, rd_stats_y, bsize); + PrintPredictionUnitStats(cpi, tile_data, x, rd_stats_y, bsize); #endif // CONFIG_COLLECT_RD_STATS == 2 - } else { - super_block_yrd(cpi, x, rd_stats_y, bsize, rd_thresh); - memset(mbmi->inter_tx_size, mbmi->tx_size, sizeof(mbmi->inter_tx_size)); - for (int i = 0; i < xd->n4_h * xd->n4_w; ++i) - set_blk_skip(x, 0, i, rd_stats_y->skip); - } - - if (rd_stats_y->rate == INT_MAX) { - av1_invalid_rd_stats(rd_stats); - // TODO(angiebird): check if we need this - // restore_dst_buf(xd, *orig_dst, num_planes); - mbmi->ref_frame[1] = ref_frame_1; - return 0; - } - - av1_merge_rd_stats(rd_stats, rd_stats_y); - - non_skip_rdcosty = RDCOST( - x->rdmult, rd_stats->rate + x->skip_cost[skip_ctx][0], rd_stats->dist); - skip_rdcosty = - RDCOST(x->rdmult, mode_rate + x->skip_cost[skip_ctx][1], rd_stats->sse); - min_rdcosty = AOMMIN(non_skip_rdcosty, skip_rdcosty); - - if (min_rdcosty > ref_best_rd) { - int64_t tokenonly_rdy = - AOMMIN(RDCOST(x->rdmult, rd_stats_y->rate, rd_stats_y->dist), - RDCOST(x->rdmult, 0, rd_stats_y->sse)); - // Invalidate rd_stats_y to skip the rest of the motion modes search - if (tokenonly_rdy - (tokenonly_rdy >> cpi->sf.adaptive_txb_search_level) > - rd_thresh) - av1_invalid_rd_stats(rd_stats_y); - mbmi->ref_frame[1] = ref_frame_1; - return 0; - } - - if (num_planes > 1) { - /* clang-format off */ - is_cost_valid_uv = - inter_block_uvrd(cpi, x, rd_stats_uv, bsize, - ref_best_rd - non_skip_rdcosty, - ref_best_rd - skip_rdcosty, FTXS_NONE); - if (!is_cost_valid_uv) { - mbmi->ref_frame[1] = ref_frame_1; - return 0; - } - /* clang-format on */ - av1_merge_rd_stats(rd_stats, rd_stats_uv); - } else { - av1_init_rd_stats(rd_stats_uv); - } - if (rd_stats->skip) { - rd_stats->rate -= rd_stats_uv->rate + rd_stats_y->rate; - rd_stats_y->rate = 0; - rd_stats_uv->rate = 0; - rd_stats->rate += x->skip_cost[skip_ctx][1]; - mbmi->skip = 0; - // here mbmi->skip temporarily plays a role as what this_skip2 does - - int64_t tmprd = RDCOST(x->rdmult, rd_stats->rate, rd_stats->dist); - if (tmprd > ref_best_rd) { - mbmi->ref_frame[1] = ref_frame_1; - return 0; - } - } else if (!xd->lossless[mbmi->segment_id] && - (RDCOST(x->rdmult, - rd_stats_y->rate + rd_stats_uv->rate + - x->skip_cost[skip_ctx][0], - rd_stats->dist) >= - RDCOST(x->rdmult, x->skip_cost[skip_ctx][1], rd_stats->sse))) { - rd_stats->rate -= rd_stats_uv->rate + rd_stats_y->rate; - rd_stats->rate += x->skip_cost[skip_ctx][1]; - rd_stats->dist = rd_stats->sse; - rd_stats_y->rate = 0; - rd_stats_uv->rate = 0; - mbmi->skip = 1; - } else { - rd_stats->rate += x->skip_cost[skip_ctx][0]; - mbmi->skip = 0; - } } else { - x->skip = 1; - mbmi->tx_size = tx_size_from_tx_mode(bsize, cm->tx_mode); - // The cost of skip bit needs to be added. - mbmi->skip = 0; - rd_stats->rate += x->skip_cost[skip_ctx][1]; + super_block_yrd(cpi, x, rd_stats_y, bsize, rd_thresh); + memset(mbmi->inter_tx_size, mbmi->tx_size, sizeof(mbmi->inter_tx_size)); + for (int i = 0; i < xd->n4_h * xd->n4_w; ++i) + set_blk_skip(x, 0, i, rd_stats_y->skip); + } - rd_stats->dist = 0; - rd_stats->sse = 0; + if (rd_stats_y->rate == INT_MAX) { + // TODO(angiebird): check if we need this + // restore_dst_buf(xd, *orig_dst, num_planes); + mbmi->ref_frame[1] = ref_frame_1; + return 0; + } + + av1_merge_rd_stats(rd_stats, rd_stats_y); + + const int64_t non_skip_rdcosty = + RDCOST(x->rdmult, rd_stats->rate + skip_flag_cost[0], rd_stats->dist); + const int64_t skip_rdcosty = + RDCOST(x->rdmult, mode_rate + skip_flag_cost[1], rd_stats->sse); + const int64_t min_rdcosty = AOMMIN(non_skip_rdcosty, skip_rdcosty); + if (min_rdcosty > ref_best_rd) { + const int64_t tokenonly_rdy = + AOMMIN(RDCOST(x->rdmult, rd_stats_y->rate, rd_stats_y->dist), + RDCOST(x->rdmult, 0, rd_stats_y->sse)); + // Invalidate rd_stats_y to skip the rest of the motion modes search + if (tokenonly_rdy - (tokenonly_rdy >> cpi->sf.prune_motion_mode_level) > + rd_thresh) + av1_invalid_rd_stats(rd_stats_y); + mbmi->ref_frame[1] = ref_frame_1; + return 0; + } + + av1_init_rd_stats(rd_stats_uv); + const int num_planes = av1_num_planes(cm); + if (num_planes > 1) { + const int is_cost_valid_uv = + super_block_uvrd(cpi, x, rd_stats_uv, bsize, ref_best_rd); + if (!is_cost_valid_uv) { + mbmi->ref_frame[1] = ref_frame_1; + return 0; + } + av1_merge_rd_stats(rd_stats, rd_stats_uv); + } + + if (rd_stats->skip) { + rd_stats->rate -= rd_stats_uv->rate + rd_stats_y->rate; rd_stats_y->rate = 0; rd_stats_uv->rate = 0; - rd_stats->skip = 1; - int64_t tmprd = RDCOST(x->rdmult, rd_stats->rate, rd_stats->dist); + rd_stats->dist = rd_stats->sse; + rd_stats_y->dist = rd_stats_y->sse; + rd_stats_uv->dist = rd_stats_uv->sse; + rd_stats->rate += skip_flag_cost[1]; + mbmi->skip = 1; + // here mbmi->skip temporarily plays a role as what this_skip2 does + + const int64_t tmprd = RDCOST(x->rdmult, rd_stats->rate, rd_stats->dist); if (tmprd > ref_best_rd) { mbmi->ref_frame[1] = ref_frame_1; return 0; } +#if CONFIG_ONE_PASS_SVM + av1_reg_stat_skipmode_update(rd_stats_y, x->rdmult); +#endif + } else if (!xd->lossless[mbmi->segment_id] && + (RDCOST(x->rdmult, + rd_stats_y->rate + rd_stats_uv->rate + skip_flag_cost[0], + rd_stats->dist) >= + RDCOST(x->rdmult, skip_flag_cost[1], rd_stats->sse))) { + rd_stats->rate -= rd_stats_uv->rate + rd_stats_y->rate; + rd_stats->rate += skip_flag_cost[1]; + rd_stats->dist = rd_stats->sse; + rd_stats_y->dist = rd_stats_y->sse; + rd_stats_uv->dist = rd_stats_uv->sse; + rd_stats_y->rate = 0; + rd_stats_uv->rate = 0; + mbmi->skip = 1; +#if CONFIG_ONE_PASS_SVM + av1_reg_stat_skipmode_update(rd_stats_y, x->rdmult); +#endif + } else { + rd_stats->rate += skip_flag_cost[0]; + mbmi->skip = 0; } + return 1; } +static INLINE bool enable_wedge_search(MACROBLOCK *const x, + const AV1_COMP *const cpi) { + // Enable wedge search if source variance and edge strength are above + // the thresholds. + return x->source_variance > cpi->sf.disable_wedge_search_var_thresh && + x->edge_strength > cpi->sf.disable_wedge_search_edge_thresh; +} + +static INLINE bool enable_wedge_interinter_search(MACROBLOCK *const x, + const AV1_COMP *const cpi) { + return enable_wedge_search(x, cpi) && cpi->oxcf.enable_interinter_wedge; +} + +static INLINE bool enable_wedge_interintra_search(MACROBLOCK *const x, + const AV1_COMP *const cpi) { + return enable_wedge_search(x, cpi) && cpi->oxcf.enable_interintra_wedge && + !cpi->sf.disable_wedge_interintra_search; +} + static int handle_inter_intra_mode(const AV1_COMP *const cpi, MACROBLOCK *const x, BLOCK_SIZE bsize, int mi_row, int mi_col, MB_MODE_INFO *mbmi, HandleInterModeArgs *args, int64_t ref_best_rd, int *rate_mv, - int *tmp_rate2, BUFFER_SET *orig_dst) { + int *tmp_rate2, const BUFFER_SET *orig_dst) { const AV1_COMMON *const cm = &cpi->common; const int num_planes = av1_num_planes(cm); MACROBLOCKD *xd = &x->e_mbd; INTERINTRA_MODE best_interintra_mode = II_DC_PRED; - int64_t rd, best_interintra_rd = INT64_MAX; + int64_t rd = INT64_MAX; + int64_t best_interintra_rd = INT64_MAX; int rmode, rate_sum; int64_t dist_sum; int tmp_rate_mv = 0; @@ -8528,61 +8938,115 @@ static int handle_inter_intra_mode(const AV1_COMP *const cpi, mbmi->ref_frame[1] = NONE_FRAME; xd->plane[0].dst.buf = tmp_buf; xd->plane[0].dst.stride = bw; - av1_build_inter_predictors_sby(cm, xd, mi_row, mi_col, NULL, bsize); + av1_enc_build_inter_predictor(cm, xd, mi_row, mi_col, NULL, bsize, + AOM_PLANE_Y, AOM_PLANE_Y); restore_dst_buf(xd, *orig_dst, num_planes); mbmi->ref_frame[1] = INTRA_FRAME; - mbmi->use_wedge_interintra = 0; best_interintra_mode = args->inter_intra_mode[mbmi->ref_frame[0]]; - int j = 0; - if (cpi->sf.reuse_inter_intra_mode == 0 || - best_interintra_mode == INTERINTRA_MODES) { - for (j = 0; j < INTERINTRA_MODES; ++j) { - mbmi->interintra_mode = (INTERINTRA_MODE)j; - rmode = interintra_mode_cost[mbmi->interintra_mode]; + + if (cpi->oxcf.enable_smooth_interintra) { + mbmi->use_wedge_interintra = 0; + int j = 0; + if (cpi->sf.reuse_inter_intra_mode == 0 || + best_interintra_mode == INTERINTRA_MODES) { + for (j = 0; j < INTERINTRA_MODES; ++j) { + if (!cpi->oxcf.enable_smooth_intra && + (INTERINTRA_MODE)j == II_SMOOTH_PRED) + continue; + mbmi->interintra_mode = (INTERINTRA_MODE)j; + rmode = interintra_mode_cost[mbmi->interintra_mode]; + av1_build_intra_predictors_for_interintra(cm, xd, bsize, 0, orig_dst, + intrapred, bw); + av1_combine_interintra(xd, bsize, 0, tmp_buf, bw, intrapred, bw); + model_rd_sb_fn[MODELRD_TYPE_INTERINTRA]( + cpi, bsize, x, xd, 0, 0, mi_row, mi_col, &rate_sum, &dist_sum, + &tmp_skip_txfm_sb, &tmp_skip_sse_sb, NULL, NULL, NULL); + rd = RDCOST(x->rdmult, tmp_rate_mv + rate_sum + rmode, dist_sum); + if (rd < best_interintra_rd) { + best_interintra_rd = rd; + best_interintra_mode = mbmi->interintra_mode; + } + } + args->inter_intra_mode[mbmi->ref_frame[0]] = best_interintra_mode; + } + assert(IMPLIES(!cpi->oxcf.enable_smooth_interintra, + best_interintra_mode != II_SMOOTH_PRED)); + rmode = interintra_mode_cost[best_interintra_mode]; + if (j == 0 || best_interintra_mode != II_SMOOTH_PRED) { + mbmi->interintra_mode = best_interintra_mode; av1_build_intra_predictors_for_interintra(cm, xd, bsize, 0, orig_dst, intrapred, bw); av1_combine_interintra(xd, bsize, 0, tmp_buf, bw, intrapred, bw); - model_rd_sb_fn[MODELRD_TYPE_INTERINTRA]( - cpi, bsize, x, xd, 0, 0, mi_row, mi_col, &rate_sum, &dist_sum, - &tmp_skip_txfm_sb, &tmp_skip_sse_sb, NULL, NULL, NULL); - rd = RDCOST(x->rdmult, tmp_rate_mv + rate_sum + rmode, dist_sum); - if (rd < best_interintra_rd) { - best_interintra_rd = rd; - best_interintra_mode = mbmi->interintra_mode; - } } - args->inter_intra_mode[mbmi->ref_frame[0]] = best_interintra_mode; - } - if (j == 0 || best_interintra_mode != II_SMOOTH_PRED) { - mbmi->interintra_mode = best_interintra_mode; - rmode = interintra_mode_cost[mbmi->interintra_mode]; - av1_build_intra_predictors_for_interintra(cm, xd, bsize, 0, orig_dst, - intrapred, bw); - av1_combine_interintra(xd, bsize, 0, tmp_buf, bw, intrapred, bw); - } - rd = estimate_yrd_for_sb(cpi, bsize, x, &rate_sum, &dist_sum, - &tmp_skip_txfm_sb, &tmp_skip_sse_sb, INT64_MAX); - if (rd != INT64_MAX) - rd = RDCOST(x->rdmult, *rate_mv + rmode + rate_sum + rwedge, dist_sum); - best_interintra_rd = rd; - if (ref_best_rd < INT64_MAX && (best_interintra_rd >> 1) > ref_best_rd) { - return -1; + + RD_STATS rd_stats; + rd = estimate_yrd_for_sb(cpi, bsize, x, INT64_MAX, &rd_stats); + if (rd != INT64_MAX) { + rd = RDCOST(x->rdmult, *rate_mv + rmode + rd_stats.rate + rwedge, + rd_stats.dist); + } + best_interintra_rd = rd; + if (ref_best_rd < INT64_MAX && + ((best_interintra_rd >> 4) * 9) > ref_best_rd) { + return -1; + } } if (is_wedge_used) { int64_t best_interintra_rd_nowedge = rd; int64_t best_interintra_rd_wedge = INT64_MAX; int_mv tmp_mv; - // Disable wedge search if source variance is small - if (x->source_variance > cpi->sf.disable_wedge_search_var_thresh) { + if (enable_wedge_interintra_search(x, cpi)) { mbmi->use_wedge_interintra = 1; rwedge = av1_cost_literal(get_interintra_wedge_bits(bsize)) + x->wedge_interintra_cost[bsize][1]; - best_interintra_rd_wedge = - pick_interintra_wedge(cpi, x, bsize, intrapred_, tmp_buf_); + if (!cpi->oxcf.enable_smooth_interintra) { + if (best_interintra_mode == INTERINTRA_MODES) { + mbmi->interintra_mode = II_SMOOTH_PRED; + best_interintra_mode = II_SMOOTH_PRED; + av1_build_intra_predictors_for_interintra(cm, xd, bsize, 0, orig_dst, + intrapred, bw); + best_interintra_rd_wedge = + pick_interintra_wedge(cpi, x, bsize, intrapred_, tmp_buf_); + int j = 0; + for (j = 0; j < INTERINTRA_MODES; ++j) { + mbmi->interintra_mode = (INTERINTRA_MODE)j; + rmode = interintra_mode_cost[mbmi->interintra_mode]; + av1_build_intra_predictors_for_interintra(cm, xd, bsize, 0, + orig_dst, intrapred, bw); + av1_combine_interintra(xd, bsize, 0, tmp_buf, bw, intrapred, bw); + model_rd_sb_fn[MODELRD_TYPE_INTERINTRA]( + cpi, bsize, x, xd, 0, 0, mi_row, mi_col, &rate_sum, &dist_sum, + &tmp_skip_txfm_sb, &tmp_skip_sse_sb, NULL, NULL, NULL); + rd = RDCOST(x->rdmult, tmp_rate_mv + rate_sum + rmode, dist_sum); + if (rd < best_interintra_rd) { + best_interintra_rd_wedge = rd; + best_interintra_mode = mbmi->interintra_mode; + } + } + args->inter_intra_mode[mbmi->ref_frame[0]] = best_interintra_mode; + mbmi->interintra_mode = best_interintra_mode; + + if (best_interintra_mode != II_SMOOTH_PRED) { + av1_build_intra_predictors_for_interintra(cm, xd, bsize, 0, + orig_dst, intrapred, bw); + } + } else { + mbmi->interintra_mode = best_interintra_mode; + av1_build_intra_predictors_for_interintra(cm, xd, bsize, 0, orig_dst, + intrapred, bw); + best_interintra_rd_wedge = + pick_interintra_wedge(cpi, x, bsize, intrapred_, tmp_buf_); + } + } else { + best_interintra_rd_wedge = + pick_interintra_wedge(cpi, x, bsize, intrapred_, tmp_buf_); + } + + rmode = interintra_mode_cost[mbmi->interintra_mode]; best_interintra_rd_wedge += RDCOST(x->rdmult, rmode + *rate_mv + rwedge, 0); rd = INT64_MAX; @@ -8597,8 +9061,8 @@ static int handle_inter_intra_mode(const AV1_COMP *const cpi, 0); if (mbmi->mv[0].as_int != tmp_mv.as_int) { mbmi->mv[0].as_int = tmp_mv.as_int; - av1_build_inter_predictors_sby(cm, xd, mi_row, mi_col, orig_dst, - bsize); + av1_enc_build_inter_predictor(cm, xd, mi_row, mi_col, orig_dst, bsize, + AOM_PLANE_Y, AOM_PLANE_Y); model_rd_sb_fn[MODELRD_TYPE_MASKED_COMPOUND]( cpi, bsize, x, xd, 0, 0, mi_row, mi_col, &rate_sum, &dist_sum, &tmp_skip_txfm_sb, &tmp_skip_sse_sb, NULL, NULL, NULL); @@ -8612,12 +9076,16 @@ static int handle_inter_intra_mode(const AV1_COMP *const cpi, av1_combine_interintra(xd, bsize, 0, tmp_buf, bw, intrapred, bw); } // Evaluate closer to true rd - rd = estimate_yrd_for_sb(cpi, bsize, x, &rate_sum, &dist_sum, - &tmp_skip_txfm_sb, &tmp_skip_sse_sb, INT64_MAX); - if (rd != INT64_MAX) - rd = RDCOST(x->rdmult, rmode + tmp_rate_mv + rwedge + rate_sum, - dist_sum); + RD_STATS rd_stats; + rd = estimate_yrd_for_sb(cpi, bsize, x, INT64_MAX, &rd_stats); + if (rd != INT64_MAX) { + rd = RDCOST(x->rdmult, rmode + tmp_rate_mv + rwedge + rd_stats.rate, + rd_stats.dist); + } best_interintra_rd_wedge = rd; + if (!cpi->oxcf.enable_smooth_interintra && + best_interintra_rd_wedge == INT64_MAX) + return -1; if (best_interintra_rd_wedge < best_interintra_rd_nowedge) { mbmi->use_wedge_interintra = 1; mbmi->mv[0].as_int = tmp_mv.as_int; @@ -8626,33 +9094,100 @@ static int handle_inter_intra_mode(const AV1_COMP *const cpi, } else { mbmi->use_wedge_interintra = 0; mbmi->mv[0].as_int = mv0.as_int; - av1_build_inter_predictors_sby(cm, xd, mi_row, mi_col, orig_dst, bsize); + av1_enc_build_inter_predictor(cm, xd, mi_row, mi_col, orig_dst, bsize, + AOM_PLANE_Y, AOM_PLANE_Y); } } else { + if (!cpi->oxcf.enable_smooth_interintra) return -1; mbmi->use_wedge_interintra = 0; } - } // if (is_interintra_wedge_used(bsize)) + } // if (is_wedge_used) if (num_planes > 1) { - av1_build_inter_predictors_sbuv(cm, xd, mi_row, mi_col, orig_dst, bsize); + av1_enc_build_inter_predictor(cm, xd, mi_row, mi_col, orig_dst, bsize, + AOM_PLANE_U, num_planes - 1); } return 0; } +// If number of valid neighbours is 1, +// 1) ROTZOOM parameters can be obtained reliably (2 parameters from +// one neighbouring MV) +// 2) For IDENTITY/TRANSLATION cases, warp can perform better due to +// a different interpolation filter being used. However the quality +// gains (due to the same) may not be much +// For above 2 cases warp evaluation is skipped + +static int check_if_optimal_warp(const AV1_COMP *cpi, + WarpedMotionParams *wm_params, + int num_proj_ref) { + int is_valid_warp = 1; + if (cpi->sf.prune_warp_using_wmtype) { + TransformationType wmtype = get_wmtype(wm_params); + if (num_proj_ref == 1) { + if (wmtype != ROTZOOM) is_valid_warp = 0; + } else { + if (wmtype < ROTZOOM) is_valid_warp = 0; + } + } + return is_valid_warp; +} + +struct obmc_check_mv_field_ctxt { + MB_MODE_INFO *current_mi; + int mv_field_check_result; +}; + +static INLINE void obmc_check_identical_mv(MACROBLOCKD *xd, int rel_mi_col, + uint8_t nb_mi_width, + MB_MODE_INFO *nb_mi, void *fun_ctxt, + const int num_planes) { + (void)xd; + (void)rel_mi_col; + (void)nb_mi_width; + (void)num_planes; + struct obmc_check_mv_field_ctxt *ctxt = + (struct obmc_check_mv_field_ctxt *)fun_ctxt; + const MB_MODE_INFO *current_mi = ctxt->current_mi; + + if (ctxt->mv_field_check_result == 0) return; + + if (nb_mi->ref_frame[0] != current_mi->ref_frame[0] || + nb_mi->mv[0].as_int != current_mi->mv[0].as_int || + nb_mi->interp_filters != current_mi->interp_filters) { + ctxt->mv_field_check_result = 0; + } + return; +} + +// Check if the neighbors' motions used by obmc have same parameters as for +// the current block. If all the parameters are identical, obmc will produce +// the same prediction as from regular bmc, therefore we can skip the +// overlapping operations for less complexity. The parameters checked include +// reference frame, motion vector, and interpolation filter. +int check_identical_obmc_mv_field(const AV1_COMMON *cm, MACROBLOCKD *xd, + int mi_row, int mi_col) { + const BLOCK_SIZE bsize = xd->mi[0]->sb_type; + struct obmc_check_mv_field_ctxt mv_field_check_ctxt = { xd->mi[0], 1 }; + + foreach_overlappable_nb_above(cm, xd, mi_col, + max_neighbor_obmc[mi_size_wide_log2[bsize]], + obmc_check_identical_mv, &mv_field_check_ctxt); + foreach_overlappable_nb_left(cm, xd, mi_row, + max_neighbor_obmc[mi_size_high_log2[bsize]], + obmc_check_identical_mv, &mv_field_check_ctxt); + + return mv_field_check_ctxt.mv_field_check_result; +} + // TODO(afergs): Refactor the MBMI references in here - there's four // TODO(afergs): Refactor optional args - add them to a struct or remove -static int64_t motion_mode_rd(const AV1_COMP *const cpi, MACROBLOCK *const x, - BLOCK_SIZE bsize, RD_STATS *rd_stats, - RD_STATS *rd_stats_y, RD_STATS *rd_stats_uv, - int *disable_skip, int mi_row, int mi_col, - HandleInterModeArgs *const args, - int64_t ref_best_rd, const int *refs, - int *rate_mv, BUFFER_SET *orig_dst -#if CONFIG_COLLECT_INTER_MODE_RD_STATS - , - TileDataEnc *tile_data, int64_t *best_est_rd, - int do_tx_search, InterModesInfo *inter_modes_info -#endif -) { +static int64_t motion_mode_rd( + const AV1_COMP *const cpi, TileDataEnc *tile_data, MACROBLOCK *const x, + BLOCK_SIZE bsize, RD_STATS *rd_stats, RD_STATS *rd_stats_y, + RD_STATS *rd_stats_uv, int *disable_skip, int mi_row, int mi_col, + HandleInterModeArgs *const args, int64_t ref_best_rd, const int *refs, + int *rate_mv, const BUFFER_SET *orig_dst, int64_t *best_est_rd, + int do_tx_search, InterModesInfo *inter_modes_info) { const AV1_COMMON *const cm = &cpi->common; const int num_planes = av1_num_planes(cm); MACROBLOCKD *xd = &x->e_mbd; @@ -8660,18 +9195,18 @@ static int64_t motion_mode_rd(const AV1_COMP *const cpi, MACROBLOCK *const x, const int is_comp_pred = has_second_ref(mbmi); const PREDICTION_MODE this_mode = mbmi->mode; const int rate2_nocoeff = rd_stats->rate; - int best_xskip, best_disable_skip = 0; + int best_xskip = 0, best_disable_skip = 0; RD_STATS best_rd_stats, best_rd_stats_y, best_rd_stats_uv; - MB_MODE_INFO base_mbmi, best_mbmi; uint8_t best_blk_skip[MAX_MIB_SIZE * MAX_MIB_SIZE]; const int rate_mv0 = *rate_mv; - - int interintra_allowed = cm->seq_params.enable_interintra_compound && - is_interintra_allowed(mbmi) && mbmi->compound_idx; + const int interintra_allowed = cm->seq_params.enable_interintra_compound && + is_interintra_allowed(mbmi) && + mbmi->compound_idx; int pts0[SAMPLES_ARRAY_SIZE], pts_inref0[SAMPLES_ARRAY_SIZE]; assert(mbmi->ref_frame[1] != INTRA_FRAME); const MV_REFERENCE_FRAME ref_frame_1 = mbmi->ref_frame[1]; + (void)tile_data; av1_invalid_rd_stats(&best_rd_stats); aom_clear_system_state(); mbmi->num_proj_ref = 1; // assume num_proj_ref >=1 @@ -8683,24 +9218,32 @@ static int64_t motion_mode_rd(const AV1_COMP *const cpi, MACROBLOCK *const x, if (last_motion_mode_allowed == WARPED_CAUSAL) { mbmi->num_proj_ref = findSamples(cm, xd, mi_row, mi_col, pts0, pts_inref0); } - int total_samples = mbmi->num_proj_ref; + const int total_samples = mbmi->num_proj_ref; if (total_samples == 0) { last_motion_mode_allowed = OBMC_CAUSAL; } - base_mbmi = *mbmi; + const MB_MODE_INFO base_mbmi = *mbmi; + MB_MODE_INFO best_mbmi; + SimpleRDState *const simple_states = &args->simple_rd_state[mbmi->ref_mv_idx]; const int switchable_rate = av1_is_interp_needed(xd) ? av1_get_switchable_rate(cm, x, xd) : 0; int64_t best_rd = INT64_MAX; int best_rate_mv = rate_mv0; + const int identical_obmc_mv_field_detected = + (cpi->sf.skip_obmc_in_uniform_mv_field || + cpi->sf.skip_wm_in_uniform_mv_field) + ? check_identical_obmc_mv_field(cm, xd, mi_row, mi_col) + : 0; for (int mode_index = (int)SIMPLE_TRANSLATION; mode_index <= (int)last_motion_mode_allowed + interintra_allowed; mode_index++) { if (args->skip_motion_mode && mode_index) continue; - int64_t tmp_rd = INT64_MAX; + if (cpi->sf.prune_single_motion_modes_by_simple_trans && + args->single_ref_first_pass && mode_index) + break; int tmp_rate2 = rate2_nocoeff; - int is_interintra_mode = mode_index > (int)last_motion_mode_allowed; - int skip_txfm_sb = 0; + const int is_interintra_mode = mode_index > (int)last_motion_mode_allowed; int tmp_rate_mv = rate_mv0; *mbmi = base_mbmi; @@ -8711,12 +9254,44 @@ static int64_t motion_mode_rd(const AV1_COMP *const cpi, MACROBLOCK *const x, assert(mbmi->ref_frame[1] != INTRA_FRAME); } + if (cpi->oxcf.enable_obmc == 0 && mbmi->motion_mode == OBMC_CAUSAL) + continue; + + if (identical_obmc_mv_field_detected) { + if (cpi->sf.skip_obmc_in_uniform_mv_field && + mbmi->motion_mode == OBMC_CAUSAL) + continue; + if (cpi->sf.skip_wm_in_uniform_mv_field && + mbmi->motion_mode == WARPED_CAUSAL) + continue; + } + if (mbmi->motion_mode == SIMPLE_TRANSLATION && !is_interintra_mode) { // SIMPLE_TRANSLATION mode: no need to recalculate. // The prediction is calculated before motion_mode_rd() is called in // handle_inter_mode() + if (cpi->sf.prune_single_motion_modes_by_simple_trans && + args->single_ref_first_pass == 0 && !is_comp_pred) { + if (simple_states->early_skipped) { + assert(simple_states->rd_stats.rdcost == INT64_MAX); + return INT64_MAX; + } + if (simple_states->rd_stats.rdcost != INT64_MAX) { + best_rd = simple_states->rd_stats.rdcost; + best_rd_stats = simple_states->rd_stats; + best_rd_stats_y = simple_states->rd_stats_y; + best_rd_stats_uv = simple_states->rd_stats_uv; + memcpy(best_blk_skip, simple_states->blk_skip, + sizeof(x->blk_skip[0]) * xd->n4_h * xd->n4_w); + best_xskip = simple_states->skip; + best_disable_skip = simple_states->disable_skip; + best_mbmi = *mbmi; + } + continue; + } + simple_states->early_skipped = 0; } else if (mbmi->motion_mode == OBMC_CAUSAL) { - uint32_t cur_mv = mbmi->mv[0].as_int; + const uint32_t cur_mv = mbmi->mv[0].as_int; assert(!is_comp_pred); if (have_newmv_in_inter_mode(this_mode)) { single_motion_search(cpi, x, bsize, mi_row, mi_col, 0, &tmp_rate_mv); @@ -8729,7 +9304,8 @@ static int64_t motion_mode_rd(const AV1_COMP *const cpi, MACROBLOCK *const x, tmp_rate2 = rate2_nocoeff - rate_mv0 + tmp_rate_mv; } if (mbmi->mv[0].as_int != cur_mv) { - av1_build_inter_predictors_sb(cm, xd, mi_row, mi_col, orig_dst, bsize); + av1_enc_build_inter_predictor(cm, xd, mi_row, mi_col, orig_dst, bsize, + 0, av1_num_planes(cm) - 1); } av1_build_obmc_inter_prediction( cm, xd, mi_row, mi_col, args->above_pred_buf, args->above_pred_stride, @@ -8757,7 +9333,12 @@ static int64_t motion_mode_rd(const AV1_COMP *const cpi, MACROBLOCK *const x, if (have_newmv_in_inter_mode(this_mode)) { const int_mv mv0 = mbmi->mv[0]; const WarpedMotionParams wm_params0 = mbmi->wm_params; - int num_proj_ref0 = mbmi->num_proj_ref; + const int num_proj_ref0 = mbmi->num_proj_ref; + + if (cpi->sf.prune_warp_using_wmtype) { + TransformationType wmtype = get_wmtype(&mbmi->wm_params); + if (wmtype < ROTZOOM) continue; + } // Refine MV in a small range. av1_refine_warped_mv(cpi, x, bsize, mi_row, mi_col, pts0, pts_inref0, @@ -8767,9 +9348,9 @@ static int64_t motion_mode_rd(const AV1_COMP *const cpi, MACROBLOCK *const x, if (mv0.as_int != mbmi->mv[0].as_int) { const int ref = refs[0]; const int_mv ref_mv = av1_get_ref_mv(x, 0); - tmp_rate_mv = - av1_mv_bit_cost(&mbmi->mv[0].as_mv, &ref_mv.as_mv, - x->nmvjointcost, x->mvcost, MV_COST_WEIGHT); + tmp_rate_mv = av1_mv_bit_cost(&mbmi->mv[0].as_mv, &ref_mv.as_mv, + x->nmv_vec_cost, x->mv_cost_stack, + MV_COST_WEIGHT); if (cpi->sf.adaptive_motion_search) x->pred_mv[ref] = mbmi->mv[0].as_mv; @@ -8786,9 +9367,13 @@ static int64_t motion_mode_rd(const AV1_COMP *const cpi, MACROBLOCK *const x, mbmi->wm_params = wm_params0; mbmi->num_proj_ref = num_proj_ref0; } + } else { + if (!check_if_optimal_warp(cpi, &mbmi->wm_params, mbmi->num_proj_ref)) + continue; } - av1_build_inter_predictors_sb(cm, xd, mi_row, mi_col, NULL, bsize); + av1_enc_build_inter_predictor(cm, xd, mi_row, mi_col, NULL, bsize, 0, + av1_num_planes(cm) - 1); } else { continue; } @@ -8799,11 +9384,7 @@ static int64_t motion_mode_rd(const AV1_COMP *const cpi, MACROBLOCK *const x, if (ret < 0) continue; } - if (!cpi->common.all_lossless) - check_block_skip(cpi, bsize, x, xd, 0, num_planes - 1, &skip_txfm_sb); - x->skip = 0; - rd_stats->dist = 0; rd_stats->sse = 0; rd_stats->skip = 1; @@ -8834,45 +9415,47 @@ static int64_t motion_mode_rd(const AV1_COMP *const cpi, MACROBLOCK *const x, } } - if (!skip_txfm_sb) { -#if CONFIG_COLLECT_INTER_MODE_RD_STATS - int64_t est_rd = 0; - int est_skip = 0; - if (cpi->sf.inter_mode_rd_model_estimation && cm->tile_cols == 1 && - cm->tile_rows == 1) { - InterModeRdModel *md = &tile_data->inter_mode_rd_models[mbmi->sb_type]; - if (md->ready) { - const int64_t curr_sse = get_sse(cpi, x); - est_rd = get_est_rd(tile_data, mbmi->sb_type, x->rdmult, curr_sse, - rd_stats->rate); - est_skip = est_rd * 0.8 > *best_est_rd; - if (est_skip) { - mbmi->ref_frame[1] = ref_frame_1; - continue; - } else { - if (est_rd < *best_est_rd) { - *best_est_rd = est_rd; - } - } - } + if (cpi->sf.model_based_motion_mode_rd_breakout && do_tx_search) { + int model_rate; + int64_t model_dist; + model_rd_sb_fn[MODELRD_TYPE_MOTION_MODE_RD]( + cpi, mbmi->sb_type, x, xd, 0, num_planes - 1, mi_row, mi_col, + &model_rate, &model_dist, NULL, NULL, NULL, NULL, NULL); + const int64_t est_rd = + RDCOST(x->rdmult, rd_stats->rate + model_rate, model_dist); + if ((est_rd >> 3) * 6 > ref_best_rd) { + mbmi->ref_frame[1] = ref_frame_1; + continue; } -#endif // CONFIG_COLLECT_INTER_MODE_RD_STATS } -#if CONFIG_COLLECT_INTER_MODE_RD_STATS if (!do_tx_search) { - const int64_t curr_sse = get_sse(cpi, x); + int64_t curr_sse; int est_residue_cost = 0; int64_t est_dist = 0; - const int has_est_rd = get_est_rate_dist(tile_data, bsize, curr_sse, - &est_residue_cost, &est_dist); - (void)has_est_rd; - assert(has_est_rd); + int64_t est_rd = 0; + if (cpi->sf.inter_mode_rd_model_estimation == 1) { + curr_sse = get_sse(cpi, x); + const int has_est_rd = get_est_rate_dist(tile_data, bsize, curr_sse, + &est_residue_cost, &est_dist); + (void)has_est_rd; + assert(has_est_rd); + } else if (cpi->sf.inter_mode_rd_model_estimation == 2) { + model_rd_sb_fn[MODELRD_TYPE_MOTION_MODE_RD]( + cpi, bsize, x, xd, 0, num_planes - 1, mi_row, mi_col, + &est_residue_cost, &est_dist, NULL, &curr_sse, NULL, NULL, NULL); + } + est_rd = RDCOST(x->rdmult, rd_stats->rate + est_residue_cost, est_dist); + if (est_rd * 0.8 > *best_est_rd) { + mbmi->ref_frame[1] = ref_frame_1; + continue; + } const int mode_rate = rd_stats->rate; rd_stats->rate += est_residue_cost; rd_stats->dist = est_dist; - rd_stats->rdcost = RDCOST(x->rdmult, rd_stats->rate, rd_stats->dist); - if (cm->reference_mode == SINGLE_REFERENCE) { + rd_stats->rdcost = est_rd; + *best_est_rd = AOMMIN(*best_est_rd, rd_stats->rdcost); + if (cm->current_frame.reference_mode == SINGLE_REFERENCE) { if (!is_comp_pred) { inter_modes_info_push(inter_modes_info, mode_rate, curr_sse, rd_stats->rdcost, mbmi); @@ -8882,37 +9465,26 @@ static int64_t motion_mode_rd(const AV1_COMP *const cpi, MACROBLOCK *const x, rd_stats->rdcost, mbmi); } } else { -#endif - int mode_rate = rd_stats->rate; - if (!txfm_search(cpi, x, bsize, mi_row, mi_col, rd_stats, rd_stats_y, - rd_stats_uv, mode_rate, ref_best_rd)) { + if (!txfm_search(cpi, tile_data, x, bsize, mi_row, mi_col, rd_stats, + rd_stats_y, rd_stats_uv, rd_stats->rate, ref_best_rd)) { if (rd_stats_y->rate == INT_MAX && mode_index == 0) { + simple_states->early_skipped = 1; return INT64_MAX; } continue; } - if (!skip_txfm_sb) { - const int64_t curr_rd = - RDCOST(x->rdmult, rd_stats->rate, rd_stats->dist); - if (curr_rd < ref_best_rd) { - ref_best_rd = curr_rd; - } - *disable_skip = 0; -#if CONFIG_COLLECT_INTER_MODE_RD_STATS - if (cpi->sf.inter_mode_rd_model_estimation) { - const int skip_ctx = av1_get_skip_context(xd); - inter_mode_data_push(tile_data, mbmi->sb_type, rd_stats->sse, - rd_stats->dist, - rd_stats_y->rate + rd_stats_uv->rate + - x->skip_cost[skip_ctx][mbmi->skip]); - } -#endif // CONFIG_COLLECT_INTER_MODE_RD_STATS - } else { - *disable_skip = 1; + + const int64_t curr_rd = RDCOST(x->rdmult, rd_stats->rate, rd_stats->dist); + ref_best_rd = AOMMIN(ref_best_rd, curr_rd); + *disable_skip = 0; + if (cpi->sf.inter_mode_rd_model_estimation == 1) { + const int skip_ctx = av1_get_skip_context(xd); + inter_mode_data_push(tile_data, mbmi->sb_type, rd_stats->sse, + rd_stats->dist, + rd_stats_y->rate + rd_stats_uv->rate + + x->skip_cost[skip_ctx][mbmi->skip]); } -#if CONFIG_COLLECT_INTER_MODE_RD_STATS } -#endif if (this_mode == GLOBALMV || this_mode == GLOBAL_GLOBALMV) { if (is_nontrans_global_motion(xd, xd->mi[0])) { @@ -8921,10 +9493,21 @@ static int64_t motion_mode_rd(const AV1_COMP *const cpi, MACROBLOCK *const x, } } - tmp_rd = RDCOST(x->rdmult, rd_stats->rate, rd_stats->dist); - if (mode_index == 0) + const int64_t tmp_rd = RDCOST(x->rdmult, rd_stats->rate, rd_stats->dist); + if (mode_index == 0) { args->simple_rd[this_mode][mbmi->ref_mv_idx][mbmi->ref_frame[0]] = tmp_rd; - if ((mode_index == 0) || (tmp_rd < best_rd)) { + if (!is_comp_pred) { + simple_states->rd_stats = *rd_stats; + simple_states->rd_stats.rdcost = tmp_rd; + simple_states->rd_stats_y = *rd_stats_y; + simple_states->rd_stats_uv = *rd_stats_uv; + memcpy(simple_states->blk_skip, x->blk_skip, + sizeof(x->blk_skip[0]) * xd->n4_h * xd->n4_w); + simple_states->skip = x->skip; + simple_states->disable_skip = *disable_skip; + } + } + if (mode_index == 0 || tmp_rd < best_rd) { best_mbmi = *mbmi; best_rd = tmp_rd; best_rd_stats = *rd_stats; @@ -8960,11 +9543,12 @@ static int64_t motion_mode_rd(const AV1_COMP *const cpi, MACROBLOCK *const x, static int64_t skip_mode_rd(RD_STATS *rd_stats, const AV1_COMP *const cpi, MACROBLOCK *const x, BLOCK_SIZE bsize, int mi_row, - int mi_col, BUFFER_SET *const orig_dst) { + int mi_col, const BUFFER_SET *const orig_dst) { const AV1_COMMON *cm = &cpi->common; const int num_planes = av1_num_planes(cm); MACROBLOCKD *const xd = &x->e_mbd; - av1_build_inter_predictors_sb(cm, xd, mi_row, mi_col, orig_dst, bsize); + av1_enc_build_inter_predictor(cm, xd, mi_row, mi_col, orig_dst, bsize, 0, + av1_num_planes(cm) - 1); int64_t total_sse = 0; for (int plane = 0; plane < num_planes; ++plane) { @@ -8976,9 +9560,44 @@ static int64_t skip_mode_rd(RD_STATS *rd_stats, const AV1_COMP *const cpi, const int bh = block_size_high[plane_bsize]; av1_subtract_plane(x, bsize, plane); - int64_t sse = aom_sum_squares_2d_i16(p->src_diff, bw, bw, bh); + int64_t sse; +#if CONFIG_ONE_PASS_SVM + if (plane == AOM_PLANE_Y && bsize >= BLOCK_8X8 && bw == bh) { + rd_stats->sse_0 = aom_sum_squares_2d_i16(p->src_diff, bw, bw / 2, bh / 2) + << 4; + rd_stats->sse_1 = + aom_sum_squares_2d_i16(p->src_diff + bw / 2, bw, bw / 2, bh / 2) << 4; + rd_stats->sse_2 = + aom_sum_squares_2d_i16(p->src_diff + bh / 2 * bw, bw, bw / 2, bh / 2) + << 4; + rd_stats->sse_3 = + aom_sum_squares_2d_i16(p->src_diff + bh / 2 * bw + bw / 2, bw, bw / 2, + bh / 2) + << 4; + + sse = + rd_stats->sse_0 + rd_stats->sse_1 + rd_stats->sse_2 + rd_stats->sse_3; + total_sse += sse; + + const int scaling_factor = MAX_MIB_SIZE * MAX_MIB_SIZE; + rd_stats->sse = sse; + rd_stats->sse_0 = rd_stats->sse_0 * scaling_factor; + rd_stats->sse_1 = rd_stats->sse_1 * scaling_factor; + rd_stats->sse_2 = rd_stats->sse_2 * scaling_factor; + rd_stats->sse_3 = rd_stats->sse_3 * scaling_factor; + rd_stats->y_sse = sse; + // TODO(chiyotsai@google.com): Don't manually set the flags + av1_reg_stat_skipmode_update(rd_stats, x->rdmult); + } else { + sse = aom_sum_squares_2d_i16(p->src_diff, bw, bw, bh); + sse = sse << 4; + total_sse += sse; + } +#else + sse = aom_sum_squares_2d_i16(p->src_diff, bw, bw, bh); sse = sse << 4; total_sse += sse; +#endif } const int skip_mode_ctx = av1_get_skip_mode_context(xd); rd_stats->dist = rd_stats->sse = total_sse; @@ -9101,22 +9720,19 @@ typedef struct { static int compound_type_rd(const AV1_COMP *const cpi, MACROBLOCK *x, BLOCK_SIZE bsize, int mi_col, int mi_row, int_mv *cur_mv, int masked_compound_used, - BUFFER_SET *orig_dst, const BUFFER_SET *tmp_dst, + const BUFFER_SET *orig_dst, + const BUFFER_SET *tmp_dst, CompoundTypeRdBuffers *buffers, int *rate_mv, int64_t *rd, RD_STATS *rd_stats, - int64_t ref_best_rd) { + int64_t ref_best_rd, int *is_luma_interp_done) { const AV1_COMMON *cm = &cpi->common; MACROBLOCKD *xd = &x->e_mbd; MB_MODE_INFO *mbmi = xd->mi[0]; const PREDICTION_MODE this_mode = mbmi->mode; const int bw = block_size_wide[bsize]; - int rate_sum, rs2; - int64_t dist_sum; - + int rs2; int_mv best_mv[2]; int best_tmp_rate_mv = *rate_mv; - int tmp_skip_txfm_sb; - int64_t tmp_skip_sse_sb; INTERINTER_COMPOUND_DATA best_compound_data; best_compound_data.type = COMPOUND_AVERAGE; uint8_t *preds0[1] = { buffers->pred0 }; @@ -9128,7 +9744,9 @@ static int compound_type_rd(const AV1_COMP *const cpi, MACROBLOCK *x, COMPOUND_TYPE cur_type; int best_compmode_interinter_cost = 0; int calc_pred_masked_compound = 1; - + int64_t comp_dist[COMPOUND_TYPES] = { INT64_MAX, INT64_MAX, INT64_MAX }; + int32_t comp_rate[COMPOUND_TYPES] = { INT_MAX, INT_MAX, INT_MAX }; + const int match_found = find_comp_rd_in_stats(x, mbmi, comp_rate, comp_dist); best_mv[0].as_int = cur_mv[0].as_int; best_mv[1].as_int = cur_mv[1].as_int; *rd = INT64_MAX; @@ -9152,12 +9770,31 @@ static int compound_type_rd(const AV1_COMP *const cpi, MACROBLOCK *x, rs2 = masked_type_cost; const int64_t mode_rd = RDCOST(x->rdmult, rs2 + rd_stats->rate, 0); if (mode_rd < ref_best_rd) { - av1_build_inter_predictors_sby(cm, xd, mi_row, mi_col, orig_dst, bsize); - int64_t est_rd = - estimate_yrd_for_sb(cpi, bsize, x, &rate_sum, &dist_sum, - &tmp_skip_txfm_sb, &tmp_skip_sse_sb, INT64_MAX); - if (est_rd != INT64_MAX) - best_rd_cur = RDCOST(x->rdmult, rs2 + *rate_mv + rate_sum, dist_sum); + // Reuse data if matching record is found + if (comp_rate[0] == INT_MAX) { + av1_enc_build_inter_predictor(cm, xd, mi_row, mi_col, orig_dst, bsize, + AOM_PLANE_Y, AOM_PLANE_Y); + *is_luma_interp_done = 1; + RD_STATS est_rd_stats; + const int64_t est_rd = + estimate_yrd_for_sb(cpi, bsize, x, INT64_MAX, &est_rd_stats); + if (comp_rate[0] != INT_MAX) { + assert(comp_rate[0] == est_rd_stats.rate); + assert(comp_dist[0] == est_rd_stats.dist); + } + if (est_rd != INT64_MAX) { + best_rd_cur = RDCOST(x->rdmult, rs2 + *rate_mv + est_rd_stats.rate, + est_rd_stats.dist); + // Backup rate and distortion for future reuse + comp_rate[0] = est_rd_stats.rate; + comp_dist[0] = est_rd_stats.dist; + } + } else { + // Calculate RD cost based on stored stats + assert(comp_dist[0] != INT64_MAX); + best_rd_cur = + RDCOST(x->rdmult, rs2 + *rate_mv + comp_rate[0], comp_dist[0]); + } } // use spare buffer for following compound type try restore_dst_buf(xd, *tmp_dst, 1); @@ -9166,13 +9803,20 @@ static int compound_type_rd(const AV1_COMP *const cpi, MACROBLOCK *x, masked_type_cost += x->comp_group_idx_cost[comp_group_idx_ctx][1]; masked_type_cost += x->compound_type_cost[bsize][cur_type - 1]; rs2 = masked_type_cost; - if (x->source_variance > cpi->sf.disable_wedge_search_var_thresh && - *rd / 3 < ref_best_rd) { - best_rd_cur = build_and_cost_compound_type( - cpi, x, cur_mv, bsize, this_mode, &rs2, *rate_mv, orig_dst, - &tmp_rate_mv, preds0, preds1, buffers->residual1, buffers->diff10, - strides, mi_row, mi_col, rd_stats->rate, ref_best_rd, - &calc_pred_masked_compound); + + if (((*rd / cpi->max_comp_type_rd_threshold_div) * + cpi->max_comp_type_rd_threshold_mul) < ref_best_rd) { + const COMPOUND_TYPE compound_type = mbmi->interinter_comp.type; + + if (!((compound_type == COMPOUND_WEDGE && + !enable_wedge_interinter_search(x, cpi)) || + (compound_type == COMPOUND_DIFFWTD && + !cpi->oxcf.enable_diff_wtd_comp))) + best_rd_cur = build_and_cost_compound_type( + cpi, x, cur_mv, bsize, this_mode, &rs2, *rate_mv, orig_dst, + &tmp_rate_mv, preds0, preds1, buffers->residual1, buffers->diff10, + strides, mi_row, mi_col, rd_stats->rate, ref_best_rd, + &calc_pred_masked_compound, comp_rate, comp_dist); } } if (best_rd_cur < *rd) { @@ -9212,6 +9856,8 @@ static int compound_type_rd(const AV1_COMP *const cpi, MACROBLOCK *x, } } restore_dst_buf(xd, *orig_dst, 1); + if (!match_found) + save_comp_rd_search_stat(x, mbmi, comp_rate, comp_dist, cur_mv); return best_compmode_interinter_cost; } @@ -9252,20 +9898,13 @@ typedef struct { int_mv mv; } inter_mode_info; -static int64_t handle_inter_mode(const AV1_COMP *const cpi, MACROBLOCK *x, - BLOCK_SIZE bsize, RD_STATS *rd_stats, - RD_STATS *rd_stats_y, RD_STATS *rd_stats_uv, - int *disable_skip, int mi_row, int mi_col, - HandleInterModeArgs *args, int64_t ref_best_rd, - uint8_t *const tmp_buf, - CompoundTypeRdBuffers *rd_buffers -#if CONFIG_COLLECT_INTER_MODE_RD_STATS - , - TileDataEnc *tile_data, int64_t *best_est_rd, - const int do_tx_search, - InterModesInfo *inter_modes_info -#endif -) { +static int64_t handle_inter_mode( + const AV1_COMP *const cpi, TileDataEnc *tile_data, MACROBLOCK *x, + BLOCK_SIZE bsize, RD_STATS *rd_stats, RD_STATS *rd_stats_y, + RD_STATS *rd_stats_uv, int *disable_skip, int mi_row, int mi_col, + HandleInterModeArgs *args, int64_t ref_best_rd, uint8_t *const tmp_buf, + CompoundTypeRdBuffers *rd_buffers, int64_t *best_est_rd, + const int do_tx_search, InterModesInfo *inter_modes_info) { const AV1_COMMON *cm = &cpi->common; const int num_planes = av1_num_planes(cm); MACROBLOCKD *xd = &x->e_mbd; @@ -9285,7 +9924,7 @@ static int64_t handle_inter_mode(const AV1_COMP *const cpi, MACROBLOCK *x, // one for future predictions. In the end, copy from tmp_buf to // dst if necessary. struct macroblockd_plane *p = xd->plane; - BUFFER_SET orig_dst = { + const BUFFER_SET orig_dst = { { p[0].dst.buf, p[1].dst.buf, p[2].dst.buf }, { p[0].dst.stride, p[1].dst.stride, p[2].dst.stride }, }; @@ -9312,8 +9951,10 @@ static int64_t handle_inter_mode(const AV1_COMP *const cpi, MACROBLOCK *x, inter_mode_info mode_info[MAX_REF_MV_SERCH]; int comp_idx; - const int search_jnt_comp = is_comp_pred & cm->seq_params.enable_jnt_comp & - (mbmi->mode != GLOBAL_GLOBALMV); + const int search_dist_wtd_comp = + is_comp_pred & cm->seq_params.order_hint_info.enable_dist_wtd_comp & + (mbmi->mode != GLOBAL_GLOBALMV) & + (cpi->sf.use_dist_wtd_comp_flag != DIST_WTD_COMP_DISABLED); // TODO(jingning): This should be deprecated shortly. const int has_nearmv = have_nearmv_in_inter_mode(mbmi->mode) ? 1 : 0; @@ -9334,7 +9975,12 @@ static int64_t handle_inter_mode(const AV1_COMP *const cpi, MACROBLOCK *x, } } } - + if (cpi->sf.prune_single_motion_modes_by_simple_trans && !is_comp_pred && + args->single_ref_first_pass == 0) { + if (args->simple_rd_state[ref_mv_idx].early_skipped) { + continue; + } + } av1_init_rd_stats(rd_stats); mbmi->interinter_comp.type = COMPOUND_AVERAGE; @@ -9364,28 +10010,20 @@ static int64_t handle_inter_mode(const AV1_COMP *const cpi, MACROBLOCK *x, continue; } - int64_t best_rd2 = INT64_MAX; - const RD_STATS backup_rd_stats = *rd_stats; - // If !search_jnt_comp, we need to force mbmi->compound_idx = 1. - for (comp_idx = 1; comp_idx >= !search_jnt_comp; --comp_idx) { + // If !search_dist_wtd_comp, we need to force mbmi->compound_idx = 1. + for (comp_idx = 1; comp_idx >= !search_dist_wtd_comp; --comp_idx) { int rs = 0; int compmode_interinter_cost = 0; mbmi->compound_idx = comp_idx; if (is_comp_pred && comp_idx == 0) { *rd_stats = backup_rd_stats; mbmi->interinter_comp.type = COMPOUND_AVERAGE; - if (mbmi->ref_frame[1] == INTRA_FRAME) mbmi->ref_frame[1] = NONE_FRAME; mbmi->num_proj_ref = 0; mbmi->motion_mode = SIMPLE_TRANSLATION; mbmi->comp_group_idx = 0; - const int comp_group_idx_ctx = get_comp_group_idx_context(xd); const int comp_index_ctx = get_comp_index_context(cm, xd); - if (masked_compound_used) { - compmode_interinter_cost += - x->comp_group_idx_cost[comp_group_idx_ctx][0]; - } compmode_interinter_cost += x->comp_idx_cost[comp_index_ctx][0]; } @@ -9400,9 +10038,16 @@ static int64_t handle_inter_mode(const AV1_COMP *const cpi, MACROBLOCK *x, rate_mv = backup_rate_mv; } - // when jnt_comp_skip_mv_search flag is on, new mv will be searched once - if (!(search_jnt_comp && cpi->sf.jnt_comp_skip_mv_search && - comp_idx == 0)) { + if (cpi->sf.prune_single_motion_modes_by_simple_trans && + args->single_ref_first_pass == 0 && !is_comp_pred) { + const int ref0 = mbmi->ref_frame[0]; + newmv_ret_val = args->single_newmv_valid[ref_mv_idx][ref0] ? 0 : 1; + cur_mv[0] = args->single_newmv[ref_mv_idx][ref0]; + rate_mv = args->single_newmv_rate[ref_mv_idx][ref0]; + } else if (!(search_dist_wtd_comp && + (cpi->sf.use_dist_wtd_comp_flag == + DIST_WTD_COMP_SKIP_MV_SEARCH) && + comp_idx == 0)) { newmv_ret_val = handle_newmv(cpi, x, bsize, cur_mv, mi_row, mi_col, &rate_mv, args); @@ -9437,9 +10082,9 @@ static int64_t handle_inter_mode(const AV1_COMP *const cpi, MACROBLOCK *x, const int compare_cost = mode_info[i].rate_mv + mode_info[i].drl_cost; const int_mv ref_mv = av1_get_ref_mv(x, 0); - this_rate_mv = av1_mv_bit_cost(&mode_info[i].mv.as_mv, - &ref_mv.as_mv, x->nmvjointcost, - x->mvcost, MV_COST_WEIGHT); + this_rate_mv = av1_mv_bit_cost( + &mode_info[i].mv.as_mv, &ref_mv.as_mv, x->nmv_vec_cost, + x->mv_cost_stack, MV_COST_WEIGHT); const int this_cost = this_rate_mv + drl_cost; if (compare_cost < this_cost) { @@ -9455,7 +10100,6 @@ static int64_t handle_inter_mode(const AV1_COMP *const cpi, MACROBLOCK *x, best_rd = RDCOST(x->rdmult, best_rd_stats.rate, best_rd_stats.dist); if (best_rd < ref_best_rd) ref_best_rd = best_rd; - skip = 1; break; } @@ -9514,6 +10158,7 @@ static int64_t handle_inter_mode(const AV1_COMP *const cpi, MACROBLOCK *x, av1_is_interp_needed(xd) && av1_is_interp_search_needed(xd); int match_found = -1; const InterpFilter assign_filter = cm->interp_filter; + int is_luma_interp_done = 0; if (cpi->sf.skip_repeat_interpolation_filter_search && need_search) { match_found = find_interp_filter_in_stats(x, mbmi); } @@ -9525,25 +10170,28 @@ static int64_t handle_inter_mode(const AV1_COMP *const cpi, MACROBLOCK *x, compmode_interinter_cost = compound_type_rd( cpi, x, bsize, mi_col, mi_row, cur_mv, masked_compound_used, &orig_dst, &tmp_dst, rd_buffers, &rate_mv, &best_rd_compound, - rd_stats, ref_best_rd); - if (ref_best_rd < INT64_MAX && best_rd_compound / 3 > ref_best_rd) { + rd_stats, ref_best_rd, &is_luma_interp_done); + if (ref_best_rd < INT64_MAX && + (best_rd_compound >> 4) * 13 > ref_best_rd) { restore_dst_buf(xd, orig_dst, num_planes); continue; } - // No need to call av1_build_inter_predictors_sby if + // No need to call av1_enc_build_inter_predictor for luma if // COMPOUND_AVERAGE is selected because it is the first // candidate in compound_type_rd, and the following // compound types searching uses tmp_dst buffer - if (mbmi->interinter_comp.type == COMPOUND_AVERAGE) { - if (num_planes > 1) - av1_build_inter_predictors_sbuv(cm, xd, mi_row, mi_col, &orig_dst, - bsize); + if (mbmi->interinter_comp.type == COMPOUND_AVERAGE && + is_luma_interp_done) { + if (num_planes > 1) { + av1_enc_build_inter_predictor(cm, xd, mi_row, mi_col, &orig_dst, + bsize, AOM_PLANE_U, num_planes - 1); + } skip_build_pred = 1; } } ret_val = interpolation_filter_search( - x, cpi, bsize, mi_row, mi_col, &tmp_dst, &orig_dst, + x, cpi, tile_data, bsize, mi_row, mi_col, &tmp_dst, &orig_dst, args->single_filter, &rd, &rs, &skip_txfm_sb, &skip_sse_sb, skip_build_pred, args, ref_best_rd); if (args->modelled_rd != NULL && !is_comp_pred) { @@ -9555,17 +10203,7 @@ static int64_t handle_inter_mode(const AV1_COMP *const cpi, MACROBLOCK *x, } else if (cpi->sf.model_based_post_interp_filter_breakout && ref_best_rd != INT64_MAX && (rd >> 3) * 3 > ref_best_rd) { restore_dst_buf(xd, orig_dst, num_planes); - if ((rd >> 3) * 2 > ref_best_rd) break; - continue; - } - - if (search_jnt_comp) { - // if 1/2 model rd is larger than best_rd in jnt_comp mode, - // use jnt_comp mode, save additional search - if ((rd >> 3) * 4 > best_rd) { - restore_dst_buf(xd, orig_dst, num_planes); - continue; - } + break; } if (!is_comp_pred) @@ -9587,7 +10225,8 @@ static int64_t handle_inter_mode(const AV1_COMP *const cpi, MACROBLOCK *x, } rd_stats->rate += compmode_interinter_cost; - if (search_jnt_comp && cpi->sf.jnt_comp_fast_tx_search && comp_idx == 0) { + if (search_dist_wtd_comp && cpi->sf.dist_wtd_comp_fast_tx_search && + comp_idx == 0) { // TODO(chengchen): this speed feature introduces big loss. // Need better estimation of rate distortion. int dummy_rate; @@ -9596,7 +10235,7 @@ static int64_t handle_inter_mode(const AV1_COMP *const cpi, MACROBLOCK *x, int64_t plane_sse[MAX_MB_PLANE] = { 0 }; int64_t plane_dist[MAX_MB_PLANE] = { 0 }; - model_rd_sb_fn[MODELRD_TYPE_JNT_COMPOUND]( + model_rd_sb_fn[MODELRD_TYPE_DIST_WTD_COMPOUND]( cpi, bsize, x, xd, 0, num_planes - 1, mi_row, mi_col, &dummy_rate, &dummy_dist, &skip_txfm_sb, &skip_sse_sb, plane_rate, plane_sse, plane_dist); @@ -9612,16 +10251,10 @@ static int64_t handle_inter_mode(const AV1_COMP *const cpi, MACROBLOCK *x, rd_stats_y->dist = plane_dist[0]; rd_stats_uv->dist = plane_dist[1] + plane_dist[2]; } else { -#if CONFIG_COLLECT_INTER_MODE_RD_STATS - ret_val = motion_mode_rd( - cpi, x, bsize, rd_stats, rd_stats_y, rd_stats_uv, disable_skip, - mi_row, mi_col, args, ref_best_rd, refs, &rate_mv, &orig_dst, - tile_data, best_est_rd, do_tx_search, inter_modes_info); -#else - ret_val = motion_mode_rd(cpi, x, bsize, rd_stats, rd_stats_y, + ret_val = motion_mode_rd(cpi, tile_data, x, bsize, rd_stats, rd_stats_y, rd_stats_uv, disable_skip, mi_row, mi_col, - args, ref_best_rd, refs, &rate_mv, &orig_dst); -#endif + args, ref_best_rd, refs, &rate_mv, &orig_dst, + best_est_rd, do_tx_search, inter_modes_info); } mode_info[ref_mv_idx].mv.as_int = mbmi->mv[0].as_int; mode_info[ref_mv_idx].rate_mv = rate_mv; @@ -9640,10 +10273,6 @@ static int64_t handle_inter_mode(const AV1_COMP *const cpi, MACROBLOCK *x, sizeof(best_blk_skip[0]) * xd->n4_h * xd->n4_w); } - if (tmp_rd < best_rd2) { - best_rd2 = tmp_rd; - } - if (tmp_rd < ref_best_rd) { ref_best_rd = tmp_rd; } @@ -9670,10 +10299,10 @@ static int64_t handle_inter_mode(const AV1_COMP *const cpi, MACROBLOCK *x, } static int64_t rd_pick_intrabc_mode_sb(const AV1_COMP *cpi, MACROBLOCK *x, - RD_STATS *rd_cost, BLOCK_SIZE bsize, + RD_STATS *rd_stats, BLOCK_SIZE bsize, int64_t best_rd) { const AV1_COMMON *const cm = &cpi->common; - if (!av1_allow_intrabc(cm)) return INT64_MAX; + if (!av1_allow_intrabc(cm) || !cpi->oxcf.enable_intrabc) return INT64_MAX; const int num_planes = av1_num_planes(cm); MACROBLOCKD *const xd = &x->e_mbd; @@ -9725,7 +10354,7 @@ static int64_t rd_pick_intrabc_mode_sb(const AV1_COMP *cpi, MACROBLOCK *x, }; MB_MODE_INFO best_mbmi = *mbmi; - RD_STATS best_rdcost = *rd_cost; + RD_STATS best_rdstats = *rd_stats; int best_skip = x->skip; uint8_t best_blk_skip[MAX_MIB_SIZE * MAX_MIB_SIZE] = { 0 }; @@ -9769,9 +10398,9 @@ static int64_t rd_pick_intrabc_mode_sb(const AV1_COMP *cpi, MACROBLOCK *x, MV mvp_full = dv_ref.as_mv; mvp_full.col >>= 3; mvp_full.row >>= 3; - int sadpb = x->sadperbit16; + const int sadpb = x->sadperbit16; int cost_list[5]; - int bestsme = av1_full_pixel_search( + const int bestsme = av1_full_pixel_search( cpi, x, bsize, &mvp_full, step_param, cpi->sf.mv.search_method, 0, sadpb, cond_cost_list(cpi, cost_list), &dv_ref.as_mv, INT_MAX, 1, (MI_SIZE * mi_col), (MI_SIZE * mi_row), 1); @@ -9779,7 +10408,7 @@ static int64_t rd_pick_intrabc_mode_sb(const AV1_COMP *cpi, MACROBLOCK *x, x->mv_limits = tmp_mv_limits; if (bestsme == INT_MAX) continue; mvp_full = x->best_mv.as_mv; - MV dv = { .row = mvp_full.row * 8, .col = mvp_full.col * 8 }; + const MV dv = { .row = mvp_full.row * 8, .col = mvp_full.col * 8 }; if (mv_check_bounds(&x->mv_limits, &dv)) continue; if (!av1_is_dv_valid(dv, cm, xd, mi_row, mi_col, bsize, cm->seq_params.mib_size_log2)) @@ -9798,74 +10427,39 @@ static int64_t rd_pick_intrabc_mode_sb(const AV1_COMP *cpi, MACROBLOCK *x, mbmi->interp_filters = av1_broadcast_interp_filter(BILINEAR); mbmi->skip = 0; x->skip = 0; - av1_build_inter_predictors_sb(cm, xd, mi_row, mi_col, NULL, bsize); + av1_enc_build_inter_predictor(cm, xd, mi_row, mi_col, NULL, bsize, 0, + av1_num_planes(cm) - 1); int *dvcost[2] = { (int *)&cpi->dv_cost[0][MV_MAX], (int *)&cpi->dv_cost[1][MV_MAX] }; // TODO(aconverse@google.com): The full motion field defining discount // in MV_COST_WEIGHT is too large. Explore other values. - int rate_mv = av1_mv_bit_cost(&dv, &dv_ref.as_mv, cpi->dv_joint_cost, - dvcost, MV_COST_WEIGHT_SUB); + const int rate_mv = av1_mv_bit_cost(&dv, &dv_ref.as_mv, cpi->dv_joint_cost, + dvcost, MV_COST_WEIGHT_SUB); const int rate_mode = x->intrabc_cost[1]; - RD_STATS rd_stats, rd_stats_uv; - av1_subtract_plane(x, bsize, 0); - if (cm->tx_mode == TX_MODE_SELECT && !xd->lossless[mbmi->segment_id]) { - // Intrabc - select_tx_type_yrd(cpi, x, &rd_stats, bsize, mi_row, mi_col, INT64_MAX); - } else { - super_block_yrd(cpi, x, &rd_stats, bsize, INT64_MAX); - memset(mbmi->inter_tx_size, mbmi->tx_size, sizeof(mbmi->inter_tx_size)); - for (int i = 0; i < xd->n4_h * xd->n4_w; ++i) - set_blk_skip(x, 0, i, rd_stats.skip); - } - if (num_planes > 1) { - super_block_uvrd(cpi, x, &rd_stats_uv, bsize, INT64_MAX); - av1_merge_rd_stats(&rd_stats, &rd_stats_uv); - } -#if CONFIG_RD_DEBUG - mbmi->rd_stats = rd_stats; -#endif - - const int skip_ctx = av1_get_skip_context(xd); - - RD_STATS rdc_noskip; - av1_init_rd_stats(&rdc_noskip); - rdc_noskip.rate = - rate_mode + rate_mv + rd_stats.rate + x->skip_cost[skip_ctx][0]; - rdc_noskip.dist = rd_stats.dist; - rdc_noskip.rdcost = RDCOST(x->rdmult, rdc_noskip.rate, rdc_noskip.dist); - if (rdc_noskip.rdcost < best_rd) { - best_rd = rdc_noskip.rdcost; + RD_STATS rd_stats_yuv, rd_stats_y, rd_stats_uv; + if (!txfm_search(cpi, NULL, x, bsize, mi_row, mi_col, &rd_stats_yuv, + &rd_stats_y, &rd_stats_uv, rate_mode + rate_mv, INT64_MAX)) + continue; + rd_stats_yuv.rdcost = + RDCOST(x->rdmult, rd_stats_yuv.rate, rd_stats_yuv.dist); + if (rd_stats_yuv.rdcost < best_rd) { + best_rd = rd_stats_yuv.rdcost; best_mbmi = *mbmi; - best_skip = x->skip; - best_rdcost = rdc_noskip; + best_skip = mbmi->skip; + best_rdstats = rd_stats_yuv; memcpy(best_blk_skip, x->blk_skip, sizeof(x->blk_skip[0]) * xd->n4_h * xd->n4_w); } - - if (!xd->lossless[mbmi->segment_id]) { - x->skip = 1; - mbmi->skip = 1; - RD_STATS rdc_skip; - av1_init_rd_stats(&rdc_skip); - rdc_skip.rate = rate_mode + rate_mv + x->skip_cost[skip_ctx][1]; - rdc_skip.dist = rd_stats.sse; - rdc_skip.rdcost = RDCOST(x->rdmult, rdc_skip.rate, rdc_skip.dist); - if (rdc_skip.rdcost < best_rd) { - best_rd = rdc_skip.rdcost; - best_mbmi = *mbmi; - best_skip = x->skip; - best_rdcost = rdc_skip; - memcpy(best_blk_skip, x->blk_skip, - sizeof(x->blk_skip[0]) * xd->n4_h * xd->n4_w); - } - } } *mbmi = best_mbmi; - *rd_cost = best_rdcost; + *rd_stats = best_rdstats; x->skip = best_skip; memcpy(x->blk_skip, best_blk_skip, sizeof(x->blk_skip[0]) * xd->n4_h * xd->n4_w); +#if CONFIG_RD_DEBUG + mbmi->rd_stats = *rd_stats; +#endif return best_rd; } @@ -10006,6 +10600,7 @@ static void rd_pick_skip_mode(RD_STATS *rd_cost, BLOCK_SIZE bsize, int mi_row, int mi_col, struct buf_2d yv12_mb[REF_FRAMES][MAX_MB_PLANE]) { const AV1_COMMON *const cm = &cpi->common; + const SkipModeInfo *const skip_mode_info = &cm->current_frame.skip_mode_info; const int num_planes = av1_num_planes(cm); MACROBLOCKD *const xd = &x->e_mbd; MB_MODE_INFO *const mbmi = xd->mi[0]; @@ -10014,13 +10609,15 @@ static void rd_pick_skip_mode(RD_STATS *rd_cost, RD_STATS skip_mode_rd_stats; av1_invalid_rd_stats(&skip_mode_rd_stats); - if (cm->ref_frame_idx_0 == INVALID_IDX || - cm->ref_frame_idx_1 == INVALID_IDX) { + if (skip_mode_info->ref_frame_idx_0 == INVALID_IDX || + skip_mode_info->ref_frame_idx_1 == INVALID_IDX) { return; } - const MV_REFERENCE_FRAME ref_frame = LAST_FRAME + cm->ref_frame_idx_0; - const MV_REFERENCE_FRAME second_ref_frame = LAST_FRAME + cm->ref_frame_idx_1; + const MV_REFERENCE_FRAME ref_frame = + LAST_FRAME + skip_mode_info->ref_frame_idx_0; + const MV_REFERENCE_FRAME second_ref_frame = + LAST_FRAME + skip_mode_info->ref_frame_idx_1; const PREDICTION_MODE this_mode = NEAREST_NEARESTMV; const int mode_index = get_prediction_mode_idx(this_mode, ref_frame, second_ref_frame); @@ -10085,7 +10682,8 @@ static void rd_pick_skip_mode(RD_STATS *rd_cost, rd_cost->dist) : INT64_MAX; - if (skip_mode_rd_stats.rdcost <= best_intra_inter_mode_cost) { + if (skip_mode_rd_stats.rdcost <= best_intra_inter_mode_cost && + (!xd->lossless[mbmi->segment_id] || skip_mode_rd_stats.dist == 0)) { assert(mode_index != -1); search_state->best_mbmode.skip_mode = 1; search_state->best_mbmode = *mbmi; @@ -10131,9 +10729,16 @@ static void rd_pick_skip_mode(RD_STATS *rd_cost, rd_cost->dist = rd_cost->sse = skip_mode_rd_stats.dist; rd_cost->rdcost = skip_mode_rd_stats.rdcost; +#if CONFIG_ONE_PASS_SVM + if (bsize >= BLOCK_8X8 && + block_size_high[bsize] == block_size_wide[bsize]) { + av1_copy_reg_stat(rd_cost, &skip_mode_rd_stats); + } +#endif + search_state->best_rd = rd_cost->rdcost; search_state->best_skip2 = 1; - search_state->best_mode_skippable = (skip_mode_rd_stats.sse == 0); + search_state->best_mode_skippable = 1; x->skip = 1; } @@ -10163,6 +10768,7 @@ static void sf_refine_fast_tx_type_search( !is_inter_mode(best_mbmode->mode)))) { int skip_blk = 0; RD_STATS rd_stats_y, rd_stats_uv; + const int skip_ctx = av1_get_skip_context(xd); x->use_default_inter_tx_type = 0; x->use_default_intra_tx_type = 0; @@ -10179,15 +10785,15 @@ static void sf_refine_fast_tx_type_search( } if (is_inter_mode(mbmi->mode)) { - av1_build_inter_predictors_sb(cm, xd, mi_row, mi_col, NULL, bsize); + av1_enc_build_inter_predictor(cm, xd, mi_row, mi_col, NULL, bsize, 0, + av1_num_planes(cm) - 1); if (mbmi->motion_mode == OBMC_CAUSAL) av1_build_obmc_inter_predictors_sb(cm, xd, mi_row, mi_col); av1_subtract_plane(x, bsize, 0); if (cm->tx_mode == TX_MODE_SELECT && !xd->lossless[mbmi->segment_id]) { - // av1_rd_pick_inter_mode_sb - select_tx_type_yrd(cpi, x, &rd_stats_y, bsize, mi_row, mi_col, - INT64_MAX); + pick_tx_size_type_yrd(cpi, x, &rd_stats_y, bsize, mi_row, mi_col, + INT64_MAX); assert(rd_stats_y.rate != INT_MAX); } else { super_block_yrd(cpi, x, &rd_stats_y, bsize, INT64_MAX); @@ -10195,32 +10801,29 @@ static void sf_refine_fast_tx_type_search( for (int i = 0; i < xd->n4_h * xd->n4_w; ++i) set_blk_skip(x, 0, i, rd_stats_y.skip); } - if (num_planes > 1) { - inter_block_uvrd(cpi, x, &rd_stats_uv, bsize, INT64_MAX, INT64_MAX, - FTXS_NONE); - } else { - av1_init_rd_stats(&rd_stats_uv); - } } else { super_block_yrd(cpi, x, &rd_stats_y, bsize, INT64_MAX); - if (num_planes > 1) { - super_block_uvrd(cpi, x, &rd_stats_uv, bsize, INT64_MAX); - } else { - av1_init_rd_stats(&rd_stats_uv); - } } - if (RDCOST(x->rdmult, rd_stats_y.rate + rd_stats_uv.rate, + if (num_planes > 1) { + super_block_uvrd(cpi, x, &rd_stats_uv, bsize, INT64_MAX); + } else { + av1_init_rd_stats(&rd_stats_uv); + } + + if (RDCOST(x->rdmult, + x->skip_cost[skip_ctx][0] + rd_stats_y.rate + rd_stats_uv.rate, (rd_stats_y.dist + rd_stats_uv.dist)) > - RDCOST(x->rdmult, 0, (rd_stats_y.sse + rd_stats_uv.sse))) { + RDCOST(x->rdmult, x->skip_cost[skip_ctx][1], + (rd_stats_y.sse + rd_stats_uv.sse))) { skip_blk = 1; - rd_stats_y.rate = x->skip_cost[av1_get_skip_context(xd)][1]; + rd_stats_y.rate = x->skip_cost[skip_ctx][1]; rd_stats_uv.rate = 0; rd_stats_y.dist = rd_stats_y.sse; rd_stats_uv.dist = rd_stats_uv.sse; } else { skip_blk = 0; - rd_stats_y.rate += x->skip_cost[av1_get_skip_context(xd)][0]; + rd_stats_y.rate += x->skip_cost[skip_ctx][0]; } if (RDCOST(x->rdmult, best_rate_y + best_rate_uv, rd_cost->dist) > @@ -10240,13 +10843,137 @@ static void sf_refine_fast_tx_type_search( } } +// Contains information on which modes to skip. The last entry contains +// information on whether the reference frames should should be skipped. +typedef struct mode_skip_mask_struct { + uint32_t mode[REF_FRAMES]; + uint16_t ref_frame1; + uint16_t ref_frame2; +} mode_skip_mask_t; + +static void init_mode_skip_mask(mode_skip_mask_t *mask, const AV1_COMP *cpi, + MACROBLOCK *x, BLOCK_SIZE bsize) { + const AV1_COMMON *const cm = &cpi->common; + const struct segmentation *const seg = &cm->seg; + MACROBLOCKD *const xd = &x->e_mbd; + MB_MODE_INFO *const mbmi = xd->mi[0]; + unsigned char segment_id = mbmi->segment_id; + const SPEED_FEATURES *const sf = &cpi->sf; + + memset(mask, 0, sizeof(*mask)); + + int min_pred_mv_sad = INT_MAX; + MV_REFERENCE_FRAME ref_frame; + for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) + min_pred_mv_sad = AOMMIN(min_pred_mv_sad, x->pred_mv_sad[ref_frame]); + + for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) { + if (!(cpi->ref_frame_flags & ref_frame_flag_list[ref_frame])) { + // Skip checking missing references in both single and compound reference + // modes. Note that a mode will be skipped iff both reference frames + // are masked out. + mask->ref_frame1 |= (1 << ref_frame); + mask->ref_frame2 |= SECOND_REF_FRAME_MASK; + } else { + // Skip fixed mv modes for poor references + if ((x->pred_mv_sad[ref_frame] >> 2) > min_pred_mv_sad) { + mask->mode[ref_frame] |= INTER_NEAREST_NEAR_ZERO; + } + } + // If the segment reference frame feature is enabled.... + // then do nothing if the current ref frame is not allowed.. + if (segfeature_active(seg, segment_id, SEG_LVL_REF_FRAME) && + get_segdata(seg, segment_id, SEG_LVL_REF_FRAME) != (int)ref_frame) { + mask->ref_frame1 |= (1 << ref_frame); + mask->ref_frame2 |= SECOND_REF_FRAME_MASK; + } + } + // Disable this drop out case if the ref frame + // segment level feature is enabled for this segment. This is to + // prevent the possibility that we end up unable to pick any mode. + if (!segfeature_active(seg, segment_id, SEG_LVL_REF_FRAME)) { + // Only consider GLOBALMV/ALTREF_FRAME for alt ref frame, + // unless ARNR filtering is enabled in which case we want + // an unfiltered alternative. We allow near/nearest as well + // because they may result in zero-zero MVs but be cheaper. + if (cpi->rc.is_src_frame_alt_ref && (cpi->oxcf.arnr_max_frames == 0)) { + mask->ref_frame1 = (1 << LAST_FRAME) | (1 << LAST2_FRAME) | + (1 << LAST3_FRAME) | (1 << BWDREF_FRAME) | + (1 << ALTREF2_FRAME) | (1 << GOLDEN_FRAME); + mask->ref_frame2 = SECOND_REF_FRAME_MASK; + // TODO(zoeliu): To further explore whether following needs to be done for + // BWDREF_FRAME as well. + mask->mode[ALTREF_FRAME] = ~INTER_NEAREST_NEAR_ZERO; + const MV_REFERENCE_FRAME tmp_ref_frames[2] = { ALTREF_FRAME, NONE_FRAME }; + int_mv near_mv, nearest_mv, global_mv; + get_this_mv(&nearest_mv, NEARESTMV, 0, 0, tmp_ref_frames, x->mbmi_ext); + get_this_mv(&near_mv, NEARMV, 0, 0, tmp_ref_frames, x->mbmi_ext); + get_this_mv(&global_mv, GLOBALMV, 0, 0, tmp_ref_frames, x->mbmi_ext); + + if (near_mv.as_int != global_mv.as_int) + mask->mode[ALTREF_FRAME] |= (1 << NEARMV); + if (nearest_mv.as_int != global_mv.as_int) + mask->mode[ALTREF_FRAME] |= (1 << NEARESTMV); + } + } + + if (cpi->rc.is_src_frame_alt_ref) { + if (sf->alt_ref_search_fp) { + assert(cpi->ref_frame_flags & ref_frame_flag_list[ALTREF_FRAME]); + mask->mode[ALTREF_FRAME] = 0; + mask->ref_frame1 = ~(1 << ALTREF_FRAME); + mask->ref_frame2 = SECOND_REF_FRAME_MASK; + } + } + + if (sf->alt_ref_search_fp) + if (!cm->show_frame && x->pred_mv_sad[GOLDEN_FRAME] < INT_MAX) + if (x->pred_mv_sad[ALTREF_FRAME] > (x->pred_mv_sad[GOLDEN_FRAME] << 1)) + mask->mode[ALTREF_FRAME] |= INTER_ALL; + + if (sf->adaptive_mode_search) { + if (cm->show_frame && !cpi->rc.is_src_frame_alt_ref && + cpi->rc.frames_since_golden >= 3) + if ((x->pred_mv_sad[GOLDEN_FRAME] >> 1) > x->pred_mv_sad[LAST_FRAME]) + mask->mode[GOLDEN_FRAME] |= INTER_ALL; + } + + if (bsize > sf->max_intra_bsize) { + mask->ref_frame1 |= (1 << INTRA_FRAME); + mask->ref_frame2 |= (1 << INTRA_FRAME); + } + + mask->mode[INTRA_FRAME] |= ~(sf->intra_y_mode_mask[max_txsize_lookup[bsize]]); + + // Disable some frames to enforce max number of reference frames requested. + // TODO(urvang): Should this logic be moved to enforce_max_ref_frames()? + const MV_REFERENCE_FRAME disable_order[] = { + LAST3_FRAME, LAST2_FRAME, ALTREF2_FRAME, BWDREF_FRAME, + // GOLDEN_FRAME, + }; + assert(cpi->oxcf.max_reference_frames <= INTER_REFS_PER_FRAME); + const int num_frames_to_disable = + INTER_REFS_PER_FRAME - cpi->oxcf.max_reference_frames; + for (int i = 0; i < num_frames_to_disable; ++i) { + const MV_REFERENCE_FRAME ref_frame_to_disable = disable_order[i]; + + mask->ref_frame1 |= (1 << ref_frame_to_disable); + mask->ref_frame2 |= (1 << ref_frame_to_disable); + + if (ref_frame_to_disable == BWDREF_FRAME && + (cpi->ref_frame_flags & AOM_BWD_FLAG)) { + mask->ref_frame1 |= (1 << ALTREF_FRAME); + mask->ref_frame2 |= (1 << ALTREF_FRAME); + } + } +} + // Please add/modify parameter setting in this function, making it consistent // and easy to read and maintain. static void set_params_rd_pick_inter_mode( const AV1_COMP *cpi, MACROBLOCK *x, HandleInterModeArgs *args, - BLOCK_SIZE bsize, int mi_row, int mi_col, uint16_t ref_frame_skip_mask[2], - uint32_t mode_skip_mask[REF_FRAMES], int skip_ref_frame_mask, - unsigned int ref_costs_single[REF_FRAMES], + BLOCK_SIZE bsize, int mi_row, int mi_col, mode_skip_mask_t *mode_skip_mask, + int skip_ref_frame_mask, unsigned int ref_costs_single[REF_FRAMES], unsigned int ref_costs_comp[REF_FRAMES][REF_FRAMES], struct buf_2d yv12_mb[REF_FRAMES][MAX_MB_PLANE]) { const AV1_COMMON *const cm = &cpi->common; @@ -10254,8 +10981,6 @@ static void set_params_rd_pick_inter_mode( MACROBLOCKD *const xd = &x->e_mbd; MB_MODE_INFO *const mbmi = xd->mi[0]; MB_MODE_INFO_EXT *const mbmi_ext = x->mbmi_ext; - const struct segmentation *const seg = &cm->seg; - const SPEED_FEATURES *const sf = &cpi->sf; unsigned char segment_id = mbmi->segment_id; int dst_width1[MAX_MB_PLANE] = { MAX_SB_SIZE, MAX_SB_SIZE, MAX_SB_SIZE }; int dst_width2[MAX_MB_PLANE] = { MAX_SB_SIZE >> 1, MAX_SB_SIZE >> 1, @@ -10267,7 +10992,7 @@ static void set_params_rd_pick_inter_mode( for (int i = 0; i < MB_MODE_COUNT; ++i) for (int k = 0; k < REF_FRAMES; ++k) args->single_filter[i][k] = SWITCHABLE; - if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + if (is_cur_buf_hbd(xd)) { int len = sizeof(uint16_t); args->above_pred_buf[0] = CONVERT_TO_BYTEPTR(x->above_pred_buf); args->above_pred_buf[1] = @@ -10316,7 +11041,7 @@ static void set_params_rd_pick_inter_mode( if (skip) continue; } } - assert(get_ref_frame_buffer(cpi, ref_frame) != NULL); + assert(get_ref_frame_yv12_buf(cm, ref_frame) != NULL); setup_buffer_ref_mvs_inter(cpi, x, ref_frame, bsize, mi_row, mi_col, yv12_mb); } @@ -10352,101 +11077,15 @@ static void set_params_rd_pick_inter_mode( av1_build_prediction_by_left_preds(cm, xd, mi_row, mi_col, args->left_pred_buf, dst_width2, dst_height2, args->left_pred_stride); - av1_setup_dst_planes(xd->plane, bsize, get_frame_new_buffer(cm), mi_row, - mi_col, 0, num_planes); + av1_setup_dst_planes(xd->plane, bsize, &cm->cur_frame->buf, mi_row, mi_col, + 0, num_planes); calc_target_weighted_pred( cm, x, xd, mi_row, mi_col, args->above_pred_buf[0], args->above_pred_stride[0], args->left_pred_buf[0], args->left_pred_stride[0]); } - int min_pred_mv_sad = INT_MAX; - for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) - min_pred_mv_sad = AOMMIN(min_pred_mv_sad, x->pred_mv_sad[ref_frame]); - - for (int i = 0; i < 2; ++i) { - ref_frame_skip_mask[i] = 0; - } - memset(mode_skip_mask, 0, REF_FRAMES * sizeof(*mode_skip_mask)); - for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) { - if (!(cpi->ref_frame_flags & ref_frame_flag_list[ref_frame])) { - // Skip checking missing references in both single and compound reference - // modes. Note that a mode will be skipped iff both reference frames - // are masked out. - ref_frame_skip_mask[0] |= (1 << ref_frame); - ref_frame_skip_mask[1] |= SECOND_REF_FRAME_MASK; - } else { - // Skip fixed mv modes for poor references - if ((x->pred_mv_sad[ref_frame] >> 2) > min_pred_mv_sad) { - mode_skip_mask[ref_frame] |= INTER_NEAREST_NEAR_ZERO; - } - } - // If the segment reference frame feature is enabled.... - // then do nothing if the current ref frame is not allowed.. - if (segfeature_active(seg, segment_id, SEG_LVL_REF_FRAME) && - get_segdata(seg, segment_id, SEG_LVL_REF_FRAME) != (int)ref_frame) { - ref_frame_skip_mask[0] |= (1 << ref_frame); - ref_frame_skip_mask[1] |= SECOND_REF_FRAME_MASK; - } - } - - // Disable this drop out case if the ref frame - // segment level feature is enabled for this segment. This is to - // prevent the possibility that we end up unable to pick any mode. - if (!segfeature_active(seg, segment_id, SEG_LVL_REF_FRAME)) { - // Only consider GLOBALMV/ALTREF_FRAME for alt ref frame, - // unless ARNR filtering is enabled in which case we want - // an unfiltered alternative. We allow near/nearest as well - // because they may result in zero-zero MVs but be cheaper. - if (cpi->rc.is_src_frame_alt_ref && (cpi->oxcf.arnr_max_frames == 0)) { - ref_frame_skip_mask[0] = (1 << LAST_FRAME) | (1 << LAST2_FRAME) | - (1 << LAST3_FRAME) | (1 << BWDREF_FRAME) | - (1 << ALTREF2_FRAME) | (1 << GOLDEN_FRAME); - ref_frame_skip_mask[1] = SECOND_REF_FRAME_MASK; - // TODO(zoeliu): To further explore whether following needs to be done for - // BWDREF_FRAME as well. - mode_skip_mask[ALTREF_FRAME] = ~INTER_NEAREST_NEAR_ZERO; - const MV_REFERENCE_FRAME tmp_ref_frames[2] = { ALTREF_FRAME, NONE_FRAME }; - int_mv near_mv, nearest_mv, global_mv; - get_this_mv(&nearest_mv, NEARESTMV, 0, 0, tmp_ref_frames, x->mbmi_ext); - get_this_mv(&near_mv, NEARMV, 0, 0, tmp_ref_frames, x->mbmi_ext); - get_this_mv(&global_mv, GLOBALMV, 0, 0, tmp_ref_frames, x->mbmi_ext); - - if (near_mv.as_int != global_mv.as_int) - mode_skip_mask[ALTREF_FRAME] |= (1 << NEARMV); - if (nearest_mv.as_int != global_mv.as_int) - mode_skip_mask[ALTREF_FRAME] |= (1 << NEARESTMV); - } - } - - if (cpi->rc.is_src_frame_alt_ref) { - if (sf->alt_ref_search_fp) { - assert(cpi->ref_frame_flags & ref_frame_flag_list[ALTREF_FRAME]); - mode_skip_mask[ALTREF_FRAME] = 0; - ref_frame_skip_mask[0] = ~(1 << ALTREF_FRAME); - ref_frame_skip_mask[1] = SECOND_REF_FRAME_MASK; - } - } - - if (sf->alt_ref_search_fp) - if (!cm->show_frame && x->pred_mv_sad[GOLDEN_FRAME] < INT_MAX) - if (x->pred_mv_sad[ALTREF_FRAME] > (x->pred_mv_sad[GOLDEN_FRAME] << 1)) - mode_skip_mask[ALTREF_FRAME] |= INTER_ALL; - - if (sf->adaptive_mode_search) { - if (cm->show_frame && !cpi->rc.is_src_frame_alt_ref && - cpi->rc.frames_since_golden >= 3) - if ((x->pred_mv_sad[GOLDEN_FRAME] >> 1) > x->pred_mv_sad[LAST_FRAME]) - mode_skip_mask[GOLDEN_FRAME] |= INTER_ALL; - } - - if (bsize > sf->max_intra_bsize) { - ref_frame_skip_mask[0] |= (1 << INTRA_FRAME); - ref_frame_skip_mask[1] |= (1 << INTRA_FRAME); - } - - mode_skip_mask[INTRA_FRAME] |= - ~(sf->intra_y_mode_mask[max_txsize_lookup[bsize]]); + init_mode_skip_mask(mode_skip_mask, cpi, x, bsize); if (cpi->sf.tx_type_search.fast_intra_tx_type_search) x->use_default_intra_tx_type = 1; @@ -10461,6 +11100,7 @@ static void set_params_rd_pick_inter_mode( x->interp_filter_stats_idx[0] = 0; x->interp_filter_stats_idx[1] = 0; } + x->comp_rd_stats_idx = 0; } static void search_palette_mode(const AV1_COMP *cpi, MACROBLOCK *x, int mi_row, @@ -10538,6 +11178,9 @@ static void search_palette_mode(const AV1_COMP *cpi, MACROBLOCK *x, int mi_row, rate2 -= rd_stats_y.rate; if (num_planes > 1) rate2 -= search_state->rate_uv_tokenonly[uv_tx]; rate2 += x->skip_cost[av1_get_skip_context(xd)][1]; +#if CONFIG_ONE_PASS_SVM + av1_reg_stat_skipmode_update(&rd_stats_y, x->rdmult); +#endif } else { rate2 += x->skip_cost[av1_get_skip_context(xd)][0]; } @@ -10554,6 +11197,9 @@ static void search_palette_mode(const AV1_COMP *cpi, MACROBLOCK *x, int mi_row, search_state->best_mode_skippable = skippable; memcpy(ctx->blk_skip, x->blk_skip, sizeof(x->blk_skip[0]) * ctx->num_4x4_blk); +#if CONFIG_ONE_PASS_SVM + av1_copy_reg_stat(rd_cost, &rd_stats_y); +#endif } } @@ -10598,6 +11244,7 @@ static void init_inter_mode_search_state(InterModeSearchState *search_state, search_state->best_intra_rd = INT64_MAX; search_state->angle_stats_ready = 0; + av1_zero(search_state->directional_mode_skip_mask); search_state->best_pred_sse = UINT_MAX; @@ -10647,23 +11294,81 @@ static void init_inter_mode_search_state(InterModeSearchState *search_state, av1_zero(search_state->single_state_modelled_cnt); } +bool mask_says_skip(const mode_skip_mask_t *mode_skip_mask, + const MV_REFERENCE_FRAME *ref_frame, + const PREDICTION_MODE this_mode) { + if (mode_skip_mask->mode[ref_frame[0]] & (1 << this_mode)) { + return true; + } + + return (mode_skip_mask->ref_frame1 & (1 << ref_frame[0])) && + (mode_skip_mask->ref_frame2 & (1 << AOMMAX(0, ref_frame[1]))); +} + +static int inter_mode_compatible_skip(const AV1_COMP *cpi, const MACROBLOCK *x, + BLOCK_SIZE bsize, int mode_index) { + const AV1_COMMON *const cm = &cpi->common; + const struct segmentation *const seg = &cm->seg; + const MV_REFERENCE_FRAME *ref_frame = av1_mode_order[mode_index].ref_frame; + const PREDICTION_MODE this_mode = av1_mode_order[mode_index].mode; + const CurrentFrame *const current_frame = &cm->current_frame; + const MACROBLOCKD *const xd = &x->e_mbd; + const MB_MODE_INFO *const mbmi = xd->mi[0]; + const unsigned char segment_id = mbmi->segment_id; + const int comp_pred = ref_frame[1] > INTRA_FRAME; + + if (comp_pred) { + if (frame_is_intra_only(cm)) return 1; + + if (current_frame->reference_mode == SINGLE_REFERENCE) return 1; + + // Skip compound inter modes if ARF is not available. + if (!(cpi->ref_frame_flags & ref_frame_flag_list[ref_frame[1]])) return 1; + + // Do not allow compound prediction if the segment level reference frame + // feature is in use as in this case there can only be one reference. + if (segfeature_active(seg, segment_id, SEG_LVL_REF_FRAME)) return 1; + + if (!is_comp_ref_allowed(bsize)) return 1; + } + + if (ref_frame[0] > INTRA_FRAME && ref_frame[1] == INTRA_FRAME) { + // Mode must by compatible + if (!is_interintra_allowed_mode(this_mode)) return 1; + if (!is_interintra_allowed_bsize(bsize)) return 1; + } + + return 0; +} + // Case 1: return 0, means don't skip this mode // Case 2: return 1, means skip this mode completely // Case 3: return 2, means skip compound only, but still try single motion modes static int inter_mode_search_order_independent_skip( const AV1_COMP *cpi, const PICK_MODE_CONTEXT *ctx, const MACROBLOCK *x, BLOCK_SIZE bsize, int mode_index, int mi_row, int mi_col, - uint32_t *mode_skip_mask, uint16_t *ref_frame_skip_mask, - InterModeSearchState *search_state) { + mode_skip_mask_t *mode_skip_mask, InterModeSearchState *search_state) { const SPEED_FEATURES *const sf = &cpi->sf; const AV1_COMMON *const cm = &cpi->common; - const struct segmentation *const seg = &cm->seg; + const OrderHintInfo *const order_hint_info = &cm->seq_params.order_hint_info; + const CurrentFrame *const current_frame = &cm->current_frame; const MACROBLOCKD *const xd = &x->e_mbd; const MB_MODE_INFO *const mbmi = xd->mi[0]; - const unsigned char segment_id = mbmi->segment_id; const MV_REFERENCE_FRAME *ref_frame = av1_mode_order[mode_index].ref_frame; const PREDICTION_MODE this_mode = av1_mode_order[mode_index].mode; + const int comp_pred = ref_frame[1] > INTRA_FRAME; int skip_motion_mode = 0; + + if (mask_says_skip(mode_skip_mask, ref_frame, this_mode)) { + return 1; + } + + // If no valid mode has been found so far in PARTITION_NONE when finding a + // valid partition is required, do not skip mode. + if (search_state->best_rd == INT64_MAX && mbmi->partition == PARTITION_NONE && + x->must_find_valid_partition) + return 0; + if (mbmi->partition != PARTITION_NONE && mbmi->partition != PARTITION_SPLIT) { const int ref_type = av1_ref_frame_type(ref_frame); int skip_ref = ctx->skip_ref_frame_mask & (1 << ref_type); @@ -10714,12 +11419,6 @@ static int inter_mode_search_order_independent_skip( if (!found) return 1; } - if (ref_frame[0] > INTRA_FRAME && ref_frame[1] == INTRA_FRAME) { - // Mode must by compatible - if (!is_interintra_allowed_mode(this_mode)) return 1; - if (!is_interintra_allowed_bsize(bsize)) return 1; - } - // This is only used in motion vector unit test. if (cpi->oxcf.motion_vector_unit_test && ref_frame[0] == INTRA_FRAME) return 1; @@ -10734,71 +11433,94 @@ static int inter_mode_search_order_independent_skip( x->source_variance < skip_intra_var_thresh) return 1; } - } else { - if (!is_comp_ref_allowed(bsize) && ref_frame[1] > INTRA_FRAME) return 1; - } - - const int comp_pred = ref_frame[1] > INTRA_FRAME; - if (comp_pred) { - if (!cpi->allow_comp_inter_inter) return 1; - - if (cm->reference_mode == SINGLE_REFERENCE) return 1; - - // Skip compound inter modes if ARF is not available. - if (!(cpi->ref_frame_flags & ref_frame_flag_list[ref_frame[1]])) return 1; - - // Do not allow compound prediction if the segment level reference frame - // feature is in use as in this case there can only be one reference. - if (segfeature_active(seg, segment_id, SEG_LVL_REF_FRAME)) return 1; } if (sf->selective_ref_frame) { - if (sf->selective_ref_frame >= 2 || x->cb_partition_scan) { + if (sf->selective_ref_frame >= 3 || x->cb_partition_scan) { if (ref_frame[0] == ALTREF2_FRAME || ref_frame[1] == ALTREF2_FRAME) if (get_relative_dist( - cm, cm->cur_frame->ref_frame_offset[ALTREF2_FRAME - LAST_FRAME], - cm->frame_offset) < 0) + order_hint_info, + cm->cur_frame->ref_order_hints[ALTREF2_FRAME - LAST_FRAME], + current_frame->order_hint) < 0) return 1; if (ref_frame[0] == BWDREF_FRAME || ref_frame[1] == BWDREF_FRAME) if (get_relative_dist( - cm, cm->cur_frame->ref_frame_offset[BWDREF_FRAME - LAST_FRAME], - cm->frame_offset) < 0) + order_hint_info, + cm->cur_frame->ref_order_hints[BWDREF_FRAME - LAST_FRAME], + current_frame->order_hint) < 0) + return 1; + } + + if (sf->selective_ref_frame >= 2 || + (sf->selective_ref_frame == 1 && comp_pred)) { + if (ref_frame[0] == LAST3_FRAME || ref_frame[1] == LAST3_FRAME) + if (get_relative_dist( + order_hint_info, + cm->cur_frame->ref_order_hints[LAST3_FRAME - LAST_FRAME], + cm->cur_frame->ref_order_hints[GOLDEN_FRAME - LAST_FRAME]) <= 0) + return 1; + if (ref_frame[0] == LAST2_FRAME || ref_frame[1] == LAST2_FRAME) + if (get_relative_dist( + order_hint_info, + cm->cur_frame->ref_order_hints[LAST2_FRAME - LAST_FRAME], + cm->cur_frame->ref_order_hints[GOLDEN_FRAME - LAST_FRAME]) <= 0) return 1; } - if (ref_frame[0] == LAST3_FRAME || ref_frame[1] == LAST3_FRAME) - if (get_relative_dist( - cm, cm->cur_frame->ref_frame_offset[LAST3_FRAME - LAST_FRAME], - cm->cur_frame->ref_frame_offset[GOLDEN_FRAME - LAST_FRAME]) <= 0) - return 1; - if (ref_frame[0] == LAST2_FRAME || ref_frame[1] == LAST2_FRAME) - if (get_relative_dist( - cm, cm->cur_frame->ref_frame_offset[LAST2_FRAME - LAST_FRAME], - cm->cur_frame->ref_frame_offset[GOLDEN_FRAME - LAST_FRAME]) <= 0) - return 1; } // One-sided compound is used only when all reference frames are one-sided. - if (sf->selective_ref_frame && comp_pred && !cpi->all_one_sided_refs) { + if ((sf->selective_ref_frame >= 2) && comp_pred && !cpi->all_one_sided_refs) { unsigned int ref_offsets[2]; for (int i = 0; i < 2; ++i) { - const int buf_idx = cm->frame_refs[ref_frame[i] - LAST_FRAME].idx; - assert(buf_idx >= 0); - ref_offsets[i] = cm->buffer_pool->frame_bufs[buf_idx].cur_frame_offset; + const RefCntBuffer *const buf = get_ref_frame_buf(cm, ref_frame[i]); + assert(buf != NULL); + ref_offsets[i] = buf->order_hint; } - if ((get_relative_dist(cm, ref_offsets[0], cm->frame_offset) <= 0 && - get_relative_dist(cm, ref_offsets[1], cm->frame_offset) <= 0) || - (get_relative_dist(cm, ref_offsets[0], cm->frame_offset) > 0 && - get_relative_dist(cm, ref_offsets[1], cm->frame_offset) > 0)) + if ((get_relative_dist(order_hint_info, ref_offsets[0], + current_frame->order_hint) <= 0 && + get_relative_dist(order_hint_info, ref_offsets[1], + current_frame->order_hint) <= 0) || + (get_relative_dist(order_hint_info, ref_offsets[0], + current_frame->order_hint) > 0 && + get_relative_dist(order_hint_info, ref_offsets[1], + current_frame->order_hint) > 0)) return 1; } - if (mode_skip_mask[ref_frame[0]] & (1 << this_mode)) { - return 1; - } - - if ((ref_frame_skip_mask[0] & (1 << ref_frame[0])) && - (ref_frame_skip_mask[1] & (1 << AOMMAX(0, ref_frame[1])))) { - return 1; + if (sf->selective_ref_frame >= 4 && comp_pred) { + // Check if one of the reference is ALTREF2_FRAME and BWDREF_FRAME is a + // valid reference. + if ((ref_frame[0] == ALTREF2_FRAME || ref_frame[1] == ALTREF2_FRAME) && + (cpi->ref_frame_flags & ref_frame_flag_list[BWDREF_FRAME])) { + // Check if both ALTREF2_FRAME and BWDREF_FRAME are future references. + if ((get_relative_dist( + order_hint_info, + cm->cur_frame->ref_order_hints[ALTREF2_FRAME - LAST_FRAME], + current_frame->order_hint) > 0) && + (get_relative_dist( + order_hint_info, + cm->cur_frame->ref_order_hints[BWDREF_FRAME - LAST_FRAME], + current_frame->order_hint) > 0)) { + // Drop ALTREF2_FRAME as a reference if BWDREF_FRAME is a closer + // reference to the current frame than ALTREF2_FRAME + if (get_relative_dist( + order_hint_info, + cm->cur_frame->ref_order_hints[ALTREF2_FRAME - LAST_FRAME], + cm->cur_frame->ref_order_hints[BWDREF_FRAME - LAST_FRAME]) >= + 0) { + const RefCntBuffer *const buf_arf2 = + get_ref_frame_buf(cm, ALTREF2_FRAME); + assert(buf_arf2 != NULL); + const RefCntBuffer *const buf_bwd = + get_ref_frame_buf(cm, BWDREF_FRAME); + assert(buf_bwd != NULL); + assert(buf_arf2->frame_rf_level == buf_bwd->frame_rf_level); + (void)buf_arf2; + (void)buf_bwd; + return 1; + } + } + } } if (skip_repeated_mv(cm, x, this_mode, ref_frame, search_state)) { @@ -10864,18 +11586,15 @@ static int64_t handle_intra_mode(InterModeSearchState *search_state, TX_SIZE uv_tx; int is_directional_mode = av1_is_directional_mode(mbmi->mode); - if (is_directional_mode && av1_use_angle_delta(bsize)) { + if (is_directional_mode && av1_use_angle_delta(bsize) && + cpi->oxcf.enable_angle_delta) { int rate_dummy; int64_t model_rd = INT64_MAX; - if (!search_state->angle_stats_ready) { + if (sf->intra_angle_estimation && !search_state->angle_stats_ready) { const int src_stride = x->plane[0].src.stride; const uint8_t *src = x->plane[0].src.buf; - if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) - highbd_angle_estimation(src, src_stride, rows, cols, bsize, - search_state->directional_mode_skip_mask); - else - angle_estimation(src, src_stride, rows, cols, bsize, - search_state->directional_mode_skip_mask); + angle_estimation(src, src_stride, rows, cols, bsize, is_cur_buf_hbd(xd), + search_state->directional_mode_skip_mask); search_state->angle_stats_ready = 1; } if (search_state->directional_mode_skip_mask[mbmi->mode]) return INT64_MAX; @@ -11110,6 +11829,8 @@ static void analyze_single_states(const AV1_COMP *cpi, for (dir = 0; dir < 2; ++dir) { int64_t best_rd; SingleInterModeState(*state)[FWD_REFS]; + const int prune_factor = + cpi->sf.prune_comp_search_by_single_result >= 2 ? 6 : 5; // Use the best rd of GLOBALMV or NEWMV to prune the unlikely // reference frames for all the modes (NEARESTMV and NEARMV may not @@ -11121,7 +11842,7 @@ static void analyze_single_states(const AV1_COMP *cpi, for (mode = 0; mode < SINGLE_INTER_MODE_NUM; ++mode) { for (i = 1; i < search_state->single_state_cnt[dir][mode]; ++i) { if (state[mode][i].rd != INT64_MAX && - (state[mode][i].rd >> 1) > best_rd) { + (state[mode][i].rd >> 3) * prune_factor > best_rd) { state[mode][i].valid = 0; } } @@ -11134,7 +11855,7 @@ static void analyze_single_states(const AV1_COMP *cpi, for (i = 1; i < search_state->single_state_modelled_cnt[dir][mode]; ++i) { if (state[mode][i].rd != INT64_MAX && - (state[mode][i].rd >> 1) > best_rd) { + (state[mode][i].rd >> 3) * prune_factor > best_rd) { state[mode][i].valid = 0; } } @@ -11338,6 +12059,44 @@ static INLINE void sf_drop_ref_analyze(InterModeSearchState *search_state, } } +// sf->prune_single_motion_modes_by_simple_trans +static int analyze_simple_trans_states(const AV1_COMP *cpi, MACROBLOCK *x) { + (void)cpi; + int64_t rdcosts[REF_FRAMES] = { INT64_MAX, INT64_MAX, INT64_MAX, INT64_MAX, + INT64_MAX, INT64_MAX, INT64_MAX, INT64_MAX }; + int skip_ref = 0; + int64_t min_rd = INT64_MAX; + for (int i = 0; i < SINGLE_REF_MODES; ++i) { + const MODE_DEFINITION *mode_order = &av1_mode_order[i]; + const MV_REFERENCE_FRAME ref_frame = mode_order->ref_frame[0]; + for (int k = 0; k < MAX_REF_MV_SERCH; ++k) { + const int64_t rd = x->simple_rd_state[i][k].rd_stats.rdcost; + rdcosts[ref_frame] = AOMMIN(rdcosts[ref_frame], rd); + min_rd = AOMMIN(min_rd, rd); + } + } + int valid_cnt = 0; + for (int i = 1; i < REF_FRAMES; ++i) { + if (rdcosts[i] == INT64_MAX) { + skip_ref |= (1 << i); + } else { + valid_cnt++; + } + } + if (valid_cnt < 2) { + return 0; + } + min_rd += (min_rd >> 1); + if (valid_cnt > 2) { + for (int i = 1; i < REF_FRAMES; ++i) { + if (rdcosts[i] > min_rd) { + skip_ref |= (1 << i); + } + } + } + return skip_ref; +} + static void alloc_compound_type_rd_buffers(AV1_COMMON *const cm, CompoundTypeRdBuffers *const bufs) { CHECK_MEM_ERROR( @@ -11377,6 +12136,7 @@ void av1_rd_pick_inter_mode_sb(AV1_COMP *cpi, TileDataEnc *tile_data, MACROBLOCKD *const xd = &x->e_mbd; MB_MODE_INFO *const mbmi = xd->mi[0]; const int try_palette = + cpi->oxcf.enable_palette && av1_allow_palette(cm->allow_screen_content_tools, mbmi->sb_type); PALETTE_MODE_INFO *const pmi = &mbmi->palette_mode_info; const struct segmentation *const seg = &cm->seg; @@ -11387,9 +12147,14 @@ void av1_rd_pick_inter_mode_sb(AV1_COMP *cpi, TileDataEnc *tile_data, unsigned int ref_costs_single[REF_FRAMES]; unsigned int ref_costs_comp[REF_FRAMES][REF_FRAMES]; int *comp_inter_cost = x->comp_inter_cost[av1_get_reference_mode_context(xd)]; - int *mode_map = tile_data->mode_map[bsize]; - uint32_t mode_skip_mask[REF_FRAMES]; - uint16_t ref_frame_skip_mask[2]; + mode_skip_mask_t mode_skip_mask; + uint8_t motion_mode_skip_mask = 0; // second pass of single ref modes +#if CONFIG_ONE_PASS_SVM + int temp_y_eob = 0, temp_y_eob_0 = 0, temp_y_eob_1 = 0, temp_y_eob_2 = 0, + temp_y_eob_3 = 0; + int64_t temp_y_rd = 0, temp_y_rd_0 = 0, temp_y_rd_1 = 0, temp_y_rd_2 = 0, + temp_y_rd_3 = 0; +#endif InterModeSearchState search_state; init_inter_mode_search_state(&search_state, cpi, tile_data, x, bsize, @@ -11405,64 +12170,71 @@ void av1_rd_pick_inter_mode_sb(AV1_COMP *cpi, TileDataEnc *tile_data, NULL, search_state.modelled_rd, { { 0 } }, INT_MAX, INT_MAX, search_state.simple_rd, - 0, interintra_modes + 0, interintra_modes, + 1, NULL }; for (i = 0; i < REF_FRAMES; ++i) x->pred_sse[i] = INT_MAX; av1_invalid_rd_stats(rd_cost); // init params, set frame modes, speed features - set_params_rd_pick_inter_mode( - cpi, x, &args, bsize, mi_row, mi_col, ref_frame_skip_mask, mode_skip_mask, - ctx->skip_ref_frame_mask, ref_costs_single, ref_costs_comp, yv12_mb); + set_params_rd_pick_inter_mode(cpi, x, &args, bsize, mi_row, mi_col, + &mode_skip_mask, ctx->skip_ref_frame_mask, + ref_costs_single, ref_costs_comp, yv12_mb); -#if CONFIG_COLLECT_INTER_MODE_RD_STATS int64_t best_est_rd = INT64_MAX; // TODO(angiebird): Turn this on when this speed feature is well tested -#if 1 const InterModeRdModel *md = &tile_data->inter_mode_rd_models[bsize]; - const int do_tx_search = !md->ready; -#else - const int do_tx_search = 1; -#endif - InterModesInfo *inter_modes_info = &tile_data->inter_modes_info; + const int do_tx_search = + !((cpi->sf.inter_mode_rd_model_estimation == 1 && md->ready) || + (cpi->sf.inter_mode_rd_model_estimation == 2 && + x->source_variance < 512)); + InterModesInfo *inter_modes_info = x->inter_modes_info; inter_modes_info->num = 0; -#endif int intra_mode_num = 0; int intra_mode_idx_ls[MAX_MODES]; int reach_first_comp_mode = 0; // Temporary buffers used by handle_inter_mode(). - // We allocate them once and reuse it in every call to that function. - // Note: Must be allocated on the heap due to large size of the arrays. - uint8_t *tmp_buf_orig; - CHECK_MEM_ERROR( - cm, tmp_buf_orig, - (uint8_t *)aom_memalign(32, 2 * MAX_MB_PLANE * MAX_SB_SQUARE)); - uint8_t *const tmp_buf = get_buf_by_bd(xd, tmp_buf_orig); + uint8_t *const tmp_buf = get_buf_by_bd(xd, x->tmp_obmc_bufs[0]); CompoundTypeRdBuffers rd_buffers; alloc_compound_type_rd_buffers(cm, &rd_buffers); for (int midx = 0; midx < MAX_MODES; ++midx) { - int mode_index = mode_map[midx]; - int64_t this_rd = INT64_MAX; - int disable_skip = 0; - int rate2 = 0, rate_y = 0, rate_uv = 0; - int64_t distortion2 = 0; - int skippable = 0; - int this_skip2 = 0; - const MODE_DEFINITION *mode_order = &av1_mode_order[mode_index]; + const MODE_DEFINITION *mode_order = &av1_mode_order[midx]; + this_mode = mode_order->mode; const MV_REFERENCE_FRAME ref_frame = mode_order->ref_frame[0]; const MV_REFERENCE_FRAME second_ref_frame = mode_order->ref_frame[1]; const int comp_pred = second_ref_frame > INTRA_FRAME; - this_mode = mode_order->mode; - init_mbmi(mbmi, mode_index, cm); - - x->skip = 0; - set_ref_ptrs(cm, xd, ref_frame, second_ref_frame); + // When single ref motion search ends: + // 1st pass: To evaluate single ref RD results and rewind to the beginning; + // 2nd pass: To continue with compound ref search. + if (sf->prune_single_motion_modes_by_simple_trans) { + if (comp_pred && args.single_ref_first_pass) { + args.single_ref_first_pass = 0; + // Reach the first comp ref mode + // Reset midx to start the 2nd pass for single ref motion search + midx = -1; + motion_mode_skip_mask = analyze_simple_trans_states(cpi, x); + continue; + } + if (!comp_pred) { // single ref mode + if (args.single_ref_first_pass) { + // clear stats + for (int k = 0; k < MAX_REF_MV_SERCH; ++k) { + x->simple_rd_state[midx][k].rd_stats.rdcost = INT64_MAX; + x->simple_rd_state[midx][k].early_skipped = 0; + } + } else { + if (motion_mode_skip_mask & (1 << ref_frame)) { + continue; + } + } + } + } // Reach the first compound prediction mode if (sf->prune_comp_search_by_single_result > 0 && comp_pred && @@ -11470,9 +12242,23 @@ void av1_rd_pick_inter_mode_sb(AV1_COMP *cpi, TileDataEnc *tile_data, analyze_single_states(cpi, &search_state); reach_first_comp_mode = 1; } + int64_t this_rd = INT64_MAX; + int disable_skip = 0; + int rate2 = 0, rate_y = 0, rate_uv = 0; + int64_t distortion2 = 0; + int skippable = 0; + int this_skip2 = 0; + + init_mbmi(mbmi, midx, cm); + + x->skip = 0; + set_ref_ptrs(cm, xd, ref_frame, second_ref_frame); + + if (inter_mode_compatible_skip(cpi, x, bsize, midx)) continue; + const int ret = inter_mode_search_order_independent_skip( - cpi, ctx, x, bsize, mode_index, mi_row, mi_col, mode_skip_mask, - ref_frame_skip_mask, &search_state); + cpi, ctx, x, bsize, midx, mi_row, mi_col, &mode_skip_mask, + &search_state); if (ret == 1) continue; args.skip_motion_mode = (ret == 2); @@ -11482,8 +12268,7 @@ void av1_rd_pick_inter_mode_sb(AV1_COMP *cpi, TileDataEnc *tile_data, } } - if (search_state.best_rd < search_state.mode_threshold[mode_index]) - continue; + if (search_state.best_rd < search_state.mode_threshold[midx]) continue; if (sf->prune_comp_search_by_single_result > 0 && comp_pred) { if (compound_skip_by_single_states(cpi, &search_state, this_mode, @@ -11497,7 +12282,9 @@ void av1_rd_pick_inter_mode_sb(AV1_COMP *cpi, TileDataEnc *tile_data, const int compmode_cost = is_comp_ref_allowed(mbmi->sb_type) ? comp_inter_cost[comp_pred] : 0; const int real_compmode_cost = - cm->reference_mode == REFERENCE_MODE_SELECT ? compmode_cost : 0; + cm->current_frame.reference_mode == REFERENCE_MODE_SELECT + ? compmode_cost + : 0; if (comp_pred) { if ((sf->mode_search_skip_flags & FLAG_SKIP_COMP_BESTINTRA) && @@ -11507,7 +12294,12 @@ void av1_rd_pick_inter_mode_sb(AV1_COMP *cpi, TileDataEnc *tile_data, } if (ref_frame == INTRA_FRAME) { - if (sf->adaptive_mode_search) + if (!cpi->oxcf.enable_smooth_intra && + (mbmi->mode == SMOOTH_PRED || mbmi->mode == SMOOTH_H_PRED || + mbmi->mode == SMOOTH_V_PRED)) + continue; + if (!cpi->oxcf.enable_paeth_intra && mbmi->mode == PAETH_PRED) continue; + if (sf->adaptive_mode_search > 1) if ((x->source_variance << num_pels_log2_lookup[bsize]) > search_state.best_pred_sse) continue; @@ -11535,7 +12327,7 @@ void av1_rd_pick_inter_mode_sb(AV1_COMP *cpi, TileDataEnc *tile_data, } if (ref_frame == INTRA_FRAME) { - intra_mode_idx_ls[intra_mode_num++] = mode_index; + intra_mode_idx_ls[intra_mode_num++] = midx; continue; } else { mbmi->angle_delta[PLANE_TYPE_Y] = 0; @@ -11554,25 +12346,28 @@ void av1_rd_pick_inter_mode_sb(AV1_COMP *cpi, TileDataEnc *tile_data, args.single_newmv_valid = search_state.single_newmv_valid; args.single_comp_cost = real_compmode_cost; args.ref_frame_cost = ref_frame_cost; -#if CONFIG_COLLECT_INTER_MODE_RD_STATS + if (midx < MAX_SINGLE_REF_MODES) { + args.simple_rd_state = x->simple_rd_state[midx]; + } this_rd = handle_inter_mode( - cpi, x, bsize, &rd_stats, &rd_stats_y, &rd_stats_uv, &disable_skip, - mi_row, mi_col, &args, ref_best_rd, tmp_buf, &rd_buffers, tile_data, - &best_est_rd, do_tx_search, inter_modes_info); -#else - this_rd = handle_inter_mode(cpi, x, bsize, &rd_stats, &rd_stats_y, - &rd_stats_uv, &disable_skip, mi_row, mi_col, - &args, ref_best_rd, tmp_buf, &rd_buffers); -#endif + cpi, tile_data, x, bsize, &rd_stats, &rd_stats_y, &rd_stats_uv, + &disable_skip, mi_row, mi_col, &args, ref_best_rd, tmp_buf, + &rd_buffers, &best_est_rd, do_tx_search, inter_modes_info); rate2 = rd_stats.rate; skippable = rd_stats.skip; distortion2 = rd_stats.dist; rate_y = rd_stats_y.rate; rate_uv = rd_stats_uv.rate; +#if CONFIG_ONE_PASS_SVM + av1_unpack_reg_stat(&rd_stats_y, &temp_y_eob, &temp_y_eob_0, + &temp_y_eob_1, &temp_y_eob_2, &temp_y_eob_3, + &temp_y_rd, &temp_y_rd_0, &temp_y_rd_1, + &temp_y_rd_2, &temp_y_rd_3); +#endif } if (sf->prune_comp_search_by_single_result > 0 && - is_inter_singleref_mode(this_mode)) { + is_inter_singleref_mode(this_mode) && args.single_ref_first_pass) { collect_single_states(x, &search_state, mbmi); } @@ -11590,11 +12385,11 @@ void av1_rd_pick_inter_mode_sb(AV1_COMP *cpi, TileDataEnc *tile_data, if (this_rd < search_state.best_rd || x->skip) { int mode_excluded = 0; if (comp_pred) { - mode_excluded = cm->reference_mode == SINGLE_REFERENCE; + mode_excluded = cm->current_frame.reference_mode == SINGLE_REFERENCE; } if (!mode_excluded) { // Note index of best mode so far - search_state.best_mode_index = mode_index; + search_state.best_mode_index = midx; if (ref_frame == INTRA_FRAME) { /* required for left and above block mv */ @@ -11610,7 +12405,6 @@ void av1_rd_pick_inter_mode_sb(AV1_COMP *cpi, TileDataEnc *tile_data, search_state.best_mbmode = *mbmi; search_state.best_skip2 = this_skip2; search_state.best_mode_skippable = skippable; -#if CONFIG_COLLECT_INTER_MODE_RD_STATS if (do_tx_search) { // When do_tx_search == 0, handle_inter_mode won't provide correct // rate_y and rate_uv because txfm_search process is replaced by @@ -11621,13 +12415,13 @@ void av1_rd_pick_inter_mode_sb(AV1_COMP *cpi, TileDataEnc *tile_data, rate_y + x->skip_cost[av1_get_skip_context(xd)][this_skip2 || skippable]; search_state.best_rate_uv = rate_uv; + +#if CONFIG_ONE_PASS_SVM + av1_set_reg_stat(rd_cost, temp_y_eob, temp_y_eob_0, temp_y_eob_1, + temp_y_eob_2, temp_y_eob_3, temp_y_rd, temp_y_rd_0, + temp_y_rd_1, temp_y_rd_2, temp_y_rd_3); +#endif } -#else // CONFIG_COLLECT_INTER_MODE_RD_STATS - search_state.best_rate_y = - rate_y + - x->skip_cost[av1_get_skip_context(xd)][this_skip2 || skippable]; - search_state.best_rate_uv = rate_uv; -#endif // CONFIG_COLLECT_INTER_MODE_RD_STATS memcpy(ctx->blk_skip, x->blk_skip, sizeof(x->blk_skip[0]) * ctx->num_4x4_blk); } @@ -11637,7 +12431,7 @@ void av1_rd_pick_inter_mode_sb(AV1_COMP *cpi, TileDataEnc *tile_data, if (!disable_skip && ref_frame != INTRA_FRAME) { int64_t single_rd, hybrid_rd, single_rate, hybrid_rate; - if (cm->reference_mode == REFERENCE_MODE_SELECT) { + if (cm->current_frame.reference_mode == REFERENCE_MODE_SELECT) { single_rate = rate2 - compmode_cost; hybrid_rate = rate2; } else { @@ -11666,24 +12460,23 @@ void av1_rd_pick_inter_mode_sb(AV1_COMP *cpi, TileDataEnc *tile_data, if (x->skip && !comp_pred) break; } - aom_free(tmp_buf_orig); - tmp_buf_orig = NULL; release_compound_type_rd_buffers(&rd_buffers); -#if CONFIG_COLLECT_INTER_MODE_RD_STATS if (!do_tx_search) { inter_modes_info_sort(inter_modes_info, inter_modes_info->rd_idx_pair_arr); search_state.best_rd = INT64_MAX; int64_t top_est_rd = - inter_modes_info->est_rd_arr[inter_modes_info->rd_idx_pair_arr[0].idx]; + inter_modes_info->num > 0 + ? inter_modes_info + ->est_rd_arr[inter_modes_info->rd_idx_pair_arr[0].idx] + : INT64_MAX; for (int j = 0; j < inter_modes_info->num; ++j) { const int data_idx = inter_modes_info->rd_idx_pair_arr[j].idx; *mbmi = inter_modes_info->mbmi_arr[data_idx]; int64_t curr_est_rd = inter_modes_info->est_rd_arr[data_idx]; - if (curr_est_rd * 0.9 > top_est_rd) { - continue; - } + if (curr_est_rd * 0.80 > top_est_rd) break; + const int mode_rate = inter_modes_info->mode_rate_arr[data_idx]; x->skip = 0; @@ -11700,14 +12493,16 @@ void av1_rd_pick_inter_mode_sb(AV1_COMP *cpi, TileDataEnc *tile_data, RD_STATS rd_stats_y; RD_STATS rd_stats_uv; - av1_build_inter_predictors_sb(cm, xd, mi_row, mi_col, NULL, bsize); + av1_enc_build_inter_predictor(cm, xd, mi_row, mi_col, NULL, bsize, 0, + av1_num_planes(cm) - 1); if (mbmi->motion_mode == OBMC_CAUSAL) av1_build_obmc_inter_predictors_sb(cm, xd, mi_row, mi_col); - if (!txfm_search(cpi, x, bsize, mi_row, mi_col, &rd_stats, &rd_stats_y, - &rd_stats_uv, mode_rate, search_state.best_rd)) { + if (!txfm_search(cpi, tile_data, x, bsize, mi_row, mi_col, &rd_stats, + &rd_stats_y, &rd_stats_uv, mode_rate, + search_state.best_rd)) { continue; - } else { + } else if (cpi->sf.inter_mode_rd_model_estimation == 1) { const int skip_ctx = av1_get_skip_context(xd); inter_mode_data_push(tile_data, mbmi->sb_type, rd_stats.sse, rd_stats.dist, @@ -11733,10 +12528,12 @@ void av1_rd_pick_inter_mode_sb(AV1_COMP *cpi, TileDataEnc *tile_data, search_state.best_rate_uv = rd_stats_uv.rate; memcpy(ctx->blk_skip, x->blk_skip, sizeof(x->blk_skip[0]) * ctx->num_4x4_blk); +#if CONFIG_ONE_PASS_SVM + av1_copy_reg_stat(rd_cost, &rd_stats_y); +#endif } } } -#endif for (int j = 0; j < intra_mode_num; ++j) { const int mode_index = intra_mode_idx_ls[j]; @@ -11775,6 +12572,9 @@ void av1_rd_pick_inter_mode_sb(AV1_COMP *cpi, TileDataEnc *tile_data, search_state.best_rate_uv = intra_rd_stats_uv.rate; memcpy(ctx->blk_skip, x->blk_skip, sizeof(x->blk_skip[0]) * ctx->num_4x4_blk); +#if CONFIG_ONE_PASS_SVM + av1_copy_reg_stat(rd_cost, &intra_rd_stats_y); +#endif } } @@ -11791,7 +12591,7 @@ void av1_rd_pick_inter_mode_sb(AV1_COMP *cpi, TileDataEnc *tile_data, } search_state.best_mbmode.skip_mode = 0; - if (cm->skip_mode_flag && + if (cm->current_frame.skip_mode_info.skip_mode_flag && !segfeature_active(seg, segment_id, SEG_LVL_REF_FRAME) && is_comp_ref_allowed(bsize)) { rd_pick_skip_mode(rd_cost, &search_state, cpi, x, bsize, mi_row, mi_col, @@ -11958,7 +12758,7 @@ void av1_rd_pick_inter_mode_sb_seg_skip(const AV1_COMP *cpi, mbmi->interp_filters = av1_broadcast_interp_filter(best_filter); rate2 += av1_get_switchable_rate(cm, x, xd); - if (cm->reference_mode == REFERENCE_MODE_SELECT) + if (cm->current_frame.reference_mode == REFERENCE_MODE_SELECT) rate2 += comp_inter_cost[comp_pred]; // Estimate the reference frame signaling cost and add it @@ -12010,7 +12810,7 @@ static INLINE void calc_target_weighted_pred_above( int32_t *wsrc = ctxt->x->wsrc_buf + (rel_mi_col * MI_SIZE); int32_t *mask = ctxt->x->mask_buf + (rel_mi_col * MI_SIZE); const uint8_t *tmp = ctxt->tmp + rel_mi_col * MI_SIZE; - const int is_hbd = (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) ? 1 : 0; + const int is_hbd = is_cur_buf_hbd(xd); if (!is_hbd) { for (int row = 0; row < ctxt->overlap; ++row) { @@ -12056,7 +12856,7 @@ static INLINE void calc_target_weighted_pred_left( int32_t *wsrc = ctxt->x->wsrc_buf + (rel_mi_row * MI_SIZE * bw); int32_t *mask = ctxt->x->mask_buf + (rel_mi_row * MI_SIZE * bw); const uint8_t *tmp = ctxt->tmp + (rel_mi_row * MI_SIZE * ctxt->tmp_stride); - const int is_hbd = (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) ? 1 : 0; + const int is_hbd = is_cur_buf_hbd(xd); if (!is_hbd) { for (int row = 0; row < nb_mi_height * MI_SIZE; ++row) { @@ -12138,7 +12938,7 @@ static void calc_target_weighted_pred(const AV1_COMMON *cm, const MACROBLOCK *x, int32_t *mask_buf = x->mask_buf; int32_t *wsrc_buf = x->wsrc_buf; - const int is_hbd = (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) ? 1 : 0; + const int is_hbd = is_cur_buf_hbd(xd); const int src_scale = AOM_BLEND_A64_MAX_ALPHA * AOM_BLEND_A64_MAX_ALPHA; // plane 0 should not be subsampled @@ -12197,3 +12997,110 @@ static void calc_target_weighted_pred(const AV1_COMMON *cm, const MACROBLOCK *x, } } } + +/* Use standard 3x3 Sobel matrix. Macro so it can be used for either high or + low bit-depth arrays. */ +#define SOBEL_X(src, stride, i, j) \ + ((src)[((i)-1) + (stride) * ((j)-1)] - \ + (src)[((i) + 1) + (stride) * ((j)-1)] + /* NOLINT */ \ + 2 * (src)[((i)-1) + (stride) * (j)] - /* NOLINT */ \ + 2 * (src)[((i) + 1) + (stride) * (j)] + /* NOLINT */ \ + (src)[((i)-1) + (stride) * ((j) + 1)] - /* NOLINT */ \ + (src)[((i) + 1) + (stride) * ((j) + 1)]) /* NOLINT */ +#define SOBEL_Y(src, stride, i, j) \ + ((src)[((i)-1) + (stride) * ((j)-1)] + \ + 2 * (src)[(i) + (stride) * ((j)-1)] + /* NOLINT */ \ + (src)[((i) + 1) + (stride) * ((j)-1)] - /* NOLINT */ \ + (src)[((i)-1) + (stride) * ((j) + 1)] - /* NOLINT */ \ + 2 * (src)[(i) + (stride) * ((j) + 1)] - /* NOLINT */ \ + (src)[((i) + 1) + (stride) * ((j) + 1)]) /* NOLINT */ + +sobel_xy sobel(const uint8_t *input, int stride, int i, int j, bool high_bd) { + int16_t s_x; + int16_t s_y; + if (high_bd) { + const uint16_t *src = CONVERT_TO_SHORTPTR(input); + s_x = SOBEL_X(src, stride, i, j); + s_y = SOBEL_Y(src, stride, i, j); + } else { + s_x = SOBEL_X(input, stride, i, j); + s_y = SOBEL_Y(input, stride, i, j); + } + sobel_xy r = { .x = s_x, .y = s_y }; + return r; +} + +// 8-tap Gaussian convolution filter with sigma = 1.3, sums to 128, +// all co-efficients must be even. +DECLARE_ALIGNED(16, static const int16_t, gauss_filter[8]) = { 2, 12, 30, 40, + 30, 12, 2, 0 }; + +void gaussian_blur(const uint8_t *src, int src_stride, int w, int h, + uint8_t *dst, bool high_bd, int bd) { + ConvolveParams conv_params = get_conv_params(0, 0, bd); + InterpFilterParams filter = { .filter_ptr = gauss_filter, + .taps = 8, + .subpel_shifts = 0, + .interp_filter = EIGHTTAP_REGULAR }; + // Requirements from the vector-optimized implementations. + assert(h % 4 == 0); + assert(w % 8 == 0); + // Because we use an eight tap filter, the stride should be at least 7 + w. + assert(src_stride >= w + 7); + if (high_bd) { + av1_highbd_convolve_2d_sr(CONVERT_TO_SHORTPTR(src), src_stride, + CONVERT_TO_SHORTPTR(dst), w, w, h, &filter, + &filter, 0, 0, &conv_params, bd); + } else { + av1_convolve_2d_sr(src, src_stride, dst, w, w, h, &filter, &filter, 0, 0, + &conv_params); + } +} + +static uint16_t edge_probability(const uint8_t *input, int w, int h, + bool high_bd, int bd) { + // The probability of an edge in the whole image is the same as the highest + // probability of an edge for any individual pixel. Use Sobel as the metric + // for finding an edge. + uint16_t highest = 0; + // Ignore the 1 pixel border around the image for the computation. + for (int j = 1; j < h - 1; ++j) { + for (int i = 1; i < w - 1; ++i) { + sobel_xy g = sobel(input, w, i, j, high_bd); + // Scale down to 8-bit to get same output regardless of bit depth. + int16_t g_x = g.x >> (bd - 8); + int16_t g_y = g.y >> (bd - 8); + uint16_t magnitude = (uint16_t)sqrt(g_x * g_x + g_y * g_y); + highest = AOMMAX(highest, magnitude); + } + } + return highest; +} + +/* Uses most of the Canny edge detection algorithm to find if there are any + * edges in the image. + */ +uint16_t av1_edge_exists(const uint8_t *src, int src_stride, int w, int h, + bool high_bd, int bd) { + if (w < 3 || h < 3) { + return 0; + } + uint8_t *blurred; + if (high_bd) { + blurred = CONVERT_TO_BYTEPTR(aom_memalign(32, sizeof(uint16_t) * w * h)); + } else { + blurred = (uint8_t *)aom_memalign(32, sizeof(uint8_t) * w * h); + } + gaussian_blur(src, src_stride, w, h, blurred, high_bd, bd); + // Skip the non-maximum suppression step in Canny edge detection. We just + // want a probability of an edge existing in the buffer, which is determined + // by the strongest edge in it -- we don't need to eliminate the weaker + // edges. Use Sobel for the edge detection. + uint16_t prob = edge_probability(blurred, w, h, high_bd, bd); + if (high_bd) { + aom_free(CONVERT_TO_SHORTPTR(blurred)); + } else { + aom_free(blurred); + } + return prob; +} diff --git a/third_party/aom/av1/encoder/rdopt.h b/third_party/aom/av1/encoder/rdopt.h index 4c11f90b8f..983d3850df 100644 --- a/third_party/aom/av1/encoder/rdopt.h +++ b/third_party/aom/av1/encoder/rdopt.h @@ -12,6 +12,8 @@ #ifndef AOM_AV1_ENCODER_RDOPT_H_ #define AOM_AV1_ENCODER_RDOPT_H_ +#include + #include "av1/common/blockd.h" #include "av1/common/txb_common.h" @@ -126,10 +128,31 @@ void av1_rd_pick_inter_mode_sb_seg_skip( struct macroblock *x, int mi_row, int mi_col, struct RD_STATS *rd_cost, BLOCK_SIZE bsize, PICK_MODE_CONTEXT *ctx, int64_t best_rd_so_far); -#if CONFIG_COLLECT_INTER_MODE_RD_STATS +/** Returns an integer indicating the strength of the edge. + * 0 means no edge found, 556 is the strength of a solid black/white edge, + * and the number may range higher if the signal is even stronger (e.g., on a + * corner). high_bd is a bool indicating the source should be treated + * as a 16-bit array. bd is the bit depth. + */ +uint16_t av1_edge_exists(const uint8_t *src, int src_stride, int w, int h, + bool high_bd, int bd); + +/** Applies a Gaussian blur with sigma = 1.3. Used by av1_edge_exists and + * tests. + */ +void gaussian_blur(const uint8_t *src, int src_stride, int w, int h, + uint8_t *dst, bool high_bd, int bd); + +/* Applies standard 3x3 Sobel matrix. */ +typedef struct { + int16_t x; + int16_t y; +} sobel_xy; + +sobel_xy sobel(const uint8_t *input, int stride, int i, int j, bool high_bd); + void av1_inter_mode_data_init(struct TileDataEnc *tile_data); void av1_inter_mode_data_fit(TileDataEnc *tile_data, int rdmult); -#endif #ifdef __cplusplus } // extern "C" diff --git a/third_party/aom/av1/encoder/reconinter_enc.c b/third_party/aom/av1/encoder/reconinter_enc.c index 23d920fc32..4b477cef62 100644 --- a/third_party/aom/av1/encoder/reconinter_enc.c +++ b/third_party/aom/av1/encoder/reconinter_enc.c @@ -138,26 +138,28 @@ static INLINE void build_inter_predictors(const AV1_COMMON *cm, MACROBLOCKD *xd, assert(bw < 8 || bh < 8); ConvolveParams conv_params = get_conv_params_no_round( 0, plane, xd->tmp_conv_dst, tmp_dst_stride, is_compound, xd->bd); - conv_params.use_jnt_comp_avg = 0; + conv_params.use_dist_wtd_comp_avg = 0; struct buf_2d *const dst_buf = &pd->dst; uint8_t *dst = dst_buf->buf + dst_buf->stride * y + x; ref = 0; - const RefBuffer *ref_buf = - &cm->frame_refs[this_mbmi->ref_frame[ref] - LAST_FRAME]; + const RefCntBuffer *ref_buf = + get_ref_frame_buf(cm, this_mbmi->ref_frame[ref]); + const struct scale_factors *ref_scale_factors = + get_ref_scale_factors_const(cm, this_mbmi->ref_frame[ref]); pd->pre[ref].buf0 = - (plane == 1) ? ref_buf->buf->u_buffer : ref_buf->buf->v_buffer; + (plane == 1) ? ref_buf->buf.u_buffer : ref_buf->buf.v_buffer; pd->pre[ref].buf = pd->pre[ref].buf0 + scaled_buffer_offset(pre_x, pre_y, - ref_buf->buf->uv_stride, - &ref_buf->sf); - pd->pre[ref].width = ref_buf->buf->uv_crop_width; - pd->pre[ref].height = ref_buf->buf->uv_crop_height; - pd->pre[ref].stride = ref_buf->buf->uv_stride; + ref_buf->buf.uv_stride, + ref_scale_factors); + pd->pre[ref].width = ref_buf->buf.uv_crop_width; + pd->pre[ref].height = ref_buf->buf.uv_crop_height; + pd->pre[ref].stride = ref_buf->buf.uv_stride; const struct scale_factors *const sf = - is_intrabc ? &cm->sf_identity : &ref_buf->sf; + is_intrabc ? &cm->sf_identity : ref_scale_factors; struct buf_2d *const pre_buf = is_intrabc ? dst_buf : &pd->pre[ref]; const MV mv = this_mbmi->mv[ref].as_mv; @@ -194,15 +196,15 @@ static INLINE void build_inter_predictors(const AV1_COMMON *cm, MACROBLOCKD *xd, { ConvolveParams conv_params = get_conv_params_no_round( 0, plane, xd->tmp_conv_dst, MAX_SB_SIZE, is_compound, xd->bd); - av1_jnt_comp_weight_assign(cm, mi, 0, &conv_params.fwd_offset, - &conv_params.bck_offset, - &conv_params.use_jnt_comp_avg, is_compound); + av1_dist_wtd_comp_weight_assign( + cm, mi, 0, &conv_params.fwd_offset, &conv_params.bck_offset, + &conv_params.use_dist_wtd_comp_avg, is_compound); struct buf_2d *const dst_buf = &pd->dst; uint8_t *const dst = dst_buf->buf; for (ref = 0; ref < 1 + is_compound; ++ref) { const struct scale_factors *const sf = - is_intrabc ? &cm->sf_identity : &xd->block_refs[ref]->sf; + is_intrabc ? &cm->sf_identity : xd->block_ref_scale_factors[ref]; struct buf_2d *const pre_buf = is_intrabc ? dst_buf : &pd->pre[ref]; const MV mv = mi->mv[ref].as_mv; @@ -235,46 +237,19 @@ static INLINE void build_inter_predictors(const AV1_COMMON *cm, MACROBLOCKD *xd, } } -static void build_inter_predictors_for_planes(const AV1_COMMON *cm, - MACROBLOCKD *xd, BLOCK_SIZE bsize, - int mi_row, int mi_col, - int plane_from, int plane_to) { - int plane; +static void build_inter_predictors_for_plane(const AV1_COMMON *cm, + MACROBLOCKD *xd, int mi_row, + int mi_col, const BUFFER_SET *ctx, + BLOCK_SIZE bsize, int plane_idx) { + const struct macroblockd_plane *pd = &xd->plane[plane_idx]; + if (!is_chroma_reference(mi_row, mi_col, bsize, pd->subsampling_x, + pd->subsampling_y)) + return; + const int mi_x = mi_col * MI_SIZE; const int mi_y = mi_row * MI_SIZE; - for (plane = plane_from; plane <= plane_to; ++plane) { - const struct macroblockd_plane *pd = &xd->plane[plane]; - const int bw = pd->width; - const int bh = pd->height; - - if (!is_chroma_reference(mi_row, mi_col, bsize, pd->subsampling_x, - pd->subsampling_y)) - continue; - - build_inter_predictors(cm, xd, plane, xd->mi[0], 0, bw, bh, mi_x, mi_y); - } -} - -void av1_build_inter_predictors_sby(const AV1_COMMON *cm, MACROBLOCKD *xd, - int mi_row, int mi_col, BUFFER_SET *ctx, - BLOCK_SIZE bsize) { - av1_build_inter_predictors_sbp(cm, xd, mi_row, mi_col, ctx, bsize, 0); -} - -void av1_build_inter_predictors_sbuv(const AV1_COMMON *cm, MACROBLOCKD *xd, - int mi_row, int mi_col, BUFFER_SET *ctx, - BLOCK_SIZE bsize) { - for (int plane_idx = 1; plane_idx < MAX_MB_PLANE; plane_idx++) { - av1_build_inter_predictors_sbp(cm, xd, mi_row, mi_col, ctx, bsize, - plane_idx); - } -} - -void av1_build_inter_predictors_sbp(const AV1_COMMON *cm, MACROBLOCKD *xd, - int mi_row, int mi_col, BUFFER_SET *ctx, - BLOCK_SIZE bsize, int plane_idx) { - build_inter_predictors_for_planes(cm, xd, bsize, mi_row, mi_col, plane_idx, - plane_idx); + build_inter_predictors(cm, xd, plane_idx, xd->mi[0], 0, pd->width, pd->height, + mi_x, mi_y); if (is_interintra_pred(xd->mi[0])) { BUFFER_SET default_ctx = { { NULL, NULL, NULL }, { 0, 0, 0 } }; @@ -289,13 +264,14 @@ void av1_build_inter_predictors_sbp(const AV1_COMMON *cm, MACROBLOCKD *xd, } } -void av1_build_inter_predictors_sb(const AV1_COMMON *cm, MACROBLOCKD *xd, - int mi_row, int mi_col, BUFFER_SET *ctx, - BLOCK_SIZE bsize) { - const int num_planes = av1_num_planes(cm); - av1_build_inter_predictors_sby(cm, xd, mi_row, mi_col, ctx, bsize); - if (num_planes > 1) - av1_build_inter_predictors_sbuv(cm, xd, mi_row, mi_col, ctx, bsize); +void av1_enc_build_inter_predictor(const AV1_COMMON *cm, MACROBLOCKD *xd, + int mi_row, int mi_col, + const BUFFER_SET *ctx, BLOCK_SIZE bsize, + int plane_from, int plane_to) { + for (int plane_idx = plane_from; plane_idx <= plane_to; ++plane_idx) { + build_inter_predictors_for_plane(cm, xd, mi_row, mi_col, ctx, bsize, + plane_idx); + } } // TODO(sarahparker): @@ -308,7 +284,7 @@ void av1_build_inter_predictor(const uint8_t *src, int src_stride, uint8_t *dst, InterpFilters interp_filters, const WarpTypesAllowed *warp_types, int p_col, int p_row, int plane, int ref, - enum mv_precision precision, int x, int y, + mv_precision precision, int x, int y, const MACROBLOCKD *xd, int can_use_previous) { const int is_q4 = precision == MV_PRECISION_Q4; const MV mv_q4 = { is_q4 ? src_mv->row : src_mv->row * 2, @@ -338,7 +314,7 @@ static INLINE void build_prediction_by_above_pred( MB_MODE_INFO backup_mbmi = *above_mbmi; av1_setup_build_prediction_by_above_pred(xd, rel_mi_col, above_mi_width, - above_mbmi, ctxt, num_planes); + &backup_mbmi, ctxt, num_planes); mi_x = above_mi_col << MI_SIZE_LOG2; mi_y = ctxt->mi_row << MI_SIZE_LOG2; @@ -351,9 +327,9 @@ static INLINE void build_prediction_by_above_pred( block_size_high[BLOCK_64X64] >> (pd->subsampling_y + 1)); if (av1_skip_u4x4_pred_in_obmc(bsize, pd, 0)) continue; - build_inter_predictors(ctxt->cm, xd, j, above_mbmi, 1, bw, bh, mi_x, mi_y); + build_inter_predictors(ctxt->cm, xd, j, &backup_mbmi, 1, bw, bh, mi_x, + mi_y); } - *above_mbmi = backup_mbmi; } void av1_build_prediction_by_above_preds(const AV1_COMMON *cm, MACROBLOCKD *xd, @@ -394,7 +370,7 @@ static INLINE void build_prediction_by_left_pred( MB_MODE_INFO backup_mbmi = *left_mbmi; av1_setup_build_prediction_by_left_pred(xd, rel_mi_row, left_mi_height, - left_mbmi, ctxt, num_planes); + &backup_mbmi, ctxt, num_planes); mi_x = ctxt->mi_col << MI_SIZE_LOG2; mi_y = left_mi_row << MI_SIZE_LOG2; const BLOCK_SIZE bsize = xd->mi[0]->sb_type; @@ -406,9 +382,9 @@ static INLINE void build_prediction_by_left_pred( int bh = (left_mi_height << MI_SIZE_LOG2) >> pd->subsampling_y; if (av1_skip_u4x4_pred_in_obmc(bsize, pd, 1)) continue; - build_inter_predictors(ctxt->cm, xd, j, left_mbmi, 1, bw, bh, mi_x, mi_y); + build_inter_predictors(ctxt->cm, xd, j, &backup_mbmi, 1, bw, bh, mi_x, + mi_y); } - *left_mbmi = backup_mbmi; } void av1_build_prediction_by_left_preds(const AV1_COMMON *cm, MACROBLOCKD *xd, @@ -451,7 +427,7 @@ void av1_build_obmc_inter_predictors_sb(const AV1_COMMON *cm, MACROBLOCKD *xd, int dst_height1[MAX_MB_PLANE] = { MAX_SB_SIZE, MAX_SB_SIZE, MAX_SB_SIZE }; int dst_height2[MAX_MB_PLANE] = { MAX_SB_SIZE, MAX_SB_SIZE, MAX_SB_SIZE }; - if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + if (is_cur_buf_hbd(xd)) { int len = sizeof(uint16_t); dst_buf1[0] = CONVERT_TO_BYTEPTR(xd->tmp_obmc_bufs[0]); dst_buf1[1] = @@ -475,7 +451,7 @@ void av1_build_obmc_inter_predictors_sb(const AV1_COMMON *cm, MACROBLOCKD *xd, dst_width1, dst_height1, dst_stride1); av1_build_prediction_by_left_preds(cm, xd, mi_row, mi_col, dst_buf2, dst_width2, dst_height2, dst_stride2); - av1_setup_dst_planes(xd->plane, xd->mi[0]->sb_type, get_frame_new_buffer(cm), + av1_setup_dst_planes(xd->plane, xd->mi[0]->sb_type, &cm->cur_frame->buf, mi_row, mi_col, 0, num_planes); av1_build_obmc_inter_prediction(cm, xd, mi_row, mi_col, dst_buf1, dst_stride1, dst_buf2, dst_stride2); @@ -492,7 +468,7 @@ static void build_inter_predictors_single_buf(MACROBLOCKD *xd, int plane, struct macroblockd_plane *const pd = &xd->plane[plane]; const MB_MODE_INFO *mi = xd->mi[0]; - const struct scale_factors *const sf = &xd->block_refs[ref]->sf; + const struct scale_factors *const sf = xd->block_ref_scale_factors[ref]; struct buf_2d *const pre_buf = &pd->pre[ref]; uint8_t *const dst = get_buf_by_bd(xd, ext_dst) + ext_dst_stride * y + x; const MV mv = mi->mv[ref].as_mv; @@ -574,37 +550,41 @@ static void build_wedge_inter_predictor_from_buf( uint8_t *const dst = dst_buf->buf + dst_buf->stride * y + x; mbmi->interinter_comp.seg_mask = xd->seg_mask; const INTERINTER_COMPOUND_DATA *comp_data = &mbmi->interinter_comp; + const int is_hbd = is_cur_buf_hbd(xd); if (is_compound && is_masked_compound_type(comp_data->type)) { if (!plane && comp_data->type == COMPOUND_DIFFWTD) { - if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) + if (is_hbd) { av1_build_compound_diffwtd_mask_highbd( comp_data->seg_mask, comp_data->mask_type, CONVERT_TO_BYTEPTR(ext_dst0), ext_dst_stride0, CONVERT_TO_BYTEPTR(ext_dst1), ext_dst_stride1, h, w, xd->bd); - else + } else { av1_build_compound_diffwtd_mask( comp_data->seg_mask, comp_data->mask_type, ext_dst0, ext_dst_stride0, ext_dst1, ext_dst_stride1, h, w); + } } - if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) + if (is_hbd) { build_masked_compound_highbd( dst, dst_buf->stride, CONVERT_TO_BYTEPTR(ext_dst0), ext_dst_stride0, CONVERT_TO_BYTEPTR(ext_dst1), ext_dst_stride1, comp_data, mbmi->sb_type, h, w, xd->bd); - else + } else { build_masked_compound(dst, dst_buf->stride, ext_dst0, ext_dst_stride0, ext_dst1, ext_dst_stride1, comp_data, mbmi->sb_type, h, w); + } } else { - if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) + if (is_hbd) { aom_highbd_convolve_copy(CONVERT_TO_BYTEPTR(ext_dst0), ext_dst_stride0, dst, dst_buf->stride, NULL, 0, NULL, 0, w, h, xd->bd); - else + } else { aom_convolve_copy(ext_dst0, ext_dst_stride0, dst, dst_buf->stride, NULL, 0, NULL, 0, w, h); + } } } diff --git a/third_party/aom/av1/encoder/reconinter_enc.h b/third_party/aom/av1/encoder/reconinter_enc.h index 10d5e8c284..5687168315 100644 --- a/third_party/aom/av1/encoder/reconinter_enc.h +++ b/third_party/aom/av1/encoder/reconinter_enc.h @@ -23,21 +23,10 @@ extern "C" { #endif -void av1_build_inter_predictors_sby(const AV1_COMMON *cm, MACROBLOCKD *xd, - int mi_row, int mi_col, BUFFER_SET *ctx, - BLOCK_SIZE bsize); - -void av1_build_inter_predictors_sbuv(const AV1_COMMON *cm, MACROBLOCKD *xd, - int mi_row, int mi_col, BUFFER_SET *ctx, - BLOCK_SIZE bsize); - -void av1_build_inter_predictors_sbp(const AV1_COMMON *cm, MACROBLOCKD *xd, - int mi_row, int mi_col, BUFFER_SET *ctx, - BLOCK_SIZE bsize, int plane_idx); - -void av1_build_inter_predictors_sb(const AV1_COMMON *cm, MACROBLOCKD *xd, - int mi_row, int mi_col, BUFFER_SET *ctx, - BLOCK_SIZE bsize); +void av1_enc_build_inter_predictor(const AV1_COMMON *cm, MACROBLOCKD *xd, + int mi_row, int mi_col, + const BUFFER_SET *ctx, BLOCK_SIZE bsize, + int plane_from, int plane_to); void av1_build_inter_predictor(const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, const MV *src_mv, @@ -46,7 +35,7 @@ void av1_build_inter_predictor(const uint8_t *src, int src_stride, uint8_t *dst, InterpFilters interp_filters, const WarpTypesAllowed *warp_types, int p_col, int p_row, int plane, int ref, - enum mv_precision precision, int x, int y, + mv_precision precision, int x, int y, const MACROBLOCKD *xd, int can_use_previous); // Detect if the block have sub-pixel level motion vectors diff --git a/third_party/aom/av1/encoder/segmentation.c b/third_party/aom/av1/encoder/segmentation.c index 2e9102745c..6d0c6549df 100644 --- a/third_party/aom/av1/encoder/segmentation.c +++ b/third_party/aom/av1/encoder/segmentation.c @@ -62,7 +62,7 @@ static void count_segs(const AV1_COMMON *cm, MACROBLOCKD *xd, no_pred_segcounts[segment_id]++; // Temporal prediction not allowed on key frames - if (cm->frame_type != KEY_FRAME) { + if (cm->current_frame.frame_type != KEY_FRAME) { const BLOCK_SIZE bsize = xd->mi[0]->sb_type; // Test to see if the segment id matches the predicted value. const int pred_segment_id = diff --git a/third_party/aom/av1/encoder/speed_features.c b/third_party/aom/av1/encoder/speed_features.c index 4c35baae01..bbb68426fa 100644 --- a/third_party/aom/av1/encoder/speed_features.c +++ b/third_party/aom/av1/encoder/speed_features.c @@ -24,6 +24,8 @@ #define DISABLE_TRELLISQ_SEARCH 0 #define MAX_MESH_SPEED 5 // Max speed setting for mesh motion method +// Max speed setting for tx domain evaluation +#define MAX_TX_DOMAIN_EVAL_SPEED 5 static MESH_PATTERN good_quality_mesh_patterns[MAX_MESH_SPEED + 1][MAX_MESH_STEP] = { { { 64, 8 }, { 28, 4 }, { 15, 1 }, { 7, 1 } }, @@ -50,6 +52,22 @@ static MESH_PATTERN intrabc_mesh_patterns[MAX_MESH_SPEED + 1][MAX_MESH_STEP] = { static uint8_t intrabc_max_mesh_pct[MAX_MESH_SPEED + 1] = { 100, 100, 100, 25, 25, 10 }; +// Threshold values to be used for pruning the txfm_domain_distortion +// based on block MSE +// TODO(any): Experiment the threshold logic based on variance metric +static unsigned int tx_domain_dist_thresholds[MAX_TX_DOMAIN_EVAL_SPEED + 1] = { + UINT_MAX, 162754, 22026, 22026, 22026, 0 +}; +// Threshold values to be used for disabling coeff RD-optimization +// based on block MSE +// TODO(any): Experiment the threshold logic based on variance metric +static unsigned int coeff_opt_dist_thresholds[5] = { UINT_MAX, 162754, 162754, + 22026, 22026 }; +// scaling values to be used for gating wedge/compound segment based on best +// approximate rd +static int comp_type_rd_threshold_mul[3] = { 1, 11, 12 }; +static int comp_type_rd_threshold_div[3] = { 3, 16, 16 }; + // Intra only frames, golden frames (except alt ref overlays) and // alt ref frames tend to be coded at a higher than ambient quality static int frame_is_boosted(const AV1_COMP *cpi) { @@ -62,7 +80,7 @@ static int frame_is_boosted(const AV1_COMP *cpi) { // partly on the screen area that over which they propogate. Propogation is // limited by transform block size but the screen area take up by a given block // size will be larger for a small image format stretched to full screen. -static BLOCK_SIZE set_partition_min_limit(AV1_COMMON *const cm) { +static BLOCK_SIZE set_partition_min_limit(const AV1_COMMON *const cm) { unsigned int screen_area = (cm->width * cm->height); // Select block size based on image format size. @@ -85,10 +103,9 @@ static int has_internal_image_edge(const AV1_COMP *cpi) { (cpi->twopass.this_frame_stats.inactive_zone_cols > 0)); } -static void set_good_speed_feature_framesize_dependent(AV1_COMP *cpi, - SPEED_FEATURES *sf, - int speed) { - AV1_COMMON *const cm = &cpi->common; +static void set_good_speed_feature_framesize_dependent( + const AV1_COMP *const cpi, SPEED_FEATURES *const sf, int speed) { + const AV1_COMMON *const cm = &cpi->common; const int is_720p_or_larger = AOMMIN(cm->width, cm->height) >= 720; const int is_480p_or_larger = AOMMIN(cm->width, cm->height) >= 480; @@ -126,6 +143,17 @@ static void set_good_speed_feature_framesize_dependent(AV1_COMP *cpi, } if (speed >= 2) { + if (is_720p_or_larger) { + sf->use_square_partition_only_threshold = BLOCK_64X64; + } else if (is_480p_or_larger) { + sf->use_square_partition_only_threshold = BLOCK_32X32; + } else { + // TODO(chiyotsai@google.com): Setting the threshold to BLOCK_16X16 incurs + // a large loss (about 0.584%). Try increasing the threshold on boosted + // frame and see if it improves the performance. + sf->use_square_partition_only_threshold = BLOCK_32X32; + } + if (is_720p_or_larger) { sf->disable_split_mask = cm->show_frame ? DISABLE_ALL_SPLIT : DISABLE_ALL_INTER_SPLIT; @@ -172,11 +200,12 @@ static void set_good_speed_feature_framesize_dependent(AV1_COMP *cpi, } } -static void set_good_speed_features_framesize_independent(AV1_COMP *cpi, - SPEED_FEATURES *sf, - int speed) { - AV1_COMMON *const cm = &cpi->common; +static void set_good_speed_features_framesize_independent( + const AV1_COMP *const cpi, SPEED_FEATURES *const sf, int speed) { + const AV1_COMMON *const cm = &cpi->common; const int boosted = frame_is_boosted(cpi); + const int is_boosted_arf2_bwd_type = + boosted || cpi->refresh_bwd_ref_frame || cpi->refresh_alt2_ref_frame; // Speed 0 for all speed features that give neutral coding performance change. sf->reduce_inter_modes = 1; @@ -184,54 +213,79 @@ static void set_good_speed_features_framesize_independent(AV1_COMP *cpi, sf->ml_prune_rect_partition = 1; sf->ml_prune_ab_partition = 1; sf->ml_prune_4_partition = 1; + sf->simple_motion_search_prune_rect = 1; sf->adaptive_txb_search_level = 1; - sf->jnt_comp_skip_mv_search = 1; + sf->use_dist_wtd_comp_flag = DIST_WTD_COMP_SKIP_MV_SEARCH; sf->model_based_prune_tx_search_level = 1; sf->model_based_post_interp_filter_breakout = 1; + sf->model_based_motion_mode_rd_breakout = 1; + + // TODO(debargha): Test, tweak and turn on either 1 or 2 sf->inter_mode_rd_model_estimation = 1; - sf->prune_ref_frame_for_rect_partitions = - !(boosted || cpi->refresh_bwd_ref_frame || cpi->refresh_alt2_ref_frame); + + sf->prune_ref_frame_for_rect_partitions = !is_boosted_arf2_bwd_type; + sf->prune_ref_mode_for_partitions = sf->prune_ref_frame_for_rect_partitions; sf->less_rectangular_check_level = 1; - sf->gm_search_type = GM_REDUCED_REF_SEARCH; + sf->gm_search_type = GM_REDUCED_REF_SEARCH_SKIP_L2_L3; sf->gm_disable_recode = 1; + sf->use_fast_interpolation_filter_search = 1; + sf->intra_tx_size_search_init_depth_sqr = 1; + sf->intra_angle_estimation = 1; + sf->selective_ref_frame = 1; + sf->prune_wedge_pred_diff_based = 1; + sf->disable_wedge_search_var_thresh = 0; + sf->disable_wedge_search_edge_thresh = 0; + sf->prune_motion_mode_level = 1; + sf->cb_pred_filter_search = 0; if (speed >= 1) { sf->gm_erroradv_type = GM_ERRORADV_TR_1; - sf->selective_ref_frame = 1; + sf->selective_ref_frame = 2; + sf->inter_tx_size_search_init_depth_rect = 1; sf->inter_tx_size_search_init_depth_sqr = 1; sf->intra_tx_size_search_init_depth_rect = 1; - sf->intra_tx_size_search_init_depth_sqr = 1; sf->tx_size_search_lgr_block = 1; if (speed >= CONFIG_2PASS_PARTITION_SEARCH_LVL) { sf->two_pass_partition_search = 1; sf->mode_pruning_based_on_two_pass_partition_search = 1; } sf->prune_ext_partition_types_search_level = 2; - sf->use_fast_interpolation_filter_search = 1; sf->skip_repeat_interpolation_filter_search = 1; sf->tx_type_search.skip_tx_search = 1; sf->tx_type_search.ml_tx_split_thresh = 40; sf->model_based_prune_tx_search_level = 0; - sf->model_based_post_interp_filter_breakout = 0; - // TODO(angiebird): Re-evaluate the impact of inter_mode_rd_model_estimation - // on speed 1 - sf->inter_mode_rd_model_estimation = 0; sf->adaptive_txb_search_level = 2; sf->use_intra_txb_hash = 1; sf->optimize_b_precheck = 1; sf->dual_sgr_penalty_level = 1; - sf->use_accurate_subpel_search = 1; + sf->use_accurate_subpel_search = USE_4_TAPS; sf->reuse_inter_intra_mode = 1; sf->prune_comp_search_by_single_result = 1; sf->skip_repeated_newmv = 1; sf->obmc_full_pixel_search_level = 1; + // TODO(anyone): Following speed feature will be further explored to + // identify the appropriate tradeoff between encoder performance and its + // speed. + sf->prune_single_motion_modes_by_simple_trans = 1; + + sf->simple_motion_search_split_only = 1; + + sf->disable_wedge_search_var_thresh = 0; + sf->disable_wedge_search_edge_thresh = 0; + sf->disable_interinter_wedge_newmv_search = boosted ? 0 : 1; + sf->prune_comp_type_by_comp_avg = 1; + sf->prune_motion_mode_level = 2; + sf->gm_search_type = GM_REDUCED_REF_SEARCH_SKIP_L2_L3_ARF2; + sf->cb_pred_filter_search = 1; + sf->use_transform_domain_distortion = boosted ? 0 : 1; + sf->perform_coeff_opt = boosted ? 0 : 1; } if (speed >= 2) { sf->gm_erroradv_type = GM_ERRORADV_TR_2; - sf->selective_ref_frame = 2; + sf->selective_ref_frame = 3; sf->fast_cdef_search = 1; sf->adaptive_rd_thresh = 1; @@ -246,10 +300,19 @@ static void set_good_speed_features_framesize_independent(AV1_COMP *cpi, // sf->auto_min_max_partition_size = RELAXED_NEIGHBORING_MIN_MAX; sf->allow_partition_search_skip = 1; sf->disable_wedge_search_var_thresh = 100; + sf->disable_wedge_search_edge_thresh = 0; + sf->disable_interinter_wedge_newmv_search = 1; sf->fast_wedge_sign_estimate = 1; + sf->disable_dual_filter = 1; + sf->use_dist_wtd_comp_flag = DIST_WTD_COMP_DISABLED; + sf->prune_comp_type_by_comp_avg = 2; + sf->cb_pred_filter_search = 0; + sf->adaptive_interp_filter_search = 1; + sf->perform_coeff_opt = boosted ? 0 : 2; } if (speed >= 3) { + sf->selective_ref_frame = 4; sf->tx_size_search_method = boosted ? USE_FULL_RD : USE_LARGESTALL; sf->less_rectangular_check_level = 2; sf->adaptive_pred_interp_filter = 1; @@ -258,26 +321,33 @@ static void set_good_speed_features_framesize_independent(AV1_COMP *cpi, // See aomedia:1778. // sf->adaptive_motion_search = 1; sf->recode_loop = ALLOW_RECODE_KFARFGF; - sf->use_transform_domain_distortion = 1; - sf->use_accurate_subpel_search = 0; + sf->use_transform_domain_distortion = boosted ? 1 : 2; + sf->use_accurate_subpel_search = USE_2_TAPS; sf->adaptive_rd_thresh = 2; sf->tx_type_search.prune_mode = PRUNE_2D_FAST; sf->gm_search_type = GM_DISABLE_SEARCH; sf->prune_comp_search_by_single_result = 2; + sf->prune_motion_mode_level = boosted ? 2 : 3; + sf->prune_warp_using_wmtype = 1; + // TODO(yunqing): evaluate this speed feature for speed 1 & 2, and combine + // it with cpi->sf.disable_wedge_search_var_thresh. + sf->disable_wedge_interintra_search = 1; + sf->perform_coeff_opt = is_boosted_arf2_bwd_type ? 2 : 3; } if (speed >= 4) { + sf->use_intra_txb_hash = 0; sf->tx_type_search.fast_intra_tx_type_search = 1; - sf->tx_type_search.fast_inter_tx_type_search = 1; sf->use_square_partition_only_threshold = boosted ? BLOCK_128X128 : BLOCK_4X4; - sf->tx_size_search_method = - frame_is_intra_only(cm) ? USE_FULL_RD : USE_LARGESTALL; sf->mv.subpel_search_method = SUBPEL_TREE_PRUNED; sf->adaptive_pred_interp_filter = 0; sf->adaptive_mode_search = 1; sf->cb_partition_search = !boosted; sf->alt_ref_search_fp = 1; + sf->skip_sharp_interp_filter_search = 1; + sf->perform_coeff_opt = is_boosted_arf2_bwd_type ? 2 : 4; + sf->adaptive_txb_search_level = boosted ? 2 : 3; } if (speed >= 5) { @@ -294,7 +364,7 @@ static void set_good_speed_features_framesize_independent(AV1_COMP *cpi, sf->mv.subpel_search_method = SUBPEL_TREE_PRUNED_MORE; sf->adaptive_rd_thresh = 4; sf->mode_search_skip_flags = - (cm->frame_type == KEY_FRAME) + (cm->current_frame.frame_type == KEY_FRAME) ? 0 : FLAG_SKIP_INTRA_DIRMISMATCH | FLAG_SKIP_INTRA_BESTINTER | FLAG_SKIP_COMP_BESTINTRA | FLAG_SKIP_INTRA_LOWVAR | @@ -327,35 +397,28 @@ static void set_good_speed_features_framesize_independent(AV1_COMP *cpi, sf->mv.search_method = FAST_HEX; sf->partition_search_type = REFERENCE_PARTITION; sf->mode_search_skip_flags |= FLAG_SKIP_INTRA_DIRMISMATCH; + // TODO(any): evaluate adaptive_mode_search=1 for speed 7 & 8 + sf->adaptive_mode_search = 2; } if (speed >= 8) { sf->mv.search_method = FAST_DIAMOND; - sf->mv.subpel_force_stop = 2; + sf->mv.subpel_force_stop = HALF_PEL; sf->lpf_pick = LPF_PICK_MINIMAL_LPF; } } -void av1_set_speed_features_framesize_dependent(AV1_COMP *cpi) { +void av1_set_speed_features_framesize_dependent(AV1_COMP *cpi, int speed) { SPEED_FEATURES *const sf = &cpi->sf; const AV1EncoderConfig *const oxcf = &cpi->oxcf; - RD_OPT *const rd = &cpi->rd; - int i; if (oxcf->mode == GOOD) { - set_good_speed_feature_framesize_dependent(cpi, sf, oxcf->speed); + set_good_speed_feature_framesize_dependent(cpi, sf, speed); } if (sf->disable_split_mask == DISABLE_ALL_SPLIT) { sf->adaptive_pred_interp_filter = 0; } - // Check for masked out split cases. - for (i = 0; i < MAX_REFS; ++i) { - if (sf->disable_split_mask & (1 << i)) { - rd->thresh_mult_sub8x8[i] = INT_MAX; - } - } - // This is only used in motion vector unit test. if (cpi->oxcf.motion_vector_unit_test == 1) cpi->find_fractional_mv_step = av1_return_max_sub_pixel_mv; @@ -363,7 +426,7 @@ void av1_set_speed_features_framesize_dependent(AV1_COMP *cpi) { cpi->find_fractional_mv_step = av1_return_min_sub_pixel_mv; } -void av1_set_speed_features_framesize_independent(AV1_COMP *cpi) { +void av1_set_speed_features_framesize_independent(AV1_COMP *cpi, int speed) { AV1_COMMON *const cm = &cpi->common; SPEED_FEATURES *const sf = &cpi->sf; MACROBLOCK *const x = &cpi->td.mb; @@ -376,7 +439,7 @@ void av1_set_speed_features_framesize_independent(AV1_COMP *cpi) { sf->recode_loop = ALLOW_RECODE; sf->mv.subpel_search_method = SUBPEL_TREE; sf->mv.subpel_iters_per_step = 2; - sf->mv.subpel_force_stop = 0; + sf->mv.subpel_force_stop = EIGHTH_PEL; #if DISABLE_TRELLISQ_SEARCH == 2 sf->optimize_coefficients = !is_lossless_requested(&cpi->oxcf) ? FINAL_PASS_TRELLIS_OPT @@ -402,6 +465,7 @@ void av1_set_speed_features_framesize_independent(AV1_COMP *cpi) { sf->tx_size_search_lgr_block = 0; sf->model_based_prune_tx_search_level = 0; sf->model_based_post_interp_filter_breakout = 0; + sf->model_based_motion_mode_rd_breakout = 0; sf->reduce_inter_modes = 0; sf->selective_ref_gm = 1; sf->adaptive_motion_search = 0; @@ -420,6 +484,7 @@ void av1_set_speed_features_framesize_independent(AV1_COMP *cpi) { sf->less_rectangular_check_level = 0; sf->use_square_partition_only_threshold = BLOCK_128X128; sf->prune_ref_frame_for_rect_partitions = 0; + sf->prune_ref_mode_for_partitions = 0; sf->auto_min_max_partition_size = NOT_IN_USE; sf->rd_auto_partition_min_limit = BLOCK_4X4; sf->default_max_partition_size = BLOCK_LARGEST; @@ -429,9 +494,11 @@ void av1_set_speed_features_framesize_independent(AV1_COMP *cpi) { sf->mode_search_skip_flags = 0; sf->disable_filter_search_var_thresh = 0; sf->allow_partition_search_skip = 0; - sf->use_accurate_subpel_search = 2; + sf->use_accurate_subpel_search = USE_8_TAPS; + sf->disable_wedge_search_edge_thresh = 0; sf->disable_wedge_search_var_thresh = 0; sf->fast_wedge_sign_estimate = 0; + sf->prune_wedge_pred_diff_based = 0; sf->drop_ref = 0; sf->skip_intra_in_interframe = 1; sf->txb_split_cap = 1; @@ -439,12 +506,17 @@ void av1_set_speed_features_framesize_independent(AV1_COMP *cpi) { sf->two_pass_partition_search = 0; sf->mode_pruning_based_on_two_pass_partition_search = 0; sf->use_intra_txb_hash = 0; - sf->use_inter_txb_hash = 1; + // TODO(any) : clean use_inter_txb_hash code + sf->use_inter_txb_hash = 0; sf->use_mb_rd_hash = 1; sf->optimize_b_precheck = 0; - sf->jnt_comp_fast_tx_search = 0; - sf->jnt_comp_skip_mv_search = 0; + sf->dist_wtd_comp_fast_tx_search = 0; + sf->use_dist_wtd_comp_flag = DIST_WTD_COMP_ENABLED; sf->reuse_inter_intra_mode = 0; + sf->intra_angle_estimation = 0; + sf->skip_obmc_in_uniform_mv_field = 0; + sf->skip_wm_in_uniform_mv_field = 0; + sf->adaptive_interp_filter_search = 0; for (i = 0; i < TX_SIZES; i++) { sf->intra_y_mode_mask[i] = INTRA_ALL; @@ -466,27 +538,48 @@ void av1_set_speed_features_framesize_independent(AV1_COMP *cpi) { sf->ml_prune_ab_partition = 0; sf->ml_prune_4_partition = 0; sf->fast_cdef_search = 0; - for (i = 0; i < PARTITION_BLOCK_SIZES; ++i) + for (i = 0; i < PARTITION_BLOCK_SIZES; ++i) { sf->ml_partition_search_breakout_thresh[i] = -1; // -1 means not enabled. + } + sf->simple_motion_search_split_only = 0; + sf->simple_motion_search_prune_rect = 0; // Set this at the appropriate speed levels sf->use_transform_domain_distortion = 0; sf->gm_search_type = GM_FULL_SEARCH; sf->gm_disable_recode = 0; sf->use_fast_interpolation_filter_search = 0; + sf->disable_dual_filter = 0; sf->skip_repeat_interpolation_filter_search = 0; sf->use_hash_based_trellis = 0; sf->prune_comp_search_by_single_result = 0; sf->skip_repeated_newmv = 0; + sf->prune_single_motion_modes_by_simple_trans = 0; // Set decoder side speed feature to use less dual sgr modes sf->dual_sgr_penalty_level = 0; + // TODO(angiebird, debargha): Re-evaluate the impact of + // inter_mode_rd_model_estimation in conjunction with + // model_based_motion_mode_rd_breakout sf->inter_mode_rd_model_estimation = 0; + sf->obmc_full_pixel_search_level = 0; + sf->skip_sharp_interp_filter_search = 0; + sf->prune_comp_type_by_comp_avg = 0; + sf->disable_interinter_wedge_newmv_search = 0; + sf->prune_motion_mode_level = 0; + sf->prune_warp_using_wmtype = 0; + + sf->disable_wedge_interintra_search = 0; + sf->perform_coeff_opt = 0; if (oxcf->mode == GOOD) - set_good_speed_features_framesize_independent(cpi, sf, oxcf->speed); + set_good_speed_features_framesize_independent(cpi, sf, speed); + + if (!cpi->seq_params_locked) { + cpi->common.seq_params.enable_dual_filter &= !sf->disable_dual_filter; + } // sf->partition_search_breakout_dist_thr is set assuming max 64x64 // blocks. Normalise this if the blocks are bigger. @@ -497,28 +590,31 @@ void av1_set_speed_features_framesize_independent(AV1_COMP *cpi) { cpi->diamond_search_sad = av1_diamond_search_sad; sf->allow_exhaustive_searches = 1; - int speed = (oxcf->speed > MAX_MESH_SPEED) ? MAX_MESH_SPEED : oxcf->speed; + + const int mesh_speed = AOMMIN(speed, MAX_MESH_SPEED); if (cpi->twopass.fr_content_type == FC_GRAPHICS_ANIMATION) sf->exhaustive_searches_thresh = (1 << 24); else sf->exhaustive_searches_thresh = (1 << 25); - sf->max_exaustive_pct = good_quality_max_mesh_pct[speed]; - if (speed > 0) + sf->max_exaustive_pct = good_quality_max_mesh_pct[mesh_speed]; + if (mesh_speed > 0) sf->exhaustive_searches_thresh = sf->exhaustive_searches_thresh << 1; for (i = 0; i < MAX_MESH_STEP; ++i) { - sf->mesh_patterns[i].range = good_quality_mesh_patterns[speed][i].range; + sf->mesh_patterns[i].range = + good_quality_mesh_patterns[mesh_speed][i].range; sf->mesh_patterns[i].interval = - good_quality_mesh_patterns[speed][i].interval; + good_quality_mesh_patterns[mesh_speed][i].interval; } if ((frame_is_intra_only(cm) && cm->allow_screen_content_tools) && (cpi->twopass.fr_content_type == FC_GRAPHICS_ANIMATION || cpi->oxcf.content == AOM_CONTENT_SCREEN)) { for (i = 0; i < MAX_MESH_STEP; ++i) { - sf->mesh_patterns[i].range = intrabc_mesh_patterns[speed][i].range; - sf->mesh_patterns[i].interval = intrabc_mesh_patterns[speed][i].interval; + sf->mesh_patterns[i].range = intrabc_mesh_patterns[mesh_speed][i].range; + sf->mesh_patterns[i].interval = + intrabc_mesh_patterns[mesh_speed][i].interval; } - sf->max_exaustive_pct = intrabc_max_mesh_pct[speed]; + sf->max_exaustive_pct = intrabc_max_mesh_pct[mesh_speed]; } // Slow quant, dct and trellis not worthwhile for first pass @@ -544,7 +640,7 @@ void av1_set_speed_features_framesize_independent(AV1_COMP *cpi) { cpi->optimize_speed_feature = oxcf->pass != 1 ? sf->optimize_coefficients : NO_TRELLIS_OPT; // FIXME: trellis not very efficient for quantisation matrices - if (cm->using_qmatrix) cpi->optimize_speed_feature = NO_TRELLIS_OPT; + if (oxcf->using_qm) cpi->optimize_speed_feature = NO_TRELLIS_OPT; if (oxcf->disable_trellis_quant) cpi->optimize_speed_feature = NO_TRELLIS_OPT; x->min_partition_size = sf->default_min_partition_size; @@ -555,10 +651,26 @@ void av1_set_speed_features_framesize_independent(AV1_COMP *cpi) { cpi->find_fractional_mv_step = av1_return_max_sub_pixel_mv; else if (cpi->oxcf.motion_vector_unit_test == 2) cpi->find_fractional_mv_step = av1_return_min_sub_pixel_mv; + cpi->max_comp_type_rd_threshold_mul = + comp_type_rd_threshold_mul[sf->prune_comp_type_by_comp_avg]; + cpi->max_comp_type_rd_threshold_div = + comp_type_rd_threshold_div[sf->prune_comp_type_by_comp_avg]; + const int tx_domain_speed = AOMMIN(speed, MAX_TX_DOMAIN_EVAL_SPEED); + cpi->tx_domain_dist_threshold = tx_domain_dist_thresholds[tx_domain_speed]; + + // assert ensures that coeff_opt_dist_thresholds is accessed correctly + assert(cpi->sf.perform_coeff_opt >= 0 && cpi->sf.perform_coeff_opt < 5); + cpi->coeff_opt_dist_threshold = + coeff_opt_dist_thresholds[cpi->sf.perform_coeff_opt]; #if CONFIG_DIST_8X8 if (sf->use_transform_domain_distortion > 0) cpi->oxcf.using_dist_8x8 = 0; if (cpi->oxcf.using_dist_8x8) x->min_partition_size = BLOCK_8X8; #endif // CONFIG_DIST_8X8 + if (cpi->oxcf.row_mt == 1 && (cpi->oxcf.max_threads > 1)) { + sf->adaptive_rd_thresh = 0; + if (sf->inter_mode_rd_model_estimation == 1) + sf->inter_mode_rd_model_estimation = 0; + } } diff --git a/third_party/aom/av1/encoder/speed_features.h b/third_party/aom/av1/encoder/speed_features.h index 41013b2e79..2216c7be04 100644 --- a/third_party/aom/av1/encoder/speed_features.h +++ b/third_party/aom/av1/encoder/speed_features.h @@ -73,7 +73,7 @@ enum { (1 << THR_ALTR) | (1 << THR_GOLD) }; -typedef enum { +enum { TXFM_CODING_SF = 1, INTER_PRED_SF = 2, INTRA_PRED_SF = 4, @@ -82,9 +82,9 @@ typedef enum { RD_SKIP_SF = 32, RESERVE_2_SF = 64, RESERVE_3_SF = 128, -} DEV_SPEED_FEATURES; +} UENUM1BYTE(DEV_SPEED_FEATURES); -typedef enum { +enum { DIAMOND = 0, NSTEP = 1, HEX = 2, @@ -92,9 +92,9 @@ typedef enum { SQUARE = 4, FAST_HEX = 5, FAST_DIAMOND = 6 -} SEARCH_METHODS; +} UENUM1BYTE(SEARCH_METHODS); -typedef enum { +enum { // No recode. DISALLOW_RECODE = 0, // Allow recode for KF and exceeding maximum frame bandwidth. @@ -103,28 +103,28 @@ typedef enum { ALLOW_RECODE_KFARFGF = 2, // Allow recode for all frames based on bitrate constraints. ALLOW_RECODE = 3, -} RECODE_LOOP_TYPE; +} UENUM1BYTE(RECODE_LOOP_TYPE); -typedef enum { +enum { SUBPEL_TREE = 0, SUBPEL_TREE_PRUNED = 1, // Prunes 1/2-pel searches SUBPEL_TREE_PRUNED_MORE = 2, // Prunes 1/2-pel searches more aggressively SUBPEL_TREE_PRUNED_EVENMORE = 3, // Prunes 1/2- and 1/4-pel searches // Other methods to come -} SUBPEL_SEARCH_METHODS; +} UENUM1BYTE(SUBPEL_SEARCH_METHODS); -typedef enum { +enum { USE_FULL_RD = 0, USE_FAST_RD, USE_LARGESTALL, -} TX_SIZE_SEARCH_METHOD; +} UENUM1BYTE(TX_SIZE_SEARCH_METHOD); -typedef enum { +enum { NOT_IN_USE = 0, RELAXED_NEIGHBORING_MIN_MAX = 1 -} AUTO_MIN_MAX_MODE; +} UENUM1BYTE(AUTO_MIN_MAX_MODE); -typedef enum { +enum { // Try the full image with different values. LPF_PICK_FROM_FULL_IMAGE, // Try a small portion of the image with different values. @@ -133,9 +133,9 @@ typedef enum { LPF_PICK_FROM_Q, // Pick 0 to disable LPF if LPF was enabled last frame LPF_PICK_MINIMAL_LPF -} LPF_PICK_METHOD; +} UENUM1BYTE(LPF_PICK_METHOD); -typedef enum { +enum { // Terminate search early based on distortion so far compared to // qp step, distortion in the neighborhood of the frame, etc. FLAG_EARLY_TERMINATE = 1 << 0, @@ -152,9 +152,9 @@ typedef enum { // Skips intra modes other than DC_PRED if the source variance is small FLAG_SKIP_INTRA_LOWVAR = 1 << 5, -} MODE_SEARCH_SKIP_LOGIC; +} UENUM1BYTE(MODE_SEARCH_SKIP_LOGIC); -typedef enum { +enum { NO_PRUNE = 0, // eliminates one tx type in vertical and horizontal direction PRUNE_ONE = 1, @@ -165,7 +165,7 @@ typedef enum { PRUNE_2D_ACCURATE = 3, // similar, but applies much more aggressive pruning to get better speed-up PRUNE_2D_FAST = 4, -} TX_TYPE_PRUNE_MODE; +} UENUM1BYTE(TX_TYPE_PRUNE_MODE); typedef struct { TX_TYPE_PRUNE_MODE prune_mode; @@ -184,7 +184,7 @@ typedef struct { int skip_tx_search; } TX_TYPE_SEARCH; -typedef enum { +enum { // Search partitions using RD criterion SEARCH_PARTITION, @@ -192,7 +192,14 @@ typedef enum { FIXED_PARTITION, REFERENCE_PARTITION -} PARTITION_SEARCH_TYPE; +} UENUM1BYTE(PARTITION_SEARCH_TYPE); + +enum { + EIGHTH_PEL, + QUARTER_PEL, + HALF_PEL, + FULL_PEL +} UENUM1BYTE(SUBPEL_FORCE_STOP); typedef struct MV_SPEED_FEATURES { // Motion search method (Diamond, NSTEP, Hex, Big Diamond, Square, etc). @@ -215,8 +222,8 @@ typedef struct MV_SPEED_FEATURES { // Maximum number of steps in logarithmic subpel search before giving up. int subpel_iters_per_step; - // Control when to stop subpel search - int subpel_force_stop; + // When to stop subpel search. + SUBPEL_FORCE_STOP subpel_force_stop; } MV_SPEED_FEATURES; #define MAX_MESH_STEP 4 @@ -226,29 +233,42 @@ typedef struct MESH_PATTERN { int interval; } MESH_PATTERN; -typedef enum { +enum { GM_FULL_SEARCH, - GM_REDUCED_REF_SEARCH, + GM_REDUCED_REF_SEARCH_SKIP_L2_L3, + GM_REDUCED_REF_SEARCH_SKIP_L2_L3_ARF2, GM_DISABLE_SEARCH -} GM_SEARCH_TYPE; +} UENUM1BYTE(GM_SEARCH_TYPE); -typedef enum { +enum { GM_ERRORADV_TR_0, GM_ERRORADV_TR_1, GM_ERRORADV_TR_2, GM_ERRORADV_TR_TYPES, -} GM_ERRORADV_TYPE; +} UENUM1BYTE(GM_ERRORADV_TYPE); -typedef enum { +enum { NO_TRELLIS_OPT, // No trellis optimization FULL_TRELLIS_OPT, // Trellis optimization in all stages FINAL_PASS_TRELLIS_OPT // Trellis optimization in only the final encode pass -} TRELLIS_OPT_TYPE; +} UENUM1BYTE(TRELLIS_OPT_TYPE); -typedef enum { +enum { FULL_TXFM_RD, LOW_TXFM_RD, -} TXFM_RD_MODEL; +} UENUM1BYTE(TXFM_RD_MODEL); + +enum { + DIST_WTD_COMP_ENABLED, + DIST_WTD_COMP_SKIP_MV_SEARCH, + DIST_WTD_COMP_DISABLED, +} UENUM1BYTE(DIST_WTD_COMP_FLAG); + +typedef enum { + FLAG_SKIP_EIGHTTAP = 1 << EIGHTTAP_REGULAR, + FLAG_SKIP_EIGHTTAP_SMOOTH = 1 << EIGHTTAP_SMOOTH, + FLAG_SKIP_EIGHTTAP_SHARP = 1 << MULTITAP_SHARP, +} INTERP_FILTER_MASK; typedef struct SPEED_FEATURES { MV_SPEED_FEATURES mv; @@ -329,11 +349,16 @@ typedef struct SPEED_FEATURES { // 1: use model based rd breakout int model_based_post_interp_filter_breakout; + // Model based breakout in motion_mode_rd + // 0: no breakout + // 1: use model based rd breakout + int model_based_motion_mode_rd_breakout; + // Used if partition_search_type = FIXED_SIZE_PARTITION BLOCK_SIZE always_this_block_size; // Drop less likely to be picked reference frames in the RD search. - // Has three levels for now: 0, 1 and 2, where higher levels prune more + // Has five levels for now: 0, 1, 2, 3 and 4, where higher levels prune more // aggressively than lower ones. (0 means no pruning). int selective_ref_frame; @@ -371,6 +396,9 @@ typedef struct SPEED_FEATURES { // Prune reference frames for rectangular partitions. int prune_ref_frame_for_rect_partitions; + // Prune ref/mode choices for partitions. + int prune_ref_mode_for_partitions; + // Sets min and max partition sizes for this superblock based on the // same superblock in last encoded frame, and the left and above neighbor. AUTO_MIN_MAX_MODE auto_min_max_partition_size; @@ -431,12 +459,19 @@ typedef struct SPEED_FEATURES { // Choose a very large value (UINT_MAX) to use 8-tap always unsigned int disable_filter_search_var_thresh; - // A source variance threshold below which wedge search is disabled + // Only enable wedge search if the edge strength is greater than + // this threshold. A value of 0 signals that this check is disabled. + unsigned int disable_wedge_search_edge_thresh; + + // Only enable wedge search if the variance is above this threshold. unsigned int disable_wedge_search_var_thresh; // Whether fast wedge sign estimate is used int fast_wedge_sign_estimate; + // Whether to prune wedge search based on predictor difference + int prune_wedge_pred_diff_based; + // These bit masks allow you to enable or disable intra modes for each // transform size separately. int intra_y_mode_mask[TX_SIZES]; @@ -473,7 +508,7 @@ typedef struct SPEED_FEATURES { // If true, sub-pixel search uses the exact convolve function used for final // encoding and decoding; otherwise, it uses bilinear interpolation. - int use_accurate_subpel_search; + SUBPEL_SEARCH_TYPE use_accurate_subpel_search; // Whether to compute distortion in the image domain (slower but // more accurate), or in the transform domain (faster but less acurate). @@ -492,6 +527,9 @@ typedef struct SPEED_FEATURES { // usually includes EIGHTTAP_REGULAR. int use_fast_interpolation_filter_search; + // Disable dual filter + int disable_dual_filter; + // Save results of interpolation_filter_search for a block // Check mv and ref_frames before search, if they are same with previous // saved results, it can be skipped. @@ -522,18 +560,23 @@ typedef struct SPEED_FEATURES { // Calculate RD cost before doing optimize_b, and skip if the cost is large. int optimize_b_precheck; - // Use model rd instead of transform search in jnt_comp - int jnt_comp_fast_tx_search; + // Use model rd instead of transform search in dist_wtd_comp + int dist_wtd_comp_fast_tx_search; - // Skip mv search in jnt_comp - int jnt_comp_skip_mv_search; + // Decide when and how to use joint_comp. + DIST_WTD_COMP_FLAG use_dist_wtd_comp_flag; // Decoder side speed feature to add penalty for use of dual-sgr filters. // Takes values 0 - 10, 0 indicating no penalty and each additional level // adding a penalty of 1% int dual_sgr_penalty_level; - // Dynamically estimate final rd from prediction error and mode cost + // 2-pass inter mode model estimation where the preliminary pass skips + // transform search and uses a model to estimate rd, while the final pass + // computes the full transform search. two types of models are supported: + // 0: not used + // 1: used with online dynamic rd model + // 2: used with static rd model int inter_mode_rd_model_estimation; // Skip some ref frames in compound motion search by single motion search @@ -543,6 +586,10 @@ typedef struct SPEED_FEATURES { // single inter mode as a group. int prune_comp_search_by_single_result; + // Skip certain motion modes (OBMC, warped, interintra) for single reference + // motion search, using the results of single ref SIMPLE_TRANSLATION + int prune_single_motion_modes_by_simple_trans; + // Reuse the inter_intra_mode search result from NEARESTMV mode to other // single ref modes int reuse_inter_intra_mode; @@ -554,12 +601,65 @@ typedef struct SPEED_FEATURES { // flag to skip NEWMV mode in drl if the motion search result is the same int skip_repeated_newmv; + + // Prune intra mode candidates based on source block gradient stats. + int intra_angle_estimation; + + // Skip obmc or warped motion mode when neighborhood motion field is + // identical + int skip_obmc_in_uniform_mv_field; + int skip_wm_in_uniform_mv_field; + + // Enable/disable ME for interinter wedge search. + int disable_interinter_wedge_newmv_search; + + // skip sharp_filter evaluation based on regular and smooth filter rd for + // dual_filter=0 case + int skip_sharp_interp_filter_search; + + // prune wedge and compound segment approximate rd evaluation based on + // compound average rd/ref_best_rd + int prune_comp_type_by_comp_avg; + + // Prune/gate motion mode evaluation based on token based rd + // during transform search for inter blocks + // Values are 0 (not used) , 1 - 3 with progressively increasing + // aggressiveness + int prune_motion_mode_level; + + // Gate warp evaluation for motions of type IDENTITY, + // TRANSLATION and AFFINE(based on number of warp neighbors) + int prune_warp_using_wmtype; + + // Perform simple_motion_search on each possible subblock and use it to prune + // PARTITION_HORZ and PARTITION_VERT. + int simple_motion_search_prune_rect; + + // Perform simple motion search before none_partition to decide if we + // want to split directly without trying other partition types. + int simple_motion_search_split_only; + + int cb_pred_filter_search; + + // adaptive interp_filter search to allow skip of certain filter types. + int adaptive_interp_filter_search; + + // mask for skip evaluation of certain interp_filter type. + INTERP_FILTER_MASK interp_filter_search_mask; + + // Enable/disable interintra wedge search. + int disable_wedge_interintra_search; + + // Flag used to control the extent of coeff R-D optimization + int perform_coeff_opt; } SPEED_FEATURES; struct AV1_COMP; -void av1_set_speed_features_framesize_independent(struct AV1_COMP *cpi); -void av1_set_speed_features_framesize_dependent(struct AV1_COMP *cpi); +void av1_set_speed_features_framesize_independent(struct AV1_COMP *cpi, + int speed); +void av1_set_speed_features_framesize_dependent(struct AV1_COMP *cpi, + int speed); #ifdef __cplusplus } // extern "C" diff --git a/third_party/aom/av1/encoder/temporal_filter.c b/third_party/aom/av1/encoder/temporal_filter.c index 75fdf02a52..385f027932 100644 --- a/third_party/aom/av1/encoder/temporal_filter.c +++ b/third_party/aom/av1/encoder/temporal_filter.c @@ -34,49 +34,466 @@ #include "aom_ports/aom_timer.h" #include "aom_scale/aom_scale.h" +#define EDGE_THRESHOLD 50 +#define SQRT_PI_BY_2 1.25331413732 + static void temporal_filter_predictors_mb_c( MACROBLOCKD *xd, uint8_t *y_mb_ptr, uint8_t *u_mb_ptr, uint8_t *v_mb_ptr, int stride, int uv_block_width, int uv_block_height, int mv_row, int mv_col, uint8_t *pred, struct scale_factors *scale, int x, int y, - int can_use_previous, int num_planes) { - const MV mv = { mv_row, mv_col }; - enum mv_precision mv_precision_uv; + int can_use_previous, int num_planes, MV *blk_mvs, int use_32x32) { + mv_precision mv_precision_uv; int uv_stride; // TODO(angiebird): change plane setting accordingly ConvolveParams conv_params = get_conv_params(0, 0, xd->bd); - const InterpFilters interp_filters = xd->mi[0]->interp_filters; + const InterpFilters interp_filters = + av1_make_interp_filters(MULTITAP_SHARP, MULTITAP_SHARP); WarpTypesAllowed warp_types; memset(&warp_types, 0, sizeof(WarpTypesAllowed)); - if (uv_block_width == 8) { + const int ssx = (uv_block_width == (BW >> 1)) ? 1 : 0; + if (ssx) { uv_stride = (stride + 1) >> 1; mv_precision_uv = MV_PRECISION_Q4; } else { uv_stride = stride; mv_precision_uv = MV_PRECISION_Q3; } - av1_build_inter_predictor(y_mb_ptr, stride, &pred[0], 16, &mv, scale, 16, 16, - &conv_params, interp_filters, &warp_types, x, y, 0, - 0, MV_PRECISION_Q3, x, y, xd, can_use_previous); + if (use_32x32) { + const MV mv = { mv_row, mv_col }; + + av1_build_inter_predictor(y_mb_ptr, stride, &pred[0], BW, &mv, scale, BW, + BH, &conv_params, interp_filters, &warp_types, x, + y, 0, 0, MV_PRECISION_Q3, x, y, xd, + can_use_previous); + if (num_planes > 1) { + av1_build_inter_predictor( + u_mb_ptr, uv_stride, &pred[BLK_PELS], uv_block_width, &mv, scale, + uv_block_width, uv_block_height, &conv_params, interp_filters, + &warp_types, x, y, 1, 0, mv_precision_uv, x, y, xd, can_use_previous); + av1_build_inter_predictor( + v_mb_ptr, uv_stride, &pred[(BLK_PELS << 1)], uv_block_width, &mv, + scale, uv_block_width, uv_block_height, &conv_params, interp_filters, + &warp_types, x, y, 2, 0, mv_precision_uv, x, y, xd, can_use_previous); + } + + return; + } + + // While use_32x32 = 0, construct the 32x32 predictor using 4 16x16 + // predictors. + int i, j, k = 0, ys = (BH >> 1), xs = (BW >> 1); + // Y predictor + for (i = 0; i < BH; i += ys) { + for (j = 0; j < BW; j += xs) { + const MV mv = blk_mvs[k]; + const int y_offset = i * stride + j; + const int p_offset = i * BW + j; + + av1_build_inter_predictor(y_mb_ptr + y_offset, stride, &pred[p_offset], + BW, &mv, scale, xs, ys, &conv_params, + interp_filters, &warp_types, x, y, 0, 0, + MV_PRECISION_Q3, x, y, xd, can_use_previous); + k++; + } + } + + // U and V predictors if (num_planes > 1) { - av1_build_inter_predictor( - u_mb_ptr, uv_stride, &pred[256], uv_block_width, &mv, scale, - uv_block_width, uv_block_height, &conv_params, interp_filters, - &warp_types, x, y, 1, 0, mv_precision_uv, x, y, xd, can_use_previous); + ys = (uv_block_height >> 1); + xs = (uv_block_width >> 1); + k = 0; - av1_build_inter_predictor( - v_mb_ptr, uv_stride, &pred[512], uv_block_width, &mv, scale, - uv_block_width, uv_block_height, &conv_params, interp_filters, - &warp_types, x, y, 2, 0, mv_precision_uv, x, y, xd, can_use_previous); + for (i = 0; i < uv_block_height; i += ys) { + for (j = 0; j < uv_block_width; j += xs) { + const MV mv = blk_mvs[k]; + const int uv_offset = i * uv_stride + j; + const int p_offset = i * uv_block_width + j; + + av1_build_inter_predictor(u_mb_ptr + uv_offset, uv_stride, + &pred[BLK_PELS + p_offset], uv_block_width, + &mv, scale, xs, ys, &conv_params, + interp_filters, &warp_types, x, y, 1, 0, + mv_precision_uv, x, y, xd, can_use_previous); + av1_build_inter_predictor( + v_mb_ptr + uv_offset, uv_stride, &pred[(BLK_PELS << 1) + p_offset], + uv_block_width, &mv, scale, xs, ys, &conv_params, interp_filters, + &warp_types, x, y, 2, 0, mv_precision_uv, x, y, xd, + can_use_previous); + k++; + } + } } } +static void apply_temporal_filter_self(const uint8_t *pred, int buf_stride, + unsigned int block_width, + unsigned int block_height, + int filter_weight, uint32_t *accumulator, + uint16_t *count) { + const int modifier = filter_weight * 16; + unsigned int i, j, k = 0; + assert(filter_weight == 2); + + for (i = 0; i < block_height; i++) { + for (j = 0; j < block_width; j++) { + const int pixel_value = pred[i * buf_stride + j]; + count[k] += modifier; + accumulator[k] += modifier * pixel_value; + ++k; + } + } +} + +static void highbd_apply_temporal_filter_self( + const uint8_t *pred8, int buf_stride, unsigned int block_width, + unsigned int block_height, int filter_weight, uint32_t *accumulator, + uint16_t *count) { + const int modifier = filter_weight * 16; + const uint16_t *pred = CONVERT_TO_SHORTPTR(pred8); + unsigned int i, j, k = 0; + assert(filter_weight == 2); + + for (i = 0; i < block_height; i++) { + for (j = 0; j < block_width; j++) { + const int pixel_value = pred[i * buf_stride + j]; + count[k] += modifier; + accumulator[k] += modifier * pixel_value; + ++k; + } + } +} + +static INLINE int mod_index(int64_t sum_dist, int index, int rounding, + int strength, int filter_weight) { + int mod = (int)(((sum_dist * 3) / index + rounding) >> strength); + mod = AOMMIN(16, mod); + mod = 16 - mod; + mod *= filter_weight; + return mod; +} + +static INLINE void calculate_squared_errors(const uint8_t *s, int s_stride, + const uint8_t *p, int p_stride, + uint16_t *diff_sse, unsigned int w, + unsigned int h) { + int idx = 0; + unsigned int i, j; + + for (i = 0; i < h; i++) { + for (j = 0; j < w; j++) { + const int16_t diff = s[i * s_stride + j] - p[i * p_stride + j]; + diff_sse[idx] = diff * diff; + idx++; + } + } +} + +static INLINE int get_filter_weight(unsigned int i, unsigned int j, + unsigned int block_height, + unsigned int block_width, int *blk_fw, + int use_32x32) { + if (use_32x32) + // blk_fw[0] ~ blk_fw[3] are the same. + return blk_fw[0]; + + int filter_weight = 0; + if (i < block_height / 2) { + if (j < block_width / 2) + filter_weight = blk_fw[0]; + else + filter_weight = blk_fw[1]; + } else { + if (j < block_width / 2) + filter_weight = blk_fw[2]; + else + filter_weight = blk_fw[3]; + } + return filter_weight; +} + +static void apply_temporal_filter( + const uint8_t *y_frame1, int y_stride, const uint8_t *y_pred, + int y_buf_stride, const uint8_t *u_frame1, const uint8_t *v_frame1, + int uv_stride, const uint8_t *u_pred, const uint8_t *v_pred, + int uv_buf_stride, unsigned int block_width, unsigned int block_height, + int ss_x, int ss_y, int strength, int *blk_fw, int use_32x32, + uint32_t *y_accumulator, uint16_t *y_count, uint32_t *u_accumulator, + uint16_t *u_count, uint32_t *v_accumulator, uint16_t *v_count) { + unsigned int i, j, k, m; + int modifier; + const int rounding = (1 << strength) >> 1; + const unsigned int uv_block_width = block_width >> ss_x; + const unsigned int uv_block_height = block_height >> ss_y; + DECLARE_ALIGNED(16, uint16_t, y_diff_sse[BLK_PELS]); + DECLARE_ALIGNED(16, uint16_t, u_diff_sse[BLK_PELS]); + DECLARE_ALIGNED(16, uint16_t, v_diff_sse[BLK_PELS]); + + int idx = 0, idy; + + memset(y_diff_sse, 0, BLK_PELS * sizeof(uint16_t)); + memset(u_diff_sse, 0, BLK_PELS * sizeof(uint16_t)); + memset(v_diff_sse, 0, BLK_PELS * sizeof(uint16_t)); + + // Calculate diff^2 for each pixel of the block. + // TODO(yunqing): the following code needs to be optimized. + calculate_squared_errors(y_frame1, y_stride, y_pred, y_buf_stride, y_diff_sse, + block_width, block_height); + calculate_squared_errors(u_frame1, uv_stride, u_pred, uv_buf_stride, + u_diff_sse, uv_block_width, uv_block_height); + calculate_squared_errors(v_frame1, uv_stride, v_pred, uv_buf_stride, + v_diff_sse, uv_block_width, uv_block_height); + + for (i = 0, k = 0, m = 0; i < block_height; i++) { + for (j = 0; j < block_width; j++) { + const int pixel_value = y_pred[i * y_buf_stride + j]; + int filter_weight = + get_filter_weight(i, j, block_height, block_width, blk_fw, use_32x32); + + // non-local mean approach + int y_index = 0; + + const int uv_r = i >> ss_y; + const int uv_c = j >> ss_x; + modifier = 0; + + for (idy = -1; idy <= 1; ++idy) { + for (idx = -1; idx <= 1; ++idx) { + const int row = (int)i + idy; + const int col = (int)j + idx; + + if (row >= 0 && row < (int)block_height && col >= 0 && + col < (int)block_width) { + modifier += y_diff_sse[row * (int)block_width + col]; + ++y_index; + } + } + } + + assert(y_index > 0); + + modifier += u_diff_sse[uv_r * uv_block_width + uv_c]; + modifier += v_diff_sse[uv_r * uv_block_width + uv_c]; + + y_index += 2; + + modifier = + (int)mod_index(modifier, y_index, rounding, strength, filter_weight); + + y_count[k] += modifier; + y_accumulator[k] += modifier * pixel_value; + + ++k; + + // Process chroma component + if (!(i & ss_y) && !(j & ss_x)) { + const int u_pixel_value = u_pred[uv_r * uv_buf_stride + uv_c]; + const int v_pixel_value = v_pred[uv_r * uv_buf_stride + uv_c]; + + // non-local mean approach + int cr_index = 0; + int u_mod = 0, v_mod = 0; + int y_diff = 0; + + for (idy = -1; idy <= 1; ++idy) { + for (idx = -1; idx <= 1; ++idx) { + const int row = uv_r + idy; + const int col = uv_c + idx; + + if (row >= 0 && row < (int)uv_block_height && col >= 0 && + col < (int)uv_block_width) { + u_mod += u_diff_sse[row * uv_block_width + col]; + v_mod += v_diff_sse[row * uv_block_width + col]; + ++cr_index; + } + } + } + + assert(cr_index > 0); + + for (idy = 0; idy < 1 + ss_y; ++idy) { + for (idx = 0; idx < 1 + ss_x; ++idx) { + const int row = (uv_r << ss_y) + idy; + const int col = (uv_c << ss_x) + idx; + y_diff += y_diff_sse[row * (int)block_width + col]; + ++cr_index; + } + } + + u_mod += y_diff; + v_mod += y_diff; + + u_mod = + (int)mod_index(u_mod, cr_index, rounding, strength, filter_weight); + v_mod = + (int)mod_index(v_mod, cr_index, rounding, strength, filter_weight); + + u_count[m] += u_mod; + u_accumulator[m] += u_mod * u_pixel_value; + v_count[m] += v_mod; + v_accumulator[m] += v_mod * v_pixel_value; + + ++m; + } // Complete YUV pixel + } + } +} + +static INLINE void highbd_calculate_squared_errors( + const uint16_t *s, int s_stride, const uint16_t *p, int p_stride, + uint32_t *diff_sse, unsigned int w, unsigned int h) { + int idx = 0; + unsigned int i, j; + + for (i = 0; i < h; i++) { + for (j = 0; j < w; j++) { + const int16_t diff = s[i * s_stride + j] - p[i * p_stride + j]; + diff_sse[idx] = diff * diff; + idx++; + } + } +} + +static void highbd_apply_temporal_filter( + const uint8_t *yf, int y_stride, const uint8_t *yp, int y_buf_stride, + const uint8_t *uf, const uint8_t *vf, int uv_stride, const uint8_t *up, + const uint8_t *vp, int uv_buf_stride, unsigned int block_width, + unsigned int block_height, int ss_x, int ss_y, int strength, int *blk_fw, + int use_32x32, uint32_t *y_accumulator, uint16_t *y_count, + uint32_t *u_accumulator, uint16_t *u_count, uint32_t *v_accumulator, + uint16_t *v_count) { + unsigned int i, j, k, m; + int64_t modifier; + const int rounding = (1 << strength) >> 1; + const unsigned int uv_block_width = block_width >> ss_x; + const unsigned int uv_block_height = block_height >> ss_y; + DECLARE_ALIGNED(16, uint32_t, y_diff_sse[BLK_PELS]); + DECLARE_ALIGNED(16, uint32_t, u_diff_sse[BLK_PELS]); + DECLARE_ALIGNED(16, uint32_t, v_diff_sse[BLK_PELS]); + + const uint16_t *y_frame1 = CONVERT_TO_SHORTPTR(yf); + const uint16_t *u_frame1 = CONVERT_TO_SHORTPTR(uf); + const uint16_t *v_frame1 = CONVERT_TO_SHORTPTR(vf); + const uint16_t *y_pred = CONVERT_TO_SHORTPTR(yp); + const uint16_t *u_pred = CONVERT_TO_SHORTPTR(up); + const uint16_t *v_pred = CONVERT_TO_SHORTPTR(vp); + int idx = 0, idy; + + memset(y_diff_sse, 0, BLK_PELS * sizeof(uint32_t)); + memset(u_diff_sse, 0, BLK_PELS * sizeof(uint32_t)); + memset(v_diff_sse, 0, BLK_PELS * sizeof(uint32_t)); + + // Calculate diff^2 for each pixel of the block. + // TODO(yunqing): the following code needs to be optimized. + highbd_calculate_squared_errors(y_frame1, y_stride, y_pred, y_buf_stride, + y_diff_sse, block_width, block_height); + highbd_calculate_squared_errors(u_frame1, uv_stride, u_pred, uv_buf_stride, + u_diff_sse, uv_block_width, uv_block_height); + highbd_calculate_squared_errors(v_frame1, uv_stride, v_pred, uv_buf_stride, + v_diff_sse, uv_block_width, uv_block_height); + + for (i = 0, k = 0, m = 0; i < block_height; i++) { + for (j = 0; j < block_width; j++) { + const int pixel_value = y_pred[i * y_buf_stride + j]; + int filter_weight = + get_filter_weight(i, j, block_height, block_width, blk_fw, use_32x32); + + // non-local mean approach + int y_index = 0; + + const int uv_r = i >> ss_y; + const int uv_c = j >> ss_x; + modifier = 0; + + for (idy = -1; idy <= 1; ++idy) { + for (idx = -1; idx <= 1; ++idx) { + const int row = (int)i + idy; + const int col = (int)j + idx; + + if (row >= 0 && row < (int)block_height && col >= 0 && + col < (int)block_width) { + modifier += y_diff_sse[row * (int)block_width + col]; + ++y_index; + } + } + } + + assert(y_index > 0); + + modifier += u_diff_sse[uv_r * uv_block_width + uv_c]; + modifier += v_diff_sse[uv_r * uv_block_width + uv_c]; + + y_index += 2; + + const int final_y_mod = + mod_index(modifier, y_index, rounding, strength, filter_weight); + + y_count[k] += final_y_mod; + y_accumulator[k] += final_y_mod * pixel_value; + + ++k; + + // Process chroma component + if (!(i & ss_y) && !(j & ss_x)) { + const int u_pixel_value = u_pred[uv_r * uv_buf_stride + uv_c]; + const int v_pixel_value = v_pred[uv_r * uv_buf_stride + uv_c]; + + // non-local mean approach + int cr_index = 0; + int64_t u_mod = 0, v_mod = 0; + int y_diff = 0; + + for (idy = -1; idy <= 1; ++idy) { + for (idx = -1; idx <= 1; ++idx) { + const int row = uv_r + idy; + const int col = uv_c + idx; + + if (row >= 0 && row < (int)uv_block_height && col >= 0 && + col < (int)uv_block_width) { + u_mod += u_diff_sse[row * uv_block_width + col]; + v_mod += v_diff_sse[row * uv_block_width + col]; + ++cr_index; + } + } + } + + assert(cr_index > 0); + + for (idy = 0; idy < 1 + ss_y; ++idy) { + for (idx = 0; idx < 1 + ss_x; ++idx) { + const int row = (uv_r << ss_y) + idy; + const int col = (uv_c << ss_x) + idx; + y_diff += y_diff_sse[row * (int)block_width + col]; + ++cr_index; + } + } + + u_mod += y_diff; + v_mod += y_diff; + + const int final_u_mod = + mod_index(u_mod, cr_index, rounding, strength, filter_weight); + const int final_v_mod = + mod_index(v_mod, cr_index, rounding, strength, filter_weight); + + u_count[m] += final_u_mod; + u_accumulator[m] += final_u_mod * u_pixel_value; + v_count[m] += final_v_mod; + v_accumulator[m] += final_v_mod * v_pixel_value; + + ++m; + } // Complete YUV pixel + } + } +} + +// Only used in single plane case void av1_temporal_filter_apply_c(uint8_t *frame1, unsigned int stride, uint8_t *frame2, unsigned int block_width, unsigned int block_height, int strength, - int filter_weight, unsigned int *accumulator, - uint16_t *count) { + int *blk_fw, int use_32x32, + unsigned int *accumulator, uint16_t *count) { unsigned int i, j, k; int modifier; int byte = 0; @@ -85,6 +502,8 @@ void av1_temporal_filter_apply_c(uint8_t *frame1, unsigned int stride, for (i = 0, k = 0; i < block_height; i++) { for (j = 0; j < block_width; j++, k++) { int pixel_value = *frame2; + int filter_weight = + get_filter_weight(i, j, block_height, block_width, blk_fw, use_32x32); // non-local mean approach int diff_sse[9] = { 0 }; @@ -133,10 +552,11 @@ void av1_temporal_filter_apply_c(uint8_t *frame1, unsigned int stride, } } +// Only used in single plane case void av1_highbd_temporal_filter_apply_c( uint8_t *frame1_8, unsigned int stride, uint8_t *frame2_8, unsigned int block_width, unsigned int block_height, int strength, - int filter_weight, unsigned int *accumulator, uint16_t *count) { + int *blk_fw, int use_32x32, unsigned int *accumulator, uint16_t *count) { uint16_t *frame1 = CONVERT_TO_SHORTPTR(frame1_8); uint16_t *frame2 = CONVERT_TO_SHORTPTR(frame2_8); unsigned int i, j, k; @@ -147,6 +567,8 @@ void av1_highbd_temporal_filter_apply_c( for (i = 0, k = 0; i < block_height; i++) { for (j = 0; j < block_width; j++, k++) { int pixel_value = *frame2; + int filter_weight = + get_filter_weight(i, j, block_height, block_width, blk_fw, use_32x32); // non-local mean approach int diff_sse[9] = { 0 }; @@ -198,8 +620,8 @@ void av1_highbd_temporal_filter_apply_c( static int temporal_filter_find_matching_mb_c(AV1_COMP *cpi, uint8_t *arf_frame_buf, uint8_t *frame_ptr_buf, - int stride, int x_pos, - int y_pos) { + int stride, int x_pos, int y_pos, + MV *blk_mvs, int *blk_bestsme) { MACROBLOCK *const x = &cpi->td.mb; MACROBLOCKD *const xd = &x->e_mbd; const MV_SPEED_FEATURES *const mv_sf = &cpi->sf.mv; @@ -232,12 +654,12 @@ static int temporal_filter_find_matching_mb_c(AV1_COMP *cpi, av1_set_mv_search_range(&x->mv_limits, &best_ref_mv1); - x->mvcost = x->mv_cost_stack; - x->nmvjointcost = x->nmv_vec_cost; - - av1_full_pixel_search(cpi, x, BLOCK_16X16, &best_ref_mv1_full, step_param, - NSTEP, 1, sadpb, cond_cost_list(cpi, cost_list), - &best_ref_mv1, 0, 0, x_pos, y_pos, 0); + // av1_full_pixel_search() parameters: best_ref_mv1_full is the start mv, and + // best_ref_mv1 is for mv rate calculation. The search result is stored in + // x->best_mv. + av1_full_pixel_search(cpi, x, TF_BLOCK, &best_ref_mv1_full, step_param, NSTEP, + 1, sadpb, cond_cost_list(cpi, cost_list), &best_ref_mv1, + 0, 0, x_pos, y_pos, 0); x->mv_limits = tmp_mv_limits; // Ignore mv costing by sending NULL pointer instead of cost array @@ -251,19 +673,63 @@ static int temporal_filter_find_matching_mb_c(AV1_COMP *cpi, x->best_mv.as_mv.row *= 8; x->best_mv.as_mv.col *= 8; - bestsme = cpi->fn_ptr[BLOCK_16X16].vf(y + offset, y_stride, src_address, - src_stride, &sse); - } else { - bestsme = cpi->find_fractional_mv_step( - x, &cpi->common, 0, 0, &best_ref_mv1, - cpi->common.allow_high_precision_mv, x->errorperbit, - &cpi->fn_ptr[BLOCK_16X16], 0, mv_sf->subpel_iters_per_step, - cond_cost_list(cpi, cost_list), NULL, NULL, &distortion, &sse, NULL, - NULL, 0, 0, 0, 0, 0); + bestsme = cpi->fn_ptr[TF_BLOCK].vf(y + offset, y_stride, src_address, + src_stride, &sse); + + x->e_mbd.mi[0]->mv[0] = x->best_mv; + + // Restore input state + x->plane[0].src = src; + xd->plane[0].pre[0] = pre; + + return bestsme; } + // find_fractional_mv_step parameters: best_ref_mv1 is for mv rate cost + // calculation. The start full mv and the search result are stored in + // x->best_mv. mi_row and mi_col are only needed for "av1_is_scaled(sf)=1" + // case. + bestsme = cpi->find_fractional_mv_step( + x, &cpi->common, 0, 0, &best_ref_mv1, cpi->common.allow_high_precision_mv, + x->errorperbit, &cpi->fn_ptr[TF_BLOCK], 0, mv_sf->subpel_iters_per_step, + cond_cost_list(cpi, cost_list), NULL, NULL, &distortion, &sse, NULL, NULL, + 0, 0, BW, BH, USE_8_TAPS, 1); + x->e_mbd.mi[0]->mv[0] = x->best_mv; + // DO motion search on 4 16x16 sub_blocks. + int i, j, k = 0; + best_ref_mv1.row = x->e_mbd.mi[0]->mv[0].as_mv.row; + best_ref_mv1.col = x->e_mbd.mi[0]->mv[0].as_mv.col; + best_ref_mv1_full.col = best_ref_mv1.col >> 3; + best_ref_mv1_full.row = best_ref_mv1.row >> 3; + + for (i = 0; i < BH; i += SUB_BH) { + for (j = 0; j < BW; j += SUB_BW) { + // Setup frame pointers + x->plane[0].src.buf = arf_frame_buf + i * stride + j; + x->plane[0].src.stride = stride; + xd->plane[0].pre[0].buf = frame_ptr_buf + i * stride + j; + xd->plane[0].pre[0].stride = stride; + + av1_set_mv_search_range(&x->mv_limits, &best_ref_mv1); + av1_full_pixel_search( + cpi, x, TF_SUB_BLOCK, &best_ref_mv1_full, step_param, NSTEP, 1, sadpb, + cond_cost_list(cpi, cost_list), &best_ref_mv1, 0, 0, x_pos, y_pos, 0); + x->mv_limits = tmp_mv_limits; + + blk_bestsme[k] = cpi->find_fractional_mv_step( + x, &cpi->common, 0, 0, &best_ref_mv1, + cpi->common.allow_high_precision_mv, x->errorperbit, + &cpi->fn_ptr[TF_SUB_BLOCK], 0, mv_sf->subpel_iters_per_step, + cond_cost_list(cpi, cost_list), NULL, NULL, &distortion, &sse, NULL, + NULL, 0, 0, SUB_BW, SUB_BH, USE_8_TAPS, 1); + + blk_mvs[k] = x->best_mv.as_mv; + k++; + } + } + // Restore input state x->plane[0].src = src; xd->plane[0].pre[0] = pre; @@ -275,37 +741,40 @@ static void temporal_filter_iterate_c(AV1_COMP *cpi, YV12_BUFFER_CONFIG **frames, int frame_count, int alt_ref_index, int strength, - struct scale_factors *scale) { + struct scale_factors *ref_scale_factors) { const AV1_COMMON *cm = &cpi->common; const int num_planes = av1_num_planes(cm); int byte; int frame; int mb_col, mb_row; - unsigned int filter_weight; - int mb_cols = (frames[alt_ref_index]->y_crop_width + 15) >> 4; - int mb_rows = (frames[alt_ref_index]->y_crop_height + 15) >> 4; + int mb_cols = (frames[alt_ref_index]->y_crop_width + BW - 1) >> BW_LOG2; + int mb_rows = (frames[alt_ref_index]->y_crop_height + BH - 1) >> BH_LOG2; int mb_y_offset = 0; int mb_uv_offset = 0; - DECLARE_ALIGNED(16, unsigned int, accumulator[16 * 16 * 3]); - DECLARE_ALIGNED(16, uint16_t, count[16 * 16 * 3]); + DECLARE_ALIGNED(16, unsigned int, accumulator[BLK_PELS * 3]); + DECLARE_ALIGNED(16, uint16_t, count[BLK_PELS * 3]); MACROBLOCKD *mbd = &cpi->td.mb.e_mbd; YV12_BUFFER_CONFIG *f = frames[alt_ref_index]; uint8_t *dst1, *dst2; - DECLARE_ALIGNED(32, uint16_t, predictor16[16 * 16 * 3]); - DECLARE_ALIGNED(32, uint8_t, predictor8[16 * 16 * 3]); + DECLARE_ALIGNED(32, uint16_t, predictor16[BLK_PELS * 3]); + DECLARE_ALIGNED(32, uint8_t, predictor8[BLK_PELS * 3]); uint8_t *predictor; - const int mb_uv_height = 16 >> mbd->plane[1].subsampling_y; - const int mb_uv_width = 16 >> mbd->plane[1].subsampling_x; + const int mb_uv_height = BH >> mbd->plane[1].subsampling_y; + const int mb_uv_width = BW >> mbd->plane[1].subsampling_x; // Save input state uint8_t *input_buffer[MAX_MB_PLANE]; int i; - if (mbd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + const int is_hbd = is_cur_buf_hbd(mbd); + if (is_hbd) { predictor = CONVERT_TO_BYTEPTR(predictor16); } else { predictor = predictor8; } + mbd->block_ref_scale_factors[0] = ref_scale_factors; + mbd->block_ref_scale_factors[1] = ref_scale_factors; + for (i = 0; i < num_planes; i++) input_buffer[i] = mbd->plane[i].pre[0].buf; for (mb_row = 0; mb_row < mb_rows; mb_row++) { @@ -321,101 +790,170 @@ static void temporal_filter_iterate_c(AV1_COMP *cpi, // To keep the mv in play for both Y and UV planes the max that it // can be on a border is therefore 16 - (2*AOM_INTERP_EXTEND+1). cpi->td.mb.mv_limits.row_min = - -((mb_row * 16) + (17 - 2 * AOM_INTERP_EXTEND)); + -((mb_row * BH) + (17 - 2 * AOM_INTERP_EXTEND)); cpi->td.mb.mv_limits.row_max = - ((mb_rows - 1 - mb_row) * 16) + (17 - 2 * AOM_INTERP_EXTEND); + ((mb_rows - 1 - mb_row) * BH) + (17 - 2 * AOM_INTERP_EXTEND); for (mb_col = 0; mb_col < mb_cols; mb_col++) { int j, k; int stride; - memset(accumulator, 0, 16 * 16 * 3 * sizeof(accumulator[0])); - memset(count, 0, 16 * 16 * 3 * sizeof(count[0])); + memset(accumulator, 0, BLK_PELS * 3 * sizeof(accumulator[0])); + memset(count, 0, BLK_PELS * 3 * sizeof(count[0])); cpi->td.mb.mv_limits.col_min = - -((mb_col * 16) + (17 - 2 * AOM_INTERP_EXTEND)); + -((mb_col * BW) + (17 - 2 * AOM_INTERP_EXTEND)); cpi->td.mb.mv_limits.col_max = - ((mb_cols - 1 - mb_col) * 16) + (17 - 2 * AOM_INTERP_EXTEND); + ((mb_cols - 1 - mb_col) * BW) + (17 - 2 * AOM_INTERP_EXTEND); for (frame = 0; frame < frame_count; frame++) { - const int thresh_low = 10000; - const int thresh_high = 20000; + // MVs for 4 16x16 sub blocks. + MV blk_mvs[4]; + // Filter weights for 4 16x16 sub blocks. + int blk_fw[4] = { 0, 0, 0, 0 }; + int use_32x32 = 0; if (frames[frame] == NULL) continue; mbd->mi[0]->mv[0].as_mv.row = 0; mbd->mi[0]->mv[0].as_mv.col = 0; mbd->mi[0]->motion_mode = SIMPLE_TRANSLATION; + blk_mvs[0] = kZeroMv; + blk_mvs[1] = kZeroMv; + blk_mvs[2] = kZeroMv; + blk_mvs[3] = kZeroMv; if (frame == alt_ref_index) { - filter_weight = 2; + blk_fw[0] = blk_fw[1] = blk_fw[2] = blk_fw[3] = 2; + use_32x32 = 1; } else { + int thresh_low = 10000; + int thresh_high = 20000; + int blk_bestsme[4] = { INT_MAX, INT_MAX, INT_MAX, INT_MAX }; + // Find best match in this frame by MC int err = temporal_filter_find_matching_mb_c( cpi, frames[alt_ref_index]->y_buffer + mb_y_offset, frames[frame]->y_buffer + mb_y_offset, frames[frame]->y_stride, - mb_col * 16, mb_row * 16); + mb_col * BW, mb_row * BH, blk_mvs, blk_bestsme); - // Assign higher weight to matching MB if it's error - // score is lower. If not applying MC default behavior - // is to weight all MBs equal. - filter_weight = err < thresh_low ? 2 : err < thresh_high ? 1 : 0; + int err16 = + blk_bestsme[0] + blk_bestsme[1] + blk_bestsme[2] + blk_bestsme[3]; + int max_err = INT_MIN, min_err = INT_MAX; + for (k = 0; k < 4; k++) { + if (min_err > blk_bestsme[k]) min_err = blk_bestsme[k]; + if (max_err < blk_bestsme[k]) max_err = blk_bestsme[k]; + } + + if (((err * 15 < (err16 << 4)) && max_err - min_err < 12000) || + ((err * 14 < (err16 << 4)) && max_err - min_err < 6000)) { + use_32x32 = 1; + // Assign higher weight to matching MB if it's error + // score is lower. If not applying MC default behavior + // is to weight all MBs equal. + blk_fw[0] = err < (thresh_low << THR_SHIFT) + ? 2 + : err < (thresh_high << THR_SHIFT) ? 1 : 0; + blk_fw[1] = blk_fw[2] = blk_fw[3] = blk_fw[0]; + } else { + use_32x32 = 0; + for (k = 0; k < 4; k++) + blk_fw[k] = blk_bestsme[k] < thresh_low + ? 2 + : blk_bestsme[k] < thresh_high ? 1 : 0; + } } - if (filter_weight != 0) { + if (blk_fw[0] || blk_fw[1] || blk_fw[2] || blk_fw[3]) { // Construct the predictors temporal_filter_predictors_mb_c( mbd, frames[frame]->y_buffer + mb_y_offset, frames[frame]->u_buffer + mb_uv_offset, frames[frame]->v_buffer + mb_uv_offset, frames[frame]->y_stride, mb_uv_width, mb_uv_height, mbd->mi[0]->mv[0].as_mv.row, - mbd->mi[0]->mv[0].as_mv.col, predictor, scale, mb_col * 16, - mb_row * 16, cm->allow_warped_motion, num_planes); + mbd->mi[0]->mv[0].as_mv.col, predictor, ref_scale_factors, + mb_col * BW, mb_row * BH, cm->allow_warped_motion, num_planes, + blk_mvs, use_32x32); // Apply the filter (YUV) - if (mbd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { - int adj_strength = strength + 2 * (mbd->bd - 8); - av1_highbd_temporal_filter_apply( - f->y_buffer + mb_y_offset, f->y_stride, predictor, 16, 16, - adj_strength, filter_weight, accumulator, count); - if (num_planes > 1) { - av1_highbd_temporal_filter_apply( - f->u_buffer + mb_uv_offset, f->uv_stride, predictor + 256, - mb_uv_width, mb_uv_height, adj_strength, filter_weight, - accumulator + 256, count + 256); - av1_highbd_temporal_filter_apply( - f->v_buffer + mb_uv_offset, f->uv_stride, predictor + 512, - mb_uv_width, mb_uv_height, adj_strength, filter_weight, - accumulator + 512, count + 512); + if (frame == alt_ref_index) { + uint8_t *pred = predictor; + uint32_t *accum = accumulator; + uint16_t *cnt = count; + int plane; + + // All 4 blk_fws are equal to 2. + for (plane = 0; plane < num_planes; ++plane) { + const int pred_stride = plane ? mb_uv_width : BW; + const unsigned int w = plane ? mb_uv_width : BW; + const unsigned int h = plane ? mb_uv_height : BH; + + if (is_hbd) { + highbd_apply_temporal_filter_self(pred, pred_stride, w, h, + blk_fw[0], accum, cnt); + } else { + apply_temporal_filter_self(pred, pred_stride, w, h, blk_fw[0], + accum, cnt); + } + + pred += BLK_PELS; + accum += BLK_PELS; + cnt += BLK_PELS; } } else { - av1_temporal_filter_apply_c(f->y_buffer + mb_y_offset, f->y_stride, - predictor, 16, 16, strength, - filter_weight, accumulator, count); - if (num_planes > 1) { - av1_temporal_filter_apply_c( - f->u_buffer + mb_uv_offset, f->uv_stride, predictor + 256, - mb_uv_width, mb_uv_height, strength, filter_weight, - accumulator + 256, count + 256); - av1_temporal_filter_apply_c( - f->v_buffer + mb_uv_offset, f->uv_stride, predictor + 512, - mb_uv_width, mb_uv_height, strength, filter_weight, - accumulator + 512, count + 512); + if (is_hbd) { + const int adj_strength = strength + 2 * (mbd->bd - 8); + + if (num_planes <= 1) { + // Single plane case + av1_highbd_temporal_filter_apply_c( + f->y_buffer + mb_y_offset, f->y_stride, predictor, BW, BH, + adj_strength, blk_fw, use_32x32, accumulator, count); + } else { + // Process 3 planes together. + highbd_apply_temporal_filter( + f->y_buffer + mb_y_offset, f->y_stride, predictor, BW, + f->u_buffer + mb_uv_offset, f->v_buffer + mb_uv_offset, + f->uv_stride, predictor + BLK_PELS, + predictor + (BLK_PELS << 1), mb_uv_width, BW, BH, + mbd->plane[1].subsampling_x, mbd->plane[1].subsampling_y, + adj_strength, blk_fw, use_32x32, accumulator, count, + accumulator + BLK_PELS, count + BLK_PELS, + accumulator + (BLK_PELS << 1), count + (BLK_PELS << 1)); + } + } else { + if (num_planes <= 1) { + // Single plane case + av1_temporal_filter_apply_c( + f->y_buffer + mb_y_offset, f->y_stride, predictor, BW, BH, + strength, blk_fw, use_32x32, accumulator, count); + } else { + // Process 3 planes together. + apply_temporal_filter( + f->y_buffer + mb_y_offset, f->y_stride, predictor, BW, + f->u_buffer + mb_uv_offset, f->v_buffer + mb_uv_offset, + f->uv_stride, predictor + BLK_PELS, + predictor + (BLK_PELS << 1), mb_uv_width, BW, BH, + mbd->plane[1].subsampling_x, mbd->plane[1].subsampling_y, + strength, blk_fw, use_32x32, accumulator, count, + accumulator + BLK_PELS, count + BLK_PELS, + accumulator + (BLK_PELS << 1), count + (BLK_PELS << 1)); + } } } } } // Normalize filter output to produce AltRef frame - if (mbd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + if (is_hbd) { uint16_t *dst1_16; uint16_t *dst2_16; dst1 = cpi->alt_ref_buffer.y_buffer; dst1_16 = CONVERT_TO_SHORTPTR(dst1); stride = cpi->alt_ref_buffer.y_stride; byte = mb_y_offset; - for (i = 0, k = 0; i < 16; i++) { - for (j = 0; j < 16; j++, k++) { + for (i = 0, k = 0; i < BH; i++) { + for (j = 0; j < BW; j++, k++) { dst1_16[byte] = (uint16_t)OD_DIVU(accumulator[k] + (count[k] >> 1), count[k]); @@ -423,7 +961,7 @@ static void temporal_filter_iterate_c(AV1_COMP *cpi, byte++; } - byte += stride - 16; + byte += stride - BW; } if (num_planes > 1) { dst1 = cpi->alt_ref_buffer.u_buffer; @@ -432,9 +970,9 @@ static void temporal_filter_iterate_c(AV1_COMP *cpi, dst2_16 = CONVERT_TO_SHORTPTR(dst2); stride = cpi->alt_ref_buffer.uv_stride; byte = mb_uv_offset; - for (i = 0, k = 256; i < mb_uv_height; i++) { + for (i = 0, k = BLK_PELS; i < mb_uv_height; i++) { for (j = 0; j < mb_uv_width; j++, k++) { - int m = k + 256; + int m = k + BLK_PELS; // U dst1_16[byte] = (uint16_t)OD_DIVU(accumulator[k] + (count[k] >> 1), count[k]); @@ -451,24 +989,24 @@ static void temporal_filter_iterate_c(AV1_COMP *cpi, dst1 = cpi->alt_ref_buffer.y_buffer; stride = cpi->alt_ref_buffer.y_stride; byte = mb_y_offset; - for (i = 0, k = 0; i < 16; i++) { - for (j = 0; j < 16; j++, k++) { + for (i = 0, k = 0; i < BH; i++) { + for (j = 0; j < BW; j++, k++) { dst1[byte] = (uint8_t)OD_DIVU(accumulator[k] + (count[k] >> 1), count[k]); // move to next pixel byte++; } - byte += stride - 16; + byte += stride - BW; } if (num_planes > 1) { dst1 = cpi->alt_ref_buffer.u_buffer; dst2 = cpi->alt_ref_buffer.v_buffer; stride = cpi->alt_ref_buffer.uv_stride; byte = mb_uv_offset; - for (i = 0, k = 256; i < mb_uv_height; i++) { + for (i = 0, k = BLK_PELS; i < mb_uv_height; i++) { for (j = 0; j < mb_uv_width; j++, k++) { - int m = k + 256; + int m = k + BLK_PELS; // U dst1[byte] = (uint8_t)OD_DIVU(accumulator[k] + (count[k] >> 1), count[k]); @@ -482,10 +1020,10 @@ static void temporal_filter_iterate_c(AV1_COMP *cpi, } } } - mb_y_offset += 16; + mb_y_offset += BW; mb_uv_offset += mb_uv_width; } - mb_y_offset += 16 * (f->y_stride - mb_cols); + mb_y_offset += BH * f->y_stride - BW * mb_cols; mb_uv_offset += mb_uv_height * f->uv_stride - mb_uv_width * mb_cols; } @@ -493,6 +1031,83 @@ static void temporal_filter_iterate_c(AV1_COMP *cpi, for (i = 0; i < num_planes; i++) mbd->plane[i].pre[0].buf = input_buffer[i]; } +// This is an adaptation of the mehtod in the following paper: +// Shen-Chuan Tai, Shih-Ming Yang, "A fast method for image noise +// estimation using Laplacian operator and adaptive edge detection," +// Proc. 3rd International Symposium on Communications, Control and +// Signal Processing, 2008, St Julians, Malta. +// +// Return noise estimate, or -1.0 if there was a failure +static double estimate_noise(const uint8_t *src, int width, int height, + int stride, int edge_thresh) { + int64_t sum = 0; + int64_t num = 0; + for (int i = 1; i < height - 1; ++i) { + for (int j = 1; j < width - 1; ++j) { + const int k = i * stride + j; + // Sobel gradients + const int Gx = (src[k - stride - 1] - src[k - stride + 1]) + + (src[k + stride - 1] - src[k + stride + 1]) + + 2 * (src[k - 1] - src[k + 1]); + const int Gy = (src[k - stride - 1] - src[k + stride - 1]) + + (src[k - stride + 1] - src[k + stride + 1]) + + 2 * (src[k - stride] - src[k + stride]); + const int Ga = abs(Gx) + abs(Gy); + if (Ga < edge_thresh) { // Smooth pixels + // Find Laplacian + const int v = + 4 * src[k] - + 2 * (src[k - 1] + src[k + 1] + src[k - stride] + src[k + stride]) + + (src[k - stride - 1] + src[k - stride + 1] + src[k + stride - 1] + + src[k + stride + 1]); + sum += abs(v); + ++num; + } + } + } + // If very few smooth pels, return -1 since the estimate is unreliable + if (num < 16) return -1.0; + + const double sigma = (double)sum / (6 * num) * SQRT_PI_BY_2; + return sigma; +} + +// Return noise estimate, or -1.0 if there was a failure +static double highbd_estimate_noise(const uint8_t *src8, int width, int height, + int stride, int bd, int edge_thresh) { + uint16_t *src = CONVERT_TO_SHORTPTR(src8); + int64_t sum = 0; + int64_t num = 0; + for (int i = 1; i < height - 1; ++i) { + for (int j = 1; j < width - 1; ++j) { + const int k = i * stride + j; + // Sobel gradients + const int Gx = (src[k - stride - 1] - src[k - stride + 1]) + + (src[k + stride - 1] - src[k + stride + 1]) + + 2 * (src[k - 1] - src[k + 1]); + const int Gy = (src[k - stride - 1] - src[k + stride - 1]) + + (src[k - stride + 1] - src[k + stride + 1]) + + 2 * (src[k - stride] - src[k + stride]); + const int Ga = ROUND_POWER_OF_TWO(abs(Gx) + abs(Gy), bd - 8); + if (Ga < edge_thresh) { // Smooth pixels + // Find Laplacian + const int v = + 4 * src[k] - + 2 * (src[k - 1] + src[k + 1] + src[k - stride] + src[k + stride]) + + (src[k - stride - 1] + src[k - stride + 1] + src[k + stride - 1] + + src[k + stride + 1]); + sum += ROUND_POWER_OF_TWO(abs(v), bd - 8); + ++num; + } + } + } + // If very few smooth pels, return -1 since the estimate is unreliable + if (num < 16) return -1.0; + + const double sigma = (double)sum / (6 * num) * SQRT_PI_BY_2; + return sigma; +} + // Apply buffer limits and context specific adjustments to arnr filter. static void adjust_arnr_filter(AV1_COMP *cpi, int distance, int group_boost, int *arnr_frames, int *arnr_strength) { @@ -517,16 +1132,44 @@ static void adjust_arnr_filter(AV1_COMP *cpi, int distance, int group_boost, frames = frames_bwd + 1 + frames_fwd; // Adjust the strength based on active max q. - if (cpi->common.current_video_frame > 1) + if (cpi->common.current_frame.frame_number > 1) q = ((int)av1_convert_qindex_to_q(cpi->rc.avg_frame_qindex[INTER_FRAME], cpi->common.seq_params.bit_depth)); else q = ((int)av1_convert_qindex_to_q(cpi->rc.avg_frame_qindex[KEY_FRAME], cpi->common.seq_params.bit_depth)); - if (q > 16) { - strength = oxcf->arnr_strength; + MACROBLOCKD *mbd = &cpi->td.mb.e_mbd; + struct lookahead_entry *buf = av1_lookahead_peek(cpi->lookahead, distance); + double noiselevel; + if (is_cur_buf_hbd(mbd)) { + noiselevel = highbd_estimate_noise( + buf->img.y_buffer, buf->img.y_crop_width, buf->img.y_crop_height, + buf->img.y_stride, mbd->bd, EDGE_THRESHOLD); } else { - strength = oxcf->arnr_strength - ((16 - q) / 2); + noiselevel = estimate_noise(buf->img.y_buffer, buf->img.y_crop_width, + buf->img.y_crop_height, buf->img.y_stride, + EDGE_THRESHOLD); + } + int adj_strength = oxcf->arnr_strength; + if (noiselevel > 0) { + // Get 4 integer adjustment levels in [-2, 1] + int noiselevel_adj; + if (noiselevel < 0.75) + noiselevel_adj = -2; + else if (noiselevel < 1.75) + noiselevel_adj = -1; + else if (noiselevel < 4.0) + noiselevel_adj = 0; + else + noiselevel_adj = 1; + adj_strength += noiselevel_adj; + } + // printf("[noise level: %g, strength = %d]\n", noiselevel, adj_strength); + + if (q > 16) { + strength = adj_strength; + } else { + strength = adj_strength - ((16 - q) / 2); if (strength < 0) strength = 0; } @@ -553,18 +1196,22 @@ void av1_temporal_filter(AV1_COMP *cpi, int distance) { int frames_to_blur_backward; int frames_to_blur_forward; struct scale_factors sf; + YV12_BUFFER_CONFIG *frames[MAX_LAG_BUFFERS] = { NULL }; const GF_GROUP *const gf_group = &cpi->twopass.gf_group; + int rdmult = 0; // Apply context specific adjustments to the arnr filter parameters. - adjust_arnr_filter(cpi, distance, rc->gfu_boost, &frames_to_blur, &strength); - // TODO(weitinglin): Currently, we enforce the filtering strength on - // extra ARFs' to be zeros. We should investigate in which - // case it is more beneficial to use non-zero strength - // filtering. if (gf_group->update_type[gf_group->index] == INTNL_ARF_UPDATE) { + // TODO(weitinglin): Currently, we enforce the filtering strength on + // extra ARFs' to be zeros. We should investigate in which + // case it is more beneficial to use non-zero strength + // filtering. strength = 0; frames_to_blur = 1; + } else { + adjust_arnr_filter(cpi, distance, rc->gfu_boost, &frames_to_blur, + &strength); } int which_arf = gf_group->arf_update_idx[gf_group->index]; @@ -597,6 +1244,12 @@ void av1_temporal_filter(AV1_COMP *cpi, int distance) { frames[0]->y_crop_width, frames[0]->y_crop_height); } + // Initialize errorperbit, sadperbit16 and sadperbit4. + rdmult = av1_compute_rd_mult_based_on_qindex(cpi, ARNR_FILT_QINDEX); + set_error_per_bit(&cpi->td.mb, rdmult); + av1_initialize_me_consts(cpi, &cpi->td.mb, ARNR_FILT_QINDEX); + av1_initialize_cost_tables(&cpi->common, &cpi->td.mb); + temporal_filter_iterate_c(cpi, frames, frames_to_blur, frames_to_blur_backward, strength, &sf); } diff --git a/third_party/aom/av1/encoder/temporal_filter.h b/third_party/aom/av1/encoder/temporal_filter.h index 2ddc68b2c9..bb26c36660 100644 --- a/third_party/aom/av1/encoder/temporal_filter.h +++ b/third_party/aom/av1/encoder/temporal_filter.h @@ -16,6 +16,20 @@ extern "C" { #endif +#define ARNR_FILT_QINDEX 128 + +// Block size used in temporal filtering +#define TF_BLOCK BLOCK_32X32 +#define BH 32 +#define BH_LOG2 5 +#define BW 32 +#define BW_LOG2 5 +#define BLK_PELS 1024 // Pixels in the block +#define THR_SHIFT 2 +#define TF_SUB_BLOCK BLOCK_16X16 +#define SUB_BH 16 +#define SUB_BW 16 + void av1_temporal_filter(AV1_COMP *cpi, int distance); #ifdef __cplusplus diff --git a/third_party/aom/av1/encoder/tokenize.c b/third_party/aom/av1/encoder/tokenize.c index 16a6a9a35a..ce1a2121a4 100644 --- a/third_party/aom/av1/encoder/tokenize.c +++ b/third_party/aom/av1/encoder/tokenize.c @@ -29,7 +29,7 @@ static int cost_and_tokenize_map(Av1ColorMapParam *param, TOKENEXTRA **t, int plane, int calc_rate, int allow_update_cdf, - FRAME_COUNTS *counts) { + FRAME_COUNTS *counts, MapCdf map_pb_cdf) { const uint8_t *const color_map = param->color_map; MapCdf map_cdf = param->map_cdf; ColorCost color_cost = param->color_cost; @@ -55,7 +55,7 @@ static int cost_and_tokenize_map(Av1ColorMapParam *param, TOKENEXTRA **t, this_rate += (*color_cost)[palette_size_idx][color_ctx][color_new_idx]; } else { (*t)->token = color_new_idx; - (*t)->color_map_cdf = map_cdf[palette_size_idx][color_ctx]; + (*t)->color_map_cdf = map_pb_cdf[palette_size_idx][color_ctx]; ++(*t); if (allow_update_cdf) update_cdf(map_cdf[palette_size_idx][color_ctx], color_new_idx, n); @@ -107,7 +107,10 @@ int av1_cost_color_map(const MACROBLOCK *const x, int plane, BLOCK_SIZE bsize, assert(plane == 0 || plane == 1); Av1ColorMapParam color_map_params; get_color_map_params(x, plane, bsize, tx_size, type, &color_map_params); - return cost_and_tokenize_map(&color_map_params, NULL, plane, 1, 0, NULL); + MapCdf map_pb_cdf = plane ? x->tile_pb_ctx->palette_uv_color_index_cdf + : x->tile_pb_ctx->palette_y_color_index_cdf; + return cost_and_tokenize_map(&color_map_params, NULL, plane, 1, 0, NULL, + map_pb_cdf); } void av1_tokenize_color_map(const MACROBLOCK *const x, int plane, @@ -121,13 +124,15 @@ void av1_tokenize_color_map(const MACROBLOCK *const x, int plane, (*t)->token = color_map_params.color_map[0]; (*t)->color_map_cdf = NULL; ++(*t); + MapCdf map_pb_cdf = plane ? x->tile_pb_ctx->palette_uv_color_index_cdf + : x->tile_pb_ctx->palette_y_color_index_cdf; cost_and_tokenize_map(&color_map_params, t, plane, 0, allow_update_cdf, - counts); + counts, map_pb_cdf); } -void tokenize_vartx(ThreadData *td, TOKENEXTRA **t, RUN_TYPE dry_run, - TX_SIZE tx_size, BLOCK_SIZE plane_bsize, int blk_row, - int blk_col, int block, int plane, void *arg) { +static void tokenize_vartx(ThreadData *td, TOKENEXTRA **t, RUN_TYPE dry_run, + TX_SIZE tx_size, BLOCK_SIZE plane_bsize, int blk_row, + int blk_col, int block, int plane, void *arg) { MACROBLOCK *const x = &td->mb; MACROBLOCKD *const xd = &x->e_mbd; MB_MODE_INFO *const mbmi = xd->mi[0]; diff --git a/third_party/aom/av1/encoder/tokenize.h b/third_party/aom/av1/encoder/tokenize.h index 63b505f36f..c80af7bd0d 100644 --- a/third_party/aom/av1/encoder/tokenize.h +++ b/third_party/aom/av1/encoder/tokenize.h @@ -38,11 +38,11 @@ struct tokenize_b_args { uint8_t allow_update_cdf; }; -typedef enum { +enum { OUTPUT_ENABLED = 0, DRY_RUN_NORMAL, DRY_RUN_COSTCOEFFS, -} RUN_TYPE; +} UENUM1BYTE(RUN_TYPE); // Note in all the tokenize functions rate if non NULL is incremented // with the coefficient token cost only if dry_run = DRY_RUN_COSTCOEFS, diff --git a/third_party/aom/av1/encoder/tpl_model.c b/third_party/aom/av1/encoder/tpl_model.c new file mode 100644 index 0000000000..3399116e9f --- /dev/null +++ b/third_party/aom/av1/encoder/tpl_model.c @@ -0,0 +1,595 @@ +/* + * Copyright (c) 2019, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "config/aom_config.h" +#include "config/aom_dsp_rtcd.h" + +#include "aom/aom_codec.h" + +#include "av1/common/onyxc_int.h" +#include "av1/common/reconintra.h" + +#include "av1/encoder/encoder.h" +#include "av1/encoder/reconinter_enc.h" + +typedef struct GF_PICTURE { + YV12_BUFFER_CONFIG *frame; + int ref_frame[7]; +} GF_PICTURE; + +static void get_quantize_error(MACROBLOCK *x, int plane, tran_low_t *coeff, + tran_low_t *qcoeff, tran_low_t *dqcoeff, + TX_SIZE tx_size, int64_t *recon_error, + int64_t *sse) { + const struct macroblock_plane *const p = &x->plane[plane]; + const SCAN_ORDER *const scan_order = &av1_default_scan_orders[tx_size]; + uint16_t eob; + int pix_num = 1 << num_pels_log2_lookup[txsize_to_bsize[tx_size]]; + const int shift = tx_size == TX_32X32 ? 0 : 2; + + av1_quantize_fp_32x32(coeff, pix_num, p->zbin_QTX, p->round_fp_QTX, + p->quant_fp_QTX, p->quant_shift_QTX, qcoeff, dqcoeff, + p->dequant_QTX, &eob, scan_order->scan, + scan_order->iscan); + + *recon_error = av1_block_error(coeff, dqcoeff, pix_num, sse) >> shift; + *recon_error = AOMMAX(*recon_error, 1); + + *sse = (*sse) >> shift; + *sse = AOMMAX(*sse, 1); +} + +static void wht_fwd_txfm(int16_t *src_diff, int bw, tran_low_t *coeff, + TX_SIZE tx_size) { + switch (tx_size) { + case TX_8X8: aom_hadamard_8x8(src_diff, bw, coeff); break; + case TX_16X16: aom_hadamard_16x16(src_diff, bw, coeff); break; + case TX_32X32: aom_hadamard_32x32(src_diff, bw, coeff); break; + default: assert(0); + } +} + +static uint32_t motion_compensated_prediction(AV1_COMP *cpi, ThreadData *td, + uint8_t *cur_frame_buf, + uint8_t *ref_frame_buf, + int stride, BLOCK_SIZE bsize, + int mi_row, int mi_col) { + AV1_COMMON *cm = &cpi->common; + MACROBLOCK *const x = &td->mb; + MACROBLOCKD *const xd = &x->e_mbd; + MV_SPEED_FEATURES *const mv_sf = &cpi->sf.mv; + const SEARCH_METHODS search_method = NSTEP; + int step_param; + int sadpb = x->sadperbit16; + uint32_t bestsme = UINT_MAX; + int distortion; + uint32_t sse; + int cost_list[5]; + const MvLimits tmp_mv_limits = x->mv_limits; + + MV best_ref_mv1 = { 0, 0 }; + MV best_ref_mv1_full; /* full-pixel value of best_ref_mv1 */ + + best_ref_mv1_full.col = best_ref_mv1.col >> 3; + best_ref_mv1_full.row = best_ref_mv1.row >> 3; + + // Setup frame pointers + x->plane[0].src.buf = cur_frame_buf; + x->plane[0].src.stride = stride; + xd->plane[0].pre[0].buf = ref_frame_buf; + xd->plane[0].pre[0].stride = stride; + + step_param = mv_sf->reduce_first_step_size; + step_param = AOMMIN(step_param, MAX_MVSEARCH_STEPS - 2); + + av1_set_mv_search_range(&x->mv_limits, &best_ref_mv1); + + av1_full_pixel_search(cpi, x, bsize, &best_ref_mv1_full, step_param, + search_method, 0, sadpb, cond_cost_list(cpi, cost_list), + &best_ref_mv1, INT_MAX, 0, (MI_SIZE * mi_col), + (MI_SIZE * mi_row), 0); + + /* restore UMV window */ + x->mv_limits = tmp_mv_limits; + + const int pw = block_size_wide[bsize]; + const int ph = block_size_high[bsize]; + bestsme = cpi->find_fractional_mv_step( + x, cm, mi_row, mi_col, &best_ref_mv1, cpi->common.allow_high_precision_mv, + x->errorperbit, &cpi->fn_ptr[bsize], 0, mv_sf->subpel_iters_per_step, + cond_cost_list(cpi, cost_list), NULL, NULL, &distortion, &sse, NULL, NULL, + 0, 0, pw, ph, 1, 1); + + return bestsme; +} + +static void mode_estimation(AV1_COMP *cpi, MACROBLOCK *x, MACROBLOCKD *xd, + struct scale_factors *sf, GF_PICTURE *gf_picture, + int frame_idx, int16_t *src_diff, tran_low_t *coeff, + tran_low_t *qcoeff, tran_low_t *dqcoeff, int mi_row, + int mi_col, BLOCK_SIZE bsize, TX_SIZE tx_size, + YV12_BUFFER_CONFIG *ref_frame[], uint8_t *predictor, + int64_t *recon_error, int64_t *sse, + TplDepStats *tpl_stats) { + AV1_COMMON *cm = &cpi->common; + ThreadData *td = &cpi->td; + + const int bw = 4 << mi_size_wide_log2[bsize]; + const int bh = 4 << mi_size_high_log2[bsize]; + const int pix_num = bw * bh; + int best_rf_idx = -1; + int_mv best_mv; + int64_t best_inter_cost = INT64_MAX; + int64_t inter_cost; + int rf_idx; + const InterpFilters kernel = + av1_make_interp_filters(EIGHTTAP_REGULAR, EIGHTTAP_REGULAR); + + int64_t best_intra_cost = INT64_MAX; + int64_t intra_cost; + PREDICTION_MODE mode; + int mb_y_offset = mi_row * MI_SIZE * xd->cur_buf->y_stride + mi_col * MI_SIZE; + MB_MODE_INFO mi_above, mi_left; + + memset(tpl_stats, 0, sizeof(*tpl_stats)); + + xd->mb_to_top_edge = -((mi_row * MI_SIZE) * 8); + xd->mb_to_bottom_edge = ((cm->mi_rows - 1 - mi_row) * MI_SIZE) * 8; + xd->mb_to_left_edge = -((mi_col * MI_SIZE) * 8); + xd->mb_to_right_edge = ((cm->mi_cols - 1 - mi_col) * MI_SIZE) * 8; + xd->above_mbmi = (mi_row > 0) ? &mi_above : NULL; + xd->left_mbmi = (mi_col > 0) ? &mi_left : NULL; + + // Intra prediction search + for (mode = DC_PRED; mode <= PAETH_PRED; ++mode) { + uint8_t *src, *dst; + int src_stride, dst_stride; + + src = xd->cur_buf->y_buffer + mb_y_offset; + src_stride = xd->cur_buf->y_stride; + + dst = &predictor[0]; + dst_stride = bw; + + xd->mi[0]->sb_type = bsize; + xd->mi[0]->ref_frame[0] = INTRA_FRAME; + + av1_predict_intra_block( + cm, xd, block_size_wide[bsize], block_size_high[bsize], tx_size, mode, + 0, 0, FILTER_INTRA_MODES, src, src_stride, dst, dst_stride, 0, 0, 0); + + if (is_cur_buf_hbd(xd)) { + aom_highbd_subtract_block(bh, bw, src_diff, bw, src, src_stride, dst, + dst_stride, xd->bd); + } else { + aom_subtract_block(bh, bw, src_diff, bw, src, src_stride, dst, + dst_stride); + } + + wht_fwd_txfm(src_diff, bw, coeff, tx_size); + + intra_cost = aom_satd(coeff, pix_num); + + if (intra_cost < best_intra_cost) best_intra_cost = intra_cost; + } + + // Motion compensated prediction + best_mv.as_int = 0; + + (void)mb_y_offset; + // Motion estimation column boundary + x->mv_limits.col_min = -((mi_col * MI_SIZE) + (17 - 2 * AOM_INTERP_EXTEND)); + x->mv_limits.col_max = + ((cm->mi_cols - 1 - mi_col) * MI_SIZE) + (17 - 2 * AOM_INTERP_EXTEND); + + for (rf_idx = 0; rf_idx < 7; ++rf_idx) { + if (ref_frame[rf_idx] == NULL) continue; + + motion_compensated_prediction(cpi, td, xd->cur_buf->y_buffer + mb_y_offset, + ref_frame[rf_idx]->y_buffer + mb_y_offset, + xd->cur_buf->y_stride, bsize, mi_row, mi_col); + + // TODO(jingning): Not yet support high bit-depth in the next three + // steps. + ConvolveParams conv_params = get_conv_params(0, 0, xd->bd); + WarpTypesAllowed warp_types; + memset(&warp_types, 0, sizeof(WarpTypesAllowed)); + + av1_build_inter_predictor( + ref_frame[rf_idx]->y_buffer + mb_y_offset, ref_frame[rf_idx]->y_stride, + &predictor[0], bw, &x->best_mv.as_mv, sf, bw, bh, &conv_params, kernel, + &warp_types, mi_col * MI_SIZE, mi_row * MI_SIZE, 0, 0, MV_PRECISION_Q3, + mi_col * MI_SIZE, mi_row * MI_SIZE, xd, 0); + if (is_cur_buf_hbd(xd)) { + aom_highbd_subtract_block( + bh, bw, src_diff, bw, xd->cur_buf->y_buffer + mb_y_offset, + xd->cur_buf->y_stride, &predictor[0], bw, xd->bd); + } else { + aom_subtract_block(bh, bw, src_diff, bw, + xd->cur_buf->y_buffer + mb_y_offset, + xd->cur_buf->y_stride, &predictor[0], bw); + } + wht_fwd_txfm(src_diff, bw, coeff, tx_size); + + inter_cost = aom_satd(coeff, pix_num); + if (inter_cost < best_inter_cost) { + best_rf_idx = rf_idx; + best_inter_cost = inter_cost; + best_mv.as_int = x->best_mv.as_int; + get_quantize_error(x, 0, coeff, qcoeff, dqcoeff, tx_size, recon_error, + sse); + } + } + best_intra_cost = AOMMAX(best_intra_cost, 1); + best_inter_cost = AOMMIN(best_intra_cost, best_inter_cost); + tpl_stats->inter_cost = best_inter_cost << TPL_DEP_COST_SCALE_LOG2; + tpl_stats->intra_cost = best_intra_cost << TPL_DEP_COST_SCALE_LOG2; + tpl_stats->mc_dep_cost = tpl_stats->intra_cost + tpl_stats->mc_flow; + + tpl_stats->ref_frame_index = gf_picture[frame_idx].ref_frame[best_rf_idx]; + tpl_stats->mv.as_int = best_mv.as_int; +} + +static int round_floor(int ref_pos, int bsize_pix) { + int round; + if (ref_pos < 0) + round = -(1 + (-ref_pos - 1) / bsize_pix); + else + round = ref_pos / bsize_pix; + + return round; +} + +static int get_overlap_area(int grid_pos_row, int grid_pos_col, int ref_pos_row, + int ref_pos_col, int block, BLOCK_SIZE bsize) { + int width = 0, height = 0; + int bw = 4 << mi_size_wide_log2[bsize]; + int bh = 4 << mi_size_high_log2[bsize]; + + switch (block) { + case 0: + width = grid_pos_col + bw - ref_pos_col; + height = grid_pos_row + bh - ref_pos_row; + break; + case 1: + width = ref_pos_col + bw - grid_pos_col; + height = grid_pos_row + bh - ref_pos_row; + break; + case 2: + width = grid_pos_col + bw - ref_pos_col; + height = ref_pos_row + bh - grid_pos_row; + break; + case 3: + width = ref_pos_col + bw - grid_pos_col; + height = ref_pos_row + bh - grid_pos_row; + break; + default: assert(0); + } + + return width * height; +} + +static void tpl_model_update_b(TplDepFrame *tpl_frame, TplDepStats *tpl_stats, + int mi_row, int mi_col, const BLOCK_SIZE bsize) { + TplDepFrame *ref_tpl_frame = &tpl_frame[tpl_stats->ref_frame_index]; + TplDepStats *ref_stats = ref_tpl_frame->tpl_stats_ptr; + MV mv = tpl_stats->mv.as_mv; + int mv_row = mv.row >> 3; + int mv_col = mv.col >> 3; + + int ref_pos_row = mi_row * MI_SIZE + mv_row; + int ref_pos_col = mi_col * MI_SIZE + mv_col; + + const int bw = 4 << mi_size_wide_log2[bsize]; + const int bh = 4 << mi_size_high_log2[bsize]; + const int mi_height = mi_size_high[bsize]; + const int mi_width = mi_size_wide[bsize]; + const int pix_num = bw * bh; + + // top-left on grid block location in pixel + int grid_pos_row_base = round_floor(ref_pos_row, bh) * bh; + int grid_pos_col_base = round_floor(ref_pos_col, bw) * bw; + int block; + + for (block = 0; block < 4; ++block) { + int grid_pos_row = grid_pos_row_base + bh * (block >> 1); + int grid_pos_col = grid_pos_col_base + bw * (block & 0x01); + + if (grid_pos_row >= 0 && grid_pos_row < ref_tpl_frame->mi_rows * MI_SIZE && + grid_pos_col >= 0 && grid_pos_col < ref_tpl_frame->mi_cols * MI_SIZE) { + int overlap_area = get_overlap_area( + grid_pos_row, grid_pos_col, ref_pos_row, ref_pos_col, block, bsize); + int ref_mi_row = round_floor(grid_pos_row, bh) * mi_height; + int ref_mi_col = round_floor(grid_pos_col, bw) * mi_width; + + int64_t mc_flow = tpl_stats->mc_dep_cost - + (tpl_stats->mc_dep_cost * tpl_stats->inter_cost) / + tpl_stats->intra_cost; + + int idx, idy; + + for (idy = 0; idy < mi_height; ++idy) { + for (idx = 0; idx < mi_width; ++idx) { + TplDepStats *des_stats = + &ref_stats[(ref_mi_row + idy) * ref_tpl_frame->stride + + (ref_mi_col + idx)]; + + des_stats->mc_flow += (mc_flow * overlap_area) / pix_num; + des_stats->mc_ref_cost += + ((tpl_stats->intra_cost - tpl_stats->inter_cost) * overlap_area) / + pix_num; + assert(overlap_area >= 0); + } + } + } + } +} + +static void tpl_model_update(TplDepFrame *tpl_frame, TplDepStats *tpl_stats, + int mi_row, int mi_col, const BLOCK_SIZE bsize) { + int idx, idy; + const int mi_height = mi_size_high[bsize]; + const int mi_width = mi_size_wide[bsize]; + + for (idy = 0; idy < mi_height; ++idy) { + for (idx = 0; idx < mi_width; ++idx) { + TplDepStats *tpl_ptr = + &tpl_stats[(mi_row + idy) * tpl_frame->stride + (mi_col + idx)]; + tpl_model_update_b(tpl_frame, tpl_ptr, mi_row + idy, mi_col + idx, + BLOCK_4X4); + } + } +} + +static void tpl_model_store(TplDepStats *tpl_stats, int mi_row, int mi_col, + BLOCK_SIZE bsize, int stride, + const TplDepStats *src_stats) { + const int mi_height = mi_size_high[bsize]; + const int mi_width = mi_size_wide[bsize]; + int idx, idy; + + int64_t intra_cost = src_stats->intra_cost / (mi_height * mi_width); + int64_t inter_cost = src_stats->inter_cost / (mi_height * mi_width); + + TplDepStats *tpl_ptr; + + intra_cost = AOMMAX(1, intra_cost); + inter_cost = AOMMAX(1, inter_cost); + + for (idy = 0; idy < mi_height; ++idy) { + tpl_ptr = &tpl_stats[(mi_row + idy) * stride + mi_col]; + for (idx = 0; idx < mi_width; ++idx) { + tpl_ptr->intra_cost = intra_cost; + tpl_ptr->inter_cost = inter_cost; + tpl_ptr->mc_dep_cost = tpl_ptr->intra_cost + tpl_ptr->mc_flow; + tpl_ptr->ref_frame_index = src_stats->ref_frame_index; + tpl_ptr->mv.as_int = src_stats->mv.as_int; + ++tpl_ptr; + } + } +} + +static void mc_flow_dispenser(AV1_COMP *cpi, GF_PICTURE *gf_picture, + int frame_idx) { + TplDepFrame *tpl_frame = &cpi->tpl_stats[frame_idx]; + YV12_BUFFER_CONFIG *this_frame = gf_picture[frame_idx].frame; + YV12_BUFFER_CONFIG *ref_frame[7] = { + NULL, NULL, NULL, NULL, NULL, NULL, NULL + }; + + AV1_COMMON *cm = &cpi->common; + struct scale_factors sf; + int rdmult, idx; + ThreadData *td = &cpi->td; + MACROBLOCK *x = &td->mb; + MACROBLOCKD *xd = &x->e_mbd; + int mi_row, mi_col; + + DECLARE_ALIGNED(16, uint16_t, predictor16[32 * 32 * 3]); + DECLARE_ALIGNED(16, uint8_t, predictor8[32 * 32 * 3]); + uint8_t *predictor; + DECLARE_ALIGNED(16, int16_t, src_diff[32 * 32]); + DECLARE_ALIGNED(16, tran_low_t, coeff[32 * 32]); + DECLARE_ALIGNED(16, tran_low_t, qcoeff[32 * 32]); + DECLARE_ALIGNED(16, tran_low_t, dqcoeff[32 * 32]); + + const BLOCK_SIZE bsize = BLOCK_32X32; + const TX_SIZE tx_size = max_txsize_lookup[bsize]; + const int mi_height = mi_size_high[bsize]; + const int mi_width = mi_size_wide[bsize]; + int64_t recon_error, sse; + + // Setup scaling factor + av1_setup_scale_factors_for_frame( + &sf, this_frame->y_crop_width, this_frame->y_crop_height, + this_frame->y_crop_width, this_frame->y_crop_height); + + if (is_cur_buf_hbd(xd)) + predictor = CONVERT_TO_BYTEPTR(predictor16); + else + predictor = predictor8; + + // Prepare reference frame pointers. If any reference frame slot is + // unavailable, the pointer will be set to Null. + for (idx = 0; idx < 7; ++idx) { + int rf_idx = gf_picture[frame_idx].ref_frame[idx]; + if (rf_idx != -1) ref_frame[idx] = gf_picture[rf_idx].frame; + } + + xd->mi = cm->mi_grid_visible; + xd->mi[0] = cm->mi; + xd->cur_buf = this_frame; + + // Get rd multiplier set up. + rdmult = (int)av1_compute_rd_mult(cpi, tpl_frame->base_qindex); + if (rdmult < 1) rdmult = 1; + set_error_per_bit(&cpi->td.mb, rdmult); + av1_initialize_me_consts(cpi, &cpi->td.mb, tpl_frame->base_qindex); + + tpl_frame->is_valid = 1; + + cm->base_qindex = tpl_frame->base_qindex; + av1_frame_init_quantizer(cpi); + + for (mi_row = 0; mi_row < cm->mi_rows; mi_row += mi_height) { + // Motion estimation row boundary + x->mv_limits.row_min = -((mi_row * MI_SIZE) + (17 - 2 * AOM_INTERP_EXTEND)); + x->mv_limits.row_max = + (cm->mi_rows - 1 - mi_row) * MI_SIZE + (17 - 2 * AOM_INTERP_EXTEND); + for (mi_col = 0; mi_col < cm->mi_cols; mi_col += mi_width) { + TplDepStats tpl_stats; + mode_estimation(cpi, x, xd, &sf, gf_picture, frame_idx, src_diff, coeff, + qcoeff, dqcoeff, mi_row, mi_col, bsize, tx_size, + ref_frame, predictor, &recon_error, &sse, &tpl_stats); + + // Motion flow dependency dispenser. + tpl_model_store(tpl_frame->tpl_stats_ptr, mi_row, mi_col, bsize, + tpl_frame->stride, &tpl_stats); + + tpl_model_update(cpi->tpl_stats, tpl_frame->tpl_stats_ptr, mi_row, mi_col, + bsize); + } + } +} + +static void init_gop_frames(AV1_COMP *cpi, GF_PICTURE *gf_picture, + const GF_GROUP *gf_group, int *tpl_group_frames, + const EncodeFrameInput *const frame_input) { + AV1_COMMON *cm = &cpi->common; + const SequenceHeader *const seq_params = &cm->seq_params; + int frame_idx = 0; + int i; + int gld_index = -1; + int alt_index = -1; + int lst_index = -1; + int extend_frame_count = 0; + int pframe_qindex = cpi->tpl_stats[2].base_qindex; + + RefCntBuffer *frame_bufs = cm->buffer_pool->frame_bufs; + int recon_frame_index[INTER_REFS_PER_FRAME + 1] = { -1, -1, -1, -1, + -1, -1, -1, -1 }; + + // TODO(jingning): To be used later for gf frame type parsing. + (void)gf_group; + + for (i = 0; i < FRAME_BUFFERS && frame_idx < INTER_REFS_PER_FRAME + 1; ++i) { + if (frame_bufs[i].ref_count == 0) { + alloc_frame_mvs(cm, &frame_bufs[i]); + if (aom_realloc_frame_buffer( + &frame_bufs[i].buf, cm->width, cm->height, + seq_params->subsampling_x, seq_params->subsampling_y, + seq_params->use_highbitdepth, cpi->oxcf.border_in_pixels, + cm->byte_alignment, NULL, NULL, NULL)) + aom_internal_error(&cm->error, AOM_CODEC_MEM_ERROR, + "Failed to allocate frame buffer"); + + recon_frame_index[frame_idx] = i; + ++frame_idx; + } + } + + for (i = 0; i < INTER_REFS_PER_FRAME + 1; ++i) { + assert(recon_frame_index[i] >= 0); + cpi->tpl_recon_frames[i] = &frame_bufs[recon_frame_index[i]].buf; + } + + *tpl_group_frames = 0; + + // Initialize Golden reference frame. + gf_picture[0].frame = NULL; + RefCntBuffer *ref_buf = get_ref_frame_buf(cm, GOLDEN_FRAME); + if (ref_buf) gf_picture[0].frame = &ref_buf->buf; + for (i = 0; i < 7; ++i) gf_picture[0].ref_frame[i] = -1; + gld_index = 0; + ++*tpl_group_frames; + + // Initialize ARF frame + gf_picture[1].frame = frame_input->source; + gf_picture[1].ref_frame[0] = gld_index; + gf_picture[1].ref_frame[1] = lst_index; + gf_picture[1].ref_frame[2] = alt_index; + // TODO(yuec) Need o figure out full AV1 reference model + for (i = 3; i < 7; ++i) gf_picture[1].ref_frame[i] = -1; + alt_index = 1; + ++*tpl_group_frames; + + // Initialize P frames + for (frame_idx = 2; frame_idx < MAX_LAG_BUFFERS; ++frame_idx) { + struct lookahead_entry *buf = + av1_lookahead_peek(cpi->lookahead, frame_idx - 2); + + if (buf == NULL) break; + + gf_picture[frame_idx].frame = &buf->img; + gf_picture[frame_idx].ref_frame[0] = gld_index; + gf_picture[frame_idx].ref_frame[1] = lst_index; + gf_picture[frame_idx].ref_frame[2] = alt_index; + for (i = 3; i < 7; ++i) gf_picture[frame_idx].ref_frame[i] = -1; + + ++*tpl_group_frames; + lst_index = frame_idx; + + if (frame_idx == cpi->rc.baseline_gf_interval + 1) break; + } + + gld_index = frame_idx; + lst_index = AOMMAX(0, frame_idx - 1); + alt_index = -1; + ++frame_idx; + + // Extend two frames outside the current gf group. + for (; frame_idx < MAX_LAG_BUFFERS && extend_frame_count < 2; ++frame_idx) { + struct lookahead_entry *buf = + av1_lookahead_peek(cpi->lookahead, frame_idx - 2); + + if (buf == NULL) break; + + cpi->tpl_stats[frame_idx].base_qindex = pframe_qindex; + + gf_picture[frame_idx].frame = &buf->img; + gf_picture[frame_idx].ref_frame[0] = gld_index; + gf_picture[frame_idx].ref_frame[1] = lst_index; + gf_picture[frame_idx].ref_frame[2] = alt_index; + for (i = 3; i < 7; ++i) gf_picture[frame_idx].ref_frame[i] = -1; + lst_index = frame_idx; + ++*tpl_group_frames; + ++extend_frame_count; + } +} + +static void init_tpl_stats(AV1_COMP *cpi) { + int frame_idx; + for (frame_idx = 0; frame_idx < MAX_LAG_BUFFERS; ++frame_idx) { + TplDepFrame *tpl_frame = &cpi->tpl_stats[frame_idx]; + memset(tpl_frame->tpl_stats_ptr, 0, + tpl_frame->height * tpl_frame->width * + sizeof(*tpl_frame->tpl_stats_ptr)); + tpl_frame->is_valid = 0; + } +} + +void av1_tpl_setup_stats(AV1_COMP *cpi, + const EncodeFrameInput *const frame_input) { + GF_PICTURE gf_picture[MAX_LAG_BUFFERS]; + const GF_GROUP *gf_group = &cpi->twopass.gf_group; + int tpl_group_frames = 0; + int frame_idx; + + init_gop_frames(cpi, gf_picture, gf_group, &tpl_group_frames, frame_input); + + init_tpl_stats(cpi); + + // Backward propagation from tpl_group_frames to 1. + for (frame_idx = tpl_group_frames - 1; frame_idx > 0; --frame_idx) + mc_flow_dispenser(cpi, gf_picture, frame_idx); +} diff --git a/third_party/aom/av1/encoder/tpl_model.h b/third_party/aom/av1/encoder/tpl_model.h new file mode 100644 index 0000000000..f6b33b07c3 --- /dev/null +++ b/third_party/aom/av1/encoder/tpl_model.h @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2019, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_AV1_ENCODER_TPL_MODEL_H_ +#define AOM_AV1_ENCODER_TPL_MODEL_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +void av1_tpl_setup_stats(AV1_COMP *cpi, + const EncodeFrameInput *const frame_input); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AOM_AV1_ENCODER_TPL_MODEL_H_ diff --git a/third_party/aom/av1/encoder/x86/av1_fwd_txfm1d_sse4.c b/third_party/aom/av1/encoder/x86/av1_fwd_txfm1d_sse4.c index 07615543c6..865ac31494 100644 --- a/third_party/aom/av1/encoder/x86/av1_fwd_txfm1d_sse4.c +++ b/third_party/aom/av1/encoder/x86/av1_fwd_txfm1d_sse4.c @@ -11,45 +11,78 @@ #include "av1/encoder/x86/av1_txfm1d_sse4.h" -void av1_fdct32_new_sse4_1(const __m128i *input, __m128i *output, - int8_t cos_bit) { +void av1_fdct32_new_sse4_1(__m128i *input, __m128i *output, int cos_bit, + const int stride) { __m128i buf0[32]; __m128i buf1[32]; const int32_t *cospi; + + int startidx = 0 * stride; + int endidx = 31 * stride; // stage 0 // stage 1 - buf1[0] = _mm_add_epi32(input[0], input[31]); - buf1[31] = _mm_sub_epi32(input[0], input[31]); - buf1[1] = _mm_add_epi32(input[1], input[30]); - buf1[30] = _mm_sub_epi32(input[1], input[30]); - buf1[2] = _mm_add_epi32(input[2], input[29]); - buf1[29] = _mm_sub_epi32(input[2], input[29]); - buf1[3] = _mm_add_epi32(input[3], input[28]); - buf1[28] = _mm_sub_epi32(input[3], input[28]); - buf1[4] = _mm_add_epi32(input[4], input[27]); - buf1[27] = _mm_sub_epi32(input[4], input[27]); - buf1[5] = _mm_add_epi32(input[5], input[26]); - buf1[26] = _mm_sub_epi32(input[5], input[26]); - buf1[6] = _mm_add_epi32(input[6], input[25]); - buf1[25] = _mm_sub_epi32(input[6], input[25]); - buf1[7] = _mm_add_epi32(input[7], input[24]); - buf1[24] = _mm_sub_epi32(input[7], input[24]); - buf1[8] = _mm_add_epi32(input[8], input[23]); - buf1[23] = _mm_sub_epi32(input[8], input[23]); - buf1[9] = _mm_add_epi32(input[9], input[22]); - buf1[22] = _mm_sub_epi32(input[9], input[22]); - buf1[10] = _mm_add_epi32(input[10], input[21]); - buf1[21] = _mm_sub_epi32(input[10], input[21]); - buf1[11] = _mm_add_epi32(input[11], input[20]); - buf1[20] = _mm_sub_epi32(input[11], input[20]); - buf1[12] = _mm_add_epi32(input[12], input[19]); - buf1[19] = _mm_sub_epi32(input[12], input[19]); - buf1[13] = _mm_add_epi32(input[13], input[18]); - buf1[18] = _mm_sub_epi32(input[13], input[18]); - buf1[14] = _mm_add_epi32(input[14], input[17]); - buf1[17] = _mm_sub_epi32(input[14], input[17]); - buf1[15] = _mm_add_epi32(input[15], input[16]); - buf1[16] = _mm_sub_epi32(input[15], input[16]); + buf1[0] = _mm_add_epi32(input[startidx], input[endidx]); + buf1[31] = _mm_sub_epi32(input[startidx], input[endidx]); + startidx += stride; + endidx -= stride; + buf1[1] = _mm_add_epi32(input[startidx], input[endidx]); + buf1[30] = _mm_sub_epi32(input[startidx], input[endidx]); + startidx += stride; + endidx -= stride; + buf1[2] = _mm_add_epi32(input[startidx], input[endidx]); + buf1[29] = _mm_sub_epi32(input[startidx], input[endidx]); + startidx += stride; + endidx -= stride; + buf1[3] = _mm_add_epi32(input[startidx], input[endidx]); + buf1[28] = _mm_sub_epi32(input[startidx], input[endidx]); + startidx += stride; + endidx -= stride; + buf1[4] = _mm_add_epi32(input[startidx], input[endidx]); + buf1[27] = _mm_sub_epi32(input[startidx], input[endidx]); + startidx += stride; + endidx -= stride; + buf1[5] = _mm_add_epi32(input[startidx], input[endidx]); + buf1[26] = _mm_sub_epi32(input[startidx], input[endidx]); + startidx += stride; + endidx -= stride; + buf1[6] = _mm_add_epi32(input[startidx], input[endidx]); + buf1[25] = _mm_sub_epi32(input[startidx], input[endidx]); + startidx += stride; + endidx -= stride; + buf1[7] = _mm_add_epi32(input[startidx], input[endidx]); + buf1[24] = _mm_sub_epi32(input[startidx], input[endidx]); + startidx += stride; + endidx -= stride; + buf1[8] = _mm_add_epi32(input[startidx], input[endidx]); + buf1[23] = _mm_sub_epi32(input[startidx], input[endidx]); + startidx += stride; + endidx -= stride; + buf1[9] = _mm_add_epi32(input[startidx], input[endidx]); + buf1[22] = _mm_sub_epi32(input[startidx], input[endidx]); + startidx += stride; + endidx -= stride; + buf1[10] = _mm_add_epi32(input[startidx], input[endidx]); + buf1[21] = _mm_sub_epi32(input[startidx], input[endidx]); + startidx += stride; + endidx -= stride; + buf1[11] = _mm_add_epi32(input[startidx], input[endidx]); + buf1[20] = _mm_sub_epi32(input[startidx], input[endidx]); + startidx += stride; + endidx -= stride; + buf1[12] = _mm_add_epi32(input[startidx], input[endidx]); + buf1[19] = _mm_sub_epi32(input[startidx], input[endidx]); + startidx += stride; + endidx -= stride; + buf1[13] = _mm_add_epi32(input[startidx], input[endidx]); + buf1[18] = _mm_sub_epi32(input[startidx], input[endidx]); + startidx += stride; + endidx -= stride; + buf1[14] = _mm_add_epi32(input[startidx], input[endidx]); + buf1[17] = _mm_sub_epi32(input[startidx], input[endidx]); + startidx += stride; + endidx -= stride; + buf1[15] = _mm_add_epi32(input[startidx], input[endidx]); + buf1[16] = _mm_sub_epi32(input[startidx], input[endidx]); // stage 2 cospi = cospi_arr(cos_bit); @@ -296,39 +329,71 @@ void av1_fdct32_new_sse4_1(const __m128i *input, __m128i *output, btf_32_sse4_1_type1(cospi[6], cospi[58], buf1[23], buf1[24], buf0[23], buf0[24], cos_bit); + startidx = 0 * stride; + endidx = 31 * stride; // stage 9 - output[0] = buf0[0]; - output[1] = buf0[16]; - output[2] = buf0[8]; - output[3] = buf0[24]; - output[4] = buf0[4]; - output[5] = buf0[20]; - output[6] = buf0[12]; - output[7] = buf0[28]; - output[8] = buf0[2]; - output[9] = buf0[18]; - output[10] = buf0[10]; - output[11] = buf0[26]; - output[12] = buf0[6]; - output[13] = buf0[22]; - output[14] = buf0[14]; - output[15] = buf0[30]; - output[16] = buf0[1]; - output[17] = buf0[17]; - output[18] = buf0[9]; - output[19] = buf0[25]; - output[20] = buf0[5]; - output[21] = buf0[21]; - output[22] = buf0[13]; - output[23] = buf0[29]; - output[24] = buf0[3]; - output[25] = buf0[19]; - output[26] = buf0[11]; - output[27] = buf0[27]; - output[28] = buf0[7]; - output[29] = buf0[23]; - output[30] = buf0[15]; - output[31] = buf0[31]; + output[startidx] = buf0[0]; + output[endidx] = buf0[31]; + startidx += stride; + endidx -= stride; + output[startidx] = buf0[16]; + output[endidx] = buf0[15]; + startidx += stride; + endidx -= stride; + output[startidx] = buf0[8]; + output[endidx] = buf0[23]; + startidx += stride; + endidx -= stride; + output[startidx] = buf0[24]; + output[endidx] = buf0[7]; + startidx += stride; + endidx -= stride; + output[startidx] = buf0[4]; + output[endidx] = buf0[27]; + startidx += stride; + endidx -= stride; + output[startidx] = buf0[20]; + output[endidx] = buf0[11]; + startidx += stride; + endidx -= stride; + output[startidx] = buf0[12]; + output[endidx] = buf0[19]; + startidx += stride; + endidx -= stride; + output[startidx] = buf0[28]; + output[endidx] = buf0[3]; + startidx += stride; + endidx -= stride; + output[startidx] = buf0[2]; + output[endidx] = buf0[29]; + startidx += stride; + endidx -= stride; + output[startidx] = buf0[18]; + output[endidx] = buf0[13]; + startidx += stride; + endidx -= stride; + output[startidx] = buf0[10]; + output[endidx] = buf0[21]; + startidx += stride; + endidx -= stride; + output[startidx] = buf0[26]; + output[endidx] = buf0[5]; + startidx += stride; + endidx -= stride; + output[startidx] = buf0[6]; + output[endidx] = buf0[25]; + startidx += stride; + endidx -= stride; + output[startidx] = buf0[22]; + output[endidx] = buf0[9]; + startidx += stride; + endidx -= stride; + output[startidx] = buf0[14]; + output[endidx] = buf0[17]; + startidx += stride; + endidx -= stride; + output[startidx] = buf0[30]; + output[endidx] = buf0[1]; } void av1_fadst4_new_sse4_1(const __m128i *input, __m128i *output, @@ -394,9 +459,8 @@ void av1_fadst4_new_sse4_1(const __m128i *input, __m128i *output, } } -void av1_fdct64_new_sse4_1(const __m128i *input, __m128i *output, - int8_t cos_bit, const int instride, - const int outstride) { +void av1_fdct64_new_sse4_1(__m128i *input, __m128i *output, int8_t cos_bit, + const int instride, const int outstride) { const int32_t *cospi = cospi_arr(cos_bit); const __m128i __rounding = _mm_set1_epi32(1 << (cos_bit - 1)); @@ -479,72 +543,136 @@ void av1_fdct64_new_sse4_1(const __m128i *input, __m128i *output, __m128i cospi_p03 = _mm_set1_epi32(cospi[3]); __m128i cospi_p61 = _mm_set1_epi32(cospi[61]); + int startidx = 0 * instride; + int endidx = 63 * instride; // stage 1 __m128i x1[64]; - x1[0] = _mm_add_epi32(input[0 * instride], input[63 * instride]); - x1[63] = _mm_sub_epi32(input[0 * instride], input[63 * instride]); - x1[1] = _mm_add_epi32(input[1 * instride], input[62 * instride]); - x1[62] = _mm_sub_epi32(input[1 * instride], input[62 * instride]); - x1[2] = _mm_add_epi32(input[2 * instride], input[61 * instride]); - x1[61] = _mm_sub_epi32(input[2 * instride], input[61 * instride]); - x1[3] = _mm_add_epi32(input[3 * instride], input[60 * instride]); - x1[60] = _mm_sub_epi32(input[3 * instride], input[60 * instride]); - x1[4] = _mm_add_epi32(input[4 * instride], input[59 * instride]); - x1[59] = _mm_sub_epi32(input[4 * instride], input[59 * instride]); - x1[5] = _mm_add_epi32(input[5 * instride], input[58 * instride]); - x1[58] = _mm_sub_epi32(input[5 * instride], input[58 * instride]); - x1[6] = _mm_add_epi32(input[6 * instride], input[57 * instride]); - x1[57] = _mm_sub_epi32(input[6 * instride], input[57 * instride]); - x1[7] = _mm_add_epi32(input[7 * instride], input[56 * instride]); - x1[56] = _mm_sub_epi32(input[7 * instride], input[56 * instride]); - x1[8] = _mm_add_epi32(input[8 * instride], input[55 * instride]); - x1[55] = _mm_sub_epi32(input[8 * instride], input[55 * instride]); - x1[9] = _mm_add_epi32(input[9 * instride], input[54 * instride]); - x1[54] = _mm_sub_epi32(input[9 * instride], input[54 * instride]); - x1[10] = _mm_add_epi32(input[10 * instride], input[53 * instride]); - x1[53] = _mm_sub_epi32(input[10 * instride], input[53 * instride]); - x1[11] = _mm_add_epi32(input[11 * instride], input[52 * instride]); - x1[52] = _mm_sub_epi32(input[11 * instride], input[52 * instride]); - x1[12] = _mm_add_epi32(input[12 * instride], input[51 * instride]); - x1[51] = _mm_sub_epi32(input[12 * instride], input[51 * instride]); - x1[13] = _mm_add_epi32(input[13 * instride], input[50 * instride]); - x1[50] = _mm_sub_epi32(input[13 * instride], input[50 * instride]); - x1[14] = _mm_add_epi32(input[14 * instride], input[49 * instride]); - x1[49] = _mm_sub_epi32(input[14 * instride], input[49 * instride]); - x1[15] = _mm_add_epi32(input[15 * instride], input[48 * instride]); - x1[48] = _mm_sub_epi32(input[15 * instride], input[48 * instride]); - x1[16] = _mm_add_epi32(input[16 * instride], input[47 * instride]); - x1[47] = _mm_sub_epi32(input[16 * instride], input[47 * instride]); - x1[17] = _mm_add_epi32(input[17 * instride], input[46 * instride]); - x1[46] = _mm_sub_epi32(input[17 * instride], input[46 * instride]); - x1[18] = _mm_add_epi32(input[18 * instride], input[45 * instride]); - x1[45] = _mm_sub_epi32(input[18 * instride], input[45 * instride]); - x1[19] = _mm_add_epi32(input[19 * instride], input[44 * instride]); - x1[44] = _mm_sub_epi32(input[19 * instride], input[44 * instride]); - x1[20] = _mm_add_epi32(input[20 * instride], input[43 * instride]); - x1[43] = _mm_sub_epi32(input[20 * instride], input[43 * instride]); - x1[21] = _mm_add_epi32(input[21 * instride], input[42 * instride]); - x1[42] = _mm_sub_epi32(input[21 * instride], input[42 * instride]); - x1[22] = _mm_add_epi32(input[22 * instride], input[41 * instride]); - x1[41] = _mm_sub_epi32(input[22 * instride], input[41 * instride]); - x1[23] = _mm_add_epi32(input[23 * instride], input[40 * instride]); - x1[40] = _mm_sub_epi32(input[23 * instride], input[40 * instride]); - x1[24] = _mm_add_epi32(input[24 * instride], input[39 * instride]); - x1[39] = _mm_sub_epi32(input[24 * instride], input[39 * instride]); - x1[25] = _mm_add_epi32(input[25 * instride], input[38 * instride]); - x1[38] = _mm_sub_epi32(input[25 * instride], input[38 * instride]); - x1[26] = _mm_add_epi32(input[26 * instride], input[37 * instride]); - x1[37] = _mm_sub_epi32(input[26 * instride], input[37 * instride]); - x1[27] = _mm_add_epi32(input[27 * instride], input[36 * instride]); - x1[36] = _mm_sub_epi32(input[27 * instride], input[36 * instride]); - x1[28] = _mm_add_epi32(input[28 * instride], input[35 * instride]); - x1[35] = _mm_sub_epi32(input[28 * instride], input[35 * instride]); - x1[29] = _mm_add_epi32(input[29 * instride], input[34 * instride]); - x1[34] = _mm_sub_epi32(input[29 * instride], input[34 * instride]); - x1[30] = _mm_add_epi32(input[30 * instride], input[33 * instride]); - x1[33] = _mm_sub_epi32(input[30 * instride], input[33 * instride]); - x1[31] = _mm_add_epi32(input[31 * instride], input[32 * instride]); - x1[32] = _mm_sub_epi32(input[31 * instride], input[32 * instride]); + x1[0] = _mm_add_epi32(input[startidx], input[endidx]); + x1[63] = _mm_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + x1[1] = _mm_add_epi32(input[startidx], input[endidx]); + x1[62] = _mm_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + x1[2] = _mm_add_epi32(input[startidx], input[endidx]); + x1[61] = _mm_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + x1[3] = _mm_add_epi32(input[startidx], input[endidx]); + x1[60] = _mm_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + x1[4] = _mm_add_epi32(input[startidx], input[endidx]); + x1[59] = _mm_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + x1[5] = _mm_add_epi32(input[startidx], input[endidx]); + x1[58] = _mm_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + x1[6] = _mm_add_epi32(input[startidx], input[endidx]); + x1[57] = _mm_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + x1[7] = _mm_add_epi32(input[startidx], input[endidx]); + x1[56] = _mm_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + x1[8] = _mm_add_epi32(input[startidx], input[endidx]); + x1[55] = _mm_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + x1[9] = _mm_add_epi32(input[startidx], input[endidx]); + x1[54] = _mm_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + x1[10] = _mm_add_epi32(input[startidx], input[endidx]); + x1[53] = _mm_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + x1[11] = _mm_add_epi32(input[startidx], input[endidx]); + x1[52] = _mm_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + x1[12] = _mm_add_epi32(input[startidx], input[endidx]); + x1[51] = _mm_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + x1[13] = _mm_add_epi32(input[startidx], input[endidx]); + x1[50] = _mm_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + x1[14] = _mm_add_epi32(input[startidx], input[endidx]); + x1[49] = _mm_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + x1[15] = _mm_add_epi32(input[startidx], input[endidx]); + x1[48] = _mm_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + x1[16] = _mm_add_epi32(input[startidx], input[endidx]); + x1[47] = _mm_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + x1[17] = _mm_add_epi32(input[startidx], input[endidx]); + x1[46] = _mm_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + x1[18] = _mm_add_epi32(input[startidx], input[endidx]); + x1[45] = _mm_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + x1[19] = _mm_add_epi32(input[startidx], input[endidx]); + x1[44] = _mm_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + x1[20] = _mm_add_epi32(input[startidx], input[endidx]); + x1[43] = _mm_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + x1[21] = _mm_add_epi32(input[startidx], input[endidx]); + x1[42] = _mm_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + x1[22] = _mm_add_epi32(input[startidx], input[endidx]); + x1[41] = _mm_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + x1[23] = _mm_add_epi32(input[startidx], input[endidx]); + x1[40] = _mm_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + x1[24] = _mm_add_epi32(input[startidx], input[endidx]); + x1[39] = _mm_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + x1[25] = _mm_add_epi32(input[startidx], input[endidx]); + x1[38] = _mm_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + x1[26] = _mm_add_epi32(input[startidx], input[endidx]); + x1[37] = _mm_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + x1[27] = _mm_add_epi32(input[startidx], input[endidx]); + x1[36] = _mm_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + x1[28] = _mm_add_epi32(input[startidx], input[endidx]); + x1[35] = _mm_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + x1[29] = _mm_add_epi32(input[startidx], input[endidx]); + x1[34] = _mm_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + x1[30] = _mm_add_epi32(input[startidx], input[endidx]); + x1[33] = _mm_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + x1[31] = _mm_add_epi32(input[startidx], input[endidx]); + x1[32] = _mm_sub_epi32(input[startidx], input[endidx]); // stage 2 __m128i x2[64]; @@ -1149,69 +1277,141 @@ void av1_fdct64_new_sse4_1(const __m128i *input, __m128i *output, btf_32_type1_sse4_1_new(cospi_p03, cospi_p61, x9[47], x9[48], x10[47], x10[48], __rounding, cos_bit); + startidx = 0 * outstride; + endidx = 63 * outstride; // stage 11 - output[0 * outstride] = x10[0]; - output[1 * outstride] = x10[32]; - output[2 * outstride] = x10[16]; - output[3 * outstride] = x10[48]; - output[4 * outstride] = x10[8]; - output[5 * outstride] = x10[40]; - output[6 * outstride] = x10[24]; - output[7 * outstride] = x10[56]; - output[8 * outstride] = x10[4]; - output[9 * outstride] = x10[36]; - output[10 * outstride] = x10[20]; - output[11 * outstride] = x10[52]; - output[12 * outstride] = x10[12]; - output[13 * outstride] = x10[44]; - output[14 * outstride] = x10[28]; - output[15 * outstride] = x10[60]; - output[16 * outstride] = x10[2]; - output[17 * outstride] = x10[34]; - output[18 * outstride] = x10[18]; - output[19 * outstride] = x10[50]; - output[20 * outstride] = x10[10]; - output[21 * outstride] = x10[42]; - output[22 * outstride] = x10[26]; - output[23 * outstride] = x10[58]; - output[24 * outstride] = x10[6]; - output[25 * outstride] = x10[38]; - output[26 * outstride] = x10[22]; - output[27 * outstride] = x10[54]; - output[28 * outstride] = x10[14]; - output[29 * outstride] = x10[46]; - output[30 * outstride] = x10[30]; - output[31 * outstride] = x10[62]; - output[32 * outstride] = x10[1]; - output[33 * outstride] = x10[33]; - output[34 * outstride] = x10[17]; - output[35 * outstride] = x10[49]; - output[36 * outstride] = x10[9]; - output[37 * outstride] = x10[41]; - output[38 * outstride] = x10[25]; - output[39 * outstride] = x10[57]; - output[40 * outstride] = x10[5]; - output[41 * outstride] = x10[37]; - output[42 * outstride] = x10[21]; - output[43 * outstride] = x10[53]; - output[44 * outstride] = x10[13]; - output[45 * outstride] = x10[45]; - output[46 * outstride] = x10[29]; - output[47 * outstride] = x10[61]; - output[48 * outstride] = x10[3]; - output[49 * outstride] = x10[35]; - output[50 * outstride] = x10[19]; - output[51 * outstride] = x10[51]; - output[52 * outstride] = x10[11]; - output[53 * outstride] = x10[43]; - output[54 * outstride] = x10[27]; - output[55 * outstride] = x10[59]; - output[56 * outstride] = x10[7]; - output[57 * outstride] = x10[39]; - output[58 * outstride] = x10[23]; - output[59 * outstride] = x10[55]; - output[60 * outstride] = x10[15]; - output[61 * outstride] = x10[47]; - output[62 * outstride] = x10[31]; - output[63 * outstride] = x10[63]; + output[startidx] = x10[0]; + output[endidx] = x10[63]; + startidx += outstride; + endidx -= outstride; + output[startidx] = x10[32]; + output[endidx] = x10[31]; + startidx += outstride; + endidx -= outstride; + output[startidx] = x10[16]; + output[endidx] = x10[47]; + startidx += outstride; + endidx -= outstride; + output[startidx] = x10[48]; + output[endidx] = x10[15]; + startidx += outstride; + endidx -= outstride; + output[startidx] = x10[8]; + output[endidx] = x10[55]; + startidx += outstride; + endidx -= outstride; + output[startidx] = x10[40]; + output[endidx] = x10[23]; + startidx += outstride; + endidx -= outstride; + output[startidx] = x10[24]; + output[endidx] = x10[39]; + startidx += outstride; + endidx -= outstride; + output[startidx] = x10[56]; + output[endidx] = x10[7]; + startidx += outstride; + endidx -= outstride; + output[startidx] = x10[4]; + output[endidx] = x10[59]; + startidx += outstride; + endidx -= outstride; + output[startidx] = x10[36]; + output[endidx] = x10[27]; + startidx += outstride; + endidx -= outstride; + output[startidx] = x10[20]; + output[endidx] = x10[43]; + startidx += outstride; + endidx -= outstride; + output[startidx] = x10[52]; + output[endidx] = x10[11]; + startidx += outstride; + endidx -= outstride; + output[startidx] = x10[12]; + output[endidx] = x10[51]; + startidx += outstride; + endidx -= outstride; + output[startidx] = x10[44]; + output[endidx] = x10[19]; + startidx += outstride; + endidx -= outstride; + output[startidx] = x10[28]; + output[endidx] = x10[35]; + startidx += outstride; + endidx -= outstride; + output[startidx] = x10[60]; + output[endidx] = x10[3]; + startidx += outstride; + endidx -= outstride; + output[startidx] = x10[2]; + output[endidx] = x10[61]; + startidx += outstride; + endidx -= outstride; + output[startidx] = x10[34]; + output[endidx] = x10[29]; + startidx += outstride; + endidx -= outstride; + output[startidx] = x10[18]; + output[endidx] = x10[45]; + startidx += outstride; + endidx -= outstride; + output[startidx] = x10[50]; + output[endidx] = x10[13]; + startidx += outstride; + endidx -= outstride; + output[startidx] = x10[10]; + output[endidx] = x10[53]; + startidx += outstride; + endidx -= outstride; + output[startidx] = x10[42]; + output[endidx] = x10[21]; + startidx += outstride; + endidx -= outstride; + output[startidx] = x10[26]; + output[endidx] = x10[37]; + startidx += outstride; + endidx -= outstride; + output[startidx] = x10[58]; + output[endidx] = x10[5]; + startidx += outstride; + endidx -= outstride; + output[startidx] = x10[6]; + output[endidx] = x10[57]; + startidx += outstride; + endidx -= outstride; + output[startidx] = x10[38]; + output[endidx] = x10[25]; + startidx += outstride; + endidx -= outstride; + output[startidx] = x10[22]; + output[endidx] = x10[41]; + startidx += outstride; + endidx -= outstride; + output[startidx] = x10[54]; + output[endidx] = x10[9]; + startidx += outstride; + endidx -= outstride; + output[startidx] = x10[14]; + output[endidx] = x10[49]; + startidx += outstride; + endidx -= outstride; + output[startidx] = x10[46]; + output[endidx] = x10[17]; + startidx += outstride; + endidx -= outstride; + output[startidx] = x10[30]; + output[endidx] = x10[33]; + startidx += outstride; + endidx -= outstride; + output[startidx] = x10[62]; + output[endidx] = x10[1]; +} + +void av1_idtx32_new_sse4_1(__m128i *input, __m128i *output, int cos_bit, + const int col_num) { + (void)cos_bit; + for (int i = 0; i < 32; i++) { + output[i * col_num] = _mm_slli_epi32(input[i * col_num], 2); + } } diff --git a/third_party/aom/av1/encoder/x86/av1_fwd_txfm2d_avx2.c b/third_party/aom/av1/encoder/x86/av1_fwd_txfm2d_avx2.c index 592462e20d..13982ccbb9 100644 --- a/third_party/aom/av1/encoder/x86/av1_fwd_txfm2d_avx2.c +++ b/third_party/aom/av1/encoder/x86/av1_fwd_txfm2d_avx2.c @@ -1436,45 +1436,6 @@ static INLINE void fidentity16x32_new_avx2(const __m256i *input, } } -static INLINE void av1_round_shift_array_32_avx2(__m256i *input, - __m256i *output, - const int size, - const int bit) { - if (bit > 0) { - int i; - for (i = 0; i < size; i++) { - output[i] = av1_round_shift_32_avx2(input[i], bit); - } - } else { - int i; - for (i = 0; i < size; i++) { - output[i] = _mm256_slli_epi32(input[i], -bit); - } - } -} - -static INLINE void av1_round_shift_rect_array_32_avx2(__m256i *input, - __m256i *output, - const int size, - const int bit) { - const __m256i sqrt2 = _mm256_set1_epi32(NewSqrt2); - if (bit > 0) { - int i; - for (i = 0; i < size; i++) { - const __m256i r0 = av1_round_shift_32_avx2(input[i], bit); - const __m256i r1 = _mm256_mullo_epi32(sqrt2, r0); - output[i] = av1_round_shift_32_avx2(r1, NewSqrt2Bits); - } - } else { - int i; - for (i = 0; i < size; i++) { - const __m256i r0 = _mm256_slli_epi32(input[i], -bit); - const __m256i r1 = _mm256_mullo_epi32(sqrt2, r0); - output[i] = av1_round_shift_32_avx2(r1, NewSqrt2Bits); - } - } -} - static INLINE void transpose_32_8x8_avx2(int stride, const __m256i *inputA, __m256i *output) { __m256i temp0 = _mm256_unpacklo_epi32(inputA[0], inputA[2]); @@ -1540,6 +1501,9 @@ static INLINE void store_rect_buffer_16bit_to_32bit_w16_avx2( } } +typedef void (*transform_1d_avx2)(const __m256i *input, __m256i *output, + int8_t cos_bit); + static const transform_1d_avx2 col_txfm16x32_arr[TX_TYPES] = { fdct16x32_new_avx2, // DCT_DCT NULL, // ADST_DCT @@ -1885,8 +1849,8 @@ static void lowbd_fwd_txfm2d_64x32_avx2(const int16_t *input, int32_t *output, } av1_fdct64_new_avx2(bufA, bufA, cos_bit_row); av1_fdct64_new_avx2(bufB, bufB, cos_bit_row); - av1_round_shift_rect_array_32_avx2(bufA, bufA, 32, -shift[2]); - av1_round_shift_rect_array_32_avx2(bufB, bufB, 32, -shift[2]); + av1_round_shift_rect_array_32_avx2(bufA, bufA, 32, -shift[2], NewSqrt2); + av1_round_shift_rect_array_32_avx2(bufB, bufB, 32, -shift[2], NewSqrt2); int32_t *output8 = output + 16 * 32 * i; for (int j = 0; j < 4; ++j) { @@ -1935,8 +1899,8 @@ static void lowbd_fwd_txfm2d_32x64_avx2(const int16_t *input, int32_t *output, } av1_fdct32_new_avx2(bufA, bufA, cos_bit_row); av1_fdct32_new_avx2(bufB, bufB, cos_bit_row); - av1_round_shift_rect_array_32_avx2(bufA, bufA, 32, -shift[2]); - av1_round_shift_rect_array_32_avx2(bufB, bufB, 32, -shift[2]); + av1_round_shift_rect_array_32_avx2(bufA, bufA, 32, -shift[2], NewSqrt2); + av1_round_shift_rect_array_32_avx2(bufB, bufB, 32, -shift[2], NewSqrt2); int32_t *output8 = output + 16 * 32 * i; for (int j = 0; j < 4; ++j) { diff --git a/third_party/aom/av1/encoder/x86/av1_fwd_txfm2d_sse4.c b/third_party/aom/av1/encoder/x86/av1_fwd_txfm2d_sse4.c index 8ec0256eb8..193f9d10af 100644 --- a/third_party/aom/av1/encoder/x86/av1_fwd_txfm2d_sse4.c +++ b/third_party/aom/av1/encoder/x86/av1_fwd_txfm2d_sse4.c @@ -29,31 +29,22 @@ static INLINE void int16_array_with_stride_to_int32_array_without_stride( } } -typedef void (*TxfmFuncSSE2)(const __m128i *input, __m128i *output, +typedef void (*TxfmFuncSSE2)(__m128i *input, __m128i *output, const int8_t cos_bit, const int8_t *stage_range); -static void fdct32_new_sse4_1(const __m128i *input, __m128i *output, +static void fdct32_new_sse4_1(__m128i *input, __m128i *output, const int8_t cos_bit, const int8_t *stage_range) { const int txfm_size = 32; const int num_per_128 = 4; - __m128i buf0[32]; - __m128i buf1[32]; int col_num = txfm_size / num_per_128; int col; (void)stage_range; for (col = 0; col < col_num; col++) { - int j; - for (j = 0; j < 32; ++j) { - buf0[j] = input[j * col_num + col]; - } - av1_fdct32_new_sse4_1(buf0, buf1, cos_bit); - for (j = 0; j < 32; ++j) { - output[j * col_num + col] = buf1[j]; - } + av1_fdct32_new_sse4_1((input + col), (output + col), cos_bit, col_num); } } -static void fdct64_new_sse4_1(const __m128i *input, __m128i *output, +static void fdct64_new_sse4_1(__m128i *input, __m128i *output, const int8_t cos_bit, const int8_t *stage_range) { const int txfm_size = 64; const int num_per_128 = 4; @@ -64,11 +55,20 @@ static void fdct64_new_sse4_1(const __m128i *input, __m128i *output, col_num); } } +static void idtx32x32_sse4_1(__m128i *input, __m128i *output, + const int8_t cos_bit, const int8_t *stage_range) { + (void)stage_range; + + for (int i = 0; i < 8; i++) { + av1_idtx32_new_sse4_1(&input[i * 32], &output[i * 32], cos_bit, 1); + } +} static INLINE TxfmFuncSSE2 fwd_txfm_type_to_func(TXFM_TYPE txfm_type) { switch (txfm_type) { case TXFM_TYPE_DCT32: return fdct32_new_sse4_1; break; case TXFM_TYPE_DCT64: return fdct64_new_sse4_1; break; + case TXFM_TYPE_IDENTITY32: return idtx32x32_sse4_1; break; default: assert(0); } return NULL; @@ -142,7 +142,7 @@ static INLINE void fwd_txfm2d_64x64_sse4_1(const int16_t *input, txfm2d_size_128 = (col_num >> 1) * (txfm_size >> 1); av1_round_shift_array_32_sse4_1(out_128, buf_128, txfm2d_size_128, -shift[2]); - transpose_32x32(buf_128, out_128); + transpose_8nx8n(buf_128, out_128, 32, 32); } void av1_fwd_txfm2d_32x32_sse4_1(const int16_t *input, int32_t *output, @@ -317,8 +317,8 @@ static void lowbd_fwd_txfm2d_32x64_sse4_1(const int16_t *input, int32_t *output, bufA[j] = _mm_cvtepi16_epi32(buf[j]); bufB[j] = _mm_cvtepi16_epi32(_mm_unpackhi_epi64(buf[j], buf[j])); } - av1_fdct32_new_sse4_1(bufA, bufA, cos_bit_row); - av1_fdct32_new_sse4_1(bufB, bufB, cos_bit_row); + av1_fdct32_new_sse4_1(bufA, bufA, cos_bit_row, 1); + av1_fdct32_new_sse4_1(bufB, bufB, cos_bit_row, 1); av1_round_shift_rect_array_32_sse4_1(bufA, bufA, 32, -shift[2], NewSqrt2); av1_round_shift_rect_array_32_sse4_1(bufB, bufB, 32, -shift[2], NewSqrt2); diff --git a/third_party/aom/av1/encoder/x86/av1_fwd_txfm_avx2.h b/third_party/aom/av1/encoder/x86/av1_fwd_txfm_avx2.h index 38707137c4..aaad76e5ae 100644 --- a/third_party/aom/av1/encoder/x86/av1_fwd_txfm_avx2.h +++ b/third_party/aom/av1/encoder/x86/av1_fwd_txfm_avx2.h @@ -13,13 +13,6 @@ #define AOM_AV1_ENCODER_X86_AV1_FWD_TXFM_AVX2_H_ #include -static INLINE __m256i av1_round_shift_32_avx2(__m256i vec, int bit) { - __m256i tmp, round; - round = _mm256_set1_epi32(1 << (bit - 1)); - tmp = _mm256_add_epi32(vec, round); - return _mm256_srai_epi32(tmp, bit); -} - // out0 = in0*w0 + in1*w1 // out1 = -in1*w0 + in0*w1 static INLINE void btf_32_avx2_type0(const int32_t w0, const int32_t w1, diff --git a/third_party/aom/av1/encoder/x86/av1_txfm1d_sse4.h b/third_party/aom/av1/encoder/x86/av1_txfm1d_sse4.h index 6df2a8bdbb..b3d5b22d79 100644 --- a/third_party/aom/av1/encoder/x86/av1_txfm1d_sse4.h +++ b/third_party/aom/av1/encoder/x86/av1_txfm1d_sse4.h @@ -26,12 +26,10 @@ void av1_fdct8_new_sse4_1(const __m128i *input, __m128i *output, const int8_t cos_bit, const int8_t *stage_range); void av1_fdct16_new_sse4_1(const __m128i *input, __m128i *output, const int8_t cos_bit, const int8_t *stage_range); -void av1_fdct32_new_sse4_1(const __m128i *input, __m128i *output, - int8_t cos_bit); -void av1_fdct64_new_sse4_1(const __m128i *input, __m128i *output, - int8_t cos_bit, const int instride, - const int outstride); - +void av1_fdct32_new_sse4_1(__m128i *input, __m128i *output, int cos_bit, + const int stride); +void av1_fdct64_new_sse4_1(__m128i *input, __m128i *output, int8_t cos_bit, + const int instride, const int outstride); void av1_fadst4_new_sse4_1(const __m128i *input, __m128i *output, const int8_t cos_bit, const int8_t *stage_range); void av1_fadst8_new_sse4_1(const __m128i *input, __m128i *output, @@ -56,6 +54,10 @@ void av1_iadst8_new_sse4_1(const __m128i *input, __m128i *output, const int8_t cos_bit, const int8_t *stage_range); void av1_iadst16_new_sse4_1(const __m128i *input, __m128i *output, const int8_t cos_bit, const int8_t *stage_range); + +void av1_idtx32_new_sse4_1(__m128i *input, __m128i *output, int cos_bit, + const int col_num); + static INLINE void transpose_32_4x4(int stride, const __m128i *input, __m128i *output) { __m128i temp0 = _mm_unpacklo_epi32(input[0 * stride], input[2 * stride]); diff --git a/third_party/aom/av1/encoder/x86/corner_match_avx2.c b/third_party/aom/av1/encoder/x86/corner_match_avx2.c new file mode 100644 index 0000000000..3351d0af19 --- /dev/null +++ b/third_party/aom/av1/encoder/x86/corner_match_avx2.c @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2018, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include +#include +#include + +#include +#include "config/av1_rtcd.h" + +#include "aom_ports/mem.h" +#include "av1/encoder/corner_match.h" + +DECLARE_ALIGNED(16, static const uint8_t, byte_mask[16]) = { + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 0, 0, 0 +}; +#if MATCH_SZ != 13 +#error "Need to change byte_mask in corner_match_sse4.c if MATCH_SZ != 13" +#endif + +/* Compute corr(im1, im2) * MATCH_SZ * stddev(im1), where the +correlation/standard deviation are taken over MATCH_SZ by MATCH_SZ windows +of each image, centered at (x1, y1) and (x2, y2) respectively. +*/ +double compute_cross_correlation_avx2(unsigned char *im1, int stride1, int x1, + int y1, unsigned char *im2, int stride2, + int x2, int y2) { + int i, stride1_i = 0, stride2_i = 0; + __m256i temp1, sum_vec, sumsq2_vec, cross_vec, v, v1_1, v2_1; + const __m128i mask = _mm_load_si128((__m128i *)byte_mask); + const __m256i zero = _mm256_setzero_si256(); + __m128i v1, v2; + + sum_vec = zero; + sumsq2_vec = zero; + cross_vec = zero; + + im1 += (y1 - MATCH_SZ_BY2) * stride1 + (x1 - MATCH_SZ_BY2); + im2 += (y2 - MATCH_SZ_BY2) * stride2 + (x2 - MATCH_SZ_BY2); + + for (i = 0; i < MATCH_SZ; ++i) { + v1 = _mm_and_si128(_mm_loadu_si128((__m128i *)&im1[stride1_i]), mask); + v1_1 = _mm256_cvtepu8_epi16(v1); + v2 = _mm_and_si128(_mm_loadu_si128((__m128i *)&im2[stride2_i]), mask); + v2_1 = _mm256_cvtepu8_epi16(v2); + + v = _mm256_insertf128_si256(_mm256_castsi128_si256(v1), v2, 1); + sumsq2_vec = _mm256_add_epi32(sumsq2_vec, _mm256_madd_epi16(v2_1, v2_1)); + + sum_vec = _mm256_add_epi16(sum_vec, _mm256_sad_epu8(v, zero)); + cross_vec = _mm256_add_epi32(cross_vec, _mm256_madd_epi16(v1_1, v2_1)); + stride1_i += stride1; + stride2_i += stride2; + } + __m256i sum_vec1 = _mm256_srli_si256(sum_vec, 8); + sum_vec = _mm256_add_epi32(sum_vec, sum_vec1); + int sum1_acc = _mm_cvtsi128_si32(_mm256_castsi256_si128(sum_vec)); + int sum2_acc = _mm256_extract_epi32(sum_vec, 4); + + __m256i unp_low = _mm256_unpacklo_epi64(sumsq2_vec, cross_vec); + __m256i unp_hig = _mm256_unpackhi_epi64(sumsq2_vec, cross_vec); + temp1 = _mm256_add_epi32(unp_low, unp_hig); + + __m128i low_sumsq = _mm256_castsi256_si128(temp1); + low_sumsq = _mm_add_epi32(low_sumsq, _mm256_extractf128_si256(temp1, 1)); + low_sumsq = _mm_add_epi32(low_sumsq, _mm_srli_epi64(low_sumsq, 32)); + int sumsq2_acc = _mm_cvtsi128_si32(low_sumsq); + int cross_acc = _mm_extract_epi32(low_sumsq, 2); + + int var2 = sumsq2_acc * MATCH_SZ_SQ - sum2_acc * sum2_acc; + int cov = cross_acc * MATCH_SZ_SQ - sum1_acc * sum2_acc; + return cov / sqrt((double)var2); +} diff --git a/third_party/aom/av1/encoder/x86/encodetxb_avx2.c b/third_party/aom/av1/encoder/x86/encodetxb_avx2.c index 7642f57d18..2621301859 100644 --- a/third_party/aom/av1/encoder/x86/encodetxb_avx2.c +++ b/third_party/aom/av1/encoder/x86/encodetxb_avx2.c @@ -26,14 +26,6 @@ void av1_txb_init_levels_avx2(const tran_low_t *const coeff, const int width, const int stride = width + TX_PAD_HOR; const __m256i y_zeros = _mm256_setzero_si256(); - const int32_t pre_len = sizeof(*levels) * TX_PAD_TOP * stride; - uint8_t *pre_buf = levels - TX_PAD_TOP * stride; - uint8_t *pre_buf_end = pre_buf + pre_len; - do { - yy_storeu_256(pre_buf, y_zeros); - pre_buf += 32; - } while (pre_buf < pre_buf_end); - const int32_t bottom_len = sizeof(*levels) * (TX_PAD_BOTTOM * stride); uint8_t *bottom_buf_end = levels + (height + TX_PAD_BOTTOM) * stride; uint8_t *bottom_buf = bottom_buf_end - ((bottom_len + 31) & (~31)); diff --git a/third_party/aom/av1/encoder/x86/encodetxb_sse4.c b/third_party/aom/av1/encoder/x86/encodetxb_sse4.c index 5e0687cd38..34c9e4f266 100644 --- a/third_party/aom/av1/encoder/x86/encodetxb_sse4.c +++ b/third_party/aom/av1/encoder/x86/encodetxb_sse4.c @@ -23,14 +23,6 @@ void av1_txb_init_levels_sse4_1(const tran_low_t *const coeff, const int width, const int stride = width + TX_PAD_HOR; const __m128i zeros = _mm_setzero_si128(); - const int32_t pre_len = sizeof(*levels) * TX_PAD_TOP * stride; - uint8_t *pre_buf = levels - TX_PAD_TOP * stride; - uint8_t *pre_buf_end = pre_buf + pre_len; - do { - _mm_storeu_si128((__m128i *)(pre_buf), zeros); - pre_buf += 16; - } while (pre_buf < pre_buf_end); - const int32_t bottom_len = sizeof(*levels) * (TX_PAD_BOTTOM * stride); uint8_t *bottom_buf = levels + stride * height; uint8_t *bottom_buf_end = bottom_buf + bottom_len; diff --git a/third_party/aom/av1/encoder/x86/highbd_fwd_txfm_avx2.c b/third_party/aom/av1/encoder/x86/highbd_fwd_txfm_avx2.c new file mode 100644 index 0000000000..24c513feaf --- /dev/null +++ b/third_party/aom/av1/encoder/x86/highbd_fwd_txfm_avx2.c @@ -0,0 +1,3170 @@ +/* + * Copyright (c) 2018, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ +#include +#include /*AVX2*/ + +#include "config/aom_config.h" +#include "config/av1_rtcd.h" +#include "av1/common/av1_txfm.h" +#include "av1/encoder/av1_fwd_txfm1d_cfg.h" +#include "aom_dsp/txfm_common.h" +#include "aom_ports/mem.h" +#include "aom_dsp/x86/txfm_common_sse2.h" +#include "aom_dsp/x86/txfm_common_avx2.h" + +static INLINE void av1_load_buffer_8x8_avx2(const int16_t *input, __m256i *out, + int stride, int flipud, int fliplr, + int shift) { + __m128i out1[8]; + if (!flipud) { + out1[0] = _mm_load_si128((const __m128i *)(input + 0 * stride)); + out1[1] = _mm_load_si128((const __m128i *)(input + 1 * stride)); + out1[2] = _mm_load_si128((const __m128i *)(input + 2 * stride)); + out1[3] = _mm_load_si128((const __m128i *)(input + 3 * stride)); + out1[4] = _mm_load_si128((const __m128i *)(input + 4 * stride)); + out1[5] = _mm_load_si128((const __m128i *)(input + 5 * stride)); + out1[6] = _mm_load_si128((const __m128i *)(input + 6 * stride)); + out1[7] = _mm_load_si128((const __m128i *)(input + 7 * stride)); + + } else { + out1[7] = _mm_load_si128((const __m128i *)(input + 0 * stride)); + out1[6] = _mm_load_si128((const __m128i *)(input + 1 * stride)); + out1[5] = _mm_load_si128((const __m128i *)(input + 2 * stride)); + out1[4] = _mm_load_si128((const __m128i *)(input + 3 * stride)); + out1[3] = _mm_load_si128((const __m128i *)(input + 4 * stride)); + out1[2] = _mm_load_si128((const __m128i *)(input + 5 * stride)); + out1[1] = _mm_load_si128((const __m128i *)(input + 6 * stride)); + out1[0] = _mm_load_si128((const __m128i *)(input + 7 * stride)); + } + if (!fliplr) { + out[0] = _mm256_cvtepi16_epi32(out1[0]); + out[1] = _mm256_cvtepi16_epi32(out1[1]); + out[2] = _mm256_cvtepi16_epi32(out1[2]); + out[3] = _mm256_cvtepi16_epi32(out1[3]); + out[4] = _mm256_cvtepi16_epi32(out1[4]); + out[5] = _mm256_cvtepi16_epi32(out1[5]); + out[6] = _mm256_cvtepi16_epi32(out1[6]); + out[7] = _mm256_cvtepi16_epi32(out1[7]); + + } else { + out[0] = _mm256_cvtepi16_epi32(mm_reverse_epi16(out1[0])); + out[1] = _mm256_cvtepi16_epi32(mm_reverse_epi16(out1[1])); + out[2] = _mm256_cvtepi16_epi32(mm_reverse_epi16(out1[2])); + out[3] = _mm256_cvtepi16_epi32(mm_reverse_epi16(out1[3])); + out[4] = _mm256_cvtepi16_epi32(mm_reverse_epi16(out1[4])); + out[5] = _mm256_cvtepi16_epi32(mm_reverse_epi16(out1[5])); + out[6] = _mm256_cvtepi16_epi32(mm_reverse_epi16(out1[6])); + out[7] = _mm256_cvtepi16_epi32(mm_reverse_epi16(out1[7])); + } + out[0] = _mm256_slli_epi32(out[0], shift); + out[1] = _mm256_slli_epi32(out[1], shift); + out[2] = _mm256_slli_epi32(out[2], shift); + out[3] = _mm256_slli_epi32(out[3], shift); + out[4] = _mm256_slli_epi32(out[4], shift); + out[5] = _mm256_slli_epi32(out[5], shift); + out[6] = _mm256_slli_epi32(out[6], shift); + out[7] = _mm256_slli_epi32(out[7], shift); +} +static INLINE void col_txfm_8x8_rounding(__m256i *in, int shift) { + const __m256i rounding = _mm256_set1_epi32(1 << (shift - 1)); + + in[0] = _mm256_add_epi32(in[0], rounding); + in[1] = _mm256_add_epi32(in[1], rounding); + in[2] = _mm256_add_epi32(in[2], rounding); + in[3] = _mm256_add_epi32(in[3], rounding); + in[4] = _mm256_add_epi32(in[4], rounding); + in[5] = _mm256_add_epi32(in[5], rounding); + in[6] = _mm256_add_epi32(in[6], rounding); + in[7] = _mm256_add_epi32(in[7], rounding); + + in[0] = _mm256_srai_epi32(in[0], shift); + in[1] = _mm256_srai_epi32(in[1], shift); + in[2] = _mm256_srai_epi32(in[2], shift); + in[3] = _mm256_srai_epi32(in[3], shift); + in[4] = _mm256_srai_epi32(in[4], shift); + in[5] = _mm256_srai_epi32(in[5], shift); + in[6] = _mm256_srai_epi32(in[6], shift); + in[7] = _mm256_srai_epi32(in[7], shift); +} +static INLINE void av1_load_buffer_8x16_avx2(const int16_t *input, __m256i *out, + int stride, int flipud, int fliplr, + int shift) { + const int16_t *topL = input; + const int16_t *botL = input + 8 * stride; + + const int16_t *tmp; + + if (flipud) { + tmp = topL; + topL = botL; + botL = tmp; + } + av1_load_buffer_8x8_avx2(topL, out, stride, flipud, fliplr, shift); + av1_load_buffer_8x8_avx2(botL, out + 8, stride, flipud, fliplr, shift); +} +static INLINE void av1_load_buffer_16xn_avx2(const int16_t *input, __m256i *out, + int stride, int height, + int outstride, int flipud, + int fliplr) { + __m256i out1[64]; + if (!flipud) { + for (int i = 0; i < height; i++) { + out1[i] = _mm256_loadu_si256((const __m256i *)(input + i * stride)); + } + } else { + for (int i = 0; i < height; i++) { + out1[(height - 1) - i] = + _mm256_loadu_si256((const __m256i *)(input + i * stride)); + } + } + if (!fliplr) { + for (int i = 0; i < height; i++) { + out[i * outstride] = + _mm256_cvtepi16_epi32(_mm256_castsi256_si128(out1[i])); + out[i * outstride + 1] = + _mm256_cvtepi16_epi32(_mm256_extractf128_si256(out1[i], 1)); + } + } else { + for (int i = 0; i < height; i++) { + out[i * outstride + 1] = _mm256_cvtepi16_epi32( + mm_reverse_epi16(_mm256_castsi256_si128(out1[i]))); + out[i * outstride + 0] = _mm256_cvtepi16_epi32( + mm_reverse_epi16(_mm256_extractf128_si256(out1[i], 1))); + } + } +} + +static void av1_fwd_txfm_transpose_8x8_avx2(const __m256i *in, __m256i *out, + const int instride, + const int outstride) { + __m256i u0, u1, u2, u3, u4, u5, u6, u7; + __m256i x0, x1; + + u0 = _mm256_unpacklo_epi32(in[0 * instride], in[1 * instride]); + u1 = _mm256_unpackhi_epi32(in[0 * instride], in[1 * instride]); + + u2 = _mm256_unpacklo_epi32(in[2 * instride], in[3 * instride]); + u3 = _mm256_unpackhi_epi32(in[2 * instride], in[3 * instride]); + + u4 = _mm256_unpacklo_epi32(in[4 * instride], in[5 * instride]); + u5 = _mm256_unpackhi_epi32(in[4 * instride], in[5 * instride]); + + u6 = _mm256_unpacklo_epi32(in[6 * instride], in[7 * instride]); + u7 = _mm256_unpackhi_epi32(in[6 * instride], in[7 * instride]); + + x0 = _mm256_unpacklo_epi64(u0, u2); + x1 = _mm256_unpacklo_epi64(u4, u6); + out[0 * outstride] = _mm256_permute2f128_si256(x0, x1, 0x20); + out[4 * outstride] = _mm256_permute2f128_si256(x0, x1, 0x31); + + x0 = _mm256_unpackhi_epi64(u0, u2); + x1 = _mm256_unpackhi_epi64(u4, u6); + out[1 * outstride] = _mm256_permute2f128_si256(x0, x1, 0x20); + out[5 * outstride] = _mm256_permute2f128_si256(x0, x1, 0x31); + + x0 = _mm256_unpacklo_epi64(u1, u3); + x1 = _mm256_unpacklo_epi64(u5, u7); + out[2 * outstride] = _mm256_permute2f128_si256(x0, x1, 0x20); + out[6 * outstride] = _mm256_permute2f128_si256(x0, x1, 0x31); + + x0 = _mm256_unpackhi_epi64(u1, u3); + x1 = _mm256_unpackhi_epi64(u5, u7); + out[3 * outstride] = _mm256_permute2f128_si256(x0, x1, 0x20); + out[7 * outstride] = _mm256_permute2f128_si256(x0, x1, 0x31); +} +static INLINE void av1_round_shift_32_8xn_avx2(__m256i *in, int size, int bit, + int stride) { + if (bit < 0) { + bit = -bit; + __m256i round = _mm256_set1_epi32(1 << (bit - 1)); + for (int i = 0; i < size; ++i) { + in[stride * i] = _mm256_add_epi32(in[stride * i], round); + in[stride * i] = _mm256_srai_epi32(in[stride * i], bit); + } + } else if (bit > 0) { + for (int i = 0; i < size; ++i) { + in[stride * i] = _mm256_slli_epi32(in[stride * i], bit); + } + } +} +static INLINE void av1_store_buffer_avx2(const __m256i *const in, int32_t *out, + const int stride, const int out_size) { + for (int i = 0; i < out_size; ++i) { + _mm256_store_si256((__m256i *)(out), in[i]); + out += stride; + } +} +static INLINE void av1_fwd_txfm_transpose_16x16_avx2(const __m256i *in, + __m256i *out) { + av1_fwd_txfm_transpose_8x8_avx2(&in[0], &out[0], 2, 2); + av1_fwd_txfm_transpose_8x8_avx2(&in[1], &out[16], 2, 2); + av1_fwd_txfm_transpose_8x8_avx2(&in[16], &out[1], 2, 2); + av1_fwd_txfm_transpose_8x8_avx2(&in[17], &out[17], 2, 2); +} + +static INLINE __m256i av1_half_btf_avx2(const __m256i *w0, const __m256i *n0, + const __m256i *w1, const __m256i *n1, + const __m256i *rounding, int bit) { + __m256i x, y; + + x = _mm256_mullo_epi32(*w0, *n0); + y = _mm256_mullo_epi32(*w1, *n1); + x = _mm256_add_epi32(x, y); + x = _mm256_add_epi32(x, *rounding); + x = _mm256_srai_epi32(x, bit); + return x; +} +#define btf_32_avx2_type0(w0, w1, in0, in1, out0, out1, bit) \ + do { \ + const __m256i ww0 = _mm256_set1_epi32(w0); \ + const __m256i ww1 = _mm256_set1_epi32(w1); \ + const __m256i in0_w0 = _mm256_mullo_epi32(in0, ww0); \ + const __m256i in1_w1 = _mm256_mullo_epi32(in1, ww1); \ + out0 = _mm256_add_epi32(in0_w0, in1_w1); \ + av1_round_shift_32_8xn_avx2(&out0, 1, -bit, 1); \ + const __m256i in0_w1 = _mm256_mullo_epi32(in0, ww1); \ + const __m256i in1_w0 = _mm256_mullo_epi32(in1, ww0); \ + out1 = _mm256_sub_epi32(in0_w1, in1_w0); \ + av1_round_shift_32_8xn_avx2(&out1, 1, -bit, 1); \ + } while (0) + +#define btf_32_type0_avx2_new(ww0, ww1, in0, in1, out0, out1, r, bit) \ + do { \ + const __m256i in0_w0 = _mm256_mullo_epi32(in0, ww0); \ + const __m256i in1_w1 = _mm256_mullo_epi32(in1, ww1); \ + out0 = _mm256_add_epi32(in0_w0, in1_w1); \ + out0 = _mm256_add_epi32(out0, r); \ + out0 = _mm256_srai_epi32(out0, bit); \ + const __m256i in0_w1 = _mm256_mullo_epi32(in0, ww1); \ + const __m256i in1_w0 = _mm256_mullo_epi32(in1, ww0); \ + out1 = _mm256_sub_epi32(in0_w1, in1_w0); \ + out1 = _mm256_add_epi32(out1, r); \ + out1 = _mm256_srai_epi32(out1, bit); \ + } while (0) + +typedef void (*transform_1d_avx2)(__m256i *in, __m256i *out, + const int8_t cos_bit, int instride, + int outstride); +static void av1_fdct8_avx2(__m256i *in, __m256i *out, const int8_t bit, + const int col_num, const int outstride) { + const int32_t *cospi = cospi_arr(bit); + const __m256i cospi32 = _mm256_set1_epi32(cospi[32]); + const __m256i cospim32 = _mm256_set1_epi32(-cospi[32]); + const __m256i cospi48 = _mm256_set1_epi32(cospi[48]); + const __m256i cospi16 = _mm256_set1_epi32(cospi[16]); + const __m256i cospi56 = _mm256_set1_epi32(cospi[56]); + const __m256i cospi8 = _mm256_set1_epi32(cospi[8]); + const __m256i cospi24 = _mm256_set1_epi32(cospi[24]); + const __m256i cospi40 = _mm256_set1_epi32(cospi[40]); + const __m256i rnding = _mm256_set1_epi32(1 << (bit - 1)); + __m256i u[8], v[8]; + for (int col = 0; col < col_num; ++col) { + u[0] = _mm256_add_epi32(in[0 * col_num + col], in[7 * col_num + col]); + v[7] = _mm256_sub_epi32(in[0 * col_num + col], in[7 * col_num + col]); + u[1] = _mm256_add_epi32(in[1 * col_num + col], in[6 * col_num + col]); + u[6] = _mm256_sub_epi32(in[1 * col_num + col], in[6 * col_num + col]); + u[2] = _mm256_add_epi32(in[2 * col_num + col], in[5 * col_num + col]); + u[5] = _mm256_sub_epi32(in[2 * col_num + col], in[5 * col_num + col]); + u[3] = _mm256_add_epi32(in[3 * col_num + col], in[4 * col_num + col]); + v[4] = _mm256_sub_epi32(in[3 * col_num + col], in[4 * col_num + col]); + v[0] = _mm256_add_epi32(u[0], u[3]); + v[3] = _mm256_sub_epi32(u[0], u[3]); + v[1] = _mm256_add_epi32(u[1], u[2]); + v[2] = _mm256_sub_epi32(u[1], u[2]); + + v[5] = _mm256_mullo_epi32(u[5], cospim32); + v[6] = _mm256_mullo_epi32(u[6], cospi32); + v[5] = _mm256_add_epi32(v[5], v[6]); + v[5] = _mm256_add_epi32(v[5], rnding); + v[5] = _mm256_srai_epi32(v[5], bit); + + u[0] = _mm256_mullo_epi32(u[5], cospi32); + v[6] = _mm256_mullo_epi32(u[6], cospim32); + v[6] = _mm256_sub_epi32(u[0], v[6]); + v[6] = _mm256_add_epi32(v[6], rnding); + v[6] = _mm256_srai_epi32(v[6], bit); + + // stage 3 + // type 0 + v[0] = _mm256_mullo_epi32(v[0], cospi32); + v[1] = _mm256_mullo_epi32(v[1], cospi32); + u[0] = _mm256_add_epi32(v[0], v[1]); + u[0] = _mm256_add_epi32(u[0], rnding); + u[0] = _mm256_srai_epi32(u[0], bit); + + u[1] = _mm256_sub_epi32(v[0], v[1]); + u[1] = _mm256_add_epi32(u[1], rnding); + u[1] = _mm256_srai_epi32(u[1], bit); + + // type 1 + v[0] = _mm256_mullo_epi32(v[2], cospi48); + v[1] = _mm256_mullo_epi32(v[3], cospi16); + u[2] = _mm256_add_epi32(v[0], v[1]); + u[2] = _mm256_add_epi32(u[2], rnding); + u[2] = _mm256_srai_epi32(u[2], bit); + + v[0] = _mm256_mullo_epi32(v[2], cospi16); + v[1] = _mm256_mullo_epi32(v[3], cospi48); + u[3] = _mm256_sub_epi32(v[1], v[0]); + u[3] = _mm256_add_epi32(u[3], rnding); + u[3] = _mm256_srai_epi32(u[3], bit); + + u[4] = _mm256_add_epi32(v[4], v[5]); + u[5] = _mm256_sub_epi32(v[4], v[5]); + u[6] = _mm256_sub_epi32(v[7], v[6]); + u[7] = _mm256_add_epi32(v[7], v[6]); + + // stage 4 + // stage 5 + v[0] = _mm256_mullo_epi32(u[4], cospi56); + v[1] = _mm256_mullo_epi32(u[7], cospi8); + v[0] = _mm256_add_epi32(v[0], v[1]); + v[0] = _mm256_add_epi32(v[0], rnding); + out[1 * outstride + col] = _mm256_srai_epi32(v[0], bit); // buf0[4] + + v[0] = _mm256_mullo_epi32(u[4], cospi8); + v[1] = _mm256_mullo_epi32(u[7], cospi56); + v[0] = _mm256_sub_epi32(v[1], v[0]); + v[0] = _mm256_add_epi32(v[0], rnding); + out[7 * outstride + col] = _mm256_srai_epi32(v[0], bit); // buf0[7] + + v[0] = _mm256_mullo_epi32(u[5], cospi24); + v[1] = _mm256_mullo_epi32(u[6], cospi40); + v[0] = _mm256_add_epi32(v[0], v[1]); + v[0] = _mm256_add_epi32(v[0], rnding); + out[5 * outstride + col] = _mm256_srai_epi32(v[0], bit); // buf0[5] + + v[0] = _mm256_mullo_epi32(u[5], cospi40); + v[1] = _mm256_mullo_epi32(u[6], cospi24); + v[0] = _mm256_sub_epi32(v[1], v[0]); + v[0] = _mm256_add_epi32(v[0], rnding); + out[3 * outstride + col] = _mm256_srai_epi32(v[0], bit); // buf0[6] + + out[0 * outstride + col] = u[0]; // buf0[0] + out[4 * outstride + col] = u[1]; // buf0[1] + out[2 * outstride + col] = u[2]; // buf0[2] + out[6 * outstride + col] = u[3]; // buf0[3] + } +} +static void av1_fadst8_avx2(__m256i *in, __m256i *out, const int8_t bit, + const int col_num, const int outstirde) { + (void)col_num; + const int32_t *cospi = cospi_arr(bit); + const __m256i cospi32 = _mm256_set1_epi32(cospi[32]); + const __m256i cospi16 = _mm256_set1_epi32(cospi[16]); + const __m256i cospim16 = _mm256_set1_epi32(-cospi[16]); + const __m256i cospi48 = _mm256_set1_epi32(cospi[48]); + const __m256i cospim48 = _mm256_set1_epi32(-cospi[48]); + const __m256i cospi4 = _mm256_set1_epi32(cospi[4]); + const __m256i cospim4 = _mm256_set1_epi32(-cospi[4]); + const __m256i cospi60 = _mm256_set1_epi32(cospi[60]); + const __m256i cospi20 = _mm256_set1_epi32(cospi[20]); + const __m256i cospim20 = _mm256_set1_epi32(-cospi[20]); + const __m256i cospi44 = _mm256_set1_epi32(cospi[44]); + const __m256i cospi28 = _mm256_set1_epi32(cospi[28]); + const __m256i cospi36 = _mm256_set1_epi32(cospi[36]); + const __m256i cospim36 = _mm256_set1_epi32(-cospi[36]); + const __m256i cospi52 = _mm256_set1_epi32(cospi[52]); + const __m256i cospim52 = _mm256_set1_epi32(-cospi[52]); + const __m256i cospi12 = _mm256_set1_epi32(cospi[12]); + const __m256i rnding = _mm256_set1_epi32(1 << (bit - 1)); + const __m256i zero = _mm256_setzero_si256(); + __m256i u0, u1, u2, u3, u4, u5, u6, u7; + __m256i v0, v1, v2, v3, v4, v5, v6, v7; + __m256i x, y; + for (int col = 0; col < col_num; ++col) { + u0 = in[0 * col_num + col]; + u1 = _mm256_sub_epi32(zero, in[7 * col_num + col]); + u2 = _mm256_sub_epi32(zero, in[3 * col_num + col]); + u3 = in[4 * col_num + col]; + u4 = _mm256_sub_epi32(zero, in[1 * col_num + col]); + u5 = in[6 * col_num + col]; + u6 = in[2 * col_num + col]; + u7 = _mm256_sub_epi32(zero, in[5 * col_num + col]); + + // stage 2 + v0 = u0; + v1 = u1; + + x = _mm256_mullo_epi32(u2, cospi32); + y = _mm256_mullo_epi32(u3, cospi32); + v2 = _mm256_add_epi32(x, y); + v2 = _mm256_add_epi32(v2, rnding); + v2 = _mm256_srai_epi32(v2, bit); + + v3 = _mm256_sub_epi32(x, y); + v3 = _mm256_add_epi32(v3, rnding); + v3 = _mm256_srai_epi32(v3, bit); + + v4 = u4; + v5 = u5; + + x = _mm256_mullo_epi32(u6, cospi32); + y = _mm256_mullo_epi32(u7, cospi32); + v6 = _mm256_add_epi32(x, y); + v6 = _mm256_add_epi32(v6, rnding); + v6 = _mm256_srai_epi32(v6, bit); + + v7 = _mm256_sub_epi32(x, y); + v7 = _mm256_add_epi32(v7, rnding); + v7 = _mm256_srai_epi32(v7, bit); + + // stage 3 + u0 = _mm256_add_epi32(v0, v2); + u1 = _mm256_add_epi32(v1, v3); + u2 = _mm256_sub_epi32(v0, v2); + u3 = _mm256_sub_epi32(v1, v3); + u4 = _mm256_add_epi32(v4, v6); + u5 = _mm256_add_epi32(v5, v7); + u6 = _mm256_sub_epi32(v4, v6); + u7 = _mm256_sub_epi32(v5, v7); + + // stage 4 + v0 = u0; + v1 = u1; + v2 = u2; + v3 = u3; + + x = _mm256_mullo_epi32(u4, cospi16); + y = _mm256_mullo_epi32(u5, cospi48); + v4 = _mm256_add_epi32(x, y); + v4 = _mm256_add_epi32(v4, rnding); + v4 = _mm256_srai_epi32(v4, bit); + + x = _mm256_mullo_epi32(u4, cospi48); + y = _mm256_mullo_epi32(u5, cospim16); + v5 = _mm256_add_epi32(x, y); + v5 = _mm256_add_epi32(v5, rnding); + v5 = _mm256_srai_epi32(v5, bit); + + x = _mm256_mullo_epi32(u6, cospim48); + y = _mm256_mullo_epi32(u7, cospi16); + v6 = _mm256_add_epi32(x, y); + v6 = _mm256_add_epi32(v6, rnding); + v6 = _mm256_srai_epi32(v6, bit); + + x = _mm256_mullo_epi32(u6, cospi16); + y = _mm256_mullo_epi32(u7, cospi48); + v7 = _mm256_add_epi32(x, y); + v7 = _mm256_add_epi32(v7, rnding); + v7 = _mm256_srai_epi32(v7, bit); + + // stage 5 + u0 = _mm256_add_epi32(v0, v4); + u1 = _mm256_add_epi32(v1, v5); + u2 = _mm256_add_epi32(v2, v6); + u3 = _mm256_add_epi32(v3, v7); + u4 = _mm256_sub_epi32(v0, v4); + u5 = _mm256_sub_epi32(v1, v5); + u6 = _mm256_sub_epi32(v2, v6); + u7 = _mm256_sub_epi32(v3, v7); + + // stage 6 + x = _mm256_mullo_epi32(u0, cospi4); + y = _mm256_mullo_epi32(u1, cospi60); + v0 = _mm256_add_epi32(x, y); + v0 = _mm256_add_epi32(v0, rnding); + v0 = _mm256_srai_epi32(v0, bit); + + x = _mm256_mullo_epi32(u0, cospi60); + y = _mm256_mullo_epi32(u1, cospim4); + v1 = _mm256_add_epi32(x, y); + v1 = _mm256_add_epi32(v1, rnding); + v1 = _mm256_srai_epi32(v1, bit); + + x = _mm256_mullo_epi32(u2, cospi20); + y = _mm256_mullo_epi32(u3, cospi44); + v2 = _mm256_add_epi32(x, y); + v2 = _mm256_add_epi32(v2, rnding); + v2 = _mm256_srai_epi32(v2, bit); + + x = _mm256_mullo_epi32(u2, cospi44); + y = _mm256_mullo_epi32(u3, cospim20); + v3 = _mm256_add_epi32(x, y); + v3 = _mm256_add_epi32(v3, rnding); + v3 = _mm256_srai_epi32(v3, bit); + + x = _mm256_mullo_epi32(u4, cospi36); + y = _mm256_mullo_epi32(u5, cospi28); + v4 = _mm256_add_epi32(x, y); + v4 = _mm256_add_epi32(v4, rnding); + v4 = _mm256_srai_epi32(v4, bit); + + x = _mm256_mullo_epi32(u4, cospi28); + y = _mm256_mullo_epi32(u5, cospim36); + v5 = _mm256_add_epi32(x, y); + v5 = _mm256_add_epi32(v5, rnding); + v5 = _mm256_srai_epi32(v5, bit); + + x = _mm256_mullo_epi32(u6, cospi52); + y = _mm256_mullo_epi32(u7, cospi12); + v6 = _mm256_add_epi32(x, y); + v6 = _mm256_add_epi32(v6, rnding); + v6 = _mm256_srai_epi32(v6, bit); + + x = _mm256_mullo_epi32(u6, cospi12); + y = _mm256_mullo_epi32(u7, cospim52); + v7 = _mm256_add_epi32(x, y); + v7 = _mm256_add_epi32(v7, rnding); + v7 = _mm256_srai_epi32(v7, bit); + + // stage 7 + out[0 * outstirde + col] = v1; + out[1 * outstirde + col] = v6; + out[2 * outstirde + col] = v3; + out[3 * outstirde + col] = v4; + out[4 * outstirde + col] = v5; + out[5 * outstirde + col] = v2; + out[6 * outstirde + col] = v7; + out[7 * outstirde + col] = v0; + } +} +static void av1_idtx8_avx2(__m256i *in, __m256i *out, const int8_t bit, + int col_num, int outstride) { + (void)bit; + (void)outstride; + int num_iters = 8 * col_num; + for (int i = 0; i < num_iters; i += 8) { + out[i] = _mm256_add_epi32(in[i], in[i]); + out[i + 1] = _mm256_add_epi32(in[i + 1], in[i + 1]); + out[i + 2] = _mm256_add_epi32(in[i + 2], in[i + 2]); + out[i + 3] = _mm256_add_epi32(in[i + 3], in[i + 3]); + out[i + 4] = _mm256_add_epi32(in[i + 4], in[i + 4]); + out[i + 5] = _mm256_add_epi32(in[i + 5], in[i + 5]); + out[i + 6] = _mm256_add_epi32(in[i + 6], in[i + 6]); + out[i + 7] = _mm256_add_epi32(in[i + 7], in[i + 7]); + } +} +void av1_fwd_txfm2d_8x8_avx2(const int16_t *input, int32_t *coeff, int stride, + TX_TYPE tx_type, int bd) { + __m256i in[8], out[8]; + const TX_SIZE tx_size = TX_8X8; + const int8_t *shift = fwd_txfm_shift_ls[tx_size]; + const int txw_idx = get_txw_idx(tx_size); + const int txh_idx = get_txh_idx(tx_size); + const int width = tx_size_wide[tx_size]; + const int width_div8 = (width >> 3); + + switch (tx_type) { + case DCT_DCT: + av1_load_buffer_8x8_avx2(input, in, stride, 0, 0, shift[0]); + av1_fdct8_avx2(in, out, fwd_cos_bit_col[txw_idx][txh_idx], width_div8, + width_div8); + col_txfm_8x8_rounding(out, -shift[1]); + av1_fwd_txfm_transpose_8x8_avx2(out, in, width_div8, width_div8); + av1_fdct8_avx2(in, out, fwd_cos_bit_row[txw_idx][txh_idx], width_div8, + width_div8); + av1_fwd_txfm_transpose_8x8_avx2(out, in, width_div8, width_div8); + av1_store_buffer_avx2(in, coeff, 8, 8); + break; + case ADST_DCT: + av1_load_buffer_8x8_avx2(input, in, stride, 0, 0, shift[0]); + av1_fadst8_avx2(in, out, fwd_cos_bit_col[txw_idx][txh_idx], width_div8, + width_div8); + col_txfm_8x8_rounding(out, -shift[1]); + av1_fwd_txfm_transpose_8x8_avx2(out, in, width_div8, width_div8); + av1_fdct8_avx2(in, out, fwd_cos_bit_row[txw_idx][txh_idx], width_div8, + width_div8); + av1_fwd_txfm_transpose_8x8_avx2(out, in, width_div8, width_div8); + av1_store_buffer_avx2(in, coeff, 8, 8); + break; + case DCT_ADST: + av1_load_buffer_8x8_avx2(input, in, stride, 0, 0, shift[0]); + av1_fdct8_avx2(in, out, fwd_cos_bit_col[txw_idx][txh_idx], width_div8, + width_div8); + col_txfm_8x8_rounding(out, -shift[1]); + av1_fwd_txfm_transpose_8x8_avx2(out, in, width_div8, width_div8); + av1_fadst8_avx2(in, out, fwd_cos_bit_row[txw_idx][txh_idx], width_div8, + width_div8); + av1_fwd_txfm_transpose_8x8_avx2(out, in, width_div8, width_div8); + av1_store_buffer_avx2(in, coeff, 8, 8); + break; + case ADST_ADST: + av1_load_buffer_8x8_avx2(input, in, stride, 0, 0, shift[0]); + av1_fadst8_avx2(in, out, fwd_cos_bit_col[txw_idx][txh_idx], width_div8, + width_div8); + col_txfm_8x8_rounding(out, -shift[1]); + av1_fwd_txfm_transpose_8x8_avx2(out, in, width_div8, width_div8); + av1_fadst8_avx2(in, out, fwd_cos_bit_row[txw_idx][txh_idx], width_div8, + width_div8); + av1_fwd_txfm_transpose_8x8_avx2(out, in, width_div8, width_div8); + av1_store_buffer_avx2(in, coeff, 8, 8); + break; + case FLIPADST_DCT: + av1_load_buffer_8x8_avx2(input, in, stride, 1, 0, shift[0]); + av1_fadst8_avx2(in, out, fwd_cos_bit_col[txw_idx][txh_idx], width_div8, + width_div8); + col_txfm_8x8_rounding(out, -shift[1]); + av1_fwd_txfm_transpose_8x8_avx2(out, in, width_div8, width_div8); + av1_fdct8_avx2(in, out, fwd_cos_bit_row[txw_idx][txh_idx], width_div8, + width_div8); + av1_fwd_txfm_transpose_8x8_avx2(out, in, width_div8, width_div8); + av1_store_buffer_avx2(in, coeff, 8, 8); + break; + case DCT_FLIPADST: + av1_load_buffer_8x8_avx2(input, in, stride, 0, 1, shift[0]); + av1_fdct8_avx2(in, out, fwd_cos_bit_col[txw_idx][txh_idx], width_div8, + width_div8); + col_txfm_8x8_rounding(out, -shift[1]); + av1_fwd_txfm_transpose_8x8_avx2(out, in, width_div8, width_div8); + av1_fadst8_avx2(in, out, fwd_cos_bit_row[txw_idx][txh_idx], width_div8, + width_div8); + av1_fwd_txfm_transpose_8x8_avx2(out, in, width_div8, width_div8); + av1_store_buffer_avx2(in, coeff, 8, 8); + break; + case FLIPADST_FLIPADST: + av1_load_buffer_8x8_avx2(input, in, stride, 1, 1, shift[0]); + av1_fadst8_avx2(in, out, fwd_cos_bit_col[txw_idx][txh_idx], width_div8, + width_div8); + col_txfm_8x8_rounding(out, -shift[1]); + av1_fwd_txfm_transpose_8x8_avx2(out, in, width_div8, width_div8); + av1_fadst8_avx2(in, out, fwd_cos_bit_row[txw_idx][txh_idx], width_div8, + width_div8); + av1_fwd_txfm_transpose_8x8_avx2(out, in, width_div8, width_div8); + av1_store_buffer_avx2(in, coeff, 8, 8); + break; + case ADST_FLIPADST: + av1_load_buffer_8x8_avx2(input, in, stride, 0, 1, shift[0]); + av1_fadst8_avx2(in, out, fwd_cos_bit_col[txw_idx][txh_idx], width_div8, + width_div8); + col_txfm_8x8_rounding(out, -shift[1]); + av1_fwd_txfm_transpose_8x8_avx2(out, in, width_div8, width_div8); + av1_fadst8_avx2(in, out, fwd_cos_bit_row[txw_idx][txh_idx], width_div8, + width_div8); + av1_fwd_txfm_transpose_8x8_avx2(out, in, width_div8, width_div8); + av1_store_buffer_avx2(in, coeff, 8, 8); + break; + case FLIPADST_ADST: + av1_load_buffer_8x8_avx2(input, in, stride, 1, 0, shift[0]); + av1_fadst8_avx2(in, out, fwd_cos_bit_col[txw_idx][txh_idx], width_div8, + width_div8); + col_txfm_8x8_rounding(out, -shift[1]); + av1_fwd_txfm_transpose_8x8_avx2(out, in, width_div8, width_div8); + av1_fadst8_avx2(in, out, fwd_cos_bit_row[txw_idx][txh_idx], width_div8, + width_div8); + av1_fwd_txfm_transpose_8x8_avx2(out, in, width_div8, width_div8); + av1_store_buffer_avx2(in, coeff, 8, 8); + break; + case IDTX: + av1_load_buffer_8x8_avx2(input, in, stride, 0, 0, shift[0]); + av1_idtx8_avx2(in, out, fwd_cos_bit_col[txw_idx][txh_idx], width_div8, + width_div8); + col_txfm_8x8_rounding(out, -shift[1]); + av1_idtx8_avx2(out, in, fwd_cos_bit_col[txw_idx][txh_idx], width_div8, + width_div8); + av1_store_buffer_avx2(in, coeff, 8, 8); + break; + case V_DCT: + av1_load_buffer_8x8_avx2(input, in, stride, 0, 0, shift[0]); + av1_fdct8_avx2(in, out, fwd_cos_bit_col[txw_idx][txh_idx], width_div8, + width_div8); + col_txfm_8x8_rounding(out, -shift[1]); + av1_idtx8_avx2(out, in, fwd_cos_bit_col[txw_idx][txh_idx], width_div8, + width_div8); + av1_store_buffer_avx2(in, coeff, 8, 8); + break; + case H_DCT: + av1_load_buffer_8x8_avx2(input, in, stride, 0, 0, shift[0]); + av1_idtx8_avx2(in, out, fwd_cos_bit_col[txw_idx][txh_idx], width_div8, + width_div8); + col_txfm_8x8_rounding(out, -shift[1]); + av1_fwd_txfm_transpose_8x8_avx2(out, in, width_div8, width_div8); + av1_fdct8_avx2(in, out, fwd_cos_bit_col[txw_idx][txh_idx], width_div8, + width_div8); + av1_fwd_txfm_transpose_8x8_avx2(out, in, width_div8, width_div8); + av1_store_buffer_avx2(in, coeff, 8, 8); + break; + case V_ADST: + av1_load_buffer_8x8_avx2(input, in, stride, 0, 0, shift[0]); + av1_fadst8_avx2(in, out, fwd_cos_bit_col[txw_idx][txh_idx], width_div8, + width_div8); + col_txfm_8x8_rounding(out, -shift[1]); + av1_idtx8_avx2(out, in, fwd_cos_bit_col[txw_idx][txh_idx], width_div8, + width_div8); + av1_store_buffer_avx2(in, coeff, 8, 8); + break; + case H_ADST: + av1_load_buffer_8x8_avx2(input, in, stride, 0, 0, shift[0]); + av1_idtx8_avx2(in, out, fwd_cos_bit_col[txw_idx][txh_idx], width_div8, + width_div8); + col_txfm_8x8_rounding(out, -shift[1]); + av1_fwd_txfm_transpose_8x8_avx2(out, in, width_div8, width_div8); + av1_fadst8_avx2(in, out, fwd_cos_bit_col[txw_idx][txh_idx], width_div8, + width_div8); + av1_fwd_txfm_transpose_8x8_avx2(out, in, width_div8, width_div8); + av1_store_buffer_avx2(in, coeff, 8, 8); + break; + case V_FLIPADST: + av1_load_buffer_8x8_avx2(input, in, stride, 1, 0, shift[0]); + av1_fadst8_avx2(in, out, fwd_cos_bit_col[txw_idx][txh_idx], width_div8, + width_div8); + col_txfm_8x8_rounding(out, -shift[1]); + av1_idtx8_avx2(out, in, fwd_cos_bit_col[txw_idx][txh_idx], width_div8, + width_div8); + av1_store_buffer_avx2(in, coeff, 8, 8); + break; + case H_FLIPADST: + av1_load_buffer_8x8_avx2(input, in, stride, 0, 1, shift[0]); + av1_idtx8_avx2(in, out, fwd_cos_bit_col[txw_idx][txh_idx], width_div8, + width_div8); + col_txfm_8x8_rounding(out, -shift[1]); + av1_fwd_txfm_transpose_8x8_avx2(out, in, width_div8, width_div8); + av1_fadst8_avx2(in, out, fwd_cos_bit_col[txw_idx][txh_idx], width_div8, + width_div8); + av1_fwd_txfm_transpose_8x8_avx2(out, in, width_div8, width_div8); + av1_store_buffer_avx2(in, coeff, 8, 8); + break; + default: assert(0); + } + (void)bd; +} + +static void av1_fdct16_avx2(__m256i *in, __m256i *out, const int8_t bit, + const int col_num, const int outstride) { + const int32_t *cospi = cospi_arr(bit); + const __m256i cospi32 = _mm256_set1_epi32(cospi[32]); + const __m256i cospim32 = _mm256_set1_epi32(-cospi[32]); + const __m256i cospi48 = _mm256_set1_epi32(cospi[48]); + const __m256i cospi16 = _mm256_set1_epi32(cospi[16]); + const __m256i cospim48 = _mm256_set1_epi32(-cospi[48]); + const __m256i cospim16 = _mm256_set1_epi32(-cospi[16]); + const __m256i cospi56 = _mm256_set1_epi32(cospi[56]); + const __m256i cospi8 = _mm256_set1_epi32(cospi[8]); + const __m256i cospi24 = _mm256_set1_epi32(cospi[24]); + const __m256i cospi40 = _mm256_set1_epi32(cospi[40]); + const __m256i cospi60 = _mm256_set1_epi32(cospi[60]); + const __m256i cospi4 = _mm256_set1_epi32(cospi[4]); + const __m256i cospi28 = _mm256_set1_epi32(cospi[28]); + const __m256i cospi36 = _mm256_set1_epi32(cospi[36]); + const __m256i cospi44 = _mm256_set1_epi32(cospi[44]); + const __m256i cospi20 = _mm256_set1_epi32(cospi[20]); + const __m256i cospi12 = _mm256_set1_epi32(cospi[12]); + const __m256i cospi52 = _mm256_set1_epi32(cospi[52]); + const __m256i rnding = _mm256_set1_epi32(1 << (bit - 1)); + __m256i u[16], v[16], x; + int col; + + // Calculate the column 0, 1, 2, 3 + for (col = 0; col < col_num; ++col) { + // stage 0 + // stage 1 + u[0] = _mm256_add_epi32(in[0 * col_num + col], in[15 * col_num + col]); + u[15] = _mm256_sub_epi32(in[0 * col_num + col], in[15 * col_num + col]); + u[1] = _mm256_add_epi32(in[1 * col_num + col], in[14 * col_num + col]); + u[14] = _mm256_sub_epi32(in[1 * col_num + col], in[14 * col_num + col]); + u[2] = _mm256_add_epi32(in[2 * col_num + col], in[13 * col_num + col]); + u[13] = _mm256_sub_epi32(in[2 * col_num + col], in[13 * col_num + col]); + u[3] = _mm256_add_epi32(in[3 * col_num + col], in[12 * col_num + col]); + u[12] = _mm256_sub_epi32(in[3 * col_num + col], in[12 * col_num + col]); + u[4] = _mm256_add_epi32(in[4 * col_num + col], in[11 * col_num + col]); + u[11] = _mm256_sub_epi32(in[4 * col_num + col], in[11 * col_num + col]); + u[5] = _mm256_add_epi32(in[5 * col_num + col], in[10 * col_num + col]); + u[10] = _mm256_sub_epi32(in[5 * col_num + col], in[10 * col_num + col]); + u[6] = _mm256_add_epi32(in[6 * col_num + col], in[9 * col_num + col]); + u[9] = _mm256_sub_epi32(in[6 * col_num + col], in[9 * col_num + col]); + u[7] = _mm256_add_epi32(in[7 * col_num + col], in[8 * col_num + col]); + u[8] = _mm256_sub_epi32(in[7 * col_num + col], in[8 * col_num + col]); + + // stage 2 + v[0] = _mm256_add_epi32(u[0], u[7]); + v[7] = _mm256_sub_epi32(u[0], u[7]); + v[1] = _mm256_add_epi32(u[1], u[6]); + v[6] = _mm256_sub_epi32(u[1], u[6]); + v[2] = _mm256_add_epi32(u[2], u[5]); + v[5] = _mm256_sub_epi32(u[2], u[5]); + v[3] = _mm256_add_epi32(u[3], u[4]); + v[4] = _mm256_sub_epi32(u[3], u[4]); + v[8] = u[8]; + v[9] = u[9]; + + v[10] = _mm256_mullo_epi32(u[10], cospim32); + x = _mm256_mullo_epi32(u[13], cospi32); + v[10] = _mm256_add_epi32(v[10], x); + v[10] = _mm256_add_epi32(v[10], rnding); + v[10] = _mm256_srai_epi32(v[10], bit); + + v[13] = _mm256_mullo_epi32(u[10], cospi32); + x = _mm256_mullo_epi32(u[13], cospim32); + v[13] = _mm256_sub_epi32(v[13], x); + v[13] = _mm256_add_epi32(v[13], rnding); + v[13] = _mm256_srai_epi32(v[13], bit); + + v[11] = _mm256_mullo_epi32(u[11], cospim32); + x = _mm256_mullo_epi32(u[12], cospi32); + v[11] = _mm256_add_epi32(v[11], x); + v[11] = _mm256_add_epi32(v[11], rnding); + v[11] = _mm256_srai_epi32(v[11], bit); + + v[12] = _mm256_mullo_epi32(u[11], cospi32); + x = _mm256_mullo_epi32(u[12], cospim32); + v[12] = _mm256_sub_epi32(v[12], x); + v[12] = _mm256_add_epi32(v[12], rnding); + v[12] = _mm256_srai_epi32(v[12], bit); + v[14] = u[14]; + v[15] = u[15]; + + // stage 3 + u[0] = _mm256_add_epi32(v[0], v[3]); + u[3] = _mm256_sub_epi32(v[0], v[3]); + u[1] = _mm256_add_epi32(v[1], v[2]); + u[2] = _mm256_sub_epi32(v[1], v[2]); + u[4] = v[4]; + + u[5] = _mm256_mullo_epi32(v[5], cospim32); + x = _mm256_mullo_epi32(v[6], cospi32); + u[5] = _mm256_add_epi32(u[5], x); + u[5] = _mm256_add_epi32(u[5], rnding); + u[5] = _mm256_srai_epi32(u[5], bit); + + u[6] = _mm256_mullo_epi32(v[5], cospi32); + x = _mm256_mullo_epi32(v[6], cospim32); + u[6] = _mm256_sub_epi32(u[6], x); + u[6] = _mm256_add_epi32(u[6], rnding); + u[6] = _mm256_srai_epi32(u[6], bit); + + u[7] = v[7]; + u[8] = _mm256_add_epi32(v[8], v[11]); + u[11] = _mm256_sub_epi32(v[8], v[11]); + u[9] = _mm256_add_epi32(v[9], v[10]); + u[10] = _mm256_sub_epi32(v[9], v[10]); + u[12] = _mm256_sub_epi32(v[15], v[12]); + u[15] = _mm256_add_epi32(v[15], v[12]); + u[13] = _mm256_sub_epi32(v[14], v[13]); + u[14] = _mm256_add_epi32(v[14], v[13]); + + // stage 4 + u[0] = _mm256_mullo_epi32(u[0], cospi32); + u[1] = _mm256_mullo_epi32(u[1], cospi32); + v[0] = _mm256_add_epi32(u[0], u[1]); + v[0] = _mm256_add_epi32(v[0], rnding); + v[0] = _mm256_srai_epi32(v[0], bit); + + v[1] = _mm256_sub_epi32(u[0], u[1]); + v[1] = _mm256_add_epi32(v[1], rnding); + v[1] = _mm256_srai_epi32(v[1], bit); + + v[2] = _mm256_mullo_epi32(u[2], cospi48); + x = _mm256_mullo_epi32(u[3], cospi16); + v[2] = _mm256_add_epi32(v[2], x); + v[2] = _mm256_add_epi32(v[2], rnding); + v[2] = _mm256_srai_epi32(v[2], bit); + + v[3] = _mm256_mullo_epi32(u[2], cospi16); + x = _mm256_mullo_epi32(u[3], cospi48); + v[3] = _mm256_sub_epi32(x, v[3]); + v[3] = _mm256_add_epi32(v[3], rnding); + v[3] = _mm256_srai_epi32(v[3], bit); + + v[4] = _mm256_add_epi32(u[4], u[5]); + v[5] = _mm256_sub_epi32(u[4], u[5]); + v[6] = _mm256_sub_epi32(u[7], u[6]); + v[7] = _mm256_add_epi32(u[7], u[6]); + v[8] = u[8]; + + v[9] = _mm256_mullo_epi32(u[9], cospim16); + x = _mm256_mullo_epi32(u[14], cospi48); + v[9] = _mm256_add_epi32(v[9], x); + v[9] = _mm256_add_epi32(v[9], rnding); + v[9] = _mm256_srai_epi32(v[9], bit); + + v[14] = _mm256_mullo_epi32(u[9], cospi48); + x = _mm256_mullo_epi32(u[14], cospim16); + v[14] = _mm256_sub_epi32(v[14], x); + v[14] = _mm256_add_epi32(v[14], rnding); + v[14] = _mm256_srai_epi32(v[14], bit); + + v[10] = _mm256_mullo_epi32(u[10], cospim48); + x = _mm256_mullo_epi32(u[13], cospim16); + v[10] = _mm256_add_epi32(v[10], x); + v[10] = _mm256_add_epi32(v[10], rnding); + v[10] = _mm256_srai_epi32(v[10], bit); + + v[13] = _mm256_mullo_epi32(u[10], cospim16); + x = _mm256_mullo_epi32(u[13], cospim48); + v[13] = _mm256_sub_epi32(v[13], x); + v[13] = _mm256_add_epi32(v[13], rnding); + v[13] = _mm256_srai_epi32(v[13], bit); + + v[11] = u[11]; + v[12] = u[12]; + v[15] = u[15]; + + // stage 5 + u[0] = v[0]; + u[1] = v[1]; + u[2] = v[2]; + u[3] = v[3]; + + u[4] = _mm256_mullo_epi32(v[4], cospi56); + x = _mm256_mullo_epi32(v[7], cospi8); + u[4] = _mm256_add_epi32(u[4], x); + u[4] = _mm256_add_epi32(u[4], rnding); + u[4] = _mm256_srai_epi32(u[4], bit); + + u[7] = _mm256_mullo_epi32(v[4], cospi8); + x = _mm256_mullo_epi32(v[7], cospi56); + u[7] = _mm256_sub_epi32(x, u[7]); + u[7] = _mm256_add_epi32(u[7], rnding); + u[7] = _mm256_srai_epi32(u[7], bit); + + u[5] = _mm256_mullo_epi32(v[5], cospi24); + x = _mm256_mullo_epi32(v[6], cospi40); + u[5] = _mm256_add_epi32(u[5], x); + u[5] = _mm256_add_epi32(u[5], rnding); + u[5] = _mm256_srai_epi32(u[5], bit); + + u[6] = _mm256_mullo_epi32(v[5], cospi40); + x = _mm256_mullo_epi32(v[6], cospi24); + u[6] = _mm256_sub_epi32(x, u[6]); + u[6] = _mm256_add_epi32(u[6], rnding); + u[6] = _mm256_srai_epi32(u[6], bit); + + u[8] = _mm256_add_epi32(v[8], v[9]); + u[9] = _mm256_sub_epi32(v[8], v[9]); + u[10] = _mm256_sub_epi32(v[11], v[10]); + u[11] = _mm256_add_epi32(v[11], v[10]); + u[12] = _mm256_add_epi32(v[12], v[13]); + u[13] = _mm256_sub_epi32(v[12], v[13]); + u[14] = _mm256_sub_epi32(v[15], v[14]); + u[15] = _mm256_add_epi32(v[15], v[14]); + + // stage 6 + v[0] = u[0]; + v[1] = u[1]; + v[2] = u[2]; + v[3] = u[3]; + v[4] = u[4]; + v[5] = u[5]; + v[6] = u[6]; + v[7] = u[7]; + + v[8] = _mm256_mullo_epi32(u[8], cospi60); + x = _mm256_mullo_epi32(u[15], cospi4); + v[8] = _mm256_add_epi32(v[8], x); + v[8] = _mm256_add_epi32(v[8], rnding); + v[8] = _mm256_srai_epi32(v[8], bit); + + v[15] = _mm256_mullo_epi32(u[8], cospi4); + x = _mm256_mullo_epi32(u[15], cospi60); + v[15] = _mm256_sub_epi32(x, v[15]); + v[15] = _mm256_add_epi32(v[15], rnding); + v[15] = _mm256_srai_epi32(v[15], bit); + + v[9] = _mm256_mullo_epi32(u[9], cospi28); + x = _mm256_mullo_epi32(u[14], cospi36); + v[9] = _mm256_add_epi32(v[9], x); + v[9] = _mm256_add_epi32(v[9], rnding); + v[9] = _mm256_srai_epi32(v[9], bit); + + v[14] = _mm256_mullo_epi32(u[9], cospi36); + x = _mm256_mullo_epi32(u[14], cospi28); + v[14] = _mm256_sub_epi32(x, v[14]); + v[14] = _mm256_add_epi32(v[14], rnding); + v[14] = _mm256_srai_epi32(v[14], bit); + + v[10] = _mm256_mullo_epi32(u[10], cospi44); + x = _mm256_mullo_epi32(u[13], cospi20); + v[10] = _mm256_add_epi32(v[10], x); + v[10] = _mm256_add_epi32(v[10], rnding); + v[10] = _mm256_srai_epi32(v[10], bit); + + v[13] = _mm256_mullo_epi32(u[10], cospi20); + x = _mm256_mullo_epi32(u[13], cospi44); + v[13] = _mm256_sub_epi32(x, v[13]); + v[13] = _mm256_add_epi32(v[13], rnding); + v[13] = _mm256_srai_epi32(v[13], bit); + + v[11] = _mm256_mullo_epi32(u[11], cospi12); + x = _mm256_mullo_epi32(u[12], cospi52); + v[11] = _mm256_add_epi32(v[11], x); + v[11] = _mm256_add_epi32(v[11], rnding); + v[11] = _mm256_srai_epi32(v[11], bit); + + v[12] = _mm256_mullo_epi32(u[11], cospi52); + x = _mm256_mullo_epi32(u[12], cospi12); + v[12] = _mm256_sub_epi32(x, v[12]); + v[12] = _mm256_add_epi32(v[12], rnding); + v[12] = _mm256_srai_epi32(v[12], bit); + + out[0 * outstride + col] = v[0]; + out[1 * outstride + col] = v[8]; + out[2 * outstride + col] = v[4]; + out[3 * outstride + col] = v[12]; + out[4 * outstride + col] = v[2]; + out[5 * outstride + col] = v[10]; + out[6 * outstride + col] = v[6]; + out[7 * outstride + col] = v[14]; + out[8 * outstride + col] = v[1]; + out[9 * outstride + col] = v[9]; + out[10 * outstride + col] = v[5]; + out[11 * outstride + col] = v[13]; + out[12 * outstride + col] = v[3]; + out[13 * outstride + col] = v[11]; + out[14 * outstride + col] = v[7]; + out[15 * outstride + col] = v[15]; + } +} +static void av1_fadst16_avx2(__m256i *in, __m256i *out, const int8_t bit, + const int num_cols, const int outstride) { + const int32_t *cospi = cospi_arr(bit); + const __m256i cospi32 = _mm256_set1_epi32(cospi[32]); + const __m256i cospi48 = _mm256_set1_epi32(cospi[48]); + const __m256i cospi16 = _mm256_set1_epi32(cospi[16]); + const __m256i cospim16 = _mm256_set1_epi32(-cospi[16]); + const __m256i cospim48 = _mm256_set1_epi32(-cospi[48]); + const __m256i cospi8 = _mm256_set1_epi32(cospi[8]); + const __m256i cospi56 = _mm256_set1_epi32(cospi[56]); + const __m256i cospim56 = _mm256_set1_epi32(-cospi[56]); + const __m256i cospim8 = _mm256_set1_epi32(-cospi[8]); + const __m256i cospi24 = _mm256_set1_epi32(cospi[24]); + const __m256i cospim24 = _mm256_set1_epi32(-cospi[24]); + const __m256i cospim40 = _mm256_set1_epi32(-cospi[40]); + const __m256i cospi40 = _mm256_set1_epi32(cospi[40]); + const __m256i cospi2 = _mm256_set1_epi32(cospi[2]); + const __m256i cospi62 = _mm256_set1_epi32(cospi[62]); + const __m256i cospim2 = _mm256_set1_epi32(-cospi[2]); + const __m256i cospi10 = _mm256_set1_epi32(cospi[10]); + const __m256i cospi54 = _mm256_set1_epi32(cospi[54]); + const __m256i cospim10 = _mm256_set1_epi32(-cospi[10]); + const __m256i cospi18 = _mm256_set1_epi32(cospi[18]); + const __m256i cospi46 = _mm256_set1_epi32(cospi[46]); + const __m256i cospim18 = _mm256_set1_epi32(-cospi[18]); + const __m256i cospi26 = _mm256_set1_epi32(cospi[26]); + const __m256i cospi38 = _mm256_set1_epi32(cospi[38]); + const __m256i cospim26 = _mm256_set1_epi32(-cospi[26]); + const __m256i cospi34 = _mm256_set1_epi32(cospi[34]); + const __m256i cospi30 = _mm256_set1_epi32(cospi[30]); + const __m256i cospim34 = _mm256_set1_epi32(-cospi[34]); + const __m256i cospi42 = _mm256_set1_epi32(cospi[42]); + const __m256i cospi22 = _mm256_set1_epi32(cospi[22]); + const __m256i cospim42 = _mm256_set1_epi32(-cospi[42]); + const __m256i cospi50 = _mm256_set1_epi32(cospi[50]); + const __m256i cospi14 = _mm256_set1_epi32(cospi[14]); + const __m256i cospim50 = _mm256_set1_epi32(-cospi[50]); + const __m256i cospi58 = _mm256_set1_epi32(cospi[58]); + const __m256i cospi6 = _mm256_set1_epi32(cospi[6]); + const __m256i cospim58 = _mm256_set1_epi32(-cospi[58]); + const __m256i rnding = _mm256_set1_epi32(1 << (bit - 1)); + const __m256i zero = _mm256_setzero_si256(); + + __m256i u[16], v[16], x, y; + int col; + + for (col = 0; col < num_cols; ++col) { + // stage 0 + // stage 1 + u[0] = in[0 * num_cols + col]; + u[1] = _mm256_sub_epi32(zero, in[15 * num_cols + col]); + u[2] = _mm256_sub_epi32(zero, in[7 * num_cols + col]); + u[3] = in[8 * num_cols + col]; + u[4] = _mm256_sub_epi32(zero, in[3 * num_cols + col]); + u[5] = in[12 * num_cols + col]; + u[6] = in[4 * num_cols + col]; + u[7] = _mm256_sub_epi32(zero, in[11 * num_cols + col]); + u[8] = _mm256_sub_epi32(zero, in[1 * num_cols + col]); + u[9] = in[14 * num_cols + col]; + u[10] = in[6 * num_cols + col]; + u[11] = _mm256_sub_epi32(zero, in[9 * num_cols + col]); + u[12] = in[2 * num_cols + col]; + u[13] = _mm256_sub_epi32(zero, in[13 * num_cols + col]); + u[14] = _mm256_sub_epi32(zero, in[5 * num_cols + col]); + u[15] = in[10 * num_cols + col]; + + // stage 2 + v[0] = u[0]; + v[1] = u[1]; + + x = _mm256_mullo_epi32(u[2], cospi32); + y = _mm256_mullo_epi32(u[3], cospi32); + v[2] = _mm256_add_epi32(x, y); + v[2] = _mm256_add_epi32(v[2], rnding); + v[2] = _mm256_srai_epi32(v[2], bit); + + v[3] = _mm256_sub_epi32(x, y); + v[3] = _mm256_add_epi32(v[3], rnding); + v[3] = _mm256_srai_epi32(v[3], bit); + + v[4] = u[4]; + v[5] = u[5]; + + x = _mm256_mullo_epi32(u[6], cospi32); + y = _mm256_mullo_epi32(u[7], cospi32); + v[6] = _mm256_add_epi32(x, y); + v[6] = _mm256_add_epi32(v[6], rnding); + v[6] = _mm256_srai_epi32(v[6], bit); + + v[7] = _mm256_sub_epi32(x, y); + v[7] = _mm256_add_epi32(v[7], rnding); + v[7] = _mm256_srai_epi32(v[7], bit); + + v[8] = u[8]; + v[9] = u[9]; + + x = _mm256_mullo_epi32(u[10], cospi32); + y = _mm256_mullo_epi32(u[11], cospi32); + v[10] = _mm256_add_epi32(x, y); + v[10] = _mm256_add_epi32(v[10], rnding); + v[10] = _mm256_srai_epi32(v[10], bit); + + v[11] = _mm256_sub_epi32(x, y); + v[11] = _mm256_add_epi32(v[11], rnding); + v[11] = _mm256_srai_epi32(v[11], bit); + + v[12] = u[12]; + v[13] = u[13]; + + x = _mm256_mullo_epi32(u[14], cospi32); + y = _mm256_mullo_epi32(u[15], cospi32); + v[14] = _mm256_add_epi32(x, y); + v[14] = _mm256_add_epi32(v[14], rnding); + v[14] = _mm256_srai_epi32(v[14], bit); + + v[15] = _mm256_sub_epi32(x, y); + v[15] = _mm256_add_epi32(v[15], rnding); + v[15] = _mm256_srai_epi32(v[15], bit); + + // stage 3 + u[0] = _mm256_add_epi32(v[0], v[2]); + u[1] = _mm256_add_epi32(v[1], v[3]); + u[2] = _mm256_sub_epi32(v[0], v[2]); + u[3] = _mm256_sub_epi32(v[1], v[3]); + u[4] = _mm256_add_epi32(v[4], v[6]); + u[5] = _mm256_add_epi32(v[5], v[7]); + u[6] = _mm256_sub_epi32(v[4], v[6]); + u[7] = _mm256_sub_epi32(v[5], v[7]); + u[8] = _mm256_add_epi32(v[8], v[10]); + u[9] = _mm256_add_epi32(v[9], v[11]); + u[10] = _mm256_sub_epi32(v[8], v[10]); + u[11] = _mm256_sub_epi32(v[9], v[11]); + u[12] = _mm256_add_epi32(v[12], v[14]); + u[13] = _mm256_add_epi32(v[13], v[15]); + u[14] = _mm256_sub_epi32(v[12], v[14]); + u[15] = _mm256_sub_epi32(v[13], v[15]); + + // stage 4 + v[0] = u[0]; + v[1] = u[1]; + v[2] = u[2]; + v[3] = u[3]; + v[4] = av1_half_btf_avx2(&cospi16, &u[4], &cospi48, &u[5], &rnding, bit); + v[5] = av1_half_btf_avx2(&cospi48, &u[4], &cospim16, &u[5], &rnding, bit); + v[6] = av1_half_btf_avx2(&cospim48, &u[6], &cospi16, &u[7], &rnding, bit); + v[7] = av1_half_btf_avx2(&cospi16, &u[6], &cospi48, &u[7], &rnding, bit); + v[8] = u[8]; + v[9] = u[9]; + v[10] = u[10]; + v[11] = u[11]; + v[12] = av1_half_btf_avx2(&cospi16, &u[12], &cospi48, &u[13], &rnding, bit); + v[13] = + av1_half_btf_avx2(&cospi48, &u[12], &cospim16, &u[13], &rnding, bit); + v[14] = + av1_half_btf_avx2(&cospim48, &u[14], &cospi16, &u[15], &rnding, bit); + v[15] = av1_half_btf_avx2(&cospi16, &u[14], &cospi48, &u[15], &rnding, bit); + + // stage 5 + u[0] = _mm256_add_epi32(v[0], v[4]); + u[1] = _mm256_add_epi32(v[1], v[5]); + u[2] = _mm256_add_epi32(v[2], v[6]); + u[3] = _mm256_add_epi32(v[3], v[7]); + u[4] = _mm256_sub_epi32(v[0], v[4]); + u[5] = _mm256_sub_epi32(v[1], v[5]); + u[6] = _mm256_sub_epi32(v[2], v[6]); + u[7] = _mm256_sub_epi32(v[3], v[7]); + u[8] = _mm256_add_epi32(v[8], v[12]); + u[9] = _mm256_add_epi32(v[9], v[13]); + u[10] = _mm256_add_epi32(v[10], v[14]); + u[11] = _mm256_add_epi32(v[11], v[15]); + u[12] = _mm256_sub_epi32(v[8], v[12]); + u[13] = _mm256_sub_epi32(v[9], v[13]); + u[14] = _mm256_sub_epi32(v[10], v[14]); + u[15] = _mm256_sub_epi32(v[11], v[15]); + + // stage 6 + v[0] = u[0]; + v[1] = u[1]; + v[2] = u[2]; + v[3] = u[3]; + v[4] = u[4]; + v[5] = u[5]; + v[6] = u[6]; + v[7] = u[7]; + v[8] = av1_half_btf_avx2(&cospi8, &u[8], &cospi56, &u[9], &rnding, bit); + v[9] = av1_half_btf_avx2(&cospi56, &u[8], &cospim8, &u[9], &rnding, bit); + v[10] = av1_half_btf_avx2(&cospi40, &u[10], &cospi24, &u[11], &rnding, bit); + v[11] = + av1_half_btf_avx2(&cospi24, &u[10], &cospim40, &u[11], &rnding, bit); + v[12] = av1_half_btf_avx2(&cospim56, &u[12], &cospi8, &u[13], &rnding, bit); + v[13] = av1_half_btf_avx2(&cospi8, &u[12], &cospi56, &u[13], &rnding, bit); + v[14] = + av1_half_btf_avx2(&cospim24, &u[14], &cospi40, &u[15], &rnding, bit); + v[15] = av1_half_btf_avx2(&cospi40, &u[14], &cospi24, &u[15], &rnding, bit); + + // stage 7 + u[0] = _mm256_add_epi32(v[0], v[8]); + u[1] = _mm256_add_epi32(v[1], v[9]); + u[2] = _mm256_add_epi32(v[2], v[10]); + u[3] = _mm256_add_epi32(v[3], v[11]); + u[4] = _mm256_add_epi32(v[4], v[12]); + u[5] = _mm256_add_epi32(v[5], v[13]); + u[6] = _mm256_add_epi32(v[6], v[14]); + u[7] = _mm256_add_epi32(v[7], v[15]); + u[8] = _mm256_sub_epi32(v[0], v[8]); + u[9] = _mm256_sub_epi32(v[1], v[9]); + u[10] = _mm256_sub_epi32(v[2], v[10]); + u[11] = _mm256_sub_epi32(v[3], v[11]); + u[12] = _mm256_sub_epi32(v[4], v[12]); + u[13] = _mm256_sub_epi32(v[5], v[13]); + u[14] = _mm256_sub_epi32(v[6], v[14]); + u[15] = _mm256_sub_epi32(v[7], v[15]); + + // stage 8 + v[0] = av1_half_btf_avx2(&cospi2, &u[0], &cospi62, &u[1], &rnding, bit); + v[1] = av1_half_btf_avx2(&cospi62, &u[0], &cospim2, &u[1], &rnding, bit); + v[2] = av1_half_btf_avx2(&cospi10, &u[2], &cospi54, &u[3], &rnding, bit); + v[3] = av1_half_btf_avx2(&cospi54, &u[2], &cospim10, &u[3], &rnding, bit); + v[4] = av1_half_btf_avx2(&cospi18, &u[4], &cospi46, &u[5], &rnding, bit); + v[5] = av1_half_btf_avx2(&cospi46, &u[4], &cospim18, &u[5], &rnding, bit); + v[6] = av1_half_btf_avx2(&cospi26, &u[6], &cospi38, &u[7], &rnding, bit); + v[7] = av1_half_btf_avx2(&cospi38, &u[6], &cospim26, &u[7], &rnding, bit); + v[8] = av1_half_btf_avx2(&cospi34, &u[8], &cospi30, &u[9], &rnding, bit); + v[9] = av1_half_btf_avx2(&cospi30, &u[8], &cospim34, &u[9], &rnding, bit); + v[10] = av1_half_btf_avx2(&cospi42, &u[10], &cospi22, &u[11], &rnding, bit); + v[11] = + av1_half_btf_avx2(&cospi22, &u[10], &cospim42, &u[11], &rnding, bit); + v[12] = av1_half_btf_avx2(&cospi50, &u[12], &cospi14, &u[13], &rnding, bit); + v[13] = + av1_half_btf_avx2(&cospi14, &u[12], &cospim50, &u[13], &rnding, bit); + v[14] = av1_half_btf_avx2(&cospi58, &u[14], &cospi6, &u[15], &rnding, bit); + v[15] = av1_half_btf_avx2(&cospi6, &u[14], &cospim58, &u[15], &rnding, bit); + + // stage 9 + out[0 * outstride + col] = v[1]; + out[1 * outstride + col] = v[14]; + out[2 * outstride + col] = v[3]; + out[3 * outstride + col] = v[12]; + out[4 * outstride + col] = v[5]; + out[5 * outstride + col] = v[10]; + out[6 * outstride + col] = v[7]; + out[7 * outstride + col] = v[8]; + out[8 * outstride + col] = v[9]; + out[9 * outstride + col] = v[6]; + out[10 * outstride + col] = v[11]; + out[11 * outstride + col] = v[4]; + out[12 * outstride + col] = v[13]; + out[13 * outstride + col] = v[2]; + out[14 * outstride + col] = v[15]; + out[15 * outstride + col] = v[0]; + } +} +static void av1_idtx16_avx2(__m256i *in, __m256i *out, const int8_t bit, + int col_num, const int outstride) { + (void)bit; + (void)outstride; + __m256i fact = _mm256_set1_epi32(2 * NewSqrt2); + __m256i offset = _mm256_set1_epi32(1 << (NewSqrt2Bits - 1)); + __m256i a_low; + + int num_iters = 16 * col_num; + for (int i = 0; i < num_iters; i++) { + a_low = _mm256_mullo_epi32(in[i], fact); + a_low = _mm256_add_epi32(a_low, offset); + out[i] = _mm256_srai_epi32(a_low, NewSqrt2Bits); + } +} +static const transform_1d_avx2 col_highbd_txfm8x16_arr[TX_TYPES] = { + av1_fdct16_avx2, // DCT_DCT + av1_fadst16_avx2, // ADST_DCT + av1_fdct16_avx2, // DCT_ADST + av1_fadst16_avx2, // ADST_ADST + av1_fadst16_avx2, // FLIPADST_DCT + av1_fdct16_avx2, // DCT_FLIPADST + av1_fadst16_avx2, // FLIPADST_FLIPADST + av1_fadst16_avx2, // ADST_FLIPADST + av1_fadst16_avx2, // FLIPADST_ADST + av1_idtx16_avx2, // IDTX + av1_fdct16_avx2, // V_DCT + av1_idtx16_avx2, // H_DCT + av1_fadst16_avx2, // V_ADST + av1_idtx16_avx2, // H_ADST + av1_fadst16_avx2, // V_FLIPADST + av1_idtx16_avx2 // H_FLIPADST +}; +static const transform_1d_avx2 row_highbd_txfm8x8_arr[TX_TYPES] = { + av1_fdct8_avx2, // DCT_DCT + av1_fdct8_avx2, // ADST_DCT + av1_fadst8_avx2, // DCT_ADST + av1_fadst8_avx2, // ADST_ADST + av1_fdct8_avx2, // FLIPADST_DCT + av1_fadst8_avx2, // DCT_FLIPADST + av1_fadst8_avx2, // FLIPADST_FLIPADST + av1_fadst8_avx2, // ADST_FLIPADST + av1_fadst8_avx2, // FLIPADST_ADST + av1_idtx8_avx2, // IDTX + av1_idtx8_avx2, // V_DCT + av1_fdct8_avx2, // H_DCT + av1_idtx8_avx2, // V_ADST + av1_fadst8_avx2, // H_ADST + av1_idtx8_avx2, // V_FLIPADST + av1_fadst8_avx2 // H_FLIPADST +}; +void av1_fwd_txfm2d_8x16_avx2(const int16_t *input, int32_t *coeff, int stride, + TX_TYPE tx_type, int bd) { + __m256i in[16], out[16]; + const int8_t *shift = fwd_txfm_shift_ls[TX_8X16]; + const int txw_idx = get_txw_idx(TX_8X16); + const int txh_idx = get_txh_idx(TX_8X16); + const transform_1d_avx2 col_txfm = col_highbd_txfm8x16_arr[tx_type]; + const transform_1d_avx2 row_txfm = row_highbd_txfm8x8_arr[tx_type]; + const int8_t bit = fwd_cos_bit_col[txw_idx][txh_idx]; + int ud_flip, lr_flip; + get_flip_cfg(tx_type, &ud_flip, &lr_flip); + + av1_load_buffer_8x16_avx2(input, in, stride, ud_flip, lr_flip, shift[0]); + col_txfm(in, out, bit, 1, 1); + col_txfm_8x8_rounding(out, -shift[1]); + col_txfm_8x8_rounding(&out[8], -shift[1]); + av1_fwd_txfm_transpose_8x8_avx2(out, in, 1, 2); + av1_fwd_txfm_transpose_8x8_avx2(&out[8], &in[1], 1, 2); + row_txfm(in, out, bit, 2, 2); + av1_fwd_txfm_transpose_8x8_avx2(out, in, 2, 1); + av1_fwd_txfm_transpose_8x8_avx2(&out[1], &in[8], 2, 1); + av1_round_shift_rect_array_32_avx2(in, in, 16, -shift[2], NewSqrt2); + av1_store_buffer_avx2(in, coeff, 8, 16); + (void)bd; +} +static const transform_1d_avx2 col_highbd_txfm8x8_arr[TX_TYPES] = { + av1_fdct8_avx2, // DCT_DCT + av1_fadst8_avx2, // ADST_DCT + av1_fdct8_avx2, // DCT_ADST + av1_fadst8_avx2, // ADST_ADST + av1_fadst8_avx2, // FLIPADST_DCT + av1_fdct8_avx2, // DCT_FLIPADST + av1_fadst8_avx2, // FLIPADST_FLIPADST + av1_fadst8_avx2, // ADST_FLIPADST + av1_fadst8_avx2, // FLIPADST_ADST + av1_idtx8_avx2, // IDTX + av1_fdct8_avx2, // V_DCT + av1_idtx8_avx2, // H_DCT + av1_fadst8_avx2, // V_ADST + av1_idtx8_avx2, // H_ADST + av1_fadst8_avx2, // V_FLIPADST + av1_idtx8_avx2 // H_FLIPADST +}; +static const transform_1d_avx2 row_highbd_txfm8x16_arr[TX_TYPES] = { + av1_fdct16_avx2, // DCT_DCT + av1_fdct16_avx2, // ADST_DCT + av1_fadst16_avx2, // DCT_ADST + av1_fadst16_avx2, // ADST_ADST + av1_fdct16_avx2, // FLIPADST_DCT + av1_fadst16_avx2, // DCT_FLIPADST + av1_fadst16_avx2, // FLIPADST_FLIPADST + av1_fadst16_avx2, // ADST_FLIPADST + av1_fadst16_avx2, // FLIPADST_ADST + av1_idtx16_avx2, // IDTX + av1_idtx16_avx2, // V_DCT + av1_fdct16_avx2, // H_DCT + av1_idtx16_avx2, // V_ADST + av1_fadst16_avx2, // H_ADST + av1_idtx16_avx2, // V_FLIPADST + av1_fadst16_avx2 // H_FLIPADST +}; +void av1_fwd_txfm2d_16x8_avx2(const int16_t *input, int32_t *coeff, int stride, + TX_TYPE tx_type, int bd) { + __m256i in[16], out[16]; + const int8_t *shift = fwd_txfm_shift_ls[TX_16X8]; + const int txw_idx = get_txw_idx(TX_16X8); + const int txh_idx = get_txh_idx(TX_16X8); + const transform_1d_avx2 col_txfm = col_highbd_txfm8x8_arr[tx_type]; + const transform_1d_avx2 row_txfm = row_highbd_txfm8x16_arr[tx_type]; + const int8_t bit = fwd_cos_bit_col[txw_idx][txh_idx]; + int ud_flip, lr_flip; + get_flip_cfg(tx_type, &ud_flip, &lr_flip); + + av1_load_buffer_16xn_avx2(input, in, stride, 8, 2, ud_flip, lr_flip); + av1_round_shift_32_8xn_avx2(in, 16, shift[0], 1); + col_txfm(in, out, bit, 2, 2); + av1_round_shift_32_8xn_avx2(out, 16, shift[1], 1); + av1_fwd_txfm_transpose_8x8_avx2(out, in, 2, 1); + av1_fwd_txfm_transpose_8x8_avx2(&out[1], &in[8], 2, 1); + row_txfm(in, out, bit, 1, 1); + av1_fwd_txfm_transpose_8x8_avx2(out, in, 1, 2); + av1_fwd_txfm_transpose_8x8_avx2(&out[8], &in[1], 1, 2); + av1_round_shift_rect_array_32_avx2(in, in, 16, -shift[2], NewSqrt2); + av1_store_buffer_avx2(in, coeff, 8, 16); + (void)bd; +} +void av1_fwd_txfm2d_16x16_avx2(const int16_t *input, int32_t *coeff, int stride, + TX_TYPE tx_type, int bd) { + __m256i in[32], out[32]; + const TX_SIZE tx_size = TX_16X16; + const int8_t *shift = fwd_txfm_shift_ls[tx_size]; + const int txw_idx = get_txw_idx(tx_size); + const int txh_idx = get_txh_idx(tx_size); + const int width = tx_size_wide[tx_size]; + const int height = tx_size_high[tx_size]; + const int width_div8 = (width >> 3); + const int width_div16 = (width >> 4); + const int size = (height << 1); + switch (tx_type) { + case DCT_DCT: + av1_load_buffer_16xn_avx2(input, in, stride, height, width_div8, 0, 0); + av1_round_shift_32_8xn_avx2(in, size, shift[0], width_div16); + av1_fdct16_avx2(in, out, fwd_cos_bit_col[txw_idx][txh_idx], width_div8, + width_div8); + av1_round_shift_32_8xn_avx2(out, size, shift[1], width_div16); + av1_fwd_txfm_transpose_16x16_avx2(out, in); + av1_fdct16_avx2(in, out, fwd_cos_bit_row[txw_idx][txh_idx], width_div8, + width_div8); + av1_fwd_txfm_transpose_16x16_avx2(out, in); + av1_store_buffer_avx2(in, coeff, 8, 32); + break; + case ADST_DCT: + av1_load_buffer_16xn_avx2(input, in, stride, height, width_div8, 0, 0); + av1_round_shift_32_8xn_avx2(in, size, shift[0], width_div16); + av1_fadst16_avx2(in, out, fwd_cos_bit_col[txw_idx][txh_idx], width_div8, + width_div8); + av1_round_shift_32_8xn_avx2(out, size, shift[1], width_div16); + av1_fwd_txfm_transpose_16x16_avx2(out, in); + av1_fdct16_avx2(in, out, fwd_cos_bit_row[txw_idx][txh_idx], width_div8, + width_div8); + av1_fwd_txfm_transpose_16x16_avx2(out, in); + av1_store_buffer_avx2(in, coeff, 8, 32); + break; + case DCT_ADST: + av1_load_buffer_16xn_avx2(input, in, stride, height, width_div8, 0, 0); + av1_round_shift_32_8xn_avx2(in, size, shift[0], width_div16); + av1_fdct16_avx2(in, out, fwd_cos_bit_col[txw_idx][txh_idx], width_div8, + width_div8); + av1_round_shift_32_8xn_avx2(out, size, shift[1], width_div16); + av1_fwd_txfm_transpose_16x16_avx2(out, in); + av1_fadst16_avx2(in, out, fwd_cos_bit_row[txw_idx][txh_idx], width_div8, + width_div8); + av1_fwd_txfm_transpose_16x16_avx2(out, in); + av1_store_buffer_avx2(in, coeff, 8, 32); + break; + case ADST_ADST: + av1_load_buffer_16xn_avx2(input, in, stride, height, width_div8, 0, 0); + av1_round_shift_32_8xn_avx2(in, size, shift[0], width_div16); + av1_fadst16_avx2(in, out, fwd_cos_bit_col[txw_idx][txh_idx], width_div8, + width_div8); + av1_round_shift_32_8xn_avx2(out, size, shift[1], width_div16); + av1_fwd_txfm_transpose_16x16_avx2(out, in); + av1_fadst16_avx2(in, out, fwd_cos_bit_row[txw_idx][txh_idx], width_div8, + width_div8); + av1_fwd_txfm_transpose_16x16_avx2(out, in); + av1_store_buffer_avx2(in, coeff, 8, 32); + break; + case FLIPADST_DCT: + av1_load_buffer_16xn_avx2(input, in, stride, height, width_div8, 1, 0); + av1_round_shift_32_8xn_avx2(in, size, shift[0], width_div16); + av1_fadst16_avx2(in, out, fwd_cos_bit_col[txw_idx][txh_idx], width_div8, + width_div8); + av1_round_shift_32_8xn_avx2(out, size, shift[1], width_div16); + av1_fwd_txfm_transpose_16x16_avx2(out, in); + av1_fdct16_avx2(in, out, fwd_cos_bit_row[txw_idx][txh_idx], width_div8, + width_div8); + av1_fwd_txfm_transpose_16x16_avx2(out, in); + av1_store_buffer_avx2(in, coeff, 8, 32); + break; + case DCT_FLIPADST: + av1_load_buffer_16xn_avx2(input, in, stride, height, width_div8, 0, 1); + av1_round_shift_32_8xn_avx2(in, size, shift[0], width_div16); + av1_fdct16_avx2(in, out, fwd_cos_bit_col[txw_idx][txh_idx], width_div8, + width_div8); + av1_round_shift_32_8xn_avx2(out, size, shift[1], width_div16); + av1_fwd_txfm_transpose_16x16_avx2(out, in); + av1_fadst16_avx2(in, out, fwd_cos_bit_row[txw_idx][txh_idx], width_div8, + width_div8); + av1_fwd_txfm_transpose_16x16_avx2(out, in); + av1_store_buffer_avx2(in, coeff, 8, 32); + break; + case FLIPADST_FLIPADST: + av1_load_buffer_16xn_avx2(input, in, stride, height, width_div8, 1, 1); + av1_round_shift_32_8xn_avx2(in, size, shift[0], width_div16); + av1_fadst16_avx2(in, out, fwd_cos_bit_col[txw_idx][txh_idx], width_div8, + width_div8); + av1_round_shift_32_8xn_avx2(out, size, shift[1], width_div16); + av1_fwd_txfm_transpose_16x16_avx2(out, in); + av1_fadst16_avx2(in, out, fwd_cos_bit_row[txw_idx][txh_idx], width_div8, + width_div8); + av1_fwd_txfm_transpose_16x16_avx2(out, in); + av1_store_buffer_avx2(in, coeff, 8, 32); + break; + case ADST_FLIPADST: + av1_load_buffer_16xn_avx2(input, in, stride, height, width_div8, 0, 1); + av1_round_shift_32_8xn_avx2(in, size, shift[0], width_div16); + av1_fadst16_avx2(in, out, fwd_cos_bit_col[txw_idx][txh_idx], width_div8, + width_div8); + av1_round_shift_32_8xn_avx2(out, size, shift[1], width_div16); + av1_fwd_txfm_transpose_16x16_avx2(out, in); + av1_fadst16_avx2(in, out, fwd_cos_bit_row[txw_idx][txh_idx], width_div8, + width_div8); + av1_fwd_txfm_transpose_16x16_avx2(out, in); + av1_store_buffer_avx2(in, coeff, 8, 32); + break; + case FLIPADST_ADST: + av1_load_buffer_16xn_avx2(input, in, stride, height, width_div8, 1, 0); + av1_round_shift_32_8xn_avx2(in, size, shift[0], width_div16); + av1_fadst16_avx2(in, out, fwd_cos_bit_col[txw_idx][txh_idx], width_div8, + width_div8); + av1_round_shift_32_8xn_avx2(out, size, shift[1], width_div16); + av1_fwd_txfm_transpose_16x16_avx2(out, in); + av1_fadst16_avx2(in, out, fwd_cos_bit_row[txw_idx][txh_idx], width_div8, + width_div8); + av1_fwd_txfm_transpose_16x16_avx2(out, in); + av1_store_buffer_avx2(in, coeff, 8, 32); + break; + case IDTX: + av1_load_buffer_16xn_avx2(input, in, stride, height, width_div8, 0, 0); + av1_round_shift_32_8xn_avx2(in, size, shift[0], width_div16); + av1_idtx16_avx2(in, out, fwd_cos_bit_col[txw_idx][txh_idx], width_div8, + width_div8); + av1_round_shift_32_8xn_avx2(out, size, shift[1], width_div16); + av1_idtx16_avx2(out, in, fwd_cos_bit_row[txw_idx][txh_idx], width_div8, + width_div8); + av1_store_buffer_avx2(in, coeff, 8, 32); + break; + case V_DCT: + av1_load_buffer_16xn_avx2(input, in, stride, height, width_div8, 0, 0); + av1_round_shift_32_8xn_avx2(in, size, shift[0], width_div16); + av1_fdct16_avx2(in, out, fwd_cos_bit_col[txw_idx][txh_idx], width_div8, + width_div8); + av1_round_shift_32_8xn_avx2(out, size, shift[1], width_div16); + av1_idtx16_avx2(out, in, fwd_cos_bit_row[txw_idx][txh_idx], width_div8, + width_div8); + av1_store_buffer_avx2(in, coeff, 8, 32); + break; + case H_DCT: + av1_load_buffer_16xn_avx2(input, in, stride, height, width_div8, 0, 0); + av1_round_shift_32_8xn_avx2(in, size, shift[0], width_div16); + av1_idtx16_avx2(in, out, fwd_cos_bit_col[txw_idx][txh_idx], width_div8, + width_div8); + av1_round_shift_32_8xn_avx2(out, size, shift[1], width_div16); + av1_fwd_txfm_transpose_16x16_avx2(out, in); + av1_fdct16_avx2(in, out, fwd_cos_bit_row[txw_idx][txh_idx], width_div8, + width_div8); + av1_fwd_txfm_transpose_16x16_avx2(out, in); + av1_store_buffer_avx2(in, coeff, 8, 32); + break; + case V_ADST: + av1_load_buffer_16xn_avx2(input, in, stride, height, width_div8, 0, 0); + av1_round_shift_32_8xn_avx2(in, size, shift[0], width_div16); + av1_fadst16_avx2(in, out, fwd_cos_bit_col[txw_idx][txh_idx], width_div8, + width_div8); + av1_round_shift_32_8xn_avx2(out, size, shift[1], width_div16); + av1_idtx16_avx2(out, in, fwd_cos_bit_row[txw_idx][txh_idx], width_div8, + width_div8); + av1_store_buffer_avx2(in, coeff, 8, 32); + break; + case H_ADST: + av1_load_buffer_16xn_avx2(input, in, stride, height, width_div8, 0, 0); + av1_round_shift_32_8xn_avx2(in, size, shift[0], width_div16); + av1_idtx16_avx2(in, out, fwd_cos_bit_col[txw_idx][txh_idx], width_div8, + width_div8); + av1_round_shift_32_8xn_avx2(out, size, shift[1], width_div16); + av1_fwd_txfm_transpose_16x16_avx2(out, in); + av1_fadst16_avx2(in, out, fwd_cos_bit_row[txw_idx][txh_idx], width_div8, + width_div8); + av1_fwd_txfm_transpose_16x16_avx2(out, in); + av1_store_buffer_avx2(in, coeff, 8, 32); + break; + case V_FLIPADST: + av1_load_buffer_16xn_avx2(input, in, stride, height, width_div8, 1, 0); + av1_round_shift_32_8xn_avx2(in, size, shift[0], width_div16); + av1_fadst16_avx2(in, out, fwd_cos_bit_col[txw_idx][txh_idx], width_div8, + width_div8); + av1_round_shift_32_8xn_avx2(out, size, shift[1], width_div16); + av1_idtx16_avx2(out, in, fwd_cos_bit_row[txw_idx][txh_idx], width_div8, + width_div8); + av1_store_buffer_avx2(in, coeff, 8, 32); + break; + case H_FLIPADST: + av1_load_buffer_16xn_avx2(input, in, stride, height, width_div8, 0, 1); + av1_round_shift_32_8xn_avx2(in, size, shift[0], width_div16); + av1_idtx16_avx2(in, out, fwd_cos_bit_col[txw_idx][txh_idx], width_div8, + width_div8); + av1_round_shift_32_8xn_avx2(out, size, shift[1], width_div16); + av1_fwd_txfm_transpose_16x16_avx2(out, in); + av1_fadst16_avx2(in, out, fwd_cos_bit_row[txw_idx][txh_idx], width_div8, + width_div8); + av1_fwd_txfm_transpose_16x16_avx2(out, in); + av1_store_buffer_avx2(in, coeff, 8, 32); + break; + default: assert(0); + } + (void)bd; +} +static INLINE void av1_fdct32_avx2(__m256i *input, __m256i *output, + const int8_t cos_bit, const int instride, + const int outstride) { + __m256i buf0[32]; + __m256i buf1[32]; + const int32_t *cospi; + int startidx = 0 * instride; + int endidx = 31 * instride; + // stage 0 + // stage 1 + buf1[0] = _mm256_add_epi32(input[startidx], input[endidx]); + buf1[31] = _mm256_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + buf1[1] = _mm256_add_epi32(input[startidx], input[endidx]); + buf1[30] = _mm256_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + buf1[2] = _mm256_add_epi32(input[startidx], input[endidx]); + buf1[29] = _mm256_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + buf1[3] = _mm256_add_epi32(input[startidx], input[endidx]); + buf1[28] = _mm256_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + buf1[4] = _mm256_add_epi32(input[startidx], input[endidx]); + buf1[27] = _mm256_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + buf1[5] = _mm256_add_epi32(input[startidx], input[endidx]); + buf1[26] = _mm256_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + buf1[6] = _mm256_add_epi32(input[startidx], input[endidx]); + buf1[25] = _mm256_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + buf1[7] = _mm256_add_epi32(input[startidx], input[endidx]); + buf1[24] = _mm256_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + buf1[8] = _mm256_add_epi32(input[startidx], input[endidx]); + buf1[23] = _mm256_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + buf1[9] = _mm256_add_epi32(input[startidx], input[endidx]); + buf1[22] = _mm256_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + buf1[10] = _mm256_add_epi32(input[startidx], input[endidx]); + buf1[21] = _mm256_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + buf1[11] = _mm256_add_epi32(input[startidx], input[endidx]); + buf1[20] = _mm256_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + buf1[12] = _mm256_add_epi32(input[startidx], input[endidx]); + buf1[19] = _mm256_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + buf1[13] = _mm256_add_epi32(input[startidx], input[endidx]); + buf1[18] = _mm256_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + buf1[14] = _mm256_add_epi32(input[startidx], input[endidx]); + buf1[17] = _mm256_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + buf1[15] = _mm256_add_epi32(input[startidx], input[endidx]); + buf1[16] = _mm256_sub_epi32(input[startidx], input[endidx]); + + // stage 2 + cospi = cospi_arr(cos_bit); + buf0[0] = _mm256_add_epi32(buf1[0], buf1[15]); + buf0[15] = _mm256_sub_epi32(buf1[0], buf1[15]); + buf0[1] = _mm256_add_epi32(buf1[1], buf1[14]); + buf0[14] = _mm256_sub_epi32(buf1[1], buf1[14]); + buf0[2] = _mm256_add_epi32(buf1[2], buf1[13]); + buf0[13] = _mm256_sub_epi32(buf1[2], buf1[13]); + buf0[3] = _mm256_add_epi32(buf1[3], buf1[12]); + buf0[12] = _mm256_sub_epi32(buf1[3], buf1[12]); + buf0[4] = _mm256_add_epi32(buf1[4], buf1[11]); + buf0[11] = _mm256_sub_epi32(buf1[4], buf1[11]); + buf0[5] = _mm256_add_epi32(buf1[5], buf1[10]); + buf0[10] = _mm256_sub_epi32(buf1[5], buf1[10]); + buf0[6] = _mm256_add_epi32(buf1[6], buf1[9]); + buf0[9] = _mm256_sub_epi32(buf1[6], buf1[9]); + buf0[7] = _mm256_add_epi32(buf1[7], buf1[8]); + buf0[8] = _mm256_sub_epi32(buf1[7], buf1[8]); + buf0[16] = buf1[16]; + buf0[17] = buf1[17]; + buf0[18] = buf1[18]; + buf0[19] = buf1[19]; + btf_32_avx2_type0(-cospi[32], cospi[32], buf1[20], buf1[27], buf0[20], + buf0[27], cos_bit); + btf_32_avx2_type0(-cospi[32], cospi[32], buf1[21], buf1[26], buf0[21], + buf0[26], cos_bit); + btf_32_avx2_type0(-cospi[32], cospi[32], buf1[22], buf1[25], buf0[22], + buf0[25], cos_bit); + btf_32_avx2_type0(-cospi[32], cospi[32], buf1[23], buf1[24], buf0[23], + buf0[24], cos_bit); + buf0[28] = buf1[28]; + buf0[29] = buf1[29]; + buf0[30] = buf1[30]; + buf0[31] = buf1[31]; + + // stage 3 + cospi = cospi_arr(cos_bit); + buf1[0] = _mm256_add_epi32(buf0[0], buf0[7]); + buf1[7] = _mm256_sub_epi32(buf0[0], buf0[7]); + buf1[1] = _mm256_add_epi32(buf0[1], buf0[6]); + buf1[6] = _mm256_sub_epi32(buf0[1], buf0[6]); + buf1[2] = _mm256_add_epi32(buf0[2], buf0[5]); + buf1[5] = _mm256_sub_epi32(buf0[2], buf0[5]); + buf1[3] = _mm256_add_epi32(buf0[3], buf0[4]); + buf1[4] = _mm256_sub_epi32(buf0[3], buf0[4]); + buf1[8] = buf0[8]; + buf1[9] = buf0[9]; + btf_32_avx2_type0(-cospi[32], cospi[32], buf0[10], buf0[13], buf1[10], + buf1[13], cos_bit); + btf_32_avx2_type0(-cospi[32], cospi[32], buf0[11], buf0[12], buf1[11], + buf1[12], cos_bit); + buf1[14] = buf0[14]; + buf1[15] = buf0[15]; + buf1[16] = _mm256_add_epi32(buf0[16], buf0[23]); + buf1[23] = _mm256_sub_epi32(buf0[16], buf0[23]); + buf1[17] = _mm256_add_epi32(buf0[17], buf0[22]); + buf1[22] = _mm256_sub_epi32(buf0[17], buf0[22]); + buf1[18] = _mm256_add_epi32(buf0[18], buf0[21]); + buf1[21] = _mm256_sub_epi32(buf0[18], buf0[21]); + buf1[19] = _mm256_add_epi32(buf0[19], buf0[20]); + buf1[20] = _mm256_sub_epi32(buf0[19], buf0[20]); + buf1[24] = _mm256_sub_epi32(buf0[31], buf0[24]); + buf1[31] = _mm256_add_epi32(buf0[31], buf0[24]); + buf1[25] = _mm256_sub_epi32(buf0[30], buf0[25]); + buf1[30] = _mm256_add_epi32(buf0[30], buf0[25]); + buf1[26] = _mm256_sub_epi32(buf0[29], buf0[26]); + buf1[29] = _mm256_add_epi32(buf0[29], buf0[26]); + buf1[27] = _mm256_sub_epi32(buf0[28], buf0[27]); + buf1[28] = _mm256_add_epi32(buf0[28], buf0[27]); + + // stage 4 + cospi = cospi_arr(cos_bit); + buf0[0] = _mm256_add_epi32(buf1[0], buf1[3]); + buf0[3] = _mm256_sub_epi32(buf1[0], buf1[3]); + buf0[1] = _mm256_add_epi32(buf1[1], buf1[2]); + buf0[2] = _mm256_sub_epi32(buf1[1], buf1[2]); + buf0[4] = buf1[4]; + btf_32_avx2_type0(-cospi[32], cospi[32], buf1[5], buf1[6], buf0[5], buf0[6], + cos_bit); + buf0[7] = buf1[7]; + buf0[8] = _mm256_add_epi32(buf1[8], buf1[11]); + buf0[11] = _mm256_sub_epi32(buf1[8], buf1[11]); + buf0[9] = _mm256_add_epi32(buf1[9], buf1[10]); + buf0[10] = _mm256_sub_epi32(buf1[9], buf1[10]); + buf0[12] = _mm256_sub_epi32(buf1[15], buf1[12]); + buf0[15] = _mm256_add_epi32(buf1[15], buf1[12]); + buf0[13] = _mm256_sub_epi32(buf1[14], buf1[13]); + buf0[14] = _mm256_add_epi32(buf1[14], buf1[13]); + buf0[16] = buf1[16]; + buf0[17] = buf1[17]; + btf_32_avx2_type0(-cospi[16], cospi[48], buf1[18], buf1[29], buf0[18], + buf0[29], cos_bit); + btf_32_avx2_type0(-cospi[16], cospi[48], buf1[19], buf1[28], buf0[19], + buf0[28], cos_bit); + btf_32_avx2_type0(-cospi[48], -cospi[16], buf1[20], buf1[27], buf0[20], + buf0[27], cos_bit); + btf_32_avx2_type0(-cospi[48], -cospi[16], buf1[21], buf1[26], buf0[21], + buf0[26], cos_bit); + buf0[22] = buf1[22]; + buf0[23] = buf1[23]; + buf0[24] = buf1[24]; + buf0[25] = buf1[25]; + buf0[30] = buf1[30]; + buf0[31] = buf1[31]; + + // stage 5 + cospi = cospi_arr(cos_bit); + btf_32_avx2_type0(cospi[32], cospi[32], buf0[0], buf0[1], buf1[0], buf1[1], + cos_bit); + btf_32_avx2_type0(cospi[16], cospi[48], buf0[3], buf0[2], buf1[2], buf1[3], + cos_bit); + buf1[4] = _mm256_add_epi32(buf0[4], buf0[5]); + buf1[5] = _mm256_sub_epi32(buf0[4], buf0[5]); + buf1[6] = _mm256_sub_epi32(buf0[7], buf0[6]); + buf1[7] = _mm256_add_epi32(buf0[7], buf0[6]); + buf1[8] = buf0[8]; + btf_32_avx2_type0(-cospi[16], cospi[48], buf0[9], buf0[14], buf1[9], buf1[14], + cos_bit); + btf_32_avx2_type0(-cospi[48], -cospi[16], buf0[10], buf0[13], buf1[10], + buf1[13], cos_bit); + buf1[11] = buf0[11]; + buf1[12] = buf0[12]; + buf1[15] = buf0[15]; + buf1[16] = _mm256_add_epi32(buf0[16], buf0[19]); + buf1[19] = _mm256_sub_epi32(buf0[16], buf0[19]); + buf1[17] = _mm256_add_epi32(buf0[17], buf0[18]); + buf1[18] = _mm256_sub_epi32(buf0[17], buf0[18]); + buf1[20] = _mm256_sub_epi32(buf0[23], buf0[20]); + buf1[23] = _mm256_add_epi32(buf0[23], buf0[20]); + buf1[21] = _mm256_sub_epi32(buf0[22], buf0[21]); + buf1[22] = _mm256_add_epi32(buf0[22], buf0[21]); + buf1[24] = _mm256_add_epi32(buf0[24], buf0[27]); + buf1[27] = _mm256_sub_epi32(buf0[24], buf0[27]); + buf1[25] = _mm256_add_epi32(buf0[25], buf0[26]); + buf1[26] = _mm256_sub_epi32(buf0[25], buf0[26]); + buf1[28] = _mm256_sub_epi32(buf0[31], buf0[28]); + buf1[31] = _mm256_add_epi32(buf0[31], buf0[28]); + buf1[29] = _mm256_sub_epi32(buf0[30], buf0[29]); + buf1[30] = _mm256_add_epi32(buf0[30], buf0[29]); + + // stage 6 + cospi = cospi_arr(cos_bit); + buf0[0] = buf1[0]; + buf0[1] = buf1[1]; + buf0[2] = buf1[2]; + buf0[3] = buf1[3]; + btf_32_avx2_type0(cospi[8], cospi[56], buf1[7], buf1[4], buf0[4], buf0[7], + cos_bit); + btf_32_avx2_type0(cospi[40], cospi[24], buf1[6], buf1[5], buf0[5], buf0[6], + cos_bit); + buf0[8] = _mm256_add_epi32(buf1[8], buf1[9]); + buf0[9] = _mm256_sub_epi32(buf1[8], buf1[9]); + buf0[10] = _mm256_sub_epi32(buf1[11], buf1[10]); + buf0[11] = _mm256_add_epi32(buf1[11], buf1[10]); + buf0[12] = _mm256_add_epi32(buf1[12], buf1[13]); + buf0[13] = _mm256_sub_epi32(buf1[12], buf1[13]); + buf0[14] = _mm256_sub_epi32(buf1[15], buf1[14]); + buf0[15] = _mm256_add_epi32(buf1[15], buf1[14]); + buf0[16] = buf1[16]; + btf_32_avx2_type0(-cospi[8], cospi[56], buf1[17], buf1[30], buf0[17], + buf0[30], cos_bit); + btf_32_avx2_type0(-cospi[56], -cospi[8], buf1[18], buf1[29], buf0[18], + buf0[29], cos_bit); + buf0[19] = buf1[19]; + buf0[20] = buf1[20]; + btf_32_avx2_type0(-cospi[40], cospi[24], buf1[21], buf1[26], buf0[21], + buf0[26], cos_bit); + btf_32_avx2_type0(-cospi[24], -cospi[40], buf1[22], buf1[25], buf0[22], + buf0[25], cos_bit); + buf0[23] = buf1[23]; + buf0[24] = buf1[24]; + buf0[27] = buf1[27]; + buf0[28] = buf1[28]; + buf0[31] = buf1[31]; + + // stage 7 + cospi = cospi_arr(cos_bit); + buf1[0] = buf0[0]; + buf1[1] = buf0[1]; + buf1[2] = buf0[2]; + buf1[3] = buf0[3]; + buf1[4] = buf0[4]; + buf1[5] = buf0[5]; + buf1[6] = buf0[6]; + buf1[7] = buf0[7]; + btf_32_avx2_type0(cospi[4], cospi[60], buf0[15], buf0[8], buf1[8], buf1[15], + cos_bit); + btf_32_avx2_type0(cospi[36], cospi[28], buf0[14], buf0[9], buf1[9], buf1[14], + cos_bit); + btf_32_avx2_type0(cospi[20], cospi[44], buf0[13], buf0[10], buf1[10], + buf1[13], cos_bit); + btf_32_avx2_type0(cospi[52], cospi[12], buf0[12], buf0[11], buf1[11], + buf1[12], cos_bit); + buf1[16] = _mm256_add_epi32(buf0[16], buf0[17]); + buf1[17] = _mm256_sub_epi32(buf0[16], buf0[17]); + buf1[18] = _mm256_sub_epi32(buf0[19], buf0[18]); + buf1[19] = _mm256_add_epi32(buf0[19], buf0[18]); + buf1[20] = _mm256_add_epi32(buf0[20], buf0[21]); + buf1[21] = _mm256_sub_epi32(buf0[20], buf0[21]); + buf1[22] = _mm256_sub_epi32(buf0[23], buf0[22]); + buf1[23] = _mm256_add_epi32(buf0[23], buf0[22]); + buf1[24] = _mm256_add_epi32(buf0[24], buf0[25]); + buf1[25] = _mm256_sub_epi32(buf0[24], buf0[25]); + buf1[26] = _mm256_sub_epi32(buf0[27], buf0[26]); + buf1[27] = _mm256_add_epi32(buf0[27], buf0[26]); + buf1[28] = _mm256_add_epi32(buf0[28], buf0[29]); + buf1[29] = _mm256_sub_epi32(buf0[28], buf0[29]); + buf1[30] = _mm256_sub_epi32(buf0[31], buf0[30]); + buf1[31] = _mm256_add_epi32(buf0[31], buf0[30]); + + // stage 8 + cospi = cospi_arr(cos_bit); + buf0[0] = buf1[0]; + buf0[1] = buf1[1]; + buf0[2] = buf1[2]; + buf0[3] = buf1[3]; + buf0[4] = buf1[4]; + buf0[5] = buf1[5]; + buf0[6] = buf1[6]; + buf0[7] = buf1[7]; + buf0[8] = buf1[8]; + buf0[9] = buf1[9]; + buf0[10] = buf1[10]; + buf0[11] = buf1[11]; + buf0[12] = buf1[12]; + buf0[13] = buf1[13]; + buf0[14] = buf1[14]; + buf0[15] = buf1[15]; + btf_32_avx2_type0(cospi[2], cospi[62], buf1[31], buf1[16], buf0[16], buf0[31], + cos_bit); + btf_32_avx2_type0(cospi[34], cospi[30], buf1[30], buf1[17], buf0[17], + buf0[30], cos_bit); + btf_32_avx2_type0(cospi[18], cospi[46], buf1[29], buf1[18], buf0[18], + buf0[29], cos_bit); + btf_32_avx2_type0(cospi[50], cospi[14], buf1[28], buf1[19], buf0[19], + buf0[28], cos_bit); + btf_32_avx2_type0(cospi[10], cospi[54], buf1[27], buf1[20], buf0[20], + buf0[27], cos_bit); + btf_32_avx2_type0(cospi[42], cospi[22], buf1[26], buf1[21], buf0[21], + buf0[26], cos_bit); + btf_32_avx2_type0(cospi[26], cospi[38], buf1[25], buf1[22], buf0[22], + buf0[25], cos_bit); + btf_32_avx2_type0(cospi[58], cospi[6], buf1[24], buf1[23], buf0[23], buf0[24], + cos_bit); + + startidx = 0 * outstride; + endidx = 31 * outstride; + // stage 9 + output[startidx] = buf0[0]; + output[endidx] = buf0[31]; + startidx += outstride; + endidx -= outstride; + output[startidx] = buf0[16]; + output[endidx] = buf0[15]; + startidx += outstride; + endidx -= outstride; + output[startidx] = buf0[8]; + output[endidx] = buf0[23]; + startidx += outstride; + endidx -= outstride; + output[startidx] = buf0[24]; + output[endidx] = buf0[7]; + startidx += outstride; + endidx -= outstride; + output[startidx] = buf0[4]; + output[endidx] = buf0[27]; + startidx += outstride; + endidx -= outstride; + output[startidx] = buf0[20]; + output[endidx] = buf0[11]; + startidx += outstride; + endidx -= outstride; + output[startidx] = buf0[12]; + output[endidx] = buf0[19]; + startidx += outstride; + endidx -= outstride; + output[startidx] = buf0[28]; + output[endidx] = buf0[3]; + startidx += outstride; + endidx -= outstride; + output[startidx] = buf0[2]; + output[endidx] = buf0[29]; + startidx += outstride; + endidx -= outstride; + output[startidx] = buf0[18]; + output[endidx] = buf0[13]; + startidx += outstride; + endidx -= outstride; + output[startidx] = buf0[10]; + output[endidx] = buf0[21]; + startidx += outstride; + endidx -= outstride; + output[startidx] = buf0[26]; + output[endidx] = buf0[5]; + startidx += outstride; + endidx -= outstride; + output[startidx] = buf0[6]; + output[endidx] = buf0[25]; + startidx += outstride; + endidx -= outstride; + output[startidx] = buf0[22]; + output[endidx] = buf0[9]; + startidx += outstride; + endidx -= outstride; + output[startidx] = buf0[14]; + output[endidx] = buf0[17]; + startidx += outstride; + endidx -= outstride; + output[startidx] = buf0[30]; + output[endidx] = buf0[1]; +} +static INLINE void idtx32x32_avx2(__m256i *input, __m256i *output, + const int8_t cos_bit, int instride, + int outstride) { + (void)cos_bit; + for (int i = 0; i < 32; i += 8) { + output[i * outstride] = _mm256_slli_epi32(input[i * instride], 2); + output[(i + 1) * outstride] = + _mm256_slli_epi32(input[(i + 1) * instride], 2); + output[(i + 2) * outstride] = + _mm256_slli_epi32(input[(i + 2) * instride], 2); + output[(i + 3) * outstride] = + _mm256_slli_epi32(input[(i + 3) * instride], 2); + output[(i + 4) * outstride] = + _mm256_slli_epi32(input[(i + 4) * instride], 2); + output[(i + 5) * outstride] = + _mm256_slli_epi32(input[(i + 5) * instride], 2); + output[(i + 6) * outstride] = + _mm256_slli_epi32(input[(i + 6) * instride], 2); + output[(i + 7) * outstride] = + _mm256_slli_epi32(input[(i + 7) * instride], 2); + } +} +static const transform_1d_avx2 col_txfm8x32_arr[TX_TYPES] = { + av1_fdct32_avx2, // DCT_DCT + NULL, // ADST_DCT + NULL, // DCT_ADST + NULL, // ADST_ADST + NULL, // FLIPADST_DCT + NULL, // DCT_FLIPADST + NULL, // FLIPADST_FLIPADST + NULL, // ADST_FLIPADST + NULL, // FLIPADST_ADST + idtx32x32_avx2, // IDTX + NULL, // V_DCT + NULL, // H_DCT + NULL, // V_ADST + NULL, // H_ADST + NULL, // V_FLIPADST + NULL // H_FLIPADST +}; +static const transform_1d_avx2 row_txfm8x32_arr[TX_TYPES] = { + av1_fdct32_avx2, // DCT_DCT + NULL, // ADST_DCT + NULL, // DCT_ADST + NULL, // ADST_ADST + NULL, // FLIPADST_DCT + NULL, // DCT_FLIPADST + NULL, // FLIPADST_FLIPADST + NULL, // ADST_FLIPADST + NULL, // FLIPADST_ADST + idtx32x32_avx2, // IDTX + NULL, // V_DCT + NULL, // H_DCT + NULL, // V_ADST + NULL, // H_ADST + NULL, // V_FLIPADST + NULL // H_FLIPADST +}; +void av1_fwd_txfm2d_32x32_avx2(const int16_t *input, int32_t *output, + int stride, TX_TYPE tx_type, int bd) { + (void)bd; + __m256i buf0[128], buf1[128]; + const int tx_size = TX_32X32; + const int8_t *shift = fwd_txfm_shift_ls[tx_size]; + const int txw_idx = get_txw_idx(tx_size); + const int txh_idx = get_txh_idx(tx_size); + const int cos_bit_col = fwd_cos_bit_col[txw_idx][txh_idx]; + const int cos_bit_row = fwd_cos_bit_row[txw_idx][txh_idx]; + const int width = tx_size_wide[tx_size]; + const int height = tx_size_high[tx_size]; + const transform_1d_avx2 col_txfm = col_txfm8x32_arr[tx_type]; + const transform_1d_avx2 row_txfm = row_txfm8x32_arr[tx_type]; + int r, c; + const int width_div16 = (width >> 4); + const int width_div8 = (width >> 3); + + for (int i = 0; i < width_div16; i++) { + av1_load_buffer_16xn_avx2(input + (i << 4), &buf0[(i << 1)], stride, height, + width_div8, 0, 0); + av1_round_shift_32_8xn_avx2(&buf0[(i << 1)], height, shift[0], width_div8); + av1_round_shift_32_8xn_avx2(&buf0[(i << 1) + 1], height, shift[0], + width_div8); + col_txfm(&buf0[(i << 1)], &buf0[(i << 1)], cos_bit_col, width_div8, + width_div8); + col_txfm(&buf0[(i << 1) + 1], &buf0[(i << 1) + 1], cos_bit_col, width_div8, + width_div8); + av1_round_shift_32_8xn_avx2(&buf0[(i << 1)], height, shift[1], width_div8); + av1_round_shift_32_8xn_avx2(&buf0[(i << 1) + 1], height, shift[1], + width_div8); + } + + for (r = 0; r < height; r += 8) { + for (c = 0; c < width_div8; c++) { + av1_fwd_txfm_transpose_8x8_avx2(&buf0[r * width_div8 + c], + &buf1[c * 8 * width_div8 + (r >> 3)], + width_div8, width_div8); + } + } + + for (int i = 0; i < width_div16; i++) { + row_txfm(&buf1[(i << 1)], &buf1[(i << 1)], cos_bit_row, width_div8, + width_div8); + row_txfm(&buf1[(i << 1) + 1], &buf1[(i << 1) + 1], cos_bit_row, width_div8, + width_div8); + av1_round_shift_32_8xn_avx2(&buf1[(i << 1)], height, shift[2], width_div8); + av1_round_shift_32_8xn_avx2(&buf1[(i << 1) + 1], height, shift[2], + width_div8); + } + + for (r = 0; r < height; r += 8) { + for (c = 0; c < width_div8; c++) { + av1_fwd_txfm_transpose_8x8_avx2(&buf1[r * width_div8 + c], + &buf0[c * 8 * width_div8 + (r >> 3)], + width_div8, width_div8); + } + } + + av1_store_buffer_avx2(buf0, output, 8, 128); +} +static INLINE void av1_fdct64_stage2_avx2(__m256i *x1, __m256i *x2, + __m256i *cospi_m32, + __m256i *cospi_p32, + const __m256i *__rounding, + int8_t cos_bit) { + x2[0] = _mm256_add_epi32(x1[0], x1[31]); + x2[31] = _mm256_sub_epi32(x1[0], x1[31]); + x2[1] = _mm256_add_epi32(x1[1], x1[30]); + x2[30] = _mm256_sub_epi32(x1[1], x1[30]); + x2[2] = _mm256_add_epi32(x1[2], x1[29]); + x2[29] = _mm256_sub_epi32(x1[2], x1[29]); + x2[3] = _mm256_add_epi32(x1[3], x1[28]); + x2[28] = _mm256_sub_epi32(x1[3], x1[28]); + x2[4] = _mm256_add_epi32(x1[4], x1[27]); + x2[27] = _mm256_sub_epi32(x1[4], x1[27]); + x2[5] = _mm256_add_epi32(x1[5], x1[26]); + x2[26] = _mm256_sub_epi32(x1[5], x1[26]); + x2[6] = _mm256_add_epi32(x1[6], x1[25]); + x2[25] = _mm256_sub_epi32(x1[6], x1[25]); + x2[7] = _mm256_add_epi32(x1[7], x1[24]); + x2[24] = _mm256_sub_epi32(x1[7], x1[24]); + x2[8] = _mm256_add_epi32(x1[8], x1[23]); + x2[23] = _mm256_sub_epi32(x1[8], x1[23]); + x2[9] = _mm256_add_epi32(x1[9], x1[22]); + x2[22] = _mm256_sub_epi32(x1[9], x1[22]); + x2[10] = _mm256_add_epi32(x1[10], x1[21]); + x2[21] = _mm256_sub_epi32(x1[10], x1[21]); + x2[11] = _mm256_add_epi32(x1[11], x1[20]); + x2[20] = _mm256_sub_epi32(x1[11], x1[20]); + x2[12] = _mm256_add_epi32(x1[12], x1[19]); + x2[19] = _mm256_sub_epi32(x1[12], x1[19]); + x2[13] = _mm256_add_epi32(x1[13], x1[18]); + x2[18] = _mm256_sub_epi32(x1[13], x1[18]); + x2[14] = _mm256_add_epi32(x1[14], x1[17]); + x2[17] = _mm256_sub_epi32(x1[14], x1[17]); + x2[15] = _mm256_add_epi32(x1[15], x1[16]); + x2[16] = _mm256_sub_epi32(x1[15], x1[16]); + x2[32] = x1[32]; + x2[33] = x1[33]; + x2[34] = x1[34]; + x2[35] = x1[35]; + x2[36] = x1[36]; + x2[37] = x1[37]; + x2[38] = x1[38]; + x2[39] = x1[39]; + btf_32_type0_avx2_new(*cospi_m32, *cospi_p32, x1[40], x1[55], x2[40], x2[55], + *__rounding, cos_bit); + btf_32_type0_avx2_new(*cospi_m32, *cospi_p32, x1[41], x1[54], x2[41], x2[54], + *__rounding, cos_bit); + btf_32_type0_avx2_new(*cospi_m32, *cospi_p32, x1[42], x1[53], x2[42], x2[53], + *__rounding, cos_bit); + btf_32_type0_avx2_new(*cospi_m32, *cospi_p32, x1[43], x1[52], x2[43], x2[52], + *__rounding, cos_bit); + btf_32_type0_avx2_new(*cospi_m32, *cospi_p32, x1[44], x1[51], x2[44], x2[51], + *__rounding, cos_bit); + btf_32_type0_avx2_new(*cospi_m32, *cospi_p32, x1[45], x1[50], x2[45], x2[50], + *__rounding, cos_bit); + btf_32_type0_avx2_new(*cospi_m32, *cospi_p32, x1[46], x1[49], x2[46], x2[49], + *__rounding, cos_bit); + btf_32_type0_avx2_new(*cospi_m32, *cospi_p32, x1[47], x1[48], x2[47], x2[48], + *__rounding, cos_bit); + x2[56] = x1[56]; + x2[57] = x1[57]; + x2[58] = x1[58]; + x2[59] = x1[59]; + x2[60] = x1[60]; + x2[61] = x1[61]; + x2[62] = x1[62]; + x2[63] = x1[63]; +} +static INLINE void av1_fdct64_stage3_avx2(__m256i *x2, __m256i *x3, + __m256i *cospi_m32, + __m256i *cospi_p32, + const __m256i *__rounding, + int8_t cos_bit) { + x3[0] = _mm256_add_epi32(x2[0], x2[15]); + x3[15] = _mm256_sub_epi32(x2[0], x2[15]); + x3[1] = _mm256_add_epi32(x2[1], x2[14]); + x3[14] = _mm256_sub_epi32(x2[1], x2[14]); + x3[2] = _mm256_add_epi32(x2[2], x2[13]); + x3[13] = _mm256_sub_epi32(x2[2], x2[13]); + x3[3] = _mm256_add_epi32(x2[3], x2[12]); + x3[12] = _mm256_sub_epi32(x2[3], x2[12]); + x3[4] = _mm256_add_epi32(x2[4], x2[11]); + x3[11] = _mm256_sub_epi32(x2[4], x2[11]); + x3[5] = _mm256_add_epi32(x2[5], x2[10]); + x3[10] = _mm256_sub_epi32(x2[5], x2[10]); + x3[6] = _mm256_add_epi32(x2[6], x2[9]); + x3[9] = _mm256_sub_epi32(x2[6], x2[9]); + x3[7] = _mm256_add_epi32(x2[7], x2[8]); + x3[8] = _mm256_sub_epi32(x2[7], x2[8]); + x3[16] = x2[16]; + x3[17] = x2[17]; + x3[18] = x2[18]; + x3[19] = x2[19]; + btf_32_type0_avx2_new(*cospi_m32, *cospi_p32, x2[20], x2[27], x3[20], x3[27], + *__rounding, cos_bit); + btf_32_type0_avx2_new(*cospi_m32, *cospi_p32, x2[21], x2[26], x3[21], x3[26], + *__rounding, cos_bit); + btf_32_type0_avx2_new(*cospi_m32, *cospi_p32, x2[22], x2[25], x3[22], x3[25], + *__rounding, cos_bit); + btf_32_type0_avx2_new(*cospi_m32, *cospi_p32, x2[23], x2[24], x3[23], x3[24], + *__rounding, cos_bit); + x3[28] = x2[28]; + x3[29] = x2[29]; + x3[30] = x2[30]; + x3[31] = x2[31]; + x3[32] = _mm256_add_epi32(x2[32], x2[47]); + x3[47] = _mm256_sub_epi32(x2[32], x2[47]); + x3[33] = _mm256_add_epi32(x2[33], x2[46]); + x3[46] = _mm256_sub_epi32(x2[33], x2[46]); + x3[34] = _mm256_add_epi32(x2[34], x2[45]); + x3[45] = _mm256_sub_epi32(x2[34], x2[45]); + x3[35] = _mm256_add_epi32(x2[35], x2[44]); + x3[44] = _mm256_sub_epi32(x2[35], x2[44]); + x3[36] = _mm256_add_epi32(x2[36], x2[43]); + x3[43] = _mm256_sub_epi32(x2[36], x2[43]); + x3[37] = _mm256_add_epi32(x2[37], x2[42]); + x3[42] = _mm256_sub_epi32(x2[37], x2[42]); + x3[38] = _mm256_add_epi32(x2[38], x2[41]); + x3[41] = _mm256_sub_epi32(x2[38], x2[41]); + x3[39] = _mm256_add_epi32(x2[39], x2[40]); + x3[40] = _mm256_sub_epi32(x2[39], x2[40]); + x3[48] = _mm256_sub_epi32(x2[63], x2[48]); + x3[63] = _mm256_add_epi32(x2[63], x2[48]); + x3[49] = _mm256_sub_epi32(x2[62], x2[49]); + x3[62] = _mm256_add_epi32(x2[62], x2[49]); + x3[50] = _mm256_sub_epi32(x2[61], x2[50]); + x3[61] = _mm256_add_epi32(x2[61], x2[50]); + x3[51] = _mm256_sub_epi32(x2[60], x2[51]); + x3[60] = _mm256_add_epi32(x2[60], x2[51]); + x3[52] = _mm256_sub_epi32(x2[59], x2[52]); + x3[59] = _mm256_add_epi32(x2[59], x2[52]); + x3[53] = _mm256_sub_epi32(x2[58], x2[53]); + x3[58] = _mm256_add_epi32(x2[58], x2[53]); + x3[54] = _mm256_sub_epi32(x2[57], x2[54]); + x3[57] = _mm256_add_epi32(x2[57], x2[54]); + x3[55] = _mm256_sub_epi32(x2[56], x2[55]); + x3[56] = _mm256_add_epi32(x2[56], x2[55]); +} +static INLINE void av1_fdct64_stage4_avx2( + __m256i *x3, __m256i *x4, __m256i *cospi_m32, __m256i *cospi_p32, + __m256i *cospi_m16, __m256i *cospi_p48, __m256i *cospi_m48, + const __m256i *__rounding, int8_t cos_bit) { + x4[0] = _mm256_add_epi32(x3[0], x3[7]); + x4[7] = _mm256_sub_epi32(x3[0], x3[7]); + x4[1] = _mm256_add_epi32(x3[1], x3[6]); + x4[6] = _mm256_sub_epi32(x3[1], x3[6]); + x4[2] = _mm256_add_epi32(x3[2], x3[5]); + x4[5] = _mm256_sub_epi32(x3[2], x3[5]); + x4[3] = _mm256_add_epi32(x3[3], x3[4]); + x4[4] = _mm256_sub_epi32(x3[3], x3[4]); + x4[8] = x3[8]; + x4[9] = x3[9]; + btf_32_type0_avx2_new(*cospi_m32, *cospi_p32, x3[10], x3[13], x4[10], x4[13], + *__rounding, cos_bit); + btf_32_type0_avx2_new(*cospi_m32, *cospi_p32, x3[11], x3[12], x4[11], x4[12], + *__rounding, cos_bit); + x4[14] = x3[14]; + x4[15] = x3[15]; + x4[16] = _mm256_add_epi32(x3[16], x3[23]); + x4[23] = _mm256_sub_epi32(x3[16], x3[23]); + x4[17] = _mm256_add_epi32(x3[17], x3[22]); + x4[22] = _mm256_sub_epi32(x3[17], x3[22]); + x4[18] = _mm256_add_epi32(x3[18], x3[21]); + x4[21] = _mm256_sub_epi32(x3[18], x3[21]); + x4[19] = _mm256_add_epi32(x3[19], x3[20]); + x4[20] = _mm256_sub_epi32(x3[19], x3[20]); + x4[24] = _mm256_sub_epi32(x3[31], x3[24]); + x4[31] = _mm256_add_epi32(x3[31], x3[24]); + x4[25] = _mm256_sub_epi32(x3[30], x3[25]); + x4[30] = _mm256_add_epi32(x3[30], x3[25]); + x4[26] = _mm256_sub_epi32(x3[29], x3[26]); + x4[29] = _mm256_add_epi32(x3[29], x3[26]); + x4[27] = _mm256_sub_epi32(x3[28], x3[27]); + x4[28] = _mm256_add_epi32(x3[28], x3[27]); + x4[32] = x3[32]; + x4[33] = x3[33]; + x4[34] = x3[34]; + x4[35] = x3[35]; + btf_32_type0_avx2_new(*cospi_m16, *cospi_p48, x3[36], x3[59], x4[36], x4[59], + *__rounding, cos_bit); + btf_32_type0_avx2_new(*cospi_m16, *cospi_p48, x3[37], x3[58], x4[37], x4[58], + *__rounding, cos_bit); + btf_32_type0_avx2_new(*cospi_m16, *cospi_p48, x3[38], x3[57], x4[38], x4[57], + *__rounding, cos_bit); + btf_32_type0_avx2_new(*cospi_m16, *cospi_p48, x3[39], x3[56], x4[39], x4[56], + *__rounding, cos_bit); + btf_32_type0_avx2_new(*cospi_m48, *cospi_m16, x3[40], x3[55], x4[40], x4[55], + *__rounding, cos_bit); + btf_32_type0_avx2_new(*cospi_m48, *cospi_m16, x3[41], x3[54], x4[41], x4[54], + *__rounding, cos_bit); + btf_32_type0_avx2_new(*cospi_m48, *cospi_m16, x3[42], x3[53], x4[42], x4[53], + *__rounding, cos_bit); + btf_32_type0_avx2_new(*cospi_m48, *cospi_m16, x3[43], x3[52], x4[43], x4[52], + *__rounding, cos_bit); + x4[44] = x3[44]; + x4[45] = x3[45]; + x4[46] = x3[46]; + x4[47] = x3[47]; + x4[48] = x3[48]; + x4[49] = x3[49]; + x4[50] = x3[50]; + x4[51] = x3[51]; + x4[60] = x3[60]; + x4[61] = x3[61]; + x4[62] = x3[62]; + x4[63] = x3[63]; +} +static INLINE void av1_fdct64_stage5_avx2( + __m256i *x4, __m256i *x5, __m256i *cospi_m32, __m256i *cospi_p32, + __m256i *cospi_m16, __m256i *cospi_p48, __m256i *cospi_m48, + const __m256i *__rounding, int8_t cos_bit) { + x5[0] = _mm256_add_epi32(x4[0], x4[3]); + x5[3] = _mm256_sub_epi32(x4[0], x4[3]); + x5[1] = _mm256_add_epi32(x4[1], x4[2]); + x5[2] = _mm256_sub_epi32(x4[1], x4[2]); + x5[4] = x4[4]; + btf_32_type0_avx2_new(*cospi_m32, *cospi_p32, x4[5], x4[6], x5[5], x5[6], + *__rounding, cos_bit); + x5[7] = x4[7]; + x5[8] = _mm256_add_epi32(x4[8], x4[11]); + x5[11] = _mm256_sub_epi32(x4[8], x4[11]); + x5[9] = _mm256_add_epi32(x4[9], x4[10]); + x5[10] = _mm256_sub_epi32(x4[9], x4[10]); + x5[12] = _mm256_sub_epi32(x4[15], x4[12]); + x5[15] = _mm256_add_epi32(x4[15], x4[12]); + x5[13] = _mm256_sub_epi32(x4[14], x4[13]); + x5[14] = _mm256_add_epi32(x4[14], x4[13]); + x5[16] = x4[16]; + x5[17] = x4[17]; + btf_32_type0_avx2_new(*cospi_m16, *cospi_p48, x4[18], x4[29], x5[18], x5[29], + *__rounding, cos_bit); + btf_32_type0_avx2_new(*cospi_m16, *cospi_p48, x4[19], x4[28], x5[19], x5[28], + *__rounding, cos_bit); + btf_32_type0_avx2_new(*cospi_m48, *cospi_m16, x4[20], x4[27], x5[20], x5[27], + *__rounding, cos_bit); + btf_32_type0_avx2_new(*cospi_m48, *cospi_m16, x4[21], x4[26], x5[21], x5[26], + *__rounding, cos_bit); + x5[22] = x4[22]; + x5[23] = x4[23]; + x5[24] = x4[24]; + x5[25] = x4[25]; + x5[30] = x4[30]; + x5[31] = x4[31]; + x5[32] = _mm256_add_epi32(x4[32], x4[39]); + x5[39] = _mm256_sub_epi32(x4[32], x4[39]); + x5[33] = _mm256_add_epi32(x4[33], x4[38]); + x5[38] = _mm256_sub_epi32(x4[33], x4[38]); + x5[34] = _mm256_add_epi32(x4[34], x4[37]); + x5[37] = _mm256_sub_epi32(x4[34], x4[37]); + x5[35] = _mm256_add_epi32(x4[35], x4[36]); + x5[36] = _mm256_sub_epi32(x4[35], x4[36]); + x5[40] = _mm256_sub_epi32(x4[47], x4[40]); + x5[47] = _mm256_add_epi32(x4[47], x4[40]); + x5[41] = _mm256_sub_epi32(x4[46], x4[41]); + x5[46] = _mm256_add_epi32(x4[46], x4[41]); + x5[42] = _mm256_sub_epi32(x4[45], x4[42]); + x5[45] = _mm256_add_epi32(x4[45], x4[42]); + x5[43] = _mm256_sub_epi32(x4[44], x4[43]); + x5[44] = _mm256_add_epi32(x4[44], x4[43]); + x5[48] = _mm256_add_epi32(x4[48], x4[55]); + x5[55] = _mm256_sub_epi32(x4[48], x4[55]); + x5[49] = _mm256_add_epi32(x4[49], x4[54]); + x5[54] = _mm256_sub_epi32(x4[49], x4[54]); + x5[50] = _mm256_add_epi32(x4[50], x4[53]); + x5[53] = _mm256_sub_epi32(x4[50], x4[53]); + x5[51] = _mm256_add_epi32(x4[51], x4[52]); + x5[52] = _mm256_sub_epi32(x4[51], x4[52]); + x5[56] = _mm256_sub_epi32(x4[63], x4[56]); + x5[63] = _mm256_add_epi32(x4[63], x4[56]); + x5[57] = _mm256_sub_epi32(x4[62], x4[57]); + x5[62] = _mm256_add_epi32(x4[62], x4[57]); + x5[58] = _mm256_sub_epi32(x4[61], x4[58]); + x5[61] = _mm256_add_epi32(x4[61], x4[58]); + x5[59] = _mm256_sub_epi32(x4[60], x4[59]); + x5[60] = _mm256_add_epi32(x4[60], x4[59]); +} +static INLINE void av1_fdct64_stage6_avx2( + __m256i *x5, __m256i *x6, __m256i *cospi_p16, __m256i *cospi_p32, + __m256i *cospi_m16, __m256i *cospi_p48, __m256i *cospi_m48, + __m256i *cospi_m08, __m256i *cospi_p56, __m256i *cospi_m56, + __m256i *cospi_m40, __m256i *cospi_p24, __m256i *cospi_m24, + const __m256i *__rounding, int8_t cos_bit) { + btf_32_type0_avx2_new(*cospi_p32, *cospi_p32, x5[0], x5[1], x6[0], x6[1], + *__rounding, cos_bit); + btf_32_type0_avx2_new(*cospi_p16, *cospi_p48, x5[3], x5[2], x6[2], x6[3], + *__rounding, cos_bit); + x6[4] = _mm256_add_epi32(x5[4], x5[5]); + x6[5] = _mm256_sub_epi32(x5[4], x5[5]); + x6[6] = _mm256_sub_epi32(x5[7], x5[6]); + x6[7] = _mm256_add_epi32(x5[7], x5[6]); + x6[8] = x5[8]; + btf_32_type0_avx2_new(*cospi_m16, *cospi_p48, x5[9], x5[14], x6[9], x6[14], + *__rounding, cos_bit); + btf_32_type0_avx2_new(*cospi_m48, *cospi_m16, x5[10], x5[13], x6[10], x6[13], + *__rounding, cos_bit); + x6[11] = x5[11]; + x6[12] = x5[12]; + x6[15] = x5[15]; + x6[16] = _mm256_add_epi32(x5[16], x5[19]); + x6[19] = _mm256_sub_epi32(x5[16], x5[19]); + x6[17] = _mm256_add_epi32(x5[17], x5[18]); + x6[18] = _mm256_sub_epi32(x5[17], x5[18]); + x6[20] = _mm256_sub_epi32(x5[23], x5[20]); + x6[23] = _mm256_add_epi32(x5[23], x5[20]); + x6[21] = _mm256_sub_epi32(x5[22], x5[21]); + x6[22] = _mm256_add_epi32(x5[22], x5[21]); + x6[24] = _mm256_add_epi32(x5[24], x5[27]); + x6[27] = _mm256_sub_epi32(x5[24], x5[27]); + x6[25] = _mm256_add_epi32(x5[25], x5[26]); + x6[26] = _mm256_sub_epi32(x5[25], x5[26]); + x6[28] = _mm256_sub_epi32(x5[31], x5[28]); + x6[31] = _mm256_add_epi32(x5[31], x5[28]); + x6[29] = _mm256_sub_epi32(x5[30], x5[29]); + x6[30] = _mm256_add_epi32(x5[30], x5[29]); + x6[32] = x5[32]; + x6[33] = x5[33]; + btf_32_type0_avx2_new(*cospi_m08, *cospi_p56, x5[34], x5[61], x6[34], x6[61], + *__rounding, cos_bit); + btf_32_type0_avx2_new(*cospi_m08, *cospi_p56, x5[35], x5[60], x6[35], x6[60], + *__rounding, cos_bit); + btf_32_type0_avx2_new(*cospi_m56, *cospi_m08, x5[36], x5[59], x6[36], x6[59], + *__rounding, cos_bit); + btf_32_type0_avx2_new(*cospi_m56, *cospi_m08, x5[37], x5[58], x6[37], x6[58], + *__rounding, cos_bit); + x6[38] = x5[38]; + x6[39] = x5[39]; + x6[40] = x5[40]; + x6[41] = x5[41]; + btf_32_type0_avx2_new(*cospi_m40, *cospi_p24, x5[42], x5[53], x6[42], x6[53], + *__rounding, cos_bit); + btf_32_type0_avx2_new(*cospi_m40, *cospi_p24, x5[43], x5[52], x6[43], x6[52], + *__rounding, cos_bit); + btf_32_type0_avx2_new(*cospi_m24, *cospi_m40, x5[44], x5[51], x6[44], x6[51], + *__rounding, cos_bit); + btf_32_type0_avx2_new(*cospi_m24, *cospi_m40, x5[45], x5[50], x6[45], x6[50], + *__rounding, cos_bit); + x6[46] = x5[46]; + x6[47] = x5[47]; + x6[48] = x5[48]; + x6[49] = x5[49]; + x6[54] = x5[54]; + x6[55] = x5[55]; + x6[56] = x5[56]; + x6[57] = x5[57]; + x6[62] = x5[62]; + x6[63] = x5[63]; +} +static INLINE void av1_fdct64_stage7_avx2( + __m256i *x6, __m256i *x7, __m256i *cospi_p08, __m256i *cospi_p56, + __m256i *cospi_p40, __m256i *cospi_p24, __m256i *cospi_m08, + __m256i *cospi_m56, __m256i *cospi_m40, __m256i *cospi_m24, + const __m256i *__rounding, int8_t cos_bit) { + x7[0] = x6[0]; + x7[1] = x6[1]; + x7[2] = x6[2]; + x7[3] = x6[3]; + btf_32_type0_avx2_new(*cospi_p08, *cospi_p56, x6[7], x6[4], x7[4], x7[7], + *__rounding, cos_bit); + btf_32_type0_avx2_new(*cospi_p40, *cospi_p24, x6[6], x6[5], x7[5], x7[6], + *__rounding, cos_bit); + x7[8] = _mm256_add_epi32(x6[8], x6[9]); + x7[9] = _mm256_sub_epi32(x6[8], x6[9]); + x7[10] = _mm256_sub_epi32(x6[11], x6[10]); + x7[11] = _mm256_add_epi32(x6[11], x6[10]); + x7[12] = _mm256_add_epi32(x6[12], x6[13]); + x7[13] = _mm256_sub_epi32(x6[12], x6[13]); + x7[14] = _mm256_sub_epi32(x6[15], x6[14]); + x7[15] = _mm256_add_epi32(x6[15], x6[14]); + x7[16] = x6[16]; + btf_32_type0_avx2_new(*cospi_m08, *cospi_p56, x6[17], x6[30], x7[17], x7[30], + *__rounding, cos_bit); + btf_32_type0_avx2_new(*cospi_m56, *cospi_m08, x6[18], x6[29], x7[18], x7[29], + *__rounding, cos_bit); + x7[19] = x6[19]; + x7[20] = x6[20]; + btf_32_type0_avx2_new(*cospi_m40, *cospi_p24, x6[21], x6[26], x7[21], x7[26], + *__rounding, cos_bit); + btf_32_type0_avx2_new(*cospi_m24, *cospi_m40, x6[22], x6[25], x7[22], x7[25], + *__rounding, cos_bit); + x7[23] = x6[23]; + x7[24] = x6[24]; + x7[27] = x6[27]; + x7[28] = x6[28]; + x7[31] = x6[31]; + x7[32] = _mm256_add_epi32(x6[32], x6[35]); + x7[35] = _mm256_sub_epi32(x6[32], x6[35]); + x7[33] = _mm256_add_epi32(x6[33], x6[34]); + x7[34] = _mm256_sub_epi32(x6[33], x6[34]); + x7[36] = _mm256_sub_epi32(x6[39], x6[36]); + x7[39] = _mm256_add_epi32(x6[39], x6[36]); + x7[37] = _mm256_sub_epi32(x6[38], x6[37]); + x7[38] = _mm256_add_epi32(x6[38], x6[37]); + x7[40] = _mm256_add_epi32(x6[40], x6[43]); + x7[43] = _mm256_sub_epi32(x6[40], x6[43]); + x7[41] = _mm256_add_epi32(x6[41], x6[42]); + x7[42] = _mm256_sub_epi32(x6[41], x6[42]); + x7[44] = _mm256_sub_epi32(x6[47], x6[44]); + x7[47] = _mm256_add_epi32(x6[47], x6[44]); + x7[45] = _mm256_sub_epi32(x6[46], x6[45]); + x7[46] = _mm256_add_epi32(x6[46], x6[45]); + x7[48] = _mm256_add_epi32(x6[48], x6[51]); + x7[51] = _mm256_sub_epi32(x6[48], x6[51]); + x7[49] = _mm256_add_epi32(x6[49], x6[50]); + x7[50] = _mm256_sub_epi32(x6[49], x6[50]); + x7[52] = _mm256_sub_epi32(x6[55], x6[52]); + x7[55] = _mm256_add_epi32(x6[55], x6[52]); + x7[53] = _mm256_sub_epi32(x6[54], x6[53]); + x7[54] = _mm256_add_epi32(x6[54], x6[53]); + x7[56] = _mm256_add_epi32(x6[56], x6[59]); + x7[59] = _mm256_sub_epi32(x6[56], x6[59]); + x7[57] = _mm256_add_epi32(x6[57], x6[58]); + x7[58] = _mm256_sub_epi32(x6[57], x6[58]); + x7[60] = _mm256_sub_epi32(x6[63], x6[60]); + x7[63] = _mm256_add_epi32(x6[63], x6[60]); + x7[61] = _mm256_sub_epi32(x6[62], x6[61]); + x7[62] = _mm256_add_epi32(x6[62], x6[61]); +} +static INLINE void av1_fdct64_stage8_avx2(__m256i *x7, __m256i *x8, + const int32_t *cospi, + const __m256i *__rounding, + int8_t cos_bit) { + __m256i cospi_p60 = _mm256_set1_epi32(cospi[60]); + __m256i cospi_p04 = _mm256_set1_epi32(cospi[4]); + __m256i cospi_p28 = _mm256_set1_epi32(cospi[28]); + __m256i cospi_p36 = _mm256_set1_epi32(cospi[36]); + __m256i cospi_p44 = _mm256_set1_epi32(cospi[44]); + __m256i cospi_p20 = _mm256_set1_epi32(cospi[20]); + __m256i cospi_p12 = _mm256_set1_epi32(cospi[12]); + __m256i cospi_p52 = _mm256_set1_epi32(cospi[52]); + __m256i cospi_m04 = _mm256_set1_epi32(-cospi[4]); + __m256i cospi_m60 = _mm256_set1_epi32(-cospi[60]); + __m256i cospi_m36 = _mm256_set1_epi32(-cospi[36]); + __m256i cospi_m28 = _mm256_set1_epi32(-cospi[28]); + __m256i cospi_m20 = _mm256_set1_epi32(-cospi[20]); + __m256i cospi_m44 = _mm256_set1_epi32(-cospi[44]); + __m256i cospi_m52 = _mm256_set1_epi32(-cospi[52]); + __m256i cospi_m12 = _mm256_set1_epi32(-cospi[12]); + + x8[0] = x7[0]; + x8[1] = x7[1]; + x8[2] = x7[2]; + x8[3] = x7[3]; + x8[4] = x7[4]; + x8[5] = x7[5]; + x8[6] = x7[6]; + x8[7] = x7[7]; + + btf_32_type0_avx2_new(cospi_p04, cospi_p60, x7[15], x7[8], x8[8], x8[15], + *__rounding, cos_bit); + btf_32_type0_avx2_new(cospi_p36, cospi_p28, x7[14], x7[9], x8[9], x8[14], + *__rounding, cos_bit); + btf_32_type0_avx2_new(cospi_p20, cospi_p44, x7[13], x7[10], x8[10], x8[13], + *__rounding, cos_bit); + btf_32_type0_avx2_new(cospi_p52, cospi_p12, x7[12], x7[11], x8[11], x8[12], + *__rounding, cos_bit); + x8[16] = _mm256_add_epi32(x7[16], x7[17]); + x8[17] = _mm256_sub_epi32(x7[16], x7[17]); + x8[18] = _mm256_sub_epi32(x7[19], x7[18]); + x8[19] = _mm256_add_epi32(x7[19], x7[18]); + x8[20] = _mm256_add_epi32(x7[20], x7[21]); + x8[21] = _mm256_sub_epi32(x7[20], x7[21]); + x8[22] = _mm256_sub_epi32(x7[23], x7[22]); + x8[23] = _mm256_add_epi32(x7[23], x7[22]); + x8[24] = _mm256_add_epi32(x7[24], x7[25]); + x8[25] = _mm256_sub_epi32(x7[24], x7[25]); + x8[26] = _mm256_sub_epi32(x7[27], x7[26]); + x8[27] = _mm256_add_epi32(x7[27], x7[26]); + x8[28] = _mm256_add_epi32(x7[28], x7[29]); + x8[29] = _mm256_sub_epi32(x7[28], x7[29]); + x8[30] = _mm256_sub_epi32(x7[31], x7[30]); + x8[31] = _mm256_add_epi32(x7[31], x7[30]); + x8[32] = x7[32]; + btf_32_type0_avx2_new(cospi_m04, cospi_p60, x7[33], x7[62], x8[33], x8[62], + *__rounding, cos_bit); + btf_32_type0_avx2_new(cospi_m60, cospi_m04, x7[34], x7[61], x8[34], x8[61], + *__rounding, cos_bit); + x8[35] = x7[35]; + x8[36] = x7[36]; + btf_32_type0_avx2_new(cospi_m36, cospi_p28, x7[37], x7[58], x8[37], x8[58], + *__rounding, cos_bit); + btf_32_type0_avx2_new(cospi_m28, cospi_m36, x7[38], x7[57], x8[38], x8[57], + *__rounding, cos_bit); + x8[39] = x7[39]; + x8[40] = x7[40]; + btf_32_type0_avx2_new(cospi_m20, cospi_p44, x7[41], x7[54], x8[41], x8[54], + *__rounding, cos_bit); + btf_32_type0_avx2_new(cospi_m44, cospi_m20, x7[42], x7[53], x8[42], x8[53], + *__rounding, cos_bit); + x8[43] = x7[43]; + x8[44] = x7[44]; + btf_32_type0_avx2_new(cospi_m52, cospi_p12, x7[45], x7[50], x8[45], x8[50], + *__rounding, cos_bit); + btf_32_type0_avx2_new(cospi_m12, cospi_m52, x7[46], x7[49], x8[46], x8[49], + *__rounding, cos_bit); + x8[47] = x7[47]; + x8[48] = x7[48]; + x8[51] = x7[51]; + x8[52] = x7[52]; + x8[55] = x7[55]; + x8[56] = x7[56]; + x8[59] = x7[59]; + x8[60] = x7[60]; + x8[63] = x7[63]; +} +static INLINE void av1_fdct64_stage9_avx2(__m256i *x8, __m256i *x9, + const int32_t *cospi, + const __m256i *__rounding, + int8_t cos_bit) { + __m256i cospi_p62 = _mm256_set1_epi32(cospi[62]); + __m256i cospi_p02 = _mm256_set1_epi32(cospi[2]); + __m256i cospi_p30 = _mm256_set1_epi32(cospi[30]); + __m256i cospi_p34 = _mm256_set1_epi32(cospi[34]); + __m256i cospi_p46 = _mm256_set1_epi32(cospi[46]); + __m256i cospi_p18 = _mm256_set1_epi32(cospi[18]); + __m256i cospi_p14 = _mm256_set1_epi32(cospi[14]); + __m256i cospi_p50 = _mm256_set1_epi32(cospi[50]); + __m256i cospi_p54 = _mm256_set1_epi32(cospi[54]); + __m256i cospi_p10 = _mm256_set1_epi32(cospi[10]); + __m256i cospi_p22 = _mm256_set1_epi32(cospi[22]); + __m256i cospi_p42 = _mm256_set1_epi32(cospi[42]); + __m256i cospi_p38 = _mm256_set1_epi32(cospi[38]); + __m256i cospi_p26 = _mm256_set1_epi32(cospi[26]); + __m256i cospi_p06 = _mm256_set1_epi32(cospi[6]); + __m256i cospi_p58 = _mm256_set1_epi32(cospi[58]); + + x9[0] = x8[0]; + x9[1] = x8[1]; + x9[2] = x8[2]; + x9[3] = x8[3]; + x9[4] = x8[4]; + x9[5] = x8[5]; + x9[6] = x8[6]; + x9[7] = x8[7]; + x9[8] = x8[8]; + x9[9] = x8[9]; + x9[10] = x8[10]; + x9[11] = x8[11]; + x9[12] = x8[12]; + x9[13] = x8[13]; + x9[14] = x8[14]; + x9[15] = x8[15]; + btf_32_type0_avx2_new(cospi_p02, cospi_p62, x8[31], x8[16], x9[16], x9[31], + *__rounding, cos_bit); + btf_32_type0_avx2_new(cospi_p34, cospi_p30, x8[30], x8[17], x9[17], x9[30], + *__rounding, cos_bit); + btf_32_type0_avx2_new(cospi_p18, cospi_p46, x8[29], x8[18], x9[18], x9[29], + *__rounding, cos_bit); + btf_32_type0_avx2_new(cospi_p50, cospi_p14, x8[28], x8[19], x9[19], x9[28], + *__rounding, cos_bit); + btf_32_type0_avx2_new(cospi_p10, cospi_p54, x8[27], x8[20], x9[20], x9[27], + *__rounding, cos_bit); + btf_32_type0_avx2_new(cospi_p42, cospi_p22, x8[26], x8[21], x9[21], x9[26], + *__rounding, cos_bit); + btf_32_type0_avx2_new(cospi_p26, cospi_p38, x8[25], x8[22], x9[22], x9[25], + *__rounding, cos_bit); + btf_32_type0_avx2_new(cospi_p58, cospi_p06, x8[24], x8[23], x9[23], x9[24], + *__rounding, cos_bit); + x9[32] = _mm256_add_epi32(x8[32], x8[33]); + x9[33] = _mm256_sub_epi32(x8[32], x8[33]); + x9[34] = _mm256_sub_epi32(x8[35], x8[34]); + x9[35] = _mm256_add_epi32(x8[35], x8[34]); + x9[36] = _mm256_add_epi32(x8[36], x8[37]); + x9[37] = _mm256_sub_epi32(x8[36], x8[37]); + x9[38] = _mm256_sub_epi32(x8[39], x8[38]); + x9[39] = _mm256_add_epi32(x8[39], x8[38]); + x9[40] = _mm256_add_epi32(x8[40], x8[41]); + x9[41] = _mm256_sub_epi32(x8[40], x8[41]); + x9[42] = _mm256_sub_epi32(x8[43], x8[42]); + x9[43] = _mm256_add_epi32(x8[43], x8[42]); + x9[44] = _mm256_add_epi32(x8[44], x8[45]); + x9[45] = _mm256_sub_epi32(x8[44], x8[45]); + x9[46] = _mm256_sub_epi32(x8[47], x8[46]); + x9[47] = _mm256_add_epi32(x8[47], x8[46]); + x9[48] = _mm256_add_epi32(x8[48], x8[49]); + x9[49] = _mm256_sub_epi32(x8[48], x8[49]); + x9[50] = _mm256_sub_epi32(x8[51], x8[50]); + x9[51] = _mm256_add_epi32(x8[51], x8[50]); + x9[52] = _mm256_add_epi32(x8[52], x8[53]); + x9[53] = _mm256_sub_epi32(x8[52], x8[53]); + x9[54] = _mm256_sub_epi32(x8[55], x8[54]); + x9[55] = _mm256_add_epi32(x8[55], x8[54]); + x9[56] = _mm256_add_epi32(x8[56], x8[57]); + x9[57] = _mm256_sub_epi32(x8[56], x8[57]); + x9[58] = _mm256_sub_epi32(x8[59], x8[58]); + x9[59] = _mm256_add_epi32(x8[59], x8[58]); + x9[60] = _mm256_add_epi32(x8[60], x8[61]); + x9[61] = _mm256_sub_epi32(x8[60], x8[61]); + x9[62] = _mm256_sub_epi32(x8[63], x8[62]); + x9[63] = _mm256_add_epi32(x8[63], x8[62]); +} +static INLINE void av1_fdct64_stage10_avx2(__m256i *x9, __m256i *x10, + const int32_t *cospi, + const __m256i *__rounding, + int8_t cos_bit) { + __m256i cospi_p63 = _mm256_set1_epi32(cospi[63]); + __m256i cospi_p01 = _mm256_set1_epi32(cospi[1]); + __m256i cospi_p31 = _mm256_set1_epi32(cospi[31]); + __m256i cospi_p33 = _mm256_set1_epi32(cospi[33]); + __m256i cospi_p47 = _mm256_set1_epi32(cospi[47]); + __m256i cospi_p17 = _mm256_set1_epi32(cospi[17]); + __m256i cospi_p15 = _mm256_set1_epi32(cospi[15]); + __m256i cospi_p49 = _mm256_set1_epi32(cospi[49]); + __m256i cospi_p55 = _mm256_set1_epi32(cospi[55]); + __m256i cospi_p09 = _mm256_set1_epi32(cospi[9]); + __m256i cospi_p23 = _mm256_set1_epi32(cospi[23]); + __m256i cospi_p41 = _mm256_set1_epi32(cospi[41]); + __m256i cospi_p39 = _mm256_set1_epi32(cospi[39]); + __m256i cospi_p25 = _mm256_set1_epi32(cospi[25]); + __m256i cospi_p07 = _mm256_set1_epi32(cospi[7]); + __m256i cospi_p57 = _mm256_set1_epi32(cospi[57]); + __m256i cospi_p59 = _mm256_set1_epi32(cospi[59]); + __m256i cospi_p05 = _mm256_set1_epi32(cospi[5]); + __m256i cospi_p27 = _mm256_set1_epi32(cospi[27]); + __m256i cospi_p37 = _mm256_set1_epi32(cospi[37]); + __m256i cospi_p43 = _mm256_set1_epi32(cospi[43]); + __m256i cospi_p21 = _mm256_set1_epi32(cospi[21]); + __m256i cospi_p11 = _mm256_set1_epi32(cospi[11]); + __m256i cospi_p53 = _mm256_set1_epi32(cospi[53]); + __m256i cospi_p51 = _mm256_set1_epi32(cospi[51]); + __m256i cospi_p13 = _mm256_set1_epi32(cospi[13]); + __m256i cospi_p19 = _mm256_set1_epi32(cospi[19]); + __m256i cospi_p45 = _mm256_set1_epi32(cospi[45]); + __m256i cospi_p35 = _mm256_set1_epi32(cospi[35]); + __m256i cospi_p29 = _mm256_set1_epi32(cospi[29]); + __m256i cospi_p03 = _mm256_set1_epi32(cospi[3]); + __m256i cospi_p61 = _mm256_set1_epi32(cospi[61]); + + x10[0] = x9[0]; + x10[1] = x9[1]; + x10[2] = x9[2]; + x10[3] = x9[3]; + x10[4] = x9[4]; + x10[5] = x9[5]; + x10[6] = x9[6]; + x10[7] = x9[7]; + x10[8] = x9[8]; + x10[9] = x9[9]; + x10[10] = x9[10]; + x10[11] = x9[11]; + x10[12] = x9[12]; + x10[13] = x9[13]; + x10[14] = x9[14]; + x10[15] = x9[15]; + x10[16] = x9[16]; + x10[17] = x9[17]; + x10[18] = x9[18]; + x10[19] = x9[19]; + x10[20] = x9[20]; + x10[21] = x9[21]; + x10[22] = x9[22]; + x10[23] = x9[23]; + x10[24] = x9[24]; + x10[25] = x9[25]; + x10[26] = x9[26]; + x10[27] = x9[27]; + x10[28] = x9[28]; + x10[29] = x9[29]; + x10[30] = x9[30]; + x10[31] = x9[31]; + btf_32_type0_avx2_new(cospi_p01, cospi_p63, x9[63], x9[32], x10[32], x10[63], + *__rounding, cos_bit); + btf_32_type0_avx2_new(cospi_p33, cospi_p31, x9[62], x9[33], x10[33], x10[62], + *__rounding, cos_bit); + btf_32_type0_avx2_new(cospi_p17, cospi_p47, x9[61], x9[34], x10[34], x10[61], + *__rounding, cos_bit); + btf_32_type0_avx2_new(cospi_p49, cospi_p15, x9[60], x9[35], x10[35], x10[60], + *__rounding, cos_bit); + btf_32_type0_avx2_new(cospi_p09, cospi_p55, x9[59], x9[36], x10[36], x10[59], + *__rounding, cos_bit); + btf_32_type0_avx2_new(cospi_p41, cospi_p23, x9[58], x9[37], x10[37], x10[58], + *__rounding, cos_bit); + btf_32_type0_avx2_new(cospi_p25, cospi_p39, x9[57], x9[38], x10[38], x10[57], + *__rounding, cos_bit); + btf_32_type0_avx2_new(cospi_p57, cospi_p07, x9[56], x9[39], x10[39], x10[56], + *__rounding, cos_bit); + btf_32_type0_avx2_new(cospi_p05, cospi_p59, x9[55], x9[40], x10[40], x10[55], + *__rounding, cos_bit); + btf_32_type0_avx2_new(cospi_p37, cospi_p27, x9[54], x9[41], x10[41], x10[54], + *__rounding, cos_bit); + btf_32_type0_avx2_new(cospi_p21, cospi_p43, x9[53], x9[42], x10[42], x10[53], + *__rounding, cos_bit); + btf_32_type0_avx2_new(cospi_p53, cospi_p11, x9[52], x9[43], x10[43], x10[52], + *__rounding, cos_bit); + btf_32_type0_avx2_new(cospi_p13, cospi_p51, x9[51], x9[44], x10[44], x10[51], + *__rounding, cos_bit); + btf_32_type0_avx2_new(cospi_p45, cospi_p19, x9[50], x9[45], x10[45], x10[50], + *__rounding, cos_bit); + btf_32_type0_avx2_new(cospi_p29, cospi_p35, x9[49], x9[46], x10[46], x10[49], + *__rounding, cos_bit); + btf_32_type0_avx2_new(cospi_p61, cospi_p03, x9[48], x9[47], x10[47], x10[48], + *__rounding, cos_bit); +} +static void av1_fdct64_avx2(__m256i *input, __m256i *output, int8_t cos_bit, + const int instride, const int outstride) { + const int32_t *cospi = cospi_arr(cos_bit); + const __m256i __rounding = _mm256_set1_epi32(1 << (cos_bit - 1)); + __m256i cospi_m32 = _mm256_set1_epi32(-cospi[32]); + __m256i cospi_p32 = _mm256_set1_epi32(cospi[32]); + __m256i cospi_m16 = _mm256_set1_epi32(-cospi[16]); + __m256i cospi_p48 = _mm256_set1_epi32(cospi[48]); + __m256i cospi_m48 = _mm256_set1_epi32(-cospi[48]); + __m256i cospi_p16 = _mm256_set1_epi32(cospi[16]); + __m256i cospi_m08 = _mm256_set1_epi32(-cospi[8]); + __m256i cospi_p56 = _mm256_set1_epi32(cospi[56]); + __m256i cospi_m56 = _mm256_set1_epi32(-cospi[56]); + __m256i cospi_m40 = _mm256_set1_epi32(-cospi[40]); + __m256i cospi_p24 = _mm256_set1_epi32(cospi[24]); + __m256i cospi_m24 = _mm256_set1_epi32(-cospi[24]); + __m256i cospi_p08 = _mm256_set1_epi32(cospi[8]); + __m256i cospi_p40 = _mm256_set1_epi32(cospi[40]); + + int startidx = 0 * instride; + int endidx = 63 * instride; + // stage 1 + __m256i x1[64]; + x1[0] = _mm256_add_epi32(input[startidx], input[endidx]); + x1[63] = _mm256_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + x1[1] = _mm256_add_epi32(input[startidx], input[endidx]); + x1[62] = _mm256_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + x1[2] = _mm256_add_epi32(input[startidx], input[endidx]); + x1[61] = _mm256_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + x1[3] = _mm256_add_epi32(input[startidx], input[endidx]); + x1[60] = _mm256_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + x1[4] = _mm256_add_epi32(input[startidx], input[endidx]); + x1[59] = _mm256_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + x1[5] = _mm256_add_epi32(input[startidx], input[endidx]); + x1[58] = _mm256_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + x1[6] = _mm256_add_epi32(input[startidx], input[endidx]); + x1[57] = _mm256_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + x1[7] = _mm256_add_epi32(input[startidx], input[endidx]); + x1[56] = _mm256_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + x1[8] = _mm256_add_epi32(input[startidx], input[endidx]); + x1[55] = _mm256_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + x1[9] = _mm256_add_epi32(input[startidx], input[endidx]); + x1[54] = _mm256_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + x1[10] = _mm256_add_epi32(input[startidx], input[endidx]); + x1[53] = _mm256_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + x1[11] = _mm256_add_epi32(input[startidx], input[endidx]); + x1[52] = _mm256_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + x1[12] = _mm256_add_epi32(input[startidx], input[endidx]); + x1[51] = _mm256_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + x1[13] = _mm256_add_epi32(input[startidx], input[endidx]); + x1[50] = _mm256_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + x1[14] = _mm256_add_epi32(input[startidx], input[endidx]); + x1[49] = _mm256_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + x1[15] = _mm256_add_epi32(input[startidx], input[endidx]); + x1[48] = _mm256_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + x1[16] = _mm256_add_epi32(input[startidx], input[endidx]); + x1[47] = _mm256_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + x1[17] = _mm256_add_epi32(input[startidx], input[endidx]); + x1[46] = _mm256_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + x1[18] = _mm256_add_epi32(input[startidx], input[endidx]); + x1[45] = _mm256_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + x1[19] = _mm256_add_epi32(input[startidx], input[endidx]); + x1[44] = _mm256_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + x1[20] = _mm256_add_epi32(input[startidx], input[endidx]); + x1[43] = _mm256_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + x1[21] = _mm256_add_epi32(input[startidx], input[endidx]); + x1[42] = _mm256_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + x1[22] = _mm256_add_epi32(input[startidx], input[endidx]); + x1[41] = _mm256_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + x1[23] = _mm256_add_epi32(input[startidx], input[endidx]); + x1[40] = _mm256_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + x1[24] = _mm256_add_epi32(input[startidx], input[endidx]); + x1[39] = _mm256_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + x1[25] = _mm256_add_epi32(input[startidx], input[endidx]); + x1[38] = _mm256_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + x1[26] = _mm256_add_epi32(input[startidx], input[endidx]); + x1[37] = _mm256_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + x1[27] = _mm256_add_epi32(input[startidx], input[endidx]); + x1[36] = _mm256_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + x1[28] = _mm256_add_epi32(input[startidx], input[endidx]); + x1[35] = _mm256_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + x1[29] = _mm256_add_epi32(input[startidx], input[endidx]); + x1[34] = _mm256_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + x1[30] = _mm256_add_epi32(input[startidx], input[endidx]); + x1[33] = _mm256_sub_epi32(input[startidx], input[endidx]); + startidx += instride; + endidx -= instride; + x1[31] = _mm256_add_epi32(input[startidx], input[endidx]); + x1[32] = _mm256_sub_epi32(input[startidx], input[endidx]); + + // stage 2 + __m256i x2[64]; + av1_fdct64_stage2_avx2(x1, x2, &cospi_m32, &cospi_p32, &__rounding, cos_bit); + // stage 3 + av1_fdct64_stage3_avx2(x2, x1, &cospi_m32, &cospi_p32, &__rounding, cos_bit); + // stage 4 + av1_fdct64_stage4_avx2(x1, x2, &cospi_m32, &cospi_p32, &cospi_m16, &cospi_p48, + &cospi_m48, &__rounding, cos_bit); + // stage 5 + av1_fdct64_stage5_avx2(x2, x1, &cospi_m32, &cospi_p32, &cospi_m16, &cospi_p48, + &cospi_m48, &__rounding, cos_bit); + // stage 6 + av1_fdct64_stage6_avx2(x1, x2, &cospi_p16, &cospi_p32, &cospi_m16, &cospi_p48, + &cospi_m48, &cospi_m08, &cospi_p56, &cospi_m56, + &cospi_m40, &cospi_p24, &cospi_m24, &__rounding, + cos_bit); + // stage 7 + av1_fdct64_stage7_avx2(x2, x1, &cospi_p08, &cospi_p56, &cospi_p40, &cospi_p24, + &cospi_m08, &cospi_m56, &cospi_m40, &cospi_m24, + &__rounding, cos_bit); + // stage 8 + av1_fdct64_stage8_avx2(x1, x2, cospi, &__rounding, cos_bit); + // stage 9 + av1_fdct64_stage9_avx2(x2, x1, cospi, &__rounding, cos_bit); + // stage 10 + av1_fdct64_stage10_avx2(x1, x2, cospi, &__rounding, cos_bit); + + startidx = 0 * outstride; + endidx = 63 * outstride; + + // stage 11 + output[startidx] = x2[0]; + output[endidx] = x2[63]; + startidx += outstride; + endidx -= outstride; + output[startidx] = x2[32]; + output[endidx] = x2[31]; + startidx += outstride; + endidx -= outstride; + output[startidx] = x2[16]; + output[endidx] = x2[47]; + startidx += outstride; + endidx -= outstride; + output[startidx] = x2[48]; + output[endidx] = x2[15]; + startidx += outstride; + endidx -= outstride; + output[startidx] = x2[8]; + output[endidx] = x2[55]; + startidx += outstride; + endidx -= outstride; + output[startidx] = x2[40]; + output[endidx] = x2[23]; + startidx += outstride; + endidx -= outstride; + output[startidx] = x2[24]; + output[endidx] = x2[39]; + startidx += outstride; + endidx -= outstride; + output[startidx] = x2[56]; + output[endidx] = x2[7]; + startidx += outstride; + endidx -= outstride; + output[startidx] = x2[4]; + output[endidx] = x2[59]; + startidx += outstride; + endidx -= outstride; + output[startidx] = x2[36]; + output[endidx] = x2[27]; + startidx += outstride; + endidx -= outstride; + output[startidx] = x2[20]; + output[endidx] = x2[43]; + startidx += outstride; + endidx -= outstride; + output[startidx] = x2[52]; + output[endidx] = x2[11]; + startidx += outstride; + endidx -= outstride; + output[startidx] = x2[12]; + output[endidx] = x2[51]; + startidx += outstride; + endidx -= outstride; + output[startidx] = x2[44]; + output[endidx] = x2[19]; + startidx += outstride; + endidx -= outstride; + output[startidx] = x2[28]; + output[endidx] = x2[35]; + startidx += outstride; + endidx -= outstride; + output[startidx] = x2[60]; + output[endidx] = x2[3]; + startidx += outstride; + endidx -= outstride; + output[startidx] = x2[2]; + output[endidx] = x2[61]; + startidx += outstride; + endidx -= outstride; + output[startidx] = x2[34]; + output[endidx] = x2[29]; + startidx += outstride; + endidx -= outstride; + output[startidx] = x2[18]; + output[endidx] = x2[45]; + startidx += outstride; + endidx -= outstride; + output[startidx] = x2[50]; + output[endidx] = x2[13]; + startidx += outstride; + endidx -= outstride; + output[startidx] = x2[10]; + output[endidx] = x2[53]; + startidx += outstride; + endidx -= outstride; + output[startidx] = x2[42]; + output[endidx] = x2[21]; + startidx += outstride; + endidx -= outstride; + output[startidx] = x2[26]; + output[endidx] = x2[37]; + startidx += outstride; + endidx -= outstride; + output[startidx] = x2[58]; + output[endidx] = x2[5]; + startidx += outstride; + endidx -= outstride; + output[startidx] = x2[6]; + output[endidx] = x2[57]; + startidx += outstride; + endidx -= outstride; + output[startidx] = x2[38]; + output[endidx] = x2[25]; + startidx += outstride; + endidx -= outstride; + output[startidx] = x2[22]; + output[endidx] = x2[41]; + startidx += outstride; + endidx -= outstride; + output[startidx] = x2[54]; + output[endidx] = x2[9]; + startidx += outstride; + endidx -= outstride; + output[startidx] = x2[14]; + output[endidx] = x2[49]; + startidx += outstride; + endidx -= outstride; + output[startidx] = x2[46]; + output[endidx] = x2[17]; + startidx += outstride; + endidx -= outstride; + output[startidx] = x2[30]; + output[endidx] = x2[33]; + startidx += outstride; + endidx -= outstride; + output[startidx] = x2[62]; + output[endidx] = x2[1]; +} +void av1_fwd_txfm2d_64x64_avx2(const int16_t *input, int32_t *output, + int stride, TX_TYPE tx_type, int bd) { + (void)bd; + (void)tx_type; + assert(tx_type == DCT_DCT); + const TX_SIZE tx_size = TX_64X64; + __m256i buf0[512], buf1[512]; + const int8_t *shift = fwd_txfm_shift_ls[tx_size]; + const int txw_idx = get_txw_idx(tx_size); + const int txh_idx = get_txh_idx(tx_size); + const int cos_bit_col = fwd_cos_bit_col[txw_idx][txh_idx]; + const int cos_bit_row = fwd_cos_bit_row[txw_idx][txh_idx]; + const int width = tx_size_wide[tx_size]; + const int height = tx_size_high[tx_size]; + const transform_1d_avx2 col_txfm = av1_fdct64_avx2; + const transform_1d_avx2 row_txfm = av1_fdct64_avx2; + const int width_div16 = (width >> 4); + const int width_div8 = (width >> 3); + int r, c; + for (int i = 0; i < width_div16; i++) { + av1_load_buffer_16xn_avx2(input + (i << 4), &buf0[i << 1], stride, height, + width_div8, 0, 0); + av1_round_shift_32_8xn_avx2(&buf0[i << 1], height, shift[0], width_div8); + av1_round_shift_32_8xn_avx2(&buf0[(i << 1) + 1], height, shift[0], + width_div8); + col_txfm(&buf0[i << 1], &buf0[i << 1], cos_bit_col, width_div8, width_div8); + col_txfm(&buf0[(i << 1) + 1], &buf0[(i << 1) + 1], cos_bit_col, width_div8, + width_div8); + av1_round_shift_32_8xn_avx2(&buf0[i << 1], height, shift[1], width_div8); + av1_round_shift_32_8xn_avx2(&buf0[(i << 1) + 1], height, shift[1], + width_div8); + } + + for (r = 0; r < height; r += 8) { + for (c = 0; c < width_div8; c++) { + av1_fwd_txfm_transpose_8x8_avx2(&buf0[r * width_div8 + c], + &buf1[c * 8 * width_div8 + (r >> 3)], + width_div8, width_div8); + } + } + + for (int i = 0; i < 2; i++) { + row_txfm(&buf1[i << 1], &buf0[i << 1], cos_bit_row, width_div8, + width_div16); + row_txfm(&buf1[(i << 1) + 1], &buf0[(i << 1) + 1], cos_bit_row, width_div8, + width_div16); + av1_round_shift_32_8xn_avx2(&buf0[i << 1], (height >> 1), shift[2], + width_div16); + av1_round_shift_32_8xn_avx2(&buf0[(i << 1) + 1], (height >> 1), shift[2], + width_div16); + } + + for (r = 0; r < (height >> 1); r += 8) { + for (c = 0; c < width_div16; c++) { + av1_fwd_txfm_transpose_8x8_avx2(&buf0[r * width_div16 + c], + &buf1[c * 8 * width_div16 + (r >> 3)], + width_div16, width_div16); + } + } + av1_store_buffer_avx2(buf1, output, 8, 128); +} diff --git a/third_party/aom/av1/encoder/x86/highbd_fwd_txfm_sse4.c b/third_party/aom/av1/encoder/x86/highbd_fwd_txfm_sse4.c index 535485ae8e..d1059779dd 100644 --- a/third_party/aom/av1/encoder/x86/highbd_fwd_txfm_sse4.c +++ b/third_party/aom/av1/encoder/x86/highbd_fwd_txfm_sse4.c @@ -59,7 +59,8 @@ static INLINE void load_buffer_4x4(const int16_t *input, __m128i *in, // shift[0] is used in load_buffer_4x4() // shift[1] is used in txfm_func_col() // shift[2] is used in txfm_func_row() -static void fdct4x4_sse4_1(__m128i *in, int bit) { +static void fdct4x4_sse4_1(__m128i *in, __m128i *out, int bit, + const int num_col) { const int32_t *cospi = cospi_arr(bit); const __m128i cospi32 = _mm_set1_epi32(cospi[32]); const __m128i cospi48 = _mm_set1_epi32(cospi[48]); @@ -69,10 +70,12 @@ static void fdct4x4_sse4_1(__m128i *in, int bit) { __m128i u0, u1, u2, u3; __m128i v0, v1, v2, v3; - s0 = _mm_add_epi32(in[0], in[3]); - s1 = _mm_add_epi32(in[1], in[2]); - s2 = _mm_sub_epi32(in[1], in[2]); - s3 = _mm_sub_epi32(in[0], in[3]); + int endidx = 3 * num_col; + s0 = _mm_add_epi32(in[0], in[endidx]); + s3 = _mm_sub_epi32(in[0], in[endidx]); + endidx -= num_col; + s1 = _mm_add_epi32(in[num_col], in[endidx]); + s2 = _mm_sub_epi32(in[num_col], in[endidx]); // btf_32_sse4_1_type0(cospi32, cospi32, s[01], u[02], bit); u0 = _mm_mullo_epi32(s0, cospi32); @@ -109,10 +112,10 @@ static void fdct4x4_sse4_1(__m128i *in, int bit) { v2 = _mm_unpacklo_epi32(u2, u3); v3 = _mm_unpackhi_epi32(u2, u3); - in[0] = _mm_unpacklo_epi64(v0, v2); - in[1] = _mm_unpackhi_epi64(v0, v2); - in[2] = _mm_unpacklo_epi64(v1, v3); - in[3] = _mm_unpackhi_epi64(v1, v3); + out[0] = _mm_unpacklo_epi64(v0, v2); + out[1] = _mm_unpackhi_epi64(v0, v2); + out[2] = _mm_unpacklo_epi64(v1, v3); + out[3] = _mm_unpackhi_epi64(v1, v3); } static INLINE void write_buffer_4x4(__m128i *res, int32_t *output) { @@ -122,7 +125,8 @@ static INLINE void write_buffer_4x4(__m128i *res, int32_t *output) { _mm_store_si128((__m128i *)(output + 3 * 4), res[3]); } -static void fadst4x4_sse4_1(__m128i *in, int bit) { +static void fadst4x4_sse4_1(__m128i *in, __m128i *out, int bit, + const int num_col) { const int32_t *sinpi = sinpi_arr(bit); const __m128i rnding = _mm_set1_epi32(1 << (bit - 1)); const __m128i sinpi1 = _mm_set1_epi32((int)sinpi[1]); @@ -135,15 +139,19 @@ static void fadst4x4_sse4_1(__m128i *in, int bit) { __m128i u0, u1, u2, u3; __m128i v0, v1, v2, v3; - s0 = _mm_mullo_epi32(in[0], sinpi1); - s1 = _mm_mullo_epi32(in[0], sinpi4); - s2 = _mm_mullo_epi32(in[1], sinpi2); - s3 = _mm_mullo_epi32(in[1], sinpi1); - s4 = _mm_mullo_epi32(in[2], sinpi3); - s5 = _mm_mullo_epi32(in[3], sinpi4); - s6 = _mm_mullo_epi32(in[3], sinpi2); - t = _mm_add_epi32(in[0], in[1]); - s7 = _mm_sub_epi32(t, in[3]); + int idx = 0 * num_col; + s0 = _mm_mullo_epi32(in[idx], sinpi1); + s1 = _mm_mullo_epi32(in[idx], sinpi4); + t = _mm_add_epi32(in[idx], in[idx + num_col]); + idx += num_col; + s2 = _mm_mullo_epi32(in[idx], sinpi2); + s3 = _mm_mullo_epi32(in[idx], sinpi1); + idx += num_col; + s4 = _mm_mullo_epi32(in[idx], sinpi3); + idx += num_col; + s5 = _mm_mullo_epi32(in[idx], sinpi4); + s6 = _mm_mullo_epi32(in[idx], sinpi2); + s7 = _mm_sub_epi32(t, in[idx]); t = _mm_add_epi32(s0, s2); x0 = _mm_add_epi32(t, s5); @@ -175,12 +183,35 @@ static void fadst4x4_sse4_1(__m128i *in, int bit) { v2 = _mm_unpacklo_epi32(u2, u3); v3 = _mm_unpackhi_epi32(u2, u3); - in[0] = _mm_unpacklo_epi64(v0, v2); - in[1] = _mm_unpackhi_epi64(v0, v2); - in[2] = _mm_unpacklo_epi64(v1, v3); - in[3] = _mm_unpackhi_epi64(v1, v3); + out[0] = _mm_unpacklo_epi64(v0, v2); + out[1] = _mm_unpackhi_epi64(v0, v2); + out[2] = _mm_unpacklo_epi64(v1, v3); + out[3] = _mm_unpackhi_epi64(v1, v3); } +static void idtx4x4_sse4_1(__m128i *in, __m128i *out, int bit, int col_num) { + (void)bit; + __m128i fact = _mm_set1_epi32(NewSqrt2); + __m128i offset = _mm_set1_epi32(1 << (NewSqrt2Bits - 1)); + __m128i a_low; + __m128i v[4]; + for (int i = 0; i < 4; i++) { + a_low = _mm_mullo_epi32(in[i * col_num], fact); + a_low = _mm_add_epi32(a_low, offset); + out[i] = _mm_srai_epi32(a_low, NewSqrt2Bits); + } + + // Transpose for 4x4 + v[0] = _mm_unpacklo_epi32(out[0], out[1]); + v[1] = _mm_unpackhi_epi32(out[0], out[1]); + v[2] = _mm_unpacklo_epi32(out[2], out[3]); + v[3] = _mm_unpackhi_epi32(out[2], out[3]); + + out[0] = _mm_unpacklo_epi64(v[0], v[2]); + out[1] = _mm_unpackhi_epi64(v[0], v[2]); + out[2] = _mm_unpacklo_epi64(v[1], v[3]); + out[3] = _mm_unpackhi_epi64(v[1], v[3]); +} void av1_fwd_txfm2d_4x4_sse4_1(const int16_t *input, int32_t *coeff, int input_stride, TX_TYPE tx_type, int bd) { __m128i in[4]; @@ -191,56 +222,98 @@ void av1_fwd_txfm2d_4x4_sse4_1(const int16_t *input, int32_t *coeff, switch (tx_type) { case DCT_DCT: load_buffer_4x4(input, in, input_stride, 0, 0, shift[0]); - fdct4x4_sse4_1(in, fwd_cos_bit_col[txw_idx][txh_idx]); - fdct4x4_sse4_1(in, fwd_cos_bit_row[txw_idx][txh_idx]); + fdct4x4_sse4_1(in, in, fwd_cos_bit_col[txw_idx][txh_idx], 1); + fdct4x4_sse4_1(in, in, fwd_cos_bit_row[txw_idx][txh_idx], 1); write_buffer_4x4(in, coeff); break; case ADST_DCT: load_buffer_4x4(input, in, input_stride, 0, 0, shift[0]); - fadst4x4_sse4_1(in, fwd_cos_bit_col[txw_idx][txh_idx]); - fdct4x4_sse4_1(in, fwd_cos_bit_row[txw_idx][txh_idx]); + fadst4x4_sse4_1(in, in, fwd_cos_bit_col[txw_idx][txh_idx], 1); + fdct4x4_sse4_1(in, in, fwd_cos_bit_row[txw_idx][txh_idx], 1); write_buffer_4x4(in, coeff); break; case DCT_ADST: load_buffer_4x4(input, in, input_stride, 0, 0, shift[0]); - fdct4x4_sse4_1(in, fwd_cos_bit_col[txw_idx][txh_idx]); - fadst4x4_sse4_1(in, fwd_cos_bit_row[txw_idx][txh_idx]); + fdct4x4_sse4_1(in, in, fwd_cos_bit_col[txw_idx][txh_idx], 1); + fadst4x4_sse4_1(in, in, fwd_cos_bit_row[txw_idx][txh_idx], 1); write_buffer_4x4(in, coeff); break; case ADST_ADST: load_buffer_4x4(input, in, input_stride, 0, 0, shift[0]); - fadst4x4_sse4_1(in, fwd_cos_bit_col[txw_idx][txh_idx]); - fadst4x4_sse4_1(in, fwd_cos_bit_row[txw_idx][txh_idx]); + fadst4x4_sse4_1(in, in, fwd_cos_bit_col[txw_idx][txh_idx], 1); + fadst4x4_sse4_1(in, in, fwd_cos_bit_row[txw_idx][txh_idx], 1); write_buffer_4x4(in, coeff); break; case FLIPADST_DCT: load_buffer_4x4(input, in, input_stride, 1, 0, shift[0]); - fadst4x4_sse4_1(in, fwd_cos_bit_col[txw_idx][txh_idx]); - fdct4x4_sse4_1(in, fwd_cos_bit_row[txw_idx][txh_idx]); + fadst4x4_sse4_1(in, in, fwd_cos_bit_col[txw_idx][txh_idx], 1); + fdct4x4_sse4_1(in, in, fwd_cos_bit_row[txw_idx][txh_idx], 1); write_buffer_4x4(in, coeff); break; case DCT_FLIPADST: load_buffer_4x4(input, in, input_stride, 0, 1, shift[0]); - fdct4x4_sse4_1(in, fwd_cos_bit_col[txw_idx][txh_idx]); - fadst4x4_sse4_1(in, fwd_cos_bit_row[txw_idx][txh_idx]); + fdct4x4_sse4_1(in, in, fwd_cos_bit_col[txw_idx][txh_idx], 1); + fadst4x4_sse4_1(in, in, fwd_cos_bit_row[txw_idx][txh_idx], 1); write_buffer_4x4(in, coeff); break; case FLIPADST_FLIPADST: load_buffer_4x4(input, in, input_stride, 1, 1, shift[0]); - fadst4x4_sse4_1(in, fwd_cos_bit_col[txw_idx][txh_idx]); - fadst4x4_sse4_1(in, fwd_cos_bit_row[txw_idx][txh_idx]); + fadst4x4_sse4_1(in, in, fwd_cos_bit_col[txw_idx][txh_idx], 1); + fadst4x4_sse4_1(in, in, fwd_cos_bit_row[txw_idx][txh_idx], 1); write_buffer_4x4(in, coeff); break; case ADST_FLIPADST: load_buffer_4x4(input, in, input_stride, 0, 1, shift[0]); - fadst4x4_sse4_1(in, fwd_cos_bit_col[txw_idx][txh_idx]); - fadst4x4_sse4_1(in, fwd_cos_bit_row[txw_idx][txh_idx]); + fadst4x4_sse4_1(in, in, fwd_cos_bit_col[txw_idx][txh_idx], 1); + fadst4x4_sse4_1(in, in, fwd_cos_bit_row[txw_idx][txh_idx], 1); write_buffer_4x4(in, coeff); break; case FLIPADST_ADST: load_buffer_4x4(input, in, input_stride, 1, 0, shift[0]); - fadst4x4_sse4_1(in, fwd_cos_bit_col[txw_idx][txh_idx]); - fadst4x4_sse4_1(in, fwd_cos_bit_row[txw_idx][txh_idx]); + fadst4x4_sse4_1(in, in, fwd_cos_bit_col[txw_idx][txh_idx], 1); + fadst4x4_sse4_1(in, in, fwd_cos_bit_row[txw_idx][txh_idx], 1); + write_buffer_4x4(in, coeff); + break; + case IDTX: + load_buffer_4x4(input, in, input_stride, 0, 0, shift[0]); + idtx4x4_sse4_1(in, in, fwd_cos_bit_col[txw_idx][txh_idx], 1); + idtx4x4_sse4_1(in, in, fwd_cos_bit_row[txw_idx][txh_idx], 1); + write_buffer_4x4(in, coeff); + break; + case V_DCT: + load_buffer_4x4(input, in, input_stride, 0, 0, shift[0]); + fdct4x4_sse4_1(in, in, fwd_cos_bit_col[txw_idx][txh_idx], 1); + idtx4x4_sse4_1(in, in, fwd_cos_bit_row[txw_idx][txh_idx], 1); + write_buffer_4x4(in, coeff); + break; + case H_DCT: + load_buffer_4x4(input, in, input_stride, 0, 0, shift[0]); + idtx4x4_sse4_1(in, in, fwd_cos_bit_row[txw_idx][txh_idx], 1); + fdct4x4_sse4_1(in, in, fwd_cos_bit_col[txw_idx][txh_idx], 1); + write_buffer_4x4(in, coeff); + break; + case V_ADST: + load_buffer_4x4(input, in, input_stride, 0, 0, shift[0]); + fadst4x4_sse4_1(in, in, fwd_cos_bit_col[txw_idx][txh_idx], 1); + idtx4x4_sse4_1(in, in, fwd_cos_bit_row[txw_idx][txh_idx], 1); + write_buffer_4x4(in, coeff); + break; + case H_ADST: + load_buffer_4x4(input, in, input_stride, 0, 0, shift[0]); + idtx4x4_sse4_1(in, in, fwd_cos_bit_row[txw_idx][txh_idx], 1); + fadst4x4_sse4_1(in, in, fwd_cos_bit_col[txw_idx][txh_idx], 1); + write_buffer_4x4(in, coeff); + break; + case V_FLIPADST: + load_buffer_4x4(input, in, input_stride, 1, 0, shift[0]); + fadst4x4_sse4_1(in, in, fwd_cos_bit_row[txw_idx][txh_idx], 1); + idtx4x4_sse4_1(in, in, fwd_cos_bit_row[txw_idx][txh_idx], 1); + write_buffer_4x4(in, coeff); + break; + case H_FLIPADST: + load_buffer_4x4(input, in, input_stride, 0, 1, shift[0]); + idtx4x4_sse4_1(in, in, fwd_cos_bit_row[txw_idx][txh_idx], 1); + fadst4x4_sse4_1(in, in, fwd_cos_bit_row[txw_idx][txh_idx], 1); write_buffer_4x4(in, coeff); break; default: assert(0); @@ -372,6 +445,28 @@ static INLINE void col_txfm_8x8_rounding(__m128i *in, int shift) { in[15] = _mm_srai_epi32(in[15], shift); } +static INLINE void col_txfm_4x8_rounding(__m128i *in, int shift) { + const __m128i rounding = _mm_set1_epi32(1 << (shift - 1)); + + in[0] = _mm_add_epi32(in[0], rounding); + in[1] = _mm_add_epi32(in[1], rounding); + in[2] = _mm_add_epi32(in[2], rounding); + in[3] = _mm_add_epi32(in[3], rounding); + in[4] = _mm_add_epi32(in[4], rounding); + in[5] = _mm_add_epi32(in[5], rounding); + in[6] = _mm_add_epi32(in[6], rounding); + in[7] = _mm_add_epi32(in[7], rounding); + + in[0] = _mm_srai_epi32(in[0], shift); + in[1] = _mm_srai_epi32(in[1], shift); + in[2] = _mm_srai_epi32(in[2], shift); + in[3] = _mm_srai_epi32(in[3], shift); + in[4] = _mm_srai_epi32(in[4], shift); + in[5] = _mm_srai_epi32(in[5], shift); + in[6] = _mm_srai_epi32(in[6], shift); + in[7] = _mm_srai_epi32(in[7], shift); +} + static INLINE void write_buffer_8x8(const __m128i *res, int32_t *output) { _mm_store_si128((__m128i *)(output + 0 * 4), res[0]); _mm_store_si128((__m128i *)(output + 1 * 4), res[1]); @@ -417,9 +512,8 @@ static INLINE void write_buffer_16x8(const __m128i *res, int32_t *output, _mm_storeu_si128((__m128i *)(output + (stride * 7) + 4), res[15]); } -static void fdct8x8_sse4_1(__m128i *in, __m128i *out, int bit, +static void fdct4x8_sse4_1(__m128i *in, __m128i *out, int bit, const int col_num) { - (void)(col_num); const int32_t *cospi = cospi_arr(bit); const __m128i cospi32 = _mm_set1_epi32(cospi[32]); const __m128i cospim32 = _mm_set1_epi32(-cospi[32]); @@ -432,17 +526,25 @@ static void fdct8x8_sse4_1(__m128i *in, __m128i *out, int bit, const __m128i rnding = _mm_set1_epi32(1 << (bit - 1)); __m128i u[8], v[8]; + int startidx = 0 * col_num; + int endidx = 7 * col_num; // Even 8 points 0, 2, ..., 14 // stage 0 // stage 1 - u[0] = _mm_add_epi32(in[0], in[14]); - v[7] = _mm_sub_epi32(in[0], in[14]); // v[7] - u[1] = _mm_add_epi32(in[2], in[12]); - u[6] = _mm_sub_epi32(in[2], in[12]); - u[2] = _mm_add_epi32(in[4], in[10]); - u[5] = _mm_sub_epi32(in[4], in[10]); - u[3] = _mm_add_epi32(in[6], in[8]); - v[4] = _mm_sub_epi32(in[6], in[8]); // v[4] + u[0] = _mm_add_epi32(in[startidx], in[endidx]); + v[7] = _mm_sub_epi32(in[startidx], in[endidx]); // v[7] + startidx += col_num; + endidx -= col_num; + u[1] = _mm_add_epi32(in[startidx], in[endidx]); + u[6] = _mm_sub_epi32(in[startidx], in[endidx]); + startidx += col_num; + endidx -= col_num; + u[2] = _mm_add_epi32(in[startidx], in[endidx]); + u[5] = _mm_sub_epi32(in[startidx], in[endidx]); + startidx += col_num; + endidx -= col_num; + u[3] = _mm_add_epi32(in[startidx], in[endidx]); + v[4] = _mm_sub_epi32(in[startidx], in[endidx]); // v[4] // stage 2 v[0] = _mm_add_epi32(u[0], u[3]); @@ -498,126 +600,40 @@ static void fdct8x8_sse4_1(__m128i *in, __m128i *out, int bit, v[1] = _mm_mullo_epi32(u[7], cospi8); v[0] = _mm_add_epi32(v[0], v[1]); v[0] = _mm_add_epi32(v[0], rnding); - out[2] = _mm_srai_epi32(v[0], bit); // buf0[4] + out[1 * col_num] = _mm_srai_epi32(v[0], bit); // buf0[4] v[0] = _mm_mullo_epi32(u[4], cospi8); v[1] = _mm_mullo_epi32(u[7], cospi56); v[0] = _mm_sub_epi32(v[1], v[0]); v[0] = _mm_add_epi32(v[0], rnding); - out[14] = _mm_srai_epi32(v[0], bit); // buf0[7] + out[7 * col_num] = _mm_srai_epi32(v[0], bit); // buf0[7] v[0] = _mm_mullo_epi32(u[5], cospi24); v[1] = _mm_mullo_epi32(u[6], cospi40); v[0] = _mm_add_epi32(v[0], v[1]); v[0] = _mm_add_epi32(v[0], rnding); - out[10] = _mm_srai_epi32(v[0], bit); // buf0[5] + out[5 * col_num] = _mm_srai_epi32(v[0], bit); // buf0[5] v[0] = _mm_mullo_epi32(u[5], cospi40); v[1] = _mm_mullo_epi32(u[6], cospi24); v[0] = _mm_sub_epi32(v[1], v[0]); v[0] = _mm_add_epi32(v[0], rnding); - out[6] = _mm_srai_epi32(v[0], bit); // buf0[6] + out[3 * col_num] = _mm_srai_epi32(v[0], bit); // buf0[6] - out[0] = u[0]; // buf0[0] - out[8] = u[1]; // buf0[1] - out[4] = u[2]; // buf0[2] - out[12] = u[3]; // buf0[3] + out[0 * col_num] = u[0]; // buf0[0] + out[4 * col_num] = u[1]; // buf0[1] + out[2 * col_num] = u[2]; // buf0[2] + out[6 * col_num] = u[3]; // buf0[3] +} - // Odd 8 points: 1, 3, ..., 15 - // stage 0 - // stage 1 - u[0] = _mm_add_epi32(in[1], in[15]); - v[7] = _mm_sub_epi32(in[1], in[15]); // v[7] - u[1] = _mm_add_epi32(in[3], in[13]); - u[6] = _mm_sub_epi32(in[3], in[13]); - u[2] = _mm_add_epi32(in[5], in[11]); - u[5] = _mm_sub_epi32(in[5], in[11]); - u[3] = _mm_add_epi32(in[7], in[9]); - v[4] = _mm_sub_epi32(in[7], in[9]); // v[4] - - // stage 2 - v[0] = _mm_add_epi32(u[0], u[3]); - v[3] = _mm_sub_epi32(u[0], u[3]); - v[1] = _mm_add_epi32(u[1], u[2]); - v[2] = _mm_sub_epi32(u[1], u[2]); - - v[5] = _mm_mullo_epi32(u[5], cospim32); - v[6] = _mm_mullo_epi32(u[6], cospi32); - v[5] = _mm_add_epi32(v[5], v[6]); - v[5] = _mm_add_epi32(v[5], rnding); - v[5] = _mm_srai_epi32(v[5], bit); - - u[0] = _mm_mullo_epi32(u[5], cospi32); - v[6] = _mm_mullo_epi32(u[6], cospim32); - v[6] = _mm_sub_epi32(u[0], v[6]); - v[6] = _mm_add_epi32(v[6], rnding); - v[6] = _mm_srai_epi32(v[6], bit); - - // stage 3 - // type 0 - v[0] = _mm_mullo_epi32(v[0], cospi32); - v[1] = _mm_mullo_epi32(v[1], cospi32); - u[0] = _mm_add_epi32(v[0], v[1]); - u[0] = _mm_add_epi32(u[0], rnding); - u[0] = _mm_srai_epi32(u[0], bit); - - u[1] = _mm_sub_epi32(v[0], v[1]); - u[1] = _mm_add_epi32(u[1], rnding); - u[1] = _mm_srai_epi32(u[1], bit); - - // type 1 - v[0] = _mm_mullo_epi32(v[2], cospi48); - v[1] = _mm_mullo_epi32(v[3], cospi16); - u[2] = _mm_add_epi32(v[0], v[1]); - u[2] = _mm_add_epi32(u[2], rnding); - u[2] = _mm_srai_epi32(u[2], bit); - - v[0] = _mm_mullo_epi32(v[2], cospi16); - v[1] = _mm_mullo_epi32(v[3], cospi48); - u[3] = _mm_sub_epi32(v[1], v[0]); - u[3] = _mm_add_epi32(u[3], rnding); - u[3] = _mm_srai_epi32(u[3], bit); - - u[4] = _mm_add_epi32(v[4], v[5]); - u[5] = _mm_sub_epi32(v[4], v[5]); - u[6] = _mm_sub_epi32(v[7], v[6]); - u[7] = _mm_add_epi32(v[7], v[6]); - - // stage 4 - // stage 5 - v[0] = _mm_mullo_epi32(u[4], cospi56); - v[1] = _mm_mullo_epi32(u[7], cospi8); - v[0] = _mm_add_epi32(v[0], v[1]); - v[0] = _mm_add_epi32(v[0], rnding); - out[3] = _mm_srai_epi32(v[0], bit); // buf0[4] - - v[0] = _mm_mullo_epi32(u[4], cospi8); - v[1] = _mm_mullo_epi32(u[7], cospi56); - v[0] = _mm_sub_epi32(v[1], v[0]); - v[0] = _mm_add_epi32(v[0], rnding); - out[15] = _mm_srai_epi32(v[0], bit); // buf0[7] - - v[0] = _mm_mullo_epi32(u[5], cospi24); - v[1] = _mm_mullo_epi32(u[6], cospi40); - v[0] = _mm_add_epi32(v[0], v[1]); - v[0] = _mm_add_epi32(v[0], rnding); - out[11] = _mm_srai_epi32(v[0], bit); // buf0[5] - - v[0] = _mm_mullo_epi32(u[5], cospi40); - v[1] = _mm_mullo_epi32(u[6], cospi24); - v[0] = _mm_sub_epi32(v[1], v[0]); - v[0] = _mm_add_epi32(v[0], rnding); - out[7] = _mm_srai_epi32(v[0], bit); // buf0[6] - - out[1] = u[0]; // buf0[0] - out[9] = u[1]; // buf0[1] - out[5] = u[2]; // buf0[2] - out[13] = u[3]; // buf0[3] +static void fdct8x8_sse4_1(__m128i *in, __m128i *out, int bit, + const int col_num) { + fdct4x8_sse4_1(in, out, bit, col_num); + fdct4x8_sse4_1(in + 1, out + 1, bit, col_num); } static void fadst8x8_sse4_1(__m128i *in, __m128i *out, int bit, const int col_num) { - (void)(col_num); const int32_t *cospi = cospi_arr(bit); const __m128i cospi32 = _mm_set1_epi32(cospi[32]); const __m128i cospi16 = _mm_set1_epi32(cospi[16]); @@ -648,17 +664,17 @@ static void fadst8x8_sse4_1(__m128i *in, __m128i *out, int bit, // Odd column: 1, 3, ..., 15 // one even column plus one odd column constructs one row (8 coeffs) // total we have 8 rows (8x8). - for (col = 0; col < 2; ++col) { + for (col = 0; col < col_num; ++col) { // stage 0 // stage 1 - u0 = in[2 * 0 + col]; - u1 = _mm_sub_epi32(zero, in[2 * 7 + col]); - u2 = _mm_sub_epi32(zero, in[2 * 3 + col]); - u3 = in[2 * 4 + col]; - u4 = _mm_sub_epi32(zero, in[2 * 1 + col]); - u5 = in[2 * 6 + col]; - u6 = in[2 * 2 + col]; - u7 = _mm_sub_epi32(zero, in[2 * 5 + col]); + u0 = in[col_num * 0 + col]; + u1 = _mm_sub_epi32(zero, in[col_num * 7 + col]); + u2 = _mm_sub_epi32(zero, in[col_num * 3 + col]); + u3 = in[col_num * 4 + col]; + u4 = _mm_sub_epi32(zero, in[col_num * 1 + col]); + u5 = in[col_num * 6 + col]; + u6 = in[col_num * 2 + col]; + u7 = _mm_sub_epi32(zero, in[col_num * 5 + col]); // stage 2 v0 = u0; @@ -787,17 +803,44 @@ static void fadst8x8_sse4_1(__m128i *in, __m128i *out, int bit, v7 = _mm_srai_epi32(v7, bit); // stage 7 - out[2 * 0 + col] = v1; - out[2 * 1 + col] = v6; - out[2 * 2 + col] = v3; - out[2 * 3 + col] = v4; - out[2 * 4 + col] = v5; - out[2 * 5 + col] = v2; - out[2 * 6 + col] = v7; - out[2 * 7 + col] = v0; + out[col_num * 0 + col] = v1; + out[col_num * 1 + col] = v6; + out[col_num * 2 + col] = v3; + out[col_num * 3 + col] = v4; + out[col_num * 4 + col] = v5; + out[col_num * 5 + col] = v2; + out[col_num * 6 + col] = v7; + out[col_num * 7 + col] = v0; } } +static void idtx8x8_sse4_1(__m128i *in, __m128i *out, int bit, int col_num) { + (void)bit; + for (int i = 0; i < col_num; i += 1) { + out[0 + 8 * i] = _mm_add_epi32(in[0 + 8 * i], in[0 + 8 * i]); + out[1 + 8 * i] = _mm_add_epi32(in[1 + 8 * i], in[1 + 8 * i]); + out[2 + 8 * i] = _mm_add_epi32(in[2 + 8 * i], in[2 + 8 * i]); + out[3 + 8 * i] = _mm_add_epi32(in[3 + 8 * i], in[3 + 8 * i]); + out[4 + 8 * i] = _mm_add_epi32(in[4 + 8 * i], in[4 + 8 * i]); + out[5 + 8 * i] = _mm_add_epi32(in[5 + 8 * i], in[5 + 8 * i]); + out[6 + 8 * i] = _mm_add_epi32(in[6 + 8 * i], in[6 + 8 * i]); + out[7 + 8 * i] = _mm_add_epi32(in[7 + 8 * i], in[7 + 8 * i]); + } +} +static void idtx32x8_sse4_1(__m128i *in, __m128i *out, int bit, int col_num) { + (void)bit; + (void)col_num; + for (int j = 0; j < 2; j++) { + out[j + 8 * 0] = _mm_add_epi32(in[j + 8 * 0], in[j + 8 * 0]); + out[j + 8 * 1] = _mm_add_epi32(in[j + 8 * 1], in[j + 8 * 1]); + out[j + 8 * 2] = _mm_add_epi32(in[j + 8 * 2], in[j + 8 * 2]); + out[j + 8 * 3] = _mm_add_epi32(in[j + 8 * 3], in[j + 8 * 3]); + out[j + 8 * 4] = _mm_add_epi32(in[j + 8 * 4], in[j + 8 * 4]); + out[j + 8 * 5] = _mm_add_epi32(in[j + 8 * 5], in[j + 8 * 5]); + out[j + 8 * 6] = _mm_add_epi32(in[j + 8 * 6], in[j + 8 * 6]); + out[j + 8 * 7] = _mm_add_epi32(in[j + 8 * 7], in[j + 8 * 7]); + } +} void av1_fwd_txfm2d_8x8_sse4_1(const int16_t *input, int32_t *coeff, int stride, TX_TYPE tx_type, int bd) { __m128i in[16], out[16]; @@ -808,82 +851,145 @@ void av1_fwd_txfm2d_8x8_sse4_1(const int16_t *input, int32_t *coeff, int stride, switch (tx_type) { case DCT_DCT: load_buffer_8x8(input, in, stride, 0, 0, shift[0]); - fdct8x8_sse4_1(in, out, fwd_cos_bit_col[txw_idx][txh_idx], 0); + fdct8x8_sse4_1(in, out, fwd_cos_bit_col[txw_idx][txh_idx], 2); col_txfm_8x8_rounding(out, -shift[1]); transpose_8x8(out, in); - fdct8x8_sse4_1(in, out, fwd_cos_bit_row[txw_idx][txh_idx], 0); + fdct8x8_sse4_1(in, out, fwd_cos_bit_row[txw_idx][txh_idx], 2); transpose_8x8(out, in); write_buffer_8x8(in, coeff); break; case ADST_DCT: load_buffer_8x8(input, in, stride, 0, 0, shift[0]); - fadst8x8_sse4_1(in, out, fwd_cos_bit_col[txw_idx][txh_idx], 0); + fadst8x8_sse4_1(in, out, fwd_cos_bit_col[txw_idx][txh_idx], 2); col_txfm_8x8_rounding(out, -shift[1]); transpose_8x8(out, in); - fdct8x8_sse4_1(in, out, fwd_cos_bit_row[txw_idx][txh_idx], 0); + fdct8x8_sse4_1(in, out, fwd_cos_bit_row[txw_idx][txh_idx], 2); transpose_8x8(out, in); write_buffer_8x8(in, coeff); break; case DCT_ADST: load_buffer_8x8(input, in, stride, 0, 0, shift[0]); - fdct8x8_sse4_1(in, out, fwd_cos_bit_col[txw_idx][txh_idx], 0); + fdct8x8_sse4_1(in, out, fwd_cos_bit_col[txw_idx][txh_idx], 2); col_txfm_8x8_rounding(out, -shift[1]); transpose_8x8(out, in); - fadst8x8_sse4_1(in, out, fwd_cos_bit_row[txw_idx][txh_idx], 0); + fadst8x8_sse4_1(in, out, fwd_cos_bit_row[txw_idx][txh_idx], 2); transpose_8x8(out, in); write_buffer_8x8(in, coeff); break; case ADST_ADST: load_buffer_8x8(input, in, stride, 0, 0, shift[0]); - fadst8x8_sse4_1(in, out, fwd_cos_bit_col[txw_idx][txh_idx], 0); + fadst8x8_sse4_1(in, out, fwd_cos_bit_col[txw_idx][txh_idx], 2); col_txfm_8x8_rounding(out, -shift[1]); transpose_8x8(out, in); - fadst8x8_sse4_1(in, out, fwd_cos_bit_row[txw_idx][txh_idx], 0); + fadst8x8_sse4_1(in, out, fwd_cos_bit_row[txw_idx][txh_idx], 2); transpose_8x8(out, in); write_buffer_8x8(in, coeff); break; case FLIPADST_DCT: load_buffer_8x8(input, in, stride, 1, 0, shift[0]); - fadst8x8_sse4_1(in, out, fwd_cos_bit_col[txw_idx][txh_idx], 0); + fadst8x8_sse4_1(in, out, fwd_cos_bit_col[txw_idx][txh_idx], 2); col_txfm_8x8_rounding(out, -shift[1]); transpose_8x8(out, in); - fdct8x8_sse4_1(in, out, fwd_cos_bit_row[txw_idx][txh_idx], 0); + fdct8x8_sse4_1(in, out, fwd_cos_bit_row[txw_idx][txh_idx], 2); transpose_8x8(out, in); write_buffer_8x8(in, coeff); break; case DCT_FLIPADST: load_buffer_8x8(input, in, stride, 0, 1, shift[0]); - fdct8x8_sse4_1(in, out, fwd_cos_bit_col[txw_idx][txh_idx], 0); + fdct8x8_sse4_1(in, out, fwd_cos_bit_col[txw_idx][txh_idx], 2); col_txfm_8x8_rounding(out, -shift[1]); transpose_8x8(out, in); - fadst8x8_sse4_1(in, out, fwd_cos_bit_row[txw_idx][txh_idx], 0); + fadst8x8_sse4_1(in, out, fwd_cos_bit_row[txw_idx][txh_idx], 2); transpose_8x8(out, in); write_buffer_8x8(in, coeff); break; case FLIPADST_FLIPADST: load_buffer_8x8(input, in, stride, 1, 1, shift[0]); - fadst8x8_sse4_1(in, out, fwd_cos_bit_col[txw_idx][txh_idx], 0); + fadst8x8_sse4_1(in, out, fwd_cos_bit_col[txw_idx][txh_idx], 2); col_txfm_8x8_rounding(out, -shift[1]); transpose_8x8(out, in); - fadst8x8_sse4_1(in, out, fwd_cos_bit_row[txw_idx][txh_idx], 0); + fadst8x8_sse4_1(in, out, fwd_cos_bit_row[txw_idx][txh_idx], 2); transpose_8x8(out, in); write_buffer_8x8(in, coeff); break; case ADST_FLIPADST: load_buffer_8x8(input, in, stride, 0, 1, shift[0]); - fadst8x8_sse4_1(in, out, fwd_cos_bit_col[txw_idx][txh_idx], 0); + fadst8x8_sse4_1(in, out, fwd_cos_bit_col[txw_idx][txh_idx], 2); col_txfm_8x8_rounding(out, -shift[1]); transpose_8x8(out, in); - fadst8x8_sse4_1(in, out, fwd_cos_bit_row[txw_idx][txh_idx], 0); + fadst8x8_sse4_1(in, out, fwd_cos_bit_row[txw_idx][txh_idx], 2); transpose_8x8(out, in); write_buffer_8x8(in, coeff); break; case FLIPADST_ADST: load_buffer_8x8(input, in, stride, 1, 0, shift[0]); - fadst8x8_sse4_1(in, out, fwd_cos_bit_col[txw_idx][txh_idx], 0); + fadst8x8_sse4_1(in, out, fwd_cos_bit_col[txw_idx][txh_idx], 2); col_txfm_8x8_rounding(out, -shift[1]); transpose_8x8(out, in); - fadst8x8_sse4_1(in, out, fwd_cos_bit_row[txw_idx][txh_idx], 0); + fadst8x8_sse4_1(in, out, fwd_cos_bit_row[txw_idx][txh_idx], 2); + transpose_8x8(out, in); + write_buffer_8x8(in, coeff); + break; + case IDTX: + load_buffer_8x8(input, in, stride, 0, 0, shift[0]); + idtx8x8_sse4_1(in, out, fwd_cos_bit_col[txw_idx][txh_idx], 2); + col_txfm_8x8_rounding(out, -shift[1]); + transpose_8x8(out, in); + idtx8x8_sse4_1(in, out, fwd_cos_bit_col[txw_idx][txh_idx], 2); + transpose_8x8(out, in); + write_buffer_8x8(in, coeff); + break; + case V_DCT: + load_buffer_8x8(input, in, stride, 0, 0, shift[0]); + fdct8x8_sse4_1(in, out, fwd_cos_bit_col[txw_idx][txh_idx], 2); + col_txfm_8x8_rounding(out, -shift[1]); + transpose_8x8(out, in); + idtx8x8_sse4_1(in, out, fwd_cos_bit_col[txw_idx][txh_idx], 2); + transpose_8x8(out, in); + write_buffer_8x8(in, coeff); + break; + case H_DCT: + load_buffer_8x8(input, in, stride, 0, 0, shift[0]); + idtx8x8_sse4_1(in, out, fwd_cos_bit_col[txw_idx][txh_idx], 2); + col_txfm_8x8_rounding(out, -shift[1]); + transpose_8x8(out, in); + fdct8x8_sse4_1(in, out, fwd_cos_bit_col[txw_idx][txh_idx], 2); + transpose_8x8(out, in); + write_buffer_8x8(in, coeff); + break; + case V_ADST: + load_buffer_8x8(input, in, stride, 0, 0, shift[0]); + fadst8x8_sse4_1(in, out, fwd_cos_bit_col[txw_idx][txh_idx], 2); + col_txfm_8x8_rounding(out, -shift[1]); + transpose_8x8(out, in); + idtx8x8_sse4_1(in, out, fwd_cos_bit_col[txw_idx][txh_idx], 2); + transpose_8x8(out, in); + write_buffer_8x8(in, coeff); + break; + case H_ADST: + load_buffer_8x8(input, in, stride, 0, 0, shift[0]); + idtx8x8_sse4_1(in, out, fwd_cos_bit_col[txw_idx][txh_idx], 2); + col_txfm_8x8_rounding(out, -shift[1]); + transpose_8x8(out, in); + fadst8x8_sse4_1(in, out, fwd_cos_bit_col[txw_idx][txh_idx], 2); + transpose_8x8(out, in); + write_buffer_8x8(in, coeff); + break; + case V_FLIPADST: + load_buffer_8x8(input, in, stride, 1, 0, shift[0]); + fadst8x8_sse4_1(in, out, fwd_cos_bit_col[txw_idx][txh_idx], 2); + col_txfm_8x8_rounding(out, -shift[1]); + transpose_8x8(out, in); + idtx8x8_sse4_1(in, out, fwd_cos_bit_col[txw_idx][txh_idx], 2); + transpose_8x8(out, in); + write_buffer_8x8(in, coeff); + break; + case H_FLIPADST: + load_buffer_8x8(input, in, stride, 0, 1, shift[0]); + idtx8x8_sse4_1(in, out, fwd_cos_bit_col[txw_idx][txh_idx], 2); + col_txfm_8x8_rounding(out, -shift[1]); + transpose_8x8(out, in); + fadst8x8_sse4_1(in, out, fwd_cos_bit_col[txw_idx][txh_idx], 2); transpose_8x8(out, in); write_buffer_8x8(in, coeff); break; @@ -986,6 +1092,90 @@ static INLINE void load_buffer_8x16(const int16_t *input, __m128i *out, load_buffer_8x8(botL, out + 16, stride, flipud, fliplr, shift); } +static INLINE void load_buffer_8x4(const int16_t *input, __m128i *out, + int stride, int flipud, int fliplr, + int shift) { + const int16_t *topL = input; + const int16_t *topR = input + 4; + + const int16_t *tmp; + + if (fliplr) { + tmp = topL; + topL = topR; + topR = tmp; + } + + load_buffer_4x4(topL, out, stride, flipud, fliplr, shift); + load_buffer_4x4(topR, out + 4, stride, flipud, fliplr, shift); +} + +static INLINE void load_buffer_16x4(const int16_t *input, __m128i *out, + int stride, int flipud, int fliplr, + int shift) { + const int16_t *topL = input; + const int16_t *topR = input + 8; + + const int16_t *tmp; + + if (fliplr) { + tmp = topL; + topL = topR; + topR = tmp; + } + + load_buffer_8x4(topL, out, stride, flipud, fliplr, shift); + load_buffer_8x4(topR, out + 8, stride, flipud, fliplr, shift); +} + +static INLINE void load_buffer_4x8(const int16_t *input, __m128i *out, + int stride, int flipud, int fliplr, + int shift) { + const int16_t *topL = input; + const int16_t *botL = input + 4 * stride; + + const int16_t *tmp; + + if (flipud) { + tmp = topL; + topL = botL; + botL = tmp; + } + + load_buffer_4x4(topL, out, stride, flipud, fliplr, shift); + load_buffer_4x4(botL, out + 4, stride, flipud, fliplr, shift); +} + +static INLINE void load_buffer_4x16(const int16_t *input, __m128i *out, + const int stride, const int flipud, + const int fliplr, const int shift) { + const int16_t *topL = input; + const int16_t *botL = input + 8 * stride; + + const int16_t *tmp; + + if (flipud) { + tmp = topL; + topL = botL; + botL = tmp; + } + load_buffer_4x8(topL, out, stride, flipud, fliplr, shift); + load_buffer_4x8(botL, out + 8, stride, flipud, fliplr, shift); +} + +static INLINE void load_buffer_32x8n(const int16_t *input, __m128i *out, + int stride, int flipud, int fliplr, + int shift, const int height) { + const int16_t *in = input; + __m128i *output = out; + for (int col = 0; col < height; col++) { + in = input + col * stride; + output = out + col * 8; + load_buffer_4x4(in, output, 4, flipud, fliplr, shift); + load_buffer_4x4((in + 16), (output + 4), 4, flipud, fliplr, shift); + } +} + static void fdct16x16_sse4_1(__m128i *in, __m128i *out, int bit, const int col_num) { const int32_t *cospi = cospi_arr(bit); @@ -1544,7 +1734,19 @@ static void write_buffer_16x16(const __m128i *in, int32_t *output) { output += size_8x8; write_buffer_8x8(&in[48], output); } +static void idtx16x16_sse4_1(__m128i *in, __m128i *out, int bit, int col_num) { + (void)bit; + __m128i fact = _mm_set1_epi32(2 * NewSqrt2); + __m128i offset = _mm_set1_epi32(1 << (NewSqrt2Bits - 1)); + __m128i a_low; + int num_iters = 16 * col_num; + for (int i = 0; i < num_iters; i++) { + a_low = _mm_mullo_epi32(in[i], fact); + a_low = _mm_add_epi32(a_low, offset); + out[i] = _mm_srai_epi32(a_low, NewSqrt2Bits); + } +} void av1_fwd_txfm2d_16x16_sse4_1(const int16_t *input, int32_t *coeff, int stride, TX_TYPE tx_type, int bd) { __m128i in[64], out[64]; @@ -1634,6 +1836,69 @@ void av1_fwd_txfm2d_16x16_sse4_1(const int16_t *input, int32_t *coeff, transpose_16x16(out, in); write_buffer_16x16(in, coeff); break; + case IDTX: + load_buffer_16x16(input, in, stride, 0, 0, shift[0]); + idtx16x16_sse4_1(in, out, fwd_cos_bit_col[txw_idx][txh_idx], col_num); + col_txfm_16x16_rounding(out, -shift[1]); + transpose_16x16(out, in); + idtx16x16_sse4_1(in, out, fwd_cos_bit_row[txw_idx][txh_idx], col_num); + transpose_16x16(out, in); + write_buffer_16x16(in, coeff); + break; + case V_DCT: + load_buffer_16x16(input, in, stride, 0, 0, shift[0]); + fdct16x16_sse4_1(in, out, fwd_cos_bit_col[txw_idx][txh_idx], col_num); + col_txfm_16x16_rounding(out, -shift[1]); + transpose_16x16(out, in); + idtx16x16_sse4_1(in, out, fwd_cos_bit_row[txw_idx][txh_idx], col_num); + transpose_16x16(out, in); + write_buffer_16x16(in, coeff); + break; + case H_DCT: + load_buffer_16x16(input, in, stride, 0, 0, shift[0]); + idtx16x16_sse4_1(in, out, fwd_cos_bit_col[txw_idx][txh_idx], col_num); + col_txfm_16x16_rounding(out, -shift[1]); + transpose_16x16(out, in); + fdct16x16_sse4_1(in, out, fwd_cos_bit_row[txw_idx][txh_idx], col_num); + transpose_16x16(out, in); + write_buffer_16x16(in, coeff); + break; + case V_ADST: + load_buffer_16x16(input, in, stride, 0, 0, shift[0]); + fadst16x16_sse4_1(in, out, fwd_cos_bit_col[txw_idx][txh_idx], col_num); + col_txfm_16x16_rounding(out, -shift[1]); + transpose_16x16(out, in); + idtx16x16_sse4_1(in, out, fwd_cos_bit_row[txw_idx][txh_idx], col_num); + transpose_16x16(out, in); + write_buffer_16x16(in, coeff); + break; + case H_ADST: + load_buffer_16x16(input, in, stride, 0, 0, shift[0]); + idtx16x16_sse4_1(in, out, fwd_cos_bit_col[txw_idx][txh_idx], col_num); + col_txfm_16x16_rounding(out, -shift[1]); + transpose_16x16(out, in); + fadst16x16_sse4_1(in, out, fwd_cos_bit_row[txw_idx][txh_idx], col_num); + transpose_16x16(out, in); + write_buffer_16x16(in, coeff); + break; + case V_FLIPADST: + load_buffer_16x16(input, in, stride, 1, 0, shift[0]); + fadst16x16_sse4_1(in, out, fwd_cos_bit_col[txw_idx][txh_idx], col_num); + col_txfm_16x16_rounding(out, -shift[1]); + transpose_16x16(out, in); + idtx16x16_sse4_1(in, out, fwd_cos_bit_row[txw_idx][txh_idx], col_num); + transpose_16x16(out, in); + write_buffer_16x16(in, coeff); + break; + case H_FLIPADST: + load_buffer_16x16(input, in, stride, 0, 1, shift[0]); + idtx16x16_sse4_1(in, out, fwd_cos_bit_col[txw_idx][txh_idx], col_num); + col_txfm_16x16_rounding(out, -shift[1]); + transpose_16x16(out, in); + fadst16x16_sse4_1(in, out, fwd_cos_bit_row[txw_idx][txh_idx], col_num); + transpose_16x16(out, in); + write_buffer_16x16(in, coeff); + break; default: assert(0); } (void)bd; @@ -1654,13 +1919,49 @@ static const fwd_transform_1d_sse4_1 col_highbd_txfm8x8_arr[TX_TYPES] = { fadst8x8_sse4_1, // FLIPADST_FLIPADST fadst8x8_sse4_1, // ADST_FLIPADST fadst8x8_sse4_1, // FLIPADST_ADST - NULL, // IDTX + idtx8x8_sse4_1, // IDTX + fdct8x8_sse4_1, // V_DCT + idtx8x8_sse4_1, // H_DCT + fadst8x8_sse4_1, // V_ADST + idtx8x8_sse4_1, // H_ADST + fadst8x8_sse4_1, // V_FLIPADST + idtx8x8_sse4_1 // H_FLIPADST +}; +static const fwd_transform_1d_sse4_1 row_highbd_txfm32x8_arr[TX_TYPES] = { + fdct8x8_sse4_1, // DCT_DCT + NULL, // ADST_DCT + NULL, // DCT_ADST + NULL, // ADST_ADST + NULL, // FLIPADST_DCT + NULL, // DCT_FLIPADST + NULL, // FLIPADST_FLIPADST + NULL, // ADST_FLIPADST + NULL, // FLIPADST-ADST + idtx32x8_sse4_1, // IDTX NULL, // V_DCT NULL, // H_DCT NULL, // V_ADST NULL, // H_ADST NULL, // V_FLIPADST - NULL // H_FLIPADST + NULL, // H_FLIPADST +}; +static const fwd_transform_1d_sse4_1 col_highbd_txfm4x8_arr[TX_TYPES] = { + fdct4x8_sse4_1, // DCT_DCT + fadst8x8_sse4_1, // ADST_DCT + fdct4x8_sse4_1, // DCT_ADST + fadst8x8_sse4_1, // ADST_ADST + fadst8x8_sse4_1, // FLIPADST_DCT + fdct4x8_sse4_1, // DCT_FLIPADST + fadst8x8_sse4_1, // FLIPADST_FLIPADST + fadst8x8_sse4_1, // ADST_FLIPADST + fadst8x8_sse4_1, // FLIPADST_ADST + idtx8x8_sse4_1, // IDTX + fdct4x8_sse4_1, // V_DCT + idtx8x8_sse4_1, // H_DCT + fadst8x8_sse4_1, // V_ADST + idtx8x8_sse4_1, // H_ADST + fadst8x8_sse4_1, // V_FLIPADST + idtx8x8_sse4_1 // H_FLIPADST }; static const fwd_transform_1d_sse4_1 row_highbd_txfm8x16_arr[TX_TYPES] = { @@ -1673,13 +1974,13 @@ static const fwd_transform_1d_sse4_1 row_highbd_txfm8x16_arr[TX_TYPES] = { fadst16x16_sse4_1, // FLIPADST_FLIPADST fadst16x16_sse4_1, // ADST_FLIPADST fadst16x16_sse4_1, // FLIPADST_ADST - NULL, // IDTX - NULL, // V_DCT - NULL, // H_DCT - NULL, // V_ADST - NULL, // H_ADST - NULL, // V_FLIPADST - NULL // H_FLIPADST + idtx16x16_sse4_1, // IDTX + idtx16x16_sse4_1, // V_DCT + fdct16x16_sse4_1, // H_DCT + idtx16x16_sse4_1, // V_ADST + fadst16x16_sse4_1, // H_ADST + idtx16x16_sse4_1, // V_FLIPADST + fadst16x16_sse4_1 // H_FLIPADST }; static const fwd_transform_1d_sse4_1 col_highbd_txfm8x16_arr[TX_TYPES] = { @@ -1692,13 +1993,13 @@ static const fwd_transform_1d_sse4_1 col_highbd_txfm8x16_arr[TX_TYPES] = { fadst16x16_sse4_1, // FLIPADST_FLIPADST fadst16x16_sse4_1, // ADST_FLIPADST fadst16x16_sse4_1, // FLIPADST_ADST - NULL, // IDTX - NULL, // V_DCT - NULL, // H_DCT - NULL, // V_ADST - NULL, // H_ADST - NULL, // V_FLIPADST - NULL // H_FLIPADST + idtx16x16_sse4_1, // IDTX + fdct16x16_sse4_1, // V_DCT + idtx16x16_sse4_1, // H_DCT + fadst16x16_sse4_1, // V_ADST + idtx16x16_sse4_1, // H_ADST + fadst16x16_sse4_1, // V_FLIPADST + idtx16x16_sse4_1 // H_FLIPADST }; static const fwd_transform_1d_sse4_1 row_highbd_txfm8x8_arr[TX_TYPES] = { fdct8x8_sse4_1, // DCT_DCT @@ -1710,13 +2011,108 @@ static const fwd_transform_1d_sse4_1 row_highbd_txfm8x8_arr[TX_TYPES] = { fadst8x8_sse4_1, // FLIPADST_FLIPADST fadst8x8_sse4_1, // ADST_FLIPADST fadst8x8_sse4_1, // FLIPADST_ADST - NULL, // IDTX - NULL, // V_DCT - NULL, // H_DCT - NULL, // V_ADST - NULL, // H_ADST - NULL, // V_FLIPADST - NULL // H_FLIPADST + idtx8x8_sse4_1, // IDTX + idtx8x8_sse4_1, // V_DCT + fdct8x8_sse4_1, // H_DCT + idtx8x8_sse4_1, // V_ADST + fadst8x8_sse4_1, // H_ADST + idtx8x8_sse4_1, // V_FLIPADST + fadst8x8_sse4_1 // H_FLIPADST +}; + +static const fwd_transform_1d_sse4_1 row_highbd_txfm4x8_arr[TX_TYPES] = { + fdct4x8_sse4_1, // DCT_DCT + fdct4x8_sse4_1, // ADST_DCT + fadst8x8_sse4_1, // DCT_ADST + fadst8x8_sse4_1, // ADST_ADST + fdct4x8_sse4_1, // FLIPADST_DCT + fadst8x8_sse4_1, // DCT_FLIPADST + fadst8x8_sse4_1, // FLIPADST_FLIPADST + fadst8x8_sse4_1, // ADST_FLIPADST + fadst8x8_sse4_1, // FLIPADST_ADST + idtx8x8_sse4_1, // IDTX + idtx8x8_sse4_1, // V_DCT + fdct4x8_sse4_1, // H_DCT + idtx8x8_sse4_1, // V_ADST + fadst8x8_sse4_1, // H_ADST + idtx8x8_sse4_1, // V_FLIPADST + fadst8x8_sse4_1 // H_FLIPADST +}; + +static const fwd_transform_1d_sse4_1 row_highbd_txfm4x4_arr[TX_TYPES] = { + fdct4x4_sse4_1, // DCT_DCT + fdct4x4_sse4_1, // ADST_DCT + fadst4x4_sse4_1, // DCT_ADST + fadst4x4_sse4_1, // ADST_ADST + fdct4x4_sse4_1, // FLIPADST_DCT + fadst4x4_sse4_1, // DCT_FLIPADST + fadst4x4_sse4_1, // FLIPADST_FLIPADST + fadst4x4_sse4_1, // ADST_FLIPADST + fadst4x4_sse4_1, // FLIPADST_ADST + idtx4x4_sse4_1, // IDTX + idtx4x4_sse4_1, // V_DCT + fdct4x4_sse4_1, // H_DCT + idtx4x4_sse4_1, // V_ADST + fadst4x4_sse4_1, // H_ADST + idtx4x4_sse4_1, // V_FLIPADST + fadst4x4_sse4_1 // H_FLIPADST +}; + +static const fwd_transform_1d_sse4_1 col_highbd_txfm4x4_arr[TX_TYPES] = { + fdct4x4_sse4_1, // DCT_DCT + fadst4x4_sse4_1, // ADST_DCT + fdct4x4_sse4_1, // DCT_ADST + fadst4x4_sse4_1, // ADST_ADST + fadst4x4_sse4_1, // FLIPADST_DCT + fdct4x4_sse4_1, // DCT_FLIPADST + fadst4x4_sse4_1, // FLIPADST_FLIPADST + fadst4x4_sse4_1, // ADST_FLIPADST + fadst4x4_sse4_1, // FLIPADST_ADST + idtx4x4_sse4_1, // IDTX + fdct4x4_sse4_1, // V_DCT + idtx4x4_sse4_1, // H_DCT + fadst4x4_sse4_1, // V_ADST + idtx4x4_sse4_1, // H_ADST + fadst4x4_sse4_1, // V_FLIPADST + idtx4x4_sse4_1 // H_FLIPADST +}; + +static const fwd_transform_1d_sse4_1 col_highbd_txfm8x32_arr[TX_TYPES] = { + av1_fdct32_new_sse4_1, // DCT_DCT + NULL, // ADST_DCT + NULL, // DCT_ADST + NULL, // ADST_ADST + NULL, // FLIPADST_DCT + NULL, // DCT_FLIPADST + NULL, // FLIPADST_FLIPADST + NULL, // ADST_FLIPADST + NULL, // FLIPADST_ADST + av1_idtx32_new_sse4_1, // IDTX + NULL, // V_DCT + NULL, // H_DCT + NULL, // V_ADST + NULL, // H_ADST + NULL, // V_FLIPADST + NULL // H_FLIPADST +}; + +static const fwd_transform_1d_sse4_1 row_highbd_txfm8x32_arr[TX_TYPES] = { + fdct16x16_sse4_1, // DCT_DCT + NULL, // ADST_DCT + NULL, // DCT_ADST + NULL, // ADST_ADST + NULL, // FLIPADST_DCT + NULL, // DCT_FLIPADST + NULL, // FLIPADST_FLIPADST + NULL, // ADST_FLIPADST + NULL, // FLIPADST_ADST + idtx16x16_sse4_1, // IDTX + NULL, // V_DCT + NULL, // H_DCT + NULL, // V_ADST + NULL, // H_ADST + NULL, // V_FLIPADST + NULL // H_FLIPADST }; void av1_fwd_txfm2d_16x8_sse4_1(const int16_t *input, int32_t *coeff, @@ -1733,7 +2129,7 @@ void av1_fwd_txfm2d_16x8_sse4_1(const int16_t *input, int32_t *coeff, for (int i = 0; i < 2; i++) { load_buffer_8x8(input + i * 8, in, stride, ud_flip, 0, shift[0]); - col_txfm(in, in, bit, 0); + col_txfm(in, in, bit, 2); col_txfm_8x8_rounding(in, -shift[1]); transpose_8x8(in, out + i * 16); } @@ -1773,7 +2169,7 @@ void av1_fwd_txfm2d_8x16_sse4_1(const int16_t *input, int32_t *coeff, transpose_8x8(in + 16, out + 16); for (int i = 0; i < 2; i++) { - row_txfm(out + i * 16, out, bit, 0); + row_txfm(out + i * 16, out, bit, 2); transpose_8x8(out, in); av1_round_shift_rect_array_32_sse4_1(in, in, 16, -shift[2], NewSqrt2); write_buffer_8x8(in, coeff + i * 64); @@ -1781,3 +2177,412 @@ void av1_fwd_txfm2d_8x16_sse4_1(const int16_t *input, int32_t *coeff, (void)bd; } + +void av1_fwd_txfm2d_4x16_sse4_1(const int16_t *input, int32_t *coeff, + int stride, TX_TYPE tx_type, int bd) { + __m128i in[16]; + __m128i *outcoeff128 = (__m128i *)coeff; + const int8_t *shift = fwd_txfm_shift_ls[TX_4X16]; + const int txw_idx = get_txw_idx(TX_4X16); + const int txh_idx = get_txh_idx(TX_4X16); + const int txfm_size_col = tx_size_wide[TX_4X16]; + const int txfm_size_row = tx_size_high[TX_4X16]; + int bitcol = fwd_cos_bit_col[txw_idx][txh_idx]; + int bitrow = fwd_cos_bit_row[txw_idx][txh_idx]; + const fwd_transform_1d_sse4_1 col_txfm = col_highbd_txfm8x16_arr[tx_type]; + const fwd_transform_1d_sse4_1 row_txfm = row_highbd_txfm4x4_arr[tx_type]; + + int ud_flip, lr_flip; + get_flip_cfg(tx_type, &ud_flip, &lr_flip); + // col transform + load_buffer_4x16(input, in, stride, ud_flip, lr_flip, shift[0]); + col_txfm(in, outcoeff128, bitcol, 1); + col_txfm_8x8_rounding(outcoeff128, -shift[1]); + transpose_8nx8n(outcoeff128, in, txfm_size_col, txfm_size_row); + + // row transform + for (int i = 0; i < txfm_size_col; i++) { + row_txfm(in + i, outcoeff128 + i * txfm_size_col, bitrow, txfm_size_col); + } + (void)bd; +} + +void av1_fwd_txfm2d_16x4_sse4_1(const int16_t *input, int32_t *coeff, + int stride, TX_TYPE tx_type, int bd) { + __m128i in[16]; + __m128i *outcoeff128 = (__m128i *)coeff; + const int8_t *shift = fwd_txfm_shift_ls[TX_16X4]; + const int txw_idx = get_txw_idx(TX_16X4); + const int txh_idx = get_txh_idx(TX_16X4); + const int txfm_size_col = tx_size_wide[TX_16X4]; + const int txfm_size_row = tx_size_high[TX_16X4]; + int bitcol = fwd_cos_bit_col[txw_idx][txh_idx]; + int bitrow = fwd_cos_bit_row[txw_idx][txh_idx]; + const fwd_transform_1d_sse4_1 col_txfm = col_highbd_txfm4x4_arr[tx_type]; + const fwd_transform_1d_sse4_1 row_txfm = row_highbd_txfm8x16_arr[tx_type]; + int ud_flip, lr_flip; + get_flip_cfg(tx_type, &ud_flip, &lr_flip); + + // col transform + load_buffer_16x4(input, in, stride, ud_flip, lr_flip, shift[0]); + + for (int i = 0; i < txfm_size_row; i++) { + col_txfm(in + i * txfm_size_row, outcoeff128 + i * txfm_size_row, bitcol, + 1); + } + col_txfm_8x8_rounding(outcoeff128, -shift[1]); + + // row transform + row_txfm(outcoeff128, in, bitrow, 1); + transpose_8nx8n(in, outcoeff128, txfm_size_row, txfm_size_col); + (void)bd; +} + +void av1_fwd_txfm2d_16x32_sse4_1(const int16_t *input, int32_t *coeff, + int stride, TX_TYPE tx_type, int bd) { + __m128i in[128]; + __m128i *outcoef128 = (__m128i *)coeff; + const int8_t *shift = fwd_txfm_shift_ls[TX_16X32]; + const int txw_idx = get_txw_idx(TX_16X32); + const int txh_idx = get_txh_idx(TX_16X32); + const fwd_transform_1d_sse4_1 col_txfm = col_highbd_txfm8x32_arr[tx_type]; + const fwd_transform_1d_sse4_1 row_txfm = row_highbd_txfm8x32_arr[tx_type]; + int bitcol = fwd_cos_bit_col[txw_idx][txh_idx]; + int bitrow = fwd_cos_bit_row[txw_idx][txh_idx]; + + // column transform + load_buffer_16x16(input, in, stride, 0, 0, shift[0]); + load_buffer_16x16(input + 16 * stride, in + 64, stride, 0, 0, shift[0]); + + for (int i = 0; i < 4; i++) { + col_txfm((in + i), (in + i), bitcol, 4); + } + col_txfm_16x16_rounding(&in[0], -shift[1]); + col_txfm_16x16_rounding(&in[64], -shift[1]); + transpose_8nx8n(in, outcoef128, 16, 32); + + // row transform + row_txfm(outcoef128, in, bitrow, 8); + transpose_8nx8n(in, outcoef128, 32, 16); + av1_round_shift_rect_array_32_sse4_1(outcoef128, outcoef128, 128, -shift[2], + NewSqrt2); + (void)bd; +} + +void av1_fwd_txfm2d_32x64_sse4_1(const int16_t *input, int32_t *coeff, + int stride, TX_TYPE tx_type, int bd) { + (void)tx_type; + __m128i in[512]; + __m128i *outcoef128 = (__m128i *)coeff; + const int8_t *shift = fwd_txfm_shift_ls[TX_32X64]; + const int txw_idx = get_txw_idx(TX_32X64); + const int txh_idx = get_txh_idx(TX_32X64); + const int txfm_size_col = tx_size_wide[TX_32X64]; + const int txfm_size_row = tx_size_high[TX_32X64]; + int bitcol = fwd_cos_bit_col[txw_idx][txh_idx]; + int bitrow = fwd_cos_bit_row[txw_idx][txh_idx]; + const int num_row = txfm_size_row >> 2; + const int num_col = txfm_size_col >> 2; + + // column transform + load_buffer_32x8n(input, in, stride, 0, 0, shift[0], txfm_size_row); + for (int i = 0; i < num_col; i++) { + av1_fdct64_new_sse4_1((in + i), (in + i), bitcol, num_col, num_col); + } + for (int i = 0; i < num_col; i++) { + col_txfm_16x16_rounding((in + i * txfm_size_row), -shift[1]); + } + transpose_8nx8n(in, outcoef128, txfm_size_col, txfm_size_row); + + // row transform + for (int i = 0; i < num_row; i++) { + av1_fdct32_new_sse4_1((outcoef128 + i), (in + i), bitrow, num_row); + } + transpose_8nx8n(in, outcoef128, txfm_size_row, txfm_size_col); + av1_round_shift_rect_array_32_sse4_1(outcoef128, outcoef128, 512, -shift[2], + NewSqrt2); + (void)bd; +} + +void av1_fwd_txfm2d_64x32_sse4_1(const int16_t *input, int32_t *coeff, + int stride, TX_TYPE tx_type, int bd) { + (void)tx_type; + __m128i in[512]; + __m128i *outcoef128 = (__m128i *)coeff; + const int8_t *shift = fwd_txfm_shift_ls[TX_64X32]; + const int txw_idx = get_txw_idx(TX_64X32); + const int txh_idx = get_txh_idx(TX_64X32); + const int txfm_size_col = tx_size_wide[TX_64X32]; + const int txfm_size_row = tx_size_high[TX_64X32]; + int bitcol = fwd_cos_bit_col[txw_idx][txh_idx]; + int bitrow = fwd_cos_bit_row[txw_idx][txh_idx]; + const int num_row = txfm_size_row >> 2; + const int num_col = txfm_size_col >> 2; + + // column transform + for (int i = 0; i < 32; i++) { + load_buffer_4x4(input + 0 + i * stride, in + 0 + i * 16, 4, 0, 0, shift[0]); + load_buffer_4x4(input + 16 + i * stride, in + 4 + i * 16, 4, 0, 0, + shift[0]); + load_buffer_4x4(input + 32 + i * stride, in + 8 + i * 16, 4, 0, 0, + shift[0]); + load_buffer_4x4(input + 48 + i * stride, in + 12 + i * 16, 4, 0, 0, + shift[0]); + } + + for (int i = 0; i < num_col; i++) { + av1_fdct32_new_sse4_1((in + i), (in + i), bitcol, num_col); + } + + for (int i = 0; i < num_row; i++) { + col_txfm_16x16_rounding((in + i * txfm_size_col), -shift[1]); + } + transpose_8nx8n(in, outcoef128, txfm_size_col, txfm_size_row); + + // row transform + for (int i = 0; i < num_row; i++) { + av1_fdct64_new_sse4_1((outcoef128 + i), (in + i), bitrow, num_row, num_row); + } + transpose_8nx8n(in, outcoef128, txfm_size_row, txfm_size_col >> 1); + av1_round_shift_rect_array_32_sse4_1(outcoef128, outcoef128, 512 >> 1, + -shift[2], NewSqrt2); + (void)bd; +} + +void av1_fwd_txfm2d_32x16_sse4_1(const int16_t *input, int32_t *coeff, + int stride, TX_TYPE tx_type, int bd) { + __m128i in[128]; + __m128i *outcoef128 = (__m128i *)coeff; + const int8_t *shift = fwd_txfm_shift_ls[TX_32X16]; + const int txw_idx = get_txw_idx(TX_32X16); + const int txh_idx = get_txh_idx(TX_32X16); + const fwd_transform_1d_sse4_1 col_txfm = row_highbd_txfm8x32_arr[tx_type]; + const fwd_transform_1d_sse4_1 row_txfm = col_highbd_txfm8x32_arr[tx_type]; + int bitcol = fwd_cos_bit_col[txw_idx][txh_idx]; + int bitrow = fwd_cos_bit_row[txw_idx][txh_idx]; + + // column transform + load_buffer_32x8n(input, in, stride, 0, 0, shift[0], 16); + col_txfm(in, in, bitcol, 8); + col_txfm_16x16_rounding(&in[0], -shift[1]); + col_txfm_16x16_rounding(&in[64], -shift[1]); + transpose_8nx8n(in, outcoef128, 32, 16); + + // row transform + for (int i = 0; i < 4; i++) { + row_txfm((outcoef128 + i), (in + i), bitrow, 4); + } + transpose_8nx8n(in, outcoef128, 16, 32); + av1_round_shift_rect_array_32_sse4_1(outcoef128, outcoef128, 128, -shift[2], + NewSqrt2); + (void)bd; +} + +void av1_fwd_txfm2d_8x32_sse4_1(const int16_t *input, int32_t *coeff, + int stride, TX_TYPE tx_type, int bd) { + __m128i in[64]; + __m128i *outcoef128 = (__m128i *)coeff; + const int8_t *shift = fwd_txfm_shift_ls[TX_8X32]; + const int txw_idx = get_txw_idx(TX_8X32); + const int txh_idx = get_txh_idx(TX_8X32); + const fwd_transform_1d_sse4_1 col_txfm = col_highbd_txfm8x32_arr[tx_type]; + const fwd_transform_1d_sse4_1 row_txfm = row_highbd_txfm32x8_arr[tx_type]; + int bitcol = fwd_cos_bit_col[txw_idx][txh_idx]; + int bitrow = fwd_cos_bit_row[txw_idx][txh_idx]; + + const int txfm_size_col = tx_size_wide[TX_8X32]; + const int txfm_size_row = tx_size_high[TX_8X32]; + const int num_col = txfm_size_col >> 2; + + // column transform + load_buffer_8x16(input, in, stride, 0, 0, shift[0]); + load_buffer_8x16(input + (txfm_size_row >> 1) * stride, in + txfm_size_row, + stride, 0, 0, shift[0]); + + for (int i = 0; i < num_col; i++) { + col_txfm((in + i), (in + i), bitcol, num_col); + } + col_txfm_16x16_rounding(in, -shift[1]); + transpose_8nx8n(in, outcoef128, txfm_size_col, txfm_size_row); + + // row transform + for (int i = 0; i < txfm_size_col; i += 2) { + row_txfm((outcoef128 + i), (in + i), bitrow, txfm_size_col); + } + transpose_8nx8n(in, outcoef128, txfm_size_row, txfm_size_col); + (void)bd; +} + +void av1_fwd_txfm2d_32x8_sse4_1(const int16_t *input, int32_t *coeff, + int stride, TX_TYPE tx_type, int bd) { + __m128i in[64]; + __m128i *outcoef128 = (__m128i *)coeff; + const int8_t *shift = fwd_txfm_shift_ls[TX_32X8]; + const int txw_idx = get_txw_idx(TX_32X8); + const int txh_idx = get_txh_idx(TX_32X8); + const fwd_transform_1d_sse4_1 col_txfm = row_highbd_txfm32x8_arr[tx_type]; + const fwd_transform_1d_sse4_1 row_txfm = col_highbd_txfm8x32_arr[tx_type]; + int bitcol = fwd_cos_bit_col[txw_idx][txh_idx]; + int bitrow = fwd_cos_bit_row[txw_idx][txh_idx]; + + const int txfm_size_col = tx_size_wide[TX_32X8]; + const int txfm_size_row = tx_size_high[TX_32X8]; + const int num_col = txfm_size_row >> 2; + + // column transform + load_buffer_32x8n(input, in, stride, 0, 0, shift[0], 8); + for (int i = 0; i < txfm_size_row; i += 2) { + col_txfm((in + i), (in + i), bitcol, txfm_size_row); + } + + col_txfm_16x16_rounding(&in[0], -shift[1]); + transpose_8nx8n(in, outcoef128, txfm_size_col, txfm_size_row); + + // row transform + for (int i = 0; i < num_col; i++) { + row_txfm((outcoef128 + i), (in + i), bitrow, num_col); + } + transpose_8nx8n(in, outcoef128, txfm_size_row, txfm_size_col); + (void)bd; +} + +void av1_fwd_txfm2d_4x8_sse4_1(const int16_t *input, int32_t *coeff, int stride, + TX_TYPE tx_type, int bd) { + __m128i in[8]; + __m128i *outcoeff128 = (__m128i *)coeff; + const int8_t *shift = fwd_txfm_shift_ls[TX_4X8]; + const int txw_idx = get_txw_idx(TX_4X8); + const int txh_idx = get_txh_idx(TX_4X8); + const int txfm_size_col = tx_size_wide[TX_4X8]; + const int txfm_size_row = tx_size_high[TX_4X8]; + int bitcol = fwd_cos_bit_col[txw_idx][txh_idx]; + int bitrow = fwd_cos_bit_row[txw_idx][txh_idx]; + const fwd_transform_1d_sse4_1 col_txfm = col_highbd_txfm4x8_arr[tx_type]; + const fwd_transform_1d_sse4_1 row_txfm = row_highbd_txfm4x4_arr[tx_type]; + + int ud_flip, lr_flip; + get_flip_cfg(tx_type, &ud_flip, &lr_flip); + + load_buffer_4x8(input, in, stride, ud_flip, lr_flip, shift[0]); + col_txfm(in, in, bitcol, 1); + col_txfm_4x8_rounding(in, -shift[1]); + transpose_8nx8n(in, outcoeff128, txfm_size_col, txfm_size_row); + + for (int i = 0; i < 2; i++) { + row_txfm(outcoeff128 + i, in + i * txfm_size_col, bitrow, 2); + } + av1_round_shift_rect_array_32_sse4_1(in, outcoeff128, txfm_size_row, + -shift[2], NewSqrt2); + (void)bd; +} + +void av1_fwd_txfm2d_8x4_sse4_1(const int16_t *input, int32_t *coeff, int stride, + TX_TYPE tx_type, int bd) { + __m128i in[8]; + __m128i *outcoeff128 = (__m128i *)coeff; + const int8_t *shift = fwd_txfm_shift_ls[TX_8X4]; + const int txw_idx = get_txw_idx(TX_8X4); + const int txh_idx = get_txh_idx(TX_8X4); + const int txfm_size_col = tx_size_wide[TX_8X4]; + const int txfm_size_row = tx_size_high[TX_8X4]; + int bitcol = fwd_cos_bit_col[txw_idx][txh_idx]; + int bitrow = fwd_cos_bit_row[txw_idx][txh_idx]; + const fwd_transform_1d_sse4_1 col_txfm = col_highbd_txfm4x4_arr[tx_type]; + const fwd_transform_1d_sse4_1 row_txfm = row_highbd_txfm4x8_arr[tx_type]; + int ud_flip, lr_flip; + get_flip_cfg(tx_type, &ud_flip, &lr_flip); + // col tranform + load_buffer_8x4(input, in, stride, ud_flip, lr_flip, shift[0]); + for (int i = 0; i < 2; i++) { + col_txfm(in + i * txfm_size_row, in + i * txfm_size_row, bitcol, 1); + } + col_txfm_4x8_rounding(in, -shift[1]); + + // row tranform + row_txfm(in, outcoeff128, bitrow, 1); + av1_round_shift_rect_array_32_sse4_1(outcoeff128, in, txfm_size_col, + -shift[2], NewSqrt2); + transpose_8nx8n(in, outcoeff128, txfm_size_row, txfm_size_col); + (void)bd; +} + +void av1_fwd_txfm2d_16x64_sse4_1(const int16_t *input, int32_t *coeff, + int stride, TX_TYPE tx_type, int bd) { + __m128i in[256]; + __m128i *outcoeff128 = (__m128i *)coeff; + const int8_t *shift = fwd_txfm_shift_ls[TX_16X64]; + const int txw_idx = get_txw_idx(TX_16X64); + const int txh_idx = get_txh_idx(TX_16X64); + const int txfm_size_col = tx_size_wide[TX_16X64]; + const int txfm_size_row = tx_size_high[TX_16X64]; + int bitcol = fwd_cos_bit_col[txw_idx][txh_idx]; + int bitrow = fwd_cos_bit_row[txw_idx][txh_idx]; + int ud_flip, lr_flip; + get_flip_cfg(tx_type, &ud_flip, &lr_flip); + const int num_col = txfm_size_col >> 2; + // col tranform + for (int i = 0; i < txfm_size_row; i += num_col) { + load_buffer_4x4(input + (i + 0) * stride, in + (i + 0) * num_col, num_col, + ud_flip, lr_flip, shift[0]); + load_buffer_4x4(input + (i + 1) * stride, in + (i + 1) * num_col, num_col, + ud_flip, lr_flip, shift[0]); + load_buffer_4x4(input + (i + 2) * stride, in + (i + 2) * num_col, num_col, + ud_flip, lr_flip, shift[0]); + load_buffer_4x4(input + (i + 3) * stride, in + (i + 3) * num_col, num_col, + ud_flip, lr_flip, shift[0]); + } + + for (int i = 0; i < num_col; i++) { + av1_fdct64_new_sse4_1(in + i, outcoeff128 + i, bitcol, num_col, num_col); + } + + col_txfm_16x16_rounding(outcoeff128, -shift[1]); + col_txfm_16x16_rounding(outcoeff128 + 64, -shift[1]); + col_txfm_16x16_rounding(outcoeff128 + 128, -shift[1]); + col_txfm_16x16_rounding(outcoeff128 + 192, -shift[1]); + + transpose_8nx8n(outcoeff128, in, txfm_size_col, 32); + fdct16x16_sse4_1(in, in, bitrow, 8); + transpose_8nx8n(in, outcoeff128, 32, txfm_size_col); + memset(coeff + txfm_size_col * 32, 0, txfm_size_col * 32 * sizeof(*coeff)); + (void)bd; +} + +void av1_fwd_txfm2d_64x16_sse4_1(const int16_t *input, int32_t *coeff, + int stride, TX_TYPE tx_type, int bd) { + __m128i in[256]; + __m128i *outcoeff128 = (__m128i *)coeff; + const int8_t *shift = fwd_txfm_shift_ls[TX_64X16]; + const int txw_idx = get_txw_idx(TX_64X16); + const int txh_idx = get_txh_idx(TX_64X16); + const int txfm_size_col = tx_size_wide[TX_64X16]; + const int txfm_size_row = tx_size_high[TX_64X16]; + int bitcol = fwd_cos_bit_col[txw_idx][txh_idx]; + int bitrow = fwd_cos_bit_row[txw_idx][txh_idx]; + int ud_flip, lr_flip; + get_flip_cfg(tx_type, &ud_flip, &lr_flip); + // col tranform + for (int i = 0; i < txfm_size_row; i++) { + load_buffer_4x4(input + 0 + i * stride, in + 0 + i * txfm_size_row, 4, + ud_flip, lr_flip, shift[0]); + load_buffer_4x4(input + 16 + i * stride, in + 4 + i * txfm_size_row, 4, + ud_flip, lr_flip, shift[0]); + load_buffer_4x4(input + 32 + i * stride, in + 8 + i * txfm_size_row, 4, + ud_flip, lr_flip, shift[0]); + load_buffer_4x4(input + 48 + i * stride, in + 12 + i * txfm_size_row, 4, + ud_flip, lr_flip, shift[0]); + } + + fdct16x16_sse4_1(in, outcoeff128, bitcol, txfm_size_row); + col_txfm_16x16_rounding(outcoeff128, -shift[1]); + col_txfm_16x16_rounding(outcoeff128 + 64, -shift[1]); + col_txfm_16x16_rounding(outcoeff128 + 128, -shift[1]); + col_txfm_16x16_rounding(outcoeff128 + 192, -shift[1]); + + transpose_8nx8n(outcoeff128, in, txfm_size_col, txfm_size_row); + for (int i = 0; i < 4; i++) { + av1_fdct64_new_sse4_1(in + i, in + i, bitrow, 4, 4); + } + transpose_8nx8n(in, outcoeff128, txfm_size_row, 32); + (void)bd; +} diff --git a/third_party/aom/av1/encoder/x86/ml_sse3.c b/third_party/aom/av1/encoder/x86/ml_sse3.c new file mode 100644 index 0000000000..c520c3c356 --- /dev/null +++ b/third_party/aom/av1/encoder/x86/ml_sse3.c @@ -0,0 +1,243 @@ +/* + * Copyright (c) 2018, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include +#include + +#include "config/av1_rtcd.h" +#include "av1/encoder/ml.h" + +// In order to avoid the high-latency of swapping between FPU and SIMD +// operations, we keep the result in a 128-bit register even though we only +// care about a single value. +static void nn_propagate_8to1(const float *const inputs, + const float *const weights, + __m128 *const output) { + const __m128 inputs_h = _mm_loadu_ps(&inputs[4]); + const __m128 inputs_l = _mm_loadu_ps(inputs); + + const __m128 weights_h = _mm_loadu_ps(&weights[4]); + const __m128 weights_l = _mm_loadu_ps(weights); + + const __m128 mul_h = _mm_mul_ps(inputs_h, weights_h); + const __m128 mul_l = _mm_mul_ps(inputs_l, weights_l); + // [7 6 5 4] [3 2 1 0] (weight and input indices) + + const __m128 vadd = _mm_add_ps(mul_l, mul_h); + // [7+3 6+2 5+1 4+0] + const __m128 hadd1 = _mm_hadd_ps(vadd, vadd); + // [7+6+3+2 5+4+1+0 7+6+3+2 5+4+1+0] + const __m128 hadd2 = _mm_hadd_ps(hadd1, hadd1); + // [7+6+5+4+3+2+1+0 7+6+5+4+3+2+1+0 7+6+5+4+3+2+1+0 7+6+5+4+3+2+1+0] + *output = _mm_add_ps(*output, hadd2); +} + +static void nn_propagate_4to1(const float *const inputs, + const float *const weights, + __m128 *const output) { + const __m128 inputs128 = _mm_loadu_ps(inputs); + + const __m128 weights128 = _mm_loadu_ps(weights); + + const __m128 mul = _mm_mul_ps(inputs128, weights128); + // [3 2 1 0] (weight and input indices) + + const __m128 hadd1 = _mm_hadd_ps(mul, mul); + // [3+2 1+0 3+2 1+0] + const __m128 hadd2 = _mm_hadd_ps(hadd1, hadd1); + // [3+2+1+0 3+2+1+0 3+2+1+0 3+2+1+0] + *output = _mm_add_ps(*output, hadd2); +} + +static void nn_propagate_4to4(const float *const inputs, + const float *const weights, __m128 *const outputs, + const int num_inputs) { + const __m128 inputs128 = _mm_loadu_ps(inputs); + + __m128 hadd[2]; + for (int i = 0; i < 2; i++) { // For each pair of outputs + const __m128 weight0 = _mm_loadu_ps(&weights[2 * i * num_inputs]); + const __m128 mul0 = _mm_mul_ps(weight0, inputs128); + const __m128 weight1 = _mm_loadu_ps(&weights[(2 * i + 1) * num_inputs]); + const __m128 mul1 = _mm_mul_ps(weight1, inputs128); + hadd[i] = _mm_hadd_ps(mul0, mul1); + } + // hadd[0] = [7+6 5+4 3+2 1+0] (weight indices) + // hadd[1] = [15+14 13+12 11+10 9+8] + + const __m128 hh = _mm_hadd_ps(hadd[0], hadd[1]); + // [15+14+13+12 11+10+9+8 7+6+5+4 3+2+1+0] + + *outputs = _mm_add_ps(*outputs, hh); +} + +static void nn_propagate_4to8(const float *const inputs, + const float *const weights, __m128 *const out_h, + __m128 *const out_l, const int num_inputs) { + const __m128 inputs128 = _mm_loadu_ps(inputs); + + __m128 hadd[4]; + for (int i = 0; i < 4; i++) { // For each pair of outputs + const __m128 weight0 = _mm_loadu_ps(&weights[2 * i * num_inputs]); + const __m128 weight1 = _mm_loadu_ps(&weights[(2 * i + 1) * num_inputs]); + const __m128 mul0 = _mm_mul_ps(inputs128, weight0); + const __m128 mul1 = _mm_mul_ps(inputs128, weight1); + hadd[i] = _mm_hadd_ps(mul0, mul1); + } + // hadd[0] = [7+6 5+4 3+2 1+0] (weight indices) + // hadd[1] = [15+14 13+12 11+10 9+8] + // hadd[2] = [23+22 21+20 19+18 17+16] + // hadd[3] = [31+30 29+28 27+26 25+24] + + const __m128 hh0 = _mm_hadd_ps(hadd[0], hadd[1]); + // [15+14+13+12 11+10+9+8 7+6+5+4 3+2+1+0] + const __m128 hh1 = _mm_hadd_ps(hadd[2], hadd[3]); + // [31+30+29+28 27+26+25+24 23+22+21+20 19+18+17+16] + + *out_h = _mm_add_ps(*out_h, hh1); + *out_l = _mm_add_ps(*out_l, hh0); +} + +static void nn_propagate_8to4(const float *const inputs, + const float *const weights, __m128 *const outputs, + const int num_inputs) { + const __m128 inputs_h = _mm_loadu_ps(inputs + 4); + const __m128 inputs_l = _mm_loadu_ps(inputs); + // [7 6 5 4] [3 2 1 0] (input indices) + + __m128 add[4]; + for (int i = 0; i < 4; i++) { // For each output: + const __m128 weight_h = _mm_loadu_ps(&weights[i * num_inputs + 4]); + const __m128 weight_l = _mm_loadu_ps(&weights[i * num_inputs]); + const __m128 mul_h = _mm_mul_ps(inputs_h, weight_h); + const __m128 mul_l = _mm_mul_ps(inputs_l, weight_l); + add[i] = _mm_add_ps(mul_l, mul_h); + } + // add[0] = [7+3 6+2 5+1 4+0] + // add[1] = [15+11 14+10 13+9 12+8] + // add[2] = [23+19 22+18 21+17 20+16] + // add[3] = [31+27 30+26 29+25 28+24] + + const __m128 hadd_h = _mm_hadd_ps(add[2], add[3]); + // [31+30+27+26 29+28+25+24 23+22+19+18 21+20+17+16] + const __m128 hadd_l = _mm_hadd_ps(add[0], add[1]); + // [15+14+11+10 13+12+9+8 7+6+3+2 5+4+1+0] + + const __m128 haddhadd = _mm_hadd_ps(hadd_l, hadd_h); + // [31+30+29+28+27+26+25+24 23+22+21+20+19+18+17+16 + // 15+14+13+12+11+10+9+8 7+6+5+4+3+2+1+0] + + *outputs = _mm_add_ps(*outputs, haddhadd); +} + +static void nn_activate8(__m128 *out_h, __m128 *out_l) { + const __m128 zero = _mm_setzero_ps(); + *out_h = _mm_max_ps(*out_h, zero); + *out_l = _mm_max_ps(*out_l, zero); +} + +static void nn_activate4(__m128 *x) { *x = _mm_max_ps(*x, _mm_setzero_ps()); } + +// Calculate prediction based on the given input features and neural net config. +// Assume there are no more than NN_MAX_NODES_PER_LAYER nodes in each hidden +// layer. +void av1_nn_predict_sse3(const float *input_nodes, + const NN_CONFIG *const nn_config, + float *const output) { + float buf[2][NN_MAX_NODES_PER_LAYER]; + int buf_index = 0; + int num_inputs = nn_config->num_inputs; + + // Hidden layers, except the final iteration is the output layer. + for (int layer = 0; layer <= nn_config->num_hidden_layers; layer++) { + const float *layer_weights = nn_config->weights[layer]; + const float *layer_bias = nn_config->bias[layer]; + bool output_layer = (layer == nn_config->num_hidden_layers); + float *const output_nodes = output_layer ? output : buf[buf_index]; + const int num_outputs = output_layer ? nn_config->num_outputs + : nn_config->num_hidden_nodes[layer]; + + if (num_inputs % 4 == 0 && num_outputs % 8 == 0) { + for (int out = 0; out < num_outputs; out += 8) { + __m128 out_h = _mm_loadu_ps(&layer_bias[out + 4]); + __m128 out_l = _mm_loadu_ps(&layer_bias[out]); + for (int in = 0; in < num_inputs; in += 4) { + nn_propagate_4to8(&input_nodes[in], + &layer_weights[out * num_inputs + in], &out_h, + &out_l, num_inputs); + } + if (!output_layer) nn_activate8(&out_h, &out_l); + _mm_storeu_ps(&output_nodes[out + 4], out_h); + _mm_storeu_ps(&output_nodes[out], out_l); + } + } else if (num_inputs % 8 == 0 && num_outputs % 4 == 0) { + for (int out = 0; out < num_outputs; out += 4) { + __m128 outputs = _mm_loadu_ps(&layer_bias[out]); + for (int in = 0; in < num_inputs; in += 8) { + nn_propagate_8to4(&input_nodes[in], + &layer_weights[out * num_inputs + in], &outputs, + num_inputs); + } + if (!output_layer) nn_activate4(&outputs); + _mm_storeu_ps(&output_nodes[out], outputs); + } + } else if (num_inputs % 4 == 0 && num_outputs % 4 == 0) { + for (int out = 0; out < num_outputs; out += 4) { + __m128 outputs = _mm_loadu_ps(&layer_bias[out]); + for (int in = 0; in < num_inputs; in += 4) { + nn_propagate_4to4(&input_nodes[in], + &layer_weights[out * num_inputs + in], &outputs, + num_inputs); + } + if (!output_layer) nn_activate4(&outputs); + _mm_storeu_ps(&output_nodes[out], outputs); + } + } else if (num_inputs % 8 == 0) { + for (int out = 0; out < num_outputs; out++) { + __m128 total = _mm_load1_ps(&layer_bias[out]); + for (int in = 0; in < num_inputs; in += 8) { + nn_propagate_8to1(&input_nodes[in], + &layer_weights[out * num_inputs + in], &total); + } + if (!output_layer) nn_activate4(&total); + output_nodes[out] = _mm_cvtss_f32(total); + } + } else if (num_inputs % 4 == 0) { + for (int out = 0; out < num_outputs; out++) { + __m128 total = _mm_load1_ps(&layer_bias[out]); + for (int in = 0; in < num_inputs; in += 4) { + nn_propagate_4to1(&input_nodes[in], + &layer_weights[out * num_inputs + in], &total); + } + if (!output_layer) nn_activate4(&total); + output_nodes[out] = _mm_cvtss_f32(total); + } + } else { + // Use SSE instructions for scalar operations to avoid the latency of + // swapping between SIMD and FPU modes. + for (int out = 0; out < num_outputs; out++) { + __m128 total = _mm_load1_ps(&layer_bias[out]); + for (int in_node = 0; in_node < num_inputs; in_node++) { + __m128 input = _mm_load1_ps(&input_nodes[in_node]); + __m128 weight = + _mm_load1_ps(&layer_weights[num_inputs * out + in_node]); + total = _mm_add_ps(total, _mm_mul_ps(input, weight)); + } + if (!output_layer) nn_activate4(&total); + output_nodes[out] = _mm_cvtss_f32(total); + } + } + input_nodes = output_nodes; + num_inputs = num_outputs; + buf_index = 1 - buf_index; + } +} diff --git a/third_party/aom/av1/encoder/x86/pickrst_avx2.c b/third_party/aom/av1/encoder/x86/pickrst_avx2.c index 06aaaa7eee..d00fca0594 100644 --- a/third_party/aom/av1/encoder/x86/pickrst_avx2.c +++ b/third_party/aom/av1/encoder/x86/pickrst_avx2.c @@ -64,14 +64,13 @@ static INLINE void acc_stat_win7_one_line_avx2( static INLINE void compute_stats_win7_opt_avx2( const uint8_t *dgd, const uint8_t *src, int h_start, int h_end, int v_start, - int v_end, int dgd_stride, int src_stride, double *M, double *H) { + int v_end, int dgd_stride, int src_stride, int64_t *M, int64_t *H) { int i, j, k, l, m, n; const int wiener_win = WIENER_WIN; const int pixel_count = (h_end - h_start) * (v_end - v_start); const int wiener_win2 = wiener_win * wiener_win; const int wiener_halfwin = (wiener_win >> 1); - const double avg = - find_average(dgd, h_start, h_end, v_start, v_end, dgd_stride); + uint8_t avg = find_average(dgd, h_start, h_end, v_start, v_end, dgd_stride); int32_t M_int32[WIENER_WIN][WIENER_WIN] = { { 0 } }; int64_t M_int64[WIENER_WIN][WIENER_WIN] = { { 0 } }; @@ -103,23 +102,281 @@ static INLINE void compute_stats_win7_opt_avx2( } } - const double avg_square_sum = avg * avg * pixel_count; + const int64_t avg_square_sum = (int64_t)avg * (int64_t)avg * pixel_count; for (k = 0; k < wiener_win; k++) { for (l = 0; l < wiener_win; l++) { const int32_t idx0 = l * wiener_win + k; - M[idx0] = M_int64[k][l] + avg_square_sum - avg * (sumX + sumY[k][l]); - double *H_ = H + idx0 * wiener_win2; + M[idx0] = + M_int64[k][l] + (avg_square_sum - (int64_t)avg * (sumX + sumY[k][l])); + int64_t *H_ = H + idx0 * wiener_win2; int64_t *H_int_ = &H_int64[idx0][0]; for (m = 0; m < wiener_win; m++) { for (n = 0; n < wiener_win; n++) { H_[m * wiener_win + n] = H_int_[n * 8 + m] + avg_square_sum - - avg * (sumY[k][l] + sumY[n][m]); + (int64_t)avg * (sumY[k][l] + sumY[n][m]); } } } } } +static INLINE void acc_stat_highbd_avx2(int64_t *dst, const uint16_t *dgd, + const __m256i *shuffle, + const __m256i *dgd_ijkl) { + // Load two 128-bit chunks from dgd + const __m256i s0 = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_loadu_si128((__m128i *)dgd)), + _mm_loadu_si128((__m128i *)(dgd + 4)), 1); + // s0 = [11 10 9 8 7 6 5 4] [7 6 5 4 3 2 1 0] as u16 (values are dgd indices) + // The weird order is so the shuffle stays within 128-bit lanes + + // Shuffle 16x u16 values within lanes according to the mask: + // [0 1 1 2 2 3 3 4] [0 1 1 2 2 3 3 4] + // (Actually we shuffle u8 values as there's no 16-bit shuffle) + const __m256i s1 = _mm256_shuffle_epi8(s0, *shuffle); + // s1 = [8 7 7 6 6 5 5 4] [4 3 3 2 2 1 1 0] as u16 (values are dgd indices) + + // Multiply 16x 16-bit integers in dgd_ijkl and s1, resulting in 16x 32-bit + // integers then horizontally add pairs of these integers resulting in 8x + // 32-bit integers + const __m256i d0 = _mm256_madd_epi16(*dgd_ijkl, s1); + // d0 = [a b c d] [e f g h] as u32 + + // Take the lower-half of d0, extend to u64, add it on to dst (H) + const __m256i d0l = _mm256_cvtepu32_epi64(_mm256_extracti128_si256(d0, 0)); + // d0l = [a b] [c d] as u64 + const __m256i dst0 = yy_loadu_256(dst); + yy_storeu_256(dst, _mm256_add_epi64(d0l, dst0)); + + // Take the upper-half of d0, extend to u64, add it on to dst (H) + const __m256i d0h = _mm256_cvtepu32_epi64(_mm256_extracti128_si256(d0, 1)); + // d0h = [e f] [g h] as u64 + const __m256i dst1 = yy_loadu_256(dst + 4); + yy_storeu_256(dst + 4, _mm256_add_epi64(d0h, dst1)); +} + +static INLINE void acc_stat_highbd_win7_one_line_avx2( + const uint16_t *dgd, const uint16_t *src, int h_start, int h_end, + int dgd_stride, const __m256i *shuffle, int32_t *sumX, + int32_t sumY[WIENER_WIN][WIENER_WIN], int64_t M_int[WIENER_WIN][WIENER_WIN], + int64_t H_int[WIENER_WIN2][WIENER_WIN * 8]) { + int j, k, l; + const int wiener_win = WIENER_WIN; + for (j = h_start; j < h_end; j += 2) { + const uint16_t X1 = src[j]; + const uint16_t X2 = src[j + 1]; + *sumX += X1 + X2; + const uint16_t *dgd_ij = dgd + j; + for (k = 0; k < wiener_win; k++) { + const uint16_t *dgd_ijk = dgd_ij + k * dgd_stride; + for (l = 0; l < wiener_win; l++) { + int64_t *H_ = &H_int[(l * wiener_win + k)][0]; + const uint16_t D1 = dgd_ijk[l]; + const uint16_t D2 = dgd_ijk[l + 1]; + sumY[k][l] += D1 + D2; + M_int[k][l] += D1 * X1 + D2 * X2; + + // Load two u16 values from dgd_ijkl combined as a u32, + // then broadcast to 8x u32 slots of a 256 + const __m256i dgd_ijkl = + _mm256_set1_epi32(*((uint32_t *)(dgd_ijk + l))); + // dgd_ijkl = [y x y x y x y x] [y x y x y x y x] where each is a u16 + + acc_stat_highbd_avx2(H_ + 0 * 8, dgd_ij + 0 * dgd_stride, shuffle, + &dgd_ijkl); + acc_stat_highbd_avx2(H_ + 1 * 8, dgd_ij + 1 * dgd_stride, shuffle, + &dgd_ijkl); + acc_stat_highbd_avx2(H_ + 2 * 8, dgd_ij + 2 * dgd_stride, shuffle, + &dgd_ijkl); + acc_stat_highbd_avx2(H_ + 3 * 8, dgd_ij + 3 * dgd_stride, shuffle, + &dgd_ijkl); + acc_stat_highbd_avx2(H_ + 4 * 8, dgd_ij + 4 * dgd_stride, shuffle, + &dgd_ijkl); + acc_stat_highbd_avx2(H_ + 5 * 8, dgd_ij + 5 * dgd_stride, shuffle, + &dgd_ijkl); + acc_stat_highbd_avx2(H_ + 6 * 8, dgd_ij + 6 * dgd_stride, shuffle, + &dgd_ijkl); + } + } + } +} + +static INLINE void compute_stats_highbd_win7_opt_avx2( + const uint8_t *dgd8, const uint8_t *src8, int h_start, int h_end, + int v_start, int v_end, int dgd_stride, int src_stride, int64_t *M, + int64_t *H, aom_bit_depth_t bit_depth) { + int i, j, k, l, m, n; + const int wiener_win = WIENER_WIN; + const int pixel_count = (h_end - h_start) * (v_end - v_start); + const int wiener_win2 = wiener_win * wiener_win; + const int wiener_halfwin = (wiener_win >> 1); + const uint16_t *src = CONVERT_TO_SHORTPTR(src8); + const uint16_t *dgd = CONVERT_TO_SHORTPTR(dgd8); + const uint16_t avg = + find_average_highbd(dgd, h_start, h_end, v_start, v_end, dgd_stride); + + int64_t M_int[WIENER_WIN][WIENER_WIN] = { { 0 } }; + int64_t H_int[WIENER_WIN2][WIENER_WIN * 8] = { { 0 } }; + int32_t sumY[WIENER_WIN][WIENER_WIN] = { { 0 } }; + int32_t sumX = 0; + const uint16_t *dgd_win = dgd - wiener_halfwin * dgd_stride - wiener_halfwin; + + const __m256i shuffle = yy_loadu_256(g_shuffle_stats_highbd_data); + for (j = v_start; j < v_end; j += 64) { + const int vert_end = AOMMIN(64, v_end - j) + j; + for (i = j; i < vert_end; i++) { + acc_stat_highbd_win7_one_line_avx2( + dgd_win + i * dgd_stride, src + i * src_stride, h_start, h_end, + dgd_stride, &shuffle, &sumX, sumY, M_int, H_int); + } + } + + uint8_t bit_depth_divider = 1; + if (bit_depth == AOM_BITS_12) + bit_depth_divider = 16; + else if (bit_depth == AOM_BITS_10) + bit_depth_divider = 4; + + const int64_t avg_square_sum = (int64_t)avg * (int64_t)avg * pixel_count; + for (k = 0; k < wiener_win; k++) { + for (l = 0; l < wiener_win; l++) { + const int32_t idx0 = l * wiener_win + k; + M[idx0] = (M_int[k][l] + + (avg_square_sum - (int64_t)avg * (sumX + sumY[k][l]))) / + bit_depth_divider; + int64_t *H_ = H + idx0 * wiener_win2; + int64_t *H_int_ = &H_int[idx0][0]; + for (m = 0; m < wiener_win; m++) { + for (n = 0; n < wiener_win; n++) { + H_[m * wiener_win + n] = + (H_int_[n * 8 + m] + + (avg_square_sum - (int64_t)avg * (sumY[k][l] + sumY[n][m]))) / + bit_depth_divider; + } + } + } + } +} + +static INLINE void acc_stat_highbd_win5_one_line_avx2( + const uint16_t *dgd, const uint16_t *src, int h_start, int h_end, + int dgd_stride, const __m256i *shuffle, int32_t *sumX, + int32_t sumY[WIENER_WIN_CHROMA][WIENER_WIN_CHROMA], + int64_t M_int[WIENER_WIN_CHROMA][WIENER_WIN_CHROMA], + int64_t H_int[WIENER_WIN2_CHROMA][WIENER_WIN_CHROMA * 8]) { + int j, k, l; + const int wiener_win = WIENER_WIN_CHROMA; + for (j = h_start; j < h_end; j += 2) { + const uint16_t X1 = src[j]; + const uint16_t X2 = src[j + 1]; + *sumX += X1 + X2; + const uint16_t *dgd_ij = dgd + j; + for (k = 0; k < wiener_win; k++) { + const uint16_t *dgd_ijk = dgd_ij + k * dgd_stride; + for (l = 0; l < wiener_win; l++) { + int64_t *H_ = &H_int[(l * wiener_win + k)][0]; + const uint16_t D1 = dgd_ijk[l]; + const uint16_t D2 = dgd_ijk[l + 1]; + sumY[k][l] += D1 + D2; + M_int[k][l] += D1 * X1 + D2 * X2; + + // Load two u16 values from dgd_ijkl combined as a u32, + // then broadcast to 8x u32 slots of a 256 + const __m256i dgd_ijkl = + _mm256_set1_epi32(*((uint32_t *)(dgd_ijk + l))); + // dgd_ijkl = [x y x y x y x y] [x y x y x y x y] where each is a u16 + + acc_stat_highbd_avx2(H_ + 0 * 8, dgd_ij + 0 * dgd_stride, shuffle, + &dgd_ijkl); + acc_stat_highbd_avx2(H_ + 1 * 8, dgd_ij + 1 * dgd_stride, shuffle, + &dgd_ijkl); + acc_stat_highbd_avx2(H_ + 2 * 8, dgd_ij + 2 * dgd_stride, shuffle, + &dgd_ijkl); + acc_stat_highbd_avx2(H_ + 3 * 8, dgd_ij + 3 * dgd_stride, shuffle, + &dgd_ijkl); + acc_stat_highbd_avx2(H_ + 4 * 8, dgd_ij + 4 * dgd_stride, shuffle, + &dgd_ijkl); + } + } + } +} + +static INLINE void compute_stats_highbd_win5_opt_avx2( + const uint8_t *dgd8, const uint8_t *src8, int h_start, int h_end, + int v_start, int v_end, int dgd_stride, int src_stride, int64_t *M, + int64_t *H, aom_bit_depth_t bit_depth) { + int i, j, k, l, m, n; + const int wiener_win = WIENER_WIN_CHROMA; + const int pixel_count = (h_end - h_start) * (v_end - v_start); + const int wiener_win2 = wiener_win * wiener_win; + const int wiener_halfwin = (wiener_win >> 1); + const uint16_t *src = CONVERT_TO_SHORTPTR(src8); + const uint16_t *dgd = CONVERT_TO_SHORTPTR(dgd8); + const uint16_t avg = + find_average_highbd(dgd, h_start, h_end, v_start, v_end, dgd_stride); + + int64_t M_int64[WIENER_WIN_CHROMA][WIENER_WIN_CHROMA] = { { 0 } }; + int64_t H_int64[WIENER_WIN2_CHROMA][WIENER_WIN_CHROMA * 8] = { { 0 } }; + int32_t sumY[WIENER_WIN_CHROMA][WIENER_WIN_CHROMA] = { { 0 } }; + int32_t sumX = 0; + const uint16_t *dgd_win = dgd - wiener_halfwin * dgd_stride - wiener_halfwin; + + const __m256i shuffle = yy_loadu_256(g_shuffle_stats_highbd_data); + for (j = v_start; j < v_end; j += 64) { + const int vert_end = AOMMIN(64, v_end - j) + j; + for (i = j; i < vert_end; i++) { + acc_stat_highbd_win5_one_line_avx2( + dgd_win + i * dgd_stride, src + i * src_stride, h_start, h_end, + dgd_stride, &shuffle, &sumX, sumY, M_int64, H_int64); + } + } + + uint8_t bit_depth_divider = 1; + if (bit_depth == AOM_BITS_12) + bit_depth_divider = 16; + else if (bit_depth == AOM_BITS_10) + bit_depth_divider = 4; + + const int64_t avg_square_sum = (int64_t)avg * (int64_t)avg * pixel_count; + for (k = 0; k < wiener_win; k++) { + for (l = 0; l < wiener_win; l++) { + const int32_t idx0 = l * wiener_win + k; + M[idx0] = (M_int64[k][l] + + (avg_square_sum - (int64_t)avg * (sumX + sumY[k][l]))) / + bit_depth_divider; + int64_t *H_ = H + idx0 * wiener_win2; + int64_t *H_int_ = &H_int64[idx0][0]; + for (m = 0; m < wiener_win; m++) { + for (n = 0; n < wiener_win; n++) { + H_[m * wiener_win + n] = + (H_int_[n * 8 + m] + + (avg_square_sum - (int64_t)avg * (sumY[k][l] + sumY[n][m]))) / + bit_depth_divider; + } + } + } + } +} + +void av1_compute_stats_highbd_avx2(int wiener_win, const uint8_t *dgd8, + const uint8_t *src8, int h_start, int h_end, + int v_start, int v_end, int dgd_stride, + int src_stride, int64_t *M, int64_t *H, + aom_bit_depth_t bit_depth) { + if (wiener_win == WIENER_WIN) { + compute_stats_highbd_win7_opt_avx2(dgd8, src8, h_start, h_end, v_start, + v_end, dgd_stride, src_stride, M, H, + bit_depth); + } else if (wiener_win == WIENER_WIN_CHROMA) { + compute_stats_highbd_win5_opt_avx2(dgd8, src8, h_start, h_end, v_start, + v_end, dgd_stride, src_stride, M, H, + bit_depth); + } else { + av1_compute_stats_highbd_c(wiener_win, dgd8, src8, h_start, h_end, v_start, + v_end, dgd_stride, src_stride, M, H, bit_depth); + } +} + static INLINE void acc_stat_win5_one_line_avx2( const uint8_t *dgd, const uint8_t *src, int h_start, int h_end, int dgd_stride, const __m128i *shuffle, int32_t *sumX, @@ -156,14 +413,13 @@ static INLINE void acc_stat_win5_one_line_avx2( static INLINE void compute_stats_win5_opt_avx2( const uint8_t *dgd, const uint8_t *src, int h_start, int h_end, int v_start, - int v_end, int dgd_stride, int src_stride, double *M, double *H) { + int v_end, int dgd_stride, int src_stride, int64_t *M, int64_t *H) { int i, j, k, l, m, n; const int wiener_win = WIENER_WIN_CHROMA; const int pixel_count = (h_end - h_start) * (v_end - v_start); const int wiener_win2 = wiener_win * wiener_win; const int wiener_halfwin = (wiener_win >> 1); - const double avg = - find_average(dgd, h_start, h_end, v_start, v_end, dgd_stride); + uint8_t avg = find_average(dgd, h_start, h_end, v_start, v_end, dgd_stride); int32_t M_int32[WIENER_WIN_CHROMA][WIENER_WIN_CHROMA] = { { 0 } }; int64_t M_int64[WIENER_WIN_CHROMA][WIENER_WIN_CHROMA] = { { 0 } }; @@ -195,17 +451,18 @@ static INLINE void compute_stats_win5_opt_avx2( } } - const double avg_square_sum = avg * avg * pixel_count; + const int64_t avg_square_sum = (int64_t)avg * (int64_t)avg * pixel_count; for (k = 0; k < wiener_win; k++) { for (l = 0; l < wiener_win; l++) { const int32_t idx0 = l * wiener_win + k; - M[idx0] = M_int64[k][l] + avg_square_sum - avg * (sumX + sumY[k][l]); - double *H_ = H + idx0 * wiener_win2; + M[idx0] = + M_int64[k][l] + (avg_square_sum - (int64_t)avg * (sumX + sumY[k][l])); + int64_t *H_ = H + idx0 * wiener_win2; int64_t *H_int_ = &H_int64[idx0][0]; for (m = 0; m < wiener_win; m++) { for (n = 0; n < wiener_win; n++) { H_[m * wiener_win + n] = H_int_[n * 8 + m] + avg_square_sum - - avg * (sumY[k][l] + sumY[n][m]); + (int64_t)avg * (sumY[k][l] + sumY[n][m]); } } } @@ -215,7 +472,7 @@ static INLINE void compute_stats_win5_opt_avx2( void av1_compute_stats_avx2(int wiener_win, const uint8_t *dgd, const uint8_t *src, int h_start, int h_end, int v_start, int v_end, int dgd_stride, - int src_stride, double *M, double *H) { + int src_stride, int64_t *M, int64_t *H) { if (wiener_win == WIENER_WIN) { compute_stats_win7_opt_avx2(dgd, src, h_start, h_end, v_start, v_end, dgd_stride, src_stride, M, H); @@ -279,7 +536,7 @@ int64_t av1_lowbd_pixel_proj_error_avx2( const int32_t u = (int32_t)(dat[k] << SGRPROJ_RST_BITS); int32_t v = xq[0] * (flt0[k] - u) + xq[1] * (flt1[k] - u); const int32_t e = ROUND_POWER_OF_TWO(v, shift) + dat[k] - src[k]; - err += e * e; + err += ((int64_t)e * e); } dat += dat_stride; src += src_stride; @@ -292,22 +549,25 @@ int64_t av1_lowbd_pixel_proj_error_avx2( sum64 = _mm256_add_epi64(sum64, sum64_0); sum64 = _mm256_add_epi64(sum64, sum64_1); } - } else if (params->r[0] > 0) { - __m256i xq_coeff = - pair_set_epi16(xq[0], (-xq[0] * (1 << SGRPROJ_RST_BITS))); + } else if (params->r[0] > 0 || params->r[1] > 0) { + const int xq_active = (params->r[0] > 0) ? xq[0] : xq[1]; + const __m256i xq_coeff = + pair_set_epi16(xq_active, (-xq_active * (1 << SGRPROJ_RST_BITS))); + const int32_t *flt = (params->r[0] > 0) ? flt0 : flt1; + const int flt_stride = (params->r[0] > 0) ? flt0_stride : flt1_stride; for (i = 0; i < height; ++i) { __m256i sum32 = _mm256_setzero_si256(); for (j = 0; j <= width - 16; j += 16) { const __m256i d0 = _mm256_cvtepu8_epi16(xx_loadu_128(dat + j)); const __m256i s0 = _mm256_cvtepu8_epi16(xx_loadu_128(src + j)); - const __m256i flt0_16b = _mm256_permute4x64_epi64( - _mm256_packs_epi32(yy_loadu_256(flt0 + j), - yy_loadu_256(flt0 + j + 8)), + const __m256i flt_16b = _mm256_permute4x64_epi64( + _mm256_packs_epi32(yy_loadu_256(flt + j), + yy_loadu_256(flt + j + 8)), 0xd8); const __m256i v0 = - _mm256_madd_epi16(xq_coeff, _mm256_unpacklo_epi16(flt0_16b, d0)); + _mm256_madd_epi16(xq_coeff, _mm256_unpacklo_epi16(flt_16b, d0)); const __m256i v1 = - _mm256_madd_epi16(xq_coeff, _mm256_unpackhi_epi16(flt0_16b, d0)); + _mm256_madd_epi16(xq_coeff, _mm256_unpackhi_epi16(flt_16b, d0)); const __m256i vr0 = _mm256_srai_epi32(_mm256_add_epi32(v0, rounding), shift); const __m256i vr1 = @@ -319,53 +579,13 @@ int64_t av1_lowbd_pixel_proj_error_avx2( } for (k = j; k < width; ++k) { const int32_t u = (int32_t)(dat[k] << SGRPROJ_RST_BITS); - int32_t v = xq[0] * (flt0[k] - u); + int32_t v = xq_active * (flt[k] - u); const int32_t e = ROUND_POWER_OF_TWO(v, shift) + dat[k] - src[k]; - err += e * e; + err += ((int64_t)e * e); } dat += dat_stride; src += src_stride; - flt0 += flt0_stride; - const __m256i sum64_0 = - _mm256_cvtepi32_epi64(_mm256_castsi256_si128(sum32)); - const __m256i sum64_1 = - _mm256_cvtepi32_epi64(_mm256_extracti128_si256(sum32, 1)); - sum64 = _mm256_add_epi64(sum64, sum64_0); - sum64 = _mm256_add_epi64(sum64, sum64_1); - } - } else if (params->r[1] > 0) { - __m256i xq_coeff = pair_set_epi16(xq[1], -(xq[1] << SGRPROJ_RST_BITS)); - for (i = 0; i < height; ++i) { - __m256i sum32 = _mm256_setzero_si256(); - for (j = 0; j <= width - 16; j += 16) { - const __m256i d0 = _mm256_cvtepu8_epi16(xx_loadu_128(dat + j)); - const __m256i s0 = _mm256_cvtepu8_epi16(xx_loadu_128(src + j)); - const __m256i flt1_16b = _mm256_permute4x64_epi64( - _mm256_packs_epi32(yy_loadu_256(flt1 + j), - yy_loadu_256(flt1 + j + 8)), - 0xd8); - const __m256i v0 = - _mm256_madd_epi16(xq_coeff, _mm256_unpacklo_epi16(flt1_16b, d0)); - const __m256i v1 = - _mm256_madd_epi16(xq_coeff, _mm256_unpackhi_epi16(flt1_16b, d0)); - const __m256i vr0 = - _mm256_srai_epi32(_mm256_add_epi32(v0, rounding), shift); - const __m256i vr1 = - _mm256_srai_epi32(_mm256_add_epi32(v1, rounding), shift); - const __m256i e0 = _mm256_sub_epi16( - _mm256_add_epi16(_mm256_packs_epi32(vr0, vr1), d0), s0); - const __m256i err0 = _mm256_madd_epi16(e0, e0); - sum32 = _mm256_add_epi32(sum32, err0); - } - for (k = j; k < width; ++k) { - const int32_t u = (int32_t)(dat[k] << SGRPROJ_RST_BITS); - int32_t v = xq[1] * (flt1[k] - u); - const int32_t e = ROUND_POWER_OF_TWO(v, shift) + dat[k] - src[k]; - err += e * e; - } - dat += dat_stride; - src += src_stride; - flt1 += flt1_stride; + flt += flt_stride; const __m256i sum64_0 = _mm256_cvtepi32_epi64(_mm256_castsi256_si128(sum32)); const __m256i sum64_1 = @@ -385,7 +605,7 @@ int64_t av1_lowbd_pixel_proj_error_avx2( } for (k = j; k < width; ++k) { const int32_t e = (int32_t)(dat[k]) - src[k]; - err += e * e; + err += ((int64_t)e * e); } dat += dat_stride; src += src_stride; @@ -401,3 +621,223 @@ int64_t av1_lowbd_pixel_proj_error_avx2( err += sum[0] + sum[1] + sum[2] + sum[3]; return err; } + +int64_t av1_highbd_pixel_proj_error_avx2( + const uint8_t *src8, int width, int height, int src_stride, + const uint8_t *dat8, int dat_stride, int32_t *flt0, int flt0_stride, + int32_t *flt1, int flt1_stride, int xq[2], const sgr_params_type *params) { + int i, j, k; + const int32_t shift = SGRPROJ_RST_BITS + SGRPROJ_PRJ_BITS; + const __m256i rounding = _mm256_set1_epi32(1 << (shift - 1)); + __m256i sum64 = _mm256_setzero_si256(); + const uint16_t *src = CONVERT_TO_SHORTPTR(src8); + const uint16_t *dat = CONVERT_TO_SHORTPTR(dat8); + int64_t err = 0; + if (params->r[0] > 0 && params->r[1] > 0) { // Both filters are enabled + const __m256i xq0 = _mm256_set1_epi32(xq[0]); + const __m256i xq1 = _mm256_set1_epi32(xq[1]); + for (i = 0; i < height; ++i) { + __m256i sum32 = _mm256_setzero_si256(); + for (j = 0; j <= width - 16; j += 16) { // Process 16 pixels at a time + // Load 16 pixels each from source image and corrupted image + const __m256i s0 = yy_loadu_256(src + j); + const __m256i d0 = yy_loadu_256(dat + j); + // s0 = [15 14 13 12 11 10 9 8] [7 6 5 4 3 2 1 0] as u16 (indices) + + // Shift-up each pixel to match filtered image scaling + const __m256i u0 = _mm256_slli_epi16(d0, SGRPROJ_RST_BITS); + + // Split u0 into two halves and pad each from u16 to i32 + const __m256i u0l = _mm256_cvtepu16_epi32(_mm256_castsi256_si128(u0)); + const __m256i u0h = + _mm256_cvtepu16_epi32(_mm256_extracti128_si256(u0, 1)); + // u0h, u0l = [15 14 13 12] [11 10 9 8], [7 6 5 4] [3 2 1 0] as u32 + + // Load 16 pixels from each filtered image + const __m256i flt0l = yy_loadu_256(flt0 + j); + const __m256i flt0h = yy_loadu_256(flt0 + j + 8); + const __m256i flt1l = yy_loadu_256(flt1 + j); + const __m256i flt1h = yy_loadu_256(flt1 + j + 8); + // flt?l, flt?h = [15 14 13 12] [11 10 9 8], [7 6 5 4] [3 2 1 0] as u32 + + // Subtract shifted corrupt image from each filtered image + const __m256i flt0l_subu = _mm256_sub_epi32(flt0l, u0l); + const __m256i flt0h_subu = _mm256_sub_epi32(flt0h, u0h); + const __m256i flt1l_subu = _mm256_sub_epi32(flt1l, u0l); + const __m256i flt1h_subu = _mm256_sub_epi32(flt1h, u0h); + + // Multiply basis vectors by appropriate coefficients + const __m256i v0l = _mm256_mullo_epi32(flt0l_subu, xq0); + const __m256i v0h = _mm256_mullo_epi32(flt0h_subu, xq0); + const __m256i v1l = _mm256_mullo_epi32(flt1l_subu, xq1); + const __m256i v1h = _mm256_mullo_epi32(flt1h_subu, xq1); + + // Add together the contributions from the two basis vectors + const __m256i vl = _mm256_add_epi32(v0l, v1l); + const __m256i vh = _mm256_add_epi32(v0h, v1h); + + // Right-shift v with appropriate rounding + const __m256i vrl = + _mm256_srai_epi32(_mm256_add_epi32(vl, rounding), shift); + const __m256i vrh = + _mm256_srai_epi32(_mm256_add_epi32(vh, rounding), shift); + // vrh, vrl = [15 14 13 12] [11 10 9 8], [7 6 5 4] [3 2 1 0] + + // Saturate each i32 to an i16 then combine both halves + // The permute (control=[3 1 2 0]) fixes weird ordering from AVX lanes + const __m256i vr = + _mm256_permute4x64_epi64(_mm256_packs_epi32(vrl, vrh), 0xd8); + // intermediate = [15 14 13 12 7 6 5 4] [11 10 9 8 3 2 1 0] + // vr = [15 14 13 12 11 10 9 8] [7 6 5 4 3 2 1 0] + + // Add twin-subspace-sgr-filter to corrupt image then subtract source + const __m256i e0 = _mm256_sub_epi16(_mm256_add_epi16(vr, d0), s0); + + // Calculate squared error and add adjacent values + const __m256i err0 = _mm256_madd_epi16(e0, e0); + + sum32 = _mm256_add_epi32(sum32, err0); + } + + const __m256i sum32l = + _mm256_cvtepu32_epi64(_mm256_castsi256_si128(sum32)); + sum64 = _mm256_add_epi64(sum64, sum32l); + const __m256i sum32h = + _mm256_cvtepu32_epi64(_mm256_extracti128_si256(sum32, 1)); + sum64 = _mm256_add_epi64(sum64, sum32h); + + // Process remaining pixels in this row (modulo 16) + for (k = j; k < width; ++k) { + const int32_t u = (int32_t)(dat[k] << SGRPROJ_RST_BITS); + int32_t v = xq[0] * (flt0[k] - u) + xq[1] * (flt1[k] - u); + const int32_t e = ROUND_POWER_OF_TWO(v, shift) + dat[k] - src[k]; + err += ((int64_t)e * e); + } + dat += dat_stride; + src += src_stride; + flt0 += flt0_stride; + flt1 += flt1_stride; + } + } else if (params->r[0] > 0 || params->r[1] > 0) { // Only one filter enabled + const int32_t xq_on = (params->r[0] > 0) ? xq[0] : xq[1]; + const __m256i xq_active = _mm256_set1_epi32(xq_on); + const __m256i xq_inactive = + _mm256_set1_epi32(-xq_on * (1 << SGRPROJ_RST_BITS)); + const int32_t *flt = (params->r[0] > 0) ? flt0 : flt1; + const int flt_stride = (params->r[0] > 0) ? flt0_stride : flt1_stride; + for (i = 0; i < height; ++i) { + __m256i sum32 = _mm256_setzero_si256(); + for (j = 0; j <= width - 16; j += 16) { + // Load 16 pixels from source image + const __m256i s0 = yy_loadu_256(src + j); + // s0 = [15 14 13 12 11 10 9 8] [7 6 5 4 3 2 1 0] as u16 + + // Load 16 pixels from corrupted image and pad each u16 to i32 + const __m256i d0 = yy_loadu_256(dat + j); + const __m256i d0h = + _mm256_cvtepu16_epi32(_mm256_extracti128_si256(d0, 1)); + const __m256i d0l = _mm256_cvtepu16_epi32(_mm256_castsi256_si128(d0)); + // d0 = [15 14 13 12 11 10 9 8] [7 6 5 4 3 2 1 0] as u16 + // d0h, d0l = [15 14 13 12] [11 10 9 8], [7 6 5 4] [3 2 1 0] as i32 + + // Load 16 pixels from the filtered image + const __m256i flth = yy_loadu_256(flt + j + 8); + const __m256i fltl = yy_loadu_256(flt + j); + // flth, fltl = [15 14 13 12] [11 10 9 8], [7 6 5 4] [3 2 1 0] as i32 + + const __m256i flth_xq = _mm256_mullo_epi32(flth, xq_active); + const __m256i fltl_xq = _mm256_mullo_epi32(fltl, xq_active); + const __m256i d0h_xq = _mm256_mullo_epi32(d0h, xq_inactive); + const __m256i d0l_xq = _mm256_mullo_epi32(d0l, xq_inactive); + + const __m256i vh = _mm256_add_epi32(flth_xq, d0h_xq); + const __m256i vl = _mm256_add_epi32(fltl_xq, d0l_xq); + + // Shift this down with appropriate rounding + const __m256i vrh = + _mm256_srai_epi32(_mm256_add_epi32(vh, rounding), shift); + const __m256i vrl = + _mm256_srai_epi32(_mm256_add_epi32(vl, rounding), shift); + // vrh, vrl = [15 14 13 12] [11 10 9 8], [7 6 5 4] [3 2 1 0] as i32 + + // Saturate each i32 to an i16 then combine both halves + // The permute (control=[3 1 2 0]) fixes weird ordering from AVX lanes + const __m256i vr = + _mm256_permute4x64_epi64(_mm256_packs_epi32(vrl, vrh), 0xd8); + // intermediate = [15 14 13 12 7 6 5 4] [11 10 9 8 3 2 1 0] as u16 + // vr = [15 14 13 12 11 10 9 8] [7 6 5 4 3 2 1 0] as u16 + + // Subtract twin-subspace-sgr filtered from source image to get error + const __m256i e0 = _mm256_sub_epi16(_mm256_add_epi16(vr, d0), s0); + + // Calculate squared error and add adjacent values + const __m256i err0 = _mm256_madd_epi16(e0, e0); + + sum32 = _mm256_add_epi32(sum32, err0); + } + + const __m256i sum32l = + _mm256_cvtepu32_epi64(_mm256_castsi256_si128(sum32)); + sum64 = _mm256_add_epi64(sum64, sum32l); + const __m256i sum32h = + _mm256_cvtepu32_epi64(_mm256_extracti128_si256(sum32, 1)); + sum64 = _mm256_add_epi64(sum64, sum32h); + + // Process remaining pixels in this row (modulo 16) + for (k = j; k < width; ++k) { + const int32_t u = (int32_t)(dat[k] << SGRPROJ_RST_BITS); + int32_t v = xq_on * (flt[k] - u); + const int32_t e = ROUND_POWER_OF_TWO(v, shift) + dat[k] - src[k]; + err += ((int64_t)e * e); + } + dat += dat_stride; + src += src_stride; + flt += flt_stride; + } + } else { // Neither filter is enabled + for (i = 0; i < height; ++i) { + __m256i sum32 = _mm256_setzero_si256(); + for (j = 0; j <= width - 32; j += 32) { + // Load 2x16 u16 from source image + const __m256i s0l = yy_loadu_256(src + j); + const __m256i s0h = yy_loadu_256(src + j + 16); + + // Load 2x16 u16 from corrupted image + const __m256i d0l = yy_loadu_256(dat + j); + const __m256i d0h = yy_loadu_256(dat + j + 16); + + // Subtract corrupted image from source image + const __m256i diffl = _mm256_sub_epi16(d0l, s0l); + const __m256i diffh = _mm256_sub_epi16(d0h, s0h); + + // Square error and add adjacent values + const __m256i err0l = _mm256_madd_epi16(diffl, diffl); + const __m256i err0h = _mm256_madd_epi16(diffh, diffh); + + sum32 = _mm256_add_epi32(sum32, err0l); + sum32 = _mm256_add_epi32(sum32, err0h); + } + + const __m256i sum32l = + _mm256_cvtepu32_epi64(_mm256_castsi256_si128(sum32)); + sum64 = _mm256_add_epi64(sum64, sum32l); + const __m256i sum32h = + _mm256_cvtepu32_epi64(_mm256_extracti128_si256(sum32, 1)); + sum64 = _mm256_add_epi64(sum64, sum32h); + + // Process remaining pixels (modulu 16) + for (k = j; k < width; ++k) { + const int32_t e = (int32_t)(dat[k]) - src[k]; + err += ((int64_t)e * e); + } + dat += dat_stride; + src += src_stride; + } + } + + // Sum 4 values from sum64l and sum64h into err + int64_t sum[4]; + yy_storeu_256(sum, sum64); + err += sum[0] + sum[1] + sum[2] + sum[3]; + return err; +} diff --git a/third_party/aom/av1/encoder/x86/pickrst_sse4.c b/third_party/aom/av1/encoder/x86/pickrst_sse4.c index 04e4d1afc4..a94e169849 100644 --- a/third_party/aom/av1/encoder/x86/pickrst_sse4.c +++ b/third_party/aom/av1/encoder/x86/pickrst_sse4.c @@ -68,13 +68,13 @@ static INLINE void acc_stat_win7_one_line_sse4_1( static INLINE void compute_stats_win7_opt_sse4_1( const uint8_t *dgd, const uint8_t *src, int h_start, int h_end, int v_start, - int v_end, int dgd_stride, int src_stride, double *M, double *H) { + int v_end, int dgd_stride, int src_stride, int64_t *M, int64_t *H) { int i, j, k, l, m, n; const int wiener_win = WIENER_WIN; const int pixel_count = (h_end - h_start) * (v_end - v_start); const int wiener_win2 = wiener_win * wiener_win; const int wiener_halfwin = (wiener_win >> 1); - const double avg = + const uint8_t avg = find_average(dgd, h_start, h_end, v_start, v_end, dgd_stride); int32_t M_int32[WIENER_WIN][WIENER_WIN] = { { 0 } }; @@ -107,23 +107,286 @@ static INLINE void compute_stats_win7_opt_sse4_1( } } - const double avg_square_sum = avg * avg * pixel_count; + const int64_t avg_square_sum = (int64_t)avg * (int64_t)avg * pixel_count; for (k = 0; k < wiener_win; k++) { for (l = 0; l < wiener_win; l++) { const int32_t idx0 = l * wiener_win + k; - M[idx0] = M_int64[k][l] + avg_square_sum - avg * (sumX + sumY[k][l]); - double *H_ = H + idx0 * wiener_win2; + M[idx0] = + M_int64[k][l] + (avg_square_sum - (int64_t)avg * (sumX + sumY[k][l])); + int64_t *H_ = H + idx0 * wiener_win2; int64_t *H_int_ = &H_int64[idx0][0]; for (m = 0; m < wiener_win; m++) { for (n = 0; n < wiener_win; n++) { H_[m * wiener_win + n] = H_int_[n * 8 + m] + avg_square_sum - - avg * (sumY[k][l] + sumY[n][m]); + (int64_t)avg * (sumY[k][l] + sumY[n][m]); } } } } } +static INLINE void acc_stat_highbd_sse41(int64_t *dst, const uint16_t *dgd, + const __m128i *shuffle, + const __m128i *dgd_ijkl) { + // Load 256 bits from dgd in two chunks + const __m128i s0l = xx_loadu_128(dgd); + const __m128i s0h = xx_loadu_128(dgd + 4); + // s0l = [7 6 5 4 3 2 1 0] as u16 values (dgd indices) + // s0h = [11 10 9 8 7 6 5 4] as u16 values (dgd indices) + // (Slightly strange order so we can apply the same shuffle to both halves) + + // Shuffle the u16 values in each half (actually using 8-bit shuffle mask) + const __m128i s1l = _mm_shuffle_epi8(s0l, *shuffle); + const __m128i s1h = _mm_shuffle_epi8(s0h, *shuffle); + // s1l = [4 3 3 2 2 1 1 0] as u16 values (dgd indices) + // s1h = [8 7 7 6 6 5 5 4] as u16 values (dgd indices) + + // Multiply s1 by dgd_ijkl resulting in 8x u32 values + // Horizontally add pairs of u32 resulting in 4x u32 + const __m128i dl = _mm_madd_epi16(*dgd_ijkl, s1l); + const __m128i dh = _mm_madd_epi16(*dgd_ijkl, s1h); + // dl = [d c b a] as u32 values + // dh = [h g f e] as u32 values + + // Add these 8x u32 results on to dst in four parts + const __m128i dll = _mm_cvtepu32_epi64(dl); + const __m128i dlh = _mm_cvtepu32_epi64(_mm_srli_si128(dl, 8)); + const __m128i dhl = _mm_cvtepu32_epi64(dh); + const __m128i dhh = _mm_cvtepu32_epi64(_mm_srli_si128(dh, 8)); + // dll = [b a] as u64 values, etc. + + const __m128i rll = _mm_add_epi64(xx_loadu_128(dst), dll); + xx_storeu_128(dst, rll); + const __m128i rlh = _mm_add_epi64(xx_loadu_128(dst + 2), dlh); + xx_storeu_128(dst + 2, rlh); + const __m128i rhl = _mm_add_epi64(xx_loadu_128(dst + 4), dhl); + xx_storeu_128(dst + 4, rhl); + const __m128i rhh = _mm_add_epi64(xx_loadu_128(dst + 6), dhh); + xx_storeu_128(dst + 6, rhh); +} + +static INLINE void acc_stat_highbd_win7_one_line_sse4_1( + const uint16_t *dgd, const uint16_t *src, int h_start, int h_end, + int dgd_stride, const __m128i *shuffle, int32_t *sumX, + int32_t sumY[WIENER_WIN][WIENER_WIN], int64_t M_int[WIENER_WIN][WIENER_WIN], + int64_t H_int[WIENER_WIN2][WIENER_WIN * 8]) { + int j, k, l; + const int wiener_win = WIENER_WIN; + for (j = h_start; j < h_end; j += 2) { + const uint16_t X1 = src[j]; + const uint16_t X2 = src[j + 1]; + *sumX += X1 + X2; + const uint16_t *dgd_ij = dgd + j; + for (k = 0; k < wiener_win; k++) { + const uint16_t *dgd_ijk = dgd_ij + k * dgd_stride; + for (l = 0; l < wiener_win; l++) { + int64_t *H_ = &H_int[(l * wiener_win + k)][0]; + const uint16_t D1 = dgd_ijk[l]; + const uint16_t D2 = dgd_ijk[l + 1]; + sumY[k][l] += D1 + D2; + M_int[k][l] += D1 * X1 + D2 * X2; + + // Load two u16 values from dgd as a single u32 + // Then broadcast to 4x u32 slots of a 128 + const __m128i dgd_ijkl = _mm_set1_epi32(*((uint32_t *)(dgd_ijk + l))); + // dgd_ijkl = [y x y x y x y x] as u16 + + acc_stat_highbd_sse41(H_ + 0 * 8, dgd_ij + 0 * dgd_stride, shuffle, + &dgd_ijkl); + acc_stat_highbd_sse41(H_ + 1 * 8, dgd_ij + 1 * dgd_stride, shuffle, + &dgd_ijkl); + acc_stat_highbd_sse41(H_ + 2 * 8, dgd_ij + 2 * dgd_stride, shuffle, + &dgd_ijkl); + acc_stat_highbd_sse41(H_ + 3 * 8, dgd_ij + 3 * dgd_stride, shuffle, + &dgd_ijkl); + acc_stat_highbd_sse41(H_ + 4 * 8, dgd_ij + 4 * dgd_stride, shuffle, + &dgd_ijkl); + acc_stat_highbd_sse41(H_ + 5 * 8, dgd_ij + 5 * dgd_stride, shuffle, + &dgd_ijkl); + acc_stat_highbd_sse41(H_ + 6 * 8, dgd_ij + 6 * dgd_stride, shuffle, + &dgd_ijkl); + } + } + } +} + +static INLINE void compute_stats_highbd_win7_opt_sse4_1( + const uint8_t *dgd8, const uint8_t *src8, int h_start, int h_end, + int v_start, int v_end, int dgd_stride, int src_stride, int64_t *M, + int64_t *H, aom_bit_depth_t bit_depth) { + int i, j, k, l, m, n; + const int wiener_win = WIENER_WIN; + const int pixel_count = (h_end - h_start) * (v_end - v_start); + const int wiener_win2 = wiener_win * wiener_win; + const int wiener_halfwin = (wiener_win >> 1); + const uint16_t *src = CONVERT_TO_SHORTPTR(src8); + const uint16_t *dgd = CONVERT_TO_SHORTPTR(dgd8); + const uint16_t avg = + find_average_highbd(dgd, h_start, h_end, v_start, v_end, dgd_stride); + + int64_t M_int[WIENER_WIN][WIENER_WIN] = { { 0 } }; + int64_t H_int[WIENER_WIN2][WIENER_WIN * 8] = { { 0 } }; + int32_t sumY[WIENER_WIN][WIENER_WIN] = { { 0 } }; + int32_t sumX = 0; + const uint16_t *dgd_win = dgd - wiener_halfwin * dgd_stride - wiener_halfwin; + + // Load just half of the 256-bit shuffle control used for the AVX2 version + const __m128i shuffle = xx_loadu_128(g_shuffle_stats_highbd_data); + for (j = v_start; j < v_end; j += 64) { + const int vert_end = AOMMIN(64, v_end - j) + j; + for (i = j; i < vert_end; i++) { + acc_stat_highbd_win7_one_line_sse4_1( + dgd_win + i * dgd_stride, src + i * src_stride, h_start, h_end, + dgd_stride, &shuffle, &sumX, sumY, M_int, H_int); + } + } + + uint8_t bit_depth_divider = 1; + if (bit_depth == AOM_BITS_12) + bit_depth_divider = 16; + else if (bit_depth == AOM_BITS_10) + bit_depth_divider = 4; + + const int64_t avg_square_sum = (int64_t)avg * (int64_t)avg * pixel_count; + for (k = 0; k < wiener_win; k++) { + for (l = 0; l < wiener_win; l++) { + const int32_t idx0 = l * wiener_win + k; + M[idx0] = (M_int[k][l] + + (avg_square_sum - (int64_t)avg * (sumX + sumY[k][l]))) / + bit_depth_divider; + int64_t *H_ = H + idx0 * wiener_win2; + int64_t *H_int_ = &H_int[idx0][0]; + for (m = 0; m < wiener_win; m++) { + for (n = 0; n < wiener_win; n++) { + H_[m * wiener_win + n] = + (H_int_[n * 8 + m] + + (avg_square_sum - (int64_t)avg * (sumY[k][l] + sumY[n][m]))) / + bit_depth_divider; + } + } + } + } +} + +static INLINE void acc_stat_highbd_win5_one_line_sse4_1( + const uint16_t *dgd, const uint16_t *src, int h_start, int h_end, + int dgd_stride, const __m128i *shuffle, int32_t *sumX, + int32_t sumY[WIENER_WIN_CHROMA][WIENER_WIN_CHROMA], + int64_t M_int[WIENER_WIN_CHROMA][WIENER_WIN_CHROMA], + int64_t H_int[WIENER_WIN2_CHROMA][WIENER_WIN_CHROMA * 8]) { + int j, k, l; + const int wiener_win = WIENER_WIN_CHROMA; + for (j = h_start; j < h_end; j += 2) { + const uint16_t X1 = src[j]; + const uint16_t X2 = src[j + 1]; + *sumX += X1 + X2; + const uint16_t *dgd_ij = dgd + j; + for (k = 0; k < wiener_win; k++) { + const uint16_t *dgd_ijk = dgd_ij + k * dgd_stride; + for (l = 0; l < wiener_win; l++) { + int64_t *H_ = &H_int[(l * wiener_win + k)][0]; + const uint16_t D1 = dgd_ijk[l]; + const uint16_t D2 = dgd_ijk[l + 1]; + sumY[k][l] += D1 + D2; + M_int[k][l] += D1 * X1 + D2 * X2; + + // Load two u16 values from dgd as a single u32 + // then broadcast to 4x u32 slots of a 128 + const __m128i dgd_ijkl = _mm_set1_epi32(*((uint32_t *)(dgd_ijk + l))); + // dgd_ijkl = [y x y x y x y x] as u16 + + acc_stat_highbd_sse41(H_ + 0 * 8, dgd_ij + 0 * dgd_stride, shuffle, + &dgd_ijkl); + acc_stat_highbd_sse41(H_ + 1 * 8, dgd_ij + 1 * dgd_stride, shuffle, + &dgd_ijkl); + acc_stat_highbd_sse41(H_ + 2 * 8, dgd_ij + 2 * dgd_stride, shuffle, + &dgd_ijkl); + acc_stat_highbd_sse41(H_ + 3 * 8, dgd_ij + 3 * dgd_stride, shuffle, + &dgd_ijkl); + acc_stat_highbd_sse41(H_ + 4 * 8, dgd_ij + 4 * dgd_stride, shuffle, + &dgd_ijkl); + } + } + } +} + +static INLINE void compute_stats_highbd_win5_opt_sse4_1( + const uint8_t *dgd8, const uint8_t *src8, int h_start, int h_end, + int v_start, int v_end, int dgd_stride, int src_stride, int64_t *M, + int64_t *H, aom_bit_depth_t bit_depth) { + int i, j, k, l, m, n; + const int wiener_win = WIENER_WIN_CHROMA; + const int pixel_count = (h_end - h_start) * (v_end - v_start); + const int wiener_win2 = wiener_win * wiener_win; + const int wiener_halfwin = (wiener_win >> 1); + const uint16_t *src = CONVERT_TO_SHORTPTR(src8); + const uint16_t *dgd = CONVERT_TO_SHORTPTR(dgd8); + const uint16_t avg = + find_average_highbd(dgd, h_start, h_end, v_start, v_end, dgd_stride); + + int64_t M_int[WIENER_WIN_CHROMA][WIENER_WIN_CHROMA] = { { 0 } }; + int64_t H_int[WIENER_WIN2_CHROMA][WIENER_WIN_CHROMA * 8] = { { 0 } }; + int32_t sumY[WIENER_WIN_CHROMA][WIENER_WIN_CHROMA] = { { 0 } }; + int32_t sumX = 0; + const uint16_t *dgd_win = dgd - wiener_halfwin * dgd_stride - wiener_halfwin; + + // Load just half of the 256-bit shuffle control used for the AVX2 version + const __m128i shuffle = xx_loadu_128(g_shuffle_stats_highbd_data); + for (j = v_start; j < v_end; j += 64) { + const int vert_end = AOMMIN(64, v_end - j) + j; + for (i = j; i < vert_end; i++) { + acc_stat_highbd_win5_one_line_sse4_1( + dgd_win + i * dgd_stride, src + i * src_stride, h_start, h_end, + dgd_stride, &shuffle, &sumX, sumY, M_int, H_int); + } + } + + uint8_t bit_depth_divider = 1; + if (bit_depth == AOM_BITS_12) + bit_depth_divider = 16; + else if (bit_depth == AOM_BITS_10) + bit_depth_divider = 4; + + const int64_t avg_square_sum = (int64_t)avg * (int64_t)avg * pixel_count; + for (k = 0; k < wiener_win; k++) { + for (l = 0; l < wiener_win; l++) { + const int32_t idx0 = l * wiener_win + k; + M[idx0] = (M_int[k][l] + + (avg_square_sum - (int64_t)avg * (sumX + sumY[k][l]))) / + bit_depth_divider; + int64_t *H_ = H + idx0 * wiener_win2; + int64_t *H_int_ = &H_int[idx0][0]; + for (m = 0; m < wiener_win; m++) { + for (n = 0; n < wiener_win; n++) { + H_[m * wiener_win + n] = + (H_int_[n * 8 + m] + + (avg_square_sum - (int64_t)avg * (sumY[k][l] + sumY[n][m]))) / + bit_depth_divider; + } + } + } + } +} + +void av1_compute_stats_highbd_sse4_1(int wiener_win, const uint8_t *dgd8, + const uint8_t *src8, int h_start, + int h_end, int v_start, int v_end, + int dgd_stride, int src_stride, int64_t *M, + int64_t *H, aom_bit_depth_t bit_depth) { + if (wiener_win == WIENER_WIN) { + compute_stats_highbd_win7_opt_sse4_1(dgd8, src8, h_start, h_end, v_start, + v_end, dgd_stride, src_stride, M, H, + bit_depth); + } else if (wiener_win == WIENER_WIN_CHROMA) { + compute_stats_highbd_win5_opt_sse4_1(dgd8, src8, h_start, h_end, v_start, + v_end, dgd_stride, src_stride, M, H, + bit_depth); + } else { + av1_compute_stats_highbd_c(wiener_win, dgd8, src8, h_start, h_end, v_start, + v_end, dgd_stride, src_stride, M, H, bit_depth); + } +} + static INLINE void acc_stat_win5_one_line_sse4_1( const uint8_t *dgd, const uint8_t *src, int h_start, int h_end, int dgd_stride, const __m128i *shuffle, int32_t *sumX, @@ -160,13 +423,13 @@ static INLINE void acc_stat_win5_one_line_sse4_1( static INLINE void compute_stats_win5_opt_sse4_1( const uint8_t *dgd, const uint8_t *src, int h_start, int h_end, int v_start, - int v_end, int dgd_stride, int src_stride, double *M, double *H) { + int v_end, int dgd_stride, int src_stride, int64_t *M, int64_t *H) { int i, j, k, l, m, n; const int wiener_win = WIENER_WIN_CHROMA; const int pixel_count = (h_end - h_start) * (v_end - v_start); const int wiener_win2 = wiener_win * wiener_win; const int wiener_halfwin = (wiener_win >> 1); - const double avg = + const uint8_t avg = find_average(dgd, h_start, h_end, v_start, v_end, dgd_stride); int32_t M_int32[WIENER_WIN_CHROMA][WIENER_WIN_CHROMA] = { { 0 } }; @@ -199,17 +462,18 @@ static INLINE void compute_stats_win5_opt_sse4_1( } } - const double avg_square_sum = avg * avg * pixel_count; + const int64_t avg_square_sum = (int64_t)avg * (int64_t)avg * pixel_count; for (k = 0; k < wiener_win; k++) { for (l = 0; l < wiener_win; l++) { const int32_t idx0 = l * wiener_win + k; - M[idx0] = M_int64[k][l] + avg_square_sum - avg * (sumX + sumY[k][l]); - double *H_ = H + idx0 * wiener_win2; + M[idx0] = + M_int64[k][l] + (avg_square_sum - (int64_t)avg * (sumX + sumY[k][l])); + int64_t *H_ = H + idx0 * wiener_win2; int64_t *H_int_ = &H_int64[idx0][0]; for (m = 0; m < wiener_win; m++) { for (n = 0; n < wiener_win; n++) { H_[m * wiener_win + n] = H_int_[n * 8 + m] + avg_square_sum - - avg * (sumY[k][l] + sumY[n][m]); + (int64_t)avg * (sumY[k][l] + sumY[n][m]); } } } @@ -218,7 +482,7 @@ static INLINE void compute_stats_win5_opt_sse4_1( void av1_compute_stats_sse4_1(int wiener_win, const uint8_t *dgd, const uint8_t *src, int h_start, int h_end, int v_start, int v_end, int dgd_stride, - int src_stride, double *M, double *H) { + int src_stride, int64_t *M, int64_t *H) { if (wiener_win == WIENER_WIN) { compute_stats_win7_opt_sse4_1(dgd, src, h_start, h_end, v_start, v_end, dgd_stride, src_stride, M, H); @@ -250,7 +514,7 @@ int64_t av1_lowbd_pixel_proj_error_sse4_1( __m128i xq_coeff = pair_set_epi16(xq[0], xq[1]); for (i = 0; i < height; ++i) { __m128i sum32 = _mm_setzero_si128(); - for (j = 0; j < width - 8; j += 8) { + for (j = 0; j <= width - 8; j += 8) { const __m128i d0 = _mm_cvtepu8_epi16(xx_loadl_64(dat + j)); const __m128i s0 = _mm_cvtepu8_epi16(xx_loadl_64(src + j)); const __m128i flt0_16b = @@ -275,7 +539,7 @@ int64_t av1_lowbd_pixel_proj_error_sse4_1( const int32_t u = (int32_t)(dat[k] << SGRPROJ_RST_BITS); int32_t v = xq[0] * (flt0[k] - u) + xq[1] * (flt1[k] - u); const int32_t e = ROUND_POWER_OF_TWO(v, shift) + dat[k] - src[k]; - err += e * e; + err += ((int64_t)e * e); } dat += dat_stride; src += src_stride; @@ -286,19 +550,23 @@ int64_t av1_lowbd_pixel_proj_error_sse4_1( sum64 = _mm_add_epi64(sum64, sum64_0); sum64 = _mm_add_epi64(sum64, sum64_1); } - } else if (params->r[0] > 0) { - __m128i xq_coeff = pair_set_epi16(xq[0], -(xq[0] << SGRPROJ_RST_BITS)); + } else if (params->r[0] > 0 || params->r[1] > 0) { + const int xq_active = (params->r[0] > 0) ? xq[0] : xq[1]; + const __m128i xq_coeff = + pair_set_epi16(xq_active, -(xq_active << SGRPROJ_RST_BITS)); + const int32_t *flt = (params->r[0] > 0) ? flt0 : flt1; + const int flt_stride = (params->r[0] > 0) ? flt0_stride : flt1_stride; for (i = 0; i < height; ++i) { __m128i sum32 = _mm_setzero_si128(); - for (j = 0; j < width - 8; j += 8) { + for (j = 0; j <= width - 8; j += 8) { const __m128i d0 = _mm_cvtepu8_epi16(xx_loadl_64(dat + j)); const __m128i s0 = _mm_cvtepu8_epi16(xx_loadl_64(src + j)); - const __m128i flt0_16b = - _mm_packs_epi32(xx_loadu_128(flt0 + j), xx_loadu_128(flt0 + j + 4)); + const __m128i flt_16b = + _mm_packs_epi32(xx_loadu_128(flt + j), xx_loadu_128(flt + j + 4)); const __m128i v0 = - _mm_madd_epi16(xq_coeff, _mm_unpacklo_epi16(flt0_16b, d0)); + _mm_madd_epi16(xq_coeff, _mm_unpacklo_epi16(flt_16b, d0)); const __m128i v1 = - _mm_madd_epi16(xq_coeff, _mm_unpackhi_epi16(flt0_16b, d0)); + _mm_madd_epi16(xq_coeff, _mm_unpackhi_epi16(flt_16b, d0)); const __m128i vr0 = _mm_srai_epi32(_mm_add_epi32(v0, rounding), shift); const __m128i vr1 = _mm_srai_epi32(_mm_add_epi32(v1, rounding), shift); const __m128i e0 = @@ -308,47 +576,13 @@ int64_t av1_lowbd_pixel_proj_error_sse4_1( } for (k = j; k < width; ++k) { const int32_t u = (int32_t)(dat[k] << SGRPROJ_RST_BITS); - int32_t v = xq[0] * (flt0[k] - u); + int32_t v = xq_active * (flt[k] - u); const int32_t e = ROUND_POWER_OF_TWO(v, shift) + dat[k] - src[k]; - err += e * e; + err += ((int64_t)e * e); } dat += dat_stride; src += src_stride; - flt0 += flt0_stride; - const __m128i sum64_0 = _mm_cvtepi32_epi64(sum32); - const __m128i sum64_1 = _mm_cvtepi32_epi64(_mm_srli_si128(sum32, 8)); - sum64 = _mm_add_epi64(sum64, sum64_0); - sum64 = _mm_add_epi64(sum64, sum64_1); - } - } else if (params->r[1] > 0) { - __m128i xq_coeff = pair_set_epi16(xq[1], -(xq[1] << SGRPROJ_RST_BITS)); - for (i = 0; i < height; ++i) { - __m128i sum32 = _mm_setzero_si128(); - for (j = 0; j < width - 8; j += 8) { - const __m128i d0 = _mm_cvtepu8_epi16(xx_loadl_64(dat + j)); - const __m128i s0 = _mm_cvtepu8_epi16(xx_loadl_64(src + j)); - const __m128i flt1_16b = - _mm_packs_epi32(xx_loadu_128(flt1 + j), xx_loadu_128(flt1 + j + 4)); - const __m128i v0 = - _mm_madd_epi16(xq_coeff, _mm_unpacklo_epi16(flt1_16b, d0)); - const __m128i v1 = - _mm_madd_epi16(xq_coeff, _mm_unpackhi_epi16(flt1_16b, d0)); - const __m128i vr0 = _mm_srai_epi32(_mm_add_epi32(v0, rounding), shift); - const __m128i vr1 = _mm_srai_epi32(_mm_add_epi32(v1, rounding), shift); - const __m128i e0 = - _mm_sub_epi16(_mm_add_epi16(_mm_packs_epi32(vr0, vr1), d0), s0); - const __m128i err0 = _mm_madd_epi16(e0, e0); - sum32 = _mm_add_epi32(sum32, err0); - } - for (k = j; k < width; ++k) { - const int32_t u = (int32_t)(dat[k] << SGRPROJ_RST_BITS); - int32_t v = xq[1] * (flt1[k] - u); - const int32_t e = ROUND_POWER_OF_TWO(v, shift) + dat[k] - src[k]; - err += e * e; - } - dat += dat_stride; - src += src_stride; - flt1 += flt1_stride; + flt += flt_stride; const __m128i sum64_0 = _mm_cvtepi32_epi64(sum32); const __m128i sum64_1 = _mm_cvtepi32_epi64(_mm_srli_si128(sum32, 8)); sum64 = _mm_add_epi64(sum64, sum64_0); @@ -357,7 +591,7 @@ int64_t av1_lowbd_pixel_proj_error_sse4_1( } else { __m128i sum32 = _mm_setzero_si128(); for (i = 0; i < height; ++i) { - for (j = 0; j < width - 16; j += 16) { + for (j = 0; j <= width - 16; j += 16) { const __m128i d = xx_loadu_128(dat + j); const __m128i s = xx_loadu_128(src + j); const __m128i d0 = _mm_cvtepu8_epi16(d); @@ -373,7 +607,7 @@ int64_t av1_lowbd_pixel_proj_error_sse4_1( } for (k = j; k < width; ++k) { const int32_t e = (int32_t)(dat[k]) - src[k]; - err += e * e; + err += ((int64_t)e * e); } dat += dat_stride; src += src_stride; @@ -387,3 +621,209 @@ int64_t av1_lowbd_pixel_proj_error_sse4_1( err += sum[0] + sum[1]; return err; } + +int64_t av1_highbd_pixel_proj_error_sse4_1( + const uint8_t *src8, int width, int height, int src_stride, + const uint8_t *dat8, int dat_stride, int32_t *flt0, int flt0_stride, + int32_t *flt1, int flt1_stride, int xq[2], const sgr_params_type *params) { + int i, j, k; + const int32_t shift = SGRPROJ_RST_BITS + SGRPROJ_PRJ_BITS; + const __m128i rounding = _mm_set1_epi32(1 << (shift - 1)); + __m128i sum64 = _mm_setzero_si128(); + const uint16_t *src = CONVERT_TO_SHORTPTR(src8); + const uint16_t *dat = CONVERT_TO_SHORTPTR(dat8); + int64_t err = 0; + if (params->r[0] > 0 && params->r[1] > 0) { // Both filters are enabled + const __m128i xq0 = _mm_set1_epi32(xq[0]); + const __m128i xq1 = _mm_set1_epi32(xq[1]); + + for (i = 0; i < height; ++i) { + __m128i sum32 = _mm_setzero_si128(); + for (j = 0; j <= width - 8; j += 8) { + // Load 8x pixels from source image + const __m128i s0 = xx_loadu_128(src + j); + // s0 = [7 6 5 4 3 2 1 0] as i16 (indices of src[]) + + // Load 8x pixels from corrupted image + const __m128i d0 = xx_loadu_128(dat + j); + // d0 = [7 6 5 4 3 2 1 0] as i16 (indices of dat[]) + + // Shift each pixel value up by SGRPROJ_RST_BITS + const __m128i u0 = _mm_slli_epi16(d0, SGRPROJ_RST_BITS); + + // Split u0 into two halves and pad each from u16 to i32 + const __m128i u0l = _mm_cvtepu16_epi32(u0); + const __m128i u0h = _mm_cvtepu16_epi32(_mm_srli_si128(u0, 8)); + // u0h = [7 6 5 4] as i32, u0l = [3 2 1 0] as i32, all dat[] indices + + // Load 8 pixels from first and second filtered images + const __m128i flt0l = xx_loadu_128(flt0 + j); + const __m128i flt0h = xx_loadu_128(flt0 + j + 4); + const __m128i flt1l = xx_loadu_128(flt1 + j); + const __m128i flt1h = xx_loadu_128(flt1 + j + 4); + // flt0 = [7 6 5 4] [3 2 1 0] as i32 (indices of flt0+j) + // flt1 = [7 6 5 4] [3 2 1 0] as i32 (indices of flt1+j) + + // Subtract shifted corrupt image from each filtered image + // This gives our two basis vectors for the projection + const __m128i flt0l_subu = _mm_sub_epi32(flt0l, u0l); + const __m128i flt0h_subu = _mm_sub_epi32(flt0h, u0h); + const __m128i flt1l_subu = _mm_sub_epi32(flt1l, u0l); + const __m128i flt1h_subu = _mm_sub_epi32(flt1h, u0h); + // flt?h_subu = [ f[7]-u[7] f[6]-u[6] f[5]-u[5] f[4]-u[4] ] as i32 + // flt?l_subu = [ f[3]-u[3] f[2]-u[2] f[1]-u[1] f[0]-u[0] ] as i32 + + // Multiply each basis vector by the corresponding coefficient + const __m128i v0l = _mm_mullo_epi32(flt0l_subu, xq0); + const __m128i v0h = _mm_mullo_epi32(flt0h_subu, xq0); + const __m128i v1l = _mm_mullo_epi32(flt1l_subu, xq1); + const __m128i v1h = _mm_mullo_epi32(flt1h_subu, xq1); + + // Add together the contribution from each scaled basis vector + const __m128i vl = _mm_add_epi32(v0l, v1l); + const __m128i vh = _mm_add_epi32(v0h, v1h); + + // Right-shift v with appropriate rounding + const __m128i vrl = _mm_srai_epi32(_mm_add_epi32(vl, rounding), shift); + const __m128i vrh = _mm_srai_epi32(_mm_add_epi32(vh, rounding), shift); + + // Saturate each i32 value to i16 and combine lower and upper halves + const __m128i vr = _mm_packs_epi32(vrl, vrh); + + // Add twin-subspace-sgr-filter to corrupt image then subtract source + const __m128i e0 = _mm_sub_epi16(_mm_add_epi16(vr, d0), s0); + + // Calculate squared error and add adjacent values + const __m128i err0 = _mm_madd_epi16(e0, e0); + + sum32 = _mm_add_epi32(sum32, err0); + } + + const __m128i sum32l = _mm_cvtepu32_epi64(sum32); + sum64 = _mm_add_epi64(sum64, sum32l); + const __m128i sum32h = _mm_cvtepu32_epi64(_mm_srli_si128(sum32, 8)); + sum64 = _mm_add_epi64(sum64, sum32h); + + // Process remaining pixels in this row (modulo 8) + for (k = j; k < width; ++k) { + const int32_t u = (int32_t)(dat[k] << SGRPROJ_RST_BITS); + int32_t v = xq[0] * (flt0[k] - u) + xq[1] * (flt1[k] - u); + const int32_t e = ROUND_POWER_OF_TWO(v, shift) + dat[k] - src[k]; + err += ((int64_t)e * e); + } + dat += dat_stride; + src += src_stride; + flt0 += flt0_stride; + flt1 += flt1_stride; + } + } else if (params->r[0] > 0 || params->r[1] > 0) { // Only one filter enabled + const int32_t xq_on = (params->r[0] > 0) ? xq[0] : xq[1]; + const __m128i xq_active = _mm_set1_epi32(xq_on); + const __m128i xq_inactive = + _mm_set1_epi32(-xq_on * (1 << SGRPROJ_RST_BITS)); + const int32_t *flt = (params->r[0] > 0) ? flt0 : flt1; + const int flt_stride = (params->r[0] > 0) ? flt0_stride : flt1_stride; + for (i = 0; i < height; ++i) { + __m128i sum32 = _mm_setzero_si128(); + for (j = 0; j <= width - 8; j += 8) { + // Load 8x pixels from source image + const __m128i s0 = xx_loadu_128(src + j); + // s0 = [7 6 5 4 3 2 1 0] as u16 (indices of src[]) + + // Load 8x pixels from corrupted image and pad each u16 to i32 + const __m128i d0 = xx_loadu_128(dat + j); + const __m128i d0h = _mm_cvtepu16_epi32(_mm_srli_si128(d0, 8)); + const __m128i d0l = _mm_cvtepu16_epi32(d0); + // d0h, d0l = [7 6 5 4], [3 2 1 0] as u32 (indices of dat[]) + + // Load 8 pixels from the filtered image + const __m128i flth = xx_loadu_128(flt + j + 4); + const __m128i fltl = xx_loadu_128(flt + j); + // flth, fltl = [7 6 5 4], [3 2 1 0] as i32 (indices of flt+j) + + const __m128i flth_xq = _mm_mullo_epi32(flth, xq_active); + const __m128i fltl_xq = _mm_mullo_epi32(fltl, xq_active); + const __m128i d0h_xq = _mm_mullo_epi32(d0h, xq_inactive); + const __m128i d0l_xq = _mm_mullo_epi32(d0l, xq_inactive); + + const __m128i vh = _mm_add_epi32(flth_xq, d0h_xq); + const __m128i vl = _mm_add_epi32(fltl_xq, d0l_xq); + // vh = [ xq0(f[7]-d[7]) xq0(f[6]-d[6]) xq0(f[5]-d[5]) xq0(f[4]-d[4]) ] + // vl = [ xq0(f[3]-d[3]) xq0(f[2]-d[2]) xq0(f[1]-d[1]) xq0(f[0]-d[0]) ] + + // Shift this down with appropriate rounding + const __m128i vrh = _mm_srai_epi32(_mm_add_epi32(vh, rounding), shift); + const __m128i vrl = _mm_srai_epi32(_mm_add_epi32(vl, rounding), shift); + + // Saturate vr0 and vr1 from i32 to i16 then pack together + const __m128i vr = _mm_packs_epi32(vrl, vrh); + + // Subtract twin-subspace-sgr filtered from source image to get error + const __m128i e0 = _mm_sub_epi16(_mm_add_epi16(vr, d0), s0); + + // Calculate squared error and add adjacent values + const __m128i err0 = _mm_madd_epi16(e0, e0); + + sum32 = _mm_add_epi32(sum32, err0); + } + + const __m128i sum32l = _mm_cvtepu32_epi64(sum32); + sum64 = _mm_add_epi64(sum64, sum32l); + const __m128i sum32h = _mm_cvtepu32_epi64(_mm_srli_si128(sum32, 8)); + sum64 = _mm_add_epi64(sum64, sum32h); + + // Process remaining pixels in this row (modulo 8) + for (k = j; k < width; ++k) { + const int32_t u = (int32_t)(dat[k] << SGRPROJ_RST_BITS); + int32_t v = xq_on * (flt[k] - u); + const int32_t e = ROUND_POWER_OF_TWO(v, shift) + dat[k] - src[k]; + err += ((int64_t)e * e); + } + dat += dat_stride; + src += src_stride; + flt += flt_stride; + } + } else { // Neither filter is enabled + for (i = 0; i < height; ++i) { + __m128i sum32 = _mm_setzero_si128(); + for (j = 0; j <= width - 16; j += 16) { + // Load 2x8 u16 from source image + const __m128i s0 = xx_loadu_128(src + j); + const __m128i s1 = xx_loadu_128(src + j + 8); + // Load 2x8 u16 from corrupted image + const __m128i d0 = xx_loadu_128(dat + j); + const __m128i d1 = xx_loadu_128(dat + j + 8); + + // Subtract corrupted image from source image + const __m128i diff0 = _mm_sub_epi16(d0, s0); + const __m128i diff1 = _mm_sub_epi16(d1, s1); + + // Square error and add adjacent values + const __m128i err0 = _mm_madd_epi16(diff0, diff0); + const __m128i err1 = _mm_madd_epi16(diff1, diff1); + + sum32 = _mm_add_epi32(sum32, err0); + sum32 = _mm_add_epi32(sum32, err1); + } + + const __m128i sum32l = _mm_cvtepu32_epi64(sum32); + sum64 = _mm_add_epi64(sum64, sum32l); + const __m128i sum32h = _mm_cvtepu32_epi64(_mm_srli_si128(sum32, 8)); + sum64 = _mm_add_epi64(sum64, sum32h); + + // Process remaining pixels (modulu 8) + for (k = j; k < width; ++k) { + const int32_t e = (int32_t)(dat[k]) - src[k]; + err += ((int64_t)e * e); + } + dat += dat_stride; + src += src_stride; + } + } + + // Sum 4 values from sum64l and sum64h into err + int64_t sum[2]; + xx_storeu_128(sum, sum64); + err += sum[0] + sum[1]; + return err; +} diff --git a/third_party/aom/av1/encoder/x86/rdopt_avx2.c b/third_party/aom/av1/encoder/x86/rdopt_avx2.c new file mode 100644 index 0000000000..f588badc7c --- /dev/null +++ b/third_party/aom/av1/encoder/x86/rdopt_avx2.c @@ -0,0 +1,256 @@ +/* + * Copyright (c) 2018, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include +#include "aom_dsp/x86/synonyms_avx2.h" +#include "aom_ports/system_state.h" + +#include "config/av1_rtcd.h" +#include "av1/encoder/rdopt.h" + +// Process horizontal and vertical correlations in a 4x4 block of pixels. +// We actually use the 4x4 pixels to calculate correlations corresponding to +// the top-left 3x3 pixels, so this function must be called with 1x1 overlap, +// moving the window along/down by 3 pixels at a time. +INLINE static void horver_correlation_4x4(const int16_t *diff, int stride, + __m256i *xy_sum_32, + __m256i *xz_sum_32, __m256i *x_sum_32, + __m256i *x2_sum_32) { + // Pixels in this 4x4 [ a b c d ] + // are referred to as: [ e f g h ] + // [ i j k l ] + // [ m n o p ] + + const __m256i pixels = _mm256_set_epi64x( + *(uint64_t *)&diff[0 * stride], *(uint64_t *)&diff[1 * stride], + *(uint64_t *)&diff[2 * stride], *(uint64_t *)&diff[3 * stride]); + // pixels = [d c b a h g f e] [l k j i p o n m] as i16 + + const __m256i slli = _mm256_slli_epi64(pixels, 16); + // slli = [c b a 0 g f e 0] [k j i 0 o n m 0] as i16 + + const __m256i madd_xy = _mm256_madd_epi16(pixels, slli); + // madd_xy = [bc+cd ab fg+gh ef] [jk+kl ij no+op mn] as i32 + *xy_sum_32 = _mm256_add_epi32(*xy_sum_32, madd_xy); + + // Permute control [3 2] [1 0] => [2 1] [0 0], 0b10010000 = 0x90 + const __m256i perm = _mm256_permute4x64_epi64(slli, 0x90); + // perm = [g f e 0 k j i 0] [o n m 0 o n m 0] as i16 + + const __m256i madd_xz = _mm256_madd_epi16(slli, perm); + // madd_xz = [cg+bf ae gk+fj ei] [ko+jn im oo+nn mm] as i32 + *xz_sum_32 = _mm256_add_epi32(*xz_sum_32, madd_xz); + + // Sum every element in slli (and then also their squares) + const __m256i madd1_slli = _mm256_madd_epi16(slli, _mm256_set1_epi16(1)); + // madd1_slli = [c+b a g+f e] [k+j i o+n m] as i32 + *x_sum_32 = _mm256_add_epi32(*x_sum_32, madd1_slli); + + const __m256i madd_slli = _mm256_madd_epi16(slli, slli); + // madd_slli = [cc+bb aa gg+ff ee] [kk+jj ii oo+nn mm] as i32 + *x2_sum_32 = _mm256_add_epi32(*x2_sum_32, madd_slli); +} + +void av1_get_horver_correlation_full_avx2(const int16_t *diff, int stride, + int width, int height, float *hcorr, + float *vcorr) { + // The following notation is used: + // x - current pixel + // y - right neighbour pixel + // z - below neighbour pixel + // w - down-right neighbour pixel + int64_t xy_sum = 0, xz_sum = 0; + int64_t x_sum = 0, x2_sum = 0; + + // Process horizontal and vertical correlations through the body in 4x4 + // blocks. This excludes the final row and column and possibly one extra + // column depending how 3 divides into width and height + int32_t xy_xz_tmp[8] = { 0 }, x_x2_tmp[8] = { 0 }; + __m256i xy_sum_32 = _mm256_setzero_si256(); + __m256i xz_sum_32 = _mm256_setzero_si256(); + __m256i x_sum_32 = _mm256_setzero_si256(); + __m256i x2_sum_32 = _mm256_setzero_si256(); + for (int i = 0; i <= height - 4; i += 3) { + for (int j = 0; j <= width - 4; j += 3) { + horver_correlation_4x4(&diff[i * stride + j], stride, &xy_sum_32, + &xz_sum_32, &x_sum_32, &x2_sum_32); + } + const __m256i hadd_xy_xz = _mm256_hadd_epi32(xy_sum_32, xz_sum_32); + // hadd_xy_xz = [ae+bf+cg ei+fj+gk ab+bc+cd ef+fg+gh] + // [im+jn+ko mm+nn+oo ij+jk+kl mn+no+op] as i32 + yy_storeu_256(xy_xz_tmp, hadd_xy_xz); + xy_sum += (int64_t)xy_xz_tmp[5] + xy_xz_tmp[4] + xy_xz_tmp[1]; + xz_sum += (int64_t)xy_xz_tmp[7] + xy_xz_tmp[6] + xy_xz_tmp[3]; + + const __m256i hadd_x_x2 = _mm256_hadd_epi32(x_sum_32, x2_sum_32); + // hadd_x_x2 = [aa+bb+cc ee+ff+gg a+b+c e+f+g] + // [ii+jj+kk mm+nn+oo i+j+k m+n+o] as i32 + yy_storeu_256(x_x2_tmp, hadd_x_x2); + x_sum += (int64_t)x_x2_tmp[5] + x_x2_tmp[4] + x_x2_tmp[1]; + x2_sum += (int64_t)x_x2_tmp[7] + x_x2_tmp[6] + x_x2_tmp[3]; + + xy_sum_32 = _mm256_setzero_si256(); + xz_sum_32 = _mm256_setzero_si256(); + x_sum_32 = _mm256_setzero_si256(); + x2_sum_32 = _mm256_setzero_si256(); + } + + // x_sum now covers every pixel except the final 1-2 rows and 1-2 cols + int64_t x_finalrow = 0, x_finalcol = 0, x2_finalrow = 0, x2_finalcol = 0; + + // Do we have 2 rows remaining or just the one? Note that width and height + // are powers of 2, so each modulo 3 must be 1 or 2. + if (height % 3 == 1) { // Just horiz corrs on the final row + const int16_t x0 = diff[(height - 1) * stride]; + x_sum += x0; + x_finalrow += x0; + x2_sum += x0 * x0; + x2_finalrow += x0 * x0; + for (int j = 0; j < width - 1; ++j) { + const int16_t x = diff[(height - 1) * stride + j]; + const int16_t y = diff[(height - 1) * stride + j + 1]; + xy_sum += x * y; + x_sum += y; + x2_sum += y * y; + x_finalrow += y; + x2_finalrow += y * y; + } + } else { // Two rows remaining to do + const int16_t x0 = diff[(height - 2) * stride]; + const int16_t z0 = diff[(height - 1) * stride]; + x_sum += x0 + z0; + x2_sum += x0 * x0 + z0 * z0; + x_finalrow += z0; + x2_finalrow += z0 * z0; + for (int j = 0; j < width - 1; ++j) { + const int16_t x = diff[(height - 2) * stride + j]; + const int16_t y = diff[(height - 2) * stride + j + 1]; + const int16_t z = diff[(height - 1) * stride + j]; + const int16_t w = diff[(height - 1) * stride + j + 1]; + + // Horizontal and vertical correlations for the penultimate row: + xy_sum += x * y; + xz_sum += x * z; + + // Now just horizontal correlations for the final row: + xy_sum += z * w; + + x_sum += y + w; + x2_sum += y * y + w * w; + x_finalrow += w; + x2_finalrow += w * w; + } + } + + // Do we have 2 columns remaining or just the one? + if (width % 3 == 1) { // Just vert corrs on the final col + const int16_t x0 = diff[width - 1]; + x_sum += x0; + x_finalcol += x0; + x2_sum += x0 * x0; + x2_finalcol += x0 * x0; + for (int i = 0; i < height - 1; ++i) { + const int16_t x = diff[i * stride + width - 1]; + const int16_t z = diff[(i + 1) * stride + width - 1]; + xz_sum += x * z; + x_finalcol += z; + x2_finalcol += z * z; + // So the bottom-right elements don't get counted twice: + if (i < height - (height % 3 == 1 ? 2 : 3)) { + x_sum += z; + x2_sum += z * z; + } + } + } else { // Two cols remaining + const int16_t x0 = diff[width - 2]; + const int16_t y0 = diff[width - 1]; + x_sum += x0 + y0; + x2_sum += x0 * x0 + y0 * y0; + x_finalcol += y0; + x2_finalcol += y0 * y0; + for (int i = 0; i < height - 1; ++i) { + const int16_t x = diff[i * stride + width - 2]; + const int16_t y = diff[i * stride + width - 1]; + const int16_t z = diff[(i + 1) * stride + width - 2]; + const int16_t w = diff[(i + 1) * stride + width - 1]; + + // Horizontal and vertical correlations for the penultimate col: + // Skip these on the last iteration of this loop if we also had two + // rows remaining, otherwise the final horizontal and vertical correlation + // get erroneously processed twice + if (i < height - 2 || height % 3 == 1) { + xy_sum += x * y; + xz_sum += x * z; + } + + x_finalcol += w; + x2_finalcol += w * w; + // So the bottom-right elements don't get counted twice: + if (i < height - (height % 3 == 1 ? 2 : 3)) { + x_sum += z + w; + x2_sum += z * z + w * w; + } + + // Now just vertical correlations for the final column: + xz_sum += y * w; + } + } + + // Calculate the simple sums and squared-sums + int64_t x_firstrow = 0, x_firstcol = 0; + int64_t x2_firstrow = 0, x2_firstcol = 0; + + for (int j = 0; j < width; ++j) { + x_firstrow += diff[j]; + x2_firstrow += diff[j] * diff[j]; + } + for (int i = 0; i < height; ++i) { + x_firstcol += diff[i * stride]; + x2_firstcol += diff[i * stride] * diff[i * stride]; + } + + int64_t xhor_sum = x_sum - x_finalcol; + int64_t xver_sum = x_sum - x_finalrow; + int64_t y_sum = x_sum - x_firstcol; + int64_t z_sum = x_sum - x_firstrow; + int64_t x2hor_sum = x2_sum - x2_finalcol; + int64_t x2ver_sum = x2_sum - x2_finalrow; + int64_t y2_sum = x2_sum - x2_firstcol; + int64_t z2_sum = x2_sum - x2_firstrow; + + aom_clear_system_state(); + + const float num_hor = (float)(height * (width - 1)); + const float num_ver = (float)((height - 1) * width); + + const float xhor_var_n = x2hor_sum - (xhor_sum * xhor_sum) / num_hor; + const float xver_var_n = x2ver_sum - (xver_sum * xver_sum) / num_ver; + + const float y_var_n = y2_sum - (y_sum * y_sum) / num_hor; + const float z_var_n = z2_sum - (z_sum * z_sum) / num_ver; + + const float xy_var_n = xy_sum - (xhor_sum * y_sum) / num_hor; + const float xz_var_n = xz_sum - (xver_sum * z_sum) / num_ver; + + if (xhor_var_n > 0 && y_var_n > 0) { + *hcorr = xy_var_n / sqrtf(xhor_var_n * y_var_n); + *hcorr = *hcorr < 0 ? 0 : *hcorr; + } else { + *hcorr = 1.0; + } + if (xver_var_n > 0 && z_var_n > 0) { + *vcorr = xz_var_n / sqrtf(xver_var_n * z_var_n); + *vcorr = *vcorr < 0 ? 0 : *vcorr; + } else { + *vcorr = 1.0; + } +} diff --git a/third_party/aom/av1/encoder/x86/rdopt_sse4.c b/third_party/aom/av1/encoder/x86/rdopt_sse4.c new file mode 100644 index 0000000000..67d94b4ca8 --- /dev/null +++ b/third_party/aom/av1/encoder/x86/rdopt_sse4.c @@ -0,0 +1,275 @@ +/* + * Copyright (c) 2018, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include +#include "aom_dsp/x86/synonyms.h" +#include "aom_ports/system_state.h" + +#include "config/av1_rtcd.h" +#include "av1/encoder/rdopt.h" + +// Process horizontal and vertical correlations in a 4x4 block of pixels. +// We actually use the 4x4 pixels to calculate correlations corresponding to +// the top-left 3x3 pixels, so this function must be called with 1x1 overlap, +// moving the window along/down by 3 pixels at a time. +INLINE static void horver_correlation_4x4(const int16_t *diff, int stride, + __m128i *xy_sum_32, + __m128i *xz_sum_32, __m128i *x_sum_32, + __m128i *x2_sum_32) { + // Pixels in this 4x4 [ a b c d ] + // are referred to as: [ e f g h ] + // [ i j k l ] + // [ m n o p ] + + const __m128i pixelsa = _mm_set_epi64x(*(uint64_t *)&diff[0 * stride], + *(uint64_t *)&diff[2 * stride]); + const __m128i pixelsb = _mm_set_epi64x(*(uint64_t *)&diff[1 * stride], + *(uint64_t *)&diff[3 * stride]); + // pixelsa = [d c b a l k j i] as i16 + // pixelsb = [h g f e p o n m] as i16 + + const __m128i slli_a = _mm_slli_epi64(pixelsa, 16); + const __m128i slli_b = _mm_slli_epi64(pixelsb, 16); + // slli_a = [c b a 0 k j i 0] as i16 + // slli_b = [g f e 0 o n m 0] as i16 + + const __m128i xy_madd_a = _mm_madd_epi16(pixelsa, slli_a); + const __m128i xy_madd_b = _mm_madd_epi16(pixelsb, slli_b); + // xy_madd_a = [bc+cd ab jk+kl ij] as i32 + // xy_madd_b = [fg+gh ef no+op mn] as i32 + + const __m128i xy32 = _mm_hadd_epi32(xy_madd_b, xy_madd_a); + // xy32 = [ab+bc+cd ij+jk+kl ef+fg+gh mn+no+op] as i32 + *xy_sum_32 = _mm_add_epi32(*xy_sum_32, xy32); + + const __m128i xz_madd_a = _mm_madd_epi16(slli_a, slli_b); + // xz_madd_a = [bf+cg ae jn+ko im] i32 + + const __m128i swap_b = _mm_srli_si128(slli_b, 8); + // swap_b = [0 0 0 0 g f e 0] as i16 + const __m128i xz_madd_b = _mm_madd_epi16(slli_a, swap_b); + // xz_madd_b = [0 0 gk+fj ei] i32 + + const __m128i xz32 = _mm_hadd_epi32(xz_madd_b, xz_madd_a); + // xz32 = [ae+bf+cg im+jn+ko 0 ei+fj+gk] i32 + *xz_sum_32 = _mm_add_epi32(*xz_sum_32, xz32); + + // Now calculate the straight sums, x_sum += a+b+c+e+f+g+i+j+k + // (sum up every element in slli_a and swap_b) + const __m128i sum_slli_a = _mm_hadd_epi16(slli_a, slli_a); + const __m128i sum_slli_a32 = _mm_cvtepi16_epi32(sum_slli_a); + // sum_slli_a32 = [c+b a k+j i] as i32 + const __m128i swap_b32 = _mm_cvtepi16_epi32(swap_b); + // swap_b32 = [g f e 0] as i32 + *x_sum_32 = _mm_add_epi32(*x_sum_32, sum_slli_a32); + *x_sum_32 = _mm_add_epi32(*x_sum_32, swap_b32); + // sum = [c+b+g a+f k+j+e i] as i32 + + // Also sum their squares + const __m128i slli_a_2 = _mm_madd_epi16(slli_a, slli_a); + const __m128i swap_b_2 = _mm_madd_epi16(swap_b, swap_b); + // slli_a_2 = [c2+b2 a2 k2+j2 i2] + // swap_b_2 = [0 0 g2+f2 e2] + const __m128i sum2 = _mm_hadd_epi32(slli_a_2, swap_b_2); + // sum2 = [0 g2+f2+e2 c2+b2+a2 k2+j2+i2] + *x2_sum_32 = _mm_add_epi32(*x2_sum_32, sum2); +} + +void av1_get_horver_correlation_full_sse4_1(const int16_t *diff, int stride, + int width, int height, float *hcorr, + float *vcorr) { + // The following notation is used: + // x - current pixel + // y - right neighbour pixel + // z - below neighbour pixel + // w - down-right neighbour pixel + int64_t xy_sum = 0, xz_sum = 0; + int64_t x_sum = 0, x2_sum = 0; + + // Process horizontal and vertical correlations through the body in 4x4 + // blocks. This excludes the final row and column and possibly one extra + // column depending how 3 divides into width and height + int32_t xy_tmp[4] = { 0 }, xz_tmp[4] = { 0 }; + int32_t x_tmp[4] = { 0 }, x2_tmp[4] = { 0 }; + __m128i xy_sum_32 = _mm_setzero_si128(); + __m128i xz_sum_32 = _mm_setzero_si128(); + __m128i x_sum_32 = _mm_setzero_si128(); + __m128i x2_sum_32 = _mm_setzero_si128(); + for (int i = 0; i <= height - 4; i += 3) { + for (int j = 0; j <= width - 4; j += 3) { + horver_correlation_4x4(&diff[i * stride + j], stride, &xy_sum_32, + &xz_sum_32, &x_sum_32, &x2_sum_32); + } + xx_storeu_128(xy_tmp, xy_sum_32); + xx_storeu_128(xz_tmp, xz_sum_32); + xx_storeu_128(x_tmp, x_sum_32); + xx_storeu_128(x2_tmp, x2_sum_32); + xy_sum += (int64_t)xy_tmp[3] + xy_tmp[2] + xy_tmp[1]; + xz_sum += (int64_t)xz_tmp[3] + xz_tmp[2] + xz_tmp[0]; + x_sum += (int64_t)x_tmp[3] + x_tmp[2] + x_tmp[1] + x_tmp[0]; + x2_sum += (int64_t)x2_tmp[2] + x2_tmp[1] + x2_tmp[0]; + xy_sum_32 = _mm_setzero_si128(); + xz_sum_32 = _mm_setzero_si128(); + x_sum_32 = _mm_setzero_si128(); + x2_sum_32 = _mm_setzero_si128(); + } + + // x_sum now covers every pixel except the final 1-2 rows and 1-2 cols + int64_t x_finalrow = 0, x_finalcol = 0, x2_finalrow = 0, x2_finalcol = 0; + + // Do we have 2 rows remaining or just the one? Note that width and height + // are powers of 2, so each modulo 3 must be 1 or 2. + if (height % 3 == 1) { // Just horiz corrs on the final row + const int16_t x0 = diff[(height - 1) * stride]; + x_sum += x0; + x_finalrow += x0; + x2_sum += x0 * x0; + x2_finalrow += x0 * x0; + for (int j = 0; j < width - 1; ++j) { + const int16_t x = diff[(height - 1) * stride + j]; + const int16_t y = diff[(height - 1) * stride + j + 1]; + xy_sum += x * y; + x_sum += y; + x2_sum += y * y; + x_finalrow += y; + x2_finalrow += y * y; + } + } else { // Two rows remaining to do + const int16_t x0 = diff[(height - 2) * stride]; + const int16_t z0 = diff[(height - 1) * stride]; + x_sum += x0 + z0; + x2_sum += x0 * x0 + z0 * z0; + x_finalrow += z0; + x2_finalrow += z0 * z0; + for (int j = 0; j < width - 1; ++j) { + const int16_t x = diff[(height - 2) * stride + j]; + const int16_t y = diff[(height - 2) * stride + j + 1]; + const int16_t z = diff[(height - 1) * stride + j]; + const int16_t w = diff[(height - 1) * stride + j + 1]; + + // Horizontal and vertical correlations for the penultimate row: + xy_sum += x * y; + xz_sum += x * z; + + // Now just horizontal correlations for the final row: + xy_sum += z * w; + + x_sum += y + w; + x2_sum += y * y + w * w; + x_finalrow += w; + x2_finalrow += w * w; + } + } + + // Do we have 2 columns remaining or just the one? + if (width % 3 == 1) { // Just vert corrs on the final col + const int16_t x0 = diff[width - 1]; + x_sum += x0; + x_finalcol += x0; + x2_sum += x0 * x0; + x2_finalcol += x0 * x0; + for (int i = 0; i < height - 1; ++i) { + const int16_t x = diff[i * stride + width - 1]; + const int16_t z = diff[(i + 1) * stride + width - 1]; + xz_sum += x * z; + x_finalcol += z; + x2_finalcol += z * z; + // So the bottom-right elements don't get counted twice: + if (i < height - (height % 3 == 1 ? 2 : 3)) { + x_sum += z; + x2_sum += z * z; + } + } + } else { // Two cols remaining + const int16_t x0 = diff[width - 2]; + const int16_t y0 = diff[width - 1]; + x_sum += x0 + y0; + x2_sum += x0 * x0 + y0 * y0; + x_finalcol += y0; + x2_finalcol += y0 * y0; + for (int i = 0; i < height - 1; ++i) { + const int16_t x = diff[i * stride + width - 2]; + const int16_t y = diff[i * stride + width - 1]; + const int16_t z = diff[(i + 1) * stride + width - 2]; + const int16_t w = diff[(i + 1) * stride + width - 1]; + + // Horizontal and vertical correlations for the penultimate col: + // Skip these on the last iteration of this loop if we also had two + // rows remaining, otherwise the final horizontal and vertical correlation + // get erroneously processed twice + if (i < height - 2 || height % 3 == 1) { + xy_sum += x * y; + xz_sum += x * z; + } + + x_finalcol += w; + x2_finalcol += w * w; + // So the bottom-right elements don't get counted twice: + if (i < height - (height % 3 == 1 ? 2 : 3)) { + x_sum += z + w; + x2_sum += z * z + w * w; + } + + // Now just vertical correlations for the final column: + xz_sum += y * w; + } + } + + // Calculate the simple sums and squared-sums + int64_t x_firstrow = 0, x_firstcol = 0; + int64_t x2_firstrow = 0, x2_firstcol = 0; + + for (int j = 0; j < width; ++j) { + x_firstrow += diff[j]; + x2_firstrow += diff[j] * diff[j]; + } + for (int i = 0; i < height; ++i) { + x_firstcol += diff[i * stride]; + x2_firstcol += diff[i * stride] * diff[i * stride]; + } + + int64_t xhor_sum = x_sum - x_finalcol; + int64_t xver_sum = x_sum - x_finalrow; + int64_t y_sum = x_sum - x_firstcol; + int64_t z_sum = x_sum - x_firstrow; + int64_t x2hor_sum = x2_sum - x2_finalcol; + int64_t x2ver_sum = x2_sum - x2_finalrow; + int64_t y2_sum = x2_sum - x2_firstcol; + int64_t z2_sum = x2_sum - x2_firstrow; + + aom_clear_system_state(); + + const float num_hor = (float)(height * (width - 1)); + const float num_ver = (float)((height - 1) * width); + + const float xhor_var_n = x2hor_sum - (xhor_sum * xhor_sum) / num_hor; + const float xver_var_n = x2ver_sum - (xver_sum * xver_sum) / num_ver; + + const float y_var_n = y2_sum - (y_sum * y_sum) / num_hor; + const float z_var_n = z2_sum - (z_sum * z_sum) / num_ver; + + const float xy_var_n = xy_sum - (xhor_sum * y_sum) / num_hor; + const float xz_var_n = xz_sum - (xver_sum * z_sum) / num_ver; + + if (xhor_var_n > 0 && y_var_n > 0) { + *hcorr = xy_var_n / sqrtf(xhor_var_n * y_var_n); + *hcorr = *hcorr < 0 ? 0 : *hcorr; + } else { + *hcorr = 1.0; + } + if (xver_var_n > 0 && z_var_n > 0) { + *vcorr = xz_var_n / sqrtf(xver_var_n * z_var_n); + *vcorr = *vcorr < 0 ? 0 : *vcorr; + } else { + *vcorr = 1.0; + } +} diff --git a/third_party/aom/av1/encoder/x86/temporal_filter_apply_sse2.asm b/third_party/aom/av1/encoder/x86/temporal_filter_apply_sse2.asm index 30983d1c10..0cae3a748a 100644 --- a/third_party/aom/av1/encoder/x86/temporal_filter_apply_sse2.asm +++ b/third_party/aom/av1/encoder/x86/temporal_filter_apply_sse2.asm @@ -16,6 +16,7 @@ SECTION .text +; TODO(yunqing) The following optimization is not used since the c code changes. ; void av1_temporal_filter_apply_sse2 | arg ; (unsigned char *frame1, | 0 ; unsigned int stride, | 1 diff --git a/third_party/aom/av1/exports_ident b/third_party/aom/av1/exports_ident new file mode 100644 index 0000000000..b523a679d5 --- /dev/null +++ b/third_party/aom/av1/exports_ident @@ -0,0 +1,2 @@ +text ifd_init +text ifd_inspect diff --git a/third_party/aom/build/cmake/aom_config_defaults.cmake b/third_party/aom/build/cmake/aom_config_defaults.cmake index a07438cfe3..47aba55af0 100644 --- a/third_party/aom/build/cmake/aom_config_defaults.cmake +++ b/third_party/aom/build/cmake/aom_config_defaults.cmake @@ -101,32 +101,22 @@ set_aom_config_var(CONFIG_DENOISE 1 NUMBER "Denoise/noise modeling support in encoder.") set_aom_config_var(CONFIG_FILEOPTIONS 1 NUMBER "Enables encoder config file support.") -set_aom_config_var(CONFIG_FIX_GF_LENGTH 1 NUMBER - "Fix the GF length if possible") set_aom_config_var(CONFIG_INSPECTION 0 NUMBER "Enables bitstream inspection.") set_aom_config_var(CONFIG_INTERNAL_STATS 0 NUMBER "Enables internal encoder stats.") -set_aom_config_var(CONFIG_LOWBITDEPTH 0 NUMBER +set_aom_config_var(CONFIG_LOWBITDEPTH 1 NUMBER "Enables 8-bit optimized pipeline.") set_aom_config_var(CONFIG_MAX_DECODE_PROFILE 2 NUMBER "Max profile to support decoding.") set_aom_config_var(CONFIG_NORMAL_TILE_MODE 0 NUMBER "Only enables normal tile mode.") -set_aom_config_var( - CONFIG_REDUCED_ENCODER_BORDER 0 NUMBER - "Enable reduced border extention for encoder. \ - Disables superres and resize support." - ) set_aom_config_var(CONFIG_SIZE_LIMIT 0 NUMBER "Limit max decode width/height.") set_aom_config_var(CONFIG_SPATIAL_RESAMPLING 1 NUMBER "Spatial resampling.") set_aom_config_var(DECODE_HEIGHT_LIMIT 0 NUMBER "Set limit for decode height.") set_aom_config_var(DECODE_WIDTH_LIMIT 0 NUMBER "Set limit for decode width.") -set_aom_config_var(CONFIG_GLOBAL_MOTION_SEARCH 1 NUMBER - "Global motion search flag.") # AV1 experiment flags. -set_aom_config_var(CONFIG_COLLECT_INTER_MODE_RD_STATS 1 NUMBER - "AV1 experiment flag.") +set_aom_config_var(CONFIG_SPEED_STATS 0 NUMBER "AV1 experiment flag.") set_aom_config_var(CONFIG_COLLECT_RD_STATS 0 NUMBER "AV1 experiment flag.") set_aom_config_var(CONFIG_DIST_8X8 0 NUMBER "AV1 experiment flag.") set_aom_config_var(CONFIG_ENTROPY_STATS 0 NUMBER "AV1 experiment flag.") @@ -135,8 +125,12 @@ set_aom_config_var(CONFIG_INTER_STATS_ONLY 0 NUMBER "AV1 experiment flag.") set_aom_config_var(CONFIG_RD_DEBUG 0 NUMBER "AV1 experiment flag.") set_aom_config_var(CONFIG_2PASS_PARTITION_SEARCH_LVL 1 NUMBER "AV1 experiment flag.") -set_aom_config_var(CONFIG_SHARP_SETTINGS 0 NUMBER - "Use sharper encoding settings") +set_aom_config_var(CONFIG_SHARP_SETTINGS 0 NUMBER "AV1 experiment flag.") +set_aom_config_var(CONFIG_ONE_PASS_SVM 0 NUMBER "AV1 experiment flag.") +set_aom_config_var(CONFIG_DISABLE_FULL_PIXEL_SPLIT_8X8 1 NUMBER + "Disable full_pixel_motion_search_based_split on BLOCK_8X8.") +set_aom_config_var(CONFIG_COLLECT_PARTITION_STATS 0 NUMBER + "Collect stats on partition decisions.") # # Variables in this section control optional features of the build system. diff --git a/third_party/aom/build/cmake/aom_configure.cmake b/third_party/aom/build/cmake/aom_configure.cmake index c0c7381e87..cb64713e58 100644 --- a/third_party/aom/build/cmake/aom_configure.cmake +++ b/third_party/aom/build/cmake/aom_configure.cmake @@ -17,9 +17,6 @@ include(FindGit) include(FindPerl) include(FindThreads) -set(AOM_SUPPORTED_CPU_TARGETS - "arm64 armv7 armv7s generic mips32 mips64 ppc x86 x86_64") - include("${AOM_ROOT}/build/cmake/aom_config_defaults.cmake") include("${AOM_ROOT}/build/cmake/aom_experiment_deps.cmake") include("${AOM_ROOT}/build/cmake/aom_optimization.cmake") @@ -70,8 +67,12 @@ if(NOT AOM_TARGET_CPU) endif() if(CMAKE_TOOLCHAIN_FILE) # Add toolchain file to config string. - file(RELATIVE_PATH toolchain_path "${AOM_CONFIG_DIR}" - "${CMAKE_TOOLCHAIN_FILE}") + if(IS_ABSOLUTE "${CMAKE_TOOLCHAIN_FILE}") + file(RELATIVE_PATH toolchain_path "${AOM_CONFIG_DIR}" + "${CMAKE_TOOLCHAIN_FILE}") + else() + set(toolchain_path "${CMAKE_TOOLCHAIN_FILE}") + endif() set(toolchain_string "-DCMAKE_TOOLCHAIN_FILE=\\\"${toolchain_path}\\\"") set(AOM_CMAKE_CONFIG "${toolchain_string} ${AOM_CMAKE_CONFIG}") else() @@ -104,7 +105,7 @@ if(NOT MSVC) # work. set(CMAKE_POSITION_INDEPENDENT_CODE ON) if("${AOM_TARGET_SYSTEM}" STREQUAL "Linux" AND "${AOM_TARGET_CPU}" MATCHES - "^armv7") + "^armv[78]") set(AOM_AS_FLAGS ${AOM_AS_FLAGS} --defsym PIC=1) else() set(AOM_AS_FLAGS ${AOM_AS_FLAGS} -DPIC) @@ -112,13 +113,6 @@ if(NOT MSVC) endif() endif() -if(NOT "${AOM_SUPPORTED_CPU_TARGETS}" MATCHES "${AOM_TARGET_CPU}") - message(FATAL_ERROR - "No RTCD support for ${AOM_TARGET_CPU}. Create it, or " - "add -DAOM_TARGET_CPU=generic to your cmake command line for a " - "generic build of libaom and tools.") -endif() - if("${AOM_TARGET_CPU}" STREQUAL "x86" OR "${AOM_TARGET_CPU}" STREQUAL "x86_64") find_program(AS_EXECUTABLE yasm $ENV{YASM_PATH}) if(NOT AS_EXECUTABLE OR ENABLE_NASM) @@ -184,6 +178,12 @@ if("${AOM_TARGET_SYSTEM}" MATCHES "Darwin\|Linux\|Windows") set(CONFIG_OS_SUPPORT 1) endif() +# The default _WIN32_WINNT value in MinGW is 0x0502 (Windows XP with SP2). Set +# it to 0x0601 (Windows 7). +if("${AOM_TARGET_SYSTEM}" STREQUAL "Windows") + add_compiler_flag_if_supported("-D_WIN32_WINNT=0x0601") +endif() + # # Fix CONFIG_* dependencies. This must be done before including cpu.cmake to # ensure RTCD_CONFIG_* are properly set. @@ -252,6 +252,7 @@ if(MSVC) endif() else() require_c_flag("-std=c99" YES) + require_cxx_flag_nomsvc("-std=c++11" YES) add_compiler_flag_if_supported("-Wall") add_compiler_flag_if_supported("-Wdisabled-optimization") add_compiler_flag_if_supported("-Wextra") @@ -266,8 +267,17 @@ else() add_compiler_flag_if_supported("-Wunused") add_compiler_flag_if_supported("-Wvla") - add_c_flag_if_supported("-Wstack-usage=100000") - add_cxx_flag_if_supported("-Wstack-usage=360000") + if(CMAKE_C_COMPILER_ID MATCHES "GNU" AND "${SANITIZE}" MATCHES + "address|undefined") + + # This combination has more stack overhead, so we account for it by + # providing higher stack limit than usual. + add_c_flag_if_supported("-Wstack-usage=170000") + add_cxx_flag_if_supported("-Wstack-usage=270000") + else() + add_c_flag_if_supported("-Wstack-usage=100000") + add_cxx_flag_if_supported("-Wstack-usage=240000") + endif() # TODO(jzern): this could be added as a cxx flags for test/*.cc only, avoiding # third_party. @@ -365,13 +375,3 @@ execute_process(COMMAND ${CMAKE_COMMAND} -DAOM_CONFIG_DIR=${AOM_CONFIG_DIR} -DGIT_EXECUTABLE=${GIT_EXECUTABLE} -DPERL_EXECUTABLE=${PERL_EXECUTABLE} -P "${AOM_ROOT}/build/cmake/version.cmake") - -if(NOT MSVC) # Generate aom.pc (pkg-config file). - execute_process(COMMAND ${CMAKE_COMMAND} -DAOM_CONFIG_DIR=${AOM_CONFIG_DIR} - -DAOM_ROOT=${AOM_ROOT} - -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX} - -DCMAKE_PROJECT_NAME=${CMAKE_PROJECT_NAME} - -DCONFIG_MULTITHREAD=${CONFIG_MULTITHREAD} - -DHAVE_PTHREAD_H=${HAVE_PTHREAD_H} -P - "${AOM_ROOT}/build/cmake/pkg_config.cmake") -endif() diff --git a/third_party/aom/build/cmake/aom_experiment_deps.cmake b/third_party/aom/build/cmake/aom_experiment_deps.cmake index 0688704e5f..2e3615791e 100644 --- a/third_party/aom/build/cmake/aom_experiment_deps.cmake +++ b/third_party/aom/build/cmake/aom_experiment_deps.cmake @@ -21,10 +21,6 @@ macro(fix_experiment_configs) change_config_and_warn(CONFIG_INSPECTION 1 CONFIG_ANALYZER) endif() - if(CONFIG_RD_DEBUG) - change_config_and_warn(CONFIG_RD_DEBUG 0 CONFIG_JNT_COMP) - endif() - if(CONFIG_DIST_8X8 AND CONFIG_MULTITHREAD) change_config_and_warn(CONFIG_DIST_8X8 0 CONFIG_MULTITHREAD) endif() diff --git a/third_party/aom/build/cmake/aom_install.cmake b/third_party/aom/build/cmake/aom_install.cmake new file mode 100644 index 0000000000..47206d8bfb --- /dev/null +++ b/third_party/aom/build/cmake/aom_install.cmake @@ -0,0 +1,90 @@ +# +# Copyright (c) 2018, Alliance for Open Media. All rights reserved +# +# This source code is subject to the terms of the BSD 2 Clause License and the +# Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License was +# not distributed with this source code in the LICENSE file, you can obtain it +# at www.aomedia.org/license/software. If the Alliance for Open Media Patent +# License 1.0 was not distributed with this source code in the PATENTS file, you +# can obtain it at www.aomedia.org/license/patent. +# +list(APPEND AOM_INSTALL_INCS "${AOM_ROOT}/aom/aom.h" + "${AOM_ROOT}/aom/aom_codec.h" "${AOM_ROOT}/aom/aom_frame_buffer.h" + "${AOM_ROOT}/aom/aom_image.h" "${AOM_ROOT}/aom/aom_integer.h" + "${AOM_ROOT}/aom/aom.h") + +if(CONFIG_AV1_DECODER) + list(APPEND AOM_INSTALL_INCS "${AOM_ROOT}/aom/aom_decoder.h" + "${AOM_ROOT}/aom/aomdx.h") +endif() + +if(CONFIG_AV1_ENCODER) + list(APPEND AOM_INSTALL_INCS "${AOM_ROOT}/aom/aomcx.h" + "${AOM_ROOT}/aom/aom_encoder.h") +endif() + +# Generate aom.pc and setup dependencies to ensure it is created when necessary. +# Note: aom.pc generation uses GNUInstallDirs: +# https://cmake.org/cmake/help/latest/module/GNUInstallDirs.html +macro(setup_aom_install_targets) + if(NOT (MSVC OR XCODE)) + include("GNUInstallDirs") + set(AOM_PKG_CONFIG_FILE "${AOM_CONFIG_DIR}/aom.pc") + + # Create a dummy library target for creating aom.pc. + create_dummy_source_file(aom_pc c AOM_PKG_CONFIG_SOURCES) + add_library(aom_pc ${AOM_PKG_CONFIG_SOURCES}) + + # Setup a rule to generate aom.pc. + add_custom_command(OUTPUT "${AOM_PKG_CONFIG_FILE}" + COMMAND + ${CMAKE_COMMAND} ARGS + -DAOM_CONFIG_DIR=${AOM_CONFIG_DIR} + -DAOM_ROOT=${AOM_ROOT} + -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX} + -DCMAKE_INSTALL_BINDIR=${CMAKE_INSTALL_BINDIR} + -DCMAKE_INSTALL_INCLUDEDIR=${CMAKE_INSTALL_INCLUDEDIR} + -DCMAKE_INSTALL_LIBDIR=${CMAKE_INSTALL_LIBDIR} + -DCMAKE_PROJECT_NAME=${CMAKE_PROJECT_NAME} + -DCONFIG_MULTITHREAD=${CONFIG_MULTITHREAD} + -DHAVE_PTHREAD_H=${HAVE_PTHREAD_H} -P + "${AOM_ROOT}/build/cmake/pkg_config.cmake" + COMMENT "Writing aom.pc" VERBATIM) + + # Explicitly add a dependency on the pkg-config file to ensure it's built. + get_property(aom_pc_sources TARGET aom_pc PROPERTY SOURCES) + set_source_files_properties(${aom_pc_sources} OBJECT_DEPENDS + "${AOM_PKG_CONFIG_FILE}") + + # Our pkg-config file carries version information: add a dependency on the + # version rule. + add_dependencies(aom_pc aom_version) + + if(CONFIG_AV1_DECODER) + if(ENABLE_EXAMPLES) + list(APPEND AOM_INSTALL_BINS aomdec) + endif() + endif() + + if(CONFIG_AV1_ENCODER) + if(ENABLE_EXAMPLES) + list(APPEND AOM_INSTALL_BINS aomenc) + endif() + endif() + + set(AOM_INSTALL_LIBS aom) + + # Setup the install rules. + install(FILES ${AOM_INSTALL_INCS} DESTINATION + "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_INCLUDEDIR}/aom") + install(FILES "${AOM_PKG_CONFIG_FILE}" DESTINATION + "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}/pkgconfig") + install(TARGETS ${AOM_INSTALL_LIBS} DESTINATION + "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}") + + if(ENABLE_EXAMPLES) + install(TARGETS ${AOM_INSTALL_BINS} DESTINATION + "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_BINDIR}") + endif() + endif() +endmacro() diff --git a/third_party/aom/build/cmake/cpu.cmake b/third_party/aom/build/cmake/cpu.cmake index 6e8089e635..ef2d7552bb 100644 --- a/third_party/aom/build/cmake/cpu.cmake +++ b/third_party/aom/build/cmake/cpu.cmake @@ -9,18 +9,7 @@ # can obtain it at www.aomedia.org/license/patent. # -if("${AOM_TARGET_CPU}" STREQUAL "arm64") - set(ARCH_ARM 1) - set(RTCD_ARCH_ARM "yes") - - if(ENABLE_NEON) - set(HAVE_NEON 1) - set(RTCD_HAVE_NEON "yes") - else() - set(HAVE_NEON 0) - set(AOM_RTCD_FLAGS ${AOM_RTCD_FLAGS} --disable-neon) - endif() -elseif("${AOM_TARGET_CPU}" MATCHES "^armv7") +if("${AOM_TARGET_CPU}" MATCHES "^arm") set(ARCH_ARM 1) set(RTCD_ARCH_ARM "yes") diff --git a/third_party/aom/build/cmake/exports.cmake b/third_party/aom/build/cmake/exports.cmake index e0813dc0f2..b6e14d917b 100644 --- a/third_party/aom/build/cmake/exports.cmake +++ b/third_party/aom/build/cmake/exports.cmake @@ -35,6 +35,7 @@ function(setup_exports_target) -DAOM_XCODE=${XCODE} -DCONFIG_NAME=$ -DCONFIG_AV1_DECODER=${CONFIG_AV1_DECODER} -DCONFIG_AV1_ENCODER=${CONFIG_AV1_ENCODER} + -DCONFIG_INSPECTION=${CONFIG_INSPECTION} -DENABLE_TESTS=${ENABLE_TESTS} -P "${AOM_ROOT}/build/cmake/generate_exports.cmake" SOURCES ${AOM_EXPORTS_SOURCES} diff --git a/third_party/aom/build/cmake/exports_sources.cmake b/third_party/aom/build/cmake/exports_sources.cmake index 576920e36c..46bf001d86 100644 --- a/third_party/aom/build/cmake/exports_sources.cmake +++ b/third_party/aom/build/cmake/exports_sources.cmake @@ -19,6 +19,9 @@ list(APPEND AOM_EXPORTS_SOURCES "${AOM_ROOT}/aom/exports_com" if(CONFIG_AV1_DECODER) list(APPEND AOM_EXPORTS_SOURCES "${AOM_ROOT}/aom/exports_dec" "${AOM_ROOT}/av1/exports_dec") + if(CONFIG_INSPECTION) + list(APPEND AOM_EXPORTS_SOURCES "${AOM_ROOT}/av1/exports_ident") + endif() endif() if(CONFIG_AV1_ENCODER) diff --git a/third_party/aom/build/cmake/pkg_config.cmake b/third_party/aom/build/cmake/pkg_config.cmake index 64e20214e9..358c1a20dd 100644 --- a/third_party/aom/build/cmake/pkg_config.cmake +++ b/third_party/aom/build/cmake/pkg_config.cmake @@ -11,6 +11,7 @@ cmake_minimum_required(VERSION 3.5) set(REQUIRED_ARGS "AOM_ROOT" "AOM_CONFIG_DIR" "CMAKE_INSTALL_PREFIX" + "CMAKE_INSTALL_BINDIR" "CMAKE_INSTALL_INCLUDEDIR" "CMAKE_INSTALL_LIBDIR" "CMAKE_PROJECT_NAME" "CONFIG_MULTITHREAD" "HAVE_PTHREAD_H") foreach(arg ${REQUIRED_ARGS}) @@ -34,25 +35,28 @@ endif() # Write pkg-config info. set(prefix "${CMAKE_INSTALL_PREFIX}") +set(bindir "${CMAKE_INSTALL_BINDIR}") +set(includedir "${CMAKE_INSTALL_INCLUDEDIR}") +set(libdir "${CMAKE_INSTALL_LIBDIR}") set(pkgconfig_file "${AOM_CONFIG_DIR}/aom.pc") string(TOLOWER ${CMAKE_PROJECT_NAME} pkg_name) file(WRITE "${pkgconfig_file}" "# libaom pkg-config.\n") file(APPEND "${pkgconfig_file}" "prefix=${prefix}\n") -file(APPEND "${pkgconfig_file}" "exec_prefix=\${prefix}/bin\n") -file(APPEND "${pkgconfig_file}" "libdir=\${prefix}/lib\n") -file(APPEND "${pkgconfig_file}" "includedir=\${prefix}/include\n\n") +file(APPEND "${pkgconfig_file}" "exec_prefix=\${prefix}\n") +file(APPEND "${pkgconfig_file}" "includedir=\${prefix}/${includedir}\n") +file(APPEND "${pkgconfig_file}" "libdir=\${exec_prefix}/${libdir}\n\n") file(APPEND "${pkgconfig_file}" "Name: ${pkg_name}\n") -file(APPEND "${pkgconfig_file}" - "Description: AV1 codec library v${aom_version}.\n") +file( + APPEND + "${pkgconfig_file}" + "Description: Alliance for Open Media AV1 codec library v${aom_version}.\n") file(APPEND "${pkgconfig_file}" "Version: ${package_version}\n") file(APPEND "${pkgconfig_file}" "Requires:\n") file(APPEND "${pkgconfig_file}" "Conflicts:\n") +file(APPEND "${pkgconfig_file}" "Libs: -L\${libdir} -l${pkg_name}\n") if(CONFIG_MULTITHREAD AND HAVE_PTHREAD_H) - file(APPEND "${pkgconfig_file}" - "Libs: -L\${prefix}/lib -l${pkg_name} -lm -lpthread\n") file(APPEND "${pkgconfig_file}" "Libs.private: -lm -lpthread\n") else() - file(APPEND "${pkgconfig_file}" "Libs: -L\${prefix}/lib -l${pkg_name} -lm\n") file(APPEND "${pkgconfig_file}" "Libs.private: -lm\n") endif() -file(APPEND "${pkgconfig_file}" "Cflags: -I\${prefix}/include\n") +file(APPEND "${pkgconfig_file}" "Cflags: -I\${includedir}\n") diff --git a/third_party/aom/build/cmake/rtcd.pl b/third_party/aom/build/cmake/rtcd.pl index 46e06907cc..dafccdca9c 100755 --- a/third_party/aom/build/cmake/rtcd.pl +++ b/third_party/aom/build/cmake/rtcd.pl @@ -426,10 +426,10 @@ if ($opts{arch} eq 'x86') { @ALL_ARCHS = filter("$opts{arch}", qw/msa/); } mips; -} elsif ($opts{arch} =~ /armv7\w?/) { +} elsif ($opts{arch} =~ /armv[78]\w?/) { @ALL_ARCHS = filter(qw/neon/); arm; -} elsif ($opts{arch} eq 'armv8' || $opts{arch} eq 'arm64' ) { +} elsif ($opts{arch} eq 'arm64' ) { @ALL_ARCHS = filter(qw/neon/); &require("neon"); arm; diff --git a/third_party/aom/build/cmake/toolchains/arm64-mingw-gcc.cmake b/third_party/aom/build/cmake/toolchains/arm64-mingw-gcc.cmake index b5b2ff1cd3..bfeac928b7 100644 --- a/third_party/aom/build/cmake/toolchains/arm64-mingw-gcc.cmake +++ b/third_party/aom/build/cmake/toolchains/arm64-mingw-gcc.cmake @@ -27,6 +27,3 @@ set(CMAKE_RANLIB ${CROSS}ranlib CACHE FILEPATH Indexer) # No runtime cpu detect for arm64-mingw-gcc. set(CONFIG_RUNTIME_CPU_DETECT 0 CACHE NUMBER "") - -# Disable the use of the gtest's CMake support. -set(AOM_DISABLE_GTEST_CMAKE 1) diff --git a/third_party/aom/build/cmake/toolchains/armv7-mingw-gcc.cmake b/third_party/aom/build/cmake/toolchains/armv7-mingw-gcc.cmake index cf06a11b38..eb488ecc1b 100644 --- a/third_party/aom/build/cmake/toolchains/armv7-mingw-gcc.cmake +++ b/third_party/aom/build/cmake/toolchains/armv7-mingw-gcc.cmake @@ -27,6 +27,3 @@ set(CMAKE_RANLIB ${CROSS}ranlib CACHE FILEPATH Indexer) # No runtime cpu detect for armv7-mingw-gcc. set(CONFIG_RUNTIME_CPU_DETECT 0 CACHE NUMBER "") - -# Disable the use of the gtest's CMake support. -set(AOM_DISABLE_GTEST_CMAKE 1) diff --git a/third_party/aom/build/cmake/toolchains/x86-mingw-gcc.cmake b/third_party/aom/build/cmake/toolchains/x86-mingw-gcc.cmake index c986c4ee36..4839c9d455 100644 --- a/third_party/aom/build/cmake/toolchains/x86-mingw-gcc.cmake +++ b/third_party/aom/build/cmake/toolchains/x86-mingw-gcc.cmake @@ -26,6 +26,3 @@ set(CMAKE_C_COMPILER ${CROSS}gcc) set(CMAKE_CXX_COMPILER ${CROSS}g++) set(CMAKE_AR ${CROSS}ar CACHE FILEPATH Archiver) set(CMAKE_RANLIB ${CROSS}ranlib CACHE FILEPATH Indexer) - -# Disable the use of the gtest's CMake support. -set(AOM_DISABLE_GTEST_CMAKE 1) diff --git a/third_party/aom/build/cmake/toolchains/x86_64-mingw-gcc.cmake b/third_party/aom/build/cmake/toolchains/x86_64-mingw-gcc.cmake index 00d94d5f1f..4b2d28debc 100644 --- a/third_party/aom/build/cmake/toolchains/x86_64-mingw-gcc.cmake +++ b/third_party/aom/build/cmake/toolchains/x86_64-mingw-gcc.cmake @@ -24,6 +24,3 @@ set(CMAKE_C_COMPILER ${CROSS}gcc) set(CMAKE_CXX_COMPILER ${CROSS}g++) set(CMAKE_AR ${CROSS}ar CACHE FILEPATH Archiver) set(CMAKE_RANLIB ${CROSS}ranlib CACHE FILEPATH Indexer) - -# Disable the use of the gtest's CMake support. -set(AOM_DISABLE_GTEST_CMAKE 1) diff --git a/third_party/aom/build/cmake/util.cmake b/third_party/aom/build/cmake/util.cmake index b70ec40137..53379410f1 100644 --- a/third_party/aom/build/cmake/util.cmake +++ b/third_party/aom/build/cmake/util.cmake @@ -16,10 +16,10 @@ set(AOM_BUILD_CMAKE_UTIL_CMAKE_ 1) # Directory where generated sources will be written. set(AOM_GEN_SRC_DIR "${AOM_CONFIG_DIR}/gen_src") -# Creates dummy source file in $AOM_CONFIG_DIR named $basename.$extension and -# returns the full path to the dummy source file via the $out_file_path -# parameter. -macro(create_dummy_source_file basename extension out_file_path) +# Creates dummy source file in $AOM_GEN_SRC_DIR named $basename.$extension and +# returns the full path to the dummy source file via appending it to the list +# variable referred to by $out_file_list_var parameter. +macro(create_dummy_source_file basename extension out_file_list_var) set(dummy_source_file "${AOM_GEN_SRC_DIR}/${basename}_dummy.${extension}") file( WRITE @@ -27,13 +27,15 @@ macro(create_dummy_source_file basename extension out_file_path) "// ${target_name} needs a ${extension} file to force link language, \n" "// or to silence a harmless CMake warning: Ignore me.\n" "void ${target_name}_dummy_function(void) {}\n") + list(APPEND "${out_file_list_var}" "${dummy_source_file}") endmacro() # Convenience function for adding a dummy source file to $target_name using # $extension as the file extension. Wraps create_dummy_source_file(). function(add_dummy_source_file_to_target target_name extension) - create_dummy_source_file("${target_name}" "${extension}" "dummy_source_file") - target_sources(${target_name} PRIVATE ${dummy_source_file}) + create_dummy_source_file("${target_name}" "${extension}" + "dummy_source_file_list") + target_sources(${target_name} PRIVATE ${dummy_source_file_list}) endfunction() # Sets the value of the variable referenced by $feature to $value, and reports diff --git a/third_party/aom/common/av1_config.c b/third_party/aom/common/av1_config.c index e8decf76fe..90955fb5b3 100644 --- a/third_party/aom/common/av1_config.c +++ b/third_party/aom/common/av1_config.c @@ -322,7 +322,7 @@ static int parse_sequence_header(const uint8_t *const buffer, size_t length, AV1C_READ_BITS_OR_RETURN_ERROR(max_frame_height_minus_1, frame_height_bits_minus_1 + 1); - int frame_id_numbers_present = 0; + uint8_t frame_id_numbers_present = 0; if (!reduced_still_picture_header) { AV1C_READ_BIT_OR_RETURN_ERROR(frame_id_numbers_present_flag); frame_id_numbers_present = frame_id_numbers_present_flag; @@ -345,7 +345,7 @@ static int parse_sequence_header(const uint8_t *const buffer, size_t length, AV1C_READ_BIT_OR_RETURN_ERROR(enable_order_hint); if (enable_order_hint) { - AV1C_READ_BIT_OR_RETURN_ERROR(enable_jnt_comp); + AV1C_READ_BIT_OR_RETURN_ERROR(enable_dist_wtd_comp); AV1C_READ_BIT_OR_RETURN_ERROR(enable_ref_frame_mvs); } diff --git a/third_party/aom/common/obudec.c b/third_party/aom/common/obudec.c index acbd12e0c8..bd9f98d3ae 100644 --- a/third_party/aom/common/obudec.c +++ b/third_party/aom/common/obudec.c @@ -12,9 +12,11 @@ #include #include #include +#include #include "common/obudec.h" +#include "aom_dsp/aom_dsp_common.h" #include "aom_ports/mem_ops.h" #include "av1/common/common.h" #include "av1/common/obu_util.h" @@ -24,8 +26,12 @@ #define OBU_HEADER_SIZE 1 #define OBU_EXTENSION_SIZE 1 #define OBU_MAX_LENGTH_FIELD_SIZE 8 + +#define OBU_MAX_HEADER_SIZE \ + (OBU_HEADER_SIZE + OBU_EXTENSION_SIZE + 2 * OBU_MAX_LENGTH_FIELD_SIZE) + #define OBU_DETECTION_SIZE \ - (OBU_HEADER_SIZE + OBU_EXTENSION_SIZE + 3 * OBU_MAX_LENGTH_FIELD_SIZE) + (OBU_HEADER_SIZE + OBU_EXTENSION_SIZE + 4 * OBU_MAX_LENGTH_FIELD_SIZE) // Reads unsigned LEB128 integer and returns 0 upon successful read and decode. // Stores raw bytes in 'value_buffer', length of the number in 'value_length', @@ -117,20 +123,20 @@ static int obudec_read_obu_header_and_size(FILE *f, size_t buffer_capacity, size_t *bytes_read, size_t *payload_length, ObuHeader *obu_header) { - const size_t kMinimumBufferSize = - (OBU_HEADER_SIZE + OBU_EXTENSION_SIZE + OBU_MAX_LENGTH_FIELD_SIZE); + const size_t kMinimumBufferSize = OBU_MAX_HEADER_SIZE; if (!f || !buffer || !bytes_read || !payload_length || !obu_header || buffer_capacity < kMinimumBufferSize) { return -1; } - size_t leb128_length = 0; + size_t leb128_length_obu = 0; + size_t leb128_length_payload = 0; uint64_t obu_size = 0; if (is_annexb) { - if (obudec_read_leb128(f, &buffer[0], &leb128_length, &obu_size) != 0) { + if (obudec_read_leb128(f, &buffer[0], &leb128_length_obu, &obu_size) != 0) { fprintf(stderr, "obudec: Failure reading OBU size length.\n"); return -1; - } else if (leb128_length == 0) { + } else if (leb128_length_obu == 0) { *payload_length = 0; return 0; } @@ -141,8 +147,8 @@ static int obudec_read_obu_header_and_size(FILE *f, size_t buffer_capacity, } size_t header_size = 0; - if (obudec_read_obu_header(f, buffer_capacity - leb128_length, is_annexb, - buffer + leb128_length, obu_header, + if (obudec_read_obu_header(f, buffer_capacity - leb128_length_obu, is_annexb, + buffer + leb128_length_obu, obu_header, &header_size) != 0) { return -1; } else if (header_size == 0) { @@ -150,7 +156,8 @@ static int obudec_read_obu_header_and_size(FILE *f, size_t buffer_capacity, return 0; } - if (is_annexb) { + if (!obu_header->has_size_field) { + assert(is_annexb); if (obu_size < header_size) { fprintf(stderr, "obudec: OBU size is too small.\n"); return -1; @@ -158,8 +165,8 @@ static int obudec_read_obu_header_and_size(FILE *f, size_t buffer_capacity, *payload_length = (size_t)obu_size - header_size; } else { uint64_t u64_payload_length = 0; - if (obudec_read_leb128(f, &buffer[header_size], &leb128_length, - &u64_payload_length) != 0) { + if (obudec_read_leb128(f, &buffer[leb128_length_obu + header_size], + &leb128_length_payload, &u64_payload_length) != 0) { fprintf(stderr, "obudec: Failure reading OBU payload length.\n"); return -1; } @@ -171,7 +178,39 @@ static int obudec_read_obu_header_and_size(FILE *f, size_t buffer_capacity, *payload_length = (size_t)u64_payload_length; } - *bytes_read = leb128_length + header_size; + *bytes_read = leb128_length_obu + header_size + leb128_length_payload; + return 0; +} + +static int obudec_grow_buffer(size_t growth_amount, uint8_t **obu_buffer, + size_t *obu_buffer_capacity) { + if (!*obu_buffer || !obu_buffer_capacity || growth_amount == 0) { + return -1; + } + + const size_t capacity = *obu_buffer_capacity; + if (SIZE_MAX - growth_amount < capacity) { + fprintf(stderr, "obudec: cannot grow buffer, capacity will roll over.\n"); + return -1; + } + + const size_t new_capacity = capacity + growth_amount; + +#if defined AOM_MAX_ALLOCABLE_MEMORY + if (new_capacity > AOM_MAX_ALLOCABLE_MEMORY) { + fprintf(stderr, "obudec: OBU size exceeds max alloc size.\n"); + return -1; + } +#endif + + uint8_t *new_buffer = (uint8_t *)realloc(*obu_buffer, new_capacity); + if (!new_buffer) { + fprintf(stderr, "obudec: Failed to allocate compressed data buffer.\n"); + return -1; + } + + *obu_buffer = new_buffer; + *obu_buffer_capacity = new_capacity; return 0; } @@ -179,12 +218,25 @@ static int obudec_read_one_obu(FILE *f, uint8_t **obu_buffer, size_t obu_bytes_buffered, size_t *obu_buffer_capacity, size_t *obu_length, ObuHeader *obu_header, int is_annexb) { - size_t available_buffer_capacity = *obu_buffer_capacity - obu_bytes_buffered; - - if (!(*obu_buffer)) return -1; + if (!f || !(*obu_buffer) || !obu_buffer_capacity || !obu_length || + !obu_header) { + return -1; + } size_t bytes_read = 0; size_t obu_payload_length = 0; + size_t available_buffer_capacity = *obu_buffer_capacity - obu_bytes_buffered; + + if (available_buffer_capacity < OBU_MAX_HEADER_SIZE) { + if (obudec_grow_buffer(AOMMAX(*obu_buffer_capacity, OBU_MAX_HEADER_SIZE), + obu_buffer, obu_buffer_capacity) != 0) { + *obu_length = bytes_read; + return -1; + } + available_buffer_capacity += + AOMMAX(*obu_buffer_capacity, OBU_MAX_HEADER_SIZE); + } + const int status = obudec_read_obu_header_and_size( f, available_buffer_capacity, is_annexb, *obu_buffer + obu_bytes_buffered, &bytes_read, &obu_payload_length, obu_header); @@ -199,28 +251,11 @@ static int obudec_read_one_obu(FILE *f, uint8_t **obu_buffer, return -1; } - if (bytes_read + obu_payload_length > available_buffer_capacity) { - // TODO(tomfinegan): Add overflow check. - const size_t new_capacity = - obu_bytes_buffered + bytes_read + 2 * obu_payload_length; - -#if defined AOM_MAX_ALLOCABLE_MEMORY - if (new_capacity > AOM_MAX_ALLOCABLE_MEMORY) { - fprintf(stderr, "obudec: OBU size exceeds max alloc size.\n"); - return -1; - } -#endif - - uint8_t *new_buffer = (uint8_t *)realloc(*obu_buffer, new_capacity); - - if (new_buffer) { - *obu_buffer = new_buffer; - *obu_buffer_capacity = new_capacity; - } else { - fprintf(stderr, "obudec: Failed to allocate compressed data buffer\n"); - *obu_length = bytes_read + obu_payload_length; - return -1; - } + if (bytes_read + obu_payload_length > available_buffer_capacity && + obudec_grow_buffer(AOMMAX(*obu_buffer_capacity, obu_payload_length), + obu_buffer, obu_buffer_capacity) != 0) { + *obu_length = bytes_read + obu_payload_length; + return -1; } if (obu_payload_length > 0 && diff --git a/third_party/aom/common/rawenc.c b/third_party/aom/common/rawenc.c index 5a2731d3ac..b72132c2e9 100644 --- a/third_party/aom/common/rawenc.c +++ b/third_party/aom/common/rawenc.c @@ -9,36 +9,88 @@ * PATENTS file, you can obtain it at www.aomedia.org/license/patent. */ +#include #include "common/rawenc.h" -void raw_write_image_file(const aom_image_t *img, const int *planes, - const int num_planes, FILE *file) { - const int bytes_per_sample = ((img->fmt & AOM_IMG_FMT_HIGHBITDEPTH) ? 2 : 1); +#define BATCH_SIZE 8 +// When writing greyscale color, batch 8 writes for low bit-depth, 4 writes +// for high bit-depth. +static const uint8_t batched[BATCH_SIZE] = { 128, 128, 128, 128, + 128, 128, 128, 128 }; +static const uint8_t batched_hbd[BATCH_SIZE] = { + 0, 128, 0, 128, 0, 128, 0, 128 +}; + +// Interface to writing to either a file or MD5Context. Takes a pointer to +// either the file or MD5Context, the buffer, the size of each element, and +// number of elements to write. Note that size and nmemb (last two args) must +// be unsigned int, as the interface to MD5Update requires that. +typedef void (*WRITER)(void *, const uint8_t *, unsigned int, unsigned int); + +static void write_file(void *fp, const uint8_t *buffer, unsigned int size, + unsigned int nmemb) { + fwrite(buffer, size, nmemb, (FILE *)fp); +} + +static void write_md5(void *md5, const uint8_t *buffer, unsigned int size, + unsigned int nmemb) { + MD5Update((MD5Context *)md5, buffer, size * nmemb); +} + +// Writes out n greyscale values. +static void write_greyscale(const bool high_bitdepth, int n, WRITER writer_func, + void *file_or_md5) { + const uint8_t *b = batched; + if (high_bitdepth) { + b = batched_hbd; + } + const int num_batched_writes = + high_bitdepth ? n / (BATCH_SIZE / 2) : n / BATCH_SIZE; + for (int i = 0; i < num_batched_writes; ++i) { + writer_func(file_or_md5, b, sizeof(uint8_t), BATCH_SIZE); + } + const int remaining = high_bitdepth ? n % (BATCH_SIZE / 2) : n % BATCH_SIZE; + for (int i = 0; i < remaining; ++i) { + if (high_bitdepth) { + writer_func(file_or_md5, batched_hbd, sizeof(uint8_t), 2); + } else { + writer_func(file_or_md5, batched, sizeof(uint8_t), 1); + } + } +} + +// Encapsulates the logic for writing raw data to either an image file or +// to an MD5 context. +static void raw_write_image_file_or_md5(const aom_image_t *img, + const int *planes, const int num_planes, + void *file_or_md5, WRITER writer_func) { + const bool high_bitdepth = img->fmt & AOM_IMG_FMT_HIGHBITDEPTH; + const int bytes_per_sample = high_bitdepth ? 2 : 1; for (int i = 0; i < num_planes; ++i) { const int plane = planes[i]; - const unsigned char *buf = img->planes[plane]; - const int stride = img->stride[plane]; const int w = aom_img_plane_width(img, plane); const int h = aom_img_plane_height(img, plane); + // If we're on a color plane and the output is monochrome, write a greyscale + // value. Since there are only YUV planes, compare against Y. + if (img->monochrome && plane != AOM_PLANE_Y) { + write_greyscale(high_bitdepth, w * h, writer_func, file_or_md5); + continue; + } + const unsigned char *buf = img->planes[plane]; + const int stride = img->stride[plane]; for (int y = 0; y < h; ++y) { - fwrite(buf, bytes_per_sample, w, file); + writer_func(file_or_md5, buf, bytes_per_sample, w); buf += stride; } } } +void raw_write_image_file(const aom_image_t *img, const int *planes, + const int num_planes, FILE *file) { + raw_write_image_file_or_md5(img, planes, num_planes, file, write_file); +} + void raw_update_image_md5(const aom_image_t *img, const int *planes, const int num_planes, MD5Context *md5) { - for (int i = 0; i < num_planes; ++i) { - const int plane = planes[i]; - const unsigned char *buf = img->planes[plane]; - const int stride = img->stride[plane]; - const int w = aom_img_plane_width(img, plane) * - ((img->fmt & AOM_IMG_FMT_HIGHBITDEPTH) ? 2 : 1); - const int h = aom_img_plane_height(img, plane); - for (int y = 0; y < h; ++y) { - MD5Update(md5, buf, w); - buf += stride; - } - } + raw_write_image_file_or_md5(img, planes, num_planes, md5, write_md5); } diff --git a/third_party/aom/common/tools_common.c b/third_party/aom/common/tools_common.c index 21cd800262..51c1c52a1a 100644 --- a/third_party/aom/common/tools_common.c +++ b/third_party/aom/common/tools_common.c @@ -149,6 +149,11 @@ const AvxInterface *get_aom_encoder_by_name(const char *name) { return NULL; } + +// large scale tile encoding +static const AvxInterface aom_lst_encoder = { "av1", LST_FOURCC, + &aom_codec_av1_cx }; +const AvxInterface *get_aom_lst_encoder(void) { return &aom_lst_encoder; } #endif // CONFIG_AV1_ENCODER #if CONFIG_AV1_DECODER @@ -283,6 +288,7 @@ static void lowbd_img_upshift(aom_image_t *dst, const aom_image_t *src, fatal("Unsupported image conversion"); } switch (src->fmt) { + case AOM_IMG_FMT_YV12: case AOM_IMG_FMT_I420: case AOM_IMG_FMT_I422: case AOM_IMG_FMT_I444: break; @@ -423,3 +429,80 @@ void aom_img_downshift(aom_image_t *dst, const aom_image_t *src, lowbd_img_downshift(dst, src, down_shift); } } + +static int img_shifted_realloc_required(const aom_image_t *img, + const aom_image_t *shifted, + aom_img_fmt_t required_fmt) { + return img->d_w != shifted->d_w || img->d_h != shifted->d_h || + required_fmt != shifted->fmt; +} + +void aom_shift_img(unsigned int output_bit_depth, aom_image_t **img_ptr, + aom_image_t **img_shifted_ptr) { + aom_image_t *img = *img_ptr; + aom_image_t *img_shifted = *img_shifted_ptr; + + const aom_img_fmt_t shifted_fmt = output_bit_depth == 8 + ? img->fmt & ~AOM_IMG_FMT_HIGHBITDEPTH + : img->fmt | AOM_IMG_FMT_HIGHBITDEPTH; + + if (shifted_fmt != img->fmt || output_bit_depth != img->bit_depth) { + if (img_shifted && + img_shifted_realloc_required(img, img_shifted, shifted_fmt)) { + aom_img_free(img_shifted); + img_shifted = NULL; + } + if (img_shifted) { + img_shifted->monochrome = img->monochrome; + } + if (!img_shifted) { + img_shifted = aom_img_alloc(NULL, shifted_fmt, img->d_w, img->d_h, 16); + img_shifted->bit_depth = output_bit_depth; + img_shifted->monochrome = img->monochrome; + img_shifted->csp = img->csp; + } + if (output_bit_depth > img->bit_depth) { + aom_img_upshift(img_shifted, img, output_bit_depth - img->bit_depth); + } else { + aom_img_downshift(img_shifted, img, img->bit_depth - output_bit_depth); + } + *img_shifted_ptr = img_shifted; + *img_ptr = img_shifted; + } +} + +// Related to I420, NV12 format has one luma "luminance" plane Y and one plane +// with U and V values interleaved. +void aom_img_write_nv12(const aom_image_t *img, FILE *file) { + // Y plane + const unsigned char *buf = img->planes[0]; + int stride = img->stride[0]; + int w = aom_img_plane_width(img, 0) * + ((img->fmt & AOM_IMG_FMT_HIGHBITDEPTH) ? 2 : 1); + int h = aom_img_plane_height(img, 0); + int x, y; + + for (y = 0; y < h; ++y) { + fwrite(buf, 1, w, file); + buf += stride; + } + + // Interleaved U and V plane + const unsigned char *ubuf = img->planes[1]; + const unsigned char *vbuf = img->planes[2]; + const size_t size = (img->fmt & AOM_IMG_FMT_HIGHBITDEPTH) ? 2 : 1; + stride = img->stride[1]; + w = aom_img_plane_width(img, 1); + h = aom_img_plane_height(img, 1); + + for (y = 0; y < h; ++y) { + for (x = 0; x < w; ++x) { + fwrite(ubuf, size, 1, file); + fwrite(vbuf, size, 1, file); + ubuf += size; + vbuf += size; + } + ubuf += (stride - w * size); + vbuf += (stride - w * size); + } +} diff --git a/third_party/aom/common/tools_common.h b/third_party/aom/common/tools_common.h index 4e1d12f4a0..d9a68f01bf 100644 --- a/third_party/aom/common/tools_common.h +++ b/third_party/aom/common/tools_common.h @@ -18,6 +18,7 @@ #include "aom/aom_codec.h" #include "aom/aom_image.h" #include "aom/aom_integer.h" +#include "aom_ports/mem.h" #include "aom_ports/msvc.h" #if CONFIG_AV1_ENCODER @@ -77,6 +78,16 @@ enum VideoFileType { FILE_TYPE_WEBM }; +// Used in lightfield example. +enum { + YUV1D, // 1D tile output for conformance test. + YUV, // Tile output in YUV format. + NV12, // Tile output in NV12 format. +} UENUM1BYTE(OUTPUT_FORMAT); + +// The fourcc for large_scale_tile encoding is "LSTC". +#define LST_FOURCC 0x4354534c + struct FileTypeDetectionBuffer { char buf[4]; size_t buf_read; @@ -142,6 +153,7 @@ typedef struct AvxInterface { int get_aom_encoder_count(void); const AvxInterface *get_aom_encoder_by_index(int i); const AvxInterface *get_aom_encoder_by_name(const char *name); +const AvxInterface *get_aom_lst_encoder(void); int get_aom_decoder_count(void); const AvxInterface *get_aom_decoder_by_index(int i); @@ -155,8 +167,13 @@ double sse_to_psnr(double samples, double peak, double mse); void aom_img_upshift(aom_image_t *dst, const aom_image_t *src, int input_shift); void aom_img_downshift(aom_image_t *dst, const aom_image_t *src, int down_shift); +void aom_shift_img(unsigned int output_bit_depth, aom_image_t **img_ptr, + aom_image_t **img_shifted_ptr); void aom_img_truncate_16_to_8(aom_image_t *dst, const aom_image_t *src); +// Output in NV12 format. +void aom_img_write_nv12(const aom_image_t *img, FILE *file); + #ifdef __cplusplus } /* extern "C" */ #endif diff --git a/third_party/aom/common/video_reader.c b/third_party/aom/common/video_reader.c index 47ad6e1891..7b021bc40f 100644 --- a/third_party/aom/common/video_reader.c +++ b/third_party/aom/common/video_reader.c @@ -121,3 +121,7 @@ FILE *aom_video_reader_get_file(AvxVideoReader *reader) { const AvxVideoInfo *aom_video_reader_get_info(AvxVideoReader *reader) { return &reader->info; } + +void aom_video_reader_set_fourcc(AvxVideoReader *reader, uint32_t fourcc) { + reader->info.codec_fourcc = fourcc; +} diff --git a/third_party/aom/common/video_reader.h b/third_party/aom/common/video_reader.h index 903deae849..9ab439e8af 100644 --- a/third_party/aom/common/video_reader.h +++ b/third_party/aom/common/video_reader.h @@ -50,6 +50,9 @@ FILE *aom_video_reader_get_file(AvxVideoReader *reader); // Fills AvxVideoInfo with information from opened video file. const AvxVideoInfo *aom_video_reader_get_info(AvxVideoReader *reader); +// Set fourcc. +void aom_video_reader_set_fourcc(AvxVideoReader *reader, uint32_t fourcc); + #ifdef __cplusplus } // extern "C" #endif diff --git a/third_party/aom/common/video_writer.c b/third_party/aom/common/video_writer.c index a7ec309fc7..2b42e365cc 100644 --- a/third_party/aom/common/video_writer.c +++ b/third_party/aom/common/video_writer.c @@ -75,3 +75,7 @@ int aom_video_writer_write_frame(AvxVideoWriter *writer, const uint8_t *buffer, return 1; } + +void aom_video_writer_set_fourcc(AvxVideoWriter *writer, uint32_t fourcc) { + writer->info.codec_fourcc = fourcc; +} diff --git a/third_party/aom/common/video_writer.h b/third_party/aom/common/video_writer.h index 3e2b6554b8..8712d47a58 100644 --- a/third_party/aom/common/video_writer.h +++ b/third_party/aom/common/video_writer.h @@ -14,7 +14,7 @@ #include "common/video_common.h" -typedef enum { kContainerIVF } AvxContainer; +enum { kContainerIVF } UENUM1BYTE(AvxContainer); struct AvxVideoWriterStruct; typedef struct AvxVideoWriterStruct AvxVideoWriter; @@ -37,6 +37,8 @@ void aom_video_writer_close(AvxVideoWriter *writer); // Writes frame bytes to the file. int aom_video_writer_write_frame(AvxVideoWriter *writer, const uint8_t *buffer, size_t size, int64_t pts); +// Set fourcc. +void aom_video_writer_set_fourcc(AvxVideoWriter *writer, uint32_t fourcc); #ifdef __cplusplus } // extern "C" diff --git a/third_party/aom/common/webmenc.cc b/third_party/aom/common/webmenc.cc index 58ab336707..6ae7df646f 100644 --- a/third_party/aom/common/webmenc.cc +++ b/third_party/aom/common/webmenc.cc @@ -11,8 +11,11 @@ #include "common/webmenc.h" +#include + #include +#include "common/av1_config.h" #include "third_party/libwebm/mkvmuxer/mkvmuxer.h" #include "third_party/libwebm/mkvmuxer/mkvmuxerutil.h" #include "third_party/libwebm/mkvmuxer/mkvwriter.h" @@ -22,17 +25,33 @@ const uint64_t kDebugTrackUid = 0xDEADBEEF; const int kVideoTrackNumber = 1; } // namespace -void write_webm_file_header(struct WebmOutputContext *webm_ctx, - const aom_codec_enc_cfg_t *cfg, - stereo_format_t stereo_fmt, unsigned int fourcc, - const struct AvxRational *par) { +int write_webm_file_header(struct WebmOutputContext *webm_ctx, + aom_codec_ctx_t *encoder_ctx, + const aom_codec_enc_cfg_t *cfg, + stereo_format_t stereo_fmt, unsigned int fourcc, + const struct AvxRational *par) { mkvmuxer::MkvWriter *const writer = new mkvmuxer::MkvWriter(webm_ctx->stream); mkvmuxer::Segment *const segment = new mkvmuxer::Segment(); - segment->Init(writer); + if (!writer || !segment) { + fprintf(stderr, "webmenc> mkvmuxer objects alloc failed, out of memory?\n"); + return -1; + } + + bool ok = segment->Init(writer); + if (!ok) { + fprintf(stderr, "webmenc> mkvmuxer Init failed.\n"); + return -1; + } + segment->set_mode(mkvmuxer::Segment::kFile); segment->OutputCues(true); mkvmuxer::SegmentInfo *const info = segment->GetSegmentInfo(); + if (!info) { + fprintf(stderr, "webmenc> Cannot retrieve Segment Info.\n"); + return -1; + } + const uint64_t kTimecodeScale = 1000000; info->set_timecode_scale(kTimecodeScale); std::string version = "aomenc"; @@ -46,13 +65,48 @@ void write_webm_file_header(struct WebmOutputContext *webm_ctx, static_cast(cfg->g_h), kVideoTrackNumber); mkvmuxer::VideoTrack *const video_track = static_cast( segment->GetTrackByNumber(video_track_id)); - video_track->SetStereoMode(stereo_fmt); - const char *codec_id; - switch (fourcc) { - case AV1_FOURCC: codec_id = "V_AV1"; break; - default: codec_id = "V_AV1"; break; + + if (!video_track) { + fprintf(stderr, "webmenc> Video track creation failed.\n"); + return -1; } - video_track->set_codec_id(codec_id); + + ok = false; + aom_fixed_buf_t *obu_sequence_header = + aom_codec_get_global_headers(encoder_ctx); + if (obu_sequence_header) { + Av1Config av1_config; + if (get_av1config_from_obu( + reinterpret_cast(obu_sequence_header->buf), + obu_sequence_header->sz, false, &av1_config) == 0) { + uint8_t av1_config_buffer[4] = { 0 }; + size_t bytes_written = 0; + if (write_av1config(&av1_config, sizeof(av1_config_buffer), + &bytes_written, av1_config_buffer) == 0) { + ok = video_track->SetCodecPrivate(av1_config_buffer, + sizeof(av1_config_buffer)); + } + } + free(obu_sequence_header->buf); + free(obu_sequence_header); + } + if (!ok) { + fprintf(stderr, "webmenc> Unable to set AV1 config.\n"); + return -1; + } + + ok = video_track->SetStereoMode(stereo_fmt); + if (!ok) { + fprintf(stderr, "webmenc> Unable to set stereo mode.\n"); + return -1; + } + + if (fourcc != AV1_FOURCC) { + fprintf(stderr, "webmenc> Unsupported codec (unknown 4 CC).\n"); + return -1; + } + video_track->set_codec_id("V_AV1"); + if (par->numerator > 1 || par->denominator > 1) { // TODO(fgalligan): Add support of DisplayUnit, Display Aspect Ratio type // to WebM format. @@ -61,16 +115,24 @@ void write_webm_file_header(struct WebmOutputContext *webm_ctx, video_track->set_display_width(display_width); video_track->set_display_height(cfg->g_h); } + if (webm_ctx->debug) { video_track->set_uid(kDebugTrackUid); } + webm_ctx->writer = writer; webm_ctx->segment = segment; + + return 0; } -void write_webm_block(struct WebmOutputContext *webm_ctx, - const aom_codec_enc_cfg_t *cfg, - const aom_codec_cx_pkt_t *pkt) { +int write_webm_block(struct WebmOutputContext *webm_ctx, + const aom_codec_enc_cfg_t *cfg, + const aom_codec_cx_pkt_t *pkt) { + if (!webm_ctx->segment) { + fprintf(stderr, "webmenc> segment is NULL.\n"); + return -1; + } mkvmuxer::Segment *const segment = reinterpret_cast(webm_ctx->segment); int64_t pts_ns = pkt->data.frame.pts * 1000000000ll * cfg->g_timebase.num / @@ -78,19 +140,34 @@ void write_webm_block(struct WebmOutputContext *webm_ctx, if (pts_ns <= webm_ctx->last_pts_ns) pts_ns = webm_ctx->last_pts_ns + 1000000; webm_ctx->last_pts_ns = pts_ns; - segment->AddFrame(static_cast(pkt->data.frame.buf), - pkt->data.frame.sz, kVideoTrackNumber, pts_ns, - pkt->data.frame.flags & AOM_FRAME_IS_KEY); + if (!segment->AddFrame(static_cast(pkt->data.frame.buf), + pkt->data.frame.sz, kVideoTrackNumber, pts_ns, + pkt->data.frame.flags & AOM_FRAME_IS_KEY)) { + fprintf(stderr, "webmenc> AddFrame failed.\n"); + return -1; + } + return 0; } -void write_webm_file_footer(struct WebmOutputContext *webm_ctx) { +int write_webm_file_footer(struct WebmOutputContext *webm_ctx) { + if (!webm_ctx->writer || !webm_ctx->segment) { + fprintf(stderr, "webmenc> segment or writer NULL.\n"); + return -1; + } mkvmuxer::MkvWriter *const writer = reinterpret_cast(webm_ctx->writer); mkvmuxer::Segment *const segment = reinterpret_cast(webm_ctx->segment); - segment->Finalize(); + const bool ok = segment->Finalize(); delete segment; delete writer; webm_ctx->writer = NULL; webm_ctx->segment = NULL; + + if (!ok) { + fprintf(stderr, "webmenc> Segment::Finalize failed.\n"); + return -1; + } + + return 0; } diff --git a/third_party/aom/common/webmenc.h b/third_party/aom/common/webmenc.h index aa9832fbae..a4aa992b02 100644 --- a/third_party/aom/common/webmenc.h +++ b/third_party/aom/common/webmenc.h @@ -30,24 +30,28 @@ struct WebmOutputContext { }; /* Stereo 3D packed frame format */ -typedef enum stereo_format { +enum { STEREO_FORMAT_MONO = 0, STEREO_FORMAT_LEFT_RIGHT = 1, STEREO_FORMAT_BOTTOM_TOP = 2, STEREO_FORMAT_TOP_BOTTOM = 3, STEREO_FORMAT_RIGHT_LEFT = 11 -} stereo_format_t; +} UENUM1BYTE(stereo_format_t); -void write_webm_file_header(struct WebmOutputContext *webm_ctx, - const aom_codec_enc_cfg_t *cfg, - stereo_format_t stereo_fmt, unsigned int fourcc, - const struct AvxRational *par); +// The following functions wrap libwebm's mkvmuxer. All functions return 0 upon +// success, or -1 upon failure. -void write_webm_block(struct WebmOutputContext *webm_ctx, - const aom_codec_enc_cfg_t *cfg, - const aom_codec_cx_pkt_t *pkt); +int write_webm_file_header(struct WebmOutputContext *webm_ctx, + aom_codec_ctx_t *encoder_ctx, + const aom_codec_enc_cfg_t *cfg, + stereo_format_t stereo_fmt, unsigned int fourcc, + const struct AvxRational *par); -void write_webm_file_footer(struct WebmOutputContext *webm_ctx); +int write_webm_block(struct WebmOutputContext *webm_ctx, + const aom_codec_enc_cfg_t *cfg, + const aom_codec_cx_pkt_t *pkt); + +int write_webm_file_footer(struct WebmOutputContext *webm_ctx); #ifdef __cplusplus } // extern "C" diff --git a/third_party/aom/common/y4menc.c b/third_party/aom/common/y4menc.c index 585d221976..e3f5d5b387 100644 --- a/third_party/aom/common/y4menc.c +++ b/third_party/aom/common/y4menc.c @@ -15,7 +15,7 @@ #include "common/y4menc.h" // Returns the Y4M name associated with the monochrome colorspace. -const char *monochrome_colorspace(unsigned int bit_depth) { +static const char *monochrome_colorspace(unsigned int bit_depth) { switch (bit_depth) { case 8: return "Cmono"; case 9: return "Cmono9"; @@ -30,12 +30,15 @@ const char *monochrome_colorspace(unsigned int bit_depth) { // image format. const char *colorspace8(aom_chroma_sample_position_t csp, aom_img_fmt_t fmt) { switch (fmt) { - case AOM_IMG_FMT_444A: return "C444alpha"; case AOM_IMG_FMT_I444: return "C444"; case AOM_IMG_FMT_I422: return "C422"; default: if (csp == AOM_CSP_VERTICAL) { return "C420mpeg2 XYSCSS=420MPEG2"; + } else if (csp == AOM_CSP_COLOCATED) { + // Note that Y4M does not have a dedicated header for colocated chroma, + // and that FFMPEG interprets C420 as C420jpeg. + return "C420"; } else { return "C420jpeg"; } @@ -43,8 +46,9 @@ const char *colorspace8(aom_chroma_sample_position_t csp, aom_img_fmt_t fmt) { } // Return the Y4M name of the colorspace, given the bit depth and image format. -const char *colorspace(unsigned int bit_depth, aom_chroma_sample_position_t csp, - aom_img_fmt_t fmt) { +static const char *colorspace(unsigned int bit_depth, + aom_chroma_sample_position_t csp, + aom_img_fmt_t fmt) { switch (bit_depth) { case 8: return colorspace8(csp, fmt); case 9: diff --git a/third_party/aom/common/y4minput.c b/third_party/aom/common/y4minput.c index eca8b1bbac..f3dfaafc68 100644 --- a/third_party/aom/common/y4minput.c +++ b/third_party/aom/common/y4minput.c @@ -16,6 +16,7 @@ #include #include "aom/aom_integer.h" +#include "aom_ports/msvc.h" #include "y4minput.h" // Reads 'size' bytes from 'file' into 'buf' with some fault tolerance. @@ -109,7 +110,8 @@ static int y4m_parse_tags(y4m_input *_y4m, char *_tags) { if (!got_par) _y4m->par_n = _y4m->par_d = 0; /*Chroma-type is not specified in older files, e.g., those generated by mplayer.*/ - if (!got_chroma) strcpy(_y4m->chroma_type, "420"); + if (!got_chroma) + snprintf(_y4m->chroma_type, sizeof(_y4m->chroma_type), "420"); return 0; } @@ -778,7 +780,7 @@ static void y4m_convert_null(y4m_input *_y4m, unsigned char *_dst, } int y4m_input_open(y4m_input *_y4m, FILE *_fin, char *_skip, int _nskip, - int only_420) { + aom_chroma_sample_position_t csp, int only_420) { char buffer[80] = { 0 }; int ret; int i; @@ -821,6 +823,19 @@ int y4m_input_open(y4m_input *_y4m, FILE *_fin, char *_skip, int _nskip, "Only progressive scan handled.\n"); return -1; } + /* Only support vertical chroma sample position if the input format is + * already 420mpeg2. Colocated is not supported in Y4M. + */ + if (csp == AOM_CSP_VERTICAL && strcmp(_y4m->chroma_type, "420mpeg2") != 0) { + fprintf(stderr, + "Vertical chroma sample position only supported " + "for 420mpeg2 input\n"); + return -1; + } + if (csp == AOM_CSP_COLOCATED) { + fprintf(stderr, "Colocated chroma sample position not supported in Y4M\n"); + return -1; + } _y4m->aom_fmt = AOM_IMG_FMT_I420; _y4m->bps = 12; _y4m->bit_depth = 8; @@ -877,7 +892,11 @@ int y4m_input_open(y4m_input *_y4m, FILE *_fin, char *_skip, int _nskip, /*Chroma filter required: read into the aux buf first.*/ _y4m->aux_buf_sz = _y4m->aux_buf_read_sz = 2 * ((_y4m->pic_w + 1) / 2) * ((_y4m->pic_h + 1) / 2); - _y4m->convert = y4m_convert_42xmpeg2_42xjpeg; + _y4m->convert = y4m_convert_null; + if (csp != AOM_CSP_VERTICAL) { + _y4m->convert = y4m_convert_42xmpeg2_42xjpeg; + snprintf(_y4m->chroma_type, sizeof(_y4m->chroma_type), "420"); + } } else if (strcmp(_y4m->chroma_type, "420paldv") == 0) { _y4m->src_c_dec_h = _y4m->dst_c_dec_h = _y4m->src_c_dec_v = _y4m->dst_c_dec_v = 2; @@ -1037,14 +1056,8 @@ int y4m_input_open(y4m_input *_y4m, FILE *_fin, char *_skip, int _nskip, _y4m->aux_buf_sz = _y4m->aux_buf_read_sz = 3 * _y4m->pic_w * _y4m->pic_h; _y4m->convert = y4m_convert_444_420jpeg; } else { - _y4m->aom_fmt = AOM_IMG_FMT_444A; - _y4m->bps = 32; - _y4m->dst_c_dec_h = _y4m->src_c_dec_h; - _y4m->dst_c_dec_v = _y4m->src_c_dec_v; - _y4m->dst_buf_read_sz = 4 * _y4m->pic_w * _y4m->pic_h; - /*Natively supported: no conversion required.*/ - _y4m->aux_buf_sz = _y4m->aux_buf_read_sz = 0; - _y4m->convert = y4m_convert_null; + fprintf(stderr, "Unsupported format: 444A\n"); + return -1; } } else if (strcmp(_y4m->chroma_type, "mono") == 0) { _y4m->src_c_dec_h = _y4m->src_c_dec_v = 0; @@ -1131,12 +1144,10 @@ int y4m_input_fetch_frame(y4m_input *_y4m, FILE *_fin, aom_image_t *_img) { c_w *= bytes_per_sample; c_h = (_y4m->pic_h + _y4m->dst_c_dec_v - 1) / _y4m->dst_c_dec_v; c_sz = c_w * c_h; - _img->stride[AOM_PLANE_Y] = _img->stride[AOM_PLANE_ALPHA] = - _y4m->pic_w * bytes_per_sample; + _img->stride[AOM_PLANE_Y] = _y4m->pic_w * bytes_per_sample; _img->stride[AOM_PLANE_U] = _img->stride[AOM_PLANE_V] = c_w; _img->planes[AOM_PLANE_Y] = _y4m->dst_buf; _img->planes[AOM_PLANE_U] = _y4m->dst_buf + pic_sz; _img->planes[AOM_PLANE_V] = _y4m->dst_buf + pic_sz + c_sz; - _img->planes[AOM_PLANE_ALPHA] = _y4m->dst_buf + pic_sz + 2 * c_sz; return 1; } diff --git a/third_party/aom/common/y4minput.h b/third_party/aom/common/y4minput.h index 01b9ce9726..f6c5a3d3ab 100644 --- a/third_party/aom/common/y4minput.h +++ b/third_party/aom/common/y4minput.h @@ -57,8 +57,14 @@ struct y4m_input { unsigned int bit_depth; }; +/** + * Open the input file, treating it as Y4M. y4m_input is filled in after + * reading it. Note that chroma-sample-position should only be set for 420 + * input, and the input chroma is shifted if necessary. The code does not + * support the conversion from co-located to vertical. + */ int y4m_input_open(y4m_input *_y4m, FILE *_fin, char *_skip, int _nskip, - int only_420); + aom_chroma_sample_position_t csp, int only_420); void y4m_input_close(y4m_input *_y4m); int y4m_input_fetch_frame(y4m_input *_y4m, FILE *_fin, aom_image_t *img); diff --git a/third_party/aom/examples/analyzer.cc b/third_party/aom/examples/analyzer.cc index 6a42eca248..261d085697 100644 --- a/third_party/aom/examples/analyzer.cc +++ b/third_party/aom/examples/analyzer.cc @@ -162,7 +162,7 @@ bool AV1Decoder::setInspectionCallback() { void AV1Decoder::inspect(void *pbi, void *data) { AV1Decoder *decoder = (AV1Decoder *)data; - ifd_inspect(&decoder->frame_data, pbi); + ifd_inspect(&decoder->frame_data, pbi, 0); } #define MIN_ZOOM (1) diff --git a/third_party/aom/examples/aom_cx_set_ref.c b/third_party/aom/examples/aom_cx_set_ref.c index 8e3d216fe6..5a78af9f78 100644 --- a/third_party/aom/examples/aom_cx_set_ref.c +++ b/third_party/aom/examples/aom_cx_set_ref.c @@ -141,9 +141,7 @@ static int encode_frame(aom_codec_ctx_t *ecodec, aom_image_t *img, if (pkt->kind == AOM_CODEC_CX_FRAME_PKT) { const int keyframe = (pkt->data.frame.flags & AOM_FRAME_IS_KEY) != 0; - if (!(pkt->data.frame.flags & AOM_FRAME_IS_FRAGMENT)) { - *frame_out += 1; - } + ++*frame_out; if (!aom_video_writer_write_frame(writer, pkt->data.frame.buf, pkt->data.frame.sz, diff --git a/third_party/aom/examples/av1_dec_fuzzer.cc b/third_party/aom/examples/av1_dec_fuzzer.cc new file mode 100644 index 0000000000..96d16a8564 --- /dev/null +++ b/third_party/aom/examples/av1_dec_fuzzer.cc @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2019, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +/* + * See build_av1_dec_fuzzer.sh for building instructions. + */ + +#include +#include +#include +#include +#include + +#include "config/aom_config.h" +#include "aom/aom_decoder.h" +#include "aom/aomdx.h" +#include "aom_ports/mem_ops.h" +#include "common/ivfdec.h" + +static void close_file(FILE *file) { fclose(file); } + +extern "C" void usage_exit(void) { exit(EXIT_FAILURE); } + +extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { + std::unique_ptr file( + fmemopen((void *)data, size, "rb"), &close_file); + if (file == nullptr) { + return 0; + } + + char header[32]; + if (fread(header, 1, 32, file.get()) != 32) { + return 0; + } + const AvxInterface *decoder = get_aom_decoder_by_name("av1"); + if (decoder == nullptr) { + return 0; + } + + aom_codec_ctx_t codec; + // Set thread count in the range [1, 64]. + const unsigned int threads = (header[0] & 0x3f) + 1; + aom_codec_dec_cfg_t cfg = { threads, 0, 0, CONFIG_LOWBITDEPTH }; + if (aom_codec_dec_init(&codec, decoder->codec_interface(), &cfg, 0)) { + return 0; + } + + uint8_t *buffer = nullptr; + size_t buffer_size = 0; + size_t frame_size = 0; + while (!ivf_read_frame(file.get(), &buffer, &frame_size, &buffer_size, + nullptr)) { + const aom_codec_err_t err = + aom_codec_decode(&codec, buffer, frame_size, nullptr); + static_cast(err); + aom_codec_iter_t iter = nullptr; + aom_image_t *img = nullptr; + while ((img = aom_codec_get_frame(&codec, &iter)) != nullptr) { + } + } + aom_codec_destroy(&codec); + free(buffer); + return 0; +} diff --git a/third_party/aom/examples/build_av1_dec_fuzzer.sh b/third_party/aom/examples/build_av1_dec_fuzzer.sh new file mode 100644 index 0000000000..86992a0990 --- /dev/null +++ b/third_party/aom/examples/build_av1_dec_fuzzer.sh @@ -0,0 +1,78 @@ +#!/bin/bash +# +# Copyright (c) 2019, Alliance for Open Media. All rights reserved +# +# This source code is subject to the terms of the BSD 2 Clause License and +# the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +# was not distributed with this source code in the LICENSE file, you can +# obtain it at www.aomedia.org/license/software. If the Alliance for Open +# Media Patent License 1.0 was not distributed with this source code in the +# PATENTS file, you can obtain it at www.aomedia.org/license/patent. +# +############################################################################### +# Fuzzer for libaom decoder. +# ========================== +# Requirements +# --------------------- +# Clang6.0 or above (must support -fsanitize=fuzzer) +# +# References: +# --------------------- +# http://llvm.org/docs/LibFuzzer.html +# https://github.com/google/oss-fuzz +# +# Steps to build / run +# --------------------- + +set -eu + +# Have a copy of AOM and a build directory ready. +if [[ $# -ne 2 ]]; then + echo "Pass in the AOM source tree as first argument, and a build directory " + echo "as the second argument. The AOM source tree can be obtained via: " + echo " git clone https://aomedia.googlesource.com/aom" + exit 2 +fi +if [[ -z "$CC" ]]; then + echo "Set the CC environment variable to point to your C compiler." + exit 2 +fi +if [[ -z "$CXX" ]]; then + echo "Set the CXX environment variable to point to your C++ compiler." + exit 2 +fi + +AOM_DIR=$1 +BUILD_DIR=$2 +# Run CMake with address sanitizer enabled and build the codec. +# Enable DO_RANGE_CHECK_CLAMP to suppress the noise of integer overflows +# in the transform functions. Also set memory limits. +EXTRA_C_FLAGS='-DDO_RANGE_CHECK_CLAMP=1 -DAOM_MAX_ALLOCABLE_MEMORY=1073741824' +cd "${BUILD_DIR}" +cmake "${AOM_DIR}" -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCONFIG_PIC=1 \ + -DCONFIG_SCALABILITY=0 -DCONFIG_LOWBITDEPTH=1 -DCONFIG_AV1_ENCODER=0 \ + -DENABLE_EXAMPLES=0 -DENABLE_DOCS=0 -DENABLE_TESTS=0 -DCONFIG_SIZE_LIMIT=1 \ + -DDECODE_HEIGHT_LIMIT=12288 -DDECODE_WIDTH_LIMIT=12288 \ + -DAOM_EXTRA_C_FLAGS="${EXTRA_C_FLAGS}" \ + -DAOM_EXTRA_CXX_FLAGS="${EXTRA_C_FLAGS}" -DSANITIZE=address + +# Build the codec. +make -j$(nproc) + +# Build some libaom utils that are not part of the core lib. +$CC -std=c99 -c -I${AOM_DIR} -I${BUILD_DIR} \ + ${AOM_DIR}/common/ivfdec.c -o ${BUILD_DIR}/ivfdec.o + +$CC -std=c99 -c -I${AOM_DIR} -I${BUILD_DIR} \ + ${AOM_DIR}/common/tools_common.c -o ${BUILD_DIR}/tools_common.o + +# Build the av1 fuzzer +$CXX -std=c++11 -DDECODER=av1 -I${AOM_DIR} -I${BUILD_DIR} \ + -fsanitize=fuzzer -Wl,--start-group \ + ${AOM_DIR}/examples/av1_dec_fuzzer.cc -o ${BUILD_DIR}/av1_dec_fuzzer \ + ${BUILD_DIR}/libaom.a ${BUILD_DIR}/ivfdec.o ${BUILD_DIR}/tools_common.o \ + -Wl,--end-group + +echo "Fuzzer built at ${BUILD_DIR}/av1_dec_fuzzer." +echo "Create a corpus directory, copy IVF files in there, and run:" +echo " av1_dec_fuzzer CORPUS_DIR" diff --git a/third_party/aom/examples/inspect.c b/third_party/aom/examples/inspect.c index 9d5f0dcfc4..050e3da3da 100644 --- a/third_party/aom/examples/inspect.c +++ b/third_party/aom/examples/inspect.c @@ -60,7 +60,9 @@ typedef enum { DUAL_FILTER_LAYER = 1 << 12, Q_INDEX_LAYER = 1 << 13, SEGMENT_ID_LAYER = 1 << 14, - ALL_LAYERS = (1 << 15) - 1 + MOTION_MODE_LAYER = 1 << 15, + COMPOUND_TYPE_LAYER = 1 << 16, + ALL_LAYERS = (1 << 17) - 1 } LayerType; static LayerType layers = 0; @@ -84,6 +86,10 @@ static const arg_def_t dump_transform_size_arg = static const arg_def_t dump_transform_type_arg = ARG_DEF("tt", "transformType", 0, "Dump Transform Type"); static const arg_def_t dump_mode_arg = ARG_DEF("m", "mode", 0, "Dump Mode"); +static const arg_def_t dump_motion_mode_arg = + ARG_DEF("mm", "motion_mode", 0, "Dump Motion Modes"); +static const arg_def_t dump_compound_type_arg = + ARG_DEF("ct", "compound_type", 0, "Dump Compound Types"); static const arg_def_t dump_uv_mode_arg = ARG_DEF("uvm", "uv_mode", 0, "Dump UV Intra Prediction Modes"); static const arg_def_t dump_skip_arg = ARG_DEF("s", "skip", 0, "Dump Skip"); @@ -101,6 +107,13 @@ static const arg_def_t dump_delta_q_arg = static const arg_def_t dump_seg_id_arg = ARG_DEF("si", "seg_id", 0, "Dump Segment ID"); static const arg_def_t usage_arg = ARG_DEF("h", "help", 0, "Help"); +static const arg_def_t skip_non_transform_arg = ARG_DEF( + "snt", "skip_non_transform", 1, "Skip is counted as a non transform."); +static const arg_def_t combined_arg = + ARG_DEF("comb", "combined", 1, "combinining parameters into one output."); + +int combined_parm_list[5]; +int combined_parm_count = 0; static const arg_def_t *main_args[] = { &limit_arg, &dump_all_arg, @@ -113,6 +126,8 @@ static const arg_def_t *main_args[] = { &limit_arg, &dump_transform_type_arg, &dump_mode_arg, &dump_uv_mode_arg, + &dump_motion_mode_arg, + &dump_compound_type_arg, &dump_skip_arg, &dump_filter_arg, &dump_cdef_arg, @@ -123,6 +138,8 @@ static const arg_def_t *main_args[] = { &limit_arg, &dump_delta_q_arg, &dump_seg_id_arg, &usage_arg, + &skip_non_transform_arg, + &combined_arg, NULL }; #define ENUM(name) \ { #name, name } @@ -150,12 +167,14 @@ const map_entry block_size_map[] = { ENUM(BLOCK_64X16), LAST_ENUM }; +#define TX_SKIP -1 + const map_entry tx_size_map[] = { ENUM(TX_4X4), ENUM(TX_8X8), ENUM(TX_16X16), ENUM(TX_32X32), ENUM(TX_64X64), ENUM(TX_4X8), ENUM(TX_8X4), ENUM(TX_8X16), ENUM(TX_16X8), ENUM(TX_16X32), ENUM(TX_32X16), ENUM(TX_32X64), ENUM(TX_64X32), ENUM(TX_4X16), ENUM(TX_16X4), ENUM(TX_8X32), - ENUM(TX_32X8), LAST_ENUM + ENUM(TX_32X8), ENUM(TX_16X64), ENUM(TX_64X16), LAST_ENUM }; const map_entry tx_type_map[] = { ENUM(DCT_DCT), @@ -193,6 +212,15 @@ const map_entry prediction_mode_map[] = { ENUM(NEW_NEWMV), ENUM(INTRA_INVALID), LAST_ENUM }; +const map_entry motion_mode_map[] = { ENUM(SIMPLE_TRANSLATION), + ENUM(OBMC_CAUSAL), // 2-sided OBMC + ENUM(WARPED_CAUSAL), // 2-sided WARPED + LAST_ENUM }; + +const map_entry compound_type_map[] = { ENUM(COMPOUND_AVERAGE), + ENUM(COMPOUND_WEDGE), + ENUM(COMPOUND_DIFFWTD), LAST_ENUM }; + const map_entry uv_prediction_mode_map[] = { ENUM(UV_DC_PRED), ENUM(UV_V_PRED), ENUM(UV_H_PRED), ENUM(UV_D45_PRED), @@ -212,6 +240,43 @@ const map_entry config_map[] = { ENUM(MI_SIZE), LAST_ENUM }; static const char *exec_name; +struct parm_offset { + char parm[60]; + char offset; +}; +struct parm_offset parm_offsets[] = { + { "blockSize", offsetof(insp_mi_data, sb_type) }, + { "transformSize", offsetof(insp_mi_data, tx_size) }, + { "transformType", offsetof(insp_mi_data, tx_type) }, + { "dualFilterType", offsetof(insp_mi_data, dual_filter_type) }, + { "mode", offsetof(insp_mi_data, mode) }, + { "uv_mode", offsetof(insp_mi_data, uv_mode) }, + { "motion_mode", offsetof(insp_mi_data, motion_mode) }, + { "compound_type", offsetof(insp_mi_data, compound_type) }, + { "referenceFrame", offsetof(insp_mi_data, ref_frame) }, + { "skip", offsetof(insp_mi_data, skip) }, +}; +int parm_count = sizeof(parm_offsets) / sizeof(parm_offsets[0]); + +int convert_to_indices(char *str, int *indices, int maxCount, int *count) { + *count = 0; + do { + char *comma = strchr(str, ','); + int length = (comma ? (int)(comma - str) : (int)strlen(str)); + int i; + for (i = 0; i < parm_count; ++i) { + if (!strncmp(str, parm_offsets[i].parm, length)) { + break; + } + } + if (i == parm_count) return 0; + indices[(*count)++] = i; + if (*count > maxCount) return 0; + str += length + 1; + } while (strlen(str) > 0); + return 1; +} + insp_frame_data frame_data; int frame_count = 0; int decoded_frame_count = 0; @@ -382,6 +447,38 @@ int put_motion_vectors(char *buffer) { return (int)(buf - buffer); } +int put_combined(char *buffer) { + const int mi_rows = frame_data.mi_rows; + const int mi_cols = frame_data.mi_cols; + char *buf = buffer; + int r, c, p; + buf += put_str(buf, " \""); + for (p = 0; p < combined_parm_count; ++p) { + if (p) buf += put_str(buf, "&"); + buf += put_str(buf, parm_offsets[combined_parm_list[p]].parm); + } + buf += put_str(buf, "\": ["); + for (r = 0; r < mi_rows; ++r) { + *(buf++) = '['; + for (c = 0; c < mi_cols; ++c) { + insp_mi_data *mi = &frame_data.mi_grid[r * mi_cols + c]; + *(buf++) = '['; + for (p = 0; p < combined_parm_count; ++p) { + if (p) *(buf++) = ','; + int16_t *v = (int16_t *)(((int8_t *)mi) + + parm_offsets[combined_parm_list[p]].offset); + buf += put_num(buf, 0, v[0], 0); + } + *(buf++) = ']'; + if (c < mi_cols - 1) *(buf++) = ','; + } + *(buf++) = ']'; + if (r < mi_rows - 1) *(buf++) = ','; + } + buf += put_str(buf, "],\n"); + return (int)(buf - buffer); +} + int put_block_info(char *buffer, const map_entry *map, const char *name, size_t offset, int len) { const int mi_rows = frame_data.mi_rows; @@ -490,9 +587,16 @@ int put_accounting(char *buffer) { } #endif +int skip_non_transform = 0; + void inspect(void *pbi, void *data) { /* Fetch frame data. */ - ifd_inspect(&frame_data, pbi); + ifd_inspect(&frame_data, pbi, skip_non_transform); + + // Show existing frames just show a reference buffer we've already decoded. + // There's no information to show. + if (frame_data.show_existing_frame) return; + (void)data; // We allocate enough space and hope we don't write out of bounds. Totally // unsafe but this speeds things up, especially when compiled to Javascript. @@ -523,6 +627,14 @@ void inspect(void *pbi, void *data) { buf += put_block_info(buf, uv_prediction_mode_map, "uv_mode", offsetof(insp_mi_data, uv_mode), 0); } + if (layers & MOTION_MODE_LAYER) { + buf += put_block_info(buf, motion_mode_map, "motion_mode", + offsetof(insp_mi_data, motion_mode), 0); + } + if (layers & COMPOUND_TYPE_LAYER) { + buf += put_block_info(buf, compound_type_map, "compound_type", + offsetof(insp_mi_data, compound_type), 0); + } if (layers & SKIP_LAYER) { buf += put_block_info(buf, skip_map, "skip", offsetof(insp_mi_data, skip), 0); @@ -554,6 +666,7 @@ void inspect(void *pbi, void *data) { if (layers & MOTION_VECTORS_LAYER) { buf += put_motion_vectors(buf); } + if (combined_parm_count > 0) buf += put_combined(buf); if (layers & REFERENCE_FRAME_LAYER) { buf += put_block_info(buf, refs_map, "referenceFrame", offsetof(insp_mi_data, ref_frame), 2); @@ -563,7 +676,8 @@ void inspect(void *pbi, void *data) { buf += put_accounting(buf); } #endif - buf += snprintf(buf, MAX_BUFFER, " \"frame\": %d,\n", decoded_frame_count); + buf += + snprintf(buf, MAX_BUFFER, " \"frame\": %d,\n", frame_data.frame_number); buf += snprintf(buf, MAX_BUFFER, " \"showFrame\": %d,\n", frame_data.show_frame); buf += snprintf(buf, MAX_BUFFER, " \"frameType\": %d,\n", @@ -618,21 +732,51 @@ int open_file(char *file) { return EXIT_SUCCESS; } +Av1DecodeReturn adr; +int have_frame = 0; +const unsigned char *frame; +const unsigned char *end_frame; +size_t frame_size = 0; + EMSCRIPTEN_KEEPALIVE int read_frame() { - if (!aom_video_reader_read_frame(reader)) return EXIT_FAILURE; img = NULL; - aom_codec_iter_t iter = NULL; - size_t frame_size = 0; - const unsigned char *frame = aom_video_reader_get_frame(reader, &frame_size); - if (aom_codec_decode(&codec, frame, (unsigned int)frame_size, NULL) != - AOM_CODEC_OK) { - die_codec(&codec, "Failed to decode frame."); - } + + // This loop skips over any frames that are show_existing_frames, as + // there is nothing to analyze. + do { + if (!have_frame) { + if (!aom_video_reader_read_frame(reader)) return EXIT_FAILURE; + frame = aom_video_reader_get_frame(reader, &frame_size); + + have_frame = 1; + end_frame = frame + frame_size; + } + + if (aom_codec_decode(&codec, frame, (unsigned int)frame_size, &adr) != + AOM_CODEC_OK) { + die_codec(&codec, "Failed to decode frame."); + } + + frame = adr.buf; + if (frame == end_frame) have_frame = 0; + } while (adr.show_existing); + int got_any_frames = 0; aom_image_t *frame_img; - while ((frame_img = aom_codec_get_frame(&codec, &iter))) { - img = frame_img; + struct av1_ref_frame ref_dec; + ref_dec.idx = adr.idx; + + // ref_dec.idx is the index to the reference buffer idx to AV1_GET_REFERENCE + // if its -1 the decoder didn't update any reference buffer and the only + // way to see the frame is aom_codec_get_frame. + if (ref_dec.idx == -1) { + aom_codec_iter_t iter = NULL; + img = frame_img = aom_codec_get_frame(&codec, &iter); + ++frame_count; + got_any_frames = 1; + } else if (!aom_codec_control(&codec, AV1_GET_REFERENCE, &ref_dec)) { + img = frame_img = &ref_dec.img; ++frame_count; got_any_frames = 1; } @@ -692,6 +836,10 @@ static void parse_args(char **argv) { layers |= MODE_LAYER; else if (arg_match(&arg, &dump_uv_mode_arg, argi)) layers |= UV_MODE_LAYER; + else if (arg_match(&arg, &dump_motion_mode_arg, argi)) + layers |= MOTION_MODE_LAYER; + else if (arg_match(&arg, &dump_compound_type_arg, argi)) + layers |= COMPOUND_TYPE_LAYER; else if (arg_match(&arg, &dump_skip_arg, argi)) layers |= SKIP_LAYER; else if (arg_match(&arg, &dump_filter_arg, argi)) @@ -718,6 +866,13 @@ static void parse_args(char **argv) { usage_exit(); else if (arg_match(&arg, &limit_arg, argi)) stop_after = arg_parse_uint(&arg); + else if (arg_match(&arg, &skip_non_transform_arg, argi)) + skip_non_transform = arg_parse_uint(&arg); + else if (arg_match(&arg, &combined_arg, argi)) + convert_to_indices( + (char *)arg.val, combined_parm_list, + sizeof(combined_parm_list) / sizeof(combined_parm_list[0]), + &combined_parm_count); else argj++; } diff --git a/third_party/aom/examples/lightfield_bitstream_parsing.c b/third_party/aom/examples/lightfield_bitstream_parsing.c index 159f1617a3..afacf44043 100644 --- a/third_party/aom/examples/lightfield_bitstream_parsing.c +++ b/third_party/aom/examples/lightfield_bitstream_parsing.c @@ -13,13 +13,28 @@ // ============================ // // This is a lightfield bitstream parsing example. It takes an input file -// containing the whole compressed lightfield bitstream(ivf file), and parses it -// and constructs and outputs a new bitstream that can be decoded by an AV1 -// decoder. The output bitstream contains reference frames(i.e. anchor frames), -// camera frame header, and tile list OBUs. num_references is the number of -// anchor frames coded at the beginning of the light field file. -// After running the lightfield encoder, run lightfield bitstream parsing: +// containing the whole compressed lightfield bitstream(ivf file) and a text +// file containing a stream of tiles to decode and then constructs and outputs +// a new bitstream that can be decoded by an AV1 decoder. The output bitstream +// contains reference frames(i.e. anchor frames), camera frame header, and +// tile list OBUs. num_references is the number of anchor frames coded at the +// beginning of the light field file. After running the lightfield encoder, +// run lightfield bitstream parsing: // examples/lightfield_bitstream_parsing vase10x10.ivf vase_tile_list.ivf 4 +// tile_list.txt +// +// The tile_list.txt is expected to be of the form: +// Frame +// +// +// ... +// Frame #include @@ -39,7 +54,7 @@ static const char *exec_name; void usage_exit(void) { - fprintf(stderr, "Usage: %s \n", + fprintf(stderr, "Usage: %s \n", exec_name); exit(EXIT_FAILURE); } @@ -47,9 +62,8 @@ void usage_exit(void) { #define ALIGN_POWER_OF_TWO(value, n) \ (((value) + ((1 << (n)) - 1)) & ~((1 << (n)) - 1)) -// SB size: 64x64 -const uint8_t output_frame_width_in_tiles_minus_1 = 512 / 64 - 1; -const uint8_t output_frame_height_in_tiles_minus_1 = 512 / 64 - 1; +const int output_frame_width = 512; +const int output_frame_height = 512; // Spec: // typedef struct { @@ -68,32 +82,6 @@ typedef struct { int tile_row; } TILE_LIST_INFO; -// M references: 0 - M-1; N images(including references): 0 - N-1; -// Note: order the image index incrementally, so that we only go through the -// bitstream once to construct the tile list. -const int num_tile_lists = 2; -const uint16_t tile_count_minus_1 = 9 - 1; -const TILE_LIST_INFO tile_list[2][9] = { - { { 16, 0, 4, 5 }, - { 83, 3, 13, 2 }, - { 57, 2, 2, 6 }, - { 31, 1, 11, 5 }, - { 2, 0, 7, 4 }, - { 77, 3, 9, 9 }, - { 49, 1, 0, 1 }, - { 6, 0, 3, 10 }, - { 63, 2, 5, 8 } }, - { { 65, 2, 11, 1 }, - { 42, 1, 3, 7 }, - { 88, 3, 8, 4 }, - { 76, 3, 1, 15 }, - { 1, 0, 2, 2 }, - { 19, 0, 5, 6 }, - { 60, 2, 4, 0 }, - { 25, 1, 11, 15 }, - { 50, 2, 5, 4 } }, -}; - static int get_image_bps(aom_img_fmt_t fmt) { switch (fmt) { case AOM_IMG_FMT_I420: return 12; @@ -107,6 +95,102 @@ static int get_image_bps(aom_img_fmt_t fmt) { return 0; } +void process_tile_list(const TILE_LIST_INFO *tiles, int num_tiles, + aom_codec_pts_t tl_pts, unsigned char **frames, + const size_t *frame_sizes, aom_codec_ctx_t *codec, + unsigned char *tl_buf, AvxVideoWriter *writer, + uint8_t output_frame_width_in_tiles_minus_1, + uint8_t output_frame_height_in_tiles_minus_1) { + unsigned char *tl = tl_buf; + struct aom_write_bit_buffer wb = { tl, 0 }; + unsigned char *saved_obu_size_loc = NULL; + uint32_t tile_list_obu_header_size = 0; + uint32_t tile_list_obu_size = 0; + int num_tiles_minus_1 = num_tiles - 1; + int i; + + // Write the tile list OBU header that is 1 byte long. + aom_wb_write_literal(&wb, 0, 1); // forbidden bit. + aom_wb_write_literal(&wb, 8, 4); // tile list OBU: "1000" + aom_wb_write_literal(&wb, 0, 1); // obu_extension = 0 + aom_wb_write_literal(&wb, 1, 1); // obu_has_size_field + aom_wb_write_literal(&wb, 0, 1); // reserved + tl++; + tile_list_obu_header_size++; + + // Write the OBU size using a fixed length_field_size of 4 bytes. + saved_obu_size_loc = tl; + // aom_wb_write_unsigned_literal(&wb, data, bits) requires that bits <= 32. + aom_wb_write_unsigned_literal(&wb, 0, 32); + tl += 4; + tile_list_obu_header_size += 4; + + // write_tile_list_obu() + aom_wb_write_literal(&wb, output_frame_width_in_tiles_minus_1, 8); + aom_wb_write_literal(&wb, output_frame_height_in_tiles_minus_1, 8); + aom_wb_write_literal(&wb, num_tiles_minus_1, 16); + tl += 4; + tile_list_obu_size += 4; + + // Write each tile's data + for (i = 0; i <= num_tiles_minus_1; i++) { + aom_tile_data tile_data = { 0, NULL, 0 }; + + int image_idx = tiles[i].image_idx; + int ref_idx = tiles[i].reference_idx; + int tc = tiles[i].tile_col; + int tr = tiles[i].tile_row; + + // Reset bit writer to the right location. + wb.bit_buffer = tl; + wb.bit_offset = 0; + + size_t frame_size = frame_sizes[image_idx]; + const unsigned char *frame = frames[image_idx]; + + aom_codec_control_(codec, AV1_SET_DECODE_TILE_ROW, tr); + aom_codec_control_(codec, AV1_SET_DECODE_TILE_COL, tc); + + aom_codec_err_t aom_status = + aom_codec_decode(codec, frame, frame_size, NULL); + if (aom_status) die_codec(codec, "Failed to decode tile."); + + aom_codec_control_(codec, AV1D_GET_TILE_DATA, &tile_data); + + // Copy over tile info. + // uint8_t anchor_frame_idx; + // uint8_t tile_row; + // uint8_t tile_col; + // uint16_t coded_tile_data_size_minus_1; + // uint8_t *coded_tile_data; + uint32_t tile_info_bytes = 5; + aom_wb_write_literal(&wb, ref_idx, 8); + aom_wb_write_literal(&wb, tr, 8); + aom_wb_write_literal(&wb, tc, 8); + aom_wb_write_literal(&wb, (int)tile_data.coded_tile_data_size - 1, 16); + tl += tile_info_bytes; + + memcpy(tl, (uint8_t *)tile_data.coded_tile_data, + tile_data.coded_tile_data_size); + tl += tile_data.coded_tile_data_size; + + tile_list_obu_size += + tile_info_bytes + (uint32_t)tile_data.coded_tile_data_size; + } + + // Write tile list OBU size. + size_t bytes_written = 0; + if (aom_uleb_encode_fixed_size(tile_list_obu_size, 4, 4, saved_obu_size_loc, + &bytes_written)) + die_codec(codec, "Failed to encode the tile list obu size."); + + // Copy the tile list. + if (!aom_video_writer_write_frame( + writer, tl_buf, tile_list_obu_header_size + tile_list_obu_size, + tl_pts)) + die_codec(codec, "Failed to copy compressed tile list."); +} + int main(int argc, char **argv) { aom_codec_ctx_t codec; AvxVideoReader *reader = NULL; @@ -114,11 +198,12 @@ int main(int argc, char **argv) { const AvxInterface *decoder = NULL; const AvxVideoInfo *info = NULL; int num_references; - int n, i; + int i; aom_codec_pts_t pts; + const char *tile_list_file = NULL; exec_name = argv[0]; - if (argc != 4) die("Invalid number of arguments."); + if (argc != 5) die("Invalid number of arguments."); reader = aom_video_reader_open(argv[1]); if (!reader) die("Failed to open %s for reading.", argv[1]); @@ -126,11 +211,15 @@ int main(int argc, char **argv) { num_references = (int)strtol(argv[3], NULL, 0); info = aom_video_reader_get_info(reader); + aom_video_reader_set_fourcc(reader, AV1_FOURCC); + // The writer to write out ivf file in tile list OBU, which can be decoded by // AV1 decoder. writer = aom_video_writer_open(argv[2], kContainerIVF, info); if (!writer) die("Failed to open %s for writing", argv[2]); + tile_list_file = argv[4]; + decoder = get_aom_decoder_by_fourcc(info->codec_fourcc); if (!decoder) die("Unknown input codec."); printf("Using %s\n", aom_codec_iface_name(decoder->codec_interface())); @@ -141,6 +230,7 @@ int main(int argc, char **argv) { // Decode anchor frames. aom_codec_control_(&codec, AV1_SET_TILE_MODE, 0); + printf("Reading %d reference images.\n", num_references); for (i = 0; i < num_references; ++i) { aom_video_reader_read_frame(reader); @@ -165,16 +255,38 @@ int main(int argc, char **argv) { // Record the offset of the first camera image. const FileOffset camera_frame_pos = ftello(infile); - // Read out the first camera frame. - aom_video_reader_read_frame(reader); + printf("Loading compressed frames into memory.\n"); + + // Count the frames in the lightfield. + int num_frames = 0; + while (aom_video_reader_read_frame(reader)) { + ++num_frames; + } + if (num_frames < 1) die("Input light field has no frames."); + + // Read all of the lightfield frames into memory. + unsigned char **frames = + (unsigned char **)malloc(num_frames * sizeof(unsigned char *)); + size_t *frame_sizes = (size_t *)malloc(num_frames * sizeof(size_t)); + // Seek to the first camera image. + fseeko(infile, camera_frame_pos, SEEK_SET); + for (int f = 0; f < num_frames; ++f) { + aom_video_reader_read_frame(reader); + size_t frame_size = 0; + const unsigned char *frame = + aom_video_reader_get_frame(reader, &frame_size); + frames[f] = (unsigned char *)malloc(frame_size * sizeof(unsigned char)); + memcpy(frames[f], frame, frame_size); + frame_sizes[f] = frame_size; + } + printf("Read %d frames.\n", num_frames); // Copy first camera frame for getting camera frame header. This is done // only once. { - size_t frame_size = 0; - const unsigned char *frame = - aom_video_reader_get_frame(reader, &frame_size); - pts = (aom_codec_pts_t)aom_video_reader_get_frame_pts(reader); + size_t frame_size = frame_sizes[0]; + const unsigned char *frame = frames[0]; + pts = num_references; aom_tile_data frame_header_info = { 0, NULL, 0 }; // Need to decode frame header to get camera frame header info. So, here @@ -231,115 +343,68 @@ int main(int argc, char **argv) { // Allocate a buffer to store tile list bitstream. const size_t data_sz = MAX_TILES * ALIGN_POWER_OF_TWO(tile_width, 5) * ALIGN_POWER_OF_TWO(tile_height, 5) * bps / 8; + unsigned char *tl_buf = (unsigned char *)malloc(data_sz); if (tl_buf == NULL) die_codec(&codec, "Failed to allocate tile list buffer."); - aom_codec_pts_t tl_pts = pts; + aom_codec_pts_t tl_pts = num_references; + const uint8_t output_frame_width_in_tiles_minus_1 = + output_frame_width / tile_width - 1; + const uint8_t output_frame_height_in_tiles_minus_1 = + output_frame_height / tile_height - 1; - // Process 1 tile list. - for (n = 0; n < num_tile_lists; n++) { - unsigned char *tl = tl_buf; - struct aom_write_bit_buffer wb = { tl, 0 }; - unsigned char *saved_obu_size_loc = NULL; - uint32_t tile_list_obu_header_size = 0; - uint32_t tile_list_obu_size = 0; - - // Write the tile list OBU header that is 1 byte long. - aom_wb_write_literal(&wb, 0, 1); // forbidden bit. - aom_wb_write_literal(&wb, 8, 4); // tile list OBU: "1000" - aom_wb_write_literal(&wb, 0, 1); // obu_extension = 0 - aom_wb_write_literal(&wb, 1, 1); // obu_has_size_field - aom_wb_write_literal(&wb, 0, 1); // reserved - tl++; - tile_list_obu_header_size++; - - // Write the OBU size using a fixed length_field_size of 4 bytes. - saved_obu_size_loc = tl; - // aom_wb_write_unsigned_literal(&wb, data, bits) requires that bits <= 32. - aom_wb_write_unsigned_literal(&wb, 0, 32); - tl += 4; - tile_list_obu_header_size += 4; - - // write_tile_list_obu() - aom_wb_write_literal(&wb, output_frame_width_in_tiles_minus_1, 8); - aom_wb_write_literal(&wb, output_frame_height_in_tiles_minus_1, 8); - aom_wb_write_literal(&wb, tile_count_minus_1, 16); - tl += 4; - tile_list_obu_size += 4; - - // Write each tile's data - for (i = 0; i <= tile_count_minus_1; i++) { - aom_tile_data tile_data = { 0, NULL, 0 }; - - int image_idx = tile_list[n][i].image_idx; - int ref_idx = tile_list[n][i].reference_idx; - int tc = tile_list[n][i].tile_col; - int tr = tile_list[n][i].tile_row; - int frame_cnt = -1; - - // Reset bit writer to the right location. - wb.bit_buffer = tl; - wb.bit_offset = 0; - - // Seek to the first camera image. - fseeko(infile, camera_frame_pos, SEEK_SET); - - // Read out the camera image - while (frame_cnt != image_idx) { - aom_video_reader_read_frame(reader); - frame_cnt++; + printf("Reading tile list from file.\n"); + char line[1024]; + FILE *tile_list_fptr = fopen(tile_list_file, "r"); + if (!tile_list_fptr) die_codec(&codec, "Failed to open tile list file."); + int num_tiles = 0; + TILE_LIST_INFO tiles[MAX_TILES]; + while ((fgets(line, 1024, tile_list_fptr)) != NULL) { + if (line[0] == 'F' || num_tiles >= MAX_TILES) { + // Flush existing tile list and start another, either because we hit a + // new render frame or because we've hit our max number of tiles per list. + if (num_tiles > 0) { + process_tile_list(tiles, num_tiles, tl_pts, frames, frame_sizes, &codec, + tl_buf, writer, output_frame_width_in_tiles_minus_1, + output_frame_height_in_tiles_minus_1); + ++tl_pts; } - - size_t frame_size = 0; - const unsigned char *frame = - aom_video_reader_get_frame(reader, &frame_size); - - aom_codec_control_(&codec, AV1_SET_DECODE_TILE_ROW, tr); - aom_codec_control_(&codec, AV1_SET_DECODE_TILE_COL, tc); - - aom_codec_err_t aom_status = - aom_codec_decode(&codec, frame, frame_size, NULL); - if (aom_status) die_codec(&codec, "Failed to decode tile."); - - aom_codec_control_(&codec, AV1D_GET_TILE_DATA, &tile_data); - - // Copy over tile info. - // uint8_t anchor_frame_idx; - // uint8_t tile_row; - // uint8_t tile_col; - // uint16_t coded_tile_data_size_minus_1; - // uint8_t *coded_tile_data; - uint32_t tile_info_bytes = 5; - aom_wb_write_literal(&wb, ref_idx, 8); - aom_wb_write_literal(&wb, tr, 8); - aom_wb_write_literal(&wb, tc, 8); - aom_wb_write_literal(&wb, (int)tile_data.coded_tile_data_size - 1, 16); - tl += tile_info_bytes; - - memcpy(tl, (uint8_t *)tile_data.coded_tile_data, - tile_data.coded_tile_data_size); - tl += tile_data.coded_tile_data_size; - - tile_list_obu_size += - tile_info_bytes + (uint32_t)tile_data.coded_tile_data_size; + num_tiles = 0; } - - // Write tile list OBU size. - size_t bytes_written = 0; - if (aom_uleb_encode_fixed_size(tile_list_obu_size, 4, 4, saved_obu_size_loc, - &bytes_written)) - die_codec(&codec, "Failed to encode the tile list obu size."); - - // Copy the tile list. - if (!aom_video_writer_write_frame( - writer, tl_buf, tile_list_obu_header_size + tile_list_obu_size, - tl_pts)) - die_codec(&codec, "Failed to copy compressed tile list."); - - tl_pts++; + if (line[0] == 'F') { + continue; + } + if (sscanf(line, "%d %d %d %d", &tiles[num_tiles].image_idx, + &tiles[num_tiles].reference_idx, &tiles[num_tiles].tile_col, + &tiles[num_tiles].tile_row) == 4) { + if (tiles[num_tiles].image_idx >= num_frames) { + die("Tile list image_idx out of bounds: %d >= %d.", + tiles[num_tiles].image_idx, num_frames); + } + if (tiles[num_tiles].reference_idx >= num_references) { + die("Tile list reference_idx out of bounds: %d >= %d.", + tiles[num_tiles].reference_idx, num_references); + } + ++num_tiles; + } + } + if (num_tiles > 0) { + // Flush out the last tile list. + process_tile_list(tiles, num_tiles, tl_pts, frames, frame_sizes, &codec, + tl_buf, writer, output_frame_width_in_tiles_minus_1, + output_frame_height_in_tiles_minus_1); + ++tl_pts; } + const int num_tile_lists = (int)(tl_pts - pts); + printf("Finished processing tile lists. Num tile lists: %d.\n", + num_tile_lists); free(tl_buf); + for (int f = 0; f < num_frames; ++f) { + free(frames[f]); + } + free(frame_sizes); + free(frames); if (aom_codec_destroy(&codec)) die_codec(&codec, "Failed to destroy codec"); aom_video_writer_close(writer); aom_video_reader_close(reader); diff --git a/third_party/aom/examples/lightfield_decoder.c b/third_party/aom/examples/lightfield_decoder.c index f5e54db7f1..7a445f003f 100644 --- a/third_party/aom/examples/lightfield_decoder.c +++ b/third_party/aom/examples/lightfield_decoder.c @@ -14,10 +14,26 @@ // // This is an example of a simple lightfield decoder. It builds upon the // simple_decoder.c example. It takes an input file containing the compressed -// data (in ivf format), treating it as a lightfield instead of a video. +// data (in ivf format), treating it as a lightfield instead of a video; and a +// text file with a list of tiles to decode. There is an optional parameter +// allowing to choose the output format, and the supported formats are +// YUV1D(default), YUV, and NV12. // After running the lightfield encoder, run lightfield decoder to decode a // batch of tiles: -// examples/lightfield_decoder vase10x10.ivf vase_reference.yuv 4 +// examples/lightfield_decoder vase10x10.ivf vase_reference.yuv 4 tile_list.txt +// 0(optional) +// The tile_list.txt is expected to be of the form: +// Frame +// +// +// ... +// Frame #include @@ -33,43 +49,109 @@ static const char *exec_name; void usage_exit(void) { - fprintf(stderr, "Usage: %s \n", exec_name); + fprintf(stderr, + "Usage: %s \n", + exec_name); exit(EXIT_FAILURE); } -// Tile list entry provided by the application -typedef struct { - int image_idx; - int reference_idx; - int tile_col; - int tile_row; -} TILE_LIST_INFO; +// Output frame size +const int output_frame_width = 512; +const int output_frame_height = 512; -// M references: 0 - M-1; N images(including references): 0 - N-1; -// Note: order the image index incrementally, so that we only go through the -// bitstream once to construct the tile list. -const int num_tile_lists = 2; -const uint16_t tile_count_minus_1 = 9 - 1; -const TILE_LIST_INFO tile_list[2][9] = { - { { 16, 0, 4, 5 }, - { 83, 3, 13, 2 }, - { 57, 2, 2, 6 }, - { 31, 1, 11, 5 }, - { 2, 0, 7, 4 }, - { 77, 3, 9, 9 }, - { 49, 1, 0, 1 }, - { 6, 0, 3, 10 }, - { 63, 2, 5, 8 } }, - { { 65, 2, 11, 1 }, - { 42, 1, 3, 7 }, - { 88, 3, 8, 4 }, - { 76, 3, 1, 15 }, - { 1, 0, 2, 2 }, - { 19, 0, 5, 6 }, - { 60, 2, 4, 0 }, - { 25, 1, 11, 15 }, - { 50, 2, 5, 4 } }, -}; +static void aom_img_copy_tile(const aom_image_t *src, const aom_image_t *dst, + int dst_row_offset, int dst_col_offset) { + const int shift = (src->fmt & AOM_IMG_FMT_HIGHBITDEPTH) ? 1 : 0; + int plane; + + for (plane = 0; plane < 3; ++plane) { + const unsigned char *src_buf = src->planes[plane]; + const int src_stride = src->stride[plane]; + unsigned char *dst_buf = dst->planes[plane]; + const int dst_stride = dst->stride[plane]; + const int roffset = + (plane > 0) ? dst_row_offset >> dst->y_chroma_shift : dst_row_offset; + const int coffset = + (plane > 0) ? dst_col_offset >> dst->x_chroma_shift : dst_col_offset; + + // col offset needs to be adjusted for HBD. + dst_buf += roffset * dst_stride + (coffset << shift); + + const int w = (aom_img_plane_width(src, plane) << shift); + const int h = aom_img_plane_height(src, plane); + int y; + + for (y = 0; y < h; ++y) { + memcpy(dst_buf, src_buf, w); + src_buf += src_stride; + dst_buf += dst_stride; + } + } +} + +void decode_tile(aom_codec_ctx_t *codec, const unsigned char *frame, + size_t frame_size, int tr, int tc, int ref_idx, + aom_image_t *reference_images, aom_image_t *output, + int *tile_idx, unsigned int *output_bit_depth, + aom_image_t **img_ptr, int output_format) { + aom_codec_control_(codec, AV1_SET_TILE_MODE, 1); + aom_codec_control_(codec, AV1D_EXT_TILE_DEBUG, 1); + aom_codec_control_(codec, AV1_SET_DECODE_TILE_ROW, tr); + aom_codec_control_(codec, AV1_SET_DECODE_TILE_COL, tc); + + av1_ref_frame_t ref; + ref.idx = 0; + ref.use_external_ref = 1; + ref.img = reference_images[ref_idx]; + if (aom_codec_control(codec, AV1_SET_REFERENCE, &ref)) { + die_codec(codec, "Failed to set reference frame."); + } + + aom_codec_err_t aom_status = aom_codec_decode(codec, frame, frame_size, NULL); + if (aom_status) die_codec(codec, "Failed to decode tile."); + + aom_codec_iter_t iter = NULL; + aom_image_t *img = aom_codec_get_frame(codec, &iter); + if (!img) die_codec(codec, "Failed to get frame."); + *img_ptr = img; + + // aom_img_alloc() sets bit_depth as follows: + // output->bit_depth = (fmt & AOM_IMG_FMT_HIGHBITDEPTH) ? 16 : 8; + // Use img->bit_depth(read from bitstream), so that aom_shift_img() + // works as expected. + output->bit_depth = img->bit_depth; + *output_bit_depth = img->bit_depth; + + if (output_format != YUV1D) { + // read out the tile size. + unsigned int tile_size = 0; + if (aom_codec_control(codec, AV1D_GET_TILE_SIZE, &tile_size)) + die_codec(codec, "Failed to get the tile size"); + const unsigned int tile_width = tile_size >> 16; + const unsigned int tile_height = tile_size & 65535; + const uint8_t output_frame_width_in_tiles = output_frame_width / tile_width; + + // Copy the tile to the output frame. + const int row_offset = + (*tile_idx / output_frame_width_in_tiles) * tile_height; + const int col_offset = + (*tile_idx % output_frame_width_in_tiles) * tile_width; + + aom_img_copy_tile(img, output, row_offset, col_offset); + (*tile_idx)++; + } +} + +static void img_write_to_file(const aom_image_t *img, FILE *file, + int output_format) { + if (output_format == YUV) + aom_img_write(img, file); + else if (output_format == NV12) + aom_img_write_nv12(img, file); + else + die("Invalid output format"); +} int main(int argc, char **argv) { FILE *outfile = NULL; @@ -78,13 +160,18 @@ int main(int argc, char **argv) { const AvxInterface *decoder = NULL; const AvxVideoInfo *info = NULL; int num_references; + aom_img_fmt_t ref_fmt = 0; aom_image_t reference_images[MAX_EXTERNAL_REFERENCES]; + aom_image_t output; + aom_image_t *output_shifted = NULL; size_t frame_size = 0; const unsigned char *frame = NULL; - int n, i, j; + int i, j; + const char *tile_list_file = NULL; + int output_format = YUV1D; exec_name = argv[0]; - if (argc != 4) die("Invalid number of arguments."); + if (argc < 5) die("Invalid number of arguments."); reader = aom_video_reader_open(argv[1]); if (!reader) die("Failed to open %s for reading.", argv[1]); @@ -93,11 +180,18 @@ int main(int argc, char **argv) { die("Failed to open %s for writing.", argv[2]); num_references = (int)strtol(argv[3], NULL, 0); + tile_list_file = argv[4]; + + if (argc > 5) output_format = (int)strtol(argv[5], NULL, 0); + if (output_format < YUV1D || output_format > NV12) + die("Output format out of range [0, 2]"); info = aom_video_reader_get_info(reader); - decoder = get_aom_decoder_by_fourcc(info->codec_fourcc); - if (!decoder) die("Unknown input codec."); + if (info->codec_fourcc == LST_FOURCC) + decoder = get_aom_decoder_by_fourcc(AV1_FOURCC); + else + die("Unknown input codec."); printf("Using %s\n", aom_codec_iface_name(decoder->codec_interface())); if (aom_codec_dec_init(&codec, decoder->codec_interface(), NULL, 0)) @@ -116,7 +210,6 @@ int main(int argc, char **argv) { die_codec(&codec, "Failed to decode frame."); if (i == 0) { - aom_img_fmt_t ref_fmt = 0; if (aom_codec_control(&codec, AV1D_GET_IMG_FORMAT, &ref_fmt)) die_codec(&codec, "Failed to get the image format"); @@ -127,7 +220,7 @@ int main(int argc, char **argv) { // Allocate memory to store decoded references. Allocate memory with the // border so that it can be used as a reference. for (j = 0; j < num_references; j++) { - unsigned int border = AOM_BORDER_IN_PIXELS; + unsigned int border = AOM_DEC_BORDER_IN_PIXELS; if (!aom_img_alloc_with_border(&reference_images[j], ref_fmt, frame_res[0], frame_res[1], 32, 8, border)) { @@ -156,50 +249,110 @@ int main(int argc, char **argv) { // Record the offset of the first camera image. const FileOffset camera_frame_pos = ftello(infile); - // Process 1 tile. - for (n = 0; n < num_tile_lists; n++) { - for (i = 0; i <= tile_count_minus_1; i++) { - int image_idx = tile_list[n][i].image_idx; - int ref_idx = tile_list[n][i].reference_idx; - int tc = tile_list[n][i].tile_col; - int tr = tile_list[n][i].tile_row; - int frame_cnt = -1; + printf("Loading compressed frames into memory.\n"); - // Seek to the first camera image. - fseeko(infile, camera_frame_pos, SEEK_SET); + // Count the frames in the lightfield. + int num_frames = 0; + while (aom_video_reader_read_frame(reader)) { + ++num_frames; + } + if (num_frames < 1) die("Input light field has no frames."); - // Read out the camera image - while (frame_cnt != image_idx) { - aom_video_reader_read_frame(reader); - frame_cnt++; + // Read all of the lightfield frames into memory. + unsigned char **frames = + (unsigned char **)malloc(num_frames * sizeof(unsigned char *)); + size_t *frame_sizes = (size_t *)malloc(num_frames * sizeof(size_t)); + // Seek to the first camera image. + fseeko(infile, camera_frame_pos, SEEK_SET); + for (int f = 0; f < num_frames; ++f) { + aom_video_reader_read_frame(reader); + frame = aom_video_reader_get_frame(reader, &frame_size); + frames[f] = (unsigned char *)malloc(frame_size * sizeof(unsigned char)); + memcpy(frames[f], frame, frame_size); + frame_sizes[f] = frame_size; + } + printf("Read %d frames.\n", num_frames); + + if (output_format != YUV1D) { + // Allocate the output frame. + aom_img_fmt_t out_fmt = ref_fmt; + if (!CONFIG_LOWBITDEPTH) out_fmt |= AOM_IMG_FMT_HIGHBITDEPTH; + if (!aom_img_alloc(&output, out_fmt, output_frame_width, + output_frame_height, 32)) + die("Failed to allocate output image."); + } + + printf("Decoding tile list from file.\n"); + char line[1024]; + FILE *tile_list_fptr = fopen(tile_list_file, "r"); + int tile_list_cnt = 0; + int tile_list_writes = 0; + int tile_idx = 0; + aom_image_t *out = NULL; + unsigned int output_bit_depth = 0; + + while ((fgets(line, 1024, tile_list_fptr)) != NULL) { + if (line[0] == 'F') { + if (output_format != YUV1D) { + // Write out the tile list. + if (tile_list_cnt) { + out = &output; + if (output_bit_depth != 0) + aom_shift_img(output_bit_depth, &out, &output_shifted); + img_write_to_file(out, outfile, output_format); + tile_list_writes++; + } + + tile_list_cnt++; + tile_idx = 0; + // Then memset the frame. + memset(output.img_data, 0, output.sz); } + continue; + } - frame = aom_video_reader_get_frame(reader, &frame_size); + int image_idx, ref_idx, tc, tr; + sscanf(line, "%d %d %d %d", &image_idx, &ref_idx, &tc, &tr); + if (image_idx >= num_frames) { + die("Tile list image_idx out of bounds: %d >= %d.", image_idx, + num_frames); + } + if (ref_idx >= num_references) { + die("Tile list ref_idx out of bounds: %d >= %d.", ref_idx, + num_references); + } + frame = frames[image_idx]; + frame_size = frame_sizes[image_idx]; - aom_codec_control_(&codec, AV1_SET_TILE_MODE, 1); - aom_codec_control_(&codec, AV1D_EXT_TILE_DEBUG, 1); - aom_codec_control_(&codec, AV1_SET_DECODE_TILE_ROW, tr); - aom_codec_control_(&codec, AV1_SET_DECODE_TILE_COL, tc); - - av1_ref_frame_t ref; - ref.idx = 0; - ref.use_external_ref = 1; - ref.img = reference_images[ref_idx]; - if (aom_codec_control(&codec, AV1_SET_REFERENCE, &ref)) { - die_codec(&codec, "Failed to set reference frame."); - } - - aom_codec_err_t aom_status = - aom_codec_decode(&codec, frame, frame_size, NULL); - if (aom_status) die_codec(&codec, "Failed to decode tile."); - - aom_codec_iter_t iter = NULL; - aom_image_t *img = aom_codec_get_frame(&codec, &iter); - aom_img_write(img, outfile); + aom_image_t *img = NULL; + decode_tile(&codec, frame, frame_size, tr, tc, ref_idx, reference_images, + &output, &tile_idx, &output_bit_depth, &img, output_format); + if (output_format == YUV1D) { + out = img; + if (output_bit_depth != 0) + aom_shift_img(output_bit_depth, &out, &output_shifted); + aom_img_write(out, outfile); } } + if (output_format != YUV1D) { + // Write out the last tile list. + if (tile_list_writes < tile_list_cnt) { + out = &output; + if (output_bit_depth != 0) + aom_shift_img(output_bit_depth, &out, &output_shifted); + img_write_to_file(out, outfile, output_format); + } + } + + if (output_shifted) aom_img_free(output_shifted); + if (output_format != YUV1D) aom_img_free(&output); for (i = 0; i < num_references; i++) aom_img_free(&reference_images[i]); + for (int f = 0; f < num_frames; ++f) { + free(frames[f]); + } + free(frame_sizes); + free(frames); if (aom_codec_destroy(&codec)) die_codec(&codec, "Failed to destroy codec"); aom_video_reader_close(reader); fclose(outfile); diff --git a/third_party/aom/examples/lightfield_encoder.c b/third_party/aom/examples/lightfield_encoder.c index e55cd5ce3c..4dd71cac83 100644 --- a/third_party/aom/examples/lightfield_encoder.c +++ b/third_party/aom/examples/lightfield_encoder.c @@ -275,9 +275,13 @@ static void pass1(aom_image_t *raw, FILE *infile, const char *outfile_name, aom_img_fmt_t ref_fmt = AOM_IMG_FMT_I420; if (!CONFIG_LOWBITDEPTH) ref_fmt |= AOM_IMG_FMT_HIGHBITDEPTH; // Allocate memory with the border so that it can be used as a reference. + int border_in_pixels = + (codec.config.enc->rc_resize_mode || codec.config.enc->rc_superres_mode) + ? AOM_BORDER_IN_PIXELS + : AOM_ENC_NO_SCALE_BORDER; for (i = 0; i < reference_image_num; i++) { if (!aom_img_alloc_with_border(&reference_images[i], ref_fmt, cfg->g_w, - cfg->g_h, 32, 8, AOM_BORDER_IN_PIXELS)) { + cfg->g_h, 32, 8, border_in_pixels)) { die("Failed to allocate image."); } } @@ -393,6 +397,10 @@ static void pass1(aom_image_t *raw, FILE *infile, const char *outfile_name, for (i = 0; i < reference_image_num; i++) aom_img_free(&reference_images[i]); if (aom_codec_destroy(&codec)) die_codec(&codec, "Failed to destroy codec."); + + // Modify large_scale_file fourcc. + if (cfg->large_scale_tile == 1) + aom_video_writer_set_fourcc(writer, LST_FOURCC); aom_video_writer_close(writer); printf("\nSecond pass complete. Processed %d frames.\n", frame_count); diff --git a/third_party/aom/examples/lightfield_tile_list_decoder.c b/third_party/aom/examples/lightfield_tile_list_decoder.c index 5556bf0e7d..87a8b436db 100644 --- a/third_party/aom/examples/lightfield_tile_list_decoder.c +++ b/third_party/aom/examples/lightfield_tile_list_decoder.c @@ -18,10 +18,12 @@ // compressed tile data. This input file is reconstructed from the encoded // lightfield ivf file, and is decodable by AV1 decoder. num_references is // the number of anchor frames coded at the beginning of the light field file. -// num_tile_lists is the number of tile lists need to be decoded. +// num_tile_lists is the number of tile lists need to be decoded. There is an +// optional parameter allowing to choose the output format, and the supported +// formats are YUV1D(default), YUV, and NV12. // Run lightfield tile list decoder to decode an AV1 tile list file: // examples/lightfield_tile_list_decoder vase_tile_list.ivf vase_tile_list.yuv -// 4 2 +// 4 2 0(optional) #include #include @@ -39,11 +41,61 @@ static const char *exec_name; void usage_exit(void) { fprintf(stderr, - "Usage: %s \n", + "Usage: %s " + "\n", exec_name); exit(EXIT_FAILURE); } +static void write_tile_yuv1d(aom_codec_ctx_t *codec, const aom_image_t *img, + FILE *file) { + // read out the tile size. + unsigned int tile_size = 0; + if (aom_codec_control(codec, AV1D_GET_TILE_SIZE, &tile_size)) + die_codec(codec, "Failed to get the tile size"); + const unsigned int tile_width = tile_size >> 16; + const unsigned int tile_height = tile_size & 65535; + const uint8_t output_frame_width_in_tiles = img->d_w / tile_width; + + unsigned int tile_count = 0; + if (aom_codec_control(codec, AV1D_GET_TILE_COUNT, &tile_count)) + die_codec(codec, "Failed to get the tile size"); + + // Write tile to file. + const int shift = (img->fmt & AOM_IMG_FMT_HIGHBITDEPTH) ? 1 : 0; + unsigned int tile_idx; + + for (tile_idx = 0; tile_idx < tile_count; ++tile_idx) { + const int row_offset = + (tile_idx / output_frame_width_in_tiles) * tile_height; + const int col_offset = + (tile_idx % output_frame_width_in_tiles) * tile_width; + int plane; + + for (plane = 0; plane < 3; ++plane) { + const unsigned char *buf = img->planes[plane]; + const int stride = img->stride[plane]; + const int roffset = + (plane > 0) ? row_offset >> img->y_chroma_shift : row_offset; + const int coffset = + (plane > 0) ? col_offset >> img->x_chroma_shift : col_offset; + const int w = (plane > 0) ? ((tile_width >> img->x_chroma_shift) << shift) + : (tile_width << shift); + const int h = + (plane > 0) ? (tile_height >> img->y_chroma_shift) : tile_height; + int y; + + // col offset needs to be adjusted for HBD. + buf += roffset * stride + (coffset << shift); + + for (y = 0; y < h; ++y) { + fwrite(buf, 1, w, file); + buf += stride; + } + } + } +} + int main(int argc, char **argv) { FILE *outfile = NULL; aom_codec_ctx_t codec; @@ -55,11 +107,12 @@ int main(int argc, char **argv) { aom_image_t reference_images[MAX_EXTERNAL_REFERENCES]; size_t frame_size = 0; const unsigned char *frame = NULL; + int output_format = YUV1D; int i, j, n; exec_name = argv[0]; - if (argc != 5) die("Invalid number of arguments."); + if (argc < 5) die("Invalid number of arguments."); reader = aom_video_reader_open(argv[1]); if (!reader) die("Failed to open %s for reading.", argv[1]); @@ -70,6 +123,10 @@ int main(int argc, char **argv) { num_references = (int)strtol(argv[3], NULL, 0); num_tile_lists = (int)strtol(argv[4], NULL, 0); + if (argc > 5) output_format = (int)strtol(argv[5], NULL, 0); + if (output_format < YUV1D || output_format > NV12) + die("Output format out of range [0, 2]"); + info = aom_video_reader_get_info(reader); decoder = get_aom_decoder_by_fourcc(info->codec_fourcc); @@ -103,7 +160,7 @@ int main(int argc, char **argv) { // Allocate memory to store decoded references. Allocate memory with the // border so that it can be used as a reference. for (j = 0; j < num_references; j++) { - unsigned int border = AOM_BORDER_IN_PIXELS; + unsigned int border = AOM_DEC_BORDER_IN_PIXELS; if (!aom_img_alloc_with_border(&reference_images[j], ref_fmt, frame_res[0], frame_res[1], 32, 8, border)) { @@ -147,9 +204,17 @@ int main(int argc, char **argv) { if (aom_codec_decode(&codec, frame, frame_size, NULL)) die_codec(&codec, "Failed to decode the tile list."); aom_codec_iter_t iter = NULL; - aom_image_t *img; - while ((img = aom_codec_get_frame(&codec, &iter))) - fwrite(img->img_data, 1, img->sz, outfile); + aom_image_t *img = aom_codec_get_frame(&codec, &iter); + if (!img) die_codec(&codec, "Failed to get frame."); + + if (output_format == YUV1D) + // write the tile to the output file in 1D format. + write_tile_yuv1d(&codec, img, outfile); + else if (output_format == YUV) + aom_img_write(img, outfile); + else + // NV12 output format + aom_img_write_nv12(img, outfile); } for (i = 0; i < num_references; i++) aom_img_free(&reference_images[i]); diff --git a/third_party/aom/examples/noise_model.c b/third_party/aom/examples/noise_model.c index 5cc6003b67..45f1b4dbaf 100644 --- a/third_party/aom/examples/noise_model.c +++ b/third_party/aom/examples/noise_model.c @@ -114,7 +114,7 @@ typedef struct { const char *debug_file; } noise_model_args_t; -void parse_args(noise_model_args_t *noise_args, int *argc, char **argv) { +static void parse_args(noise_model_args_t *noise_args, int *argc, char **argv) { struct arg arg; static const arg_def_t *main_args[] = { &help, &input_arg, diff --git a/third_party/aom/test/acm_random.h b/third_party/aom/test/acm_random.h index 0a8317fd57..a14b671ffe 100644 --- a/third_party/aom/test/acm_random.h +++ b/third_party/aom/test/acm_random.h @@ -26,6 +26,7 @@ class ACMRandom { void Reset(int seed) { random_.Reseed(seed); } + // Generates a random 31-bit unsigned integer from [0, 2^31). uint32_t Rand31(void) { return random_.Generate(testing::internal::Random::kMaxRange); } diff --git a/third_party/aom/test/active_map_test.cc b/third_party/aom/test/active_map_test.cc index a2b0546edb..0f8a7329e0 100644 --- a/third_party/aom/test/active_map_test.cc +++ b/third_party/aom/test/active_map_test.cc @@ -37,7 +37,7 @@ class ActiveMapTest virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video, ::libaom_test::Encoder *encoder) { - if (video->frame() == 1) { + if (video->frame() == 0) { encoder->Control(AOME_SET_CPUUSED, cpu_used_); } else if (video->frame() == 3) { aom_active_map_t map = aom_active_map_t(); diff --git a/third_party/aom/test/aq_segment_test.cc b/third_party/aom/test/aq_segment_test.cc index bbb5027d4d..51557a5cd9 100644 --- a/third_party/aom/test/aq_segment_test.cc +++ b/third_party/aom/test/aq_segment_test.cc @@ -35,7 +35,7 @@ class AqSegmentTest virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video, ::libaom_test::Encoder *encoder) { - if (video->frame() == 1) { + if (video->frame() == 0) { encoder->Control(AOME_SET_CPUUSED, set_cpu_used_); encoder->Control(AV1E_SET_AQ_MODE, aq_mode_); encoder->Control(AV1E_SET_DELTAQ_MODE, deltaq_mode_); diff --git a/third_party/aom/test/arf_freq_test.cc b/third_party/aom/test/arf_freq_test.cc index 083f4022f4..50b478ba65 100644 --- a/third_party/aom/test/arf_freq_test.cc +++ b/third_party/aom/test/arf_freq_test.cc @@ -9,6 +9,8 @@ * PATENTS file, you can obtain it at www.aomedia.org/license/patent. */ +#include + #include "third_party/googletest/src/googletest/include/gtest/gtest.h" #include "test/codec_factory.h" @@ -185,7 +187,7 @@ TEST_P(ArfFreqTestLarge, MinArfFreqTest) { init_flags_ = AOM_CODEC_USE_PSNR; if (cfg_.g_bit_depth > 8) init_flags_ |= AOM_CODEC_USE_HIGHBITDEPTH; - testing::internal::scoped_ptr video; + std::unique_ptr video; if (is_extension_y4m(test_video_param_.filename)) { video.reset(new libaom_test::Y4mVideoSource(test_video_param_.filename, 0, kFrames)); diff --git a/third_party/aom/test/av1_convolve_2d_test.cc b/third_party/aom/test/av1_convolve_2d_test.cc index 03286260e8..b0cef81532 100644 --- a/third_party/aom/test/av1_convolve_2d_test.cc +++ b/third_party/aom/test/av1_convolve_2d_test.cc @@ -12,13 +12,14 @@ #include "third_party/googletest/src/googletest/include/gtest/gtest.h" #include "test/av1_convolve_2d_test_util.h" -using ::testing::make_tuple; -using ::testing::tuple; using libaom_test::ACMRandom; using libaom_test::AV1Convolve2D::AV1Convolve2DSrTest; using libaom_test::AV1Convolve2D::AV1JntConvolve2DTest; using libaom_test::AV1HighbdConvolve2D::AV1HighbdConvolve2DSrTest; using libaom_test::AV1HighbdConvolve2D::AV1HighbdJntConvolve2DTest; +using ::testing::make_tuple; +using ::testing::tuple; + namespace { TEST_P(AV1Convolve2DSrTest, DISABLED_Speed) { RunSpeedTest(GET_PARAM(0)); } @@ -89,68 +90,72 @@ INSTANTIATE_TEST_CASE_P(NEON_COPY, AV1Convolve2DSrTest, TEST_P(AV1JntConvolve2DTest, CheckOutput) { RunCheckOutput(GET_PARAM(0)); } TEST_P(AV1JntConvolve2DTest, DISABLED_Speed) { RunSpeedTest(GET_PARAM(0)); } -INSTANTIATE_TEST_CASE_P( - C_COPY, AV1JntConvolve2DTest, - libaom_test::AV1Convolve2D::BuildParams(av1_jnt_convolve_2d_copy_c, 0, 0)); +INSTANTIATE_TEST_CASE_P(C_COPY, AV1JntConvolve2DTest, + libaom_test::AV1Convolve2D::BuildParams( + av1_dist_wtd_convolve_2d_copy_c, 0, 0)); INSTANTIATE_TEST_CASE_P( C_X, AV1JntConvolve2DTest, - libaom_test::AV1Convolve2D::BuildParams(av1_jnt_convolve_x_c, 1, 0)); + libaom_test::AV1Convolve2D::BuildParams(av1_dist_wtd_convolve_x_c, 1, 0)); INSTANTIATE_TEST_CASE_P( C_Y, AV1JntConvolve2DTest, - libaom_test::AV1Convolve2D::BuildParams(av1_jnt_convolve_y_c, 0, 1)); + libaom_test::AV1Convolve2D::BuildParams(av1_dist_wtd_convolve_y_c, 0, 1)); #if HAVE_SSE2 INSTANTIATE_TEST_CASE_P(SSE2_COPY, AV1JntConvolve2DTest, libaom_test::AV1Convolve2D::BuildParams( - av1_jnt_convolve_2d_copy_sse2, 0, 0)); -#if HAVE_SSE4_1 -INSTANTIATE_TEST_CASE_P( - SSE2_X, AV1JntConvolve2DTest, - libaom_test::AV1Convolve2D::BuildParams(av1_jnt_convolve_x_sse2, 1, 0)); + av1_dist_wtd_convolve_2d_copy_sse2, 0, 0)); +INSTANTIATE_TEST_CASE_P(SSE2, AV1JntConvolve2DTest, + libaom_test::AV1Convolve2D::BuildParams( + av1_dist_wtd_convolve_2d_sse2, 1, 1)); -INSTANTIATE_TEST_CASE_P( - SSE2_Y, AV1JntConvolve2DTest, - libaom_test::AV1Convolve2D::BuildParams(av1_jnt_convolve_y_sse2, 0, 1)); +INSTANTIATE_TEST_CASE_P(SSE2_X, AV1JntConvolve2DTest, + libaom_test::AV1Convolve2D::BuildParams( + av1_dist_wtd_convolve_x_sse2, 1, 0)); -INSTANTIATE_TEST_CASE_P( - SSSE3, AV1JntConvolve2DTest, - libaom_test::AV1Convolve2D::BuildParams(av1_jnt_convolve_2d_ssse3, 1, 1)); +INSTANTIATE_TEST_CASE_P(SSE2_Y, AV1JntConvolve2DTest, + libaom_test::AV1Convolve2D::BuildParams( + av1_dist_wtd_convolve_y_sse2, 0, 1)); + +#if HAVE_SSSE3 +INSTANTIATE_TEST_CASE_P(SSSE3, AV1JntConvolve2DTest, + libaom_test::AV1Convolve2D::BuildParams( + av1_dist_wtd_convolve_2d_ssse3, 1, 1)); #if HAVE_AVX2 INSTANTIATE_TEST_CASE_P(AVX2_COPY, AV1JntConvolve2DTest, libaom_test::AV1Convolve2D::BuildParams( - av1_jnt_convolve_2d_copy_avx2, 0, 0)); -INSTANTIATE_TEST_CASE_P( - AVX2_X, AV1JntConvolve2DTest, - libaom_test::AV1Convolve2D::BuildParams(av1_jnt_convolve_x_avx2, 1, 0)); + av1_dist_wtd_convolve_2d_copy_avx2, 0, 0)); +INSTANTIATE_TEST_CASE_P(AVX2_X, AV1JntConvolve2DTest, + libaom_test::AV1Convolve2D::BuildParams( + av1_dist_wtd_convolve_x_avx2, 1, 0)); -INSTANTIATE_TEST_CASE_P( - AVX2_Y, AV1JntConvolve2DTest, - libaom_test::AV1Convolve2D::BuildParams(av1_jnt_convolve_y_avx2, 0, 1)); +INSTANTIATE_TEST_CASE_P(AVX2_Y, AV1JntConvolve2DTest, + libaom_test::AV1Convolve2D::BuildParams( + av1_dist_wtd_convolve_y_avx2, 0, 1)); -INSTANTIATE_TEST_CASE_P( - AVX2, AV1JntConvolve2DTest, - libaom_test::AV1Convolve2D::BuildParams(av1_jnt_convolve_2d_avx2, 1, 1)); +INSTANTIATE_TEST_CASE_P(AVX2, AV1JntConvolve2DTest, + libaom_test::AV1Convolve2D::BuildParams( + av1_dist_wtd_convolve_2d_avx2, 1, 1)); #endif // HAVE_AVX2 -#endif // HAVE_SSE4_1 +#endif // HAVE_SSSE3 #endif // HAVE_SSE2 #if HAVE_NEON INSTANTIATE_TEST_CASE_P(NEON_COPY, AV1JntConvolve2DTest, libaom_test::AV1Convolve2D::BuildParams( - av1_jnt_convolve_2d_copy_neon, 0, 0)); + av1_dist_wtd_convolve_2d_copy_neon, 0, 0)); -INSTANTIATE_TEST_CASE_P( - NEON, AV1JntConvolve2DTest, - libaom_test::AV1Convolve2D::BuildParams(av1_jnt_convolve_2d_neon, 1, 1)); -INSTANTIATE_TEST_CASE_P( - NEON_X, AV1JntConvolve2DTest, - libaom_test::AV1Convolve2D::BuildParams(av1_jnt_convolve_x_neon, 1, 0)); +INSTANTIATE_TEST_CASE_P(NEON, AV1JntConvolve2DTest, + libaom_test::AV1Convolve2D::BuildParams( + av1_dist_wtd_convolve_2d_neon, 1, 1)); +INSTANTIATE_TEST_CASE_P(NEON_X, AV1JntConvolve2DTest, + libaom_test::AV1Convolve2D::BuildParams( + av1_dist_wtd_convolve_x_neon, 1, 0)); -INSTANTIATE_TEST_CASE_P( - NEON_Y, AV1JntConvolve2DTest, - libaom_test::AV1Convolve2D::BuildParams(av1_jnt_convolve_y_neon, 0, 1)); +INSTANTIATE_TEST_CASE_P(NEON_Y, AV1JntConvolve2DTest, + libaom_test::AV1Convolve2D::BuildParams( + av1_dist_wtd_convolve_y_neon, 0, 1)); #endif // HAVE_NEON TEST_P(AV1HighbdConvolve2DSrTest, CheckOutput) { RunCheckOutput(GET_PARAM(1)); } @@ -209,41 +214,41 @@ TEST_P(AV1HighbdJntConvolve2DTest, DISABLED_Speed) { INSTANTIATE_TEST_CASE_P(C_X, AV1HighbdJntConvolve2DTest, libaom_test::AV1HighbdConvolve2D::BuildParams( - av1_highbd_jnt_convolve_x_c, 1, 0)); + av1_highbd_dist_wtd_convolve_x_c, 1, 0)); INSTANTIATE_TEST_CASE_P(C_Y, AV1HighbdJntConvolve2DTest, libaom_test::AV1HighbdConvolve2D::BuildParams( - av1_highbd_jnt_convolve_y_c, 0, 1)); + av1_highbd_dist_wtd_convolve_y_c, 0, 1)); INSTANTIATE_TEST_CASE_P(C_COPY, AV1HighbdJntConvolve2DTest, libaom_test::AV1HighbdConvolve2D::BuildParams( - av1_highbd_jnt_convolve_2d_copy_c, 0, 0)); + av1_highbd_dist_wtd_convolve_2d_copy_c, 0, 0)); #if HAVE_SSE4_1 INSTANTIATE_TEST_CASE_P(SSE4_1_COPY, AV1HighbdJntConvolve2DTest, libaom_test::AV1HighbdConvolve2D::BuildParams( - av1_highbd_jnt_convolve_2d_copy_sse4_1, 0, 0)); + av1_highbd_dist_wtd_convolve_2d_copy_sse4_1, 0, 0)); INSTANTIATE_TEST_CASE_P(SSE4_1, AV1HighbdJntConvolve2DTest, libaom_test::AV1HighbdConvolve2D::BuildParams( - av1_highbd_jnt_convolve_2d_sse4_1, 1, 1)); + av1_highbd_dist_wtd_convolve_2d_sse4_1, 1, 1)); INSTANTIATE_TEST_CASE_P(SSE4_1_X, AV1HighbdJntConvolve2DTest, libaom_test::AV1HighbdConvolve2D::BuildParams( - av1_highbd_jnt_convolve_x_sse4_1, 1, 0)); + av1_highbd_dist_wtd_convolve_x_sse4_1, 1, 0)); INSTANTIATE_TEST_CASE_P(SSE4_1_Y, AV1HighbdJntConvolve2DTest, libaom_test::AV1HighbdConvolve2D::BuildParams( - av1_highbd_jnt_convolve_y_sse4_1, 0, 1)); + av1_highbd_dist_wtd_convolve_y_sse4_1, 0, 1)); #if HAVE_AVX2 INSTANTIATE_TEST_CASE_P(AVX2_COPY, AV1HighbdJntConvolve2DTest, libaom_test::AV1HighbdConvolve2D::BuildParams( - av1_highbd_jnt_convolve_2d_copy_avx2, 0, 0)); + av1_highbd_dist_wtd_convolve_2d_copy_avx2, 0, 0)); INSTANTIATE_TEST_CASE_P(AVX2, AV1HighbdJntConvolve2DTest, libaom_test::AV1HighbdConvolve2D::BuildParams( - av1_highbd_jnt_convolve_2d_avx2, 1, 1)); + av1_highbd_dist_wtd_convolve_2d_avx2, 1, 1)); INSTANTIATE_TEST_CASE_P(AVX2_X, AV1HighbdJntConvolve2DTest, libaom_test::AV1HighbdConvolve2D::BuildParams( - av1_highbd_jnt_convolve_x_avx2, 1, 0)); + av1_highbd_dist_wtd_convolve_x_avx2, 1, 0)); INSTANTIATE_TEST_CASE_P(AVX2_Y, AV1HighbdJntConvolve2DTest, libaom_test::AV1HighbdConvolve2D::BuildParams( - av1_highbd_jnt_convolve_y_avx2, 0, 1)); + av1_highbd_dist_wtd_convolve_y_avx2, 0, 1)); #endif // HAVE_AVX2 #endif // HAVE_SSE4_1 } // namespace diff --git a/third_party/aom/test/av1_convolve_2d_test_util.cc b/third_party/aom/test/av1_convolve_2d_test_util.cc index 409fd23e10..9cfe3e6b33 100644 --- a/third_party/aom/test/av1_convolve_2d_test_util.cc +++ b/third_party/aom/test/av1_convolve_2d_test_util.cc @@ -200,9 +200,9 @@ void AV1JntConvolve2DTest::RunCheckOutput(convolve_2d_func test_impl) { ConvolveParams conv_params2 = get_conv_params_no_round(do_average, 0, output2, MAX_SB_SIZE, 1, 8); - // Test special case where jnt_comp_avg is not used - conv_params1.use_jnt_comp_avg = 0; - conv_params2.use_jnt_comp_avg = 0; + // Test special case where dist_wtd_comp_avg is not used + conv_params1.use_dist_wtd_comp_avg = 0; + conv_params2.use_dist_wtd_comp_avg = 0; const int subx_range = has_subx ? 16 : 1; const int suby_range = has_suby ? 16 : 1; @@ -211,9 +211,10 @@ void AV1JntConvolve2DTest::RunCheckOutput(convolve_2d_func test_impl) { // Choose random locations within the source block const int offset_r = 3 + rnd_.PseudoUniform(h - out_h - 7); const int offset_c = 3 + rnd_.PseudoUniform(w - out_w - 7); - av1_jnt_convolve_2d_c(input + offset_r * w + offset_c, w, output8_1, - MAX_SB_SIZE, out_w, out_h, filter_params_x, - filter_params_y, subx, suby, &conv_params1); + av1_dist_wtd_convolve_2d_c(input + offset_r * w + offset_c, w, + output8_1, MAX_SB_SIZE, out_w, out_h, + filter_params_x, filter_params_y, subx, + suby, &conv_params1); test_impl(input + offset_r * w + offset_c, w, output8_2, MAX_SB_SIZE, out_w, out_h, filter_params_x, filter_params_y, subx, suby, &conv_params2); @@ -222,7 +223,7 @@ void AV1JntConvolve2DTest::RunCheckOutput(convolve_2d_func test_impl) { for (int j = 0; j < out_w; ++j) { int idx = i * MAX_SB_SIZE + j; ASSERT_EQ(output1[idx], output2[idx]) - << "Mismatch at unit tests for av1_jnt_convolve_2d\n" + << "Mismatch at unit tests for av1_dist_wtd_convolve_2d\n" << out_w << "x" << out_h << " Pixel mismatch at index " << idx << " = (" << i << ", " << j << "), sub pixel offset = (" << suby << ", " << subx << ")"; @@ -247,8 +248,8 @@ void AV1JntConvolve2DTest::RunCheckOutput(convolve_2d_func test_impl) { // Test different combination of fwd and bck offset weights for (int k = 0; k < 2; ++k) { for (int l = 0; l < 4; ++l) { - conv_params1.use_jnt_comp_avg = 1; - conv_params2.use_jnt_comp_avg = 1; + conv_params1.use_dist_wtd_comp_avg = 1; + conv_params2.use_dist_wtd_comp_avg = 1; conv_params1.fwd_offset = quant_dist_lookup_table[k][l][0]; conv_params1.bck_offset = quant_dist_lookup_table[k][l][1]; conv_params2.fwd_offset = quant_dist_lookup_table[k][l][0]; @@ -259,10 +260,10 @@ void AV1JntConvolve2DTest::RunCheckOutput(convolve_2d_func test_impl) { // Choose random locations within the source block const int offset_r = 3 + rnd_.PseudoUniform(h - out_h - 7); const int offset_c = 3 + rnd_.PseudoUniform(w - out_w - 7); - av1_jnt_convolve_2d_c(input + offset_r * w + offset_c, w, - output8_1, MAX_SB_SIZE, out_w, out_h, - filter_params_x, filter_params_y, subx, - suby, &conv_params1); + av1_dist_wtd_convolve_2d_c(input + offset_r * w + offset_c, w, + output8_1, MAX_SB_SIZE, out_w, out_h, + filter_params_x, filter_params_y, + subx, suby, &conv_params1); test_impl(input + offset_r * w + offset_c, w, output8_2, MAX_SB_SIZE, out_w, out_h, filter_params_x, filter_params_y, subx, suby, &conv_params2); @@ -272,7 +273,7 @@ void AV1JntConvolve2DTest::RunCheckOutput(convolve_2d_func test_impl) { int idx = i * MAX_SB_SIZE + j; ASSERT_EQ(output1[idx], output2[idx]) << "Mismatch at unit tests for " - "av1_jnt_convolve_2d\n" + "av1_dist_wtd_convolve_2d\n" << out_w << "x" << out_h << " Pixel mismatch at index " << idx << " = (" << i << ", " << j << "), sub pixel offset = (" << suby << ", " << subx @@ -333,7 +334,7 @@ void AV1JntConvolve2DTest::RunSpeedTest(convolve_2d_func test_impl) { ConvolveParams conv_params = get_conv_params_no_round(do_average, 0, output, MAX_SB_SIZE, 1, 8); - conv_params.use_jnt_comp_avg = 0; + conv_params.use_dist_wtd_comp_avg = 0; // Choose random locations within the source block const int offset_r = 3 + rnd_.PseudoUniform(h - out_h - 7); @@ -540,8 +541,8 @@ void AV1HighbdJntConvolve2DTest::RunSpeedTest( ConvolveParams conv_params = get_conv_params_no_round(do_average, 0, output, MAX_SB_SIZE, 1, bd); - // Test special case where jnt_comp_avg is not used - conv_params.use_jnt_comp_avg = 0; + // Test special case where dist_wtd_comp_avg is not used + conv_params.use_dist_wtd_comp_avg = 0; subx = 0; suby = 0; @@ -601,9 +602,9 @@ void AV1HighbdJntConvolve2DTest::RunCheckOutput( ConvolveParams conv_params2 = get_conv_params_no_round( do_average, 0, output2, MAX_SB_SIZE, 1, bd); - // Test special case where jnt_comp_avg is not used - conv_params1.use_jnt_comp_avg = 0; - conv_params2.use_jnt_comp_avg = 0; + // Test special case where dist_wtd_comp_avg is not used + conv_params1.use_dist_wtd_comp_avg = 0; + conv_params2.use_dist_wtd_comp_avg = 0; const int subx_range = has_subx ? 16 : 1; const int suby_range = has_suby ? 16 : 1; @@ -612,10 +613,10 @@ void AV1HighbdJntConvolve2DTest::RunCheckOutput( // Choose random locations within the source block const int offset_r = 3 + rnd_.PseudoUniform(h - out_h - 7); const int offset_c = 3 + rnd_.PseudoUniform(w - out_w - 7); - av1_highbd_jnt_convolve_2d_c(input + offset_r * w + offset_c, w, - output16_1, MAX_SB_SIZE, out_w, out_h, - filter_params_x, filter_params_y, subx, - suby, &conv_params1, bd); + av1_highbd_dist_wtd_convolve_2d_c( + input + offset_r * w + offset_c, w, output16_1, MAX_SB_SIZE, + out_w, out_h, filter_params_x, filter_params_y, subx, suby, + &conv_params1, bd); test_impl(input + offset_r * w + offset_c, w, output16_2, MAX_SB_SIZE, out_w, out_h, filter_params_x, filter_params_y, subx, suby, &conv_params2, bd); @@ -648,8 +649,8 @@ void AV1HighbdJntConvolve2DTest::RunCheckOutput( // Test different combination of fwd and bck offset weights for (int k = 0; k < 2; ++k) { for (int l = 0; l < 4; ++l) { - conv_params1.use_jnt_comp_avg = 1; - conv_params2.use_jnt_comp_avg = 1; + conv_params1.use_dist_wtd_comp_avg = 1; + conv_params2.use_dist_wtd_comp_avg = 1; conv_params1.fwd_offset = quant_dist_lookup_table[k][l][0]; conv_params1.bck_offset = quant_dist_lookup_table[k][l][1]; conv_params2.fwd_offset = quant_dist_lookup_table[k][l][0]; @@ -662,7 +663,7 @@ void AV1HighbdJntConvolve2DTest::RunCheckOutput( // Choose random locations within the source block const int offset_r = 3 + rnd_.PseudoUniform(h - out_h - 7); const int offset_c = 3 + rnd_.PseudoUniform(w - out_w - 7); - av1_highbd_jnt_convolve_2d_c( + av1_highbd_dist_wtd_convolve_2d_c( input + offset_r * w + offset_c, w, output16_1, MAX_SB_SIZE, out_w, out_h, filter_params_x, filter_params_y, subx, suby, &conv_params1, bd); diff --git a/third_party/aom/test/av1_convolve_scale_test.cc b/third_party/aom/test/av1_convolve_scale_test.cc index 3b1698eeb4..a933fc9238 100644 --- a/third_party/aom/test/av1_convolve_scale_test.cc +++ b/third_party/aom/test/av1_convolve_scale_test.cc @@ -32,9 +32,9 @@ const int kHPad = 32; const int kXStepQn = 16; const int kYStepQn = 20; +using libaom_test::ACMRandom; using ::testing::make_tuple; using ::testing::tuple; -using libaom_test::ACMRandom; enum NTaps { EIGHT_TAP, TEN_TAP, TWELVE_TAP }; int NTapsToInt(NTaps ntaps) { return 8 + static_cast(ntaps) * 2; } @@ -286,13 +286,13 @@ class ConvolveScaleTestBase : public ::testing::Test { } void SetConvParamOffset(int i, int j, int is_compound, int do_average, - int use_jnt_comp_avg) { + int use_dist_wtd_comp_avg) { if (i == -1 && j == -1) { - convolve_params_.use_jnt_comp_avg = use_jnt_comp_avg; + convolve_params_.use_dist_wtd_comp_avg = use_dist_wtd_comp_avg; convolve_params_.is_compound = is_compound; convolve_params_.do_average = do_average; } else { - convolve_params_.use_jnt_comp_avg = use_jnt_comp_avg; + convolve_params_.use_dist_wtd_comp_avg = use_dist_wtd_comp_avg; convolve_params_.fwd_offset = quant_dist_lookup_table[i][j][0]; convolve_params_.bck_offset = quant_dist_lookup_table[i][j][1]; convolve_params_.is_compound = is_compound; @@ -312,12 +312,12 @@ class ConvolveScaleTestBase : public ::testing::Test { is_compound = 1; for (int do_average = 0; do_average < 2; do_average++) { - for (int use_jnt_comp_avg = 0; use_jnt_comp_avg < 2; - use_jnt_comp_avg++) { + for (int use_dist_wtd_comp_avg = 0; use_dist_wtd_comp_avg < 2; + use_dist_wtd_comp_avg++) { for (int j = 0; j < 2; ++j) { for (int k = 0; k < 4; ++k) { SetConvParamOffset(j, k, is_compound, do_average, - use_jnt_comp_avg); + use_dist_wtd_comp_avg); Prep(&rnd); RunOne(true); RunOne(false); diff --git a/third_party/aom/test/av1_encoder_parms_get_to_decoder.cc b/third_party/aom/test/av1_encoder_parms_get_to_decoder.cc index e8470e5d50..76b82f58f7 100644 --- a/third_party/aom/test/av1_encoder_parms_get_to_decoder.cc +++ b/third_party/aom/test/av1_encoder_parms_get_to_decoder.cc @@ -9,6 +9,8 @@ * PATENTS file, you can obtain it at www.aomedia.org/license/patent. */ +#include + #include "third_party/googletest/src/googletest/include/gtest/gtest.h" #include "test/codec_factory.h" @@ -95,7 +97,7 @@ class AVxEncoderParmsGetToDecoder virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video, ::libaom_test::Encoder *encoder) { - if (video->frame() == 1) { + if (video->frame() == 0) { encoder->Control(AV1E_SET_COLOR_PRIMARIES, encode_parms.color_primaries); encoder->Control(AV1E_SET_TRANSFER_CHARACTERISTICS, encode_parms.transfer_characteristics); @@ -146,7 +148,7 @@ class AVxEncoderParmsGetToDecoder TEST_P(AVxEncoderParmsGetToDecoder, BitstreamParms) { init_flags_ = AOM_CODEC_USE_PSNR; - testing::internal::scoped_ptr video( + std::unique_ptr video( new libaom_test::Y4mVideoSource(test_video_.name, 0, test_video_.frames)); ASSERT_TRUE(video.get() != NULL); diff --git a/third_party/aom/test/av1_fwd_txfm1d_test.cc b/third_party/aom/test/av1_fwd_txfm1d_test.cc index 49a6668793..863cb396ae 100644 --- a/third_party/aom/test/av1_fwd_txfm1d_test.cc +++ b/third_party/aom/test/av1_fwd_txfm1d_test.cc @@ -13,12 +13,12 @@ #include "test/av1_txfm_test.h" using libaom_test::ACMRandom; +using libaom_test::input_base; +using libaom_test::reference_hybrid_1d; using libaom_test::TYPE_ADST; using libaom_test::TYPE_DCT; using libaom_test::TYPE_IDTX; using libaom_test::TYPE_TXFM; -using libaom_test::input_base; -using libaom_test::reference_hybrid_1d; namespace { const int txfm_type_num = 3; @@ -56,7 +56,7 @@ TEST(av1_fwd_txfm1d, av1_cospi_arr_data) { for (int i = 0; i < 7; i++) { for (int j = 0; j < 64; j++) { EXPECT_EQ(av1_cospi_arr_data[i][j], - (int32_t)round(cos(M_PI * j / 128) * (1 << (cos_bit_min + i)))); + (int32_t)round(cos(PI * j / 128) * (1 << (cos_bit_min + i)))); } } } diff --git a/third_party/aom/test/av1_fwd_txfm2d_test.cc b/third_party/aom/test/av1_fwd_txfm2d_test.cc index 75f20536b2..d55085f069 100644 --- a/third_party/aom/test/av1_fwd_txfm2d_test.cc +++ b/third_party/aom/test/av1_fwd_txfm2d_test.cc @@ -23,10 +23,10 @@ #include "av1/encoder/hybrid_fwd_txfm.h" using libaom_test::ACMRandom; -using libaom_test::TYPE_TXFM; using libaom_test::bd; using libaom_test::compute_avg_abs_error; using libaom_test::input_base; +using libaom_test::TYPE_TXFM; using std::vector; @@ -507,5 +507,12 @@ INSTANTIATE_TEST_CASE_P(SSE4_1, AV1HighbdFwdTxfm2dTest, Combine(ValuesIn(Highbd_fwd_txfm_for_sse4_1), Values(av1_highbd_fwd_txfm))); #endif // HAVE_SSE4_1 +#if HAVE_AVX2 +static TX_SIZE Highbd_fwd_txfm_for_avx2[] = { TX_8X8, TX_16X16, TX_32X32, + TX_64X64, TX_8X16, TX_16X8 }; +INSTANTIATE_TEST_CASE_P(AVX2, AV1HighbdFwdTxfm2dTest, + Combine(ValuesIn(Highbd_fwd_txfm_for_avx2), + Values(av1_highbd_fwd_txfm))); +#endif // HAVE_AVX2 } // namespace diff --git a/third_party/aom/test/av1_highbd_iht_test.cc b/third_party/aom/test/av1_highbd_iht_test.cc index 2d6490c2ad..7f077b657c 100644 --- a/third_party/aom/test/av1_highbd_iht_test.cc +++ b/third_party/aom/test/av1_highbd_iht_test.cc @@ -25,8 +25,8 @@ namespace { -using ::testing::tuple; using libaom_test::ACMRandom; +using ::testing::tuple; typedef void (*HbdHtFunc)(const int16_t *input, int32_t *output, int stride, TX_TYPE tx_type, int bd); diff --git a/third_party/aom/test/av1_horz_only_frame_superres_test.cc b/third_party/aom/test/av1_horz_only_frame_superres_test.cc index fd77ef35d7..ffc9136762 100644 --- a/third_party/aom/test/av1_horz_only_frame_superres_test.cc +++ b/third_party/aom/test/av1_horz_only_frame_superres_test.cc @@ -30,9 +30,9 @@ const int kPerfIters = 1000; const int kVPad = 32; const int kHPad = 32; +using libaom_test::ACMRandom; using ::testing::make_tuple; using ::testing::tuple; -using libaom_test::ACMRandom; template class TestImage { diff --git a/third_party/aom/test/av1_inv_txfm2d_test.cc b/third_party/aom/test/av1_inv_txfm2d_test.cc index 11e231ba64..5432130d76 100644 --- a/third_party/aom/test/av1_inv_txfm2d_test.cc +++ b/third_party/aom/test/av1_inv_txfm2d_test.cc @@ -24,11 +24,11 @@ #include "test/util.h" using libaom_test::ACMRandom; -using libaom_test::InvTxfm2dFunc; -using libaom_test::LbdInvTxfm2dFunc; using libaom_test::bd; using libaom_test::compute_avg_abs_error; using libaom_test::input_base; +using libaom_test::InvTxfm2dFunc; +using libaom_test::LbdInvTxfm2dFunc; using ::testing::Combine; using ::testing::Range; diff --git a/third_party/aom/test/av1_nn_predict_test.cc b/third_party/aom/test/av1_nn_predict_test.cc new file mode 100644 index 0000000000..0574a15ad3 --- /dev/null +++ b/third_party/aom/test/av1_nn_predict_test.cc @@ -0,0 +1,215 @@ +/* + * Copyright (c) 2018, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" + +#include "aom/aom_integer.h" +#include "aom_ports/aom_timer.h" +#include "av1/encoder/ml.h" +#include "config/aom_config.h" +#include "config/aom_dsp_rtcd.h" +#include "config/av1_rtcd.h" +#include "test/util.h" +#include "test/register_state_check.h" +#include "test/acm_random.h" +#include "test/clear_system_state.h" + +namespace { +typedef void (*NnPredict_Func)(const float *const input_nodes, + const NN_CONFIG *const nn_config, + float *const output); + +typedef ::testing::tuple NnPredictTestParam; + +const float epsilon = 1e-3f; // Error threshold for functional equivalence + +class NnPredictTest : public ::testing::TestWithParam { + public: + virtual void SetUp() { + const int MAX_NODES2 = NN_MAX_NODES_PER_LAYER * NN_MAX_NODES_PER_LAYER; + // Allocate two massive buffers on the heap for edge weights and node bias + // Then set-up the double-dimension arrays pointing into the big buffers + weights_buf = (float *)aom_malloc(MAX_NODES2 * (NN_MAX_HIDDEN_LAYERS + 1) * + sizeof(*weights_buf)); + bias_buf = + (float *)aom_malloc(NN_MAX_NODES_PER_LAYER * + (NN_MAX_HIDDEN_LAYERS + 1) * sizeof(*bias_buf)); + ASSERT_NE(weights_buf, nullptr); + ASSERT_NE(bias_buf, nullptr); + for (int i = 0; i < NN_MAX_HIDDEN_LAYERS + 1; i++) { + weights[i] = &weights_buf[i * MAX_NODES2]; + bias[i] = &bias_buf[i * NN_MAX_NODES_PER_LAYER]; + } + target_func_ = GET_PARAM(0); + } + virtual void TearDown() { + aom_free(weights_buf); + aom_free(bias_buf); + } + void RunNnPredictTest(const NN_CONFIG *const shape); + void RunNnPredictSpeedTest(const NN_CONFIG *const shape, const int run_times); + void RunNnPredictTest_all(const NN_CONFIG *const shapes, + const int num_shapes); + void RunNnPredictSpeedTest_all(const NN_CONFIG *const shapes, + const int num_shapes, const int run_times); + + private: + NnPredict_Func target_func_; + libaom_test::ACMRandom rng_; + float *weights[NN_MAX_HIDDEN_LAYERS + 1] = { 0 }; + float *bias[NN_MAX_HIDDEN_LAYERS + 1] = { 0 }; + float *weights_buf = nullptr, *bias_buf = nullptr; +}; + +void NnPredictTest::RunNnPredictTest(const NN_CONFIG *const shape) { + libaom_test::ClearSystemState(); + float inputs[NN_MAX_NODES_PER_LAYER] = { 0 }; + float outputs_test[NN_MAX_NODES_PER_LAYER] = { 0 }; + float outputs_ref[NN_MAX_NODES_PER_LAYER] = { 0 }; + + NN_CONFIG nn_config; + memcpy(&nn_config, shape, sizeof(nn_config)); + + char shape_str[32] = { 0 }; + snprintf(shape_str, sizeof(shape_str), "%d", shape->num_inputs); + for (int layer = 0; layer < shape->num_hidden_layers; layer++) + snprintf(&shape_str[strlen(shape_str)], + sizeof(shape_str) - strlen(shape_str), "x%d", + shape->num_hidden_nodes[layer]); + snprintf(&shape_str[strlen(shape_str)], sizeof(shape_str) - strlen(shape_str), + "x%d", shape->num_outputs); + + for (int i = 0; i < NN_MAX_HIDDEN_LAYERS + 1; i++) { + nn_config.weights[i] = weights[i]; + nn_config.bias[i] = bias[i]; + } + + for (int iter = 0; iter < 10000 && !HasFatalFailure(); ++iter) { + for (int node = 0; node < shape->num_inputs; node++) { + inputs[node] = ((float)rng_.Rand31() - (1 << 30)) / (1u << 31); + } + for (int layer = 0; layer < shape->num_hidden_layers; layer++) { + for (int node = 0; node < NN_MAX_NODES_PER_LAYER; node++) { + bias[layer][node] = ((float)rng_.Rand31() - (1 << 30)) / (1u << 31); + } + for (int node = 0; node < NN_MAX_NODES_PER_LAYER * NN_MAX_NODES_PER_LAYER; + node++) { + weights[layer][node] = ((float)rng_.Rand31() - (1 << 30)) / (1u << 31); + } + } + // Now the outputs: + int layer = shape->num_hidden_layers; + for (int node = 0; node < NN_MAX_NODES_PER_LAYER; node++) { + bias[layer][node] = ((float)rng_.Rand31() - (1 << 30)) / (1u << 31); + } + for (int node = 0; node < NN_MAX_NODES_PER_LAYER * NN_MAX_NODES_PER_LAYER; + node++) { + weights[layer][node] = ((float)rng_.Rand31() - (1 << 30)) / (1u << 31); + } + + av1_nn_predict_c(inputs, &nn_config, outputs_ref); + target_func_(inputs, &nn_config, outputs_test); + libaom_test::ClearSystemState(); + + for (int node = 0; node < shape->num_outputs; node++) { + if (outputs_ref[node] < epsilon) { + ASSERT_LE(outputs_test[node], epsilon) + << "Reference output was near-zero, test output was not (" + << shape_str << ")"; + } else { + const float error = outputs_ref[node] - outputs_test[node]; + const float relative_error = fabsf(error / outputs_ref[node]); + ASSERT_LE(relative_error, epsilon) + << "Excessive relative error between reference and test (" + << shape_str << ")"; + } + } + } +} + +void NnPredictTest::RunNnPredictSpeedTest(const NN_CONFIG *const shape, + const int run_times) { + libaom_test::ClearSystemState(); + float inputs[NN_MAX_NODES_PER_LAYER] = { 0 }; + float outputs_test[NN_MAX_NODES_PER_LAYER] = { 0 }; + float outputs_ref[NN_MAX_NODES_PER_LAYER] = { 0 }; + + NN_CONFIG nn_config; + memcpy(&nn_config, shape, sizeof(nn_config)); + + for (int i = 0; i < NN_MAX_HIDDEN_LAYERS; i++) { + nn_config.weights[i] = weights[i]; + nn_config.bias[i] = bias[i]; + } + // Don't bother actually changing the values for inputs/weights/bias: it + // shouldn't make any difference for a speed test. + + aom_usec_timer timer; + aom_usec_timer_start(&timer); + for (int i = 0; i < run_times; ++i) { + av1_nn_predict_c(inputs, &nn_config, outputs_ref); + } + aom_usec_timer_mark(&timer); + const double time1 = static_cast(aom_usec_timer_elapsed(&timer)); + aom_usec_timer_start(&timer); + for (int i = 0; i < run_times; ++i) { + target_func_(inputs, &nn_config, outputs_test); + } + aom_usec_timer_mark(&timer); + libaom_test::ClearSystemState(); + const double time2 = static_cast(aom_usec_timer_elapsed(&timer)); + + printf("%d", shape->num_inputs); + for (int layer = 0; layer < shape->num_hidden_layers; layer++) + printf("x%d", shape->num_hidden_nodes[layer]); + printf("x%d: ", shape->num_outputs); + printf("%7.2f/%7.2fns (%3.2f)\n", time1, time2, time1 / time2); +} + +// This is all the neural network shapes observed executed in a few different +// runs of the encoder. It also conveniently covers all the kernels +// implemented. +static const NN_CONFIG shapes[] = { + { 10, 16, 1, { 64 }, { 0 }, { 0 } }, { 12, 1, 1, { 12 }, { 0 }, { 0 } }, + { 12, 1, 1, { 24 }, { 0 }, { 0 } }, { 12, 1, 1, { 32 }, { 0 }, { 0 } }, + { 18, 4, 1, { 24 }, { 0 }, { 0 } }, { 18, 4, 1, { 32 }, { 0 }, { 0 } }, + { 4, 1, 1, { 16 }, { 0 }, { 0 } }, { 8, 1, 1, { 16 }, { 0 }, { 0 } }, + { 8, 4, 1, { 16 }, { 0 }, { 0 } }, { 8, 1, 1, { 24 }, { 0 }, { 0 } }, + { 8, 1, 1, { 32 }, { 0 }, { 0 } }, { 8, 1, 1, { 64 }, { 0 }, { 0 } }, + { 9, 3, 1, { 32 }, { 0 }, { 0 } }, { 4, 4, 1, { 8 }, { 0 }, { 0 } }, +}; + +void NnPredictTest::RunNnPredictTest_all(const NN_CONFIG *const shapes, + const int num_shapes) { + for (int i = 0; i < num_shapes; i++) RunNnPredictTest(&shapes[i]); +} + +void NnPredictTest::RunNnPredictSpeedTest_all(const NN_CONFIG *const shapes, + const int num_shapes, + const int run_times) { + for (int i = 0; i < num_shapes; i++) + NnPredictTest::RunNnPredictSpeedTest(&shapes[i], run_times); +} + +TEST_P(NnPredictTest, RandomValues) { + RunNnPredictTest_all(shapes, sizeof(shapes) / sizeof(*shapes)); +} + +TEST_P(NnPredictTest, DISABLED_Speed) { + RunNnPredictSpeedTest_all(shapes, sizeof(shapes) / sizeof(*shapes), 10000000); +} + +#if HAVE_SSE3 +INSTANTIATE_TEST_CASE_P(SSE3, NnPredictTest, + ::testing::Values(av1_nn_predict_sse3)); +#endif + +} // namespace diff --git a/third_party/aom/test/av1_round_shift_array_test.cc b/third_party/aom/test/av1_round_shift_array_test.cc index 181a394604..61dbed51f4 100644 --- a/third_party/aom/test/av1_round_shift_array_test.cc +++ b/third_party/aom/test/av1_round_shift_array_test.cc @@ -13,7 +13,7 @@ #include #include -#include "config/aom_dsp_rtcd.h" +#include "config/av1_rtcd.h" #include "aom_mem/aom_mem.h" #include "aom_ports/aom_timer.h" diff --git a/third_party/aom/test/av1_txfm_test.cc b/third_party/aom/test/av1_txfm_test.cc index d5b0ce3255..abbc475918 100644 --- a/third_party/aom/test/av1_txfm_test.cc +++ b/third_party/aom/test/av1_txfm_test.cc @@ -94,7 +94,7 @@ double Sqrt2 = pow(2, 0.5); double invSqrt2 = 1 / pow(2, 0.5); double dct_matrix(double n, double k, int size) { - return cos(M_PI * (2 * n + 1) * k / (2 * size)); + return cos(PI * (2 * n + 1) * k / (2 * size)); } void reference_dct_1d(const double *in, double *out, int size) { @@ -179,7 +179,7 @@ void reference_adst_1d(const double *in, double *out, int size) { for (int k = 0; k < size; ++k) { out[k] = 0; for (int n = 0; n < size; ++n) { - out[k] += in[n] * sin(M_PI * (2 * n + 1) * (2 * k + 1) / (4 * size)); + out[k] += in[n] * sin(PI * (2 * n + 1) * (2 * k + 1) / (4 * size)); } } } diff --git a/third_party/aom/test/av1_txfm_test.h b/third_party/aom/test/av1_txfm_test.h index a181647411..5a56d28f1a 100644 --- a/third_party/aom/test/av1_txfm_test.h +++ b/third_party/aom/test/av1_txfm_test.h @@ -29,14 +29,14 @@ #include "av1/common/enums.h" namespace libaom_test { -typedef enum { +enum { TYPE_DCT = 0, TYPE_ADST, TYPE_IDTX, TYPE_IDCT, TYPE_IADST, TYPE_LAST -} TYPE_TXFM; +} UENUM1BYTE(TYPE_TXFM); int get_txfm1d_size(TX_SIZE tx_size); diff --git a/third_party/aom/test/blend_a64_mask_test.cc b/third_party/aom/test/blend_a64_mask_test.cc index 66ca6fc5f3..75925337f5 100644 --- a/third_party/aom/test/blend_a64_mask_test.cc +++ b/third_party/aom/test/blend_a64_mask_test.cc @@ -86,6 +86,7 @@ class BlendA64MaskTest : public FunctionEquivalenceTest { w_ = block_size_wide[block_size]; h_ = block_size_high[block_size]; run_times = run_times > 1 ? run_times / w_ : 1; + ASSERT_GT(run_times, 0); subx_ = subx; suby_ = suby; @@ -248,13 +249,13 @@ TEST_P(BlendA64MaskTest8B, DISABLED_Speed) { INSTANTIATE_TEST_CASE_P(SSE4_1, BlendA64MaskTest8B, ::testing::Values(TestFuncs( aom_blend_a64_mask_c, aom_blend_a64_mask_sse4_1))); -#endif // HAVE_AVX2 +#endif // HAVE_SSE4_1 #if HAVE_AVX2 INSTANTIATE_TEST_CASE_P(AVX2, BlendA64MaskTest8B, ::testing::Values(TestFuncs(aom_blend_a64_mask_sse4_1, aom_blend_a64_mask_avx2))); -#endif // HAVE_SSE4_1 +#endif // HAVE_AVX2 ////////////////////////////////////////////////////////////////////////////// // 8 bit _d16 version @@ -482,6 +483,7 @@ class BlendA64MaskTestHBD_d16 static const int kSrcMaxBitsMaskHBD = (1 << 16) - 1; void Execute(const uint16_t *p_src0, const uint16_t *p_src1, int run_times) { + ASSERT_GT(run_times, 0) << "Cannot run 0 iterations of the test."; ConvolveParams conv_params; conv_params.round_0 = (bit_depth_ == 12) ? ROUND0_BITS + 2 : ROUND0_BITS; conv_params.round_1 = COMPOUND_ROUND1_BITS; @@ -566,11 +568,45 @@ TEST_P(BlendA64MaskTestHBD_d16, DISABLED_SaturatedValues) { } } } +TEST_P(BlendA64MaskTestHBD_d16, DISABLED_Speed) { + const int kRunTimes = 10000000; + for (int bsize = 0; bsize < BLOCK_SIZES_ALL; ++bsize) { + for (bit_depth_ = 8; bit_depth_ <= 12; bit_depth_ += 2) { + for (int i = 0; i < kBufSize; ++i) { + dst_ref_[i] = rng_.Rand12() % (1 << bit_depth_); + dst_tst_[i] = rng_.Rand12() % (1 << bit_depth_); + + src0_[i] = rng_.Rand16(); + src1_[i] = rng_.Rand16(); + } + + for (int i = 0; i < kMaxMaskSize; ++i) + mask_[i] = rng_(AOM_BLEND_A64_MAX_ALPHA + 1); + + RunOneTest(bsize, 1, 1, kRunTimes); + RunOneTest(bsize, 0, 0, kRunTimes); + } + } +} INSTANTIATE_TEST_CASE_P( C, BlendA64MaskTestHBD_d16, ::testing::Values(TestFuncsHBD_d16(aom_highbd_blend_a64_d16_mask_c, NULL))); +#if HAVE_SSE4_1 +INSTANTIATE_TEST_CASE_P( + SSE4_1, BlendA64MaskTestHBD_d16, + ::testing::Values(TestFuncsHBD_d16(aom_highbd_blend_a64_d16_mask_c, + aom_highbd_blend_a64_d16_mask_sse4_1))); +#endif // HAVE_SSE4_1 + +#if HAVE_AVX2 +INSTANTIATE_TEST_CASE_P( + AVX2, BlendA64MaskTestHBD_d16, + ::testing::Values(TestFuncsHBD_d16(aom_highbd_blend_a64_d16_mask_c, + aom_highbd_blend_a64_d16_mask_avx2))); +#endif // HAVE_AVX2 + // TODO(slavarnway): Enable the following in the avx2 commit. (56501) #if 0 #if HAVE_AVX2 diff --git a/third_party/aom/test/borders_test.cc b/third_party/aom/test/borders_test.cc index 893237ef30..31eacab12e 100644 --- a/third_party/aom/test/borders_test.cc +++ b/third_party/aom/test/borders_test.cc @@ -33,7 +33,7 @@ class BordersTestLarge virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video, ::libaom_test::Encoder *encoder) { - if (video->frame() == 1) { + if (video->frame() == 0) { encoder->Control(AOME_SET_CPUUSED, 1); encoder->Control(AOME_SET_ENABLEAUTOALTREF, 1); encoder->Control(AOME_SET_ARNR_MAXFRAMES, 7); diff --git a/third_party/aom/test/cdef_test.cc b/third_party/aom/test/cdef_test.cc index becc072918..8201818795 100644 --- a/third_party/aom/test/cdef_test.cc +++ b/third_party/aom/test/cdef_test.cc @@ -116,7 +116,7 @@ void test_cdef(int bsize, int iterations, cdef_filter_block_func cdef, ref_cdef(depth == 8 ? (uint8_t *)ref_d : 0, ref_d, size, s + CDEF_HBORDER + CDEF_VBORDER * CDEF_BSTRIDE, pristrength, secstrength, dir, pridamping, - secdamping, bsize, (1 << depth) - 1, depth - 8); + secdamping, bsize, depth - 8); // If cdef and ref_cdef are the same, we're just testing // speed if (cdef != ref_cdef) @@ -124,7 +124,7 @@ void test_cdef(int bsize, int iterations, cdef_filter_block_func cdef, cdef(depth == 8 ? (uint8_t *)d : 0, d, size, s + CDEF_HBORDER + CDEF_VBORDER * CDEF_BSTRIDE, pristrength, secstrength, dir, pridamping, - secdamping, bsize, (1 << depth) - 1, depth - 8)); + secdamping, bsize, depth - 8)); if (ref_cdef != cdef) { for (pos = 0; pos < max_pos && !error; pos++) { error = ref_d[pos] != d[pos]; diff --git a/third_party/aom/test/cfl_test.cc b/third_party/aom/test/cfl_test.cc index e4d438d6ab..f087dd9fbd 100644 --- a/third_party/aom/test/cfl_test.cc +++ b/third_party/aom/test/cfl_test.cc @@ -84,7 +84,7 @@ class CFLTest { tx_size = tx; width = tx_size_wide[tx_size]; height = tx_size_high[tx_size]; - rnd(ACMRandom::DeterministicSeed()); + rnd.Reset(ACMRandom::DeterministicSeed()); } protected: diff --git a/third_party/aom/test/comp_avg_pred_test.cc b/third_party/aom/test/comp_avg_pred_test.cc index 9ad8973f08..3e5632e7eb 100644 --- a/third_party/aom/test/comp_avg_pred_test.cc +++ b/third_party/aom/test/comp_avg_pred_test.cc @@ -11,62 +11,66 @@ #include "test/comp_avg_pred_test.h" +using libaom_test::ACMRandom; +using libaom_test::AV1DISTWTDCOMPAVG::AV1DISTWTDCOMPAVGTest; +using libaom_test::AV1DISTWTDCOMPAVG::AV1DISTWTDCOMPAVGUPSAMPLEDTest; +using libaom_test::AV1DISTWTDCOMPAVG::AV1HighBDDISTWTDCOMPAVGTest; +using libaom_test::AV1DISTWTDCOMPAVG::AV1HighBDDISTWTDCOMPAVGUPSAMPLEDTest; using ::testing::make_tuple; using ::testing::tuple; -using libaom_test::ACMRandom; -using libaom_test::AV1JNTCOMPAVG::AV1HighBDJNTCOMPAVGTest; -using libaom_test::AV1JNTCOMPAVG::AV1HighBDJNTCOMPAVGUPSAMPLEDTest; -using libaom_test::AV1JNTCOMPAVG::AV1JNTCOMPAVGTest; -using libaom_test::AV1JNTCOMPAVG::AV1JNTCOMPAVGUPSAMPLEDTest; namespace { -TEST_P(AV1JNTCOMPAVGTest, DISABLED_Speed) { RunSpeedTest(GET_PARAM(0)); } +TEST_P(AV1DISTWTDCOMPAVGTest, DISABLED_Speed) { RunSpeedTest(GET_PARAM(0)); } -TEST_P(AV1JNTCOMPAVGTest, CheckOutput) { RunCheckOutput(GET_PARAM(0)); } +TEST_P(AV1DISTWTDCOMPAVGTest, CheckOutput) { RunCheckOutput(GET_PARAM(0)); } #if HAVE_SSSE3 -INSTANTIATE_TEST_CASE_P( - SSSE3, AV1JNTCOMPAVGTest, - libaom_test::AV1JNTCOMPAVG::BuildParams(aom_jnt_comp_avg_pred_ssse3)); +INSTANTIATE_TEST_CASE_P(SSSE3, AV1DISTWTDCOMPAVGTest, + libaom_test::AV1DISTWTDCOMPAVG::BuildParams( + aom_dist_wtd_comp_avg_pred_ssse3)); #endif -TEST_P(AV1JNTCOMPAVGUPSAMPLEDTest, DISABLED_Speed) { +TEST_P(AV1DISTWTDCOMPAVGUPSAMPLEDTest, DISABLED_Speed) { RunSpeedTest(GET_PARAM(0)); } -TEST_P(AV1JNTCOMPAVGUPSAMPLEDTest, CheckOutput) { +TEST_P(AV1DISTWTDCOMPAVGUPSAMPLEDTest, CheckOutput) { RunCheckOutput(GET_PARAM(0)); } #if HAVE_SSSE3 -INSTANTIATE_TEST_CASE_P(SSSE3, AV1JNTCOMPAVGUPSAMPLEDTest, - libaom_test::AV1JNTCOMPAVG::BuildParams( - aom_jnt_comp_avg_upsampled_pred_ssse3)); +INSTANTIATE_TEST_CASE_P(SSSE3, AV1DISTWTDCOMPAVGUPSAMPLEDTest, + libaom_test::AV1DISTWTDCOMPAVG::BuildParams( + aom_dist_wtd_comp_avg_upsampled_pred_ssse3)); #endif -TEST_P(AV1HighBDJNTCOMPAVGTest, DISABLED_Speed) { RunSpeedTest(GET_PARAM(1)); } - -TEST_P(AV1HighBDJNTCOMPAVGTest, CheckOutput) { RunCheckOutput(GET_PARAM(1)); } - -#if HAVE_SSE2 -INSTANTIATE_TEST_CASE_P(SSE2, AV1HighBDJNTCOMPAVGTest, - libaom_test::AV1JNTCOMPAVG::BuildParams( - aom_highbd_jnt_comp_avg_pred_sse2, 1)); -#endif - -TEST_P(AV1HighBDJNTCOMPAVGUPSAMPLEDTest, DISABLED_Speed) { +TEST_P(AV1HighBDDISTWTDCOMPAVGTest, DISABLED_Speed) { RunSpeedTest(GET_PARAM(1)); } -TEST_P(AV1HighBDJNTCOMPAVGUPSAMPLEDTest, CheckOutput) { +TEST_P(AV1HighBDDISTWTDCOMPAVGTest, CheckOutput) { RunCheckOutput(GET_PARAM(1)); } #if HAVE_SSE2 -INSTANTIATE_TEST_CASE_P(SSE2, AV1HighBDJNTCOMPAVGUPSAMPLEDTest, - libaom_test::AV1JNTCOMPAVG::BuildParams( - aom_highbd_jnt_comp_avg_upsampled_pred_sse2)); +INSTANTIATE_TEST_CASE_P(SSE2, AV1HighBDDISTWTDCOMPAVGTest, + libaom_test::AV1DISTWTDCOMPAVG::BuildParams( + aom_highbd_dist_wtd_comp_avg_pred_sse2, 1)); +#endif + +TEST_P(AV1HighBDDISTWTDCOMPAVGUPSAMPLEDTest, DISABLED_Speed) { + RunSpeedTest(GET_PARAM(1)); +} + +TEST_P(AV1HighBDDISTWTDCOMPAVGUPSAMPLEDTest, CheckOutput) { + RunCheckOutput(GET_PARAM(1)); +} + +#if HAVE_SSE2 +INSTANTIATE_TEST_CASE_P(SSE2, AV1HighBDDISTWTDCOMPAVGUPSAMPLEDTest, + libaom_test::AV1DISTWTDCOMPAVG::BuildParams( + aom_highbd_dist_wtd_comp_avg_upsampled_pred_sse2)); #endif } // namespace diff --git a/third_party/aom/test/comp_avg_pred_test.h b/third_party/aom/test/comp_avg_pred_test.h index 9661dd9f54..01ea35d59b 100644 --- a/third_party/aom/test/comp_avg_pred_test.h +++ b/third_party/aom/test/comp_avg_pred_test.h @@ -25,72 +25,73 @@ namespace libaom_test { const int kMaxSize = 128 + 32; // padding -namespace AV1JNTCOMPAVG { +namespace AV1DISTWTDCOMPAVG { -typedef void (*jntcompavg_func)(uint8_t *comp_pred, const uint8_t *pred, - int width, int height, const uint8_t *ref, - int ref_stride, - const JNT_COMP_PARAMS *jcp_param); +typedef void (*distwtdcompavg_func)(uint8_t *comp_pred, const uint8_t *pred, + int width, int height, const uint8_t *ref, + int ref_stride, + const DIST_WTD_COMP_PARAMS *jcp_param); -typedef void (*jntcompavgupsampled_func)( +typedef void (*distwtdcompavgupsampled_func)( MACROBLOCKD *xd, const struct AV1Common *const cm, int mi_row, int mi_col, const MV *const mv, uint8_t *comp_pred, const uint8_t *pred, int width, int height, int subpel_x_q3, int subpel_y_q3, const uint8_t *ref, - int ref_stride, const JNT_COMP_PARAMS *jcp_param, int subpel_search); + int ref_stride, const DIST_WTD_COMP_PARAMS *jcp_param, int subpel_search); -typedef void (*highbdjntcompavgupsampled_func)( +typedef void (*highbddistwtdcompavgupsampled_func)( MACROBLOCKD *xd, const struct AV1Common *const cm, int mi_row, int mi_col, const MV *const mv, uint8_t *comp_pred8, const uint8_t *pred8, int width, int height, int subpel_x_q3, int subpel_y_q3, const uint8_t *ref8, - int ref_stride, int bd, const JNT_COMP_PARAMS *jcp_param, + int ref_stride, int bd, const DIST_WTD_COMP_PARAMS *jcp_param, int subpel_search); -typedef ::testing::tuple JNTCOMPAVGParam; +typedef ::testing::tuple DISTWTDCOMPAVGParam; -typedef ::testing::tuple - JNTCOMPAVGUPSAMPLEDParam; +typedef ::testing::tuple + DISTWTDCOMPAVGUPSAMPLEDParam; -typedef ::testing::tuple - HighbdJNTCOMPAVGParam; +typedef ::testing::tuple + HighbdDISTWTDCOMPAVGParam; -typedef ::testing::tuple - HighbdJNTCOMPAVGUPSAMPLEDParam; +typedef ::testing::tuple + HighbdDISTWTDCOMPAVGUPSAMPLEDParam; -::testing::internal::ParamGenerator BuildParams( - jntcompavg_func filter) { +::testing::internal::ParamGenerator BuildParams( + distwtdcompavg_func filter) { return ::testing::Combine(::testing::Values(filter), ::testing::Range(BLOCK_4X4, BLOCK_SIZES_ALL)); } -::testing::internal::ParamGenerator BuildParams( - jntcompavgupsampled_func filter) { +::testing::internal::ParamGenerator BuildParams( + distwtdcompavgupsampled_func filter) { return ::testing::Combine(::testing::Values(filter), ::testing::Range(BLOCK_4X4, BLOCK_SIZES_ALL)); } -::testing::internal::ParamGenerator BuildParams( - jntcompavg_func filter, int is_hbd) { +::testing::internal::ParamGenerator BuildParams( + distwtdcompavg_func filter, int is_hbd) { (void)is_hbd; return ::testing::Combine(::testing::Range(8, 13, 2), ::testing::Values(filter), ::testing::Range(BLOCK_4X4, BLOCK_SIZES_ALL)); } -::testing::internal::ParamGenerator BuildParams( - highbdjntcompavgupsampled_func filter) { +::testing::internal::ParamGenerator +BuildParams(highbddistwtdcompavgupsampled_func filter) { return ::testing::Combine(::testing::Range(8, 13, 2), ::testing::Values(filter), ::testing::Range(BLOCK_4X4, BLOCK_SIZES_ALL)); } -class AV1JNTCOMPAVGTest : public ::testing::TestWithParam { +class AV1DISTWTDCOMPAVGTest + : public ::testing::TestWithParam { public: - ~AV1JNTCOMPAVGTest() {} + ~AV1DISTWTDCOMPAVGTest() {} void SetUp() { rnd_.Reset(ACMRandom::DeterministicSeed()); } void TearDown() { libaom_test::ClearSystemState(); } protected: - void RunCheckOutput(jntcompavg_func test_impl) { + void RunCheckOutput(distwtdcompavg_func test_impl) { const int w = kMaxSize, h = kMaxSize; const int block_idx = GET_PARAM(1); @@ -107,27 +108,27 @@ class AV1JNTCOMPAVGTest : public ::testing::TestWithParam { const int in_w = block_size_wide[block_idx]; const int in_h = block_size_high[block_idx]; - JNT_COMP_PARAMS jnt_comp_params; - jnt_comp_params.use_jnt_comp_avg = 1; + DIST_WTD_COMP_PARAMS dist_wtd_comp_params; + dist_wtd_comp_params.use_dist_wtd_comp_avg = 1; for (int ii = 0; ii < 2; ii++) { for (int jj = 0; jj < 4; jj++) { - jnt_comp_params.fwd_offset = quant_dist_lookup_table[ii][jj][0]; - jnt_comp_params.bck_offset = quant_dist_lookup_table[ii][jj][1]; + dist_wtd_comp_params.fwd_offset = quant_dist_lookup_table[ii][jj][0]; + dist_wtd_comp_params.bck_offset = quant_dist_lookup_table[ii][jj][1]; const int offset_r = 3 + rnd_.PseudoUniform(h - in_h - 7); const int offset_c = 3 + rnd_.PseudoUniform(w - in_w - 7); - aom_jnt_comp_avg_pred_c(output, pred8 + offset_r * w + offset_c, in_w, - in_h, ref8 + offset_r * w + offset_c, in_w, - &jnt_comp_params); + aom_dist_wtd_comp_avg_pred_c(output, pred8 + offset_r * w + offset_c, + in_w, in_h, ref8 + offset_r * w + offset_c, + in_w, &dist_wtd_comp_params); test_impl(output2, pred8 + offset_r * w + offset_c, in_w, in_h, - ref8 + offset_r * w + offset_c, in_w, &jnt_comp_params); + ref8 + offset_r * w + offset_c, in_w, &dist_wtd_comp_params); for (int i = 0; i < in_h; ++i) { for (int j = 0; j < in_w; ++j) { int idx = i * in_w + j; ASSERT_EQ(output[idx], output2[idx]) - << "Mismatch at unit tests for AV1JNTCOMPAVGTest\n" + << "Mismatch at unit tests for AV1DISTWTDCOMPAVGTest\n" << in_w << "x" << in_h << " Pixel mismatch at index " << idx << " = (" << i << ", " << j << ")"; } @@ -135,7 +136,7 @@ class AV1JNTCOMPAVGTest : public ::testing::TestWithParam { } } } - void RunSpeedTest(jntcompavg_func test_impl) { + void RunSpeedTest(distwtdcompavg_func test_impl) { const int w = kMaxSize, h = kMaxSize; const int block_idx = GET_PARAM(1); @@ -152,49 +153,49 @@ class AV1JNTCOMPAVGTest : public ::testing::TestWithParam { const int in_w = block_size_wide[block_idx]; const int in_h = block_size_high[block_idx]; - JNT_COMP_PARAMS jnt_comp_params; - jnt_comp_params.use_jnt_comp_avg = 1; + DIST_WTD_COMP_PARAMS dist_wtd_comp_params; + dist_wtd_comp_params.use_dist_wtd_comp_avg = 1; - jnt_comp_params.fwd_offset = quant_dist_lookup_table[0][0][0]; - jnt_comp_params.bck_offset = quant_dist_lookup_table[0][0][1]; + dist_wtd_comp_params.fwd_offset = quant_dist_lookup_table[0][0][0]; + dist_wtd_comp_params.bck_offset = quant_dist_lookup_table[0][0][1]; const int num_loops = 1000000000 / (in_w + in_h); aom_usec_timer timer; aom_usec_timer_start(&timer); for (int i = 0; i < num_loops; ++i) - aom_jnt_comp_avg_pred_c(output, pred8, in_w, in_h, ref8, in_w, - &jnt_comp_params); + aom_dist_wtd_comp_avg_pred_c(output, pred8, in_w, in_h, ref8, in_w, + &dist_wtd_comp_params); aom_usec_timer_mark(&timer); const int elapsed_time = static_cast(aom_usec_timer_elapsed(&timer)); - printf("jntcompavg c_code %3dx%-3d: %7.2f us\n", in_w, in_h, + printf("distwtdcompavg c_code %3dx%-3d: %7.2f us\n", in_w, in_h, 1000.0 * elapsed_time / num_loops); aom_usec_timer timer1; aom_usec_timer_start(&timer1); for (int i = 0; i < num_loops; ++i) - test_impl(output2, pred8, in_w, in_h, ref8, in_w, &jnt_comp_params); + test_impl(output2, pred8, in_w, in_h, ref8, in_w, &dist_wtd_comp_params); aom_usec_timer_mark(&timer1); const int elapsed_time1 = static_cast(aom_usec_timer_elapsed(&timer1)); - printf("jntcompavg test_code %3dx%-3d: %7.2f us\n", in_w, in_h, + printf("distwtdcompavg test_code %3dx%-3d: %7.2f us\n", in_w, in_h, 1000.0 * elapsed_time1 / num_loops); } libaom_test::ACMRandom rnd_; -}; // class AV1JNTCOMPAVGTest +}; // class AV1DISTWTDCOMPAVGTest -class AV1JNTCOMPAVGUPSAMPLEDTest - : public ::testing::TestWithParam { +class AV1DISTWTDCOMPAVGUPSAMPLEDTest + : public ::testing::TestWithParam { public: - ~AV1JNTCOMPAVGUPSAMPLEDTest() {} + ~AV1DISTWTDCOMPAVGUPSAMPLEDTest() {} void SetUp() { rnd_.Reset(ACMRandom::DeterministicSeed()); } void TearDown() { libaom_test::ClearSystemState(); } protected: - void RunCheckOutput(jntcompavgupsampled_func test_impl) { + void RunCheckOutput(distwtdcompavgupsampled_func test_impl) { const int w = kMaxSize, h = kMaxSize; const int block_idx = GET_PARAM(1); @@ -211,37 +212,40 @@ class AV1JNTCOMPAVGUPSAMPLEDTest const int in_w = block_size_wide[block_idx]; const int in_h = block_size_high[block_idx]; - JNT_COMP_PARAMS jnt_comp_params; - jnt_comp_params.use_jnt_comp_avg = 1; + DIST_WTD_COMP_PARAMS dist_wtd_comp_params; + dist_wtd_comp_params.use_dist_wtd_comp_avg = 1; int sub_x_q3, sub_y_q3; int subpel_search; - for (subpel_search = 1; subpel_search <= 2; ++subpel_search) { + for (subpel_search = USE_4_TAPS; subpel_search <= USE_8_TAPS; + ++subpel_search) { for (sub_x_q3 = 0; sub_x_q3 < 8; ++sub_x_q3) { for (sub_y_q3 = 0; sub_y_q3 < 8; ++sub_y_q3) { for (int ii = 0; ii < 2; ii++) { for (int jj = 0; jj < 4; jj++) { - jnt_comp_params.fwd_offset = quant_dist_lookup_table[ii][jj][0]; - jnt_comp_params.bck_offset = quant_dist_lookup_table[ii][jj][1]; + dist_wtd_comp_params.fwd_offset = + quant_dist_lookup_table[ii][jj][0]; + dist_wtd_comp_params.bck_offset = + quant_dist_lookup_table[ii][jj][1]; const int offset_r = 3 + rnd_.PseudoUniform(h - in_h - 7); const int offset_c = 3 + rnd_.PseudoUniform(w - in_w - 7); - aom_jnt_comp_avg_upsampled_pred_c( + aom_dist_wtd_comp_avg_upsampled_pred_c( NULL, NULL, 0, 0, NULL, output, pred8 + offset_r * w + offset_c, in_w, in_h, sub_x_q3, sub_y_q3, ref8 + offset_r * w + offset_c, in_w, - &jnt_comp_params, subpel_search); + &dist_wtd_comp_params, subpel_search); test_impl(NULL, NULL, 0, 0, NULL, output2, pred8 + offset_r * w + offset_c, in_w, in_h, sub_x_q3, sub_y_q3, ref8 + offset_r * w + offset_c, in_w, - &jnt_comp_params, subpel_search); + &dist_wtd_comp_params, subpel_search); for (int i = 0; i < in_h; ++i) { for (int j = 0; j < in_w; ++j) { int idx = i * in_w + j; ASSERT_EQ(output[idx], output2[idx]) << "Mismatch at unit tests for " - "AV1JNTCOMPAVGUPSAMPLEDTest\n" + "AV1DISTWTDCOMPAVGUPSAMPLEDTest\n" << in_w << "x" << in_h << " Pixel mismatch at index " << idx << " = (" << i << ", " << j << "), sub pixel offset = (" << sub_y_q3 << ", " @@ -254,7 +258,7 @@ class AV1JNTCOMPAVGUPSAMPLEDTest } } } - void RunSpeedTest(jntcompavgupsampled_func test_impl) { + void RunSpeedTest(distwtdcompavgupsampled_func test_impl) { const int w = kMaxSize, h = kMaxSize; const int block_idx = GET_PARAM(1); @@ -271,11 +275,11 @@ class AV1JNTCOMPAVGUPSAMPLEDTest const int in_w = block_size_wide[block_idx]; const int in_h = block_size_high[block_idx]; - JNT_COMP_PARAMS jnt_comp_params; - jnt_comp_params.use_jnt_comp_avg = 1; + DIST_WTD_COMP_PARAMS dist_wtd_comp_params; + dist_wtd_comp_params.use_dist_wtd_comp_avg = 1; - jnt_comp_params.fwd_offset = quant_dist_lookup_table[0][0][0]; - jnt_comp_params.bck_offset = quant_dist_lookup_table[0][0][1]; + dist_wtd_comp_params.fwd_offset = quant_dist_lookup_table[0][0][0]; + dist_wtd_comp_params.bck_offset = quant_dist_lookup_table[0][0][1]; int sub_x_q3 = 0; int sub_y_q3 = 0; @@ -283,16 +287,16 @@ class AV1JNTCOMPAVGUPSAMPLEDTest const int num_loops = 1000000000 / (in_w + in_h); aom_usec_timer timer; aom_usec_timer_start(&timer); - int subpel_search = 2; // set to 1 to test 4-tap filter. + int subpel_search = USE_8_TAPS; // set to USE_4_TAPS to test 4-tap filter. for (int i = 0; i < num_loops; ++i) - aom_jnt_comp_avg_upsampled_pred_c(NULL, NULL, 0, 0, NULL, output, pred8, - in_w, in_h, sub_x_q3, sub_y_q3, ref8, - in_w, &jnt_comp_params, subpel_search); + aom_dist_wtd_comp_avg_upsampled_pred_c( + NULL, NULL, 0, 0, NULL, output, pred8, in_w, in_h, sub_x_q3, sub_y_q3, + ref8, in_w, &dist_wtd_comp_params, subpel_search); aom_usec_timer_mark(&timer); const int elapsed_time = static_cast(aom_usec_timer_elapsed(&timer)); - printf("jntcompavgupsampled c_code %3dx%-3d: %7.2f us\n", in_w, in_h, + printf("distwtdcompavgupsampled c_code %3dx%-3d: %7.2f us\n", in_w, in_h, 1000.0 * elapsed_time / num_loops); aom_usec_timer timer1; @@ -300,27 +304,27 @@ class AV1JNTCOMPAVGUPSAMPLEDTest for (int i = 0; i < num_loops; ++i) test_impl(NULL, NULL, 0, 0, NULL, output2, pred8, in_w, in_h, sub_x_q3, - sub_y_q3, ref8, in_w, &jnt_comp_params, subpel_search); + sub_y_q3, ref8, in_w, &dist_wtd_comp_params, subpel_search); aom_usec_timer_mark(&timer1); const int elapsed_time1 = static_cast(aom_usec_timer_elapsed(&timer1)); - printf("jntcompavgupsampled test_code %3dx%-3d: %7.2f us\n", in_w, in_h, + printf("distwtdcompavgupsampled test_code %3dx%-3d: %7.2f us\n", in_w, in_h, 1000.0 * elapsed_time1 / num_loops); } libaom_test::ACMRandom rnd_; -}; // class AV1JNTCOMPAVGUPSAMPLEDTest +}; // class AV1DISTWTDCOMPAVGUPSAMPLEDTest -class AV1HighBDJNTCOMPAVGTest - : public ::testing::TestWithParam { +class AV1HighBDDISTWTDCOMPAVGTest + : public ::testing::TestWithParam { public: - ~AV1HighBDJNTCOMPAVGTest() {} + ~AV1HighBDDISTWTDCOMPAVGTest() {} void SetUp() { rnd_.Reset(ACMRandom::DeterministicSeed()); } void TearDown() { libaom_test::ClearSystemState(); } protected: - void RunCheckOutput(jntcompavg_func test_impl) { + void RunCheckOutput(distwtdcompavg_func test_impl) { const int w = kMaxSize, h = kMaxSize; const int block_idx = GET_PARAM(2); const int bd = GET_PARAM(0); @@ -337,31 +341,31 @@ class AV1HighBDJNTCOMPAVGTest const int in_w = block_size_wide[block_idx]; const int in_h = block_size_high[block_idx]; - JNT_COMP_PARAMS jnt_comp_params; - jnt_comp_params.use_jnt_comp_avg = 1; + DIST_WTD_COMP_PARAMS dist_wtd_comp_params; + dist_wtd_comp_params.use_dist_wtd_comp_avg = 1; for (int ii = 0; ii < 2; ii++) { for (int jj = 0; jj < 4; jj++) { - jnt_comp_params.fwd_offset = quant_dist_lookup_table[ii][jj][0]; - jnt_comp_params.bck_offset = quant_dist_lookup_table[ii][jj][1]; + dist_wtd_comp_params.fwd_offset = quant_dist_lookup_table[ii][jj][0]; + dist_wtd_comp_params.bck_offset = quant_dist_lookup_table[ii][jj][1]; const int offset_r = 3 + rnd_.PseudoUniform(h - in_h - 7); const int offset_c = 3 + rnd_.PseudoUniform(w - in_w - 7); - aom_highbd_jnt_comp_avg_pred_c( + aom_highbd_dist_wtd_comp_avg_pred_c( CONVERT_TO_BYTEPTR(output), CONVERT_TO_BYTEPTR(pred8) + offset_r * w + offset_c, in_w, in_h, CONVERT_TO_BYTEPTR(ref8) + offset_r * w + offset_c, in_w, - &jnt_comp_params); + &dist_wtd_comp_params); test_impl(CONVERT_TO_BYTEPTR(output2), CONVERT_TO_BYTEPTR(pred8) + offset_r * w + offset_c, in_w, in_h, CONVERT_TO_BYTEPTR(ref8) + offset_r * w + offset_c, - in_w, &jnt_comp_params); + in_w, &dist_wtd_comp_params); for (int i = 0; i < in_h; ++i) { for (int j = 0; j < in_w; ++j) { int idx = i * in_w + j; ASSERT_EQ(output[idx], output2[idx]) - << "Mismatch at unit tests for AV1HighBDJNTCOMPAVGTest\n" + << "Mismatch at unit tests for AV1HighBDDISTWTDCOMPAVGTest\n" << in_w << "x" << in_h << " Pixel mismatch at index " << idx << " = (" << i << ", " << j << ")"; } @@ -369,7 +373,7 @@ class AV1HighBDJNTCOMPAVGTest } } } - void RunSpeedTest(jntcompavg_func test_impl) { + void RunSpeedTest(distwtdcompavg_func test_impl) { const int w = kMaxSize, h = kMaxSize; const int block_idx = GET_PARAM(2); const int bd = GET_PARAM(0); @@ -386,24 +390,24 @@ class AV1HighBDJNTCOMPAVGTest const int in_w = block_size_wide[block_idx]; const int in_h = block_size_high[block_idx]; - JNT_COMP_PARAMS jnt_comp_params; - jnt_comp_params.use_jnt_comp_avg = 1; + DIST_WTD_COMP_PARAMS dist_wtd_comp_params; + dist_wtd_comp_params.use_dist_wtd_comp_avg = 1; - jnt_comp_params.fwd_offset = quant_dist_lookup_table[0][0][0]; - jnt_comp_params.bck_offset = quant_dist_lookup_table[0][0][1]; + dist_wtd_comp_params.fwd_offset = quant_dist_lookup_table[0][0][0]; + dist_wtd_comp_params.bck_offset = quant_dist_lookup_table[0][0][1]; const int num_loops = 1000000000 / (in_w + in_h); aom_usec_timer timer; aom_usec_timer_start(&timer); for (int i = 0; i < num_loops; ++i) - aom_highbd_jnt_comp_avg_pred_c( + aom_highbd_dist_wtd_comp_avg_pred_c( CONVERT_TO_BYTEPTR(output), CONVERT_TO_BYTEPTR(pred8), in_w, in_h, - CONVERT_TO_BYTEPTR(ref8), in_w, &jnt_comp_params); + CONVERT_TO_BYTEPTR(ref8), in_w, &dist_wtd_comp_params); aom_usec_timer_mark(&timer); const int elapsed_time = static_cast(aom_usec_timer_elapsed(&timer)); - printf("highbdjntcompavg c_code %3dx%-3d: %7.2f us\n", in_w, in_h, + printf("highbddistwtdcompavg c_code %3dx%-3d: %7.2f us\n", in_w, in_h, 1000.0 * elapsed_time / num_loops); aom_usec_timer timer1; @@ -411,33 +415,33 @@ class AV1HighBDJNTCOMPAVGTest for (int i = 0; i < num_loops; ++i) test_impl(CONVERT_TO_BYTEPTR(output2), CONVERT_TO_BYTEPTR(pred8), in_w, - in_h, CONVERT_TO_BYTEPTR(ref8), in_w, &jnt_comp_params); + in_h, CONVERT_TO_BYTEPTR(ref8), in_w, &dist_wtd_comp_params); aom_usec_timer_mark(&timer1); const int elapsed_time1 = static_cast(aom_usec_timer_elapsed(&timer1)); - printf("highbdjntcompavg test_code %3dx%-3d: %7.2f us\n", in_w, in_h, + printf("highbddistwtdcompavg test_code %3dx%-3d: %7.2f us\n", in_w, in_h, 1000.0 * elapsed_time1 / num_loops); } libaom_test::ACMRandom rnd_; -}; // class AV1HighBDJNTCOMPAVGTest +}; // class AV1HighBDDISTWTDCOMPAVGTest -class AV1HighBDJNTCOMPAVGUPSAMPLEDTest - : public ::testing::TestWithParam { +class AV1HighBDDISTWTDCOMPAVGUPSAMPLEDTest + : public ::testing::TestWithParam { public: - ~AV1HighBDJNTCOMPAVGUPSAMPLEDTest() {} + ~AV1HighBDDISTWTDCOMPAVGUPSAMPLEDTest() {} void SetUp() { rnd_.Reset(ACMRandom::DeterministicSeed()); } void TearDown() { libaom_test::ClearSystemState(); } protected: - void RunCheckOutput(highbdjntcompavgupsampled_func test_impl) { + void RunCheckOutput(highbddistwtdcompavgupsampled_func test_impl) { const int w = kMaxSize, h = kMaxSize; const int block_idx = GET_PARAM(2); const int bd = GET_PARAM(0); uint16_t pred8[kMaxSize * kMaxSize]; uint16_t ref8[kMaxSize * kMaxSize]; - uint16_t output[kMaxSize * kMaxSize]; - uint16_t output2[kMaxSize * kMaxSize]; + DECLARE_ALIGNED(16, uint16_t, output[kMaxSize * kMaxSize]); + DECLARE_ALIGNED(16, uint16_t, output2[kMaxSize * kMaxSize]); for (int i = 0; i < h; ++i) for (int j = 0; j < w; ++j) { @@ -447,39 +451,42 @@ class AV1HighBDJNTCOMPAVGUPSAMPLEDTest const int in_w = block_size_wide[block_idx]; const int in_h = block_size_high[block_idx]; - JNT_COMP_PARAMS jnt_comp_params; - jnt_comp_params.use_jnt_comp_avg = 1; + DIST_WTD_COMP_PARAMS dist_wtd_comp_params; + dist_wtd_comp_params.use_dist_wtd_comp_avg = 1; int sub_x_q3, sub_y_q3; int subpel_search; - for (subpel_search = 1; subpel_search <= 2; ++subpel_search) { + for (subpel_search = USE_4_TAPS; subpel_search <= USE_8_TAPS; + ++subpel_search) { for (sub_x_q3 = 0; sub_x_q3 < 8; ++sub_x_q3) { for (sub_y_q3 = 0; sub_y_q3 < 8; ++sub_y_q3) { for (int ii = 0; ii < 2; ii++) { for (int jj = 0; jj < 4; jj++) { - jnt_comp_params.fwd_offset = quant_dist_lookup_table[ii][jj][0]; - jnt_comp_params.bck_offset = quant_dist_lookup_table[ii][jj][1]; + dist_wtd_comp_params.fwd_offset = + quant_dist_lookup_table[ii][jj][0]; + dist_wtd_comp_params.bck_offset = + quant_dist_lookup_table[ii][jj][1]; const int offset_r = 3 + rnd_.PseudoUniform(h - in_h - 7); const int offset_c = 3 + rnd_.PseudoUniform(w - in_w - 7); - aom_highbd_jnt_comp_avg_upsampled_pred_c( + aom_highbd_dist_wtd_comp_avg_upsampled_pred_c( NULL, NULL, 0, 0, NULL, CONVERT_TO_BYTEPTR(output), CONVERT_TO_BYTEPTR(pred8) + offset_r * w + offset_c, in_w, in_h, sub_x_q3, sub_y_q3, CONVERT_TO_BYTEPTR(ref8) + offset_r * w + offset_c, in_w, bd, - &jnt_comp_params, subpel_search); + &dist_wtd_comp_params, subpel_search); test_impl(NULL, NULL, 0, 0, NULL, CONVERT_TO_BYTEPTR(output2), CONVERT_TO_BYTEPTR(pred8) + offset_r * w + offset_c, in_w, in_h, sub_x_q3, sub_y_q3, CONVERT_TO_BYTEPTR(ref8) + offset_r * w + offset_c, - in_w, bd, &jnt_comp_params, subpel_search); + in_w, bd, &dist_wtd_comp_params, subpel_search); for (int i = 0; i < in_h; ++i) { for (int j = 0; j < in_w; ++j) { int idx = i * in_w + j; ASSERT_EQ(output[idx], output2[idx]) << "Mismatch at unit tests for " - "AV1HighBDJNTCOMPAVGUPSAMPLEDTest\n" + "AV1HighBDDISTWTDCOMPAVGUPSAMPLEDTest\n" << in_w << "x" << in_h << " Pixel mismatch at index " << idx << " = (" << i << ", " << j << "), sub pixel offset = (" << sub_y_q3 << ", " @@ -492,14 +499,14 @@ class AV1HighBDJNTCOMPAVGUPSAMPLEDTest } } } - void RunSpeedTest(highbdjntcompavgupsampled_func test_impl) { + void RunSpeedTest(highbddistwtdcompavgupsampled_func test_impl) { const int w = kMaxSize, h = kMaxSize; const int block_idx = GET_PARAM(2); const int bd = GET_PARAM(0); uint16_t pred8[kMaxSize * kMaxSize]; uint16_t ref8[kMaxSize * kMaxSize]; - uint16_t output[kMaxSize * kMaxSize]; - uint16_t output2[kMaxSize * kMaxSize]; + DECLARE_ALIGNED(16, uint16_t, output[kMaxSize * kMaxSize]); + DECLARE_ALIGNED(16, uint16_t, output2[kMaxSize * kMaxSize]); for (int i = 0; i < h; ++i) for (int j = 0; j < w; ++j) { @@ -509,27 +516,28 @@ class AV1HighBDJNTCOMPAVGUPSAMPLEDTest const int in_w = block_size_wide[block_idx]; const int in_h = block_size_high[block_idx]; - JNT_COMP_PARAMS jnt_comp_params; - jnt_comp_params.use_jnt_comp_avg = 1; + DIST_WTD_COMP_PARAMS dist_wtd_comp_params; + dist_wtd_comp_params.use_dist_wtd_comp_avg = 1; - jnt_comp_params.fwd_offset = quant_dist_lookup_table[0][0][0]; - jnt_comp_params.bck_offset = quant_dist_lookup_table[0][0][1]; + dist_wtd_comp_params.fwd_offset = quant_dist_lookup_table[0][0][0]; + dist_wtd_comp_params.bck_offset = quant_dist_lookup_table[0][0][1]; int sub_x_q3 = 0; int sub_y_q3 = 0; const int num_loops = 1000000000 / (in_w + in_h); aom_usec_timer timer; aom_usec_timer_start(&timer); - int subpel_search = 2; // set to 1 to test 4-tap filter. + int subpel_search = USE_8_TAPS; // set to USE_4_TAPS to test 4-tap filter. for (int i = 0; i < num_loops; ++i) - aom_highbd_jnt_comp_avg_upsampled_pred_c( + aom_highbd_dist_wtd_comp_avg_upsampled_pred_c( NULL, NULL, 0, 0, NULL, CONVERT_TO_BYTEPTR(output), CONVERT_TO_BYTEPTR(pred8), in_w, in_h, sub_x_q3, sub_y_q3, - CONVERT_TO_BYTEPTR(ref8), in_w, bd, &jnt_comp_params, subpel_search); + CONVERT_TO_BYTEPTR(ref8), in_w, bd, &dist_wtd_comp_params, + subpel_search); aom_usec_timer_mark(&timer); const int elapsed_time = static_cast(aom_usec_timer_elapsed(&timer)); - printf("highbdjntcompavgupsampled c_code %3dx%-3d: %7.2f us\n", in_w, in_h, - 1000.0 * elapsed_time / num_loops); + printf("highbddistwtdcompavgupsampled c_code %3dx%-3d: %7.2f us\n", in_w, + in_h, 1000.0 * elapsed_time / num_loops); aom_usec_timer timer1; aom_usec_timer_start(&timer1); @@ -537,19 +545,19 @@ class AV1HighBDJNTCOMPAVGUPSAMPLEDTest for (int i = 0; i < num_loops; ++i) test_impl(NULL, NULL, 0, 0, NULL, CONVERT_TO_BYTEPTR(output2), CONVERT_TO_BYTEPTR(pred8), in_w, in_h, sub_x_q3, sub_y_q3, - CONVERT_TO_BYTEPTR(ref8), in_w, bd, &jnt_comp_params, + CONVERT_TO_BYTEPTR(ref8), in_w, bd, &dist_wtd_comp_params, subpel_search); aom_usec_timer_mark(&timer1); const int elapsed_time1 = static_cast(aom_usec_timer_elapsed(&timer1)); - printf("highbdjntcompavgupsampled test_code %3dx%-3d: %7.2f us\n", in_w, + printf("highbddistwtdcompavgupsampled test_code %3dx%-3d: %7.2f us\n", in_w, in_h, 1000.0 * elapsed_time1 / num_loops); } libaom_test::ACMRandom rnd_; -}; // class AV1HighBDJNTCOMPAVGUPSAMPLEDTest +}; // class AV1HighBDDISTWTDCOMPAVGUPSAMPLEDTest -} // namespace AV1JNTCOMPAVG +} // namespace AV1DISTWTDCOMPAVG } // namespace libaom_test #endif // AOM_TEST_COMP_AVG_PRED_TEST_H_ diff --git a/third_party/aom/test/comp_mask_variance_test.cc b/third_party/aom/test/comp_mask_variance_test.cc index 34be2aa6db..53ba56c819 100644 --- a/third_party/aom/test/comp_mask_variance_test.cc +++ b/third_party/aom/test/comp_mask_variance_test.cc @@ -34,7 +34,7 @@ typedef void (*comp_mask_pred_func)(uint8_t *comp_pred, const uint8_t *pred, int ref_stride, const uint8_t *mask, int mask_stride, int invert_mask); -#if HAVE_SSSE3 || HAVE_AV2 +#if HAVE_SSSE3 || HAVE_SSE2 || HAVE_AV2 const BLOCK_SIZE kValidBlockSize[] = { BLOCK_8X8, BLOCK_8X16, BLOCK_8X32, BLOCK_16X8, BLOCK_16X16, BLOCK_16X32, BLOCK_32X8, BLOCK_32X16, BLOCK_32X32, @@ -191,7 +191,8 @@ void AV1CompMaskUpVarianceTest::RunCheckOutput(comp_mask_pred_func test_impl, const int h = block_size_high[bsize]; int wedge_types = (1 << get_wedge_bits_lookup(bsize)); int subpel_search; - for (subpel_search = 1; subpel_search <= 2; ++subpel_search) { + for (subpel_search = USE_4_TAPS; subpel_search <= USE_8_TAPS; + ++subpel_search) { // loop through subx and suby for (int sub = 0; sub < 8 * 8; ++sub) { int subx = sub & 0x7; @@ -231,7 +232,7 @@ void AV1CompMaskUpVarianceTest::RunSpeedTest(comp_mask_pred_func test_impl, const int num_loops = 1000000000 / (w + h); comp_mask_pred_func funcs[2] = { &aom_comp_mask_pred_c, test_impl }; double elapsed_time[2] = { 0 }; - int subpel_search = 2; // set to 1 to test 4-tap filter. + int subpel_search = USE_8_TAPS; // set to USE_4_TAPS to test 4-tap filter. for (int i = 0; i < 2; ++i) { aom_usec_timer timer; aom_usec_timer_start(&timer); @@ -458,6 +459,7 @@ AV1HighbdCompMaskUpVarianceTest::~AV1HighbdCompMaskUpVarianceTest() { ; } void AV1HighbdCompMaskUpVarianceTest::RunCheckOutput( highbd_comp_mask_pred_func test_impl, BLOCK_SIZE bsize, int inv) { + (void)test_impl; int bd_ = GET_PARAM(2); const int w = block_size_wide[bsize]; const int h = block_size_high[bsize]; @@ -480,19 +482,24 @@ void AV1HighbdCompMaskUpVarianceTest::RunCheckOutput( const uint8_t *mask = av1_get_contiguous_soft_mask(wedge_index, 1, bsize); - aom_highbd_comp_mask_pred = aom_highbd_comp_mask_pred_c; // ref - aom_highbd_comp_mask_upsampled_pred( - NULL, NULL, 0, 0, NULL, CONVERT_TO_BYTEPTR(comp_pred1_), - CONVERT_TO_BYTEPTR(pred_), w, h, subx, suby, - CONVERT_TO_BYTEPTR(ref_), MAX_SB_SIZE, mask, w, inv, bd_, - subpel_search); + // ref + aom_highbd_upsampled_pred_c( + NULL, NULL, 0, 0, NULL, CONVERT_TO_BYTEPTR(comp_pred1_), w, h, subx, + suby, CONVERT_TO_BYTEPTR(ref_), MAX_SB_SIZE, bd_, subpel_search); + + aom_highbd_comp_mask_pred_c( + CONVERT_TO_BYTEPTR(comp_pred1_), CONVERT_TO_BYTEPTR(pred_), w, h, + CONVERT_TO_BYTEPTR(comp_pred1_), w, mask, w, inv); + + // test + aom_highbd_upsampled_pred( + NULL, NULL, 0, 0, NULL, CONVERT_TO_BYTEPTR(comp_pred2_), w, h, subx, + suby, CONVERT_TO_BYTEPTR(ref_), MAX_SB_SIZE, bd_, subpel_search); + + aom_highbd_comp_mask_pred( + CONVERT_TO_BYTEPTR(comp_pred2_), CONVERT_TO_BYTEPTR(pred_), w, h, + CONVERT_TO_BYTEPTR(comp_pred2_), w, mask, w, inv); - aom_highbd_comp_mask_pred = test_impl; // test - aom_highbd_comp_mask_upsampled_pred( - NULL, NULL, 0, 0, NULL, CONVERT_TO_BYTEPTR(comp_pred2_), - CONVERT_TO_BYTEPTR(pred_), w, h, subx, suby, - CONVERT_TO_BYTEPTR(ref_), MAX_SB_SIZE, mask, w, inv, bd_, - subpel_search); ASSERT_EQ(CheckResult(w, h), true) << " wedge " << wedge_index << " inv " << inv << "sub (" << subx << "," << suby << ")"; diff --git a/third_party/aom/test/convolve_test.cc b/third_party/aom/test/convolve_test.cc index de3f47628a..760d2c579d 100644 --- a/third_party/aom/test/convolve_test.cc +++ b/third_party/aom/test/convolve_test.cc @@ -273,6 +273,8 @@ class ConvolveTest : public ::testing::TestWithParam { input_ = reinterpret_cast( aom_memalign(kDataAlignment, kInputBufferSize + 1)) + 1; + ref8_ = reinterpret_cast( + aom_memalign(kDataAlignment, kOutputStride * kMaxDimension)); output_ = reinterpret_cast( aom_memalign(kDataAlignment, kOutputBufferSize)); output_ref_ = reinterpret_cast( @@ -280,6 +282,8 @@ class ConvolveTest : public ::testing::TestWithParam { input16_ = reinterpret_cast(aom_memalign( kDataAlignment, (kInputBufferSize + 1) * sizeof(uint16_t))) + 1; + ref16_ = reinterpret_cast(aom_memalign( + kDataAlignment, kOutputStride * kMaxDimension * sizeof(uint16_t))); output16_ = reinterpret_cast( aom_memalign(kDataAlignment, (kOutputBufferSize) * sizeof(uint16_t))); output16_ref_ = reinterpret_cast( @@ -291,12 +295,16 @@ class ConvolveTest : public ::testing::TestWithParam { static void TearDownTestCase() { aom_free(input_ - 1); input_ = NULL; + aom_free(ref8_); + ref8_ = NULL; aom_free(output_); output_ = NULL; aom_free(output_ref_); output_ref_ = NULL; aom_free(input16_ - 1); input16_ = NULL; + aom_free(ref16_); + ref16_ = NULL; aom_free(output16_); output16_ = NULL; aom_free(output16_ref_); @@ -449,18 +457,22 @@ class ConvolveTest : public ::testing::TestWithParam { const ConvolveFunctions *UUT_; static uint8_t *input_; + static uint8_t *ref8_; static uint8_t *output_; static uint8_t *output_ref_; static uint16_t *input16_; + static uint16_t *ref16_; static uint16_t *output16_; static uint16_t *output16_ref_; int mask_; }; uint8_t *ConvolveTest::input_ = NULL; +uint8_t *ConvolveTest::ref8_ = NULL; uint8_t *ConvolveTest::output_ = NULL; uint8_t *ConvolveTest::output_ref_ = NULL; uint16_t *ConvolveTest::input16_ = NULL; +uint16_t *ConvolveTest::ref16_ = NULL; uint16_t *ConvolveTest::output16_ = NULL; uint16_t *ConvolveTest::output16_ref_ = NULL; @@ -486,26 +498,28 @@ const int kNumFilterBanks = SWITCHABLE_FILTERS; const int kNumFilters = 16; TEST(ConvolveTest, FiltersWontSaturateWhenAddedPairwise) { - for (int filter_bank = 0; filter_bank < kNumFilterBanks; ++filter_bank) { - const InterpFilter filter = (InterpFilter)filter_bank; - const InterpKernel *filters = - (const InterpKernel *)av1_get_interp_filter_kernel(filter); - const InterpFilterParams *filter_params = - av1_get_interp_filter_params_with_block_size(filter, 8); - if (filter_params->taps != SUBPEL_TAPS) continue; - for (int i = 0; i < kNumFilters; i++) { - const int p0 = filters[i][0] + filters[i][1]; - const int p1 = filters[i][2] + filters[i][3]; - const int p2 = filters[i][4] + filters[i][5]; - const int p3 = filters[i][6] + filters[i][7]; - EXPECT_LE(p0, 128); - EXPECT_LE(p1, 128); - EXPECT_LE(p2, 128); - EXPECT_LE(p3, 128); - EXPECT_LE(p0 + p3, 128); - EXPECT_LE(p0 + p3 + p1, 128); - EXPECT_LE(p0 + p3 + p1 + p2, 128); - EXPECT_EQ(p0 + p1 + p2 + p3, 128); + int subpel_search; + for (subpel_search = USE_4_TAPS; subpel_search <= USE_8_TAPS; + ++subpel_search) { + for (int filter_bank = 0; filter_bank < kNumFilterBanks; ++filter_bank) { + const InterpFilter filter = (InterpFilter)filter_bank; + const InterpKernel *filters = + (const InterpKernel *)av1_get_interp_filter_kernel(filter, + subpel_search); + for (int i = 0; i < kNumFilters; i++) { + const int p0 = filters[i][0] + filters[i][1]; + const int p1 = filters[i][2] + filters[i][3]; + const int p2 = filters[i][4] + filters[i][5]; + const int p3 = filters[i][6] + filters[i][7]; + EXPECT_LE(p0, 128); + EXPECT_LE(p1, 128); + EXPECT_LE(p2, 128); + EXPECT_LE(p3, 128); + EXPECT_LE(p0 + p3, 128); + EXPECT_LE(p0 + p3 + p1, 128); + EXPECT_LE(p0 + p3 + p1 + p2, 128); + EXPECT_EQ(p0 + p1 + p2 + p3, 128); + } } } } @@ -515,53 +529,51 @@ const int16_t kInvalidFilter[8] = { 0 }; TEST_P(ConvolveTest, MatchesReferenceSubpixelFilter) { uint8_t *const in = input(); uint8_t *const out = output(); - uint8_t ref8[kOutputStride * kMaxDimension]; - uint16_t ref16[kOutputStride * kMaxDimension]; uint8_t *ref; if (UUT_->use_highbd_ == 0) { - ref = ref8; + ref = ref8_; } else { - ref = CONVERT_TO_BYTEPTR(ref16); + ref = CONVERT_TO_BYTEPTR(ref16_); } + int subpel_search; + for (subpel_search = USE_4_TAPS; subpel_search <= USE_8_TAPS; + ++subpel_search) { + for (int filter_bank = 0; filter_bank < kNumFilterBanks; ++filter_bank) { + const InterpFilter filter = (InterpFilter)filter_bank; + const InterpKernel *filters = + (const InterpKernel *)av1_get_interp_filter_kernel(filter, + subpel_search); + for (int filter_x = 0; filter_x < kNumFilters; ++filter_x) { + for (int filter_y = 0; filter_y < kNumFilters; ++filter_y) { + wrapper_filter_block2d_8_c(in, kInputStride, filters[filter_x], + filters[filter_y], ref, kOutputStride, + Width(), Height()); - for (int filter_bank = 0; filter_bank < kNumFilterBanks; ++filter_bank) { - const InterpFilter filter = (InterpFilter)filter_bank; - const InterpKernel *filters = - (const InterpKernel *)av1_get_interp_filter_kernel(filter); - const InterpFilterParams *filter_params = - av1_get_interp_filter_params_with_block_size(filter, 8); - if (filter_params->taps != SUBPEL_TAPS) continue; + if (filter_x && filter_y) + continue; + else if (filter_y) + ASM_REGISTER_STATE_CHECK( + UUT_->v8_(in, kInputStride, out, kOutputStride, kInvalidFilter, + 16, filters[filter_y], 16, Width(), Height())); + else if (filter_x) + ASM_REGISTER_STATE_CHECK(UUT_->h8_( + in, kInputStride, out, kOutputStride, filters[filter_x], 16, + kInvalidFilter, 16, Width(), Height())); + else + ASM_REGISTER_STATE_CHECK(UUT_->copy_( + in, kInputStride, out, kOutputStride, kInvalidFilter, 0, + kInvalidFilter, 0, Width(), Height())); - for (int filter_x = 0; filter_x < kNumFilters; ++filter_x) { - for (int filter_y = 0; filter_y < kNumFilters; ++filter_y) { - wrapper_filter_block2d_8_c(in, kInputStride, filters[filter_x], - filters[filter_y], ref, kOutputStride, - Width(), Height()); + CheckGuardBlocks(); - if (filter_x && filter_y) - continue; - else if (filter_y) - ASM_REGISTER_STATE_CHECK( - UUT_->v8_(in, kInputStride, out, kOutputStride, kInvalidFilter, - 16, filters[filter_y], 16, Width(), Height())); - else if (filter_x) - ASM_REGISTER_STATE_CHECK( - UUT_->h8_(in, kInputStride, out, kOutputStride, filters[filter_x], - 16, kInvalidFilter, 16, Width(), Height())); - else - ASM_REGISTER_STATE_CHECK( - UUT_->copy_(in, kInputStride, out, kOutputStride, kInvalidFilter, - 0, kInvalidFilter, 0, Width(), Height())); - - CheckGuardBlocks(); - - for (int y = 0; y < Height(); ++y) - for (int x = 0; x < Width(); ++x) - ASSERT_EQ(lookup(ref, y * kOutputStride + x), - lookup(out, y * kOutputStride + x)) - << "mismatch at (" << x << "," << y << "), " - << "filters (" << filter_bank << "," << filter_x << "," - << filter_y << ")"; + for (int y = 0; y < Height(); ++y) + for (int x = 0; x < Width(); ++x) + ASSERT_EQ(lookup(ref, y * kOutputStride + x), + lookup(out, y * kOutputStride + x)) + << "mismatch at (" << x << "," << y << "), " + << "filters (" << filter_bank << "," << filter_x << "," + << filter_y << ")"; + } } } } @@ -570,13 +582,11 @@ TEST_P(ConvolveTest, MatchesReferenceSubpixelFilter) { TEST_P(ConvolveTest, FilterExtremes) { uint8_t *const in = input(); uint8_t *const out = output(); - uint8_t ref8[kOutputStride * kMaxDimension]; - uint16_t ref16[kOutputStride * kMaxDimension]; uint8_t *ref; if (UUT_->use_highbd_ == 0) { - ref = ref8; + ref = ref8_; } else { - ref = CONVERT_TO_BYTEPTR(ref16); + ref = CONVERT_TO_BYTEPTR(ref16_); } // Populate ref and out with some random data @@ -609,41 +619,43 @@ TEST_P(ConvolveTest, FilterExtremes) { seed_val++; } if (axis) seed_val += 8; + int subpel_search; + for (subpel_search = USE_4_TAPS; subpel_search <= USE_8_TAPS; + ++subpel_search) { + for (int filter_bank = 0; filter_bank < kNumFilterBanks; + ++filter_bank) { + const InterpFilter filter = (InterpFilter)filter_bank; + const InterpKernel *filters = + (const InterpKernel *)av1_get_interp_filter_kernel(filter, + subpel_search); + for (int filter_x = 0; filter_x < kNumFilters; ++filter_x) { + for (int filter_y = 0; filter_y < kNumFilters; ++filter_y) { + wrapper_filter_block2d_8_c(in, kInputStride, filters[filter_x], + filters[filter_y], ref, kOutputStride, + Width(), Height()); + if (filter_x && filter_y) + continue; + else if (filter_y) + ASM_REGISTER_STATE_CHECK(UUT_->v8_( + in, kInputStride, out, kOutputStride, kInvalidFilter, 16, + filters[filter_y], 16, Width(), Height())); + else if (filter_x) + ASM_REGISTER_STATE_CHECK(UUT_->h8_( + in, kInputStride, out, kOutputStride, filters[filter_x], 16, + kInvalidFilter, 16, Width(), Height())); + else + ASM_REGISTER_STATE_CHECK(UUT_->copy_( + in, kInputStride, out, kOutputStride, kInvalidFilter, 0, + kInvalidFilter, 0, Width(), Height())); - for (int filter_bank = 0; filter_bank < kNumFilterBanks; ++filter_bank) { - const InterpFilter filter = (InterpFilter)filter_bank; - const InterpKernel *filters = - (const InterpKernel *)av1_get_interp_filter_kernel(filter); - const InterpFilterParams *filter_params = - av1_get_interp_filter_params_with_block_size(filter, 8); - if (filter_params->taps != SUBPEL_TAPS) continue; - for (int filter_x = 0; filter_x < kNumFilters; ++filter_x) { - for (int filter_y = 0; filter_y < kNumFilters; ++filter_y) { - wrapper_filter_block2d_8_c(in, kInputStride, filters[filter_x], - filters[filter_y], ref, kOutputStride, - Width(), Height()); - if (filter_x && filter_y) - continue; - else if (filter_y) - ASM_REGISTER_STATE_CHECK(UUT_->v8_( - in, kInputStride, out, kOutputStride, kInvalidFilter, 16, - filters[filter_y], 16, Width(), Height())); - else if (filter_x) - ASM_REGISTER_STATE_CHECK(UUT_->h8_( - in, kInputStride, out, kOutputStride, filters[filter_x], 16, - kInvalidFilter, 16, Width(), Height())); - else - ASM_REGISTER_STATE_CHECK(UUT_->copy_( - in, kInputStride, out, kOutputStride, kInvalidFilter, 0, - kInvalidFilter, 0, Width(), Height())); - - for (int y = 0; y < Height(); ++y) - for (int x = 0; x < Width(); ++x) - ASSERT_EQ(lookup(ref, y * kOutputStride + x), - lookup(out, y * kOutputStride + x)) - << "mismatch at (" << x << "," << y << "), " - << "filters (" << filter_bank << "," << filter_x << "," - << filter_y << ")"; + for (int y = 0; y < Height(); ++y) + for (int x = 0; x < Width(); ++x) + ASSERT_EQ(lookup(ref, y * kOutputStride + x), + lookup(out, y * kOutputStride + x)) + << "mismatch at (" << x << "," << y << "), " + << "filters (" << filter_bank << "," << filter_x << "," + << filter_y << ")"; + } } } } @@ -674,13 +686,11 @@ TEST_P(ConvolveTest, DISABLED_Copy_Speed) { TEST_P(ConvolveTest, DISABLED_Speed) { uint8_t *const in = input(); uint8_t *const out = output(); - uint8_t ref8[kOutputStride * kMaxDimension]; - uint16_t ref16[kOutputStride * kMaxDimension]; uint8_t *ref; if (UUT_->use_highbd_ == 0) { - ref = ref8; + ref = ref8_; } else { - ref = CONVERT_TO_BYTEPTR(ref16); + ref = CONVERT_TO_BYTEPTR(ref16_); } // Populate ref and out with some random data @@ -700,7 +710,7 @@ TEST_P(ConvolveTest, DISABLED_Speed) { const InterpFilter filter = (InterpFilter)1; const InterpKernel *filters = - (const InterpKernel *)av1_get_interp_filter_kernel(filter); + (const InterpKernel *)av1_get_interp_filter_kernel(filter, USE_8_TAPS); wrapper_filter_average_block2d_8_c(in, kInputStride, filters[1], filters[1], out, kOutputStride, Width(), Height()); @@ -712,11 +722,8 @@ TEST_P(ConvolveTest, DISABLED_Speed) { for (int filter_bank = 0; filter_bank < kNumFilterBanks; ++filter_bank) { const InterpFilter filter = (InterpFilter)filter_bank; const InterpKernel *filters = - (const InterpKernel *)av1_get_interp_filter_kernel(filter); - const InterpFilterParams *filter_params = - av1_get_interp_filter_params_with_block_size(filter, 8); - if (filter_params->taps != SUBPEL_TAPS) continue; - + (const InterpKernel *)av1_get_interp_filter_kernel(filter, + USE_8_TAPS); for (int filter_x = 0; filter_x < kNumFilters; ++filter_x) { for (int filter_y = 0; filter_y < kNumFilters; ++filter_y) { if (filter_x && filter_y) continue; @@ -805,18 +812,22 @@ const ConvolveParam kArrayConvolve_c[] = { INSTANTIATE_TEST_CASE_P(C, ConvolveTest, ::testing::ValuesIn(kArrayConvolve_c)); #if HAVE_SSE2 && ARCH_X86_64 -const ConvolveFunctions convolve8_sse2(wrap_convolve_copy_sse2_8, - wrap_convolve8_horiz_sse2_8, - wrap_convolve8_vert_sse2_8, 8); -const ConvolveFunctions convolve10_sse2(wrap_convolve_copy_sse2_10, - wrap_convolve8_horiz_sse2_10, - wrap_convolve8_vert_sse2_10, 10); -const ConvolveFunctions convolve12_sse2(wrap_convolve_copy_sse2_12, - wrap_convolve8_horiz_sse2_12, - wrap_convolve8_vert_sse2_12, 12); +const ConvolveFunctions convolve8_sse2(aom_convolve_copy_c, + aom_convolve8_horiz_sse2, + aom_convolve8_vert_sse2, 0); +const ConvolveFunctions wrap_convolve8_sse2(wrap_convolve_copy_sse2_8, + wrap_convolve8_horiz_sse2_8, + wrap_convolve8_vert_sse2_8, 8); +const ConvolveFunctions wrap_convolve10_sse2(wrap_convolve_copy_sse2_10, + wrap_convolve8_horiz_sse2_10, + wrap_convolve8_vert_sse2_10, 10); +const ConvolveFunctions wrap_convolve12_sse2(wrap_convolve_copy_sse2_12, + wrap_convolve8_horiz_sse2_12, + wrap_convolve8_vert_sse2_12, 12); const ConvolveParam kArrayConvolve_sse2[] = { ALL_SIZES(convolve8_sse2), - ALL_SIZES(convolve10_sse2), - ALL_SIZES(convolve12_sse2) }; + ALL_SIZES(wrap_convolve8_sse2), + ALL_SIZES(wrap_convolve10_sse2), + ALL_SIZES(wrap_convolve12_sse2) }; INSTANTIATE_TEST_CASE_P(SSE2, ConvolveTest, ::testing::ValuesIn(kArrayConvolve_sse2)); #endif diff --git a/third_party/aom/test/corner_match_test.cc b/third_party/aom/test/corner_match_test.cc index 58e3139c5f..af2baa7b14 100644 --- a/third_party/aom/test/corner_match_test.cc +++ b/third_party/aom/test/corner_match_test.cc @@ -24,9 +24,13 @@ namespace AV1CornerMatch { using libaom_test::ACMRandom; +typedef double (*ComputeCrossCorrFunc)(unsigned char *im1, int stride1, int x1, + int y1, unsigned char *im2, int stride2, + int x2, int y2); + using ::testing::make_tuple; using ::testing::tuple; -typedef tuple CornerMatchParam; +typedef tuple CornerMatchParam; class AV1CornerMatchTest : public ::testing::TestWithParam { public: @@ -36,19 +40,24 @@ class AV1CornerMatchTest : public ::testing::TestWithParam { virtual void TearDown(); protected: - void RunCheckOutput(); + void RunCheckOutput(int run_times); + ComputeCrossCorrFunc target_func; libaom_test::ACMRandom rnd_; }; AV1CornerMatchTest::~AV1CornerMatchTest() {} -void AV1CornerMatchTest::SetUp() { rnd_.Reset(ACMRandom::DeterministicSeed()); } +void AV1CornerMatchTest::SetUp() { + rnd_.Reset(ACMRandom::DeterministicSeed()); + target_func = GET_PARAM(1); +} void AV1CornerMatchTest::TearDown() { libaom_test::ClearSystemState(); } -void AV1CornerMatchTest::RunCheckOutput() { +void AV1CornerMatchTest::RunCheckOutput(int run_times) { const int w = 128, h = 128; const int num_iters = 10000; int i, j; + aom_usec_timer ref_timer, test_timer; uint8_t *input1 = new uint8_t[w * h]; uint8_t *input2 = new uint8_t[w * h]; @@ -80,21 +89,54 @@ void AV1CornerMatchTest::RunCheckOutput() { double res_c = compute_cross_correlation_c(input1, w, x1, y1, input2, w, x2, y2); - double res_sse4 = - compute_cross_correlation_sse4_1(input1, w, x1, y1, input2, w, x2, y2); + double res_simd = target_func(input1, w, x1, y1, input2, w, x2, y2); - ASSERT_EQ(res_sse4, res_c); + if (run_times > 1) { + aom_usec_timer_start(&ref_timer); + for (j = 0; j < run_times; j++) { + compute_cross_correlation_c(input1, w, x1, y1, input2, w, x2, y2); + } + aom_usec_timer_mark(&ref_timer); + const int elapsed_time_c = + static_cast(aom_usec_timer_elapsed(&ref_timer)); + + aom_usec_timer_start(&test_timer); + for (j = 0; j < run_times; j++) { + target_func(input1, w, x1, y1, input2, w, x2, y2); + } + aom_usec_timer_mark(&test_timer); + const int elapsed_time_simd = + static_cast(aom_usec_timer_elapsed(&test_timer)); + + printf( + "c_time=%d \t simd_time=%d \t " + "gain=%d\n", + elapsed_time_c, elapsed_time_simd, + (elapsed_time_c / elapsed_time_simd)); + } else { + ASSERT_EQ(res_simd, res_c); + } } - delete[] input1; delete[] input2; } -TEST_P(AV1CornerMatchTest, CheckOutput) { RunCheckOutput(); } +TEST_P(AV1CornerMatchTest, CheckOutput) { RunCheckOutput(1); } +TEST_P(AV1CornerMatchTest, DISABLED_Speed) { RunCheckOutput(100000); } -INSTANTIATE_TEST_CASE_P(SSE4_1, AV1CornerMatchTest, - ::testing::Values(make_tuple(0), make_tuple(1))); +#if HAVE_SSE4_1 +INSTANTIATE_TEST_CASE_P( + SSE4_1, AV1CornerMatchTest, + ::testing::Values(make_tuple(0, compute_cross_correlation_sse4_1), + make_tuple(1, compute_cross_correlation_sse4_1))); +#endif +#if HAVE_AVX2 +INSTANTIATE_TEST_CASE_P( + AVX2, AV1CornerMatchTest, + ::testing::Values(make_tuple(0, compute_cross_correlation_avx2), + make_tuple(1, compute_cross_correlation_avx2))); +#endif } // namespace AV1CornerMatch } // namespace test_libaom diff --git a/third_party/aom/test/cpu_speed_test.cc b/third_party/aom/test/cpu_speed_test.cc index 8ea3e69650..2a164974b0 100644 --- a/third_party/aom/test/cpu_speed_test.cc +++ b/third_party/aom/test/cpu_speed_test.cc @@ -46,7 +46,7 @@ class CpuSpeedTest virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video, ::libaom_test::Encoder *encoder) { - if (video->frame() == 1) { + if (video->frame() == 0) { encoder->Control(AOME_SET_CPUUSED, set_cpu_used_); encoder->Control(AV1E_SET_TUNE_CONTENT, tune_content_); if (encoding_mode_ != ::libaom_test::kRealTime) { diff --git a/third_party/aom/test/decode_multithreaded_test.cc b/third_party/aom/test/decode_multithreaded_test.cc index cea1d144f1..92253ede81 100644 --- a/third_party/aom/test/decode_multithreaded_test.cc +++ b/third_party/aom/test/decode_multithreaded_test.cc @@ -76,7 +76,7 @@ class AV1DecodeMultiThreadedTest virtual void PreEncodeFrameHook(libaom_test::VideoSource *video, libaom_test::Encoder *encoder) { - if (video->frame() == 1) { + if (video->frame() == 0) { encoder->Control(AV1E_SET_TILE_COLUMNS, n_tile_cols_); encoder->Control(AV1E_SET_TILE_ROWS, n_tile_rows_); encoder->Control(AV1E_SET_NUM_TG, n_tile_groups_); diff --git a/third_party/aom/test/decode_perf_test.cc b/third_party/aom/test/decode_perf_test.cc index bb7b00032e..2f67342570 100644 --- a/third_party/aom/test/decode_perf_test.cc +++ b/third_party/aom/test/decode_perf_test.cc @@ -122,7 +122,7 @@ class AV1NewEncodeDecodePerfTest virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video, ::libaom_test::Encoder *encoder) { - if (video->frame() == 1) { + if (video->frame() == 0) { encoder->Control(AOME_SET_CPUUSED, speed_); encoder->Control(AV1E_SET_FRAME_PARALLEL_DECODING, 1); encoder->Control(AV1E_SET_TILE_COLUMNS, 2); diff --git a/third_party/aom/test/dr_prediction_test.cc b/third_party/aom/test/dr_prediction_test.cc index ff2c1de4e9..603694f3ae 100644 --- a/third_party/aom/test/dr_prediction_test.cc +++ b/third_party/aom/test/dr_prediction_test.cc @@ -1,4 +1,4 @@ -/* +/* * Copyright (c) 2018, Alliance for Open Media. All rights reserved * * This source code is subject to the terms of the BSD 2 Clause License and @@ -135,7 +135,7 @@ struct DrPredFunc { template class DrPredTest : public ::testing::TestWithParam > { protected: - static const int kMaxNumTests = 100000; + static const int kMaxNumTests = 10000; static const int kIterations = 10; static const int kDstStride = 64; static const int kDstSize = kDstStride * kDstStride; @@ -162,6 +162,7 @@ class DrPredTest : public ::testing::TestWithParam > { for (int i = 0; i < kDstSize; ++i) { dst_ref_[i] = 0; + dst_tst_[i] = 0; } } @@ -170,7 +171,7 @@ class DrPredTest : public ::testing::TestWithParam > { void Predict(bool speedtest, int tx) { const int kNumTests = speedtest ? kMaxNumTests : 1; aom_usec_timer timer; - + int tst_time = 0; aom_usec_timer_start(&timer); for (int k = 0; k < kNumTests; ++k) { params_.ref_fn(dst_ref_, dst_stride_, bw_, bh_, above_, left_, @@ -179,32 +180,39 @@ class DrPredTest : public ::testing::TestWithParam > { aom_usec_timer_mark(&timer); const int ref_time = static_cast(aom_usec_timer_elapsed(&timer)); - aom_usec_timer_start(&timer); if (params_.tst_fn) { + aom_usec_timer_start(&timer); for (int k = 0; k < kNumTests; ++k) { ASM_REGISTER_STATE_CHECK(params_.tst_fn(dst_tst_, dst_stride_, bw_, bh_, above_, left_, upsample_above_, upsample_left_, dx_, dy_, bd_)); } + aom_usec_timer_mark(&timer); + tst_time = static_cast(aom_usec_timer_elapsed(&timer)); + } else { + for (int i = 0; i < kDstSize; ++i) { + dst_ref_[i] = dst_tst_[i]; + } } - aom_usec_timer_mark(&timer); - const int tst_time = static_cast(aom_usec_timer_elapsed(&timer)); OutputTimes(kNumTests, ref_time, tst_time, tx); } - void RunTest(bool speedtest, int p_angle) { - for (int i = 0; i < kBufSize; ++i) { - above_data_[i] = left_data_[i] = (1 << bd_) - 1; + void RunTest(bool speedtest, bool needsaturation, int p_angle) { + if (needsaturation) { + for (int i = 0; i < kBufSize; ++i) { + above_data_[i] = left_data_[i] = (1 << bd_) - 1; + } } - for (int tx = 0; tx < TX_SIZES_ALL; ++tx) { if (params_.tst_fn == NULL) { for (int i = 0; i < kDstSize; ++i) { dst_tst_[i] = (1 << bd_) - 1; + dst_ref_[i] = (1 << bd_) - 1; } } else { for (int i = 0; i < kDstSize; ++i) { + dst_ref_[i] = 0; dst_tst_[i] = 0; } } @@ -282,26 +290,11 @@ class DrPredTest : public ::testing::TestWithParam > { class LowbdDrPredTest : public DrPredTest {}; TEST_P(LowbdDrPredTest, SaturatedValues) { - for (int iter = 0; iter < kIterations && !HasFatalFailure(); ++iter) { - enable_upsample_ = iter & 1; + for (enable_upsample_ = 0; enable_upsample_ < 2; ++enable_upsample_) { for (int angle = start_angle_; angle < stop_angle_; ++angle) { dx_ = av1_get_dx(angle); dy_ = av1_get_dy(angle); - if (dx_ && dy_) RunTest(false, angle); - } - } -} - -TEST_P(LowbdDrPredTest, DISABLED_Speed) { - const int angles[] = { 3, 45, 87 }; - for (enable_upsample_ = 0; enable_upsample_ < 2; ++enable_upsample_) { - for (int i = 0; i < 3; ++i) { - const int angle = angles[i] + start_angle_; - dx_ = av1_get_dx(angle); - dy_ = av1_get_dy(angle); - printf("enable_upsample: %d angle: %d ~~~~~~~~~~~~~~~\n", - enable_upsample_, angle); - if (dx_ && dy_) RunTest(true, angle); + if (dx_ && dy_) RunTest(false, true, angle); } } } @@ -320,26 +313,11 @@ INSTANTIATE_TEST_CASE_P( class HighbdDrPredTest : public DrPredTest {}; TEST_P(HighbdDrPredTest, SaturatedValues) { - for (int iter = 0; iter < kIterations && !HasFatalFailure(); ++iter) { - enable_upsample_ = iter & 1; + for (enable_upsample_ = 0; enable_upsample_ < 2; ++enable_upsample_) { for (int angle = start_angle_; angle < stop_angle_; ++angle) { dx_ = av1_get_dx(angle); dy_ = av1_get_dy(angle); - if (dx_ && dy_) RunTest(false, angle); - } - } -} - -TEST_P(HighbdDrPredTest, DISABLED_Speed) { - const int angles[] = { 3, 45, 87 }; - for (enable_upsample_ = 0; enable_upsample_ < 2; ++enable_upsample_) { - for (int i = 0; i < 3; ++i) { - const int angle = angles[i] + start_angle_; - dx_ = av1_get_dx(angle); - dy_ = av1_get_dy(angle); - printf("enable_upsample: %d angle: %d ~~~~~~~~~~~~~~~\n", - enable_upsample_, angle); - if (dx_ && dy_) RunTest(true, angle); + if (dx_ && dy_) RunTest(false, true, angle); } } } @@ -366,4 +344,112 @@ INSTANTIATE_TEST_CASE_P( DrPredFunc(&z3_wrapper_hbd, NULL, AOM_BITS_12, kZ3Start))); +#if HAVE_AVX2 +INSTANTIATE_TEST_CASE_P( + AVX2, LowbdDrPredTest, + ::testing::Values(DrPredFunc(&z1_wrapper, + &z1_wrapper, + AOM_BITS_8, kZ1Start), + DrPredFunc(&z2_wrapper, + &z2_wrapper, + AOM_BITS_8, kZ2Start), + DrPredFunc(&z3_wrapper, + &z3_wrapper, + AOM_BITS_8, kZ3Start))); + +TEST_P(LowbdDrPredTest, DISABLED_Speed) { + const int angles[] = { 3, 45, 87 }; + for (enable_upsample_ = 0; enable_upsample_ < 2; ++enable_upsample_) { + for (int i = 0; i < 3; ++i) { + const int angle = angles[i] + start_angle_; + dx_ = av1_get_dx(angle); + dy_ = av1_get_dy(angle); + printf("enable_upsample: %d angle: %d ~~~~~~~~~~~~~~~\n", + enable_upsample_, angle); + if (dx_ && dy_) RunTest(true, false, angle); + } + } +} + +TEST_P(LowbdDrPredTest, OperationCheck) { + if (params_.tst_fn == NULL) return; + // const int angles[] = { 3, 45, 81, 87, 93, 100, 145, 187, 199, 260 }; + for (enable_upsample_ = 0; enable_upsample_ < 2; ++enable_upsample_) { + for (int angle = start_angle_; angle < stop_angle_; ++angle) { + dx_ = av1_get_dx(angle); + dy_ = av1_get_dy(angle); + if (dx_ && dy_) RunTest(false, false, angle); + } + } +} + +INSTANTIATE_TEST_CASE_P( + AVX2, HighbdDrPredTest, + ::testing::Values(DrPredFunc( + &z1_wrapper_hbd, + &z1_wrapper_hbd, + AOM_BITS_8, kZ1Start), + DrPredFunc( + &z1_wrapper_hbd, + &z1_wrapper_hbd, + AOM_BITS_10, kZ1Start), + DrPredFunc( + &z1_wrapper_hbd, + &z1_wrapper_hbd, + AOM_BITS_12, kZ1Start), + /* TODO(https://crbug.com/aomedia/2288) + DrPredFunc( + &z2_wrapper_hbd, + &z2_wrapper_hbd, + AOM_BITS_8, kZ2Start), + DrPredFunc( + &z2_wrapper_hbd, + &z2_wrapper_hbd, + AOM_BITS_10, kZ2Start), + DrPredFunc( + &z2_wrapper_hbd, + &z2_wrapper_hbd, + AOM_BITS_12, kZ2Start), + */ + DrPredFunc( + &z3_wrapper_hbd, + &z3_wrapper_hbd, + AOM_BITS_8, kZ3Start), + DrPredFunc( + &z3_wrapper_hbd, + &z3_wrapper_hbd, + AOM_BITS_10, kZ3Start), + DrPredFunc( + &z3_wrapper_hbd, + &z3_wrapper_hbd, + AOM_BITS_12, kZ3Start))); + +TEST_P(HighbdDrPredTest, DISABLED_Speed) { + const int angles[] = { 3, 45, 87 }; + for (enable_upsample_ = 0; enable_upsample_ < 2; ++enable_upsample_) { + for (int i = 0; i < 3; ++i) { + int angle = angles[i] + start_angle_; + dx_ = av1_get_dx(angle); + dy_ = av1_get_dy(angle); + printf("enable_upsample: %d angle: %d ~~~~~~~~~~~~~~~\n", + enable_upsample_, angle); + if (dx_ && dy_) RunTest(true, false, angle); + } + } +} + +TEST_P(HighbdDrPredTest, OperationCheck) { + if (params_.tst_fn == NULL) return; + // const int angles[] = { 3, 45, 81, 87, 93, 100, 145, 187, 199, 260 }; + for (enable_upsample_ = 0; enable_upsample_ < 2; ++enable_upsample_) { + for (int angle = start_angle_; angle < stop_angle_; angle++) { + dx_ = av1_get_dx(angle); + dy_ = av1_get_dy(angle); + if (dx_ && dy_) RunTest(false, false, angle); + } + } +} + +#endif // HAVE_AVX2 + } // namespace diff --git a/third_party/aom/test/ec_test.cc b/third_party/aom/test/ec_test.cc index e6a5ea63bf..853abcbc5a 100644 --- a/third_party/aom/test/ec_test.cc +++ b/third_party/aom/test/ec_test.cc @@ -22,7 +22,6 @@ TEST(EC_TEST, random_ec_test) { int sz; int i; int ret; - unsigned int sym; unsigned int seed; unsigned char *ptr; uint32_t ptr_sz; @@ -90,6 +89,8 @@ TEST(EC_TEST, random_ec_test) { << " (Random seed: " << seed << ").\n"; for (j = 0; j < sz; j++) { int dec_method; + unsigned int sym = data[j] + 1; // Initialize sym to an invalid value. + if (CDF_SHIFT == 0) { dec_method = 3 + (rand() & 1); } else { diff --git a/third_party/aom/test/edge_detect_test.cc b/third_party/aom/test/edge_detect_test.cc new file mode 100644 index 0000000000..47466cb872 --- /dev/null +++ b/third_party/aom/test/edge_detect_test.cc @@ -0,0 +1,340 @@ +/* + * Copyright (c) 2018, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include "aom_mem/aom_mem.h" +#include "av1/encoder/rdopt.h" +#include "test/util.h" +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" + +namespace { + +using ::testing::get; +using ::testing::tuple; + +static int get_pix(uint8_t *buf, int i, bool high_bd) { + if (high_bd) { + return *CONVERT_TO_SHORTPTR(buf + i); + } else { + return buf[i]; + } +} + +/** Get the (i, j) value from the input; if i or j is outside of the width + * or height, the nearest pixel value is returned. + */ +static int get_nearest_pix(const int *buf, int w, int h, int i, int j) { + int offset = AOMMAX(AOMMIN(i, w - 1), 0) + w * AOMMAX(AOMMIN(j, h - 1), 0); + return buf[offset]; +} + +/** Given the image data, creates a new image with padded values, so an + * 8-tap filter can be convolved. The padded value is the same as the closest + * value in the image. Returns a pointer to the start of the image in the + * padded data. Must be freed with free_pad_8tap. The output will be either + * 8-bit or 16-bit, depending on the high bit-depth (high_bd) field. + */ +static uint8_t *pad_8tap_convolve(const int *data, int w, int h, bool high_bd) { + // SIMD optimizations require the width to be a multiple of 8 and the height + // to be multiples of 4. + assert(w % 8 == 0); + assert(h % 4 == 0); + // For an 8-tap filter, we need to pad with 3 lines on top and on the left, + // and 4 lines on the right and bottom, for 7 extra lines. + const int pad_w = w + 7; + const int pad_h = h + 7; + + uint8_t *dst; + if (high_bd) { + dst = + CONVERT_TO_BYTEPTR(aom_memalign(32, sizeof(uint16_t) * pad_w * pad_h)); + } else { + dst = (uint8_t *)aom_memalign(32, sizeof(uint8_t) * pad_w * pad_h); + } + for (int j = 0; j < pad_h; ++j) { + for (int i = 0; i < pad_w; ++i) { + const int v = get_nearest_pix(data, w, h, i - 3, j - 3); + if (high_bd) { + *CONVERT_TO_SHORTPTR(dst + i + j * pad_w) = v; + } else { + dst[i + j * pad_w] = v; + } + } + } + return dst + (w + 7) * 3 + 3; +} + +static int stride_8tap(int width) { return width + 7; } + +static void free_pad_8tap(uint8_t *padded, int width, bool high_bd) { + if (high_bd) { + aom_free(CONVERT_TO_SHORTPTR(padded - (width + 7) * 3 - 3)); + } else { + aom_free(padded - (width + 7) * 3 - 3); + } +} + +static uint8_t *malloc_bd(int num_entries, bool high_bd) { + const int bytes_per_entry = high_bd ? sizeof(uint16_t) : sizeof(uint8_t); + + uint8_t *buf = (uint8_t *)aom_memalign(32, bytes_per_entry * num_entries); + if (high_bd) { + return CONVERT_TO_BYTEPTR(buf); + } else { + return buf; + } +} + +static void free_bd(uint8_t *p, bool high_bd) { + if (high_bd) { + aom_free(CONVERT_TO_SHORTPTR(p)); + } else { + aom_free(p); + } +} + +class EdgeDetectBrightnessTest : + // Parameters are (brightness, width, height, high bit depth representation, + // bit depth). + public ::testing::TestWithParam > { + protected: + void SetUp() override { + // Allocate a (width by height) array of luma values in orig_. + // padded_ will be filled by the pad() call, which adds a border around + // the orig_. The output_ array has enough space for the computation. + const int brightness = GET_PARAM(0); + const int width = GET_PARAM(1); + const int height = GET_PARAM(2); + const bool high_bd = GET_PARAM(3); + + // Create the padded image of uniform brightness. + int *orig = (int *)malloc(width * height * sizeof(int)); + for (int i = 0; i < width * height; ++i) { + orig[i] = brightness; + } + input_ = pad_8tap_convolve(orig, width, height, high_bd); + free(orig); + output_ = malloc_bd(width * height, high_bd); + } + + void TearDown() override { + const int width = GET_PARAM(1); + const bool high_bd = GET_PARAM(3); + free_pad_8tap(input_, width, high_bd); + free_bd(output_, high_bd); + } + + // Skip the tests where brightness exceeds the bit-depth; we run into this + // issue because of gtest's limitation on valid combinations of test + // parameters. Also skip the tests where bit depth is greater than 8, but + // high bit depth representation is not set. + bool should_skip() const { + const int brightness = GET_PARAM(0); + const int bd = GET_PARAM(4); + if (brightness >= (1 << bd)) { + return true; + } + const bool high_bd = GET_PARAM(3); + if (bd > 8 && !high_bd) { + return true; + } + return false; + } + + uint8_t *input_; + uint8_t *output_; +}; + +TEST_P(EdgeDetectBrightnessTest, BlurUniformBrightness) { + // Some combination of parameters are non-sensical, due to limitations + // of the testing framework. Ignore these. + if (should_skip()) { + return; + } + + // For varying levels of brightness, the algorithm should + // produce the same output. + const int brightness = GET_PARAM(0); + const int width = GET_PARAM(1); + const int height = GET_PARAM(2); + const bool high_bd = GET_PARAM(3); + const int bd = GET_PARAM(4); + + gaussian_blur(input_, stride_8tap(width), width, height, output_, high_bd, + bd); + for (int i = 0; i < width * height; ++i) { + ASSERT_EQ(brightness, get_pix(output_, i, high_bd)); + } +} + +// No edges on a uniformly bright image. +TEST_P(EdgeDetectBrightnessTest, DetectUniformBrightness) { + if (should_skip()) { + return; + } + const int width = GET_PARAM(1); + const int height = GET_PARAM(2); + const bool high_bd = GET_PARAM(3); + const int bd = GET_PARAM(4); + + ASSERT_EQ(0, av1_edge_exists(input_, stride_8tap(width), width, height, + high_bd, bd)); +} + +INSTANTIATE_TEST_CASE_P(ImageBrightnessTests, EdgeDetectBrightnessTest, + ::testing::Combine( + // Brightness + ::testing::Values(0, 1, 2, 127, 128, 129, 254, 255, + 256, 511, 512, 1023, 1024, 2048, + 4095), + // Width + ::testing::Values(8, 16, 32), + // Height + ::testing::Values(4, 8, 12, 32), + // High bit depth representation + ::testing::Bool(), + // Bit depth + ::testing::Values(8, 10, 12))); + +class EdgeDetectImageTest : + // Parameters are (width, height, high bit depth representation, bit depth). + public ::testing::TestWithParam > { + protected: + // Skip the tests where bit depth is greater than 8, but high bit depth + // representation is not set (limitation of testing framework). + bool should_skip() const { + const bool high_bd = GET_PARAM(2); + const int bd = GET_PARAM(3); + return bd > 8 && !high_bd; + } +}; + +// Generate images with black on one side and white on the other. +TEST_P(EdgeDetectImageTest, BlackWhite) { + // Some combination of parameters are non-sensical, due to limitations + // of the testing framework. Ignore these. + if (should_skip()) { + return; + } + + const int width = GET_PARAM(0); + const int height = GET_PARAM(1); + const bool high_bd = GET_PARAM(2); + const int bd = GET_PARAM(3); + + const int white = (1 << bd) - 1; + int *orig = (int *)malloc(width * height * sizeof(int)); + for (int j = 0; j < height; ++j) { + for (int i = 0; i < width; ++i) { + if (i < width / 2) { + orig[i + j * width] = 0; + } else { + orig[i + j * width] = white; + } + } + } + uint8_t *padded = pad_8tap_convolve(orig, width, height, high_bd); + free(orig); + // Value should be between 556 and 560. + ASSERT_LE(556, av1_edge_exists(padded, stride_8tap(width), width, height, + high_bd, bd)); + ASSERT_GE(560, av1_edge_exists(padded, stride_8tap(width), width, height, + high_bd, bd)); + + free_pad_8tap(padded, width, high_bd); +} + +// Hardcoded blur tests. +static const int luma[32] = { 241, 147, 7, 90, 184, 103, 28, 186, + 2, 248, 49, 242, 114, 146, 127, 22, + 121, 228, 167, 108, 158, 174, 41, 168, + 214, 99, 184, 109, 114, 247, 117, 119 }; +static const uint8_t expected[] = { 161, 138, 119, 118, 123, 118, 113, 122, + 143, 140, 134, 133, 134, 126, 116, 114, + 147, 149, 145, 142, 143, 138, 126, 118, + 164, 156, 148, 144, 148, 148, 138, 126 }; + +static void hardcoded_blur_test_aux(const bool high_bd) { + const int w = 8; + const int h = 4; + for (int bd = 8; bd <= 12; bd += 2) { + // Skip the tests where bit depth is greater than 8, but high bit depth + // representation is not set. + if (bd > 8 && !high_bd) { + break; + } + uint8_t *output = malloc_bd(w * h, high_bd); + uint8_t *padded = pad_8tap_convolve(luma, w, h, high_bd); + gaussian_blur(padded, stride_8tap(w), w, h, output, high_bd, bd); + for (int i = 0; i < w * h; ++i) { + ASSERT_EQ(expected[i], get_pix(output, i, high_bd)); + } + free_pad_8tap(padded, w, high_bd); + free_bd(output, high_bd); + + // If we multiply the inputs by a constant factor, the output should not + // vary more than 0.5 * factor. + for (int c = 2; c < (1 << (bd - 8)); ++c) { + int scaled_luma[32]; + for (int i = 0; i < 32; ++i) { + scaled_luma[i] = luma[i] * c; + } + uint8_t *output = malloc_bd(w * h, high_bd); + uint8_t *padded = pad_8tap_convolve(scaled_luma, w, h, high_bd); + gaussian_blur(padded, stride_8tap(w), w, h, output, high_bd, bd); + for (int i = 0; i < w * h; ++i) { + ASSERT_GE(c / 2, abs(expected[i] * c - get_pix(output, i, high_bd))); + } + free_pad_8tap(padded, w, high_bd); + free_bd(output, high_bd); + } + } +} + +TEST(EdgeDetectImageTest, HardcodedBlurTest) { + hardcoded_blur_test_aux(false); + hardcoded_blur_test_aux(true); +} + +TEST(EdgeDetectImageTest, SobelTest) { + // Randomly generated 3x3. Compute Sobel for middle value. + const uint8_t buf[9] = { 241, 147, 7, 90, 184, 103, 28, 186, 2 }; + const int stride = 3; + bool high_bd = false; + sobel_xy result = sobel(buf, stride, 1, 1, high_bd); + ASSERT_EQ(234, result.x); + ASSERT_EQ(140, result.y); + + // Verify it works for 8-bit values in a high bit-depth buffer. + const uint16_t buf8_16[9] = { 241, 147, 7, 90, 184, 103, 28, 186, 2 }; + high_bd = true; + result = sobel(CONVERT_TO_BYTEPTR(buf8_16), stride, 1, 1, high_bd); + ASSERT_EQ(234, result.x); + ASSERT_EQ(140, result.y); + + // Verify it works for high bit-depth values as well. + const uint16_t buf16[9] = { 241, 147, 7, 90, 184, 2003, 1028, 186, 2 }; + result = sobel(CONVERT_TO_BYTEPTR(buf16), stride, 1, 1, high_bd); + ASSERT_EQ(-2566, result.x); + ASSERT_EQ(-860, result.y); +} + +INSTANTIATE_TEST_CASE_P(EdgeDetectImages, EdgeDetectImageTest, + ::testing::Combine( + // Width + ::testing::Values(8, 16, 32), + // Height + ::testing::Values(4, 8, 12, 32), + // High bit depth representation + ::testing::Bool(), + // Bit depth + ::testing::Values(8, 10, 12))); +} // namespace diff --git a/third_party/aom/test/encode_test_driver.cc b/third_party/aom/test/encode_test_driver.cc index f3d61dc36a..d06168fec7 100644 --- a/third_party/aom/test/encode_test_driver.cc +++ b/third_party/aom/test/encode_test_driver.cc @@ -9,6 +9,7 @@ * PATENTS file, you can obtain it at www.aomedia.org/license/patent. */ +#include #include #include "third_party/googletest/src/googletest/include/gtest/gtest.h" @@ -192,7 +193,7 @@ void EncoderTest::RunLoop(VideoSource *video) { cfg_.g_pass = AOM_RC_LAST_PASS; BeginPassHook(pass); - testing::internal::scoped_ptr encoder( + std::unique_ptr encoder( codec_->CreateEncoder(cfg_, init_flags_, &stats_)); ASSERT_TRUE(encoder.get() != NULL); @@ -205,7 +206,7 @@ void EncoderTest::RunLoop(VideoSource *video) { ASSERT_FALSE(::testing::Test::HasFatalFailure()); - testing::internal::scoped_ptr decoder( + std::unique_ptr decoder( codec_->CreateDecoder(dec_cfg, 0 /* flags */)); #if CONFIG_AV1_DECODER if (decoder->IsAV1()) { diff --git a/third_party/aom/test/end_to_end_test.cc b/third_party/aom/test/end_to_end_test.cc index 1ac0ae9312..6ea09a6d9d 100644 --- a/third_party/aom/test/end_to_end_test.cc +++ b/third_party/aom/test/end_to_end_test.cc @@ -9,6 +9,8 @@ * PATENTS file, you can obtain it at www.aomedia.org/license/patent. */ +#include + #include "third_party/googletest/src/googletest/include/gtest/gtest.h" #include "test/codec_factory.h" @@ -51,6 +53,13 @@ typedef struct { unsigned int profile; } TestVideoParam; +std::ostream &operator<<(std::ostream &os, const TestVideoParam &test_arg) { + return os << "TestVideoParam { filename:" << test_arg.filename + << " input_bit_depth:" << test_arg.input_bit_depth + << " fmt:" << test_arg.fmt << " bit_depth:" << test_arg.bit_depth + << " profile:" << test_arg.profile << "}"; +} + const TestVideoParam kTestVectors[] = { { "park_joy_90p_8_420.y4m", 8, AOM_IMG_FMT_I420, AOM_BITS_8, 0 }, { "park_joy_90p_8_422.y4m", 8, AOM_IMG_FMT_I422, AOM_BITS_8, 2 }, @@ -120,7 +129,7 @@ class EndToEndTest virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video, ::libaom_test::Encoder *encoder) { - if (video->frame() == 1) { + if (video->frame() == 0) { encoder->Control(AV1E_SET_FRAME_PARALLEL_DECODING, 1); encoder->Control(AV1E_SET_TILE_COLUMNS, 4); encoder->Control(AOME_SET_CPUUSED, cpu_used_); @@ -155,7 +164,7 @@ class EndToEndTest init_flags_ = AOM_CODEC_USE_PSNR; if (cfg_.g_bit_depth > 8) init_flags_ |= AOM_CODEC_USE_HIGHBITDEPTH; - testing::internal::scoped_ptr video; + std::unique_ptr video; if (is_extension_y4m(test_video_param_.filename)) { video.reset(new libaom_test::Y4mVideoSource(test_video_param_.filename, 0, kFrames)); diff --git a/third_party/aom/test/error_resilience_test.cc b/third_party/aom/test/error_resilience_test.cc index 13ac0bf93d..10bcd6734c 100644 --- a/third_party/aom/test/error_resilience_test.cc +++ b/third_party/aom/test/error_resilience_test.cc @@ -145,6 +145,23 @@ class ErrorResilienceTestLarge } } + virtual void FramePktHook(const aom_codec_cx_pkt_t *pkt) { + // Check that the encode frame flags are correctly reflected + // in the output frame flags. + const int encode_flags = pkt->data.frame.flags >> 16; + if ((encode_flags & (AOM_EFLAG_NO_UPD_LAST | AOM_EFLAG_NO_UPD_GF | + AOM_EFLAG_NO_UPD_ARF)) == + (AOM_EFLAG_NO_UPD_LAST | AOM_EFLAG_NO_UPD_GF | AOM_EFLAG_NO_UPD_ARF)) { + ASSERT_TRUE(!!(pkt->data.frame.flags & AOM_FRAME_IS_DROPPABLE)); + } + if (encode_flags & AOM_EFLAG_SET_S_FRAME) { + ASSERT_TRUE(!!(pkt->data.frame.flags & AOM_FRAME_IS_SWITCH)); + } + if (encode_flags & AOM_EFLAG_ERROR_RESILIENT) { + ASSERT_TRUE(!!(pkt->data.frame.flags & AOM_FRAME_IS_ERROR_RESILIENT)); + } + } + double GetAveragePsnr() const { if (nframes_) return psnr_ / nframes_; return 0.0; @@ -342,7 +359,7 @@ TEST_P(ErrorResilienceTestLarge, DropFramesWithoutRecovery) { // Set an arbitrary set of error frames same as droppable frames. unsigned int num_droppable_frames = 3; - unsigned int droppable_frame_list[] = { 5, 10, 13 }; + unsigned int droppable_frame_list[] = { 5, 11, 13 }; SetDroppableFrames(num_droppable_frames, droppable_frame_list); SetErrorFrames(num_droppable_frames, droppable_frame_list); ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); diff --git a/third_party/aom/test/ethread_test.cc b/third_party/aom/test/ethread_test.cc index d9ac782828..306cc2f3a8 100644 --- a/third_party/aom/test/ethread_test.cc +++ b/third_party/aom/test/ethread_test.cc @@ -20,14 +20,15 @@ namespace { class AVxEncoderThreadTest - : public ::libaom_test::CodecTestWith4Params, + : public ::libaom_test::CodecTestWith5Params, public ::libaom_test::EncoderTest { protected: AVxEncoderThreadTest() : EncoderTest(GET_PARAM(0)), encoder_initialized_(false), encoding_mode_(GET_PARAM(1)), set_cpu_used_(GET_PARAM(2)), - tile_cols_(GET_PARAM(3)), tile_rows_(GET_PARAM(4)) { + tile_cols_(GET_PARAM(3)), tile_rows_(GET_PARAM(4)), + row_mt_(GET_PARAM(5)) { init_flags_ = AOM_CODEC_USE_PSNR; aom_codec_dec_cfg_t cfg = aom_codec_dec_cfg_t(); cfg.w = 1280; @@ -119,87 +120,87 @@ class AVxEncoderThreadTest "niklas_640_480_30.yuv", AOM_IMG_FMT_I420, 640, 480, 30, 1, 15, 21); cfg_.rc_target_bitrate = 1000; - // Encode using single thread. - row_mt_ = 0; - cfg_.g_threads = 1; - init_flags_ = AOM_CODEC_USE_PSNR; - ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); - std::vector single_thr_size_enc; - std::vector single_thr_md5_enc; - std::vector single_thr_md5_dec; - single_thr_size_enc = size_enc_; - single_thr_md5_enc = md5_enc_; - single_thr_md5_dec = md5_dec_; - size_enc_.clear(); - md5_enc_.clear(); - md5_dec_.clear(); + if (row_mt_ == 0) { + // Encode using single thread. + cfg_.g_threads = 1; + init_flags_ = AOM_CODEC_USE_PSNR; + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + std::vector single_thr_size_enc; + std::vector single_thr_md5_enc; + std::vector single_thr_md5_dec; + single_thr_size_enc = size_enc_; + single_thr_md5_enc = md5_enc_; + single_thr_md5_dec = md5_dec_; + size_enc_.clear(); + md5_enc_.clear(); + md5_dec_.clear(); - // Encode using multiple threads. - cfg_.g_threads = 4; - ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); - std::vector multi_thr_size_enc; - std::vector multi_thr_md5_enc; - std::vector multi_thr_md5_dec; - multi_thr_size_enc = size_enc_; - multi_thr_md5_enc = md5_enc_; - multi_thr_md5_dec = md5_dec_; - size_enc_.clear(); - md5_enc_.clear(); - md5_dec_.clear(); + // Encode using multiple threads. + cfg_.g_threads = 4; + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + std::vector multi_thr_size_enc; + std::vector multi_thr_md5_enc; + std::vector multi_thr_md5_dec; + multi_thr_size_enc = size_enc_; + multi_thr_md5_enc = md5_enc_; + multi_thr_md5_dec = md5_dec_; + size_enc_.clear(); + md5_enc_.clear(); + md5_dec_.clear(); - // Check that the vectors are equal. - ASSERT_EQ(single_thr_size_enc, multi_thr_size_enc); - ASSERT_EQ(single_thr_md5_enc, multi_thr_md5_enc); - ASSERT_EQ(single_thr_md5_dec, multi_thr_md5_dec); + // Check that the vectors are equal. + ASSERT_EQ(single_thr_size_enc, multi_thr_size_enc); + ASSERT_EQ(single_thr_md5_enc, multi_thr_md5_enc); + ASSERT_EQ(single_thr_md5_dec, multi_thr_md5_dec); + } else if (row_mt_ == 1) { + // Encode using multiple threads row-mt enabled. + cfg_.g_threads = 2; + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + std::vector multi_thr2_row_mt_size_enc; + std::vector multi_thr2_row_mt_md5_enc; + std::vector multi_thr2_row_mt_md5_dec; + multi_thr2_row_mt_size_enc = size_enc_; + multi_thr2_row_mt_md5_enc = md5_enc_; + multi_thr2_row_mt_md5_dec = md5_dec_; + size_enc_.clear(); + md5_enc_.clear(); + md5_dec_.clear(); - // Encode using multiple threads row-mt enabled. - row_mt_ = 1; - cfg_.g_threads = 2; - ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); - std::vector multi_thr2_row_mt_size_enc; - std::vector multi_thr2_row_mt_md5_enc; - std::vector multi_thr2_row_mt_md5_dec; - multi_thr2_row_mt_size_enc = size_enc_; - multi_thr2_row_mt_md5_enc = md5_enc_; - multi_thr2_row_mt_md5_dec = md5_dec_; - size_enc_.clear(); - md5_enc_.clear(); - md5_dec_.clear(); + // Disable threads=3 test for now to reduce the time so that the nightly + // test would not time out. + // cfg_.g_threads = 3; + // ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + // std::vector multi_thr3_row_mt_size_enc; + // std::vector multi_thr3_row_mt_md5_enc; + // std::vector multi_thr3_row_mt_md5_dec; + // multi_thr3_row_mt_size_enc = size_enc_; + // multi_thr3_row_mt_md5_enc = md5_enc_; + // multi_thr3_row_mt_md5_dec = md5_dec_; + // size_enc_.clear(); + // md5_enc_.clear(); + // md5_dec_.clear(); + // Check that the vectors are equal. + // ASSERT_EQ(multi_thr3_row_mt_size_enc, multi_thr2_row_mt_size_enc); + // ASSERT_EQ(multi_thr3_row_mt_md5_enc, multi_thr2_row_mt_md5_enc); + // ASSERT_EQ(multi_thr3_row_mt_md5_dec, multi_thr2_row_mt_md5_dec); - // Disable threads=3 test for now to reduce the time so that the nightly - // test would not time out. - // cfg_.g_threads = 3; - // ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); - // std::vector multi_thr3_row_mt_size_enc; - // std::vector multi_thr3_row_mt_md5_enc; - // std::vector multi_thr3_row_mt_md5_dec; - // multi_thr3_row_mt_size_enc = size_enc_; - // multi_thr3_row_mt_md5_enc = md5_enc_; - // multi_thr3_row_mt_md5_dec = md5_dec_; - // size_enc_.clear(); - // md5_enc_.clear(); - // md5_dec_.clear(); - // Check that the vectors are equal. - // ASSERT_EQ(multi_thr3_row_mt_size_enc, multi_thr2_row_mt_size_enc); - // ASSERT_EQ(multi_thr3_row_mt_md5_enc, multi_thr2_row_mt_md5_enc); - // ASSERT_EQ(multi_thr3_row_mt_md5_dec, multi_thr2_row_mt_md5_dec); + cfg_.g_threads = 4; + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + std::vector multi_thr4_row_mt_size_enc; + std::vector multi_thr4_row_mt_md5_enc; + std::vector multi_thr4_row_mt_md5_dec; + multi_thr4_row_mt_size_enc = size_enc_; + multi_thr4_row_mt_md5_enc = md5_enc_; + multi_thr4_row_mt_md5_dec = md5_dec_; + size_enc_.clear(); + md5_enc_.clear(); + md5_dec_.clear(); - cfg_.g_threads = 4; - ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); - std::vector multi_thr4_row_mt_size_enc; - std::vector multi_thr4_row_mt_md5_enc; - std::vector multi_thr4_row_mt_md5_dec; - multi_thr4_row_mt_size_enc = size_enc_; - multi_thr4_row_mt_md5_enc = md5_enc_; - multi_thr4_row_mt_md5_dec = md5_dec_; - size_enc_.clear(); - md5_enc_.clear(); - md5_dec_.clear(); - - // Check that the vectors are equal. - ASSERT_EQ(multi_thr4_row_mt_size_enc, multi_thr2_row_mt_size_enc); - ASSERT_EQ(multi_thr4_row_mt_md5_enc, multi_thr2_row_mt_md5_enc); - ASSERT_EQ(multi_thr4_row_mt_md5_dec, multi_thr2_row_mt_md5_dec); + // Check that the vectors are equal. + ASSERT_EQ(multi_thr4_row_mt_size_enc, multi_thr2_row_mt_size_enc); + ASSERT_EQ(multi_thr4_row_mt_md5_enc, multi_thr2_row_mt_md5_enc); + ASSERT_EQ(multi_thr4_row_mt_md5_dec, multi_thr2_row_mt_md5_dec); + } } bool encoder_initialized_; @@ -233,14 +234,15 @@ TEST_P(AVxEncoderThreadTestLarge, EncoderResultTest) { AV1_INSTANTIATE_TEST_CASE(AVxEncoderThreadTest, ::testing::Values(::libaom_test::kTwoPassGood), ::testing::Range(2, 4), ::testing::Values(0, 2), - ::testing::Values(0, 1)); + ::testing::Values(0, 1), ::testing::Values(0, 1)); // Test cpu_used 0 and 1. AV1_INSTANTIATE_TEST_CASE(AVxEncoderThreadTestLarge, ::testing::Values(::libaom_test::kTwoPassGood, ::libaom_test::kOnePassGood), ::testing::Range(0, 2), ::testing::Values(0, 1, 2, 6), - ::testing::Values(0, 1, 2, 6)); + ::testing::Values(0, 1, 2, 6), + ::testing::Values(0, 1)); class AVxEncoderThreadLSTest : public AVxEncoderThreadTest { virtual void SetTileSize(libaom_test::Encoder *encoder) { @@ -269,5 +271,5 @@ AV1_INSTANTIATE_TEST_CASE(AVxEncoderThreadLSTestLarge, ::testing::Values(::libaom_test::kTwoPassGood, ::libaom_test::kOnePassGood), ::testing::Range(0, 4), ::testing::Values(0, 6), - ::testing::Values(0, 6)); + ::testing::Values(0, 6), ::testing::Values(0, 1)); } // namespace diff --git a/third_party/aom/test/external_frame_buffer_test.cc b/third_party/aom/test/external_frame_buffer_test.cc index c2af059a4e..4938a64bbb 100644 --- a/third_party/aom/test/external_frame_buffer_test.cc +++ b/third_party/aom/test/external_frame_buffer_test.cc @@ -8,8 +8,9 @@ * be found in the AUTHORS file in the root of the source tree. */ +#include #include - +#include "common/tools_common.h" #include "config/aom_config.h" #include "test/codec_factory.h" #include "test/decode_test_driver.h" @@ -57,7 +58,7 @@ class ExternalFrameBufferList { // Searches the frame buffer list for a free frame buffer. Makes sure // that the frame buffer is at least |min_size| in bytes. Marks that the - // frame buffer is in use by libvpx. Finally sets |fb| to point to the + // frame buffer is in use by libaom. Finally sets |fb| to point to the // external frame buffer. Returns < 0 on an error. int GetFreeFrameBuffer(size_t min_size, aom_codec_frame_buffer_t *fb) { EXPECT_TRUE(fb != NULL); @@ -113,9 +114,9 @@ class ExternalFrameBufferList { return 0; } - // Checks that the ximage data is contained within the external frame buffer - // private data passed back in the ximage. - void CheckXImageFrameBuffer(const aom_image_t *img) { + // Checks that the aom_image_t data is contained within the external frame + // buffer private data passed back in the aom_image_t. + void CheckImageFrameBuffer(const aom_image_t *img) { if (img->fb_priv != NULL) { const struct ExternalFrameBuffer *const ext_fb = reinterpret_cast(img->fb_priv); @@ -157,7 +158,7 @@ class ExternalFrameBufferList { #if CONFIG_WEBM_IO -// Callback used by libvpx to request the application to return a frame +// Callback used by libaom to request the application to return a frame // buffer of at least |min_size| in bytes. int get_aom_frame_buffer(void *user_priv, size_t min_size, aom_codec_frame_buffer_t *fb) { @@ -166,7 +167,7 @@ int get_aom_frame_buffer(void *user_priv, size_t min_size, return fb_list->GetFreeFrameBuffer(min_size, fb); } -// Callback used by libvpx to tell the application that |fb| is not needed +// Callback used by libaom to tell the application that |fb| is not needed // anymore. int release_aom_frame_buffer(void *user_priv, aom_codec_frame_buffer_t *fb) { ExternalFrameBufferList *const fb_list = @@ -217,7 +218,7 @@ class ExternalFrameBufferMD5Test const libaom_test::CompressedVideoSource &video, libaom_test::Decoder *decoder) { if (num_buffers_ > 0 && video.frame_number() == 0) { - // Have libvpx use frame buffers we create. + // Have libaom use frame buffers we create. ASSERT_TRUE(fb_list_.CreateBufferList(num_buffers_)); ASSERT_EQ(AOM_CODEC_OK, decoder->SetFrameBufferFunctions(GetAV1FrameBuffer, @@ -243,7 +244,23 @@ class ExternalFrameBufferMD5Test expected_md5[32] = '\0'; ::libaom_test::MD5 md5_res; - md5_res.Add(&img); +#if !CONFIG_LOWBITDEPTH + const aom_img_fmt_t shifted_fmt = + (aom_img_fmt)(img.fmt & ~AOM_IMG_FMT_HIGHBITDEPTH); + if (img.bit_depth == 8 && shifted_fmt != img.fmt) { + aom_image_t *img_shifted = + aom_img_alloc(NULL, shifted_fmt, img.d_w, img.d_h, 16); + img_shifted->bit_depth = img.bit_depth; + img_shifted->monochrome = img.monochrome; + aom_img_downshift(img_shifted, &img, 0); + md5_res.Add(img_shifted); + aom_img_free(img_shifted); + } else { +#endif + md5_res.Add(&img); +#if !CONFIG_LOWBITDEPTH + } +#endif const char *const actual_md5 = md5_res.Get(); // Check md5 match. @@ -279,16 +296,16 @@ class ExternalFrameBufferMD5Test }; #if CONFIG_WEBM_IO -const char kAV1TestFile[] = "av1-1-b8-01-size-226x226.ivf"; +const char kAV1TestFile[] = "av1-1-b8-03-sizeup.mkv"; const char kAV1NonRefTestFile[] = "av1-1-b8-01-size-226x226.ivf"; -// Class for testing passing in external frame buffers to libvpx. +// Class for testing passing in external frame buffers to libaom. class ExternalFrameBufferTest : public ::testing::Test { protected: ExternalFrameBufferTest() : video_(NULL), decoder_(NULL), num_buffers_(0) {} virtual void SetUp() { - video_ = new libaom_test::IVFVideoSource(kAV1TestFile); + video_ = new libaom_test::WebMVideoSource(kAV1TestFile); ASSERT_TRUE(video_ != NULL); video_->Init(); video_->Begin(); @@ -305,7 +322,7 @@ class ExternalFrameBufferTest : public ::testing::Test { video_ = NULL; } - // Passes the external frame buffer information to libvpx. + // Passes the external frame buffer information to libaom. aom_codec_err_t SetFrameBufferFunctions( int num_buffers, aom_get_frame_buffer_cb_fn_t cb_get, aom_release_frame_buffer_cb_fn_t cb_release) { @@ -342,11 +359,11 @@ class ExternalFrameBufferTest : public ::testing::Test { // Get decompressed data while ((img = dec_iter.Next()) != NULL) { - fb_list_.CheckXImageFrameBuffer(img); + fb_list_.CheckImageFrameBuffer(img); } } - libaom_test::IVFVideoSource *video_; + libaom_test::CompressedVideoSource *video_; libaom_test::AV1Decoder *decoder_; int num_buffers_; ExternalFrameBufferList fb_list_; @@ -373,12 +390,13 @@ class ExternalFrameBufferNonRefTest : public ExternalFrameBufferTest { #endif // CONFIG_WEBM_IO // This test runs through the set of test vectors, and decodes them. -// Libvpx will call into the application to allocate a frame buffer when +// Libaom will call into the application to allocate a frame buffer when // needed. The md5 checksums are computed for each frame in the video file. // If md5 checksums match the correct md5 data, then the test is passed. // Otherwise, the test failed. -TEST_P(ExternalFrameBufferMD5Test, DISABLED_ExtFBMD5Match) { +TEST_P(ExternalFrameBufferMD5Test, ExtFBMD5Match) { const std::string filename = GET_PARAM(kVideoNameParam); + aom_codec_dec_cfg_t cfg = aom_codec_dec_cfg_t(); // Number of buffers equals #AOM_MAXIMUM_REF_BUFFERS + // #AOM_MAXIMUM_WORK_BUFFERS + four jitter buffers. @@ -388,7 +406,7 @@ TEST_P(ExternalFrameBufferMD5Test, DISABLED_ExtFBMD5Match) { set_num_buffers(num_buffers); // Open compressed video file. - testing::internal::scoped_ptr video; + std::unique_ptr video; if (filename.substr(filename.length() - 3, 3) == "ivf") { video.reset(new libaom_test::IVFVideoSource(filename)); } else { @@ -407,8 +425,12 @@ TEST_P(ExternalFrameBufferMD5Test, DISABLED_ExtFBMD5Match) { const std::string md5_filename = filename + ".md5"; OpenMD5File(md5_filename); + // Set decode config. + cfg.allow_lowbitdepth = CONFIG_LOWBITDEPTH; + set_cfg(cfg); + // Decode frame, and check the md5 matching. - ASSERT_NO_FATAL_FAILURE(RunLoop(video.get())); + ASSERT_NO_FATAL_FAILURE(RunLoop(video.get(), cfg)); } #if CONFIG_WEBM_IO @@ -434,7 +456,7 @@ TEST_F(ExternalFrameBufferTest, EightJitterBuffers) { ASSERT_EQ(AOM_CODEC_OK, DecodeRemainingFrames()); } -TEST_F(ExternalFrameBufferTest, DISABLED_NotEnoughBuffers) { +TEST_F(ExternalFrameBufferTest, NotEnoughBuffers) { // Minimum number of external frame buffers for AV1 is // #AOM_MAXIMUM_REF_BUFFERS + #AOM_MAXIMUM_WORK_BUFFERS. Most files will // only use 5 frame buffers at one time. @@ -448,7 +470,7 @@ TEST_F(ExternalFrameBufferTest, DISABLED_NotEnoughBuffers) { ASSERT_EQ(AOM_CODEC_MEM_ERROR, DecodeRemainingFrames()); } -TEST_F(ExternalFrameBufferTest, DISABLED_NoRelease) { +TEST_F(ExternalFrameBufferTest, NoRelease) { const int num_buffers = AOM_MAXIMUM_REF_BUFFERS + AOM_MAXIMUM_WORK_BUFFERS; ASSERT_EQ(AOM_CODEC_OK, SetFrameBufferFunctions(num_buffers, get_aom_frame_buffer, diff --git a/third_party/aom/test/filterintra_test.cc b/third_party/aom/test/filterintra_test.cc index 5971349406..93e26ae46b 100644 --- a/third_party/aom/test/filterintra_test.cc +++ b/third_party/aom/test/filterintra_test.cc @@ -21,8 +21,8 @@ namespace { -using ::testing::tuple; using libaom_test::ACMRandom; +using ::testing::tuple; typedef void (*Predictor)(uint8_t *dst, ptrdiff_t stride, TX_SIZE tx_size, const uint8_t *above, const uint8_t *left, int mode); diff --git a/third_party/aom/test/frame_size_tests.cc b/third_party/aom/test/frame_size_tests.cc index eaf0b8370a..1546012a30 100644 --- a/third_party/aom/test/frame_size_tests.cc +++ b/third_party/aom/test/frame_size_tests.cc @@ -35,7 +35,7 @@ class AV1FrameSizeTests : public ::testing::Test, virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video, ::libaom_test::Encoder *encoder) { - if (video->frame() == 1) { + if (video->frame() == 0) { encoder->Control(AOME_SET_CPUUSED, 7); encoder->Control(AOME_SET_ENABLEAUTOALTREF, 1); encoder->Control(AOME_SET_ARNR_MAXFRAMES, 7); diff --git a/third_party/aom/test/hiprec_convolve_test.cc b/third_party/aom/test/hiprec_convolve_test.cc index f94a0730c1..dcf85230ed 100644 --- a/third_party/aom/test/hiprec_convolve_test.cc +++ b/third_party/aom/test/hiprec_convolve_test.cc @@ -12,11 +12,11 @@ #include "third_party/googletest/src/googletest/include/gtest/gtest.h" #include "test/hiprec_convolve_test_util.h" -using ::testing::make_tuple; -using ::testing::tuple; using libaom_test::ACMRandom; using libaom_test::AV1HighbdHiprecConvolve::AV1HighbdHiprecConvolveTest; using libaom_test::AV1HiprecConvolve::AV1HiprecConvolveTest; +using ::testing::make_tuple; +using ::testing::tuple; namespace { diff --git a/third_party/aom/test/hiprec_convolve_test_util.cc b/third_party/aom/test/hiprec_convolve_test_util.cc index 2672bcec3d..f5bf56e907 100644 --- a/third_party/aom/test/hiprec_convolve_test_util.cc +++ b/third_party/aom/test/hiprec_convolve_test_util.cc @@ -31,7 +31,7 @@ static void generate_kernels(ACMRandom *rnd, InterpKernel hkernel, hkernel[2] = hkernel[4] = WIENER_FILT_TAP2_MINV + rnd->PseudoUniform(WIENER_FILT_TAP2_MAXV + 1 - WIENER_FILT_TAP2_MINV); - hkernel[3] = -(hkernel[0] + hkernel[1] + hkernel[2]); + hkernel[3] = -2 * (hkernel[0] + hkernel[1] + hkernel[2]); hkernel[7] = 0; vkernel[0] = vkernel[6] = @@ -43,7 +43,7 @@ static void generate_kernels(ACMRandom *rnd, InterpKernel hkernel, vkernel[2] = vkernel[4] = WIENER_FILT_TAP2_MINV + rnd->PseudoUniform(WIENER_FILT_TAP2_MAXV + 1 - WIENER_FILT_TAP2_MINV); - vkernel[3] = -(vkernel[0] + vkernel[1] + vkernel[2]); + vkernel[3] = -2 * (vkernel[0] + vkernel[1] + vkernel[2]); vkernel[7] = 0; } diff --git a/third_party/aom/test/horver_correlation_test.cc b/third_party/aom/test/horver_correlation_test.cc new file mode 100644 index 0000000000..72fd46ac23 --- /dev/null +++ b/third_party/aom/test/horver_correlation_test.cc @@ -0,0 +1,146 @@ +/* + * Copyright (c) 2018, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" + +#include "test/acm_random.h" +#include "test/register_state_check.h" +#include "test/util.h" + +#include "config/aom_config.h" +#include "config/aom_dsp_rtcd.h" +#include "config/av1_rtcd.h" + +#include "aom/aom_integer.h" + +using libaom_test::ACMRandom; + +namespace { +typedef void (*HorverFunc)(const int16_t *diff, int stride, int w, int h, + float *hcorr, float *vcorr); + +typedef ::testing::tuple HorverTestParam; + +class HorverTest : public ::testing::TestWithParam { + public: + virtual void SetUp() { + data_buf_ = (int16_t *)aom_malloc(MAX_SB_SQUARE * sizeof(int16_t)); + ASSERT_NE(data_buf_, nullptr); + target_func_ = GET_PARAM(0); + } + virtual void TearDown() { aom_free(data_buf_); } + void RunHorverTest(void); + void RunHorverTest_ExtremeValues(void); + void RunHorverSpeedTest(int run_times); + + private: + HorverFunc target_func_; + ACMRandom rng_; + int16_t *data_buf_; +}; + +void HorverTest::RunHorverTest(void) { + for (int block_size = 0; block_size < BLOCK_SIZES_ALL; block_size++) { + const int w = block_size_wide[block_size]; + const int h = block_size_high[block_size]; + for (int iter = 0; iter < 1000 && !HasFatalFailure(); ++iter) { + float hcorr_ref = 0.0, vcorr_ref = 0.0; + float hcorr_test = 0.0, vcorr_test = 0.0; + + for (int i = 0; i < MAX_SB_SQUARE; ++i) { + data_buf_[i] = (rng_.Rand16() % (1 << 12)) - (1 << 11); + } + + av1_get_horver_correlation_full_c(data_buf_, MAX_SB_SIZE, w, h, + &hcorr_ref, &vcorr_ref); + + target_func_(data_buf_, MAX_SB_SIZE, w, h, &hcorr_test, &vcorr_test); + + ASSERT_LE(fabs(hcorr_ref - hcorr_test), 1e-6) + << "hcorr incorrect (" << w << "x" << h << ")"; + ASSERT_LE(fabs(vcorr_ref - vcorr_test), 1e-6) + << "vcorr incorrect (" << w << "x" << h << ")"; + } + // printf("(%3dx%-3d) passed\n", w, h); + } +} + +void HorverTest::RunHorverSpeedTest(int run_times) { + for (int i = 0; i < MAX_SB_SQUARE; ++i) { + data_buf_[i] = rng_.Rand16() % (1 << 12); + } + + for (int block_size = 0; block_size < BLOCK_SIZES_ALL; block_size++) { + const int w = block_size_wide[block_size]; + const int h = block_size_high[block_size]; + float hcorr_ref = 0.0, vcorr_ref = 0.0; + float hcorr_test = 0.0, vcorr_test = 0.0; + + aom_usec_timer timer; + aom_usec_timer_start(&timer); + for (int i = 0; i < run_times; ++i) { + av1_get_horver_correlation_full_c(data_buf_, MAX_SB_SIZE, w, h, + &hcorr_ref, &vcorr_ref); + } + aom_usec_timer_mark(&timer); + const double time1 = static_cast(aom_usec_timer_elapsed(&timer)); + aom_usec_timer_start(&timer); + for (int i = 0; i < run_times; ++i) { + target_func_(data_buf_, MAX_SB_SIZE, w, h, &hcorr_test, &vcorr_test); + } + aom_usec_timer_mark(&timer); + const double time2 = static_cast(aom_usec_timer_elapsed(&timer)); + + printf("%3dx%-3d:%7.2f/%7.2fns (%3.2f)\n", w, h, time1, time2, + time1 / time2); + } +} + +void HorverTest::RunHorverTest_ExtremeValues(void) { + for (int i = 0; i < MAX_SB_SQUARE; ++i) { + // Most of get_horver_test is squaring and summing, so simply saturating + // the whole buffer is mostly likely to cause an overflow. + data_buf_[i] = (1 << 12) - 1; + } + + for (int block_size = 0; block_size < BLOCK_SIZES_ALL; block_size++) { + const int w = block_size_wide[block_size]; + const int h = block_size_high[block_size]; + float hcorr_ref = 0.0, vcorr_ref = 0.0; + float hcorr_test = 0.0, vcorr_test = 0.0; + + av1_get_horver_correlation_full_c(data_buf_, MAX_SB_SIZE, w, h, &hcorr_ref, + &vcorr_ref); + target_func_(data_buf_, MAX_SB_SIZE, w, h, &hcorr_test, &vcorr_test); + + ASSERT_LE(fabs(hcorr_ref - hcorr_test), 1e-6) << "hcorr incorrect"; + ASSERT_LE(fabs(vcorr_ref - vcorr_test), 1e-6) << "vcorr incorrect"; + } +} + +TEST_P(HorverTest, RandomValues) { RunHorverTest(); } + +TEST_P(HorverTest, ExtremeValues) { RunHorverTest_ExtremeValues(); } + +TEST_P(HorverTest, DISABLED_Speed) { RunHorverSpeedTest(100000); } + +#if HAVE_SSE4_1 +INSTANTIATE_TEST_CASE_P( + SSE4_1, HorverTest, + ::testing::Values(av1_get_horver_correlation_full_sse4_1)); +#endif // HAVE_SSE4_1 + +#if HAVE_AVX2 +INSTANTIATE_TEST_CASE_P( + AVX2, HorverTest, ::testing::Values(av1_get_horver_correlation_full_avx2)); +#endif // HAVE_AVX2 + +} // namespace diff --git a/third_party/aom/test/horz_superres_test.cc b/third_party/aom/test/horz_superres_test.cc index 973f55b668..1627684b9e 100644 --- a/third_party/aom/test/horz_superres_test.cc +++ b/third_party/aom/test/horz_superres_test.cc @@ -9,6 +9,8 @@ * PATENTS file, you can obtain it at www.aomedia.org/license/patent. */ +#include + #include "third_party/googletest/src/googletest/include/gtest/gtest.h" #include "av1/encoder/encoder.h" @@ -125,7 +127,7 @@ class HorzSuperresEndToEndTest virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video, ::libaom_test::Encoder *encoder) { - if (video->frame() == 1) { + if (video->frame() == 0) { encoder->Control(AV1E_SET_FRAME_PARALLEL_DECODING, 1); encoder->Control(AV1E_SET_TILE_COLUMNS, 4); @@ -152,7 +154,7 @@ class HorzSuperresEndToEndTest double GetPsnrThreshold() { return kPSNRThresholds[test_video_idx_]; } void DoTest() { - testing::internal::scoped_ptr video; + std::unique_ptr video; video.reset(new libaom_test::Y4mVideoSource(test_video_param_.filename, 0, test_video_param_.limit)); ASSERT_TRUE(video.get() != NULL); @@ -248,7 +250,7 @@ class HorzSuperresQThreshEndToEndTest virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video, ::libaom_test::Encoder *encoder) { - if (video->frame() == 1) { + if (video->frame() == 0) { encoder->Control(AV1E_SET_FRAME_PARALLEL_DECODING, 1); encoder->Control(AV1E_SET_TILE_COLUMNS, 0); @@ -275,7 +277,7 @@ class HorzSuperresQThreshEndToEndTest double GetPsnrThreshold() { return kPSNRThresholds[test_video_idx_]; } void DoTest() { - testing::internal::scoped_ptr video; + std::unique_ptr video; video.reset(new libaom_test::Y4mVideoSource(test_video_param_.filename, 0, test_video_param_.limit)); ASSERT_TRUE(video.get() != NULL); diff --git a/third_party/aom/test/intrapred_test.cc b/third_party/aom/test/intrapred_test.cc index 1a1c0fc421..43ab773b61 100644 --- a/third_party/aom/test/intrapred_test.cc +++ b/third_party/aom/test/intrapred_test.cc @@ -197,8 +197,8 @@ TEST_P(LowbdIntraPredTest, Bitexact) { highbd_entry(type, 32, 16, opt, bd), highbd_entry(type, 32, 32, opt, bd) #endif - // --------------------------------------------------------------------------- - // Low Bit Depth Tests +// --------------------------------------------------------------------------- +// Low Bit Depth Tests #define lowbd_entry(type, width, height, opt) \ IntraPredFunc(&aom_##type##_predictor_##width##x##height##_##opt, \ diff --git a/third_party/aom/test/invalid_file_test.cc b/third_party/aom/test/invalid_file_test.cc index 5b4f5a6c3d..0478c7e5e1 100644 --- a/third_party/aom/test/invalid_file_test.cc +++ b/third_party/aom/test/invalid_file_test.cc @@ -44,6 +44,11 @@ class InvalidFileTest : public ::libaom_test::DecoderTest, << "Result file open failed. Filename: " << res_file_name; } + virtual void DecompressedFrameHook(const aom_image_t &img, + const unsigned int /*frame_number*/) { + EXPECT_NE(img.fb_priv, nullptr); + } + virtual bool HandleDecodeResult( const aom_codec_err_t res_dec, const libaom_test::CompressedVideoSource &video, @@ -108,12 +113,22 @@ TEST_P(InvalidFileTest, ReturnCode) { RunTest(); } const DecodeParam kAV1InvalidFileTests[] = { { 1, "invalid-bug-1814.ivf" }, + { 1, "invalid-chromium-906381.ivf" }, + { 1, "invalid-oss-fuzz-9288.ivf" }, { 4, "invalid-oss-fuzz-9463.ivf" }, { 1, "invalid-oss-fuzz-9482.ivf" }, { 1, "invalid-oss-fuzz-9720.ivf" }, { 1, "invalid-oss-fuzz-10061.ivf" }, { 1, "invalid-oss-fuzz-10117-mc-buf-use-highbd.ivf" }, { 1, "invalid-oss-fuzz-10227.ivf" }, + { 1, "invalid-oss-fuzz-10389.ivf" }, + { 4, "invalid-oss-fuzz-10555.ivf" }, + { 1, "invalid-oss-fuzz-10705.ivf" }, + { 1, "invalid-oss-fuzz-10723.ivf" }, + { 1, "invalid-oss-fuzz-10779.ivf" }, + { 1, "invalid-oss-fuzz-11477.ivf" }, + { 1, "invalid-oss-fuzz-11479.ivf" }, + { 1, "invalid-oss-fuzz-11523.ivf" }, }; AV1_INSTANTIATE_TEST_CASE(InvalidFileTest, diff --git a/third_party/aom/test/lightfield_test.sh b/third_party/aom/test/lightfield_test.sh index b957a6b794..19b6934a3b 100755 --- a/third_party/aom/test/lightfield_test.sh +++ b/third_party/aom/test/lightfield_test.sh @@ -50,13 +50,14 @@ lightfield_test() { # be decoded by an AV1 decoder. local bs_decoder="${LIBAOM_BIN_PATH}/lightfield_bitstream_parsing${AOM_TEST_EXE_SUFFIX}" local tl_file="${AOM_TEST_OUTPUT_DIR}/vase_tile_list.ivf" + local tl_text_file="${LIBAOM_TEST_DATA_PATH}/vase10x10_tiles.txt" if [ ! -x "${bs_decoder}" ]; then elog "${bs_decoder} does not exist or is not executable." return 1 fi eval "${AOM_TEST_PREFIX}" "${bs_decoder}" "${lf_file}" "${tl_file}" \ - "${num_references}" ${devnull} + "${num_references}" "${tl_text_file}" ${devnull} [ -e "${tl_file}" ] || return 1 @@ -82,7 +83,7 @@ lightfield_test() { fi eval "${AOM_TEST_PREFIX}" "${ref_decoder}" "${lf_file}" "${tl_reffile}" \ - "${num_references}" ${devnull} + "${num_references}" "${tl_text_file}" ${devnull} [ -e "${tl_reffile}" ] || return 1 diff --git a/third_party/aom/test/lossless_test.cc b/third_party/aom/test/lossless_test.cc index 3f8e89c815..71ae5e72be 100644 --- a/third_party/aom/test/lossless_test.cc +++ b/third_party/aom/test/lossless_test.cc @@ -40,7 +40,7 @@ class LosslessTestLarge virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video, ::libaom_test::Encoder *encoder) { - if (video->frame() == 1) { + if (video->frame() == 0) { // Only call Control if quantizer > 0 to verify that using quantizer // alone will activate lossless if (cfg_.rc_max_quantizer > 0 || cfg_.rc_min_quantizer > 0) { diff --git a/third_party/aom/test/motion_vector_test.cc b/third_party/aom/test/motion_vector_test.cc index 27eb938930..2636c39aa2 100644 --- a/third_party/aom/test/motion_vector_test.cc +++ b/third_party/aom/test/motion_vector_test.cc @@ -9,6 +9,8 @@ * PATENTS file, you can obtain it at www.aomedia.org/license/patent. */ +#include + #include "third_party/googletest/src/googletest/include/gtest/gtest.h" #include "test/codec_factory.h" @@ -60,7 +62,7 @@ class MotionVectorTestLarge virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video, ::libaom_test::Encoder *encoder) { - if (video->frame() == 1) { + if (video->frame() == 0) { encoder->Control(AOME_SET_CPUUSED, cpu_used_); encoder->Control(AV1E_ENABLE_MOTION_VECTOR_UNIT_TEST, mv_test_mode_); if (encoding_mode_ != ::libaom_test::kRealTime) { @@ -90,7 +92,7 @@ TEST_P(MotionVectorTestLarge, OverallTest) { cfg_.g_profile = 0; init_flags_ = AOM_CODEC_USE_PSNR; - testing::internal::scoped_ptr video; + std::unique_ptr video; video.reset(new libaom_test::YUVVideoSource( "niklas_640_480_30.yuv", AOM_IMG_FMT_I420, width, height, 30, 1, 0, 3)); diff --git a/third_party/aom/test/pickrst_test.cc b/third_party/aom/test/pickrst_test.cc index 040e8e8b76..0aa49b63f9 100644 --- a/third_party/aom/test/pickrst_test.cc +++ b/third_party/aom/test/pickrst_test.cc @@ -11,19 +11,20 @@ #include "third_party/googletest/src/googletest/include/gtest/gtest.h" -#include "test/function_equivalence_test.h" #include "test/register_state_check.h" +#include "test/acm_random.h" +#include "test/util.h" #include "config/aom_config.h" #include "config/aom_dsp_rtcd.h" #include "aom/aom_integer.h" +#include "aom_ports/aom_timer.h" #include "av1/encoder/pickrst.h" -using libaom_test::FunctionEquivalenceTest; #define MAX_DATA_BLOCK 384 -namespace { +namespace pickrst_test_lowbd { static const int kIterations = 100; typedef int64_t (*lowbd_pixel_proj_error_func)( @@ -31,8 +32,6 @@ typedef int64_t (*lowbd_pixel_proj_error_func)( const uint8_t *dat8, int dat_stride, int32_t *flt0, int flt0_stride, int32_t *flt1, int flt1_stride, int xq[2], const sgr_params_type *params); -typedef libaom_test::FuncParam TestFuncs; - //////////////////////////////////////////////////////////////////////////////// // 8 bit //////////////////////////////////////////////////////////////////////////////// @@ -46,13 +45,17 @@ class PixelProjErrorTest virtual void SetUp() { target_func_ = GET_PARAM(0); src_ = (uint8_t *)(aom_malloc(MAX_DATA_BLOCK * MAX_DATA_BLOCK * - sizeof(uint8_t))); + sizeof(*src_))); + ASSERT_NE(src_, nullptr); dgd_ = (uint8_t *)(aom_malloc(MAX_DATA_BLOCK * MAX_DATA_BLOCK * - sizeof(uint8_t))); + sizeof(*dgd_))); + ASSERT_NE(dgd_, nullptr); flt0_ = (int32_t *)(aom_malloc(MAX_DATA_BLOCK * MAX_DATA_BLOCK * - sizeof(int32_t))); + sizeof(*flt0_))); + ASSERT_NE(flt0_, nullptr); flt1_ = (int32_t *)(aom_malloc(MAX_DATA_BLOCK * MAX_DATA_BLOCK * - sizeof(int32_t))); + sizeof(*flt1_))); + ASSERT_NE(flt1_, nullptr); } virtual void TearDown() { aom_free(src_); @@ -60,19 +63,19 @@ class PixelProjErrorTest aom_free(flt0_); aom_free(flt1_); } - void runPixelProjErrorTest(int32_t run_times); - void runPixelProjErrorTest_ExtremeValues(); + void RunPixelProjErrorTest(int32_t run_times); + void RunPixelProjErrorTest_ExtremeValues(); private: lowbd_pixel_proj_error_func target_func_; - ACMRandom rng_; + libaom_test::ACMRandom rng_; uint8_t *src_; uint8_t *dgd_; int32_t *flt0_; int32_t *flt1_; }; -void PixelProjErrorTest::runPixelProjErrorTest(int32_t run_times) { +void PixelProjErrorTest::RunPixelProjErrorTest(int32_t run_times) { int h_end = run_times != 1 ? 128 : (rng_.Rand16() % MAX_DATA_BLOCK) + 1; int v_end = run_times != 1 ? 128 : (rng_.Rand16() % MAX_DATA_BLOCK) + 1; const int dgd_stride = MAX_DATA_BLOCK; @@ -124,7 +127,7 @@ void PixelProjErrorTest::runPixelProjErrorTest(int32_t run_times) { } } -void PixelProjErrorTest::runPixelProjErrorTest_ExtremeValues() { +void PixelProjErrorTest::RunPixelProjErrorTest_ExtremeValues() { const int h_start = 0; int h_end = 192; const int v_start = 0; @@ -165,13 +168,13 @@ void PixelProjErrorTest::runPixelProjErrorTest_ExtremeValues() { } } -TEST_P(PixelProjErrorTest, RandomValues) { runPixelProjErrorTest(1); } +TEST_P(PixelProjErrorTest, RandomValues) { RunPixelProjErrorTest(1); } TEST_P(PixelProjErrorTest, ExtremeValues) { - runPixelProjErrorTest_ExtremeValues(); + RunPixelProjErrorTest_ExtremeValues(); } -TEST_P(PixelProjErrorTest, DISABLED_Speed) { runPixelProjErrorTest(200000); } +TEST_P(PixelProjErrorTest, DISABLED_Speed) { RunPixelProjErrorTest(200000); } #if HAVE_SSE4_1 INSTANTIATE_TEST_CASE_P(SSE4_1, PixelProjErrorTest, @@ -184,4 +187,171 @@ INSTANTIATE_TEST_CASE_P(AVX2, PixelProjErrorTest, ::testing::Values(av1_lowbd_pixel_proj_error_avx2)); #endif // HAVE_AVX2 -} // namespace +} // namespace pickrst_test_lowbd + +namespace pickrst_test_highbd { +static const int kIterations = 100; + +typedef int64_t (*highbd_pixel_proj_error_func)( + const uint8_t *src8, int width, int height, int src_stride, + const uint8_t *dat8, int dat_stride, int32_t *flt0, int flt0_stride, + int32_t *flt1, int flt1_stride, int xq[2], const sgr_params_type *params); + +//////////////////////////////////////////////////////////////////////////////// +// High bit-depth +//////////////////////////////////////////////////////////////////////////////// + +typedef ::testing::tuple + PixelProjErrorTestParam; + +class PixelProjHighbdErrorTest + : public ::testing::TestWithParam { + public: + virtual void SetUp() { + target_func_ = GET_PARAM(0); + src_ = + (uint16_t *)aom_malloc(MAX_DATA_BLOCK * MAX_DATA_BLOCK * sizeof(*src_)); + ASSERT_NE(src_, nullptr); + dgd_ = + (uint16_t *)aom_malloc(MAX_DATA_BLOCK * MAX_DATA_BLOCK * sizeof(*dgd_)); + ASSERT_NE(dgd_, nullptr); + flt0_ = + (int32_t *)aom_malloc(MAX_DATA_BLOCK * MAX_DATA_BLOCK * sizeof(*flt0_)); + ASSERT_NE(flt0_, nullptr); + flt1_ = + (int32_t *)aom_malloc(MAX_DATA_BLOCK * MAX_DATA_BLOCK * sizeof(*flt1_)); + ASSERT_NE(flt1_, nullptr); + } + virtual void TearDown() { + aom_free(src_); + aom_free(dgd_); + aom_free(flt0_); + aom_free(flt1_); + } + void RunPixelProjErrorTest(int32_t run_times); + void RunPixelProjErrorTest_ExtremeValues(); + + private: + highbd_pixel_proj_error_func target_func_; + libaom_test::ACMRandom rng_; + uint16_t *src_; + uint16_t *dgd_; + int32_t *flt0_; + int32_t *flt1_; +}; + +void PixelProjHighbdErrorTest::RunPixelProjErrorTest(int32_t run_times) { + int h_end = run_times != 1 ? 128 : (rng_.Rand16() % MAX_DATA_BLOCK) + 1; + int v_end = run_times != 1 ? 128 : (rng_.Rand16() % MAX_DATA_BLOCK) + 1; + const int dgd_stride = MAX_DATA_BLOCK; + const int src_stride = MAX_DATA_BLOCK; + const int flt0_stride = MAX_DATA_BLOCK; + const int flt1_stride = MAX_DATA_BLOCK; + sgr_params_type params; + int xq[2]; + const int iters = run_times == 1 ? kIterations : 4; + for (int iter = 0; iter < iters && !HasFatalFailure(); ++iter) { + int64_t err_ref = 0, err_test = 1; + for (int i = 0; i < MAX_DATA_BLOCK * MAX_DATA_BLOCK; ++i) { + dgd_[i] = rng_.Rand16() % (1 << 12); + src_[i] = rng_.Rand16() % (1 << 12); + flt0_[i] = rng_.Rand15Signed(); + flt1_[i] = rng_.Rand15Signed(); + } + xq[0] = rng_.Rand8() % (1 << SGRPROJ_PRJ_BITS); + xq[1] = rng_.Rand8() % (1 << SGRPROJ_PRJ_BITS); + params.r[0] = run_times == 1 ? (rng_.Rand8() % MAX_RADIUS) : (iter % 2); + params.r[1] = run_times == 1 ? (rng_.Rand8() % MAX_RADIUS) : (iter / 2); + params.s[0] = run_times == 1 ? (rng_.Rand8() % MAX_RADIUS) : (iter % 2); + params.s[1] = run_times == 1 ? (rng_.Rand8() % MAX_RADIUS) : (iter / 2); + uint8_t *dgd8 = CONVERT_TO_BYTEPTR(dgd_); + uint8_t *src8 = CONVERT_TO_BYTEPTR(src_); + + aom_usec_timer timer; + aom_usec_timer_start(&timer); + for (int i = 0; i < run_times; ++i) { + err_ref = av1_highbd_pixel_proj_error_c( + src8, h_end, v_end, src_stride, dgd8, dgd_stride, flt0_, flt0_stride, + flt1_, flt1_stride, xq, ¶ms); + } + aom_usec_timer_mark(&timer); + const double time1 = static_cast(aom_usec_timer_elapsed(&timer)); + aom_usec_timer_start(&timer); + for (int i = 0; i < run_times; ++i) { + err_test = + target_func_(src8, h_end, v_end, src_stride, dgd8, dgd_stride, flt0_, + flt0_stride, flt1_, flt1_stride, xq, ¶ms); + } + aom_usec_timer_mark(&timer); + const double time2 = static_cast(aom_usec_timer_elapsed(&timer)); + if (run_times > 10) { + printf("r0 %d r1 %d %3dx%-3d:%7.2f/%7.2fns (%3.2f)\n", params.r[0], + params.r[1], h_end, v_end, time1, time2, time1 / time2); + } + ASSERT_EQ(err_ref, err_test); + } +} + +void PixelProjHighbdErrorTest::RunPixelProjErrorTest_ExtremeValues() { + const int h_start = 0; + int h_end = 192; + const int v_start = 0; + int v_end = 192; + const int dgd_stride = MAX_DATA_BLOCK; + const int src_stride = MAX_DATA_BLOCK; + const int flt0_stride = MAX_DATA_BLOCK; + const int flt1_stride = MAX_DATA_BLOCK; + sgr_params_type params; + int xq[2]; + const int iters = kIterations; + for (int iter = 0; iter < iters && !HasFatalFailure(); ++iter) { + int64_t err_ref = 0, err_test = 1; + for (int i = 0; i < MAX_DATA_BLOCK * MAX_DATA_BLOCK; ++i) { + dgd_[i] = 0; + src_[i] = (1 << 12) - 1; + flt0_[i] = rng_.Rand15Signed(); + flt1_[i] = rng_.Rand15Signed(); + } + xq[0] = rng_.Rand8() % (1 << SGRPROJ_PRJ_BITS); + xq[1] = rng_.Rand8() % (1 << SGRPROJ_PRJ_BITS); + params.r[0] = rng_.Rand8() % MAX_RADIUS; + params.r[1] = rng_.Rand8() % MAX_RADIUS; + params.s[0] = rng_.Rand8() % MAX_RADIUS; + params.s[1] = rng_.Rand8() % MAX_RADIUS; + uint8_t *dgd8 = CONVERT_TO_BYTEPTR(dgd_); + uint8_t *src8 = CONVERT_TO_BYTEPTR(src_); + + err_ref = av1_highbd_pixel_proj_error_c( + src8, h_end - h_start, v_end - v_start, src_stride, dgd8, dgd_stride, + flt0_, flt0_stride, flt1_, flt1_stride, xq, ¶ms); + + err_test = target_func_(src8, h_end - h_start, v_end - v_start, src_stride, + dgd8, dgd_stride, flt0_, flt0_stride, flt1_, + flt1_stride, xq, ¶ms); + + ASSERT_EQ(err_ref, err_test); + } +} + +TEST_P(PixelProjHighbdErrorTest, RandomValues) { RunPixelProjErrorTest(1); } + +TEST_P(PixelProjHighbdErrorTest, ExtremeValues) { + RunPixelProjErrorTest_ExtremeValues(); +} + +TEST_P(PixelProjHighbdErrorTest, DISABLED_Speed) { + RunPixelProjErrorTest(200000); +} + +#if HAVE_SSE4_1 +INSTANTIATE_TEST_CASE_P(SSE4_1, PixelProjHighbdErrorTest, + ::testing::Values(av1_highbd_pixel_proj_error_sse4_1)); +#endif // HAVE_SSE4_1 + +#if HAVE_AVX2 + +INSTANTIATE_TEST_CASE_P(AVX2, PixelProjHighbdErrorTest, + ::testing::Values(av1_highbd_pixel_proj_error_avx2)); +#endif // HAVE_AVX2 + +} // namespace pickrst_test_highbd diff --git a/third_party/aom/test/qm_test.cc b/third_party/aom/test/qm_test.cc index c87506b41a..d1dfbb849b 100644 --- a/third_party/aom/test/qm_test.cc +++ b/third_party/aom/test/qm_test.cc @@ -33,7 +33,7 @@ class QMTest virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video, ::libaom_test::Encoder *encoder) { - if (video->frame() == 1) { + if (video->frame() == 0) { encoder->Control(AOME_SET_CPUUSED, set_cpu_used_); encoder->Control(AV1E_SET_ENABLE_QM, 1); encoder->Control(AV1E_SET_QM_MIN, qm_min_); diff --git a/third_party/aom/test/quantize_func_test.cc b/third_party/aom/test/quantize_func_test.cc index 554d0c7214..2975c1dff5 100644 --- a/third_party/aom/test/quantize_func_test.cc +++ b/third_party/aom/test/quantize_func_test.cc @@ -63,7 +63,7 @@ void highbd_quan64x64_wrapper(QUAN_PARAM_LIST) { HBD_QUAN_FUNC; } -typedef enum { TYPE_B, TYPE_DC, TYPE_FP } QuantType; +enum { TYPE_B, TYPE_DC, TYPE_FP } UENUM1BYTE(QuantType); using ::testing::tuple; typedef tuple @@ -286,19 +286,29 @@ TEST_P(QuantizeTest, DISABLED_Speed) { const int16_t *quant_shift = qtab_->quant.y_quant_shift[q]; const int16_t *dequant = qtab_->dequant.y_dequant_QTX[q]; const int kNumTests = 5000000; - aom_usec_timer timer; + aom_usec_timer timer, simd_timer; FillCoeffRandom(); aom_usec_timer_start(&timer); for (int n = 0; n < kNumTests; ++n) { - quant_(coeff_ptr, n_coeffs, zbin, round_fp, quant_fp, quant_shift, qcoeff, - dqcoeff, dequant, eob, sc->scan, sc->iscan); + quant_ref_(coeff_ptr, n_coeffs, zbin, round_fp, quant_fp, quant_shift, + qcoeff, dqcoeff, dequant, eob, sc->scan, sc->iscan); } aom_usec_timer_mark(&timer); + aom_usec_timer_start(&simd_timer); + for (int n = 0; n < kNumTests; ++n) { + quant_(coeff_ptr, n_coeffs, zbin, round_fp, quant_fp, quant_shift, qcoeff, + dqcoeff, dequant, eob, sc->scan, sc->iscan); + } + aom_usec_timer_mark(&simd_timer); + const int elapsed_time = static_cast(aom_usec_timer_elapsed(&timer)); - printf("Elapsed time: %d us\n", elapsed_time); + const int simd_elapsed_time = + static_cast(aom_usec_timer_elapsed(&simd_timer)); + printf("c_time = %d \t simd_time = %d \t Gain = %d \n", elapsed_time, + simd_elapsed_time, (elapsed_time / simd_elapsed_time)); } using ::testing::make_tuple; @@ -398,14 +408,12 @@ INSTANTIATE_TEST_CASE_P(SSE2, QuantizeTest, INSTANTIATE_TEST_CASE_P( SSSE3, QuantizeTest, ::testing::Values(make_tuple(&aom_quantize_b_c, &aom_quantize_b_ssse3, - TX_16X16, TYPE_B, AOM_BITS_8))); - -// Like libvpx, the ssse3 and avx quantize tests do not pass. -// https://bugs.chromium.org/p/webm/issues/detail?id=1448 -INSTANTIATE_TEST_CASE_P( - DISABLED_SSSE3_32x32, QuantizeTest, - ::testing::Values(make_tuple(&aom_quantize_b_32x32_c, - &aom_quantize_b_32x32_ssse3, TX_16X16, TYPE_B, + TX_16X16, TYPE_B, AOM_BITS_8), + make_tuple(&aom_quantize_b_32x32_c, + &aom_quantize_b_32x32_ssse3, TX_32X32, TYPE_B, + AOM_BITS_8), + make_tuple(&aom_quantize_b_64x64_c, + &aom_quantize_b_64x64_ssse3, TX_64X64, TYPE_B, AOM_BITS_8))); #endif // HAVE_SSSE3 && ARCH_X86_64 @@ -413,13 +421,11 @@ INSTANTIATE_TEST_CASE_P( #if HAVE_AVX && ARCH_X86_64 INSTANTIATE_TEST_CASE_P( AVX, QuantizeTest, - ::testing::Values( - make_tuple(&aom_quantize_b_c, &aom_quantize_b_avx, TX_16X16, TYPE_B, - AOM_BITS_8), - // Although these tests will not pass against _c, test them against each - // other so there is some minor checking. - make_tuple(&aom_quantize_b_32x32_ssse3, &aom_quantize_b_32x32_avx, - TX_32X32, TYPE_B, AOM_BITS_8))); + ::testing::Values(make_tuple(&aom_quantize_b_c, &aom_quantize_b_avx, + TX_16X16, TYPE_B, AOM_BITS_8), + make_tuple(&aom_quantize_b_32x32_c, + &aom_quantize_b_32x32_avx, TX_32X32, TYPE_B, + AOM_BITS_8))); #endif // HAVE_AVX && ARCH_X86_64 } // namespace diff --git a/third_party/aom/test/resize_test.cc b/third_party/aom/test/resize_test.cc index b270b8362f..34964f0a30 100644 --- a/third_party/aom/test/resize_test.cc +++ b/third_party/aom/test/resize_test.cc @@ -297,7 +297,7 @@ class ResizeInternalTestLarge : public ResizeTest { virtual void PSNRPktHook(const aom_codec_cx_pkt_t *pkt) { if (frame0_psnr_ == 0.) frame0_psnr_ = pkt->data.psnr.psnr[0]; - EXPECT_NEAR(pkt->data.psnr.psnr[0], frame0_psnr_, 2.5); + EXPECT_NEAR(pkt->data.psnr.psnr[0], frame0_psnr_, 3.0); } #if WRITE_COMPRESSED_STREAM diff --git a/third_party/aom/test/sad_test.cc b/third_party/aom/test/sad_test.cc index 845fe79da1..87dbb33afe 100644 --- a/third_party/aom/test/sad_test.cc +++ b/third_party/aom/test/sad_test.cc @@ -35,22 +35,25 @@ typedef uint32_t (*SadMxNAvgFunc)(const uint8_t *src_ptr, int src_stride, const uint8_t *second_pred); typedef ::testing::tuple SadMxNAvgParam; -typedef void (*JntCompAvgFunc)(uint8_t *comp_pred, const uint8_t *pred, - int width, int height, const uint8_t *ref, - int ref_stride, - const JNT_COMP_PARAMS *jcp_param); -typedef ::testing::tuple JntCompAvgParam; +typedef void (*DistWtdCompAvgFunc)(uint8_t *comp_pred, const uint8_t *pred, + int width, int height, const uint8_t *ref, + int ref_stride, + const DIST_WTD_COMP_PARAMS *jcp_param); +typedef ::testing::tuple DistWtdCompAvgParam; -typedef unsigned int (*JntSadMxhFunc)(const uint8_t *src_ptr, int src_stride, - const uint8_t *ref_ptr, int ref_stride, - int width, int height); -typedef ::testing::tuple JntSadMxhParam; +typedef unsigned int (*DistWtdSadMxhFunc)(const uint8_t *src_ptr, + int src_stride, + const uint8_t *ref_ptr, + int ref_stride, int width, + int height); +typedef ::testing::tuple DistWtdSadMxhParam; -typedef uint32_t (*JntSadMxNAvgFunc)(const uint8_t *src_ptr, int src_stride, - const uint8_t *ref_ptr, int ref_stride, - const uint8_t *second_pred, - const JNT_COMP_PARAMS *jcp_param); -typedef ::testing::tuple JntSadMxNAvgParam; +typedef uint32_t (*DistWtdSadMxNAvgFunc)(const uint8_t *src_ptr, int src_stride, + const uint8_t *ref_ptr, int ref_stride, + const uint8_t *second_pred, + const DIST_WTD_COMP_PARAMS *jcp_param); +typedef ::testing::tuple + DistWtdSadMxNAvgParam; typedef void (*SadMxNx4Func)(const uint8_t *src_ptr, int src_stride, const uint8_t *const ref_ptr[], int ref_stride, @@ -203,7 +206,7 @@ class SADTestBase : public ::testing::Test { return sad; } - void ReferenceJntCompAvg(int block_idx) { + void ReferenceDistWtdCompAvg(int block_idx) { const uint8_t *const reference8 = GetReference(block_idx); const uint8_t *const second_pred8 = second_pred_; uint8_t *const comp_pred8 = comp_pred_; @@ -228,7 +231,7 @@ class SADTestBase : public ::testing::Test { } } - unsigned int ReferenceJntSADavg(int block_idx) { + unsigned int ReferenceDistWtdSADavg(int block_idx) { unsigned int sad = 0; const uint8_t *const reference8 = GetReference(block_idx); const uint8_t *const source8 = source_data_; @@ -305,7 +308,7 @@ class SADTestBase : public ::testing::Test { static uint8_t *comp_pred_test_; static uint8_t *comp_pred8_test_; static uint16_t *comp_pred16_test_; - JNT_COMP_PARAMS jcp_param_; + DIST_WTD_COMP_PARAMS jcp_param_; ACMRandom rnd_; }; @@ -391,13 +394,15 @@ class SADavgTest : public ::testing::WithParamInterface, } }; -class JntCompAvgTest : public ::testing::WithParamInterface, - public SADTestBase { +class DistWtdCompAvgTest + : public ::testing::WithParamInterface, + public SADTestBase { public: - JntCompAvgTest() : SADTestBase(GET_PARAM(0), GET_PARAM(1), GET_PARAM(3)) {} + DistWtdCompAvgTest() + : SADTestBase(GET_PARAM(0), GET_PARAM(1), GET_PARAM(3)) {} protected: - void jnt_comp_avg(int block_idx) { + void dist_wtd_comp_avg(int block_idx) { const uint8_t *const reference = GetReference(block_idx); ASM_REGISTER_STATE_CHECK(GET_PARAM(2)(comp_pred_test_, second_pred_, width_, @@ -411,8 +416,8 @@ class JntCompAvgTest : public ::testing::WithParamInterface, jcp_param_.fwd_offset = quant_dist_lookup_table[j][i][0]; jcp_param_.bck_offset = quant_dist_lookup_table[j][i][1]; - ReferenceJntCompAvg(0); - jnt_comp_avg(0); + ReferenceDistWtdCompAvg(0); + dist_wtd_comp_avg(0); for (int y = 0; y < height_; ++y) for (int x = 0; x < width_; ++x) @@ -423,10 +428,10 @@ class JntCompAvgTest : public ::testing::WithParamInterface, } }; -class JntSADTest : public ::testing::WithParamInterface, - public SADTestBase { +class DistWtdSADTest : public ::testing::WithParamInterface, + public SADTestBase { public: - JntSADTest() : SADTestBase(GET_PARAM(0), GET_PARAM(1), GET_PARAM(3)) {} + DistWtdSADTest() : SADTestBase(GET_PARAM(0), GET_PARAM(1), GET_PARAM(3)) {} protected: unsigned int SAD(int block_idx) { @@ -455,13 +460,14 @@ class JntSADTest : public ::testing::WithParamInterface, } }; -class JntSADavgTest : public ::testing::WithParamInterface, - public SADTestBase { +class DistWtdSADavgTest + : public ::testing::WithParamInterface, + public SADTestBase { public: - JntSADavgTest() : SADTestBase(GET_PARAM(0), GET_PARAM(1), GET_PARAM(3)) {} + DistWtdSADavgTest() : SADTestBase(GET_PARAM(0), GET_PARAM(1), GET_PARAM(3)) {} protected: - unsigned int jnt_SAD_avg(int block_idx) { + unsigned int dist_wtd_SAD_avg(int block_idx) { unsigned int ret; const uint8_t *const reference = GetReference(block_idx); @@ -477,8 +483,8 @@ class JntSADavgTest : public ::testing::WithParamInterface, jcp_param_.fwd_offset = quant_dist_lookup_table[j][i][0]; jcp_param_.bck_offset = quant_dist_lookup_table[j][i][1]; - const unsigned int reference_sad = ReferenceJntSADavg(0); - const unsigned int exp_sad = jnt_SAD_avg(0); + const unsigned int reference_sad = ReferenceDistWtdSADavg(0); + const unsigned int exp_sad = dist_wtd_SAD_avg(0); ASSERT_EQ(reference_sad, exp_sad); } @@ -608,19 +614,19 @@ TEST_P(SADavgTest, ShortSrc) { source_stride_ = tmp_stride; } -TEST_P(JntCompAvgTest, MaxRef) { +TEST_P(DistWtdCompAvgTest, MaxRef) { FillConstant(reference_data_, reference_stride_, mask_); FillConstant(second_pred_, width_, 0); CheckCompAvg(); } -TEST_P(JntCompAvgTest, MaxSecondPred) { +TEST_P(DistWtdCompAvgTest, MaxSecondPred) { FillConstant(reference_data_, reference_stride_, 0); FillConstant(second_pred_, width_, mask_); CheckCompAvg(); } -TEST_P(JntCompAvgTest, ShortRef) { +TEST_P(DistWtdCompAvgTest, ShortRef) { const int tmp_stride = reference_stride_; reference_stride_ >>= 1; FillRandom(reference_data_, reference_stride_); @@ -629,7 +635,7 @@ TEST_P(JntCompAvgTest, ShortRef) { reference_stride_ = tmp_stride; } -TEST_P(JntCompAvgTest, UnalignedRef) { +TEST_P(DistWtdCompAvgTest, UnalignedRef) { // The reference frame, but not the source frame, may be unaligned for // certain types of searches. const int tmp_stride = reference_stride_; @@ -640,19 +646,19 @@ TEST_P(JntCompAvgTest, UnalignedRef) { reference_stride_ = tmp_stride; } -TEST_P(JntSADTest, MaxRef) { +TEST_P(DistWtdSADTest, MaxRef) { FillConstant(source_data_, source_stride_, 0); FillConstant(reference_data_, reference_stride_, mask_); CheckSAD(); } -TEST_P(JntSADTest, MaxSrc) { +TEST_P(DistWtdSADTest, MaxSrc) { FillConstant(source_data_, source_stride_, mask_); FillConstant(reference_data_, reference_stride_, 0); CheckSAD(); } -TEST_P(JntSADTest, ShortRef) { +TEST_P(DistWtdSADTest, ShortRef) { const int tmp_stride = reference_stride_; reference_stride_ >>= 1; FillRandom(source_data_, source_stride_); @@ -661,7 +667,7 @@ TEST_P(JntSADTest, ShortRef) { reference_stride_ = tmp_stride; } -TEST_P(JntSADTest, UnalignedRef) { +TEST_P(DistWtdSADTest, UnalignedRef) { // The reference frame, but not the source frame, may be unaligned for // certain types of searches. const int tmp_stride = reference_stride_; @@ -672,7 +678,7 @@ TEST_P(JntSADTest, UnalignedRef) { reference_stride_ = tmp_stride; } -TEST_P(JntSADTest, ShortSrc) { +TEST_P(DistWtdSADTest, ShortSrc) { const int tmp_stride = source_stride_; source_stride_ >>= 1; int test_count = 2000; @@ -685,20 +691,20 @@ TEST_P(JntSADTest, ShortSrc) { source_stride_ = tmp_stride; } -TEST_P(JntSADavgTest, MaxRef) { +TEST_P(DistWtdSADavgTest, MaxRef) { FillConstant(source_data_, source_stride_, 0); FillConstant(reference_data_, reference_stride_, mask_); FillConstant(second_pred_, width_, 0); CheckSAD(); } -TEST_P(JntSADavgTest, MaxSrc) { +TEST_P(DistWtdSADavgTest, MaxSrc) { FillConstant(source_data_, source_stride_, mask_); FillConstant(reference_data_, reference_stride_, 0); FillConstant(second_pred_, width_, 0); CheckSAD(); } -TEST_P(JntSADavgTest, ShortRef) { +TEST_P(DistWtdSADavgTest, ShortRef) { const int tmp_stride = reference_stride_; reference_stride_ >>= 1; FillRandom(source_data_, source_stride_); @@ -708,7 +714,7 @@ TEST_P(JntSADavgTest, ShortRef) { reference_stride_ = tmp_stride; } -TEST_P(JntSADavgTest, UnalignedRef) { +TEST_P(DistWtdSADavgTest, UnalignedRef) { // The reference frame, but not the source frame, may be unaligned for // certain types of searches. const int tmp_stride = reference_stride_; @@ -720,7 +726,7 @@ TEST_P(JntSADavgTest, UnalignedRef) { reference_stride_ = tmp_stride; } -TEST_P(JntSADavgTest, ShortSrc) { +TEST_P(DistWtdSADavgTest, ShortSrc) { const int tmp_stride = source_stride_; source_stride_ >>= 1; int test_count = 2000; @@ -947,47 +953,48 @@ const SadMxNAvgParam avg_c_tests[] = { INSTANTIATE_TEST_CASE_P(C, SADavgTest, ::testing::ValuesIn(avg_c_tests)); // TODO(chengchen): add highbd tests -const JntCompAvgParam jnt_comp_avg_c_tests[] = { - make_tuple(128, 128, &aom_jnt_comp_avg_pred_c, -1), - make_tuple(128, 64, &aom_jnt_comp_avg_pred_c, -1), - make_tuple(64, 128, &aom_jnt_comp_avg_pred_c, -1), - make_tuple(64, 64, &aom_jnt_comp_avg_pred_c, -1), - make_tuple(64, 32, &aom_jnt_comp_avg_pred_c, -1), - make_tuple(32, 64, &aom_jnt_comp_avg_pred_c, -1), - make_tuple(32, 32, &aom_jnt_comp_avg_pred_c, -1), - make_tuple(32, 16, &aom_jnt_comp_avg_pred_c, -1), - make_tuple(16, 32, &aom_jnt_comp_avg_pred_c, -1), - make_tuple(16, 16, &aom_jnt_comp_avg_pred_c, -1), - make_tuple(16, 8, &aom_jnt_comp_avg_pred_c, -1), - make_tuple(8, 16, &aom_jnt_comp_avg_pred_c, -1), - make_tuple(8, 8, &aom_jnt_comp_avg_pred_c, -1), - make_tuple(8, 4, &aom_jnt_comp_avg_pred_c, -1), - make_tuple(4, 8, &aom_jnt_comp_avg_pred_c, -1), - make_tuple(4, 4, &aom_jnt_comp_avg_pred_c, -1), +const DistWtdCompAvgParam dist_wtd_comp_avg_c_tests[] = { + make_tuple(128, 128, &aom_dist_wtd_comp_avg_pred_c, -1), + make_tuple(128, 64, &aom_dist_wtd_comp_avg_pred_c, -1), + make_tuple(64, 128, &aom_dist_wtd_comp_avg_pred_c, -1), + make_tuple(64, 64, &aom_dist_wtd_comp_avg_pred_c, -1), + make_tuple(64, 32, &aom_dist_wtd_comp_avg_pred_c, -1), + make_tuple(32, 64, &aom_dist_wtd_comp_avg_pred_c, -1), + make_tuple(32, 32, &aom_dist_wtd_comp_avg_pred_c, -1), + make_tuple(32, 16, &aom_dist_wtd_comp_avg_pred_c, -1), + make_tuple(16, 32, &aom_dist_wtd_comp_avg_pred_c, -1), + make_tuple(16, 16, &aom_dist_wtd_comp_avg_pred_c, -1), + make_tuple(16, 8, &aom_dist_wtd_comp_avg_pred_c, -1), + make_tuple(8, 16, &aom_dist_wtd_comp_avg_pred_c, -1), + make_tuple(8, 8, &aom_dist_wtd_comp_avg_pred_c, -1), + make_tuple(8, 4, &aom_dist_wtd_comp_avg_pred_c, -1), + make_tuple(4, 8, &aom_dist_wtd_comp_avg_pred_c, -1), + make_tuple(4, 4, &aom_dist_wtd_comp_avg_pred_c, -1), }; -INSTANTIATE_TEST_CASE_P(C, JntCompAvgTest, - ::testing::ValuesIn(jnt_comp_avg_c_tests)); +INSTANTIATE_TEST_CASE_P(C, DistWtdCompAvgTest, + ::testing::ValuesIn(dist_wtd_comp_avg_c_tests)); -const JntSadMxNAvgParam jnt_avg_c_tests[] = { - make_tuple(128, 128, &aom_jnt_sad128x128_avg_c, -1), - make_tuple(128, 64, &aom_jnt_sad128x64_avg_c, -1), - make_tuple(64, 128, &aom_jnt_sad64x128_avg_c, -1), - make_tuple(64, 64, &aom_jnt_sad64x64_avg_c, -1), - make_tuple(64, 32, &aom_jnt_sad64x32_avg_c, -1), - make_tuple(32, 64, &aom_jnt_sad32x64_avg_c, -1), - make_tuple(32, 32, &aom_jnt_sad32x32_avg_c, -1), - make_tuple(32, 16, &aom_jnt_sad32x16_avg_c, -1), - make_tuple(16, 32, &aom_jnt_sad16x32_avg_c, -1), - make_tuple(16, 16, &aom_jnt_sad16x16_avg_c, -1), - make_tuple(16, 8, &aom_jnt_sad16x8_avg_c, -1), - make_tuple(8, 16, &aom_jnt_sad8x16_avg_c, -1), - make_tuple(8, 8, &aom_jnt_sad8x8_avg_c, -1), - make_tuple(8, 4, &aom_jnt_sad8x4_avg_c, -1), - make_tuple(4, 8, &aom_jnt_sad4x8_avg_c, -1), - make_tuple(4, 4, &aom_jnt_sad4x4_avg_c, -1), +const DistWtdSadMxNAvgParam dist_wtd_avg_c_tests[] = { + make_tuple(128, 128, &aom_dist_wtd_sad128x128_avg_c, -1), + make_tuple(128, 64, &aom_dist_wtd_sad128x64_avg_c, -1), + make_tuple(64, 128, &aom_dist_wtd_sad64x128_avg_c, -1), + make_tuple(64, 64, &aom_dist_wtd_sad64x64_avg_c, -1), + make_tuple(64, 32, &aom_dist_wtd_sad64x32_avg_c, -1), + make_tuple(32, 64, &aom_dist_wtd_sad32x64_avg_c, -1), + make_tuple(32, 32, &aom_dist_wtd_sad32x32_avg_c, -1), + make_tuple(32, 16, &aom_dist_wtd_sad32x16_avg_c, -1), + make_tuple(16, 32, &aom_dist_wtd_sad16x32_avg_c, -1), + make_tuple(16, 16, &aom_dist_wtd_sad16x16_avg_c, -1), + make_tuple(16, 8, &aom_dist_wtd_sad16x8_avg_c, -1), + make_tuple(8, 16, &aom_dist_wtd_sad8x16_avg_c, -1), + make_tuple(8, 8, &aom_dist_wtd_sad8x8_avg_c, -1), + make_tuple(8, 4, &aom_dist_wtd_sad8x4_avg_c, -1), + make_tuple(4, 8, &aom_dist_wtd_sad4x8_avg_c, -1), + make_tuple(4, 4, &aom_dist_wtd_sad4x4_avg_c, -1), }; -INSTANTIATE_TEST_CASE_P(C, JntSADavgTest, ::testing::ValuesIn(jnt_avg_c_tests)); +INSTANTIATE_TEST_CASE_P(C, DistWtdSADavgTest, + ::testing::ValuesIn(dist_wtd_avg_c_tests)); const SadMxNx4Param x4d_c_tests[] = { make_tuple(128, 128, &aom_sad128x128x4d_c, -1), @@ -1251,7 +1258,7 @@ INSTANTIATE_TEST_CASE_P(SSE2, SADx4Test, ::testing::ValuesIn(x4d_sse2_tests)); #if HAVE_SSSE3 // Note: These are named sse2, but part of ssse3 file and only built and linked // when ssse3 is enabled. -const JntSadMxhParam jnt_sad_sse2_tests[] = { +const DistWtdSadMxhParam dist_wtd_sad_sse2_tests[] = { make_tuple(4, 4, &aom_sad4xh_sse2, -1), make_tuple(4, 8, &aom_sad4xh_sse2, -1), make_tuple(8, 4, &aom_sad8xh_sse2, -1), @@ -1275,8 +1282,8 @@ const JntSadMxhParam jnt_sad_sse2_tests[] = { make_tuple(16, 64, &aom_sad16xh_sse2, -1), make_tuple(64, 16, &aom_sad64xh_sse2, -1), }; -INSTANTIATE_TEST_CASE_P(SSE2, JntSADTest, - ::testing::ValuesIn(jnt_sad_sse2_tests)); +INSTANTIATE_TEST_CASE_P(SSE2, DistWtdSADTest, + ::testing::ValuesIn(dist_wtd_sad_sse2_tests)); #endif // HAVE_SSSE3 @@ -1285,49 +1292,49 @@ INSTANTIATE_TEST_CASE_P(SSE2, JntSADTest, #endif // HAVE_SSE3 #if HAVE_SSSE3 -const JntCompAvgParam jnt_comp_avg_ssse3_tests[] = { - make_tuple(128, 128, &aom_jnt_comp_avg_pred_ssse3, -1), - make_tuple(128, 64, &aom_jnt_comp_avg_pred_ssse3, -1), - make_tuple(64, 128, &aom_jnt_comp_avg_pred_ssse3, -1), - make_tuple(64, 64, &aom_jnt_comp_avg_pred_ssse3, -1), - make_tuple(64, 32, &aom_jnt_comp_avg_pred_ssse3, -1), - make_tuple(32, 64, &aom_jnt_comp_avg_pred_ssse3, -1), - make_tuple(32, 32, &aom_jnt_comp_avg_pred_ssse3, -1), - make_tuple(32, 16, &aom_jnt_comp_avg_pred_ssse3, -1), - make_tuple(16, 32, &aom_jnt_comp_avg_pred_ssse3, -1), - make_tuple(16, 16, &aom_jnt_comp_avg_pred_ssse3, -1), - make_tuple(16, 8, &aom_jnt_comp_avg_pred_ssse3, -1), - make_tuple(8, 16, &aom_jnt_comp_avg_pred_ssse3, -1), - make_tuple(8, 8, &aom_jnt_comp_avg_pred_ssse3, -1), - make_tuple(8, 4, &aom_jnt_comp_avg_pred_ssse3, -1), - make_tuple(4, 8, &aom_jnt_comp_avg_pred_ssse3, -1), - make_tuple(4, 4, &aom_jnt_comp_avg_pred_ssse3, -1), - make_tuple(16, 16, &aom_jnt_comp_avg_pred_ssse3, -1), +const DistWtdCompAvgParam dist_wtd_comp_avg_ssse3_tests[] = { + make_tuple(128, 128, &aom_dist_wtd_comp_avg_pred_ssse3, -1), + make_tuple(128, 64, &aom_dist_wtd_comp_avg_pred_ssse3, -1), + make_tuple(64, 128, &aom_dist_wtd_comp_avg_pred_ssse3, -1), + make_tuple(64, 64, &aom_dist_wtd_comp_avg_pred_ssse3, -1), + make_tuple(64, 32, &aom_dist_wtd_comp_avg_pred_ssse3, -1), + make_tuple(32, 64, &aom_dist_wtd_comp_avg_pred_ssse3, -1), + make_tuple(32, 32, &aom_dist_wtd_comp_avg_pred_ssse3, -1), + make_tuple(32, 16, &aom_dist_wtd_comp_avg_pred_ssse3, -1), + make_tuple(16, 32, &aom_dist_wtd_comp_avg_pred_ssse3, -1), + make_tuple(16, 16, &aom_dist_wtd_comp_avg_pred_ssse3, -1), + make_tuple(16, 8, &aom_dist_wtd_comp_avg_pred_ssse3, -1), + make_tuple(8, 16, &aom_dist_wtd_comp_avg_pred_ssse3, -1), + make_tuple(8, 8, &aom_dist_wtd_comp_avg_pred_ssse3, -1), + make_tuple(8, 4, &aom_dist_wtd_comp_avg_pred_ssse3, -1), + make_tuple(4, 8, &aom_dist_wtd_comp_avg_pred_ssse3, -1), + make_tuple(4, 4, &aom_dist_wtd_comp_avg_pred_ssse3, -1), + make_tuple(16, 16, &aom_dist_wtd_comp_avg_pred_ssse3, -1), }; -INSTANTIATE_TEST_CASE_P(SSSE3, JntCompAvgTest, - ::testing::ValuesIn(jnt_comp_avg_ssse3_tests)); +INSTANTIATE_TEST_CASE_P(SSSE3, DistWtdCompAvgTest, + ::testing::ValuesIn(dist_wtd_comp_avg_ssse3_tests)); -const JntSadMxNAvgParam jnt_avg_ssse3_tests[] = { - make_tuple(128, 128, &aom_jnt_sad128x128_avg_ssse3, -1), - make_tuple(128, 64, &aom_jnt_sad128x64_avg_ssse3, -1), - make_tuple(64, 128, &aom_jnt_sad64x128_avg_ssse3, -1), - make_tuple(64, 64, &aom_jnt_sad64x64_avg_ssse3, -1), - make_tuple(64, 32, &aom_jnt_sad64x32_avg_ssse3, -1), - make_tuple(32, 64, &aom_jnt_sad32x64_avg_ssse3, -1), - make_tuple(32, 32, &aom_jnt_sad32x32_avg_ssse3, -1), - make_tuple(32, 16, &aom_jnt_sad32x16_avg_ssse3, -1), - make_tuple(16, 32, &aom_jnt_sad16x32_avg_ssse3, -1), - make_tuple(16, 16, &aom_jnt_sad16x16_avg_ssse3, -1), - make_tuple(16, 8, &aom_jnt_sad16x8_avg_ssse3, -1), - make_tuple(8, 16, &aom_jnt_sad8x16_avg_ssse3, -1), - make_tuple(8, 8, &aom_jnt_sad8x8_avg_ssse3, -1), - make_tuple(8, 4, &aom_jnt_sad8x4_avg_ssse3, -1), - make_tuple(4, 8, &aom_jnt_sad4x8_avg_ssse3, -1), - make_tuple(4, 4, &aom_jnt_sad4x4_avg_ssse3, -1), +const DistWtdSadMxNAvgParam dist_wtd_avg_ssse3_tests[] = { + make_tuple(128, 128, &aom_dist_wtd_sad128x128_avg_ssse3, -1), + make_tuple(128, 64, &aom_dist_wtd_sad128x64_avg_ssse3, -1), + make_tuple(64, 128, &aom_dist_wtd_sad64x128_avg_ssse3, -1), + make_tuple(64, 64, &aom_dist_wtd_sad64x64_avg_ssse3, -1), + make_tuple(64, 32, &aom_dist_wtd_sad64x32_avg_ssse3, -1), + make_tuple(32, 64, &aom_dist_wtd_sad32x64_avg_ssse3, -1), + make_tuple(32, 32, &aom_dist_wtd_sad32x32_avg_ssse3, -1), + make_tuple(32, 16, &aom_dist_wtd_sad32x16_avg_ssse3, -1), + make_tuple(16, 32, &aom_dist_wtd_sad16x32_avg_ssse3, -1), + make_tuple(16, 16, &aom_dist_wtd_sad16x16_avg_ssse3, -1), + make_tuple(16, 8, &aom_dist_wtd_sad16x8_avg_ssse3, -1), + make_tuple(8, 16, &aom_dist_wtd_sad8x16_avg_ssse3, -1), + make_tuple(8, 8, &aom_dist_wtd_sad8x8_avg_ssse3, -1), + make_tuple(8, 4, &aom_dist_wtd_sad8x4_avg_ssse3, -1), + make_tuple(4, 8, &aom_dist_wtd_sad4x8_avg_ssse3, -1), + make_tuple(4, 4, &aom_dist_wtd_sad4x4_avg_ssse3, -1), }; -INSTANTIATE_TEST_CASE_P(SSSE3, JntSADavgTest, - ::testing::ValuesIn(jnt_avg_ssse3_tests)); +INSTANTIATE_TEST_CASE_P(SSSE3, DistWtdSADavgTest, + ::testing::ValuesIn(dist_wtd_avg_ssse3_tests)); #endif // HAVE_SSSE3 #if HAVE_SSE4_1 diff --git a/third_party/aom/test/selfguided_filter_test.cc b/third_party/aom/test/selfguided_filter_test.cc index d2d5c6105d..464a58f3fd 100644 --- a/third_party/aom/test/selfguided_filter_test.cc +++ b/third_party/aom/test/selfguided_filter_test.cc @@ -26,9 +26,9 @@ namespace { +using libaom_test::ACMRandom; using ::testing::make_tuple; using ::testing::tuple; -using libaom_test::ACMRandom; typedef void (*SgrFunc)(const uint8_t *dat8, int width, int height, int stride, int eps, const int *xqd, uint8_t *dst8, int dst_stride, diff --git a/third_party/aom/test/simd_cmp_impl.h b/third_party/aom/test/simd_cmp_impl.h index b98af9aade..2aa02c864f 100644 --- a/third_party/aom/test/simd_cmp_impl.h +++ b/third_party/aom/test/simd_cmp_impl.h @@ -471,7 +471,7 @@ typedef struct { #define MAP(name) \ { \ - #name, reinterpret_cast < fptr > (c_##name), \ +#name, reinterpret_cast < fptr > (c_##name), \ reinterpret_cast < fptr > (name) \ } diff --git a/third_party/aom/test/sum_squares_test.cc b/third_party/aom/test/sum_squares_test.cc index f109984986..f26a64654d 100644 --- a/third_party/aom/test/sum_squares_test.cc +++ b/third_party/aom/test/sum_squares_test.cc @@ -27,6 +27,10 @@ using libaom_test::ACMRandom; using libaom_test::FunctionEquivalenceTest; +using ::testing::Combine; +using ::testing::Range; +using ::testing::Values; +using ::testing::ValuesIn; namespace { const int kNumIterations = 10000; @@ -225,4 +229,170 @@ INSTANTIATE_TEST_CASE_P(SSE2, SumSquares1DTest, aom_sum_squares_i16_c, aom_sum_squares_i16_sse2))); #endif // HAVE_SSE2 + +typedef int64_t (*sse_func)(const uint8_t *a, int a_stride, const uint8_t *b, + int b_stride, int width, int height); +typedef libaom_test::FuncParam TestSSEFuncs; + +typedef ::testing::tuple SSETestParam; + +class SSETest : public ::testing::TestWithParam { + public: + virtual ~SSETest() {} + virtual void SetUp() { + params_ = GET_PARAM(0); + width_ = GET_PARAM(1); + isHbd_ = params_.ref_func == aom_highbd_sse_c; + rnd_.Reset(ACMRandom::DeterministicSeed()); + src_ = reinterpret_cast(aom_memalign(32, 256 * 256 * 2)); + ref_ = reinterpret_cast(aom_memalign(32, 256 * 256 * 2)); + ASSERT_TRUE(src_ != NULL); + ASSERT_TRUE(ref_ != NULL); + } + + virtual void TearDown() { + libaom_test::ClearSystemState(); + aom_free(src_); + aom_free(ref_); + } + void RunTest(int isRandom, int width, int height, int run_times); + + void GenRandomData(int width, int height, int stride) { + uint16_t *pSrc = (uint16_t *)src_; + uint16_t *pRef = (uint16_t *)ref_; + const int msb = 11; // Up to 12 bit input + const int limit = 1 << (msb + 1); + for (int ii = 0; ii < height; ii++) { + for (int jj = 0; jj < width; jj++) { + if (!isHbd_) { + src_[ii * stride + jj] = rnd_.Rand8(); + ref_[ii * stride + jj] = rnd_.Rand8(); + } else { + pSrc[ii * stride + jj] = rnd_(limit); + pRef[ii * stride + jj] = rnd_(limit); + } + } + } + } + + void GenExtremeData(int width, int height, int stride, uint8_t *data, + int16_t val) { + uint16_t *pData = (uint16_t *)data; + for (int ii = 0; ii < height; ii++) { + for (int jj = 0; jj < width; jj++) { + if (!isHbd_) { + data[ii * stride + jj] = (uint8_t)val; + } else { + pData[ii * stride + jj] = val; + } + } + } + } + + protected: + int isHbd_; + int width_; + TestSSEFuncs params_; + uint8_t *src_; + uint8_t *ref_; + ACMRandom rnd_; +}; + +void SSETest::RunTest(int isRandom, int width, int height, int run_times) { + int failed = 0; + aom_usec_timer ref_timer, test_timer; + for (int k = 0; k < 3; k++) { + int stride = 4 << rnd_(7); // Up to 256 stride + while (stride < width) { // Make sure it's valid + stride = 4 << rnd_(7); + } + if (isRandom) { + GenRandomData(width, height, stride); + } else { + const int msb = isHbd_ ? 12 : 8; // Up to 12 bit input + const int limit = (1 << msb) - 1; + if (k == 0) { + GenExtremeData(width, height, stride, src_, 0); + GenExtremeData(width, height, stride, ref_, limit); + } else { + GenExtremeData(width, height, stride, src_, limit); + GenExtremeData(width, height, stride, ref_, 0); + } + } + int64_t res_ref, res_tst; + uint8_t *pSrc = src_; + uint8_t *pRef = ref_; + if (isHbd_) { + pSrc = CONVERT_TO_BYTEPTR(src_); + pRef = CONVERT_TO_BYTEPTR(ref_); + } + res_ref = params_.ref_func(pSrc, stride, pRef, stride, width, height); + res_tst = params_.tst_func(pSrc, stride, pRef, stride, width, height); + if (run_times > 1) { + aom_usec_timer_start(&ref_timer); + for (int j = 0; j < run_times; j++) { + params_.ref_func(pSrc, stride, pRef, stride, width, height); + } + aom_usec_timer_mark(&ref_timer); + const int elapsed_time_c = + static_cast(aom_usec_timer_elapsed(&ref_timer)); + + aom_usec_timer_start(&test_timer); + for (int j = 0; j < run_times; j++) { + params_.tst_func(pSrc, stride, pRef, stride, width, height); + } + aom_usec_timer_mark(&test_timer); + const int elapsed_time_simd = + static_cast(aom_usec_timer_elapsed(&test_timer)); + + printf( + "c_time=%d \t simd_time=%d \t " + "gain=%d\n", + elapsed_time_c, elapsed_time_simd, + (elapsed_time_c / elapsed_time_simd)); + } else { + if (!failed) { + failed = res_ref != res_tst; + EXPECT_EQ(res_ref, res_tst) + << "Error:" << (isHbd_ ? "hbd " : " ") << k << " SSE Test [" + << width << "x" << height + << "] C output does not match optimized output."; + } + } + } +} + +TEST_P(SSETest, OperationCheck) { + for (int height = 4; height <= 128; height += 4) { + RunTest(1, width_, height, 1); // GenRandomData + } +} + +TEST_P(SSETest, ExtremeValues) { + for (int height = 4; height <= 128; height += 4) { + RunTest(0, width_, height, 1); + } +} + +TEST_P(SSETest, DISABLED_Speed) { + for (int height = 4; height <= 128; height += 4) { + RunTest(1, width_, height, 100); + } +} +#if HAVE_SSE4_1 +TestSSEFuncs sse_sse4[] = { TestSSEFuncs(&aom_sse_c, &aom_sse_sse4_1), + TestSSEFuncs(&aom_highbd_sse_c, + &aom_highbd_sse_sse4_1) }; +INSTANTIATE_TEST_CASE_P(SSE4_1, SSETest, + Combine(ValuesIn(sse_sse4), Range(4, 129, 4))); +#endif // HAVE_SSE4_1 + +#if HAVE_AVX2 + +TestSSEFuncs sse_avx2[] = { TestSSEFuncs(&aom_sse_c, &aom_sse_avx2), + TestSSEFuncs(&aom_highbd_sse_c, + &aom_highbd_sse_avx2) }; +INSTANTIATE_TEST_CASE_P(AVX2, SSETest, + Combine(ValuesIn(sse_avx2), Range(4, 129, 4))); +#endif // HAVE_AVX2 } // namespace diff --git a/third_party/aom/test/superframe_test.cc b/third_party/aom/test/superframe_test.cc index 7be18f72a3..2cec95aa52 100644 --- a/third_party/aom/test/superframe_test.cc +++ b/third_party/aom/test/superframe_test.cc @@ -45,7 +45,7 @@ class SuperframeTest virtual void PreEncodeFrameHook(libaom_test::VideoSource *video, libaom_test::Encoder *encoder) { - if (video->frame() == 1) { + if (video->frame() == 0) { encoder->Control(AOME_SET_ENABLEAUTOALTREF, 1); encoder->Control(AOME_SET_CPUUSED, 2); encoder->Control(AV1E_SET_TILE_COLUMNS, n_tile_cols_); diff --git a/third_party/aom/test/test-data.sha1 b/third_party/aom/test/test-data.sha1 index b6ee34701a..95342a8c38 100644 --- a/third_party/aom/test/test-data.sha1 +++ b/third_party/aom/test/test-data.sha1 @@ -2,12 +2,32 @@ d5dfb0151c9051f8c85999255645d7a23916d3c0 *hantro_collage_w352h288.yuv b87815bf86020c592ccc7a846ba2e28ec8043902 *hantro_odd.yuv 26b7f64399b84db4b4c9c915d743ec5c2619d4b9 *invalid-bug-1814.ivf d3964f9dad9f60363c81b688324d95b4ec7c8038 *invalid-bug-1814.ivf.res +09aa07e5325b3bb5462182eb30b8ecc914630740 *invalid-chromium-906381.ivf +09d2af8dd22201dd8d48e5dcfcaed281ff9422c7 *invalid-chromium-906381.ivf.res fa06784f23751d8c37be94160fb821e855199af4 *invalid-oss-fuzz-10061.ivf b055f06b9a95aaa5697fa26497b592a47843a7c8 *invalid-oss-fuzz-10061.ivf.res c9e06c4c7fb7d69fd635a1f606a5e478d60e99cf *invalid-oss-fuzz-10117-mc-buf-use-highbd.ivf 88e18e61bd2b7457b4c71ebefbdff0029c41cc04 *invalid-oss-fuzz-10117-mc-buf-use-highbd.ivf.res 91a5bedeb4832c1c2900736cc0f644bb63971bbc *invalid-oss-fuzz-10227.ivf b055f06b9a95aaa5697fa26497b592a47843a7c8 *invalid-oss-fuzz-10227.ivf.res +b2d0a29a65879436bf483d04865faca7d11cc2ee *invalid-oss-fuzz-10389.ivf +9655e6275888547ecd1f14e20e08ce4891372e76 *invalid-oss-fuzz-10389.ivf.res +11df8e9a068669c678097d460b63609d3da73828 *invalid-oss-fuzz-10555.ivf +b055f06b9a95aaa5697fa26497b592a47843a7c8 *invalid-oss-fuzz-10555.ivf.res +cf5945085fe85456a1f74bf4cc7998b88b3f4b62 *invalid-oss-fuzz-10705.ivf +758671858368ffd2a2c0727898de5661f7cf7d68 *invalid-oss-fuzz-10705.ivf.res +88e29851122cca3f336824f7fa4d9f757f91110c *invalid-oss-fuzz-10723.ivf +1af486cd2cc83ebeddc76ca7a1c512cc0ec568d5 *invalid-oss-fuzz-10723.ivf.res +0784acc8931090ec24eba752d6c27e359e68fe7d *invalid-oss-fuzz-10779.ivf +5d9474c0309b7ca09a182d888f73b37a8fe1362c *invalid-oss-fuzz-10779.ivf.res +7d37be9357f89a100ced694aee1ca5a6fad35ba9 *invalid-oss-fuzz-11477.ivf +15932651aacfc4622f0910f728f3f95e08e1753d *invalid-oss-fuzz-11477.ivf.res +1674787c38ddf82a2e5c804203f04f56a304e8e0 *invalid-oss-fuzz-11479.ivf +1af486cd2cc83ebeddc76ca7a1c512cc0ec568d5 *invalid-oss-fuzz-11479.ivf.res +b1a45514f0c59be03c9991cd04882426b9b930fa *invalid-oss-fuzz-11523.ivf +7c44ac1723c14d98bcb888fbf118c959511519ba *invalid-oss-fuzz-11523.ivf.res +ccbe4081557eb44820a0e6337c4a094421826b9a *invalid-oss-fuzz-9288.ivf +67c54283fe1a26ccf02cc991e4f9a1eea3ac5e78 *invalid-oss-fuzz-9288.ivf.res c0960f032484579f967881cc025b71cfd7a79ee1 *invalid-oss-fuzz-9463.ivf d3964f9dad9f60363c81b688324d95b4ec7c8038 *invalid-oss-fuzz-9463.ivf.res f448caf378e250b7eea4fa2d1c3cd7ef4a3211ce *invalid-oss-fuzz-9482.ivf @@ -505,3 +525,10 @@ e69e41fee40b408b6eebcc79f266a95f2ee24f9e *av1-1-b8-03-sizedown.mkv 8de81b170635d456602dc8923a8b39c534d01fa8 *av1-1-b8-03-sizeup.mkv.md5 d3ed7de0aa8c155fe35e0f5f4203240710d31383 *park_joy_90p_8_420_monochrome.y4m 5b3f0907407b809aa66b62cb080feda8c92454ca *park_joy_90p_8_420_vertical_csp.y4m +caf8b6a5f1a5bcb38afae8a54a08c4f4459aafa3 *vase10x10_tiles.txt +e14825f50ff845b8a6932c64cb254007a0b5e3a1 *av1-1-b8-22-svc-L2T1.ivf +0f75f2ac44e61fc83be70c955410fa378e433237 *av1-1-b8-22-svc-L2T1.ivf.md5 +e94687eb0e90179b3800b6d5e11eb7e9bfb34eec *av1-1-b8-22-svc-L1T2.ivf +2bc12b16385ea14323bc79607fb8dfbd7edaf8ef *av1-1-b8-22-svc-L1T2.ivf.md5 +32ef2f14ee9cb11a24a22934f4c065e926e5d236 *av1-1-b8-22-svc-L2T2.ivf +f476a10ff06d750129f8229755d51e17ff141b2a *av1-1-b8-22-svc-L2T2.ivf.md5 diff --git a/third_party/aom/test/test.cmake b/third_party/aom/test/test.cmake index b16ae14c3d..1b11373c76 100644 --- a/third_party/aom/test/test.cmake +++ b/third_party/aom/test/test.cmake @@ -170,6 +170,7 @@ if(NOT BUILD_SHARED_LIBS) "${AOM_ROOT}/test/av1_fwd_txfm2d_test.cc" "${AOM_ROOT}/test/av1_inv_txfm1d_test.cc" "${AOM_ROOT}/test/av1_inv_txfm2d_test.cc" + "${AOM_ROOT}/test/av1_nn_predict_test.cc" "${AOM_ROOT}/test/av1_round_shift_array_test.cc" "${AOM_ROOT}/test/av1_txfm_test.cc" "${AOM_ROOT}/test/av1_txfm_test.h" @@ -179,10 +180,12 @@ if(NOT BUILD_SHARED_LIBS) "${AOM_ROOT}/test/comp_avg_pred_test.cc" "${AOM_ROOT}/test/comp_avg_pred_test.h" "${AOM_ROOT}/test/comp_mask_variance_test.cc" + "${AOM_ROOT}/test/edge_detect_test.cc" "${AOM_ROOT}/test/encodetxb_test.cc" "${AOM_ROOT}/test/error_block_test.cc" "${AOM_ROOT}/test/fft_test.cc" "${AOM_ROOT}/test/fwht4x4_test.cc" + "${AOM_ROOT}/test/horver_correlation_test.cc" "${AOM_ROOT}/test/masked_sad_test.cc" "${AOM_ROOT}/test/masked_variance_test.cc" "${AOM_ROOT}/test/motion_vector_test.cc" @@ -230,13 +233,6 @@ if(ENABLE_TESTS) "make sure it's in your PATH.") endif() - if(MSVC) # Force static run time to avoid collisions with googletest. - include("${AOM_ROOT}/build/cmake/msvc_runtime.cmake") - if(BUILD_SHARED_LIBS) - set(AOM_DISABLE_GTEST_CMAKE 1) - endif() - endif() - if(BUILD_SHARED_LIBS AND APPLE) # Silence an RPATH warning. set(CMAKE_MACOSX_RPATH 1) endif() @@ -244,15 +240,16 @@ if(ENABLE_TESTS) include_directories( "${AOM_ROOT}/third_party/googletest/src/googletest/include") - if(AOM_DISABLE_GTEST_CMAKE) - include_directories("${AOM_ROOT}/third_party/googletest/src/googletest") - add_library( - gtest - STATIC - "${AOM_ROOT}/third_party/googletest/src/googletest/src/gtest-all.cc") + include_directories("${AOM_ROOT}/third_party/googletest/src/googletest") + add_library( + aom_gtest + STATIC "${AOM_ROOT}/third_party/googletest/src/googletest/src/gtest-all.cc") + if(MSVC OR WIN32) + target_compile_definitions(aom_gtest PRIVATE GTEST_OS_WINDOWS=1) + elseif(CONFIG_MULTITHREAD AND CMAKE_USE_PTHREADS_INIT) + target_compile_definitions(aom_gtest PRIVATE GTEST_HAS_PTHREAD=1) else() - add_subdirectory("${AOM_ROOT}/third_party/googletest/src/googletest" - EXCLUDE_FROM_ALL) + target_compile_definitions(aom_gtest PRIVATE GTEST_HAS_PTHREAD=0) endif() endif() @@ -304,12 +301,12 @@ function(setup_aom_test_targets) add_executable(test_intra_pred_speed ${AOM_TEST_INTRA_PRED_SPEED_SOURCES} $) target_link_libraries(test_intra_pred_speed ${AOM_LIB_LINK_TYPE} aom - gtest) + aom_gtest) list(APPEND AOM_APP_TARGETS test_intra_pred_speed) endif() endif() - target_link_libraries(test_libaom ${AOM_LIB_LINK_TYPE} aom gtest) + target_link_libraries(test_libaom ${AOM_LIB_LINK_TYPE} aom aom_gtest) if(CONFIG_LIBYUV) target_sources(test_libaom PRIVATE $) diff --git a/third_party/aom/test/test_data_util.cmake b/third_party/aom/test/test_data_util.cmake index 45c951478a..6d684cbc27 100644 --- a/third_party/aom/test/test_data_util.cmake +++ b/third_party/aom/test/test_data_util.cmake @@ -12,20 +12,6 @@ list(APPEND AOM_TEST_DATA_FILE_NAMES "hantro_collage_w352h288.yuv" "hantro_odd.yuv" - "invalid-bug-1814.ivf" - "invalid-bug-1814.ivf.res" - "invalid-oss-fuzz-10061.ivf" - "invalid-oss-fuzz-10061.ivf.res" - "invalid-oss-fuzz-10117-mc-buf-use-highbd.ivf" - "invalid-oss-fuzz-10117-mc-buf-use-highbd.ivf.res" - "invalid-oss-fuzz-10227.ivf" - "invalid-oss-fuzz-10227.ivf.res" - "invalid-oss-fuzz-9463.ivf" - "invalid-oss-fuzz-9463.ivf.res" - "invalid-oss-fuzz-9482.ivf" - "invalid-oss-fuzz-9482.ivf.res" - "invalid-oss-fuzz-9720.ivf" - "invalid-oss-fuzz-9720.ivf.res" "park_joy_90p_10_420.y4m" "park_joy_90p_10_422.y4m" "park_joy_90p_10_444.y4m" @@ -43,7 +29,8 @@ list(APPEND AOM_TEST_DATA_FILE_NAMES "rush_hour_444.y4m" "screendata.y4m" "niklas_640_480_30.yuv" - "vase10x10.yuv") + "vase10x10.yuv" + "vase10x10_tiles.txt") if(ENABLE_DECODE_PERF_TESTS AND CONFIG_AV1_ENCODER) list(APPEND AOM_TEST_DATA_FILE_NAMES "niklas_1280_720_30.yuv") @@ -512,7 +499,47 @@ if(CONFIG_AV1_DECODER) "av1-1-b8-03-sizeup.mkv" "av1-1-b8-03-sizeup.mkv.md5" "av1-1-b8-03-sizedown.mkv" - "av1-1-b8-03-sizedown.mkv.md5") + "av1-1-b8-03-sizedown.mkv.md5" + "av1-1-b8-22-svc-L2T1.ivf" + "av1-1-b8-22-svc-L2T1.ivf.md5" + "av1-1-b8-22-svc-L1T2.ivf" + "av1-1-b8-22-svc-L1T2.ivf.md5" + "av1-1-b8-22-svc-L2T2.ivf" + "av1-1-b8-22-svc-L2T2.ivf.md5" + "invalid-bug-1814.ivf" + "invalid-bug-1814.ivf.res" + "invalid-chromium-906381.ivf" + "invalid-chromium-906381.ivf.res" + "invalid-oss-fuzz-10061.ivf" + "invalid-oss-fuzz-10061.ivf.res" + "invalid-oss-fuzz-10117-mc-buf-use-highbd.ivf" + "invalid-oss-fuzz-10117-mc-buf-use-highbd.ivf.res" + "invalid-oss-fuzz-10227.ivf" + "invalid-oss-fuzz-10227.ivf.res" + "invalid-oss-fuzz-10389.ivf" + "invalid-oss-fuzz-10389.ivf.res" + "invalid-oss-fuzz-10555.ivf" + "invalid-oss-fuzz-10555.ivf.res" + "invalid-oss-fuzz-10705.ivf" + "invalid-oss-fuzz-10705.ivf.res" + "invalid-oss-fuzz-10723.ivf" + "invalid-oss-fuzz-10723.ivf.res" + "invalid-oss-fuzz-10779.ivf" + "invalid-oss-fuzz-10779.ivf.res" + "invalid-oss-fuzz-11477.ivf" + "invalid-oss-fuzz-11477.ivf.res" + "invalid-oss-fuzz-11479.ivf" + "invalid-oss-fuzz-11479.ivf.res" + "invalid-oss-fuzz-11523.ivf" + "invalid-oss-fuzz-11523.ivf.res" + "invalid-oss-fuzz-9288.ivf" + "invalid-oss-fuzz-9288.ivf.res" + "invalid-oss-fuzz-9463.ivf" + "invalid-oss-fuzz-9463.ivf.res" + "invalid-oss-fuzz-9482.ivf" + "invalid-oss-fuzz-9482.ivf.res" + "invalid-oss-fuzz-9720.ivf" + "invalid-oss-fuzz-9720.ivf.res") endif() if(ENABLE_ENCODE_PERF_TESTS AND CONFIG_AV1_ENCODER) diff --git a/third_party/aom/test/test_vector_test.cc b/third_party/aom/test/test_vector_test.cc index 286988b170..9f070b9b24 100644 --- a/third_party/aom/test/test_vector_test.cc +++ b/third_party/aom/test/test_vector_test.cc @@ -11,6 +11,7 @@ #include #include +#include #include #include #include "third_party/googletest/src/googletest/include/gtest/gtest.h" @@ -116,7 +117,7 @@ TEST_P(TestVectorTest, MD5Match) { SCOPED_TRACE(str); // Open compressed video file. - testing::internal::scoped_ptr video; + std::unique_ptr video; if (filename.substr(filename.length() - 3, 3) == "ivf") { video.reset(new libaom_test::IVFVideoSource(filename)); } else if (filename.substr(filename.length() - 4, 4) == "webm" || diff --git a/third_party/aom/test/test_vectors.cc b/third_party/aom/test/test_vectors.cc index 71e431e18b..d2f333f74b 100644 --- a/third_party/aom/test/test_vectors.cc +++ b/third_party/aom/test/test_vectors.cc @@ -132,7 +132,8 @@ const char *const kAV1TestVectors[] = { "av1-1-b8-01-size-66x32.ivf", "av1-1-b8-01-size-66x34.ivf", "av1-1-b8-01-size-66x64.ivf", "av1-1-b8-01-size-66x66.ivf", "av1-1-b8-02-allintra.ivf", "av1-1-b8-03-sizedown.mkv", - "av1-1-b8-03-sizeup.mkv" + "av1-1-b8-03-sizeup.mkv", "av1-1-b8-22-svc-L1T2.ivf", + "av1-1-b8-22-svc-L2T1.ivf", "av1-1-b8-22-svc-L2T2.ivf" }; const int kNumAV1TestVectors = NELEMENTS(kAV1TestVectors); #endif // CONFIG_AV1_DECODER diff --git a/third_party/aom/test/tile_independence_test.cc b/third_party/aom/test/tile_independence_test.cc index cf534c0c59..4f7c4a475e 100644 --- a/third_party/aom/test/tile_independence_test.cc +++ b/third_party/aom/test/tile_independence_test.cc @@ -59,7 +59,7 @@ class TileIndependenceTest virtual void PreEncodeFrameHook(libaom_test::VideoSource *video, libaom_test::Encoder *encoder) { - if (video->frame() == 1) { + if (video->frame() == 0) { encoder->Control(AV1E_SET_TILE_COLUMNS, n_tile_cols_); encoder->Control(AV1E_SET_TILE_ROWS, n_tile_rows_); SetCpuUsed(encoder); diff --git a/third_party/aom/test/variance_test.cc b/third_party/aom/test/variance_test.cc index 0df314b0fd..d57fef99a1 100644 --- a/third_party/aom/test/variance_test.cc +++ b/third_party/aom/test/variance_test.cc @@ -43,10 +43,10 @@ typedef unsigned int (*SubpixAvgVarMxNFunc)(const uint8_t *a, int a_stride, typedef unsigned int (*Get4x4SseFunc)(const uint8_t *a, int a_stride, const uint8_t *b, int b_stride); typedef unsigned int (*SumOfSquaresFunction)(const int16_t *src); -typedef unsigned int (*JntSubpixAvgVarMxNFunc)( +typedef unsigned int (*DistWtdSubpixAvgVarMxNFunc)( const uint8_t *a, int a_stride, int xoffset, int yoffset, const uint8_t *b, int b_stride, uint32_t *sse, const uint8_t *second_pred, - const JNT_COMP_PARAMS *jcp_param); + const DIST_WTD_COMP_PARAMS *jcp_param); typedef uint32_t (*ObmcSubpelVarFunc)(const uint8_t *pre, int pre_stride, int xoffset, int yoffset, const int32_t *wsrc, const int32_t *mask, @@ -216,10 +216,10 @@ static uint32_t subpel_avg_variance_ref(const uint8_t *ref, const uint8_t *src, return static_cast(sse - ((se * se) >> (l2w + l2h))); } -static uint32_t jnt_subpel_avg_variance_ref( +static uint32_t dist_wtd_subpel_avg_variance_ref( const uint8_t *ref, const uint8_t *src, const uint8_t *second_pred, int l2w, int l2h, int xoff, int yoff, uint32_t *sse_ptr, bool use_high_bit_depth, - aom_bit_depth_t bit_depth, JNT_COMP_PARAMS *jcp_param) { + aom_bit_depth_t bit_depth, DIST_WTD_COMP_PARAMS *jcp_param) { int64_t se = 0; uint64_t sse = 0; const int w = 1 << l2w; @@ -709,7 +709,7 @@ class SubpelVarianceTest uint8_t *ref_; uint8_t *sec_; TestParams params_; - JNT_COMP_PARAMS jcp_param_; + DIST_WTD_COMP_PARAMS jcp_param_; // some relay helpers bool use_high_bit_depth() const { return params_.use_high_bit_depth; } @@ -820,7 +820,7 @@ void SubpelVarianceTest::RefTest() { } template <> -void SubpelVarianceTest::RefTest() { +void SubpelVarianceTest::RefTest() { for (int x = 0; x < 8; ++x) { for (int y = 0; y < 8; ++y) { if (!use_high_bit_depth()) { @@ -849,7 +849,7 @@ void SubpelVarianceTest::RefTest() { ASM_REGISTER_STATE_CHECK(var1 = params_.func(ref_, width() + 0, x, y, src_, width(), &sse1, sec_, &jcp_param_)); - var2 = jnt_subpel_avg_variance_ref( + var2 = dist_wtd_subpel_avg_variance_ref( ref_, src_, sec_, params_.log2width, params_.log2height, x, y, &sse2, use_high_bit_depth(), params_.bit_depth, &jcp_param_); EXPECT_EQ(sse1, sse2) << "at position " << x << ", " << y; @@ -1022,7 +1022,8 @@ typedef MainTestClass AvxMseTest; typedef MainTestClass AvxVarianceTest; typedef SubpelVarianceTest AvxSubpelVarianceTest; typedef SubpelVarianceTest AvxSubpelAvgVarianceTest; -typedef SubpelVarianceTest AvxJntSubpelAvgVarianceTest; +typedef SubpelVarianceTest + AvxDistWtdSubpelAvgVarianceTest; typedef ObmcVarianceTest AvxObmcSubpelVarianceTest; TEST_P(AvxSseTest, RefSse) { RefTestSse(); } @@ -1039,7 +1040,7 @@ TEST_P(SumOfSquaresTest, Ref) { RefTest(); } TEST_P(AvxSubpelVarianceTest, Ref) { RefTest(); } TEST_P(AvxSubpelVarianceTest, ExtremeRef) { ExtremeRefTest(); } TEST_P(AvxSubpelAvgVarianceTest, Ref) { RefTest(); } -TEST_P(AvxJntSubpelAvgVarianceTest, Ref) { RefTest(); } +TEST_P(AvxDistWtdSubpelAvgVarianceTest, Ref) { RefTest(); } TEST_P(AvxObmcSubpelVarianceTest, Ref) { RefTest(); } TEST_P(AvxObmcSubpelVarianceTest, ExtremeRef) { ExtremeRefTest(); } TEST_P(AvxObmcSubpelVarianceTest, DISABLED_Speed) { SpeedTest(); } @@ -1121,36 +1122,35 @@ INSTANTIATE_TEST_CASE_P( SubpelAvgVarianceParams(2, 3, &aom_sub_pixel_avg_variance4x8_c, 0), SubpelAvgVarianceParams(2, 2, &aom_sub_pixel_avg_variance4x4_c, 0))); -typedef TestParams JntSubpelAvgVarianceParams; +typedef TestParams DistWtdSubpelAvgVarianceParams; INSTANTIATE_TEST_CASE_P( - C, AvxJntSubpelAvgVarianceTest, - ::testing::Values( - JntSubpelAvgVarianceParams(6, 6, &aom_jnt_sub_pixel_avg_variance64x64_c, - 0), - JntSubpelAvgVarianceParams(6, 5, &aom_jnt_sub_pixel_avg_variance64x32_c, - 0), - JntSubpelAvgVarianceParams(5, 6, &aom_jnt_sub_pixel_avg_variance32x64_c, - 0), - JntSubpelAvgVarianceParams(5, 5, &aom_jnt_sub_pixel_avg_variance32x32_c, - 0), - JntSubpelAvgVarianceParams(5, 4, &aom_jnt_sub_pixel_avg_variance32x16_c, - 0), - JntSubpelAvgVarianceParams(4, 5, &aom_jnt_sub_pixel_avg_variance16x32_c, - 0), - JntSubpelAvgVarianceParams(4, 4, &aom_jnt_sub_pixel_avg_variance16x16_c, - 0), - JntSubpelAvgVarianceParams(4, 3, &aom_jnt_sub_pixel_avg_variance16x8_c, - 0), - JntSubpelAvgVarianceParams(3, 4, &aom_jnt_sub_pixel_avg_variance8x16_c, - 0), - JntSubpelAvgVarianceParams(3, 3, &aom_jnt_sub_pixel_avg_variance8x8_c, - 0), - JntSubpelAvgVarianceParams(3, 2, &aom_jnt_sub_pixel_avg_variance8x4_c, - 0), - JntSubpelAvgVarianceParams(2, 3, &aom_jnt_sub_pixel_avg_variance4x8_c, - 0), - JntSubpelAvgVarianceParams(2, 2, &aom_jnt_sub_pixel_avg_variance4x4_c, - 0))); + C, AvxDistWtdSubpelAvgVarianceTest, + ::testing::Values(DistWtdSubpelAvgVarianceParams( + 6, 6, &aom_dist_wtd_sub_pixel_avg_variance64x64_c, 0), + DistWtdSubpelAvgVarianceParams( + 6, 5, &aom_dist_wtd_sub_pixel_avg_variance64x32_c, 0), + DistWtdSubpelAvgVarianceParams( + 5, 6, &aom_dist_wtd_sub_pixel_avg_variance32x64_c, 0), + DistWtdSubpelAvgVarianceParams( + 5, 5, &aom_dist_wtd_sub_pixel_avg_variance32x32_c, 0), + DistWtdSubpelAvgVarianceParams( + 5, 4, &aom_dist_wtd_sub_pixel_avg_variance32x16_c, 0), + DistWtdSubpelAvgVarianceParams( + 4, 5, &aom_dist_wtd_sub_pixel_avg_variance16x32_c, 0), + DistWtdSubpelAvgVarianceParams( + 4, 4, &aom_dist_wtd_sub_pixel_avg_variance16x16_c, 0), + DistWtdSubpelAvgVarianceParams( + 4, 3, &aom_dist_wtd_sub_pixel_avg_variance16x8_c, 0), + DistWtdSubpelAvgVarianceParams( + 3, 4, &aom_dist_wtd_sub_pixel_avg_variance8x16_c, 0), + DistWtdSubpelAvgVarianceParams( + 3, 3, &aom_dist_wtd_sub_pixel_avg_variance8x8_c, 0), + DistWtdSubpelAvgVarianceParams( + 3, 2, &aom_dist_wtd_sub_pixel_avg_variance8x4_c, 0), + DistWtdSubpelAvgVarianceParams( + 2, 3, &aom_dist_wtd_sub_pixel_avg_variance4x8_c, 0), + DistWtdSubpelAvgVarianceParams( + 2, 2, &aom_dist_wtd_sub_pixel_avg_variance4x4_c, 0))); INSTANTIATE_TEST_CASE_P( C, AvxObmcSubpelVarianceTest, @@ -1840,44 +1840,34 @@ INSTANTIATE_TEST_CASE_P( 0))); INSTANTIATE_TEST_CASE_P( - SSSE3, AvxJntSubpelAvgVarianceTest, + SSSE3, AvxDistWtdSubpelAvgVarianceTest, ::testing::Values( - JntSubpelAvgVarianceParams(6, 6, - &aom_jnt_sub_pixel_avg_variance64x64_ssse3, - 0), - JntSubpelAvgVarianceParams(6, 5, - &aom_jnt_sub_pixel_avg_variance64x32_ssse3, - 0), - JntSubpelAvgVarianceParams(5, 6, - &aom_jnt_sub_pixel_avg_variance32x64_ssse3, - 0), - JntSubpelAvgVarianceParams(5, 5, - &aom_jnt_sub_pixel_avg_variance32x32_ssse3, - 0), - JntSubpelAvgVarianceParams(5, 4, - &aom_jnt_sub_pixel_avg_variance32x16_ssse3, - 0), - JntSubpelAvgVarianceParams(4, 5, - &aom_jnt_sub_pixel_avg_variance16x32_ssse3, - 0), - JntSubpelAvgVarianceParams(4, 4, - &aom_jnt_sub_pixel_avg_variance16x16_ssse3, - 0), - JntSubpelAvgVarianceParams(4, 3, - &aom_jnt_sub_pixel_avg_variance16x8_ssse3, - 0), - JntSubpelAvgVarianceParams(3, 4, - &aom_jnt_sub_pixel_avg_variance8x16_ssse3, - 0), - JntSubpelAvgVarianceParams(3, 3, - &aom_jnt_sub_pixel_avg_variance8x8_ssse3, 0), - JntSubpelAvgVarianceParams(3, 2, - &aom_jnt_sub_pixel_avg_variance8x4_ssse3, 0), - JntSubpelAvgVarianceParams(2, 3, - &aom_jnt_sub_pixel_avg_variance4x8_ssse3, 0), - JntSubpelAvgVarianceParams(2, 2, - &aom_jnt_sub_pixel_avg_variance4x4_ssse3, - 0))); + DistWtdSubpelAvgVarianceParams( + 6, 6, &aom_dist_wtd_sub_pixel_avg_variance64x64_ssse3, 0), + DistWtdSubpelAvgVarianceParams( + 6, 5, &aom_dist_wtd_sub_pixel_avg_variance64x32_ssse3, 0), + DistWtdSubpelAvgVarianceParams( + 5, 6, &aom_dist_wtd_sub_pixel_avg_variance32x64_ssse3, 0), + DistWtdSubpelAvgVarianceParams( + 5, 5, &aom_dist_wtd_sub_pixel_avg_variance32x32_ssse3, 0), + DistWtdSubpelAvgVarianceParams( + 5, 4, &aom_dist_wtd_sub_pixel_avg_variance32x16_ssse3, 0), + DistWtdSubpelAvgVarianceParams( + 4, 5, &aom_dist_wtd_sub_pixel_avg_variance16x32_ssse3, 0), + DistWtdSubpelAvgVarianceParams( + 4, 4, &aom_dist_wtd_sub_pixel_avg_variance16x16_ssse3, 0), + DistWtdSubpelAvgVarianceParams( + 4, 3, &aom_dist_wtd_sub_pixel_avg_variance16x8_ssse3, 0), + DistWtdSubpelAvgVarianceParams( + 3, 4, &aom_dist_wtd_sub_pixel_avg_variance8x16_ssse3, 0), + DistWtdSubpelAvgVarianceParams( + 3, 3, &aom_dist_wtd_sub_pixel_avg_variance8x8_ssse3, 0), + DistWtdSubpelAvgVarianceParams( + 3, 2, &aom_dist_wtd_sub_pixel_avg_variance8x4_ssse3, 0), + DistWtdSubpelAvgVarianceParams( + 2, 3, &aom_dist_wtd_sub_pixel_avg_variance4x8_ssse3, 0), + DistWtdSubpelAvgVarianceParams( + 2, 2, &aom_dist_wtd_sub_pixel_avg_variance4x4_ssse3, 0))); #endif // HAVE_SSSE3 #if HAVE_SSE4_1 diff --git a/third_party/aom/test/warp_filter_test.cc b/third_party/aom/test/warp_filter_test.cc index 19a4e8b6a1..d7b3ec9fb2 100644 --- a/third_party/aom/test/warp_filter_test.cc +++ b/third_party/aom/test/warp_filter_test.cc @@ -10,11 +10,11 @@ */ #include "third_party/googletest/src/googletest/include/gtest/gtest.h" #include "test/warp_filter_test_util.h" -using ::testing::make_tuple; -using ::testing::tuple; using libaom_test::ACMRandom; using libaom_test::AV1HighbdWarpFilter::AV1HighbdWarpFilterTest; using libaom_test::AV1WarpFilter::AV1WarpFilterTest; +using ::testing::make_tuple; +using ::testing::tuple; namespace { diff --git a/third_party/aom/test/warp_filter_test_util.cc b/third_party/aom/test/warp_filter_test_util.cc index 69b2ed4afe..9208af84e3 100644 --- a/third_party/aom/test/warp_filter_test_util.cc +++ b/third_party/aom/test/warp_filter_test_util.cc @@ -149,7 +149,7 @@ void AV1WarpFilterTest::RunSpeedTest(warp_affine_func test_impl) { int do_average = 0; conv_params = get_conv_params_no_round(do_average, 0, dsta, out_w, 1, bd); - conv_params.use_jnt_comp_avg = 0; + conv_params.use_dist_wtd_comp_avg = 0; const int num_loops = 1000000000 / (out_w + out_h); aom_usec_timer timer; @@ -222,9 +222,9 @@ void AV1WarpFilterTest::RunCheckOutput(warp_affine_func test_impl) { conv_params = get_conv_params(0, 0, bd); } if (jj >= 4) { - conv_params.use_jnt_comp_avg = 0; + conv_params.use_dist_wtd_comp_avg = 0; } else { - conv_params.use_jnt_comp_avg = 1; + conv_params.use_dist_wtd_comp_avg = 1; conv_params.fwd_offset = quant_dist_lookup_table[ii][jj][0]; conv_params.bck_offset = quant_dist_lookup_table[ii][jj][1]; } @@ -236,9 +236,9 @@ void AV1WarpFilterTest::RunCheckOutput(warp_affine_func test_impl) { get_conv_params_no_round(do_average, 0, dstb, out_w, 1, bd); } if (jj >= 4) { - conv_params.use_jnt_comp_avg = 0; + conv_params.use_dist_wtd_comp_avg = 0; } else { - conv_params.use_jnt_comp_avg = 1; + conv_params.use_dist_wtd_comp_avg = 1; conv_params.fwd_offset = quant_dist_lookup_table[ii][jj][0]; conv_params.bck_offset = quant_dist_lookup_table[ii][jj][1]; } @@ -342,7 +342,7 @@ void AV1HighbdWarpFilterTest::RunSpeedTest(highbd_warp_affine_func test_impl) { sub_x = 0; sub_y = 0; int do_average = 0; - conv_params.use_jnt_comp_avg = 0; + conv_params.use_dist_wtd_comp_avg = 0; conv_params = get_conv_params_no_round(do_average, 0, dsta, out_w, 1, bd); const int num_loops = 1000000000 / (out_w + out_h); @@ -419,9 +419,9 @@ void AV1HighbdWarpFilterTest::RunCheckOutput( conv_params = get_conv_params(0, 0, bd); } if (jj >= 4) { - conv_params.use_jnt_comp_avg = 0; + conv_params.use_dist_wtd_comp_avg = 0; } else { - conv_params.use_jnt_comp_avg = 1; + conv_params.use_dist_wtd_comp_avg = 1; conv_params.fwd_offset = quant_dist_lookup_table[ii][jj][0]; conv_params.bck_offset = quant_dist_lookup_table[ii][jj][1]; } @@ -436,9 +436,9 @@ void AV1HighbdWarpFilterTest::RunCheckOutput( get_conv_params_no_round(do_average, 0, dstb, out_w, 1, bd); } if (jj >= 4) { - conv_params.use_jnt_comp_avg = 0; + conv_params.use_dist_wtd_comp_avg = 0; } else { - conv_params.use_jnt_comp_avg = 1; + conv_params.use_dist_wtd_comp_avg = 1; conv_params.fwd_offset = quant_dist_lookup_table[ii][jj][0]; conv_params.bck_offset = quant_dist_lookup_table[ii][jj][1]; } diff --git a/third_party/aom/test/wiener_test.cc b/third_party/aom/test/wiener_test.cc index dfec09119d..8f49af6213 100644 --- a/third_party/aom/test/wiener_test.cc +++ b/third_party/aom/test/wiener_test.cc @@ -13,32 +13,33 @@ #include "third_party/googletest/src/googletest/include/gtest/gtest.h" -#include "test/function_equivalence_test.h" #include "test/register_state_check.h" +#include "test/acm_random.h" +#include "test/util.h" #include "config/aom_config.h" #include "config/aom_dsp_rtcd.h" #include "aom/aom_integer.h" +#include "aom_ports/aom_timer.h" #include "av1/encoder/pickrst.h" #define MAX_WIENER_BLOCK 384 #define MAX_DATA_BLOCK (MAX_WIENER_BLOCK + WIENER_WIN) -using libaom_test::FunctionEquivalenceTest; -namespace { +// 8-bit-depth tests +namespace wiener_lowbd { static void compute_stats_win_opt_c(int wiener_win, const uint8_t *dgd, const uint8_t *src, int h_start, int h_end, int v_start, int v_end, int dgd_stride, - int src_stride, double *M, double *H) { + int src_stride, int64_t *M, int64_t *H) { ASSERT_TRUE(wiener_win == WIENER_WIN || wiener_win == WIENER_WIN_CHROMA); int i, j, k, l, m, n; const int pixel_count = (h_end - h_start) * (v_end - v_start); const int wiener_win2 = wiener_win * wiener_win; const int wiener_halfwin = (wiener_win >> 1); - const double avg = - find_average(dgd, h_start, h_end, v_start, v_end, dgd_stride); + uint8_t avg = find_average(dgd, h_start, h_end, v_start, v_end, dgd_stride); std::vector > M_int(wiener_win, std::vector(wiener_win, 0)); @@ -75,16 +76,16 @@ static void compute_stats_win_opt_c(int wiener_win, const uint8_t *dgd, } } - const double avg_square_sum = avg * avg * pixel_count; + const int64_t avg_square_sum = (int64_t)avg * (int64_t)avg * pixel_count; for (k = 0; k < wiener_win; k++) { for (l = 0; l < wiener_win; l++) { M[l * wiener_win + k] = - M_int[l][k] + avg_square_sum - avg * (sumX + sumY[k][l]); + M_int[l][k] + avg_square_sum - (int64_t)avg * (sumX + sumY[k][l]); for (m = 0; m < wiener_win; m++) { for (n = 0; n < wiener_win; n++) { H[(l * wiener_win + k) * wiener_win2 + m * wiener_win + n] = H_int[(l * wiener_win + k)][n * 8 + m] + avg_square_sum - - avg * (sumY[k][l] + sumY[n][m]); + (int64_t)avg * (sumY[k][l] + sumY[n][m]); } } } @@ -93,7 +94,8 @@ static void compute_stats_win_opt_c(int wiener_win, const uint8_t *dgd, void compute_stats_opt_c(int wiener_win, const uint8_t *dgd, const uint8_t *src, int h_start, int h_end, int v_start, int v_end, - int dgd_stride, int src_stride, double *M, double *H) { + int dgd_stride, int src_stride, int64_t *M, + int64_t *H) { if (wiener_win == WIENER_WIN || wiener_win == WIENER_WIN_CHROMA) { compute_stats_win_opt_c(wiener_win, dgd, src, h_start, h_end, v_start, v_end, dgd_stride, src_stride, M, H); @@ -104,13 +106,10 @@ void compute_stats_opt_c(int wiener_win, const uint8_t *dgd, const uint8_t *src, } static const int kIterations = 100; -static const double min_error = (double)(0.01); typedef void (*compute_stats_Func)(int wiener_win, const uint8_t *dgd, const uint8_t *src, int h_start, int h_end, int v_start, int v_end, int dgd_stride, - int src_stride, double *M, double *H); - -typedef libaom_test::FuncParam TestFuncs; + int src_stride, int64_t *M, int64_t *H); //////////////////////////////////////////////////////////////////////////////// // 8 bit @@ -120,24 +119,34 @@ typedef ::testing::tuple WienerTestParam; class WienerTest : public ::testing::TestWithParam { public: - virtual void SetUp() { target_func_ = GET_PARAM(0); } - void runWienerTest(const int32_t wiener_win, int32_t run_times); - void runWienerTest_ExtremeValues(const int32_t wiener_win); + virtual void SetUp() { + src_buf = (uint8_t *)aom_memalign( + 32, MAX_DATA_BLOCK * MAX_DATA_BLOCK * sizeof(*src_buf)); + dgd_buf = (uint8_t *)aom_memalign( + 32, MAX_DATA_BLOCK * MAX_DATA_BLOCK * sizeof(*dgd_buf)); + target_func_ = GET_PARAM(0); + } + virtual void TearDown() { + aom_free(src_buf); + aom_free(dgd_buf); + } + void RunWienerTest(const int32_t wiener_win, int32_t run_times); + void RunWienerTest_ExtremeValues(const int32_t wiener_win); private: compute_stats_Func target_func_; - ACMRandom rng_; + libaom_test::ACMRandom rng_; + uint8_t *src_buf; + uint8_t *dgd_buf; }; -void WienerTest::runWienerTest(const int32_t wiener_win, int32_t run_times) { +void WienerTest::RunWienerTest(const int32_t wiener_win, int32_t run_times) { const int32_t wiener_halfwin = wiener_win >> 1; const int32_t wiener_win2 = wiener_win * wiener_win; - DECLARE_ALIGNED(32, uint8_t, dgd_buf[MAX_DATA_BLOCK * MAX_DATA_BLOCK]); - DECLARE_ALIGNED(32, uint8_t, src_buf[MAX_DATA_BLOCK * MAX_DATA_BLOCK]); - DECLARE_ALIGNED(32, double, M_ref[WIENER_WIN2]); - DECLARE_ALIGNED(32, double, H_ref[WIENER_WIN2 * WIENER_WIN2]); - DECLARE_ALIGNED(32, double, M_test[WIENER_WIN2]); - DECLARE_ALIGNED(32, double, H_test[WIENER_WIN2 * WIENER_WIN2]); + DECLARE_ALIGNED(32, int64_t, M_ref[WIENER_WIN2]); + DECLARE_ALIGNED(32, int64_t, H_ref[WIENER_WIN2 * WIENER_WIN2]); + DECLARE_ALIGNED(32, int64_t, M_test[WIENER_WIN2]); + DECLARE_ALIGNED(32, int64_t, H_test[WIENER_WIN2 * WIENER_WIN2]); const int h_start = ((rng_.Rand16() % (MAX_WIENER_BLOCK / 2)) & (~7)); int h_end = run_times != 1 ? 256 : ((rng_.Rand16() % MAX_WIENER_BLOCK) & (~7)) + 8; @@ -177,19 +186,18 @@ void WienerTest::runWienerTest(const int32_t wiener_win, int32_t run_times) { } int failed = 0; for (int i = 0; i < wiener_win2; ++i) { - if (fabs(M_ref[i] - M_test[i]) > min_error) { + if (M_ref[i] != M_test[i]) { failed = 1; - printf("win %d M iter %d [%4d] ref %6.0f test %6.0f \n", wiener_win, - iter, i, M_ref[i], M_test[i]); + printf("win %d M iter %d [%4d] ref %6" PRId64 " test %6" PRId64 " \n", + wiener_win, iter, i, M_ref[i], M_test[i]); break; } } - // ASSERT_EQ(failed, 0); for (int i = 0; i < wiener_win2 * wiener_win2; ++i) { - if (fabs(H_ref[i] - H_test[i]) > min_error) { + if (H_ref[i] != H_test[i]) { failed = 1; - printf("win %d H iter %d [%4d] ref %6.0f test %6.0f \n", wiener_win, - iter, i, H_ref[i], H_test[i]); + printf("win %d H iter %d [%4d] ref %6" PRId64 " test %6" PRId64 " \n", + wiener_win, iter, i, H_ref[i], H_test[i]); break; } } @@ -197,15 +205,13 @@ void WienerTest::runWienerTest(const int32_t wiener_win, int32_t run_times) { } } -void WienerTest::runWienerTest_ExtremeValues(const int32_t wiener_win) { +void WienerTest::RunWienerTest_ExtremeValues(const int32_t wiener_win) { const int32_t wiener_halfwin = wiener_win >> 1; const int32_t wiener_win2 = wiener_win * wiener_win; - DECLARE_ALIGNED(32, uint8_t, dgd_buf[MAX_DATA_BLOCK * MAX_DATA_BLOCK]); - DECLARE_ALIGNED(32, uint8_t, src_buf[MAX_DATA_BLOCK * MAX_DATA_BLOCK]); - DECLARE_ALIGNED(32, double, M_ref[WIENER_WIN2]); - DECLARE_ALIGNED(32, double, H_ref[WIENER_WIN2 * WIENER_WIN2]); - DECLARE_ALIGNED(32, double, M_test[WIENER_WIN2]); - DECLARE_ALIGNED(32, double, H_test[WIENER_WIN2 * WIENER_WIN2]); + DECLARE_ALIGNED(32, int64_t, M_ref[WIENER_WIN2]); + DECLARE_ALIGNED(32, int64_t, H_ref[WIENER_WIN2 * WIENER_WIN2]); + DECLARE_ALIGNED(32, int64_t, M_test[WIENER_WIN2]); + DECLARE_ALIGNED(32, int64_t, H_test[WIENER_WIN2 * WIENER_WIN2]); const int h_start = 16; const int h_end = MAX_WIENER_BLOCK; const int v_start = 16; @@ -229,19 +235,18 @@ void WienerTest::runWienerTest_ExtremeValues(const int32_t wiener_win) { int failed = 0; for (int i = 0; i < wiener_win2; ++i) { - if (fabs(M_ref[i] - M_test[i]) > min_error) { + if (M_ref[i] != M_test[i]) { failed = 1; - printf("win %d M iter %d [%4d] ref %6.0f test %6.0f \n", wiener_win, - iter, i, M_ref[i], M_test[i]); + printf("win %d M iter %d [%4d] ref %6" PRId64 " test %6" PRId64 " \n", + wiener_win, iter, i, M_ref[i], M_test[i]); break; } } - // ASSERT_EQ(failed, 0); for (int i = 0; i < wiener_win2 * wiener_win2; ++i) { - if (fabs(H_ref[i] - H_test[i]) > min_error) { + if (H_ref[i] != H_test[i]) { failed = 1; - printf("win %d H iter %d [%4d] ref %6.0f test %6.0f \n", wiener_win, - iter, i, H_ref[i], H_test[i]); + printf("win %d H iter %d [%4d] ref %6" PRId64 " test %6" PRId64 " \n", + wiener_win, iter, i, H_ref[i], H_test[i]); break; } } @@ -250,18 +255,18 @@ void WienerTest::runWienerTest_ExtremeValues(const int32_t wiener_win) { } TEST_P(WienerTest, RandomValues) { - runWienerTest(WIENER_WIN, 1); - runWienerTest(WIENER_WIN_CHROMA, 1); + RunWienerTest(WIENER_WIN, 1); + RunWienerTest(WIENER_WIN_CHROMA, 1); } TEST_P(WienerTest, ExtremeValues) { - runWienerTest_ExtremeValues(WIENER_WIN); - runWienerTest_ExtremeValues(WIENER_WIN_CHROMA); + RunWienerTest_ExtremeValues(WIENER_WIN); + RunWienerTest_ExtremeValues(WIENER_WIN_CHROMA); } TEST_P(WienerTest, DISABLED_Speed) { - runWienerTest(WIENER_WIN, 200); - runWienerTest(WIENER_WIN_CHROMA, 200); + RunWienerTest(WIENER_WIN, 200); + RunWienerTest(WIENER_WIN_CHROMA, 200); } INSTANTIATE_TEST_CASE_P(C, WienerTest, ::testing::Values(compute_stats_opt_c)); @@ -277,4 +282,303 @@ INSTANTIATE_TEST_CASE_P(AVX2, WienerTest, ::testing::Values(av1_compute_stats_avx2)); #endif // HAVE_AVX2 -} // namespace +} // namespace wiener_lowbd + +// High bit-depth tests: +namespace wiener_highbd { + +static void compute_stats_highbd_win_opt_c(int wiener_win, const uint8_t *dgd8, + const uint8_t *src8, int h_start, + int h_end, int v_start, int v_end, + int dgd_stride, int src_stride, + int64_t *M, int64_t *H, + aom_bit_depth_t bit_depth) { + ASSERT_TRUE(wiener_win == WIENER_WIN || wiener_win == WIENER_WIN_CHROMA); + int i, j, k, l, m, n; + const int pixel_count = (h_end - h_start) * (v_end - v_start); + const int wiener_win2 = wiener_win * wiener_win; + const int wiener_halfwin = (wiener_win >> 1); + const uint16_t *src = CONVERT_TO_SHORTPTR(src8); + const uint16_t *dgd = CONVERT_TO_SHORTPTR(dgd8); + const uint16_t avg = + find_average_highbd(dgd, h_start, h_end, v_start, v_end, dgd_stride); + + std::vector > M_int(wiener_win, + std::vector(wiener_win, 0)); + std::vector > H_int( + wiener_win * wiener_win, std::vector(wiener_win * 8, 0)); + std::vector > sumY(wiener_win, + std::vector(wiener_win, 0)); + + memset(M, 0, sizeof(*M) * wiener_win2); + memset(H, 0, sizeof(*H) * wiener_win2 * wiener_win2); + + int64_t sumX = 0; + const uint16_t *dgd_win = dgd - wiener_halfwin * dgd_stride - wiener_halfwin; + + for (i = v_start; i < v_end; i++) { + for (j = h_start; j < h_end; j += 2) { + const uint16_t X1 = src[i * src_stride + j]; + const uint16_t X2 = src[i * src_stride + j + 1]; + sumX += X1 + X2; + + const uint16_t *dgd_ij = dgd_win + i * dgd_stride + j; + for (k = 0; k < wiener_win; k++) { + for (l = 0; l < wiener_win; l++) { + const uint16_t *dgd_ijkl = dgd_ij + k * dgd_stride + l; + int64_t *H_int_temp = &H_int[(l * wiener_win + k)][0]; + const uint16_t D1 = dgd_ijkl[0]; + const uint16_t D2 = dgd_ijkl[1]; + sumY[k][l] += D1 + D2; + M_int[l][k] += D1 * X1 + D2 * X2; + for (m = 0; m < wiener_win; m++) { + for (n = 0; n < wiener_win; n++) { + H_int_temp[m * 8 + n] += D1 * dgd_ij[n + dgd_stride * m] + + D2 * dgd_ij[n + dgd_stride * m + 1]; + } + } + } + } + } + } + + uint8_t bit_depth_divider = 1; + if (bit_depth == AOM_BITS_12) + bit_depth_divider = 16; + else if (bit_depth == AOM_BITS_10) + bit_depth_divider = 4; + + const int64_t avg_square_sum = (int64_t)avg * (int64_t)avg * pixel_count; + for (k = 0; k < wiener_win; k++) { + for (l = 0; l < wiener_win; l++) { + M[l * wiener_win + k] = + (M_int[l][k] + + (avg_square_sum - (int64_t)avg * (sumX + sumY[k][l]))) / + bit_depth_divider; + for (m = 0; m < wiener_win; m++) { + for (n = 0; n < wiener_win; n++) { + H[(l * wiener_win + k) * wiener_win2 + m * wiener_win + n] = + (H_int[(l * wiener_win + k)][n * 8 + m] + + (avg_square_sum - (int64_t)avg * (sumY[k][l] + sumY[n][m]))) / + bit_depth_divider; + } + } + } + } +} + +void compute_stats_highbd_opt_c(int wiener_win, const uint8_t *dgd, + const uint8_t *src, int h_start, int h_end, + int v_start, int v_end, int dgd_stride, + int src_stride, int64_t *M, int64_t *H, + aom_bit_depth_t bit_depth) { + if (wiener_win == WIENER_WIN || wiener_win == WIENER_WIN_CHROMA) { + compute_stats_highbd_win_opt_c(wiener_win, dgd, src, h_start, h_end, + v_start, v_end, dgd_stride, src_stride, M, H, + bit_depth); + } else { + av1_compute_stats_highbd_c(wiener_win, dgd, src, h_start, h_end, v_start, + v_end, dgd_stride, src_stride, M, H, bit_depth); + } +} + +static const int kIterations = 100; +typedef void (*compute_stats_Func)(int wiener_win, const uint8_t *dgd, + const uint8_t *src, int h_start, int h_end, + int v_start, int v_end, int dgd_stride, + int src_stride, int64_t *M, int64_t *H, + aom_bit_depth_t bit_depth); + +typedef ::testing::tuple WienerTestParam; + +class WienerTestHighbd : public ::testing::TestWithParam { + public: + virtual void SetUp() { + src_buf = (uint16_t *)aom_memalign( + 32, MAX_DATA_BLOCK * MAX_DATA_BLOCK * sizeof(*src_buf)); + dgd_buf = (uint16_t *)aom_memalign( + 32, MAX_DATA_BLOCK * MAX_DATA_BLOCK * sizeof(*dgd_buf)); + target_func_ = GET_PARAM(0); + } + virtual void TearDown() { + aom_free(src_buf); + aom_free(dgd_buf); + } + void RunWienerTest(const int32_t wiener_win, int32_t run_times, + aom_bit_depth_t bit_depth); + void RunWienerTest_ExtremeValues(const int32_t wiener_win, + aom_bit_depth_t bit_depth); + + private: + compute_stats_Func target_func_; + libaom_test::ACMRandom rng_; + uint16_t *src_buf; + uint16_t *dgd_buf; +}; + +void WienerTestHighbd::RunWienerTest(const int32_t wiener_win, + int32_t run_times, + aom_bit_depth_t bit_depth) { + const int32_t wiener_halfwin = wiener_win >> 1; + const int32_t wiener_win2 = wiener_win * wiener_win; + DECLARE_ALIGNED(32, int64_t, M_ref[WIENER_WIN2]); + DECLARE_ALIGNED(32, int64_t, H_ref[WIENER_WIN2 * WIENER_WIN2]); + DECLARE_ALIGNED(32, int64_t, M_test[WIENER_WIN2]); + DECLARE_ALIGNED(32, int64_t, H_test[WIENER_WIN2 * WIENER_WIN2]); + const int h_start = ((rng_.Rand16() % (MAX_WIENER_BLOCK / 2)) & (~7)); + const int h_end = + run_times != 1 ? 256 : ((rng_.Rand16() % MAX_WIENER_BLOCK) & (~7)) + 8; + const int v_start = ((rng_.Rand16() % (MAX_WIENER_BLOCK / 2)) & (~7)); + const int v_end = + run_times != 1 ? 256 : ((rng_.Rand16() % MAX_WIENER_BLOCK) & (~7)) + 8; + const int dgd_stride = h_end; + const int src_stride = MAX_DATA_BLOCK; + const int iters = run_times == 1 ? kIterations : 2; + for (int iter = 0; iter < iters && !HasFatalFailure(); ++iter) { + for (int i = 0; i < MAX_DATA_BLOCK * MAX_DATA_BLOCK; ++i) { + dgd_buf[i] = rng_.Rand16() % (1 << bit_depth); + src_buf[i] = rng_.Rand16() % (1 << bit_depth); + } + const uint8_t *dgd8 = CONVERT_TO_BYTEPTR( + dgd_buf + wiener_halfwin * MAX_DATA_BLOCK + wiener_halfwin); + const uint8_t *src8 = CONVERT_TO_BYTEPTR(src_buf); + + aom_usec_timer timer; + aom_usec_timer_start(&timer); + for (int i = 0; i < run_times; ++i) { + av1_compute_stats_highbd_c(wiener_win, dgd8, src8, h_start, h_end, + v_start, v_end, dgd_stride, src_stride, M_ref, + H_ref, bit_depth); + } + aom_usec_timer_mark(&timer); + const double time1 = static_cast(aom_usec_timer_elapsed(&timer)); + aom_usec_timer_start(&timer); + for (int i = 0; i < run_times; ++i) { + target_func_(wiener_win, dgd8, src8, h_start, h_end, v_start, v_end, + dgd_stride, src_stride, M_test, H_test, bit_depth); + } + aom_usec_timer_mark(&timer); + const double time2 = static_cast(aom_usec_timer_elapsed(&timer)); + if (run_times > 10) { + printf("win %d bd %d %3dx%-3d:%7.2f/%7.2fns", wiener_win, bit_depth, + h_end, v_end, time1, time2); + printf("(%3.2f)\n", time1 / time2); + } + int failed = 0; + for (int i = 0; i < wiener_win2; ++i) { + if (M_ref[i] != M_test[i]) { + failed = 1; + printf("win %d bd %d M iter %d [%4d] ref %6" PRId64 " test %6" PRId64 + " \n", + wiener_win, bit_depth, iter, i, M_ref[i], M_test[i]); + break; + } + } + for (int i = 0; i < wiener_win2 * wiener_win2; ++i) { + if (H_ref[i] != H_test[i]) { + failed = 1; + printf("win %d bd %d H iter %d [%4d] ref %6" PRId64 " test %6" PRId64 + " \n", + wiener_win, bit_depth, iter, i, H_ref[i], H_test[i]); + break; + } + } + ASSERT_EQ(failed, 0); + } +} + +void WienerTestHighbd::RunWienerTest_ExtremeValues(const int32_t wiener_win, + aom_bit_depth_t bit_depth) { + const int32_t wiener_halfwin = wiener_win >> 1; + const int32_t wiener_win2 = wiener_win * wiener_win; + DECLARE_ALIGNED(32, int64_t, M_ref[WIENER_WIN2]); + DECLARE_ALIGNED(32, int64_t, H_ref[WIENER_WIN2 * WIENER_WIN2]); + DECLARE_ALIGNED(32, int64_t, M_test[WIENER_WIN2]); + DECLARE_ALIGNED(32, int64_t, H_test[WIENER_WIN2 * WIENER_WIN2]); + const int h_start = 16; + const int h_end = MAX_WIENER_BLOCK; + const int v_start = 16; + const int v_end = MAX_WIENER_BLOCK; + const int dgd_stride = h_end; + const int src_stride = MAX_DATA_BLOCK; + const int iters = 1; + for (int iter = 0; iter < iters && !HasFatalFailure(); ++iter) { + for (int i = 0; i < MAX_DATA_BLOCK * MAX_DATA_BLOCK; ++i) { + dgd_buf[i] = ((uint16_t)1 << bit_depth) - 1; + src_buf[i] = ((uint16_t)1 << bit_depth) - 1; + } + const uint8_t *dgd8 = CONVERT_TO_BYTEPTR( + dgd_buf + wiener_halfwin * MAX_DATA_BLOCK + wiener_halfwin); + const uint8_t *src8 = CONVERT_TO_BYTEPTR(src_buf); + + av1_compute_stats_highbd_c(wiener_win, dgd8, src8, h_start, h_end, v_start, + v_end, dgd_stride, src_stride, M_ref, H_ref, + bit_depth); + + target_func_(wiener_win, dgd8, src8, h_start, h_end, v_start, v_end, + dgd_stride, src_stride, M_test, H_test, bit_depth); + + int failed = 0; + for (int i = 0; i < wiener_win2; ++i) { + if (M_ref[i] != M_test[i]) { + failed = 1; + printf("win %d bd %d M iter %d [%4d] ref %6" PRId64 " test %6" PRId64 + " \n", + wiener_win, bit_depth, iter, i, M_ref[i], M_test[i]); + break; + } + } + for (int i = 0; i < wiener_win2 * wiener_win2; ++i) { + if (H_ref[i] != H_test[i]) { + failed = 1; + printf("win %d bd %d H iter %d [%4d] ref %6" PRId64 " test %6" PRId64 + " \n", + wiener_win, bit_depth, iter, i, H_ref[i], H_test[i]); + break; + } + } + ASSERT_EQ(failed, 0); + } +} + +TEST_P(WienerTestHighbd, RandomValues) { + RunWienerTest(WIENER_WIN, 1, AOM_BITS_8); + RunWienerTest(WIENER_WIN_CHROMA, 1, AOM_BITS_8); + RunWienerTest(WIENER_WIN, 1, AOM_BITS_10); + RunWienerTest(WIENER_WIN_CHROMA, 1, AOM_BITS_10); + RunWienerTest(WIENER_WIN, 1, AOM_BITS_12); + RunWienerTest(WIENER_WIN_CHROMA, 1, AOM_BITS_12); +} + +TEST_P(WienerTestHighbd, ExtremeValues) { + RunWienerTest_ExtremeValues(WIENER_WIN, AOM_BITS_8); + RunWienerTest_ExtremeValues(WIENER_WIN_CHROMA, AOM_BITS_8); + RunWienerTest_ExtremeValues(WIENER_WIN, AOM_BITS_10); + RunWienerTest_ExtremeValues(WIENER_WIN_CHROMA, AOM_BITS_10); + RunWienerTest_ExtremeValues(WIENER_WIN, AOM_BITS_12); + RunWienerTest_ExtremeValues(WIENER_WIN_CHROMA, AOM_BITS_12); +} + +TEST_P(WienerTestHighbd, DISABLED_Speed) { + RunWienerTest(WIENER_WIN, 200, AOM_BITS_8); + RunWienerTest(WIENER_WIN_CHROMA, 200, AOM_BITS_8); + RunWienerTest(WIENER_WIN, 200, AOM_BITS_10); + RunWienerTest(WIENER_WIN_CHROMA, 200, AOM_BITS_10); + RunWienerTest(WIENER_WIN, 200, AOM_BITS_12); + RunWienerTest(WIENER_WIN_CHROMA, 200, AOM_BITS_12); +} + +INSTANTIATE_TEST_CASE_P(C, WienerTestHighbd, + ::testing::Values(compute_stats_highbd_opt_c)); + +#if HAVE_SSE4_1 +INSTANTIATE_TEST_CASE_P(SSE4_1, WienerTestHighbd, + ::testing::Values(av1_compute_stats_highbd_sse4_1)); +#endif // HAVE_SSE4_1 + +#if HAVE_AVX2 +INSTANTIATE_TEST_CASE_P(AVX2, WienerTestHighbd, + ::testing::Values(av1_compute_stats_highbd_avx2)); +#endif // HAVE_AVX2 + +} // namespace wiener_highbd diff --git a/third_party/aom/test/y4m_video_source.h b/third_party/aom/test/y4m_video_source.h index 3dea901e6d..63f74f567f 100644 --- a/third_party/aom/test/y4m_video_source.h +++ b/third_party/aom/test/y4m_video_source.h @@ -11,6 +11,7 @@ #ifndef AOM_TEST_Y4M_VIDEO_SOURCE_H_ #define AOM_TEST_Y4M_VIDEO_SOURCE_H_ #include +#include #include #include "common/y4minput.h" @@ -41,7 +42,8 @@ class Y4mVideoSource : public VideoSource { virtual void ReadSourceToStart() { ASSERT_TRUE(input_file_ != NULL); - ASSERT_FALSE(y4m_input_open(&y4m_, input_file_, NULL, 0, 0)); + ASSERT_FALSE( + y4m_input_open(&y4m_, input_file_, NULL, 0, AOM_CSP_UNKNOWN, 0)); framerate_numerator_ = y4m_.fps_n; framerate_denominator_ = y4m_.fps_d; frame_ = 0; @@ -109,7 +111,7 @@ class Y4mVideoSource : public VideoSource { std::string file_name_; FILE *input_file_; - testing::internal::scoped_ptr img_; + std::unique_ptr img_; unsigned int start_; unsigned int limit_; unsigned int frame_; diff --git a/third_party/aom/third_party/libwebm/README.libaom b/third_party/aom/third_party/libwebm/README.libaom index bd288d201d..17b2f47152 100644 --- a/third_party/aom/third_party/libwebm/README.libaom +++ b/third_party/aom/third_party/libwebm/README.libaom @@ -1,5 +1,5 @@ URL: https://chromium.googlesource.com/webm/libwebm -Version: af81f26025b7435fa9a14ad07c58b44cf9280430 +Version: 9f23fbc50e7a76c815b1d3f0309abe1066301331 License: BSD License File: LICENSE.txt @@ -7,8 +7,6 @@ Description: libwebm is used to handle WebM container I/O. Local Changes: -Add av1 codec as an eligible codec for webm: - https://aomedia-review.googlesource.com/c/aom/+/15103 Only keep: - Android.mk - AUTHORS.TXT diff --git a/third_party/aom/third_party/libwebm/common/file_util.cc b/third_party/aom/third_party/libwebm/common/file_util.cc index 618ffc087f..e6109d5a0e 100644 --- a/third_party/aom/third_party/libwebm/common/file_util.cc +++ b/third_party/aom/third_party/libwebm/common/file_util.cc @@ -46,7 +46,7 @@ std::string GetTempFileName() { errno_t err = tmpnam_s(tmp_file_name); #else char* fname_pointer = tmpnam(tmp_file_name); - errno_t err = (fname_pointer == &tmp_file_name[0]) ? 0 : -1; + int err = (fname_pointer == &tmp_file_name[0]) ? 0 : -1; #endif if (err == 0) { return std::string(tmp_file_name); diff --git a/third_party/aom/third_party/libwebm/common/webmids.h b/third_party/aom/third_party/libwebm/common/webmids.h index 89d722a71b..fc0c208140 100644 --- a/third_party/aom/third_party/libwebm/common/webmids.h +++ b/third_party/aom/third_party/libwebm/common/webmids.h @@ -93,6 +93,7 @@ enum MkvId { kMkvDisplayHeight = 0x54BA, kMkvDisplayUnit = 0x54B2, kMkvAspectRatioType = 0x54B3, + kMkvColourSpace = 0x2EB524, kMkvFrameRate = 0x2383E3, // end video // colour diff --git a/third_party/aom/third_party/libwebm/mkvmuxer/mkvmuxer.cc b/third_party/aom/third_party/libwebm/mkvmuxer/mkvmuxer.cc index bae2c99b87..5120312119 100644 --- a/third_party/aom/third_party/libwebm/mkvmuxer/mkvmuxer.cc +++ b/third_party/aom/third_party/libwebm/mkvmuxer/mkvmuxer.cc @@ -773,6 +773,14 @@ bool Track::Write(IMkvWriter* writer) const { if (!type_ || !codec_id_) return false; + // AV1 tracks require a CodecPrivate. See + // https://github.com/Matroska-Org/matroska-specification/blob/av1-mappin/codec/av1.md + // TODO(tomfinegan): Update the above link to the AV1 Matroska mappings to + // point to a stable version once it is finalized, or our own WebM mappings + // page on webmproject.org should we decide to release them. + if (!strcmp(codec_id_, Tracks::kAv1CodecId) && !codec_private_) + return false; + // |size| may be bigger than what is written out in this function because // derived classes may write out more data in the Track element. const uint64_t payload_size = PayloadSize(); @@ -1027,19 +1035,16 @@ bool MasteringMetadata::Write(IMkvWriter* writer) const { !WriteEbmlElement(writer, libwebm::kMkvLuminanceMin, luminance_min_)) { return false; } - if (r_ && - !r_->Write(writer, libwebm::kMkvPrimaryRChromaticityX, - libwebm::kMkvPrimaryRChromaticityY)) { + if (r_ && !r_->Write(writer, libwebm::kMkvPrimaryRChromaticityX, + libwebm::kMkvPrimaryRChromaticityY)) { return false; } - if (g_ && - !g_->Write(writer, libwebm::kMkvPrimaryGChromaticityX, - libwebm::kMkvPrimaryGChromaticityY)) { + if (g_ && !g_->Write(writer, libwebm::kMkvPrimaryGChromaticityX, + libwebm::kMkvPrimaryGChromaticityY)) { return false; } - if (b_ && - !b_->Write(writer, libwebm::kMkvPrimaryBChromaticityX, - libwebm::kMkvPrimaryBChromaticityY)) { + if (b_ && !b_->Write(writer, libwebm::kMkvPrimaryBChromaticityX, + libwebm::kMkvPrimaryBChromaticityY)) { return false; } if (white_point_ && @@ -1421,6 +1426,7 @@ VideoTrack::VideoTrack(unsigned int* seed) stereo_mode_(0), alpha_mode_(0), width_(0), + colour_space_(NULL), colour_(NULL), projection_(NULL) {} @@ -1518,6 +1524,10 @@ bool VideoTrack::Write(IMkvWriter* writer) const { static_cast(alpha_mode_))) return false; } + if (colour_space_) { + if (!WriteEbmlElement(writer, libwebm::kMkvColourSpace, colour_space_)) + return false; + } if (frame_rate_ > 0.0) { if (!WriteEbmlElement(writer, libwebm::kMkvFrameRate, static_cast(frame_rate_))) { @@ -1542,6 +1552,22 @@ bool VideoTrack::Write(IMkvWriter* writer) const { return true; } +void VideoTrack::set_colour_space(const char* colour_space) { + if (colour_space) { + delete[] colour_space_; + + const size_t length = strlen(colour_space) + 1; + colour_space_ = new (std::nothrow) char[length]; // NOLINT + if (colour_space_) { +#ifdef _MSC_VER + strcpy_s(colour_space_, length, colour_space); +#else + strcpy(colour_space_, colour_space); +#endif + } + } +} + bool VideoTrack::SetColour(const Colour& colour) { std::unique_ptr colour_ptr(new Colour()); if (!colour_ptr.get()) @@ -1625,6 +1651,8 @@ uint64_t VideoTrack::VideoPayloadSize() const { if (frame_rate_ > 0.0) size += EbmlElementSize(libwebm::kMkvFrameRate, static_cast(frame_rate_)); + if (colour_space_) + size += EbmlElementSize(libwebm::kMkvColourSpace, colour_space_); if (colour_) size += colour_->ColourSize(); if (projection_) @@ -1702,10 +1730,9 @@ bool AudioTrack::Write(IMkvWriter* writer) const { const char Tracks::kOpusCodecId[] = "A_OPUS"; const char Tracks::kVorbisCodecId[] = "A_VORBIS"; +const char Tracks::kAv1CodecId[] = "V_AV1"; const char Tracks::kVp8CodecId[] = "V_VP8"; const char Tracks::kVp9CodecId[] = "V_VP9"; -const char Tracks::kVp10CodecId[] = "V_VP10"; -const char Tracks::kAV1CodecId[] = "V_AV1"; const char Tracks::kWebVttCaptionsId[] = "D_WEBVTT/CAPTIONS"; const char Tracks::kWebVttDescriptionsId[] = "D_WEBVTT/DESCRIPTIONS"; const char Tracks::kWebVttMetadataId[] = "D_WEBVTT/METADATA"; @@ -4161,15 +4188,15 @@ bool Segment::WriteFramesLessThan(uint64_t timestamp) { } bool Segment::DocTypeIsWebm() const { - const int kNumCodecIds = 10; + const int kNumCodecIds = 9; // TODO(vigneshv): Tweak .clang-format. const char* kWebmCodecIds[kNumCodecIds] = { Tracks::kOpusCodecId, Tracks::kVorbisCodecId, - Tracks::kVp8CodecId, Tracks::kVp9CodecId, - Tracks::kVp10CodecId, Tracks::kAV1CodecId, - Tracks::kWebVttCaptionsId, Tracks::kWebVttDescriptionsId, - Tracks::kWebVttMetadataId, Tracks::kWebVttSubtitlesId}; + Tracks::kAv1CodecId, Tracks::kVp8CodecId, + Tracks::kVp9CodecId, Tracks::kWebVttCaptionsId, + Tracks::kWebVttDescriptionsId, Tracks::kWebVttMetadataId, + Tracks::kWebVttSubtitlesId}; const int num_tracks = static_cast(tracks_.track_entries_size()); for (int track_index = 0; track_index < num_tracks; ++track_index) { diff --git a/third_party/aom/third_party/libwebm/mkvmuxer/mkvmuxer.h b/third_party/aom/third_party/libwebm/mkvmuxer/mkvmuxer.h index 9e817bced4..f2db377145 100644 --- a/third_party/aom/third_party/libwebm/mkvmuxer/mkvmuxer.h +++ b/third_party/aom/third_party/libwebm/mkvmuxer/mkvmuxer.h @@ -795,6 +795,8 @@ class VideoTrack : public Track { uint64_t alpha_mode() { return alpha_mode_; } void set_width(uint64_t width) { width_ = width; } uint64_t width() const { return width_; } + void set_colour_space(const char* colour_space); + const char* colour_space() const { return colour_space_; } Colour* colour() { return colour_; } @@ -824,6 +826,7 @@ class VideoTrack : public Track { uint64_t stereo_mode_; uint64_t alpha_mode_; uint64_t width_; + char* colour_space_; Colour* colour_; Projection* projection_; @@ -871,10 +874,9 @@ class Tracks { static const char kOpusCodecId[]; static const char kVorbisCodecId[]; + static const char kAv1CodecId[]; static const char kVp8CodecId[]; static const char kVp9CodecId[]; - static const char kVp10CodecId[]; - static const char kAV1CodecId[]; static const char kWebVttCaptionsId[]; static const char kWebVttDescriptionsId[]; static const char kWebVttMetadataId[]; diff --git a/third_party/aom/third_party/libwebm/mkvmuxer/mkvmuxerutil.cc b/third_party/aom/third_party/libwebm/mkvmuxer/mkvmuxerutil.cc index 355d4e22b3..3bff7cd51a 100644 --- a/third_party/aom/third_party/libwebm/mkvmuxer/mkvmuxerutil.cc +++ b/third_party/aom/third_party/libwebm/mkvmuxer/mkvmuxerutil.cc @@ -136,9 +136,8 @@ uint64 WriteBlock(IMkvWriter* writer, const Frame* const frame, int64 timecode, return false; } - if (!frame->is_key() && - !WriteEbmlElement(writer, libwebm::kMkvReferenceBlock, - reference_block_timestamp)) { + if (!frame->is_key() && !WriteEbmlElement(writer, libwebm::kMkvReferenceBlock, + reference_block_timestamp)) { return false; } diff --git a/third_party/aom/third_party/libwebm/mkvmuxer/mkvwriter.cc b/third_party/aom/third_party/libwebm/mkvmuxer/mkvwriter.cc index 84655d802a..d668384d85 100644 --- a/third_party/aom/third_party/libwebm/mkvmuxer/mkvwriter.cc +++ b/third_party/aom/third_party/libwebm/mkvmuxer/mkvwriter.cc @@ -78,6 +78,8 @@ int32 MkvWriter::Position(int64 position) { #ifdef _MSC_VER return _fseeki64(file_, position, SEEK_SET); +#elif defined(_WIN32) + return fseeko64(file_, static_cast(position), SEEK_SET); #else return fseeko(file_, static_cast(position), SEEK_SET); #endif diff --git a/third_party/aom/third_party/libwebm/mkvparser/mkvparser.cc b/third_party/aom/third_party/libwebm/mkvparser/mkvparser.cc index e7b76f7da1..9c78ead2b0 100644 --- a/third_party/aom/third_party/libwebm/mkvparser/mkvparser.cc +++ b/third_party/aom/third_party/libwebm/mkvparser/mkvparser.cc @@ -36,8 +36,6 @@ inline bool isnan(double val) { return std::isnan(val); } inline bool isinf(double val) { return std::isinf(val); } #endif // MSC_COMPAT -IMkvReader::~IMkvReader() {} - template Type* SafeArrayAlloc(unsigned long long num_elements, unsigned long long element_size) { @@ -5274,6 +5272,7 @@ bool Projection::Parse(IMkvReader* reader, long long start, long long size, VideoTrack::VideoTrack(Segment* pSegment, long long element_start, long long element_size) : Track(pSegment, element_start, element_size), + m_colour_space(NULL), m_colour(NULL), m_projection(NULL) {} @@ -5299,6 +5298,7 @@ long VideoTrack::Parse(Segment* pSegment, const Info& info, long long stereo_mode = 0; double rate = 0.0; + char* colour_space = NULL; IMkvReader* const pReader = pSegment->m_pReader; @@ -5312,7 +5312,7 @@ long VideoTrack::Parse(Segment* pSegment, const Info& info, const long long stop = pos + s.size; Colour* colour = NULL; - Projection* projection = NULL; + std::unique_ptr projection_ptr; while (pos < stop) { long long id, size; @@ -5364,8 +5364,16 @@ long VideoTrack::Parse(Segment* pSegment, const Info& info, if (!Colour::Parse(pReader, pos, size, &colour)) return E_FILE_FORMAT_INVALID; } else if (id == libwebm::kMkvProjection) { - if (!Projection::Parse(pReader, pos, size, &projection)) + Projection* projection = NULL; + if (!Projection::Parse(pReader, pos, size, &projection)) { return E_FILE_FORMAT_INVALID; + } else { + projection_ptr.reset(projection); + } + } else if (id == libwebm::kMkvColourSpace) { + const long status = UnserializeString(pReader, pos, size, colour_space); + if (status < 0) + return status; } pos += size; // consume payload @@ -5397,7 +5405,8 @@ long VideoTrack::Parse(Segment* pSegment, const Info& info, pTrack->m_stereo_mode = stereo_mode; pTrack->m_rate = rate; pTrack->m_colour = colour; - pTrack->m_projection = projection; + pTrack->m_colour_space = colour_space; + pTrack->m_projection = projection_ptr.release(); pResult = pTrack; return 0; // success diff --git a/third_party/aom/third_party/libwebm/mkvparser/mkvparser.h b/third_party/aom/third_party/libwebm/mkvparser/mkvparser.h index 26c2b7e5eb..848d01f03e 100644 --- a/third_party/aom/third_party/libwebm/mkvparser/mkvparser.h +++ b/third_party/aom/third_party/libwebm/mkvparser/mkvparser.h @@ -22,7 +22,7 @@ class IMkvReader { virtual int Length(long long* total, long long* available) = 0; protected: - virtual ~IMkvReader(); + virtual ~IMkvReader() {} }; template @@ -527,6 +527,8 @@ class VideoTrack : public Track { Projection* GetProjection() const; + const char* GetColourSpace() const { return m_colour_space; } + private: long long m_width; long long m_height; @@ -534,7 +536,7 @@ class VideoTrack : public Track { long long m_display_height; long long m_display_unit; long long m_stereo_mode; - + char* m_colour_space; double m_rate; Colour* m_colour; diff --git a/third_party/aom/third_party/libwebm/mkvparser/mkvreader.cc b/third_party/aom/third_party/libwebm/mkvparser/mkvreader.cc index 23d68f5089..9d19c1be56 100644 --- a/third_party/aom/third_party/libwebm/mkvparser/mkvreader.cc +++ b/third_party/aom/third_party/libwebm/mkvparser/mkvreader.cc @@ -118,6 +118,8 @@ int MkvReader::Read(long long offset, long len, unsigned char* buffer) { if (status) return -1; // error +#elif defined(_WIN32) + fseeko64(m_file, static_cast(offset), SEEK_SET); #else fseeko(m_file, static_cast(offset), SEEK_SET); #endif diff --git a/third_party/aom/tools/txfm_analyzer/txfm_graph.h b/third_party/aom/tools/txfm_analyzer/txfm_graph.h index 2e3c9551e2..8dc36146dd 100644 --- a/third_party/aom/tools/txfm_analyzer/txfm_graph.h +++ b/third_party/aom/tools/txfm_analyzer/txfm_graph.h @@ -23,7 +23,6 @@ struct Node { int visited; }; -#define PI (3.141592653589793238462643383279502884) #define STAGENUM (10) #define NODENUM (32) #define COS_MOD (128)