From ef35212f8b4ebc73a4b3bd1402cb469a4b5c35e8 Mon Sep 17 00:00:00 2001 From: Roy Tam Date: Fri, 19 Apr 2019 13:08:28 +0800 Subject: [PATCH] update libaom to rev 76574b6c09515d6687ebfa9760319e521f5abeb3 (without moz.build and aom_ports/aom_once.h) --- media/libaom/README_MCP | 2 +- .../libaom/config/linux/arm/config/av1_rtcd.h | 5 +- .../config/linux/ia32/config/av1_rtcd.h | 7 +- .../libaom/config/linux/x64/config/av1_rtcd.h | 7 +- media/libaom/config/mac/x64/config/av1_rtcd.h | 7 +- .../libaom/config/win/ia32/config/av1_rtcd.h | 7 +- .../config/win/mingw32/config/av1_rtcd.h | 7 +- .../config/win/mingw64/config/av1_rtcd.h | 7 +- media/libaom/config/win/x64/config/av1_rtcd.h | 7 +- third_party/aom/.clang-format | 45 +- third_party/aom/aom/aomcx.h | 27 +- third_party/aom/aom_dsp/aom_dsp.cmake | 1 + third_party/aom/aom_dsp/aom_dsp_rtcd_defs.pl | 5 +- third_party/aom/aom_dsp/grain_synthesis.c | 2 +- third_party/aom/aom_dsp/noise_model.c | 4 + third_party/aom/aom_dsp/psnrhvs.c | 12 +- .../aom/aom_dsp/x86/adaptive_quantize_avx2.c | 244 ++ .../aom/aom_dsp/x86/adaptive_quantize_sse2.c | 390 +- .../x86/aom_subpixel_8t_intrin_ssse3.c | 36 +- third_party/aom/aom_dsp/x86/convolve_avx2.h | 25 + third_party/aom/aom_dsp/x86/intrapred_avx2.c | 2122 ++++++----- third_party/aom/aom_dsp/x86/quantize_x86.h | 57 + third_party/aom/apps/aomenc.c | 16 +- third_party/aom/av1/av1.cmake | 1 + third_party/aom/av1/av1_cx_iface.c | 79 +- third_party/aom/av1/common/av1_loopfilter.c | 7 +- third_party/aom/av1/common/av1_rtcd_defs.pl | 9 +- third_party/aom/av1/common/blockd.c | 2 + third_party/aom/av1/common/blockd.h | 6 +- third_party/aom/av1/common/convolve.c | 2 + third_party/aom/av1/common/entropymode.c | 112 +- third_party/aom/av1/common/enums.h | 23 +- third_party/aom/av1/common/mv.h | 1 - third_party/aom/av1/common/mvref_common.c | 142 +- third_party/aom/av1/common/mvref_common.h | 16 +- third_party/aom/av1/common/onyxc_int.h | 4 + third_party/aom/av1/common/pred_common.h | 1 + third_party/aom/av1/common/reconinter.c | 1 + third_party/aom/av1/common/reconintra.c | 12 +- third_party/aom/av1/common/resize.c | 4 - third_party/aom/av1/common/restoration.c | 2 +- third_party/aom/av1/common/restoration.h | 1 + third_party/aom/av1/common/scale.h | 2 + third_party/aom/av1/common/scan.c | 12 +- third_party/aom/av1/common/token_cdfs.h | 3362 ++++++++--------- third_party/aom/av1/common/warped_motion.c | 1 - .../aom/av1/common/x86/av1_inv_txfm_avx2.c | 10 +- .../aom/av1/common/x86/av1_inv_txfm_ssse3.c | 54 +- .../av1/common/x86/highbd_warp_plane_sse4.c | 18 +- .../aom/av1/common/x86/intra_edge_sse4.c | 8 +- .../aom/av1/common/x86/warp_plane_sse4.c | 6 +- .../aom/av1/common/x86/wiener_convolve_avx2.c | 345 +- third_party/aom/av1/decoder/decodeframe.c | 13 +- third_party/aom/av1/decoder/decodemv.c | 10 +- third_party/aom/av1/decoder/decodetxb.c | 1 + third_party/aom/av1/decoder/obu.c | 19 - third_party/aom/av1/encoder/aq_variance.c | 8 +- third_party/aom/av1/encoder/aq_variance.h | 4 +- third_party/aom/av1/encoder/av1_quantize.c | 2 +- third_party/aom/av1/encoder/bitstream.c | 48 +- third_party/aom/av1/encoder/block.h | 4 +- third_party/aom/av1/encoder/context_tree.c | 5 +- third_party/aom/av1/encoder/context_tree.h | 15 +- third_party/aom/av1/encoder/encode_strategy.c | 5 +- third_party/aom/av1/encoder/encode_strategy.h | 3 +- third_party/aom/av1/encoder/encodeframe.c | 1073 ++---- third_party/aom/av1/encoder/encodeframe.h | 4 +- third_party/aom/av1/encoder/encodemb.c | 6 +- third_party/aom/av1/encoder/encoder.c | 90 +- third_party/aom/av1/encoder/encoder.h | 31 +- third_party/aom/av1/encoder/encodetxb.c | 69 +- third_party/aom/av1/encoder/encodetxb.h | 6 +- third_party/aom/av1/encoder/firstpass.h | 3 + third_party/aom/av1/encoder/gop_structure.c | 136 + third_party/aom/av1/encoder/level.c | 181 +- third_party/aom/av1/encoder/level.h | 10 +- third_party/aom/av1/encoder/mcomp.c | 18 +- third_party/aom/av1/encoder/mcomp.h | 1 + third_party/aom/av1/encoder/nonrd_pickmode.c | 1364 +++++++ .../aom/av1/encoder/partition_model_weights.h | 1455 ++++++- .../aom/av1/encoder/partition_strategy.c | 894 ++++- .../aom/av1/encoder/partition_strategy.h | 100 +- third_party/aom/av1/encoder/picklpf.c | 2 +- third_party/aom/av1/encoder/pickrst.c | 30 +- third_party/aom/av1/encoder/pustats.h | 16 +- third_party/aom/av1/encoder/ransac.c | 2 + third_party/aom/av1/encoder/ratectrl.c | 41 + third_party/aom/av1/encoder/rd.c | 39 +- third_party/aom/av1/encoder/rd.h | 13 +- third_party/aom/av1/encoder/rdopt.c | 351 +- third_party/aom/av1/encoder/rdopt.h | 7 + third_party/aom/av1/encoder/reconinter_enc.c | 2 + third_party/aom/av1/encoder/segmentation.c | 1 + third_party/aom/av1/encoder/speed_features.c | 66 +- third_party/aom/av1/encoder/speed_features.h | 30 +- third_party/aom/av1/encoder/temporal_filter.c | 6 +- third_party/aom/av1/encoder/tokenize.c | 3 + third_party/aom/av1/encoder/tpl_model.c | 42 +- third_party/aom/av1/encoder/var_based_part.c | 107 +- .../aom/av1/encoder/x86/corner_match_avx2.c | 6 +- .../aom/av1/encoder/x86/corner_match_sse4.c | 6 +- .../aom/av1/encoder/x86/pickrst_avx2.c | 26 +- .../encoder/x86/temporal_filter_constants.h | 26 +- third_party/aom/build/cmake/sanitizers.cmake | 27 +- third_party/aom/common/av1_config.c | 12 +- third_party/aom/common/obudec.c | 14 +- third_party/aom/examples/inspect.c | 6 +- third_party/aom/test/aom_integer_test.cc | 6 +- third_party/aom/test/av1_config_test.cc | 18 +- third_party/aom/test/av1_inv_txfm1d_test.cc | 6 +- third_party/aom/test/av1_inv_txfm2d_test.cc | 114 +- third_party/aom/test/cfl_test.cc | 53 +- third_party/aom/test/convolve_test.cc | 6 +- third_party/aom/test/corner_match_test.cc | 8 +- third_party/aom/test/dr_prediction_test.cc | 4 +- .../aom/test/hiprec_convolve_test_util.cc | 4 +- third_party/aom/test/quantize_func_test.cc | 195 +- third_party/aom/test/rt_end_to_end_test.cc | 2 +- third_party/aom/test/simd_cmp_impl.h | 7 +- third_party/aom/test/test.cmake | 3 +- third_party/aom/test/time_stamp_test.cc | 105 + .../aom/test/yuv_temporal_filter_test.cc | 6 +- .../aom/third_party/libwebm/README.libaom | 2 +- .../third_party/libwebm/common/file_util.cc | 4 +- .../libwebm/mkvmuxer/mkvmuxerutil.cc | 8 +- .../libwebm/mkvmuxer/mkvmuxerutil.h | 3 + .../libwebm/mkvparser/mkvparser.cc | 10 +- third_party/aom/tools/aom_entropy_optimizer.c | 6 +- 128 files changed, 9378 insertions(+), 4936 deletions(-) create mode 100644 third_party/aom/aom_dsp/x86/adaptive_quantize_avx2.c create mode 100644 third_party/aom/av1/encoder/nonrd_pickmode.c create mode 100644 third_party/aom/test/time_stamp_test.cc diff --git a/media/libaom/README_MCP b/media/libaom/README_MCP index 59cf02f240..2ac84e7e1a 100644 --- a/media/libaom/README_MCP +++ b/media/libaom/README_MCP @@ -10,4 +10,4 @@ The upstream aom git repository is: https://aomedia.googlesource.com/aom -The git commit ID used was 5bdd95475e07ac8641114cb139105b8d734a1b60. +The git commit ID used was 76574b6c09515d6687ebfa9760319e521f5abeb3. diff --git a/media/libaom/config/linux/arm/config/av1_rtcd.h b/media/libaom/config/linux/arm/config/av1_rtcd.h index d51b6816cf..6b07131f75 100644 --- a/media/libaom/config/linux/arm/config/av1_rtcd.h +++ b/media/libaom/config/linux/arm/config/av1_rtcd.h @@ -268,8 +268,7 @@ void av1_inv_txfm2d_add_8x8_c(const int32_t *input, uint16_t *output, int stride #define av1_inv_txfm2d_add_8x8 av1_inv_txfm2d_add_8x8_c void av1_inv_txfm_add_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -void av1_inv_txfm_add_neon(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); -RTCD_EXTERN void (*av1_inv_txfm_add)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +#define av1_inv_txfm_add av1_inv_txfm_add_c void av1_round_shift_array_c(int32_t *arr, int size, int bit); void av1_round_shift_array_neon(int32_t *arr, int size, int bit); @@ -383,8 +382,6 @@ static void setup_rtcd_internal(void) if (flags & HAS_NEON) av1_dist_wtd_convolve_x = av1_dist_wtd_convolve_x_neon; av1_dist_wtd_convolve_y = av1_dist_wtd_convolve_y_c; if (flags & HAS_NEON) av1_dist_wtd_convolve_y = av1_dist_wtd_convolve_y_neon; - av1_inv_txfm_add = av1_inv_txfm_add_c; - if (flags & HAS_NEON) av1_inv_txfm_add = av1_inv_txfm_add_neon; av1_round_shift_array = av1_round_shift_array_c; if (flags & HAS_NEON) av1_round_shift_array = av1_round_shift_array_neon; av1_selfguided_restoration = av1_selfguided_restoration_c; diff --git a/media/libaom/config/linux/ia32/config/av1_rtcd.h b/media/libaom/config/linux/ia32/config/av1_rtcd.h index ea1cd37366..4938770b1b 100644 --- a/media/libaom/config/linux/ia32/config/av1_rtcd.h +++ b/media/libaom/config/linux/ia32/config/av1_rtcd.h @@ -214,7 +214,8 @@ void av1_highbd_dr_prediction_z1_avx2(uint16_t *dst, ptrdiff_t stride, int bw, i RTCD_EXTERN void (*av1_highbd_dr_prediction_z1)(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_above, int dx, int dy, int bd); void av1_highbd_dr_prediction_z2_c(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_above, int upsample_left, int dx, int dy, int bd); -#define av1_highbd_dr_prediction_z2 av1_highbd_dr_prediction_z2_c +void av1_highbd_dr_prediction_z2_avx2(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_above, int upsample_left, int dx, int dy, int bd); +RTCD_EXTERN void (*av1_highbd_dr_prediction_z2)(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_above, int upsample_left, int dx, int dy, int bd); void av1_highbd_dr_prediction_z3_c(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_left, int dx, int dy, int bd); void av1_highbd_dr_prediction_z3_avx2(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_left, int dx, int dy, int bd); @@ -324,6 +325,7 @@ RTCD_EXTERN void (*av1_inv_txfm2d_add_8x8)(const int32_t *input, uint16_t *outpu void av1_inv_txfm_add_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); void av1_inv_txfm_add_ssse3(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +void av1_inv_txfm_add_avx2(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); RTCD_EXTERN void (*av1_inv_txfm_add)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); void av1_round_shift_array_c(int32_t *arr, int size, int bit); @@ -526,6 +528,8 @@ static void setup_rtcd_internal(void) if (flags & HAS_AVX2) av1_highbd_dist_wtd_convolve_y = av1_highbd_dist_wtd_convolve_y_avx2; av1_highbd_dr_prediction_z1 = av1_highbd_dr_prediction_z1_c; if (flags & HAS_AVX2) av1_highbd_dr_prediction_z1 = av1_highbd_dr_prediction_z1_avx2; + av1_highbd_dr_prediction_z2 = av1_highbd_dr_prediction_z2_c; + if (flags & HAS_AVX2) av1_highbd_dr_prediction_z2 = av1_highbd_dr_prediction_z2_avx2; av1_highbd_dr_prediction_z3 = av1_highbd_dr_prediction_z3_c; if (flags & HAS_AVX2) av1_highbd_dr_prediction_z3 = av1_highbd_dr_prediction_z3_avx2; av1_highbd_inv_txfm_add = av1_highbd_inv_txfm_add_c; @@ -553,6 +557,7 @@ static void setup_rtcd_internal(void) if (flags & HAS_SSE4_1) av1_inv_txfm2d_add_8x8 = av1_inv_txfm2d_add_8x8_sse4_1; av1_inv_txfm_add = av1_inv_txfm_add_c; if (flags & HAS_SSSE3) av1_inv_txfm_add = av1_inv_txfm_add_ssse3; + if (flags & HAS_AVX2) av1_inv_txfm_add = av1_inv_txfm_add_avx2; av1_round_shift_array = av1_round_shift_array_c; if (flags & HAS_SSE4_1) av1_round_shift_array = av1_round_shift_array_sse4_1; av1_selfguided_restoration = av1_selfguided_restoration_c; diff --git a/media/libaom/config/linux/x64/config/av1_rtcd.h b/media/libaom/config/linux/x64/config/av1_rtcd.h index c4954fef53..794f7a5196 100644 --- a/media/libaom/config/linux/x64/config/av1_rtcd.h +++ b/media/libaom/config/linux/x64/config/av1_rtcd.h @@ -217,7 +217,8 @@ void av1_highbd_dr_prediction_z1_avx2(uint16_t *dst, ptrdiff_t stride, int bw, i RTCD_EXTERN void (*av1_highbd_dr_prediction_z1)(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_above, int dx, int dy, int bd); void av1_highbd_dr_prediction_z2_c(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_above, int upsample_left, int dx, int dy, int bd); -#define av1_highbd_dr_prediction_z2 av1_highbd_dr_prediction_z2_c +void av1_highbd_dr_prediction_z2_avx2(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_above, int upsample_left, int dx, int dy, int bd); +RTCD_EXTERN void (*av1_highbd_dr_prediction_z2)(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_above, int upsample_left, int dx, int dy, int bd); void av1_highbd_dr_prediction_z3_c(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_left, int dx, int dy, int bd); void av1_highbd_dr_prediction_z3_avx2(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_left, int dx, int dy, int bd); @@ -327,6 +328,7 @@ RTCD_EXTERN void (*av1_inv_txfm2d_add_8x8)(const int32_t *input, uint16_t *outpu void av1_inv_txfm_add_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); void av1_inv_txfm_add_ssse3(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +void av1_inv_txfm_add_avx2(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); RTCD_EXTERN void (*av1_inv_txfm_add)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); void av1_round_shift_array_c(int32_t *arr, int size, int bit); @@ -520,6 +522,8 @@ static void setup_rtcd_internal(void) if (flags & HAS_AVX2) av1_highbd_dist_wtd_convolve_y = av1_highbd_dist_wtd_convolve_y_avx2; av1_highbd_dr_prediction_z1 = av1_highbd_dr_prediction_z1_c; if (flags & HAS_AVX2) av1_highbd_dr_prediction_z1 = av1_highbd_dr_prediction_z1_avx2; + av1_highbd_dr_prediction_z2 = av1_highbd_dr_prediction_z2_c; + if (flags & HAS_AVX2) av1_highbd_dr_prediction_z2 = av1_highbd_dr_prediction_z2_avx2; av1_highbd_dr_prediction_z3 = av1_highbd_dr_prediction_z3_c; if (flags & HAS_AVX2) av1_highbd_dr_prediction_z3 = av1_highbd_dr_prediction_z3_avx2; av1_highbd_inv_txfm_add = av1_highbd_inv_txfm_add_c; @@ -547,6 +551,7 @@ static void setup_rtcd_internal(void) if (flags & HAS_SSE4_1) av1_inv_txfm2d_add_8x8 = av1_inv_txfm2d_add_8x8_sse4_1; av1_inv_txfm_add = av1_inv_txfm_add_c; if (flags & HAS_SSSE3) av1_inv_txfm_add = av1_inv_txfm_add_ssse3; + if (flags & HAS_AVX2) av1_inv_txfm_add = av1_inv_txfm_add_avx2; av1_round_shift_array = av1_round_shift_array_c; if (flags & HAS_SSE4_1) av1_round_shift_array = av1_round_shift_array_sse4_1; av1_selfguided_restoration = av1_selfguided_restoration_c; diff --git a/media/libaom/config/mac/x64/config/av1_rtcd.h b/media/libaom/config/mac/x64/config/av1_rtcd.h index c4954fef53..794f7a5196 100644 --- a/media/libaom/config/mac/x64/config/av1_rtcd.h +++ b/media/libaom/config/mac/x64/config/av1_rtcd.h @@ -217,7 +217,8 @@ void av1_highbd_dr_prediction_z1_avx2(uint16_t *dst, ptrdiff_t stride, int bw, i RTCD_EXTERN void (*av1_highbd_dr_prediction_z1)(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_above, int dx, int dy, int bd); void av1_highbd_dr_prediction_z2_c(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_above, int upsample_left, int dx, int dy, int bd); -#define av1_highbd_dr_prediction_z2 av1_highbd_dr_prediction_z2_c +void av1_highbd_dr_prediction_z2_avx2(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_above, int upsample_left, int dx, int dy, int bd); +RTCD_EXTERN void (*av1_highbd_dr_prediction_z2)(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_above, int upsample_left, int dx, int dy, int bd); void av1_highbd_dr_prediction_z3_c(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_left, int dx, int dy, int bd); void av1_highbd_dr_prediction_z3_avx2(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_left, int dx, int dy, int bd); @@ -327,6 +328,7 @@ RTCD_EXTERN void (*av1_inv_txfm2d_add_8x8)(const int32_t *input, uint16_t *outpu void av1_inv_txfm_add_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); void av1_inv_txfm_add_ssse3(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +void av1_inv_txfm_add_avx2(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); RTCD_EXTERN void (*av1_inv_txfm_add)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); void av1_round_shift_array_c(int32_t *arr, int size, int bit); @@ -520,6 +522,8 @@ static void setup_rtcd_internal(void) if (flags & HAS_AVX2) av1_highbd_dist_wtd_convolve_y = av1_highbd_dist_wtd_convolve_y_avx2; av1_highbd_dr_prediction_z1 = av1_highbd_dr_prediction_z1_c; if (flags & HAS_AVX2) av1_highbd_dr_prediction_z1 = av1_highbd_dr_prediction_z1_avx2; + av1_highbd_dr_prediction_z2 = av1_highbd_dr_prediction_z2_c; + if (flags & HAS_AVX2) av1_highbd_dr_prediction_z2 = av1_highbd_dr_prediction_z2_avx2; av1_highbd_dr_prediction_z3 = av1_highbd_dr_prediction_z3_c; if (flags & HAS_AVX2) av1_highbd_dr_prediction_z3 = av1_highbd_dr_prediction_z3_avx2; av1_highbd_inv_txfm_add = av1_highbd_inv_txfm_add_c; @@ -547,6 +551,7 @@ static void setup_rtcd_internal(void) if (flags & HAS_SSE4_1) av1_inv_txfm2d_add_8x8 = av1_inv_txfm2d_add_8x8_sse4_1; av1_inv_txfm_add = av1_inv_txfm_add_c; if (flags & HAS_SSSE3) av1_inv_txfm_add = av1_inv_txfm_add_ssse3; + if (flags & HAS_AVX2) av1_inv_txfm_add = av1_inv_txfm_add_avx2; av1_round_shift_array = av1_round_shift_array_c; if (flags & HAS_SSE4_1) av1_round_shift_array = av1_round_shift_array_sse4_1; av1_selfguided_restoration = av1_selfguided_restoration_c; diff --git a/media/libaom/config/win/ia32/config/av1_rtcd.h b/media/libaom/config/win/ia32/config/av1_rtcd.h index ea1cd37366..4938770b1b 100644 --- a/media/libaom/config/win/ia32/config/av1_rtcd.h +++ b/media/libaom/config/win/ia32/config/av1_rtcd.h @@ -214,7 +214,8 @@ void av1_highbd_dr_prediction_z1_avx2(uint16_t *dst, ptrdiff_t stride, int bw, i RTCD_EXTERN void (*av1_highbd_dr_prediction_z1)(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_above, int dx, int dy, int bd); void av1_highbd_dr_prediction_z2_c(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_above, int upsample_left, int dx, int dy, int bd); -#define av1_highbd_dr_prediction_z2 av1_highbd_dr_prediction_z2_c +void av1_highbd_dr_prediction_z2_avx2(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_above, int upsample_left, int dx, int dy, int bd); +RTCD_EXTERN void (*av1_highbd_dr_prediction_z2)(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_above, int upsample_left, int dx, int dy, int bd); void av1_highbd_dr_prediction_z3_c(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_left, int dx, int dy, int bd); void av1_highbd_dr_prediction_z3_avx2(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_left, int dx, int dy, int bd); @@ -324,6 +325,7 @@ RTCD_EXTERN void (*av1_inv_txfm2d_add_8x8)(const int32_t *input, uint16_t *outpu void av1_inv_txfm_add_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); void av1_inv_txfm_add_ssse3(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +void av1_inv_txfm_add_avx2(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); RTCD_EXTERN void (*av1_inv_txfm_add)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); void av1_round_shift_array_c(int32_t *arr, int size, int bit); @@ -526,6 +528,8 @@ static void setup_rtcd_internal(void) if (flags & HAS_AVX2) av1_highbd_dist_wtd_convolve_y = av1_highbd_dist_wtd_convolve_y_avx2; av1_highbd_dr_prediction_z1 = av1_highbd_dr_prediction_z1_c; if (flags & HAS_AVX2) av1_highbd_dr_prediction_z1 = av1_highbd_dr_prediction_z1_avx2; + av1_highbd_dr_prediction_z2 = av1_highbd_dr_prediction_z2_c; + if (flags & HAS_AVX2) av1_highbd_dr_prediction_z2 = av1_highbd_dr_prediction_z2_avx2; av1_highbd_dr_prediction_z3 = av1_highbd_dr_prediction_z3_c; if (flags & HAS_AVX2) av1_highbd_dr_prediction_z3 = av1_highbd_dr_prediction_z3_avx2; av1_highbd_inv_txfm_add = av1_highbd_inv_txfm_add_c; @@ -553,6 +557,7 @@ static void setup_rtcd_internal(void) if (flags & HAS_SSE4_1) av1_inv_txfm2d_add_8x8 = av1_inv_txfm2d_add_8x8_sse4_1; av1_inv_txfm_add = av1_inv_txfm_add_c; if (flags & HAS_SSSE3) av1_inv_txfm_add = av1_inv_txfm_add_ssse3; + if (flags & HAS_AVX2) av1_inv_txfm_add = av1_inv_txfm_add_avx2; av1_round_shift_array = av1_round_shift_array_c; if (flags & HAS_SSE4_1) av1_round_shift_array = av1_round_shift_array_sse4_1; av1_selfguided_restoration = av1_selfguided_restoration_c; diff --git a/media/libaom/config/win/mingw32/config/av1_rtcd.h b/media/libaom/config/win/mingw32/config/av1_rtcd.h index ea1cd37366..4938770b1b 100644 --- a/media/libaom/config/win/mingw32/config/av1_rtcd.h +++ b/media/libaom/config/win/mingw32/config/av1_rtcd.h @@ -214,7 +214,8 @@ void av1_highbd_dr_prediction_z1_avx2(uint16_t *dst, ptrdiff_t stride, int bw, i RTCD_EXTERN void (*av1_highbd_dr_prediction_z1)(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_above, int dx, int dy, int bd); void av1_highbd_dr_prediction_z2_c(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_above, int upsample_left, int dx, int dy, int bd); -#define av1_highbd_dr_prediction_z2 av1_highbd_dr_prediction_z2_c +void av1_highbd_dr_prediction_z2_avx2(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_above, int upsample_left, int dx, int dy, int bd); +RTCD_EXTERN void (*av1_highbd_dr_prediction_z2)(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_above, int upsample_left, int dx, int dy, int bd); void av1_highbd_dr_prediction_z3_c(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_left, int dx, int dy, int bd); void av1_highbd_dr_prediction_z3_avx2(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_left, int dx, int dy, int bd); @@ -324,6 +325,7 @@ RTCD_EXTERN void (*av1_inv_txfm2d_add_8x8)(const int32_t *input, uint16_t *outpu void av1_inv_txfm_add_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); void av1_inv_txfm_add_ssse3(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +void av1_inv_txfm_add_avx2(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); RTCD_EXTERN void (*av1_inv_txfm_add)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); void av1_round_shift_array_c(int32_t *arr, int size, int bit); @@ -526,6 +528,8 @@ static void setup_rtcd_internal(void) if (flags & HAS_AVX2) av1_highbd_dist_wtd_convolve_y = av1_highbd_dist_wtd_convolve_y_avx2; av1_highbd_dr_prediction_z1 = av1_highbd_dr_prediction_z1_c; if (flags & HAS_AVX2) av1_highbd_dr_prediction_z1 = av1_highbd_dr_prediction_z1_avx2; + av1_highbd_dr_prediction_z2 = av1_highbd_dr_prediction_z2_c; + if (flags & HAS_AVX2) av1_highbd_dr_prediction_z2 = av1_highbd_dr_prediction_z2_avx2; av1_highbd_dr_prediction_z3 = av1_highbd_dr_prediction_z3_c; if (flags & HAS_AVX2) av1_highbd_dr_prediction_z3 = av1_highbd_dr_prediction_z3_avx2; av1_highbd_inv_txfm_add = av1_highbd_inv_txfm_add_c; @@ -553,6 +557,7 @@ static void setup_rtcd_internal(void) if (flags & HAS_SSE4_1) av1_inv_txfm2d_add_8x8 = av1_inv_txfm2d_add_8x8_sse4_1; av1_inv_txfm_add = av1_inv_txfm_add_c; if (flags & HAS_SSSE3) av1_inv_txfm_add = av1_inv_txfm_add_ssse3; + if (flags & HAS_AVX2) av1_inv_txfm_add = av1_inv_txfm_add_avx2; av1_round_shift_array = av1_round_shift_array_c; if (flags & HAS_SSE4_1) av1_round_shift_array = av1_round_shift_array_sse4_1; av1_selfguided_restoration = av1_selfguided_restoration_c; diff --git a/media/libaom/config/win/mingw64/config/av1_rtcd.h b/media/libaom/config/win/mingw64/config/av1_rtcd.h index c4954fef53..794f7a5196 100644 --- a/media/libaom/config/win/mingw64/config/av1_rtcd.h +++ b/media/libaom/config/win/mingw64/config/av1_rtcd.h @@ -217,7 +217,8 @@ void av1_highbd_dr_prediction_z1_avx2(uint16_t *dst, ptrdiff_t stride, int bw, i RTCD_EXTERN void (*av1_highbd_dr_prediction_z1)(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_above, int dx, int dy, int bd); void av1_highbd_dr_prediction_z2_c(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_above, int upsample_left, int dx, int dy, int bd); -#define av1_highbd_dr_prediction_z2 av1_highbd_dr_prediction_z2_c +void av1_highbd_dr_prediction_z2_avx2(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_above, int upsample_left, int dx, int dy, int bd); +RTCD_EXTERN void (*av1_highbd_dr_prediction_z2)(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_above, int upsample_left, int dx, int dy, int bd); void av1_highbd_dr_prediction_z3_c(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_left, int dx, int dy, int bd); void av1_highbd_dr_prediction_z3_avx2(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_left, int dx, int dy, int bd); @@ -327,6 +328,7 @@ RTCD_EXTERN void (*av1_inv_txfm2d_add_8x8)(const int32_t *input, uint16_t *outpu void av1_inv_txfm_add_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); void av1_inv_txfm_add_ssse3(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +void av1_inv_txfm_add_avx2(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); RTCD_EXTERN void (*av1_inv_txfm_add)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); void av1_round_shift_array_c(int32_t *arr, int size, int bit); @@ -520,6 +522,8 @@ static void setup_rtcd_internal(void) if (flags & HAS_AVX2) av1_highbd_dist_wtd_convolve_y = av1_highbd_dist_wtd_convolve_y_avx2; av1_highbd_dr_prediction_z1 = av1_highbd_dr_prediction_z1_c; if (flags & HAS_AVX2) av1_highbd_dr_prediction_z1 = av1_highbd_dr_prediction_z1_avx2; + av1_highbd_dr_prediction_z2 = av1_highbd_dr_prediction_z2_c; + if (flags & HAS_AVX2) av1_highbd_dr_prediction_z2 = av1_highbd_dr_prediction_z2_avx2; av1_highbd_dr_prediction_z3 = av1_highbd_dr_prediction_z3_c; if (flags & HAS_AVX2) av1_highbd_dr_prediction_z3 = av1_highbd_dr_prediction_z3_avx2; av1_highbd_inv_txfm_add = av1_highbd_inv_txfm_add_c; @@ -547,6 +551,7 @@ static void setup_rtcd_internal(void) if (flags & HAS_SSE4_1) av1_inv_txfm2d_add_8x8 = av1_inv_txfm2d_add_8x8_sse4_1; av1_inv_txfm_add = av1_inv_txfm_add_c; if (flags & HAS_SSSE3) av1_inv_txfm_add = av1_inv_txfm_add_ssse3; + if (flags & HAS_AVX2) av1_inv_txfm_add = av1_inv_txfm_add_avx2; av1_round_shift_array = av1_round_shift_array_c; if (flags & HAS_SSE4_1) av1_round_shift_array = av1_round_shift_array_sse4_1; av1_selfguided_restoration = av1_selfguided_restoration_c; diff --git a/media/libaom/config/win/x64/config/av1_rtcd.h b/media/libaom/config/win/x64/config/av1_rtcd.h index c4954fef53..794f7a5196 100644 --- a/media/libaom/config/win/x64/config/av1_rtcd.h +++ b/media/libaom/config/win/x64/config/av1_rtcd.h @@ -217,7 +217,8 @@ void av1_highbd_dr_prediction_z1_avx2(uint16_t *dst, ptrdiff_t stride, int bw, i RTCD_EXTERN void (*av1_highbd_dr_prediction_z1)(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_above, int dx, int dy, int bd); void av1_highbd_dr_prediction_z2_c(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_above, int upsample_left, int dx, int dy, int bd); -#define av1_highbd_dr_prediction_z2 av1_highbd_dr_prediction_z2_c +void av1_highbd_dr_prediction_z2_avx2(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_above, int upsample_left, int dx, int dy, int bd); +RTCD_EXTERN void (*av1_highbd_dr_prediction_z2)(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_above, int upsample_left, int dx, int dy, int bd); void av1_highbd_dr_prediction_z3_c(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_left, int dx, int dy, int bd); void av1_highbd_dr_prediction_z3_avx2(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_left, int dx, int dy, int bd); @@ -327,6 +328,7 @@ RTCD_EXTERN void (*av1_inv_txfm2d_add_8x8)(const int32_t *input, uint16_t *outpu void av1_inv_txfm_add_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); void av1_inv_txfm_add_ssse3(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); +void av1_inv_txfm_add_avx2(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); RTCD_EXTERN void (*av1_inv_txfm_add)(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param); void av1_round_shift_array_c(int32_t *arr, int size, int bit); @@ -520,6 +522,8 @@ static void setup_rtcd_internal(void) if (flags & HAS_AVX2) av1_highbd_dist_wtd_convolve_y = av1_highbd_dist_wtd_convolve_y_avx2; av1_highbd_dr_prediction_z1 = av1_highbd_dr_prediction_z1_c; if (flags & HAS_AVX2) av1_highbd_dr_prediction_z1 = av1_highbd_dr_prediction_z1_avx2; + av1_highbd_dr_prediction_z2 = av1_highbd_dr_prediction_z2_c; + if (flags & HAS_AVX2) av1_highbd_dr_prediction_z2 = av1_highbd_dr_prediction_z2_avx2; av1_highbd_dr_prediction_z3 = av1_highbd_dr_prediction_z3_c; if (flags & HAS_AVX2) av1_highbd_dr_prediction_z3 = av1_highbd_dr_prediction_z3_avx2; av1_highbd_inv_txfm_add = av1_highbd_inv_txfm_add_c; @@ -547,6 +551,7 @@ static void setup_rtcd_internal(void) if (flags & HAS_SSE4_1) av1_inv_txfm2d_add_8x8 = av1_inv_txfm2d_add_8x8_sse4_1; av1_inv_txfm_add = av1_inv_txfm_add_c; if (flags & HAS_SSSE3) av1_inv_txfm_add = av1_inv_txfm_add_ssse3; + if (flags & HAS_AVX2) av1_inv_txfm_add = av1_inv_txfm_add_avx2; av1_round_shift_array = av1_round_shift_array_c; if (flags & HAS_SSE4_1) av1_round_shift_array = av1_round_shift_array_sse4_1; av1_selfguided_restoration = av1_selfguided_restoration_c; diff --git a/third_party/aom/.clang-format b/third_party/aom/.clang-format index e76a526e4b..a378820073 100644 --- a/third_party/aom/.clang-format +++ b/third_party/aom/.clang-format @@ -1,7 +1,7 @@ --- Language: Cpp # BasedOnStyle: Google -# Generated with clang-format 5.0.0 +# Generated with clang-format 7.0.1 AccessModifierOffset: -1 AlignAfterOpenBracket: Align AlignConsecutiveAssignments: false @@ -12,7 +12,6 @@ AlignTrailingComments: true AllowAllParametersOfDeclarationOnNextLine: true AllowShortBlocksOnASingleLine: false AllowShortCaseLabelsOnASingleLine: true -AllowShortFunctionsOnASingleLine: All AllowShortIfStatementsOnASingleLine: true AllowShortLoopsOnASingleLine: true AlwaysBreakAfterDefinitionReturnType: None @@ -30,6 +29,7 @@ BraceWrapping: AfterObjCDeclaration: false AfterStruct: false AfterUnion: false + AfterExternBlock: false BeforeCatch: false BeforeElse: false IndentBraces: false @@ -39,6 +39,7 @@ BraceWrapping: BreakBeforeBinaryOperators: None BreakBeforeBraces: Attach BreakBeforeInheritanceComma: false +BreakInheritanceList: BeforeColon BreakBeforeTernaryOperators: true BreakConstructorInitializersBeforeComma: false BreakConstructorInitializers: BeforeColon @@ -59,7 +60,10 @@ ForEachMacros: - foreach - Q_FOREACH - BOOST_FOREACH +IncludeBlocks: Preserve IncludeCategories: + - Regex: '^' + Priority: 2 - Regex: '^<.*\.h>' Priority: 1 - Regex: '^<.*' @@ -68,6 +72,7 @@ IncludeCategories: Priority: 3 IncludeIsMainRegex: '([-_](test|unittest))?$' IndentCaseLabels: true +IndentPPDirectives: None IndentWidth: 2 IndentWrappedFunctionNames: false JavaScriptQuotes: Leave @@ -77,6 +82,7 @@ MacroBlockBegin: '' MacroBlockEnd: '' MaxEmptyLinesToKeep: 1 NamespaceIndentation: None +ObjCBinPackProtocolList: Never ObjCBlockIndentWidth: 2 ObjCSpaceAfterProperty: false ObjCSpaceBeforeProtocolList: false @@ -85,20 +91,53 @@ PenaltyBreakBeforeFirstCallParameter: 1 PenaltyBreakComment: 300 PenaltyBreakFirstLessLess: 120 PenaltyBreakString: 1000 +PenaltyBreakTemplateDeclaration: 10 PenaltyExcessCharacter: 1000000 PenaltyReturnTypeOnItsOwnLine: 200 PointerAlignment: Right +RawStringFormats: + - Language: Cpp + Delimiters: + - cc + - CC + - cpp + - Cpp + - CPP + - 'c++' + - 'C++' + CanonicalDelimiter: '' + BasedOnStyle: google + - Language: TextProto + Delimiters: + - pb + - PB + - proto + - PROTO + EnclosingFunctions: + - EqualsProto + - EquivToProto + - PARSE_PARTIAL_TEXT_PROTO + - PARSE_TEST_PROTO + - PARSE_TEXT_PROTO + - ParseTextOrDie + - ParseTextProtoOrDie + CanonicalDelimiter: '' + BasedOnStyle: google ReflowComments: true SortIncludes: false SortUsingDeclarations: true SpaceAfterCStyleCast: false SpaceAfterTemplateKeyword: true SpaceBeforeAssignmentOperators: true +SpaceBeforeCpp11BracedList: false +SpaceBeforeCtorInitializerColon: true +SpaceBeforeInheritanceColon: true SpaceBeforeParens: ControlStatements +SpaceBeforeRangeBasedForLoopColon: true SpaceInEmptyParentheses: false SpacesBeforeTrailingComments: 2 SpacesInAngles: false -SpacesInContainerLiterals: true +SpacesInContainerLiterals: false SpacesInCStyleCastParentheses: false SpacesInParentheses: false SpacesInSquareBrackets: false diff --git a/third_party/aom/aom/aomcx.h b/third_party/aom/aom/aomcx.h index da7498f6f6..2cf4ad731f 100644 --- a/third_party/aom/aom/aomcx.h +++ b/third_party/aom/aom/aomcx.h @@ -220,8 +220,8 @@ enum aome_enc_control_id { /*!\brief Codec control function to set constrained quality level. * - * \attention For this value to be used aom_codec_enc_cfg_t::g_usage must be - * set to #AOM_CQ. + * \attention For this value to be used aom_codec_enc_cfg_t::rc_end_usage + * must be set to #AOM_CQ. * \note Valid range: 0..63 */ AOME_SET_CQ_LEVEL, @@ -584,7 +584,8 @@ enum aome_enc_control_id { * * 0 = apply trellis quantization * 1 = do not apply trellis quantization - * 2 = disable trellis quantization partially + * 2 = disable trellis quantization in rd search + * 3 = disable trellis quantization in estimate yrd * * By default, the encoder applies optimization on quantized * coefficients. @@ -989,16 +990,23 @@ enum aome_enc_control_id { /*!\brief Codec control function to set the delta q mode * - * AV1 has a segment based feature that allows encoder to adaptively change - * quantization parameter for each segment within a frame to improve the - * subjective quality. the delta q mode is added on top of segment based - * feature, and allows control per 64x64 q and lf delta.This control makes - * encoder operate in one of the several DELTA_Q_modes supported. + * AV1 supports a delta q mode feature, that allows modulating q per + * superblock. This control makes encoder operate in one of several + * DELTA_Q_modes supported: + * 0: Not Supported + * 1: Use modulation to maximize objective quality + * 2: Use modulation to maximize perceptual quality * * By default, encoder operates with DELTAQ_Mode 0(deltaq signaling off). */ AV1E_SET_DELTAQ_MODE, + /*!\brief Codec control function to turn on/off loopfilter modulation + * when delta q modulation is enabled. Note AV1 only supports loopfilter + * modulation when delta q modulation is enabled as well. + */ + AV1E_SET_DELTALF_MODE, + /*!\brief Codec control function to set the single tile decoding mode to 0 or * 1. * @@ -1405,6 +1413,9 @@ AOM_CTRL_USE_TYPE(AV1E_SET_AQ_MODE, unsigned int) AOM_CTRL_USE_TYPE(AV1E_SET_DELTAQ_MODE, unsigned int) #define AOM_CTRL_AV1E_SET_DELTAQ_MODE +AOM_CTRL_USE_TYPE(AV1E_SET_DELTALF_MODE, unsigned int) +#define AOM_CTRL_AV1E_SET_DELTALF_MODE + AOM_CTRL_USE_TYPE(AV1E_SET_FRAME_PERIODIC_BOOST, unsigned int) #define AOM_CTRL_AV1E_SET_FRAME_PERIODIC_BOOST diff --git a/third_party/aom/aom_dsp/aom_dsp.cmake b/third_party/aom/aom_dsp/aom_dsp.cmake index abf6a60562..4829ee79c0 100644 --- a/third_party/aom/aom_dsp/aom_dsp.cmake +++ b/third_party/aom/aom_dsp/aom_dsp.cmake @@ -207,6 +207,7 @@ if(CONFIG_AV1_ENCODER) "${AOM_ROOT}/aom_dsp/x86/masked_sad_intrin_avx2.c" "${AOM_ROOT}/aom_dsp/x86/subtract_avx2.c" "${AOM_ROOT}/aom_dsp/x86/highbd_quantize_intrin_avx2.c" + "${AOM_ROOT}/aom_dsp/x86/adaptive_quantize_avx2.c" "${AOM_ROOT}/aom_dsp/x86/sad4d_avx2.c" "${AOM_ROOT}/aom_dsp/x86/sad_avx2.c" "${AOM_ROOT}/aom_dsp/x86/sad_highbd_avx2.c" diff --git a/third_party/aom/aom_dsp/aom_dsp_rtcd_defs.pl b/third_party/aom/aom_dsp/aom_dsp_rtcd_defs.pl index f56a1170c2..a0ca999528 100755 --- a/third_party/aom/aom_dsp/aom_dsp_rtcd_defs.pl +++ b/third_party/aom/aom_dsp/aom_dsp_rtcd_defs.pl @@ -519,7 +519,7 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") { specialize qw/aom_quantize_b sse2/, "$ssse3_x86_64", "$avx_x86_64"; add_proto qw/void aom_quantize_b_adaptive/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan"; - specialize qw/aom_quantize_b_adaptive sse2/; + specialize qw/aom_quantize_b_adaptive sse2 avx2/; add_proto qw/void aom_quantize_b_32x32/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan"; specialize qw/aom_quantize_b_32x32/, "$ssse3_x86_64", "$avx_x86_64"; @@ -529,6 +529,9 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") { add_proto qw/void aom_quantize_b_64x64/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan"; specialize qw/aom_quantize_b_64x64 ssse3/; + + add_proto qw/void aom_quantize_b_64x64_adaptive/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan"; + specialize qw/aom_quantize_b_64x64_adaptive sse2/; } # CONFIG_AV1_ENCODER if (aom_config("CONFIG_AV1_ENCODER") eq "yes") { diff --git a/third_party/aom/aom_dsp/grain_synthesis.c b/third_party/aom/aom_dsp/grain_synthesis.c index 4b94dbc1ac..626eb76af0 100644 --- a/third_party/aom/aom_dsp/grain_synthesis.c +++ b/third_party/aom/aom_dsp/grain_synthesis.c @@ -1078,7 +1078,7 @@ int av1_add_film_grain_run(const aom_film_grain_t *params, uint8_t *luma, const int grain_center = 128 << (bit_depth - 8); grain_min = 0 - grain_center; - grain_max = (256 << (bit_depth - 8)) - 1 - grain_center; + grain_max = grain_center - 1; init_arrays(params, luma_stride, chroma_stride, &pred_pos_luma, &pred_pos_chroma, &luma_grain_block, &cb_grain_block, diff --git a/third_party/aom/aom_dsp/noise_model.c b/third_party/aom/aom_dsp/noise_model.c index 2faee85063..528efd94b1 100644 --- a/third_party/aom/aom_dsp/noise_model.c +++ b/third_party/aom/aom_dsp/noise_model.c @@ -214,6 +214,7 @@ static void set_chroma_coefficient_fallback_soln(aom_equation_system_t *eqns) { int aom_noise_strength_lut_init(aom_noise_strength_lut_t *lut, int num_points) { if (!lut) return 0; + lut->num_points = 0; lut->points = (double(*)[2])aom_malloc(num_points * sizeof(*lut->points)); if (!lut->points) return 0; lut->num_points = num_points; @@ -426,6 +427,9 @@ int aom_flat_block_finder_init(aom_flat_block_finder_t *block_finder, double *AtA_inv = 0; double *A = 0; int x = 0, y = 0, i = 0, j = 0; + block_finder->A = NULL; + block_finder->AtA_inv = NULL; + if (!equation_system_init(&eqns, kLowPolyNumParams)) { fprintf(stderr, "Failed to init equation system for block_size=%d\n", block_size); diff --git a/third_party/aom/aom_dsp/psnrhvs.c b/third_party/aom/aom_dsp/psnrhvs.c index 30fe21d9ce..2c3d5f7895 100644 --- a/third_party/aom/aom_dsp/psnrhvs.c +++ b/third_party/aom/aom_dsp/psnrhvs.c @@ -140,7 +140,7 @@ static double calc_psnrhvs(const unsigned char *src, int _systride, been normalized and then squared." Their CSF matrix (from PSNR-HVS) was also constructed from the JPEG matrices. I can not find any obvious scheme of normalizing to produce their table, but if I multiply their - CSF by 0.38857 and square the result I get their masking table. + CSF by 0.3885746225901003 and square the result I get their masking table. I have no idea where this constant comes from, but deviating from it too greatly hurts MOS agreement. @@ -148,11 +148,15 @@ static double calc_psnrhvs(const unsigned char *src, int _systride, Jaakko Astola, Vladimir Lukin, "On between-coefficient contrast masking of DCT basis functions", CD-ROM Proceedings of the Third International Workshop on Video Processing and Quality Metrics for Consumer - Electronics VPQM-07, Scottsdale, Arizona, USA, 25-26 January, 2007, 4 p.*/ + Electronics VPQM-07, Scottsdale, Arizona, USA, 25-26 January, 2007, 4 p. + + Suggested in aomedia issue#2363: + 0.3885746225901003 is a reciprocal of the maximum coefficient (2.573509) + of the old JPEG based matrix from the paper. Since you are not using that, + divide by actual maximum coefficient. */ for (x = 0; x < 8; x++) for (y = 0; y < 8; y++) - mask[x][y] = - (_csf[x][y] * 0.3885746225901003) * (_csf[x][y] * 0.3885746225901003); + mask[x][y] = (_csf[x][y] / _csf[1][0]) * (_csf[x][y] / _csf[1][0]); for (y = 0; y < _h - 7; y += _step) { for (x = 0; x < _w - 7; x += _step) { int i; diff --git a/third_party/aom/aom_dsp/x86/adaptive_quantize_avx2.c b/third_party/aom/aom_dsp/x86/adaptive_quantize_avx2.c new file mode 100644 index 0000000000..3335087198 --- /dev/null +++ b/third_party/aom/aom_dsp/x86/adaptive_quantize_avx2.c @@ -0,0 +1,244 @@ +/* + * Copyright (c) 2019, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include +#include "config/aom_dsp_rtcd.h" +#include "aom/aom_integer.h" +#include "av1/encoder/av1_quantize.h" +#include "aom_dsp/x86/quantize_x86.h" + +static INLINE void load_b_values_avx2(const int16_t *zbin_ptr, __m256i *zbin, + const int16_t *round_ptr, __m256i *round, + const int16_t *quant_ptr, __m256i *quant, + const int16_t *dequant_ptr, + __m256i *dequant, + const int16_t *shift_ptr, + __m256i *shift) { + *zbin = _mm256_castsi128_si256(_mm_load_si128((const __m128i *)zbin_ptr)); + *zbin = _mm256_permute4x64_epi64(*zbin, 0x54); + *zbin = _mm256_sub_epi16(*zbin, _mm256_set1_epi16(1)); + *round = _mm256_castsi128_si256(_mm_load_si128((const __m128i *)round_ptr)); + *round = _mm256_permute4x64_epi64(*round, 0x54); + *quant = _mm256_castsi128_si256(_mm_load_si128((const __m128i *)quant_ptr)); + *quant = _mm256_permute4x64_epi64(*quant, 0x54); + *dequant = + _mm256_castsi128_si256(_mm_load_si128((const __m128i *)dequant_ptr)); + *dequant = _mm256_permute4x64_epi64(*dequant, 0x54); + *shift = _mm256_castsi128_si256(_mm_load_si128((const __m128i *)shift_ptr)); + *shift = _mm256_permute4x64_epi64(*shift, 0x54); +} + +static INLINE __m256i load_coefficients_avx2(const tran_low_t *coeff_ptr) { + const __m256i coeff1 = _mm256_load_si256((__m256i *)(coeff_ptr)); + const __m256i coeff2 = _mm256_load_si256((__m256i *)(coeff_ptr + 8)); + return _mm256_packs_epi32(coeff1, coeff2); +} + +static INLINE void update_mask1_avx2(__m256i *cmp_mask, + const int16_t *iscan_ptr, int *is_found, + __m256i *mask) { + __m256i temp_mask = _mm256_setzero_si256(); + if (_mm256_movemask_epi8(*cmp_mask)) { + __m256i iscan = _mm256_loadu_si256((const __m256i *)(iscan_ptr)); + temp_mask = _mm256_and_si256(*cmp_mask, iscan); + *is_found = 1; + } + *mask = _mm256_max_epi16(temp_mask, *mask); +} + +static INLINE void update_mask0_avx2(__m256i *qcoeff, __m256i *threshold, + const int16_t *iscan_ptr, int *is_found, + __m256i *mask) { + __m256i zero = _mm256_setzero_si256(); + __m256i coeff[2], cmp_mask0, cmp_mask1; + coeff[0] = _mm256_unpacklo_epi16(*qcoeff, zero); + coeff[1] = _mm256_unpackhi_epi16(*qcoeff, zero); + coeff[0] = _mm256_slli_epi32(coeff[0], AOM_QM_BITS); + cmp_mask0 = _mm256_cmpgt_epi32(coeff[0], threshold[0]); + coeff[1] = _mm256_slli_epi32(coeff[1], AOM_QM_BITS); + cmp_mask1 = _mm256_cmpgt_epi32(coeff[1], threshold[1]); + cmp_mask0 = + _mm256_permute4x64_epi64(_mm256_packs_epi32(cmp_mask0, cmp_mask1), 0xd8); + update_mask1_avx2(&cmp_mask0, iscan_ptr, is_found, mask); +} + +static INLINE void calculate_qcoeff_avx2(__m256i *coeff, const __m256i *round, + const __m256i *quant, + const __m256i *shift) { + __m256i tmp, qcoeff; + qcoeff = _mm256_adds_epi16(*coeff, *round); + tmp = _mm256_mulhi_epi16(qcoeff, *quant); + qcoeff = _mm256_add_epi16(tmp, qcoeff); + *coeff = _mm256_mulhi_epi16(qcoeff, *shift); +} + +static INLINE __m256i calculate_dqcoeff_avx2(__m256i qcoeff, __m256i dequant) { + return _mm256_mullo_epi16(qcoeff, dequant); +} + +static INLINE void store_coefficients_avx2(__m256i coeff_vals, + tran_low_t *coeff_ptr) { + __m256i coeff_sign = _mm256_srai_epi16(coeff_vals, 15); + __m256i coeff_vals_lo = _mm256_unpacklo_epi16(coeff_vals, coeff_sign); + __m256i coeff_vals_hi = _mm256_unpackhi_epi16(coeff_vals, coeff_sign); + _mm256_store_si256((__m256i *)(coeff_ptr), coeff_vals_lo); + _mm256_store_si256((__m256i *)(coeff_ptr + 8), coeff_vals_hi); +} + +void aom_quantize_b_adaptive_avx2( + const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, + const int16_t *round_ptr, const int16_t *quant_ptr, + const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, + tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, + const int16_t *scan, const int16_t *iscan) { + int index = 16; + int non_zero_count = 0; + int non_zero_count_prescan_add_zero = 0; + int is_found0 = 0, is_found1 = 0; + int eob = -1; + const __m256i zero = _mm256_setzero_si256(); + __m256i zbin, round, quant, dequant, shift; + __m256i coeff, qcoeff; + __m256i cmp_mask, mask0 = zero, mask1 = zero; + __m128i temp_mask0, temp_mask1; + int prescan_add[2]; + int thresh[2]; + const qm_val_t wt = (1 << AOM_QM_BITS); + for (int i = 0; i < 2; ++i) { + prescan_add[i] = ROUND_POWER_OF_TWO(dequant_ptr[i] * EOB_FACTOR, 7); + thresh[i] = (zbin_ptr[i] * wt + prescan_add[i]) - 1; + } + __m256i threshold[2]; + threshold[0] = _mm256_set1_epi32(thresh[0]); + threshold[1] = _mm256_set1_epi32(thresh[1]); + threshold[0] = _mm256_blend_epi32(threshold[0], threshold[1], 0xfe); + +#if SKIP_EOB_FACTOR_ADJUST + int first = -1; +#endif + + // Setup global values. + load_b_values_avx2(zbin_ptr, &zbin, round_ptr, &round, quant_ptr, &quant, + dequant_ptr, &dequant, quant_shift_ptr, &shift); + + // Do DC and first 15 AC. + coeff = load_coefficients_avx2(coeff_ptr); + qcoeff = _mm256_abs_epi16(coeff); + update_mask0_avx2(&qcoeff, threshold, iscan, &is_found0, &mask0); + __m256i temp0 = _mm256_cmpgt_epi16(qcoeff, zbin); + zbin = _mm256_unpackhi_epi64(zbin, zbin); + cmp_mask = _mm256_permute4x64_epi64(temp0, 0xd8); + update_mask1_avx2(&cmp_mask, iscan, &is_found1, &mask1); + threshold[0] = threshold[1]; + if (_mm256_movemask_epi8(cmp_mask) == 0) { + _mm256_store_si256((__m256i *)(qcoeff_ptr), zero); + _mm256_store_si256((__m256i *)(qcoeff_ptr + 8), zero); + _mm256_store_si256((__m256i *)(dqcoeff_ptr), zero); + _mm256_store_si256((__m256i *)(dqcoeff_ptr + 8), zero); + round = _mm256_unpackhi_epi64(round, round); + quant = _mm256_unpackhi_epi64(quant, quant); + shift = _mm256_unpackhi_epi64(shift, shift); + dequant = _mm256_unpackhi_epi64(dequant, dequant); + } else { + calculate_qcoeff_avx2(&qcoeff, &round, &quant, &shift); + round = _mm256_unpackhi_epi64(round, round); + quant = _mm256_unpackhi_epi64(quant, quant); + shift = _mm256_unpackhi_epi64(shift, shift); + // Reinsert signs + qcoeff = _mm256_sign_epi16(qcoeff, coeff); + // Mask out zbin threshold coeffs + qcoeff = _mm256_and_si256(qcoeff, temp0); + store_coefficients_avx2(qcoeff, qcoeff_ptr); + coeff = calculate_dqcoeff_avx2(qcoeff, dequant); + dequant = _mm256_unpackhi_epi64(dequant, dequant); + store_coefficients_avx2(coeff, dqcoeff_ptr); + } + + // AC only loop. + while (index < n_coeffs) { + coeff = load_coefficients_avx2(coeff_ptr + index); + qcoeff = _mm256_abs_epi16(coeff); + update_mask0_avx2(&qcoeff, threshold, iscan + index, &is_found0, &mask0); + temp0 = _mm256_cmpgt_epi16(qcoeff, zbin); + cmp_mask = _mm256_permute4x64_epi64(temp0, 0xd8); + update_mask1_avx2(&cmp_mask, iscan + index, &is_found1, &mask1); + if (_mm256_movemask_epi8(cmp_mask) == 0) { + _mm256_store_si256((__m256i *)(qcoeff_ptr + index), zero); + _mm256_store_si256((__m256i *)(qcoeff_ptr + index + 8), zero); + _mm256_store_si256((__m256i *)(dqcoeff_ptr + index), zero); + _mm256_store_si256((__m256i *)(dqcoeff_ptr + index + 8), zero); + index += 16; + continue; + } + calculate_qcoeff_avx2(&qcoeff, &round, &quant, &shift); + qcoeff = _mm256_sign_epi16(qcoeff, coeff); + qcoeff = _mm256_and_si256(qcoeff, temp0); + store_coefficients_avx2(qcoeff, qcoeff_ptr + index); + coeff = calculate_dqcoeff_avx2(qcoeff, dequant); + store_coefficients_avx2(coeff, dqcoeff_ptr + index); + index += 16; + } + if (is_found0) { + temp_mask0 = _mm_max_epi16(_mm256_castsi256_si128(mask0), + _mm256_extracti128_si256(mask0, 1)); + non_zero_count = calculate_non_zero_count(temp_mask0); + } + if (is_found1) { + temp_mask1 = _mm_max_epi16(_mm256_castsi256_si128(mask1), + _mm256_extracti128_si256(mask1, 1)); + non_zero_count_prescan_add_zero = calculate_non_zero_count(temp_mask1); + } + + for (int i = non_zero_count_prescan_add_zero - 1; i >= non_zero_count; i--) { + const int rc = scan[i]; + qcoeff_ptr[rc] = 0; + dqcoeff_ptr[rc] = 0; + } + + for (int i = non_zero_count - 1; i >= 0; i--) { + const int rc = scan[i]; + if (qcoeff_ptr[rc]) { + eob = i; + break; + } + } + + *eob_ptr = eob + 1; +#if SKIP_EOB_FACTOR_ADJUST + // TODO(Aniket): Experiment the following loop with intrinsic by combining + // with the quantization loop above + for (int i = 0; i < non_zero_count; i++) { + const int rc = scan[i]; + const int qcoeff0 = qcoeff_ptr[rc]; + if (qcoeff0) { + first = i; + break; + } + } + if ((*eob_ptr - 1) >= 0 && first == (*eob_ptr - 1)) { + const int rc = scan[(*eob_ptr - 1)]; + if (qcoeff_ptr[rc] == 1 || qcoeff_ptr[rc] == -1) { + const int coeff0 = coeff_ptr[rc] * wt; + const int coeff_sign = (coeff0 >> 31); + const int abs_coeff = (coeff0 ^ coeff_sign) - coeff_sign; + const int factor = EOB_FACTOR + SKIP_EOB_FACTOR_ADJUST; + const int prescan_add_val = + ROUND_POWER_OF_TWO(dequant_ptr[rc != 0] * factor, 7); + if (abs_coeff < + (zbin_ptr[rc != 0] * (1 << AOM_QM_BITS) + prescan_add_val)) { + qcoeff_ptr[rc] = 0; + dqcoeff_ptr[rc] = 0; + *eob_ptr = 0; + } + } + } +#endif +} diff --git a/third_party/aom/aom_dsp/x86/adaptive_quantize_sse2.c b/third_party/aom/aom_dsp/x86/adaptive_quantize_sse2.c index 3822c27aaf..2fc700d7d7 100644 --- a/third_party/aom/aom_dsp/x86/adaptive_quantize_sse2.c +++ b/third_party/aom/aom_dsp/x86/adaptive_quantize_sse2.c @@ -22,41 +22,31 @@ void aom_quantize_b_adaptive_sse2( const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan) { - const __m128i zero = _mm_setzero_si128(); int index = 16; - int non_zero_count = (int)n_coeffs; + int non_zero_count = 0; + int non_zero_count_prescan_add_zero = 0; + int is_found0 = 0, is_found1 = 0; + int eob = -1; + const __m128i zero = _mm_setzero_si128(); __m128i zbin, round, quant, dequant, shift; __m128i coeff0, coeff1, coeff0_sign, coeff1_sign; __m128i qcoeff0, qcoeff1; __m128i cmp_mask0, cmp_mask1; - __m128i eob = zero, eob0, prescan0, prescan1, all_zero; - const int zbins[2] = { ROUND_POWER_OF_TWO(zbin_ptr[0], 0), - ROUND_POWER_OF_TWO(zbin_ptr[1], 0) }; + __m128i all_zero; + __m128i mask0 = zero, mask1 = zero; int prescan_add[2]; - for (int i = 0; i < 2; ++i) + int thresh[4]; + const qm_val_t wt = (1 << AOM_QM_BITS); + for (int i = 0; i < 2; ++i) { prescan_add[i] = ROUND_POWER_OF_TWO(dequant_ptr[i] * EOB_FACTOR, 7); - - // max buffer is of size 256 as this functions calls with - // maximum n_coeffs as 256 - int16_t prescan[256]; - memset(prescan, -1, n_coeffs * sizeof(int16_t)); - - // TODO(Aniket): Experiment the following loop with intrinsic - for (int i = (int)n_coeffs - 1; i >= 0; i--) { - const int rc = scan[i]; - const qm_val_t wt = 1 << AOM_QM_BITS; - const int coeff = coeff_ptr[rc] * wt; - const int coeff_sign = (coeff >> 31); - const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign; - const int prescan_add_val = prescan_add[rc != 0]; - if (abs_coeff < (zbins[rc != 0] * (1 << AOM_QM_BITS) + prescan_add_val)) { - prescan[rc] = 0; - non_zero_count--; - } else { - break; - } + thresh[i] = (zbin_ptr[i] * wt + prescan_add[i]) - 1; } + thresh[2] = thresh[3] = thresh[1]; + __m128i threshold[2]; + threshold[0] = _mm_loadu_si128((__m128i *)&thresh[0]); + threshold[1] = _mm_unpackhi_epi64(threshold[0], threshold[0]); + #if SKIP_EOB_FACTOR_ADJUST int first = -1; #endif @@ -74,13 +64,15 @@ void aom_quantize_b_adaptive_sse2( qcoeff0 = invert_sign_sse2(coeff0, coeff0_sign); qcoeff1 = invert_sign_sse2(coeff1, coeff1_sign); - prescan0 = _mm_loadu_si128((const __m128i *)prescan); - prescan1 = _mm_loadu_si128((const __m128i *)(prescan + 8)); + update_mask0(&qcoeff0, &qcoeff1, threshold, iscan, &is_found0, &mask0); - cmp_mask0 = _mm_and_si128(prescan0, _mm_cmpgt_epi16(qcoeff0, zbin)); + cmp_mask0 = _mm_cmpgt_epi16(qcoeff0, zbin); zbin = _mm_unpackhi_epi64(zbin, zbin); // Switch DC to AC - cmp_mask1 = _mm_and_si128(prescan1, _mm_cmpgt_epi16(qcoeff1, zbin)); + cmp_mask1 = _mm_cmpgt_epi16(qcoeff1, zbin); + update_mask1(&cmp_mask0, &cmp_mask1, iscan, &is_found1, &mask1); + + threshold[0] = threshold[1]; all_zero = _mm_or_si128(cmp_mask0, cmp_mask1); if (_mm_movemask_epi8(all_zero) == 0) { _mm_store_si128((__m128i *)(qcoeff_ptr), zero); @@ -121,13 +113,9 @@ void aom_quantize_b_adaptive_sse2( store_coefficients(coeff0, dqcoeff_ptr); store_coefficients(coeff1, dqcoeff_ptr + 8); - - eob = scan_for_eob(&coeff0, &coeff1, cmp_mask0, cmp_mask1, iscan, 0, zero); } // AC only loop. - // TODO(Aniket): Reduce the processing of coeff quatization - // based on eob logic while (index < n_coeffs) { coeff0 = load_coefficients(coeff_ptr + index); coeff1 = load_coefficients(coeff_ptr + index + 8); @@ -137,11 +125,13 @@ void aom_quantize_b_adaptive_sse2( qcoeff0 = invert_sign_sse2(coeff0, coeff0_sign); qcoeff1 = invert_sign_sse2(coeff1, coeff1_sign); - prescan0 = _mm_loadu_si128((const __m128i *)(prescan + index)); - prescan1 = _mm_loadu_si128((const __m128i *)(prescan + index + 8)); + update_mask0(&qcoeff0, &qcoeff1, threshold, iscan + index, &is_found0, + &mask0); - cmp_mask0 = _mm_and_si128(prescan0, _mm_cmpgt_epi16(qcoeff0, zbin)); - cmp_mask1 = _mm_and_si128(prescan1, _mm_cmpgt_epi16(qcoeff1, zbin)); + cmp_mask0 = _mm_cmpgt_epi16(qcoeff0, zbin); + cmp_mask1 = _mm_cmpgt_epi16(qcoeff1, zbin); + + update_mask1(&cmp_mask0, &cmp_mask1, iscan + index, &is_found1, &mask1); all_zero = _mm_or_si128(cmp_mask0, cmp_mask1); if (_mm_movemask_epi8(all_zero) == 0) { @@ -174,14 +164,27 @@ void aom_quantize_b_adaptive_sse2( store_coefficients(coeff0, dqcoeff_ptr + index); store_coefficients(coeff1, dqcoeff_ptr + index + 8); - eob0 = scan_for_eob(&coeff0, &coeff1, cmp_mask0, cmp_mask1, iscan, index, - zero); - eob = _mm_max_epi16(eob, eob0); index += 16; } + if (is_found0) non_zero_count = calculate_non_zero_count(mask0); + if (is_found1) + non_zero_count_prescan_add_zero = calculate_non_zero_count(mask1); - *eob_ptr = accumulate_eob(eob); + for (int i = non_zero_count_prescan_add_zero - 1; i >= non_zero_count; i--) { + const int rc = scan[i]; + qcoeff_ptr[rc] = 0; + dqcoeff_ptr[rc] = 0; + } + for (int i = non_zero_count - 1; i >= 0; i--) { + const int rc = scan[i]; + if (qcoeff_ptr[rc]) { + eob = i; + break; + } + } + + *eob_ptr = eob + 1; #if SKIP_EOB_FACTOR_ADJUST // TODO(Aniket): Experiment the following loop with intrinsic by combining // with the quantization loop above @@ -196,14 +199,14 @@ void aom_quantize_b_adaptive_sse2( if ((*eob_ptr - 1) >= 0 && first == (*eob_ptr - 1)) { const int rc = scan[(*eob_ptr - 1)]; if (qcoeff_ptr[rc] == 1 || qcoeff_ptr[rc] == -1) { - const qm_val_t wt = (1 << AOM_QM_BITS); const int coeff = coeff_ptr[rc] * wt; const int coeff_sign = (coeff >> 31); const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign; const int factor = EOB_FACTOR + SKIP_EOB_FACTOR_ADJUST; const int prescan_add_val = ROUND_POWER_OF_TWO(dequant_ptr[rc != 0] * factor, 7); - if (abs_coeff < (zbins[rc != 0] * (1 << AOM_QM_BITS) + prescan_add_val)) { + if (abs_coeff < + (zbin_ptr[rc != 0] * (1 << AOM_QM_BITS) + prescan_add_val)) { qcoeff_ptr[rc] = 0; dqcoeff_ptr[rc] = 0; *eob_ptr = 0; @@ -220,8 +223,11 @@ void aom_quantize_b_32x32_adaptive_sse2( tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan) { int index = 16; - int non_zero_count = (int)n_coeffs; const int log_scale = 1; + int non_zero_count = 0; + int non_zero_count_prescan_add_zero = 0; + int is_found0 = 0, is_found1 = 0; + int eob = -1; const __m128i zero = _mm_setzero_si128(); const __m128i one = _mm_set1_epi16(1); const __m128i log_scale_vec = _mm_set1_epi16(log_scale); @@ -229,34 +235,23 @@ void aom_quantize_b_32x32_adaptive_sse2( __m128i coeff0, coeff1, coeff0_sign, coeff1_sign; __m128i qcoeff0, qcoeff1; __m128i cmp_mask0, cmp_mask1; - __m128i eob = zero, eob0, prescan0, prescan1, all_zero; + __m128i all_zero; + __m128i mask0 = zero, mask1 = zero; + const int zbins[2] = { ROUND_POWER_OF_TWO(zbin_ptr[0], log_scale), ROUND_POWER_OF_TWO(zbin_ptr[1], log_scale) }; - int prescan_add[2]; - for (int i = 0; i < 2; ++i) + int thresh[4]; + const qm_val_t wt = (1 << AOM_QM_BITS); + for (int i = 0; i < 2; ++i) { prescan_add[i] = ROUND_POWER_OF_TWO(dequant_ptr[i] * EOB_FACTOR, 7); - - // max buffer is of size 1024 as this functions calls with - // maximum n_coeffs as 1024 - int16_t prescan[1024]; - memset(prescan, -1, n_coeffs * sizeof(int16_t)); - - // TODO(Aniket): Experiment the following loop with intrinsic - for (int i = (int)n_coeffs - 1; i >= 0; i--) { - const int rc = scan[i]; - const qm_val_t wt = 1 << AOM_QM_BITS; - const int coeff = coeff_ptr[rc] * wt; - const int coeff_sign = (coeff >> 31); - const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign; - const int prescan_add_val = prescan_add[rc != 0]; - if (abs_coeff < (zbins[rc != 0] * (1 << AOM_QM_BITS) + prescan_add_val)) { - prescan[rc] = 0; - non_zero_count--; - } else { - break; - } + thresh[i] = (zbins[i] * wt + prescan_add[i]) - 1; } + thresh[2] = thresh[3] = thresh[1]; + __m128i threshold[2]; + threshold[0] = _mm_loadu_si128((__m128i *)&thresh[0]); + threshold[1] = _mm_unpackhi_epi64(threshold[0], threshold[0]); + #if SKIP_EOB_FACTOR_ADJUST int first = -1; #endif @@ -273,6 +268,7 @@ void aom_quantize_b_32x32_adaptive_sse2( zbin = _mm_srli_epi16(zbin, log_scale); round = _mm_srli_epi16(round, log_scale); zbin = _mm_sub_epi16(zbin, one); + // Do DC and first 15 AC. coeff0 = load_coefficients(coeff_ptr); coeff1 = load_coefficients(coeff_ptr + 8); @@ -282,13 +278,15 @@ void aom_quantize_b_32x32_adaptive_sse2( qcoeff0 = invert_sign_sse2(coeff0, coeff0_sign); qcoeff1 = invert_sign_sse2(coeff1, coeff1_sign); - prescan0 = _mm_loadu_si128((const __m128i *)prescan); - prescan1 = _mm_loadu_si128((const __m128i *)(prescan + 8)); + update_mask0(&qcoeff0, &qcoeff1, threshold, iscan, &is_found0, &mask0); - cmp_mask0 = _mm_and_si128(prescan0, _mm_cmpgt_epi16(qcoeff0, zbin)); + cmp_mask0 = _mm_cmpgt_epi16(qcoeff0, zbin); zbin = _mm_unpackhi_epi64(zbin, zbin); // Switch DC to AC - cmp_mask1 = _mm_and_si128(prescan1, _mm_cmpgt_epi16(qcoeff1, zbin)); + cmp_mask1 = _mm_cmpgt_epi16(qcoeff1, zbin); + update_mask1(&cmp_mask0, &cmp_mask1, iscan, &is_found1, &mask1); + + threshold[0] = threshold[1]; all_zero = _mm_or_si128(cmp_mask0, cmp_mask1); if (_mm_movemask_epi8(all_zero) == 0) { _mm_store_si128((__m128i *)(qcoeff_ptr), zero); @@ -305,11 +303,9 @@ void aom_quantize_b_32x32_adaptive_sse2( dequant = _mm_unpackhi_epi64(dequant, dequant); } else { calculate_qcoeff_log_scale(&qcoeff0, round, quant, &shift, &log_scale); - round = _mm_unpackhi_epi64(round, round); quant = _mm_unpackhi_epi64(quant, quant); shift = _mm_unpackhi_epi64(shift, shift); - calculate_qcoeff_log_scale(&qcoeff1, round, quant, &shift, &log_scale); // Reinsert signs @@ -328,14 +324,9 @@ void aom_quantize_b_32x32_adaptive_sse2( dequant = _mm_unpackhi_epi64(dequant, dequant); calculate_dqcoeff_and_store_log_scale(qcoeff1, dequant, zero, dqcoeff_ptr + 8, &log_scale); - - eob = - scan_for_eob(&qcoeff0, &qcoeff1, cmp_mask0, cmp_mask1, iscan, 0, zero); } // AC only loop. - // TODO(Aniket): Reduce the processing of coeff quatization - // based on eob logic while (index < n_coeffs) { coeff0 = load_coefficients(coeff_ptr + index); coeff1 = load_coefficients(coeff_ptr + index + 8); @@ -345,11 +336,13 @@ void aom_quantize_b_32x32_adaptive_sse2( qcoeff0 = invert_sign_sse2(coeff0, coeff0_sign); qcoeff1 = invert_sign_sse2(coeff1, coeff1_sign); - prescan0 = _mm_loadu_si128((const __m128i *)(prescan + index)); - prescan1 = _mm_loadu_si128((const __m128i *)(prescan + index + 8)); + update_mask0(&qcoeff0, &qcoeff1, threshold, iscan + index, &is_found0, + &mask0); - cmp_mask0 = _mm_and_si128(prescan0, _mm_cmpgt_epi16(qcoeff0, zbin)); - cmp_mask1 = _mm_and_si128(prescan1, _mm_cmpgt_epi16(qcoeff1, zbin)); + cmp_mask0 = _mm_cmpgt_epi16(qcoeff0, zbin); + cmp_mask1 = _mm_cmpgt_epi16(qcoeff1, zbin); + + update_mask1(&cmp_mask0, &cmp_mask1, iscan + index, &is_found1, &mask1); all_zero = _mm_or_si128(cmp_mask0, cmp_mask1); if (_mm_movemask_epi8(all_zero) == 0) { @@ -380,15 +373,235 @@ void aom_quantize_b_32x32_adaptive_sse2( dqcoeff_ptr + index, &log_scale); calculate_dqcoeff_and_store_log_scale(qcoeff1, dequant, zero, dqcoeff_ptr + index + 8, &log_scale); - - eob0 = scan_for_eob(&qcoeff0, &qcoeff1, cmp_mask0, cmp_mask1, iscan, index, - zero); - eob = _mm_max_epi16(eob, eob0); index += 16; } + if (is_found0) non_zero_count = calculate_non_zero_count(mask0); + if (is_found1) + non_zero_count_prescan_add_zero = calculate_non_zero_count(mask1); - *eob_ptr = accumulate_eob(eob); + for (int i = non_zero_count_prescan_add_zero - 1; i >= non_zero_count; i--) { + const int rc = scan[i]; + qcoeff_ptr[rc] = 0; + dqcoeff_ptr[rc] = 0; + } + for (int i = non_zero_count - 1; i >= 0; i--) { + const int rc = scan[i]; + if (qcoeff_ptr[rc]) { + eob = i; + break; + } + } + + *eob_ptr = eob + 1; +#if SKIP_EOB_FACTOR_ADJUST + // TODO(Aniket): Experiment the following loop with intrinsic by combining + // with the quantization loop above + for (int i = 0; i < non_zero_count; i++) { + const int rc = scan[i]; + const int qcoeff = qcoeff_ptr[rc]; + if (qcoeff) { + first = i; + break; + } + } + if ((*eob_ptr - 1) >= 0 && first == (*eob_ptr - 1)) { + const int rc = scan[(*eob_ptr - 1)]; + if (qcoeff_ptr[rc] == 1 || qcoeff_ptr[rc] == -1) { + const int coeff = coeff_ptr[rc] * wt; + const int coeff_sign = (coeff >> 31); + const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign; + const int factor = EOB_FACTOR + SKIP_EOB_FACTOR_ADJUST; + const int prescan_add_val = + ROUND_POWER_OF_TWO(dequant_ptr[rc != 0] * factor, 7); + if (abs_coeff < (zbins[rc != 0] * (1 << AOM_QM_BITS) + prescan_add_val)) { + qcoeff_ptr[rc] = 0; + dqcoeff_ptr[rc] = 0; + *eob_ptr = 0; + } + } + } +#endif +} + +void aom_quantize_b_64x64_adaptive_sse2( + const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, + const int16_t *round_ptr, const int16_t *quant_ptr, + const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, + tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, + const int16_t *scan, const int16_t *iscan) { + int index = 16; + const int log_scale = 2; + int non_zero_count = 0; + int non_zero_count_prescan_add_zero = 0; + int is_found0 = 0, is_found1 = 0; + int eob = -1; + const __m128i zero = _mm_setzero_si128(); + const __m128i one = _mm_set1_epi16(1); + const __m128i log_scale_vec = _mm_set1_epi16(log_scale); + __m128i zbin, round, quant, dequant, shift; + __m128i coeff0, coeff1, coeff0_sign, coeff1_sign; + __m128i qcoeff0, qcoeff1; + __m128i cmp_mask0, cmp_mask1; + __m128i all_zero; + __m128i mask0 = zero, mask1 = zero; + + const int zbins[2] = { ROUND_POWER_OF_TWO(zbin_ptr[0], log_scale), + ROUND_POWER_OF_TWO(zbin_ptr[1], log_scale) }; + int prescan_add[2]; + int thresh[4]; + const qm_val_t wt = (1 << AOM_QM_BITS); + for (int i = 0; i < 2; ++i) { + prescan_add[i] = ROUND_POWER_OF_TWO(dequant_ptr[i] * EOB_FACTOR, 7); + thresh[i] = (zbins[i] * wt + prescan_add[i]) - 1; + } + thresh[2] = thresh[3] = thresh[1]; + __m128i threshold[2]; + threshold[0] = _mm_loadu_si128((__m128i *)&thresh[0]); + threshold[1] = _mm_unpackhi_epi64(threshold[0], threshold[0]); + +#if SKIP_EOB_FACTOR_ADJUST + int first = -1; +#endif + // Setup global values. + zbin = _mm_load_si128((const __m128i *)zbin_ptr); + round = _mm_load_si128((const __m128i *)round_ptr); + quant = _mm_load_si128((const __m128i *)quant_ptr); + dequant = _mm_load_si128((const __m128i *)dequant_ptr); + shift = _mm_load_si128((const __m128i *)quant_shift_ptr); + + // Shift with rounding. + zbin = _mm_add_epi16(zbin, log_scale_vec); + round = _mm_add_epi16(round, log_scale_vec); + zbin = _mm_srli_epi16(zbin, log_scale); + round = _mm_srli_epi16(round, log_scale); + zbin = _mm_sub_epi16(zbin, one); + + // Do DC and first 15 AC. + coeff0 = load_coefficients(coeff_ptr); + coeff1 = load_coefficients(coeff_ptr + 8); + + coeff0_sign = _mm_srai_epi16(coeff0, 15); + coeff1_sign = _mm_srai_epi16(coeff1, 15); + qcoeff0 = invert_sign_sse2(coeff0, coeff0_sign); + qcoeff1 = invert_sign_sse2(coeff1, coeff1_sign); + + update_mask0(&qcoeff0, &qcoeff1, threshold, iscan, &is_found0, &mask0); + + cmp_mask0 = _mm_cmpgt_epi16(qcoeff0, zbin); + zbin = _mm_unpackhi_epi64(zbin, zbin); // Switch DC to AC + cmp_mask1 = _mm_cmpgt_epi16(qcoeff1, zbin); + + update_mask1(&cmp_mask0, &cmp_mask1, iscan, &is_found1, &mask1); + + threshold[0] = threshold[1]; + all_zero = _mm_or_si128(cmp_mask0, cmp_mask1); + if (_mm_movemask_epi8(all_zero) == 0) { + _mm_store_si128((__m128i *)(qcoeff_ptr), zero); + _mm_store_si128((__m128i *)(qcoeff_ptr + 4), zero); + _mm_store_si128((__m128i *)(qcoeff_ptr + 8), zero); + _mm_store_si128((__m128i *)(qcoeff_ptr + 12), zero); + _mm_store_si128((__m128i *)(dqcoeff_ptr), zero); + _mm_store_si128((__m128i *)(dqcoeff_ptr + 4), zero); + _mm_store_si128((__m128i *)(dqcoeff_ptr + 8), zero); + _mm_store_si128((__m128i *)(dqcoeff_ptr + 12), zero); + round = _mm_unpackhi_epi64(round, round); + quant = _mm_unpackhi_epi64(quant, quant); + shift = _mm_unpackhi_epi64(shift, shift); + dequant = _mm_unpackhi_epi64(dequant, dequant); + } else { + calculate_qcoeff_log_scale(&qcoeff0, round, quant, &shift, &log_scale); + round = _mm_unpackhi_epi64(round, round); + quant = _mm_unpackhi_epi64(quant, quant); + shift = _mm_unpackhi_epi64(shift, shift); + calculate_qcoeff_log_scale(&qcoeff1, round, quant, &shift, &log_scale); + + // Reinsert signs + qcoeff0 = invert_sign_sse2(qcoeff0, coeff0_sign); + qcoeff1 = invert_sign_sse2(qcoeff1, coeff1_sign); + + // Mask out zbin threshold coeffs + qcoeff0 = _mm_and_si128(qcoeff0, cmp_mask0); + qcoeff1 = _mm_and_si128(qcoeff1, cmp_mask1); + + store_coefficients(qcoeff0, qcoeff_ptr); + store_coefficients(qcoeff1, qcoeff_ptr + 8); + + calculate_dqcoeff_and_store_log_scale(qcoeff0, dequant, zero, dqcoeff_ptr, + &log_scale); + dequant = _mm_unpackhi_epi64(dequant, dequant); + calculate_dqcoeff_and_store_log_scale(qcoeff1, dequant, zero, + dqcoeff_ptr + 8, &log_scale); + } + + // AC only loop. + while (index < n_coeffs) { + coeff0 = load_coefficients(coeff_ptr + index); + coeff1 = load_coefficients(coeff_ptr + index + 8); + + coeff0_sign = _mm_srai_epi16(coeff0, 15); + coeff1_sign = _mm_srai_epi16(coeff1, 15); + qcoeff0 = invert_sign_sse2(coeff0, coeff0_sign); + qcoeff1 = invert_sign_sse2(coeff1, coeff1_sign); + + update_mask0(&qcoeff0, &qcoeff1, threshold, iscan + index, &is_found0, + &mask0); + + cmp_mask0 = _mm_cmpgt_epi16(qcoeff0, zbin); + cmp_mask1 = _mm_cmpgt_epi16(qcoeff1, zbin); + + update_mask1(&cmp_mask0, &cmp_mask1, iscan + index, &is_found1, &mask1); + + all_zero = _mm_or_si128(cmp_mask0, cmp_mask1); + if (_mm_movemask_epi8(all_zero) == 0) { + _mm_store_si128((__m128i *)(qcoeff_ptr + index), zero); + _mm_store_si128((__m128i *)(qcoeff_ptr + index + 4), zero); + _mm_store_si128((__m128i *)(qcoeff_ptr + index + 8), zero); + _mm_store_si128((__m128i *)(qcoeff_ptr + index + 12), zero); + _mm_store_si128((__m128i *)(dqcoeff_ptr + index), zero); + _mm_store_si128((__m128i *)(dqcoeff_ptr + index + 4), zero); + _mm_store_si128((__m128i *)(dqcoeff_ptr + index + 8), zero); + _mm_store_si128((__m128i *)(dqcoeff_ptr + index + 12), zero); + index += 16; + continue; + } + calculate_qcoeff_log_scale(&qcoeff0, round, quant, &shift, &log_scale); + calculate_qcoeff_log_scale(&qcoeff1, round, quant, &shift, &log_scale); + + qcoeff0 = invert_sign_sse2(qcoeff0, coeff0_sign); + qcoeff1 = invert_sign_sse2(qcoeff1, coeff1_sign); + + qcoeff0 = _mm_and_si128(qcoeff0, cmp_mask0); + qcoeff1 = _mm_and_si128(qcoeff1, cmp_mask1); + + store_coefficients(qcoeff0, qcoeff_ptr + index); + store_coefficients(qcoeff1, qcoeff_ptr + index + 8); + + calculate_dqcoeff_and_store_log_scale(qcoeff0, dequant, zero, + dqcoeff_ptr + index, &log_scale); + calculate_dqcoeff_and_store_log_scale(qcoeff1, dequant, zero, + dqcoeff_ptr + index + 8, &log_scale); + index += 16; + } + if (is_found0) non_zero_count = calculate_non_zero_count(mask0); + if (is_found1) + non_zero_count_prescan_add_zero = calculate_non_zero_count(mask1); + + for (int i = non_zero_count_prescan_add_zero - 1; i >= non_zero_count; i--) { + const int rc = scan[i]; + qcoeff_ptr[rc] = 0; + dqcoeff_ptr[rc] = 0; + } + + for (int i = non_zero_count - 1; i >= 0; i--) { + const int rc = scan[i]; + if (qcoeff_ptr[rc]) { + eob = i; + break; + } + } + + *eob_ptr = eob + 1; #if SKIP_EOB_FACTOR_ADJUST // TODO(Aniket): Experiment the following loop with intrinsic by combining // with the quantization loop above @@ -403,7 +616,6 @@ void aom_quantize_b_32x32_adaptive_sse2( if ((*eob_ptr - 1) >= 0 && first == (*eob_ptr - 1)) { const int rc = scan[(*eob_ptr - 1)]; if (qcoeff_ptr[rc] == 1 || qcoeff_ptr[rc] == -1) { - const qm_val_t wt = (1 << AOM_QM_BITS); const int coeff = coeff_ptr[rc] * wt; const int coeff_sign = (coeff >> 31); const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign; diff --git a/third_party/aom/aom_dsp/x86/aom_subpixel_8t_intrin_ssse3.c b/third_party/aom/aom_dsp/x86/aom_subpixel_8t_intrin_ssse3.c index 79324f5457..f64b821ea4 100644 --- a/third_party/aom/aom_dsp/x86/aom_subpixel_8t_intrin_ssse3.c +++ b/third_party/aom/aom_dsp/x86/aom_subpixel_8t_intrin_ssse3.c @@ -20,30 +20,30 @@ #include "aom_ports/emmintrin_compat.h" // filters only for the 4_h8 convolution -DECLARE_ALIGNED(16, static const uint8_t, filt1_4_h8[16]) = { - 0, 1, 1, 2, 2, 3, 3, 4, 2, 3, 3, 4, 4, 5, 5, 6 -}; +DECLARE_ALIGNED(16, static const uint8_t, filt1_4_h8[16]) = { 0, 1, 1, 2, 2, 3, + 3, 4, 2, 3, 3, 4, + 4, 5, 5, 6 }; -DECLARE_ALIGNED(16, static const uint8_t, filt2_4_h8[16]) = { - 4, 5, 5, 6, 6, 7, 7, 8, 6, 7, 7, 8, 8, 9, 9, 10 -}; +DECLARE_ALIGNED(16, static const uint8_t, filt2_4_h8[16]) = { 4, 5, 5, 6, 6, 7, + 7, 8, 6, 7, 7, 8, + 8, 9, 9, 10 }; // filters for 8_h8 and 16_h8 -DECLARE_ALIGNED(16, static const uint8_t, filt1_global[16]) = { - 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8 -}; +DECLARE_ALIGNED(16, static const uint8_t, + filt1_global[16]) = { 0, 1, 1, 2, 2, 3, 3, 4, + 4, 5, 5, 6, 6, 7, 7, 8 }; -DECLARE_ALIGNED(16, static const uint8_t, filt2_global[16]) = { - 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10 -}; +DECLARE_ALIGNED(16, static const uint8_t, + filt2_global[16]) = { 2, 3, 3, 4, 4, 5, 5, 6, + 6, 7, 7, 8, 8, 9, 9, 10 }; -DECLARE_ALIGNED(16, static const uint8_t, filt3_global[16]) = { - 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12 -}; +DECLARE_ALIGNED(16, static const uint8_t, + filt3_global[16]) = { 4, 5, 5, 6, 6, 7, 7, 8, + 8, 9, 9, 10, 10, 11, 11, 12 }; -DECLARE_ALIGNED(16, static const uint8_t, filt4_global[16]) = { - 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14 -}; +DECLARE_ALIGNED(16, static const uint8_t, + filt4_global[16]) = { 6, 7, 7, 8, 8, 9, 9, 10, + 10, 11, 11, 12, 12, 13, 13, 14 }; DECLARE_ALIGNED(32, static const uint8_t, filt_h4[]) = { 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 0, 1, 1, diff --git a/third_party/aom/aom_dsp/x86/convolve_avx2.h b/third_party/aom/aom_dsp/x86/convolve_avx2.h index 4a1068e835..200024b620 100644 --- a/third_party/aom/aom_dsp/x86/convolve_avx2.h +++ b/third_party/aom/aom_dsp/x86/convolve_avx2.h @@ -34,6 +34,31 @@ DECLARE_ALIGNED(32, static const uint8_t, filt4_d4_global_avx2[]) = { 2, 3, 4, 5, 3, 4, 5, 6, 4, 5, 6, 7, 5, 6, 7, 8, }; +DECLARE_ALIGNED(32, static const uint8_t, filt_center_global_avx2[32]) = { + 3, 255, 4, 255, 5, 255, 6, 255, 7, 255, 8, 255, 9, 255, 10, 255, + 3, 255, 4, 255, 5, 255, 6, 255, 7, 255, 8, 255, 9, 255, 10, 255 +}; + +DECLARE_ALIGNED(32, static const uint8_t, + filt1_global_avx2[32]) = { 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, + 6, 6, 7, 7, 8, 0, 1, 1, 2, 2, 3, + 3, 4, 4, 5, 5, 6, 6, 7, 7, 8 }; + +DECLARE_ALIGNED(32, static const uint8_t, + filt2_global_avx2[32]) = { 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, + 8, 8, 9, 9, 10, 2, 3, 3, 4, 4, 5, + 5, 6, 6, 7, 7, 8, 8, 9, 9, 10 }; + +DECLARE_ALIGNED(32, static const uint8_t, filt3_global_avx2[32]) = { + 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, + 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12 +}; + +DECLARE_ALIGNED(32, static const uint8_t, filt4_global_avx2[32]) = { + 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, + 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14 +}; + #define CONVOLVE_SR_HORIZONTAL_FILTER_8TAP \ for (i = 0; i < (im_h - 2); i += 2) { \ __m256i data = _mm256_castsi128_si256( \ diff --git a/third_party/aom/aom_dsp/x86/intrapred_avx2.c b/third_party/aom/aom_dsp/x86/intrapred_avx2.c index 17f35a0b13..b8e23ec9b7 100644 --- a/third_party/aom/aom_dsp/x86/intrapred_avx2.c +++ b/third_party/aom/aom_dsp/x86/intrapred_avx2.c @@ -64,6 +64,74 @@ static INLINE void row_store_64xh(const __m256i *r, int height, uint8_t *dst, } } +static DECLARE_ALIGNED(16, uint8_t, HighbdLoadMaskx[8][16]) = { + { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }, + { 0, 1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13 }, + { 0, 1, 0, 1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 }, + { 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }, + { 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 4, 5, 6, 7 }, + { 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 4, 5 }, + { 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 2, 3 }, + { 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1 }, +}; + +static DECLARE_ALIGNED(16, uint8_t, HighbdEvenOddMaskx4[4][16]) = { + { 0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15 }, + { 0, 1, 2, 3, 6, 7, 10, 11, 14, 15, 4, 5, 8, 9, 12, 13 }, + { 0, 1, 0, 1, 4, 5, 8, 9, 12, 13, 0, 1, 6, 7, 10, 11 }, + { 0, 1, 0, 1, 0, 1, 6, 7, 10, 11, 14, 15, 0, 1, 8, 9 } +}; + +static DECLARE_ALIGNED(16, uint8_t, HighbdEvenOddMaskx[8][32]) = { + { 0, 1, 4, 5, 8, 9, 12, 13, 16, 17, 20, 21, 24, 25, 28, 29, + 2, 3, 6, 7, 10, 11, 14, 15, 18, 19, 22, 23, 26, 27, 30, 31 }, + { 0, 1, 2, 3, 6, 7, 10, 11, 14, 15, 18, 19, 22, 23, 26, 27, + 0, 1, 4, 5, 8, 9, 12, 13, 16, 17, 20, 21, 24, 25, 28, 29 }, + { 0, 1, 0, 1, 4, 5, 8, 9, 12, 13, 16, 17, 20, 21, 24, 25, + 0, 1, 0, 1, 6, 7, 10, 11, 14, 15, 18, 19, 22, 23, 26, 27 }, + { 0, 1, 0, 1, 0, 1, 6, 7, 10, 11, 14, 15, 18, 19, 22, 23, + 0, 1, 0, 1, 0, 1, 8, 9, 12, 13, 16, 17, 20, 21, 24, 25 }, + { 0, 1, 0, 1, 0, 1, 0, 1, 8, 9, 12, 13, 16, 17, 20, 21, + 0, 1, 0, 1, 0, 1, 0, 1, 10, 11, 14, 15, 18, 19, 22, 23 }, + { 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 10, 11, 14, 15, 18, 19, + 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 12, 13, 16, 17, 20, 21 }, + { 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 12, 13, 16, 17, + 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 14, 15, 18, 19 }, + { 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 14, 15, + 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 16, 17 } +}; + +static DECLARE_ALIGNED(32, uint16_t, HighbdBaseMask[17][16]) = { + { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 0xffff, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 0xffff, 0xffff, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 0xffff, 0xffff, 0xffff, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 0xffff, 0xffff, 0xffff, 0xffff, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0 }, + { 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0, 0, 0, 0, 0, 0, 0, + 0, 0 }, + { 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0, 0, 0, 0, + 0, 0, 0, 0 }, + { 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0, + 0, 0, 0, 0, 0, 0 }, + { 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, + 0xffff, 0, 0, 0, 0, 0, 0 }, + { 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, + 0xffff, 0xffff, 0, 0, 0, 0, 0 }, + { 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, + 0xffff, 0xffff, 0xffff, 0, 0, 0, 0 }, + { 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, + 0xffff, 0xffff, 0xffff, 0xffff, 0, 0, 0 }, + { 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, + 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0, 0 }, + { 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, + 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0 }, + { 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, + 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff } +}; + static INLINE void highbd_transpose16x4_8x8_sse2(__m128i *x, __m128i *d) { __m128i r0, r1, r2, r3, r4, r5, r6, r7, r8, r9, r10, r11, r12, r13, r14, r15; @@ -999,13 +1067,83 @@ static AOM_FORCE_INLINE void highbd_dr_prediction_z1_4xN_internal_avx2( int N, __m128i *dst, const uint16_t *above, int upsample_above, int dx) { const int frac_bits = 6 - upsample_above; const int max_base_x = ((N + 4) - 1) << upsample_above; - int x; - // a assert(dx > 0); + + assert(dx > 0); // pre-filter above pixels // store in temp buffers: // above[x] * 32 + 16 // above[x+1] - above[x] - // final pixels will be caluculated as: + // final pixels will be calculated as: + // (above[x] * 32 + 16 + (above[x+1] - above[x]) * shift) >> 5 + __m256i a0, a1, a32, a16; + __m256i diff, c3f; + __m128i a_mbase_x, max_base_x128, base_inc128, mask128; + __m128i a0_128, a1_128; + a16 = _mm256_set1_epi16(16); + a_mbase_x = _mm_set1_epi16(above[max_base_x]); + max_base_x128 = _mm_set1_epi16(max_base_x); + c3f = _mm256_set1_epi16(0x3f); + + int x = dx; + for (int r = 0; r < N; r++) { + __m256i b, res, shift; + __m128i res1; + + int base = x >> frac_bits; + if (base >= max_base_x) { + for (int i = r; i < N; ++i) { + dst[i] = a_mbase_x; // save 4 values + } + return; + } + + a0_128 = _mm_loadu_si128((__m128i *)(above + base)); + a1_128 = _mm_loadu_si128((__m128i *)(above + base + 1)); + + if (upsample_above) { + a0_128 = _mm_shuffle_epi8(a0_128, *(__m128i *)HighbdEvenOddMaskx4[0]); + a1_128 = _mm_srli_si128(a0_128, 8); + + base_inc128 = _mm_setr_epi16(base, base + 2, base + 4, base + 6, base + 8, + base + 10, base + 12, base + 14); + shift = _mm256_srli_epi16( + _mm256_and_si256( + _mm256_slli_epi16(_mm256_set1_epi16(x), upsample_above), + _mm256_set1_epi16(0x3f)), + 1); + } else { + base_inc128 = _mm_setr_epi16(base, base + 1, base + 2, base + 3, base + 4, + base + 5, base + 6, base + 7); + shift = _mm256_srli_epi16(_mm256_and_si256(_mm256_set1_epi16(x), c3f), 1); + } + a0 = _mm256_castsi128_si256(a0_128); + a1 = _mm256_castsi128_si256(a1_128); + diff = _mm256_sub_epi16(a1, a0); // a[x+1] - a[x] + a32 = _mm256_slli_epi16(a0, 5); // a[x] * 32 + a32 = _mm256_add_epi16(a32, a16); // a[x] * 32 + 16 + + b = _mm256_mullo_epi16(diff, shift); + res = _mm256_add_epi16(a32, b); + res = _mm256_srli_epi16(res, 5); + res1 = _mm256_castsi256_si128(res); + + mask128 = _mm_cmpgt_epi16(max_base_x128, base_inc128); + dst[r] = _mm_blendv_epi8(a_mbase_x, res1, mask128); + x += dx; + } +} + +static AOM_FORCE_INLINE void highbd_dr_prediction_32bit_z1_4xN_internal_avx2( + int N, __m128i *dst, const uint16_t *above, int upsample_above, int dx) { + const int frac_bits = 6 - upsample_above; + const int max_base_x = ((N + 4) - 1) << upsample_above; + + assert(dx > 0); + // pre-filter above pixels + // store in temp buffers: + // above[x] * 32 + 16 + // above[x+1] - above[x] + // final pixels will be calculated as: // (above[x] * 32 + 16 + (above[x+1] - above[x]) * shift) >> 5 __m256i a0, a1, a32, a16; __m256i diff; @@ -1015,7 +1153,7 @@ static AOM_FORCE_INLINE void highbd_dr_prediction_z1_4xN_internal_avx2( a_mbase_x = _mm_set1_epi16(above[max_base_x]); max_base_x128 = _mm_set1_epi32(max_base_x); - x = dx; + int x = dx; for (int r = 0; r < N; r++) { __m256i b, res, shift; __m128i res1; @@ -1068,28 +1206,32 @@ static AOM_FORCE_INLINE void highbd_dr_prediction_z1_4xN_internal_avx2( static void highbd_dr_prediction_z1_4xN_avx2(int N, uint16_t *dst, ptrdiff_t stride, const uint16_t *above, - int upsample_above, int dx) { + int upsample_above, int dx, + int bd) { __m128i dstvec[16]; - - highbd_dr_prediction_z1_4xN_internal_avx2(N, dstvec, above, upsample_above, - dx); + if (bd < 12) { + highbd_dr_prediction_z1_4xN_internal_avx2(N, dstvec, above, upsample_above, + dx); + } else { + highbd_dr_prediction_32bit_z1_4xN_internal_avx2(N, dstvec, above, + upsample_above, dx); + } for (int i = 0; i < N; i++) { _mm_storel_epi64((__m128i *)(dst + stride * i), dstvec[i]); } } -static AOM_FORCE_INLINE void highbd_dr_prediction_z1_8xN_internal_avx2( +static AOM_FORCE_INLINE void highbd_dr_prediction_32bit_z1_8xN_internal_avx2( int N, __m128i *dst, const uint16_t *above, int upsample_above, int dx) { const int frac_bits = 6 - upsample_above; const int max_base_x = ((8 + N) - 1) << upsample_above; - int x; - // a assert(dx > 0); + assert(dx > 0); // pre-filter above pixels // store in temp buffers: // above[x] * 32 + 16 // above[x+1] - above[x] - // final pixels will be caluculated as: + // final pixels will be calculated as: // (above[x] * 32 + 16 + (above[x+1] - above[x]) * shift) >> 5 __m256i a0, a1, a0_1, a1_1, a32, a16; __m256i a_mbase_x, diff, max_base_x256, base_inc256, mask256; @@ -1098,7 +1240,7 @@ static AOM_FORCE_INLINE void highbd_dr_prediction_z1_8xN_internal_avx2( a_mbase_x = _mm256_set1_epi16(above[max_base_x]); max_base_x256 = _mm256_set1_epi32(max_base_x); - x = dx; + int x = dx; for (int r = 0; r < N; r++) { __m256i b, res, res1, shift; @@ -1162,22 +1304,108 @@ static AOM_FORCE_INLINE void highbd_dr_prediction_z1_8xN_internal_avx2( } } +static AOM_FORCE_INLINE void highbd_dr_prediction_z1_8xN_internal_avx2( + int N, __m128i *dst, const uint16_t *above, int upsample_above, int dx) { + const int frac_bits = 6 - upsample_above; + const int max_base_x = ((8 + N) - 1) << upsample_above; + + assert(dx > 0); + // pre-filter above pixels + // store in temp buffers: + // above[x] * 32 + 16 + // above[x+1] - above[x] + // final pixels will be calculated as: + // (above[x] * 32 + 16 + (above[x+1] - above[x]) * shift) >> 5 + __m256i a0, a1, a32, a16, c3f; + __m256i a_mbase_x, diff, max_base_x256, base_inc256, mask256; + __m128i a0_x128, a1_x128; + + a16 = _mm256_set1_epi16(16); + a_mbase_x = _mm256_set1_epi16(above[max_base_x]); + max_base_x256 = _mm256_set1_epi16(max_base_x); + c3f = _mm256_set1_epi16(0x3f); + + int x = dx; + for (int r = 0; r < N; r++) { + __m256i b, res, res1, shift; + + int base = x >> frac_bits; + if (base >= max_base_x) { + for (int i = r; i < N; ++i) { + dst[i] = _mm256_castsi256_si128(a_mbase_x); // save 8 values + } + return; + } + + a0_x128 = _mm_loadu_si128((__m128i *)(above + base)); + if (upsample_above) { + __m128i mask, atmp0, atmp1, atmp2, atmp3; + a1_x128 = _mm_loadu_si128((__m128i *)(above + base + 8)); + atmp0 = _mm_shuffle_epi8(a0_x128, *(__m128i *)HighbdEvenOddMaskx[0]); + atmp1 = _mm_shuffle_epi8(a1_x128, *(__m128i *)HighbdEvenOddMaskx[0]); + atmp2 = + _mm_shuffle_epi8(a0_x128, *(__m128i *)(HighbdEvenOddMaskx[0] + 16)); + atmp3 = + _mm_shuffle_epi8(a1_x128, *(__m128i *)(HighbdEvenOddMaskx[0] + 16)); + mask = + _mm_cmpgt_epi8(*(__m128i *)HighbdEvenOddMaskx[0], _mm_set1_epi8(15)); + a0_x128 = _mm_blendv_epi8(atmp0, atmp1, mask); + mask = _mm_cmpgt_epi8(*(__m128i *)(HighbdEvenOddMaskx[0] + 16), + _mm_set1_epi8(15)); + a1_x128 = _mm_blendv_epi8(atmp2, atmp3, mask); + + base_inc256 = _mm256_setr_epi16(base, base + 2, base + 4, base + 6, + base + 8, base + 10, base + 12, base + 14, + 0, 0, 0, 0, 0, 0, 0, 0); + shift = _mm256_srli_epi16( + _mm256_and_si256( + _mm256_slli_epi16(_mm256_set1_epi16(x), upsample_above), c3f), + 1); + } else { + a1_x128 = _mm_loadu_si128((__m128i *)(above + base + 1)); + base_inc256 = _mm256_setr_epi16(base, base + 1, base + 2, base + 3, + base + 4, base + 5, base + 6, base + 7, 0, + 0, 0, 0, 0, 0, 0, 0); + shift = _mm256_srli_epi16(_mm256_and_si256(_mm256_set1_epi16(x), c3f), 1); + } + a0 = _mm256_castsi128_si256(a0_x128); + a1 = _mm256_castsi128_si256(a1_x128); + + diff = _mm256_sub_epi16(a1, a0); // a[x+1] - a[x] + a32 = _mm256_slli_epi16(a0, 5); // a[x] * 32 + a32 = _mm256_add_epi16(a32, a16); // a[x] * 32 + 16 + + b = _mm256_mullo_epi16(diff, shift); + res = _mm256_add_epi16(a32, b); + res = _mm256_srli_epi16(res, 5); + + mask256 = _mm256_cmpgt_epi16(max_base_x256, base_inc256); + res1 = _mm256_blendv_epi8(a_mbase_x, res, mask256); + dst[r] = _mm256_castsi256_si128(res1); + x += dx; + } +} + static void highbd_dr_prediction_z1_8xN_avx2(int N, uint16_t *dst, ptrdiff_t stride, const uint16_t *above, - int upsample_above, int dx) { + int upsample_above, int dx, + int bd) { __m128i dstvec[32]; - - highbd_dr_prediction_z1_8xN_internal_avx2(N, dstvec, above, upsample_above, - dx); + if (bd < 12) { + highbd_dr_prediction_z1_8xN_internal_avx2(N, dstvec, above, upsample_above, + dx); + } else { + highbd_dr_prediction_32bit_z1_8xN_internal_avx2(N, dstvec, above, + upsample_above, dx); + } for (int i = 0; i < N; i++) { _mm_storeu_si128((__m128i *)(dst + stride * i), dstvec[i]); } } -static AOM_FORCE_INLINE void highbd_dr_prediction_z1_16xN_internal_avx2( +static AOM_FORCE_INLINE void highbd_dr_prediction_32bit_z1_16xN_internal_avx2( int N, __m256i *dstvec, const uint16_t *above, int upsample_above, int dx) { - int x; // here upsample_above is 0 by design of av1_use_intra_edge_upsample (void)upsample_above; const int frac_bits = 6; @@ -1187,7 +1415,7 @@ static AOM_FORCE_INLINE void highbd_dr_prediction_z1_16xN_internal_avx2( // store in temp buffers: // above[x] * 32 + 16 // above[x+1] - above[x] - // final pixels will be caluculated as: + // final pixels will be calculated as: // (above[x] * 32 + 16 + (above[x+1] - above[x]) * shift) >> 5 __m256i a0, a0_1, a1, a1_1, a32, a16; __m256i a_mbase_x, diff, max_base_x256, base_inc256, mask256; @@ -1196,7 +1424,7 @@ static AOM_FORCE_INLINE void highbd_dr_prediction_z1_16xN_internal_avx2( a_mbase_x = _mm256_set1_epi16(above[max_base_x]); max_base_x256 = _mm256_set1_epi16(max_base_x); - x = dx; + int x = dx; for (int r = 0; r < N; r++) { __m256i b, res[2], res1; @@ -1255,21 +1483,82 @@ static AOM_FORCE_INLINE void highbd_dr_prediction_z1_16xN_internal_avx2( } } +static AOM_FORCE_INLINE void highbd_dr_prediction_z1_16xN_internal_avx2( + int N, __m256i *dstvec, const uint16_t *above, int upsample_above, int dx) { + // here upsample_above is 0 by design of av1_use_intra_edge_upsample + (void)upsample_above; + const int frac_bits = 6; + const int max_base_x = ((16 + N) - 1); + + // pre-filter above pixels + // store in temp buffers: + // above[x] * 32 + 16 + // above[x+1] - above[x] + // final pixels will be calculated as: + // (above[x] * 32 + 16 + (above[x+1] - above[x]) * shift) >> 5 + __m256i a0, a1, a32, a16, c3f; + __m256i a_mbase_x, diff, max_base_x256, base_inc256, mask256; + + a16 = _mm256_set1_epi16(16); + a_mbase_x = _mm256_set1_epi16(above[max_base_x]); + max_base_x256 = _mm256_set1_epi16(max_base_x); + c3f = _mm256_set1_epi16(0x3f); + + int x = dx; + for (int r = 0; r < N; r++) { + __m256i b, res; + + int base = x >> frac_bits; + if (base >= max_base_x) { + for (int i = r; i < N; ++i) { + dstvec[i] = a_mbase_x; // save 16 values + } + return; + } + __m256i shift = + _mm256_srli_epi16(_mm256_and_si256(_mm256_set1_epi16(x), c3f), 1); + + a0 = _mm256_loadu_si256((__m256i *)(above + base)); + a1 = _mm256_loadu_si256((__m256i *)(above + base + 1)); + + diff = _mm256_sub_epi16(a1, a0); // a[x+1] - a[x] + a32 = _mm256_slli_epi16(a0, 5); // a[x] * 32 + a32 = _mm256_add_epi16(a32, a16); // a[x] * 32 + 16 + b = _mm256_mullo_epi16(diff, shift); + + res = _mm256_add_epi16(a32, b); + res = _mm256_srli_epi16(res, 5); // 16 16bit values + + base_inc256 = _mm256_setr_epi16(base, base + 1, base + 2, base + 3, + base + 4, base + 5, base + 6, base + 7, + base + 8, base + 9, base + 10, base + 11, + base + 12, base + 13, base + 14, base + 15); + mask256 = _mm256_cmpgt_epi16(max_base_x256, base_inc256); + dstvec[r] = _mm256_blendv_epi8(a_mbase_x, res, mask256); + x += dx; + } +} + static void highbd_dr_prediction_z1_16xN_avx2(int N, uint16_t *dst, ptrdiff_t stride, const uint16_t *above, - int upsample_above, int dx) { + int upsample_above, int dx, + int bd) { __m256i dstvec[64]; - highbd_dr_prediction_z1_16xN_internal_avx2(N, dstvec, above, upsample_above, - dx); + if (bd < 12) { + highbd_dr_prediction_z1_16xN_internal_avx2(N, dstvec, above, upsample_above, + dx); + } else { + highbd_dr_prediction_32bit_z1_16xN_internal_avx2(N, dstvec, above, + upsample_above, dx); + } for (int i = 0; i < N; i++) { _mm256_storeu_si256((__m256i *)(dst + stride * i), dstvec[i]); } } -static AOM_FORCE_INLINE void highbd_dr_prediction_z1_32xN_internal_avx2( +static AOM_FORCE_INLINE void highbd_dr_prediction_32bit_z1_32xN_internal_avx2( int N, __m256i *dstvec, const uint16_t *above, int upsample_above, int dx) { - int x; // here upsample_above is 0 by design of av1_use_intra_edge_upsample (void)upsample_above; const int frac_bits = 6; @@ -1279,16 +1568,17 @@ static AOM_FORCE_INLINE void highbd_dr_prediction_z1_32xN_internal_avx2( // store in temp buffers: // above[x] * 32 + 16 // above[x+1] - above[x] - // final pixels will be caluculated as: + // final pixels will be calculated as: // (above[x] * 32 + 16 + (above[x+1] - above[x]) * shift) >> 5 - __m256i a0, a0_1, a1, a1_1, a32, a16; + __m256i a0, a0_1, a1, a1_1, a32, a16, c3f; __m256i a_mbase_x, diff, max_base_x256, base_inc256, mask256; a16 = _mm256_set1_epi32(16); a_mbase_x = _mm256_set1_epi16(above[max_base_x]); max_base_x256 = _mm256_set1_epi16(max_base_x); + c3f = _mm256_set1_epi16(0x3f); - x = dx; + int x = dx; for (int r = 0; r < N; r++) { __m256i b, res[2], res1; @@ -1301,8 +1591,8 @@ static AOM_FORCE_INLINE void highbd_dr_prediction_z1_32xN_internal_avx2( return; } - __m256i shift = _mm256_srli_epi32( - _mm256_and_si256(_mm256_set1_epi32(x), _mm256_set1_epi32(0x3f)), 1); + __m256i shift = + _mm256_srli_epi32(_mm256_and_si256(_mm256_set1_epi32(x), c3f), 1); for (int j = 0; j < 32; j += 16) { int mdif = max_base_x - (base + j); @@ -1354,10 +1644,83 @@ static AOM_FORCE_INLINE void highbd_dr_prediction_z1_32xN_internal_avx2( mask256 = _mm256_cmpgt_epi16(max_base_x256, base_inc256); res1 = _mm256_blendv_epi8(a_mbase_x, res1, mask256); } - if (!j) + if (!j) { dstvec[r] = res1; - else + } else { dstvec[r + N] = res1; + } + } + x += dx; + } +} + +static AOM_FORCE_INLINE void highbd_dr_prediction_z1_32xN_internal_avx2( + int N, __m256i *dstvec, const uint16_t *above, int upsample_above, int dx) { + // here upsample_above is 0 by design of av1_use_intra_edge_upsample + (void)upsample_above; + const int frac_bits = 6; + const int max_base_x = ((32 + N) - 1); + + // pre-filter above pixels + // store in temp buffers: + // above[x] * 32 + 16 + // above[x+1] - above[x] + // final pixels will be calculated as: + // (above[x] * 32 + 16 + (above[x+1] - above[x]) * shift) >> 5 + __m256i a0, a1, a32, a16, c3f; + __m256i a_mbase_x, diff, max_base_x256, base_inc256, mask256; + + a16 = _mm256_set1_epi16(16); + a_mbase_x = _mm256_set1_epi16(above[max_base_x]); + max_base_x256 = _mm256_set1_epi16(max_base_x); + c3f = _mm256_set1_epi16(0x3f); + + int x = dx; + for (int r = 0; r < N; r++) { + __m256i b, res; + + int base = x >> frac_bits; + if (base >= max_base_x) { + for (int i = r; i < N; ++i) { + dstvec[i] = a_mbase_x; // save 32 values + dstvec[i + N] = a_mbase_x; + } + return; + } + + __m256i shift = + _mm256_srli_epi16(_mm256_and_si256(_mm256_set1_epi16(x), c3f), 1); + + for (int j = 0; j < 32; j += 16) { + int mdif = max_base_x - (base + j); + if (mdif <= 0) { + res = a_mbase_x; + } else { + a0 = _mm256_loadu_si256((__m256i *)(above + base + j)); + a1 = _mm256_loadu_si256((__m256i *)(above + base + 1 + j)); + + diff = _mm256_sub_epi16(a1, a0); // a[x+1] - a[x] + a32 = _mm256_slli_epi16(a0, 5); // a[x] * 32 + a32 = _mm256_add_epi16(a32, a16); // a[x] * 32 + 16 + b = _mm256_mullo_epi16(diff, shift); + + res = _mm256_add_epi16(a32, b); + res = _mm256_srli_epi16(res, 5); + + base_inc256 = _mm256_setr_epi16( + base + j, base + j + 1, base + j + 2, base + j + 3, base + j + 4, + base + j + 5, base + j + 6, base + j + 7, base + j + 8, + base + j + 9, base + j + 10, base + j + 11, base + j + 12, + base + j + 13, base + j + 14, base + j + 15); + + mask256 = _mm256_cmpgt_epi16(max_base_x256, base_inc256); + res = _mm256_blendv_epi8(a_mbase_x, res, mask256); + } + if (!j) { + dstvec[r] = res; + } else { + dstvec[r + N] = res; + } } x += dx; } @@ -1366,23 +1729,27 @@ static AOM_FORCE_INLINE void highbd_dr_prediction_z1_32xN_internal_avx2( static void highbd_dr_prediction_z1_32xN_avx2(int N, uint16_t *dst, ptrdiff_t stride, const uint16_t *above, - int upsample_above, int dx) { + int upsample_above, int dx, + int bd) { __m256i dstvec[128]; - - highbd_dr_prediction_z1_32xN_internal_avx2(N, dstvec, above, upsample_above, - dx); + if (bd < 12) { + highbd_dr_prediction_z1_32xN_internal_avx2(N, dstvec, above, upsample_above, + dx); + } else { + highbd_dr_prediction_32bit_z1_32xN_internal_avx2(N, dstvec, above, + upsample_above, dx); + } for (int i = 0; i < N; i++) { _mm256_storeu_si256((__m256i *)(dst + stride * i), dstvec[i]); _mm256_storeu_si256((__m256i *)(dst + stride * i + 16), dstvec[i + N]); } } -static void highbd_dr_prediction_z1_64xN_avx2(int N, uint16_t *dst, - ptrdiff_t stride, - const uint16_t *above, - int upsample_above, int dx) { - int x; - +static void highbd_dr_prediction_32bit_z1_64xN_avx2(int N, uint16_t *dst, + ptrdiff_t stride, + const uint16_t *above, + int upsample_above, + int dx) { // here upsample_above is 0 by design of av1_use_intra_edge_upsample (void)upsample_above; const int frac_bits = 6; @@ -1392,7 +1759,7 @@ static void highbd_dr_prediction_z1_64xN_avx2(int N, uint16_t *dst, // store in temp buffers: // above[x] * 32 + 16 // above[x+1] - above[x] - // final pixels will be caluculated as: + // final pixels will be calculated as: // (above[x] * 32 + 16 + (above[x+1] - above[x]) * shift) >> 5 __m256i a0, a0_1, a1, a1_1, a32, a16; __m256i a_mbase_x, diff, max_base_x256, base_inc256, mask256; @@ -1401,7 +1768,7 @@ static void highbd_dr_prediction_z1_64xN_avx2(int N, uint16_t *dst, a_mbase_x = _mm256_set1_epi16(above[max_base_x]); max_base_x256 = _mm256_set1_epi16(max_base_x); - x = dx; + int x = dx; for (int r = 0; r < N; r++, dst += stride) { __m256i b, res[2], res1; @@ -1477,6 +1844,79 @@ static void highbd_dr_prediction_z1_64xN_avx2(int N, uint16_t *dst, } } +static void highbd_dr_prediction_z1_64xN_avx2(int N, uint16_t *dst, + ptrdiff_t stride, + const uint16_t *above, + int upsample_above, int dx) { + // here upsample_above is 0 by design of av1_use_intra_edge_upsample + (void)upsample_above; + const int frac_bits = 6; + const int max_base_x = ((64 + N) - 1); + + // pre-filter above pixels + // store in temp buffers: + // above[x] * 32 + 16 + // above[x+1] - above[x] + // final pixels will be calculated as: + // (above[x] * 32 + 16 + (above[x+1] - above[x]) * shift) >> 5 + __m256i a0, a1, a32, a16, c3f; + __m256i a_mbase_x, diff, max_base_x256, base_inc256, mask256; + + a16 = _mm256_set1_epi16(16); + a_mbase_x = _mm256_set1_epi16(above[max_base_x]); + max_base_x256 = _mm256_set1_epi16(max_base_x); + c3f = _mm256_set1_epi16(0x3f); + + int x = dx; + for (int r = 0; r < N; r++, dst += stride) { + __m256i b, res; + + int base = x >> frac_bits; + if (base >= max_base_x) { + for (int i = r; i < N; ++i) { + _mm256_storeu_si256((__m256i *)dst, a_mbase_x); // save 32 values + _mm256_storeu_si256((__m256i *)(dst + 16), a_mbase_x); + _mm256_storeu_si256((__m256i *)(dst + 32), a_mbase_x); + _mm256_storeu_si256((__m256i *)(dst + 48), a_mbase_x); + dst += stride; + } + return; + } + + __m256i shift = + _mm256_srli_epi16(_mm256_and_si256(_mm256_set1_epi16(x), c3f), 1); + + for (int j = 0; j < 64; j += 16) { + int mdif = max_base_x - (base + j); + if (mdif <= 0) { + _mm256_storeu_si256((__m256i *)(dst + j), a_mbase_x); + } else { + a0 = _mm256_loadu_si256((__m256i *)(above + base + j)); + a1 = _mm256_loadu_si256((__m256i *)(above + base + 1 + j)); + + diff = _mm256_sub_epi16(a1, a0); // a[x+1] - a[x] + a32 = _mm256_slli_epi16(a0, 5); // a[x] * 32 + a32 = _mm256_add_epi16(a32, a16); // a[x] * 32 + 16 + b = _mm256_mullo_epi16(diff, shift); + + res = _mm256_add_epi16(a32, b); + res = _mm256_srli_epi16(res, 5); + + base_inc256 = _mm256_setr_epi16( + base + j, base + j + 1, base + j + 2, base + j + 3, base + j + 4, + base + j + 5, base + j + 6, base + j + 7, base + j + 8, + base + j + 9, base + j + 10, base + j + 11, base + j + 12, + base + j + 13, base + j + 14, base + j + 15); + + mask256 = _mm256_cmpgt_epi16(max_base_x256, base_inc256); + res = _mm256_blendv_epi8(a_mbase_x, res, mask256); + _mm256_storeu_si256((__m256i *)(dst + j), res); // 16 16bit values + } + } + x += dx; + } +} + // Directional prediction, zone 1: 0 < angle < 90 void av1_highbd_dr_prediction_z1_avx2(uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, @@ -1484,182 +1924,61 @@ void av1_highbd_dr_prediction_z1_avx2(uint16_t *dst, ptrdiff_t stride, int bw, int dx, int dy, int bd) { (void)left; (void)dy; - (void)bd; switch (bw) { case 4: highbd_dr_prediction_z1_4xN_avx2(bh, dst, stride, above, upsample_above, - dx); + dx, bd); break; case 8: highbd_dr_prediction_z1_8xN_avx2(bh, dst, stride, above, upsample_above, - dx); + dx, bd); break; case 16: highbd_dr_prediction_z1_16xN_avx2(bh, dst, stride, above, upsample_above, - dx); + dx, bd); break; case 32: highbd_dr_prediction_z1_32xN_avx2(bh, dst, stride, above, upsample_above, - dx); + dx, bd); break; case 64: - highbd_dr_prediction_z1_64xN_avx2(bh, dst, stride, above, upsample_above, - dx); + if (bd < 12) { + highbd_dr_prediction_z1_64xN_avx2(bh, dst, stride, above, + upsample_above, dx); + } else { + highbd_dr_prediction_32bit_z1_64xN_avx2(bh, dst, stride, above, + upsample_above, dx); + } break; default: break; } return; } -static void highbd_transpose_TX_8X8(const uint16_t *src, ptrdiff_t pitchSrc, - uint16_t *dst, ptrdiff_t pitchDst) { - __m128i r0, r1, r2, r3, r4, r5, r6, r7, r0_Lo, r1_Lo, r2_Lo, r3_Lo, r4_Lo, - r5_Lo, r6_Lo; - r0 = _mm_load_si128( - (__m128i *)(src + 0 * pitchSrc)); // 07,06,05,04,03,02,01,00 - r1 = _mm_load_si128( - (__m128i *)(src + 1 * pitchSrc)); // 17,16,15,14,13,12,11,10 - r2 = _mm_load_si128( - (__m128i *)(src + 2 * pitchSrc)); // 27,26,25,24,23,22,21,20 - r3 = _mm_load_si128( - (__m128i *)(src + 3 * pitchSrc)); // 37,36,35,34,33,32,31,30 - r4 = _mm_load_si128( - (__m128i *)(src + 4 * pitchSrc)); // 47,46,45,44,43,42,41,40 - r5 = _mm_load_si128( - (__m128i *)(src + 5 * pitchSrc)); // 57,56,55,54,53,52,51,50 - r6 = _mm_load_si128( - (__m128i *)(src + 6 * pitchSrc)); // 67,66,65,64,63,62,61,60 - r7 = _mm_load_si128( - (__m128i *)(src + 7 * pitchSrc)); // 77,76,75,74,73,72,71,70 - - r0_Lo = _mm_unpacklo_epi16(r0, r1); - r2_Lo = _mm_unpacklo_epi16(r2, r3); - r4_Lo = _mm_unpacklo_epi16(r4, r5); - r6_Lo = _mm_unpacklo_epi16(r6, r7); - - r1_Lo = r0_Lo; - r0_Lo = _mm_unpacklo_epi32(r0_Lo, r2_Lo); - r1_Lo = _mm_unpackhi_epi32(r1_Lo, r2_Lo); - r5_Lo = r4_Lo; - r4_Lo = _mm_unpacklo_epi32(r4_Lo, r6_Lo); - r5_Lo = _mm_unpackhi_epi32(r5_Lo, r6_Lo); - r2_Lo = r0_Lo; - r0_Lo = _mm_unpacklo_epi64(r0_Lo, r4_Lo); // 64 - r2_Lo = _mm_unpackhi_epi64(r2_Lo, r4_Lo); - r3_Lo = r1_Lo; - r1_Lo = _mm_unpacklo_epi64(r1_Lo, r5_Lo); - r3_Lo = _mm_unpackhi_epi64(r3_Lo, r5_Lo); - - _mm_storeu_si128((__m128i *)(dst + 0 * pitchDst), r0_Lo); - _mm_storeu_si128((__m128i *)(dst + 1 * pitchDst), r2_Lo); - _mm_storeu_si128((__m128i *)(dst + 2 * pitchDst), r1_Lo); - _mm_storeu_si128((__m128i *)(dst + 3 * pitchDst), r3_Lo); - - r0 = _mm_unpackhi_epi16(r0, r1); - r2 = _mm_unpackhi_epi16(r2, r3); - r4 = _mm_unpackhi_epi16(r4, r5); - r6 = _mm_unpackhi_epi16(r6, r7); - - r1 = r0; - r0 = _mm_unpacklo_epi32(r0, r2); - r1 = _mm_unpackhi_epi32(r1, r2); - r5 = r4; - r4 = _mm_unpacklo_epi32(r4, r6); - r5 = _mm_unpackhi_epi32(r5, r6); - r2 = r0; - r0 = _mm_unpacklo_epi64(r0, r4); - r2 = _mm_unpackhi_epi64(r2, r4); - r3 = r1; - r1 = _mm_unpacklo_epi64(r1, r5); - r3 = _mm_unpackhi_epi64(r3, r5); - - _mm_storeu_si128((__m128i *)(dst + 4 * pitchDst), r0); - _mm_storeu_si128((__m128i *)(dst + 5 * pitchDst), r2); - _mm_storeu_si128((__m128i *)(dst + 6 * pitchDst), r1); - _mm_storeu_si128((__m128i *)(dst + 7 * pitchDst), r3); +static void highbd_transpose_TX_16X16(const uint16_t *src, ptrdiff_t pitchSrc, + uint16_t *dst, ptrdiff_t pitchDst) { + __m256i r[16]; + __m256i d[16]; + for (int j = 0; j < 16; j++) { + r[j] = _mm256_loadu_si256((__m256i *)(src + j * pitchSrc)); + } + highbd_transpose16x16_avx2(r, d); + for (int j = 0; j < 16; j++) { + _mm256_storeu_si256((__m256i *)(dst + j * pitchDst), d[j]); + } } -static uint8_t HighbdLoadMaskx[8][16] = { - { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }, - { 0, 1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13 }, - { 0, 1, 0, 1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 }, - { 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }, - { 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 4, 5, 6, 7 }, - { 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 4, 5 }, - { 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 2, 3 }, - { 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1 }, -}; +static void highbd_transpose(const uint16_t *src, ptrdiff_t pitchSrc, + uint16_t *dst, ptrdiff_t pitchDst, int width, + int height) { + for (int j = 0; j < height; j += 16) + for (int i = 0; i < width; i += 16) + highbd_transpose_TX_16X16(src + i * pitchSrc + j, pitchSrc, + dst + j * pitchDst + i, pitchDst); +} -static uint8_t HighbdEvenOddMaskx4[8][16] = { - { 0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, - 15 }, // 0=0,1, 1=2,3, 2=4,5, 3=6,7, 4=8,9, 5=10,11, 6=12,13, 7=14,15, - // >7=0,1 - { 0, 1, 2, 3, 6, 7, 10, 11, 14, 15, 4, 5, 8, 9, 12, 13 }, - { 0, 1, 0, 1, 4, 5, 8, 9, 12, 13, 0, 1, 6, 7, 10, 11 }, - { 0, 1, 0, 1, 0, 1, 6, 7, 10, 11, 14, 15, 0, 1, 8, 9 }, - { 0, 1, 0, 1, 0, 1, 0, 1, 8, 9, 12, 13, 0, 1, 0, 1 }, - { 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 10, 11, 14, 15, 0, 1 }, - { 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 12, 13, 0, 1 }, - { 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 14, 15 } -}; - -static uint16_t HighbdEvenOddMaskx8_2[8][16] = { - { 0, 2, 4, 6, 8, 10, 12, 14 }, { 2, 2, 4, 6, 8, 10, 12, 14 }, - { 4, 4, 4, 6, 8, 10, 12, 14 }, { 6, 6, 6, 6, 8, 10, 12, 14 }, - { 8, 8, 8, 8, 8, 10, 12, 14 }, { 10, 10, 10, 10, 10, 10, 12, 14 }, - { 12, 12, 12, 12, 12, 12, 12, 14 }, { 14, 14, 14, 14, 14, 14, 14, 14 }, -}; - -static uint16_t HighbdBaseMask[17][16] = { - { - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - }, - { 0xffff, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, - { 0xffff, 0xffff, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, - { 0xffff, 0xffff, 0xffff, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, - { 0xffff, 0xffff, 0xffff, 0xffff, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, - { 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, - { 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0 }, - { 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0, 0, 0, 0, 0, 0, 0, - 0, 0 }, - { 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0, 0, 0, 0, - 0, 0, 0, 0 }, - { 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0, - 0, 0, 0, 0, 0, 0 }, - { 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, - 0xffff, 0, 0, 0, 0, 0, 0 }, - { 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, - 0xffff, 0xffff, 0, 0, 0, 0, 0 }, - { 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, - 0xffff, 0xffff, 0xffff, 0, 0, 0, 0 }, - { 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, - 0xffff, 0xffff, 0xffff, 0xffff, 0, 0, 0 }, - { 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, - 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0, 0 }, - { 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, - 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0 }, - { 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, - 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff } -}; - -static void highbd_dr_prediction_z2_Nx4_avx2( +static void highbd_dr_prediction_32bit_z2_Nx4_avx2( int N, uint16_t *dst, ptrdiff_t stride, const uint16_t *above, const uint16_t *left, int upsample_above, int upsample_left, int dx, int dy) { @@ -1668,12 +1987,12 @@ static void highbd_dr_prediction_z2_Nx4_avx2( const int frac_bits_x = 6 - upsample_above; const int frac_bits_y = 6 - upsample_left; - // a assert(dx > 0); + assert(dx > 0); // pre-filter above pixels // store in temp buffers: // above[x] * 32 + 16 // above[x+1] - above[x] - // final pixels will be caluculated as: + // final pixels will be calculated as: // (above[x] * 32 + 16 + (above[x+1] - above[x]) * shift) >> 5 __m256i a0_x, a1_x, a32, a16; __m256i diff; @@ -1785,6 +2104,130 @@ static void highbd_dr_prediction_z2_Nx4_avx2( } } +static void highbd_dr_prediction_z2_Nx4_avx2( + int N, uint16_t *dst, ptrdiff_t stride, const uint16_t *above, + const uint16_t *left, int upsample_above, int upsample_left, int dx, + int dy) { + const int min_base_x = -(1 << upsample_above); + const int min_base_y = -(1 << upsample_left); + const int frac_bits_x = 6 - upsample_above; + const int frac_bits_y = 6 - upsample_left; + + assert(dx > 0); + // pre-filter above pixels + // store in temp buffers: + // above[x] * 32 + 16 + // above[x+1] - above[x] + // final pixels will be calculated as: + // (above[x] * 32 + 16 + (above[x+1] - above[x]) * shift) >> 5 + __m256i a0_x, a1_x, a32, a16; + __m256i diff; + __m128i c3f, min_base_y128; + + a16 = _mm256_set1_epi16(16); + c3f = _mm_set1_epi16(0x3f); + min_base_y128 = _mm_set1_epi16(min_base_y); + + for (int r = 0; r < N; r++) { + __m256i b, res, shift; + __m128i resx, resy, resxy; + __m128i a0_x128, a1_x128; + int y = r + 1; + int base_x = (-y * dx) >> frac_bits_x; + int base_shift = 0; + if (base_x < (min_base_x - 1)) { + base_shift = (min_base_x - base_x - 1) >> upsample_above; + } + int base_min_diff = + (min_base_x - base_x + upsample_above) >> upsample_above; + if (base_min_diff > 4) { + base_min_diff = 4; + } else { + if (base_min_diff < 0) base_min_diff = 0; + } + + if (base_shift > 3) { + a0_x = _mm256_setzero_si256(); + a1_x = _mm256_setzero_si256(); + shift = _mm256_setzero_si256(); + } else { + a0_x128 = _mm_loadu_si128((__m128i *)(above + base_x + base_shift)); + if (upsample_above) { + a0_x128 = _mm_shuffle_epi8(a0_x128, + *(__m128i *)HighbdEvenOddMaskx4[base_shift]); + a1_x128 = _mm_srli_si128(a0_x128, 8); + + shift = _mm256_castsi128_si256(_mm_srli_epi16( + _mm_and_si128( + _mm_slli_epi16(_mm_setr_epi16(-y * dx, (1 << 6) - y * dx, + (2 << 6) - y * dx, + (3 << 6) - y * dx, 0, 0, 0, 0), + upsample_above), + c3f), + 1)); + } else { + a0_x128 = + _mm_shuffle_epi8(a0_x128, *(__m128i *)HighbdLoadMaskx[base_shift]); + a1_x128 = _mm_srli_si128(a0_x128, 2); + + shift = _mm256_castsi128_si256(_mm_srli_epi16( + _mm_and_si128( + _mm_setr_epi16(-y * dx, (1 << 6) - y * dx, (2 << 6) - y * dx, + (3 << 6) - y * dx, 0, 0, 0, 0), + c3f), + 1)); + } + a0_x = _mm256_castsi128_si256(a0_x128); + a1_x = _mm256_castsi128_si256(a1_x128); + } + // y calc + __m128i a0_y, a1_y, shifty; + if (base_x < min_base_x) { + __m128i r6, c1234, dy128, y_c128, base_y_c128, mask128; + DECLARE_ALIGNED(32, int16_t, base_y_c[8]); + r6 = _mm_set1_epi16(r << 6); + dy128 = _mm_set1_epi16(dy); + c1234 = _mm_setr_epi16(1, 2, 3, 4, 0, 0, 0, 0); + y_c128 = _mm_sub_epi16(r6, _mm_mullo_epi16(c1234, dy128)); + base_y_c128 = _mm_srai_epi16(y_c128, frac_bits_y); + mask128 = _mm_cmpgt_epi16(min_base_y128, base_y_c128); + base_y_c128 = _mm_andnot_si128(mask128, base_y_c128); + _mm_store_si128((__m128i *)base_y_c, base_y_c128); + + a0_y = _mm_setr_epi16(left[base_y_c[0]], left[base_y_c[1]], + left[base_y_c[2]], left[base_y_c[3]], 0, 0, 0, 0); + a1_y = _mm_setr_epi16(left[base_y_c[0] + 1], left[base_y_c[1] + 1], + left[base_y_c[2] + 1], left[base_y_c[3] + 1], 0, 0, + 0, 0); + + if (upsample_left) { + shifty = _mm_srli_epi16( + _mm_and_si128(_mm_slli_epi16(y_c128, upsample_left), c3f), 1); + } else { + shifty = _mm_srli_epi16(_mm_and_si128(y_c128, c3f), 1); + } + a0_x = _mm256_inserti128_si256(a0_x, a0_y, 1); + a1_x = _mm256_inserti128_si256(a1_x, a1_y, 1); + shift = _mm256_inserti128_si256(shift, shifty, 1); + } + + diff = _mm256_sub_epi16(a1_x, a0_x); // a[x+1] - a[x] + a32 = _mm256_slli_epi16(a0_x, 5); // a[x] * 32 + a32 = _mm256_add_epi16(a32, a16); // a[x] * 32 + 16 + + b = _mm256_mullo_epi16(diff, shift); + res = _mm256_add_epi16(a32, b); + res = _mm256_srli_epi16(res, 5); + + resx = _mm256_castsi256_si128(res); + resy = _mm256_extracti128_si256(res, 1); + resxy = + _mm_blendv_epi8(resx, resy, *(__m128i *)HighbdBaseMask[base_min_diff]); + _mm_storel_epi64((__m128i *)(dst), resxy); + dst += stride; + } +} + static void highbd_dr_prediction_32bit_z2_Nx8_avx2( int N, uint16_t *dst, ptrdiff_t stride, const uint16_t *above, const uint16_t *left, int upsample_above, int upsample_left, int dx, @@ -1798,7 +2241,7 @@ static void highbd_dr_prediction_32bit_z2_Nx8_avx2( // store in temp buffers: // above[x] * 32 + 16 // above[x+1] - above[x] - // final pixels will be caluculated as: + // final pixels will be calculated as: // (above[x] * 32 + 16 + (above[x+1] - above[x]) * shift) >> 5 __m256i a0_x, a1_x, a0_y, a1_y, a32, a16, c3f, min_base_y256; __m256i diff; @@ -1828,25 +2271,24 @@ static void highbd_dr_prediction_32bit_z2_Nx8_avx2( if (base_shift > 7) { resx = _mm_setzero_si128(); } else { + a0_x128 = _mm_loadu_si128((__m128i *)(above + base_x + base_shift)); if (upsample_above) { - a0_x128 = _mm_setr_epi16( - above[base_x + HighbdEvenOddMaskx8_2[base_shift][0]], - above[base_x + HighbdEvenOddMaskx8_2[base_shift][1]], - above[base_x + HighbdEvenOddMaskx8_2[base_shift][2]], - above[base_x + HighbdEvenOddMaskx8_2[base_shift][3]], - above[base_x + HighbdEvenOddMaskx8_2[base_shift][4]], - above[base_x + HighbdEvenOddMaskx8_2[base_shift][5]], - above[base_x + HighbdEvenOddMaskx8_2[base_shift][6]], - above[base_x + HighbdEvenOddMaskx8_2[base_shift][7]]); - a1_x128 = _mm_setr_epi16( - above[base_x + 1 + HighbdEvenOddMaskx8_2[base_shift][0]], - above[base_x + 1 + HighbdEvenOddMaskx8_2[base_shift][1]], - above[base_x + 1 + HighbdEvenOddMaskx8_2[base_shift][2]], - above[base_x + 1 + HighbdEvenOddMaskx8_2[base_shift][3]], - above[base_x + 1 + HighbdEvenOddMaskx8_2[base_shift][4]], - above[base_x + 1 + HighbdEvenOddMaskx8_2[base_shift][5]], - above[base_x + 1 + HighbdEvenOddMaskx8_2[base_shift][6]], - above[base_x + 1 + HighbdEvenOddMaskx8_2[base_shift][7]]); + __m128i mask, atmp0, atmp1, atmp2, atmp3; + a1_x128 = _mm_loadu_si128((__m128i *)(above + base_x + 8 + base_shift)); + atmp0 = _mm_shuffle_epi8(a0_x128, + *(__m128i *)HighbdEvenOddMaskx[base_shift]); + atmp1 = _mm_shuffle_epi8(a1_x128, + *(__m128i *)HighbdEvenOddMaskx[base_shift]); + atmp2 = _mm_shuffle_epi8( + a0_x128, *(__m128i *)(HighbdEvenOddMaskx[base_shift] + 16)); + atmp3 = _mm_shuffle_epi8( + a1_x128, *(__m128i *)(HighbdEvenOddMaskx[base_shift] + 16)); + mask = _mm_cmpgt_epi8(*(__m128i *)HighbdEvenOddMaskx[base_shift], + _mm_set1_epi8(15)); + a0_x128 = _mm_blendv_epi8(atmp0, atmp1, mask); + mask = _mm_cmpgt_epi8(*(__m128i *)(HighbdEvenOddMaskx[base_shift] + 16), + _mm_set1_epi8(15)); + a1_x128 = _mm_blendv_epi8(atmp2, atmp3, mask); shift = _mm256_srli_epi32( _mm256_and_si256( _mm256_slli_epi32( @@ -1858,7 +2300,6 @@ static void highbd_dr_prediction_32bit_z2_Nx8_avx2( c3f), 1); } else { - a0_x128 = _mm_loadu_si128((__m128i *)(above + base_x + base_shift)); a1_x128 = _mm_loadu_si128((__m128i *)(above + base_x + 1 + base_shift)); a0_x128 = _mm_shuffle_epi8(a0_x128, *(__m128i *)HighbdLoadMaskx[base_shift]); @@ -1874,7 +2315,6 @@ static void highbd_dr_prediction_32bit_z2_Nx8_avx2( c3f), 1); } - a0_x = _mm256_cvtepu16_epi32(a0_x128); a1_x = _mm256_cvtepu16_epi32(a1_x128); @@ -1951,7 +2391,7 @@ static void highbd_dr_prediction_z2_Nx8_avx2( // store in temp buffers: // above[x] * 32 + 16 // above[x+1] - above[x] - // final pixels will be caluculated as: + // final pixels will be calculated as: // (above[x] * 32 + 16 + (above[x+1] - above[x]) * shift) >> 5 __m128i c3f, min_base_y128; __m256i a0_x, a1_x, diff, a32, a16; @@ -1983,25 +2423,25 @@ static void highbd_dr_prediction_z2_Nx8_avx2( a1_x = _mm256_setzero_si256(); shift = _mm256_setzero_si256(); } else { + a0_x128 = _mm_loadu_si128((__m128i *)(above + base_x + base_shift)); if (upsample_above) { - a0_x128 = _mm_setr_epi16( - above[base_x + HighbdEvenOddMaskx8_2[base_shift][0]], - above[base_x + HighbdEvenOddMaskx8_2[base_shift][1]], - above[base_x + HighbdEvenOddMaskx8_2[base_shift][2]], - above[base_x + HighbdEvenOddMaskx8_2[base_shift][3]], - above[base_x + HighbdEvenOddMaskx8_2[base_shift][4]], - above[base_x + HighbdEvenOddMaskx8_2[base_shift][5]], - above[base_x + HighbdEvenOddMaskx8_2[base_shift][6]], - above[base_x + HighbdEvenOddMaskx8_2[base_shift][7]]); - a1_x128 = _mm_setr_epi16( - above[base_x + 1 + HighbdEvenOddMaskx8_2[base_shift][0]], - above[base_x + 1 + HighbdEvenOddMaskx8_2[base_shift][1]], - above[base_x + 1 + HighbdEvenOddMaskx8_2[base_shift][2]], - above[base_x + 1 + HighbdEvenOddMaskx8_2[base_shift][3]], - above[base_x + 1 + HighbdEvenOddMaskx8_2[base_shift][4]], - above[base_x + 1 + HighbdEvenOddMaskx8_2[base_shift][5]], - above[base_x + 1 + HighbdEvenOddMaskx8_2[base_shift][6]], - above[base_x + 1 + HighbdEvenOddMaskx8_2[base_shift][7]]); + __m128i mask, atmp0, atmp1, atmp2, atmp3; + a1_x128 = _mm_loadu_si128((__m128i *)(above + base_x + 8 + base_shift)); + atmp0 = _mm_shuffle_epi8(a0_x128, + *(__m128i *)HighbdEvenOddMaskx[base_shift]); + atmp1 = _mm_shuffle_epi8(a1_x128, + *(__m128i *)HighbdEvenOddMaskx[base_shift]); + atmp2 = _mm_shuffle_epi8( + a0_x128, *(__m128i *)(HighbdEvenOddMaskx[base_shift] + 16)); + atmp3 = _mm_shuffle_epi8( + a1_x128, *(__m128i *)(HighbdEvenOddMaskx[base_shift] + 16)); + mask = _mm_cmpgt_epi8(*(__m128i *)HighbdEvenOddMaskx[base_shift], + _mm_set1_epi8(15)); + a0_x128 = _mm_blendv_epi8(atmp0, atmp1, mask); + mask = _mm_cmpgt_epi8(*(__m128i *)(HighbdEvenOddMaskx[base_shift] + 16), + _mm_set1_epi8(15)); + a1_x128 = _mm_blendv_epi8(atmp2, atmp3, mask); + shift = _mm256_castsi128_si256(_mm_srli_epi16( _mm_and_si128( _mm_slli_epi16( @@ -2013,7 +2453,6 @@ static void highbd_dr_prediction_z2_Nx8_avx2( c3f), 1)); } else { - a0_x128 = _mm_loadu_si128((__m128i *)(above + base_x + base_shift)); a1_x128 = _mm_loadu_si128((__m128i *)(above + base_x + 1 + base_shift)); a0_x128 = _mm_shuffle_epi8(a0_x128, *(__m128i *)HighbdLoadMaskx[base_shift]); @@ -2035,7 +2474,7 @@ static void highbd_dr_prediction_z2_Nx8_avx2( // y calc __m128i a0_y, a1_y, shifty; if (base_x < min_base_x) { - DECLARE_ALIGNED(32, int16_t, base_y_c[16]); + DECLARE_ALIGNED(32, int16_t, base_y_c[8]); __m128i r6, c1234, dy128, y_c128, base_y_c128, mask128; r6 = _mm_set1_epi16(r << 6); dy128 = _mm_set1_epi16(dy); @@ -2101,22 +2540,31 @@ static void highbd_dr_prediction_32bit_z2_HxW_avx2( // store in temp buffers: // above[x] * 32 + 16 // above[x+1] - above[x] - // final pixels will be caluculated as: + // final pixels will be calculated as: // (above[x] * 32 + 16 + (above[x+1] - above[x]) * shift) >> 5 - __m256i a0_x, a1_x, a0_y, a1_y, a32, a0_1_x, a1_1_x, a16; - __m256i diff, min_base_y256, c3f; + __m256i a0_x, a1_x, a0_y, a1_y, a32, a0_1_x, a1_1_x, a16, c1; + __m256i diff, min_base_y256, c3f, dy256, c1234, c0123, c8; __m128i a0_x128, a1_x128, a0_1_x128, a1_1_x128; + DECLARE_ALIGNED(32, int, base_y_c[16]); a16 = _mm256_set1_epi32(16); + c1 = _mm256_srli_epi32(a16, 4); + c8 = _mm256_srli_epi32(a16, 1); min_base_y256 = _mm256_set1_epi16(min_base_y); c3f = _mm256_set1_epi32(0x3f); + dy256 = _mm256_set1_epi32(dy); + c0123 = _mm256_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7); + c1234 = _mm256_add_epi32(c0123, c1); for (int r = 0; r < H; r++) { - __m256i b, res, shift; + __m256i b, res, shift, ydx; __m256i resx[2], resy[2]; - __m256i resxy; + __m256i resxy, j256, r6; for (int j = 0; j < W; j += 16) { + j256 = _mm256_set1_epi32(j); int y = r + 1; + ydx = _mm256_set1_epi32(y * dx); + int base_x = (-y * dx) >> frac_bits_x; int base_shift = 0; if ((base_x + j) < (min_base_x - 1)) { @@ -2143,15 +2591,9 @@ static void highbd_dr_prediction_32bit_z2_HxW_avx2( a0_x = _mm256_cvtepu16_epi32(a0_x128); a1_x = _mm256_cvtepu16_epi32(a1_x128); + r6 = _mm256_slli_epi32(_mm256_add_epi32(c0123, j256), 6); shift = _mm256_srli_epi32( - _mm256_and_si256( - _mm256_setr_epi32( - ((0 + j) << 6) - y * dx, ((1 + j) << 6) - y * dx, - ((2 + j) << 6) - y * dx, ((3 + j) << 6) - y * dx, - ((4 + j) << 6) - y * dx, ((5 + j) << 6) - y * dx, - ((6 + j) << 6) - y * dx, ((7 + j) << 6) - y * dx), - c3f), - 1); + _mm256_and_si256(_mm256_sub_epi32(r6, ydx), c3f), 1); diff = _mm256_sub_epi32(a1_x, a0_x); // a[x+1] - a[x] a32 = _mm256_slli_epi32(a0_x, 5); // a[x] * 32 @@ -2183,15 +2625,10 @@ static void highbd_dr_prediction_32bit_z2_HxW_avx2( a0_1_x = _mm256_cvtepu16_epi32(a0_1_x128); a1_1_x = _mm256_cvtepu16_epi32(a1_1_x128); + r6 = _mm256_slli_epi32( + _mm256_add_epi32(c0123, _mm256_add_epi32(j256, c8)), 6); shift = _mm256_srli_epi32( - _mm256_and_si256( - _mm256_setr_epi32( - ((8 + j) << 6) - y * dx, ((9 + j) << 6) - y * dx, - ((10 + j) << 6) - y * dx, ((11 + j) << 6) - y * dx, - ((12 + j) << 6) - y * dx, ((13 + j) << 6) - y * dx, - ((14 + j) << 6) - y * dx, ((15 + j) << 6) - y * dx), - c3f), - 1); + _mm256_and_si256(_mm256_sub_epi32(r6, ydx), c3f), 1); diff = _mm256_sub_epi32(a1_1_x, a0_1_x); // a[x+1] - a[x] a32 = _mm256_slli_epi32(a0_1_x, 5); // a[x] * 32 @@ -2209,20 +2646,17 @@ static void highbd_dr_prediction_32bit_z2_HxW_avx2( 1); // 16 16bit values // y calc + resy[0] = _mm256_setzero_si256(); if ((base_x < min_base_x)) { - DECLARE_ALIGNED(32, int, base_y_c[16]); - __m256i r6, c256, dy256, y_c256, y_c_1_256, base_y_c256, mask256; + __m256i c256, y_c256, y_c_1_256, base_y_c256, mask256; r6 = _mm256_set1_epi32(r << 6); - dy256 = _mm256_set1_epi32(dy); - c256 = _mm256_setr_epi32(1 + j, 2 + j, 3 + j, 4 + j, 5 + j, 6 + j, - 7 + j, 8 + j); + c256 = _mm256_add_epi32(j256, c1234); y_c256 = _mm256_sub_epi32(r6, _mm256_mullo_epi32(c256, dy256)); base_y_c256 = _mm256_srai_epi32(y_c256, frac_bits_y); mask256 = _mm256_cmpgt_epi32(min_base_y256, base_y_c256); base_y_c256 = _mm256_andnot_si256(mask256, base_y_c256); _mm256_store_si256((__m256i *)base_y_c, base_y_c256); - c256 = _mm256_setr_epi32(9 + j, 10 + j, 11 + j, 12 + j, 13 + j, 14 + j, - 15 + j, 16 + j); + c256 = _mm256_add_epi32(c256, c8); y_c_1_256 = _mm256_sub_epi32(r6, _mm256_mullo_epi32(c256, dy256)); base_y_c256 = _mm256_srai_epi32(y_c_1_256, frac_bits_y); mask256 = _mm256_cmpgt_epi32(min_base_y256, base_y_c256); @@ -2276,9 +2710,8 @@ static void highbd_dr_prediction_32bit_z2_HxW_avx2( resy[0] = _mm256_inserti128_si256(resy[0], _mm256_castsi256_si128(resy[1]), 1); // 16 16bit values - } else { - resy[0] = resx[0]; } + resxy = _mm256_blendv_epi8(resx[0], resy[0], *(__m256i *)HighbdBaseMask[base_min_diff]); _mm256_storeu_si256((__m256i *)(dst + j), resxy); @@ -2304,23 +2737,31 @@ static void highbd_dr_prediction_z2_HxW_avx2( // store in temp buffers: // above[x] * 32 + 16 // above[x+1] - above[x] - // final pixels will be caluculated as: + // final pixels will be calculated as: // (above[x] * 32 + 16 + (above[x+1] - above[x]) * shift) >> 5 - __m256i a0_x, a1_x, a32, a16, c3f; - __m256i diff, min_base_y256; + __m256i a0_x, a1_x, a32, a16, c3f, c1; + __m256i diff, min_base_y256, dy256, c1234, c0123; + DECLARE_ALIGNED(32, int16_t, base_y_c[16]); a16 = _mm256_set1_epi16(16); + c1 = _mm256_srli_epi16(a16, 4); min_base_y256 = _mm256_set1_epi16(min_base_y); c3f = _mm256_set1_epi16(0x3f); + dy256 = _mm256_set1_epi16(dy); + c0123 = + _mm256_setr_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + c1234 = _mm256_add_epi16(c0123, c1); for (int r = 0; r < H; r++) { __m256i b, res, shift; - __m256i resx, resy; - __m256i resxy; - __m128i a0_x128, a1_x128, a0_1_x128, a1_1_x128, shiftx; + __m256i resx, resy, ydx; + __m256i resxy, j256, r6; + __m128i a0_x128, a1_x128, a0_1_x128, a1_1_x128; + int y = r + 1; + ydx = _mm256_set1_epi16(y * dx); for (int j = 0; j < W; j += 16) { - int y = r + 1; + j256 = _mm256_set1_epi16(j); int base_x = (-y * dx) >> frac_bits_x; int base_shift = 0; if ((base_x + j) < (min_base_x - 1)) { @@ -2333,11 +2774,7 @@ static void highbd_dr_prediction_z2_HxW_avx2( if (base_min_diff < 0) base_min_diff = 0; } - if (base_shift > 7) { - a0_x = _mm256_setzero_si256(); - a1_x = _mm256_setzero_si256(); - shift = _mm256_setzero_si256(); - } else { + if (base_shift < 8) { a0_x128 = _mm_loadu_si128((__m128i *)(above + base_x + base_shift + j)); a1_x128 = _mm_loadu_si128((__m128i *)(above + base_x + base_shift + 1 + j)); @@ -2348,45 +2785,31 @@ static void highbd_dr_prediction_z2_HxW_avx2( a0_x = _mm256_castsi128_si256(a0_x128); a1_x = _mm256_castsi128_si256(a1_x128); - - shift = _mm256_castsi128_si256(_mm_srli_epi16( - _mm_and_si128(_mm_setr_epi16( - ((0 + j) << 6) - y * dx, ((1 + j) << 6) - y * dx, - ((2 + j) << 6) - y * dx, ((3 + j) << 6) - y * dx, - ((4 + j) << 6) - y * dx, ((5 + j) << 6) - y * dx, - ((6 + j) << 6) - y * dx, ((7 + j) << 6) - y * dx), - _mm256_castsi256_si128(c3f)), - 1)); + } else { + a0_x = _mm256_setzero_si256(); + a1_x = _mm256_setzero_si256(); } - base_shift = 0; - if ((base_x + j + 8) < (min_base_x - 1)) { - base_shift = (min_base_x - (base_x + j + 8) - 1); + int base_shift1 = 0; + if (base_shift > 8) { + base_shift1 = base_shift - 8; } - if (base_shift <= 7) { + if (base_shift1 < 8) { a0_1_x128 = - _mm_loadu_si128((__m128i *)(above + base_x + base_shift + 8 + j)); + _mm_loadu_si128((__m128i *)(above + base_x + base_shift1 + 8 + j)); a1_1_x128 = - _mm_loadu_si128((__m128i *)(above + base_x + base_shift + 9 + j)); + _mm_loadu_si128((__m128i *)(above + base_x + base_shift1 + 9 + j)); a0_1_x128 = _mm_shuffle_epi8(a0_1_x128, - *(__m128i *)HighbdLoadMaskx[base_shift]); + *(__m128i *)HighbdLoadMaskx[base_shift1]); a1_1_x128 = _mm_shuffle_epi8(a1_1_x128, - *(__m128i *)HighbdLoadMaskx[base_shift]); - - shiftx = _mm_srli_epi16( - _mm_and_si128( - _mm_setr_epi16( - ((8 + j) << 6) - y * dx, ((9 + j) << 6) - y * dx, - ((10 + j) << 6) - y * dx, ((11 + j) << 6) - y * dx, - ((12 + j) << 6) - y * dx, ((13 + j) << 6) - y * dx, - ((14 + j) << 6) - y * dx, ((15 + j) << 6) - y * dx), - _mm256_castsi256_si128(c3f)), - 1); + *(__m128i *)HighbdLoadMaskx[base_shift1]); a0_x = _mm256_inserti128_si256(a0_x, a0_1_x128, 1); a1_x = _mm256_inserti128_si256(a1_x, a1_1_x128, 1); - shift = _mm256_inserti128_si256(shift, shiftx, 1); } + r6 = _mm256_slli_epi16(_mm256_add_epi16(c0123, j256), 6); + shift = _mm256_srli_epi16( + _mm256_and_si256(_mm256_sub_epi16(r6, ydx), c3f), 1); diff = _mm256_sub_epi16(a1_x, a0_x); // a[x+1] - a[x] a32 = _mm256_slli_epi16(a0_x, 5); // a[x] * 32 @@ -2397,15 +2820,12 @@ static void highbd_dr_prediction_z2_HxW_avx2( resx = _mm256_srli_epi16(res, 5); // 16 16-bit values // y calc + resy = _mm256_setzero_si256(); __m256i a0_y, a1_y, shifty; if ((base_x < min_base_x)) { - DECLARE_ALIGNED(32, int16_t, base_y_c[16]); - __m256i r6, c256, dy256, y_c256, base_y_c256, mask256, mul16; + __m256i c256, y_c256, base_y_c256, mask256, mul16; r6 = _mm256_set1_epi16(r << 6); - dy256 = _mm256_set1_epi16(dy); - c256 = _mm256_setr_epi16(1 + j, 2 + j, 3 + j, 4 + j, 5 + j, 6 + j, - 7 + j, 8 + j, 9 + j, 10 + j, 11 + j, 12 + j, - 13 + j, 14 + j, 15 + j, 16 + j); + c256 = _mm256_add_epi16(j256, c1234); mul16 = _mm256_min_epu16(_mm256_mullo_epi16(c256, dy256), _mm256_srli_epi16(min_base_y256, 1)); y_c256 = _mm256_sub_epi16(r6, mul16); @@ -2421,14 +2841,16 @@ static void highbd_dr_prediction_z2_HxW_avx2( left[base_y_c[9]], left[base_y_c[10]], left[base_y_c[11]], left[base_y_c[12]], left[base_y_c[13]], left[base_y_c[14]], left[base_y_c[15]]); + base_y_c256 = _mm256_add_epi16(base_y_c256, c1); + _mm256_store_si256((__m256i *)base_y_c, base_y_c256); + a1_y = _mm256_setr_epi16( - left[base_y_c[0] + 1], left[base_y_c[1] + 1], left[base_y_c[2] + 1], - left[base_y_c[3] + 1], left[base_y_c[4] + 1], left[base_y_c[5] + 1], - left[base_y_c[6] + 1], left[base_y_c[7] + 1], left[base_y_c[8] + 1], - left[base_y_c[9] + 1], left[base_y_c[10] + 1], - left[base_y_c[11] + 1], left[base_y_c[12] + 1], - left[base_y_c[13] + 1], left[base_y_c[14] + 1], - left[base_y_c[15] + 1]); + left[base_y_c[0]], left[base_y_c[1]], left[base_y_c[2]], + left[base_y_c[3]], left[base_y_c[4]], left[base_y_c[5]], + left[base_y_c[6]], left[base_y_c[7]], left[base_y_c[8]], + left[base_y_c[9]], left[base_y_c[10]], left[base_y_c[11]], + left[base_y_c[12]], left[base_y_c[13]], left[base_y_c[14]], + left[base_y_c[15]]); shifty = _mm256_srli_epi16(_mm256_and_si256(y_c256, c3f), 1); @@ -2439,8 +2861,6 @@ static void highbd_dr_prediction_z2_HxW_avx2( b = _mm256_mullo_epi16(diff, shifty); res = _mm256_add_epi16(a32, b); resy = _mm256_srli_epi16(res, 5); - } else { - resy = _mm256_setzero_si256(); } resxy = _mm256_blendv_epi8(resx, resy, @@ -2462,8 +2882,14 @@ void av1_highbd_dr_prediction_z2_avx2(uint16_t *dst, ptrdiff_t stride, int bw, assert(dy > 0); switch (bw) { case 4: - highbd_dr_prediction_z2_Nx4_avx2(bh, dst, stride, above, left, - upsample_above, upsample_left, dx, dy); + if (bd < 12) { + highbd_dr_prediction_z2_Nx4_avx2(bh, dst, stride, above, left, + upsample_above, upsample_left, dx, dy); + } else { + highbd_dr_prediction_32bit_z2_Nx4_avx2(bh, dst, stride, above, left, + upsample_above, upsample_left, + dx, dy); + } break; case 8: if (bd < 12) { @@ -2488,21 +2914,19 @@ void av1_highbd_dr_prediction_z2_avx2(uint16_t *dst, ptrdiff_t stride, int bw, } } -static void highbd_transpose(const uint16_t *src, ptrdiff_t pitchSrc, - uint16_t *dst, ptrdiff_t pitchDst, int width, - int height) { - for (int j = 0; j < height; j += 8) - for (int i = 0; i < width; i += 8) - highbd_transpose_TX_8X8(src + i * pitchSrc + j, pitchSrc, - dst + j * pitchDst + i, pitchDst); -} - +// Directional prediction, zone 3 functions static void highbd_dr_prediction_z3_4x4_avx2(uint16_t *dst, ptrdiff_t stride, const uint16_t *left, - int upsample_left, int dy) { + int upsample_left, int dy, + int bd) { __m128i dstvec[4], d[4]; - - highbd_dr_prediction_z1_4xN_internal_avx2(4, dstvec, left, upsample_left, dy); + if (bd < 12) { + highbd_dr_prediction_z1_4xN_internal_avx2(4, dstvec, left, upsample_left, + dy); + } else { + highbd_dr_prediction_32bit_z1_4xN_internal_avx2(4, dstvec, left, + upsample_left, dy); + } highbd_transpose4x8_8x4_low_sse2(&dstvec[0], &dstvec[1], &dstvec[2], &dstvec[3], &d[0], &d[1], &d[2], &d[3]); _mm_storel_epi64((__m128i *)(dst + 0 * stride), d[0]); @@ -2514,10 +2938,16 @@ static void highbd_dr_prediction_z3_4x4_avx2(uint16_t *dst, ptrdiff_t stride, static void highbd_dr_prediction_z3_8x8_avx2(uint16_t *dst, ptrdiff_t stride, const uint16_t *left, - int upsample_left, int dy) { + int upsample_left, int dy, + int bd) { __m128i dstvec[8], d[8]; - - highbd_dr_prediction_z1_8xN_internal_avx2(8, dstvec, left, upsample_left, dy); + if (bd < 12) { + highbd_dr_prediction_z1_8xN_internal_avx2(8, dstvec, left, upsample_left, + dy); + } else { + highbd_dr_prediction_32bit_z1_8xN_internal_avx2(8, dstvec, left, + upsample_left, dy); + } highbd_transpose8x8_sse2(&dstvec[0], &dstvec[1], &dstvec[2], &dstvec[3], &dstvec[4], &dstvec[5], &dstvec[6], &dstvec[7], &d[0], &d[1], &d[2], &d[3], &d[4], &d[5], &d[6], @@ -2529,10 +2959,17 @@ static void highbd_dr_prediction_z3_8x8_avx2(uint16_t *dst, ptrdiff_t stride, static void highbd_dr_prediction_z3_4x8_avx2(uint16_t *dst, ptrdiff_t stride, const uint16_t *left, - int upsample_left, int dy) { + int upsample_left, int dy, + int bd) { __m128i dstvec[4], d[8]; + if (bd < 12) { + highbd_dr_prediction_z1_8xN_internal_avx2(4, dstvec, left, upsample_left, + dy); + } else { + highbd_dr_prediction_32bit_z1_8xN_internal_avx2(4, dstvec, left, + upsample_left, dy); + } - highbd_dr_prediction_z1_8xN_internal_avx2(4, dstvec, left, upsample_left, dy); highbd_transpose4x8_8x4_sse2(&dstvec[0], &dstvec[1], &dstvec[2], &dstvec[3], &d[0], &d[1], &d[2], &d[3], &d[4], &d[5], &d[6], &d[7]); @@ -2543,10 +2980,17 @@ static void highbd_dr_prediction_z3_4x8_avx2(uint16_t *dst, ptrdiff_t stride, static void highbd_dr_prediction_z3_8x4_avx2(uint16_t *dst, ptrdiff_t stride, const uint16_t *left, - int upsample_left, int dy) { + int upsample_left, int dy, + int bd) { __m128i dstvec[8], d[4]; + if (bd < 12) { + highbd_dr_prediction_z1_4xN_internal_avx2(8, dstvec, left, upsample_left, + dy); + } else { + highbd_dr_prediction_32bit_z1_4xN_internal_avx2(8, dstvec, left, + upsample_left, dy); + } - highbd_dr_prediction_z1_4xN_internal_avx2(8, dstvec, left, upsample_left, dy); highbd_transpose8x8_low_sse2(&dstvec[0], &dstvec[1], &dstvec[2], &dstvec[3], &dstvec[4], &dstvec[5], &dstvec[6], &dstvec[7], &d[0], &d[1], &d[2], &d[3]); @@ -2558,11 +3002,16 @@ static void highbd_dr_prediction_z3_8x4_avx2(uint16_t *dst, ptrdiff_t stride, static void highbd_dr_prediction_z3_8x16_avx2(uint16_t *dst, ptrdiff_t stride, const uint16_t *left, - int upsample_left, int dy) { + int upsample_left, int dy, + int bd) { __m256i dstvec[8], d[8]; - - highbd_dr_prediction_z1_16xN_internal_avx2(8, dstvec, left, upsample_left, - dy); + if (bd < 12) { + highbd_dr_prediction_z1_16xN_internal_avx2(8, dstvec, left, upsample_left, + dy); + } else { + highbd_dr_prediction_32bit_z1_16xN_internal_avx2(8, dstvec, left, + upsample_left, dy); + } highbd_transpose8x16_16x8_avx2(dstvec, d); for (int i = 0; i < 8; i++) { _mm_storeu_si128((__m128i *)(dst + i * stride), @@ -2576,11 +3025,16 @@ static void highbd_dr_prediction_z3_8x16_avx2(uint16_t *dst, ptrdiff_t stride, static void highbd_dr_prediction_z3_16x8_avx2(uint16_t *dst, ptrdiff_t stride, const uint16_t *left, - int upsample_left, int dy) { + int upsample_left, int dy, + int bd) { __m128i dstvec[16], d[16]; - - highbd_dr_prediction_z1_8xN_internal_avx2(16, dstvec, left, upsample_left, - dy); + if (bd < 12) { + highbd_dr_prediction_z1_8xN_internal_avx2(16, dstvec, left, upsample_left, + dy); + } else { + highbd_dr_prediction_32bit_z1_8xN_internal_avx2(16, dstvec, left, + upsample_left, dy); + } for (int i = 0; i < 16; i += 8) { highbd_transpose8x8_sse2(&dstvec[0 + i], &dstvec[1 + i], &dstvec[2 + i], &dstvec[3 + i], &dstvec[4 + i], &dstvec[5 + i], @@ -2596,11 +3050,16 @@ static void highbd_dr_prediction_z3_16x8_avx2(uint16_t *dst, ptrdiff_t stride, static void highbd_dr_prediction_z3_4x16_avx2(uint16_t *dst, ptrdiff_t stride, const uint16_t *left, - int upsample_left, int dy) { + int upsample_left, int dy, + int bd) { __m256i dstvec[4], d[4], d1; - - highbd_dr_prediction_z1_16xN_internal_avx2(4, dstvec, left, upsample_left, - dy); + if (bd < 12) { + highbd_dr_prediction_z1_16xN_internal_avx2(4, dstvec, left, upsample_left, + dy); + } else { + highbd_dr_prediction_32bit_z1_16xN_internal_avx2(4, dstvec, left, + upsample_left, dy); + } highbd_transpose4x16_avx2(dstvec, d); for (int i = 0; i < 4; i++) { _mm_storel_epi64((__m128i *)(dst + i * stride), @@ -2617,11 +3076,16 @@ static void highbd_dr_prediction_z3_4x16_avx2(uint16_t *dst, ptrdiff_t stride, static void highbd_dr_prediction_z3_16x4_avx2(uint16_t *dst, ptrdiff_t stride, const uint16_t *left, - int upsample_left, int dy) { + int upsample_left, int dy, + int bd) { __m128i dstvec[16], d[8]; - - highbd_dr_prediction_z1_4xN_internal_avx2(16, dstvec, left, upsample_left, - dy); + if (bd < 12) { + highbd_dr_prediction_z1_4xN_internal_avx2(16, dstvec, left, upsample_left, + dy); + } else { + highbd_dr_prediction_32bit_z1_4xN_internal_avx2(16, dstvec, left, + upsample_left, dy); + } highbd_transpose16x4_8x8_sse2(dstvec, d); _mm_storeu_si128((__m128i *)(dst + 0 * stride), d[0]); @@ -2636,11 +3100,17 @@ static void highbd_dr_prediction_z3_16x4_avx2(uint16_t *dst, ptrdiff_t stride, static void highbd_dr_prediction_z3_8x32_avx2(uint16_t *dst, ptrdiff_t stride, const uint16_t *left, - int upsample_left, int dy) { + int upsample_left, int dy, + int bd) { __m256i dstvec[16], d[16]; + if (bd < 12) { + highbd_dr_prediction_z1_32xN_internal_avx2(8, dstvec, left, upsample_left, + dy); + } else { + highbd_dr_prediction_32bit_z1_32xN_internal_avx2(8, dstvec, left, + upsample_left, dy); + } - highbd_dr_prediction_z1_32xN_internal_avx2(8, dstvec, left, upsample_left, - dy); for (int i = 0; i < 16; i += 8) { highbd_transpose8x16_16x8_avx2(dstvec + i, d + i); } @@ -2665,11 +3135,17 @@ static void highbd_dr_prediction_z3_8x32_avx2(uint16_t *dst, ptrdiff_t stride, static void highbd_dr_prediction_z3_32x8_avx2(uint16_t *dst, ptrdiff_t stride, const uint16_t *left, - int upsample_left, int dy) { + int upsample_left, int dy, + int bd) { __m128i dstvec[32], d[32]; + if (bd < 12) { + highbd_dr_prediction_z1_8xN_internal_avx2(32, dstvec, left, upsample_left, + dy); + } else { + highbd_dr_prediction_32bit_z1_8xN_internal_avx2(32, dstvec, left, + upsample_left, dy); + } - highbd_dr_prediction_z1_8xN_internal_avx2(32, dstvec, left, upsample_left, - dy); for (int i = 0; i < 32; i += 8) { highbd_transpose8x8_sse2(&dstvec[0 + i], &dstvec[1 + i], &dstvec[2 + i], &dstvec[3 + i], &dstvec[4 + i], &dstvec[5 + i], @@ -2687,11 +3163,17 @@ static void highbd_dr_prediction_z3_32x8_avx2(uint16_t *dst, ptrdiff_t stride, static void highbd_dr_prediction_z3_16x16_avx2(uint16_t *dst, ptrdiff_t stride, const uint16_t *left, - int upsample_left, int dy) { + int upsample_left, int dy, + int bd) { __m256i dstvec[16], d[16]; + if (bd < 12) { + highbd_dr_prediction_z1_16xN_internal_avx2(16, dstvec, left, upsample_left, + dy); + } else { + highbd_dr_prediction_32bit_z1_16xN_internal_avx2(16, dstvec, left, + upsample_left, dy); + } - highbd_dr_prediction_z1_16xN_internal_avx2(16, dstvec, left, upsample_left, - dy); highbd_transpose16x16_avx2(dstvec, d); for (int i = 0; i < 16; i++) { @@ -2701,12 +3183,16 @@ static void highbd_dr_prediction_z3_16x16_avx2(uint16_t *dst, ptrdiff_t stride, static void highbd_dr_prediction_z3_32x32_avx2(uint16_t *dst, ptrdiff_t stride, const uint16_t *left, - int upsample_left, int dy) { + int upsample_left, int dy, + int bd) { __m256i dstvec[64], d[16]; - - highbd_dr_prediction_z1_32xN_internal_avx2(32, dstvec, left, upsample_left, - dy); - + if (bd < 12) { + highbd_dr_prediction_z1_32xN_internal_avx2(32, dstvec, left, upsample_left, + dy); + } else { + highbd_dr_prediction_32bit_z1_32xN_internal_avx2(32, dstvec, left, + upsample_left, dy); + } highbd_transpose16x16_avx2(dstvec, d); for (int j = 0; j < 16; j++) { _mm256_storeu_si256((__m256i *)(dst + j * stride), d[j]); @@ -2727,19 +3213,30 @@ static void highbd_dr_prediction_z3_32x32_avx2(uint16_t *dst, ptrdiff_t stride, static void highbd_dr_prediction_z3_64x64_avx2(uint16_t *dst, ptrdiff_t stride, const uint16_t *left, - int upsample_left, int dy) { + int upsample_left, int dy, + int bd) { DECLARE_ALIGNED(16, uint16_t, dstT[64 * 64]); - highbd_dr_prediction_z1_64xN_avx2(64, dstT, 64, left, upsample_left, dy); + if (bd < 12) { + highbd_dr_prediction_z1_64xN_avx2(64, dstT, 64, left, upsample_left, dy); + } else { + highbd_dr_prediction_32bit_z1_64xN_avx2(64, dstT, 64, left, upsample_left, + dy); + } highbd_transpose(dstT, 64, dst, stride, 64, 64); } static void highbd_dr_prediction_z3_16x32_avx2(uint16_t *dst, ptrdiff_t stride, const uint16_t *left, - int upsample_left, int dy) { + int upsample_left, int dy, + int bd) { __m256i dstvec[32], d[32]; - - highbd_dr_prediction_z1_32xN_internal_avx2(16, dstvec, left, upsample_left, - dy); + if (bd < 12) { + highbd_dr_prediction_z1_32xN_internal_avx2(16, dstvec, left, upsample_left, + dy); + } else { + highbd_dr_prediction_32bit_z1_32xN_internal_avx2(16, dstvec, left, + upsample_left, dy); + } for (int i = 0; i < 32; i += 8) { highbd_transpose8x16_16x8_avx2(dstvec + i, d + i); } @@ -2764,11 +3261,16 @@ static void highbd_dr_prediction_z3_16x32_avx2(uint16_t *dst, ptrdiff_t stride, static void highbd_dr_prediction_z3_32x16_avx2(uint16_t *dst, ptrdiff_t stride, const uint16_t *left, - int upsample_left, int dy) { + int upsample_left, int dy, + int bd) { __m256i dstvec[32], d[16]; - - highbd_dr_prediction_z1_16xN_internal_avx2(32, dstvec, left, upsample_left, - dy); + if (bd < 12) { + highbd_dr_prediction_z1_16xN_internal_avx2(32, dstvec, left, upsample_left, + dy); + } else { + highbd_dr_prediction_32bit_z1_16xN_internal_avx2(32, dstvec, left, + upsample_left, dy); + } for (int i = 0; i < 32; i += 16) { highbd_transpose16x16_avx2((dstvec + i), d); for (int j = 0; j < 16; j++) { @@ -2779,36 +3281,54 @@ static void highbd_dr_prediction_z3_32x16_avx2(uint16_t *dst, ptrdiff_t stride, static void highbd_dr_prediction_z3_32x64_avx2(uint16_t *dst, ptrdiff_t stride, const uint16_t *left, - int upsample_left, int dy) { + int upsample_left, int dy, + int bd) { uint16_t dstT[64 * 32]; - highbd_dr_prediction_z1_64xN_avx2(32, dstT, 64, left, upsample_left, dy); + if (bd < 12) { + highbd_dr_prediction_z1_64xN_avx2(32, dstT, 64, left, upsample_left, dy); + } else { + highbd_dr_prediction_32bit_z1_64xN_avx2(32, dstT, 64, left, upsample_left, + dy); + } highbd_transpose(dstT, 64, dst, stride, 32, 64); } static void highbd_dr_prediction_z3_64x32_avx2(uint16_t *dst, ptrdiff_t stride, const uint16_t *left, - int upsample_left, int dy) { + int upsample_left, int dy, + int bd) { DECLARE_ALIGNED(16, uint16_t, dstT[32 * 64]); - highbd_dr_prediction_z1_32xN_avx2(64, dstT, 32, left, upsample_left, dy); + highbd_dr_prediction_z1_32xN_avx2(64, dstT, 32, left, upsample_left, dy, bd); highbd_transpose(dstT, 32, dst, stride, 64, 32); return; } static void highbd_dr_prediction_z3_16x64_avx2(uint16_t *dst, ptrdiff_t stride, const uint16_t *left, - int upsample_left, int dy) { + int upsample_left, int dy, + int bd) { DECLARE_ALIGNED(16, uint16_t, dstT[64 * 16]); - highbd_dr_prediction_z1_64xN_avx2(16, dstT, 64, left, upsample_left, dy); + if (bd < 12) { + highbd_dr_prediction_z1_64xN_avx2(16, dstT, 64, left, upsample_left, dy); + } else { + highbd_dr_prediction_32bit_z1_64xN_avx2(16, dstT, 64, left, upsample_left, + dy); + } highbd_transpose(dstT, 64, dst, stride, 16, 64); } static void highbd_dr_prediction_z3_64x16_avx2(uint16_t *dst, ptrdiff_t stride, const uint16_t *left, - int upsample_left, int dy) { + int upsample_left, int dy, + int bd) { __m256i dstvec[64], d[16]; - - highbd_dr_prediction_z1_16xN_internal_avx2(64, dstvec, left, upsample_left, - dy); + if (bd < 12) { + highbd_dr_prediction_z1_16xN_internal_avx2(64, dstvec, left, upsample_left, + dy); + } else { + highbd_dr_prediction_32bit_z1_16xN_internal_avx2(64, dstvec, left, + upsample_left, dy); + } for (int i = 0; i < 64; i += 16) { highbd_transpose16x16_avx2((dstvec + i), d); for (int j = 0; j < 16; j++) { @@ -2823,28 +3343,30 @@ void av1_highbd_dr_prediction_z3_avx2(uint16_t *dst, ptrdiff_t stride, int bw, int dx, int dy, int bd) { (void)above; (void)dx; - (void)bd; + assert(dx == 1); assert(dy > 0); if (bw == bh) { switch (bw) { case 4: - highbd_dr_prediction_z3_4x4_avx2(dst, stride, left, upsample_left, dy); + highbd_dr_prediction_z3_4x4_avx2(dst, stride, left, upsample_left, dy, + bd); break; case 8: - highbd_dr_prediction_z3_8x8_avx2(dst, stride, left, upsample_left, dy); + highbd_dr_prediction_z3_8x8_avx2(dst, stride, left, upsample_left, dy, + bd); break; case 16: - highbd_dr_prediction_z3_16x16_avx2(dst, stride, left, upsample_left, - dy); + highbd_dr_prediction_z3_16x16_avx2(dst, stride, left, upsample_left, dy, + bd); break; case 32: - highbd_dr_prediction_z3_32x32_avx2(dst, stride, left, upsample_left, - dy); + highbd_dr_prediction_z3_32x32_avx2(dst, stride, left, upsample_left, dy, + bd); break; case 64: - highbd_dr_prediction_z3_64x64_avx2(dst, stride, left, upsample_left, - dy); + highbd_dr_prediction_z3_64x64_avx2(dst, stride, left, upsample_left, dy, + bd); break; } } else { @@ -2853,34 +3375,34 @@ void av1_highbd_dr_prediction_z3_avx2(uint16_t *dst, ptrdiff_t stride, int bw, switch (bw) { case 4: highbd_dr_prediction_z3_4x8_avx2(dst, stride, left, upsample_left, - dy); + dy, bd); break; case 8: highbd_dr_prediction_z3_8x16_avx2(dst, stride, left, upsample_left, - dy); + dy, bd); break; case 16: highbd_dr_prediction_z3_16x32_avx2(dst, stride, left, upsample_left, - dy); + dy, bd); break; case 32: highbd_dr_prediction_z3_32x64_avx2(dst, stride, left, upsample_left, - dy); + dy, bd); break; } } else { switch (bw) { case 4: highbd_dr_prediction_z3_4x16_avx2(dst, stride, left, upsample_left, - dy); + dy, bd); break; case 8: highbd_dr_prediction_z3_8x32_avx2(dst, stride, left, upsample_left, - dy); + dy, bd); break; case 16: highbd_dr_prediction_z3_16x64_avx2(dst, stride, left, upsample_left, - dy); + dy, bd); break; } } @@ -2889,34 +3411,34 @@ void av1_highbd_dr_prediction_z3_avx2(uint16_t *dst, ptrdiff_t stride, int bw, switch (bh) { case 4: highbd_dr_prediction_z3_8x4_avx2(dst, stride, left, upsample_left, - dy); + dy, bd); break; case 8: highbd_dr_prediction_z3_16x8_avx2(dst, stride, left, upsample_left, - dy); + dy, bd); break; case 16: highbd_dr_prediction_z3_32x16_avx2(dst, stride, left, upsample_left, - dy); + dy, bd); break; case 32: highbd_dr_prediction_z3_64x32_avx2(dst, stride, left, upsample_left, - dy); + dy, bd); break; } } else { switch (bh) { case 4: highbd_dr_prediction_z3_16x4_avx2(dst, stride, left, upsample_left, - dy); + dy, bd); break; case 8: highbd_dr_prediction_z3_32x8_avx2(dst, stride, left, upsample_left, - dy); + dy, bd); break; case 16: highbd_dr_prediction_z3_64x16_avx2(dst, stride, left, upsample_left, - dy); + dy, bd); break; } } @@ -2926,7 +3448,7 @@ void av1_highbd_dr_prediction_z3_avx2(uint16_t *dst, ptrdiff_t stride, int bw, } // Low bit depth functions -static uint8_t BaseMask[33][32] = { +static DECLARE_ALIGNED(32, uint8_t, BaseMask[33][32]) = { { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, { 0xff, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -3018,17 +3540,48 @@ static uint8_t BaseMask[33][32] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, }; -static AOM_FORCE_INLINE void dr_prediction_z1_4xN_internal_avx2( - int N, __m128i *dst, const uint8_t *above, int upsample_above, int dx) { +static DECLARE_ALIGNED(16, uint8_t, LoadMaskx[16][16]) = { + { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }, + { 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14 }, + { 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13 }, + { 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 }, + { 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 }, + { 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }, + { 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }, + { 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8 }, + { 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7 }, + { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6 }, + { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5 }, + { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4 }, + { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3 }, + { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2 }, + { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1 }, + { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, +}; + +static DECLARE_ALIGNED(16, uint8_t, EvenOddMaskx[8][16]) = { + { 0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15 }, + { 0, 1, 3, 5, 7, 9, 11, 13, 0, 2, 4, 6, 8, 10, 12, 14 }, + { 0, 0, 2, 4, 6, 8, 10, 12, 0, 0, 3, 5, 7, 9, 11, 13 }, + { 0, 0, 0, 3, 5, 7, 9, 11, 0, 0, 0, 4, 6, 8, 10, 12 }, + { 0, 0, 0, 0, 4, 6, 8, 10, 0, 0, 0, 0, 5, 7, 9, 11 }, + { 0, 0, 0, 0, 0, 5, 7, 9, 0, 0, 0, 0, 0, 6, 8, 10 }, + { 0, 0, 0, 0, 0, 0, 6, 8, 0, 0, 0, 0, 0, 0, 7, 9 }, + { 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 8 } +}; + +static AOM_FORCE_INLINE void dr_prediction_z1_HxW_internal_avx2( + int H, int W, __m128i *dst, const uint8_t *above, int upsample_above, + int dx) { const int frac_bits = 6 - upsample_above; - const int max_base_x = ((N + 4) - 1) << upsample_above; - int x; - // a assert(dx > 0); + const int max_base_x = ((W + H) - 1) << upsample_above; + + assert(dx > 0); // pre-filter above pixels // store in temp buffers: // above[x] * 32 + 16 // above[x+1] - above[x] - // final pixels will be caluculated as: + // final pixels will be calculated as: // (above[x] * 32 + 16 + (above[x+1] - above[x]) * shift) >> 5 __m256i a0, a1, a32, a16; __m256i diff, c3f; @@ -3038,28 +3591,26 @@ static AOM_FORCE_INLINE void dr_prediction_z1_4xN_internal_avx2( a_mbase_x = _mm_set1_epi8(above[max_base_x]); c3f = _mm256_set1_epi16(0x3f); - x = dx; - for (int r = 0; r < N; r++) { + int x = dx; + for (int r = 0; r < W; r++) { __m256i b, res, shift; __m128i res1, a0_128, a1_128; int base = x >> frac_bits; int base_max_diff = (max_base_x - base) >> upsample_above; if (base_max_diff <= 0) { - for (int i = r; i < N; ++i) { + for (int i = r; i < W; ++i) { dst[i] = a_mbase_x; // save 4 values } return; } - if (base_max_diff > 4) base_max_diff = 4; + if (base_max_diff > H) base_max_diff = H; a0_128 = _mm_loadu_si128((__m128i *)(above + base)); - a1_128 = _mm_srli_si128(a0_128, 1); + a1_128 = _mm_loadu_si128((__m128i *)(above + base + 1)); if (upsample_above) { - a0_128 = _mm_shuffle_epi8( - a0_128, - _mm_setr_epi8(0, 2, 4, 6, 1, 3, 5, 7, 8, 10, 12, 14, 9, 11, 13, 15)); - a1_128 = _mm_srli_si128(a0_128, 4); + a0_128 = _mm_shuffle_epi8(a0_128, *(__m128i *)EvenOddMaskx[0]); + a1_128 = _mm_srli_si128(a0_128, 8); shift = _mm256_srli_epi16( _mm256_and_si256( @@ -3079,8 +3630,10 @@ static AOM_FORCE_INLINE void dr_prediction_z1_4xN_internal_avx2( res = _mm256_add_epi16(a32, b); res = _mm256_srli_epi16(res, 5); - res1 = _mm256_castsi256_si128(res); - res1 = _mm_packus_epi16(res1, res1); + res = _mm256_packus_epi16( + res, _mm256_castsi128_si256( + _mm256_extracti128_si256(res, 1))); // goto 8 bit + res1 = _mm256_castsi256_si128(res); // 16 8bit values dst[r] = _mm_blendv_epi8(a_mbase_x, res1, *(__m128i *)BaseMask[base_max_diff]); @@ -3093,191 +3646,29 @@ static void dr_prediction_z1_4xN_avx2(int N, uint8_t *dst, ptrdiff_t stride, int dx) { __m128i dstvec[16]; - dr_prediction_z1_4xN_internal_avx2(N, dstvec, above, upsample_above, dx); + dr_prediction_z1_HxW_internal_avx2(4, N, dstvec, above, upsample_above, dx); for (int i = 0; i < N; i++) { *(uint32_t *)(dst + stride * i) = _mm_cvtsi128_si32(dstvec[i]); } } -static AOM_FORCE_INLINE void dr_prediction_z1_8xN_internal_avx2( - int N, __m128i *dst, const uint8_t *above, int upsample_above, int dx) { - const int frac_bits = 6 - upsample_above; - const int max_base_x = ((8 + N) - 1) << upsample_above; - - int x; - // pre-filter above pixels - // store in temp buffers: - // above[x] * 32 + 16 - // above[x+1] - above[x] - // final pixels will be caluculated as: - // (above[x] * 32 + 16 + (above[x+1] - above[x]) * shift) >> 5 - __m256i a0, a1, a0_1, a1_1, a32, a16, diff, c3f; - __m128i a_mbase_x; - - a16 = _mm256_set1_epi32(16); - a_mbase_x = _mm_set1_epi8(above[max_base_x]); - c3f = _mm256_set1_epi32(0x3f); - - x = dx; - for (int r = 0; r < N; r++) { - __m256i b, res, res1, shift; - __m128i res128; - - int base = x >> frac_bits; - int base_max_diff = (max_base_x - base) >> upsample_above; - if (base_max_diff <= 0) { - for (int i = r; i < N; ++i) { - dst[i] = a_mbase_x; // save 16 values, 8 to be used furter - } - return; - } - if (base_max_diff > 8) base_max_diff = 8; - - a0 = _mm256_cvtepu8_epi32(_mm_loadu_si128((__m128i *)(above + base))); - a1 = _mm256_cvtepu8_epi32(_mm_loadu_si128((__m128i *)(above + base + 1))); - - if (upsample_above) { - a0 = _mm256_permutevar8x32_epi32( - a0, _mm256_set_epi32(7, 5, 3, 1, 6, 4, 2, 0)); - a1 = _mm256_castsi128_si256(_mm256_extracti128_si256(a0, 1)); - - a0_1 = - _mm256_cvtepu8_epi32(_mm_loadu_si128((__m128i *)(above + base + 8))); - a0_1 = _mm256_permutevar8x32_epi32( - a0_1, _mm256_set_epi32(7, 5, 3, 1, 6, 4, 2, 0)); - a1_1 = _mm256_castsi128_si256(_mm256_extracti128_si256(a0_1, 1)); - - a0 = _mm256_inserti128_si256(a0, _mm256_castsi256_si128(a0_1), 1); - a1 = _mm256_inserti128_si256(a1, _mm256_castsi256_si128(a1_1), 1); - - shift = _mm256_srli_epi32( - _mm256_and_si256( - _mm256_slli_epi32(_mm256_set1_epi32(x), upsample_above), c3f), - 1); - } else { - shift = _mm256_srli_epi32(_mm256_and_si256(_mm256_set1_epi32(x), c3f), 1); - } - - diff = _mm256_sub_epi32(a1, a0); // a[x+1] - a[x] - a32 = _mm256_slli_epi32(a0, 5); // a[x] * 32 - a32 = _mm256_add_epi32(a32, a16); // a[x] * 32 + 16 - - b = _mm256_mullo_epi32(diff, shift); - res = _mm256_add_epi32(a32, b); - res = _mm256_srli_epi32(res, 5); - - res1 = _mm256_packus_epi32( - res, _mm256_castsi128_si256( - _mm256_extracti128_si256(res, 1))); // goto 16 bit - - res128 = _mm_packus_epi16(_mm256_castsi256_si128(res1), - _mm256_castsi256_si128(res1)); // goto 8 bit - - res128 = - _mm_blendv_epi8(a_mbase_x, res128, *(__m128i *)BaseMask[base_max_diff]); - dst[r] = res128; - x += dx; - } -} - static void dr_prediction_z1_8xN_avx2(int N, uint8_t *dst, ptrdiff_t stride, const uint8_t *above, int upsample_above, int dx) { __m128i dstvec[32]; - dr_prediction_z1_8xN_internal_avx2(N, dstvec, above, upsample_above, dx); + dr_prediction_z1_HxW_internal_avx2(8, N, dstvec, above, upsample_above, dx); for (int i = 0; i < N; i++) { _mm_storel_epi64((__m128i *)(dst + stride * i), dstvec[i]); } } -static AOM_FORCE_INLINE void dr_prediction_z1_16xN_internal_avx2( - int N, __m128i *dstvec, const uint8_t *above, int upsample_above, int dx) { - int x; - // here upsample_above is 0 by design of av1_use_intra_edge_upsample - (void)upsample_above; - const int frac_bits = 6; - const int max_base_x = ((16 + N) - 1); - - // pre-filter above pixels - // store in temp buffers: - // above[x] * 32 + 16 - // above[x+1] - above[x] - // final pixels will be caluculated as: - // (above[x] * 32 + 16 + (above[x+1] - above[x]) * shift) >> 5 - __m256i a0, a0_1, a1, a1_1, diff, a32, a16, c3f; - __m128i a_mbase_x; - - a16 = _mm256_set1_epi32(16); - a_mbase_x = _mm_set1_epi8((uint8_t)above[max_base_x]); - c3f = _mm256_set1_epi32(0x3f); - - x = dx; - for (int r = 0; r < N; r++) { - __m256i b, res[2]; - __m128i res128[2]; - int base = x >> frac_bits; - int base_max_diff = (max_base_x - base); - if (base_max_diff <= 0) { - for (int i = r; i < N; ++i) { - dstvec[i] = a_mbase_x; // save 16 values - } - return; - } - __m256i shift = - _mm256_srli_epi32(_mm256_and_si256(_mm256_set1_epi32(x), c3f), 1); - - a0 = _mm256_cvtepu8_epi32(_mm_loadu_si128((__m128i *)(above + base))); - a1 = _mm256_cvtepu8_epi32(_mm_loadu_si128((__m128i *)(above + base + 1))); - - diff = _mm256_sub_epi32(a1, a0); // a[x+1] - a[x] - a32 = _mm256_slli_epi32(a0, 5); // a[x] * 32 - a32 = _mm256_add_epi32(a32, a16); // a[x] * 32 + 16 - b = _mm256_mullo_epi32(diff, shift); - - res[0] = _mm256_add_epi32(a32, b); - res[0] = _mm256_srli_epi32(res[0], 5); - res[0] = _mm256_packus_epi32( - res[0], _mm256_castsi128_si256(_mm256_extracti128_si256(res[0], 1))); - res128[0] = _mm_packus_epi16(_mm256_castsi256_si128(res[0]), - _mm256_castsi256_si128(res[0])); // goto 8 bit - - if (base_max_diff > 8) { - if (base_max_diff > 16) base_max_diff = 16; - a0_1 = - _mm256_cvtepu8_epi32(_mm_loadu_si128((__m128i *)(above + base + 8))); - a1_1 = - _mm256_cvtepu8_epi32(_mm_loadu_si128((__m128i *)(above + base + 9))); - - diff = _mm256_sub_epi32(a1_1, a0_1); // a[x+1] - a[x] - a32 = _mm256_slli_epi32(a0_1, 5); // a[x] * 32 - a32 = _mm256_add_epi32(a32, a16); // a[x] * 32 + 16 - b = _mm256_mullo_epi32(diff, shift); - - res[1] = _mm256_add_epi32(a32, b); - res[1] = _mm256_srli_epi32(res[1], 5); - res[1] = _mm256_packus_epi32( - res[1], _mm256_castsi128_si256(_mm256_extracti128_si256(res[1], 1))); - res128[1] = - _mm_packus_epi16(_mm256_castsi256_si128(res[1]), - _mm256_castsi256_si128(res[1])); // goto 8 bit - - } else { - res128[1] = a_mbase_x; - } - res128[0] = _mm_unpacklo_epi64(res128[0], res128[1]); // 16 8bit values - - dstvec[r] = _mm_blendv_epi8(a_mbase_x, res128[0], - *(__m128i *)BaseMask[base_max_diff]); - x += dx; - } -} static void dr_prediction_z1_16xN_avx2(int N, uint8_t *dst, ptrdiff_t stride, const uint8_t *above, int upsample_above, int dx) { __m128i dstvec[64]; - dr_prediction_z1_16xN_internal_avx2(N, dstvec, above, upsample_above, dx); + dr_prediction_z1_HxW_internal_avx2(16, N, dstvec, above, upsample_above, dx); for (int i = 0; i < N; i++) { _mm_storeu_si128((__m128i *)(dst + stride * i), dstvec[i]); } @@ -3285,7 +3676,6 @@ static void dr_prediction_z1_16xN_avx2(int N, uint8_t *dst, ptrdiff_t stride, static AOM_FORCE_INLINE void dr_prediction_z1_32xN_internal_avx2( int N, __m256i *dstvec, const uint8_t *above, int upsample_above, int dx) { - int x; // here upsample_above is 0 by design of av1_use_intra_edge_upsample (void)upsample_above; const int frac_bits = 6; @@ -3295,18 +3685,19 @@ static AOM_FORCE_INLINE void dr_prediction_z1_32xN_internal_avx2( // store in temp buffers: // above[x] * 32 + 16 // above[x+1] - above[x] - // final pixels will be caluculated as: + // final pixels will be calculated as: // (above[x] * 32 + 16 + (above[x+1] - above[x]) * shift) >> 5 - __m256i a0, a0_1, a1, a1_1, a32, a16; + __m256i a0, a1, a32, a16; __m256i a_mbase_x, diff, c3f; - a16 = _mm256_set1_epi32(16); + a16 = _mm256_set1_epi16(16); a_mbase_x = _mm256_set1_epi8(above[max_base_x]); - c3f = _mm256_set1_epi32(0x3f); + c3f = _mm256_set1_epi16(0x3f); - x = dx; + int x = dx; for (int r = 0; r < N; r++) { - __m256i b, res[2], res16[2]; + __m256i b, res, res16[2]; + __m128i a0_128, a1_128; int base = x >> frac_bits; int base_max_diff = (max_base_x - base); @@ -3318,54 +3709,28 @@ static AOM_FORCE_INLINE void dr_prediction_z1_32xN_internal_avx2( } if (base_max_diff > 32) base_max_diff = 32; __m256i shift = - _mm256_srli_epi32(_mm256_and_si256(_mm256_set1_epi32(x), c3f), 1); + _mm256_srli_epi16(_mm256_and_si256(_mm256_set1_epi16(x), c3f), 1); for (int j = 0, jj = 0; j < 32; j += 16, jj++) { int mdiff = base_max_diff - j; if (mdiff <= 0) { res16[jj] = a_mbase_x; } else { - a0 = _mm256_cvtepu8_epi32( - _mm_loadu_si128((__m128i *)(above + base + j))); - a1 = _mm256_cvtepu8_epi32( - _mm_loadu_si128((__m128i *)(above + base + 1 + j))); + a0_128 = _mm_loadu_si128((__m128i *)(above + base + j)); + a1_128 = _mm_loadu_si128((__m128i *)(above + base + j + 1)); + a0 = _mm256_cvtepu8_epi16(a0_128); + a1 = _mm256_cvtepu8_epi16(a1_128); - diff = _mm256_sub_epi32(a1, a0); // a[x+1] - a[x] - a32 = _mm256_slli_epi32(a0, 5); // a[x] * 32 - a32 = _mm256_add_epi32(a32, a16); // a[x] * 32 + 16 - b = _mm256_mullo_epi32(diff, shift); + diff = _mm256_sub_epi16(a1, a0); // a[x+1] - a[x] + a32 = _mm256_slli_epi16(a0, 5); // a[x] * 32 + a32 = _mm256_add_epi16(a32, a16); // a[x] * 32 + 16 + b = _mm256_mullo_epi16(diff, shift); - res[0] = _mm256_add_epi32(a32, b); - res[0] = _mm256_srli_epi32(res[0], 5); - res[0] = _mm256_packus_epi32( - res[0], - _mm256_castsi128_si256(_mm256_extracti128_si256(res[0], 1))); - - // goto 8 bit - res[0] = _mm256_packus_epi16(res[0], res[0]); - - if (mdiff > 8) { - a0_1 = _mm256_cvtepu8_epi32( - _mm_loadu_si128((__m128i *)(above + base + 8 + j))); - a1_1 = _mm256_cvtepu8_epi32( - _mm_loadu_si128((__m128i *)(above + base + 9 + j))); - - diff = _mm256_sub_epi32(a1_1, a0_1); // a[x+1] - a[x] - a32 = _mm256_slli_epi32(a0_1, 5); // a[x] * 32 - a32 = _mm256_add_epi32(a32, a16); // a[x] * 32 + 16 - b = _mm256_mullo_epi32(diff, shift); - - res[1] = _mm256_add_epi32(a32, b); - res[1] = _mm256_srli_epi32(res[1], 5); - res[1] = _mm256_packus_epi32( - res[1], - _mm256_castsi128_si256(_mm256_extracti128_si256(res[1], 1))); - res[1] = _mm256_packus_epi16(res[1], res[1]); - // goto 8 bit - } else { - res[1] = a_mbase_x; - } - res16[jj] = _mm256_unpacklo_epi64(res[0], res[1]); // 16 8bit values + res = _mm256_add_epi16(a32, b); + res = _mm256_srli_epi16(res, 5); + res16[jj] = _mm256_packus_epi16( + res, _mm256_castsi128_si256( + _mm256_extracti128_si256(res, 1))); // 16 8bit values } } res16[1] = @@ -3392,7 +3757,6 @@ static void dr_prediction_z1_32xN_avx2(int N, uint8_t *dst, ptrdiff_t stride, static void dr_prediction_z1_64xN_avx2(int N, uint8_t *dst, ptrdiff_t stride, const uint8_t *above, int upsample_above, int dx) { - int x; // here upsample_above is 0 by design of av1_use_intra_edge_upsample (void)upsample_above; const int frac_bits = 6; @@ -3402,22 +3766,20 @@ static void dr_prediction_z1_64xN_avx2(int N, uint8_t *dst, ptrdiff_t stride, // store in temp buffers: // above[x] * 32 + 16 // above[x+1] - above[x] - // final pixels will be caluculated as: + // final pixels will be calculated as: // (above[x] * 32 + 16 + (above[x+1] - above[x]) * shift) >> 5 - __m256i a0, a0_1, a1, a1_1, a32, a16; + __m256i a0, a1, a32, a16; __m256i a_mbase_x, diff, c3f; __m128i max_base_x128, base_inc128, mask128; - a16 = _mm256_set1_epi32(16); + a16 = _mm256_set1_epi16(16); a_mbase_x = _mm256_set1_epi8(above[max_base_x]); max_base_x128 = _mm_set1_epi8(max_base_x); - c3f = _mm256_set1_epi32(0x3f); + c3f = _mm256_set1_epi16(0x3f); - x = dx; + int x = dx; for (int r = 0; r < N; r++, dst += stride) { - __m256i b, res[2]; - __m128i res1; - + __m256i b, res; int base = x >> frac_bits; if (base >= max_base_x) { for (int i = r; i < N; ++i) { @@ -3429,9 +3791,9 @@ static void dr_prediction_z1_64xN_avx2(int N, uint8_t *dst, ptrdiff_t stride, } __m256i shift = - _mm256_srli_epi32(_mm256_and_si256(_mm256_set1_epi32(x), c3f), 1); + _mm256_srli_epi16(_mm256_and_si256(_mm256_set1_epi16(x), c3f), 1); - __m128i a0_128, a0_1_128, a1_128, a1_1_128; + __m128i a0_128, a1_128, res128; for (int j = 0; j < 64; j += 16) { int mdif = max_base_x - (base + j); if (mdif <= 0) { @@ -3440,46 +3802,19 @@ static void dr_prediction_z1_64xN_avx2(int N, uint8_t *dst, ptrdiff_t stride, } else { a0_128 = _mm_loadu_si128((__m128i *)(above + base + j)); a1_128 = _mm_loadu_si128((__m128i *)(above + base + 1 + j)); - a0 = _mm256_cvtepu8_epi32(a0_128); - a1 = _mm256_cvtepu8_epi32(a1_128); + a0 = _mm256_cvtepu8_epi16(a0_128); + a1 = _mm256_cvtepu8_epi16(a1_128); - diff = _mm256_sub_epi32(a1, a0); // a[x+1] - a[x] - a32 = _mm256_slli_epi32(a0, 5); // a[x] * 32 - a32 = _mm256_add_epi32(a32, a16); // a[x] * 32 + 16 - b = _mm256_mullo_epi32(diff, shift); + diff = _mm256_sub_epi16(a1, a0); // a[x+1] - a[x] + a32 = _mm256_slli_epi16(a0, 5); // a[x] * 32 + a32 = _mm256_add_epi16(a32, a16); // a[x] * 32 + 16 + b = _mm256_mullo_epi16(diff, shift); - res[0] = _mm256_add_epi32(a32, b); - res[0] = _mm256_srli_epi32(res[0], 5); - res[0] = _mm256_packus_epi32( - res[0], - _mm256_castsi128_si256(_mm256_extracti128_si256(res[0], 1))); - // goto 8 bit - res[0] = _mm256_packus_epi16(res[0], res[0]); - - if (mdif > 8) { - a0_1_128 = _mm_loadu_si128((__m128i *)(above + base + 8 + j)); - a1_1_128 = _mm_loadu_si128((__m128i *)(above + base + 9 + j)); - a0_1 = _mm256_cvtepu8_epi32(a0_1_128); - a1_1 = _mm256_cvtepu8_epi32(a1_1_128); - - diff = _mm256_sub_epi32(a1_1, a0_1); // a[x+1] - a[x] - a32 = _mm256_slli_epi32(a0_1, 5); // a[x] * 32 - a32 = _mm256_add_epi32(a32, a16); // a[x] * 32 + 16 - b = _mm256_mullo_epi32(diff, shift); - - res[1] = _mm256_add_epi32(a32, b); - res[1] = _mm256_srli_epi32(res[1], 5); - res[1] = _mm256_packus_epi32( - res[1], - _mm256_castsi128_si256(_mm256_extracti128_si256(res[1], 1))); - res[1] = _mm256_packus_epi16(res[1], res[1]); - - } else { - res[1] = a_mbase_x; - } - res1 = _mm_unpacklo_epi64( - _mm256_castsi256_si128(res[0]), - _mm256_castsi256_si128(res[1])); // 16 8bit values + res = _mm256_add_epi16(a32, b); + res = _mm256_srli_epi16(res, 5); + res = _mm256_packus_epi16( + res, _mm256_castsi128_si256( + _mm256_extracti128_si256(res, 1))); // 16 8bit values base_inc128 = _mm_setr_epi8( base + j, base + j + 1, base + j + 2, base + j + 3, base + j + 4, @@ -3489,9 +3824,9 @@ static void dr_prediction_z1_64xN_avx2(int N, uint8_t *dst, ptrdiff_t stride, mask128 = _mm_cmpgt_epi8(_mm_subs_epu8(max_base_x128, base_inc128), _mm_setzero_si128()); - res1 = - _mm_blendv_epi8(_mm256_castsi256_si128(a_mbase_x), res1, mask128); - _mm_storeu_si128((__m128i *)(dst + j), res1); + res128 = _mm_blendv_epi8(_mm256_castsi256_si128(a_mbase_x), + _mm256_castsi256_si128(res), mask128); + _mm_storeu_si128((__m128i *)(dst + j), res128); } } x += dx; @@ -3525,39 +3860,6 @@ void av1_dr_prediction_z1_avx2(uint8_t *dst, ptrdiff_t stride, int bw, int bh, return; } -static uint8_t LoadMaskx[8][16] = { - { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }, - { 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14 }, - { 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13 }, - { 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 }, - { 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 }, - { 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }, - { 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }, - { 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8 }, -}; - -static uint8_t EvenOddMaskx4[8][16] = { - { 0, 2, 4, 6, 1, 3, 5, 7, 0, 0, 0, 0, 0, 0, 0, 0 }, - { 0, 1, 3, 5, 7, 2, 4, 6, 8, 0, 0, 0, 0, 0, 0, 0 }, - { 0, 0, 2, 4, 6, 8, 3, 5, 7, 9, 0, 0, 0, 0, 0, 0 }, - { 0, 0, 0, 3, 5, 7, 9, 4, 6, 8, 10, 0, 0, 0, 0, 0 }, - { 0, 0, 0, 0, 4, 6, 8, 10, 5, 7, 9, 11, 0, 0, 0, 0 }, - { 0, 0, 0, 0, 0, 5, 7, 9, 11, 6, 8, 10, 12, 0, 0, 0 }, - { 0, 0, 0, 0, 0, 0, 6, 8, 10, 12, 7, 9, 11, 13, 0, 0 }, - { 0, 0, 0, 0, 0, 0, 0, 7, 9, 11, 13, 8, 10, 12, 14, 0 } -}; - -static uint8_t EvenOddMaskx[8][16] = { - { 0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 0, 0, 0, 0 }, - { 0, 1, 3, 5, 7, 9, 11, 13, 15, 2, 4, 6, 8, 0, 0, 0 }, - { 0, 0, 2, 4, 6, 8, 10, 12, 14, 3, 5, 7, 9, 0, 0, 0 }, - { 0, 0, 0, 3, 5, 7, 9, 11, 13, 15, 4, 6, 8, 10, 0 }, - { 0, 0, 0, 0, 4, 6, 8, 10, 12, 14, 5, 7, 9, 11, 0, 0 }, - { 0, 0, 0, 0, 0, 5, 7, 9, 11, 13, 15, 6, 8, 10, 12, 0 }, - { 0, 0, 0, 0, 0, 0, 6, 8, 10, 12, 14, 7, 9, 11, 13, 0 }, - { 0, 0, 0, 0, 0, 0, 0, 7, 9, 11, 13, 15, 8, 10, 12, 14 } -}; - static void dr_prediction_z2_Nx4_avx2(int N, uint8_t *dst, ptrdiff_t stride, const uint8_t *above, const uint8_t *left, int upsample_above, int upsample_left, @@ -3567,22 +3869,24 @@ static void dr_prediction_z2_Nx4_avx2(int N, uint8_t *dst, ptrdiff_t stride, const int frac_bits_x = 6 - upsample_above; const int frac_bits_y = 6 - upsample_left; - // a assert(dx > 0); + assert(dx > 0); // pre-filter above pixels // store in temp buffers: // above[x] * 32 + 16 // above[x+1] - above[x] - // final pixels will be caluculated as: + // final pixels will be calculated as: // (above[x] * 32 + 16 + (above[x+1] - above[x]) * shift) >> 5 - __m256i a0_x, a1_x, a32, a16, diff; - __m128i c3f, min_base_y128; + __m128i a0_x, a1_x, a32, a16, diff; + __m128i c3f, min_base_y128, c1234, dy128; - a16 = _mm256_set1_epi32(16); - c3f = _mm_set1_epi32(0x3f); - min_base_y128 = _mm_set1_epi32(min_base_y); + a16 = _mm_set1_epi16(16); + c3f = _mm_set1_epi16(0x3f); + min_base_y128 = _mm_set1_epi16(min_base_y); + c1234 = _mm_setr_epi16(0, 1, 2, 3, 4, 0, 0, 0); + dy128 = _mm_set1_epi16(dy); for (int r = 0; r < N; r++) { - __m256i b, res, shift; + __m128i b, res, shift, r6, ydx; __m128i resx, resy, resxy; __m128i a0_x128, a1_x128; int y = r + 1; @@ -3600,82 +3904,73 @@ static void dr_prediction_z2_Nx4_avx2(int N, uint8_t *dst, ptrdiff_t stride, } if (base_shift > 3) { - a0_x = _mm256_setzero_si256(); - a1_x = _mm256_setzero_si256(); - shift = _mm256_setzero_si256(); + a0_x = _mm_setzero_si128(); + a1_x = _mm_setzero_si128(); + shift = _mm_setzero_si128(); } else { a0_x128 = _mm_loadu_si128((__m128i *)(above + base_x + base_shift)); + ydx = _mm_set1_epi16(y * dx); + r6 = _mm_slli_epi16(c1234, 6); + if (upsample_above) { a0_x128 = - _mm_shuffle_epi8(a0_x128, *(__m128i *)EvenOddMaskx4[base_shift]); - a1_x128 = _mm_srli_si128(a0_x128, 4); + _mm_shuffle_epi8(a0_x128, *(__m128i *)EvenOddMaskx[base_shift]); + a1_x128 = _mm_srli_si128(a0_x128, 8); - shift = _mm256_castsi128_si256(_mm_srli_epi32( + shift = _mm_srli_epi16( _mm_and_si128( - _mm_slli_epi32( - _mm_setr_epi32(-y * dx, (1 << 6) - y * dx, - (2 << 6) - y * dx, (3 << 6) - y * dx), - upsample_above), - c3f), - 1)); + _mm_slli_epi16(_mm_sub_epi16(r6, ydx), upsample_above), c3f), + 1); } else { a0_x128 = _mm_shuffle_epi8(a0_x128, *(__m128i *)LoadMaskx[base_shift]); a1_x128 = _mm_srli_si128(a0_x128, 1); - shift = _mm256_castsi128_si256(_mm_srli_epi32( - _mm_and_si128(_mm_setr_epi32(-y * dx, (1 << 6) - y * dx, - (2 << 6) - y * dx, (3 << 6) - y * dx), - c3f), - 1)); + shift = _mm_srli_epi16(_mm_and_si128(_mm_sub_epi16(r6, ydx), c3f), 1); } - a0_x = _mm256_cvtepu8_epi32(a0_x128); - a1_x = _mm256_cvtepu8_epi32(a1_x128); + a0_x = _mm_cvtepu8_epi16(a0_x128); + a1_x = _mm_cvtepu8_epi16(a1_x128); } // y calc __m128i a0_y, a1_y, shifty; if (base_x < min_base_x) { - DECLARE_ALIGNED(32, int, base_y_c[4]); - __m128i r6, c1234, dy128, y_c128, base_y_c128, mask128; - r6 = _mm_set1_epi32(r << 6); - dy128 = _mm_set1_epi32(dy); - c1234 = _mm_setr_epi32(1, 2, 3, 4); - y_c128 = _mm_sub_epi32(r6, _mm_mullo_epi32(c1234, dy128)); - base_y_c128 = _mm_srai_epi32(y_c128, frac_bits_y); - mask128 = _mm_cmpgt_epi32(min_base_y128, base_y_c128); + DECLARE_ALIGNED(32, int16_t, base_y_c[8]); + __m128i y_c128, base_y_c128, mask128, c1234_; + c1234_ = _mm_srli_si128(c1234, 2); + r6 = _mm_set1_epi16(r << 6); + y_c128 = _mm_sub_epi16(r6, _mm_mullo_epi16(c1234_, dy128)); + base_y_c128 = _mm_srai_epi16(y_c128, frac_bits_y); + mask128 = _mm_cmpgt_epi16(min_base_y128, base_y_c128); base_y_c128 = _mm_andnot_si128(mask128, base_y_c128); _mm_store_si128((__m128i *)base_y_c, base_y_c128); - a0_y = _mm_setr_epi32(left[base_y_c[0]], left[base_y_c[1]], - left[base_y_c[2]], left[base_y_c[3]]); - a1_y = _mm_setr_epi32(left[base_y_c[0] + 1], left[base_y_c[1] + 1], - left[base_y_c[2] + 1], left[base_y_c[3] + 1]); + a0_y = _mm_setr_epi16(left[base_y_c[0]], left[base_y_c[1]], + left[base_y_c[2]], left[base_y_c[3]], 0, 0, 0, 0); + base_y_c128 = _mm_add_epi16(base_y_c128, _mm_srli_epi16(a16, 4)); + _mm_store_si128((__m128i *)base_y_c, base_y_c128); + a1_y = _mm_setr_epi16(left[base_y_c[0]], left[base_y_c[1]], + left[base_y_c[2]], left[base_y_c[3]], 0, 0, 0, 0); if (upsample_left) { - shifty = _mm_srli_epi32( - _mm_and_si128(_mm_slli_epi32(y_c128, upsample_left), c3f), 1); + shifty = _mm_srli_epi16( + _mm_and_si128(_mm_slli_epi16(y_c128, upsample_left), c3f), 1); } else { - shifty = _mm_srli_epi32(_mm_and_si128(y_c128, c3f), 1); + shifty = _mm_srli_epi16(_mm_and_si128(y_c128, c3f), 1); } - a0_x = _mm256_inserti128_si256(a0_x, a0_y, 1); - a1_x = _mm256_inserti128_si256(a1_x, a1_y, 1); - shift = _mm256_inserti128_si256(shift, shifty, 1); + a0_x = _mm_unpacklo_epi64(a0_x, a0_y); + a1_x = _mm_unpacklo_epi64(a1_x, a1_y); + shift = _mm_unpacklo_epi64(shift, shifty); } - diff = _mm256_sub_epi32(a1_x, a0_x); // a[x+1] - a[x] - a32 = _mm256_slli_epi32(a0_x, 5); // a[x] * 32 - a32 = _mm256_add_epi32(a32, a16); // a[x] * 32 + 16 + diff = _mm_sub_epi16(a1_x, a0_x); // a[x+1] - a[x] + a32 = _mm_slli_epi16(a0_x, 5); // a[x] * 32 + a32 = _mm_add_epi16(a32, a16); // a[x] * 32 + 16 - b = _mm256_mullo_epi32(diff, shift); - res = _mm256_add_epi32(a32, b); - res = _mm256_srli_epi32(res, 5); + b = _mm_mullo_epi16(diff, shift); + res = _mm_add_epi16(a32, b); + res = _mm_srli_epi16(res, 5); - resx = _mm256_castsi256_si128(res); - resx = _mm_packus_epi32(resx, resx); - resx = _mm_packus_epi16(resx, resx); - - resy = _mm256_extracti128_si256(res, 1); - resy = _mm_packus_epi32(resy, resy); - resy = _mm_packus_epi16(resy, resy); + resx = _mm_packus_epi16(res, res); + resy = _mm_srli_si128(resx, 4); resxy = _mm_blendv_epi8(resx, resy, *(__m128i *)BaseMask[base_min_diff]); *(uint32_t *)(dst) = _mm_cvtsi128_si32(resxy); @@ -3696,19 +3991,22 @@ static void dr_prediction_z2_Nx8_avx2(int N, uint8_t *dst, ptrdiff_t stride, // store in temp buffers: // above[x] * 32 + 16 // above[x+1] - above[x] - // final pixels will be caluculated as: + // final pixels will be calculated as: // (above[x] * 32 + 16 + (above[x+1] - above[x]) * shift) >> 5 __m256i diff, a32, a16; __m256i a0_x, a1_x; __m128i a0_x128, a1_x128, min_base_y128, c3f; + __m128i c1234, dy128; a16 = _mm256_set1_epi16(16); c3f = _mm_set1_epi16(0x3f); min_base_y128 = _mm_set1_epi16(min_base_y); + dy128 = _mm_set1_epi16(dy); + c1234 = _mm_setr_epi16(1, 2, 3, 4, 5, 6, 7, 8); for (int r = 0; r < N; r++) { __m256i b, res, shift; - __m128i resx, resy, resxy; + __m128i resx, resy, resxy, r6, ydx; int y = r + 1; int base_x = (-y * dx) >> frac_bits_x; @@ -3730,34 +4028,24 @@ static void dr_prediction_z2_Nx8_avx2(int N, uint8_t *dst, ptrdiff_t stride, shift = _mm256_setzero_si256(); } else { a0_x128 = _mm_loadu_si128((__m128i *)(above + base_x + base_shift)); - a1_x128 = _mm_loadu_si128((__m128i *)(above + base_x + 1 + base_shift)); + ydx = _mm_set1_epi16(y * dx); + r6 = _mm_slli_epi16(_mm_srli_si128(c1234, 2), 6); if (upsample_above) { a0_x128 = _mm_shuffle_epi8(a0_x128, *(__m128i *)EvenOddMaskx[base_shift]); - a1_x128 = - _mm_shuffle_epi8(a1_x128, *(__m128i *)EvenOddMaskx[base_shift]); + a1_x128 = _mm_srli_si128(a0_x128, 8); shift = _mm256_castsi128_si256(_mm_srli_epi16( _mm_and_si128( - _mm_slli_epi16( - _mm_setr_epi16(-y * dx, (1 << 6) - y * dx, - (2 << 6) - y * dx, (3 << 6) - y * dx, - (4 << 6) - y * dx, (5 << 6) - y * dx, - (6 << 6) - y * dx, (7 << 6) - y * dx), - upsample_above), - c3f), + _mm_slli_epi16(_mm_sub_epi16(r6, ydx), upsample_above), c3f), 1)); } else { + a1_x128 = _mm_srli_si128(a0_x128, 1); a0_x128 = _mm_shuffle_epi8(a0_x128, *(__m128i *)LoadMaskx[base_shift]); a1_x128 = _mm_shuffle_epi8(a1_x128, *(__m128i *)LoadMaskx[base_shift]); - shift = _mm256_castsi128_si256(_mm_srli_epi16( - _mm_and_si128(_mm_setr_epi16(-y * dx, (1 << 6) - y * dx, - (2 << 6) - y * dx, (3 << 6) - y * dx, - (4 << 6) - y * dx, (5 << 6) - y * dx, - (6 << 6) - y * dx, (7 << 6) - y * dx), - c3f), - 1)); + shift = _mm256_castsi128_si256( + _mm_srli_epi16(_mm_and_si128(_mm_sub_epi16(r6, ydx), c3f), 1)); } a0_x = _mm256_castsi128_si256(_mm_cvtepu8_epi16(a0_x128)); a1_x = _mm256_castsi128_si256(_mm_cvtepu8_epi16(a1_x128)); @@ -3767,10 +4055,8 @@ static void dr_prediction_z2_Nx8_avx2(int N, uint8_t *dst, ptrdiff_t stride, __m128i a0_y, a1_y, shifty; if (base_x < min_base_x) { DECLARE_ALIGNED(32, int16_t, base_y_c[16]); - __m128i r6, c1234, dy128, y_c128, base_y_c128, mask128; + __m128i y_c128, base_y_c128, mask128; r6 = _mm_set1_epi16(r << 6); - dy128 = _mm_set1_epi16(dy); - c1234 = _mm_setr_epi16(1, 2, 3, 4, 5, 6, 7, 8); y_c128 = _mm_sub_epi16(r6, _mm_mullo_epi16(c1234, dy128)); base_y_c128 = _mm_srai_epi16(y_c128, frac_bits_y); mask128 = _mm_cmpgt_epi16(min_base_y128, base_y_c128); @@ -3781,10 +4067,14 @@ static void dr_prediction_z2_Nx8_avx2(int N, uint8_t *dst, ptrdiff_t stride, left[base_y_c[2]], left[base_y_c[3]], left[base_y_c[4]], left[base_y_c[5]], left[base_y_c[6]], left[base_y_c[7]]); - a1_y = _mm_setr_epi16(left[base_y_c[0] + 1], left[base_y_c[1] + 1], - left[base_y_c[2] + 1], left[base_y_c[3] + 1], - left[base_y_c[4] + 1], left[base_y_c[5] + 1], - left[base_y_c[6] + 1], left[base_y_c[7] + 1]); + base_y_c128 = _mm_add_epi16( + base_y_c128, _mm_srli_epi16(_mm256_castsi256_si128(a16), 4)); + _mm_store_si128((__m128i *)base_y_c, base_y_c128); + + a1_y = _mm_setr_epi16(left[base_y_c[0]], left[base_y_c[1]], + left[base_y_c[2]], left[base_y_c[3]], + left[base_y_c[4]], left[base_y_c[5]], + left[base_y_c[6]], left[base_y_c[7]]); if (upsample_left) { shifty = _mm_srli_epi16( @@ -3830,28 +4120,30 @@ static void dr_prediction_z2_HxW_avx2(int H, int W, uint8_t *dst, const int frac_bits_x = 6; const int frac_bits_y = 6; - // pre-filter above pixels - // store in temp buffers: - // above[x] * 32 + 16 - // above[x+1] - above[x] - // final pixels will be caluculated as: - // (above[x] * 32 + 16 + (above[x+1] - above[x]) * shift) >> 5 - __m256i a0_x, a1_x, a0_y, a1_y, a32, a16; - __m256i diff, min_base_y256, c3f, shifty; - __m128i a0_x128, a1_x128, a0_1_x128, a1_1_x128, a0_1_x, a1_1_x, shiftx; + __m256i a0_x, a1_x, a0_y, a1_y, a32, a16, c1234, c0123; + __m256i diff, min_base_y256, c3f, shifty, dy256, c1; + __m128i a0_x128, a1_x128; + DECLARE_ALIGNED(32, int16_t, base_y_c[16]); a16 = _mm256_set1_epi16(16); + c1 = _mm256_srli_epi16(a16, 4); min_base_y256 = _mm256_set1_epi16(min_base_y); c3f = _mm256_set1_epi16(0x3f); + dy256 = _mm256_set1_epi16(dy); + c0123 = + _mm256_setr_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + c1234 = _mm256_add_epi16(c0123, c1); for (int r = 0; r < H; r++) { - __m256i b, res, shift; + __m256i b, res, shift, j256, r6, ydx; __m128i resx, resy; __m128i resxy; - for (int j = 0; j < W; j += 16) { - int y = r + 1; - int base_x = (-y * dx) >> frac_bits_x; + int y = r + 1; + ydx = _mm256_set1_epi16(y * dx); + int base_x = (-y * dx) >> frac_bits_x; + for (int j = 0; j < W; j += 16) { + j256 = _mm256_set1_epi16(j); int base_shift = 0; if ((base_x + j) < (min_base_x - 1)) { base_shift = (min_base_x - (base_x + j) - 1); @@ -3862,11 +4154,8 @@ static void dr_prediction_z2_HxW_avx2(int H, int W, uint8_t *dst, } else { if (base_min_diff < 0) base_min_diff = 0; } - if (base_shift > 7) { - a0_x = _mm256_setzero_si256(); - a1_x = _mm256_setzero_si256(); - shift = _mm256_setzero_si256(); - } else { + + if (base_shift < 16) { a0_x128 = _mm_loadu_si128((__m128i *)(above + base_x + base_shift + j)); a1_x128 = _mm_loadu_si128((__m128i *)(above + base_x + base_shift + 1 + j)); @@ -3876,67 +4165,28 @@ static void dr_prediction_z2_HxW_avx2(int H, int W, uint8_t *dst, a0_x = _mm256_cvtepu8_epi16(a0_x128); a1_x = _mm256_cvtepu8_epi16(a1_x128); - shift = _mm256_castsi128_si256(_mm_srli_epi16( - _mm_and_si128(_mm_setr_epi16( - ((0 + j) << 6) - y * dx, ((1 + j) << 6) - y * dx, - ((2 + j) << 6) - y * dx, ((3 + j) << 6) - y * dx, - ((4 + j) << 6) - y * dx, ((5 + j) << 6) - y * dx, - ((6 + j) << 6) - y * dx, ((7 + j) << 6) - y * dx), - _mm256_castsi256_si128(c3f)), - 1)); + r6 = _mm256_slli_epi16(_mm256_add_epi16(c0123, j256), 6); + shift = _mm256_srli_epi16( + _mm256_and_si256(_mm256_sub_epi16(r6, ydx), c3f), 1); + + diff = _mm256_sub_epi16(a1_x, a0_x); // a[x+1] - a[x] + a32 = _mm256_slli_epi16(a0_x, 5); // a[x] * 32 + a32 = _mm256_add_epi16(a32, a16); // a[x] * 32 + 16 + + b = _mm256_mullo_epi16(diff, shift); + res = _mm256_add_epi16(a32, b); + res = _mm256_srli_epi16(res, 5); // 16 16-bit values + resx = _mm256_castsi256_si128(_mm256_packus_epi16( + res, _mm256_castsi128_si256(_mm256_extracti128_si256(res, 1)))); + } else { + resx = _mm_setzero_si128(); } - base_shift = 0; - if ((base_x + j + 8) < (min_base_x - 1)) { - base_shift = (min_base_x - (base_x + j + 8) - 1); - } - if (base_shift <= 7) { - a0_1_x128 = - _mm_loadu_si128((__m128i *)(above + base_x + base_shift + 8 + j)); - a1_1_x128 = - _mm_loadu_si128((__m128i *)(above + base_x + base_shift + 9 + j)); - a0_1_x128 = - _mm_shuffle_epi8(a0_1_x128, *(__m128i *)LoadMaskx[base_shift]); - a1_1_x128 = - _mm_shuffle_epi8(a1_1_x128, *(__m128i *)LoadMaskx[base_shift]); - - a0_1_x = _mm_cvtepu8_epi16(a0_1_x128); - a1_1_x = _mm_cvtepu8_epi16(a1_1_x128); - - shiftx = _mm_srli_epi16( - _mm_and_si128( - _mm_setr_epi16( - ((8 + j) << 6) - y * dx, ((9 + j) << 6) - y * dx, - ((10 + j) << 6) - y * dx, ((11 + j) << 6) - y * dx, - ((12 + j) << 6) - y * dx, ((13 + j) << 6) - y * dx, - ((14 + j) << 6) - y * dx, ((15 + j) << 6) - y * dx), - _mm256_castsi256_si128(c3f)), - 1); - - a0_x = _mm256_inserti128_si256(a0_x, a0_1_x, 1); - a1_x = _mm256_inserti128_si256(a1_x, a1_1_x, 1); - shift = _mm256_inserti128_si256(shift, shiftx, 1); - } - - diff = _mm256_sub_epi16(a1_x, a0_x); // a[x+1] - a[x] - a32 = _mm256_slli_epi16(a0_x, 5); // a[x] * 32 - a32 = _mm256_add_epi16(a32, a16); // a[x] * 32 + 16 - - b = _mm256_mullo_epi16(diff, shift); - res = _mm256_add_epi16(a32, b); - res = _mm256_srli_epi16(res, 5); // 16 16-bit values - resx = _mm256_castsi256_si128(_mm256_packus_epi16( - res, _mm256_castsi128_si256(_mm256_extracti128_si256(res, 1)))); - // y calc - if ((base_x < min_base_x)) { - DECLARE_ALIGNED(32, int16_t, base_y_c[16]); - __m256i r6, c256, dy256, y_c256, base_y_c256, mask256, mul16; + if (base_x < min_base_x) { + __m256i c256, y_c256, base_y_c256, mask256, mul16; r6 = _mm256_set1_epi16(r << 6); - dy256 = _mm256_set1_epi16(dy); - c256 = _mm256_setr_epi16(1 + j, 2 + j, 3 + j, 4 + j, 5 + j, 6 + j, - 7 + j, 8 + j, 9 + j, 10 + j, 11 + j, 12 + j, - 13 + j, 14 + j, 15 + j, 16 + j); + c256 = _mm256_add_epi16(j256, c1234); mul16 = _mm256_min_epu16(_mm256_mullo_epi16(c256, dy256), _mm256_srli_epi16(min_base_y256, 1)); y_c256 = _mm256_sub_epi16(r6, mul16); @@ -3953,14 +4203,16 @@ static void dr_prediction_z2_HxW_avx2(int H, int W, uint8_t *dst, left[base_y_c[9]], left[base_y_c[10]], left[base_y_c[11]], left[base_y_c[12]], left[base_y_c[13]], left[base_y_c[14]], left[base_y_c[15]]); + base_y_c256 = _mm256_add_epi16(base_y_c256, c1); + _mm256_store_si256((__m256i *)base_y_c, base_y_c256); + a1_y = _mm256_setr_epi16( - left[base_y_c[0] + 1], left[base_y_c[1] + 1], left[base_y_c[2] + 1], - left[base_y_c[3] + 1], left[base_y_c[4] + 1], left[base_y_c[5] + 1], - left[base_y_c[6] + 1], left[base_y_c[7] + 1], left[base_y_c[8] + 1], - left[base_y_c[9] + 1], left[base_y_c[10] + 1], - left[base_y_c[11] + 1], left[base_y_c[12] + 1], - left[base_y_c[13] + 1], left[base_y_c[14] + 1], - left[base_y_c[15] + 1]); + left[base_y_c[0]], left[base_y_c[1]], left[base_y_c[2]], + left[base_y_c[3]], left[base_y_c[4]], left[base_y_c[5]], + left[base_y_c[6]], left[base_y_c[7]], left[base_y_c[8]], + left[base_y_c[9]], left[base_y_c[10]], left[base_y_c[11]], + left[base_y_c[12]], left[base_y_c[13]], left[base_y_c[14]], + left[base_y_c[15]]); shifty = _mm256_srli_epi16(_mm256_and_si256(y_c256, c3f), 1); @@ -3973,7 +4225,6 @@ static void dr_prediction_z2_HxW_avx2(int H, int W, uint8_t *dst, res = _mm256_srli_epi16(res, 5); // 16 16-bit values resy = _mm256_castsi256_si128(_mm256_packus_epi16( res, _mm256_castsi128_si256(_mm256_extracti128_si256(res, 1)))); - } else { resy = _mm_setzero_si128(); } @@ -4227,38 +4478,25 @@ static INLINE void transpose16x16_sse2(__m128i *x, __m128i *d) { d[15] = _mm_unpackhi_epi64(w7, w15); } -static void transpose_TX_8X8(const uint8_t *src, ptrdiff_t pitchSrc, - uint8_t *dst, ptrdiff_t pitchDst) { - __m128i r0, r1, r2, r3, r4, r5, r6, r7; - __m128i d0d1, d2d3, d4d5, d6d7; - r0 = _mm_loadl_epi64((__m128i *)(src + 0 * pitchSrc)); - r1 = _mm_loadl_epi64((__m128i *)(src + 1 * pitchSrc)); - r2 = _mm_loadl_epi64((__m128i *)(src + 2 * pitchSrc)); - r3 = _mm_loadl_epi64((__m128i *)(src + 3 * pitchSrc)); - r4 = _mm_loadl_epi64((__m128i *)(src + 4 * pitchSrc)); - r5 = _mm_loadl_epi64((__m128i *)(src + 5 * pitchSrc)); - r6 = _mm_loadl_epi64((__m128i *)(src + 6 * pitchSrc)); - r7 = _mm_loadl_epi64((__m128i *)(src + 7 * pitchSrc)); - - transpose8x8_sse2(&r0, &r1, &r2, &r3, &r4, &r5, &r6, &r7, &d0d1, &d2d3, &d4d5, - &d6d7); - - _mm_storel_epi64((__m128i *)(dst + 0 * pitchDst), d0d1); - _mm_storel_epi64((__m128i *)(dst + 1 * pitchDst), _mm_srli_si128(d0d1, 8)); - _mm_storel_epi64((__m128i *)(dst + 2 * pitchDst), d2d3); - _mm_storel_epi64((__m128i *)(dst + 3 * pitchDst), _mm_srli_si128(d2d3, 8)); - _mm_storel_epi64((__m128i *)(dst + 4 * pitchDst), d4d5); - _mm_storel_epi64((__m128i *)(dst + 5 * pitchDst), _mm_srli_si128(d4d5, 8)); - _mm_storel_epi64((__m128i *)(dst + 6 * pitchDst), d6d7); - _mm_storel_epi64((__m128i *)(dst + 7 * pitchDst), _mm_srli_si128(d6d7, 8)); +static void transpose_TX_16X16(const uint8_t *src, ptrdiff_t pitchSrc, + uint8_t *dst, ptrdiff_t pitchDst) { + __m128i r[16]; + __m128i d[16]; + for (int j = 0; j < 16; j++) { + r[j] = _mm_loadu_si128((__m128i *)(src + j * pitchSrc)); + } + transpose16x16_sse2(r, d); + for (int j = 0; j < 16; j++) { + _mm_storeu_si128((__m128i *)(dst + j * pitchDst), d[j]); + } } static void transpose(const uint8_t *src, ptrdiff_t pitchSrc, uint8_t *dst, ptrdiff_t pitchDst, int width, int height) { - for (int j = 0; j < height; j += 8) - for (int i = 0; i < width; i += 8) - transpose_TX_8X8(src + i * pitchSrc + j, pitchSrc, dst + j * pitchDst + i, - pitchDst); + for (int j = 0; j < height; j += 16) + for (int i = 0; i < width; i += 16) + transpose_TX_16X16(src + i * pitchSrc + j, pitchSrc, + dst + j * pitchDst + i, pitchDst); } static void dr_prediction_z3_4x4_avx2(uint8_t *dst, ptrdiff_t stride, @@ -4266,7 +4504,7 @@ static void dr_prediction_z3_4x4_avx2(uint8_t *dst, ptrdiff_t stride, int dy) { __m128i dstvec[4], d[4]; - dr_prediction_z1_4xN_internal_avx2(4, dstvec, left, upsample_left, dy); + dr_prediction_z1_HxW_internal_avx2(4, 4, dstvec, left, upsample_left, dy); transpose4x8_8x4_low_sse2(&dstvec[0], &dstvec[1], &dstvec[2], &dstvec[3], &d[0], &d[1], &d[2], &d[3]); @@ -4282,7 +4520,7 @@ static void dr_prediction_z3_8x8_avx2(uint8_t *dst, ptrdiff_t stride, int dy) { __m128i dstvec[8], d[8]; - dr_prediction_z1_8xN_internal_avx2(8, dstvec, left, upsample_left, dy); + dr_prediction_z1_HxW_internal_avx2(8, 8, dstvec, left, upsample_left, dy); transpose8x8_sse2(&dstvec[0], &dstvec[1], &dstvec[2], &dstvec[3], &dstvec[4], &dstvec[5], &dstvec[6], &dstvec[7], &d[0], &d[1], &d[2], &d[3]); @@ -4302,7 +4540,7 @@ static void dr_prediction_z3_4x8_avx2(uint8_t *dst, ptrdiff_t stride, int dy) { __m128i dstvec[4], d[8]; - dr_prediction_z1_8xN_internal_avx2(4, dstvec, left, upsample_left, dy); + dr_prediction_z1_HxW_internal_avx2(8, 4, dstvec, left, upsample_left, dy); transpose4x8_8x4_sse2(&dstvec[0], &dstvec[1], &dstvec[2], &dstvec[3], &d[0], &d[1], &d[2], &d[3], &d[4], &d[5], &d[6], &d[7]); for (int i = 0; i < 8; i++) { @@ -4315,7 +4553,7 @@ static void dr_prediction_z3_8x4_avx2(uint8_t *dst, ptrdiff_t stride, int dy) { __m128i dstvec[8], d[4]; - dr_prediction_z1_4xN_internal_avx2(8, dstvec, left, upsample_left, dy); + dr_prediction_z1_HxW_internal_avx2(4, 8, dstvec, left, upsample_left, dy); transpose8x8_low_sse2(&dstvec[0], &dstvec[1], &dstvec[2], &dstvec[3], &dstvec[4], &dstvec[5], &dstvec[6], &dstvec[7], &d[0], &d[1], &d[2], &d[3]); @@ -4330,7 +4568,7 @@ static void dr_prediction_z3_8x16_avx2(uint8_t *dst, ptrdiff_t stride, int dy) { __m128i dstvec[8], d[8]; - dr_prediction_z1_16xN_internal_avx2(8, dstvec, left, upsample_left, dy); + dr_prediction_z1_HxW_internal_avx2(16, 8, dstvec, left, upsample_left, dy); transpose8x16_16x8_sse2(dstvec, dstvec + 1, dstvec + 2, dstvec + 3, dstvec + 4, dstvec + 5, dstvec + 6, dstvec + 7, d, d + 1, d + 2, d + 3, d + 4, d + 5, d + 6, d + 7); @@ -4346,7 +4584,7 @@ static void dr_prediction_z3_16x8_avx2(uint8_t *dst, ptrdiff_t stride, int dy) { __m128i dstvec[16], d[16]; - dr_prediction_z1_8xN_internal_avx2(16, dstvec, left, upsample_left, dy); + dr_prediction_z1_HxW_internal_avx2(8, 16, dstvec, left, upsample_left, dy); transpose16x8_8x16_sse2( &dstvec[0], &dstvec[1], &dstvec[2], &dstvec[3], &dstvec[4], &dstvec[5], &dstvec[6], &dstvec[7], &dstvec[8], &dstvec[9], &dstvec[10], &dstvec[11], @@ -4363,7 +4601,7 @@ static void dr_prediction_z3_4x16_avx2(uint8_t *dst, ptrdiff_t stride, int dy) { __m128i dstvec[4], d[16]; - dr_prediction_z1_16xN_internal_avx2(4, dstvec, left, upsample_left, dy); + dr_prediction_z1_HxW_internal_avx2(16, 4, dstvec, left, upsample_left, dy); transpose4x16_sse2(dstvec, d); for (int i = 0; i < 16; i++) { *(uint32_t *)(dst + stride * i) = _mm_cvtsi128_si32(d[i]); @@ -4375,7 +4613,7 @@ static void dr_prediction_z3_16x4_avx2(uint8_t *dst, ptrdiff_t stride, int dy) { __m128i dstvec[16], d[8]; - dr_prediction_z1_4xN_internal_avx2(16, dstvec, left, upsample_left, dy); + dr_prediction_z1_HxW_internal_avx2(4, 16, dstvec, left, upsample_left, dy); for (int i = 4; i < 8; i++) { d[i] = _mm_setzero_si128(); } @@ -4416,7 +4654,7 @@ static void dr_prediction_z3_32x8_avx2(uint8_t *dst, ptrdiff_t stride, int dy) { __m128i dstvec[32], d[16]; - dr_prediction_z1_8xN_internal_avx2(32, dstvec, left, upsample_left, dy); + dr_prediction_z1_HxW_internal_avx2(8, 32, dstvec, left, upsample_left, dy); transpose16x8_8x16_sse2( &dstvec[0], &dstvec[1], &dstvec[2], &dstvec[3], &dstvec[4], &dstvec[5], @@ -4442,7 +4680,7 @@ static void dr_prediction_z3_16x16_avx2(uint8_t *dst, ptrdiff_t stride, int dy) { __m128i dstvec[16], d[16]; - dr_prediction_z1_16xN_internal_avx2(16, dstvec, left, upsample_left, dy); + dr_prediction_z1_HxW_internal_avx2(16, 16, dstvec, left, upsample_left, dy); transpose16x16_sse2(dstvec, d); for (int i = 0; i < 16; i++) { @@ -4501,7 +4739,7 @@ static void dr_prediction_z3_32x16_avx2(uint8_t *dst, ptrdiff_t stride, int dy) { __m128i dstvec[32], d[16]; - dr_prediction_z1_16xN_internal_avx2(32, dstvec, left, upsample_left, dy); + dr_prediction_z1_HxW_internal_avx2(16, 32, dstvec, left, upsample_left, dy); for (int i = 0; i < 32; i += 16) { transpose16x16_sse2((dstvec + i), d); for (int j = 0; j < 16; j++) { @@ -4540,7 +4778,7 @@ static void dr_prediction_z3_64x16_avx2(uint8_t *dst, ptrdiff_t stride, int dy) { __m128i dstvec[64], d[16]; - dr_prediction_z1_16xN_internal_avx2(64, dstvec, left, upsample_left, dy); + dr_prediction_z1_HxW_internal_avx2(16, 64, dstvec, left, upsample_left, dy); for (int i = 0; i < 64; i += 16) { transpose16x16_sse2((dstvec + i), d); for (int j = 0; j < 16; j++) { diff --git a/third_party/aom/aom_dsp/x86/quantize_x86.h b/third_party/aom/aom_dsp/x86/quantize_x86.h index b2de01bf27..5b040a278a 100644 --- a/third_party/aom/aom_dsp/x86/quantize_x86.h +++ b/third_party/aom/aom_dsp/x86/quantize_x86.h @@ -143,3 +143,60 @@ static INLINE void store_coefficients(__m128i coeff_vals, _mm_store_si128((__m128i *)(coeff_ptr), coeff_vals_1); _mm_store_si128((__m128i *)(coeff_ptr + 4), coeff_vals_2); } + +static INLINE void update_mask1(__m128i *cmp_mask0, __m128i *cmp_mask1, + const int16_t *iscan_ptr, int *is_found, + __m128i *mask) { + __m128i all_zero; + __m128i temp_mask = _mm_setzero_si128(); + all_zero = _mm_or_si128(*cmp_mask0, *cmp_mask1); + if (_mm_movemask_epi8(all_zero)) { + __m128i iscan0 = _mm_load_si128((const __m128i *)(iscan_ptr)); + __m128i mask0 = _mm_and_si128(*cmp_mask0, iscan0); + __m128i iscan1 = _mm_load_si128((const __m128i *)(iscan_ptr + 8)); + __m128i mask1 = _mm_and_si128(*cmp_mask1, iscan1); + temp_mask = _mm_max_epi16(mask0, mask1); + *is_found = 1; + } + *mask = _mm_max_epi16(temp_mask, *mask); +} + +static INLINE void update_mask0(__m128i *qcoeff0, __m128i *qcoeff1, + __m128i *threshold, const int16_t *iscan_ptr, + int *is_found, __m128i *mask) { + __m128i zero = _mm_setzero_si128(); + __m128i coeff[4], cmp_mask0, cmp_mask1, cmp_mask2, cmp_mask3; + + coeff[0] = _mm_unpacklo_epi16(*qcoeff0, zero); + coeff[1] = _mm_unpackhi_epi16(*qcoeff0, zero); + coeff[2] = _mm_unpacklo_epi16(*qcoeff1, zero); + coeff[3] = _mm_unpackhi_epi16(*qcoeff1, zero); + + coeff[0] = _mm_slli_epi32(coeff[0], AOM_QM_BITS); + cmp_mask0 = _mm_cmpgt_epi32(coeff[0], threshold[0]); + coeff[1] = _mm_slli_epi32(coeff[1], AOM_QM_BITS); + cmp_mask1 = _mm_cmpgt_epi32(coeff[1], threshold[1]); + coeff[2] = _mm_slli_epi32(coeff[2], AOM_QM_BITS); + cmp_mask2 = _mm_cmpgt_epi32(coeff[2], threshold[1]); + coeff[3] = _mm_slli_epi32(coeff[3], AOM_QM_BITS); + cmp_mask3 = _mm_cmpgt_epi32(coeff[3], threshold[1]); + + cmp_mask0 = _mm_packs_epi32(cmp_mask0, cmp_mask1); + cmp_mask1 = _mm_packs_epi32(cmp_mask2, cmp_mask3); + + update_mask1(&cmp_mask0, &cmp_mask1, iscan_ptr, is_found, mask); +} + +static INLINE int calculate_non_zero_count(__m128i mask) { + __m128i mask0, mask1; + int non_zero_count = 0; + mask0 = _mm_unpackhi_epi64(mask, mask); + mask1 = _mm_max_epi16(mask0, mask); + mask0 = _mm_shuffle_epi32(mask1, 1); + mask0 = _mm_max_epi16(mask0, mask1); + mask1 = _mm_srli_epi32(mask0, 16); + mask0 = _mm_max_epi16(mask0, mask1); + non_zero_count = _mm_extract_epi16(mask0, 0) + 1; + + return non_zero_count; +} diff --git a/third_party/aom/apps/aomenc.c b/third_party/aom/apps/aomenc.c index 08bf08d5b1..cfbf8e2459 100644 --- a/third_party/aom/apps/aomenc.c +++ b/third_party/aom/apps/aomenc.c @@ -549,8 +549,9 @@ static const arg_def_t enable_angle_delta = "Enable intra angle delta (0: false, 1: true (default))"); static const arg_def_t disable_trellis_quant = ARG_DEF(NULL, "disable-trellis-quant", 1, - "Disable trellis optimization of quantized coefficients (0: false (" - "default) 1: true 2: partial true)"); + "Disable trellis optimization of quantized coefficients (0: false " + "1: true 2: true for rd search 3: true for estimate yrd serch " + "(default))"); static const arg_def_t enable_qm = ARG_DEF(NULL, "enable-qm", 1, "Enable quantisation matrices (0: false (default), 1: true)"); @@ -628,9 +629,12 @@ static const arg_def_t aq_mode = ARG_DEF( NULL, "aq-mode", 1, "Adaptive quantization mode (0: off (default), 1: variance 2: complexity, " "3: cyclic refresh)"); -static const arg_def_t deltaq_mode = ARG_DEF( - NULL, "deltaq-mode", 1, - "Delta qindex mode (0: off (default), 1: deltaq 2: deltaq + deltalf)"); +static const arg_def_t deltaq_mode = + ARG_DEF(NULL, "deltaq-mode", 1, + "Delta qindex mode (0: off (default), 1: deltaq pred efficiency, " + "2: deltaq perceptual)"); +static const arg_def_t deltalf_mode = ARG_DEF( + NULL, "delta-lf-mode", 1, "Enable delta-lf-mode (0: off (default), 1: on)"); static const arg_def_t frame_periodic_boost = ARG_DEF(NULL, "frame-boost", 1, "Enable frame periodic boost (0: off (default), 1: on)"); @@ -839,6 +843,7 @@ static const arg_def_t *av1_args[] = { &cpu_used_av1, &error_resilient_mode, &aq_mode, &deltaq_mode, + &deltalf_mode, &frame_periodic_boost, &noise_sens, &tune_content, @@ -938,6 +943,7 @@ static const int av1_arg_ctrl_map[] = { AOME_SET_CPUUSED, AV1E_SET_ERROR_RESILIENT_MODE, AV1E_SET_AQ_MODE, AV1E_SET_DELTAQ_MODE, + AV1E_SET_DELTALF_MODE, AV1E_SET_FRAME_PERIODIC_BOOST, AV1E_SET_NOISE_SENSITIVITY, AV1E_SET_TUNE_CONTENT, diff --git a/third_party/aom/av1/av1.cmake b/third_party/aom/av1/av1.cmake index fb9678a8ea..d5b441333a 100644 --- a/third_party/aom/av1/av1.cmake +++ b/third_party/aom/av1/av1.cmake @@ -188,6 +188,7 @@ list(APPEND AOM_AV1_ENCODER_SOURCES "${AOM_ROOT}/av1/encoder/rd.c" "${AOM_ROOT}/av1/encoder/rd.h" "${AOM_ROOT}/av1/encoder/rdopt.c" + "${AOM_ROOT}/av1/encoder/nonrd_pickmode.c" "${AOM_ROOT}/av1/encoder/rdopt.h" "${AOM_ROOT}/av1/encoder/reconinter_enc.c" "${AOM_ROOT}/av1/encoder/reconinter_enc.h" diff --git a/third_party/aom/av1/av1_cx_iface.c b/third_party/aom/av1/av1_cx_iface.c index e8cd508cc4..e800287a35 100644 --- a/third_party/aom/av1/av1_cx_iface.c +++ b/third_party/aom/av1/av1_cx_iface.c @@ -72,6 +72,7 @@ struct av1_extracfg { int enable_dual_filter; AQ_MODE aq_mode; DELTAQ_MODE deltaq_mode; + int deltalf_mode; unsigned int frame_periodic_boost; aom_bit_depth_t bit_depth; aom_tune_content content; @@ -169,7 +170,7 @@ static struct av1_extracfg default_extra_cfg = { !CONFIG_SHARP_SETTINGS, // enable_cdef 1, // enable_restoration 1, // enable_obmc - 0, // disable_trellis_quant + 3, // disable_trellis_quant 0, // enable_qm DEFAULT_QM_Y, // qm_y DEFAULT_QM_U, // qm_u @@ -186,6 +187,7 @@ static struct av1_extracfg default_extra_cfg = { 1, // enable dual filter NO_AQ, // aq_mode NO_DELTA_Q, // deltaq_mode + 0, // delta lf mode 0, // frame_periodic_delta_q AOM_BITS_8, // Bit depth AOM_CONTENT_DEFAULT, // content @@ -261,6 +263,9 @@ struct aom_codec_alg_priv { aom_codec_priv_t base; aom_codec_enc_cfg_t cfg; struct av1_extracfg extra_cfg; + aom_rational64_t timestamp_ratio; + aom_codec_pts_t pts_offset; + unsigned char pts_offset_initialized; AV1EncoderConfig oxcf; AV1_COMP *cpi; unsigned char *cx_data; @@ -278,6 +283,23 @@ struct aom_codec_alg_priv { BufferPool *buffer_pool; }; +static INLINE int gcd(int64_t a, int b) { + int remainder; // remainder + while (b > 0) { + remainder = (int)(a % b); + a = b; + b = remainder; + } + + return (int)a; +} + +static INLINE void reduce_ratio(aom_rational64_t *ratio) { + const int denom = gcd(ratio->num, ratio->den); + ratio->num /= denom; + ratio->den /= denom; +} + static aom_codec_err_t update_error_state( aom_codec_alg_priv_t *ctx, const struct aom_internal_error_info *error) { const aom_codec_err_t res = error->error_code; @@ -324,7 +346,8 @@ static aom_codec_err_t validate_config(aom_codec_alg_priv_t *ctx, RANGE_CHECK_HI(cfg, rc_min_quantizer, cfg->rc_max_quantizer); RANGE_CHECK_BOOL(extra_cfg, lossless); RANGE_CHECK_HI(extra_cfg, aq_mode, AQ_MODE_COUNT - 1); - RANGE_CHECK_HI(extra_cfg, deltaq_mode, DELTAQ_MODE_COUNT - 1); + RANGE_CHECK_HI(extra_cfg, deltaq_mode, DELTA_Q_MODE_COUNT - 1); + RANGE_CHECK_HI(extra_cfg, deltalf_mode, 1); RANGE_CHECK_HI(extra_cfg, frame_periodic_boost, 1); RANGE_CHECK_HI(cfg, g_usage, 1); RANGE_CHECK_HI(cfg, g_threads, MAX_NUM_THREADS); @@ -683,9 +706,6 @@ static aom_codec_err_t set_encoder_config( } } - oxcf->enable_tpl_model = - extra_cfg->enable_tpl_model && (oxcf->superres_mode == SUPERRES_NONE); - oxcf->maximum_buffer_size_ms = is_vbr ? 240000 : cfg->rc_buf_sz; oxcf->starting_buffer_level_ms = is_vbr ? 60000 : cfg->rc_buf_initial_sz; oxcf->optimal_buffer_level_ms = is_vbr ? 60000 : cfg->rc_buf_optimal_sz; @@ -833,8 +853,23 @@ static aom_codec_err_t set_encoder_config( oxcf->timing_info_present = 0; } + oxcf->enable_tpl_model = + extra_cfg->enable_tpl_model && (oxcf->superres_mode == SUPERRES_NONE); + oxcf->aq_mode = extra_cfg->aq_mode; oxcf->deltaq_mode = extra_cfg->deltaq_mode; + // Turn on tpl model for deltaq_mode == DELTA_Q_OBJECTIVE and no + // superres. If superres is being used on the other hand, turn + // delta_q off. + if (oxcf->deltaq_mode == DELTA_Q_OBJECTIVE) { + if (oxcf->superres_mode == SUPERRES_NONE) + oxcf->enable_tpl_model = 1; + else + oxcf->deltaq_mode = NO_DELTA_Q; + } + + oxcf->deltalf_mode = + (oxcf->deltaq_mode != NO_DELTA_Q) && extra_cfg->deltalf_mode; oxcf->save_as_annexb = cfg->save_as_annexb; @@ -1507,6 +1542,13 @@ static aom_codec_err_t ctrl_set_deltaq_mode(aom_codec_alg_priv_t *ctx, return update_extra_cfg(ctx, &extra_cfg); } +static aom_codec_err_t ctrl_set_deltalf_mode(aom_codec_alg_priv_t *ctx, + va_list args) { + struct av1_extracfg extra_cfg = ctx->extra_cfg; + extra_cfg.deltalf_mode = CAST(AV1E_SET_DELTALF_MODE, args); + return update_extra_cfg(ctx, &extra_cfg); +} + static aom_codec_err_t ctrl_set_min_gf_interval(aom_codec_alg_priv_t *ctx, va_list args) { struct av1_extracfg extra_cfg = ctx->extra_cfg; @@ -1596,6 +1638,11 @@ static aom_codec_err_t encoder_init(aom_codec_ctx_t *ctx, res = validate_config(priv, &priv->cfg, &priv->extra_cfg); if (res == AOM_CODEC_OK) { + priv->timestamp_ratio.den = priv->cfg.g_timebase.den; + priv->timestamp_ratio.num = + (int64_t)priv->cfg.g_timebase.num * TICKS_PER_SEC; + reduce_ratio(&priv->timestamp_ratio); + set_encoder_config(&priv->oxcf, &priv->cfg, &priv->extra_cfg); priv->oxcf.use_highbitdepth = (ctx->init_flags & AOM_CODEC_USE_HIGHBITDEPTH) ? 1 : 0; @@ -1643,7 +1690,8 @@ static aom_codec_err_t encoder_encode(aom_codec_alg_priv_t *ctx, const size_t kMinCompressedSize = 8192; volatile aom_codec_err_t res = AOM_CODEC_OK; AV1_COMP *const cpi = ctx->cpi; - const aom_rational_t *const timebase = &ctx->cfg.g_timebase; + const aom_rational64_t *const timestamp_ratio = &ctx->timestamp_ratio; + volatile aom_codec_pts_t ptsvol = pts; if (cpi == NULL) return AOM_CODEC_INVALID_PARAM; @@ -1670,6 +1718,12 @@ static aom_codec_err_t encoder_encode(aom_codec_alg_priv_t *ctx, av1_change_config(ctx->cpi, &ctx->oxcf); } + if (!ctx->pts_offset_initialized) { + ctx->pts_offset = ptsvol; + ctx->pts_offset_initialized = 1; + } + ptsvol -= ctx->pts_offset; + aom_codec_pkt_list_init(&ctx->pkt_list); volatile aom_enc_frame_flags_t flags = enc_flags; @@ -1700,9 +1754,9 @@ static aom_codec_err_t encoder_encode(aom_codec_alg_priv_t *ctx, } if (res == AOM_CODEC_OK) { - int64_t dst_time_stamp = timebase_units_to_ticks(timebase, pts); + int64_t dst_time_stamp = timebase_units_to_ticks(timestamp_ratio, ptsvol); int64_t dst_end_time_stamp = - timebase_units_to_ticks(timebase, pts + duration); + timebase_units_to_ticks(timestamp_ratio, ptsvol + duration); // Set up internal flags if (ctx->base.init_flags & AOM_CODEC_USE_PSNR) cpi->b_calculate_psnr = 1; @@ -1751,7 +1805,7 @@ static aom_codec_err_t encoder_encode(aom_codec_alg_priv_t *ctx, !is_frame_visible && -1 != av1_get_compressed_data(cpi, &lib_flags, &frame_size, cx_data, &dst_time_stamp, &dst_end_time_stamp, - !img, timebase)) { + !img, timestamp_ratio)) { cpi->seq_params_locked = 1; if (frame_size) { if (ctx->pending_cx_data == 0) ctx->pending_cx_data = cx_data; @@ -1846,7 +1900,9 @@ static aom_codec_err_t encoder_encode(aom_codec_alg_priv_t *ctx, pkt.data.frame.partition_id = -1; pkt.data.frame.vis_frame_size = frame_size; - pkt.data.frame.pts = ticks_to_timebase_units(timebase, dst_time_stamp); + pkt.data.frame.pts = + ticks_to_timebase_units(timestamp_ratio, dst_time_stamp) + + ctx->pts_offset; pkt.data.frame.flags = get_frame_pkt_flags(cpi, lib_flags); if (has_fwd_keyframe) { // If one of the invisible frames in the packet is a keyframe, set @@ -1854,7 +1910,7 @@ static aom_codec_err_t encoder_encode(aom_codec_alg_priv_t *ctx, pkt.data.frame.flags |= AOM_FRAME_IS_DELAYED_RANDOM_ACCESS_POINT; } pkt.data.frame.duration = (uint32_t)ticks_to_timebase_units( - timebase, dst_end_time_stamp - dst_time_stamp); + timestamp_ratio, dst_end_time_stamp - dst_time_stamp); aom_codec_pkt_list_add(&ctx->pkt_list.head, &pkt); @@ -2231,6 +2287,7 @@ static aom_codec_ctrl_fn_map_t encoder_ctrl_maps[] = { { AV1E_SET_COEFF_COST_UPD_FREQ, ctrl_set_coeff_cost_upd_freq }, { AV1E_SET_MODE_COST_UPD_FREQ, ctrl_set_mode_cost_upd_freq }, { AV1E_SET_DELTAQ_MODE, ctrl_set_deltaq_mode }, + { AV1E_SET_DELTALF_MODE, ctrl_set_deltalf_mode }, { AV1E_SET_FRAME_PERIODIC_BOOST, ctrl_set_frame_periodic_boost }, { AV1E_SET_TUNE_CONTENT, ctrl_set_tune_content }, { AV1E_SET_CDF_UPDATE_MODE, ctrl_set_cdf_update_mode }, diff --git a/third_party/aom/av1/common/av1_loopfilter.c b/third_party/aom/av1/common/av1_loopfilter.c index 0aa1f9b907..bd99cb8b7c 100644 --- a/third_party/aom/av1/common/av1_loopfilter.c +++ b/third_party/aom/av1/common/av1_loopfilter.c @@ -28,9 +28,9 @@ static const SEG_LVL_FEATURES seg_lvl_lf_lut[MAX_MB_PLANE][2] = { { SEG_LVL_ALT_LF_V, SEG_LVL_ALT_LF_V } }; -static const int delta_lf_id_lut[MAX_MB_PLANE][2] = { - { 0, 1 }, { 2, 2 }, { 3, 3 } -}; +static const int delta_lf_id_lut[MAX_MB_PLANE][2] = { { 0, 1 }, + { 2, 2 }, + { 3, 3 } }; enum { VERT_EDGE = 0, HORZ_EDGE = 1, NUM_EDGE_DIRS } UENUM1BYTE(EDGE_DIR); @@ -2044,6 +2044,7 @@ static TX_SIZE set_lpf_parameters( const BLOCK_SIZE bsize = get_plane_block_size(mbmi->sb_type, plane_ptr->subsampling_x, plane_ptr->subsampling_y); + assert(bsize < BLOCK_SIZES_ALL); const int prediction_masks = edge_dir == VERT_EDGE ? block_size_wide[bsize] - 1 : block_size_high[bsize] - 1; diff --git a/third_party/aom/av1/common/av1_rtcd_defs.pl b/third_party/aom/av1/common/av1_rtcd_defs.pl index aca5ec7dea..f4cdfa54dc 100755 --- a/third_party/aom/av1/common/av1_rtcd_defs.pl +++ b/third_party/aom/av1/common/av1_rtcd_defs.pl @@ -111,9 +111,8 @@ specialize qw/av1_highbd_convolve8_vert/, "$sse2_x86_64"; #inv txfm add_proto qw/void av1_inv_txfm_add/, "const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param"; -# TODO(http://crbug.com/aomedia/2350): avx2 is disabled due to test vector -# mismatches. -specialize qw/av1_inv_txfm_add ssse3 neon/; +# TODO(yunqing): Disable NEON version due to test vector mismatch. +specialize qw/av1_inv_txfm_add ssse3 avx2/; # neon/; add_proto qw/void av1_highbd_inv_txfm_add/, "const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param"; # TODO(http://crbug.com/aomedia/2350): avx2 is disabled due to test vector @@ -164,9 +163,7 @@ add_proto qw/void av1_inv_txfm2d_add_32x8/, "const int32_t *input, uint16_t *out add_proto qw/void av1_highbd_dr_prediction_z1/, "uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_above, int dx, int dy, int bd"; specialize qw/av1_highbd_dr_prediction_z1 avx2/; add_proto qw/void av1_highbd_dr_prediction_z2/, "uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_above, int upsample_left, int dx, int dy, int bd"; -# TODO(niva213@gmail.com): Re-enable avx2 after fixing valgrind issue -# https://crbug.com/aomedia/2316 -# specialize qw/av1_highbd_dr_prediction_z2 avx2/; +specialize qw/av1_highbd_dr_prediction_z2 avx2/; add_proto qw/void av1_highbd_dr_prediction_z3/, "uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_left, int dx, int dy, int bd"; specialize qw/av1_highbd_dr_prediction_z3 avx2/; diff --git a/third_party/aom/av1/common/blockd.c b/third_party/aom/av1/common/blockd.c index 2e796b6560..148f66bb17 100644 --- a/third_party/aom/av1/common/blockd.c +++ b/third_party/aom/av1/common/blockd.c @@ -61,6 +61,8 @@ void av1_reset_skip_context(MACROBLOCKD *xd, int mi_row, int mi_col, int i; int nplanes; int chroma_ref; + assert(bsize < BLOCK_SIZES_ALL); + chroma_ref = is_chroma_reference(mi_row, mi_col, bsize, xd->plane[1].subsampling_x, xd->plane[1].subsampling_y); diff --git a/third_party/aom/av1/common/blockd.h b/third_party/aom/av1/common/blockd.h index 91ef3df84b..f5c70fe52d 100644 --- a/third_party/aom/av1/common/blockd.h +++ b/third_party/aom/av1/common/blockd.h @@ -187,9 +187,7 @@ typedef struct RD_STATS { int64_t rdcost; int64_t sse; int skip; // sse should equal to dist when skip == 1 - int64_t ref_rdcost; int zero_rate; - uint8_t invalid_rate; #if CONFIG_RD_DEBUG int txb_coeff_cost[MAX_MB_PLANE]; int txb_coeff_cost_map[MAX_MB_PLANE][TXB_COEFF_COST_MAP_SIZE] @@ -537,6 +535,7 @@ typedef struct macroblockd { uint8_t ref_mv_count[MODE_CTX_REF_FRAMES]; CANDIDATE_MV ref_mv_stack[MODE_CTX_REF_FRAMES][MAX_REF_MV_STACK_SIZE]; + uint16_t weight[MODE_CTX_REF_FRAMES][MAX_REF_MV_STACK_SIZE]; uint8_t is_sec_rect; // Counts of each reference frame in the above and left neighboring blocks. @@ -786,6 +785,8 @@ static INLINE BLOCK_SIZE get_plane_block_size(BLOCK_SIZE bsize, int subsampling_x, int subsampling_y) { if (bsize == BLOCK_INVALID) return BLOCK_INVALID; + assert(subsampling_x >= 0 && subsampling_x < 2); + assert(subsampling_y >= 0 && subsampling_y < 2); return ss_size_lookup[bsize][subsampling_x][subsampling_y]; } @@ -1006,6 +1007,7 @@ static INLINE int get_vartx_max_txsize(const MACROBLOCKD *xd, BLOCK_SIZE bsize, } static INLINE int is_motion_variation_allowed_bsize(BLOCK_SIZE bsize) { + assert(bsize < BLOCK_SIZES_ALL); return AOMMIN(block_size_wide[bsize], block_size_high[bsize]) >= 8; } diff --git a/third_party/aom/av1/common/convolve.c b/third_party/aom/av1/common/convolve.c index 5a55ece671..85baf45ef2 100644 --- a/third_party/aom/av1/common/convolve.c +++ b/third_party/aom/av1/common/convolve.c @@ -121,6 +121,7 @@ void av1_convolve_2d_sr_c(const uint8_t *src, int src_stride, uint8_t *dst, int16_t im_block[(MAX_SB_SIZE + MAX_FILTER_TAP - 1) * MAX_SB_SIZE]; int im_h = h + filter_params_y->taps - 1; int im_stride = w; + assert(w <= MAX_SB_SIZE && h <= MAX_SB_SIZE); const int fo_vert = filter_params_y->taps / 2 - 1; const int fo_horiz = filter_params_x->taps / 2 - 1; const int bd = 8; @@ -704,6 +705,7 @@ void av1_highbd_convolve_2d_sr_c(const uint16_t *src, int src_stride, int16_t im_block[(MAX_SB_SIZE + MAX_FILTER_TAP - 1) * MAX_SB_SIZE]; int im_h = h + filter_params_y->taps - 1; int im_stride = w; + assert(w <= MAX_SB_SIZE && h <= MAX_SB_SIZE); const int fo_vert = filter_params_y->taps / 2 - 1; const int fo_horiz = filter_params_x->taps / 2 - 1; const int bits = diff --git a/third_party/aom/av1/common/entropymode.c b/third_party/aom/av1/common/entropymode.c index 90702ac7d2..af3123a69c 100644 --- a/third_party/aom/av1/common/entropymode.c +++ b/third_party/aom/av1/common/entropymode.c @@ -435,16 +435,16 @@ static const aom_cdf_prob { AOM_CDF3(601, 943) }, { AOM_CDF3(14969, 21398) } }; -static const aom_cdf_prob default_newmv_cdf[NEWMV_MODE_CONTEXTS][CDF_SIZE(2)] = - { { AOM_CDF2(24035) }, { AOM_CDF2(16630) }, { AOM_CDF2(15339) }, - { AOM_CDF2(8386) }, { AOM_CDF2(12222) }, { AOM_CDF2(4676) } }; +static const aom_cdf_prob default_newmv_cdf[NEWMV_MODE_CONTEXTS][CDF_SIZE( + 2)] = { { AOM_CDF2(24035) }, { AOM_CDF2(16630) }, { AOM_CDF2(15339) }, + { AOM_CDF2(8386) }, { AOM_CDF2(12222) }, { AOM_CDF2(4676) } }; static const aom_cdf_prob default_zeromv_cdf[GLOBALMV_MODE_CONTEXTS][CDF_SIZE( 2)] = { { AOM_CDF2(2175) }, { AOM_CDF2(1054) } }; -static const aom_cdf_prob default_refmv_cdf[REFMV_MODE_CONTEXTS][CDF_SIZE(2)] = - { { AOM_CDF2(23974) }, { AOM_CDF2(24188) }, { AOM_CDF2(17848) }, - { AOM_CDF2(28622) }, { AOM_CDF2(24312) }, { AOM_CDF2(19923) } }; +static const aom_cdf_prob default_refmv_cdf[REFMV_MODE_CONTEXTS][CDF_SIZE( + 2)] = { { AOM_CDF2(23974) }, { AOM_CDF2(24188) }, { AOM_CDF2(17848) }, + { AOM_CDF2(28622) }, { AOM_CDF2(24312) }, { AOM_CDF2(19923) } }; static const aom_cdf_prob default_drl_cdf[DRL_MODE_CONTEXTS][CDF_SIZE(2)] = { { AOM_CDF2(13104) }, { AOM_CDF2(24560) }, { AOM_CDF2(18945) } @@ -470,11 +470,11 @@ static const aom_cdf_prob default_interintra_cdf[BLOCK_SIZE_GROUPS][CDF_SIZE( { AOM_CDF2(30237) } }; static const aom_cdf_prob - default_interintra_mode_cdf[BLOCK_SIZE_GROUPS][CDF_SIZE(INTERINTRA_MODES)] = - { { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(1875, 11082, 27332) }, - { AOM_CDF4(2473, 9996, 26388) }, - { AOM_CDF4(4238, 11537, 25926) } }; + default_interintra_mode_cdf[BLOCK_SIZE_GROUPS][CDF_SIZE( + INTERINTRA_MODES)] = { { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(1875, 11082, 27332) }, + { AOM_CDF4(2473, 9996, 26388) }, + { AOM_CDF4(4238, 11537, 25926) } }; static const aom_cdf_prob default_wedge_interintra_cdf[BLOCK_SIZES_ALL][CDF_SIZE(2)] = { @@ -500,51 +500,51 @@ static const aom_cdf_prob default_compound_type_cdf[BLOCK_SIZES_ALL][CDF_SIZE( { AOM_CDF2(16384) } }; -static const aom_cdf_prob default_wedge_idx_cdf[BLOCK_SIZES_ALL][CDF_SIZE(16)] = - { { AOM_CDF16(2048, 4096, 6144, 8192, 10240, 12288, 14336, 16384, 18432, - 20480, 22528, 24576, 26624, 28672, 30720) }, - { AOM_CDF16(2048, 4096, 6144, 8192, 10240, 12288, 14336, 16384, 18432, - 20480, 22528, 24576, 26624, 28672, 30720) }, - { AOM_CDF16(2048, 4096, 6144, 8192, 10240, 12288, 14336, 16384, 18432, - 20480, 22528, 24576, 26624, 28672, 30720) }, - { AOM_CDF16(2438, 4440, 6599, 8663, 11005, 12874, 15751, 18094, 20359, - 22362, 24127, 25702, 27752, 29450, 31171) }, - { AOM_CDF16(806, 3266, 6005, 6738, 7218, 7367, 7771, 14588, 16323, 17367, - 18452, 19422, 22839, 26127, 29629) }, - { AOM_CDF16(2779, 3738, 4683, 7213, 7775, 8017, 8655, 14357, 17939, 21332, - 24520, 27470, 29456, 30529, 31656) }, - { AOM_CDF16(1684, 3625, 5675, 7108, 9302, 11274, 14429, 17144, 19163, - 20961, 22884, 24471, 26719, 28714, 30877) }, - { AOM_CDF16(1142, 3491, 6277, 7314, 8089, 8355, 9023, 13624, 15369, 16730, - 18114, 19313, 22521, 26012, 29550) }, - { AOM_CDF16(2742, 4195, 5727, 8035, 8980, 9336, 10146, 14124, 17270, - 20533, 23434, 25972, 27944, 29570, 31416) }, - { AOM_CDF16(1727, 3948, 6101, 7796, 9841, 12344, 15766, 18944, 20638, - 22038, 23963, 25311, 26988, 28766, 31012) }, - { AOM_CDF16(2048, 4096, 6144, 8192, 10240, 12288, 14336, 16384, 18432, - 20480, 22528, 24576, 26624, 28672, 30720) }, - { AOM_CDF16(2048, 4096, 6144, 8192, 10240, 12288, 14336, 16384, 18432, - 20480, 22528, 24576, 26624, 28672, 30720) }, - { AOM_CDF16(2048, 4096, 6144, 8192, 10240, 12288, 14336, 16384, 18432, - 20480, 22528, 24576, 26624, 28672, 30720) }, - { AOM_CDF16(2048, 4096, 6144, 8192, 10240, 12288, 14336, 16384, 18432, - 20480, 22528, 24576, 26624, 28672, 30720) }, - { AOM_CDF16(2048, 4096, 6144, 8192, 10240, 12288, 14336, 16384, 18432, - 20480, 22528, 24576, 26624, 28672, 30720) }, - { AOM_CDF16(2048, 4096, 6144, 8192, 10240, 12288, 14336, 16384, 18432, - 20480, 22528, 24576, 26624, 28672, 30720) }, - { AOM_CDF16(2048, 4096, 6144, 8192, 10240, 12288, 14336, 16384, 18432, - 20480, 22528, 24576, 26624, 28672, 30720) }, - { AOM_CDF16(2048, 4096, 6144, 8192, 10240, 12288, 14336, 16384, 18432, - 20480, 22528, 24576, 26624, 28672, 30720) }, - { AOM_CDF16(154, 987, 1925, 2051, 2088, 2111, 2151, 23033, 23703, 24284, - 24985, 25684, 27259, 28883, 30911) }, - { AOM_CDF16(1135, 1322, 1493, 2635, 2696, 2737, 2770, 21016, 22935, 25057, - 27251, 29173, 30089, 30960, 31933) }, - { AOM_CDF16(2048, 4096, 6144, 8192, 10240, 12288, 14336, 16384, 18432, - 20480, 22528, 24576, 26624, 28672, 30720) }, - { AOM_CDF16(2048, 4096, 6144, 8192, 10240, 12288, 14336, 16384, 18432, - 20480, 22528, 24576, 26624, 28672, 30720) } }; +static const aom_cdf_prob default_wedge_idx_cdf[BLOCK_SIZES_ALL][CDF_SIZE( + 16)] = { { AOM_CDF16(2048, 4096, 6144, 8192, 10240, 12288, 14336, 16384, + 18432, 20480, 22528, 24576, 26624, 28672, 30720) }, + { AOM_CDF16(2048, 4096, 6144, 8192, 10240, 12288, 14336, 16384, + 18432, 20480, 22528, 24576, 26624, 28672, 30720) }, + { AOM_CDF16(2048, 4096, 6144, 8192, 10240, 12288, 14336, 16384, + 18432, 20480, 22528, 24576, 26624, 28672, 30720) }, + { AOM_CDF16(2438, 4440, 6599, 8663, 11005, 12874, 15751, 18094, + 20359, 22362, 24127, 25702, 27752, 29450, 31171) }, + { AOM_CDF16(806, 3266, 6005, 6738, 7218, 7367, 7771, 14588, 16323, + 17367, 18452, 19422, 22839, 26127, 29629) }, + { AOM_CDF16(2779, 3738, 4683, 7213, 7775, 8017, 8655, 14357, 17939, + 21332, 24520, 27470, 29456, 30529, 31656) }, + { AOM_CDF16(1684, 3625, 5675, 7108, 9302, 11274, 14429, 17144, + 19163, 20961, 22884, 24471, 26719, 28714, 30877) }, + { AOM_CDF16(1142, 3491, 6277, 7314, 8089, 8355, 9023, 13624, 15369, + 16730, 18114, 19313, 22521, 26012, 29550) }, + { AOM_CDF16(2742, 4195, 5727, 8035, 8980, 9336, 10146, 14124, + 17270, 20533, 23434, 25972, 27944, 29570, 31416) }, + { AOM_CDF16(1727, 3948, 6101, 7796, 9841, 12344, 15766, 18944, + 20638, 22038, 23963, 25311, 26988, 28766, 31012) }, + { AOM_CDF16(2048, 4096, 6144, 8192, 10240, 12288, 14336, 16384, + 18432, 20480, 22528, 24576, 26624, 28672, 30720) }, + { AOM_CDF16(2048, 4096, 6144, 8192, 10240, 12288, 14336, 16384, + 18432, 20480, 22528, 24576, 26624, 28672, 30720) }, + { AOM_CDF16(2048, 4096, 6144, 8192, 10240, 12288, 14336, 16384, + 18432, 20480, 22528, 24576, 26624, 28672, 30720) }, + { AOM_CDF16(2048, 4096, 6144, 8192, 10240, 12288, 14336, 16384, + 18432, 20480, 22528, 24576, 26624, 28672, 30720) }, + { AOM_CDF16(2048, 4096, 6144, 8192, 10240, 12288, 14336, 16384, + 18432, 20480, 22528, 24576, 26624, 28672, 30720) }, + { AOM_CDF16(2048, 4096, 6144, 8192, 10240, 12288, 14336, 16384, + 18432, 20480, 22528, 24576, 26624, 28672, 30720) }, + { AOM_CDF16(2048, 4096, 6144, 8192, 10240, 12288, 14336, 16384, + 18432, 20480, 22528, 24576, 26624, 28672, 30720) }, + { AOM_CDF16(2048, 4096, 6144, 8192, 10240, 12288, 14336, 16384, + 18432, 20480, 22528, 24576, 26624, 28672, 30720) }, + { AOM_CDF16(154, 987, 1925, 2051, 2088, 2111, 2151, 23033, 23703, + 24284, 24985, 25684, 27259, 28883, 30911) }, + { AOM_CDF16(1135, 1322, 1493, 2635, 2696, 2737, 2770, 21016, 22935, + 25057, 27251, 29173, 30089, 30960, 31933) }, + { AOM_CDF16(2048, 4096, 6144, 8192, 10240, 12288, 14336, 16384, + 18432, 20480, 22528, 24576, 26624, 28672, 30720) }, + { AOM_CDF16(2048, 4096, 6144, 8192, 10240, 12288, 14336, 16384, + 18432, 20480, 22528, 24576, 26624, 28672, 30720) } }; static const aom_cdf_prob default_motion_mode_cdf[BLOCK_SIZES_ALL][CDF_SIZE( MOTION_MODES)] = { { AOM_CDF3(10923, 21845) }, { AOM_CDF3(10923, 21845) }, diff --git a/third_party/aom/av1/common/enums.h b/third_party/aom/av1/common/enums.h index fbacc894d4..500c56de45 100644 --- a/third_party/aom/av1/common/enums.h +++ b/third_party/aom/av1/common/enums.h @@ -526,7 +526,9 @@ enum { #define DELTA_Q_SMALL 3 #define DELTA_Q_PROBS (DELTA_Q_SMALL) -#define DEFAULT_DELTA_Q_RES 4 +#define DEFAULT_DELTA_Q_RES_PERCEPTUAL 4 +#define DEFAULT_DELTA_Q_RES_OBJECTIVE 8 + #define DELTA_LF_SMALL 3 #define DELTA_LF_PROBS (DELTA_LF_SMALL) #define DEFAULT_DELTA_LF_RES 2 @@ -625,6 +627,25 @@ enum { RESTORE_TYPES = 4, } UENUM1BYTE(RestorationType); +// Picture prediction structures (0-12 are predefined) in scalability metadata. +enum { + SCALABILITY_L1T2 = 0, + SCALABILITY_L1T3 = 1, + SCALABILITY_L2T1 = 2, + SCALABILITY_L2T2 = 3, + SCALABILITY_L2T3 = 4, + SCALABILITY_S2T1 = 5, + SCALABILITY_S2T2 = 6, + SCALABILITY_S2T3 = 7, + SCALABILITY_L2T1h = 8, + SCALABILITY_L2T2h = 9, + SCALABILITY_L2T3h = 10, + SCALABILITY_S2T1h = 11, + SCALABILITY_S2T2h = 12, + SCALABILITY_S2T3h = 13, + SCALABILITY_SS = 14 +} UENUM1BYTE(SCALABILITY_STRUCTURES); + #define SUPERRES_SCALE_BITS 3 #define SUPERRES_SCALE_DENOMINATOR_MIN (SCALE_NUMERATOR + 1) diff --git a/third_party/aom/av1/common/mv.h b/third_party/aom/av1/common/mv.h index d097f9e5f1..fe821eeed6 100644 --- a/third_party/aom/av1/common/mv.h +++ b/third_party/aom/av1/common/mv.h @@ -277,7 +277,6 @@ static INLINE TransformationType get_wmtype(const WarpedMotionParams *gm) { typedef struct candidate_mv { int_mv this_mv; int_mv comp_mv; - int weight; } CANDIDATE_MV; static INLINE int is_zero_mv(const MV *mv) { diff --git a/third_party/aom/av1/common/mvref_common.c b/third_party/aom/av1/common/mvref_common.c index e38891f265..c4ca859a2f 100644 --- a/third_party/aom/av1/common/mvref_common.c +++ b/third_party/aom/av1/common/mvref_common.c @@ -74,8 +74,9 @@ void av1_copy_frame_mvs(const AV1_COMMON *const cm, static void add_ref_mv_candidate( const MB_MODE_INFO *const candidate, const MV_REFERENCE_FRAME rf[2], uint8_t *refmv_count, uint8_t *ref_match_count, uint8_t *newmv_count, - CANDIDATE_MV *ref_mv_stack, int_mv *gm_mv_candidates, - const WarpedMotionParams *gm_params, int col, int weight) { + CANDIDATE_MV *ref_mv_stack, uint16_t *ref_mv_weight, + int_mv *gm_mv_candidates, const WarpedMotionParams *gm_params, int col, + uint16_t weight) { if (!is_inter_block(candidate)) return; // for intrabc int index = 0, ref; assert(weight % 2 == 0); @@ -93,12 +94,12 @@ static void add_ref_mv_candidate( for (index = 0; index < *refmv_count; ++index) if (ref_mv_stack[index].this_mv.as_int == this_refmv.as_int) break; - if (index < *refmv_count) ref_mv_stack[index].weight += weight; + if (index < *refmv_count) ref_mv_weight[index] += weight; // Add a new item to the list. if (index == *refmv_count && *refmv_count < MAX_REF_MV_STACK_SIZE) { ref_mv_stack[index].this_mv = this_refmv; - ref_mv_stack[index].weight = weight; + ref_mv_weight[index] = weight; ++(*refmv_count); } if (have_newmv_in_inter_mode(candidate->mode)) ++*newmv_count; @@ -122,13 +123,13 @@ static void add_ref_mv_candidate( (ref_mv_stack[index].comp_mv.as_int == this_refmv[1].as_int)) break; - if (index < *refmv_count) ref_mv_stack[index].weight += weight; + if (index < *refmv_count) ref_mv_weight[index] += weight; // Add a new item to the list. if (index == *refmv_count && *refmv_count < MAX_REF_MV_STACK_SIZE) { ref_mv_stack[index].this_mv = this_refmv[0]; ref_mv_stack[index].comp_mv = this_refmv[1]; - ref_mv_stack[index].weight = weight; + ref_mv_weight[index] = weight; ++(*refmv_count); } if (have_newmv_in_inter_mode(candidate->mode)) ++*newmv_count; @@ -140,10 +141,10 @@ static void add_ref_mv_candidate( static void scan_row_mbmi(const AV1_COMMON *cm, const MACROBLOCKD *xd, int mi_row, int mi_col, const MV_REFERENCE_FRAME rf[2], int row_offset, - CANDIDATE_MV *ref_mv_stack, uint8_t *refmv_count, - uint8_t *ref_match_count, uint8_t *newmv_count, - int_mv *gm_mv_candidates, int max_row_offset, - int *processed_rows) { + CANDIDATE_MV *ref_mv_stack, uint16_t *ref_mv_weight, + uint8_t *refmv_count, uint8_t *ref_match_count, + uint8_t *newmv_count, int_mv *gm_mv_candidates, + int max_row_offset, int *processed_rows) { int end_mi = AOMMIN(xd->n4_w, cm->mi_cols - mi_col); end_mi = AOMMIN(end_mi, mi_size_wide[BLOCK_64X64]); const int n8_w_8 = mi_size_wide[BLOCK_8X8]; @@ -169,10 +170,10 @@ static void scan_row_mbmi(const AV1_COMMON *cm, const MACROBLOCKD *xd, else if (abs(row_offset) > 1) len = AOMMAX(len, n8_w_8); - int weight = 2; + uint16_t weight = 2; if (xd->n4_w >= n8_w_8 && xd->n4_w <= n4_w) { - int inc = AOMMIN(-max_row_offset + row_offset + 1, - mi_size_high[candidate_bsize]); + uint16_t inc = AOMMIN(-max_row_offset + row_offset + 1, + mi_size_high[candidate_bsize]); // Obtain range used in weight calculation. weight = AOMMAX(weight, inc); // Update processed rows. @@ -180,8 +181,9 @@ static void scan_row_mbmi(const AV1_COMMON *cm, const MACROBLOCKD *xd, } add_ref_mv_candidate(candidate, rf, refmv_count, ref_match_count, - newmv_count, ref_mv_stack, gm_mv_candidates, - cm->global_motion, col_offset + i, len * weight); + newmv_count, ref_mv_stack, ref_mv_weight, + gm_mv_candidates, cm->global_motion, col_offset + i, + len * weight); i += len; } @@ -190,10 +192,10 @@ static void scan_row_mbmi(const AV1_COMMON *cm, const MACROBLOCKD *xd, static void scan_col_mbmi(const AV1_COMMON *cm, const MACROBLOCKD *xd, int mi_row, int mi_col, const MV_REFERENCE_FRAME rf[2], int col_offset, - CANDIDATE_MV *ref_mv_stack, uint8_t *refmv_count, - uint8_t *ref_match_count, uint8_t *newmv_count, - int_mv *gm_mv_candidates, int max_col_offset, - int *processed_cols) { + CANDIDATE_MV *ref_mv_stack, uint16_t *ref_mv_weight, + uint8_t *refmv_count, uint8_t *ref_match_count, + uint8_t *newmv_count, int_mv *gm_mv_candidates, + int max_col_offset, int *processed_cols) { int end_mi = AOMMIN(xd->n4_h, cm->mi_rows - mi_row); end_mi = AOMMIN(end_mi, mi_size_high[BLOCK_64X64]); const int n8_h_8 = mi_size_high[BLOCK_8X8]; @@ -229,8 +231,9 @@ static void scan_col_mbmi(const AV1_COMMON *cm, const MACROBLOCKD *xd, } add_ref_mv_candidate(candidate, rf, refmv_count, ref_match_count, - newmv_count, ref_mv_stack, gm_mv_candidates, - cm->global_motion, col_offset, len * weight); + newmv_count, ref_mv_stack, ref_mv_weight, + gm_mv_candidates, cm->global_motion, col_offset, + len * weight); i += len; } @@ -240,8 +243,8 @@ static void scan_blk_mbmi(const AV1_COMMON *cm, const MACROBLOCKD *xd, const int mi_row, const int mi_col, const MV_REFERENCE_FRAME rf[2], int row_offset, int col_offset, CANDIDATE_MV *ref_mv_stack, - uint8_t *ref_match_count, uint8_t *newmv_count, - int_mv *gm_mv_candidates, + uint16_t *ref_mv_weight, uint8_t *ref_match_count, + uint8_t *newmv_count, int_mv *gm_mv_candidates, uint8_t refmv_count[MODE_CTX_REF_FRAMES]) { const TileInfo *const tile = &xd->tile; POSITION mi_pos; @@ -255,8 +258,9 @@ static void scan_blk_mbmi(const AV1_COMMON *cm, const MACROBLOCKD *xd, const int len = mi_size_wide[BLOCK_8X8]; add_ref_mv_candidate(candidate, rf, refmv_count, ref_match_count, - newmv_count, ref_mv_stack, gm_mv_candidates, - cm->global_motion, mi_pos.col, 2 * len); + newmv_count, ref_mv_stack, ref_mv_weight, + gm_mv_candidates, cm->global_motion, mi_pos.col, + 2 * len); } // Analyze a single 8x8 block motion information. } @@ -328,10 +332,11 @@ static int add_tpl_ref_mv(const AV1_COMMON *cm, const MACROBLOCKD *xd, int blk_row, int blk_col, int_mv *gm_mv_candidates, uint8_t refmv_count[MODE_CTX_REF_FRAMES], CANDIDATE_MV ref_mv_stacks[][MAX_REF_MV_STACK_SIZE], + uint16_t ref_mv_weights[][MAX_REF_MV_STACK_SIZE], int16_t *mode_context) { POSITION mi_pos; int idx; - const int weight_unit = 1; // mi_size_wide[BLOCK_8X8]; + const uint16_t weight_unit = 1; // mi_size_wide[BLOCK_8X8]; mi_pos.row = (mi_row & 0x01) ? blk_row : blk_row + 1; mi_pos.col = (mi_col & 0x01) ? blk_col : blk_col + 1; @@ -352,7 +357,7 @@ static int add_tpl_ref_mv(const AV1_COMMON *cm, const MACROBLOCKD *xd, int cur_offset_0 = get_relative_dist(&cm->seq_params.order_hint_info, cur_frame_index, frame0_index); CANDIDATE_MV *ref_mv_stack = ref_mv_stacks[rf[0]]; - + uint16_t *ref_mv_weight = ref_mv_weights[rf[0]]; if (prev_frame_mvs->mfmv0.as_int != INVALID_MV) { int_mv this_refmv; @@ -369,12 +374,12 @@ static int add_tpl_ref_mv(const AV1_COMMON *cm, const MACROBLOCKD *xd, for (idx = 0; idx < refmv_count[rf[0]]; ++idx) if (this_refmv.as_int == ref_mv_stack[idx].this_mv.as_int) break; - if (idx < refmv_count[rf[0]]) ref_mv_stack[idx].weight += 2 * weight_unit; + if (idx < refmv_count[rf[0]]) ref_mv_weight[idx] += 2 * weight_unit; if (idx == refmv_count[rf[0]] && refmv_count[rf[0]] < MAX_REF_MV_STACK_SIZE) { ref_mv_stack[idx].this_mv.as_int = this_refmv.as_int; - ref_mv_stack[idx].weight = 2 * weight_unit; + ref_mv_weight[idx] = 2 * weight_unit; ++(refmv_count[rf[0]]); } return 1; @@ -392,7 +397,7 @@ static int add_tpl_ref_mv(const AV1_COMMON *cm, const MACROBLOCKD *xd, int cur_offset_1 = get_relative_dist(&cm->seq_params.order_hint_info, cur_frame_index, frame1_index); CANDIDATE_MV *ref_mv_stack = ref_mv_stacks[ref_frame]; - + uint16_t *ref_mv_weight = ref_mv_weights[ref_frame]; if (prev_frame_mvs->mfmv0.as_int != INVALID_MV) { int_mv this_refmv; int_mv comp_refmv; @@ -418,14 +423,13 @@ static int add_tpl_ref_mv(const AV1_COMMON *cm, const MACROBLOCKD *xd, comp_refmv.as_int == ref_mv_stack[idx].comp_mv.as_int) break; - if (idx < refmv_count[ref_frame]) - ref_mv_stack[idx].weight += 2 * weight_unit; + if (idx < refmv_count[ref_frame]) ref_mv_weight[idx] += 2 * weight_unit; if (idx == refmv_count[ref_frame] && refmv_count[ref_frame] < MAX_REF_MV_STACK_SIZE) { ref_mv_stack[idx].this_mv.as_int = this_refmv.as_int; ref_mv_stack[idx].comp_mv.as_int = comp_refmv.as_int; - ref_mv_stack[idx].weight = 2 * weight_unit; + ref_mv_weight[idx] = 2 * weight_unit; ++(refmv_count[ref_frame]); } return 1; @@ -462,7 +466,8 @@ static void process_compound_ref_mv_candidate( static void process_single_ref_mv_candidate( const MB_MODE_INFO *const candidate, const AV1_COMMON *const cm, MV_REFERENCE_FRAME ref_frame, uint8_t refmv_count[MODE_CTX_REF_FRAMES], - CANDIDATE_MV ref_mv_stack[][MAX_REF_MV_STACK_SIZE]) { + CANDIDATE_MV ref_mv_stack[][MAX_REF_MV_STACK_SIZE], + uint16_t ref_mv_weight[][MAX_REF_MV_STACK_SIZE]) { for (int rf_idx = 0; rf_idx < 2; ++rf_idx) { if (candidate->ref_frame[rf_idx] > INTRA_FRAME) { int_mv this_mv = candidate->mv[rf_idx]; @@ -482,7 +487,7 @@ static void process_single_ref_mv_candidate( // TODO(jingning): Set an arbitrary small number here. The weight // doesn't matter as long as it is properly initialized. - ref_mv_stack[ref_frame][stack_idx].weight = 2; + ref_mv_weight[ref_frame][stack_idx] = 2; ++refmv_count[ref_frame]; } } @@ -493,6 +498,7 @@ static void setup_ref_mv_list( const AV1_COMMON *cm, const MACROBLOCKD *xd, MV_REFERENCE_FRAME ref_frame, uint8_t refmv_count[MODE_CTX_REF_FRAMES], CANDIDATE_MV ref_mv_stack[][MAX_REF_MV_STACK_SIZE], + uint16_t ref_mv_weight[][MAX_REF_MV_STACK_SIZE], int_mv mv_ref_list[][MAX_MV_REF_CANDIDATES], int_mv *gm_mv_candidates, int mi_row, int mi_col, int16_t *mode_context) { const int bs = AOMMAX(xd->n4_w, xd->n4_h); @@ -536,25 +542,28 @@ static void setup_ref_mv_list( // Scan the first above row mode info. row_offset = -1; if (abs(max_row_offset) >= 1) scan_row_mbmi(cm, xd, mi_row, mi_col, rf, -1, ref_mv_stack[ref_frame], - &refmv_count[ref_frame], &row_match_count, &newmv_count, - gm_mv_candidates, max_row_offset, &processed_rows); + ref_mv_weight[ref_frame], &refmv_count[ref_frame], + &row_match_count, &newmv_count, gm_mv_candidates, + max_row_offset, &processed_rows); // Scan the first left column mode info. col_offset = -1; if (abs(max_col_offset) >= 1) scan_col_mbmi(cm, xd, mi_row, mi_col, rf, -1, ref_mv_stack[ref_frame], - &refmv_count[ref_frame], &col_match_count, &newmv_count, - gm_mv_candidates, max_col_offset, &processed_cols); + ref_mv_weight[ref_frame], &refmv_count[ref_frame], + &col_match_count, &newmv_count, gm_mv_candidates, + max_col_offset, &processed_cols); // Check top-right boundary if (has_tr) scan_blk_mbmi(cm, xd, mi_row, mi_col, rf, -1, xd->n4_w, - ref_mv_stack[ref_frame], &row_match_count, &newmv_count, - gm_mv_candidates, &refmv_count[ref_frame]); + ref_mv_stack[ref_frame], ref_mv_weight[ref_frame], + &row_match_count, &newmv_count, gm_mv_candidates, + &refmv_count[ref_frame]); const uint8_t nearest_match = (row_match_count > 0) + (col_match_count > 0); const uint8_t nearest_refmv_count = refmv_count[ref_frame]; // TODO(yunqing): for comp_search, do it for all 3 cases. for (int idx = 0; idx < nearest_refmv_count; ++idx) - ref_mv_stack[ref_frame][idx].weight += REF_CAT_LEVEL; + ref_mv_weight[ref_frame][idx] += REF_CAT_LEVEL; if (cm->allow_ref_frame_mvs) { int is_available = 0; @@ -584,7 +593,7 @@ static void setup_ref_mv_list( for (int blk_col = 0; blk_col < blk_col_end; blk_col += step_w) { int ret = add_tpl_ref_mv(cm, xd, mi_row, mi_col, ref_frame, blk_row, blk_col, gm_mv_candidates, refmv_count, - ref_mv_stack, mode_context); + ref_mv_stack, ref_mv_weight, mode_context); if (blk_row == 0 && blk_col == 0) is_available = ret; } } @@ -597,7 +606,8 @@ static void setup_ref_mv_list( if (!check_sb_border(mi_row, mi_col, blk_row, blk_col)) continue; add_tpl_ref_mv(cm, xd, mi_row, mi_col, ref_frame, blk_row, blk_col, - gm_mv_candidates, refmv_count, ref_mv_stack, mode_context); + gm_mv_candidates, refmv_count, ref_mv_stack, ref_mv_weight, + mode_context); } } @@ -605,8 +615,8 @@ static void setup_ref_mv_list( // Scan the second outer area. scan_blk_mbmi(cm, xd, mi_row, mi_col, rf, -1, -1, ref_mv_stack[ref_frame], - &row_match_count, &dummy_newmv_count, gm_mv_candidates, - &refmv_count[ref_frame]); + ref_mv_weight[ref_frame], &row_match_count, &dummy_newmv_count, + gm_mv_candidates, &refmv_count[ref_frame]); for (int idx = 2; idx <= MVREF_ROW_COLS; ++idx) { const int row_offset = -(idx << 1) + 1 + row_adj; @@ -615,16 +625,18 @@ static void setup_ref_mv_list( if (abs(row_offset) <= abs(max_row_offset) && abs(row_offset) > processed_rows) scan_row_mbmi(cm, xd, mi_row, mi_col, rf, row_offset, - ref_mv_stack[ref_frame], &refmv_count[ref_frame], - &row_match_count, &dummy_newmv_count, gm_mv_candidates, - max_row_offset, &processed_rows); + ref_mv_stack[ref_frame], ref_mv_weight[ref_frame], + &refmv_count[ref_frame], &row_match_count, + &dummy_newmv_count, gm_mv_candidates, max_row_offset, + &processed_rows); if (abs(col_offset) <= abs(max_col_offset) && abs(col_offset) > processed_cols) scan_col_mbmi(cm, xd, mi_row, mi_col, rf, col_offset, - ref_mv_stack[ref_frame], &refmv_count[ref_frame], - &col_match_count, &dummy_newmv_count, gm_mv_candidates, - max_col_offset, &processed_cols); + ref_mv_stack[ref_frame], ref_mv_weight[ref_frame], + &refmv_count[ref_frame], &col_match_count, + &dummy_newmv_count, gm_mv_candidates, max_col_offset, + &processed_cols); } const uint8_t ref_match_count = (row_match_count > 0) + (col_match_count > 0); @@ -661,11 +673,13 @@ static void setup_ref_mv_list( while (len > 0) { int nr_len = 0; for (int idx = 1; idx < len; ++idx) { - if (ref_mv_stack[ref_frame][idx - 1].weight < - ref_mv_stack[ref_frame][idx].weight) { + if (ref_mv_weight[ref_frame][idx - 1] < ref_mv_weight[ref_frame][idx]) { CANDIDATE_MV tmp_mv = ref_mv_stack[ref_frame][idx - 1]; + uint16_t tmp_ref_mv_weight = ref_mv_weight[ref_frame][idx - 1]; ref_mv_stack[ref_frame][idx - 1] = ref_mv_stack[ref_frame][idx]; ref_mv_stack[ref_frame][idx] = tmp_mv; + ref_mv_weight[ref_frame][idx - 1] = ref_mv_weight[ref_frame][idx]; + ref_mv_weight[ref_frame][idx] = tmp_ref_mv_weight; nr_len = idx; } } @@ -676,11 +690,13 @@ static void setup_ref_mv_list( while (len > nearest_refmv_count) { int nr_len = nearest_refmv_count; for (int idx = nearest_refmv_count + 1; idx < len; ++idx) { - if (ref_mv_stack[ref_frame][idx - 1].weight < - ref_mv_stack[ref_frame][idx].weight) { + if (ref_mv_weight[ref_frame][idx - 1] < ref_mv_weight[ref_frame][idx]) { CANDIDATE_MV tmp_mv = ref_mv_stack[ref_frame][idx - 1]; + uint16_t tmp_ref_mv_weight = ref_mv_weight[ref_frame][idx - 1]; ref_mv_stack[ref_frame][idx - 1] = ref_mv_stack[ref_frame][idx]; ref_mv_stack[ref_frame][idx] = tmp_mv; + ref_mv_weight[ref_frame][idx - 1] = ref_mv_weight[ref_frame][idx]; + ref_mv_weight[ref_frame][idx] = tmp_ref_mv_weight; nr_len = idx; } } @@ -745,7 +761,7 @@ static void setup_ref_mv_list( ref_mv_stack[ref_frame][refmv_count[ref_frame]].comp_mv = comp_list[0][1]; } - ref_mv_stack[ref_frame][refmv_count[ref_frame]].weight = 2; + ref_mv_weight[ref_frame][refmv_count[ref_frame]] = 2; ++refmv_count[ref_frame]; } else { for (int idx = 0; idx < MAX_MV_REF_CANDIDATES; ++idx) { @@ -753,7 +769,7 @@ static void setup_ref_mv_list( comp_list[idx][0]; ref_mv_stack[ref_frame][refmv_count[ref_frame]].comp_mv = comp_list[idx][1]; - ref_mv_stack[ref_frame][refmv_count[ref_frame]].weight = 2; + ref_mv_weight[ref_frame][refmv_count[ref_frame]] = 2; ++refmv_count[ref_frame]; } } @@ -779,7 +795,7 @@ static void setup_ref_mv_list( refmv_count[ref_frame] < MAX_MV_REF_CANDIDATES;) { const MB_MODE_INFO *const candidate = xd->mi[-xd->mi_stride + idx]; process_single_ref_mv_candidate(candidate, cm, ref_frame, refmv_count, - ref_mv_stack); + ref_mv_stack, ref_mv_weight); idx += mi_size_wide[candidate->sb_type]; } @@ -787,7 +803,7 @@ static void setup_ref_mv_list( refmv_count[ref_frame] < MAX_MV_REF_CANDIDATES;) { const MB_MODE_INFO *const candidate = xd->mi[idx * xd->mi_stride - 1]; process_single_ref_mv_candidate(candidate, cm, ref_frame, refmv_count, - ref_mv_stack); + ref_mv_stack, ref_mv_weight); idx += mi_size_high[candidate->sb_type]; } @@ -813,6 +829,7 @@ void av1_find_mv_refs(const AV1_COMMON *cm, const MACROBLOCKD *xd, MB_MODE_INFO *mi, MV_REFERENCE_FRAME ref_frame, uint8_t ref_mv_count[MODE_CTX_REF_FRAMES], CANDIDATE_MV ref_mv_stack[][MAX_REF_MV_STACK_SIZE], + uint16_t ref_mv_weight[][MAX_REF_MV_STACK_SIZE], int_mv mv_ref_list[][MAX_MV_REF_CANDIDATES], int_mv *global_mvs, int mi_row, int mi_col, int16_t *mode_context) { @@ -848,8 +865,9 @@ void av1_find_mv_refs(const AV1_COMMON *cm, const MACROBLOCKD *xd, zeromv[0].as_int = zeromv[1].as_int = 0; } - setup_ref_mv_list(cm, xd, ref_frame, ref_mv_count, ref_mv_stack, mv_ref_list, - zeromv, mi_row, mi_col, mode_context); + setup_ref_mv_list(cm, xd, ref_frame, ref_mv_count, ref_mv_stack, + ref_mv_weight, mv_ref_list, zeromv, mi_row, mi_col, + mode_context); } void av1_find_best_ref_mvs(int allow_hp, int_mv *mvlist, int_mv *nearest_mv, diff --git a/third_party/aom/av1/common/mvref_common.h b/third_party/aom/av1/common/mvref_common.h index 0aa9d38931..4b535a53ca 100644 --- a/third_party/aom/av1/common/mvref_common.h +++ b/third_party/aom/av1/common/mvref_common.h @@ -189,18 +189,17 @@ static INLINE int16_t av1_mode_context_analyzer( return comp_ctx; } -static INLINE uint8_t av1_drl_ctx(const CANDIDATE_MV *ref_mv_stack, - int ref_idx) { - if (ref_mv_stack[ref_idx].weight >= REF_CAT_LEVEL && - ref_mv_stack[ref_idx + 1].weight >= REF_CAT_LEVEL) +static INLINE uint8_t av1_drl_ctx(const uint16_t *ref_mv_weight, int ref_idx) { + if (ref_mv_weight[ref_idx] >= REF_CAT_LEVEL && + ref_mv_weight[ref_idx + 1] >= REF_CAT_LEVEL) return 0; - if (ref_mv_stack[ref_idx].weight >= REF_CAT_LEVEL && - ref_mv_stack[ref_idx + 1].weight < REF_CAT_LEVEL) + if (ref_mv_weight[ref_idx] >= REF_CAT_LEVEL && + ref_mv_weight[ref_idx + 1] < REF_CAT_LEVEL) return 1; - if (ref_mv_stack[ref_idx].weight < REF_CAT_LEVEL && - ref_mv_stack[ref_idx + 1].weight < REF_CAT_LEVEL) + if (ref_mv_weight[ref_idx] < REF_CAT_LEVEL && + ref_mv_weight[ref_idx + 1] < REF_CAT_LEVEL) return 2; return 0; @@ -251,6 +250,7 @@ void av1_find_mv_refs(const AV1_COMMON *cm, const MACROBLOCKD *xd, MB_MODE_INFO *mi, MV_REFERENCE_FRAME ref_frame, uint8_t ref_mv_count[MODE_CTX_REF_FRAMES], CANDIDATE_MV ref_mv_stack[][MAX_REF_MV_STACK_SIZE], + uint16_t ref_mv_weight[][MAX_REF_MV_STACK_SIZE], int_mv mv_ref_list[][MAX_MV_REF_CANDIDATES], int_mv *global_mvs, int mi_row, int mi_col, int16_t *mode_context); diff --git a/third_party/aom/av1/common/onyxc_int.h b/third_party/aom/av1/common/onyxc_int.h index 8117dfc7cf..e6c30c67ec 100644 --- a/third_party/aom/av1/common/onyxc_int.h +++ b/third_party/aom/av1/common/onyxc_int.h @@ -917,6 +917,7 @@ static INLINE void update_partition_context(MACROBLOCKD *xd, int mi_row, static INLINE int is_chroma_reference(int mi_row, int mi_col, BLOCK_SIZE bsize, int subsampling_x, int subsampling_y) { + assert(bsize < BLOCK_SIZES_ALL); const int bw = mi_size_wide[bsize]; const int bh = mi_size_high[bsize]; int ref_pos = ((mi_row & 0x01) || !(bh & 0x01) || !subsampling_y) && @@ -926,6 +927,8 @@ static INLINE int is_chroma_reference(int mi_row, int mi_col, BLOCK_SIZE bsize, static INLINE BLOCK_SIZE scale_chroma_bsize(BLOCK_SIZE bsize, int subsampling_x, int subsampling_y) { + assert(subsampling_x >= 0 && subsampling_x < 2); + assert(subsampling_y >= 0 && subsampling_y < 2); BLOCK_SIZE bs = bsize; switch (bsize) { case BLOCK_4X4: @@ -1076,6 +1079,7 @@ static INLINE int partition_cdf_length(BLOCK_SIZE bsize) { static INLINE int max_block_wide(const MACROBLOCKD *xd, BLOCK_SIZE bsize, int plane) { + assert(bsize < BLOCK_SIZES_ALL); int max_blocks_wide = block_size_wide[bsize]; const struct macroblockd_plane *const pd = &xd->plane[plane]; diff --git a/third_party/aom/av1/common/pred_common.h b/third_party/aom/av1/common/pred_common.h index d9b30a97ad..a681e4e0d6 100644 --- a/third_party/aom/av1/common/pred_common.h +++ b/third_party/aom/av1/common/pred_common.h @@ -182,6 +182,7 @@ int av1_get_palette_cache(const MACROBLOCKD *const xd, int plane, uint16_t *cache); static INLINE int av1_get_palette_bsize_ctx(BLOCK_SIZE bsize) { + assert(bsize < BLOCK_SIZES_ALL); return num_pels_log2_lookup[bsize] - num_pels_log2_lookup[BLOCK_8X8]; } diff --git a/third_party/aom/av1/common/reconinter.c b/third_party/aom/av1/common/reconinter.c index ea351cf698..69da102fd6 100644 --- a/third_party/aom/av1/common/reconinter.c +++ b/third_party/aom/av1/common/reconinter.c @@ -1140,6 +1140,7 @@ void av1_build_interintra_predictors_sbp(const AV1_COMMON *cm, MACROBLOCKD *xd, uint8_t *pred, int stride, const BUFFER_SET *ctx, int plane, BLOCK_SIZE bsize) { + assert(bsize < BLOCK_SIZES_ALL); if (is_cur_buf_hbd(xd)) { DECLARE_ALIGNED(16, uint16_t, intrapredictor[MAX_SB_SQUARE]); av1_build_intra_predictors_for_interintra( diff --git a/third_party/aom/av1/common/reconintra.c b/third_party/aom/av1/common/reconintra.c index 559e49921b..128baa21c3 100644 --- a/third_party/aom/av1/common/reconintra.c +++ b/third_party/aom/av1/common/reconintra.c @@ -1015,9 +1015,9 @@ static int intra_edge_filter_strength(int bs0, int bs1, int delta, int type) { void av1_filter_intra_edge_c(uint8_t *p, int sz, int strength) { if (!strength) return; - const int kernel[INTRA_EDGE_FILT][INTRA_EDGE_TAPS] = { - { 0, 4, 8, 4, 0 }, { 0, 5, 6, 5, 0 }, { 2, 4, 4, 4, 2 } - }; + const int kernel[INTRA_EDGE_FILT][INTRA_EDGE_TAPS] = { { 0, 4, 8, 4, 0 }, + { 0, 5, 6, 5, 0 }, + { 2, 4, 4, 4, 2 } }; const int filt = strength - 1; uint8_t edge[129]; @@ -1048,9 +1048,9 @@ static void filter_intra_edge_corner(uint8_t *p_above, uint8_t *p_left) { void av1_filter_intra_edge_high_c(uint16_t *p, int sz, int strength) { if (!strength) return; - const int kernel[INTRA_EDGE_FILT][INTRA_EDGE_TAPS] = { - { 0, 4, 8, 4, 0 }, { 0, 5, 6, 5, 0 }, { 2, 4, 4, 4, 2 } - }; + const int kernel[INTRA_EDGE_FILT][INTRA_EDGE_TAPS] = { { 0, 4, 8, 4, 0 }, + { 0, 5, 6, 5, 0 }, + { 2, 4, 4, 4, 2 } }; const int filt = strength - 1; uint16_t edge[129]; diff --git a/third_party/aom/av1/common/resize.c b/third_party/aom/av1/common/resize.c index 8b24ed01b9..d668eaea56 100644 --- a/third_party/aom/av1/common/resize.c +++ b/third_party/aom/av1/common/resize.c @@ -431,7 +431,6 @@ static int32_t get_upscale_convolve_x0(int in_length, int out_length, return (int32_t)((uint32_t)x0 & RS_SCALE_SUBPEL_MASK); } -#ifndef __clang_analyzer__ static void down2_symeven(const uint8_t *const input, int length, uint8_t *output) { // Actual filter len = 2 * filter_len_half. @@ -486,7 +485,6 @@ static void down2_symeven(const uint8_t *const input, int length, } } } -#endif static void down2_symodd(const uint8_t *const input, int length, uint8_t *output) { @@ -852,7 +850,6 @@ static void highbd_interpolate(const uint16_t *const input, int in_length, &interp_filters[0][0], SUBPEL_TAPS); } -#ifndef __clang_analyzer__ static void highbd_down2_symeven(const uint16_t *const input, int length, uint16_t *output, int bd) { // Actual filter len = 2 * filter_len_half. @@ -960,7 +957,6 @@ static void highbd_down2_symodd(const uint16_t *const input, int length, } } } -#endif static void highbd_resize_multistep(const uint16_t *const input, int length, uint16_t *output, int olength, diff --git a/third_party/aom/av1/common/restoration.c b/third_party/aom/av1/common/restoration.c index 9e472b8a6e..69d898f1f6 100644 --- a/third_party/aom/av1/common/restoration.c +++ b/third_party/aom/av1/common/restoration.c @@ -1146,7 +1146,7 @@ void av1_loop_restoration_copy_planes(AV1LrStruct *loop_rest_ctxt, static const copy_fun copy_funs[3] = { aom_yv12_partial_coloc_copy_y, aom_yv12_partial_coloc_copy_u, aom_yv12_partial_coloc_copy_v }; - + assert(num_planes <= 3); for (int plane = 0; plane < num_planes; ++plane) { if (cm->rst_info[plane].frame_restoration_type == RESTORE_NONE) continue; AV1PixelRect tile_rect = loop_rest_ctxt->ctxt[plane].tile_rect; diff --git a/third_party/aom/av1/common/restoration.h b/third_party/aom/av1/common/restoration.h index 6d6ba37ec7..81ad3ba423 100644 --- a/third_party/aom/av1/common/restoration.h +++ b/third_party/aom/av1/common/restoration.h @@ -122,6 +122,7 @@ extern "C" { // If WIENER_WIN_CHROMA == WIENER_WIN - 2, that implies 5x5 filters are used for // chroma. To use 7x7 for chroma set WIENER_WIN_CHROMA to WIENER_WIN. #define WIENER_WIN_CHROMA (WIENER_WIN - 2) +#define WIENER_WIN_REDUCED (WIENER_WIN - 2) #define WIENER_WIN2_CHROMA ((WIENER_WIN_CHROMA) * (WIENER_WIN_CHROMA)) #define WIENER_FILT_PREC_BITS 7 diff --git a/third_party/aom/av1/common/scale.h b/third_party/aom/av1/common/scale.h index 748e958c35..16b40bde8f 100644 --- a/third_party/aom/av1/common/scale.h +++ b/third_party/aom/av1/common/scale.h @@ -45,11 +45,13 @@ void av1_setup_scale_factors_for_frame(struct scale_factors *sf, int other_w, int other_h, int this_w, int this_h); static INLINE int av1_is_valid_scale(const struct scale_factors *sf) { + assert(sf != NULL); return sf->x_scale_fp != REF_INVALID_SCALE && sf->y_scale_fp != REF_INVALID_SCALE; } static INLINE int av1_is_scaled(const struct scale_factors *sf) { + assert(sf != NULL); return av1_is_valid_scale(sf) && (sf->x_scale_fp != REF_NO_SCALE || sf->y_scale_fp != REF_NO_SCALE); } diff --git a/third_party/aom/av1/common/scan.c b/third_party/aom/av1/common/scan.c index 31a787b537..3ba42f070f 100644 --- a/third_party/aom/av1/common/scan.c +++ b/third_party/aom/av1/common/scan.c @@ -14,9 +14,9 @@ #include "av1/common/common_data.h" #include "av1/common/scan.h" -DECLARE_ALIGNED(16, static const int16_t, default_scan_4x4[16]) = { - 0, 1, 4, 8, 5, 2, 3, 6, 9, 12, 13, 10, 7, 11, 14, 15 -}; +DECLARE_ALIGNED(16, static const int16_t, + default_scan_4x4[16]) = { 0, 1, 4, 8, 5, 2, 3, 6, + 9, 12, 13, 10, 7, 11, 14, 15 }; DECLARE_ALIGNED(16, static const int16_t, mcol_scan_4x4[16]) = { 0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15, @@ -2376,9 +2376,9 @@ DECLARE_ALIGNED(16, static const int16_t, 990, 1021, 991, 1022, 0, 0 }; -DECLARE_ALIGNED(16, static const int16_t, av1_default_iscan_4x4[16]) = { - 0, 1, 5, 6, 2, 4, 7, 12, 3, 8, 11, 13, 9, 10, 14, 15 -}; +DECLARE_ALIGNED(16, static const int16_t, + av1_default_iscan_4x4[16]) = { 0, 1, 5, 6, 2, 4, 7, 12, + 3, 8, 11, 13, 9, 10, 14, 15 }; DECLARE_ALIGNED(16, static const int16_t, av1_mcol_iscan_4x4[16]) = { 0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15, diff --git a/third_party/aom/av1/common/token_cdfs.h b/third_party/aom/av1/common/token_cdfs.h index 53e956450b..f1edda58d7 100644 --- a/third_party/aom/av1/common/token_cdfs.h +++ b/third_party/aom/av1/common/token_cdfs.h @@ -1707,1687 +1707,1687 @@ static const aom_cdf_prob av1_default_coeff_lps_multi_cdfs static const aom_cdf_prob av1_default_coeff_base_multi_cdfs [TOKEN_CDF_Q_CTXS][TX_SIZES][PLANE_TYPES][SIG_COEF_CONTEXTS] - [CDF_SIZE(NUM_BASE_LEVELS + 2)] = - { { { { { AOM_CDF4(4034, 8930, 12727) }, - { AOM_CDF4(18082, 29741, 31877) }, - { AOM_CDF4(12596, 26124, 30493) }, - { AOM_CDF4(9446, 21118, 27005) }, - { AOM_CDF4(6308, 15141, 21279) }, - { AOM_CDF4(2463, 6357, 9783) }, - { AOM_CDF4(20667, 30546, 31929) }, - { AOM_CDF4(13043, 26123, 30134) }, - { AOM_CDF4(8151, 18757, 24778) }, - { AOM_CDF4(5255, 12839, 18632) }, - { AOM_CDF4(2820, 7206, 11161) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(15736, 27553, 30604) }, - { AOM_CDF4(11210, 23794, 28787) }, - { AOM_CDF4(5947, 13874, 19701) }, - { AOM_CDF4(4215, 9323, 13891) }, - { AOM_CDF4(2833, 6462, 10059) }, - { AOM_CDF4(19605, 30393, 31582) }, - { AOM_CDF4(13523, 26252, 30248) }, - { AOM_CDF4(8446, 18622, 24512) }, - { AOM_CDF4(3818, 10343, 15974) }, - { AOM_CDF4(1481, 4117, 6796) }, - { AOM_CDF4(22649, 31302, 32190) }, - { AOM_CDF4(14829, 27127, 30449) }, - { AOM_CDF4(8313, 17702, 23304) }, - { AOM_CDF4(3022, 8301, 12786) }, - { AOM_CDF4(1536, 4412, 7184) }, - { AOM_CDF4(22354, 29774, 31372) }, - { AOM_CDF4(14723, 25472, 29214) }, - { AOM_CDF4(6673, 13745, 18662) }, - { AOM_CDF4(2068, 5766, 9322) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) } }, - { { AOM_CDF4(6302, 16444, 21761) }, - { AOM_CDF4(23040, 31538, 32475) }, - { AOM_CDF4(15196, 28452, 31496) }, - { AOM_CDF4(10020, 22946, 28514) }, - { AOM_CDF4(6533, 16862, 23501) }, - { AOM_CDF4(3538, 9816, 15076) }, - { AOM_CDF4(24444, 31875, 32525) }, - { AOM_CDF4(15881, 28924, 31635) }, - { AOM_CDF4(9922, 22873, 28466) }, - { AOM_CDF4(6527, 16966, 23691) }, - { AOM_CDF4(4114, 11303, 17220) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(20201, 30770, 32209) }, - { AOM_CDF4(14754, 28071, 31258) }, - { AOM_CDF4(8378, 20186, 26517) }, - { AOM_CDF4(5916, 15299, 21978) }, - { AOM_CDF4(4268, 11583, 17901) }, - { AOM_CDF4(24361, 32025, 32581) }, - { AOM_CDF4(18673, 30105, 31943) }, - { AOM_CDF4(10196, 22244, 27576) }, - { AOM_CDF4(5495, 14349, 20417) }, - { AOM_CDF4(2676, 7415, 11498) }, - { AOM_CDF4(24678, 31958, 32585) }, - { AOM_CDF4(18629, 29906, 31831) }, - { AOM_CDF4(9364, 20724, 26315) }, - { AOM_CDF4(4641, 12318, 18094) }, - { AOM_CDF4(2758, 7387, 11579) }, - { AOM_CDF4(25433, 31842, 32469) }, - { AOM_CDF4(18795, 29289, 31411) }, - { AOM_CDF4(7644, 17584, 23592) }, - { AOM_CDF4(3408, 9014, 15047) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) } } }, - { { { AOM_CDF4(4536, 10072, 14001) }, - { AOM_CDF4(25459, 31416, 32206) }, - { AOM_CDF4(16605, 28048, 30818) }, - { AOM_CDF4(11008, 22857, 27719) }, - { AOM_CDF4(6915, 16268, 22315) }, - { AOM_CDF4(2625, 6812, 10537) }, - { AOM_CDF4(24257, 31788, 32499) }, - { AOM_CDF4(16880, 29454, 31879) }, - { AOM_CDF4(11958, 25054, 29778) }, - { AOM_CDF4(7916, 18718, 25084) }, - { AOM_CDF4(3383, 8777, 13446) }, - { AOM_CDF4(22720, 31603, 32393) }, - { AOM_CDF4(14960, 28125, 31335) }, - { AOM_CDF4(9731, 22210, 27928) }, - { AOM_CDF4(6304, 15832, 22277) }, - { AOM_CDF4(2910, 7818, 12166) }, - { AOM_CDF4(20375, 30627, 32131) }, - { AOM_CDF4(13904, 27284, 30887) }, - { AOM_CDF4(9368, 21558, 27144) }, - { AOM_CDF4(5937, 14966, 21119) }, - { AOM_CDF4(2667, 7225, 11319) }, - { AOM_CDF4(23970, 31470, 32378) }, - { AOM_CDF4(17173, 29734, 32018) }, - { AOM_CDF4(12795, 25441, 29965) }, - { AOM_CDF4(8981, 19680, 25893) }, - { AOM_CDF4(4728, 11372, 16902) }, - { AOM_CDF4(24287, 31797, 32439) }, - { AOM_CDF4(16703, 29145, 31696) }, - { AOM_CDF4(10833, 23554, 28725) }, - { AOM_CDF4(6468, 16566, 23057) }, - { AOM_CDF4(2415, 6562, 10278) }, - { AOM_CDF4(26610, 32395, 32659) }, - { AOM_CDF4(18590, 30498, 32117) }, - { AOM_CDF4(12420, 25756, 29950) }, - { AOM_CDF4(7639, 18746, 24710) }, - { AOM_CDF4(3001, 8086, 12347) }, - { AOM_CDF4(25076, 32064, 32580) }, - { AOM_CDF4(17946, 30128, 32028) }, - { AOM_CDF4(12024, 24985, 29378) }, - { AOM_CDF4(7517, 18390, 24304) }, - { AOM_CDF4(3243, 8781, 13331) }, - { AOM_CDF4(8192, 16384, 24576) } }, - { { AOM_CDF4(6037, 16771, 21957) }, - { AOM_CDF4(24774, 31704, 32426) }, - { AOM_CDF4(16830, 28589, 31056) }, - { AOM_CDF4(10602, 22828, 27760) }, - { AOM_CDF4(6733, 16829, 23071) }, - { AOM_CDF4(3250, 8914, 13556) }, - { AOM_CDF4(25582, 32220, 32668) }, - { AOM_CDF4(18659, 30342, 32223) }, - { AOM_CDF4(12546, 26149, 30515) }, - { AOM_CDF4(8420, 20451, 26801) }, - { AOM_CDF4(4636, 12420, 18344) }, - { AOM_CDF4(27581, 32362, 32639) }, - { AOM_CDF4(18987, 30083, 31978) }, - { AOM_CDF4(11327, 24248, 29084) }, - { AOM_CDF4(7264, 17719, 24120) }, - { AOM_CDF4(3995, 10768, 16169) }, - { AOM_CDF4(25893, 31831, 32487) }, - { AOM_CDF4(16577, 28587, 31379) }, - { AOM_CDF4(10189, 22748, 28182) }, - { AOM_CDF4(6832, 17094, 23556) }, - { AOM_CDF4(3708, 10110, 15334) }, - { AOM_CDF4(25904, 32282, 32656) }, - { AOM_CDF4(19721, 30792, 32276) }, - { AOM_CDF4(12819, 26243, 30411) }, - { AOM_CDF4(8572, 20614, 26891) }, - { AOM_CDF4(5364, 14059, 20467) }, - { AOM_CDF4(26580, 32438, 32677) }, - { AOM_CDF4(20852, 31225, 32340) }, - { AOM_CDF4(12435, 25700, 29967) }, - { AOM_CDF4(8691, 20825, 26976) }, - { AOM_CDF4(4446, 12209, 17269) }, - { AOM_CDF4(27350, 32429, 32696) }, - { AOM_CDF4(21372, 30977, 32272) }, - { AOM_CDF4(12673, 25270, 29853) }, - { AOM_CDF4(9208, 20925, 26640) }, - { AOM_CDF4(5018, 13351, 18732) }, - { AOM_CDF4(27351, 32479, 32713) }, - { AOM_CDF4(21398, 31209, 32387) }, - { AOM_CDF4(12162, 25047, 29842) }, - { AOM_CDF4(7896, 18691, 25319) }, - { AOM_CDF4(4670, 12882, 18881) }, - { AOM_CDF4(8192, 16384, 24576) } } }, - { { { AOM_CDF4(5487, 10460, 13708) }, - { AOM_CDF4(21597, 28303, 30674) }, - { AOM_CDF4(11037, 21953, 26476) }, - { AOM_CDF4(8147, 17962, 22952) }, - { AOM_CDF4(5242, 13061, 18532) }, - { AOM_CDF4(1889, 5208, 8182) }, - { AOM_CDF4(26774, 32133, 32590) }, - { AOM_CDF4(17844, 29564, 31767) }, - { AOM_CDF4(11690, 24438, 29171) }, - { AOM_CDF4(7542, 18215, 24459) }, - { AOM_CDF4(2993, 8050, 12319) }, - { AOM_CDF4(28023, 32328, 32591) }, - { AOM_CDF4(18651, 30126, 31954) }, - { AOM_CDF4(12164, 25146, 29589) }, - { AOM_CDF4(7762, 18530, 24771) }, - { AOM_CDF4(3492, 9183, 13920) }, - { AOM_CDF4(27591, 32008, 32491) }, - { AOM_CDF4(17149, 28853, 31510) }, - { AOM_CDF4(11485, 24003, 28860) }, - { AOM_CDF4(7697, 18086, 24210) }, - { AOM_CDF4(3075, 7999, 12218) }, - { AOM_CDF4(28268, 32482, 32654) }, - { AOM_CDF4(19631, 31051, 32404) }, - { AOM_CDF4(13860, 27260, 31020) }, - { AOM_CDF4(9605, 21613, 27594) }, - { AOM_CDF4(4876, 12162, 17908) }, - { AOM_CDF4(27248, 32316, 32576) }, - { AOM_CDF4(18955, 30457, 32075) }, - { AOM_CDF4(11824, 23997, 28795) }, - { AOM_CDF4(7346, 18196, 24647) }, - { AOM_CDF4(3403, 9247, 14111) }, - { AOM_CDF4(29711, 32655, 32735) }, - { AOM_CDF4(21169, 31394, 32417) }, - { AOM_CDF4(13487, 27198, 30957) }, - { AOM_CDF4(8828, 21683, 27614) }, - { AOM_CDF4(4270, 11451, 17038) }, - { AOM_CDF4(28708, 32578, 32731) }, - { AOM_CDF4(20120, 31241, 32482) }, - { AOM_CDF4(13692, 27550, 31321) }, - { AOM_CDF4(9418, 22514, 28439) }, - { AOM_CDF4(4999, 13283, 19462) }, - { AOM_CDF4(8192, 16384, 24576) } }, - { { AOM_CDF4(5673, 14302, 19711) }, - { AOM_CDF4(26251, 30701, 31834) }, - { AOM_CDF4(12782, 23783, 27803) }, - { AOM_CDF4(9127, 20657, 25808) }, - { AOM_CDF4(6368, 16208, 21462) }, - { AOM_CDF4(2465, 7177, 10822) }, - { AOM_CDF4(29961, 32563, 32719) }, - { AOM_CDF4(18318, 29891, 31949) }, - { AOM_CDF4(11361, 24514, 29357) }, - { AOM_CDF4(7900, 19603, 25607) }, - { AOM_CDF4(4002, 10590, 15546) }, - { AOM_CDF4(29637, 32310, 32595) }, - { AOM_CDF4(18296, 29913, 31809) }, - { AOM_CDF4(10144, 21515, 26871) }, - { AOM_CDF4(5358, 14322, 20394) }, - { AOM_CDF4(3067, 8362, 13346) }, - { AOM_CDF4(28652, 32470, 32676) }, - { AOM_CDF4(17538, 30771, 32209) }, - { AOM_CDF4(13924, 26882, 30494) }, - { AOM_CDF4(10496, 22837, 27869) }, - { AOM_CDF4(7236, 16396, 21621) }, - { AOM_CDF4(30743, 32687, 32746) }, - { AOM_CDF4(23006, 31676, 32489) }, - { AOM_CDF4(14494, 27828, 31120) }, - { AOM_CDF4(10174, 22801, 28352) }, - { AOM_CDF4(6242, 15281, 21043) }, - { AOM_CDF4(25817, 32243, 32720) }, - { AOM_CDF4(18618, 31367, 32325) }, - { AOM_CDF4(13997, 28318, 31878) }, - { AOM_CDF4(12255, 26534, 31383) }, - { AOM_CDF4(9561, 21588, 28450) }, - { AOM_CDF4(28188, 32635, 32724) }, - { AOM_CDF4(22060, 32365, 32728) }, - { AOM_CDF4(18102, 30690, 32528) }, - { AOM_CDF4(14196, 28864, 31999) }, - { AOM_CDF4(12262, 25792, 30865) }, - { AOM_CDF4(24176, 32109, 32628) }, - { AOM_CDF4(18280, 29681, 31963) }, - { AOM_CDF4(10205, 23703, 29664) }, - { AOM_CDF4(7889, 20025, 27676) }, - { AOM_CDF4(6060, 16743, 23970) }, - { AOM_CDF4(8192, 16384, 24576) } } }, - { { { AOM_CDF4(5141, 7096, 8260) }, - { AOM_CDF4(27186, 29022, 29789) }, - { AOM_CDF4(6668, 12568, 15682) }, - { AOM_CDF4(2172, 6181, 8638) }, - { AOM_CDF4(1126, 3379, 4531) }, - { AOM_CDF4(443, 1361, 2254) }, - { AOM_CDF4(26083, 31153, 32436) }, - { AOM_CDF4(13486, 24603, 28483) }, - { AOM_CDF4(6508, 14840, 19910) }, - { AOM_CDF4(3386, 8800, 13286) }, - { AOM_CDF4(1530, 4322, 7054) }, - { AOM_CDF4(29639, 32080, 32548) }, - { AOM_CDF4(15897, 27552, 30290) }, - { AOM_CDF4(8588, 20047, 25383) }, - { AOM_CDF4(4889, 13339, 19269) }, - { AOM_CDF4(2240, 6871, 10498) }, - { AOM_CDF4(28165, 32197, 32517) }, - { AOM_CDF4(20735, 30427, 31568) }, - { AOM_CDF4(14325, 24671, 27692) }, - { AOM_CDF4(5119, 12554, 17805) }, - { AOM_CDF4(1810, 5441, 8261) }, - { AOM_CDF4(31212, 32724, 32748) }, - { AOM_CDF4(23352, 31766, 32545) }, - { AOM_CDF4(14669, 27570, 31059) }, - { AOM_CDF4(8492, 20894, 27272) }, - { AOM_CDF4(3644, 10194, 15204) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) } }, - { { AOM_CDF4(2461, 7013, 9371) }, - { AOM_CDF4(24749, 29600, 30986) }, - { AOM_CDF4(9466, 19037, 22417) }, - { AOM_CDF4(3584, 9280, 14400) }, - { AOM_CDF4(1505, 3929, 5433) }, - { AOM_CDF4(677, 1500, 2736) }, - { AOM_CDF4(23987, 30702, 32117) }, - { AOM_CDF4(13554, 24571, 29263) }, - { AOM_CDF4(6211, 14556, 21155) }, - { AOM_CDF4(3135, 10972, 15625) }, - { AOM_CDF4(2435, 7127, 11427) }, - { AOM_CDF4(31300, 32532, 32550) }, - { AOM_CDF4(14757, 30365, 31954) }, - { AOM_CDF4(4405, 11612, 18553) }, - { AOM_CDF4(580, 4132, 7322) }, - { AOM_CDF4(1695, 10169, 14124) }, - { AOM_CDF4(30008, 32282, 32591) }, - { AOM_CDF4(19244, 30108, 31748) }, - { AOM_CDF4(11180, 24158, 29555) }, - { AOM_CDF4(5650, 14972, 19209) }, - { AOM_CDF4(2114, 5109, 8456) }, - { AOM_CDF4(31856, 32716, 32748) }, - { AOM_CDF4(23012, 31664, 32572) }, - { AOM_CDF4(13694, 26656, 30636) }, - { AOM_CDF4(8142, 19508, 26093) }, - { AOM_CDF4(4253, 10955, 16724) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) } } }, - { { { AOM_CDF4(601, 983, 1311) }, - { AOM_CDF4(18725, 23406, 28087) }, - { AOM_CDF4(5461, 8192, 10923) }, - { AOM_CDF4(3781, 15124, 21425) }, - { AOM_CDF4(2587, 7761, 12072) }, - { AOM_CDF4(106, 458, 810) }, - { AOM_CDF4(22282, 29710, 31894) }, - { AOM_CDF4(8508, 20926, 25984) }, - { AOM_CDF4(3726, 12713, 18083) }, - { AOM_CDF4(1620, 7112, 10893) }, - { AOM_CDF4(729, 2236, 3495) }, - { AOM_CDF4(30163, 32474, 32684) }, - { AOM_CDF4(18304, 30464, 32000) }, - { AOM_CDF4(11443, 26526, 29647) }, - { AOM_CDF4(6007, 15292, 21299) }, - { AOM_CDF4(2234, 6703, 8937) }, - { AOM_CDF4(30954, 32177, 32571) }, - { AOM_CDF4(17363, 29562, 31076) }, - { AOM_CDF4(9686, 22464, 27410) }, - { AOM_CDF4(8192, 16384, 21390) }, - { AOM_CDF4(1755, 8046, 11264) }, - { AOM_CDF4(31168, 32734, 32748) }, - { AOM_CDF4(22486, 31441, 32471) }, - { AOM_CDF4(12833, 25627, 29738) }, - { AOM_CDF4(6980, 17379, 23122) }, - { AOM_CDF4(3111, 8887, 13479) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) } }, - { { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) } } } }, - { { { { AOM_CDF4(6041, 11854, 15927) }, - { AOM_CDF4(20326, 30905, 32251) }, - { AOM_CDF4(14164, 26831, 30725) }, - { AOM_CDF4(9760, 20647, 26585) }, - { AOM_CDF4(6416, 14953, 21219) }, - { AOM_CDF4(2966, 7151, 10891) }, - { AOM_CDF4(23567, 31374, 32254) }, - { AOM_CDF4(14978, 27416, 30946) }, - { AOM_CDF4(9434, 20225, 26254) }, - { AOM_CDF4(6658, 14558, 20535) }, - { AOM_CDF4(3916, 8677, 12989) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(18088, 29545, 31587) }, - { AOM_CDF4(13062, 25843, 30073) }, - { AOM_CDF4(8940, 16827, 22251) }, - { AOM_CDF4(7654, 13220, 17973) }, - { AOM_CDF4(5733, 10316, 14456) }, - { AOM_CDF4(22879, 31388, 32114) }, - { AOM_CDF4(15215, 27993, 30955) }, - { AOM_CDF4(9397, 19445, 24978) }, - { AOM_CDF4(3442, 9813, 15344) }, - { AOM_CDF4(1368, 3936, 6532) }, - { AOM_CDF4(25494, 32033, 32406) }, - { AOM_CDF4(16772, 27963, 30718) }, - { AOM_CDF4(9419, 18165, 23260) }, - { AOM_CDF4(2677, 7501, 11797) }, - { AOM_CDF4(1516, 4344, 7170) }, - { AOM_CDF4(26556, 31454, 32101) }, - { AOM_CDF4(17128, 27035, 30108) }, - { AOM_CDF4(8324, 15344, 20249) }, - { AOM_CDF4(1903, 5696, 9469) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) } }, - { { AOM_CDF4(8455, 19003, 24368) }, - { AOM_CDF4(23563, 32021, 32604) }, - { AOM_CDF4(16237, 29446, 31935) }, - { AOM_CDF4(10724, 23999, 29358) }, - { AOM_CDF4(6725, 17528, 24416) }, - { AOM_CDF4(3927, 10927, 16825) }, - { AOM_CDF4(26313, 32288, 32634) }, - { AOM_CDF4(17430, 30095, 32095) }, - { AOM_CDF4(11116, 24606, 29679) }, - { AOM_CDF4(7195, 18384, 25269) }, - { AOM_CDF4(4726, 12852, 19315) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(22822, 31648, 32483) }, - { AOM_CDF4(16724, 29633, 31929) }, - { AOM_CDF4(10261, 23033, 28725) }, - { AOM_CDF4(7029, 17840, 24528) }, - { AOM_CDF4(4867, 13886, 21502) }, - { AOM_CDF4(25298, 31892, 32491) }, - { AOM_CDF4(17809, 29330, 31512) }, - { AOM_CDF4(9668, 21329, 26579) }, - { AOM_CDF4(4774, 12956, 18976) }, - { AOM_CDF4(2322, 7030, 11540) }, - { AOM_CDF4(25472, 31920, 32543) }, - { AOM_CDF4(17957, 29387, 31632) }, - { AOM_CDF4(9196, 20593, 26400) }, - { AOM_CDF4(4680, 12705, 19202) }, - { AOM_CDF4(2917, 8456, 13436) }, - { AOM_CDF4(26471, 32059, 32574) }, - { AOM_CDF4(18458, 29783, 31909) }, - { AOM_CDF4(8400, 19464, 25956) }, - { AOM_CDF4(3812, 10973, 17206) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) } } }, - { { { AOM_CDF4(6779, 13743, 17678) }, - { AOM_CDF4(24806, 31797, 32457) }, - { AOM_CDF4(17616, 29047, 31372) }, - { AOM_CDF4(11063, 23175, 28003) }, - { AOM_CDF4(6521, 16110, 22324) }, - { AOM_CDF4(2764, 7504, 11654) }, - { AOM_CDF4(25266, 32367, 32637) }, - { AOM_CDF4(19054, 30553, 32175) }, - { AOM_CDF4(12139, 25212, 29807) }, - { AOM_CDF4(7311, 18162, 24704) }, - { AOM_CDF4(3397, 9164, 14074) }, - { AOM_CDF4(25988, 32208, 32522) }, - { AOM_CDF4(16253, 28912, 31526) }, - { AOM_CDF4(9151, 21387, 27372) }, - { AOM_CDF4(5688, 14915, 21496) }, - { AOM_CDF4(2717, 7627, 12004) }, - { AOM_CDF4(23144, 31855, 32443) }, - { AOM_CDF4(16070, 28491, 31325) }, - { AOM_CDF4(8702, 20467, 26517) }, - { AOM_CDF4(5243, 13956, 20367) }, - { AOM_CDF4(2621, 7335, 11567) }, - { AOM_CDF4(26636, 32340, 32630) }, - { AOM_CDF4(19990, 31050, 32341) }, - { AOM_CDF4(13243, 26105, 30315) }, - { AOM_CDF4(8588, 19521, 25918) }, - { AOM_CDF4(4717, 11585, 17304) }, - { AOM_CDF4(25844, 32292, 32582) }, - { AOM_CDF4(19090, 30635, 32097) }, - { AOM_CDF4(11963, 24546, 28939) }, - { AOM_CDF4(6218, 16087, 22354) }, - { AOM_CDF4(2340, 6608, 10426) }, - { AOM_CDF4(28046, 32576, 32694) }, - { AOM_CDF4(21178, 31313, 32296) }, - { AOM_CDF4(13486, 26184, 29870) }, - { AOM_CDF4(7149, 17871, 23723) }, - { AOM_CDF4(2833, 7958, 12259) }, - { AOM_CDF4(27710, 32528, 32686) }, - { AOM_CDF4(20674, 31076, 32268) }, - { AOM_CDF4(12413, 24955, 29243) }, - { AOM_CDF4(6676, 16927, 23097) }, - { AOM_CDF4(2966, 8333, 12919) }, - { AOM_CDF4(8192, 16384, 24576) } }, - { { AOM_CDF4(8639, 19339, 24429) }, - { AOM_CDF4(24404, 31837, 32525) }, - { AOM_CDF4(16997, 29425, 31784) }, - { AOM_CDF4(11253, 24234, 29149) }, - { AOM_CDF4(6751, 17394, 24028) }, - { AOM_CDF4(3490, 9830, 15191) }, - { AOM_CDF4(26283, 32471, 32714) }, - { AOM_CDF4(19599, 31168, 32442) }, - { AOM_CDF4(13146, 26954, 30893) }, - { AOM_CDF4(8214, 20588, 26890) }, - { AOM_CDF4(4699, 13081, 19300) }, - { AOM_CDF4(28212, 32458, 32669) }, - { AOM_CDF4(18594, 30316, 32100) }, - { AOM_CDF4(11219, 24408, 29234) }, - { AOM_CDF4(6865, 17656, 24149) }, - { AOM_CDF4(3678, 10362, 16006) }, - { AOM_CDF4(25825, 32136, 32616) }, - { AOM_CDF4(17313, 29853, 32021) }, - { AOM_CDF4(11197, 24471, 29472) }, - { AOM_CDF4(6947, 17781, 24405) }, - { AOM_CDF4(3768, 10660, 16261) }, - { AOM_CDF4(27352, 32500, 32706) }, - { AOM_CDF4(20850, 31468, 32469) }, - { AOM_CDF4(14021, 27707, 31133) }, - { AOM_CDF4(8964, 21748, 27838) }, - { AOM_CDF4(5437, 14665, 21187) }, - { AOM_CDF4(26304, 32492, 32698) }, - { AOM_CDF4(20409, 31380, 32385) }, - { AOM_CDF4(13682, 27222, 30632) }, - { AOM_CDF4(8974, 21236, 26685) }, - { AOM_CDF4(4234, 11665, 16934) }, - { AOM_CDF4(26273, 32357, 32711) }, - { AOM_CDF4(20672, 31242, 32441) }, - { AOM_CDF4(14172, 27254, 30902) }, - { AOM_CDF4(9870, 21898, 27275) }, - { AOM_CDF4(5164, 13506, 19270) }, - { AOM_CDF4(26725, 32459, 32728) }, - { AOM_CDF4(20991, 31442, 32527) }, - { AOM_CDF4(13071, 26434, 30811) }, - { AOM_CDF4(8184, 20090, 26742) }, - { AOM_CDF4(4803, 13255, 19895) }, - { AOM_CDF4(8192, 16384, 24576) } } }, - { { { AOM_CDF4(7555, 14942, 18501) }, - { AOM_CDF4(24410, 31178, 32287) }, - { AOM_CDF4(14394, 26738, 30253) }, - { AOM_CDF4(8413, 19554, 25195) }, - { AOM_CDF4(4766, 12924, 18785) }, - { AOM_CDF4(2029, 5806, 9207) }, - { AOM_CDF4(26776, 32364, 32663) }, - { AOM_CDF4(18732, 29967, 31931) }, - { AOM_CDF4(11005, 23786, 28852) }, - { AOM_CDF4(6466, 16909, 23510) }, - { AOM_CDF4(3044, 8638, 13419) }, - { AOM_CDF4(29208, 32582, 32704) }, - { AOM_CDF4(20068, 30857, 32208) }, - { AOM_CDF4(12003, 25085, 29595) }, - { AOM_CDF4(6947, 17750, 24189) }, - { AOM_CDF4(3245, 9103, 14007) }, - { AOM_CDF4(27359, 32465, 32669) }, - { AOM_CDF4(19421, 30614, 32174) }, - { AOM_CDF4(11915, 25010, 29579) }, - { AOM_CDF4(6950, 17676, 24074) }, - { AOM_CDF4(3007, 8473, 13096) }, - { AOM_CDF4(29002, 32676, 32735) }, - { AOM_CDF4(22102, 31849, 32576) }, - { AOM_CDF4(14408, 28009, 31405) }, - { AOM_CDF4(9027, 21679, 27931) }, - { AOM_CDF4(4694, 12678, 18748) }, - { AOM_CDF4(28216, 32528, 32682) }, - { AOM_CDF4(20849, 31264, 32318) }, - { AOM_CDF4(12756, 25815, 29751) }, - { AOM_CDF4(7565, 18801, 24923) }, - { AOM_CDF4(3509, 9533, 14477) }, - { AOM_CDF4(30133, 32687, 32739) }, - { AOM_CDF4(23063, 31910, 32515) }, - { AOM_CDF4(14588, 28051, 31132) }, - { AOM_CDF4(9085, 21649, 27457) }, - { AOM_CDF4(4261, 11654, 17264) }, - { AOM_CDF4(29518, 32691, 32748) }, - { AOM_CDF4(22451, 31959, 32613) }, - { AOM_CDF4(14864, 28722, 31700) }, - { AOM_CDF4(9695, 22964, 28716) }, - { AOM_CDF4(4932, 13358, 19502) }, - { AOM_CDF4(8192, 16384, 24576) } }, - { { AOM_CDF4(6465, 16958, 21688) }, - { AOM_CDF4(25199, 31514, 32360) }, - { AOM_CDF4(14774, 27149, 30607) }, - { AOM_CDF4(9257, 21438, 26972) }, - { AOM_CDF4(5723, 15183, 21882) }, - { AOM_CDF4(3150, 8879, 13731) }, - { AOM_CDF4(26989, 32262, 32682) }, - { AOM_CDF4(17396, 29937, 32085) }, - { AOM_CDF4(11387, 24901, 29784) }, - { AOM_CDF4(7289, 18821, 25548) }, - { AOM_CDF4(3734, 10577, 16086) }, - { AOM_CDF4(29728, 32501, 32695) }, - { AOM_CDF4(17431, 29701, 31903) }, - { AOM_CDF4(9921, 22826, 28300) }, - { AOM_CDF4(5896, 15434, 22068) }, - { AOM_CDF4(3430, 9646, 14757) }, - { AOM_CDF4(28614, 32511, 32705) }, - { AOM_CDF4(19364, 30638, 32263) }, - { AOM_CDF4(13129, 26254, 30402) }, - { AOM_CDF4(8754, 20484, 26440) }, - { AOM_CDF4(4378, 11607, 17110) }, - { AOM_CDF4(30292, 32671, 32744) }, - { AOM_CDF4(21780, 31603, 32501) }, - { AOM_CDF4(14314, 27829, 31291) }, - { AOM_CDF4(9611, 22327, 28263) }, - { AOM_CDF4(4890, 13087, 19065) }, - { AOM_CDF4(25862, 32567, 32733) }, - { AOM_CDF4(20794, 32050, 32567) }, - { AOM_CDF4(17243, 30625, 32254) }, - { AOM_CDF4(13283, 27628, 31474) }, - { AOM_CDF4(9669, 22532, 28918) }, - { AOM_CDF4(27435, 32697, 32748) }, - { AOM_CDF4(24922, 32390, 32714) }, - { AOM_CDF4(21449, 31504, 32536) }, - { AOM_CDF4(16392, 29729, 31832) }, - { AOM_CDF4(11692, 24884, 29076) }, - { AOM_CDF4(24193, 32290, 32735) }, - { AOM_CDF4(18909, 31104, 32563) }, - { AOM_CDF4(12236, 26841, 31403) }, - { AOM_CDF4(8171, 21840, 29082) }, - { AOM_CDF4(7224, 17280, 25275) }, - { AOM_CDF4(8192, 16384, 24576) } } }, - { { { AOM_CDF4(3078, 6839, 9890) }, - { AOM_CDF4(13837, 20450, 24479) }, - { AOM_CDF4(5914, 14222, 19328) }, - { AOM_CDF4(3866, 10267, 14762) }, - { AOM_CDF4(2612, 7208, 11042) }, - { AOM_CDF4(1067, 2991, 4776) }, - { AOM_CDF4(25817, 31646, 32529) }, - { AOM_CDF4(13708, 26338, 30385) }, - { AOM_CDF4(7328, 18585, 24870) }, - { AOM_CDF4(4691, 13080, 19276) }, - { AOM_CDF4(1825, 5253, 8352) }, - { AOM_CDF4(29386, 32315, 32624) }, - { AOM_CDF4(17160, 29001, 31360) }, - { AOM_CDF4(9602, 21862, 27396) }, - { AOM_CDF4(5915, 15772, 22148) }, - { AOM_CDF4(2786, 7779, 12047) }, - { AOM_CDF4(29246, 32450, 32663) }, - { AOM_CDF4(18696, 29929, 31818) }, - { AOM_CDF4(10510, 23369, 28560) }, - { AOM_CDF4(6229, 16499, 23125) }, - { AOM_CDF4(2608, 7448, 11705) }, - { AOM_CDF4(30753, 32710, 32748) }, - { AOM_CDF4(21638, 31487, 32503) }, - { AOM_CDF4(12937, 26854, 30870) }, - { AOM_CDF4(8182, 20596, 26970) }, - { AOM_CDF4(3637, 10269, 15497) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) } }, - { { AOM_CDF4(5244, 12150, 16906) }, - { AOM_CDF4(20486, 26858, 29701) }, - { AOM_CDF4(7756, 18317, 23735) }, - { AOM_CDF4(3452, 9256, 13146) }, - { AOM_CDF4(2020, 5206, 8229) }, - { AOM_CDF4(1801, 4993, 7903) }, - { AOM_CDF4(27051, 31858, 32531) }, - { AOM_CDF4(15988, 27531, 30619) }, - { AOM_CDF4(9188, 21484, 26719) }, - { AOM_CDF4(6273, 17186, 23800) }, - { AOM_CDF4(3108, 9355, 14764) }, - { AOM_CDF4(31076, 32520, 32680) }, - { AOM_CDF4(18119, 30037, 31850) }, - { AOM_CDF4(10244, 22969, 27472) }, - { AOM_CDF4(4692, 14077, 19273) }, - { AOM_CDF4(3694, 11677, 17556) }, - { AOM_CDF4(30060, 32581, 32720) }, - { AOM_CDF4(21011, 30775, 32120) }, - { AOM_CDF4(11931, 24820, 29289) }, - { AOM_CDF4(7119, 17662, 24356) }, - { AOM_CDF4(3833, 10706, 16304) }, - { AOM_CDF4(31954, 32731, 32748) }, - { AOM_CDF4(23913, 31724, 32489) }, - { AOM_CDF4(15520, 28060, 31286) }, - { AOM_CDF4(11517, 23008, 28571) }, - { AOM_CDF4(6193, 14508, 20629) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) } } }, - { { { AOM_CDF4(1035, 2807, 4156) }, - { AOM_CDF4(13162, 18138, 20939) }, - { AOM_CDF4(2696, 6633, 8755) }, - { AOM_CDF4(1373, 4161, 6853) }, - { AOM_CDF4(1099, 2746, 4716) }, - { AOM_CDF4(340, 1021, 1599) }, - { AOM_CDF4(22826, 30419, 32135) }, - { AOM_CDF4(10395, 21762, 26942) }, - { AOM_CDF4(4726, 12407, 17361) }, - { AOM_CDF4(2447, 7080, 10593) }, - { AOM_CDF4(1227, 3717, 6011) }, - { AOM_CDF4(28156, 31424, 31934) }, - { AOM_CDF4(16915, 27754, 30373) }, - { AOM_CDF4(9148, 20990, 26431) }, - { AOM_CDF4(5950, 15515, 21148) }, - { AOM_CDF4(2492, 7327, 11526) }, - { AOM_CDF4(30602, 32477, 32670) }, - { AOM_CDF4(20026, 29955, 31568) }, - { AOM_CDF4(11220, 23628, 28105) }, - { AOM_CDF4(6652, 17019, 22973) }, - { AOM_CDF4(3064, 8536, 13043) }, - { AOM_CDF4(31769, 32724, 32748) }, - { AOM_CDF4(22230, 30887, 32373) }, - { AOM_CDF4(12234, 25079, 29731) }, - { AOM_CDF4(7326, 18816, 25353) }, - { AOM_CDF4(3933, 10907, 16616) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) } }, - { { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) } } } }, - { { { { AOM_CDF4(8896, 16227, 20630) }, - { AOM_CDF4(23629, 31782, 32527) }, - { AOM_CDF4(15173, 27755, 31321) }, - { AOM_CDF4(10158, 21233, 27382) }, - { AOM_CDF4(6420, 14857, 21558) }, - { AOM_CDF4(3269, 8155, 12646) }, - { AOM_CDF4(24835, 32009, 32496) }, - { AOM_CDF4(16509, 28421, 31579) }, - { AOM_CDF4(10957, 21514, 27418) }, - { AOM_CDF4(7881, 15930, 22096) }, - { AOM_CDF4(5388, 10960, 15918) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(20745, 30773, 32093) }, - { AOM_CDF4(15200, 27221, 30861) }, - { AOM_CDF4(13032, 20873, 25667) }, - { AOM_CDF4(12285, 18663, 23494) }, - { AOM_CDF4(11563, 17481, 21489) }, - { AOM_CDF4(26260, 31982, 32320) }, - { AOM_CDF4(15397, 28083, 31100) }, - { AOM_CDF4(9742, 19217, 24824) }, - { AOM_CDF4(3261, 9629, 15362) }, - { AOM_CDF4(1480, 4322, 7499) }, - { AOM_CDF4(27599, 32256, 32460) }, - { AOM_CDF4(16857, 27659, 30774) }, - { AOM_CDF4(9551, 18290, 23748) }, - { AOM_CDF4(3052, 8933, 14103) }, - { AOM_CDF4(2021, 5910, 9787) }, - { AOM_CDF4(29005, 32015, 32392) }, - { AOM_CDF4(17677, 27694, 30863) }, - { AOM_CDF4(9204, 17356, 23219) }, - { AOM_CDF4(2403, 7516, 12814) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) } }, - { { AOM_CDF4(10808, 22056, 26896) }, - { AOM_CDF4(25739, 32313, 32676) }, - { AOM_CDF4(17288, 30203, 32221) }, - { AOM_CDF4(11359, 24878, 29896) }, - { AOM_CDF4(6949, 17767, 24893) }, - { AOM_CDF4(4287, 11796, 18071) }, - { AOM_CDF4(27880, 32521, 32705) }, - { AOM_CDF4(19038, 31004, 32414) }, - { AOM_CDF4(12564, 26345, 30768) }, - { AOM_CDF4(8269, 19947, 26779) }, - { AOM_CDF4(5674, 14657, 21674) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(25742, 32319, 32671) }, - { AOM_CDF4(19557, 31164, 32454) }, - { AOM_CDF4(13381, 26381, 30755) }, - { AOM_CDF4(10101, 21466, 26722) }, - { AOM_CDF4(9209, 19650, 26825) }, - { AOM_CDF4(27107, 31917, 32432) }, - { AOM_CDF4(18056, 28893, 31203) }, - { AOM_CDF4(10200, 21434, 26764) }, - { AOM_CDF4(4660, 12913, 19502) }, - { AOM_CDF4(2368, 6930, 12504) }, - { AOM_CDF4(26960, 32158, 32613) }, - { AOM_CDF4(18628, 30005, 32031) }, - { AOM_CDF4(10233, 22442, 28232) }, - { AOM_CDF4(5471, 14630, 21516) }, - { AOM_CDF4(3235, 10767, 17109) }, - { AOM_CDF4(27696, 32440, 32692) }, - { AOM_CDF4(20032, 31167, 32438) }, - { AOM_CDF4(8700, 21341, 28442) }, - { AOM_CDF4(5662, 14831, 21795) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) } } }, - { { { AOM_CDF4(9704, 17294, 21132) }, - { AOM_CDF4(26762, 32278, 32633) }, - { AOM_CDF4(18382, 29620, 31819) }, - { AOM_CDF4(10891, 23475, 28723) }, - { AOM_CDF4(6358, 16583, 23309) }, - { AOM_CDF4(3248, 9118, 14141) }, - { AOM_CDF4(27204, 32573, 32699) }, - { AOM_CDF4(19818, 30824, 32329) }, - { AOM_CDF4(11772, 25120, 30041) }, - { AOM_CDF4(6995, 18033, 25039) }, - { AOM_CDF4(3752, 10442, 16098) }, - { AOM_CDF4(27222, 32256, 32559) }, - { AOM_CDF4(15356, 28399, 31475) }, - { AOM_CDF4(8821, 20635, 27057) }, - { AOM_CDF4(5511, 14404, 21239) }, - { AOM_CDF4(2935, 8222, 13051) }, - { AOM_CDF4(24875, 32120, 32529) }, - { AOM_CDF4(15233, 28265, 31445) }, - { AOM_CDF4(8605, 20570, 26932) }, - { AOM_CDF4(5431, 14413, 21196) }, - { AOM_CDF4(2994, 8341, 13223) }, - { AOM_CDF4(28201, 32604, 32700) }, - { AOM_CDF4(21041, 31446, 32456) }, - { AOM_CDF4(13221, 26213, 30475) }, - { AOM_CDF4(8255, 19385, 26037) }, - { AOM_CDF4(4930, 12585, 18830) }, - { AOM_CDF4(28768, 32448, 32627) }, - { AOM_CDF4(19705, 30561, 32021) }, - { AOM_CDF4(11572, 23589, 28220) }, - { AOM_CDF4(5532, 15034, 21446) }, - { AOM_CDF4(2460, 7150, 11456) }, - { AOM_CDF4(29874, 32619, 32699) }, - { AOM_CDF4(21621, 31071, 32201) }, - { AOM_CDF4(12511, 24747, 28992) }, - { AOM_CDF4(6281, 16395, 22748) }, - { AOM_CDF4(3246, 9278, 14497) }, - { AOM_CDF4(29715, 32625, 32712) }, - { AOM_CDF4(20958, 31011, 32283) }, - { AOM_CDF4(11233, 23671, 28806) }, - { AOM_CDF4(6012, 16128, 22868) }, - { AOM_CDF4(3427, 9851, 15414) }, - { AOM_CDF4(8192, 16384, 24576) } }, - { { AOM_CDF4(11016, 22111, 26794) }, - { AOM_CDF4(25946, 32357, 32677) }, - { AOM_CDF4(17890, 30452, 32252) }, - { AOM_CDF4(11678, 25142, 29816) }, - { AOM_CDF4(6720, 17534, 24584) }, - { AOM_CDF4(4230, 11665, 17820) }, - { AOM_CDF4(28400, 32623, 32747) }, - { AOM_CDF4(21164, 31668, 32575) }, - { AOM_CDF4(13572, 27388, 31182) }, - { AOM_CDF4(8234, 20750, 27358) }, - { AOM_CDF4(5065, 14055, 20897) }, - { AOM_CDF4(28981, 32547, 32705) }, - { AOM_CDF4(18681, 30543, 32239) }, - { AOM_CDF4(10919, 24075, 29286) }, - { AOM_CDF4(6431, 17199, 24077) }, - { AOM_CDF4(3819, 10464, 16618) }, - { AOM_CDF4(26870, 32467, 32693) }, - { AOM_CDF4(19041, 30831, 32347) }, - { AOM_CDF4(11794, 25211, 30016) }, - { AOM_CDF4(6888, 18019, 24970) }, - { AOM_CDF4(4370, 12363, 18992) }, - { AOM_CDF4(29578, 32670, 32744) }, - { AOM_CDF4(23159, 32007, 32613) }, - { AOM_CDF4(15315, 28669, 31676) }, - { AOM_CDF4(9298, 22607, 28782) }, - { AOM_CDF4(6144, 15913, 22968) }, - { AOM_CDF4(28110, 32499, 32669) }, - { AOM_CDF4(21574, 30937, 32015) }, - { AOM_CDF4(12759, 24818, 28727) }, - { AOM_CDF4(6545, 16761, 23042) }, - { AOM_CDF4(3649, 10597, 16833) }, - { AOM_CDF4(28163, 32552, 32728) }, - { AOM_CDF4(22101, 31469, 32464) }, - { AOM_CDF4(13160, 25472, 30143) }, - { AOM_CDF4(7303, 18684, 25468) }, - { AOM_CDF4(5241, 13975, 20955) }, - { AOM_CDF4(28400, 32631, 32744) }, - { AOM_CDF4(22104, 31793, 32603) }, - { AOM_CDF4(13557, 26571, 30846) }, - { AOM_CDF4(7749, 19861, 26675) }, - { AOM_CDF4(4873, 14030, 21234) }, - { AOM_CDF4(8192, 16384, 24576) } } }, - { { { AOM_CDF4(9800, 17635, 21073) }, - { AOM_CDF4(26153, 31885, 32527) }, - { AOM_CDF4(15038, 27852, 31006) }, - { AOM_CDF4(8718, 20564, 26486) }, - { AOM_CDF4(5128, 14076, 20514) }, - { AOM_CDF4(2636, 7566, 11925) }, - { AOM_CDF4(27551, 32504, 32701) }, - { AOM_CDF4(18310, 30054, 32100) }, - { AOM_CDF4(10211, 23420, 29082) }, - { AOM_CDF4(6222, 16876, 23916) }, - { AOM_CDF4(3462, 9954, 15498) }, - { AOM_CDF4(29991, 32633, 32721) }, - { AOM_CDF4(19883, 30751, 32201) }, - { AOM_CDF4(11141, 24184, 29285) }, - { AOM_CDF4(6420, 16940, 23774) }, - { AOM_CDF4(3392, 9753, 15118) }, - { AOM_CDF4(28465, 32616, 32712) }, - { AOM_CDF4(19850, 30702, 32244) }, - { AOM_CDF4(10983, 24024, 29223) }, - { AOM_CDF4(6294, 16770, 23582) }, - { AOM_CDF4(3244, 9283, 14509) }, - { AOM_CDF4(30023, 32717, 32748) }, - { AOM_CDF4(22940, 32032, 32626) }, - { AOM_CDF4(14282, 27928, 31473) }, - { AOM_CDF4(8562, 21327, 27914) }, - { AOM_CDF4(4846, 13393, 19919) }, - { AOM_CDF4(29981, 32590, 32695) }, - { AOM_CDF4(20465, 30963, 32166) }, - { AOM_CDF4(11479, 23579, 28195) }, - { AOM_CDF4(5916, 15648, 22073) }, - { AOM_CDF4(3031, 8605, 13398) }, - { AOM_CDF4(31146, 32691, 32739) }, - { AOM_CDF4(23106, 31724, 32444) }, - { AOM_CDF4(13783, 26738, 30439) }, - { AOM_CDF4(7852, 19468, 25807) }, - { AOM_CDF4(3860, 11124, 16853) }, - { AOM_CDF4(31014, 32724, 32748) }, - { AOM_CDF4(23629, 32109, 32628) }, - { AOM_CDF4(14747, 28115, 31403) }, - { AOM_CDF4(8545, 21242, 27478) }, - { AOM_CDF4(4574, 12781, 19067) }, - { AOM_CDF4(8192, 16384, 24576) } }, - { { AOM_CDF4(9185, 19694, 24688) }, - { AOM_CDF4(26081, 31985, 32621) }, - { AOM_CDF4(16015, 29000, 31787) }, - { AOM_CDF4(10542, 23690, 29206) }, - { AOM_CDF4(6732, 17945, 24677) }, - { AOM_CDF4(3916, 11039, 16722) }, - { AOM_CDF4(28224, 32566, 32744) }, - { AOM_CDF4(19100, 31138, 32485) }, - { AOM_CDF4(12528, 26620, 30879) }, - { AOM_CDF4(7741, 20277, 26885) }, - { AOM_CDF4(4566, 12845, 18990) }, - { AOM_CDF4(29933, 32593, 32718) }, - { AOM_CDF4(17670, 30333, 32155) }, - { AOM_CDF4(10385, 23600, 28909) }, - { AOM_CDF4(6243, 16236, 22407) }, - { AOM_CDF4(3976, 10389, 16017) }, - { AOM_CDF4(28377, 32561, 32738) }, - { AOM_CDF4(19366, 31175, 32482) }, - { AOM_CDF4(13327, 27175, 31094) }, - { AOM_CDF4(8258, 20769, 27143) }, - { AOM_CDF4(4703, 13198, 19527) }, - { AOM_CDF4(31086, 32706, 32748) }, - { AOM_CDF4(22853, 31902, 32583) }, - { AOM_CDF4(14759, 28186, 31419) }, - { AOM_CDF4(9284, 22382, 28348) }, - { AOM_CDF4(5585, 15192, 21868) }, - { AOM_CDF4(28291, 32652, 32746) }, - { AOM_CDF4(19849, 32107, 32571) }, - { AOM_CDF4(14834, 26818, 29214) }, - { AOM_CDF4(10306, 22594, 28672) }, - { AOM_CDF4(6615, 17384, 23384) }, - { AOM_CDF4(28947, 32604, 32745) }, - { AOM_CDF4(25625, 32289, 32646) }, - { AOM_CDF4(18758, 28672, 31403) }, - { AOM_CDF4(10017, 23430, 28523) }, - { AOM_CDF4(6862, 15269, 22131) }, - { AOM_CDF4(23933, 32509, 32739) }, - { AOM_CDF4(19927, 31495, 32631) }, - { AOM_CDF4(11903, 26023, 30621) }, - { AOM_CDF4(7026, 20094, 27252) }, - { AOM_CDF4(5998, 18106, 24437) }, - { AOM_CDF4(8192, 16384, 24576) } } }, - { { { AOM_CDF4(4456, 11274, 15533) }, - { AOM_CDF4(21219, 29079, 31616) }, - { AOM_CDF4(11173, 23774, 28567) }, - { AOM_CDF4(7282, 18293, 24263) }, - { AOM_CDF4(4890, 13286, 19115) }, - { AOM_CDF4(1890, 5508, 8659) }, - { AOM_CDF4(26651, 32136, 32647) }, - { AOM_CDF4(14630, 28254, 31455) }, - { AOM_CDF4(8716, 21287, 27395) }, - { AOM_CDF4(5615, 15331, 22008) }, - { AOM_CDF4(2675, 7700, 12150) }, - { AOM_CDF4(29954, 32526, 32690) }, - { AOM_CDF4(16126, 28982, 31633) }, - { AOM_CDF4(9030, 21361, 27352) }, - { AOM_CDF4(5411, 14793, 21271) }, - { AOM_CDF4(2943, 8422, 13163) }, - { AOM_CDF4(29539, 32601, 32730) }, - { AOM_CDF4(18125, 30385, 32201) }, - { AOM_CDF4(10422, 24090, 29468) }, - { AOM_CDF4(6468, 17487, 24438) }, - { AOM_CDF4(2970, 8653, 13531) }, - { AOM_CDF4(30912, 32715, 32748) }, - { AOM_CDF4(20666, 31373, 32497) }, - { AOM_CDF4(12509, 26640, 30917) }, - { AOM_CDF4(8058, 20629, 27290) }, - { AOM_CDF4(4231, 12006, 18052) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) } }, - { { AOM_CDF4(10202, 20633, 25484) }, - { AOM_CDF4(27336, 31445, 32352) }, - { AOM_CDF4(12420, 24384, 28552) }, - { AOM_CDF4(7648, 18115, 23856) }, - { AOM_CDF4(5662, 14341, 19902) }, - { AOM_CDF4(3611, 10328, 15390) }, - { AOM_CDF4(30945, 32616, 32736) }, - { AOM_CDF4(18682, 30505, 32253) }, - { AOM_CDF4(11513, 25336, 30203) }, - { AOM_CDF4(7449, 19452, 26148) }, - { AOM_CDF4(4482, 13051, 18886) }, - { AOM_CDF4(32022, 32690, 32747) }, - { AOM_CDF4(18578, 30501, 32146) }, - { AOM_CDF4(11249, 23368, 28631) }, - { AOM_CDF4(5645, 16958, 22158) }, - { AOM_CDF4(5009, 11444, 16637) }, - { AOM_CDF4(31357, 32710, 32748) }, - { AOM_CDF4(21552, 31494, 32504) }, - { AOM_CDF4(13891, 27677, 31340) }, - { AOM_CDF4(9051, 22098, 28172) }, - { AOM_CDF4(5190, 13377, 19486) }, - { AOM_CDF4(32364, 32740, 32748) }, - { AOM_CDF4(24839, 31907, 32551) }, - { AOM_CDF4(17160, 28779, 31696) }, - { AOM_CDF4(12452, 24137, 29602) }, - { AOM_CDF4(6165, 15389, 22477) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) } } }, - { { { AOM_CDF4(2575, 7281, 11077) }, - { AOM_CDF4(14002, 20866, 25402) }, - { AOM_CDF4(6343, 15056, 19658) }, - { AOM_CDF4(4474, 11858, 17041) }, - { AOM_CDF4(2865, 8299, 12534) }, - { AOM_CDF4(1344, 3949, 6391) }, - { AOM_CDF4(24720, 31239, 32459) }, - { AOM_CDF4(12585, 25356, 29968) }, - { AOM_CDF4(7181, 18246, 24444) }, - { AOM_CDF4(5025, 13667, 19885) }, - { AOM_CDF4(2521, 7304, 11605) }, - { AOM_CDF4(29908, 32252, 32584) }, - { AOM_CDF4(17421, 29156, 31575) }, - { AOM_CDF4(9889, 22188, 27782) }, - { AOM_CDF4(5878, 15647, 22123) }, - { AOM_CDF4(2814, 8665, 13323) }, - { AOM_CDF4(30183, 32568, 32713) }, - { AOM_CDF4(18528, 30195, 32049) }, - { AOM_CDF4(10982, 24606, 29657) }, - { AOM_CDF4(6957, 18165, 25231) }, - { AOM_CDF4(3508, 10118, 15468) }, - { AOM_CDF4(31761, 32736, 32748) }, - { AOM_CDF4(21041, 31328, 32546) }, - { AOM_CDF4(12568, 26732, 31166) }, - { AOM_CDF4(8052, 20720, 27733) }, - { AOM_CDF4(4336, 12192, 18396) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) } }, - { { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) } } } }, - { { { { AOM_CDF4(7062, 16472, 22319) }, - { AOM_CDF4(24538, 32261, 32674) }, - { AOM_CDF4(13675, 28041, 31779) }, - { AOM_CDF4(8590, 20674, 27631) }, - { AOM_CDF4(5685, 14675, 22013) }, - { AOM_CDF4(3655, 9898, 15731) }, - { AOM_CDF4(26493, 32418, 32658) }, - { AOM_CDF4(16376, 29342, 32090) }, - { AOM_CDF4(10594, 22649, 28970) }, - { AOM_CDF4(8176, 17170, 24303) }, - { AOM_CDF4(5605, 12694, 19139) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(23888, 31902, 32542) }, - { AOM_CDF4(18612, 29687, 31987) }, - { AOM_CDF4(16245, 24852, 29249) }, - { AOM_CDF4(15765, 22608, 27559) }, - { AOM_CDF4(19895, 24699, 27510) }, - { AOM_CDF4(28401, 32212, 32457) }, - { AOM_CDF4(15274, 27825, 30980) }, - { AOM_CDF4(9364, 18128, 24332) }, - { AOM_CDF4(2283, 8193, 15082) }, - { AOM_CDF4(1228, 3972, 7881) }, - { AOM_CDF4(29455, 32469, 32620) }, - { AOM_CDF4(17981, 28245, 31388) }, - { AOM_CDF4(10921, 20098, 26240) }, - { AOM_CDF4(3743, 11829, 18657) }, - { AOM_CDF4(2374, 9593, 15715) }, - { AOM_CDF4(31068, 32466, 32635) }, - { AOM_CDF4(20321, 29572, 31971) }, - { AOM_CDF4(10771, 20255, 27119) }, - { AOM_CDF4(2795, 10410, 17361) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) } }, - { { AOM_CDF4(9320, 22102, 27840) }, - { AOM_CDF4(27057, 32464, 32724) }, - { AOM_CDF4(16331, 30268, 32309) }, - { AOM_CDF4(10319, 23935, 29720) }, - { AOM_CDF4(6189, 16448, 24106) }, - { AOM_CDF4(3589, 10884, 18808) }, - { AOM_CDF4(29026, 32624, 32748) }, - { AOM_CDF4(19226, 31507, 32587) }, - { AOM_CDF4(12692, 26921, 31203) }, - { AOM_CDF4(7049, 19532, 27635) }, - { AOM_CDF4(7727, 15669, 23252) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(28056, 32625, 32748) }, - { AOM_CDF4(22383, 32075, 32669) }, - { AOM_CDF4(15417, 27098, 31749) }, - { AOM_CDF4(18127, 26493, 27190) }, - { AOM_CDF4(5461, 16384, 21845) }, - { AOM_CDF4(27982, 32091, 32584) }, - { AOM_CDF4(19045, 29868, 31972) }, - { AOM_CDF4(10397, 22266, 27932) }, - { AOM_CDF4(5990, 13697, 21500) }, - { AOM_CDF4(1792, 6912, 15104) }, - { AOM_CDF4(28198, 32501, 32718) }, - { AOM_CDF4(21534, 31521, 32569) }, - { AOM_CDF4(11109, 25217, 30017) }, - { AOM_CDF4(5671, 15124, 26151) }, - { AOM_CDF4(4681, 14043, 18725) }, - { AOM_CDF4(28688, 32580, 32741) }, - { AOM_CDF4(22576, 32079, 32661) }, - { AOM_CDF4(10627, 22141, 28340) }, - { AOM_CDF4(9362, 14043, 28087) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) } } }, - { { { AOM_CDF4(7754, 16948, 22142) }, - { AOM_CDF4(25670, 32330, 32691) }, - { AOM_CDF4(15663, 29225, 31994) }, - { AOM_CDF4(9878, 23288, 29158) }, - { AOM_CDF4(6419, 17088, 24336) }, - { AOM_CDF4(3859, 11003, 17039) }, - { AOM_CDF4(27562, 32595, 32725) }, - { AOM_CDF4(17575, 30588, 32399) }, - { AOM_CDF4(10819, 24838, 30309) }, - { AOM_CDF4(7124, 18686, 25916) }, - { AOM_CDF4(4479, 12688, 19340) }, - { AOM_CDF4(28385, 32476, 32673) }, - { AOM_CDF4(15306, 29005, 31938) }, - { AOM_CDF4(8937, 21615, 28322) }, - { AOM_CDF4(5982, 15603, 22786) }, - { AOM_CDF4(3620, 10267, 16136) }, - { AOM_CDF4(27280, 32464, 32667) }, - { AOM_CDF4(15607, 29160, 32004) }, - { AOM_CDF4(9091, 22135, 28740) }, - { AOM_CDF4(6232, 16632, 24020) }, - { AOM_CDF4(4047, 11377, 17672) }, - { AOM_CDF4(29220, 32630, 32718) }, - { AOM_CDF4(19650, 31220, 32462) }, - { AOM_CDF4(13050, 26312, 30827) }, - { AOM_CDF4(9228, 20870, 27468) }, - { AOM_CDF4(6146, 15149, 21971) }, - { AOM_CDF4(30169, 32481, 32623) }, - { AOM_CDF4(17212, 29311, 31554) }, - { AOM_CDF4(9911, 21311, 26882) }, - { AOM_CDF4(4487, 13314, 20372) }, - { AOM_CDF4(2570, 7772, 12889) }, - { AOM_CDF4(30924, 32613, 32708) }, - { AOM_CDF4(19490, 30206, 32107) }, - { AOM_CDF4(11232, 23998, 29276) }, - { AOM_CDF4(6769, 17955, 25035) }, - { AOM_CDF4(4398, 12623, 19214) }, - { AOM_CDF4(30609, 32627, 32722) }, - { AOM_CDF4(19370, 30582, 32287) }, - { AOM_CDF4(10457, 23619, 29409) }, - { AOM_CDF4(6443, 17637, 24834) }, - { AOM_CDF4(4645, 13236, 20106) }, - { AOM_CDF4(8192, 16384, 24576) } }, - { { AOM_CDF4(8626, 20271, 26216) }, - { AOM_CDF4(26707, 32406, 32711) }, - { AOM_CDF4(16999, 30329, 32286) }, - { AOM_CDF4(11445, 25123, 30286) }, - { AOM_CDF4(6411, 18828, 25601) }, - { AOM_CDF4(6801, 12458, 20248) }, - { AOM_CDF4(29918, 32682, 32748) }, - { AOM_CDF4(20649, 31739, 32618) }, - { AOM_CDF4(12879, 27773, 31581) }, - { AOM_CDF4(7896, 21751, 28244) }, - { AOM_CDF4(5260, 14870, 23698) }, - { AOM_CDF4(29252, 32593, 32731) }, - { AOM_CDF4(17072, 30460, 32294) }, - { AOM_CDF4(10653, 24143, 29365) }, - { AOM_CDF4(6536, 17490, 23983) }, - { AOM_CDF4(4929, 13170, 20085) }, - { AOM_CDF4(28137, 32518, 32715) }, - { AOM_CDF4(18171, 30784, 32407) }, - { AOM_CDF4(11437, 25436, 30459) }, - { AOM_CDF4(7252, 18534, 26176) }, - { AOM_CDF4(4126, 13353, 20978) }, - { AOM_CDF4(31162, 32726, 32748) }, - { AOM_CDF4(23017, 32222, 32701) }, - { AOM_CDF4(15629, 29233, 32046) }, - { AOM_CDF4(9387, 22621, 29480) }, - { AOM_CDF4(6922, 17616, 25010) }, - { AOM_CDF4(28838, 32265, 32614) }, - { AOM_CDF4(19701, 30206, 31920) }, - { AOM_CDF4(11214, 22410, 27933) }, - { AOM_CDF4(5320, 14177, 23034) }, - { AOM_CDF4(5049, 12881, 17827) }, - { AOM_CDF4(27484, 32471, 32734) }, - { AOM_CDF4(21076, 31526, 32561) }, - { AOM_CDF4(12707, 26303, 31211) }, - { AOM_CDF4(8169, 21722, 28219) }, - { AOM_CDF4(6045, 19406, 27042) }, - { AOM_CDF4(27753, 32572, 32745) }, - { AOM_CDF4(20832, 31878, 32653) }, - { AOM_CDF4(13250, 27356, 31674) }, - { AOM_CDF4(7718, 21508, 29858) }, - { AOM_CDF4(7209, 18350, 25559) }, - { AOM_CDF4(8192, 16384, 24576) } } }, - { { { AOM_CDF4(7876, 16901, 21741) }, - { AOM_CDF4(24001, 31898, 32625) }, - { AOM_CDF4(14529, 27959, 31451) }, - { AOM_CDF4(8273, 20818, 27258) }, - { AOM_CDF4(5278, 14673, 21510) }, - { AOM_CDF4(2983, 8843, 14039) }, - { AOM_CDF4(28016, 32574, 32732) }, - { AOM_CDF4(17471, 30306, 32301) }, - { AOM_CDF4(10224, 24063, 29728) }, - { AOM_CDF4(6602, 17954, 25052) }, - { AOM_CDF4(4002, 11585, 17759) }, - { AOM_CDF4(30190, 32634, 32739) }, - { AOM_CDF4(17497, 30282, 32270) }, - { AOM_CDF4(10229, 23729, 29538) }, - { AOM_CDF4(6344, 17211, 24440) }, - { AOM_CDF4(3849, 11189, 17108) }, - { AOM_CDF4(28570, 32583, 32726) }, - { AOM_CDF4(17521, 30161, 32238) }, - { AOM_CDF4(10153, 23565, 29378) }, - { AOM_CDF4(6455, 17341, 24443) }, - { AOM_CDF4(3907, 11042, 17024) }, - { AOM_CDF4(30689, 32715, 32748) }, - { AOM_CDF4(21546, 31840, 32610) }, - { AOM_CDF4(13547, 27581, 31459) }, - { AOM_CDF4(8912, 21757, 28309) }, - { AOM_CDF4(5548, 15080, 22046) }, - { AOM_CDF4(30783, 32540, 32685) }, - { AOM_CDF4(17540, 29528, 31668) }, - { AOM_CDF4(10160, 21468, 26783) }, - { AOM_CDF4(4724, 13393, 20054) }, - { AOM_CDF4(2702, 8174, 13102) }, - { AOM_CDF4(31648, 32686, 32742) }, - { AOM_CDF4(20954, 31094, 32337) }, - { AOM_CDF4(12420, 25698, 30179) }, - { AOM_CDF4(7304, 19320, 26248) }, - { AOM_CDF4(4366, 12261, 18864) }, - { AOM_CDF4(31581, 32723, 32748) }, - { AOM_CDF4(21373, 31586, 32525) }, - { AOM_CDF4(12744, 26625, 30885) }, - { AOM_CDF4(7431, 20322, 26950) }, - { AOM_CDF4(4692, 13323, 20111) }, - { AOM_CDF4(8192, 16384, 24576) } }, - { { AOM_CDF4(7833, 18369, 24095) }, - { AOM_CDF4(26650, 32273, 32702) }, - { AOM_CDF4(16371, 29961, 32191) }, - { AOM_CDF4(11055, 24082, 29629) }, - { AOM_CDF4(6892, 18644, 25400) }, - { AOM_CDF4(5006, 13057, 19240) }, - { AOM_CDF4(29834, 32666, 32748) }, - { AOM_CDF4(19577, 31335, 32570) }, - { AOM_CDF4(12253, 26509, 31122) }, - { AOM_CDF4(7991, 20772, 27711) }, - { AOM_CDF4(5677, 15910, 23059) }, - { AOM_CDF4(30109, 32532, 32720) }, - { AOM_CDF4(16747, 30166, 32252) }, - { AOM_CDF4(10134, 23542, 29184) }, - { AOM_CDF4(5791, 16176, 23556) }, - { AOM_CDF4(4362, 10414, 17284) }, - { AOM_CDF4(29492, 32626, 32748) }, - { AOM_CDF4(19894, 31402, 32525) }, - { AOM_CDF4(12942, 27071, 30869) }, - { AOM_CDF4(8346, 21216, 27405) }, - { AOM_CDF4(6572, 17087, 23859) }, - { AOM_CDF4(32035, 32735, 32748) }, - { AOM_CDF4(22957, 31838, 32618) }, - { AOM_CDF4(14724, 28572, 31772) }, - { AOM_CDF4(10364, 23999, 29553) }, - { AOM_CDF4(7004, 18433, 25655) }, - { AOM_CDF4(27528, 32277, 32681) }, - { AOM_CDF4(16959, 31171, 32096) }, - { AOM_CDF4(10486, 23593, 27962) }, - { AOM_CDF4(8192, 16384, 23211) }, - { AOM_CDF4(8937, 17873, 20852) }, - { AOM_CDF4(27715, 32002, 32615) }, - { AOM_CDF4(15073, 29491, 31676) }, - { AOM_CDF4(11264, 24576, 28672) }, - { AOM_CDF4(2341, 18725, 23406) }, - { AOM_CDF4(7282, 18204, 25486) }, - { AOM_CDF4(28547, 32213, 32657) }, - { AOM_CDF4(20788, 29773, 32239) }, - { AOM_CDF4(6780, 21469, 30508) }, - { AOM_CDF4(5958, 14895, 23831) }, - { AOM_CDF4(16384, 21845, 27307) }, - { AOM_CDF4(8192, 16384, 24576) } } }, - { { { AOM_CDF4(5992, 14304, 19765) }, - { AOM_CDF4(22612, 31238, 32456) }, - { AOM_CDF4(13456, 27162, 31087) }, - { AOM_CDF4(8001, 20062, 26504) }, - { AOM_CDF4(5168, 14105, 20764) }, - { AOM_CDF4(2632, 7771, 12385) }, - { AOM_CDF4(27034, 32344, 32709) }, - { AOM_CDF4(15850, 29415, 31997) }, - { AOM_CDF4(9494, 22776, 28841) }, - { AOM_CDF4(6151, 16830, 23969) }, - { AOM_CDF4(3461, 10039, 15722) }, - { AOM_CDF4(30134, 32569, 32731) }, - { AOM_CDF4(15638, 29422, 31945) }, - { AOM_CDF4(9150, 21865, 28218) }, - { AOM_CDF4(5647, 15719, 22676) }, - { AOM_CDF4(3402, 9772, 15477) }, - { AOM_CDF4(28530, 32586, 32735) }, - { AOM_CDF4(17139, 30298, 32292) }, - { AOM_CDF4(10200, 24039, 29685) }, - { AOM_CDF4(6419, 17674, 24786) }, - { AOM_CDF4(3544, 10225, 15824) }, - { AOM_CDF4(31333, 32726, 32748) }, - { AOM_CDF4(20618, 31487, 32544) }, - { AOM_CDF4(12901, 27217, 31232) }, - { AOM_CDF4(8624, 21734, 28171) }, - { AOM_CDF4(5104, 14191, 20748) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) } }, - { { AOM_CDF4(11206, 21090, 26561) }, - { AOM_CDF4(28759, 32279, 32671) }, - { AOM_CDF4(14171, 27952, 31569) }, - { AOM_CDF4(9743, 22907, 29141) }, - { AOM_CDF4(6871, 17886, 24868) }, - { AOM_CDF4(4960, 13152, 19315) }, - { AOM_CDF4(31077, 32661, 32748) }, - { AOM_CDF4(19400, 31195, 32515) }, - { AOM_CDF4(12752, 26858, 31040) }, - { AOM_CDF4(8370, 22098, 28591) }, - { AOM_CDF4(5457, 15373, 22298) }, - { AOM_CDF4(31697, 32706, 32748) }, - { AOM_CDF4(17860, 30657, 32333) }, - { AOM_CDF4(12510, 24812, 29261) }, - { AOM_CDF4(6180, 19124, 24722) }, - { AOM_CDF4(5041, 13548, 17959) }, - { AOM_CDF4(31552, 32716, 32748) }, - { AOM_CDF4(21908, 31769, 32623) }, - { AOM_CDF4(14470, 28201, 31565) }, - { AOM_CDF4(9493, 22982, 28608) }, - { AOM_CDF4(6858, 17240, 24137) }, - { AOM_CDF4(32543, 32752, 32756) }, - { AOM_CDF4(24286, 32097, 32666) }, - { AOM_CDF4(15958, 29217, 32024) }, - { AOM_CDF4(10207, 24234, 29958) }, - { AOM_CDF4(6929, 18305, 25652) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) } } }, - { { { AOM_CDF4(4137, 10847, 15682) }, - { AOM_CDF4(17824, 27001, 30058) }, - { AOM_CDF4(10204, 22796, 28291) }, - { AOM_CDF4(6076, 15935, 22125) }, - { AOM_CDF4(3852, 10937, 16816) }, - { AOM_CDF4(2252, 6324, 10131) }, - { AOM_CDF4(25840, 32016, 32662) }, - { AOM_CDF4(15109, 28268, 31531) }, - { AOM_CDF4(9385, 22231, 28340) }, - { AOM_CDF4(6082, 16672, 23479) }, - { AOM_CDF4(3318, 9427, 14681) }, - { AOM_CDF4(30594, 32574, 32718) }, - { AOM_CDF4(16836, 29552, 31859) }, - { AOM_CDF4(9556, 22542, 28356) }, - { AOM_CDF4(6305, 16725, 23540) }, - { AOM_CDF4(3376, 9895, 15184) }, - { AOM_CDF4(29383, 32617, 32745) }, - { AOM_CDF4(18891, 30809, 32401) }, - { AOM_CDF4(11688, 25942, 30687) }, - { AOM_CDF4(7468, 19469, 26651) }, - { AOM_CDF4(3909, 11358, 17012) }, - { AOM_CDF4(31564, 32736, 32748) }, - { AOM_CDF4(20906, 31611, 32600) }, - { AOM_CDF4(13191, 27621, 31537) }, - { AOM_CDF4(8768, 22029, 28676) }, - { AOM_CDF4(5079, 14109, 20906) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) } }, - { { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) }, - { AOM_CDF4(8192, 16384, 24576) } } } } }; + [CDF_SIZE(NUM_BASE_LEVELS + + 2)] = { { { { { AOM_CDF4(4034, 8930, 12727) }, + { AOM_CDF4(18082, 29741, 31877) }, + { AOM_CDF4(12596, 26124, 30493) }, + { AOM_CDF4(9446, 21118, 27005) }, + { AOM_CDF4(6308, 15141, 21279) }, + { AOM_CDF4(2463, 6357, 9783) }, + { AOM_CDF4(20667, 30546, 31929) }, + { AOM_CDF4(13043, 26123, 30134) }, + { AOM_CDF4(8151, 18757, 24778) }, + { AOM_CDF4(5255, 12839, 18632) }, + { AOM_CDF4(2820, 7206, 11161) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(15736, 27553, 30604) }, + { AOM_CDF4(11210, 23794, 28787) }, + { AOM_CDF4(5947, 13874, 19701) }, + { AOM_CDF4(4215, 9323, 13891) }, + { AOM_CDF4(2833, 6462, 10059) }, + { AOM_CDF4(19605, 30393, 31582) }, + { AOM_CDF4(13523, 26252, 30248) }, + { AOM_CDF4(8446, 18622, 24512) }, + { AOM_CDF4(3818, 10343, 15974) }, + { AOM_CDF4(1481, 4117, 6796) }, + { AOM_CDF4(22649, 31302, 32190) }, + { AOM_CDF4(14829, 27127, 30449) }, + { AOM_CDF4(8313, 17702, 23304) }, + { AOM_CDF4(3022, 8301, 12786) }, + { AOM_CDF4(1536, 4412, 7184) }, + { AOM_CDF4(22354, 29774, 31372) }, + { AOM_CDF4(14723, 25472, 29214) }, + { AOM_CDF4(6673, 13745, 18662) }, + { AOM_CDF4(2068, 5766, 9322) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) } }, + { { AOM_CDF4(6302, 16444, 21761) }, + { AOM_CDF4(23040, 31538, 32475) }, + { AOM_CDF4(15196, 28452, 31496) }, + { AOM_CDF4(10020, 22946, 28514) }, + { AOM_CDF4(6533, 16862, 23501) }, + { AOM_CDF4(3538, 9816, 15076) }, + { AOM_CDF4(24444, 31875, 32525) }, + { AOM_CDF4(15881, 28924, 31635) }, + { AOM_CDF4(9922, 22873, 28466) }, + { AOM_CDF4(6527, 16966, 23691) }, + { AOM_CDF4(4114, 11303, 17220) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(20201, 30770, 32209) }, + { AOM_CDF4(14754, 28071, 31258) }, + { AOM_CDF4(8378, 20186, 26517) }, + { AOM_CDF4(5916, 15299, 21978) }, + { AOM_CDF4(4268, 11583, 17901) }, + { AOM_CDF4(24361, 32025, 32581) }, + { AOM_CDF4(18673, 30105, 31943) }, + { AOM_CDF4(10196, 22244, 27576) }, + { AOM_CDF4(5495, 14349, 20417) }, + { AOM_CDF4(2676, 7415, 11498) }, + { AOM_CDF4(24678, 31958, 32585) }, + { AOM_CDF4(18629, 29906, 31831) }, + { AOM_CDF4(9364, 20724, 26315) }, + { AOM_CDF4(4641, 12318, 18094) }, + { AOM_CDF4(2758, 7387, 11579) }, + { AOM_CDF4(25433, 31842, 32469) }, + { AOM_CDF4(18795, 29289, 31411) }, + { AOM_CDF4(7644, 17584, 23592) }, + { AOM_CDF4(3408, 9014, 15047) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) } } }, + { { { AOM_CDF4(4536, 10072, 14001) }, + { AOM_CDF4(25459, 31416, 32206) }, + { AOM_CDF4(16605, 28048, 30818) }, + { AOM_CDF4(11008, 22857, 27719) }, + { AOM_CDF4(6915, 16268, 22315) }, + { AOM_CDF4(2625, 6812, 10537) }, + { AOM_CDF4(24257, 31788, 32499) }, + { AOM_CDF4(16880, 29454, 31879) }, + { AOM_CDF4(11958, 25054, 29778) }, + { AOM_CDF4(7916, 18718, 25084) }, + { AOM_CDF4(3383, 8777, 13446) }, + { AOM_CDF4(22720, 31603, 32393) }, + { AOM_CDF4(14960, 28125, 31335) }, + { AOM_CDF4(9731, 22210, 27928) }, + { AOM_CDF4(6304, 15832, 22277) }, + { AOM_CDF4(2910, 7818, 12166) }, + { AOM_CDF4(20375, 30627, 32131) }, + { AOM_CDF4(13904, 27284, 30887) }, + { AOM_CDF4(9368, 21558, 27144) }, + { AOM_CDF4(5937, 14966, 21119) }, + { AOM_CDF4(2667, 7225, 11319) }, + { AOM_CDF4(23970, 31470, 32378) }, + { AOM_CDF4(17173, 29734, 32018) }, + { AOM_CDF4(12795, 25441, 29965) }, + { AOM_CDF4(8981, 19680, 25893) }, + { AOM_CDF4(4728, 11372, 16902) }, + { AOM_CDF4(24287, 31797, 32439) }, + { AOM_CDF4(16703, 29145, 31696) }, + { AOM_CDF4(10833, 23554, 28725) }, + { AOM_CDF4(6468, 16566, 23057) }, + { AOM_CDF4(2415, 6562, 10278) }, + { AOM_CDF4(26610, 32395, 32659) }, + { AOM_CDF4(18590, 30498, 32117) }, + { AOM_CDF4(12420, 25756, 29950) }, + { AOM_CDF4(7639, 18746, 24710) }, + { AOM_CDF4(3001, 8086, 12347) }, + { AOM_CDF4(25076, 32064, 32580) }, + { AOM_CDF4(17946, 30128, 32028) }, + { AOM_CDF4(12024, 24985, 29378) }, + { AOM_CDF4(7517, 18390, 24304) }, + { AOM_CDF4(3243, 8781, 13331) }, + { AOM_CDF4(8192, 16384, 24576) } }, + { { AOM_CDF4(6037, 16771, 21957) }, + { AOM_CDF4(24774, 31704, 32426) }, + { AOM_CDF4(16830, 28589, 31056) }, + { AOM_CDF4(10602, 22828, 27760) }, + { AOM_CDF4(6733, 16829, 23071) }, + { AOM_CDF4(3250, 8914, 13556) }, + { AOM_CDF4(25582, 32220, 32668) }, + { AOM_CDF4(18659, 30342, 32223) }, + { AOM_CDF4(12546, 26149, 30515) }, + { AOM_CDF4(8420, 20451, 26801) }, + { AOM_CDF4(4636, 12420, 18344) }, + { AOM_CDF4(27581, 32362, 32639) }, + { AOM_CDF4(18987, 30083, 31978) }, + { AOM_CDF4(11327, 24248, 29084) }, + { AOM_CDF4(7264, 17719, 24120) }, + { AOM_CDF4(3995, 10768, 16169) }, + { AOM_CDF4(25893, 31831, 32487) }, + { AOM_CDF4(16577, 28587, 31379) }, + { AOM_CDF4(10189, 22748, 28182) }, + { AOM_CDF4(6832, 17094, 23556) }, + { AOM_CDF4(3708, 10110, 15334) }, + { AOM_CDF4(25904, 32282, 32656) }, + { AOM_CDF4(19721, 30792, 32276) }, + { AOM_CDF4(12819, 26243, 30411) }, + { AOM_CDF4(8572, 20614, 26891) }, + { AOM_CDF4(5364, 14059, 20467) }, + { AOM_CDF4(26580, 32438, 32677) }, + { AOM_CDF4(20852, 31225, 32340) }, + { AOM_CDF4(12435, 25700, 29967) }, + { AOM_CDF4(8691, 20825, 26976) }, + { AOM_CDF4(4446, 12209, 17269) }, + { AOM_CDF4(27350, 32429, 32696) }, + { AOM_CDF4(21372, 30977, 32272) }, + { AOM_CDF4(12673, 25270, 29853) }, + { AOM_CDF4(9208, 20925, 26640) }, + { AOM_CDF4(5018, 13351, 18732) }, + { AOM_CDF4(27351, 32479, 32713) }, + { AOM_CDF4(21398, 31209, 32387) }, + { AOM_CDF4(12162, 25047, 29842) }, + { AOM_CDF4(7896, 18691, 25319) }, + { AOM_CDF4(4670, 12882, 18881) }, + { AOM_CDF4(8192, 16384, 24576) } } }, + { { { AOM_CDF4(5487, 10460, 13708) }, + { AOM_CDF4(21597, 28303, 30674) }, + { AOM_CDF4(11037, 21953, 26476) }, + { AOM_CDF4(8147, 17962, 22952) }, + { AOM_CDF4(5242, 13061, 18532) }, + { AOM_CDF4(1889, 5208, 8182) }, + { AOM_CDF4(26774, 32133, 32590) }, + { AOM_CDF4(17844, 29564, 31767) }, + { AOM_CDF4(11690, 24438, 29171) }, + { AOM_CDF4(7542, 18215, 24459) }, + { AOM_CDF4(2993, 8050, 12319) }, + { AOM_CDF4(28023, 32328, 32591) }, + { AOM_CDF4(18651, 30126, 31954) }, + { AOM_CDF4(12164, 25146, 29589) }, + { AOM_CDF4(7762, 18530, 24771) }, + { AOM_CDF4(3492, 9183, 13920) }, + { AOM_CDF4(27591, 32008, 32491) }, + { AOM_CDF4(17149, 28853, 31510) }, + { AOM_CDF4(11485, 24003, 28860) }, + { AOM_CDF4(7697, 18086, 24210) }, + { AOM_CDF4(3075, 7999, 12218) }, + { AOM_CDF4(28268, 32482, 32654) }, + { AOM_CDF4(19631, 31051, 32404) }, + { AOM_CDF4(13860, 27260, 31020) }, + { AOM_CDF4(9605, 21613, 27594) }, + { AOM_CDF4(4876, 12162, 17908) }, + { AOM_CDF4(27248, 32316, 32576) }, + { AOM_CDF4(18955, 30457, 32075) }, + { AOM_CDF4(11824, 23997, 28795) }, + { AOM_CDF4(7346, 18196, 24647) }, + { AOM_CDF4(3403, 9247, 14111) }, + { AOM_CDF4(29711, 32655, 32735) }, + { AOM_CDF4(21169, 31394, 32417) }, + { AOM_CDF4(13487, 27198, 30957) }, + { AOM_CDF4(8828, 21683, 27614) }, + { AOM_CDF4(4270, 11451, 17038) }, + { AOM_CDF4(28708, 32578, 32731) }, + { AOM_CDF4(20120, 31241, 32482) }, + { AOM_CDF4(13692, 27550, 31321) }, + { AOM_CDF4(9418, 22514, 28439) }, + { AOM_CDF4(4999, 13283, 19462) }, + { AOM_CDF4(8192, 16384, 24576) } }, + { { AOM_CDF4(5673, 14302, 19711) }, + { AOM_CDF4(26251, 30701, 31834) }, + { AOM_CDF4(12782, 23783, 27803) }, + { AOM_CDF4(9127, 20657, 25808) }, + { AOM_CDF4(6368, 16208, 21462) }, + { AOM_CDF4(2465, 7177, 10822) }, + { AOM_CDF4(29961, 32563, 32719) }, + { AOM_CDF4(18318, 29891, 31949) }, + { AOM_CDF4(11361, 24514, 29357) }, + { AOM_CDF4(7900, 19603, 25607) }, + { AOM_CDF4(4002, 10590, 15546) }, + { AOM_CDF4(29637, 32310, 32595) }, + { AOM_CDF4(18296, 29913, 31809) }, + { AOM_CDF4(10144, 21515, 26871) }, + { AOM_CDF4(5358, 14322, 20394) }, + { AOM_CDF4(3067, 8362, 13346) }, + { AOM_CDF4(28652, 32470, 32676) }, + { AOM_CDF4(17538, 30771, 32209) }, + { AOM_CDF4(13924, 26882, 30494) }, + { AOM_CDF4(10496, 22837, 27869) }, + { AOM_CDF4(7236, 16396, 21621) }, + { AOM_CDF4(30743, 32687, 32746) }, + { AOM_CDF4(23006, 31676, 32489) }, + { AOM_CDF4(14494, 27828, 31120) }, + { AOM_CDF4(10174, 22801, 28352) }, + { AOM_CDF4(6242, 15281, 21043) }, + { AOM_CDF4(25817, 32243, 32720) }, + { AOM_CDF4(18618, 31367, 32325) }, + { AOM_CDF4(13997, 28318, 31878) }, + { AOM_CDF4(12255, 26534, 31383) }, + { AOM_CDF4(9561, 21588, 28450) }, + { AOM_CDF4(28188, 32635, 32724) }, + { AOM_CDF4(22060, 32365, 32728) }, + { AOM_CDF4(18102, 30690, 32528) }, + { AOM_CDF4(14196, 28864, 31999) }, + { AOM_CDF4(12262, 25792, 30865) }, + { AOM_CDF4(24176, 32109, 32628) }, + { AOM_CDF4(18280, 29681, 31963) }, + { AOM_CDF4(10205, 23703, 29664) }, + { AOM_CDF4(7889, 20025, 27676) }, + { AOM_CDF4(6060, 16743, 23970) }, + { AOM_CDF4(8192, 16384, 24576) } } }, + { { { AOM_CDF4(5141, 7096, 8260) }, + { AOM_CDF4(27186, 29022, 29789) }, + { AOM_CDF4(6668, 12568, 15682) }, + { AOM_CDF4(2172, 6181, 8638) }, + { AOM_CDF4(1126, 3379, 4531) }, + { AOM_CDF4(443, 1361, 2254) }, + { AOM_CDF4(26083, 31153, 32436) }, + { AOM_CDF4(13486, 24603, 28483) }, + { AOM_CDF4(6508, 14840, 19910) }, + { AOM_CDF4(3386, 8800, 13286) }, + { AOM_CDF4(1530, 4322, 7054) }, + { AOM_CDF4(29639, 32080, 32548) }, + { AOM_CDF4(15897, 27552, 30290) }, + { AOM_CDF4(8588, 20047, 25383) }, + { AOM_CDF4(4889, 13339, 19269) }, + { AOM_CDF4(2240, 6871, 10498) }, + { AOM_CDF4(28165, 32197, 32517) }, + { AOM_CDF4(20735, 30427, 31568) }, + { AOM_CDF4(14325, 24671, 27692) }, + { AOM_CDF4(5119, 12554, 17805) }, + { AOM_CDF4(1810, 5441, 8261) }, + { AOM_CDF4(31212, 32724, 32748) }, + { AOM_CDF4(23352, 31766, 32545) }, + { AOM_CDF4(14669, 27570, 31059) }, + { AOM_CDF4(8492, 20894, 27272) }, + { AOM_CDF4(3644, 10194, 15204) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) } }, + { { AOM_CDF4(2461, 7013, 9371) }, + { AOM_CDF4(24749, 29600, 30986) }, + { AOM_CDF4(9466, 19037, 22417) }, + { AOM_CDF4(3584, 9280, 14400) }, + { AOM_CDF4(1505, 3929, 5433) }, + { AOM_CDF4(677, 1500, 2736) }, + { AOM_CDF4(23987, 30702, 32117) }, + { AOM_CDF4(13554, 24571, 29263) }, + { AOM_CDF4(6211, 14556, 21155) }, + { AOM_CDF4(3135, 10972, 15625) }, + { AOM_CDF4(2435, 7127, 11427) }, + { AOM_CDF4(31300, 32532, 32550) }, + { AOM_CDF4(14757, 30365, 31954) }, + { AOM_CDF4(4405, 11612, 18553) }, + { AOM_CDF4(580, 4132, 7322) }, + { AOM_CDF4(1695, 10169, 14124) }, + { AOM_CDF4(30008, 32282, 32591) }, + { AOM_CDF4(19244, 30108, 31748) }, + { AOM_CDF4(11180, 24158, 29555) }, + { AOM_CDF4(5650, 14972, 19209) }, + { AOM_CDF4(2114, 5109, 8456) }, + { AOM_CDF4(31856, 32716, 32748) }, + { AOM_CDF4(23012, 31664, 32572) }, + { AOM_CDF4(13694, 26656, 30636) }, + { AOM_CDF4(8142, 19508, 26093) }, + { AOM_CDF4(4253, 10955, 16724) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) } } }, + { { { AOM_CDF4(601, 983, 1311) }, + { AOM_CDF4(18725, 23406, 28087) }, + { AOM_CDF4(5461, 8192, 10923) }, + { AOM_CDF4(3781, 15124, 21425) }, + { AOM_CDF4(2587, 7761, 12072) }, + { AOM_CDF4(106, 458, 810) }, + { AOM_CDF4(22282, 29710, 31894) }, + { AOM_CDF4(8508, 20926, 25984) }, + { AOM_CDF4(3726, 12713, 18083) }, + { AOM_CDF4(1620, 7112, 10893) }, + { AOM_CDF4(729, 2236, 3495) }, + { AOM_CDF4(30163, 32474, 32684) }, + { AOM_CDF4(18304, 30464, 32000) }, + { AOM_CDF4(11443, 26526, 29647) }, + { AOM_CDF4(6007, 15292, 21299) }, + { AOM_CDF4(2234, 6703, 8937) }, + { AOM_CDF4(30954, 32177, 32571) }, + { AOM_CDF4(17363, 29562, 31076) }, + { AOM_CDF4(9686, 22464, 27410) }, + { AOM_CDF4(8192, 16384, 21390) }, + { AOM_CDF4(1755, 8046, 11264) }, + { AOM_CDF4(31168, 32734, 32748) }, + { AOM_CDF4(22486, 31441, 32471) }, + { AOM_CDF4(12833, 25627, 29738) }, + { AOM_CDF4(6980, 17379, 23122) }, + { AOM_CDF4(3111, 8887, 13479) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) } }, + { { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) } } } }, + { { { { AOM_CDF4(6041, 11854, 15927) }, + { AOM_CDF4(20326, 30905, 32251) }, + { AOM_CDF4(14164, 26831, 30725) }, + { AOM_CDF4(9760, 20647, 26585) }, + { AOM_CDF4(6416, 14953, 21219) }, + { AOM_CDF4(2966, 7151, 10891) }, + { AOM_CDF4(23567, 31374, 32254) }, + { AOM_CDF4(14978, 27416, 30946) }, + { AOM_CDF4(9434, 20225, 26254) }, + { AOM_CDF4(6658, 14558, 20535) }, + { AOM_CDF4(3916, 8677, 12989) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(18088, 29545, 31587) }, + { AOM_CDF4(13062, 25843, 30073) }, + { AOM_CDF4(8940, 16827, 22251) }, + { AOM_CDF4(7654, 13220, 17973) }, + { AOM_CDF4(5733, 10316, 14456) }, + { AOM_CDF4(22879, 31388, 32114) }, + { AOM_CDF4(15215, 27993, 30955) }, + { AOM_CDF4(9397, 19445, 24978) }, + { AOM_CDF4(3442, 9813, 15344) }, + { AOM_CDF4(1368, 3936, 6532) }, + { AOM_CDF4(25494, 32033, 32406) }, + { AOM_CDF4(16772, 27963, 30718) }, + { AOM_CDF4(9419, 18165, 23260) }, + { AOM_CDF4(2677, 7501, 11797) }, + { AOM_CDF4(1516, 4344, 7170) }, + { AOM_CDF4(26556, 31454, 32101) }, + { AOM_CDF4(17128, 27035, 30108) }, + { AOM_CDF4(8324, 15344, 20249) }, + { AOM_CDF4(1903, 5696, 9469) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) } }, + { { AOM_CDF4(8455, 19003, 24368) }, + { AOM_CDF4(23563, 32021, 32604) }, + { AOM_CDF4(16237, 29446, 31935) }, + { AOM_CDF4(10724, 23999, 29358) }, + { AOM_CDF4(6725, 17528, 24416) }, + { AOM_CDF4(3927, 10927, 16825) }, + { AOM_CDF4(26313, 32288, 32634) }, + { AOM_CDF4(17430, 30095, 32095) }, + { AOM_CDF4(11116, 24606, 29679) }, + { AOM_CDF4(7195, 18384, 25269) }, + { AOM_CDF4(4726, 12852, 19315) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(22822, 31648, 32483) }, + { AOM_CDF4(16724, 29633, 31929) }, + { AOM_CDF4(10261, 23033, 28725) }, + { AOM_CDF4(7029, 17840, 24528) }, + { AOM_CDF4(4867, 13886, 21502) }, + { AOM_CDF4(25298, 31892, 32491) }, + { AOM_CDF4(17809, 29330, 31512) }, + { AOM_CDF4(9668, 21329, 26579) }, + { AOM_CDF4(4774, 12956, 18976) }, + { AOM_CDF4(2322, 7030, 11540) }, + { AOM_CDF4(25472, 31920, 32543) }, + { AOM_CDF4(17957, 29387, 31632) }, + { AOM_CDF4(9196, 20593, 26400) }, + { AOM_CDF4(4680, 12705, 19202) }, + { AOM_CDF4(2917, 8456, 13436) }, + { AOM_CDF4(26471, 32059, 32574) }, + { AOM_CDF4(18458, 29783, 31909) }, + { AOM_CDF4(8400, 19464, 25956) }, + { AOM_CDF4(3812, 10973, 17206) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) } } }, + { { { AOM_CDF4(6779, 13743, 17678) }, + { AOM_CDF4(24806, 31797, 32457) }, + { AOM_CDF4(17616, 29047, 31372) }, + { AOM_CDF4(11063, 23175, 28003) }, + { AOM_CDF4(6521, 16110, 22324) }, + { AOM_CDF4(2764, 7504, 11654) }, + { AOM_CDF4(25266, 32367, 32637) }, + { AOM_CDF4(19054, 30553, 32175) }, + { AOM_CDF4(12139, 25212, 29807) }, + { AOM_CDF4(7311, 18162, 24704) }, + { AOM_CDF4(3397, 9164, 14074) }, + { AOM_CDF4(25988, 32208, 32522) }, + { AOM_CDF4(16253, 28912, 31526) }, + { AOM_CDF4(9151, 21387, 27372) }, + { AOM_CDF4(5688, 14915, 21496) }, + { AOM_CDF4(2717, 7627, 12004) }, + { AOM_CDF4(23144, 31855, 32443) }, + { AOM_CDF4(16070, 28491, 31325) }, + { AOM_CDF4(8702, 20467, 26517) }, + { AOM_CDF4(5243, 13956, 20367) }, + { AOM_CDF4(2621, 7335, 11567) }, + { AOM_CDF4(26636, 32340, 32630) }, + { AOM_CDF4(19990, 31050, 32341) }, + { AOM_CDF4(13243, 26105, 30315) }, + { AOM_CDF4(8588, 19521, 25918) }, + { AOM_CDF4(4717, 11585, 17304) }, + { AOM_CDF4(25844, 32292, 32582) }, + { AOM_CDF4(19090, 30635, 32097) }, + { AOM_CDF4(11963, 24546, 28939) }, + { AOM_CDF4(6218, 16087, 22354) }, + { AOM_CDF4(2340, 6608, 10426) }, + { AOM_CDF4(28046, 32576, 32694) }, + { AOM_CDF4(21178, 31313, 32296) }, + { AOM_CDF4(13486, 26184, 29870) }, + { AOM_CDF4(7149, 17871, 23723) }, + { AOM_CDF4(2833, 7958, 12259) }, + { AOM_CDF4(27710, 32528, 32686) }, + { AOM_CDF4(20674, 31076, 32268) }, + { AOM_CDF4(12413, 24955, 29243) }, + { AOM_CDF4(6676, 16927, 23097) }, + { AOM_CDF4(2966, 8333, 12919) }, + { AOM_CDF4(8192, 16384, 24576) } }, + { { AOM_CDF4(8639, 19339, 24429) }, + { AOM_CDF4(24404, 31837, 32525) }, + { AOM_CDF4(16997, 29425, 31784) }, + { AOM_CDF4(11253, 24234, 29149) }, + { AOM_CDF4(6751, 17394, 24028) }, + { AOM_CDF4(3490, 9830, 15191) }, + { AOM_CDF4(26283, 32471, 32714) }, + { AOM_CDF4(19599, 31168, 32442) }, + { AOM_CDF4(13146, 26954, 30893) }, + { AOM_CDF4(8214, 20588, 26890) }, + { AOM_CDF4(4699, 13081, 19300) }, + { AOM_CDF4(28212, 32458, 32669) }, + { AOM_CDF4(18594, 30316, 32100) }, + { AOM_CDF4(11219, 24408, 29234) }, + { AOM_CDF4(6865, 17656, 24149) }, + { AOM_CDF4(3678, 10362, 16006) }, + { AOM_CDF4(25825, 32136, 32616) }, + { AOM_CDF4(17313, 29853, 32021) }, + { AOM_CDF4(11197, 24471, 29472) }, + { AOM_CDF4(6947, 17781, 24405) }, + { AOM_CDF4(3768, 10660, 16261) }, + { AOM_CDF4(27352, 32500, 32706) }, + { AOM_CDF4(20850, 31468, 32469) }, + { AOM_CDF4(14021, 27707, 31133) }, + { AOM_CDF4(8964, 21748, 27838) }, + { AOM_CDF4(5437, 14665, 21187) }, + { AOM_CDF4(26304, 32492, 32698) }, + { AOM_CDF4(20409, 31380, 32385) }, + { AOM_CDF4(13682, 27222, 30632) }, + { AOM_CDF4(8974, 21236, 26685) }, + { AOM_CDF4(4234, 11665, 16934) }, + { AOM_CDF4(26273, 32357, 32711) }, + { AOM_CDF4(20672, 31242, 32441) }, + { AOM_CDF4(14172, 27254, 30902) }, + { AOM_CDF4(9870, 21898, 27275) }, + { AOM_CDF4(5164, 13506, 19270) }, + { AOM_CDF4(26725, 32459, 32728) }, + { AOM_CDF4(20991, 31442, 32527) }, + { AOM_CDF4(13071, 26434, 30811) }, + { AOM_CDF4(8184, 20090, 26742) }, + { AOM_CDF4(4803, 13255, 19895) }, + { AOM_CDF4(8192, 16384, 24576) } } }, + { { { AOM_CDF4(7555, 14942, 18501) }, + { AOM_CDF4(24410, 31178, 32287) }, + { AOM_CDF4(14394, 26738, 30253) }, + { AOM_CDF4(8413, 19554, 25195) }, + { AOM_CDF4(4766, 12924, 18785) }, + { AOM_CDF4(2029, 5806, 9207) }, + { AOM_CDF4(26776, 32364, 32663) }, + { AOM_CDF4(18732, 29967, 31931) }, + { AOM_CDF4(11005, 23786, 28852) }, + { AOM_CDF4(6466, 16909, 23510) }, + { AOM_CDF4(3044, 8638, 13419) }, + { AOM_CDF4(29208, 32582, 32704) }, + { AOM_CDF4(20068, 30857, 32208) }, + { AOM_CDF4(12003, 25085, 29595) }, + { AOM_CDF4(6947, 17750, 24189) }, + { AOM_CDF4(3245, 9103, 14007) }, + { AOM_CDF4(27359, 32465, 32669) }, + { AOM_CDF4(19421, 30614, 32174) }, + { AOM_CDF4(11915, 25010, 29579) }, + { AOM_CDF4(6950, 17676, 24074) }, + { AOM_CDF4(3007, 8473, 13096) }, + { AOM_CDF4(29002, 32676, 32735) }, + { AOM_CDF4(22102, 31849, 32576) }, + { AOM_CDF4(14408, 28009, 31405) }, + { AOM_CDF4(9027, 21679, 27931) }, + { AOM_CDF4(4694, 12678, 18748) }, + { AOM_CDF4(28216, 32528, 32682) }, + { AOM_CDF4(20849, 31264, 32318) }, + { AOM_CDF4(12756, 25815, 29751) }, + { AOM_CDF4(7565, 18801, 24923) }, + { AOM_CDF4(3509, 9533, 14477) }, + { AOM_CDF4(30133, 32687, 32739) }, + { AOM_CDF4(23063, 31910, 32515) }, + { AOM_CDF4(14588, 28051, 31132) }, + { AOM_CDF4(9085, 21649, 27457) }, + { AOM_CDF4(4261, 11654, 17264) }, + { AOM_CDF4(29518, 32691, 32748) }, + { AOM_CDF4(22451, 31959, 32613) }, + { AOM_CDF4(14864, 28722, 31700) }, + { AOM_CDF4(9695, 22964, 28716) }, + { AOM_CDF4(4932, 13358, 19502) }, + { AOM_CDF4(8192, 16384, 24576) } }, + { { AOM_CDF4(6465, 16958, 21688) }, + { AOM_CDF4(25199, 31514, 32360) }, + { AOM_CDF4(14774, 27149, 30607) }, + { AOM_CDF4(9257, 21438, 26972) }, + { AOM_CDF4(5723, 15183, 21882) }, + { AOM_CDF4(3150, 8879, 13731) }, + { AOM_CDF4(26989, 32262, 32682) }, + { AOM_CDF4(17396, 29937, 32085) }, + { AOM_CDF4(11387, 24901, 29784) }, + { AOM_CDF4(7289, 18821, 25548) }, + { AOM_CDF4(3734, 10577, 16086) }, + { AOM_CDF4(29728, 32501, 32695) }, + { AOM_CDF4(17431, 29701, 31903) }, + { AOM_CDF4(9921, 22826, 28300) }, + { AOM_CDF4(5896, 15434, 22068) }, + { AOM_CDF4(3430, 9646, 14757) }, + { AOM_CDF4(28614, 32511, 32705) }, + { AOM_CDF4(19364, 30638, 32263) }, + { AOM_CDF4(13129, 26254, 30402) }, + { AOM_CDF4(8754, 20484, 26440) }, + { AOM_CDF4(4378, 11607, 17110) }, + { AOM_CDF4(30292, 32671, 32744) }, + { AOM_CDF4(21780, 31603, 32501) }, + { AOM_CDF4(14314, 27829, 31291) }, + { AOM_CDF4(9611, 22327, 28263) }, + { AOM_CDF4(4890, 13087, 19065) }, + { AOM_CDF4(25862, 32567, 32733) }, + { AOM_CDF4(20794, 32050, 32567) }, + { AOM_CDF4(17243, 30625, 32254) }, + { AOM_CDF4(13283, 27628, 31474) }, + { AOM_CDF4(9669, 22532, 28918) }, + { AOM_CDF4(27435, 32697, 32748) }, + { AOM_CDF4(24922, 32390, 32714) }, + { AOM_CDF4(21449, 31504, 32536) }, + { AOM_CDF4(16392, 29729, 31832) }, + { AOM_CDF4(11692, 24884, 29076) }, + { AOM_CDF4(24193, 32290, 32735) }, + { AOM_CDF4(18909, 31104, 32563) }, + { AOM_CDF4(12236, 26841, 31403) }, + { AOM_CDF4(8171, 21840, 29082) }, + { AOM_CDF4(7224, 17280, 25275) }, + { AOM_CDF4(8192, 16384, 24576) } } }, + { { { AOM_CDF4(3078, 6839, 9890) }, + { AOM_CDF4(13837, 20450, 24479) }, + { AOM_CDF4(5914, 14222, 19328) }, + { AOM_CDF4(3866, 10267, 14762) }, + { AOM_CDF4(2612, 7208, 11042) }, + { AOM_CDF4(1067, 2991, 4776) }, + { AOM_CDF4(25817, 31646, 32529) }, + { AOM_CDF4(13708, 26338, 30385) }, + { AOM_CDF4(7328, 18585, 24870) }, + { AOM_CDF4(4691, 13080, 19276) }, + { AOM_CDF4(1825, 5253, 8352) }, + { AOM_CDF4(29386, 32315, 32624) }, + { AOM_CDF4(17160, 29001, 31360) }, + { AOM_CDF4(9602, 21862, 27396) }, + { AOM_CDF4(5915, 15772, 22148) }, + { AOM_CDF4(2786, 7779, 12047) }, + { AOM_CDF4(29246, 32450, 32663) }, + { AOM_CDF4(18696, 29929, 31818) }, + { AOM_CDF4(10510, 23369, 28560) }, + { AOM_CDF4(6229, 16499, 23125) }, + { AOM_CDF4(2608, 7448, 11705) }, + { AOM_CDF4(30753, 32710, 32748) }, + { AOM_CDF4(21638, 31487, 32503) }, + { AOM_CDF4(12937, 26854, 30870) }, + { AOM_CDF4(8182, 20596, 26970) }, + { AOM_CDF4(3637, 10269, 15497) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) } }, + { { AOM_CDF4(5244, 12150, 16906) }, + { AOM_CDF4(20486, 26858, 29701) }, + { AOM_CDF4(7756, 18317, 23735) }, + { AOM_CDF4(3452, 9256, 13146) }, + { AOM_CDF4(2020, 5206, 8229) }, + { AOM_CDF4(1801, 4993, 7903) }, + { AOM_CDF4(27051, 31858, 32531) }, + { AOM_CDF4(15988, 27531, 30619) }, + { AOM_CDF4(9188, 21484, 26719) }, + { AOM_CDF4(6273, 17186, 23800) }, + { AOM_CDF4(3108, 9355, 14764) }, + { AOM_CDF4(31076, 32520, 32680) }, + { AOM_CDF4(18119, 30037, 31850) }, + { AOM_CDF4(10244, 22969, 27472) }, + { AOM_CDF4(4692, 14077, 19273) }, + { AOM_CDF4(3694, 11677, 17556) }, + { AOM_CDF4(30060, 32581, 32720) }, + { AOM_CDF4(21011, 30775, 32120) }, + { AOM_CDF4(11931, 24820, 29289) }, + { AOM_CDF4(7119, 17662, 24356) }, + { AOM_CDF4(3833, 10706, 16304) }, + { AOM_CDF4(31954, 32731, 32748) }, + { AOM_CDF4(23913, 31724, 32489) }, + { AOM_CDF4(15520, 28060, 31286) }, + { AOM_CDF4(11517, 23008, 28571) }, + { AOM_CDF4(6193, 14508, 20629) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) } } }, + { { { AOM_CDF4(1035, 2807, 4156) }, + { AOM_CDF4(13162, 18138, 20939) }, + { AOM_CDF4(2696, 6633, 8755) }, + { AOM_CDF4(1373, 4161, 6853) }, + { AOM_CDF4(1099, 2746, 4716) }, + { AOM_CDF4(340, 1021, 1599) }, + { AOM_CDF4(22826, 30419, 32135) }, + { AOM_CDF4(10395, 21762, 26942) }, + { AOM_CDF4(4726, 12407, 17361) }, + { AOM_CDF4(2447, 7080, 10593) }, + { AOM_CDF4(1227, 3717, 6011) }, + { AOM_CDF4(28156, 31424, 31934) }, + { AOM_CDF4(16915, 27754, 30373) }, + { AOM_CDF4(9148, 20990, 26431) }, + { AOM_CDF4(5950, 15515, 21148) }, + { AOM_CDF4(2492, 7327, 11526) }, + { AOM_CDF4(30602, 32477, 32670) }, + { AOM_CDF4(20026, 29955, 31568) }, + { AOM_CDF4(11220, 23628, 28105) }, + { AOM_CDF4(6652, 17019, 22973) }, + { AOM_CDF4(3064, 8536, 13043) }, + { AOM_CDF4(31769, 32724, 32748) }, + { AOM_CDF4(22230, 30887, 32373) }, + { AOM_CDF4(12234, 25079, 29731) }, + { AOM_CDF4(7326, 18816, 25353) }, + { AOM_CDF4(3933, 10907, 16616) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) } }, + { { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) } } } }, + { { { { AOM_CDF4(8896, 16227, 20630) }, + { AOM_CDF4(23629, 31782, 32527) }, + { AOM_CDF4(15173, 27755, 31321) }, + { AOM_CDF4(10158, 21233, 27382) }, + { AOM_CDF4(6420, 14857, 21558) }, + { AOM_CDF4(3269, 8155, 12646) }, + { AOM_CDF4(24835, 32009, 32496) }, + { AOM_CDF4(16509, 28421, 31579) }, + { AOM_CDF4(10957, 21514, 27418) }, + { AOM_CDF4(7881, 15930, 22096) }, + { AOM_CDF4(5388, 10960, 15918) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(20745, 30773, 32093) }, + { AOM_CDF4(15200, 27221, 30861) }, + { AOM_CDF4(13032, 20873, 25667) }, + { AOM_CDF4(12285, 18663, 23494) }, + { AOM_CDF4(11563, 17481, 21489) }, + { AOM_CDF4(26260, 31982, 32320) }, + { AOM_CDF4(15397, 28083, 31100) }, + { AOM_CDF4(9742, 19217, 24824) }, + { AOM_CDF4(3261, 9629, 15362) }, + { AOM_CDF4(1480, 4322, 7499) }, + { AOM_CDF4(27599, 32256, 32460) }, + { AOM_CDF4(16857, 27659, 30774) }, + { AOM_CDF4(9551, 18290, 23748) }, + { AOM_CDF4(3052, 8933, 14103) }, + { AOM_CDF4(2021, 5910, 9787) }, + { AOM_CDF4(29005, 32015, 32392) }, + { AOM_CDF4(17677, 27694, 30863) }, + { AOM_CDF4(9204, 17356, 23219) }, + { AOM_CDF4(2403, 7516, 12814) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) } }, + { { AOM_CDF4(10808, 22056, 26896) }, + { AOM_CDF4(25739, 32313, 32676) }, + { AOM_CDF4(17288, 30203, 32221) }, + { AOM_CDF4(11359, 24878, 29896) }, + { AOM_CDF4(6949, 17767, 24893) }, + { AOM_CDF4(4287, 11796, 18071) }, + { AOM_CDF4(27880, 32521, 32705) }, + { AOM_CDF4(19038, 31004, 32414) }, + { AOM_CDF4(12564, 26345, 30768) }, + { AOM_CDF4(8269, 19947, 26779) }, + { AOM_CDF4(5674, 14657, 21674) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(25742, 32319, 32671) }, + { AOM_CDF4(19557, 31164, 32454) }, + { AOM_CDF4(13381, 26381, 30755) }, + { AOM_CDF4(10101, 21466, 26722) }, + { AOM_CDF4(9209, 19650, 26825) }, + { AOM_CDF4(27107, 31917, 32432) }, + { AOM_CDF4(18056, 28893, 31203) }, + { AOM_CDF4(10200, 21434, 26764) }, + { AOM_CDF4(4660, 12913, 19502) }, + { AOM_CDF4(2368, 6930, 12504) }, + { AOM_CDF4(26960, 32158, 32613) }, + { AOM_CDF4(18628, 30005, 32031) }, + { AOM_CDF4(10233, 22442, 28232) }, + { AOM_CDF4(5471, 14630, 21516) }, + { AOM_CDF4(3235, 10767, 17109) }, + { AOM_CDF4(27696, 32440, 32692) }, + { AOM_CDF4(20032, 31167, 32438) }, + { AOM_CDF4(8700, 21341, 28442) }, + { AOM_CDF4(5662, 14831, 21795) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) } } }, + { { { AOM_CDF4(9704, 17294, 21132) }, + { AOM_CDF4(26762, 32278, 32633) }, + { AOM_CDF4(18382, 29620, 31819) }, + { AOM_CDF4(10891, 23475, 28723) }, + { AOM_CDF4(6358, 16583, 23309) }, + { AOM_CDF4(3248, 9118, 14141) }, + { AOM_CDF4(27204, 32573, 32699) }, + { AOM_CDF4(19818, 30824, 32329) }, + { AOM_CDF4(11772, 25120, 30041) }, + { AOM_CDF4(6995, 18033, 25039) }, + { AOM_CDF4(3752, 10442, 16098) }, + { AOM_CDF4(27222, 32256, 32559) }, + { AOM_CDF4(15356, 28399, 31475) }, + { AOM_CDF4(8821, 20635, 27057) }, + { AOM_CDF4(5511, 14404, 21239) }, + { AOM_CDF4(2935, 8222, 13051) }, + { AOM_CDF4(24875, 32120, 32529) }, + { AOM_CDF4(15233, 28265, 31445) }, + { AOM_CDF4(8605, 20570, 26932) }, + { AOM_CDF4(5431, 14413, 21196) }, + { AOM_CDF4(2994, 8341, 13223) }, + { AOM_CDF4(28201, 32604, 32700) }, + { AOM_CDF4(21041, 31446, 32456) }, + { AOM_CDF4(13221, 26213, 30475) }, + { AOM_CDF4(8255, 19385, 26037) }, + { AOM_CDF4(4930, 12585, 18830) }, + { AOM_CDF4(28768, 32448, 32627) }, + { AOM_CDF4(19705, 30561, 32021) }, + { AOM_CDF4(11572, 23589, 28220) }, + { AOM_CDF4(5532, 15034, 21446) }, + { AOM_CDF4(2460, 7150, 11456) }, + { AOM_CDF4(29874, 32619, 32699) }, + { AOM_CDF4(21621, 31071, 32201) }, + { AOM_CDF4(12511, 24747, 28992) }, + { AOM_CDF4(6281, 16395, 22748) }, + { AOM_CDF4(3246, 9278, 14497) }, + { AOM_CDF4(29715, 32625, 32712) }, + { AOM_CDF4(20958, 31011, 32283) }, + { AOM_CDF4(11233, 23671, 28806) }, + { AOM_CDF4(6012, 16128, 22868) }, + { AOM_CDF4(3427, 9851, 15414) }, + { AOM_CDF4(8192, 16384, 24576) } }, + { { AOM_CDF4(11016, 22111, 26794) }, + { AOM_CDF4(25946, 32357, 32677) }, + { AOM_CDF4(17890, 30452, 32252) }, + { AOM_CDF4(11678, 25142, 29816) }, + { AOM_CDF4(6720, 17534, 24584) }, + { AOM_CDF4(4230, 11665, 17820) }, + { AOM_CDF4(28400, 32623, 32747) }, + { AOM_CDF4(21164, 31668, 32575) }, + { AOM_CDF4(13572, 27388, 31182) }, + { AOM_CDF4(8234, 20750, 27358) }, + { AOM_CDF4(5065, 14055, 20897) }, + { AOM_CDF4(28981, 32547, 32705) }, + { AOM_CDF4(18681, 30543, 32239) }, + { AOM_CDF4(10919, 24075, 29286) }, + { AOM_CDF4(6431, 17199, 24077) }, + { AOM_CDF4(3819, 10464, 16618) }, + { AOM_CDF4(26870, 32467, 32693) }, + { AOM_CDF4(19041, 30831, 32347) }, + { AOM_CDF4(11794, 25211, 30016) }, + { AOM_CDF4(6888, 18019, 24970) }, + { AOM_CDF4(4370, 12363, 18992) }, + { AOM_CDF4(29578, 32670, 32744) }, + { AOM_CDF4(23159, 32007, 32613) }, + { AOM_CDF4(15315, 28669, 31676) }, + { AOM_CDF4(9298, 22607, 28782) }, + { AOM_CDF4(6144, 15913, 22968) }, + { AOM_CDF4(28110, 32499, 32669) }, + { AOM_CDF4(21574, 30937, 32015) }, + { AOM_CDF4(12759, 24818, 28727) }, + { AOM_CDF4(6545, 16761, 23042) }, + { AOM_CDF4(3649, 10597, 16833) }, + { AOM_CDF4(28163, 32552, 32728) }, + { AOM_CDF4(22101, 31469, 32464) }, + { AOM_CDF4(13160, 25472, 30143) }, + { AOM_CDF4(7303, 18684, 25468) }, + { AOM_CDF4(5241, 13975, 20955) }, + { AOM_CDF4(28400, 32631, 32744) }, + { AOM_CDF4(22104, 31793, 32603) }, + { AOM_CDF4(13557, 26571, 30846) }, + { AOM_CDF4(7749, 19861, 26675) }, + { AOM_CDF4(4873, 14030, 21234) }, + { AOM_CDF4(8192, 16384, 24576) } } }, + { { { AOM_CDF4(9800, 17635, 21073) }, + { AOM_CDF4(26153, 31885, 32527) }, + { AOM_CDF4(15038, 27852, 31006) }, + { AOM_CDF4(8718, 20564, 26486) }, + { AOM_CDF4(5128, 14076, 20514) }, + { AOM_CDF4(2636, 7566, 11925) }, + { AOM_CDF4(27551, 32504, 32701) }, + { AOM_CDF4(18310, 30054, 32100) }, + { AOM_CDF4(10211, 23420, 29082) }, + { AOM_CDF4(6222, 16876, 23916) }, + { AOM_CDF4(3462, 9954, 15498) }, + { AOM_CDF4(29991, 32633, 32721) }, + { AOM_CDF4(19883, 30751, 32201) }, + { AOM_CDF4(11141, 24184, 29285) }, + { AOM_CDF4(6420, 16940, 23774) }, + { AOM_CDF4(3392, 9753, 15118) }, + { AOM_CDF4(28465, 32616, 32712) }, + { AOM_CDF4(19850, 30702, 32244) }, + { AOM_CDF4(10983, 24024, 29223) }, + { AOM_CDF4(6294, 16770, 23582) }, + { AOM_CDF4(3244, 9283, 14509) }, + { AOM_CDF4(30023, 32717, 32748) }, + { AOM_CDF4(22940, 32032, 32626) }, + { AOM_CDF4(14282, 27928, 31473) }, + { AOM_CDF4(8562, 21327, 27914) }, + { AOM_CDF4(4846, 13393, 19919) }, + { AOM_CDF4(29981, 32590, 32695) }, + { AOM_CDF4(20465, 30963, 32166) }, + { AOM_CDF4(11479, 23579, 28195) }, + { AOM_CDF4(5916, 15648, 22073) }, + { AOM_CDF4(3031, 8605, 13398) }, + { AOM_CDF4(31146, 32691, 32739) }, + { AOM_CDF4(23106, 31724, 32444) }, + { AOM_CDF4(13783, 26738, 30439) }, + { AOM_CDF4(7852, 19468, 25807) }, + { AOM_CDF4(3860, 11124, 16853) }, + { AOM_CDF4(31014, 32724, 32748) }, + { AOM_CDF4(23629, 32109, 32628) }, + { AOM_CDF4(14747, 28115, 31403) }, + { AOM_CDF4(8545, 21242, 27478) }, + { AOM_CDF4(4574, 12781, 19067) }, + { AOM_CDF4(8192, 16384, 24576) } }, + { { AOM_CDF4(9185, 19694, 24688) }, + { AOM_CDF4(26081, 31985, 32621) }, + { AOM_CDF4(16015, 29000, 31787) }, + { AOM_CDF4(10542, 23690, 29206) }, + { AOM_CDF4(6732, 17945, 24677) }, + { AOM_CDF4(3916, 11039, 16722) }, + { AOM_CDF4(28224, 32566, 32744) }, + { AOM_CDF4(19100, 31138, 32485) }, + { AOM_CDF4(12528, 26620, 30879) }, + { AOM_CDF4(7741, 20277, 26885) }, + { AOM_CDF4(4566, 12845, 18990) }, + { AOM_CDF4(29933, 32593, 32718) }, + { AOM_CDF4(17670, 30333, 32155) }, + { AOM_CDF4(10385, 23600, 28909) }, + { AOM_CDF4(6243, 16236, 22407) }, + { AOM_CDF4(3976, 10389, 16017) }, + { AOM_CDF4(28377, 32561, 32738) }, + { AOM_CDF4(19366, 31175, 32482) }, + { AOM_CDF4(13327, 27175, 31094) }, + { AOM_CDF4(8258, 20769, 27143) }, + { AOM_CDF4(4703, 13198, 19527) }, + { AOM_CDF4(31086, 32706, 32748) }, + { AOM_CDF4(22853, 31902, 32583) }, + { AOM_CDF4(14759, 28186, 31419) }, + { AOM_CDF4(9284, 22382, 28348) }, + { AOM_CDF4(5585, 15192, 21868) }, + { AOM_CDF4(28291, 32652, 32746) }, + { AOM_CDF4(19849, 32107, 32571) }, + { AOM_CDF4(14834, 26818, 29214) }, + { AOM_CDF4(10306, 22594, 28672) }, + { AOM_CDF4(6615, 17384, 23384) }, + { AOM_CDF4(28947, 32604, 32745) }, + { AOM_CDF4(25625, 32289, 32646) }, + { AOM_CDF4(18758, 28672, 31403) }, + { AOM_CDF4(10017, 23430, 28523) }, + { AOM_CDF4(6862, 15269, 22131) }, + { AOM_CDF4(23933, 32509, 32739) }, + { AOM_CDF4(19927, 31495, 32631) }, + { AOM_CDF4(11903, 26023, 30621) }, + { AOM_CDF4(7026, 20094, 27252) }, + { AOM_CDF4(5998, 18106, 24437) }, + { AOM_CDF4(8192, 16384, 24576) } } }, + { { { AOM_CDF4(4456, 11274, 15533) }, + { AOM_CDF4(21219, 29079, 31616) }, + { AOM_CDF4(11173, 23774, 28567) }, + { AOM_CDF4(7282, 18293, 24263) }, + { AOM_CDF4(4890, 13286, 19115) }, + { AOM_CDF4(1890, 5508, 8659) }, + { AOM_CDF4(26651, 32136, 32647) }, + { AOM_CDF4(14630, 28254, 31455) }, + { AOM_CDF4(8716, 21287, 27395) }, + { AOM_CDF4(5615, 15331, 22008) }, + { AOM_CDF4(2675, 7700, 12150) }, + { AOM_CDF4(29954, 32526, 32690) }, + { AOM_CDF4(16126, 28982, 31633) }, + { AOM_CDF4(9030, 21361, 27352) }, + { AOM_CDF4(5411, 14793, 21271) }, + { AOM_CDF4(2943, 8422, 13163) }, + { AOM_CDF4(29539, 32601, 32730) }, + { AOM_CDF4(18125, 30385, 32201) }, + { AOM_CDF4(10422, 24090, 29468) }, + { AOM_CDF4(6468, 17487, 24438) }, + { AOM_CDF4(2970, 8653, 13531) }, + { AOM_CDF4(30912, 32715, 32748) }, + { AOM_CDF4(20666, 31373, 32497) }, + { AOM_CDF4(12509, 26640, 30917) }, + { AOM_CDF4(8058, 20629, 27290) }, + { AOM_CDF4(4231, 12006, 18052) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) } }, + { { AOM_CDF4(10202, 20633, 25484) }, + { AOM_CDF4(27336, 31445, 32352) }, + { AOM_CDF4(12420, 24384, 28552) }, + { AOM_CDF4(7648, 18115, 23856) }, + { AOM_CDF4(5662, 14341, 19902) }, + { AOM_CDF4(3611, 10328, 15390) }, + { AOM_CDF4(30945, 32616, 32736) }, + { AOM_CDF4(18682, 30505, 32253) }, + { AOM_CDF4(11513, 25336, 30203) }, + { AOM_CDF4(7449, 19452, 26148) }, + { AOM_CDF4(4482, 13051, 18886) }, + { AOM_CDF4(32022, 32690, 32747) }, + { AOM_CDF4(18578, 30501, 32146) }, + { AOM_CDF4(11249, 23368, 28631) }, + { AOM_CDF4(5645, 16958, 22158) }, + { AOM_CDF4(5009, 11444, 16637) }, + { AOM_CDF4(31357, 32710, 32748) }, + { AOM_CDF4(21552, 31494, 32504) }, + { AOM_CDF4(13891, 27677, 31340) }, + { AOM_CDF4(9051, 22098, 28172) }, + { AOM_CDF4(5190, 13377, 19486) }, + { AOM_CDF4(32364, 32740, 32748) }, + { AOM_CDF4(24839, 31907, 32551) }, + { AOM_CDF4(17160, 28779, 31696) }, + { AOM_CDF4(12452, 24137, 29602) }, + { AOM_CDF4(6165, 15389, 22477) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) } } }, + { { { AOM_CDF4(2575, 7281, 11077) }, + { AOM_CDF4(14002, 20866, 25402) }, + { AOM_CDF4(6343, 15056, 19658) }, + { AOM_CDF4(4474, 11858, 17041) }, + { AOM_CDF4(2865, 8299, 12534) }, + { AOM_CDF4(1344, 3949, 6391) }, + { AOM_CDF4(24720, 31239, 32459) }, + { AOM_CDF4(12585, 25356, 29968) }, + { AOM_CDF4(7181, 18246, 24444) }, + { AOM_CDF4(5025, 13667, 19885) }, + { AOM_CDF4(2521, 7304, 11605) }, + { AOM_CDF4(29908, 32252, 32584) }, + { AOM_CDF4(17421, 29156, 31575) }, + { AOM_CDF4(9889, 22188, 27782) }, + { AOM_CDF4(5878, 15647, 22123) }, + { AOM_CDF4(2814, 8665, 13323) }, + { AOM_CDF4(30183, 32568, 32713) }, + { AOM_CDF4(18528, 30195, 32049) }, + { AOM_CDF4(10982, 24606, 29657) }, + { AOM_CDF4(6957, 18165, 25231) }, + { AOM_CDF4(3508, 10118, 15468) }, + { AOM_CDF4(31761, 32736, 32748) }, + { AOM_CDF4(21041, 31328, 32546) }, + { AOM_CDF4(12568, 26732, 31166) }, + { AOM_CDF4(8052, 20720, 27733) }, + { AOM_CDF4(4336, 12192, 18396) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) } }, + { { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) } } } }, + { { { { AOM_CDF4(7062, 16472, 22319) }, + { AOM_CDF4(24538, 32261, 32674) }, + { AOM_CDF4(13675, 28041, 31779) }, + { AOM_CDF4(8590, 20674, 27631) }, + { AOM_CDF4(5685, 14675, 22013) }, + { AOM_CDF4(3655, 9898, 15731) }, + { AOM_CDF4(26493, 32418, 32658) }, + { AOM_CDF4(16376, 29342, 32090) }, + { AOM_CDF4(10594, 22649, 28970) }, + { AOM_CDF4(8176, 17170, 24303) }, + { AOM_CDF4(5605, 12694, 19139) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(23888, 31902, 32542) }, + { AOM_CDF4(18612, 29687, 31987) }, + { AOM_CDF4(16245, 24852, 29249) }, + { AOM_CDF4(15765, 22608, 27559) }, + { AOM_CDF4(19895, 24699, 27510) }, + { AOM_CDF4(28401, 32212, 32457) }, + { AOM_CDF4(15274, 27825, 30980) }, + { AOM_CDF4(9364, 18128, 24332) }, + { AOM_CDF4(2283, 8193, 15082) }, + { AOM_CDF4(1228, 3972, 7881) }, + { AOM_CDF4(29455, 32469, 32620) }, + { AOM_CDF4(17981, 28245, 31388) }, + { AOM_CDF4(10921, 20098, 26240) }, + { AOM_CDF4(3743, 11829, 18657) }, + { AOM_CDF4(2374, 9593, 15715) }, + { AOM_CDF4(31068, 32466, 32635) }, + { AOM_CDF4(20321, 29572, 31971) }, + { AOM_CDF4(10771, 20255, 27119) }, + { AOM_CDF4(2795, 10410, 17361) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) } }, + { { AOM_CDF4(9320, 22102, 27840) }, + { AOM_CDF4(27057, 32464, 32724) }, + { AOM_CDF4(16331, 30268, 32309) }, + { AOM_CDF4(10319, 23935, 29720) }, + { AOM_CDF4(6189, 16448, 24106) }, + { AOM_CDF4(3589, 10884, 18808) }, + { AOM_CDF4(29026, 32624, 32748) }, + { AOM_CDF4(19226, 31507, 32587) }, + { AOM_CDF4(12692, 26921, 31203) }, + { AOM_CDF4(7049, 19532, 27635) }, + { AOM_CDF4(7727, 15669, 23252) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(28056, 32625, 32748) }, + { AOM_CDF4(22383, 32075, 32669) }, + { AOM_CDF4(15417, 27098, 31749) }, + { AOM_CDF4(18127, 26493, 27190) }, + { AOM_CDF4(5461, 16384, 21845) }, + { AOM_CDF4(27982, 32091, 32584) }, + { AOM_CDF4(19045, 29868, 31972) }, + { AOM_CDF4(10397, 22266, 27932) }, + { AOM_CDF4(5990, 13697, 21500) }, + { AOM_CDF4(1792, 6912, 15104) }, + { AOM_CDF4(28198, 32501, 32718) }, + { AOM_CDF4(21534, 31521, 32569) }, + { AOM_CDF4(11109, 25217, 30017) }, + { AOM_CDF4(5671, 15124, 26151) }, + { AOM_CDF4(4681, 14043, 18725) }, + { AOM_CDF4(28688, 32580, 32741) }, + { AOM_CDF4(22576, 32079, 32661) }, + { AOM_CDF4(10627, 22141, 28340) }, + { AOM_CDF4(9362, 14043, 28087) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) } } }, + { { { AOM_CDF4(7754, 16948, 22142) }, + { AOM_CDF4(25670, 32330, 32691) }, + { AOM_CDF4(15663, 29225, 31994) }, + { AOM_CDF4(9878, 23288, 29158) }, + { AOM_CDF4(6419, 17088, 24336) }, + { AOM_CDF4(3859, 11003, 17039) }, + { AOM_CDF4(27562, 32595, 32725) }, + { AOM_CDF4(17575, 30588, 32399) }, + { AOM_CDF4(10819, 24838, 30309) }, + { AOM_CDF4(7124, 18686, 25916) }, + { AOM_CDF4(4479, 12688, 19340) }, + { AOM_CDF4(28385, 32476, 32673) }, + { AOM_CDF4(15306, 29005, 31938) }, + { AOM_CDF4(8937, 21615, 28322) }, + { AOM_CDF4(5982, 15603, 22786) }, + { AOM_CDF4(3620, 10267, 16136) }, + { AOM_CDF4(27280, 32464, 32667) }, + { AOM_CDF4(15607, 29160, 32004) }, + { AOM_CDF4(9091, 22135, 28740) }, + { AOM_CDF4(6232, 16632, 24020) }, + { AOM_CDF4(4047, 11377, 17672) }, + { AOM_CDF4(29220, 32630, 32718) }, + { AOM_CDF4(19650, 31220, 32462) }, + { AOM_CDF4(13050, 26312, 30827) }, + { AOM_CDF4(9228, 20870, 27468) }, + { AOM_CDF4(6146, 15149, 21971) }, + { AOM_CDF4(30169, 32481, 32623) }, + { AOM_CDF4(17212, 29311, 31554) }, + { AOM_CDF4(9911, 21311, 26882) }, + { AOM_CDF4(4487, 13314, 20372) }, + { AOM_CDF4(2570, 7772, 12889) }, + { AOM_CDF4(30924, 32613, 32708) }, + { AOM_CDF4(19490, 30206, 32107) }, + { AOM_CDF4(11232, 23998, 29276) }, + { AOM_CDF4(6769, 17955, 25035) }, + { AOM_CDF4(4398, 12623, 19214) }, + { AOM_CDF4(30609, 32627, 32722) }, + { AOM_CDF4(19370, 30582, 32287) }, + { AOM_CDF4(10457, 23619, 29409) }, + { AOM_CDF4(6443, 17637, 24834) }, + { AOM_CDF4(4645, 13236, 20106) }, + { AOM_CDF4(8192, 16384, 24576) } }, + { { AOM_CDF4(8626, 20271, 26216) }, + { AOM_CDF4(26707, 32406, 32711) }, + { AOM_CDF4(16999, 30329, 32286) }, + { AOM_CDF4(11445, 25123, 30286) }, + { AOM_CDF4(6411, 18828, 25601) }, + { AOM_CDF4(6801, 12458, 20248) }, + { AOM_CDF4(29918, 32682, 32748) }, + { AOM_CDF4(20649, 31739, 32618) }, + { AOM_CDF4(12879, 27773, 31581) }, + { AOM_CDF4(7896, 21751, 28244) }, + { AOM_CDF4(5260, 14870, 23698) }, + { AOM_CDF4(29252, 32593, 32731) }, + { AOM_CDF4(17072, 30460, 32294) }, + { AOM_CDF4(10653, 24143, 29365) }, + { AOM_CDF4(6536, 17490, 23983) }, + { AOM_CDF4(4929, 13170, 20085) }, + { AOM_CDF4(28137, 32518, 32715) }, + { AOM_CDF4(18171, 30784, 32407) }, + { AOM_CDF4(11437, 25436, 30459) }, + { AOM_CDF4(7252, 18534, 26176) }, + { AOM_CDF4(4126, 13353, 20978) }, + { AOM_CDF4(31162, 32726, 32748) }, + { AOM_CDF4(23017, 32222, 32701) }, + { AOM_CDF4(15629, 29233, 32046) }, + { AOM_CDF4(9387, 22621, 29480) }, + { AOM_CDF4(6922, 17616, 25010) }, + { AOM_CDF4(28838, 32265, 32614) }, + { AOM_CDF4(19701, 30206, 31920) }, + { AOM_CDF4(11214, 22410, 27933) }, + { AOM_CDF4(5320, 14177, 23034) }, + { AOM_CDF4(5049, 12881, 17827) }, + { AOM_CDF4(27484, 32471, 32734) }, + { AOM_CDF4(21076, 31526, 32561) }, + { AOM_CDF4(12707, 26303, 31211) }, + { AOM_CDF4(8169, 21722, 28219) }, + { AOM_CDF4(6045, 19406, 27042) }, + { AOM_CDF4(27753, 32572, 32745) }, + { AOM_CDF4(20832, 31878, 32653) }, + { AOM_CDF4(13250, 27356, 31674) }, + { AOM_CDF4(7718, 21508, 29858) }, + { AOM_CDF4(7209, 18350, 25559) }, + { AOM_CDF4(8192, 16384, 24576) } } }, + { { { AOM_CDF4(7876, 16901, 21741) }, + { AOM_CDF4(24001, 31898, 32625) }, + { AOM_CDF4(14529, 27959, 31451) }, + { AOM_CDF4(8273, 20818, 27258) }, + { AOM_CDF4(5278, 14673, 21510) }, + { AOM_CDF4(2983, 8843, 14039) }, + { AOM_CDF4(28016, 32574, 32732) }, + { AOM_CDF4(17471, 30306, 32301) }, + { AOM_CDF4(10224, 24063, 29728) }, + { AOM_CDF4(6602, 17954, 25052) }, + { AOM_CDF4(4002, 11585, 17759) }, + { AOM_CDF4(30190, 32634, 32739) }, + { AOM_CDF4(17497, 30282, 32270) }, + { AOM_CDF4(10229, 23729, 29538) }, + { AOM_CDF4(6344, 17211, 24440) }, + { AOM_CDF4(3849, 11189, 17108) }, + { AOM_CDF4(28570, 32583, 32726) }, + { AOM_CDF4(17521, 30161, 32238) }, + { AOM_CDF4(10153, 23565, 29378) }, + { AOM_CDF4(6455, 17341, 24443) }, + { AOM_CDF4(3907, 11042, 17024) }, + { AOM_CDF4(30689, 32715, 32748) }, + { AOM_CDF4(21546, 31840, 32610) }, + { AOM_CDF4(13547, 27581, 31459) }, + { AOM_CDF4(8912, 21757, 28309) }, + { AOM_CDF4(5548, 15080, 22046) }, + { AOM_CDF4(30783, 32540, 32685) }, + { AOM_CDF4(17540, 29528, 31668) }, + { AOM_CDF4(10160, 21468, 26783) }, + { AOM_CDF4(4724, 13393, 20054) }, + { AOM_CDF4(2702, 8174, 13102) }, + { AOM_CDF4(31648, 32686, 32742) }, + { AOM_CDF4(20954, 31094, 32337) }, + { AOM_CDF4(12420, 25698, 30179) }, + { AOM_CDF4(7304, 19320, 26248) }, + { AOM_CDF4(4366, 12261, 18864) }, + { AOM_CDF4(31581, 32723, 32748) }, + { AOM_CDF4(21373, 31586, 32525) }, + { AOM_CDF4(12744, 26625, 30885) }, + { AOM_CDF4(7431, 20322, 26950) }, + { AOM_CDF4(4692, 13323, 20111) }, + { AOM_CDF4(8192, 16384, 24576) } }, + { { AOM_CDF4(7833, 18369, 24095) }, + { AOM_CDF4(26650, 32273, 32702) }, + { AOM_CDF4(16371, 29961, 32191) }, + { AOM_CDF4(11055, 24082, 29629) }, + { AOM_CDF4(6892, 18644, 25400) }, + { AOM_CDF4(5006, 13057, 19240) }, + { AOM_CDF4(29834, 32666, 32748) }, + { AOM_CDF4(19577, 31335, 32570) }, + { AOM_CDF4(12253, 26509, 31122) }, + { AOM_CDF4(7991, 20772, 27711) }, + { AOM_CDF4(5677, 15910, 23059) }, + { AOM_CDF4(30109, 32532, 32720) }, + { AOM_CDF4(16747, 30166, 32252) }, + { AOM_CDF4(10134, 23542, 29184) }, + { AOM_CDF4(5791, 16176, 23556) }, + { AOM_CDF4(4362, 10414, 17284) }, + { AOM_CDF4(29492, 32626, 32748) }, + { AOM_CDF4(19894, 31402, 32525) }, + { AOM_CDF4(12942, 27071, 30869) }, + { AOM_CDF4(8346, 21216, 27405) }, + { AOM_CDF4(6572, 17087, 23859) }, + { AOM_CDF4(32035, 32735, 32748) }, + { AOM_CDF4(22957, 31838, 32618) }, + { AOM_CDF4(14724, 28572, 31772) }, + { AOM_CDF4(10364, 23999, 29553) }, + { AOM_CDF4(7004, 18433, 25655) }, + { AOM_CDF4(27528, 32277, 32681) }, + { AOM_CDF4(16959, 31171, 32096) }, + { AOM_CDF4(10486, 23593, 27962) }, + { AOM_CDF4(8192, 16384, 23211) }, + { AOM_CDF4(8937, 17873, 20852) }, + { AOM_CDF4(27715, 32002, 32615) }, + { AOM_CDF4(15073, 29491, 31676) }, + { AOM_CDF4(11264, 24576, 28672) }, + { AOM_CDF4(2341, 18725, 23406) }, + { AOM_CDF4(7282, 18204, 25486) }, + { AOM_CDF4(28547, 32213, 32657) }, + { AOM_CDF4(20788, 29773, 32239) }, + { AOM_CDF4(6780, 21469, 30508) }, + { AOM_CDF4(5958, 14895, 23831) }, + { AOM_CDF4(16384, 21845, 27307) }, + { AOM_CDF4(8192, 16384, 24576) } } }, + { { { AOM_CDF4(5992, 14304, 19765) }, + { AOM_CDF4(22612, 31238, 32456) }, + { AOM_CDF4(13456, 27162, 31087) }, + { AOM_CDF4(8001, 20062, 26504) }, + { AOM_CDF4(5168, 14105, 20764) }, + { AOM_CDF4(2632, 7771, 12385) }, + { AOM_CDF4(27034, 32344, 32709) }, + { AOM_CDF4(15850, 29415, 31997) }, + { AOM_CDF4(9494, 22776, 28841) }, + { AOM_CDF4(6151, 16830, 23969) }, + { AOM_CDF4(3461, 10039, 15722) }, + { AOM_CDF4(30134, 32569, 32731) }, + { AOM_CDF4(15638, 29422, 31945) }, + { AOM_CDF4(9150, 21865, 28218) }, + { AOM_CDF4(5647, 15719, 22676) }, + { AOM_CDF4(3402, 9772, 15477) }, + { AOM_CDF4(28530, 32586, 32735) }, + { AOM_CDF4(17139, 30298, 32292) }, + { AOM_CDF4(10200, 24039, 29685) }, + { AOM_CDF4(6419, 17674, 24786) }, + { AOM_CDF4(3544, 10225, 15824) }, + { AOM_CDF4(31333, 32726, 32748) }, + { AOM_CDF4(20618, 31487, 32544) }, + { AOM_CDF4(12901, 27217, 31232) }, + { AOM_CDF4(8624, 21734, 28171) }, + { AOM_CDF4(5104, 14191, 20748) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) } }, + { { AOM_CDF4(11206, 21090, 26561) }, + { AOM_CDF4(28759, 32279, 32671) }, + { AOM_CDF4(14171, 27952, 31569) }, + { AOM_CDF4(9743, 22907, 29141) }, + { AOM_CDF4(6871, 17886, 24868) }, + { AOM_CDF4(4960, 13152, 19315) }, + { AOM_CDF4(31077, 32661, 32748) }, + { AOM_CDF4(19400, 31195, 32515) }, + { AOM_CDF4(12752, 26858, 31040) }, + { AOM_CDF4(8370, 22098, 28591) }, + { AOM_CDF4(5457, 15373, 22298) }, + { AOM_CDF4(31697, 32706, 32748) }, + { AOM_CDF4(17860, 30657, 32333) }, + { AOM_CDF4(12510, 24812, 29261) }, + { AOM_CDF4(6180, 19124, 24722) }, + { AOM_CDF4(5041, 13548, 17959) }, + { AOM_CDF4(31552, 32716, 32748) }, + { AOM_CDF4(21908, 31769, 32623) }, + { AOM_CDF4(14470, 28201, 31565) }, + { AOM_CDF4(9493, 22982, 28608) }, + { AOM_CDF4(6858, 17240, 24137) }, + { AOM_CDF4(32543, 32752, 32756) }, + { AOM_CDF4(24286, 32097, 32666) }, + { AOM_CDF4(15958, 29217, 32024) }, + { AOM_CDF4(10207, 24234, 29958) }, + { AOM_CDF4(6929, 18305, 25652) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) } } }, + { { { AOM_CDF4(4137, 10847, 15682) }, + { AOM_CDF4(17824, 27001, 30058) }, + { AOM_CDF4(10204, 22796, 28291) }, + { AOM_CDF4(6076, 15935, 22125) }, + { AOM_CDF4(3852, 10937, 16816) }, + { AOM_CDF4(2252, 6324, 10131) }, + { AOM_CDF4(25840, 32016, 32662) }, + { AOM_CDF4(15109, 28268, 31531) }, + { AOM_CDF4(9385, 22231, 28340) }, + { AOM_CDF4(6082, 16672, 23479) }, + { AOM_CDF4(3318, 9427, 14681) }, + { AOM_CDF4(30594, 32574, 32718) }, + { AOM_CDF4(16836, 29552, 31859) }, + { AOM_CDF4(9556, 22542, 28356) }, + { AOM_CDF4(6305, 16725, 23540) }, + { AOM_CDF4(3376, 9895, 15184) }, + { AOM_CDF4(29383, 32617, 32745) }, + { AOM_CDF4(18891, 30809, 32401) }, + { AOM_CDF4(11688, 25942, 30687) }, + { AOM_CDF4(7468, 19469, 26651) }, + { AOM_CDF4(3909, 11358, 17012) }, + { AOM_CDF4(31564, 32736, 32748) }, + { AOM_CDF4(20906, 31611, 32600) }, + { AOM_CDF4(13191, 27621, 31537) }, + { AOM_CDF4(8768, 22029, 28676) }, + { AOM_CDF4(5079, 14109, 20906) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) } }, + { { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) }, + { AOM_CDF4(8192, 16384, 24576) } } } } }; static const aom_cdf_prob av1_default_coeff_base_eob_multi_cdfs [TOKEN_CDF_Q_CTXS][TX_SIZES][PLANE_TYPES][SIG_COEF_CONTEXTS_EOB][CDF_SIZE( diff --git a/third_party/aom/av1/common/warped_motion.c b/third_party/aom/av1/common/warped_motion.c index e232e10828..0c91cfdbab 100644 --- a/third_party/aom/av1/common/warped_motion.c +++ b/third_party/aom/av1/common/warped_motion.c @@ -574,7 +574,6 @@ static int64_t highbd_warp_error( CONVERT_TO_BYTEPTR(tmp), j, i, warp_w, warp_h, WARP_ERROR_BLOCK, subsampling_x, subsampling_y, bd, &conv_params); - gm_sumerr += highbd_frame_error( tmp, WARP_ERROR_BLOCK, CONVERT_TO_SHORTPTR(dst8) + j + i * p_stride, warp_w, warp_h, p_stride, bd); diff --git a/third_party/aom/av1/common/x86/av1_inv_txfm_avx2.c b/third_party/aom/av1/common/x86/av1_inv_txfm_avx2.c index cf1f947d30..3f5ad89a9f 100644 --- a/third_party/aom/av1/common/x86/av1_inv_txfm_avx2.c +++ b/third_party/aom/av1/common/x86/av1_inv_txfm_avx2.c @@ -1638,6 +1638,7 @@ static INLINE void lowbd_inv_txfm2d_add_no_identity_avx2( assert(row_txfm != NULL); int ud_flip, lr_flip; get_flip_cfg(tx_type, &ud_flip, &lr_flip); + const __m256i scale0 = _mm256_set1_epi16(1 << (15 + shift[0])); for (int i = 0; i < buf_size_nonzero_h_div16; i++) { __m256i buf0[64]; const int32_t *input_row = input + (i << 4) * input_stride; @@ -1652,7 +1653,9 @@ static INLINE void lowbd_inv_txfm2d_add_no_identity_avx2( round_shift_avx2(buf0, buf0, input_stride); // rect special code } row_txfm(buf0, buf0, cos_bit_row); - round_shift_16bit_w16_avx2(buf0, txfm_size_col, shift[0]); + for (int j = 0; j < txfm_size_col; ++j) { + buf0[j] = _mm256_mulhrs_epi16(buf0[j], scale0); + } __m256i *buf1_cur = buf1 + (i << 4); if (lr_flip) { @@ -1668,10 +1671,13 @@ static INLINE void lowbd_inv_txfm2d_add_no_identity_avx2( } } } + const __m256i scale1 = _mm256_set1_epi16(1 << (15 + shift[1])); for (int i = 0; i < buf_size_w_div16; i++) { __m256i *buf1_cur = buf1 + i * txfm_size_row; col_txfm(buf1_cur, buf1_cur, cos_bit_col); - round_shift_16bit_w16_avx2(buf1_cur, txfm_size_row, shift[1]); + for (int j = 0; j < txfm_size_row; ++j) { + buf1_cur[j] = _mm256_mulhrs_epi16(buf1_cur[j], scale1); + } } for (int i = 0; i < buf_size_w_div16; i++) { lowbd_write_buffer_16xn_avx2(buf1 + i * txfm_size_row, output + 16 * i, diff --git a/third_party/aom/av1/common/x86/av1_inv_txfm_ssse3.c b/third_party/aom/av1/common/x86/av1_inv_txfm_ssse3.c index de0a5617f6..2208a912ac 100644 --- a/third_party/aom/av1/common/x86/av1_inv_txfm_ssse3.c +++ b/third_party/aom/av1/common/x86/av1_inv_txfm_ssse3.c @@ -2820,8 +2820,22 @@ static void lowbd_inv_txfm2d_add_4x16_ssse3(const int32_t *input, load_buffer_32bit_to_16bit_w4(input_cur, txfm_size_col, buf_cur, row_one_loop); transpose_16bit_4x8(buf_cur, buf_cur); - row_txfm(buf_cur, buf_cur, cos_bit_row); - round_shift_16bit_ssse3(buf_cur, row_one_loop, shift[0]); + if (row_txfm == iidentity4_new_ssse3) { + const __m128i scale = pair_set_epi16(NewSqrt2, 3 << (NewSqrt2Bits - 1)); + const __m128i ones = _mm_set1_epi16(1); + for (int j = 0; j < 4; ++j) { + const __m128i buf_lo = _mm_unpacklo_epi16(buf_cur[j], ones); + const __m128i buf_hi = _mm_unpackhi_epi16(buf_cur[j], ones); + const __m128i buf_32_lo = + _mm_srai_epi32(_mm_madd_epi16(buf_lo, scale), (NewSqrt2Bits + 1)); + const __m128i buf_32_hi = + _mm_srai_epi32(_mm_madd_epi16(buf_hi, scale), (NewSqrt2Bits + 1)); + buf_cur[j] = _mm_packs_epi32(buf_32_lo, buf_32_hi); + } + } else { + row_txfm(buf_cur, buf_cur, cos_bit_row); + round_shift_16bit_ssse3(buf_cur, row_one_loop, shift[0]); + } if (lr_flip) { __m128i temp[8]; flip_buf_sse2(buf_cur, temp, txfm_size_col); @@ -2867,8 +2881,22 @@ static void lowbd_inv_txfm2d_add_16x4_ssse3(const int32_t *input, txfm_size_row); transpose_16bit_8x4(buf_cur, buf_cur); } - row_txfm(buf, buf, cos_bit_row); - round_shift_16bit_ssse3(buf, txfm_size_col, shift[0]); + if (row_txfm == iidentity16_new_ssse3) { + const __m128i scale = pair_set_epi16(2 * NewSqrt2, 3 << (NewSqrt2Bits - 1)); + const __m128i ones = _mm_set1_epi16(1); + for (int j = 0; j < 16; ++j) { + const __m128i buf_lo = _mm_unpacklo_epi16(buf[j], ones); + const __m128i buf_hi = _mm_unpackhi_epi16(buf[j], ones); + const __m128i buf_32_lo = + _mm_srai_epi32(_mm_madd_epi16(buf_lo, scale), (NewSqrt2Bits + 1)); + const __m128i buf_32_hi = + _mm_srai_epi32(_mm_madd_epi16(buf_hi, scale), (NewSqrt2Bits + 1)); + buf[j] = _mm_packs_epi32(buf_32_lo, buf_32_hi); + } + } else { + row_txfm(buf, buf, cos_bit_row); + round_shift_16bit_ssse3(buf, txfm_size_col, shift[0]); + } if (lr_flip) { __m128i temp[16]; flip_buf_sse2(buf, temp, 16); @@ -2916,22 +2944,14 @@ void av1_lowbd_inv_txfm2d_add_ssse3(const int32_t *input, uint8_t *output, break; } } + void av1_inv_txfm_add_ssse3(const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param) { - const TX_TYPE tx_type = txfm_param->tx_type; if (!txfm_param->lossless) { - switch (txfm_param->tx_size) { - case TX_4X16: - case TX_16X4: - // TODO(http://crbug.com/aomedia/2350): the ssse3 versions cause test - // vector mismatches. - av1_inv_txfm_add_c(dqcoeff, dst, stride, txfm_param); - break; - default: - av1_lowbd_inv_txfm2d_add_ssse3(dqcoeff, dst, stride, tx_type, - txfm_param->tx_size, txfm_param->eob); - break; - } + const TX_TYPE tx_type = txfm_param->tx_type; + av1_lowbd_inv_txfm2d_add_ssse3(dqcoeff, dst, stride, tx_type, + txfm_param->tx_size, txfm_param->eob); + } else { av1_inv_txfm_add_c(dqcoeff, dst, stride, txfm_param); } diff --git a/third_party/aom/av1/common/x86/highbd_warp_plane_sse4.c b/third_party/aom/av1/common/x86/highbd_warp_plane_sse4.c index 3765c5ef19..81b2316a41 100644 --- a/third_party/aom/av1/common/x86/highbd_warp_plane_sse4.c +++ b/third_party/aom/av1/common/x86/highbd_warp_plane_sse4.c @@ -15,9 +15,9 @@ #include "av1/common/warped_motion.h" -static const uint8_t warp_highbd_arrange_bytes[16] = { - 0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15 -}; +static const uint8_t warp_highbd_arrange_bytes[16] = { 0, 2, 4, 6, 8, 10, + 12, 14, 1, 3, 5, 7, + 9, 11, 13, 15 }; static const uint8_t highbd_shuffle_alpha0_mask0[16] = { 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3 @@ -25,12 +25,12 @@ static const uint8_t highbd_shuffle_alpha0_mask0[16] = { static const uint8_t highbd_shuffle_alpha0_mask1[16] = { 4, 5, 6, 7, 4, 5, 6, 7, 4, 5, 6, 7, 4, 5, 6, 7 }; -static const uint8_t highbd_shuffle_alpha0_mask2[16] = { - 8, 9, 10, 11, 8, 9, 10, 11, 8, 9, 10, 11, 8, 9, 10, 11 -}; -static const uint8_t highbd_shuffle_alpha0_mask3[16] = { - 12, 13, 14, 15, 12, 13, 14, 15, 12, 13, 14, 15, 12, 13, 14, 15 -}; +static const uint8_t highbd_shuffle_alpha0_mask2[16] = { 8, 9, 10, 11, 8, 9, + 10, 11, 8, 9, 10, 11, + 8, 9, 10, 11 }; +static const uint8_t highbd_shuffle_alpha0_mask3[16] = { 12, 13, 14, 15, 12, 13, + 14, 15, 12, 13, 14, 15, + 12, 13, 14, 15 }; static INLINE void highbd_prepare_horizontal_filter_coeff(int alpha, int sx, __m128i *coeff) { diff --git a/third_party/aom/av1/common/x86/intra_edge_sse4.c b/third_party/aom/av1/common/x86/intra_edge_sse4.c index 0c857b5835..fc69f41d79 100644 --- a/third_party/aom/av1/common/x86/intra_edge_sse4.c +++ b/third_party/aom/av1/common/x86/intra_edge_sse4.c @@ -212,10 +212,10 @@ void av1_upsample_intra_edge_sse4_1(uint8_t *p, int sz) { { -1, 9, 9, -1, -1, 9, 9, -1, -1, 9, 9, -1, -1, 9, 9, -1 } }; - DECLARE_ALIGNED(16, static const int8_t, v_const[2][16]) = { - { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 }, - { 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 } - }; + DECLARE_ALIGNED( + 16, static const int8_t, + v_const[2][16]) = { { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 }, + { 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 } }; // Extend first/last samples (upper-left p[-1], last p[sz-1]) // to support 4-tap filter diff --git a/third_party/aom/av1/common/x86/warp_plane_sse4.c b/third_party/aom/av1/common/x86/warp_plane_sse4.c index 4532d17ced..e8473705a6 100644 --- a/third_party/aom/av1/common/x86/warp_plane_sse4.c +++ b/third_party/aom/av1/common/x86/warp_plane_sse4.c @@ -221,9 +221,9 @@ static const uint8_t shuffle_gamma0_mask1[16] = { 4, 5, 6, 7, 4, 5, 6, 7, 4, 5, 6, 7, 4, 5, 6, 7 }; static const uint8_t shuffle_gamma0_mask2[16] = { 8, 9, 10, 11, 8, 9, 10, 11, 8, 9, 10, 11, 8, 9, 10, 11 }; -static const uint8_t shuffle_gamma0_mask3[16] = { - 12, 13, 14, 15, 12, 13, 14, 15, 12, 13, 14, 15, 12, 13, 14, 15 -}; +static const uint8_t shuffle_gamma0_mask3[16] = { 12, 13, 14, 15, 12, 13, + 14, 15, 12, 13, 14, 15, + 12, 13, 14, 15 }; static INLINE void filter_src_pixels(__m128i src, __m128i *tmp, __m128i *coeff, const int offset_bits_horiz, diff --git a/third_party/aom/av1/common/x86/wiener_convolve_avx2.c b/third_party/aom/av1/common/x86/wiener_convolve_avx2.c index 87a6e12396..b7ac683836 100644 --- a/third_party/aom/av1/common/x86/wiener_convolve_avx2.c +++ b/third_party/aom/av1/common/x86/wiener_convolve_avx2.c @@ -17,6 +17,7 @@ #include "av1/common/convolve.h" #include "aom_dsp/aom_dsp_common.h" #include "aom_dsp/aom_filter.h" +#include "aom_dsp/x86/convolve_avx2.h" #include "aom_dsp/x86/synonyms.h" #include "aom_dsp/x86/synonyms_avx2.h" @@ -25,6 +26,20 @@ // on the left. // A row of, say, 8-bit pixels with values p0, p1, p2, ..., p30, p31 will be // loaded and stored as [ p31 ... p17 p16 ][ p15 ... p1 p0 ]. + +// Exploiting the range of wiener filter coefficients, +// horizontal filtering can be done in 16 bit intermediate precision. +// The details are as follows : +// Consider the horizontal wiener filter coefficients of the following form : +// [C0, C1, C2, 2^(FILTER_BITS) -2 * (C0 + C1 + C2), C2, C1, C0] +// Subtracting 2^(FILTER_BITS) from the centre tap we get the following : +// [C0, C1, C2, -2 * (C0 + C1 + C2), C2, C1, C0] +// The sum of the product "C0 * p0 + C1 * p1 + C2 * p2 -2 * (C0 + C1 + C2) * p3 +// + C2 * p4 + C1 * p5 + C0 * p6" would be in the range of signed 16 bit +// precision. Finally, after rounding the above result by round_0, we multiply +// the centre pixel by 2^(FILTER_BITS - round_0) and add it to get the +// horizontal filter output. + void av1_wiener_convolve_add_src_avx2(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, @@ -37,224 +52,190 @@ void av1_wiener_convolve_add_src_avx2(const uint8_t *src, ptrdiff_t src_stride, (void)x_step_q4; (void)y_step_q4; - DECLARE_ALIGNED(32, uint16_t, - temp[(MAX_SB_SIZE + SUBPEL_TAPS - 1) * MAX_SB_SIZE]); - int intermediate_height = h + SUBPEL_TAPS - 2; - memset(temp + (intermediate_height * MAX_SB_SIZE), 0, MAX_SB_SIZE); - const int center_tap = ((SUBPEL_TAPS - 1) / 2); + DECLARE_ALIGNED(32, int16_t, im_block[(MAX_SB_SIZE + SUBPEL_TAPS) * 8]); + int im_h = h + SUBPEL_TAPS - 2; + int im_stride = 8; + memset(im_block + (im_h * im_stride), 0, MAX_SB_SIZE); + int i, j; + const int center_tap = (SUBPEL_TAPS - 1) / 2; const uint8_t *const src_ptr = src - center_tap * src_stride - center_tap; - const __m128i zero_128 = _mm_setzero_si128(); - const __m256i zero_256 = _mm256_setzero_si256(); + __m256i filt[4], coeffs_h[4], coeffs_v[4], filt_center; - // Add an offset to account for the "add_src" part of the convolve function. - const __m128i offset = _mm_insert_epi16(zero_128, 1 << FILTER_BITS, 3); + assert(conv_params->round_0 > 0); - const __m256i clamp_low = zero_256; + filt[0] = _mm256_load_si256((__m256i const *)filt1_global_avx2); + filt[1] = _mm256_load_si256((__m256i const *)filt2_global_avx2); + filt[2] = _mm256_load_si256((__m256i const *)filt3_global_avx2); + filt[3] = _mm256_load_si256((__m256i const *)filt4_global_avx2); + + filt_center = _mm256_load_si256((__m256i const *)filt_center_global_avx2); + + const __m128i coeffs_x = _mm_loadu_si128((__m128i *)filter_x); + const __m256i filter_coeffs_x = _mm256_broadcastsi128_si256(coeffs_x); + + // coeffs 0 1 0 1 0 1 0 1 + coeffs_h[0] = + _mm256_shuffle_epi8(filter_coeffs_x, _mm256_set1_epi16(0x0200u)); + // coeffs 2 3 2 3 2 3 2 3 + coeffs_h[1] = + _mm256_shuffle_epi8(filter_coeffs_x, _mm256_set1_epi16(0x0604u)); + // coeffs 4 5 4 5 4 5 4 5 + coeffs_h[2] = + _mm256_shuffle_epi8(filter_coeffs_x, _mm256_set1_epi16(0x0a08u)); + // coeffs 6 7 6 7 6 7 6 7 + coeffs_h[3] = + _mm256_shuffle_epi8(filter_coeffs_x, _mm256_set1_epi16(0x0e0cu)); + + const __m256i round_const_h = + _mm256_set1_epi16((1 << (conv_params->round_0 - 1))); + const __m256i round_const_horz = + _mm256_set1_epi16((1 << (bd + FILTER_BITS - conv_params->round_0 - 1))); + const __m256i clamp_low = _mm256_setzero_si256(); const __m256i clamp_high = _mm256_set1_epi16(WIENER_CLAMP_LIMIT(conv_params->round_0, bd) - 1); + const __m128i round_shift_h = _mm_cvtsi32_si128(conv_params->round_0); - /* Horizontal filter */ - { - // coeffs [ f7 f6 f5 f4 f3 f2 f1 f0 ] - const __m128i coeffs_x = _mm_add_epi16(xx_loadu_128(filter_x), offset); + // Add an offset to account for the "add_src" part of the convolve function. + const __m128i zero_128 = _mm_setzero_si128(); + const __m128i offset_0 = _mm_insert_epi16(zero_128, 1 << FILTER_BITS, 3); + const __m128i coeffs_y = _mm_add_epi16(xx_loadu_128(filter_y), offset_0); - // coeffs [ f3 f2 f3 f2 f1 f0 f1 f0 ] - const __m128i coeffs_0123 = _mm_unpacklo_epi32(coeffs_x, coeffs_x); - // coeffs [ f7 f6 f7 f6 f5 f4 f5 f4 ] - const __m128i coeffs_4567 = _mm_unpackhi_epi32(coeffs_x, coeffs_x); + const __m256i filter_coeffs_y = _mm256_broadcastsi128_si256(coeffs_y); - // coeffs [ f1 f0 f1 f0 f1 f0 f1 f0 ] - const __m128i coeffs_01_128 = _mm_unpacklo_epi64(coeffs_0123, coeffs_0123); - // coeffs [ f3 f2 f3 f2 f3 f2 f3 f2 ] - const __m128i coeffs_23_128 = _mm_unpackhi_epi64(coeffs_0123, coeffs_0123); - // coeffs [ f5 f4 f5 f4 f5 f4 f5 f4 ] - const __m128i coeffs_45_128 = _mm_unpacklo_epi64(coeffs_4567, coeffs_4567); - // coeffs [ f7 f6 f7 f6 f7 f6 f7 f6 ] - const __m128i coeffs_67_128 = _mm_unpackhi_epi64(coeffs_4567, coeffs_4567); + // coeffs 0 1 0 1 0 1 0 1 + coeffs_v[0] = _mm256_shuffle_epi32(filter_coeffs_y, 0x00); + // coeffs 2 3 2 3 2 3 2 3 + coeffs_v[1] = _mm256_shuffle_epi32(filter_coeffs_y, 0x55); + // coeffs 4 5 4 5 4 5 4 5 + coeffs_v[2] = _mm256_shuffle_epi32(filter_coeffs_y, 0xaa); + // coeffs 6 7 6 7 6 7 6 7 + coeffs_v[3] = _mm256_shuffle_epi32(filter_coeffs_y, 0xff); - // coeffs [ f1 f0 f1 f0 f1 f0 f1 f0 ][ f1 f0 f1 f0 f1 f0 f1 f0 ] - const __m256i coeffs_01 = yy_set_m128i(coeffs_01_128, coeffs_01_128); - // coeffs [ f3 f2 f3 f2 f3 f2 f3 f2 ][ f3 f2 f3 f2 f3 f2 f3 f2 ] - const __m256i coeffs_23 = yy_set_m128i(coeffs_23_128, coeffs_23_128); - // coeffs [ f5 f4 f5 f4 f5 f4 f5 f4 ][ f5 f4 f5 f4 f5 f4 f5 f4 ] - const __m256i coeffs_45 = yy_set_m128i(coeffs_45_128, coeffs_45_128); - // coeffs [ f7 f6 f7 f6 f7 f6 f7 f6 ][ f7 f6 f7 f6 f7 f6 f7 f6 ] - const __m256i coeffs_67 = yy_set_m128i(coeffs_67_128, coeffs_67_128); + const __m256i round_const_v = + _mm256_set1_epi32((1 << (conv_params->round_1 - 1)) - + (1 << (bd + conv_params->round_1 - 1))); + const __m128i round_shift_v = _mm_cvtsi32_si128(conv_params->round_1); - const __m256i round_const = _mm256_set1_epi32( - (1 << (conv_params->round_0 - 1)) + (1 << (bd + FILTER_BITS - 1))); + for (j = 0; j < w; j += 8) { + for (i = 0; i < im_h; i += 2) { + __m256i data = _mm256_castsi128_si256( + _mm_loadu_si128((__m128i *)&src_ptr[(i * src_stride) + j])); - for (int i = 0; i < intermediate_height; ++i) { - for (int j = 0; j < w; j += 16) { - const uint8_t *data_ij = src_ptr + i * src_stride + j; + // Load the next line + if (i + 1 < im_h) + data = _mm256_inserti128_si256( + data, + _mm_loadu_si128( + (__m128i *)&src_ptr[(i * src_stride) + j + src_stride]), + 1); - // Load 8-bit src data - const __m128i data_0 = xx_loadu_128(data_ij + 0); - const __m128i data_1 = xx_loadu_128(data_ij + 1); - const __m128i data_2 = xx_loadu_128(data_ij + 2); - const __m128i data_3 = xx_loadu_128(data_ij + 3); - const __m128i data_4 = xx_loadu_128(data_ij + 4); - const __m128i data_5 = xx_loadu_128(data_ij + 5); - const __m128i data_6 = xx_loadu_128(data_ij + 6); - const __m128i data_7 = xx_loadu_128(data_ij + 7); + __m256i res = convolve_lowbd_x(data, coeffs_h, filt); - // (Zero-)Extend 8-bit data to 16-bit data - const __m256i src_0 = _mm256_cvtepu8_epi16(data_0); - const __m256i src_1 = _mm256_cvtepu8_epi16(data_1); - const __m256i src_2 = _mm256_cvtepu8_epi16(data_2); - const __m256i src_3 = _mm256_cvtepu8_epi16(data_3); - const __m256i src_4 = _mm256_cvtepu8_epi16(data_4); - const __m256i src_5 = _mm256_cvtepu8_epi16(data_5); - const __m256i src_6 = _mm256_cvtepu8_epi16(data_6); - const __m256i src_7 = _mm256_cvtepu8_epi16(data_7); + res = + _mm256_sra_epi16(_mm256_add_epi16(res, round_const_h), round_shift_h); - // Multiply src data by filter coeffs and sum pairs - const __m256i res_0 = _mm256_madd_epi16(src_0, coeffs_01); - const __m256i res_1 = _mm256_madd_epi16(src_1, coeffs_01); - const __m256i res_2 = _mm256_madd_epi16(src_2, coeffs_23); - const __m256i res_3 = _mm256_madd_epi16(src_3, coeffs_23); - const __m256i res_4 = _mm256_madd_epi16(src_4, coeffs_45); - const __m256i res_5 = _mm256_madd_epi16(src_5, coeffs_45); - const __m256i res_6 = _mm256_madd_epi16(src_6, coeffs_67); - const __m256i res_7 = _mm256_madd_epi16(src_7, coeffs_67); + __m256i data_0 = _mm256_shuffle_epi8(data, filt_center); - // Calculate scalar product for even- and odd-indices separately, - // increasing to 32-bit precision - const __m256i res_even_sum = _mm256_add_epi32( - _mm256_add_epi32(res_0, res_4), _mm256_add_epi32(res_2, res_6)); - const __m256i res_odd_sum = _mm256_add_epi32( - _mm256_add_epi32(res_1, res_5), _mm256_add_epi32(res_3, res_7)); - - const __m256i res_even = _mm256_srai_epi32( - _mm256_add_epi32(res_even_sum, round_const), conv_params->round_0); - const __m256i res_odd = _mm256_srai_epi32( - _mm256_add_epi32(res_odd_sum, round_const), conv_params->round_0); - - // Reduce to 16-bit precision and pack even- and odd-index results - // back into one register. The _mm256_packs_epi32 intrinsic returns - // a register with the pixels ordered as follows: - // [ 15 13 11 9 14 12 10 8 ] [ 7 5 3 1 6 4 2 0 ] - const __m256i res = _mm256_packs_epi32(res_even, res_odd); - const __m256i res_clamped = - _mm256_min_epi16(_mm256_max_epi16(res, clamp_low), clamp_high); - - // Store in a temporary array - yy_storeu_256(temp + i * MAX_SB_SIZE + j, res_clamped); - } + // multiply the center pixel by 2^(FILTER_BITS - round_0) and add it to + // the result + data_0 = _mm256_slli_epi16(data_0, FILTER_BITS - conv_params->round_0); + res = _mm256_add_epi16(res, data_0); + res = _mm256_add_epi16(res, round_const_horz); + const __m256i res_clamped = + _mm256_min_epi16(_mm256_max_epi16(res, clamp_low), clamp_high); + _mm256_store_si256((__m256i *)&im_block[i * im_stride], res_clamped); } - } - /* Vertical filter */ - { - // coeffs [ g7 g6 g5 g4 g3 g2 g1 g0 ] - const __m128i coeffs_y = _mm_add_epi16(xx_loadu_128(filter_y), offset); + /* Vertical filter */ + { + __m256i src_0 = _mm256_loadu_si256((__m256i *)(im_block + 0 * im_stride)); + __m256i src_1 = _mm256_loadu_si256((__m256i *)(im_block + 1 * im_stride)); + __m256i src_2 = _mm256_loadu_si256((__m256i *)(im_block + 2 * im_stride)); + __m256i src_3 = _mm256_loadu_si256((__m256i *)(im_block + 3 * im_stride)); + __m256i src_4 = _mm256_loadu_si256((__m256i *)(im_block + 4 * im_stride)); + __m256i src_5 = _mm256_loadu_si256((__m256i *)(im_block + 5 * im_stride)); - // coeffs [ g3 g2 g3 g2 g1 g0 g1 g0 ] - const __m128i coeffs_0123 = _mm_unpacklo_epi32(coeffs_y, coeffs_y); - // coeffs [ g7 g6 g7 g6 g5 g4 g5 g4 ] - const __m128i coeffs_4567 = _mm_unpackhi_epi32(coeffs_y, coeffs_y); + __m256i s[8]; + s[0] = _mm256_unpacklo_epi16(src_0, src_1); + s[1] = _mm256_unpacklo_epi16(src_2, src_3); + s[2] = _mm256_unpacklo_epi16(src_4, src_5); - // coeffs [ g1 g0 g1 g0 g1 g0 g1 g0 ] - const __m128i coeffs_01_128 = _mm_unpacklo_epi64(coeffs_0123, coeffs_0123); - // coeffs [ g3 g2 g3 g2 g3 g2 g3 g2 ] - const __m128i coeffs_23_128 = _mm_unpackhi_epi64(coeffs_0123, coeffs_0123); - // coeffs [ g5 g4 g5 g4 g5 g4 g5 g4 ] - const __m128i coeffs_45_128 = _mm_unpacklo_epi64(coeffs_4567, coeffs_4567); - // coeffs [ g7 g6 g7 g6 g7 g6 g7 g6 ] - const __m128i coeffs_67_128 = _mm_unpackhi_epi64(coeffs_4567, coeffs_4567); + s[4] = _mm256_unpackhi_epi16(src_0, src_1); + s[5] = _mm256_unpackhi_epi16(src_2, src_3); + s[6] = _mm256_unpackhi_epi16(src_4, src_5); - // coeffs [ g1 g0 g1 g0 g1 g0 g1 g0 ][ g1 g0 g1 g0 g1 g0 g1 g0 ] - const __m256i coeffs_01 = yy_set_m128i(coeffs_01_128, coeffs_01_128); - // coeffs [ g3 g2 g3 g2 g3 g2 g3 g2 ][ g3 g2 g3 g2 g3 g2 g3 g2 ] - const __m256i coeffs_23 = yy_set_m128i(coeffs_23_128, coeffs_23_128); - // coeffs [ g5 g4 g5 g4 g5 g4 g5 g4 ][ g5 g4 g5 g4 g5 g4 g5 g4 ] - const __m256i coeffs_45 = yy_set_m128i(coeffs_45_128, coeffs_45_128); - // coeffs [ g7 g6 g7 g6 g7 g6 g7 g6 ][ g7 g6 g7 g6 g7 g6 g7 g6 ] - const __m256i coeffs_67 = yy_set_m128i(coeffs_67_128, coeffs_67_128); + for (i = 0; i < h - 1; i += 2) { + const int16_t *data = &im_block[i * im_stride]; - const __m256i round_const = - _mm256_set1_epi32((1 << (conv_params->round_1 - 1)) - - (1 << (bd + conv_params->round_1 - 1))); + const __m256i s6 = + _mm256_loadu_si256((__m256i *)(data + 6 * im_stride)); + const __m256i s7 = + _mm256_loadu_si256((__m256i *)(data + 7 * im_stride)); - for (int i = 0; i < h; ++i) { - for (int j = 0; j < w; j += 16) { - const uint16_t *data_ij = temp + i * MAX_SB_SIZE + j; + s[3] = _mm256_unpacklo_epi16(s6, s7); + s[7] = _mm256_unpackhi_epi16(s6, s7); - // Load 16-bit data from the output of the horizontal filter in - // which the pixels are ordered as follows: - // [ 15 13 11 9 14 12 10 8 ] [ 7 5 3 1 6 4 2 0 ] - const __m256i data_0 = yy_loadu_256(data_ij + 0 * MAX_SB_SIZE); - const __m256i data_1 = yy_loadu_256(data_ij + 1 * MAX_SB_SIZE); - const __m256i data_2 = yy_loadu_256(data_ij + 2 * MAX_SB_SIZE); - const __m256i data_3 = yy_loadu_256(data_ij + 3 * MAX_SB_SIZE); - const __m256i data_4 = yy_loadu_256(data_ij + 4 * MAX_SB_SIZE); - const __m256i data_5 = yy_loadu_256(data_ij + 5 * MAX_SB_SIZE); - const __m256i data_6 = yy_loadu_256(data_ij + 6 * MAX_SB_SIZE); - const __m256i data_7 = yy_loadu_256(data_ij + 7 * MAX_SB_SIZE); + __m256i res_a = convolve(s, coeffs_v); + __m256i res_b = convolve(s + 4, coeffs_v); - // Filter the even-indices, increasing to 32-bit precision - const __m256i src_0 = _mm256_unpacklo_epi16(data_0, data_1); - const __m256i src_2 = _mm256_unpacklo_epi16(data_2, data_3); - const __m256i src_4 = _mm256_unpacklo_epi16(data_4, data_5); - const __m256i src_6 = _mm256_unpacklo_epi16(data_6, data_7); + const __m256i res_a_round = _mm256_sra_epi32( + _mm256_add_epi32(res_a, round_const_v), round_shift_v); + const __m256i res_b_round = _mm256_sra_epi32( + _mm256_add_epi32(res_b, round_const_v), round_shift_v); - const __m256i res_0 = _mm256_madd_epi16(src_0, coeffs_01); - const __m256i res_2 = _mm256_madd_epi16(src_2, coeffs_23); - const __m256i res_4 = _mm256_madd_epi16(src_4, coeffs_45); - const __m256i res_6 = _mm256_madd_epi16(src_6, coeffs_67); + /* rounding code */ + // 16 bit conversion + const __m256i res_16bit = _mm256_packs_epi32(res_a_round, res_b_round); + // 8 bit conversion and saturation to uint8 + const __m256i res_8b = _mm256_packus_epi16(res_16bit, res_16bit); - const __m256i res_even = _mm256_add_epi32( - _mm256_add_epi32(res_0, res_2), _mm256_add_epi32(res_4, res_6)); + const __m128i res_0 = _mm256_castsi256_si128(res_8b); + const __m128i res_1 = _mm256_extracti128_si256(res_8b, 1); - // Filter the odd-indices, increasing to 32-bit precision - const __m256i src_1 = _mm256_unpackhi_epi16(data_0, data_1); - const __m256i src_3 = _mm256_unpackhi_epi16(data_2, data_3); - const __m256i src_5 = _mm256_unpackhi_epi16(data_4, data_5); - const __m256i src_7 = _mm256_unpackhi_epi16(data_6, data_7); + // Store values into the destination buffer + __m128i *const p_0 = (__m128i *)&dst[i * dst_stride + j]; + __m128i *const p_1 = (__m128i *)&dst[i * dst_stride + j + dst_stride]; - const __m256i res_1 = _mm256_madd_epi16(src_1, coeffs_01); - const __m256i res_3 = _mm256_madd_epi16(src_3, coeffs_23); - const __m256i res_5 = _mm256_madd_epi16(src_5, coeffs_45); - const __m256i res_7 = _mm256_madd_epi16(src_7, coeffs_67); + _mm_storel_epi64(p_0, res_0); + _mm_storel_epi64(p_1, res_1); - const __m256i res_odd = _mm256_add_epi32( - _mm256_add_epi32(res_1, res_3), _mm256_add_epi32(res_5, res_7)); + s[0] = s[1]; + s[1] = s[2]; + s[2] = s[3]; - // Pixels are currently in the following order: - // res_even order: [ 14 12 10 8 ] [ 6 4 2 0 ] - // res_odd order: [ 15 13 11 9 ] [ 7 5 3 1 ] - // - // Rearrange the pixels into the following order: - // res_lo order: [ 11 10 9 8 ] [ 3 2 1 0 ] - // res_hi order: [ 15 14 13 12 ] [ 7 6 5 4 ] - const __m256i res_lo = _mm256_unpacklo_epi32(res_even, res_odd); - const __m256i res_hi = _mm256_unpackhi_epi32(res_even, res_odd); + s[4] = s[5]; + s[5] = s[6]; + s[6] = s[7]; + } + if (h - i) { + s[0] = _mm256_permute2x128_si256(s[0], s[4], 0x20); + s[1] = _mm256_permute2x128_si256(s[1], s[5], 0x20); + s[2] = _mm256_permute2x128_si256(s[2], s[6], 0x20); - const __m256i res_lo_round = _mm256_srai_epi32( - _mm256_add_epi32(res_lo, round_const), conv_params->round_1); - const __m256i res_hi_round = _mm256_srai_epi32( - _mm256_add_epi32(res_hi, round_const), conv_params->round_1); + const int16_t *data = &im_block[i * im_stride]; + const __m128i s6_ = _mm_loadu_si128((__m128i *)(data + 6 * im_stride)); + const __m128i s7_ = _mm_loadu_si128((__m128i *)(data + 7 * im_stride)); - // Reduce to 16-bit precision and pack into the correct order: - // [ 15 14 13 12 11 10 9 8 ][ 7 6 5 4 3 2 1 0 ] - const __m256i res_16bit = - _mm256_packs_epi32(res_lo_round, res_hi_round); + __m128i s3 = _mm_unpacklo_epi16(s6_, s7_); + __m128i s7 = _mm_unpackhi_epi16(s6_, s7_); - // Reduce to 8-bit precision. This messes up the order: - // [ - - - - - - - - 15 14 13 12 11 10 9 8 ] - // [ - - - - - - - - 7 6 5 4 3 2 1 0 ] - const __m256i res_8bit = - _mm256_packus_epi16(res_16bit, zero_256 /* don't care value */); + s[3] = _mm256_inserti128_si256(_mm256_castsi128_si256(s3), s7, 1); + __m256i convolveres = convolve(s, coeffs_v); - // Swap the two central 32-bit values to get the order: - // [ - - - - - - - - - - - - - - - - ] - // [ 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1 0 ] - const __m256i res_8bit2 = _mm256_permute4x64_epi64(res_8bit, 0xd8); + const __m256i res_round = _mm256_sra_epi32( + _mm256_add_epi32(convolveres, round_const_v), round_shift_v); - // Store the lower 128-bit lane in the dst array - xx_storeu_128(dst + i * dst_stride + j, - _mm256_castsi256_si128(res_8bit2)); + /* rounding code */ + // 16 bit conversion + __m128i reslo = _mm256_castsi256_si128(res_round); + __m128i reshi = _mm256_extracti128_si256(res_round, 1); + const __m128i res_16bit = _mm_packus_epi32(reslo, reshi); + + // 8 bit conversion and saturation to uint8 + const __m128i res_8b = _mm_packus_epi16(res_16bit, res_16bit); + __m128i *const p_0 = (__m128i *)&dst[i * dst_stride + j]; + _mm_storel_epi64(p_0, res_8b); } } } diff --git a/third_party/aom/av1/decoder/decodeframe.c b/third_party/aom/av1/decoder/decodeframe.c index b7fc3709e0..d03601f1e0 100644 --- a/third_party/aom/av1/decoder/decodeframe.c +++ b/third_party/aom/av1/decoder/decodeframe.c @@ -1734,6 +1734,7 @@ static PARTITION_TYPE read_partition(MACROBLOCKD *xd, int mi_row, int mi_col, static void decode_partition(AV1Decoder *const pbi, ThreadData *const td, int mi_row, int mi_col, aom_reader *reader, BLOCK_SIZE bsize, int parse_decode_flag) { + assert(bsize < BLOCK_SIZES_ALL); AV1_COMMON *const cm = &pbi->common; MACROBLOCKD *const xd = &td->xd; const int bw = mi_size_wide[bsize]; @@ -1751,9 +1752,9 @@ static void decode_partition(AV1Decoder *const pbi, ThreadData *const td, // 01 - do parse only // 10 - do decode only // 11 - do parse and decode - static const block_visitor_fn_t block_visit[4] = { - NULL, parse_decode_block, decode_block, parse_decode_block - }; + static const block_visitor_fn_t block_visit[4] = { NULL, parse_decode_block, + decode_block, + parse_decode_block }; if (parse_decode_flag & 1) { const int num_planes = av1_num_planes(cm); @@ -1778,7 +1779,11 @@ static void decode_partition(AV1Decoder *const pbi, ThreadData *const td, partition = get_partition(cm, mi_row, mi_col, bsize); } subsize = get_partition_subsize(bsize, partition); - + if (subsize == BLOCK_INVALID) { + aom_internal_error(xd->error_info, AOM_CODEC_CORRUPT_FRAME, + "Partition is invalid for block size %dx%d", + block_size_wide[bsize], block_size_high[bsize]); + } // Check the bitstream is conformant: if there is subsampling on the // chroma planes, subsize must subsample to a valid block size. const struct macroblockd_plane *const pd_u = &xd->plane[1]; diff --git a/third_party/aom/av1/decoder/decodemv.c b/third_party/aom/av1/decoder/decodemv.c index 2791f3a0da..311ee76c1a 100644 --- a/third_party/aom/av1/decoder/decodemv.c +++ b/third_party/aom/av1/decoder/decodemv.c @@ -183,7 +183,7 @@ static void read_drl_idx(FRAME_CONTEXT *ec_ctx, MACROBLOCKD *xd, if (mbmi->mode == NEWMV || mbmi->mode == NEW_NEWMV) { for (int idx = 0; idx < 2; ++idx) { if (xd->ref_mv_count[ref_frame_type] > idx + 1) { - uint8_t drl_ctx = av1_drl_ctx(xd->ref_mv_stack[ref_frame_type], idx); + uint8_t drl_ctx = av1_drl_ctx(xd->weight[ref_frame_type], idx); int drl_idx = aom_read_symbol(r, ec_ctx->drl_cdf[drl_ctx], 2, ACCT_STR); mbmi->ref_mv_idx = idx + drl_idx; if (!drl_idx) return; @@ -196,7 +196,7 @@ static void read_drl_idx(FRAME_CONTEXT *ec_ctx, MACROBLOCKD *xd, // mode is factored in. for (int idx = 1; idx < 3; ++idx) { if (xd->ref_mv_count[ref_frame_type] > idx + 1) { - uint8_t drl_ctx = av1_drl_ctx(xd->ref_mv_stack[ref_frame_type], idx); + uint8_t drl_ctx = av1_drl_ctx(xd->weight[ref_frame_type], idx); int drl_idx = aom_read_symbol(r, ec_ctx->drl_cdf[drl_ctx], 2, ACCT_STR); mbmi->ref_mv_idx = idx + drl_idx - 1; if (!drl_idx) return; @@ -680,8 +680,8 @@ static void read_intrabc_info(AV1_COMMON *const cm, MACROBLOCKD *const xd, int_mv ref_mvs[INTRA_FRAME + 1][MAX_MV_REF_CANDIDATES]; av1_find_mv_refs(cm, xd, mbmi, INTRA_FRAME, xd->ref_mv_count, - xd->ref_mv_stack, ref_mvs, /*global_mvs=*/NULL, mi_row, - mi_col, inter_mode_ctx); + xd->ref_mv_stack, xd->weight, ref_mvs, /*global_mvs=*/NULL, + mi_row, mi_col, inter_mode_ctx); int_mv nearestmv, nearmv; @@ -1271,7 +1271,7 @@ static void read_inter_block_mode_info(AV1Decoder *const pbi, MV_REFERENCE_FRAME ref_frame = av1_ref_frame_type(mbmi->ref_frame); av1_find_mv_refs(cm, xd, mbmi, ref_frame, xd->ref_mv_count, xd->ref_mv_stack, - ref_mvs, /*global_mvs=*/NULL, mi_row, mi_col, + xd->weight, ref_mvs, /*global_mvs=*/NULL, mi_row, mi_col, inter_mode_ctx); int mode_ctx = av1_mode_context_analyzer(inter_mode_ctx, mbmi->ref_frame); diff --git a/third_party/aom/av1/decoder/decodetxb.c b/third_party/aom/av1/decoder/decodetxb.c index 223e32e990..71930dc756 100644 --- a/third_party/aom/av1/decoder/decodetxb.c +++ b/third_party/aom/av1/decoder/decodetxb.c @@ -336,6 +336,7 @@ void av1_read_coeffs_txb_facade(const AV1_COMMON *const cm, struct macroblockd_plane *const pd = &xd->plane[plane]; const BLOCK_SIZE bsize = mbmi->sb_type; + assert(bsize < BLOCK_SIZES_ALL); const BLOCK_SIZE plane_bsize = get_plane_block_size(bsize, pd->subsampling_x, pd->subsampling_y); diff --git a/third_party/aom/av1/decoder/obu.c b/third_party/aom/av1/decoder/obu.c index aaea572708..f4ea4af543 100644 --- a/third_party/aom/av1/decoder/obu.c +++ b/third_party/aom/av1/decoder/obu.c @@ -25,25 +25,6 @@ #include "av1/decoder/decodeframe.h" #include "av1/decoder/obu.h" -// Picture prediction structures (0-12 are predefined) in scalability metadata. -enum { - SCALABILITY_L1T2 = 0, - SCALABILITY_L1T3 = 1, - SCALABILITY_L2T1 = 2, - SCALABILITY_L2T2 = 3, - SCALABILITY_L2T3 = 4, - SCALABILITY_S2T1 = 5, - SCALABILITY_S2T2 = 6, - SCALABILITY_S2T3 = 7, - SCALABILITY_L2T1h = 8, - SCALABILITY_L2T2h = 9, - SCALABILITY_L2T3h = 10, - SCALABILITY_S2T1h = 11, - SCALABILITY_S2T2h = 12, - SCALABILITY_S2T3h = 13, - SCALABILITY_SS = 14 -} UENUM1BYTE(SCALABILITY_STRUCTURES); - aom_codec_err_t aom_get_num_layers_from_operating_point_idc( int operating_point_idc, unsigned int *number_spatial_layers, unsigned int *number_temporal_layers) { diff --git a/third_party/aom/av1/encoder/aq_variance.c b/third_party/aom/av1/encoder/aq_variance.c index d5729482fa..46b4971d58 100644 --- a/third_party/aom/av1/encoder/aq_variance.c +++ b/third_party/aom/av1/encoder/aq_variance.c @@ -180,12 +180,12 @@ int av1_block_wavelet_energy_level(const AV1_COMP *cpi, MACROBLOCK *x, return clamp((int)round(energy), ENERGY_MIN, ENERGY_MAX); } -int av1_compute_deltaq_from_energy_level(const AV1_COMP *const cpi, - int block_var_level) { +int av1_compute_q_from_energy_level_deltaq_mode(const AV1_COMP *const cpi, + int block_var_level) { int rate_level; const AV1_COMMON *const cm = &cpi->common; - if (DELTAQ_MODULATION == 1) { + if (DELTA_Q_PERCEPTUAL_MODULATION == 1) { ENERGY_IN_BOUNDS(block_var_level); rate_level = SEGMENT_ID(block_var_level); } else { @@ -198,5 +198,5 @@ int av1_compute_deltaq_from_energy_level(const AV1_COMP *const cpi, if ((cm->base_qindex != 0) && ((cm->base_qindex + qindex_delta) == 0)) { qindex_delta = -cm->base_qindex + 1; } - return qindex_delta; + return cm->base_qindex + qindex_delta; } diff --git a/third_party/aom/av1/encoder/aq_variance.h b/third_party/aom/av1/encoder/aq_variance.h index 2d22b663e5..543eb0b511 100644 --- a/third_party/aom/av1/encoder/aq_variance.h +++ b/third_party/aom/av1/encoder/aq_variance.h @@ -21,8 +21,8 @@ extern "C" { void av1_vaq_frame_setup(AV1_COMP *cpi); int av1_log_block_var(const AV1_COMP *cpi, MACROBLOCK *x, BLOCK_SIZE bs); -int av1_compute_deltaq_from_energy_level(const AV1_COMP *const cpi, - int block_var_level); +int av1_compute_q_from_energy_level_deltaq_mode(const AV1_COMP *const cpi, + int block_var_level); int av1_block_wavelet_energy_level(const AV1_COMP *cpi, MACROBLOCK *x, BLOCK_SIZE bs); diff --git a/third_party/aom/av1/encoder/av1_quantize.c b/third_party/aom/av1/encoder/av1_quantize.c index ff1342c0ff..ec3fdfad0c 100644 --- a/third_party/aom/av1/encoder/av1_quantize.c +++ b/third_party/aom/av1/encoder/av1_quantize.c @@ -288,7 +288,7 @@ void av1_quantize_b_facade(const tran_low_t *coeff_ptr, intptr_t n_coeffs, eob_ptr, sc->scan, sc->iscan); break; case 2: - aom_quantize_b_64x64_adaptive_c( + aom_quantize_b_64x64_adaptive( coeff_ptr, n_coeffs, p->zbin_QTX, p->round_QTX, p->quant_QTX, p->quant_shift_QTX, qcoeff_ptr, dqcoeff_ptr, p->dequant_QTX, eob_ptr, sc->scan, sc->iscan); diff --git a/third_party/aom/av1/encoder/bitstream.c b/third_party/aom/av1/encoder/bitstream.c index cbac2b26a7..f80f46b633 100644 --- a/third_party/aom/av1/encoder/bitstream.c +++ b/third_party/aom/av1/encoder/bitstream.c @@ -106,8 +106,7 @@ static void write_drl_idx(FRAME_CONTEXT *ec_ctx, const MB_MODE_INFO *mbmi, int idx; for (idx = 0; idx < 2; ++idx) { if (mbmi_ext->ref_mv_count[ref_frame_type] > idx + 1) { - uint8_t drl_ctx = - av1_drl_ctx(mbmi_ext->ref_mv_stack[ref_frame_type], idx); + uint8_t drl_ctx = av1_drl_ctx(mbmi_ext->weight[ref_frame_type], idx); aom_write_symbol(w, mbmi->ref_mv_idx != idx, ec_ctx->drl_cdf[drl_ctx], 2); @@ -122,8 +121,7 @@ static void write_drl_idx(FRAME_CONTEXT *ec_ctx, const MB_MODE_INFO *mbmi, // TODO(jingning): Temporary solution to compensate the NEARESTMV offset. for (idx = 1; idx < 3; ++idx) { if (mbmi_ext->ref_mv_count[ref_frame_type] > idx + 1) { - uint8_t drl_ctx = - av1_drl_ctx(mbmi_ext->ref_mv_stack[ref_frame_type], idx); + uint8_t drl_ctx = av1_drl_ctx(mbmi_ext->weight[ref_frame_type], idx); aom_write_symbol(w, mbmi->ref_mv_idx != (idx - 1), ec_ctx->drl_cdf[drl_ctx], 2); if (mbmi->ref_mv_idx == (idx - 1)) return; @@ -369,16 +367,16 @@ static void pack_txb_tokens(aom_writer *w, AV1_COMMON *cm, MACROBLOCK *const x, blk_col)]; if (tx_size == plane_tx_size || plane) { + const CB_COEFF_BUFFER *cb_coef_buff = x->cb_coef_buff; const int txb_offset = x->mbmi_ext->cb_offset / (TX_SIZE_W_MIN * TX_SIZE_H_MIN); - tran_low_t *tcoeff_txb = - x->mbmi_ext->cb_coef_buff->tcoeff[plane] + x->mbmi_ext->cb_offset; - uint16_t *eob_txb = x->mbmi_ext->cb_coef_buff->eobs[plane] + txb_offset; - uint8_t *txb_skip_ctx_txb = - x->mbmi_ext->cb_coef_buff->txb_skip_ctx[plane] + txb_offset; - int *dc_sign_ctx_txb = - x->mbmi_ext->cb_coef_buff->dc_sign_ctx[plane] + txb_offset; - tran_low_t *tcoeff = BLOCK_OFFSET(tcoeff_txb, block); + const tran_low_t *tcoeff_txb = + cb_coef_buff->tcoeff[plane] + x->mbmi_ext->cb_offset; + const uint16_t *eob_txb = cb_coef_buff->eobs[plane] + txb_offset; + const uint8_t *txb_skip_ctx_txb = + cb_coef_buff->txb_skip_ctx[plane] + txb_offset; + const int *dc_sign_ctx_txb = cb_coef_buff->dc_sign_ctx[plane] + txb_offset; + const tran_low_t *tcoeff = BLOCK_OFFSET(tcoeff_txb, block); const uint16_t eob = eob_txb[block]; TXB_CTX txb_ctx = { txb_skip_ctx_txb[block], dc_sign_ctx_txb[block] }; av1_write_coeffs_txb(cm, xd, w, blk_row, blk_col, plane, tx_size, tcoeff, @@ -1373,6 +1371,7 @@ static void write_inter_txb_coeff(AV1_COMMON *const cm, MACROBLOCK *const x, MACROBLOCKD *const xd = &x->e_mbd; const struct macroblockd_plane *const pd = &xd->plane[plane]; const BLOCK_SIZE bsize = mbmi->sb_type; + assert(bsize < BLOCK_SIZES_ALL); const BLOCK_SIZE bsizec = scale_chroma_bsize(bsize, pd->subsampling_x, pd->subsampling_y); @@ -1392,6 +1391,7 @@ static void write_inter_txb_coeff(AV1_COMMON *const cm, MACROBLOCK *const x, int blk_row, blk_col; + assert(plane_bsize < BLOCK_SIZES_ALL); const int num_4x4_w = block_size_wide[plane_bsize] >> tx_size_wide_log2[0]; const int num_4x4_h = block_size_high[plane_bsize] >> tx_size_high_log2[0]; @@ -1498,9 +1498,9 @@ static void write_modes_b(AV1_COMP *cpi, const TileInfo *const tile, int mi_col) { write_mbmi_b(cpi, tile, w, mi_row, mi_col); - AV1_COMMON *cm = &cpi->common; + const AV1_COMMON *cm = &cpi->common; MACROBLOCKD *xd = &cpi->td.mb.e_mbd; - MB_MODE_INFO *mbmi = xd->mi[0]; + const MB_MODE_INFO *mbmi = xd->mi[0]; for (int plane = 0; plane < AOMMIN(2, av1_num_planes(cm)); ++plane) { const uint8_t palette_size_plane = mbmi->palette_mode_info.palette_size[plane]; @@ -1516,10 +1516,10 @@ static void write_modes_b(AV1_COMP *cpi, const TileInfo *const tile, } } - BLOCK_SIZE bsize = mbmi->sb_type; - int is_inter_tx = is_inter_block(mbmi) || is_intrabc_block(mbmi); - int skip = mbmi->skip; - int segment_id = mbmi->segment_id; + const BLOCK_SIZE bsize = mbmi->sb_type; + const int is_inter_tx = is_inter_block(mbmi); + const int skip = mbmi->skip; + const int segment_id = mbmi->segment_id; if (cm->tx_mode == TX_MODE_SELECT && block_signals_txsize(bsize) && !(is_inter_tx && skip) && !xd->lossless[segment_id]) { if (is_inter_tx) { // This implies skip flag is 0. @@ -1528,17 +1528,17 @@ static void write_modes_b(AV1_COMP *cpi, const TileInfo *const tile, const int txbw = tx_size_wide_unit[max_tx_size]; const int width = block_size_wide[bsize] >> tx_size_wide_log2[0]; const int height = block_size_high[bsize] >> tx_size_high_log2[0]; - int idx, idy; - for (idy = 0; idy < height; idy += txbh) - for (idx = 0; idx < width; idx += txbw) + for (int idy = 0; idy < height; idy += txbh) { + for (int idx = 0; idx < width; idx += txbw) { write_tx_size_vartx(xd, mbmi, max_tx_size, 0, idy, idx, w); + } + } } else { write_selected_tx_size(xd, w); set_txfm_ctxs(mbmi->tx_size, xd->n4_w, xd->n4_h, 0, xd); } } else { - set_txfm_ctxs(mbmi->tx_size, xd->n4_w, xd->n4_h, - skip && is_inter_block(mbmi), xd); + set_txfm_ctxs(mbmi->tx_size, xd->n4_w, xd->n4_h, skip && is_inter_tx, xd); } write_tokens_b(cpi, tile, w, tok, tok_end, mi_row, mi_col); @@ -1587,6 +1587,7 @@ static void write_modes_sb(AV1_COMP *const cpi, const TileInfo *const tile, int mi_col, BLOCK_SIZE bsize) { const AV1_COMMON *const cm = &cpi->common; MACROBLOCKD *const xd = &cpi->td.mb.e_mbd; + assert(bsize < BLOCK_SIZES_ALL); const int hbs = mi_size_wide[bsize] / 2; const int quarter_step = mi_size_wide[bsize] / 4; int i; @@ -1711,6 +1712,7 @@ static void write_modes(AV1_COMP *const cpi, const TileInfo *const tile, for (mi_col = mi_col_start; mi_col < mi_col_end; mi_col += cm->seq_params.mib_size) { + cpi->td.mb.cb_coef_buff = av1_get_cb_coeff_buffer(cpi, mi_row, mi_col); write_modes_sb(cpi, tile, w, &tok, tok_end, mi_row, mi_col, cm->seq_params.sb_size); } diff --git a/third_party/aom/av1/encoder/block.h b/third_party/aom/av1/encoder/block.h index 96b0991dde..266f702af9 100644 --- a/third_party/aom/av1/encoder/block.h +++ b/third_party/aom/av1/encoder/block.h @@ -75,8 +75,8 @@ typedef struct { typedef struct { // TODO(angiebird): Reduce the buffer size according to sb_type - CB_COEFF_BUFFER *cb_coef_buff; CANDIDATE_MV ref_mv_stack[MODE_CTX_REF_FRAMES][MAX_REF_MV_STACK_SIZE]; + uint16_t weight[MODE_CTX_REF_FRAMES][MAX_REF_MV_STACK_SIZE]; int_mv global_mvs[REF_FRAMES]; int cb_offset; int16_t mode_context[MODE_CTX_REF_FRAMES]; @@ -433,6 +433,8 @@ struct macroblock { // [Saved stat index] COMP_RD_STATS comp_rd_stats[MAX_COMP_RD_STATS]; int comp_rd_stats_idx; + + CB_COEFF_BUFFER *cb_coef_buff; }; static INLINE int is_rect_tx_allowed_bsize(BLOCK_SIZE bsize) { diff --git a/third_party/aom/av1/encoder/context_tree.c b/third_party/aom/av1/encoder/context_tree.c index 40df6c1dae..1ee68adea9 100644 --- a/third_party/aom/av1/encoder/context_tree.c +++ b/third_party/aom/av1/encoder/context_tree.c @@ -216,7 +216,6 @@ void av1_copy_tree_context(PICK_MODE_CONTEXT *dst_ctx, dst_ctx->mbmi_ext = src_ctx->mbmi_ext; dst_ctx->num_4x4_blk = src_ctx->num_4x4_blk; - dst_ctx->skip = src_ctx->skip; dst_ctx->skippable = src_ctx->skippable; dst_ctx->best_mode_index = src_ctx->best_mode_index; @@ -227,9 +226,7 @@ void av1_copy_tree_context(PICK_MODE_CONTEXT *dst_ctx, dst_ctx->comp_pred_diff = src_ctx->comp_pred_diff; dst_ctx->single_pred_diff = src_ctx->single_pred_diff; - dst_ctx->rate = src_ctx->rate; - dst_ctx->dist = src_ctx->dist; - dst_ctx->rdcost = src_ctx->rdcost; + dst_ctx->rd_stats = src_ctx->rd_stats; dst_ctx->rd_mode_is_ready = src_ctx->rd_mode_is_ready; memcpy(dst_ctx->pred_mv, src_ctx->pred_mv, sizeof(MV) * REF_FRAMES); diff --git a/third_party/aom/av1/encoder/context_tree.h b/third_party/aom/av1/encoder/context_tree.h index 205ac8ad3d..1225520139 100644 --- a/third_party/aom/av1/encoder/context_tree.h +++ b/third_party/aom/av1/encoder/context_tree.h @@ -38,8 +38,6 @@ enum { typedef struct { MB_MODE_INFO mic; MB_MODE_INFO_EXT mbmi_ext; - int64_t dist; - int64_t rdcost; uint8_t *color_index_map[2]; uint8_t *blk_skip; @@ -50,7 +48,6 @@ typedef struct { uint8_t *txb_entropy_ctx[MAX_MB_PLANE]; int num_4x4_blk; - int skip; // For current partition, only if all Y, U, and V transform blocks' // coefficients are quantized to 0, skippable is set to 1. int skippable; @@ -59,9 +56,7 @@ typedef struct { int comp_pred_diff; int single_pred_diff; - // TODO(jingning) Use RD_COST struct here instead. This involves a boarder - // scope of refactoring. - int rate; + RD_STATS rd_stats; int rd_mode_is_ready; // Flag to indicate whether rd pick mode decision has // been made. @@ -99,7 +94,15 @@ typedef struct PC_TREE { PC_TREE_STATS pc_tree_stats; CB_TREE_SEARCH cb_search_range; int index; + + // Simple motion search_features MV mv_ref_fulls[REF_FRAMES]; + unsigned int sms_none_feat[2]; + unsigned int sms_split_feat[8]; + unsigned int sms_rect_feat[8]; + int sms_none_valid; + int sms_split_valid; + int sms_rect_valid; } PC_TREE; void av1_setup_pc_tree(struct AV1Common *cm, struct ThreadData *td); diff --git a/third_party/aom/av1/encoder/encode_strategy.c b/third_party/aom/av1/encoder/encode_strategy.c index e9d6ee7707..20687f3c23 100644 --- a/third_party/aom/av1/encoder/encode_strategy.c +++ b/third_party/aom/av1/encoder/encode_strategy.c @@ -947,7 +947,8 @@ static int get_refresh_frame_flags(const AV1_COMP *const cpi, int av1_encode_strategy(AV1_COMP *const cpi, size_t *const size, uint8_t *const dest, unsigned int *frame_flags, int64_t *const time_stamp, int64_t *const time_end, - const aom_rational_t *const timebase, int flush) { + const aom_rational64_t *const timestamp_ratio, + int flush) { const AV1EncoderConfig *const oxcf = &cpi->oxcf; AV1_COMMON *const cm = &cpi->common; @@ -1027,7 +1028,7 @@ int av1_encode_strategy(AV1_COMP *const cpi, size_t *const size, cm->seq_params.film_grain_params_present; } // only one operating point supported now - const int64_t pts64 = ticks_to_timebase_units(timebase, *time_stamp); + const int64_t pts64 = ticks_to_timebase_units(timestamp_ratio, *time_stamp); if (pts64 < 0 || pts64 > UINT32_MAX) return AOM_CODEC_ERROR; cpi->common.frame_presentation_time = (uint32_t)pts64; } diff --git a/third_party/aom/av1/encoder/encode_strategy.h b/third_party/aom/av1/encoder/encode_strategy.h index 6830e44468..7601547564 100644 --- a/third_party/aom/av1/encoder/encode_strategy.h +++ b/third_party/aom/av1/encoder/encode_strategy.h @@ -29,7 +29,8 @@ extern "C" { int av1_encode_strategy(AV1_COMP *const cpi, size_t *const size, uint8_t *const dest, unsigned int *frame_flags, int64_t *const time_stamp, int64_t *const time_end, - const aom_rational_t *const timebase, int flush); + const aom_rational64_t *const timestamp_ratio, + int flush); // Set individual buffer update flags based on frame reference type. // force_refresh_all is used when we have a KEY_FRAME or S_FRAME. It forces all diff --git a/third_party/aom/av1/encoder/encodeframe.c b/third_party/aom/av1/encoder/encodeframe.c index 2952184cb5..926e30ac09 100644 --- a/third_party/aom/av1/encoder/encodeframe.c +++ b/third_party/aom/av1/encoder/encodeframe.c @@ -68,10 +68,6 @@ static void encode_superblock(const AV1_COMP *const cpi, TileDataEnc *tile_data, ThreadData *td, TOKENEXTRA **t, RUN_TYPE dry_run, int mi_row, int mi_col, BLOCK_SIZE bsize, int *rate); -static int ml_predict_breakout(const AV1_COMP *const cpi, BLOCK_SIZE bsize, - const MACROBLOCK *const x, - const RD_STATS *const rd_stats, - unsigned int pb_source_variance); // This is used as a reference when computing the source variance for the // purposes of activity masking. @@ -142,6 +138,8 @@ static const uint16_t AV1_HIGH_VAR_OFFS_12[MAX_SB_SIZE] = { 128 * 16, 128 * 16 }; +enum { PICK_MODE_RD = 0, PICK_MODE_NONRD, PICK_MODE_FAST_NONRD }; + unsigned int av1_get_sby_perpixel_variance(const AV1_COMP *cpi, const struct buf_2d *ref, BLOCK_SIZE bs) { @@ -213,6 +211,7 @@ static void set_offsets_without_segment_id(const AV1_COMP *const cpi, const AV1_COMMON *const cm = &cpi->common; const int num_planes = av1_num_planes(cm); MACROBLOCKD *const xd = &x->e_mbd; + assert(bsize < BLOCK_SIZES_ALL); const int mi_width = mi_size_wide[bsize]; const int mi_height = mi_size_high[bsize]; @@ -355,7 +354,7 @@ static void update_state(const AV1_COMP *const cpi, memcpy(x->blk_skip, ctx->blk_skip, sizeof(x->blk_skip[0]) * ctx->num_4x4_blk); - x->skip = ctx->skip; + x->skip = ctx->rd_stats.skip; // If segmentation in use if (seg->enabled) { @@ -371,7 +370,8 @@ static void update_state(const AV1_COMP *const cpi, // and then update the quantizer. if (cpi->oxcf.aq_mode == CYCLIC_REFRESH_AQ) { av1_cyclic_refresh_update_segment(cpi, mi_addr, mi_row, mi_col, bsize, - ctx->rate, ctx->dist, x->skip); + ctx->rd_stats.rate, ctx->rd_stats.dist, + x->skip); } if (mi_addr->uv_mode == UV_CFL_PRED && !is_cfl_allowed(xd)) mi_addr->uv_mode = UV_DC_PRED; @@ -507,7 +507,7 @@ static void pick_sb_modes(AV1_COMP *const cpi, TileDataEnc *tile_data, MACROBLOCK *const x, int mi_row, int mi_col, RD_STATS *rd_cost, PARTITION_TYPE partition, BLOCK_SIZE bsize, PICK_MODE_CONTEXT *ctx, - int64_t best_rd, int use_nonrd_pick_mode) { + int64_t best_rd, int pick_mode_type) { AV1_COMMON *const cm = &cpi->common; const int num_planes = av1_num_planes(cm); TileInfo *const tile_info = &tile_data->tile_info; @@ -525,8 +525,8 @@ static void pick_sb_modes(AV1_COMP *const cpi, TileDataEnc *tile_data, #endif if (best_rd < 0) { - ctx->rdcost = INT64_MAX; - ctx->skip = 0; + ctx->rd_stats.rdcost = INT64_MAX; + ctx->rd_stats.skip = 0; av1_invalid_rd_stats(rd_cost); return; } @@ -541,9 +541,9 @@ static void pick_sb_modes(AV1_COMP *const cpi, TileDataEnc *tile_data, assert(ctx_mbmi->sb_type == bsize); assert(ctx_mbmi->partition == partition); *mbmi = *ctx_mbmi; - rd_cost->rate = ctx->rate; - rd_cost->dist = ctx->dist; - rd_cost->rdcost = ctx->rdcost; + rd_cost->rate = ctx->rd_stats.rate; + rd_cost->dist = ctx->rd_stats.dist; + rd_cost->rdcost = ctx->rd_stats.rdcost; } else { mbmi->sb_type = bsize; mbmi->partition = partition; @@ -579,7 +579,7 @@ static void pick_sb_modes(AV1_COMP *const cpi, TileDataEnc *tile_data, xd->plane[1].subsampling_y); if (ctx->rd_mode_is_ready) { - x->skip = ctx->skip; + x->skip = ctx->rd_stats.skip; *x->mbmi_ext = ctx->mbmi_ext; return; } @@ -632,7 +632,7 @@ static void pick_sb_modes(AV1_COMP *const cpi, TileDataEnc *tile_data, x->rdmult = x->cb_rdmult; } - if (deltaq_mode > 0) x->rdmult = set_deltaq_rdmult(cpi, xd); + if (deltaq_mode != NO_DELTA_Q) x->rdmult = set_deltaq_rdmult(cpi, xd); // Find best coding mode & reconstruct the MB so it is available // as a predictor for MBs that follow in the SB @@ -655,12 +655,20 @@ static void pick_sb_modes(AV1_COMP *const cpi, TileDataEnc *tile_data, } else { // TODO(kyslov): do the same for pick_intra_mode and // pick_inter_mode_sb_seg_skip - if (use_nonrd_pick_mode) { - av1_nonrd_pick_inter_mode_sb(cpi, tile_data, x, mi_row, mi_col, rd_cost, - bsize, ctx, best_rd); - } else { - av1_rd_pick_inter_mode_sb(cpi, tile_data, x, mi_row, mi_col, rd_cost, - bsize, ctx, best_rd); + switch (pick_mode_type) { + case PICK_MODE_RD: + av1_rd_pick_inter_mode_sb(cpi, tile_data, x, mi_row, mi_col, rd_cost, + bsize, ctx, best_rd); + break; + case PICK_MODE_NONRD: + av1_nonrd_pick_inter_mode_sb(cpi, tile_data, x, mi_row, mi_col, + rd_cost, bsize, ctx, best_rd); + break; + case PICK_MODE_FAST_NONRD: + av1_fast_nonrd_pick_inter_mode_sb(cpi, tile_data, x, mi_row, mi_col, + rd_cost, bsize, ctx, best_rd); + break; + default: assert(0 && "Unknown pick mode type."); } } #if CONFIG_COLLECT_COMPONENT_TIMING @@ -683,9 +691,9 @@ static void pick_sb_modes(AV1_COMP *const cpi, TileDataEnc *tile_data, // refactored to provide proper exit/return handle. if (rd_cost->rate == INT_MAX) rd_cost->rdcost = INT64_MAX; - ctx->rate = rd_cost->rate; - ctx->dist = rd_cost->dist; - ctx->rdcost = rd_cost->rdcost; + ctx->rd_stats.rate = rd_cost->rate; + ctx->rd_stats.dist = rd_cost->dist; + ctx->rd_stats.rdcost = rd_cost->rdcost; #if CONFIG_COLLECT_COMPONENT_TIMING end_timing(cpi, rd_pick_sb_modes_time); @@ -1350,7 +1358,7 @@ static void update_stats(const AV1_COMMON *const cm, TileDataEnc *tile_data, if (mbmi_ext->ref_mv_count[ref_frame_type] > idx + 1) { #if CONFIG_ENTROPY_STATS uint8_t drl_ctx = - av1_drl_ctx(mbmi_ext->ref_mv_stack[ref_frame_type], idx); + av1_drl_ctx(mbmi_ext->weight[ref_frame_type], idx); ++counts->drl_mode[drl_ctx][mbmi->ref_mv_idx != idx]; #endif @@ -1367,7 +1375,7 @@ static void update_stats(const AV1_COMMON *const cm, TileDataEnc *tile_data, if (mbmi_ext->ref_mv_count[ref_frame_type] > idx + 1) { #if CONFIG_ENTROPY_STATS uint8_t drl_ctx = - av1_drl_ctx(mbmi_ext->ref_mv_stack[ref_frame_type], idx); + av1_drl_ctx(mbmi_ext->weight[ref_frame_type], idx); ++counts->drl_mode[drl_ctx][mbmi->ref_mv_idx != idx - 1]; #endif @@ -1477,11 +1485,15 @@ static void encode_b(const AV1_COMP *const cpi, TileDataEnc *tile_data, mbmi->partition = partition; update_state(cpi, tile_data, td, ctx, mi_row, mi_col, bsize, dry_run); if (cpi->oxcf.enable_tpl_model && cpi->oxcf.aq_mode == NO_AQ && - cpi->oxcf.deltaq_mode == 0) { + cpi->oxcf.deltaq_mode == NO_DELTA_Q) { x->rdmult = x->cb_rdmult; } - if (!dry_run) av1_set_coeff_buffer(cpi, x, mi_row, mi_col); + if (!dry_run) { + x->mbmi_ext->cb_offset = x->cb_offset; + assert(x->cb_offset < + (1 << num_pels_log2_lookup[cpi->common.seq_params.sb_size])); + } encode_superblock(cpi, tile_data, td, tp, dry_run, mi_row, mi_col, bsize, rate); @@ -1758,7 +1770,7 @@ static void rd_use_partition(AV1_COMP *cpi, ThreadData *td, mi_row + hbs < cm->mi_rows && mi_col + hbs < cm->mi_cols) { pc_tree->partitioning = PARTITION_NONE; pick_sb_modes(cpi, tile_data, x, mi_row, mi_col, &none_rdc, - PARTITION_NONE, bsize, ctx_none, INT64_MAX, 0); + PARTITION_NONE, bsize, ctx_none, INT64_MAX, PICK_MODE_RD); if (none_rdc.rate < INT_MAX) { none_rdc.rate += x->partition_cost[pl][PARTITION_NONE]; @@ -1774,12 +1786,12 @@ static void rd_use_partition(AV1_COMP *cpi, ThreadData *td, switch (partition) { case PARTITION_NONE: pick_sb_modes(cpi, tile_data, x, mi_row, mi_col, &last_part_rdc, - PARTITION_NONE, bsize, ctx_none, INT64_MAX, 0); + PARTITION_NONE, bsize, ctx_none, INT64_MAX, PICK_MODE_RD); break; case PARTITION_HORZ: pick_sb_modes(cpi, tile_data, x, mi_row, mi_col, &last_part_rdc, PARTITION_HORZ, subsize, &pc_tree->horizontal[0], INT64_MAX, - 0); + PICK_MODE_RD); if (last_part_rdc.rate != INT_MAX && bsize >= BLOCK_8X8 && mi_row + hbs < cm->mi_rows) { RD_STATS tmp_rdc; @@ -1790,7 +1802,7 @@ static void rd_use_partition(AV1_COMP *cpi, ThreadData *td, mi_col, subsize, NULL); pick_sb_modes(cpi, tile_data, x, mi_row + hbs, mi_col, &tmp_rdc, PARTITION_HORZ, subsize, &pc_tree->horizontal[1], - INT64_MAX, 0); + INT64_MAX, PICK_MODE_RD); if (tmp_rdc.rate == INT_MAX || tmp_rdc.dist == INT64_MAX) { av1_invalid_rd_stats(&last_part_rdc); break; @@ -1803,7 +1815,7 @@ static void rd_use_partition(AV1_COMP *cpi, ThreadData *td, case PARTITION_VERT: pick_sb_modes(cpi, tile_data, x, mi_row, mi_col, &last_part_rdc, PARTITION_VERT, subsize, &pc_tree->vertical[0], INT64_MAX, - 0); + PICK_MODE_RD); if (last_part_rdc.rate != INT_MAX && bsize >= BLOCK_8X8 && mi_col + hbs < cm->mi_cols) { RD_STATS tmp_rdc; @@ -1814,7 +1826,8 @@ static void rd_use_partition(AV1_COMP *cpi, ThreadData *td, mi_col, subsize, NULL); pick_sb_modes(cpi, tile_data, x, mi_row, mi_col + hbs, &tmp_rdc, PARTITION_VERT, subsize, - &pc_tree->vertical[bsize > BLOCK_8X8], INT64_MAX, 0); + &pc_tree->vertical[bsize > BLOCK_8X8], INT64_MAX, + PICK_MODE_RD); if (tmp_rdc.rate == INT_MAX || tmp_rdc.dist == INT64_MAX) { av1_invalid_rd_stats(&last_part_rdc); break; @@ -1890,7 +1903,7 @@ static void rd_use_partition(AV1_COMP *cpi, ThreadData *td, pc_tree->split[i]->partitioning = PARTITION_NONE; pick_sb_modes(cpi, tile_data, x, mi_row + y_idx, mi_col + x_idx, &tmp_rdc, PARTITION_SPLIT, split_subsize, &pc_tree->split[i]->none, - INT64_MAX, 0); + INT64_MAX, PICK_MODE_RD); restore_context(x, &x_ctx, mi_row, mi_col, bsize, num_planes); if (tmp_rdc.rate == INT_MAX || tmp_rdc.dist == INT64_MAX) { @@ -1951,128 +1964,59 @@ static void rd_use_partition(AV1_COMP *cpi, ThreadData *td, *dist = chosen_rdc.dist; } -// TODO(kyslov): now this is very similar to rd_use_partition (except that -// doesn't do extra search arounf suggested partitioning) -// consider passing a flag to select non-rd path (similar to -// encode_sb_row) static void nonrd_use_partition(AV1_COMP *cpi, ThreadData *td, TileDataEnc *tile_data, MB_MODE_INFO **mib, TOKENEXTRA **tp, int mi_row, int mi_col, - BLOCK_SIZE bsize, int *rate, int64_t *dist, - int do_recon, PC_TREE *pc_tree) { + BLOCK_SIZE bsize, PC_TREE *pc_tree) { AV1_COMMON *const cm = &cpi->common; - const int num_planes = av1_num_planes(cm); TileInfo *const tile_info = &tile_data->tile_info; + const SPEED_FEATURES *const sf = &cpi->sf; MACROBLOCK *const x = &td->mb; MACROBLOCKD *const xd = &x->e_mbd; + // Only square blocks from 8x8 to 128x128 are supported + assert(bsize >= BLOCK_8X8 && bsize <= BLOCK_128X128); const int bs = mi_size_wide[bsize]; const int hbs = bs / 2; - int i; - const int pl = (bsize >= BLOCK_8X8) - ? partition_plane_context(xd, mi_row, mi_col, bsize) - : 0; const PARTITION_TYPE partition = (bsize >= BLOCK_8X8) ? get_partition(cm, mi_row, mi_col, bsize) : PARTITION_NONE; const BLOCK_SIZE subsize = get_partition_subsize(bsize, partition); - RD_SEARCH_MACROBLOCK_CONTEXT x_ctx; - RD_STATS last_part_rdc; - PICK_MODE_CONTEXT *ctx_none = &pc_tree->none; + RD_STATS dummy_cost; if (mi_row >= cm->mi_rows || mi_col >= cm->mi_cols) return; assert(mi_size_wide[bsize] == mi_size_high[bsize]); - av1_invalid_rd_stats(&last_part_rdc); - pc_tree->partitioning = partition; xd->above_txfm_context = cm->above_txfm_context[tile_info->tile_row] + mi_col; xd->left_txfm_context = xd->left_txfm_context_buffer + (mi_row & MAX_MIB_MASK); - save_context(x, &x_ctx, mi_row, mi_col, bsize, num_planes); - - if (bsize == BLOCK_16X16 && cpi->vaq_refresh) { - set_offsets(cpi, tile_info, x, mi_row, mi_col, bsize); - x->mb_energy = av1_log_block_var(cpi, x, bsize); - } switch (partition) { case PARTITION_NONE: - pick_sb_modes(cpi, tile_data, x, mi_row, mi_col, &last_part_rdc, - PARTITION_NONE, bsize, ctx_none, INT64_MAX, 1); + pick_sb_modes(cpi, tile_data, x, mi_row, mi_col, &dummy_cost, + PARTITION_NONE, bsize, &pc_tree->none, INT64_MAX, + sf->use_fast_nonrd_pick_mode ? PICK_MODE_FAST_NONRD + : PICK_MODE_NONRD); + encode_b(cpi, tile_data, td, tp, mi_row, mi_col, 0, bsize, partition, + &pc_tree->none, NULL); break; + // TODO(kyslov@) Add HORZ and VERT partitions case PARTITION_HORZ: - pick_sb_modes(cpi, tile_data, x, mi_row, mi_col, &last_part_rdc, - PARTITION_HORZ, subsize, &pc_tree->horizontal[0], INT64_MAX, - 1); - if (last_part_rdc.rate != INT_MAX && bsize >= BLOCK_8X8 && - mi_row + hbs < cm->mi_rows) { - RD_STATS tmp_rdc; - const PICK_MODE_CONTEXT *const ctx_h = &pc_tree->horizontal[0]; - av1_init_rd_stats(&tmp_rdc); - update_state(cpi, tile_data, td, ctx_h, mi_row, mi_col, subsize, 1); - encode_superblock(cpi, tile_data, td, tp, DRY_RUN_NORMAL, mi_row, - mi_col, subsize, NULL); - pick_sb_modes(cpi, tile_data, x, mi_row + hbs, mi_col, &tmp_rdc, - PARTITION_HORZ, subsize, &pc_tree->horizontal[1], - INT64_MAX, 1); - if (tmp_rdc.rate == INT_MAX || tmp_rdc.dist == INT64_MAX) { - av1_invalid_rd_stats(&last_part_rdc); - break; - } - last_part_rdc.rate += tmp_rdc.rate; - last_part_rdc.dist += tmp_rdc.dist; - last_part_rdc.rdcost += tmp_rdc.rdcost; - } - break; case PARTITION_VERT: - pick_sb_modes(cpi, tile_data, x, mi_row, mi_col, &last_part_rdc, - PARTITION_VERT, subsize, &pc_tree->vertical[0], INT64_MAX, - 1); - if (last_part_rdc.rate != INT_MAX && bsize >= BLOCK_8X8 && - mi_col + hbs < cm->mi_cols) { - RD_STATS tmp_rdc; - const PICK_MODE_CONTEXT *const ctx_v = &pc_tree->vertical[0]; - av1_init_rd_stats(&tmp_rdc); - update_state(cpi, tile_data, td, ctx_v, mi_row, mi_col, subsize, 1); - encode_superblock(cpi, tile_data, td, tp, DRY_RUN_NORMAL, mi_row, - mi_col, subsize, NULL); - pick_sb_modes(cpi, tile_data, x, mi_row, mi_col + hbs, &tmp_rdc, - PARTITION_VERT, subsize, - &pc_tree->vertical[bsize > BLOCK_8X8], INT64_MAX, 1); - if (tmp_rdc.rate == INT_MAX || tmp_rdc.dist == INT64_MAX) { - av1_invalid_rd_stats(&last_part_rdc); - break; - } - last_part_rdc.rate += tmp_rdc.rate; - last_part_rdc.dist += tmp_rdc.dist; - last_part_rdc.rdcost += tmp_rdc.rdcost; - } + assert(0 && "Cannot yet handle non-square partition types"); break; case PARTITION_SPLIT: - last_part_rdc.rate = 0; - last_part_rdc.dist = 0; - last_part_rdc.rdcost = 0; - for (i = 0; i < 4; i++) { + for (int i = 0; i < 4; i++) { int x_idx = (i & 1) * hbs; int y_idx = (i >> 1) * hbs; int jj = i >> 1, ii = i & 0x01; - RD_STATS tmp_rdc; if ((mi_row + y_idx >= cm->mi_rows) || (mi_col + x_idx >= cm->mi_cols)) continue; - - av1_init_rd_stats(&tmp_rdc); nonrd_use_partition( cpi, td, tile_data, mib + jj * hbs * cm->mi_stride + ii * hbs, tp, - mi_row + y_idx, mi_col + x_idx, subsize, &tmp_rdc.rate, - &tmp_rdc.dist, i != 3, pc_tree->split[i]); - if (tmp_rdc.rate == INT_MAX || tmp_rdc.dist == INT64_MAX) { - av1_invalid_rd_stats(&last_part_rdc); - break; - } - last_part_rdc.rate += tmp_rdc.rate; - last_part_rdc.dist += tmp_rdc.dist; + mi_row + y_idx, mi_col + x_idx, subsize, pc_tree->split[i]); } break; case PARTITION_VERT_A: @@ -2084,37 +2028,8 @@ static void nonrd_use_partition(AV1_COMP *cpi, ThreadData *td, assert(0 && "Cannot handle extended partition types"); default: assert(0); break; } - - if (last_part_rdc.rate < INT_MAX) { - last_part_rdc.rate += x->partition_cost[pl][partition]; - last_part_rdc.rdcost = - RDCOST(x->rdmult, last_part_rdc.rate, last_part_rdc.dist); - } - - restore_context(x, &x_ctx, mi_row, mi_col, bsize, num_planes); - - // We must have chosen a partitioning and encoding or we'll fail later on. - // No other opportunities for success. - if (bsize == cm->seq_params.sb_size) - assert(last_part_rdc.rate < INT_MAX && last_part_rdc.dist < INT64_MAX); - - if (do_recon) { - if (bsize == cm->seq_params.sb_size) { - // NOTE: To get estimate for rate due to the tokens, use: - // int rate_coeffs = 0; - // encode_sb(cpi, td, tile_data, tp, mi_row, mi_col, DRY_RUN_COSTCOEFFS, - // bsize, pc_tree, &rate_coeffs); - x->cb_offset = 0; - encode_sb(cpi, td, tile_data, tp, mi_row, mi_col, OUTPUT_ENABLED, bsize, - pc_tree, NULL); - } else { - encode_sb(cpi, td, tile_data, tp, mi_row, mi_col, DRY_RUN_NORMAL, bsize, - pc_tree, NULL); - } - } - - *rate = last_part_rdc.rate; - *dist = last_part_rdc.dist; + if (partition != PARTITION_SPLIT || bsize == BLOCK_8X8) + update_partition_context(xd, mi_row, mi_col, subsize, bsize); } // Checks to see if a super block is on a horizontal image edge. @@ -2186,34 +2101,29 @@ static INLINE void load_pred_mv(MACROBLOCK *x, static int rd_try_subblock(AV1_COMP *const cpi, ThreadData *td, TileDataEnc *tile_data, TOKENEXTRA **tp, int is_last, int mi_row, int mi_col, BLOCK_SIZE subsize, - RD_STATS *best_rdc, RD_STATS *sum_rdc, - RD_STATS *this_rdc, PARTITION_TYPE partition, + int64_t best_rdcost, RD_STATS *sum_rdc, + PARTITION_TYPE partition, PICK_MODE_CONTEXT *prev_ctx, PICK_MODE_CONTEXT *this_ctx) { -#define RTS_X_RATE_NOCOEF_ARG -#define RTS_MAX_RDCOST best_rdc->rdcost - MACROBLOCK *const x = &td->mb; if (cpi->sf.adaptive_motion_search) load_pred_mv(x, prev_ctx); - const int64_t rdcost_remaining = best_rdc->rdcost == INT64_MAX - ? INT64_MAX - : (best_rdc->rdcost - sum_rdc->rdcost); + const int64_t rdcost_remaining = + best_rdcost == INT64_MAX ? INT64_MAX : (best_rdcost - sum_rdc->rdcost); + RD_STATS this_rdc; + pick_sb_modes(cpi, tile_data, x, mi_row, mi_col, &this_rdc, partition, + subsize, this_ctx, rdcost_remaining, PICK_MODE_RD); - pick_sb_modes(cpi, tile_data, x, mi_row, mi_col, this_rdc, - RTS_X_RATE_NOCOEF_ARG partition, subsize, this_ctx, - rdcost_remaining, 0); - - if (this_rdc->rate == INT_MAX) { + if (this_rdc.rate == INT_MAX) { sum_rdc->rdcost = INT64_MAX; } else { - sum_rdc->rate += this_rdc->rate; - sum_rdc->dist += this_rdc->dist; - sum_rdc->rdcost += this_rdc->rdcost; + sum_rdc->rate += this_rdc.rate; + sum_rdc->dist += this_rdc.dist; + sum_rdc->rdcost += this_rdc.rdcost; } - if (sum_rdc->rdcost >= RTS_MAX_RDCOST) return 0; + if (sum_rdc->rdcost >= best_rdcost) return 0; if (!is_last) { update_state(cpi, tile_data, td, this_ctx, mi_row, mi_col, subsize, 1); @@ -2222,9 +2132,6 @@ static int rd_try_subblock(AV1_COMP *const cpi, ThreadData *td, } return 1; - -#undef RTS_X_RATE_NOCOEF_ARG -#undef RTS_MAX_RDCOST } static void rd_test_partition3(AV1_COMP *const cpi, ThreadData *td, @@ -2236,52 +2143,39 @@ static void rd_test_partition3(AV1_COMP *const cpi, ThreadData *td, int mi_row0, int mi_col0, BLOCK_SIZE subsize0, int mi_row1, int mi_col1, BLOCK_SIZE subsize1, int mi_row2, int mi_col2, BLOCK_SIZE subsize2) { - MACROBLOCK *const x = &td->mb; - MACROBLOCKD *const xd = &x->e_mbd; - RD_STATS sum_rdc, this_rdc; -#define RTP_STX_TRY_ARGS - int pl = partition_plane_context(xd, mi_row, mi_col, bsize); + const MACROBLOCK *const x = &td->mb; + const MACROBLOCKD *const xd = &x->e_mbd; + const int pl = partition_plane_context(xd, mi_row, mi_col, bsize); + RD_STATS sum_rdc; av1_init_rd_stats(&sum_rdc); sum_rdc.rate = x->partition_cost[pl][partition]; sum_rdc.rdcost = RDCOST(x->rdmult, sum_rdc.rate, 0); if (!rd_try_subblock(cpi, td, tile_data, tp, 0, mi_row0, mi_col0, subsize0, - best_rdc, &sum_rdc, &this_rdc, - RTP_STX_TRY_ARGS partition, ctx, &ctxs[0])) + best_rdc->rdcost, &sum_rdc, partition, ctx, &ctxs[0])) return; if (!rd_try_subblock(cpi, td, tile_data, tp, 0, mi_row1, mi_col1, subsize1, - best_rdc, &sum_rdc, &this_rdc, - RTP_STX_TRY_ARGS partition, &ctxs[0], &ctxs[1])) + best_rdc->rdcost, &sum_rdc, partition, &ctxs[0], + &ctxs[1])) return; - // With the new layout of mixed partitions for PARTITION_HORZ_B and - // PARTITION_VERT_B, the last subblock might start past halfway through the - // main block, so we might signal it even though the subblock lies strictly - // outside the image. In that case, we won't spend any bits coding it and the - // difference (obviously) doesn't contribute to the error. - const int try_block2 = 1; - if (try_block2 && - !rd_try_subblock(cpi, td, tile_data, tp, 1, mi_row2, mi_col2, subsize2, - best_rdc, &sum_rdc, &this_rdc, - RTP_STX_TRY_ARGS partition, &ctxs[1], &ctxs[2])) + if (!rd_try_subblock(cpi, td, tile_data, tp, 1, mi_row2, mi_col2, subsize2, + best_rdc->rdcost, &sum_rdc, partition, &ctxs[1], + &ctxs[2])) return; if (sum_rdc.rdcost >= best_rdc->rdcost) return; - sum_rdc.rdcost = RDCOST(x->rdmult, sum_rdc.rate, sum_rdc.dist); - if (sum_rdc.rdcost >= best_rdc->rdcost) return; *best_rdc = sum_rdc; pc_tree->partitioning = partition; - -#undef RTP_STX_TRY_ARGS } static void reset_partition(PC_TREE *pc_tree, BLOCK_SIZE bsize) { pc_tree->partitioning = PARTITION_NONE; pc_tree->cb_search_range = SEARCH_FULL_PLANE; - pc_tree->none.skip = 0; + pc_tree->none.rd_stats.skip = 0; pc_tree->pc_tree_stats.valid = 0; pc_tree->pc_tree_stats.split = 0; @@ -2340,8 +2234,8 @@ static void rd_pick_sqr_partition(AV1_COMP *const cpi, ThreadData *td, (void)split_rd; if (best_rd < 0) { - pc_tree->none.rdcost = INT64_MAX; - pc_tree->none.skip = 0; + pc_tree->none.rd_stats.rdcost = INT64_MAX; + pc_tree->none.rd_stats.skip = 0; av1_invalid_rd_stats(rd_cost); return; } @@ -2420,10 +2314,10 @@ static void rd_pick_sqr_partition(AV1_COMP *const cpi, ThreadData *td, best_rdc.rdcost == INT64_MAX ? INT64_MAX : (best_rdc.rdcost - partition_rd_cost); pick_sb_modes(cpi, tile_data, x, mi_row, mi_col, &this_rdc, PARTITION_NONE, - bsize, ctx_none, best_remain_rdcost, 0); + bsize, ctx_none, best_remain_rdcost, PICK_MODE_RD); - pc_tree->pc_tree_stats.rdcost = ctx_none->rdcost; - pc_tree->pc_tree_stats.skip = ctx_none->skip; + pc_tree->pc_tree_stats.rdcost = ctx_none->rd_stats.rdcost; + pc_tree->pc_tree_stats.skip = ctx_none->rd_stats.skip; if (none_rd) *none_rd = this_rdc.rdcost; if (this_rdc.rate != INT_MAX) { @@ -2456,8 +2350,8 @@ static void rd_pick_sqr_partition(AV1_COMP *const cpi, ThreadData *td, // values as in rd_pick_partition. Retraining the model and tuning the // threshold values might be helpful to improve the speed. if (use_ml_based_breakout) { - if (ml_predict_breakout(cpi, bsize, x, &this_rdc, - x->source_variance)) { + if (av1_ml_predict_breakout(cpi, bsize, x, &this_rdc, + x->source_variance)) { do_square_split = 0; } } @@ -2524,7 +2418,7 @@ static void rd_pick_sqr_partition(AV1_COMP *const cpi, ThreadData *td, pc_tree->pc_tree_stats.sub_block_rdcost[idx] = this_rdc.rdcost; pc_tree->pc_tree_stats.sub_block_skip[idx] = - pc_tree->split[idx]->none.skip; + pc_tree->split[idx]->none.rd_stats.skip; if (this_rdc.rate == INT_MAX) { sum_rdc.rdcost = INT64_MAX; @@ -2602,488 +2496,6 @@ static void rd_pick_sqr_partition(AV1_COMP *const cpi, ThreadData *td, } } -// split_score indicates confidence of picking split partition; -// none_score indicates confidence of picking none partition; -#define FEATURE_SIZE 19 -static int ml_prune_2pass_split_partition(const PC_TREE_STATS *pc_tree_stats, - BLOCK_SIZE bsize, int *split_score, - int *none_score) { - if (!pc_tree_stats->valid) return 0; - const float *split_weights = NULL; - const float *none_weights = NULL; - switch (bsize) { - case BLOCK_4X4: break; - case BLOCK_8X8: - split_weights = av1_2pass_split_partition_weights_8; - none_weights = av1_2pass_none_partition_weights_8; - break; - case BLOCK_16X16: - split_weights = av1_2pass_split_partition_weights_16; - none_weights = av1_2pass_none_partition_weights_16; - break; - case BLOCK_32X32: - split_weights = av1_2pass_split_partition_weights_32; - none_weights = av1_2pass_none_partition_weights_32; - break; - case BLOCK_64X64: - split_weights = av1_2pass_split_partition_weights_64; - none_weights = av1_2pass_none_partition_weights_64; - break; - case BLOCK_128X128: - split_weights = av1_2pass_split_partition_weights_128; - none_weights = av1_2pass_none_partition_weights_128; - break; - default: assert(0 && "Unexpected bsize."); - } - if (!split_weights || !none_weights) return 0; - - aom_clear_system_state(); - - float features[FEATURE_SIZE]; - int feature_index = 0; - features[feature_index++] = (float)pc_tree_stats->split; - features[feature_index++] = (float)pc_tree_stats->skip; - const int rdcost = (int)AOMMIN(INT_MAX, pc_tree_stats->rdcost); - const int rd_valid = rdcost > 0 && rdcost < 1000000000; - features[feature_index++] = (float)rd_valid; - for (int i = 0; i < 4; ++i) { - features[feature_index++] = (float)pc_tree_stats->sub_block_split[i]; - features[feature_index++] = (float)pc_tree_stats->sub_block_skip[i]; - const int sub_rdcost = - (int)AOMMIN(INT_MAX, pc_tree_stats->sub_block_rdcost[i]); - const int sub_rd_valid = sub_rdcost > 0 && sub_rdcost < 1000000000; - features[feature_index++] = (float)sub_rd_valid; - // Ratio between the sub-block RD and the whole-block RD. - float rd_ratio = 1.0f; - if (rd_valid && sub_rd_valid && sub_rdcost < rdcost) - rd_ratio = (float)sub_rdcost / (float)rdcost; - features[feature_index++] = rd_ratio; - } - assert(feature_index == FEATURE_SIZE); - - float score_1 = split_weights[FEATURE_SIZE]; - float score_2 = none_weights[FEATURE_SIZE]; - for (int i = 0; i < FEATURE_SIZE; ++i) { - score_1 += features[i] * split_weights[i]; - score_2 += features[i] * none_weights[i]; - } - *split_score = (int)(score_1 * 100); - *none_score = (int)(score_2 * 100); - return 1; -} -#undef FEATURE_SIZE - -static void ml_prune_rect_partition(const AV1_COMP *const cpi, - const MACROBLOCK *const x, BLOCK_SIZE bsize, - int64_t best_rd, int64_t none_rd, - int64_t *split_rd, - int *const dst_prune_horz, - int *const dst_prune_vert) { - if (bsize < BLOCK_8X8 || best_rd >= 1000000000) return; - best_rd = AOMMAX(best_rd, 1); - const NN_CONFIG *nn_config = NULL; - const float prob_thresholds[5] = { 0.01f, 0.01f, 0.004f, 0.002f, 0.002f }; - float cur_thresh = 0.0f; - switch (bsize) { - case BLOCK_8X8: - nn_config = &av1_rect_partition_nnconfig_8; - cur_thresh = prob_thresholds[0]; - break; - case BLOCK_16X16: - nn_config = &av1_rect_partition_nnconfig_16; - cur_thresh = prob_thresholds[1]; - break; - case BLOCK_32X32: - nn_config = &av1_rect_partition_nnconfig_32; - cur_thresh = prob_thresholds[2]; - break; - case BLOCK_64X64: - nn_config = &av1_rect_partition_nnconfig_64; - cur_thresh = prob_thresholds[3]; - break; - case BLOCK_128X128: - nn_config = &av1_rect_partition_nnconfig_128; - cur_thresh = prob_thresholds[4]; - break; - default: assert(0 && "Unexpected bsize."); - } - if (!nn_config) return; - aom_clear_system_state(); - - // 1. Compute input features - float features[9]; - - // RD cost ratios - for (int i = 0; i < 5; i++) features[i] = 1.0f; - if (none_rd > 0 && none_rd < 1000000000) - features[0] = (float)none_rd / (float)best_rd; - for (int i = 0; i < 4; i++) { - if (split_rd[i] > 0 && split_rd[i] < 1000000000) - features[1 + i] = (float)split_rd[i] / (float)best_rd; - } - - // Variance ratios - const MACROBLOCKD *const xd = &x->e_mbd; - int whole_block_variance; - if (is_cur_buf_hbd(xd)) { - whole_block_variance = av1_high_get_sby_perpixel_variance( - cpi, &x->plane[0].src, bsize, xd->bd); - } else { - whole_block_variance = - av1_get_sby_perpixel_variance(cpi, &x->plane[0].src, bsize); - } - whole_block_variance = AOMMAX(whole_block_variance, 1); - - int split_variance[4]; - const BLOCK_SIZE subsize = get_partition_subsize(bsize, PARTITION_SPLIT); - struct buf_2d buf; - buf.stride = x->plane[0].src.stride; - const int bw = block_size_wide[bsize]; - for (int i = 0; i < 4; ++i) { - const int x_idx = (i & 1) * bw / 2; - const int y_idx = (i >> 1) * bw / 2; - buf.buf = x->plane[0].src.buf + x_idx + y_idx * buf.stride; - if (is_cur_buf_hbd(xd)) { - split_variance[i] = - av1_high_get_sby_perpixel_variance(cpi, &buf, subsize, xd->bd); - } else { - split_variance[i] = av1_get_sby_perpixel_variance(cpi, &buf, subsize); - } - } - - for (int i = 0; i < 4; i++) - features[5 + i] = (float)split_variance[i] / (float)whole_block_variance; - - // 2. Do the prediction and prune 0-2 partitions based on their probabilities - float raw_scores[3] = { 0.0f }; - av1_nn_predict(features, nn_config, raw_scores); - aom_clear_system_state(); - float probs[3] = { 0.0f }; - av1_nn_softmax(raw_scores, probs, 3); - - // probs[0] is the probability of the fact that both rectangular partitions - // are worse than current best_rd - if (probs[1] <= cur_thresh) (*dst_prune_horz) = 1; - if (probs[2] <= cur_thresh) (*dst_prune_vert) = 1; -} - -// Use a ML model to predict if horz_a, horz_b, vert_a, and vert_b should be -// considered. -static void ml_prune_ab_partition(BLOCK_SIZE bsize, int part_ctx, int var_ctx, - int64_t best_rd, int64_t horz_rd[2], - int64_t vert_rd[2], int64_t split_rd[4], - int *const horza_partition_allowed, - int *const horzb_partition_allowed, - int *const verta_partition_allowed, - int *const vertb_partition_allowed) { - if (bsize < BLOCK_8X8 || best_rd >= 1000000000) return; - const NN_CONFIG *nn_config = NULL; - switch (bsize) { - case BLOCK_8X8: nn_config = NULL; break; - case BLOCK_16X16: nn_config = &av1_ab_partition_nnconfig_16; break; - case BLOCK_32X32: nn_config = &av1_ab_partition_nnconfig_32; break; - case BLOCK_64X64: nn_config = &av1_ab_partition_nnconfig_64; break; - case BLOCK_128X128: nn_config = &av1_ab_partition_nnconfig_128; break; - default: assert(0 && "Unexpected bsize."); - } - if (!nn_config) return; - - aom_clear_system_state(); - - // Generate features. - float features[10]; - int feature_index = 0; - features[feature_index++] = (float)part_ctx; - features[feature_index++] = (float)var_ctx; - const int rdcost = (int)AOMMIN(INT_MAX, best_rd); - int sub_block_rdcost[8] = { 0 }; - int rd_index = 0; - for (int i = 0; i < 2; ++i) { - if (horz_rd[i] > 0 && horz_rd[i] < 1000000000) - sub_block_rdcost[rd_index] = (int)horz_rd[i]; - ++rd_index; - } - for (int i = 0; i < 2; ++i) { - if (vert_rd[i] > 0 && vert_rd[i] < 1000000000) - sub_block_rdcost[rd_index] = (int)vert_rd[i]; - ++rd_index; - } - for (int i = 0; i < 4; ++i) { - if (split_rd[i] > 0 && split_rd[i] < 1000000000) - sub_block_rdcost[rd_index] = (int)split_rd[i]; - ++rd_index; - } - for (int i = 0; i < 8; ++i) { - // Ratio between the sub-block RD and the whole-block RD. - float rd_ratio = 1.0f; - if (sub_block_rdcost[i] > 0 && sub_block_rdcost[i] < rdcost) - rd_ratio = (float)sub_block_rdcost[i] / (float)rdcost; - features[feature_index++] = rd_ratio; - } - assert(feature_index == 10); - - // Calculate scores using the NN model. - float score[16] = { 0.0f }; - av1_nn_predict(features, nn_config, score); - aom_clear_system_state(); - int int_score[16]; - int max_score = -1000; - for (int i = 0; i < 16; ++i) { - int_score[i] = (int)(100 * score[i]); - max_score = AOMMAX(int_score[i], max_score); - } - - // Make decisions based on the model scores. - int thresh = max_score; - switch (bsize) { - case BLOCK_16X16: thresh -= 150; break; - case BLOCK_32X32: thresh -= 100; break; - default: break; - } - *horza_partition_allowed = 0; - *horzb_partition_allowed = 0; - *verta_partition_allowed = 0; - *vertb_partition_allowed = 0; - for (int i = 0; i < 16; ++i) { - if (int_score[i] >= thresh) { - if ((i >> 0) & 1) *horza_partition_allowed = 1; - if ((i >> 1) & 1) *horzb_partition_allowed = 1; - if ((i >> 2) & 1) *verta_partition_allowed = 1; - if ((i >> 3) & 1) *vertb_partition_allowed = 1; - } - } -} - -#define FEATURES 18 -#define LABELS 4 -// Use a ML model to predict if horz4 and vert4 should be considered. -static void ml_prune_4_partition(const AV1_COMP *const cpi, MACROBLOCK *const x, - BLOCK_SIZE bsize, int part_ctx, - int64_t best_rd, int64_t horz_rd[2], - int64_t vert_rd[2], int64_t split_rd[4], - int *const partition_horz4_allowed, - int *const partition_vert4_allowed, - unsigned int pb_source_variance, int mi_row, - int mi_col) { - if (best_rd >= 1000000000) return; - const NN_CONFIG *nn_config = NULL; - switch (bsize) { - case BLOCK_16X16: nn_config = &av1_4_partition_nnconfig_16; break; - case BLOCK_32X32: nn_config = &av1_4_partition_nnconfig_32; break; - case BLOCK_64X64: nn_config = &av1_4_partition_nnconfig_64; break; - default: assert(0 && "Unexpected bsize."); - } - if (!nn_config) return; - - aom_clear_system_state(); - - // Generate features. - float features[FEATURES]; - int feature_index = 0; - features[feature_index++] = (float)part_ctx; - features[feature_index++] = (float)get_unsigned_bits(pb_source_variance); - - const int rdcost = (int)AOMMIN(INT_MAX, best_rd); - int sub_block_rdcost[8] = { 0 }; - int rd_index = 0; - for (int i = 0; i < 2; ++i) { - if (horz_rd[i] > 0 && horz_rd[i] < 1000000000) - sub_block_rdcost[rd_index] = (int)horz_rd[i]; - ++rd_index; - } - for (int i = 0; i < 2; ++i) { - if (vert_rd[i] > 0 && vert_rd[i] < 1000000000) - sub_block_rdcost[rd_index] = (int)vert_rd[i]; - ++rd_index; - } - for (int i = 0; i < 4; ++i) { - if (split_rd[i] > 0 && split_rd[i] < 1000000000) - sub_block_rdcost[rd_index] = (int)split_rd[i]; - ++rd_index; - } - for (int i = 0; i < 8; ++i) { - // Ratio between the sub-block RD and the whole-block RD. - float rd_ratio = 1.0f; - if (sub_block_rdcost[i] > 0 && sub_block_rdcost[i] < rdcost) - rd_ratio = (float)sub_block_rdcost[i] / (float)rdcost; - features[feature_index++] = rd_ratio; - } - - // Get variance of the 1:4 and 4:1 sub-blocks. - unsigned int horz_4_source_var[4] = { 0 }; - unsigned int vert_4_source_var[4] = { 0 }; - { - BLOCK_SIZE horz_4_bs = get_partition_subsize(bsize, PARTITION_HORZ_4); - BLOCK_SIZE vert_4_bs = get_partition_subsize(bsize, PARTITION_VERT_4); - av1_setup_src_planes(x, cpi->source, mi_row, mi_col, - av1_num_planes(&cpi->common), bsize); - const int src_stride = x->plane[0].src.stride; - const uint8_t *src = x->plane[0].src.buf; - const MACROBLOCKD *const xd = &x->e_mbd; - for (int i = 0; i < 4; ++i) { - const uint8_t *horz_src = - src + i * block_size_high[horz_4_bs] * src_stride; - const uint8_t *vert_src = src + i * block_size_wide[vert_4_bs]; - unsigned int horz_var, vert_var, sse; - if (is_cur_buf_hbd(xd)) { - switch (xd->bd) { - case 10: - horz_var = cpi->fn_ptr[horz_4_bs].vf( - horz_src, src_stride, CONVERT_TO_BYTEPTR(AV1_HIGH_VAR_OFFS_10), - 0, &sse); - vert_var = cpi->fn_ptr[vert_4_bs].vf( - vert_src, src_stride, CONVERT_TO_BYTEPTR(AV1_HIGH_VAR_OFFS_10), - 0, &sse); - break; - case 12: - horz_var = cpi->fn_ptr[horz_4_bs].vf( - horz_src, src_stride, CONVERT_TO_BYTEPTR(AV1_HIGH_VAR_OFFS_12), - 0, &sse); - vert_var = cpi->fn_ptr[vert_4_bs].vf( - vert_src, src_stride, CONVERT_TO_BYTEPTR(AV1_HIGH_VAR_OFFS_12), - 0, &sse); - break; - case 8: - default: - horz_var = cpi->fn_ptr[horz_4_bs].vf( - horz_src, src_stride, CONVERT_TO_BYTEPTR(AV1_HIGH_VAR_OFFS_8), - 0, &sse); - vert_var = cpi->fn_ptr[vert_4_bs].vf( - vert_src, src_stride, CONVERT_TO_BYTEPTR(AV1_HIGH_VAR_OFFS_8), - 0, &sse); - break; - } - horz_4_source_var[i] = - ROUND_POWER_OF_TWO(horz_var, num_pels_log2_lookup[horz_4_bs]); - vert_4_source_var[i] = - ROUND_POWER_OF_TWO(vert_var, num_pels_log2_lookup[vert_4_bs]); - } else { - horz_var = cpi->fn_ptr[horz_4_bs].vf(horz_src, src_stride, AV1_VAR_OFFS, - 0, &sse); - vert_var = cpi->fn_ptr[vert_4_bs].vf(vert_src, src_stride, AV1_VAR_OFFS, - 0, &sse); - horz_4_source_var[i] = - ROUND_POWER_OF_TWO(horz_var, num_pels_log2_lookup[horz_4_bs]); - vert_4_source_var[i] = - ROUND_POWER_OF_TWO(vert_var, num_pels_log2_lookup[vert_4_bs]); - } - } - } - - const float denom = (float)(pb_source_variance + 1); - const float low_b = 0.1f; - const float high_b = 10.0f; - for (int i = 0; i < 4; ++i) { - // Ratio between the 4:1 sub-block variance and the whole-block variance. - float var_ratio = (float)(horz_4_source_var[i] + 1) / denom; - if (var_ratio < low_b) var_ratio = low_b; - if (var_ratio > high_b) var_ratio = high_b; - features[feature_index++] = var_ratio; - } - for (int i = 0; i < 4; ++i) { - // Ratio between the 1:4 sub-block RD and the whole-block RD. - float var_ratio = (float)(vert_4_source_var[i] + 1) / denom; - if (var_ratio < low_b) var_ratio = low_b; - if (var_ratio > high_b) var_ratio = high_b; - features[feature_index++] = var_ratio; - } - assert(feature_index == FEATURES); - - // Calculate scores using the NN model. - float score[LABELS] = { 0.0f }; - av1_nn_predict(features, nn_config, score); - aom_clear_system_state(); - int int_score[LABELS]; - int max_score = -1000; - for (int i = 0; i < LABELS; ++i) { - int_score[i] = (int)(100 * score[i]); - max_score = AOMMAX(int_score[i], max_score); - } - - // Make decisions based on the model scores. - int thresh = max_score; - switch (bsize) { - case BLOCK_16X16: thresh -= 500; break; - case BLOCK_32X32: thresh -= 500; break; - case BLOCK_64X64: thresh -= 200; break; - default: break; - } - *partition_horz4_allowed = 0; - *partition_vert4_allowed = 0; - for (int i = 0; i < LABELS; ++i) { - if (int_score[i] >= thresh) { - if ((i >> 0) & 1) *partition_horz4_allowed = 1; - if ((i >> 1) & 1) *partition_vert4_allowed = 1; - } - } -} -#undef FEATURES -#undef LABELS - -#define FEATURES 4 -// ML-based partition search breakout. -static int ml_predict_breakout(const AV1_COMP *const cpi, BLOCK_SIZE bsize, - const MACROBLOCK *const x, - const RD_STATS *const rd_stats, - unsigned int pb_source_variance) { - const NN_CONFIG *nn_config = NULL; - int thresh = 0; - switch (bsize) { - case BLOCK_8X8: - nn_config = &av1_partition_breakout_nnconfig_8; - thresh = cpi->sf.ml_partition_search_breakout_thresh[0]; - break; - case BLOCK_16X16: - nn_config = &av1_partition_breakout_nnconfig_16; - thresh = cpi->sf.ml_partition_search_breakout_thresh[1]; - break; - case BLOCK_32X32: - nn_config = &av1_partition_breakout_nnconfig_32; - thresh = cpi->sf.ml_partition_search_breakout_thresh[2]; - break; - case BLOCK_64X64: - nn_config = &av1_partition_breakout_nnconfig_64; - thresh = cpi->sf.ml_partition_search_breakout_thresh[3]; - break; - case BLOCK_128X128: - nn_config = &av1_partition_breakout_nnconfig_128; - thresh = cpi->sf.ml_partition_search_breakout_thresh[4]; - break; - default: assert(0 && "Unexpected bsize."); - } - if (!nn_config || thresh < 0) return 0; - - // Generate feature values. - float features[FEATURES]; - int feature_index = 0; - aom_clear_system_state(); - - const int num_pels_log2 = num_pels_log2_lookup[bsize]; - float rate_f = (float)AOMMIN(rd_stats->rate, INT_MAX); - rate_f = ((float)x->rdmult / 128.0f / 512.0f / (float)(1 << num_pels_log2)) * - rate_f; - features[feature_index++] = rate_f; - - const float dist_f = - (float)(AOMMIN(rd_stats->dist, INT_MAX) >> num_pels_log2); - features[feature_index++] = dist_f; - - features[feature_index++] = (float)pb_source_variance; - - const int dc_q = (int)x->plane[0].dequant_QTX[0]; - features[feature_index++] = (float)(dc_q * dc_q) / 256.0f; - assert(feature_index == FEATURES); - - // Calculate score using the NN model. - float score = 0.0f; - av1_nn_predict(features, nn_config, &score); - aom_clear_system_state(); - - // Make decision. - return (int)(score * 100) >= thresh; -} -#undef FEATURES - // Record the ref frames that have been selected by square partition blocks. static void update_picked_ref_frames_mask(MACROBLOCK *const x, int ref_type, BLOCK_SIZE bsize, int mib_size, @@ -3103,9 +2515,6 @@ static void update_picked_ref_frames_mask(MACROBLOCK *const x, int ref_type, // TODO(jinging,jimbankoski,rbultje): properly skip partition types that are // unlikely to be selected depending on previous rate-distortion optimization // results, for encoding speed-up. -// TODO(chiyotsai@google.com): Move these ml related varables to a seprate file -// to separate low level ml logic from partition logic -#define NUM_SIMPLE_MOTION_FEATURES 28 static void rd_pick_partition(AV1_COMP *const cpi, ThreadData *td, TileDataEnc *tile_data, TOKENEXTRA **tp, int mi_row, int mi_col, BLOCK_SIZE bsize, @@ -3147,8 +2556,8 @@ static void rd_pick_partition(AV1_COMP *const cpi, ThreadData *td, BLOCK_SIZE bsize2 = get_partition_subsize(bsize, PARTITION_SPLIT); if (best_rd < 0) { - pc_tree->none.rdcost = INT64_MAX; - pc_tree->none.skip = 0; + pc_tree->none.rd_stats.rdcost = INT64_MAX; + pc_tree->none.rd_stats.skip = 0; av1_invalid_rd_stats(rd_cost); return; } @@ -3235,7 +2644,7 @@ static void rd_pick_partition(AV1_COMP *const cpi, ThreadData *td, if (bsize > BLOCK_4X4 && x->use_cb_search_range) { int split_score = 0; int none_score = 0; - const int score_valid = ml_prune_2pass_split_partition( + const int score_valid = av1_ml_prune_2pass_split_partition( &pc_tree->pc_tree_stats, bsize, &split_score, &none_score); if (score_valid) { { @@ -3292,14 +2701,14 @@ static void rd_pick_partition(AV1_COMP *const cpi, ThreadData *td, // Use simple_motion_search to prune partitions. This must be done prior to // PARTITION_SPLIT to propagate the initial mvs to a smaller blocksize. const int try_split_only = - cpi->sf.simple_motion_search_split_only && bsize >= BLOCK_8X8 && - do_square_split && mi_row + mi_size_high[bsize] <= cm->mi_rows && + cpi->sf.simple_motion_search_split && do_square_split && + bsize >= BLOCK_8X8 && mi_row + mi_size_high[bsize] <= cm->mi_rows && mi_col + mi_size_wide[bsize] <= cm->mi_cols && !frame_is_intra_only(cm) && !av1_superres_scaled(cm); if (try_split_only) { av1_simple_motion_search_based_split( - cpi, x, mi_row, mi_col, bsize, &partition_none_allowed, + cpi, x, pc_tree, mi_row, mi_col, bsize, &partition_none_allowed, &partition_horz_allowed, &partition_vert_allowed, &do_rectangular_split, &do_square_split); } @@ -3311,15 +2720,11 @@ static void rd_pick_partition(AV1_COMP *const cpi, ThreadData *td, (prune_horz && prune_vert)) && (partition_horz_allowed || partition_vert_allowed) && bsize >= BLOCK_8X8; - float simple_motion_features[NUM_SIMPLE_MOTION_FEATURES] = { 0.0f }; - int simple_motion_features_are_valid = 0; - if (try_prune_rect) { av1_simple_motion_search_prune_part( cpi, x, pc_tree, mi_row, mi_col, bsize, &partition_none_allowed, &partition_horz_allowed, &partition_vert_allowed, &do_square_split, - &do_rectangular_split, &prune_horz, &prune_vert, simple_motion_features, - &simple_motion_features_are_valid); + &do_rectangular_split, &prune_horz, &prune_vert); } // Max and min square partition levels are defined as the partition nodes that @@ -3385,6 +2790,7 @@ BEGIN_PARTITION_SEARCH: // PARTITION_NONE if (is_le_min_sq_part && has_rows && has_cols) partition_none_allowed = 1; + int64_t part_none_rd = INT64_MAX; if (!terminate_partition_search && partition_none_allowed && !is_gt_max_sq_part) { int pt_cost = 0; @@ -3405,7 +2811,7 @@ BEGIN_PARTITION_SEARCH: } #endif pick_sb_modes(cpi, tile_data, x, mi_row, mi_col, &this_rdc, PARTITION_NONE, - bsize, ctx_none, best_remain_rdcost, 0); + bsize, ctx_none, best_remain_rdcost, PICK_MODE_RD); #if CONFIG_COLLECT_PARTITION_STATS if (partition_timer_on) { aom_usec_timer_mark(&partition_timer); @@ -3429,6 +2835,7 @@ BEGIN_PARTITION_SEARCH: this_rdc.rdcost = RDCOST(x->rdmult, this_rdc.rate, this_rdc.dist); } + part_none_rd = this_rdc.rdcost; if (this_rdc.rdcost < best_rdc.rdcost) { // Adjust dist breakout threshold according to the partition size. const int64_t dist_breakout_thr = @@ -3448,8 +2855,8 @@ BEGIN_PARTITION_SEARCH: bsize <= cpi->sf.use_square_partition_only_threshold && bsize > BLOCK_4X4 && xd->bd == 8; if (use_ml_based_breakout) { - if (ml_predict_breakout(cpi, bsize, x, &this_rdc, - pb_source_variance)) { + if (av1_ml_predict_breakout(cpi, bsize, x, &this_rdc, + pb_source_variance)) { do_square_split = 0; do_rectangular_split = 0; } @@ -3473,10 +2880,9 @@ BEGIN_PARTITION_SEARCH: this_rdc.rdcost < INT64_MAX && this_rdc.rdcost >= 0 && this_rdc.rate < INT_MAX && this_rdc.rate >= 0 && (do_square_split || do_rectangular_split)) { - av1_simple_motion_search_early_term_none( - cpi, x, pc_tree, mi_row, mi_col, bsize, &this_rdc, - &terminate_partition_search, simple_motion_features, - &simple_motion_features_are_valid); + av1_simple_motion_search_early_term_none(cpi, x, pc_tree, mi_row, + mi_col, bsize, &this_rdc, + &terminate_partition_search); } } } @@ -3488,6 +2894,7 @@ BEGIN_PARTITION_SEARCH: if (cpi->sf.adaptive_motion_search) store_pred_mv(x, ctx_none); // PARTITION_SPLIT + int64_t part_split_rd = INT64_MAX; if ((!terminate_partition_search && do_square_split) || is_gt_max_sq_part) { av1_init_rd_stats(&sum_rdc); subsize = get_partition_subsize(bsize, PARTITION_SPLIT); @@ -3548,9 +2955,9 @@ BEGIN_PARTITION_SEARCH: #endif const int reached_last_index = (idx == 4); + part_split_rd = sum_rdc.rdcost; if (reached_last_index && sum_rdc.rdcost < best_rdc.rdcost) { sum_rdc.rdcost = RDCOST(x->rdmult, sum_rdc.rate, sum_rdc.dist); - if (sum_rdc.rdcost < best_rdc.rdcost) { best_rdc = sum_rdc; pc_tree->partitioning = PARTITION_SPLIT; @@ -3565,12 +2972,22 @@ BEGIN_PARTITION_SEARCH: restore_context(x, &x_ctx, mi_row, mi_col, bsize, num_planes); } // if (do_split) - if (cpi->sf.ml_prune_rect_partition && !frame_is_intra_only(cm) && + if (cpi->sf.ml_early_term_after_part_split_level && + !frame_is_intra_only(cm) && !terminate_partition_search && + do_rectangular_split && + (partition_horz_allowed || partition_vert_allowed)) { + av1_ml_early_term_after_split(cpi, x, pc_tree, bsize, best_rdc.rdcost, + part_none_rd, part_split_rd, split_rd, mi_row, + mi_col, &terminate_partition_search); + } + + if (!cpi->sf.ml_early_term_after_part_split_level && + cpi->sf.ml_prune_rect_partition && !frame_is_intra_only(cm) && (partition_horz_allowed || partition_vert_allowed) && - !(prune_horz || prune_vert)) { + !(prune_horz || prune_vert) && !terminate_partition_search) { av1_setup_src_planes(x, cpi->source, mi_row, mi_col, num_planes, bsize); - ml_prune_rect_partition(cpi, x, bsize, best_rdc.rdcost, cur_none_rd, - split_rd, &prune_horz, &prune_vert); + av1_ml_prune_rect_partition(cpi, x, bsize, best_rdc.rdcost, cur_none_rd, + split_rd, &prune_horz, &prune_vert); } // PARTITION_HORZ @@ -3599,7 +3016,8 @@ BEGIN_PARTITION_SEARCH: } #endif pick_sb_modes(cpi, tile_data, x, mi_row, mi_col, &this_rdc, PARTITION_HORZ, - subsize, &pc_tree->horizontal[0], best_remain_rdcost, 0); + subsize, &pc_tree->horizontal[0], best_remain_rdcost, + PICK_MODE_RD); if (this_rdc.rate == INT_MAX) { sum_rdc.rdcost = INT64_MAX; @@ -3631,7 +3049,7 @@ BEGIN_PARTITION_SEARCH: } pick_sb_modes(cpi, tile_data, x, mi_row + mi_step, mi_col, &this_rdc, PARTITION_HORZ, subsize, &pc_tree->horizontal[1], - best_rdc.rdcost - sum_rdc.rdcost, 0); + best_rdc.rdcost - sum_rdc.rdcost, PICK_MODE_RD); horz_rd[1] = this_rdc.rdcost; if (this_rdc.rate == INT_MAX) { @@ -3690,7 +3108,8 @@ BEGIN_PARTITION_SEARCH: } #endif pick_sb_modes(cpi, tile_data, x, mi_row, mi_col, &this_rdc, PARTITION_VERT, - subsize, &pc_tree->vertical[0], best_remain_rdcost, 0); + subsize, &pc_tree->vertical[0], best_remain_rdcost, + PICK_MODE_RD); if (this_rdc.rate == INT_MAX) { sum_rdc.rdcost = INT64_MAX; @@ -3721,7 +3140,7 @@ BEGIN_PARTITION_SEARCH: } pick_sb_modes(cpi, tile_data, x, mi_row, mi_col + mi_step, &this_rdc, PARTITION_VERT, subsize, &pc_tree->vertical[1], - best_rdc.rdcost - sum_rdc.rdcost, 0); + best_rdc.rdcost - sum_rdc.rdcost, PICK_MODE_RD); vert_rd[1] = this_rdc.rdcost; if (this_rdc.rate == INT_MAX) { @@ -3861,11 +3280,11 @@ BEGIN_PARTITION_SEARCH: // TODO(huisu@google.com): x->source_variance may not be the current // block's variance. The correct one to use is pb_source_variance. Need to // re-train the model to fix it. - ml_prune_ab_partition(bsize, pc_tree->partitioning, - get_unsigned_bits(x->source_variance), - best_rdc.rdcost, horz_rd, vert_rd, split_rd, - &horza_partition_allowed, &horzb_partition_allowed, - &verta_partition_allowed, &vertb_partition_allowed); + av1_ml_prune_ab_partition( + bsize, pc_tree->partitioning, get_unsigned_bits(x->source_variance), + best_rdc.rdcost, horz_rd, vert_rd, split_rd, &horza_partition_allowed, + &horzb_partition_allowed, &verta_partition_allowed, + &vertb_partition_allowed); } horza_partition_allowed &= cpi->oxcf.enable_ab_partitions; @@ -4067,10 +3486,10 @@ BEGIN_PARTITION_SEARCH: } if (cpi->sf.ml_prune_4_partition && partition4_allowed && partition_horz_allowed && partition_vert_allowed) { - ml_prune_4_partition(cpi, x, bsize, pc_tree->partitioning, best_rdc.rdcost, - horz_rd, vert_rd, split_rd, &partition_horz4_allowed, - &partition_vert4_allowed, pb_source_variance, mi_row, - mi_col); + av1_ml_prune_4_partition(cpi, x, bsize, pc_tree->partitioning, + best_rdc.rdcost, horz_rd, vert_rd, split_rd, + &partition_horz4_allowed, &partition_vert4_allowed, + pb_source_variance, mi_row, mi_col); } #if CONFIG_DIST_8X8 @@ -4116,7 +3535,7 @@ BEGIN_PARTITION_SEARCH: ctx_this->rd_mode_is_ready = 0; if (!rd_try_subblock(cpi, td, tile_data, tp, (i == 3), this_mi_row, - mi_col, subsize, &best_rdc, &sum_rdc, &this_rdc, + mi_col, subsize, best_rdc.rdcost, &sum_rdc, PARTITION_HORZ_4, ctx_prev, ctx_this)) break; @@ -4171,7 +3590,7 @@ BEGIN_PARTITION_SEARCH: ctx_this->rd_mode_is_ready = 0; if (!rd_try_subblock(cpi, td, tile_data, tp, (i == 3), mi_row, - this_mi_col, subsize, &best_rdc, &sum_rdc, &this_rdc, + this_mi_col, subsize, best_rdc.rdcost, &sum_rdc, PARTITION_VERT_4, ctx_prev, ctx_this)) break; @@ -4335,6 +3754,53 @@ static int get_rdmult_delta(AV1_COMP *cpi, BLOCK_SIZE bsize, int mi_row, return dr; } +static int get_q_for_deltaq_objective(AV1_COMP *const cpi, BLOCK_SIZE bsize, + int mi_row, int mi_col) { + AV1_COMMON *const cm = &cpi->common; + TplDepFrame *tpl_frame = &cpi->tpl_stats[cpi->twopass.gf_group.index]; + TplDepStats *tpl_stats = tpl_frame->tpl_stats_ptr; + int tpl_stride = tpl_frame->stride; + int64_t intra_cost = 0; + int64_t mc_dep_cost = 0; + int mi_wide = mi_size_wide[bsize]; + int mi_high = mi_size_high[bsize]; + int row, col; + + double r0, rk, beta; + + if (tpl_frame->is_valid == 0) return cm->base_qindex; + + if (cpi->common.show_frame) return cm->base_qindex; + + if (cpi->twopass.gf_group.index >= MAX_LAG_BUFFERS) return cm->base_qindex; + + for (row = mi_row; row < mi_row + mi_high; ++row) { + for (col = mi_col; col < mi_col + mi_wide; ++col) { + TplDepStats *this_stats = &tpl_stats[row * tpl_stride + col]; + + if (row >= cm->mi_rows || col >= cm->mi_cols) continue; + + intra_cost += this_stats->intra_cost; + mc_dep_cost += this_stats->mc_dep_cost; + } + } + + aom_clear_system_state(); + + r0 = cpi->rd.r0; + rk = (double)intra_cost / mc_dep_cost; + beta = r0 / rk; + + int offset = -(int)(log(beta) * 8.0); + offset = AOMMIN(offset, 16); + offset = AOMMAX(offset, -16); + int qindex = cm->base_qindex + offset; + qindex = AOMMIN(qindex, MAXQ); + qindex = AOMMAX(qindex, MINQ); + + return qindex; +} + static void setup_delta_q(AV1_COMP *const cpi, MACROBLOCK *const x, const TileInfo *const tile_info, int mi_row, int mi_col, int num_planes) { @@ -4347,36 +3813,50 @@ static void setup_delta_q(AV1_COMP *const cpi, MACROBLOCK *const x, // Delta-q modulation based on variance av1_setup_src_planes(x, cpi->source, mi_row, mi_col, num_planes, sb_size); - int offset_qindex; - if (DELTAQ_MODULATION == 1) { - const int block_wavelet_energy_level = - av1_block_wavelet_energy_level(cpi, x, sb_size); - x->sb_energy_level = block_wavelet_energy_level; - offset_qindex = - av1_compute_deltaq_from_energy_level(cpi, block_wavelet_energy_level); - } else { - const int block_var_level = av1_log_block_var(cpi, x, sb_size); - x->sb_energy_level = block_var_level; - offset_qindex = av1_compute_deltaq_from_energy_level(cpi, block_var_level); + int current_qindex = cm->base_qindex; + if (cpi->oxcf.deltaq_mode == DELTA_Q_PERCEPTUAL) { + if (DELTA_Q_PERCEPTUAL_MODULATION == 1) { + const int block_wavelet_energy_level = + av1_block_wavelet_energy_level(cpi, x, sb_size); + x->sb_energy_level = block_wavelet_energy_level; + current_qindex = av1_compute_q_from_energy_level_deltaq_mode( + cpi, block_wavelet_energy_level); + } else { + const int block_var_level = av1_log_block_var(cpi, x, sb_size); + x->sb_energy_level = block_var_level; + current_qindex = + av1_compute_q_from_energy_level_deltaq_mode(cpi, block_var_level); + } + } else if (cpi->oxcf.deltaq_mode == DELTA_Q_OBJECTIVE) { + assert(cpi->oxcf.enable_tpl_model); + // Setup deltaq based on tpl stats + current_qindex = get_q_for_deltaq_objective(cpi, sb_size, mi_row, mi_col); } + const int qmask = ~(delta_q_info->delta_q_res - 1); - int current_qindex = - clamp(cm->base_qindex + offset_qindex, delta_q_info->delta_q_res, - 256 - delta_q_info->delta_q_res); - current_qindex = - ((current_qindex - cm->base_qindex + delta_q_info->delta_q_res / 2) & - qmask) + - cm->base_qindex; + current_qindex = clamp(current_qindex, delta_q_info->delta_q_res, + 256 - delta_q_info->delta_q_res); + + const int sign_deltaq_index = + current_qindex - xd->current_qindex >= 0 ? 1 : -1; + + const int deltaq_deadzone = 0; // delta_q_info->delta_q_res / 2; + int abs_deltaq_index = abs(current_qindex - xd->current_qindex); + abs_deltaq_index = (abs_deltaq_index + deltaq_deadzone) & qmask; + current_qindex = xd->current_qindex + sign_deltaq_index * abs_deltaq_index; + current_qindex = AOMMAX(current_qindex, MINQ + 1); assert(current_qindex > 0); xd->delta_qindex = current_qindex - cm->base_qindex; set_offsets(cpi, tile_info, x, mi_row, mi_col, sb_size); xd->mi[0]->current_qindex = current_qindex; av1_init_plane_quantizers(cpi, x, xd->mi[0]->segment_id); - if (cpi->oxcf.deltaq_mode == DELTA_Q_LF) { + x->rdmult = set_deltaq_rdmult(cpi, xd); + + if (cpi->oxcf.deltaq_mode != NO_DELTA_Q && cpi->oxcf.deltalf_mode) { const int lfmask = ~(delta_q_info->delta_lf_res - 1); const int delta_lf_from_base = - ((offset_qindex / 2 + delta_q_info->delta_lf_res / 2) & lfmask); + ((xd->delta_qindex / 2 + delta_q_info->delta_lf_res / 2) & lfmask); // pre-set the delta lf for loop filter. Note that this value is set // before mi is assigned for each block in current superblock @@ -4647,6 +4127,19 @@ static void avg_cdf_symbols(FRAME_CONTEXT *ctx_left, FRAME_CONTEXT *ctx_tr, CFL_ALPHABET_SIZE); } +static void adjust_rdmult_tpl_model(AV1_COMP *cpi, MACROBLOCK *x, int mi_row, + int mi_col) { + const BLOCK_SIZE sb_size = cpi->common.seq_params.sb_size; + const int orig_rdmult = cpi->rd.RDMULT; + x->cb_rdmult = orig_rdmult; + if (cpi->twopass.gf_group.index > 0 && cpi->oxcf.enable_tpl_model && + cpi->oxcf.aq_mode == NO_AQ && cpi->oxcf.deltaq_mode == NO_DELTA_Q) { + const int dr = get_rdmult_delta(cpi, sb_size, mi_row, mi_col, orig_rdmult); + x->rdmult = dr; + x->cb_rdmult = x->rdmult; + } +} + static void encode_sb_row(AV1_COMP *cpi, ThreadData *td, TileDataEnc *tile_data, int mi_row, TOKENEXTRA **tp, int use_nonrd_mode) { AV1_COMMON *const cm = &cpi->common; @@ -4743,18 +4236,19 @@ static void encode_sb_row(AV1_COMP *cpi, ThreadData *td, TileDataEnc *tile_data, av1_zero(x->txb_rd_record_32X32); av1_zero(x->txb_rd_record_64X64); av1_zero(x->txb_rd_record_intra); + + av1_zero(x->picked_ref_frames_mask); + + av1_zero(x->pred_mv); } - - av1_zero(x->picked_ref_frames_mask); - - av1_zero(x->pred_mv); PC_TREE *const pc_root = td->pc_root[mib_size_log2 - MIN_MIB_SIZE_LOG2]; pc_root->index = 0; - if ((sf->simple_motion_search_prune_rect || + if ((sf->simple_motion_search_split || + sf->simple_motion_search_prune_rect || sf->simple_motion_search_early_term_none || sf->firstpass_simple_motion_search_early_term) && - !frame_is_intra_only(cm)) { + !frame_is_intra_only(cm) && !use_nonrd_mode) { init_simple_motion_search_mvs(pc_root); } @@ -4773,6 +4267,8 @@ static void encode_sb_row(AV1_COMP *cpi, ThreadData *td, TileDataEnc *tile_data, if (cm->delta_q_info.delta_q_present_flag) setup_delta_q(cpi, x, tile_info, mi_row, mi_col, num_planes); + td->mb.cb_coef_buff = av1_get_cb_coeff_buffer(cpi, mi_row, mi_col); + int dummy_rate; int64_t dummy_dist; RD_STATS dummy_rdc; @@ -4781,12 +4277,14 @@ static void encode_sb_row(AV1_COMP *cpi, ThreadData *td, TileDataEnc *tile_data, x->source_variance = UINT_MAX; x->simple_motion_pred_sse = UINT_MAX; if (sf->partition_search_type == FIXED_PARTITION || seg_skip) { + adjust_rdmult_tpl_model(cpi, x, mi_row, mi_col); set_offsets(cpi, tile_info, x, mi_row, mi_col, sb_size); const BLOCK_SIZE bsize = seg_skip ? sb_size : sf->always_this_block_size; set_fixed_partitioning(cpi, tile_info, mi, mi_row, mi_col, bsize); rd_use_partition(cpi, td, tile_data, mi, tp, mi_row, mi_col, sb_size, &dummy_rate, &dummy_dist, 1, pc_root); } else if (cpi->partition_search_skippable_frame) { + adjust_rdmult_tpl_model(cpi, x, mi_row, mi_col); set_offsets(cpi, tile_info, x, mi_row, mi_col, sb_size); const BLOCK_SIZE bsize = get_rd_var_based_fixed_partition(cpi, x, mi_row, mi_col); @@ -4797,21 +4295,11 @@ static void encode_sb_row(AV1_COMP *cpi, ThreadData *td, TileDataEnc *tile_data, use_nonrd_mode) { set_offsets(cpi, tile_info, x, mi_row, mi_col, sb_size); av1_choose_var_based_partitioning(cpi, tile_info, x, mi_row, mi_col); + td->mb.cb_offset = 0; nonrd_use_partition(cpi, td, tile_data, mi, tp, mi_row, mi_col, sb_size, - &dummy_rate, &dummy_dist, 1, pc_root); - + pc_root); } else { - const int orig_rdmult = cpi->rd.RDMULT; - x->cb_rdmult = orig_rdmult; - if (cpi->twopass.gf_group.index > 0 && cpi->oxcf.enable_tpl_model && - cpi->oxcf.aq_mode == NO_AQ && cpi->oxcf.deltaq_mode == 0) { - const int dr = - get_rdmult_delta(cpi, BLOCK_128X128, mi_row, mi_col, orig_rdmult); - - x->cb_rdmult = dr; - x->rdmult = x->cb_rdmult; - } - + adjust_rdmult_tpl_model(cpi, x, mi_row, mi_col); reset_partition(pc_root, sb_size); x->use_cb_search_range = 0; #if CONFIG_COLLECT_COMPONENT_TIMING @@ -5307,7 +4795,10 @@ static void encode_frame_internal(AV1_COMP *cpi) { x->using_dist_8x8 = cpi->oxcf.using_dist_8x8; x->tune_metric = cpi->oxcf.tuning; #endif - cm->setup_mi(cm); + + if (!cpi->sf.use_nonrd_pick_mode) { + cm->setup_mi(cm); + } xd->mi = cm->mi_grid_visible; xd->mi[0] = cm->mi; @@ -5316,7 +4807,8 @@ static void encode_frame_internal(AV1_COMP *cpi) { av1_zero(rdc->comp_pred_diff); // Two pass partition search can be enabled/disabled for different frames. // Reset this data at frame level to avoid any incorrect usage. - init_first_partition_pass_stats_tables(cpi, x->first_partition_pass_stats); + if (!cpi->sf.use_nonrd_pick_mode) + init_first_partition_pass_stats_tables(cpi, x->first_partition_pass_stats); // Reset the flag. cpi->intrabc_used = 0; @@ -5327,7 +4819,8 @@ static void encode_frame_internal(AV1_COMP *cpi) { cm->allow_intrabc &= (cpi->oxcf.enable_intrabc); - if (cpi->oxcf.pass != 1 && av1_use_hash_me(cm)) { + if (cpi->oxcf.pass != 1 && av1_use_hash_me(cm) && + !cpi->sf.use_nonrd_pick_mode) { // add to hash table const int pic_width = cpi->source->y_crop_width; const int pic_height = cpi->source->y_crop_height; @@ -5420,11 +4913,16 @@ static void encode_frame_internal(AV1_COMP *cpi) { cm->tx_mode = select_tx_mode(cpi); // Fix delta q resolution for the moment - cm->delta_q_info.delta_q_res = DEFAULT_DELTA_Q_RES; + cm->delta_q_info.delta_q_res = 0; + if (cpi->oxcf.deltaq_mode == DELTA_Q_OBJECTIVE) + cm->delta_q_info.delta_q_res = DEFAULT_DELTA_Q_RES_OBJECTIVE; + else if (cpi->oxcf.deltaq_mode == DELTA_Q_PERCEPTUAL) + cm->delta_q_info.delta_q_res = DEFAULT_DELTA_Q_RES_PERCEPTUAL; // Set delta_q_present_flag before it is used for the first time cm->delta_q_info.delta_lf_res = DEFAULT_DELTA_LF_RES; cm->delta_q_info.delta_q_present_flag = cpi->oxcf.deltaq_mode != NO_DELTA_Q; - cm->delta_q_info.delta_lf_present_flag = cpi->oxcf.deltaq_mode == DELTA_Q_LF; + cm->delta_q_info.delta_lf_present_flag = + cpi->oxcf.deltaq_mode != NO_DELTA_Q && cpi->oxcf.deltalf_mode; cm->delta_q_info.delta_lf_multi = DEFAULT_DELTA_LF_MULTI; // update delta_q_present_flag and delta_lf_present_flag based on // base_qindex @@ -5672,6 +5170,8 @@ static void encode_frame_internal(AV1_COMP *cpi) { if (cm->allow_intrabc) cm->delta_q_info.delta_lf_present_flag = 0; } +#define CHECK_PRECOMPUTED_REF_FRAME_MAP 0 + void av1_encode_frame(AV1_COMP *cpi) { AV1_COMMON *const cm = &cpi->common; CurrentFrame *const current_frame = &cm->current_frame; @@ -5698,6 +5198,43 @@ void av1_encode_frame(AV1_COMP *cpi) { enforce_max_ref_frames(cpi); av1_setup_frame_sign_bias(cm); +#if CHECK_PRECOMPUTED_REF_FRAME_MAP + GF_GROUP *gf_group = &cpi->twopass.gf_group; + // TODO(yuec): The check is disabled on OVERLAY frames for now, because info + // in cpi->gf_group has been refreshed for the next GOP when the check is + // performed for OVERLAY frames. Since we have not support inter-GOP ref + // frame map computation, the precomputed ref map for an OVERLAY frame is all + // -1 at this point (although it is meaning before gf_group is refreshed). + if (!frame_is_intra_only(cm) && gf_group->index != 0) { + const RefCntBuffer *const golden_buf = get_ref_frame_buf(cm, GOLDEN_FRAME); + + if (golden_buf) { + const int golden_order_hint = golden_buf->order_hint; + + for (int ref = LAST_FRAME; ref < EXTREF_FRAME; ++ref) { + const RefCntBuffer *const buf = get_ref_frame_buf(cm, ref); + const int ref_disp_idx_precomputed = + gf_group->ref_frame_disp_idx[gf_group->index][ref - LAST_FRAME]; + + (void)ref_disp_idx_precomputed; + + if (buf != NULL) { + const int ref_disp_idx = + get_relative_dist(&cm->seq_params.order_hint_info, + buf->order_hint, golden_order_hint); + + if (ref_disp_idx >= 0) + assert(ref_disp_idx == ref_disp_idx_precomputed); + else + assert(ref_disp_idx_precomputed == -1); + } else { + assert(ref_disp_idx_precomputed == -1); + } + } + } + } +#endif + #if CONFIG_MISMATCH_DEBUG mismatch_reset_frame(num_planes); #else diff --git a/third_party/aom/av1/encoder/encodeframe.h b/third_party/aom/av1/encoder/encodeframe.h index 3b1730d323..e4c4841058 100644 --- a/third_party/aom/av1/encoder/encodeframe.h +++ b/third_party/aom/av1/encoder/encodeframe.h @@ -20,7 +20,9 @@ extern "C" { #endif -#define DELTAQ_MODULATION 1 // 0: variance based, 1: wavelet AC energy based +#define DELTA_Q_PERCEPTUAL_MODULATION \ + 1 // 0: variance based + // 1: wavelet AC energy based struct macroblock; struct yv12_buffer_config; diff --git a/third_party/aom/av1/encoder/encodemb.c b/third_party/aom/av1/encoder/encodemb.c index 8e9da6166b..10d7b25bdc 100644 --- a/third_party/aom/av1/encoder/encodemb.c +++ b/third_party/aom/av1/encoder/encodemb.c @@ -86,6 +86,7 @@ void av1_subtract_txb(MACROBLOCK *x, int plane, BLOCK_SIZE plane_bsize, void av1_subtract_plane(MACROBLOCK *x, BLOCK_SIZE bsize, int plane) { struct macroblock_plane *const p = &x->plane[plane]; const struct macroblockd_plane *const pd = &x->e_mbd.plane[plane]; + assert(bsize < BLOCK_SIZES_ALL); const BLOCK_SIZE plane_bsize = get_plane_block_size(bsize, pd->subsampling_x, pd->subsampling_y); const int bw = block_size_wide[plane_bsize]; @@ -478,10 +479,11 @@ void av1_encode_sb(const struct AV1_COMP *cpi, MACROBLOCK *x, BLOCK_SIZE bsize, if (x->skip) return; + assert(bsize < BLOCK_SIZES_ALL); + for (plane = 0; plane < num_planes; ++plane) { const int subsampling_x = xd->plane[plane].subsampling_x; const int subsampling_y = xd->plane[plane].subsampling_y; - if (!is_chroma_reference(mi_row, mi_col, bsize, subsampling_x, subsampling_y)) continue; @@ -493,6 +495,7 @@ void av1_encode_sb(const struct AV1_COMP *cpi, MACROBLOCK *x, BLOCK_SIZE bsize, const struct macroblockd_plane *const pd = &xd->plane[plane]; const BLOCK_SIZE plane_bsize = get_plane_block_size(bsizec, pd->subsampling_x, pd->subsampling_y); + assert(plane_bsize < BLOCK_SIZES_ALL); const int mi_width = block_size_wide[plane_bsize] >> tx_size_wide_log2[0]; const int mi_height = block_size_high[plane_bsize] >> tx_size_high_log2[0]; const TX_SIZE max_tx_size = get_vartx_max_txsize(xd, plane_bsize, plane); @@ -638,6 +641,7 @@ void av1_encode_intra_block_plane(const struct AV1_COMP *cpi, MACROBLOCK *x, const MACROBLOCKD *const xd = &x->e_mbd; ENTROPY_CONTEXT ta[MAX_MIB_SIZE] = { 0 }; ENTROPY_CONTEXT tl[MAX_MIB_SIZE] = { 0 }; + assert(bsize < BLOCK_SIZES_ALL); struct encode_b_args arg = { cpi, x, NULL, &(xd->mi[0]->skip), ta, tl, enable_optimize_b diff --git a/third_party/aom/av1/encoder/encoder.c b/third_party/aom/av1/encoder/encoder.c index 7716ee2000..2f2eebc2a1 100644 --- a/third_party/aom/av1/encoder/encoder.c +++ b/third_party/aom/av1/encoder/encoder.c @@ -330,15 +330,13 @@ static BLOCK_SIZE select_sb_size(const AV1_COMP *const cpi) { #endif // When superres / resize is on, 'cm->width / height' can change between - // calls, so we don't apply this heuristic there. Also, this heuristic gives - // compression gain for speed >= 2 only. - // Things break if superblock size changes per-frame which is why this - // heuristic is set based on configured speed rather than actual - // speed-features (which may change per-frame in future) + // calls, so we don't apply this heuristic there. + // Things break if superblock size changes between the first pass and second + // pass encoding, which is why this heuristic is not configured as a + // speed-feature. if (cpi->oxcf.superres_mode == SUPERRES_NONE && - cpi->oxcf.resize_mode == RESIZE_NONE && cpi->oxcf.speed >= 2) { - return (cm->width >= 480 && cm->height >= 360) ? BLOCK_128X128 - : BLOCK_64X64; + cpi->oxcf.resize_mode == RESIZE_NONE && cpi->oxcf.speed >= 1) { + return AOMMIN(cm->width, cm->height) > 480 ? BLOCK_128X128 : BLOCK_64X64; } return BLOCK_128X128; @@ -357,10 +355,11 @@ static void setup_frame(AV1_COMP *cpi) { av1_setup_past_independence(cm); } - if (cm->current_frame.frame_type == KEY_FRAME && cm->show_frame) { - set_sb_size(&cm->seq_params, select_sb_size(cpi)); - } else if (frame_is_sframe(cm)) { - set_sb_size(&cm->seq_params, select_sb_size(cpi)); + if ((cm->current_frame.frame_type == KEY_FRAME && cm->show_frame) || + frame_is_sframe(cm)) { + if (!cpi->seq_params_locked) { + set_sb_size(&cm->seq_params, select_sb_size(cpi)); + } } else { const RefCntBuffer *const primary_ref_buf = get_primary_ref_frame_buf(cm); if (primary_ref_buf == NULL) { @@ -2548,19 +2547,6 @@ void av1_change_config(struct AV1_COMP *cpi, const AV1EncoderConfig *oxcf) { } } -static void init_level_info(AV1LevelInfo *level_info) { - memset(level_info, 0, MAX_NUM_OPERATING_POINTS * sizeof(*level_info)); - for (int i = 0; i < MAX_NUM_OPERATING_POINTS; ++i) { - AV1LevelSpec *const level_spec = &level_info[i].level_spec; - level_spec->level = SEQ_LEVEL_MAX; - AV1LevelStats *const level_stats = &level_info[i].level_stats; - level_stats->min_cropped_tile_width = INT_MAX; - level_stats->min_cropped_tile_height = INT_MAX; - level_stats->tile_width_is_valid = 1; - level_stats->min_cr = 1e8; - } -} - AV1_COMP *av1_create_compressor(AV1EncoderConfig *oxcf, BufferPool *const pool) { unsigned int i; @@ -2622,7 +2608,7 @@ AV1_COMP *av1_create_compressor(AV1EncoderConfig *oxcf, cpi->refresh_alt_ref_frame = 0; - init_level_info(cpi->level_info); + av1_init_level_info(cpi->level_info); cpi->b_calculate_psnr = CONFIG_INTERNAL_STATS; #if CONFIG_INTERNAL_STATS @@ -3522,7 +3508,7 @@ static void set_screen_content_options(AV1_COMP *cpi) { // IntraBC would force loop filters off, so we use more strict rules that also // requires that the block has high variance. cm->allow_intrabc = cm->allow_screen_content_tools && - counts_2 * blk_h * blk_w * 15 > width * height; + counts_2 * blk_h * blk_w * 12 > width * height; } static void set_size_independent_vars(AV1_COMP *cpi) { @@ -3568,18 +3554,30 @@ static void set_size_dependent_vars(AV1_COMP *cpi, int *q, int *bottom_index, } static void init_motion_estimation(AV1_COMP *cpi) { - int y_stride = cpi->scaled_source.y_stride; - int y_stride_src = (cpi->oxcf.resize_mode || cpi->oxcf.superres_mode) - ? y_stride - : cpi->lookahead->buf->img.y_stride; + const int y_stride = cpi->scaled_source.y_stride; + const int y_stride_src = (cpi->oxcf.resize_mode || cpi->oxcf.superres_mode) + ? y_stride + : cpi->lookahead->buf->img.y_stride; + // Update if ss_cfg is uninitialized or the current frame has a new stride + const int should_update = !cpi->ss_cfg[SS_CFG_SRC].stride || + !cpi->ss_cfg[SS_CFG_LOOKAHEAD].stride || + (y_stride != cpi->ss_cfg[SS_CFG_SRC].stride); - if (cpi->sf.mv.search_method == NSTEP) { - av1_init3smotion_compensation(&cpi->ss_cfg[SS_CFG_SRC], y_stride); - av1_init3smotion_compensation(&cpi->ss_cfg[SS_CFG_LOOKAHEAD], y_stride_src); - } else if (cpi->sf.mv.search_method == DIAMOND) { + if (!should_update) { + return; + } + + if (cpi->sf.mv.search_method == DIAMOND) { av1_init_dsmotion_compensation(&cpi->ss_cfg[SS_CFG_SRC], y_stride); av1_init_dsmotion_compensation(&cpi->ss_cfg[SS_CFG_LOOKAHEAD], y_stride_src); + } else { + // Update the offsets in search_sites as y_stride can change due to scaled + // references. This update allows NSTEP to be used on scaled references as + // long as sf.mv.search_method is not DIAMOND. Currently in the codebae, + // sf.mv.search_method is never set to DIAMOND. + av1_init3smotion_compensation(&cpi->ss_cfg[SS_CFG_SRC], y_stride); + av1_init3smotion_compensation(&cpi->ss_cfg[SS_CFG_LOOKAHEAD], y_stride_src); } } @@ -3838,8 +3836,9 @@ static uint8_t calculate_next_superres_scale(AV1_COMP *cpi) { if (cpi->common.allow_screen_content_tools) break; // Don't use for inter frames. if (!frame_is_intra_only(&cpi->common)) break; - // Don't use for keyframes that can be used as references. - if (cpi->rc.frames_to_key != 1) break; + // Don't use for keyframes that can be used as references, except when + // using AOM_Q mode. + if (cpi->rc.frames_to_key != 1 && cpi->oxcf.rc_mode != AOM_Q) break; // Now decide the use of superres based on 'q'. int bottom_index, top_index; @@ -5422,7 +5421,7 @@ static void compute_internal_stats(AV1_COMP *cpi, int frame_bytes) { int av1_get_compressed_data(AV1_COMP *cpi, unsigned int *frame_flags, size_t *size, uint8_t *dest, int64_t *time_stamp, int64_t *time_end, int flush, - const aom_rational_t *timebase) { + const aom_rational64_t *timestamp_ratio) { const AV1EncoderConfig *const oxcf = &cpi->oxcf; AV1_COMMON *const cm = &cpi->common; @@ -5458,8 +5457,9 @@ int av1_get_compressed_data(AV1_COMP *cpi, unsigned int *frame_flags, if (assign_cur_frame_new_fb(cm) == NULL) return AOM_CODEC_ERROR; - const int result = av1_encode_strategy(cpi, size, dest, frame_flags, - time_stamp, time_end, timebase, flush); + const int result = + av1_encode_strategy(cpi, size, dest, frame_flags, time_stamp, time_end, + timestamp_ratio, flush); if (result != AOM_CODEC_OK && result != -1) { return AOM_CODEC_ERROR; } else if (result == -1) { @@ -5469,14 +5469,20 @@ int av1_get_compressed_data(AV1_COMP *cpi, unsigned int *frame_flags, #if CONFIG_INTERNAL_STATS aom_usec_timer_mark(&cmptimer); cpi->time_compress_data += aom_usec_timer_elapsed(&cmptimer); -#endif +#endif // CONFIG_INTERNAL_STATS if (cpi->b_calculate_psnr) { if (cm->show_existing_frame || (oxcf->pass != 1 && cm->show_frame)) { generate_psnr_packet(cpi); } } - if (cpi->keep_level_stats && oxcf->pass != 1) + + if (cpi->keep_level_stats && oxcf->pass != 1) { + // Initialize level info. at the beginning of each sequence. + if (cm->current_frame.frame_type == KEY_FRAME && cm->show_frame) { + av1_init_level_info(cpi->level_info); + } av1_update_level_info(cpi, *size, *time_stamp, *time_end); + } #if CONFIG_INTERNAL_STATS if (oxcf->pass != 1) { diff --git a/third_party/aom/av1/encoder/encoder.h b/third_party/aom/av1/encoder/encoder.h index bf02394a5e..a9e1379cd2 100644 --- a/third_party/aom/av1/encoder/encoder.h +++ b/third_party/aom/av1/encoder/encoder.h @@ -56,6 +56,13 @@ extern "C" { #endif +// Rational number with an int64 numerator +// This structure holds a fractional value +typedef struct aom_rational64 { + int64_t num; // fraction numerator + int den; // fraction denominator +} aom_rational64_t; // alias for struct aom_rational + typedef struct { int nmv_vec_cost[MV_JOINTS]; int nmv_costs[2][MV_VALS]; @@ -110,9 +117,9 @@ enum { } UENUM1BYTE(AQ_MODE); enum { NO_DELTA_Q = 0, - DELTA_Q_ONLY = 1, - DELTA_Q_LF = 2, - DELTAQ_MODE_COUNT // This should always be the last member of the enum + DELTA_Q_OBJECTIVE = 1, // Modulation to improve objective quality + DELTA_Q_PERCEPTUAL = 2, // Modulation to improve perceptual quality + DELTA_Q_MODE_COUNT // This should always be the last member of the enum } UENUM1BYTE(DELTAQ_MODE); enum { @@ -237,6 +244,7 @@ typedef struct AV1EncoderConfig { int cq_level; AQ_MODE aq_mode; // Adaptive Quantization mode DELTAQ_MODE deltaq_mode; + int deltalf_mode; int enable_cdef; int enable_restoration; int enable_obmc; @@ -1060,7 +1068,7 @@ int av1_receive_raw_frame(AV1_COMP *cpi, aom_enc_frame_flags_t frame_flags, int av1_get_compressed_data(AV1_COMP *cpi, unsigned int *frame_flags, size_t *size, uint8_t *dest, int64_t *time_stamp, int64_t *time_end, int flush, - const aom_rational_t *timebase); + const aom_rational64_t *timebase); int av1_encode(AV1_COMP *const cpi, uint8_t *const dest, const EncodeFrameInput *const frame_input, @@ -1099,15 +1107,16 @@ int av1_convert_sect5obus_to_annexb(uint8_t *buffer, size_t *input_size); // av1 uses 10,000,000 ticks/second as time stamp #define TICKS_PER_SEC 10000000LL -static INLINE int64_t timebase_units_to_ticks(const aom_rational_t *timebase, - int64_t n) { - return n * TICKS_PER_SEC * timebase->num / timebase->den; +static INLINE int64_t +timebase_units_to_ticks(const aom_rational64_t *timestamp_ratio, int64_t n) { + return n * timestamp_ratio->num / timestamp_ratio->den; } -static INLINE int64_t ticks_to_timebase_units(const aom_rational_t *timebase, - int64_t n) { - const int64_t round = TICKS_PER_SEC * timebase->num / 2 - 1; - return (n * timebase->den + round) / timebase->num / TICKS_PER_SEC; +static INLINE int64_t +ticks_to_timebase_units(const aom_rational64_t *timestamp_ratio, int64_t n) { + int64_t round = timestamp_ratio->num / 2; + if (round > 0) --round; + return (n * timestamp_ratio->den + round) / timestamp_ratio->num; } static INLINE int frame_is_kf_gf_arf(const AV1_COMP *cpi) { diff --git a/third_party/aom/av1/encoder/encodetxb.c b/third_party/aom/av1/encoder/encodetxb.c index 37f4bb9f79..a64e4b0c1a 100644 --- a/third_party/aom/av1/encoder/encodetxb.c +++ b/third_party/aom/av1/encoder/encodetxb.c @@ -73,17 +73,6 @@ void av1_alloc_txb_buf(AV1_COMP *cpi) { void av1_free_txb_buf(AV1_COMP *cpi) { aom_free(cpi->coeff_buffer_base); } -void av1_set_coeff_buffer(const AV1_COMP *const cpi, MACROBLOCK *const x, - int mi_row, int mi_col) { - const AV1_COMMON *const cm = &cpi->common; - int mib_size_log2 = cm->seq_params.mib_size_log2; - int stride = (cm->mi_cols >> mib_size_log2) + 1; - int offset = (mi_row >> mib_size_log2) * stride + (mi_col >> mib_size_log2); - x->mbmi_ext->cb_coef_buff = &cpi->coeff_buffer_base[offset]; - x->mbmi_ext->cb_offset = x->cb_offset; - assert(x->cb_offset < (1 << num_pels_log2_lookup[cm->seq_params.sb_size])); -} - static void write_golomb(aom_writer *w, int level) { int x = level + 1; int i = x; @@ -638,17 +627,17 @@ static void write_coeffs_txb_wrap(const AV1_COMMON *cm, MACROBLOCK *x, aom_writer *w, int plane, int block, int blk_row, int blk_col, TX_SIZE tx_size) { MACROBLOCKD *xd = &x->e_mbd; + const CB_COEFF_BUFFER *cb_coef_buff = x->cb_coef_buff; const int txb_offset = x->mbmi_ext->cb_offset / (TX_SIZE_W_MIN * TX_SIZE_H_MIN); - tran_low_t *tcoeff_txb = - x->mbmi_ext->cb_coef_buff->tcoeff[plane] + x->mbmi_ext->cb_offset; - uint16_t *eob_txb = x->mbmi_ext->cb_coef_buff->eobs[plane] + txb_offset; - uint8_t *txb_skip_ctx_txb = - x->mbmi_ext->cb_coef_buff->txb_skip_ctx[plane] + txb_offset; - int *dc_sign_ctx_txb = - x->mbmi_ext->cb_coef_buff->dc_sign_ctx[plane] + txb_offset; - tran_low_t *tcoeff = BLOCK_OFFSET(tcoeff_txb, block); - uint16_t eob = eob_txb[block]; + const tran_low_t *tcoeff_txb = + cb_coef_buff->tcoeff[plane] + x->mbmi_ext->cb_offset; + const uint16_t *eob_txb = cb_coef_buff->eobs[plane] + txb_offset; + const uint8_t *txb_skip_ctx_txb = + cb_coef_buff->txb_skip_ctx[plane] + txb_offset; + const int *dc_sign_ctx_txb = cb_coef_buff->dc_sign_ctx[plane] + txb_offset; + const tran_low_t *tcoeff = BLOCK_OFFSET(tcoeff_txb, block); + const uint16_t eob = eob_txb[block]; TXB_CTX txb_ctx = { txb_skip_ctx_txb[block], dc_sign_ctx_txb[block] }; av1_write_coeffs_txb(cm, xd, w, blk_row, blk_col, plane, tx_size, tcoeff, eob, &txb_ctx); @@ -1690,15 +1679,15 @@ int av1_optimize_txb_new(const struct AV1_COMP *cpi, MACROBLOCK *x, int plane, const LV_MAP_EOB_COST *txb_eob_costs = &x->eob_costs[eob_multi_size][plane_type]; - const int rshift = - (sharpness + - (cpi->oxcf.aq_mode == VARIANCE_AQ && mbmi->segment_id < 4 - ? 7 - mbmi->segment_id - : 2) + - (cpi->oxcf.aq_mode != VARIANCE_AQ && - cpi->oxcf.deltaq_mode > NO_DELTA_Q && x->sb_energy_level < 0 - ? (3 - x->sb_energy_level) - : 0)); + const int rshift = (sharpness + + (cpi->oxcf.aq_mode == VARIANCE_AQ && mbmi->segment_id < 4 + ? 7 - mbmi->segment_id + : 2) + + (cpi->oxcf.aq_mode != VARIANCE_AQ && + cpi->oxcf.deltaq_mode == DELTA_Q_PERCEPTUAL && + x->sb_energy_level < 0 + ? (3 - x->sb_energy_level) + : 0)); const int64_t rdmult = (((int64_t)x->rdmult * (plane_rd_mult[is_inter][plane_type] << (2 * (xd->bd - 8)))) + @@ -2008,11 +1997,11 @@ void av1_update_and_record_txb_context(int plane, int block, int blk_row, 2); } + CB_COEFF_BUFFER *cb_coef_buff = x->cb_coef_buff; const int txb_offset = x->mbmi_ext->cb_offset / (TX_SIZE_W_MIN * TX_SIZE_H_MIN); - uint16_t *eob_txb = x->mbmi_ext->cb_coef_buff->eobs[plane] + txb_offset; - uint8_t *txb_skip_ctx_txb = - x->mbmi_ext->cb_coef_buff->txb_skip_ctx[plane] + txb_offset; + uint16_t *eob_txb = cb_coef_buff->eobs[plane] + txb_offset; + uint8_t *txb_skip_ctx_txb = cb_coef_buff->txb_skip_ctx[plane] + txb_offset; txb_skip_ctx_txb[block] = txb_ctx.txb_skip_ctx; eob_txb[block] = eob; @@ -2021,8 +2010,7 @@ void av1_update_and_record_txb_context(int plane, int block, int blk_row, return; } - tran_low_t *tcoeff_txb = - x->mbmi_ext->cb_coef_buff->tcoeff[plane] + x->mbmi_ext->cb_offset; + tran_low_t *tcoeff_txb = cb_coef_buff->tcoeff[plane] + x->mbmi_ext->cb_offset; tran_low_t *tcoeff = BLOCK_OFFSET(tcoeff_txb, block); const int segment_id = mbmi->segment_id; const int seg_eob = av1_get_tx_eob(&cpi->common.seg, segment_id, tx_size); @@ -2116,8 +2104,7 @@ void av1_update_and_record_txb_context(int plane, int block, int blk_row, #endif // CONFIG_ENTROPY_STATS if (allow_update_cdf) update_cdf(ec_ctx->dc_sign_cdf[plane_type][dc_sign_ctx], dc_sign, 2); - int *dc_sign_ctx_txb = - x->mbmi_ext->cb_coef_buff->dc_sign_ctx[plane] + txb_offset; + int *dc_sign_ctx_txb = cb_coef_buff->dc_sign_ctx[plane] + txb_offset; dc_sign_ctx_txb[block] = dc_sign_ctx; } @@ -2155,3 +2142,13 @@ void av1_update_txb_context(const AV1_COMP *cpi, ThreadData *td, assert(0); } } + +CB_COEFF_BUFFER *av1_get_cb_coeff_buffer(const struct AV1_COMP *cpi, int mi_row, + int mi_col) { + const AV1_COMMON *const cm = &cpi->common; + const int mib_size_log2 = cm->seq_params.mib_size_log2; + const int stride = (cm->mi_cols >> mib_size_log2) + 1; + const int offset = + (mi_row >> mib_size_log2) * stride + (mi_col >> mib_size_log2); + return cpi->coeff_buffer_base + offset; +} diff --git a/third_party/aom/av1/encoder/encodetxb.h b/third_party/aom/av1/encoder/encodetxb.h index 06825908d0..12c78c87ed 100644 --- a/third_party/aom/av1/encoder/encodetxb.h +++ b/third_party/aom/av1/encoder/encodetxb.h @@ -71,15 +71,15 @@ void av1_update_and_record_txb_context(int plane, int block, int blk_row, int blk_col, BLOCK_SIZE plane_bsize, TX_SIZE tx_size, void *arg); -void av1_set_coeff_buffer(const AV1_COMP *const cpi, MACROBLOCK *const x, - int mi_row, int mi_col); - void hbt_destroy(); int av1_optimize_txb_new(const struct AV1_COMP *cpi, MACROBLOCK *x, int plane, int block, TX_SIZE tx_size, TX_TYPE tx_type, const TXB_CTX *const txb_ctx, int *rate_cost, int sharpness, int fast_mode); +CB_COEFF_BUFFER *av1_get_cb_coeff_buffer(const struct AV1_COMP *cpi, int mi_row, + int mi_col); + // These numbers are empirically obtained. static const int plane_rd_mult[REF_TYPES][PLANE_TYPES] = { { 17, 13 }, diff --git a/third_party/aom/av1/encoder/firstpass.h b/third_party/aom/av1/encoder/firstpass.h index 1b8636c66f..7897015af7 100644 --- a/third_party/aom/av1/encoder/firstpass.h +++ b/third_party/aom/av1/encoder/firstpass.h @@ -119,7 +119,10 @@ typedef struct { unsigned char arf_src_offset[MAX_STATIC_GF_GROUP_LENGTH + 1]; unsigned char arf_update_idx[MAX_STATIC_GF_GROUP_LENGTH + 1]; unsigned char arf_pos_in_gf[MAX_STATIC_GF_GROUP_LENGTH + 1]; + unsigned char frame_disp_idx[MAX_STATIC_GF_GROUP_LENGTH + 1]; unsigned char pyramid_level[MAX_STATIC_GF_GROUP_LENGTH + 1]; + int ref_frame_disp_idx[MAX_STATIC_GF_GROUP_LENGTH + 1][REF_FRAMES]; + int ref_frame_gop_idx[MAX_STATIC_GF_GROUP_LENGTH + 1][REF_FRAMES]; unsigned char pyramid_height; unsigned char pyramid_lvl_nodes[MAX_PYRAMID_LVL]; int bit_allocation[MAX_STATIC_GF_GROUP_LENGTH + 1]; diff --git a/third_party/aom/av1/encoder/gop_structure.c b/third_party/aom/av1/encoder/gop_structure.c index 73cb0ed421..af9da59157 100644 --- a/third_party/aom/av1/encoder/gop_structure.c +++ b/third_party/aom/av1/encoder/gop_structure.c @@ -42,6 +42,7 @@ static void set_multi_layer_params(GF_GROUP *const gf_group, int start, int end, gf_group->arf_src_offset[*frame_ind] = 0; gf_group->arf_pos_in_gf[*frame_ind] = 0; gf_group->arf_update_idx[*frame_ind] = arf_ind; + gf_group->frame_disp_idx[*frame_ind] = start; gf_group->pyramid_level[*frame_ind] = MIN_PYRAMID_LVL; ++gf_group->pyramid_lvl_nodes[MIN_PYRAMID_LVL]; ++(*frame_ind); @@ -55,6 +56,7 @@ static void set_multi_layer_params(GF_GROUP *const gf_group, int start, int end, gf_group->arf_src_offset[*frame_ind] = m - start - 1; gf_group->arf_pos_in_gf[*frame_ind] = 0; gf_group->arf_update_idx[*frame_ind] = 1; // mark all internal ARF 1 + gf_group->frame_disp_idx[*frame_ind] = m; gf_group->pyramid_level[*frame_ind] = level; ++gf_group->pyramid_lvl_nodes[level]; ++(*frame_ind); @@ -67,6 +69,7 @@ static void set_multi_layer_params(GF_GROUP *const gf_group, int start, int end, gf_group->arf_src_offset[*frame_ind] = 0; gf_group->arf_pos_in_gf[*frame_ind] = arf_pos_in_gf; // For bit allocation. gf_group->arf_update_idx[*frame_ind] = 1; + gf_group->frame_disp_idx[*frame_ind] = m; gf_group->pyramid_level[*frame_ind] = MIN_PYRAMID_LVL; ++(*frame_ind); @@ -101,6 +104,7 @@ static int construct_multi_layer_gf_structure( gf_group->arf_src_offset[frame_index] = gf_interval - 1; gf_group->arf_pos_in_gf[frame_index] = 0; gf_group->arf_update_idx[frame_index] = 0; + gf_group->frame_disp_idx[frame_index] = gf_interval; gf_group->pyramid_level[frame_index] = gf_group->pyramid_height; ++frame_index; } @@ -165,6 +169,136 @@ static int get_pyramid_height(const AV1_COMP *const cpi) { cpi->oxcf.gf_max_pyr_height); } +#define REF_IDX(ref) ((ref)-LAST_FRAME) + +static INLINE void reset_ref_frame_idx(int *ref_idx, int reset_value) { + for (int i = 0; i < REF_FRAMES; ++i) ref_idx[i] = reset_value; +} + +static INLINE void set_ref_frame_disp_idx(GF_GROUP *const gf_group) { + for (int i = 0; i <= gf_group->size; ++i) { + for (int ref = 0; ref < INTER_REFS_PER_FRAME + 1; ++ref) { + int ref_gop_idx = gf_group->ref_frame_gop_idx[i][ref]; + if (ref_gop_idx == -1) { + gf_group->ref_frame_disp_idx[i][ref] = -1; + } else { + gf_group->ref_frame_disp_idx[i][ref] = + gf_group->frame_disp_idx[ref_gop_idx]; + } + } + } +} + +static void set_gop_ref_frame_map(GF_GROUP *const gf_group) { + // Initialize the reference slots as all -1. + for (int frame_idx = 0; frame_idx <= gf_group->size; ++frame_idx) + reset_ref_frame_idx(gf_group->ref_frame_gop_idx[frame_idx], -1); + + // Set the map for frames in the current gop + for (int frame_idx = 0; frame_idx < gf_group->size; ++frame_idx) { + const FRAME_UPDATE_TYPE update_type = gf_group->update_type[frame_idx]; + // TODO(yuec): need to figure out how to determine + // (1) whether a KEY_FRAME has show_frame on + // (2) whether a frame with INTNL_OVERLAY_UPDATE type has + // show_existing_frame on + const int show_frame = + update_type != ARF_UPDATE && update_type != INTNL_ARF_UPDATE; + const int show_existing_frame = + update_type == OVERLAY_UPDATE || update_type == INTNL_OVERLAY_UPDATE; + + int this_ref_map[INTER_REFS_PER_FRAME + 1]; + memcpy(this_ref_map, gf_group->ref_frame_gop_idx[frame_idx], + sizeof(this_ref_map)); + int *next_ref_map = &gf_group->ref_frame_gop_idx[frame_idx + 1][0]; + + switch (update_type) { + case KF_UPDATE: + if (show_frame) { + reset_ref_frame_idx(this_ref_map, frame_idx); + } else { + this_ref_map[REF_IDX(LAST3_FRAME)] = frame_idx; + this_ref_map[REF_IDX(EXTREF_FRAME)] = frame_idx; + this_ref_map[REF_IDX(ALTREF2_FRAME)] = frame_idx; + this_ref_map[REF_IDX(GOLDEN_FRAME)] = frame_idx; + this_ref_map[REF_IDX(ALTREF_FRAME)] = frame_idx; + } + break; + case LF_UPDATE: this_ref_map[REF_IDX(LAST3_FRAME)] = frame_idx; break; + case GF_UPDATE: + this_ref_map[REF_IDX(LAST3_FRAME)] = frame_idx; + this_ref_map[REF_IDX(GOLDEN_FRAME)] = frame_idx; + break; + case OVERLAY_UPDATE: + this_ref_map[REF_IDX(ALTREF_FRAME)] = frame_idx; + break; + case ARF_UPDATE: this_ref_map[REF_IDX(ALTREF_FRAME)] = frame_idx; break; + case INTNL_OVERLAY_UPDATE: + if (!show_existing_frame) + this_ref_map[REF_IDX(LAST3_FRAME)] = frame_idx; + break; + case INTNL_ARF_UPDATE: + this_ref_map[REF_IDX(EXTREF_FRAME)] = frame_idx; + break; + default: assert(0); break; + } + + memcpy(next_ref_map, this_ref_map, sizeof(this_ref_map)); + + switch (update_type) { + case LF_UPDATE: + case GF_UPDATE: + next_ref_map[REF_IDX(LAST3_FRAME)] = this_ref_map[REF_IDX(LAST2_FRAME)]; + next_ref_map[REF_IDX(LAST2_FRAME)] = this_ref_map[REF_IDX(LAST_FRAME)]; + next_ref_map[REF_IDX(LAST_FRAME)] = this_ref_map[REF_IDX(LAST3_FRAME)]; + break; + case INTNL_OVERLAY_UPDATE: + if (!show_existing_frame) { + next_ref_map[REF_IDX(LAST3_FRAME)] = + this_ref_map[REF_IDX(LAST2_FRAME)]; + next_ref_map[REF_IDX(LAST2_FRAME)] = + this_ref_map[REF_IDX(LAST_FRAME)]; + next_ref_map[REF_IDX(LAST_FRAME)] = + this_ref_map[REF_IDX(LAST3_FRAME)]; + } else { + next_ref_map[REF_IDX(LAST_FRAME)] = + this_ref_map[REF_IDX(BWDREF_FRAME)]; + next_ref_map[REF_IDX(LAST2_FRAME)] = + this_ref_map[REF_IDX(LAST_FRAME)]; + next_ref_map[REF_IDX(LAST3_FRAME)] = + this_ref_map[REF_IDX(LAST2_FRAME)]; + next_ref_map[REF_IDX(BWDREF_FRAME)] = + this_ref_map[REF_IDX(ALTREF2_FRAME)]; + next_ref_map[REF_IDX(ALTREF2_FRAME)] = + this_ref_map[REF_IDX(EXTREF_FRAME)]; + next_ref_map[REF_IDX(EXTREF_FRAME)] = + this_ref_map[REF_IDX(LAST3_FRAME)]; + } + break; + case INTNL_ARF_UPDATE: + if (!show_existing_frame) { + next_ref_map[REF_IDX(BWDREF_FRAME)] = + this_ref_map[REF_IDX(EXTREF_FRAME)]; + next_ref_map[REF_IDX(ALTREF2_FRAME)] = + this_ref_map[REF_IDX(BWDREF_FRAME)]; + next_ref_map[REF_IDX(EXTREF_FRAME)] = + this_ref_map[REF_IDX(ALTREF2_FRAME)]; + } + break; + case OVERLAY_UPDATE: + next_ref_map[REF_IDX(ALTREF_FRAME)] = + this_ref_map[REF_IDX(GOLDEN_FRAME)]; + next_ref_map[REF_IDX(GOLDEN_FRAME)] = + this_ref_map[REF_IDX(ALTREF_FRAME)]; + break; + default: break; + } + } + + // Set the map in display order index by converting from gop indices in the + // above map + set_ref_frame_disp_idx(gf_group); +} + void av1_gop_setup_structure(AV1_COMP *cpi, const EncodeFrameParams *const frame_params) { RATE_CONTROL *const rc = &cpi->rc; @@ -186,6 +320,8 @@ void av1_gop_setup_structure(AV1_COMP *cpi, gf_group->arf_update_idx[gf_group->size] = 0; gf_group->arf_pos_in_gf[gf_group->size] = 0; + set_gop_ref_frame_map(gf_group); + #if CHECK_GF_PARAMETER check_frame_params(gf_group, rc->baseline_gf_interval); #endif diff --git a/third_party/aom/av1/encoder/level.c b/third_party/aom/av1/encoder/level.c index 5e49ec0e89..13e14e2bf2 100644 --- a/third_party/aom/av1/encoder/level.c +++ b/third_party/aom/av1/encoder/level.c @@ -221,9 +221,13 @@ typedef enum { LUMA_PIC_SIZE_TOO_LARGE, LUMA_PIC_H_SIZE_TOO_LARGE, LUMA_PIC_V_SIZE_TOO_LARGE, + LUMA_PIC_H_SIZE_TOO_SMALL, + LUMA_PIC_V_SIZE_TOO_SMALL, TOO_MANY_TILE_COLUMNS, TOO_MANY_TILES, + TILE_RATE_TOO_HIGH, TILE_TOO_LARGE, + SUPERRES_TILE_WIDTH_TOO_LARGE, CROPPED_TILE_WIDTH_TOO_SMALL, CROPPED_TILE_HEIGHT_TOO_SMALL, TILE_WIDTH_INVALID, @@ -231,6 +235,8 @@ typedef enum { DISPLAY_RATE_TOO_HIGH, DECODE_RATE_TOO_HIGH, CR_TOO_SMALL, + TILE_SIZE_HEADER_RATE_TOO_HIGH, + BITRATE_TOO_HIGH, TARGET_LEVEL_FAIL_IDS, TARGET_LEVEL_OK, @@ -240,33 +246,91 @@ static const char *level_fail_messages[TARGET_LEVEL_FAIL_IDS] = { "The picture size is too large.", "The picture width is too large.", "The picture height is too large.", + "The picture width is too small.", + "The picture height is too small.", "Too many tile columns are used.", "Too many tiles are used.", + "The tile rate is too high.", "The tile size is too large.", - "The cropped tile width is less than 8", - "The cropped tile height is less than 8", - "The tile width is invalid", - "The frame header rate is too high", - "The display luma sample rate is too high", - "The decoded luma sample rate is too high", - "The compression ratio is too small", + "The superres tile width is too large.", + "The cropped tile width is less than 8.", + "The cropped tile height is less than 8.", + "The tile width is invalid.", + "The frame header rate is too high.", + "The display luma sample rate is too high.", + "The decoded luma sample rate is too high.", + "The compression ratio is too small.", + "The product of max tile size and header rate is too high.", + "The bitrate is too high.", }; +void av1_init_level_info(AV1LevelInfo *level_info) { + memset(level_info, 0, MAX_NUM_OPERATING_POINTS * sizeof(*level_info)); + for (int i = 0; i < MAX_NUM_OPERATING_POINTS; ++i) { + AV1LevelSpec *const level_spec = &level_info[i].level_spec; + level_spec->level = SEQ_LEVEL_MAX; + AV1LevelStats *const level_stats = &level_info[i].level_stats; + level_stats->min_cropped_tile_width = INT_MAX; + level_stats->min_cropped_tile_height = INT_MAX; + level_stats->min_frame_width = INT_MAX; + level_stats->min_frame_height = INT_MAX; + level_stats->tile_width_is_valid = 1; + level_stats->min_cr = 1e8; + } +} + static double get_min_cr(const AV1LevelSpec *const level_spec, int tier, int is_still_picture, int64_t decoded_sample_rate) { if (is_still_picture) return 0.8; + if (level_spec->level < SEQ_LEVEL_4_0) tier = 0; const double min_cr_basis = tier ? level_spec->high_cr : level_spec->main_cr; const double speed_adj = (double)decoded_sample_rate / level_spec->max_display_rate; return AOMMAX(min_cr_basis * speed_adj, 0.8); } +static void get_temporal_parallel_params(int scalability_mode_idc, + int *temporal_parallel_num, + int *temporal_parallel_denom) { + if (scalability_mode_idc < 0) { + *temporal_parallel_num = 1; + *temporal_parallel_denom = 1; + return; + } + + // TODO(huisu@): handle scalability cases. + if (scalability_mode_idc == SCALABILITY_SS) { + (void)scalability_mode_idc; + } else { + (void)scalability_mode_idc; + } +} + +static double get_max_bitrate(const AV1LevelSpec *const level_spec, int tier, + BITSTREAM_PROFILE profile) { + if (level_spec->level < SEQ_LEVEL_4_0) tier = 0; + const double bitrate_basis = + (tier ? level_spec->high_mbps : level_spec->main_mbps) * 1e6; + const double bitrate_profile_factor = + profile == PROFILE_0 ? 1.0 : (profile == PROFILE_1 ? 2.0 : 3.0); + return bitrate_basis * bitrate_profile_factor; +} + +#define MAX_TILE_SIZE (4096 * 2304) +#define MIN_CROPPED_TILE_WIDTH 8 +#define MIN_CROPPED_TILE_HEIGHT 8 +#define MIN_FRAME_WIDTH 16 +#define MIN_FRAME_HEIGHT 16 +#define MAX_TILE_SIZE_HEADER_RATE_PRODUCT 588251136 + static TARGET_LEVEL_FAIL_ID check_level_constraints( const AV1LevelSpec *const target_level_spec, const AV1LevelSpec *const level_spec, - const AV1LevelStats *const level_stats, int tier, int is_still_picture) { + const AV1LevelStats *const level_stats, int tier, int is_still_picture, + BITSTREAM_PROFILE profile) { const double min_cr = get_min_cr(target_level_spec, tier, is_still_picture, level_spec->max_decode_rate); + const double max_bitrate = get_max_bitrate(target_level_spec, tier, profile); TARGET_LEVEL_FAIL_ID fail_id = TARGET_LEVEL_OK; do { @@ -310,21 +374,41 @@ static TARGET_LEVEL_FAIL_ID check_level_constraints( break; } - if (level_stats->max_tile_size > 4096 * 2304) { + if (level_spec->max_tile_rate > target_level_spec->max_tiles * 120) { + fail_id = TILE_RATE_TOO_HIGH; + break; + } + + if (level_stats->max_tile_size > MAX_TILE_SIZE) { fail_id = TILE_TOO_LARGE; break; } - if (level_stats->min_cropped_tile_width < 8) { + if (level_stats->max_superres_tile_width > MAX_TILE_WIDTH) { + fail_id = SUPERRES_TILE_WIDTH_TOO_LARGE; + break; + } + + if (level_stats->min_cropped_tile_width < MIN_CROPPED_TILE_WIDTH) { fail_id = CROPPED_TILE_WIDTH_TOO_SMALL; break; } - if (level_stats->min_cropped_tile_height < 8) { + if (level_stats->min_cropped_tile_height < MIN_CROPPED_TILE_HEIGHT) { fail_id = CROPPED_TILE_HEIGHT_TOO_SMALL; break; } + if (level_stats->min_frame_width < MIN_FRAME_WIDTH) { + fail_id = LUMA_PIC_H_SIZE_TOO_SMALL; + break; + } + + if (level_stats->min_frame_height < MIN_FRAME_HEIGHT) { + fail_id = LUMA_PIC_V_SIZE_TOO_SMALL; + break; + } + if (!level_stats->tile_width_is_valid) { fail_id = TILE_WIDTH_INVALID; break; @@ -334,6 +418,25 @@ static TARGET_LEVEL_FAIL_ID check_level_constraints( fail_id = CR_TOO_SMALL; break; } + + if ((double)level_stats->max_bitrate > max_bitrate) { + fail_id = BITRATE_TOO_HIGH; + break; + } + + if (target_level_spec->level > SEQ_LEVEL_5_1) { + int temporal_parallel_num; + int temporal_parallel_denom; + const int scalability_mode_idc = -1; + get_temporal_parallel_params(scalability_mode_idc, &temporal_parallel_num, + &temporal_parallel_denom); + const int val = level_stats->max_tile_size * level_spec->max_header_rate * + temporal_parallel_denom / temporal_parallel_num; + if (val > MAX_TILE_SIZE_HEADER_RATE_PRODUCT) { + fail_id = TILE_SIZE_HEADER_RATE_TOO_HIGH; + break; + } + } } while (0); return fail_id; @@ -349,14 +452,17 @@ static INLINE int is_in_operating_point(int operating_point, } static void get_tile_stats(const AV1_COMP *const cpi, int *max_tile_size, + int *max_superres_tile_width, int *min_cropped_tile_width, int *min_cropped_tile_height, int *tile_width_valid) { const AV1_COMMON *const cm = &cpi->common; const int tile_cols = cm->tile_cols; const int tile_rows = cm->tile_rows; + const int superres_scale_denominator = cm->superres_scale_denominator; *max_tile_size = 0; + *max_superres_tile_width = 0; *min_cropped_tile_width = INT_MAX; *min_cropped_tile_height = INT_MAX; *tile_width_valid = 1; @@ -372,6 +478,11 @@ static void get_tile_stats(const AV1_COMP *const cpi, int *max_tile_size, const int tile_size = tile_width * tile_height; *max_tile_size = AOMMAX(*max_tile_size, tile_size); + const int supperres_tile_width = + tile_width * superres_scale_denominator / SCALE_NUMERATOR; + *max_superres_tile_width = + AOMMAX(*max_superres_tile_width, supperres_tile_width); + const int cropped_tile_width = cm->width - tile_info->mi_col_start * MI_SIZE; const int cropped_tile_height = @@ -392,8 +503,9 @@ static void get_tile_stats(const AV1_COMP *const cpi, int *max_tile_size, } } -static int store_frame_record(int64_t ts_start, int64_t ts_end, int pic_size, - int frame_header_count, int show_frame, +static int store_frame_record(int64_t ts_start, int64_t ts_end, + size_t encoded_size, int pic_size, + int frame_header_count, int tiles, int show_frame, int show_existing_frame, FrameWindowBuffer *const buffer) { if (buffer->num < FRAME_WINDOW_SIZE) { @@ -405,8 +517,10 @@ static int store_frame_record(int64_t ts_start, int64_t ts_end, int pic_size, FrameRecord *const record = &buffer->buf[new_idx]; record->ts_start = ts_start; record->ts_end = ts_end; + record->encoded_size_in_bytes = encoded_size; record->pic_size = pic_size; record->frame_header_count = frame_header_count; + record->tiles = tiles; record->show_frame = show_frame; record->show_existing_frame = show_existing_frame; @@ -439,12 +553,15 @@ static int count_frames(const FrameWindowBuffer *const buffer, // Scan previously encoded frames and update level metrics accordingly. static void scan_past_frames(const FrameWindowBuffer *const buffer, int num_frames_to_scan, - AV1LevelSpec *const level_spec) { + AV1LevelSpec *const level_spec, + AV1LevelStats *const level_stats) { const int num_frames_in_buffer = buffer->num; int index = (buffer->start + num_frames_in_buffer - 1) % FRAME_WINDOW_SIZE; int frame_headers = 0; + int tiles = 0; int64_t display_samples = 0; int64_t decoded_samples = 0; + size_t encoded_size_in_bytes = 0; for (int i = 0; i < AOMMIN(num_frames_in_buffer, num_frames_to_scan); ++i) { const FrameRecord *const record = &buffer->buf[index]; if (!record->show_existing_frame) { @@ -454,6 +571,8 @@ static void scan_past_frames(const FrameWindowBuffer *const buffer, if (record->show_frame) { display_samples += record->pic_size; } + tiles += record->tiles; + encoded_size_in_bytes += record->encoded_size_in_bytes; --index; if (index < 0) index = FRAME_WINDOW_SIZE - 1; } @@ -463,12 +582,16 @@ static void scan_past_frames(const FrameWindowBuffer *const buffer, AOMMAX(level_spec->max_display_rate, display_samples); level_spec->max_decode_rate = AOMMAX(level_spec->max_decode_rate, decoded_samples); + level_spec->max_tile_rate = AOMMAX(level_spec->max_tile_rate, tiles); + level_stats->max_bitrate = + AOMMAX(level_stats->max_bitrate, (int)encoded_size_in_bytes * 8); } void av1_update_level_info(AV1_COMP *cpi, size_t size, int64_t ts_start, int64_t ts_end) { AV1_COMMON *const cm = &cpi->common; const int upscaled_width = cm->superres_upscaled_width; + const int width = cm->width; const int height = cm->height; const int tile_cols = cm->tile_cols; const int tile_rows = cm->tile_rows; @@ -480,8 +603,8 @@ void av1_update_level_info(AV1_COMP *cpi, size_t size, int64_t ts_start, // Store info. of current frame into FrameWindowBuffer. FrameWindowBuffer *const buffer = &cpi->frame_window_buffer; - store_frame_record(ts_start, ts_end, luma_pic_size, frame_header_count, - show_frame, show_existing_frame, buffer); + store_frame_record(ts_start, ts_end, size, luma_pic_size, frame_header_count, + tiles, show_frame, show_existing_frame, buffer); // Count the number of frames encoded in the past 1 second. const int encoded_frames_in_last_second = show_frame ? count_frames(buffer, TICKS_PER_SEC) : 0; @@ -489,9 +612,11 @@ void av1_update_level_info(AV1_COMP *cpi, size_t size, int64_t ts_start, int max_tile_size; int min_cropped_tile_width; int min_cropped_tile_height; + int max_superres_tile_width; int tile_width_is_valid; - get_tile_stats(cpi, &max_tile_size, &min_cropped_tile_width, - &min_cropped_tile_height, &tile_width_is_valid); + get_tile_stats(cpi, &max_tile_size, &max_superres_tile_width, + &min_cropped_tile_width, &min_cropped_tile_height, + &tile_width_is_valid); const SequenceHeader *const seq_params = &cm->seq_params; const BITSTREAM_PROFILE profile = seq_params->profile; @@ -524,12 +649,16 @@ void av1_update_level_info(AV1_COMP *cpi, size_t size, int64_t ts_start, level_stats->max_tile_size = AOMMAX(level_stats->max_tile_size, max_tile_size); + level_stats->max_superres_tile_width = + AOMMAX(level_stats->max_superres_tile_width, max_superres_tile_width); level_stats->min_cropped_tile_width = AOMMIN(level_stats->min_cropped_tile_width, min_cropped_tile_width); level_stats->min_cropped_tile_height = AOMMIN(level_stats->min_cropped_tile_height, min_cropped_tile_height); level_stats->tile_width_is_valid &= tile_width_is_valid; - level_stats->total_compressed_size += frame_compressed_size; + level_stats->min_frame_width = AOMMIN(level_stats->min_frame_width, width); + level_stats->min_frame_height = + AOMMIN(level_stats->min_frame_height, height); if (show_frame) level_stats->total_time_encoded = total_time_encoded; level_stats->min_cr = AOMMIN(level_stats->min_cr, compression_ratio); @@ -545,7 +674,8 @@ void av1_update_level_info(AV1_COMP *cpi, size_t size, int64_t ts_start, level_spec->max_tiles = AOMMAX(level_spec->max_tiles, tiles); if (show_frame) { - scan_past_frames(buffer, encoded_frames_in_last_second, level_spec); + scan_past_frames(buffer, encoded_frames_in_last_second, level_spec, + level_stats); } // Check whether target level is met. @@ -554,8 +684,9 @@ void av1_update_level_info(AV1_COMP *cpi, size_t size, int64_t ts_start, const AV1LevelSpec *const target_level_spec = av1_level_defs + target_seq_level_idx; const int tier = seq_params->tier[i]; - const TARGET_LEVEL_FAIL_ID fail_id = check_level_constraints( - target_level_spec, level_spec, level_stats, tier, is_still_picture); + const TARGET_LEVEL_FAIL_ID fail_id = + check_level_constraints(target_level_spec, level_spec, level_stats, + tier, is_still_picture, profile); if (fail_id != TARGET_LEVEL_OK) { const int target_level_major = 2 + (target_seq_level_idx >> 2); const int target_level_minor = target_seq_level_idx & 3; @@ -578,6 +709,7 @@ aom_codec_err_t av1_get_seq_level_idx(const AV1_COMP *cpi, int *seq_level_idx) { } const int is_still_picture = seq_params->still_picture; + const BITSTREAM_PROFILE profile = seq_params->profile; for (int op = 0; op < seq_params->operating_points_cnt_minus_1 + 1; ++op) { seq_level_idx[op] = (int)SEQ_LEVEL_MAX; const int tier = seq_params->tier[op]; @@ -586,8 +718,9 @@ aom_codec_err_t av1_get_seq_level_idx(const AV1_COMP *cpi, int *seq_level_idx) { const AV1LevelSpec *const level_spec = &level_info->level_spec; for (int level = 0; level < SEQ_LEVELS; ++level) { const AV1LevelSpec *const target_level_spec = av1_level_defs + level; - const TARGET_LEVEL_FAIL_ID fail_id = check_level_constraints( - target_level_spec, level_spec, level_stats, tier, is_still_picture); + const TARGET_LEVEL_FAIL_ID fail_id = + check_level_constraints(target_level_spec, level_spec, level_stats, + tier, is_still_picture, profile); if (fail_id == TARGET_LEVEL_OK) { seq_level_idx[op] = level; break; diff --git a/third_party/aom/av1/encoder/level.h b/third_party/aom/av1/encoder/level.h index 8fff600590..c4e20c896e 100644 --- a/third_party/aom/av1/encoder/level.h +++ b/third_party/aom/av1/encoder/level.h @@ -23,6 +23,7 @@ typedef struct { int max_h_size; int max_v_size; int max_header_rate; + int max_tile_rate; int max_tiles; int max_tile_cols; int64_t max_display_rate; @@ -36,8 +37,10 @@ typedef struct { typedef struct { int64_t ts_start; int64_t ts_end; + size_t encoded_size_in_bytes; int pic_size; int frame_header_count; + int tiles; int show_frame; int show_existing_frame; } FrameRecord; @@ -52,11 +55,14 @@ typedef struct { // Used to keep track of AV1 Level Stats. Currently unimplemented. typedef struct { - uint64_t total_compressed_size; + int max_bitrate; // In bps. int max_tile_size; + int max_superres_tile_width; int min_cropped_tile_width; int min_cropped_tile_height; int tile_width_is_valid; + int min_frame_width; + int min_frame_height; double total_time_encoded; double min_cr; } AV1LevelStats; @@ -66,6 +72,8 @@ typedef struct { AV1LevelSpec level_spec; } AV1LevelInfo; +void av1_init_level_info(AV1LevelInfo *level_info); + void av1_update_level_info(struct AV1_COMP *cpi, size_t size, int64_t ts_start, int64_t ts_end); diff --git a/third_party/aom/av1/encoder/mcomp.c b/third_party/aom/av1/encoder/mcomp.c index c9573bdedd..1a64d1a0c5 100644 --- a/third_party/aom/av1/encoder/mcomp.c +++ b/third_party/aom/av1/encoder/mcomp.c @@ -123,6 +123,7 @@ void av1_init_dsmotion_compensation(search_site_config *cfg, int stride) { cfg->ss[0].mv.col = cfg->ss[0].mv.row = 0; cfg->ss[0].offset = 0; + cfg->stride = stride; for (len = MAX_FIRST_STEP; len > 0; len /= 2) { // Generate offsets for 4 search sites per step. @@ -144,6 +145,7 @@ void av1_init3smotion_compensation(search_site_config *cfg, int stride) { cfg->ss[0].mv.col = cfg->ss[0].mv.row = 0; cfg->ss[0].offset = 0; + cfg->stride = stride; for (len = MAX_FIRST_STEP; len > 0; len /= 2) { // Generate offsets for 8 search sites per step. @@ -2017,8 +2019,8 @@ int av1_refining_search_8p_c(MACROBLOCK *x, int error_per_bit, int search_range, MV *best_mv = &x->best_mv.as_mv; unsigned int best_sad = INT_MAX; int i, j; - uint8_t do_refine_search_grid[SEARCH_GRID_STRIDE_8P * SEARCH_GRID_STRIDE_8P] = - { 0 }; + uint8_t do_refine_search_grid[SEARCH_GRID_STRIDE_8P * + SEARCH_GRID_STRIDE_8P] = { 0 }; int grid_center = SEARCH_GRID_CENTER_8P; int grid_coord = grid_center; @@ -3095,20 +3097,20 @@ void av1_simple_motion_search(AV1_COMP *const cpi, MACROBLOCK *x, int mi_row, MV ref_mv = { 0, 0 }; const int step_param = cpi->mv_step_param; const MvLimits tmp_mv_limits = x->mv_limits; - const SEARCH_METHODS search_methods = NSTEP; + const SEARCH_METHODS search_methods = cpi->sf.mv.search_method; const int do_mesh_search = 0; const int sadpb = x->sadperbit16; int cost_list[5]; const int ref_idx = 0; int var; + av1_setup_pre_planes(xd, ref_idx, yv12, mi_row, mi_col, + get_ref_scale_factors(cm, ref), num_planes); + set_ref_ptrs(cm, xd, mbmi->ref_frame[0], mbmi->ref_frame[1]); if (scaled_ref_frame) { backup_yv12 = xd->plane[AOM_PLANE_Y].pre[ref_idx]; av1_setup_pre_planes(xd, ref_idx, scaled_ref_frame, mi_row, mi_col, NULL, num_planes); - } else { - av1_setup_pre_planes(xd, ref_idx, yv12, mi_row, mi_col, - get_ref_scale_factors(cm, ref), num_planes); } // This overwrites the mv_limits so we will need to restore it later. @@ -3122,6 +3124,9 @@ void av1_simple_motion_search(AV1_COMP *const cpi, MACROBLOCK *x, int mi_row, const int use_subpel_search = var < INT_MAX && !cpi->common.cur_frame_force_integer_mv && use_subpixel; + if (scaled_ref_frame) { + xd->plane[AOM_PLANE_Y].pre[ref_idx] = backup_yv12; + } if (use_subpel_search) { int not_used = 0; if (cpi->sf.use_accurate_subpel_search) { @@ -3151,7 +3156,6 @@ void av1_simple_motion_search(AV1_COMP *const cpi, MACROBLOCK *x, int mi_row, mbmi->mv[0].as_mv = x->best_mv.as_mv; // Get a copy of the prediction output - set_ref_ptrs(cm, xd, mbmi->ref_frame[0], mbmi->ref_frame[1]); av1_enc_build_inter_predictor(cm, xd, mi_row, mi_col, NULL, bsize, AOM_PLANE_Y, AOM_PLANE_Y); diff --git a/third_party/aom/av1/encoder/mcomp.h b/third_party/aom/av1/encoder/mcomp.h index 71547da8d9..de4d931c30 100644 --- a/third_party/aom/av1/encoder/mcomp.h +++ b/third_party/aom/av1/encoder/mcomp.h @@ -47,6 +47,7 @@ typedef struct search_site_config { search_site ss[8 * MAX_MVSEARCH_STEPS + 1]; int ss_count; int searches_per_step; + int stride; } search_site_config; typedef struct { diff --git a/third_party/aom/av1/encoder/nonrd_pickmode.c b/third_party/aom/av1/encoder/nonrd_pickmode.c new file mode 100644 index 0000000000..443fee6644 --- /dev/null +++ b/third_party/aom/av1/encoder/nonrd_pickmode.c @@ -0,0 +1,1364 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + + */ + +#include +#include +#include +#include + +#include "config/aom_dsp_rtcd.h" +#include "config/av1_rtcd.h" + +#include "aom_dsp/aom_dsp_common.h" +#include "aom_dsp/blend.h" +#include "aom_mem/aom_mem.h" +#include "aom_ports/aom_timer.h" +#include "aom_ports/mem.h" +#include "aom_ports/system_state.h" + +#include "av1/common/mvref_common.h" +#include "av1/common/pred_common.h" +#include "av1/common/reconinter.h" + +#include "av1/encoder/encodemv.h" +#include "av1/encoder/rdopt.h" +#include "av1/encoder/reconinter_enc.h" + +extern int g_pick_inter_mode_cnt; +typedef struct { + uint8_t *data; + int stride; + int in_use; +} PRED_BUFFER; + +typedef struct { + PRED_BUFFER *best_pred; + PREDICTION_MODE best_mode; + TX_SIZE best_tx_size; + TX_SIZE best_intra_tx_size; + MV_REFERENCE_FRAME best_ref_frame; + MV_REFERENCE_FRAME best_second_ref_frame; + uint8_t best_mode_skip_txfm; + InterpFilters best_pred_filter; +} BEST_PICKMODE; + +typedef struct { + MV_REFERENCE_FRAME ref_frame; + PREDICTION_MODE pred_mode; +} REF_MODE; + +#define RT_INTER_MODES 9 +static const REF_MODE ref_mode_set[RT_INTER_MODES] = { + { LAST_FRAME, NEARESTMV }, { LAST_FRAME, NEARMV }, + { LAST_FRAME, NEWMV }, { GOLDEN_FRAME, NEARESTMV }, + { GOLDEN_FRAME, NEARMV }, { GOLDEN_FRAME, NEWMV }, + { ALTREF_FRAME, NEARESTMV }, { ALTREF_FRAME, NEARMV }, + { ALTREF_FRAME, NEWMV } +}; + +static const THR_MODES mode_idx[REF_FRAMES][4] = { + { THR_DC, THR_V_PRED, THR_H_PRED, THR_SMOOTH }, + { THR_NEARESTMV, THR_NEARMV, THR_GLOBALMV, THR_NEWMV }, + { THR_NEARESTG, THR_NEARG, THR_GLOBALMV, THR_NEWG }, + { THR_NEARESTA, THR_NEARA, THR_GLOBALMV, THR_NEWA }, +}; + +typedef struct { + PREDICTION_MODE mode; + MV_REFERENCE_FRAME ref_frame[2]; +} MODE_DEFINITION; + +static const MODE_DEFINITION av1_mode_order[MAX_MODES] = { + { NEARESTMV, { LAST_FRAME, NONE_FRAME } }, + { NEARESTMV, { LAST2_FRAME, NONE_FRAME } }, + { NEARESTMV, { LAST3_FRAME, NONE_FRAME } }, + { NEARESTMV, { BWDREF_FRAME, NONE_FRAME } }, + { NEARESTMV, { ALTREF2_FRAME, NONE_FRAME } }, + { NEARESTMV, { ALTREF_FRAME, NONE_FRAME } }, + { NEARESTMV, { GOLDEN_FRAME, NONE_FRAME } }, + + { NEWMV, { LAST_FRAME, NONE_FRAME } }, + { NEWMV, { LAST2_FRAME, NONE_FRAME } }, + { NEWMV, { LAST3_FRAME, NONE_FRAME } }, + { NEWMV, { BWDREF_FRAME, NONE_FRAME } }, + { NEWMV, { ALTREF2_FRAME, NONE_FRAME } }, + { NEWMV, { ALTREF_FRAME, NONE_FRAME } }, + { NEWMV, { GOLDEN_FRAME, NONE_FRAME } }, + + { NEARMV, { LAST_FRAME, NONE_FRAME } }, + { NEARMV, { LAST2_FRAME, NONE_FRAME } }, + { NEARMV, { LAST3_FRAME, NONE_FRAME } }, + { NEARMV, { BWDREF_FRAME, NONE_FRAME } }, + { NEARMV, { ALTREF2_FRAME, NONE_FRAME } }, + { NEARMV, { ALTREF_FRAME, NONE_FRAME } }, + { NEARMV, { GOLDEN_FRAME, NONE_FRAME } }, + + { GLOBALMV, { LAST_FRAME, NONE_FRAME } }, + { GLOBALMV, { LAST2_FRAME, NONE_FRAME } }, + { GLOBALMV, { LAST3_FRAME, NONE_FRAME } }, + { GLOBALMV, { BWDREF_FRAME, NONE_FRAME } }, + { GLOBALMV, { ALTREF2_FRAME, NONE_FRAME } }, + { GLOBALMV, { GOLDEN_FRAME, NONE_FRAME } }, + { GLOBALMV, { ALTREF_FRAME, NONE_FRAME } }, + + // TODO(kyslov): May need to reconsider the order on the modes to check + + { NEAREST_NEARESTMV, { LAST_FRAME, ALTREF_FRAME } }, + { NEAREST_NEARESTMV, { LAST2_FRAME, ALTREF_FRAME } }, + { NEAREST_NEARESTMV, { LAST3_FRAME, ALTREF_FRAME } }, + { NEAREST_NEARESTMV, { GOLDEN_FRAME, ALTREF_FRAME } }, + { NEAREST_NEARESTMV, { LAST_FRAME, BWDREF_FRAME } }, + { NEAREST_NEARESTMV, { LAST2_FRAME, BWDREF_FRAME } }, + { NEAREST_NEARESTMV, { LAST3_FRAME, BWDREF_FRAME } }, + { NEAREST_NEARESTMV, { GOLDEN_FRAME, BWDREF_FRAME } }, + { NEAREST_NEARESTMV, { LAST_FRAME, ALTREF2_FRAME } }, + { NEAREST_NEARESTMV, { LAST2_FRAME, ALTREF2_FRAME } }, + { NEAREST_NEARESTMV, { LAST3_FRAME, ALTREF2_FRAME } }, + { NEAREST_NEARESTMV, { GOLDEN_FRAME, ALTREF2_FRAME } }, + + { NEAREST_NEARESTMV, { LAST_FRAME, LAST2_FRAME } }, + { NEAREST_NEARESTMV, { LAST_FRAME, LAST3_FRAME } }, + { NEAREST_NEARESTMV, { LAST_FRAME, GOLDEN_FRAME } }, + { NEAREST_NEARESTMV, { BWDREF_FRAME, ALTREF_FRAME } }, + + { NEAR_NEARMV, { LAST_FRAME, ALTREF_FRAME } }, + { NEW_NEARESTMV, { LAST_FRAME, ALTREF_FRAME } }, + { NEAREST_NEWMV, { LAST_FRAME, ALTREF_FRAME } }, + { NEW_NEARMV, { LAST_FRAME, ALTREF_FRAME } }, + { NEAR_NEWMV, { LAST_FRAME, ALTREF_FRAME } }, + { NEW_NEWMV, { LAST_FRAME, ALTREF_FRAME } }, + { GLOBAL_GLOBALMV, { LAST_FRAME, ALTREF_FRAME } }, + + { NEAR_NEARMV, { LAST2_FRAME, ALTREF_FRAME } }, + { NEW_NEARESTMV, { LAST2_FRAME, ALTREF_FRAME } }, + { NEAREST_NEWMV, { LAST2_FRAME, ALTREF_FRAME } }, + { NEW_NEARMV, { LAST2_FRAME, ALTREF_FRAME } }, + { NEAR_NEWMV, { LAST2_FRAME, ALTREF_FRAME } }, + { NEW_NEWMV, { LAST2_FRAME, ALTREF_FRAME } }, + { GLOBAL_GLOBALMV, { LAST2_FRAME, ALTREF_FRAME } }, + + { NEAR_NEARMV, { LAST3_FRAME, ALTREF_FRAME } }, + { NEW_NEARESTMV, { LAST3_FRAME, ALTREF_FRAME } }, + { NEAREST_NEWMV, { LAST3_FRAME, ALTREF_FRAME } }, + { NEW_NEARMV, { LAST3_FRAME, ALTREF_FRAME } }, + { NEAR_NEWMV, { LAST3_FRAME, ALTREF_FRAME } }, + { NEW_NEWMV, { LAST3_FRAME, ALTREF_FRAME } }, + { GLOBAL_GLOBALMV, { LAST3_FRAME, ALTREF_FRAME } }, + + { NEAR_NEARMV, { GOLDEN_FRAME, ALTREF_FRAME } }, + { NEW_NEARESTMV, { GOLDEN_FRAME, ALTREF_FRAME } }, + { NEAREST_NEWMV, { GOLDEN_FRAME, ALTREF_FRAME } }, + { NEW_NEARMV, { GOLDEN_FRAME, ALTREF_FRAME } }, + { NEAR_NEWMV, { GOLDEN_FRAME, ALTREF_FRAME } }, + { NEW_NEWMV, { GOLDEN_FRAME, ALTREF_FRAME } }, + { GLOBAL_GLOBALMV, { GOLDEN_FRAME, ALTREF_FRAME } }, + + { NEAR_NEARMV, { LAST_FRAME, BWDREF_FRAME } }, + { NEW_NEARESTMV, { LAST_FRAME, BWDREF_FRAME } }, + { NEAREST_NEWMV, { LAST_FRAME, BWDREF_FRAME } }, + { NEW_NEARMV, { LAST_FRAME, BWDREF_FRAME } }, + { NEAR_NEWMV, { LAST_FRAME, BWDREF_FRAME } }, + { NEW_NEWMV, { LAST_FRAME, BWDREF_FRAME } }, + { GLOBAL_GLOBALMV, { LAST_FRAME, BWDREF_FRAME } }, + + { NEAR_NEARMV, { LAST2_FRAME, BWDREF_FRAME } }, + { NEW_NEARESTMV, { LAST2_FRAME, BWDREF_FRAME } }, + { NEAREST_NEWMV, { LAST2_FRAME, BWDREF_FRAME } }, + { NEW_NEARMV, { LAST2_FRAME, BWDREF_FRAME } }, + { NEAR_NEWMV, { LAST2_FRAME, BWDREF_FRAME } }, + { NEW_NEWMV, { LAST2_FRAME, BWDREF_FRAME } }, + { GLOBAL_GLOBALMV, { LAST2_FRAME, BWDREF_FRAME } }, + + { NEAR_NEARMV, { LAST3_FRAME, BWDREF_FRAME } }, + { NEW_NEARESTMV, { LAST3_FRAME, BWDREF_FRAME } }, + { NEAREST_NEWMV, { LAST3_FRAME, BWDREF_FRAME } }, + { NEW_NEARMV, { LAST3_FRAME, BWDREF_FRAME } }, + { NEAR_NEWMV, { LAST3_FRAME, BWDREF_FRAME } }, + { NEW_NEWMV, { LAST3_FRAME, BWDREF_FRAME } }, + { GLOBAL_GLOBALMV, { LAST3_FRAME, BWDREF_FRAME } }, + + { NEAR_NEARMV, { GOLDEN_FRAME, BWDREF_FRAME } }, + { NEW_NEARESTMV, { GOLDEN_FRAME, BWDREF_FRAME } }, + { NEAREST_NEWMV, { GOLDEN_FRAME, BWDREF_FRAME } }, + { NEW_NEARMV, { GOLDEN_FRAME, BWDREF_FRAME } }, + { NEAR_NEWMV, { GOLDEN_FRAME, BWDREF_FRAME } }, + { NEW_NEWMV, { GOLDEN_FRAME, BWDREF_FRAME } }, + { GLOBAL_GLOBALMV, { GOLDEN_FRAME, BWDREF_FRAME } }, + + { NEAR_NEARMV, { LAST_FRAME, ALTREF2_FRAME } }, + { NEW_NEARESTMV, { LAST_FRAME, ALTREF2_FRAME } }, + { NEAREST_NEWMV, { LAST_FRAME, ALTREF2_FRAME } }, + { NEW_NEARMV, { LAST_FRAME, ALTREF2_FRAME } }, + { NEAR_NEWMV, { LAST_FRAME, ALTREF2_FRAME } }, + { NEW_NEWMV, { LAST_FRAME, ALTREF2_FRAME } }, + { GLOBAL_GLOBALMV, { LAST_FRAME, ALTREF2_FRAME } }, + + { NEAR_NEARMV, { LAST2_FRAME, ALTREF2_FRAME } }, + { NEW_NEARESTMV, { LAST2_FRAME, ALTREF2_FRAME } }, + { NEAREST_NEWMV, { LAST2_FRAME, ALTREF2_FRAME } }, + { NEW_NEARMV, { LAST2_FRAME, ALTREF2_FRAME } }, + { NEAR_NEWMV, { LAST2_FRAME, ALTREF2_FRAME } }, + { NEW_NEWMV, { LAST2_FRAME, ALTREF2_FRAME } }, + { GLOBAL_GLOBALMV, { LAST2_FRAME, ALTREF2_FRAME } }, + + { NEAR_NEARMV, { LAST3_FRAME, ALTREF2_FRAME } }, + { NEW_NEARESTMV, { LAST3_FRAME, ALTREF2_FRAME } }, + { NEAREST_NEWMV, { LAST3_FRAME, ALTREF2_FRAME } }, + { NEW_NEARMV, { LAST3_FRAME, ALTREF2_FRAME } }, + { NEAR_NEWMV, { LAST3_FRAME, ALTREF2_FRAME } }, + { NEW_NEWMV, { LAST3_FRAME, ALTREF2_FRAME } }, + { GLOBAL_GLOBALMV, { LAST3_FRAME, ALTREF2_FRAME } }, + + { NEAR_NEARMV, { GOLDEN_FRAME, ALTREF2_FRAME } }, + { NEW_NEARESTMV, { GOLDEN_FRAME, ALTREF2_FRAME } }, + { NEAREST_NEWMV, { GOLDEN_FRAME, ALTREF2_FRAME } }, + { NEW_NEARMV, { GOLDEN_FRAME, ALTREF2_FRAME } }, + { NEAR_NEWMV, { GOLDEN_FRAME, ALTREF2_FRAME } }, + { NEW_NEWMV, { GOLDEN_FRAME, ALTREF2_FRAME } }, + { GLOBAL_GLOBALMV, { GOLDEN_FRAME, ALTREF2_FRAME } }, + + { NEAR_NEARMV, { LAST_FRAME, LAST2_FRAME } }, + { NEW_NEARESTMV, { LAST_FRAME, LAST2_FRAME } }, + { NEAREST_NEWMV, { LAST_FRAME, LAST2_FRAME } }, + { NEW_NEARMV, { LAST_FRAME, LAST2_FRAME } }, + { NEAR_NEWMV, { LAST_FRAME, LAST2_FRAME } }, + { NEW_NEWMV, { LAST_FRAME, LAST2_FRAME } }, + { GLOBAL_GLOBALMV, { LAST_FRAME, LAST2_FRAME } }, + + { NEAR_NEARMV, { LAST_FRAME, LAST3_FRAME } }, + { NEW_NEARESTMV, { LAST_FRAME, LAST3_FRAME } }, + { NEAREST_NEWMV, { LAST_FRAME, LAST3_FRAME } }, + { NEW_NEARMV, { LAST_FRAME, LAST3_FRAME } }, + { NEAR_NEWMV, { LAST_FRAME, LAST3_FRAME } }, + { NEW_NEWMV, { LAST_FRAME, LAST3_FRAME } }, + { GLOBAL_GLOBALMV, { LAST_FRAME, LAST3_FRAME } }, + + { NEAR_NEARMV, { LAST_FRAME, GOLDEN_FRAME } }, + { NEW_NEARESTMV, { LAST_FRAME, GOLDEN_FRAME } }, + { NEAREST_NEWMV, { LAST_FRAME, GOLDEN_FRAME } }, + { NEW_NEARMV, { LAST_FRAME, GOLDEN_FRAME } }, + { NEAR_NEWMV, { LAST_FRAME, GOLDEN_FRAME } }, + { NEW_NEWMV, { LAST_FRAME, GOLDEN_FRAME } }, + { GLOBAL_GLOBALMV, { LAST_FRAME, GOLDEN_FRAME } }, + + { NEAR_NEARMV, { BWDREF_FRAME, ALTREF_FRAME } }, + { NEW_NEARESTMV, { BWDREF_FRAME, ALTREF_FRAME } }, + { NEAREST_NEWMV, { BWDREF_FRAME, ALTREF_FRAME } }, + { NEW_NEARMV, { BWDREF_FRAME, ALTREF_FRAME } }, + { NEAR_NEWMV, { BWDREF_FRAME, ALTREF_FRAME } }, + { NEW_NEWMV, { BWDREF_FRAME, ALTREF_FRAME } }, + { GLOBAL_GLOBALMV, { BWDREF_FRAME, ALTREF_FRAME } }, + + // intra modes + { DC_PRED, { INTRA_FRAME, NONE_FRAME } }, + { PAETH_PRED, { INTRA_FRAME, NONE_FRAME } }, + { SMOOTH_PRED, { INTRA_FRAME, NONE_FRAME } }, + { SMOOTH_V_PRED, { INTRA_FRAME, NONE_FRAME } }, + { SMOOTH_H_PRED, { INTRA_FRAME, NONE_FRAME } }, + { H_PRED, { INTRA_FRAME, NONE_FRAME } }, + { V_PRED, { INTRA_FRAME, NONE_FRAME } }, + { D135_PRED, { INTRA_FRAME, NONE_FRAME } }, + { D203_PRED, { INTRA_FRAME, NONE_FRAME } }, + { D157_PRED, { INTRA_FRAME, NONE_FRAME } }, + { D67_PRED, { INTRA_FRAME, NONE_FRAME } }, + { D113_PRED, { INTRA_FRAME, NONE_FRAME } }, + { D45_PRED, { INTRA_FRAME, NONE_FRAME } }, +}; + +enum { + // INTER_ALL = (1 << NEARESTMV) | (1 << NEARMV) | (1 << NEWMV), + INTER_NEAREST = (1 << NEARESTMV), + INTER_NEAREST_NEW = (1 << NEARESTMV) | (1 << NEWMV), + INTER_NEAREST_NEAR = (1 << NEARESTMV) | (1 << NEARMV), + INTER_NEAR_NEW = (1 << NEARMV) | (1 << NEWMV), +}; + +static INLINE void init_best_pickmode(BEST_PICKMODE *bp) { + bp->best_mode = NEARESTMV; + bp->best_ref_frame = LAST_FRAME; + bp->best_tx_size = TX_SIZES; + bp->best_intra_tx_size = TX_SIZES; + bp->best_pred_filter = EIGHTTAP_REGULAR; + bp->best_mode_skip_txfm = 0; + bp->best_second_ref_frame = NONE_FRAME; + bp->best_pred = NULL; +} + +static int combined_motion_search(AV1_COMP *cpi, MACROBLOCK *x, + BLOCK_SIZE bsize, int mi_row, int mi_col, + int_mv *tmp_mv, int *rate_mv, + int64_t best_rd_sofar, int use_base_mv) { + MACROBLOCKD *xd = &x->e_mbd; + const AV1_COMMON *cm = &cpi->common; + const int num_planes = av1_num_planes(cm); + MB_MODE_INFO *mi = xd->mi[0]; + struct buf_2d backup_yv12[MAX_MB_PLANE] = { { 0, 0, 0, 0, 0 } }; + int step_param = cpi->sf.mv.reduce_first_step_size; + const int sadpb = x->sadperbit16; + MV mvp_full; + const int ref = mi->ref_frame[0]; + const MV ref_mv = av1_get_ref_mv(x, mi->ref_mv_idx).as_mv; + MV center_mv; + int dis; + const MvLimits tmp_mv_limits = x->mv_limits; + int rv = 0; + int cost_list[5]; + int search_subpel = 1; + const YV12_BUFFER_CONFIG *scaled_ref_frame = + av1_get_scaled_ref_frame(cpi, ref); + + step_param = AOMMIN(step_param, cpi->mv_step_param); + if (scaled_ref_frame) { + int i; + // Swap out the reference frame for a version that's been scaled to + // match the resolution of the current frame, allowing the existing + // motion search code to be used without additional modifications. + for (i = 0; i < MAX_MB_PLANE; i++) backup_yv12[i] = xd->plane[i].pre[0]; + av1_setup_pre_planes(xd, 0, scaled_ref_frame, mi_row, mi_col, NULL, + num_planes); + } + av1_set_mv_search_range(&x->mv_limits, &ref_mv); + + mvp_full = ref_mv; + + mvp_full.col >>= 3; + mvp_full.row >>= 3; + + if (!use_base_mv) + center_mv = ref_mv; + else + center_mv = tmp_mv->as_mv; + + av1_full_pixel_search( + cpi, x, bsize, &mvp_full, step_param, cpi->sf.mv.search_method, 0, sadpb, + cond_cost_list(cpi, cost_list), ¢er_mv, INT_MAX, 0, + (MI_SIZE * mi_col), (MI_SIZE * mi_row), 0, &cpi->ss_cfg[SS_CFG_SRC]); + + x->mv_limits = tmp_mv_limits; + *tmp_mv = x->best_mv; + // calculate the bit cost on motion vector + mvp_full.row = tmp_mv->as_mv.row * 8; + mvp_full.col = tmp_mv->as_mv.col * 8; + + *rate_mv = av1_mv_bit_cost(&mvp_full, &ref_mv, x->nmv_vec_cost, + x->mv_cost_stack, MV_COST_WEIGHT); + + // TODO(kyslov) Account for Rate Mode! + rv = !(RDCOST(x->rdmult, (*rate_mv), 0) > best_rd_sofar); + + if (rv && search_subpel) { + SUBPEL_FORCE_STOP subpel_force_stop = cpi->sf.mv.subpel_force_stop; + cpi->find_fractional_mv_step( + x, cm, mi_row, mi_col, &ref_mv, cpi->common.allow_high_precision_mv, + x->errorperbit, &cpi->fn_ptr[bsize], subpel_force_stop, + cpi->sf.mv.subpel_iters_per_step, cond_cost_list(cpi, cost_list), + x->nmv_vec_cost, x->mv_cost_stack, &dis, &x->pred_sse[ref], NULL, NULL, + 0, 0, 0, 0, 0, 1); + *tmp_mv = x->best_mv; + *rate_mv = av1_mv_bit_cost(&tmp_mv->as_mv, &ref_mv, x->nmv_vec_cost, + x->mv_cost_stack, MV_COST_WEIGHT); + } + + if (scaled_ref_frame) { + int i; + for (i = 0; i < MAX_MB_PLANE; i++) xd->plane[i].pre[0] = backup_yv12[i]; + } + return rv; +} + +static int search_new_mv(AV1_COMP *cpi, MACROBLOCK *x, + int_mv frame_mv[][REF_FRAMES], + MV_REFERENCE_FRAME ref_frame, int gf_temporal_ref, + BLOCK_SIZE bsize, int mi_row, int mi_col, + int best_pred_sad, int *rate_mv, + int64_t best_sse_sofar, RD_STATS *best_rdc) { + MACROBLOCKD *const xd = &x->e_mbd; + MB_MODE_INFO *const mi = xd->mi[0]; + AV1_COMMON *cm = &cpi->common; + (void)best_sse_sofar; + if (ref_frame > LAST_FRAME && gf_temporal_ref && + cpi->oxcf.rc_mode == AOM_CBR) { + int tmp_sad; + int dis; + int cost_list[5] = { INT_MAX, INT_MAX, INT_MAX, INT_MAX, INT_MAX }; + + if (bsize < BLOCK_16X16) return -1; + + tmp_sad = av1_int_pro_motion_estimation( + cpi, x, bsize, mi_row, mi_col, + &x->mbmi_ext->ref_mv_stack[ref_frame][0].this_mv.as_mv); + + if (tmp_sad > x->pred_mv_sad[LAST_FRAME]) return -1; + if (tmp_sad + (num_pels_log2_lookup[bsize] << 4) > best_pred_sad) return -1; + + frame_mv[NEWMV][ref_frame].as_int = mi->mv[0].as_int; + MV ref_mv = av1_get_ref_mv(x, 0).as_mv; + + *rate_mv = + av1_mv_bit_cost(&frame_mv[NEWMV][ref_frame].as_mv, &ref_mv, + x->nmv_vec_cost, x->mv_cost_stack, MV_COST_WEIGHT); + frame_mv[NEWMV][ref_frame].as_mv.row >>= 3; + frame_mv[NEWMV][ref_frame].as_mv.col >>= 3; + + cpi->find_fractional_mv_step( + x, cm, mi_row, mi_col, &ref_mv, cm->allow_high_precision_mv, + x->errorperbit, &cpi->fn_ptr[bsize], cpi->sf.mv.subpel_force_stop, + cpi->sf.mv.subpel_iters_per_step, cond_cost_list(cpi, cost_list), + x->nmv_vec_cost, x->mv_cost_stack, &dis, &x->pred_sse[ref_frame], NULL, + NULL, 0, 0, 0, 0, 0, 1); + } else if (!combined_motion_search(cpi, x, bsize, mi_row, mi_col, + &frame_mv[NEWMV][ref_frame], rate_mv, + best_rdc->rdcost, 0)) { + return -1; + } + + return 0; +} + +static INLINE void find_predictors( + AV1_COMP *cpi, MACROBLOCK *x, MV_REFERENCE_FRAME ref_frame, + int_mv frame_mv[MB_MODE_COUNT][REF_FRAMES], int const_motion[REF_FRAMES], + int *ref_frame_skip_mask, const int flag_list[4], TileDataEnc *tile_data, + int mi_row, int mi_col, struct buf_2d yv12_mb[4][MAX_MB_PLANE], + BLOCK_SIZE bsize, int force_skip_low_temp_var, int comp_pred_allowed) { + AV1_COMMON *const cm = &cpi->common; + MACROBLOCKD *const xd = &x->e_mbd; + MB_MODE_INFO *const mbmi = xd->mi[0]; + MB_MODE_INFO_EXT *const mbmi_ext = x->mbmi_ext; + const YV12_BUFFER_CONFIG *yv12 = get_ref_frame_yv12_buf(cm, ref_frame); + const int num_planes = av1_num_planes(cm); + (void)tile_data; + (void)const_motion; + (void)comp_pred_allowed; + + x->pred_mv_sad[ref_frame] = INT_MAX; + frame_mv[NEWMV][ref_frame].as_int = INVALID_MV; + // TODO(kyslov) this needs various further optimizations. to be continued.. + if ((cpi->ref_frame_flags & flag_list[ref_frame]) && (yv12 != NULL)) { + const struct scale_factors *const sf = + get_ref_scale_factors_const(cm, ref_frame); + av1_setup_pred_block(xd, yv12_mb[ref_frame], yv12, mi_row, mi_col, sf, sf, + num_planes); + av1_find_mv_refs(cm, xd, mbmi, ref_frame, mbmi_ext->ref_mv_count, + mbmi_ext->ref_mv_stack, mbmi_ext->weight, NULL, + mbmi_ext->global_mvs, mi_row, mi_col, + mbmi_ext->mode_context); + av1_find_best_ref_mvs_from_stack(cm->allow_high_precision_mv, mbmi_ext, + ref_frame, &frame_mv[NEARESTMV][ref_frame], + &frame_mv[NEARMV][ref_frame], 0); + // Early exit for golden frame if force_skip_low_temp_var is set. + if (!av1_is_scaled(sf) && bsize >= BLOCK_8X8 && + !(force_skip_low_temp_var && ref_frame == GOLDEN_FRAME)) { + av1_mv_pred(cpi, x, yv12_mb[ref_frame][0].buf, yv12->y_stride, ref_frame, + bsize); + } + } else { + *ref_frame_skip_mask |= (1 << ref_frame); + } + av1_count_overlappable_neighbors(cm, xd, mi_row, mi_col); + mbmi->num_proj_ref = 1; +} + +static void estimate_ref_frame_costs( + const AV1_COMMON *cm, const MACROBLOCKD *xd, const MACROBLOCK *x, + int segment_id, unsigned int *ref_costs_single, + unsigned int (*ref_costs_comp)[REF_FRAMES]) { + int seg_ref_active = + segfeature_active(&cm->seg, segment_id, SEG_LVL_REF_FRAME); + if (seg_ref_active) { + memset(ref_costs_single, 0, REF_FRAMES * sizeof(*ref_costs_single)); + int ref_frame; + for (ref_frame = 0; ref_frame < REF_FRAMES; ++ref_frame) + memset(ref_costs_comp[ref_frame], 0, + REF_FRAMES * sizeof((*ref_costs_comp)[0])); + } else { + int intra_inter_ctx = av1_get_intra_inter_context(xd); + ref_costs_single[INTRA_FRAME] = x->intra_inter_cost[intra_inter_ctx][0]; + unsigned int base_cost = x->intra_inter_cost[intra_inter_ctx][1]; + + for (int i = LAST_FRAME; i <= ALTREF_FRAME; ++i) + ref_costs_single[i] = base_cost; + + const int ctx_p1 = av1_get_pred_context_single_ref_p1(xd); + const int ctx_p2 = av1_get_pred_context_single_ref_p2(xd); + const int ctx_p3 = av1_get_pred_context_single_ref_p3(xd); + const int ctx_p4 = av1_get_pred_context_single_ref_p4(xd); + const int ctx_p5 = av1_get_pred_context_single_ref_p5(xd); + const int ctx_p6 = av1_get_pred_context_single_ref_p6(xd); + + // Determine cost of a single ref frame, where frame types are represented + // by a tree: + // Level 0: add cost whether this ref is a forward or backward ref + ref_costs_single[LAST_FRAME] += x->single_ref_cost[ctx_p1][0][0]; + ref_costs_single[LAST2_FRAME] += x->single_ref_cost[ctx_p1][0][0]; + ref_costs_single[LAST3_FRAME] += x->single_ref_cost[ctx_p1][0][0]; + ref_costs_single[GOLDEN_FRAME] += x->single_ref_cost[ctx_p1][0][0]; + ref_costs_single[BWDREF_FRAME] += x->single_ref_cost[ctx_p1][0][1]; + ref_costs_single[ALTREF2_FRAME] += x->single_ref_cost[ctx_p1][0][1]; + ref_costs_single[ALTREF_FRAME] += x->single_ref_cost[ctx_p1][0][1]; + + // Level 1: if this ref is forward ref, + // add cost whether it is last/last2 or last3/golden + ref_costs_single[LAST_FRAME] += x->single_ref_cost[ctx_p3][2][0]; + ref_costs_single[LAST2_FRAME] += x->single_ref_cost[ctx_p3][2][0]; + ref_costs_single[LAST3_FRAME] += x->single_ref_cost[ctx_p3][2][1]; + ref_costs_single[GOLDEN_FRAME] += x->single_ref_cost[ctx_p3][2][1]; + + // Level 1: if this ref is backward ref + // then add cost whether this ref is altref or backward ref + ref_costs_single[BWDREF_FRAME] += x->single_ref_cost[ctx_p2][1][0]; + ref_costs_single[ALTREF2_FRAME] += x->single_ref_cost[ctx_p2][1][0]; + ref_costs_single[ALTREF_FRAME] += x->single_ref_cost[ctx_p2][1][1]; + + // Level 2: further add cost whether this ref is last or last2 + ref_costs_single[LAST_FRAME] += x->single_ref_cost[ctx_p4][3][0]; + ref_costs_single[LAST2_FRAME] += x->single_ref_cost[ctx_p4][3][1]; + + // Level 2: last3 or golden + ref_costs_single[LAST3_FRAME] += x->single_ref_cost[ctx_p5][4][0]; + ref_costs_single[GOLDEN_FRAME] += x->single_ref_cost[ctx_p5][4][1]; + + // Level 2: bwdref or altref2 + ref_costs_single[BWDREF_FRAME] += x->single_ref_cost[ctx_p6][5][0]; + ref_costs_single[ALTREF2_FRAME] += x->single_ref_cost[ctx_p6][5][1]; + + if (cm->current_frame.reference_mode != SINGLE_REFERENCE) { + // Similar to single ref, determine cost of compound ref frames. + // cost_compound_refs = cost_first_ref + cost_second_ref + const int bwdref_comp_ctx_p = av1_get_pred_context_comp_bwdref_p(xd); + const int bwdref_comp_ctx_p1 = av1_get_pred_context_comp_bwdref_p1(xd); + const int ref_comp_ctx_p = av1_get_pred_context_comp_ref_p(xd); + const int ref_comp_ctx_p1 = av1_get_pred_context_comp_ref_p1(xd); + const int ref_comp_ctx_p2 = av1_get_pred_context_comp_ref_p2(xd); + + const int comp_ref_type_ctx = av1_get_comp_reference_type_context(xd); + unsigned int ref_bicomp_costs[REF_FRAMES] = { 0 }; + + ref_bicomp_costs[LAST_FRAME] = ref_bicomp_costs[LAST2_FRAME] = + ref_bicomp_costs[LAST3_FRAME] = ref_bicomp_costs[GOLDEN_FRAME] = + base_cost + x->comp_ref_type_cost[comp_ref_type_ctx][1]; + ref_bicomp_costs[BWDREF_FRAME] = ref_bicomp_costs[ALTREF2_FRAME] = 0; + ref_bicomp_costs[ALTREF_FRAME] = 0; + + // cost of first ref frame + ref_bicomp_costs[LAST_FRAME] += x->comp_ref_cost[ref_comp_ctx_p][0][0]; + ref_bicomp_costs[LAST2_FRAME] += x->comp_ref_cost[ref_comp_ctx_p][0][0]; + ref_bicomp_costs[LAST3_FRAME] += x->comp_ref_cost[ref_comp_ctx_p][0][1]; + ref_bicomp_costs[GOLDEN_FRAME] += x->comp_ref_cost[ref_comp_ctx_p][0][1]; + + ref_bicomp_costs[LAST_FRAME] += x->comp_ref_cost[ref_comp_ctx_p1][1][0]; + ref_bicomp_costs[LAST2_FRAME] += x->comp_ref_cost[ref_comp_ctx_p1][1][1]; + + ref_bicomp_costs[LAST3_FRAME] += x->comp_ref_cost[ref_comp_ctx_p2][2][0]; + ref_bicomp_costs[GOLDEN_FRAME] += x->comp_ref_cost[ref_comp_ctx_p2][2][1]; + + // cost of second ref frame + ref_bicomp_costs[BWDREF_FRAME] += + x->comp_bwdref_cost[bwdref_comp_ctx_p][0][0]; + ref_bicomp_costs[ALTREF2_FRAME] += + x->comp_bwdref_cost[bwdref_comp_ctx_p][0][0]; + ref_bicomp_costs[ALTREF_FRAME] += + x->comp_bwdref_cost[bwdref_comp_ctx_p][0][1]; + + ref_bicomp_costs[BWDREF_FRAME] += + x->comp_bwdref_cost[bwdref_comp_ctx_p1][1][0]; + ref_bicomp_costs[ALTREF2_FRAME] += + x->comp_bwdref_cost[bwdref_comp_ctx_p1][1][1]; + + // cost: if one ref frame is forward ref, the other ref is backward ref + int ref0, ref1; + for (ref0 = LAST_FRAME; ref0 <= GOLDEN_FRAME; ++ref0) { + for (ref1 = BWDREF_FRAME; ref1 <= ALTREF_FRAME; ++ref1) { + ref_costs_comp[ref0][ref1] = + ref_bicomp_costs[ref0] + ref_bicomp_costs[ref1]; + } + } + + // cost: if both ref frames are the same side. + const int uni_comp_ref_ctx_p = av1_get_pred_context_uni_comp_ref_p(xd); + const int uni_comp_ref_ctx_p1 = av1_get_pred_context_uni_comp_ref_p1(xd); + const int uni_comp_ref_ctx_p2 = av1_get_pred_context_uni_comp_ref_p2(xd); + ref_costs_comp[LAST_FRAME][LAST2_FRAME] = + base_cost + x->comp_ref_type_cost[comp_ref_type_ctx][0] + + x->uni_comp_ref_cost[uni_comp_ref_ctx_p][0][0] + + x->uni_comp_ref_cost[uni_comp_ref_ctx_p1][1][0]; + ref_costs_comp[LAST_FRAME][LAST3_FRAME] = + base_cost + x->comp_ref_type_cost[comp_ref_type_ctx][0] + + x->uni_comp_ref_cost[uni_comp_ref_ctx_p][0][0] + + x->uni_comp_ref_cost[uni_comp_ref_ctx_p1][1][1] + + x->uni_comp_ref_cost[uni_comp_ref_ctx_p2][2][0]; + ref_costs_comp[LAST_FRAME][GOLDEN_FRAME] = + base_cost + x->comp_ref_type_cost[comp_ref_type_ctx][0] + + x->uni_comp_ref_cost[uni_comp_ref_ctx_p][0][0] + + x->uni_comp_ref_cost[uni_comp_ref_ctx_p1][1][1] + + x->uni_comp_ref_cost[uni_comp_ref_ctx_p2][2][1]; + ref_costs_comp[BWDREF_FRAME][ALTREF_FRAME] = + base_cost + x->comp_ref_type_cost[comp_ref_type_ctx][0] + + x->uni_comp_ref_cost[uni_comp_ref_ctx_p][0][1]; + } else { + int ref0, ref1; + for (ref0 = LAST_FRAME; ref0 <= GOLDEN_FRAME; ++ref0) { + for (ref1 = BWDREF_FRAME; ref1 <= ALTREF_FRAME; ++ref1) + ref_costs_comp[ref0][ref1] = 512; + } + ref_costs_comp[LAST_FRAME][LAST2_FRAME] = 512; + ref_costs_comp[LAST_FRAME][LAST3_FRAME] = 512; + ref_costs_comp[LAST_FRAME][GOLDEN_FRAME] = 512; + ref_costs_comp[BWDREF_FRAME][ALTREF_FRAME] = 512; + } + } +} + +static void model_rd_from_sse(const AV1_COMP *const cpi, + const MACROBLOCK *const x, BLOCK_SIZE plane_bsize, + int plane, int64_t sse, int num_samples, + int *rate, int64_t *dist) { + (void)num_samples; + (void)cpi; + const MACROBLOCKD *const xd = &x->e_mbd; + const struct macroblockd_plane *const pd = &xd->plane[plane]; + const int dequant_shift = (is_cur_buf_hbd(xd)) ? xd->bd - 5 : 3; + + av1_model_rd_from_var_lapndz(sse, num_pels_log2_lookup[plane_bsize], + pd->dequant_Q3[1] >> dequant_shift, rate, dist); + *dist <<= 4; +} + +static void model_rd_for_sb(const AV1_COMP *const cpi, BLOCK_SIZE bsize, + MACROBLOCK *x, MACROBLOCKD *xd, int plane_from, + int plane_to, int mi_row, int mi_col, + int *out_rate_sum, int64_t *out_dist_sum, + int *skip_txfm_sb, int64_t *skip_sse_sb, + int *plane_rate, int64_t *plane_sse, + int64_t *plane_dist) { + // Note our transform coeffs are 8 times an orthogonal transform. + // Hence quantizer step is also 8 times. To get effective quantizer + // we need to divide by 8 before sending to modeling function. + int plane; + (void)mi_row; + (void)mi_col; + const int ref = xd->mi[0]->ref_frame[0]; + + int64_t rate_sum = 0; + int64_t dist_sum = 0; + int64_t total_sse = 0; + assert(bsize < BLOCK_SIZES_ALL); + + for (plane = plane_from; plane <= plane_to; ++plane) { + struct macroblock_plane *const p = &x->plane[plane]; + struct macroblockd_plane *const pd = &xd->plane[plane]; + const BLOCK_SIZE plane_bsize = + get_plane_block_size(bsize, pd->subsampling_x, pd->subsampling_y); + const int bw = block_size_wide[plane_bsize]; + const int bh = block_size_high[plane_bsize]; + int64_t sse; + int rate; + int64_t dist; + + if (x->skip_chroma_rd && plane) continue; + + if (is_cur_buf_hbd(xd)) { + sse = aom_highbd_sse(p->src.buf, p->src.stride, pd->dst.buf, + pd->dst.stride, bw, bh); + } else { + sse = aom_sse(p->src.buf, p->src.stride, pd->dst.buf, pd->dst.stride, bw, + bh); + } + sse = ROUND_POWER_OF_TWO(sse, (xd->bd - 8) * 2); + + model_rd_from_sse(cpi, x, plane_bsize, plane, sse, bw * bh, &rate, &dist); + + if (plane == 0) x->pred_sse[ref] = (unsigned int)AOMMIN(sse, UINT_MAX); + + total_sse += sse; + rate_sum += rate; + dist_sum += dist; + if (plane_rate) plane_rate[plane] = rate; + if (plane_sse) plane_sse[plane] = sse; + if (plane_dist) plane_dist[plane] = dist; + assert(rate_sum >= 0); + } + + if (skip_txfm_sb) *skip_txfm_sb = total_sse == 0; + if (skip_sse_sb) *skip_sse_sb = total_sse << 4; + rate_sum = AOMMIN(rate_sum, INT_MAX); + *out_rate_sum = (int)rate_sum; + *out_dist_sum = dist_sum; +} + +static void block_yrd(AV1_COMP *cpi, MACROBLOCK *x, int mi_row, int mi_col, + RD_STATS *this_rdc, int *skippable, int64_t *sse, + BLOCK_SIZE bsize, TX_SIZE tx_size, int rd_computed) { + MACROBLOCKD *xd = &x->e_mbd; + const struct macroblockd_plane *pd = &xd->plane[0]; + struct macroblock_plane *const p = &x->plane[0]; + const int num_4x4_w = mi_size_wide[bsize]; + const int num_4x4_h = mi_size_high[bsize]; + const int step = 1 << (tx_size << 1); + const int block_step = (1 << tx_size); + int block = 0, r, c; + const int max_blocks_wide = + num_4x4_w + (xd->mb_to_right_edge >= 0 ? 0 : xd->mb_to_right_edge >> 5); + const int max_blocks_high = + num_4x4_h + (xd->mb_to_bottom_edge >= 0 ? 0 : xd->mb_to_bottom_edge >> 5); + int eob_cost = 0; + const int bw = 4 * num_4x4_w; + const int bh = 4 * num_4x4_h; + + assert(tx_size > 0 && tx_size <= 4); + + (void)mi_row; + (void)mi_col; + (void)rd_computed; + (void)cpi; + + aom_subtract_block(bh, bw, p->src_diff, bw, p->src.buf, p->src.stride, + pd->dst.buf, pd->dst.stride); + *skippable = 1; + // Keep track of the row and column of the blocks we use so that we know + // if we are in the unrestricted motion border. + for (r = 0; r < max_blocks_high; r += block_step) { + for (c = 0; c < num_4x4_w; c += block_step) { + if (c < max_blocks_wide) { + const SCAN_ORDER *const scan_order = &av1_default_scan_orders[tx_size]; + tran_low_t *const coeff = BLOCK_OFFSET(p->coeff, block); + tran_low_t *const qcoeff = BLOCK_OFFSET(p->qcoeff, block); + tran_low_t *const dqcoeff = BLOCK_OFFSET(pd->dqcoeff, block); + uint16_t *const eob = &p->eobs[block]; + const int diff_stride = bw; + const int16_t *src_diff; + src_diff = &p->src_diff[(r * diff_stride + c) << 2]; + + switch (tx_size) { + case TX_64X64: + assert(0); // Not implemented + break; + case TX_32X32: + aom_hadamard_32x32(src_diff, diff_stride, coeff); + av1_quantize_fp(coeff, 32 * 32, p->zbin_QTX, p->round_fp_QTX, + p->quant_fp_QTX, p->quant_shift_QTX, qcoeff, + dqcoeff, p->dequant_QTX, eob, scan_order->scan, + scan_order->iscan); + break; + case TX_16X16: + aom_hadamard_16x16(src_diff, diff_stride, coeff); + av1_quantize_fp(coeff, 16 * 16, p->zbin_QTX, p->round_fp_QTX, + p->quant_fp_QTX, p->quant_shift_QTX, qcoeff, + dqcoeff, p->dequant_QTX, eob, scan_order->scan, + scan_order->iscan); + break; + case TX_8X8: + aom_hadamard_8x8(src_diff, diff_stride, coeff); + av1_quantize_fp(coeff, 8 * 8, p->zbin_QTX, p->round_fp_QTX, + p->quant_fp_QTX, p->quant_shift_QTX, qcoeff, + dqcoeff, p->dequant_QTX, eob, scan_order->scan, + scan_order->iscan); + break; + default: assert(0); break; + } + *skippable &= (*eob == 0); + eob_cost += 1; + } + block += step; + } + } + + this_rdc->rate = 0; + if (*sse < INT64_MAX) { + *sse = (*sse << 6) >> 2; + if (*skippable) { + this_rdc->dist = *sse; + return; + } + } + + block = 0; + this_rdc->dist = 0; + for (r = 0; r < max_blocks_high; r += block_step) { + for (c = 0; c < num_4x4_w; c += block_step) { + if (c < max_blocks_wide) { + int64_t dummy; + tran_low_t *const coeff = BLOCK_OFFSET(p->coeff, block); + tran_low_t *const qcoeff = BLOCK_OFFSET(p->qcoeff, block); + tran_low_t *const dqcoeff = BLOCK_OFFSET(pd->dqcoeff, block); + uint16_t *const eob = &p->eobs[block]; + + if (*eob == 1) + this_rdc->rate += (int)abs(qcoeff[0]); + else if (*eob > 1) + this_rdc->rate += aom_satd(qcoeff, step << 4); + + this_rdc->dist += + av1_block_error(coeff, dqcoeff, step << 4, &dummy) >> 2; + } + block += step; + } + } + + // If skippable is set, rate gets clobbered later. + this_rdc->rate <<= (2 + AV1_PROB_COST_SHIFT); + this_rdc->rate += (eob_cost << AV1_PROB_COST_SHIFT); +} + +static INLINE void init_mbmi(MB_MODE_INFO *mbmi, int mode_index, + const AV1_COMMON *cm) { + PALETTE_MODE_INFO *const pmi = &mbmi->palette_mode_info; + PREDICTION_MODE this_mode = av1_mode_order[mode_index].mode; + mbmi->ref_mv_idx = 0; + mbmi->mode = this_mode; + mbmi->uv_mode = UV_DC_PRED; + mbmi->ref_frame[0] = av1_mode_order[mode_index].ref_frame[0]; + mbmi->ref_frame[1] = av1_mode_order[mode_index].ref_frame[1]; + pmi->palette_size[0] = 0; + pmi->palette_size[1] = 0; + mbmi->filter_intra_mode_info.use_filter_intra = 0; + mbmi->mv[0].as_int = mbmi->mv[1].as_int = 0; + mbmi->motion_mode = SIMPLE_TRANSLATION; + mbmi->num_proj_ref = 1; + mbmi->interintra_mode = 0; + set_default_interp_filters(mbmi, cm->interp_filter); +} + +static void store_coding_context(MACROBLOCK *x, PICK_MODE_CONTEXT *ctx, + int mode_index) { + MACROBLOCKD *const xd = &x->e_mbd; + + // Take a snapshot of the coding context so it can be + // restored if we decide to encode this way + ctx->rd_stats.skip = x->skip; + memcpy(ctx->blk_skip, x->blk_skip, sizeof(x->blk_skip[0]) * ctx->num_4x4_blk); + ctx->skippable = x->skip; + ctx->best_mode_index = mode_index; + ctx->mic = *xd->mi[0]; + ctx->mbmi_ext = *x->mbmi_ext; + ctx->comp_pred_diff = 0; + ctx->hybrid_pred_diff = 0; + ctx->single_pred_diff = 0; +} + +static int get_pred_buffer(PRED_BUFFER *p, int len) { + int i; + + for (i = 0; i < len; i++) { + if (!p[i].in_use) { + p[i].in_use = 1; + return i; + } + } + return -1; +} + +static void free_pred_buffer(PRED_BUFFER *p) { + if (p != NULL) p->in_use = 0; +} + +static int cost_mv_ref(const MACROBLOCK *const x, PREDICTION_MODE mode, + int16_t mode_context) { + if (is_inter_compound_mode(mode)) { + return x + ->inter_compound_mode_cost[mode_context][INTER_COMPOUND_OFFSET(mode)]; + } + + int mode_cost = 0; + int16_t mode_ctx = mode_context & NEWMV_CTX_MASK; + + assert(is_inter_mode(mode)); + + if (mode == NEWMV) { + mode_cost = x->newmv_mode_cost[mode_ctx][0]; + return mode_cost; + } else { + mode_cost = x->newmv_mode_cost[mode_ctx][1]; + mode_ctx = (mode_context >> GLOBALMV_OFFSET) & GLOBALMV_CTX_MASK; + + if (mode == GLOBALMV) { + mode_cost += x->zeromv_mode_cost[mode_ctx][0]; + return mode_cost; + } else { + mode_cost += x->zeromv_mode_cost[mode_ctx][1]; + mode_ctx = (mode_context >> REFMV_OFFSET) & REFMV_CTX_MASK; + mode_cost += x->refmv_mode_cost[mode_ctx][mode != NEARESTMV]; + return mode_cost; + } + } +} + +static void newmv_diff_bias(MACROBLOCKD *xd, PREDICTION_MODE this_mode, + RD_STATS *this_rdc, BLOCK_SIZE bsize, int mv_row, + int mv_col, int is_last_frame) { + // Bias against MVs associated with NEWMV mode that are very different from + // top/left neighbors. + if (this_mode == NEWMV) { + int al_mv_average_row; + int al_mv_average_col; + int left_row, left_col; + int row_diff, col_diff; + int above_mv_valid = 0; + int left_mv_valid = 0; + int above_row = 0; + int above_col = 0; + + if (xd->above_mbmi) { + above_mv_valid = xd->above_mbmi->mv[0].as_int != INVALID_MV; + above_row = xd->above_mbmi->mv[0].as_mv.row; + above_col = xd->above_mbmi->mv[0].as_mv.col; + } + if (xd->left_mbmi) { + left_mv_valid = xd->left_mbmi->mv[0].as_int != INVALID_MV; + left_row = xd->left_mbmi->mv[0].as_mv.row; + left_col = xd->left_mbmi->mv[0].as_mv.col; + } + if (above_mv_valid && left_mv_valid) { + al_mv_average_row = (above_row + left_row + 1) >> 1; + al_mv_average_col = (above_col + left_col + 1) >> 1; + } else if (above_mv_valid) { + al_mv_average_row = above_row; + al_mv_average_col = above_col; + } else if (left_mv_valid) { + al_mv_average_row = left_row; + al_mv_average_col = left_col; + } else { + al_mv_average_row = al_mv_average_col = 0; + } + row_diff = (al_mv_average_row - mv_row); + col_diff = (al_mv_average_col - mv_col); + if (row_diff > 48 || row_diff < -48 || col_diff > 48 || col_diff < -48) { + if (bsize > BLOCK_32X32) + this_rdc->rdcost = this_rdc->rdcost << 1; + else + this_rdc->rdcost = 5 * this_rdc->rdcost >> 2; + } + } + if (bsize >= BLOCK_16X16 && is_last_frame && mv_row < 16 && mv_row > -16 && + mv_col < 16 && mv_col > -16) + this_rdc->rdcost = 7 * (this_rdc->rdcost >> 3); +} + +void av1_fast_nonrd_pick_inter_mode_sb(AV1_COMP *cpi, TileDataEnc *tile_data, + MACROBLOCK *x, int mi_row, int mi_col, + RD_STATS *rd_cost, BLOCK_SIZE bsize, + PICK_MODE_CONTEXT *ctx, + int64_t best_rd_so_far) { + AV1_COMMON *const cm = &cpi->common; + MACROBLOCKD *const xd = &x->e_mbd; + MB_MODE_INFO *const mi = xd->mi[0]; + struct macroblockd_plane *const pd = &xd->plane[0]; + + BEST_PICKMODE best_pickmode; + int inter_mode_mask[BLOCK_SIZES]; + + MV_REFERENCE_FRAME ref_frame; + MV_REFERENCE_FRAME usable_ref_frame, second_ref_frame; + int_mv frame_mv[MB_MODE_COUNT][REF_FRAMES]; + uint8_t mode_checked[MB_MODE_COUNT][REF_FRAMES]; + struct buf_2d yv12_mb[6][MAX_MB_PLANE]; + static const int flag_list[5] = { 0, AOM_LAST_FLAG, AOM_LAST2_FLAG, + AOM_LAST3_FLAG, AOM_GOLD_FLAG }; + RD_STATS this_rdc, best_rdc; + // var_y and sse_y are saved to be used in skipping checking + int64_t sse_y = UINT_MAX; + const int *const rd_threshes = cpi->rd.threshes[mi->segment_id][bsize]; + const int *const rd_thresh_freq_fact = tile_data->thresh_freq_fact[bsize]; + + InterpFilter filter_ref; + int const_motion[REF_FRAMES] = { 0 }; + int ref_frame_skip_mask = 0; + int idx; + int best_pred_sad = INT_MAX; + int best_early_term = 0; + unsigned int ref_costs_single[REF_FRAMES], + ref_costs_comp[REF_FRAMES][REF_FRAMES]; + int use_golden_nonzeromv = 1; + int force_skip_low_temp_var = 0; + int skip_ref_find_pred[5] = { 0 }; + int64_t best_sse_sofar = INT64_MAX; + int gf_temporal_ref = 0; + const struct segmentation *const seg = &cm->seg; + int comp_modes = 0; + int num_inter_modes = RT_INTER_MODES; + unsigned char segment_id = mi->segment_id; + InterpFilter best_filter = EIGHTTAP_REGULAR; + PRED_BUFFER tmp[4]; + DECLARE_ALIGNED(16, uint8_t, pred_buf[3 * 64 * 64]); + PRED_BUFFER *this_mode_pred = NULL; + const int reuse_inter_pred = cpi->sf.reuse_inter_pred_nonrd; + const int bh = block_size_high[bsize]; + const int bw = block_size_wide[bsize]; + const int pixels_in_block = bh * bw; + struct buf_2d orig_dst = pd->dst; + + (void)best_rd_so_far; + + init_best_pickmode(&best_pickmode); + + for (int i = 0; i < BLOCK_SIZES; ++i) inter_mode_mask[i] = INTER_ALL; + + // TODO(kyslov) Move this to Speed Features + inter_mode_mask[BLOCK_128X128] = INTER_NEAREST_NEAR; + + x->source_variance = UINT_MAX; + + struct scale_factors *const sf_last = get_ref_scale_factors(cm, LAST_FRAME); + struct scale_factors *const sf_golden = + get_ref_scale_factors(cm, GOLDEN_FRAME); + gf_temporal_ref = 1; + // For temporal long term prediction, check that the golden reference + // is same scale as last reference, otherwise disable. + if ((sf_last->x_scale_fp != sf_golden->x_scale_fp) || + (sf_last->y_scale_fp != sf_golden->y_scale_fp)) { + gf_temporal_ref = 0; + } + + av1_collect_neighbors_ref_counts(xd); + av1_count_overlappable_neighbors(cm, xd, mi_row, mi_col); + + estimate_ref_frame_costs(cm, xd, x, segment_id, ref_costs_single, + ref_costs_comp); + + memset(&mode_checked[0][0], 0, MB_MODE_COUNT * REF_FRAMES); + if (reuse_inter_pred) { + int i; + for (i = 0; i < 3; i++) { + tmp[i].data = &pred_buf[pixels_in_block * i]; + tmp[i].stride = bw; + tmp[i].in_use = 0; + } + tmp[3].data = pd->dst.buf; + tmp[3].stride = pd->dst.stride; + tmp[3].in_use = 0; + } + + x->skip = 0; + + // Instead of using av1_get_pred_context_switchable_interp(xd) to assign + // filter_ref, we use a less strict condition on assigning filter_ref. + // This is to reduce the probabily of entering the flow of not assigning + // filter_ref and then skip filter search. + filter_ref = cm->interp_filter; + + // initialize mode decisions + av1_invalid_rd_stats(&best_rdc); + av1_invalid_rd_stats(rd_cost); + mi->sb_type = bsize; + mi->ref_frame[0] = NONE_FRAME; + mi->ref_frame[1] = NONE_FRAME; + + x->source_variance = + av1_get_sby_perpixel_variance(cpi, &x->plane[0].src, bsize); + +// TODO(kyslov) Refine logic of selecting REF FRAME SET. +// For now only LAST_FRAME is used +#if 0 + if (cpi->rc.frames_since_golden == 0 && gf_temporal_ref) { + usable_ref_frame = LAST_FRAME; + } else { + usable_ref_frame = GOLDEN_FRAME; + } + + force_skip_low_temp_var = get_force_skip_low_temp_var(&x->variance_low[0], + mi_row, mi_col, bsize); + // If force_skip_low_temp_var is set, and for short circuit mode = 1 and 3, + // skip golden reference. + if (force_skip_low_temp_var) { + usable_ref_frame = LAST_FRAME; + } + + if (!((cpi->ref_frame_flags & flag_list[GOLDEN_FRAME]) && + !force_skip_low_temp_var)) + use_golden_nonzeromv = 0; + + // If the segment reference frame feature is enabled and it's set to GOLDEN + // reference, then make sure we don't skip checking GOLDEN, this is to + // prevent possibility of not picking any mode. + if (segfeature_active(seg, mi->segment_id, SEG_LVL_REF_FRAME) && + get_segdata(seg, mi->segment_id, SEG_LVL_REF_FRAME) == GOLDEN_FRAME) { + usable_ref_frame = GOLDEN_FRAME; + skip_ref_find_pred[GOLDEN_FRAME] = 0; + } +#endif + usable_ref_frame = LAST_FRAME; + + for (ref_frame = LAST_FRAME; ref_frame <= usable_ref_frame; ++ref_frame) { + // Skip find_predictor if the reference frame is not in the + // ref_frame_flags (i.e., not used as a reference for this frame). + skip_ref_find_pred[ref_frame] = + !(cpi->ref_frame_flags & flag_list[ref_frame]); + if (!skip_ref_find_pred[ref_frame]) { + find_predictors(cpi, x, ref_frame, frame_mv, const_motion, + &ref_frame_skip_mask, flag_list, tile_data, mi_row, + mi_col, yv12_mb, bsize, force_skip_low_temp_var, + comp_modes > 0); + } + } + + for (idx = 0; idx < num_inter_modes; ++idx) { + int rate_mv = 0; + int mode_rd_thresh; + int mode_index; + int i; + int64_t this_sse; + int is_skippable; + int this_early_term = 0; + int rd_computed = 0; + int inter_mv_mode = 0; + int skip_this_mv = 0; + int comp_pred = 0; + int force_mv_inter_layer = 0; + PREDICTION_MODE this_mode; + MB_MODE_INFO_EXT *const mbmi_ext = x->mbmi_ext; + second_ref_frame = NONE_FRAME; + + this_mode = ref_mode_set[idx].pred_mode; + ref_frame = ref_mode_set[idx].ref_frame; + init_mbmi(mi, idx, cm); + + mi->tx_size = AOMMIN(AOMMIN(max_txsize_lookup[bsize], + tx_mode_to_biggest_tx_size[cm->tx_mode]), + TX_16X16); + memset(mi->inter_tx_size, mi->tx_size, sizeof(mi->inter_tx_size)); + memset(mi->txk_type, DCT_DCT, sizeof(mi->txk_type[0]) * TXK_TYPE_BUF_LEN); + av1_zero(x->blk_skip); + + if (ref_frame > usable_ref_frame) continue; + if (skip_ref_find_pred[ref_frame]) continue; + + // If the segment reference frame feature is enabled then do nothing if the + // current ref frame is not allowed. + if (segfeature_active(seg, mi->segment_id, SEG_LVL_REF_FRAME) && + get_segdata(seg, mi->segment_id, SEG_LVL_REF_FRAME) != (int)ref_frame) + continue; + + if (!(cpi->ref_frame_flags & flag_list[ref_frame])) continue; + + if (x->source_variance == 0 && frame_mv[this_mode][ref_frame].as_int != 0) { + continue; + } + + if (!(inter_mode_mask[bsize] & (1 << this_mode))) continue; + + if (const_motion[ref_frame] && this_mode == NEARMV) continue; + + // Skip non-zeromv mode search for golden frame if force_skip_low_temp_var + // is set. If nearestmv for golden frame is 0, zeromv mode will be skipped + // later. + if (!force_mv_inter_layer && force_skip_low_temp_var && + ref_frame == GOLDEN_FRAME && + frame_mv[this_mode][ref_frame].as_int != 0) { + continue; + } + +// TODO(kyslov) Refine logic of pruning reference . +// For now only LAST_FRAME is used +#if 0 + if (x->content_state_sb != kVeryHighSad && + (cpi->sf.short_circuit_low_temp_var >= 2 || + (cpi->sf.short_circuit_low_temp_var == 1 && bsize == BLOCK_64X64)) + && force_skip_low_temp_var && ref_frame == LAST_FRAME && this_mode == + NEWMV) { continue; + } + + // Disable this drop out case if the ref frame segment level feature is + // enabled for this segment. This is to prevent the possibility that we + end + // up unable to pick any mode. + if (!segfeature_active(seg, mi->segment_id, SEG_LVL_REF_FRAME)) { + if (sf->reference_masking && + !(frame_mv[this_mode][ref_frame].as_int == 0 && + ref_frame == LAST_FRAME)) { + if (usable_ref_frame < ALTREF_FRAME) { + if (!force_skip_low_temp_var && usable_ref_frame > LAST_FRAME) { + i = (ref_frame == LAST_FRAME) ? GOLDEN_FRAME : LAST_FRAME; + if ((cpi->ref_frame_flags & flag_list[i])) + if (x->pred_mv_sad[ref_frame] > (x->pred_mv_sad[i] << 1)) + ref_frame_skip_mask |= (1 << ref_frame); + } + } else if (!cpi->rc.is_src_frame_alt_ref && + !(frame_mv[this_mode][ref_frame].as_int == 0 && + ref_frame == ALTREF_FRAME)) { + int ref1 = (ref_frame == GOLDEN_FRAME) ? LAST_FRAME : + GOLDEN_FRAME; int ref2 = (ref_frame == ALTREF_FRAME) ? LAST_FRAME : + ALTREF_FRAME; if (((cpi->ref_frame_flags & flag_list[ref1]) && + (x->pred_mv_sad[ref_frame] > (x->pred_mv_sad[ref1] << 1))) || + ((cpi->ref_frame_flags & flag_list[ref2]) && + (x->pred_mv_sad[ref_frame] > (x->pred_mv_sad[ref2] << 1)))) + ref_frame_skip_mask |= (1 << ref_frame); + } + } + if (ref_frame_skip_mask & (1 << ref_frame)) continue; + } +#endif + + // Select prediction reference frames. + for (i = 0; i < MAX_MB_PLANE; i++) { + xd->plane[i].pre[0] = yv12_mb[ref_frame][i]; + } + + mi->ref_frame[0] = ref_frame; + mi->ref_frame[1] = second_ref_frame; + set_ref_ptrs(cm, xd, ref_frame, second_ref_frame); + + mode_index = mode_idx[ref_frame][INTER_OFFSET(this_mode)]; + mode_rd_thresh = best_pickmode.best_mode_skip_txfm + ? rd_threshes[mode_index] << 1 + : rd_threshes[mode_index]; + + if (rd_less_than_thresh(best_rdc.rdcost, mode_rd_thresh, + rd_thresh_freq_fact[mode_index])) + if (frame_mv[this_mode][ref_frame].as_int != 0) continue; + + if (this_mode == NEWMV && !force_mv_inter_layer) { + if (search_new_mv(cpi, x, frame_mv, ref_frame, gf_temporal_ref, bsize, + mi_row, mi_col, best_pred_sad, &rate_mv, best_sse_sofar, + &best_rdc)) + continue; + } + + for (inter_mv_mode = NEARESTMV; inter_mv_mode <= NEWMV; inter_mv_mode++) { + if (inter_mv_mode == this_mode || comp_pred) continue; + if (mode_checked[inter_mv_mode][ref_frame] && + frame_mv[this_mode][ref_frame].as_int == + frame_mv[inter_mv_mode][ref_frame].as_int && + frame_mv[inter_mv_mode][ref_frame].as_int == 0) { + skip_this_mv = 1; + break; + } + } + + if (skip_this_mv) continue; + + // If use_golden_nonzeromv is false, NEWMV mode is skipped for golden, no + // need to compute best_pred_sad which is only used to skip golden NEWMV. + if (use_golden_nonzeromv && this_mode == NEWMV && ref_frame == LAST_FRAME && + frame_mv[NEWMV][LAST_FRAME].as_int != INVALID_MV) { + const int pre_stride = xd->plane[0].pre[0].stride; + const uint8_t *const pre_buf = + xd->plane[0].pre[0].buf + + (frame_mv[NEWMV][LAST_FRAME].as_mv.row >> 3) * pre_stride + + (frame_mv[NEWMV][LAST_FRAME].as_mv.col >> 3); + best_pred_sad = cpi->fn_ptr[bsize].sdf( + x->plane[0].src.buf, x->plane[0].src.stride, pre_buf, pre_stride); + x->pred_mv_sad[LAST_FRAME] = best_pred_sad; + } + + if (this_mode != NEARESTMV && !comp_pred && + frame_mv[this_mode][ref_frame].as_int == + frame_mv[NEARESTMV][ref_frame].as_int) + continue; + + mi->mode = this_mode; + mi->mv[0].as_int = frame_mv[this_mode][ref_frame].as_int; + mi->mv[1].as_int = 0; + if (reuse_inter_pred) { + if (!this_mode_pred) { + this_mode_pred = &tmp[3]; + } else { + this_mode_pred = &tmp[get_pred_buffer(tmp, 3)]; + pd->dst.buf = this_mode_pred->data; + pd->dst.stride = bw; + } + } + + // TODO(kyslov) bring back filter search + mi->interp_filters = + (filter_ref == SWITCHABLE) ? EIGHTTAP_REGULAR : filter_ref; + av1_enc_build_inter_predictor(cm, xd, mi_row, mi_col, NULL, bsize, + AOM_PLANE_Y, AOM_PLANE_Y); + + // TODO(kyslov) For large partition blocks, extra testing needs to be done + + rd_computed = 1; + model_rd_for_sb(cpi, bsize, x, xd, AOM_PLANE_Y, AOM_PLANE_Y, mi_row, mi_col, + &this_rdc.rate, &this_rdc.dist, &this_rdc.skip, NULL, NULL, + &sse_y, NULL); + + if (sse_y < best_sse_sofar) best_sse_sofar = sse_y; + + const int skip_ctx = av1_get_skip_context(xd); + const int skip_cost = x->skip_cost[skip_ctx][1]; + + this_sse = (int64_t)sse_y; + + block_yrd(cpi, x, mi_row, mi_col, &this_rdc, &is_skippable, &this_sse, + bsize, mi->tx_size, rd_computed); + + x->skip = is_skippable; + if (is_skippable) { + this_rdc.rate = skip_cost; + } else { + if (RDCOST(x->rdmult, this_rdc.rate, this_rdc.dist) >= + RDCOST(x->rdmult, 0, this_sse)) { + x->skip = 1; + this_rdc.rate = skip_cost; + this_rdc.dist = this_sse; + } + } + + // TODO(kyslov) account for UV prediction cost + + this_rdc.rate += rate_mv; + const int16_t mode_ctx = + av1_mode_context_analyzer(mbmi_ext->mode_context, mi->ref_frame); + this_rdc.rate += cost_mv_ref(x, this_mode, mode_ctx); + + this_rdc.rate += ref_costs_single[ref_frame]; + + this_rdc.rdcost = RDCOST(x->rdmult, this_rdc.rate, this_rdc.dist); + if (cpi->oxcf.rc_mode == AOM_CBR && cpi->oxcf.speed >= 8) { + newmv_diff_bias(xd, this_mode, &this_rdc, bsize, + frame_mv[this_mode][ref_frame].as_mv.row, + frame_mv[this_mode][ref_frame].as_mv.col, + ref_frame == LAST_FRAME); + } + + mode_checked[this_mode][ref_frame] = 1; + + if (this_rdc.rdcost < best_rdc.rdcost) { + best_rdc = this_rdc; + best_early_term = this_early_term; + best_pickmode.best_mode = this_mode; + best_pickmode.best_pred_filter = mi->interp_filters; + best_pickmode.best_tx_size = mi->tx_size; + best_pickmode.best_ref_frame = ref_frame; + best_pickmode.best_mode_skip_txfm = x->skip; + best_pickmode.best_second_ref_frame = second_ref_frame; + if (reuse_inter_pred) { + free_pred_buffer(best_pickmode.best_pred); + best_pickmode.best_pred = this_mode_pred; + } + } else { + if (reuse_inter_pred) free_pred_buffer(this_mode_pred); + } + if (best_early_term && idx > 0) { + x->skip = 1; + break; + } + } + + mi->mode = best_pickmode.best_mode; + mi->interp_filters = av1_broadcast_interp_filter(best_filter); + mi->tx_size = best_pickmode.best_tx_size; + mi->ref_frame[0] = best_pickmode.best_ref_frame; + mi->mv[0].as_int = + frame_mv[best_pickmode.best_mode][best_pickmode.best_ref_frame].as_int; + mi->ref_frame[1] = best_pickmode.best_second_ref_frame; + pd->dst = orig_dst; + if (reuse_inter_pred && best_pickmode.best_pred != NULL) { + PRED_BUFFER *const best_pred = best_pickmode.best_pred; + if (best_pred->data != orig_dst.buf && is_inter_mode(mi->mode)) { + aom_convolve_copy(best_pred->data, best_pred->stride, pd->dst.buf, + pd->dst.stride, 0, 0, 0, 0, bw, bh); + } + } + + store_coding_context(x, ctx, mi->mode); + *rd_cost = best_rdc; +} diff --git a/third_party/aom/av1/encoder/partition_model_weights.h b/third_party/aom/av1/encoder/partition_model_weights.h index b754c88813..ac2890c158 100644 --- a/third_party/aom/av1/encoder/partition_model_weights.h +++ b/third_party/aom/av1/encoder/partition_model_weights.h @@ -2442,6 +2442,7 @@ static const NN_CONFIG av1_rect_partition_nnconfig_128 = { #undef LABEL_SIZE // Below are the models used for simple_motion_search_based_split +// The model for the faster model static const float av1_simple_motion_search_based_split_thresh_128 = 2.0f; static const float av1_simple_motion_search_based_split_thresh_64 = 2.0f; static const float av1_simple_motion_search_based_split_thresh_32 = 2.0f; @@ -2700,49 +2701,671 @@ static const NN_CONFIG av1_simple_motion_search_based_split_nn_config_16 = { #undef NUM_LAYER_0_UNITS_16 #undef NUM_LOGITS_16 -#if !CONFIG_DISABLE_FULL_PIXEL_SPLIT_8X8 -// BLOCK_8X8 +// Slower higher quality split model +// Threshold from BLOCK_128X128 to BLOCK_8X88 +static const float av1_simple_motion_search_split_midres_thresh[5] = { + 5.839480f, // p = 0.997098 + 1.877167f, // p = 0.867285 + 3.073499f, // p = 0.955783 + 1.405601f, // p = 0.803071 + 2.555636f, // p = 0.927951 +}; + +static const float av1_simple_motion_search_split_midres_no_thresh[5] = { + -3.38168078f, // p = 0.032872917 + -4.08610739f, // p = 0.14393017 + -1.7830237f, // p = 0.15270848 + -100.0f, // p = 0 + -100.0f, // p = 0 +}; + +static const float av1_simple_motion_search_split_lowres_thresh[5] = { + 1.40402595879f, // 0.8028197 + 4.72845183649f, // 0.99123732 + 1.86517797783f, // 0.86589934 + 1.58715223005f, // 0.83021506 + 7.22695596987f, // 0.9992738 +}; + +static const float av1_simple_motion_search_split_lowres_no_thresh[5] = { + -100.0f, -100.0f, -100.0f, -100.0f, -100.0f, +}; + +static const float av1_simple_motion_search_split_mean_128[17] = { + 14.119120f, 14.087010f, 12.016185f, 11.966075f, 12.042454f, 11.994805f, + 12.152105f, 12.100394f, 12.178377f, 12.128937f, 4.779944f, 0.714786f, + 3.535450f, 3.566207f, 0.835913f, 3.315452f, 3.302908f, +}; + +static const float av1_simple_motion_search_split_std_128[17] = { + 1.832420f, 1.835338f, 2.019207f, 2.020793f, 2.008731f, 2.008403f, + 1.900999f, 1.907081f, 1.908915f, 1.913122f, 2.109345f, 0.451517f, + 1.407097f, 1.372501f, 0.370355f, 1.321495f, 1.319665f, +}; + +static const float av1_simple_motion_search_split_mean_64[17] = { + 12.363721f, 12.314348f, 10.404341f, 10.333541f, 10.405775f, 10.336996f, + 10.402246f, 10.330084f, 10.405584f, 10.334330f, 4.554232f, 0.896393f, + 2.819613f, 2.855845f, 0.926296f, 2.808782f, 2.798229f, +}; + +static const float av1_simple_motion_search_split_std_64[17] = { + 1.878920f, 1.882255f, 1.950167f, 1.953289f, 1.913869f, 1.914781f, + 1.920096f, 1.924454f, 1.880200f, 1.882499f, 2.050922f, 0.304750f, + 1.144391f, 1.125088f, 0.261289f, 1.145059f, 1.131215f, +}; + +static const float av1_simple_motion_search_split_mean_32[17] = { + 10.750278f, 10.679627f, 8.745625f, 8.644149f, 8.757436f, 8.656657f, + 8.759780f, 8.656299f, 8.772563f, 8.669839f, 4.208026f, 0.958573f, + 2.308769f, 2.347375f, 0.961685f, 2.323464f, 2.296322f, +}; + +static const float av1_simple_motion_search_split_std_32[17] = { + 1.879269f, 1.883531f, 1.935828f, 1.935677f, 1.915823f, 1.914773f, + 1.909733f, 1.910315f, 1.890451f, 1.890032f, 1.913318f, 0.199276f, + 0.988825f, 0.972115f, 0.191956f, 0.977131f, 0.951418f, +}; + +static const float av1_simple_motion_search_split_mean_16[17] = { + 9.076768f, 8.974986f, 7.078364f, 6.926072f, 7.088739f, 6.936111f, + 7.096697f, 6.942841f, 7.114978f, 6.961046f, 3.865480f, 0.982632f, + 1.886023f, 1.912892f, 0.981492f, 1.926059f, 1.891233f, +}; + +static const float av1_simple_motion_search_split_std_16[17] = { + 1.922965f, 1.925609f, 1.851980f, 1.847558f, 1.848410f, 1.843990f, + 1.843931f, 1.839582f, 1.840304f, 1.836144f, 1.760042f, 0.130639f, + 0.841086f, 0.833523f, 0.134780f, 0.840790f, 0.831309f, +}; + +static const float av1_simple_motion_search_split_mean_8[17] = { + 7.120238f, 6.957731f, 5.176309f, 4.889594f, 5.178396f, 4.886607f, + 5.195322f, 4.905566f, 5.198845f, 4.904745f, 3.648933f, 0.993198f, + 1.496831f, 1.520804f, 0.991864f, 1.489763f, 1.460761f, +}; + +static const float av1_simple_motion_search_split_std_8[17] = { + 1.698498f, 1.696000f, 1.629605f, 1.614641f, 1.632476f, 1.618831f, + 1.618352f, 1.603742f, 1.623089f, 1.609674f, 1.668587f, 0.082193f, + 0.759407f, 0.759684f, 0.089830f, 0.742797f, 0.730632f, +}; + +static const float *const av1_simple_motion_search_split_mean[5] = { + av1_simple_motion_search_split_mean_128, + av1_simple_motion_search_split_mean_64, + av1_simple_motion_search_split_mean_32, + av1_simple_motion_search_split_mean_16, + av1_simple_motion_search_split_mean_8, +}; + +static const float *const av1_simple_motion_search_split_std[5] = { + av1_simple_motion_search_split_std_128, av1_simple_motion_search_split_std_64, + av1_simple_motion_search_split_std_32, av1_simple_motion_search_split_std_16, + av1_simple_motion_search_split_std_8, +}; + +#define NUM_HIDDEN_LAYERS_128 1 +#define NUM_FEATURES_128 17 +#define NUM_LAYER_0_UNITS_128 20 +#define NUM_LOGITS_128 1 + +static const float av1_simple_motion_search_split_hiddenlayer_0_kernel_128[] = { + 0.24095f, -0.397761f, -0.388619f, -0.0629548f, -0.44577f, 0.688212f, + -0.20889f, -1.08227f, -0.0313894f, -0.615505f, -0.401839f, 0.40233f, + -0.171305f, 0.439803f, 1.58527f, -0.968535f, -1.29255f, 1.14846f, + 0.885777f, 0.116412f, -0.225704f, 0.316506f, 0.793951f, -0.63591f, + 0.097789f, -0.327027f, -0.778396f, -0.231667f, -0.9622f, 1.0044f, + 0.32594f, 0.179768f, -0.115529f, -0.499395f, -1.14727f, -1.26111f, + 0.269818f, -0.0882028f, -0.349107f, 0.100901f, 0.0249506f, 0.528929f, + 0.113961f, 0.929794f, 0.242494f, -0.122828f, -0.0477379f, 0.170659f, + 0.0500187f, 0.28859f, 0.78783f, 0.482412f, 0.795298f, 0.179517f, + 0.453911f, -0.298029f, -0.903332f, 0.510615f, 0.691994f, 0.433383f, + -0.140802f, -1.11635f, -0.547326f, 1.11318f, 0.71905f, 0.978538f, + 0.097444f, -0.0386012f, 0.713599f, 0.465164f, 0.391278f, -0.472864f, + 0.230224f, -0.279508f, 0.558192f, -0.468625f, 0.55995f, -0.57507f, + -1.39947f, -0.755819f, -1.04512f, -0.411552f, -0.830444f, -0.106571f, + -0.0972184f, 0.251842f, 0.269955f, 0.230492f, -0.290581f, -0.484799f, + 0.0151041f, 0.171047f, 0.829999f, -0.384581f, 0.220301f, -0.121687f, + 1.88848f, -0.482809f, -0.48185f, 1.34482f, -0.716438f, -0.284482f, + -1.78592f, -1.29333f, 0.886867f, 0.80106f, 0.456415f, 0.649095f, + 0.231093f, 0.361562f, 0.290018f, 0.128009f, -0.196343f, 0.0607802f, + 0.576761f, -0.0413836f, 0.0300984f, -0.318998f, 0.204434f, -0.712524f, + 0.833394f, -0.81168f, 0.765488f, -0.720973f, 1.12866f, -0.838694f, + 1.295f, -0.159127f, 1.05404f, 0.736519f, 0.248662f, 0.229233f, + 0.0434302f, 0.0551856f, 0.197862f, 0.354823f, -0.32429f, -0.227353f, + -0.132198f, -0.438118f, -0.210401f, -0.81046f, 0.653555f, 0.826737f, + 0.154235f, 0.228945f, 0.123089f, 0.614964f, -0.0940471f, -0.00676807f, + 0.24996f, 0.949233f, 0.746526f, -0.044474f, 0.386414f, 0.503221f, + 0.155133f, -0.698848f, -0.735356f, -0.255091f, 0.413235f, -0.335295f, + -0.145757f, 0.326299f, -0.602629f, -0.844474f, -0.346722f, -0.42598f, + -0.491016f, -0.447732f, -0.965366f, -0.0242841f, 0.836606f, -0.104877f, + 1.23236f, 0.683986f, 0.787005f, -0.0253437f, 1.2145f, 1.29554f, + -1.24302f, -0.229495f, 0.439415f, 0.885087f, -0.408704f, -0.119299f, + -0.0960972f, 0.60148f, 0.683271f, -0.057129f, -0.180295f, -0.264815f, + -0.363184f, 0.638271f, 0.631083f, -0.252899f, -0.164364f, -1.31274f, + 0.354408f, 0.0429172f, 0.371154f, -1.0978f, 0.0433642f, -0.467394f, + -0.706572f, 1.57198f, -0.0701271f, 1.93149f, -0.446267f, 1.4519f, + -1.29567f, 0.309978f, -0.878062f, 0.891494f, 0.364005f, -0.209611f, + -0.125927f, 0.184097f, 0.0629695f, -0.43375f, -0.0980562f, 1.08547f, + 0.578312f, 0.16566f, -0.198852f, -0.241854f, -0.523934f, -0.206037f, + -0.867721f, 1.00041f, 1.09848f, -2.12562f, -0.19992f, -0.186128f, + -0.03507f, 0.0484884f, 0.160856f, 0.10802f, -0.805141f, -1.06902f, + 0.290363f, 0.0222096f, -0.849266f, 0.112932f, 0.148682f, -0.0457585f, + 1.139f, 1.79141f, 0.194122f, -0.342508f, -0.403572f, 0.133678f, + 0.217553f, -0.263759f, 0.18441f, 0.254529f, 0.0471115f, 0.733178f, + -0.416205f, 0.441447f, -0.443335f, 0.725005f, -0.78946f, 0.71301f, + -0.644969f, 1.5445f, 0.365277f, -0.455775f, -0.365066f, 0.4742f, + -0.381714f, -0.545794f, -0.0464861f, -0.222768f, -0.0106466f, -0.069743f, + 0.0335566f, 0.378348f, -0.249663f, 0.922286f, 0.125711f, -0.894619f, + 0.444682f, 0.447893f, -1.98936f, -1.41978f, 0.0406667f, -0.199928f, + -0.199786f, 0.463481f, 0.334931f, -0.396222f, -0.0732259f, 0.796684f, + -0.140817f, -0.26878f, 0.194642f, 0.895784f, -0.369976f, -2.26981f, + -0.0791776f, -0.0492268f, 0.6715f, 0.281805f, 0.0156664f, -0.779785f, + 0.17743f, 0.188786f, -0.588077f, -0.359153f, 0.258319f, 0.881688f, + 0.846894f, 1.00292f, 0.838134f, 0.680632f, 0.273098f, -0.329261f, + 0.217757f, -0.506726f, -0.336523f, -0.695875f, -0.252006f, 0.751216f, + 0.334409f, -0.0151467f, 0.0885474f, 0.0973114f, -0.248754f, -0.263716f, + 0.369906f, -0.213749f, -0.0355395f, -0.137799f, 2.43233f, -0.944233f, + -0.745167f, 0.318558f, 0.316608f, 0.568678f +}; + +static const float av1_simple_motion_search_split_hiddenlayer_0_bias_128[] = { + 0.821344f, 1.11542f, -1.24172f, 1.03642f, 1.13511f, + 1.16414f, -0.278655f, -1.35558f, -1.26788f, -1.63189f, + -0.323271f, 1.21319f, -0.888415f, 0.987145f, -1.16767f, + 0.255833f, -0.1392f, 1.43265f, -1.54952f, 1.65159f +}; + +static const float av1_simple_motion_search_split_logits_kernel_128[] = { + 0.3565753f, 0.5490161f, -1.015597f, 0.565366f, 0.751604f, + 0.922747f, -1.931846f, 1.759353f, -0.7362949f, 0.5707034f, + -1.092127f, 0.936767f, 2.034499f, 2.08148f, 0.9509507f, + -1.342504f, -0.834566f, 0.618184f, 0.844113f, 1.182693f +}; + +static const float av1_simple_motion_search_split_logits_bias_128[] = { + 1.819351f +}; + +static const NN_CONFIG av1_simple_motion_search_split_nn_config_128 = { + NUM_FEATURES_128, + NUM_LOGITS_128, + NUM_HIDDEN_LAYERS_128, + { + NUM_LAYER_0_UNITS_128, + }, + { + av1_simple_motion_search_split_hiddenlayer_0_kernel_128, + av1_simple_motion_search_split_logits_kernel_128, + }, + { + av1_simple_motion_search_split_hiddenlayer_0_bias_128, + av1_simple_motion_search_split_logits_bias_128, + }, +}; + +#undef NUM_HIDDEN_LAYERS_128 +#undef NUM_FEATURES_128 +#undef NUM_LAYER_0_UNITS_128 +#undef NUM_LOGITS_128 + +#define NUM_HIDDEN_LAYERS_64 1 +#define NUM_FEATURES_64 17 +#define NUM_LAYER_0_UNITS_64 24 +#define NUM_LOGITS_64 1 + +static const float av1_simple_motion_search_split_hiddenlayer_0_kernel_64[] = { + -1.40663f, -0.851503f, -0.0613111f, 0.741591f, 0.302754f, + 0.184001f, 0.0474853f, 0.371096f, 0.0541624f, 0.381508f, + 0.355427f, 0.0428822f, 0.154916f, -0.00490099f, 0.025484f, + 0.0208921f, 0.140596f, -0.292525f, -0.459067f, -0.081393f, + 0.109824f, -0.290183f, 0.720236f, 0.385835f, -0.150643f, + -0.078518f, 0.0979819f, -0.102135f, 0.137152f, -0.0786457f, + 0.0171441f, 0.991338f, -0.546583f, -1.0714f, -0.0842851f, + 0.244072f, 0.427379f, 0.146775f, -0.921613f, -0.912093f, + 0.393566f, -0.232375f, 0.19963f, 0.312355f, 0.55659f, + -0.104714f, -0.137563f, 0.0985237f, 0.0788307f, -0.225514f, + 0.0228832f, -0.288733f, -0.00737685f, -0.711657f, -0.256796f, + 0.0869605f, 0.583977f, 0.384306f, 1.46692f, -0.741126f, + -0.21105f, -0.276604f, -0.0151463f, -0.0227997f, -0.0403232f, + 0.044122f, 0.0185784f, -0.0451951f, 0.00489513f, -0.387131f, + 0.0966724f, -0.599174f, -0.00243351f, -0.21439f, 0.302043f, + 0.130334f, -0.191251f, 0.863261f, -1.50112f, 0.00901057f, + 0.000324294f, -0.0572545f, 0.0117685f, -0.0734682f, -0.0570435f, + -0.126253f, 1.2313f, -0.328267f, 0.211788f, -0.175438f, + -0.0419298f, 0.166447f, -0.178739f, -0.326221f, -0.0439188f, + 1.01182f, -0.390678f, -0.426343f, 0.0944665f, -0.225042f, + -0.183344f, 0.0500763f, -0.377393f, -0.673401f, -0.436907f, + -0.00366876f, -0.363412f, 0.195194f, 0.250248f, -0.397193f, + -0.0917222f, -0.0221579f, 1.7693f, -0.0694484f, -0.0410764f, + -0.134571f, -0.159992f, -0.170359f, -0.249333f, -0.128056f, + -0.617054f, -0.808701f, -0.540642f, 0.396391f, 0.147787f, + 0.346916f, 0.709852f, 0.116064f, 0.0509731f, 0.073713f, + -0.365082f, -1.09287f, -0.618214f, 0.20545f, 0.126161f, + -0.140012f, 0.62592f, 0.316326f, -0.392765f, -0.15934f, + 0.337617f, -0.41669f, -0.295225f, 0.0602025f, -0.0150657f, + -0.319629f, 0.783729f, -0.0661199f, -0.362657f, 0.390042f, + -0.043614f, -0.0414596f, 0.121155f, -0.309775f, -0.284761f, + -0.243932f, 0.279855f, -0.266823f, 0.734824f, -0.164028f, + 0.261776f, -0.105585f, 0.10733f, -0.180469f, 1.18875f, + -1.12836f, -0.173008f, 0.150221f, 0.111598f, 0.148306f, + -1.2833f, -1.06346f, 0.233546f, 0.16432f, 0.00142378f, + 0.340574f, -0.0140885f, 0.634761f, -0.122096f, 0.821487f, + 0.421424f, -0.0256687f, -0.035503f, -0.0453547f, -0.0215179f, + -0.0671277f, -0.0486862f, -0.962761f, -0.208383f, 0.109573f, + -0.210668f, -0.176485f, 0.421279f, 0.41605f, 0.342084f, + 0.619364f, 0.103718f, -0.00341643f, 0.00266677f, 0.249089f, + -0.22848f, -0.0368968f, 1.12092f, -0.64912f, -0.456579f, + 0.477823f, 0.418345f, 1.41515f, 0.0936279f, 0.886155f, + -0.785656f, -0.217109f, -0.561829f, -0.286435f, -0.884068f, + -0.148839f, -0.282848f, 0.0683745f, 0.0962815f, -0.111975f, + 0.0509158f, -0.211274f, 0.744909f, -0.8982f, 0.315232f, + -0.78624f, 0.598387f, -0.530952f, 0.677357f, 0.0371339f, + 0.99209f, -0.681899f, -0.291416f, -0.224822f, -0.26049f, + -0.0436525f, -0.380004f, -0.27187f, 0.534779f, 0.717939f, + 0.418197f, -0.152539f, -0.0684039f, -0.186308f, -0.0653121f, + 0.194145f, -0.196367f, 0.256997f, -0.726269f, -0.307672f, + -0.153362f, 0.450827f, 0.708842f, -0.0667079f, 0.555564f, + 0.0486892f, 0.0715072f, -0.7211f, -0.849797f, 0.0650271f, + 1.2747f, -0.646738f, -0.53042f, 0.182197f, 0.928203f, + 0.180621f, -0.00640791f, -0.171416f, 0.092688f, -0.391275f, + -0.0650657f, 0.0843773f, 0.170824f, 0.378085f, 0.0596657f, + 0.844398f, -1.3083f, -1.27828f, -0.199179f, 0.557855f, + 0.241479f, 0.385804f, 0.169533f, -0.0028072f, 0.0538041f, + 0.00136234f, 0.0130481f, 0.0349449f, -0.0366494f, -0.000474055f, + 0.437956f, 0.286724f, -0.298187f, 0.461967f, 0.43065f, + -0.0877194f, -0.19133f, 0.379121f, -0.687751f, -1.64077f, + -0.375191f, -0.336836f, -0.323904f, -0.101859f, 0.0126672f, + -0.346332f, 0.112303f, -0.863336f, 0.155538f, 0.366509f, + -0.0976829f, 0.635278f, -0.681967f, -0.527729f, 0.591839f, + 0.366678f, 0.189981f, 0.0208007f, -0.565809f, 0.70183f, + -0.282844f, -0.327485f, 0.347243f, -1.13014f, -0.373378f, + -0.514978f, 0.662994f, -0.144931f, 0.1402f, -0.820049f, + 0.711498f, 0.681156f, 1.06515f, -0.423409f, -0.0392664f, + 0.0675396f, -0.0508602f, 0.0431443f, 0.0212639f, -0.0279887f, + -0.62611f, -0.202064f, 0.701934f, 1.28452f, -0.00858481f, + -0.517249f, 0.0615832f, -0.260215f, 0.0949119f, -0.28423f, + -0.39573f, -0.0574246f, -0.318658f, 0.0601775f, -0.0629386f, + -0.134208f, 0.111686f, -0.23355f, 0.078667f, 0.741023f, + 0.828523f, -0.345067f, -0.315135f, -0.0957154f, 0.522825f, + -0.190057f, -0.473789f, -0.390489f, 0.200677f, -0.0271802f, + 0.110336f, 0.493302f, 0.663126f, 0.570148f, -0.380042f, + -0.437349f, -0.660884f, 0.301908f, 0.0644179f, 0.172494f, + 0.461917f, 0.330938f, -0.140041f, -0.0430205f, -1.51003f, + -0.410984f, -0.182161f, 0.0235313f, -0.364849f, 0.154183f, + -0.592465f, 0.272701f, 0.192389f, -0.0497777f, -0.924467f, + -0.179513f, -0.592217f, 0.436363f, -0.0716164f, 0.189094f, + -0.574697f, -0.304303f, 0.326441f, -0.0865553f, 0.735948f, + 0.266912f, 0.435824f, -0.123322f +}; + +static const float av1_simple_motion_search_split_hiddenlayer_0_bias_64[] = { + -1.19333f, 1.01834f, -1.10844f, 0.0454873f, -1.45506f, 0.580864f, + -0.040979f, -0.505681f, -1.15072f, 0.692697f, -0.520812f, -0.479384f, + 0.529652f, 0.507252f, -1.08619f, 0.0586375f, 0.0929614f, -0.46753f, + -0.701857f, -0.362933f, -0.291983f, -0.133933f, -0.0131351f, -0.267582f +}; + +static const float av1_simple_motion_search_split_logits_kernel_64[] = { + -3.32501f, 0.43082f, -1.060692f, 1.328908f, 0.8892894f, 0.6488833f, + -1.096516f, -0.664786f, -1.301339f, 0.508805f, -2.128406f, -0.757304f, + 0.383839f, 0.694763f, -0.591725f, 0.770385f, 1.021594f, 0.589181f, + -0.76238f, 1.488826f, 0.709135f, -0.575738f, 0.26421759f, -0.2484219f +}; + +static const float av1_simple_motion_search_split_logits_bias_64[] = { + 0.699037f +}; + +static const NN_CONFIG av1_simple_motion_search_split_nn_config_64 = { + NUM_FEATURES_64, + NUM_LOGITS_64, + NUM_HIDDEN_LAYERS_64, + { + NUM_LAYER_0_UNITS_64, + }, + { + av1_simple_motion_search_split_hiddenlayer_0_kernel_64, + av1_simple_motion_search_split_logits_kernel_64, + }, + { + av1_simple_motion_search_split_hiddenlayer_0_bias_64, + av1_simple_motion_search_split_logits_bias_64, + }, +}; + +#undef NUM_HIDDEN_LAYERS_64 +#undef NUM_FEATURES_64 +#undef NUM_LAYER_0_UNITS_64 +#undef NUM_LOGITS_64 + +#define NUM_HIDDEN_LAYERS_32 1 +#define NUM_FEATURES_32 17 +#define NUM_LAYER_0_UNITS_32 20 +#define NUM_LOGITS_32 1 + +static const float av1_simple_motion_search_split_hiddenlayer_0_kernel_32[] = { + -0.980626f, -0.946611f, 0.103761f, 0.408899f, 0.498149f, + 0.0490161f, 0.253279f, 0.332029f, 0.00367441f, 0.364401f, + -0.236433f, 0.0592119f, -0.0978848f, 0.159733f, -0.018052f, + -1.10726f, 1.16167f, -0.244982f, -0.147819f, -0.147095f, + 0.111404f, -0.349502f, 0.441178f, 0.0984191f, -0.135537f, + -0.0423312f, 0.0123079f, 0.358012f, -0.266796f, 0.0125811f, + 0.196563f, 0.337093f, -1.07266f, -1.25134f, 0.57337f, + -0.521717f, 0.259824f, 0.537383f, -0.463688f, -0.336128f, + 0.373385f, 0.483443f, -0.229293f, -0.33373f, -0.656021f, + 0.768647f, 0.179279f, 0.315415f, 0.187749f, 1.07839f, + 0.0626629f, -0.230299f, 0.662606f, -0.414154f, 0.459334f, + -0.6312f, 0.427704f, -0.249849f, 0.701056f, -0.707969f, + 0.057401f, 0.620434f, 0.665748f, -0.501356f, -0.230685f, + 0.0722371f, -0.0988625f, -0.114035f, -0.653799f, 0.571353f, + 0.268276f, 1.13251f, -1.0695f, -0.225607f, -0.984355f, + -0.42213f, 0.300422f, 1.21492f, -0.139931f, -0.000726004f, + 0.045964f, -0.0817352f, -0.0278813f, -0.0102341f, -0.0144087f, + -0.475882f, 1.20682f, -0.359919f, 0.277189f, -0.166401f, + 0.599211f, -0.129872f, 0.574211f, -0.247573f, 0.824405f, + -1.53329f, -0.202151f, -0.328698f, -0.516322f, -0.281416f, + -0.383651f, -0.252862f, -0.43185f, 0.456802f, -0.430055f, + -0.55245f, -0.6884f, -0.541456f, -0.281376f, 1.10425f, + -0.140706f, 1.59816f, -0.0343895f, -0.00920039f, -0.0307667f, + 0.0560132f, -0.0340302f, -0.10848f, 0.0593314f, -0.951795f, + 0.876831f, -1.00548f, -0.566244f, 0.430061f, 1.10109f, + -0.634212f, -0.0755369f, -0.108953f, 1.03191f, 0.109036f, + -0.0415309f, 0.0681162f, -0.0611775f, -0.0231938f, 0.0973158f, + -0.0558169f, -0.823484f, -0.918509f, 0.16756f, 0.27087f, + 0.286074f, 0.174069f, 0.1304f, 0.386074f, 0.433953f, + 0.0291467f, -1.74087f, 0.0296094f, -0.00793714f, -0.13041f, + 0.00990992f, -0.0137848f, -0.0742606f, -0.251029f, -0.645316f, + 0.640029f, 0.550607f, 0.470097f, 0.549451f, -0.285723f, + -0.164759f, -0.128166f, -0.391496f, -0.80287f, 0.0769472f, + 1.34391f, 0.0215005f, 0.0669497f, 0.131919f, 0.291674f, + 0.0952889f, -0.677953f, -0.364054f, 0.144823f, 0.246198f, + -0.12393f, 0.363661f, 0.215091f, -0.239658f, 0.18491f, + 0.118703f, 0.0064156f, 1.38619f, -1.3845f, 0.0567323f, + 1.20812f, -0.720374f, -1.92158f, -1.48657f, 0.335601f, + 0.409379f, 0.373618f, 0.231274f, 0.292194f, 0.368619f, + 0.2398f, 0.473579f, 0.83402f, -0.0133751f, -0.00344358f, + 2.20688e-05f, 0.00836757f, 0.00405377f, 0.0110539f, -0.260154f, + 0.192112f, -0.666986f, 0.302875f, -0.113302f, 0.17882f, + -0.221493f, 0.146161f, -0.448697f, 0.584187f, 0.122109f, + 0.989981f, -1.14706f, -0.734042f, 0.0638213f, 0.213357f, + 0.068543f, -0.808558f, 0.404741f, 0.808313f, 1.57523f, + -0.113448f, 0.254102f, -0.350065f, -0.615f, 0.0753549f, + -0.540936f, -0.0250732f, -0.225681f, -0.161384f, 0.0128342f, + -0.0933368f, -0.286904f, 0.130133f, -0.874747f, 0.392585f, + -0.493135f, 0.169708f, 0.0909804f, 1.89921f, -0.469954f, + 0.65165f, -0.953401f, -0.21595f, -0.37479f, 0.0451146f, + 0.0234621f, -0.0596903f, -0.0682308f, -0.0830426f, 0.130011f, + -0.409141f, 0.0627038f, -0.581148f, -0.513922f, 0.631676f, + 0.0637034f, 0.0539081f, 0.0638872f, 0.515863f, -0.0123463f, + 0.177238f, 0.279506f, -0.930345f, 1.23726f, 0.202851f, + 0.708792f, -0.445086f, -0.0267075f, -0.913822f, -0.0714978f, + -0.281107f, -0.0770565f, -0.23086f, -0.165893f, -0.319683f, + 0.216235f, -0.490999f, 2.04841f, -0.0524071f, -0.239043f, + -0.0526375f, 0.023002f, -0.132685f, -0.155354f, -0.186503f, + -0.904296f, 0.166478f, 0.063268f, -0.302842f, -0.27179f, + -0.428299f, 0.50193f, 0.480717f, -0.864275f, 0.317096f, + 0.40698f, 0.0286107f, 0.189432f, -0.0374374f, 0.0671728f, + 0.203681f, -0.457959f, -0.155776f, 0.340948f, 0.542841f, + 0.342675f, -0.000952399f, 0.470957f, 0.744418f, -1.11763f, + -0.658812f, -0.044832f, 0.0688237f, -0.357766f, 0.428662f, + -0.087152f, -0.291903f, 0.373244f, -0.587853f, 0.415895f, + -0.535694f, 0.621785f, -0.143648f, 0.0451373f, 0.00068827f, + 1.84432f, -1.26239f, -0.432087f, -0.152307f, 0.0293551f, + 0.184744f, -0.0173156f, -0.00572154f, -0.0305062f, -0.0900071f +}; + +static const float av1_simple_motion_search_split_hiddenlayer_0_bias_32[] = { + 0.160011f, 0.903856f, -0.13738f, 0.358221f, -0.0906044f, + -0.606558f, -0.0215651f, -0.03377f, -1.67017f, -0.144554f, + -0.201482f, -0.87719f, 0.639815f, -0.51976f, -0.309922f, + -1.33421f, 0.721328f, -0.889354f, -1.7158f, -0.285963f +}; + +static const float av1_simple_motion_search_split_logits_kernel_32[] = { + -0.2745374f, 0.333548f, -0.2437388f, 0.288009f, 0.55635f, + 0.4560176f, 0.2970518f, 0.391192f, 1.311854f, -0.231219f, + -0.2968651f, -1.819984f, 0.2775824f, 0.28929857f, 0.419126f, + -0.32868411f, -0.916399f, -0.1921077f, -0.617489f, 0.637953f +}; + +static const float av1_simple_motion_search_split_logits_bias_32[] = { + 0.208473f +}; + +static const NN_CONFIG av1_simple_motion_search_split_nn_config_32 = { + NUM_FEATURES_32, + NUM_LOGITS_32, + NUM_HIDDEN_LAYERS_32, + { + NUM_LAYER_0_UNITS_32, + }, + { + av1_simple_motion_search_split_hiddenlayer_0_kernel_32, + av1_simple_motion_search_split_logits_kernel_32, + }, + { + av1_simple_motion_search_split_hiddenlayer_0_bias_32, + av1_simple_motion_search_split_logits_bias_32, + }, +}; + +#undef NUM_HIDDEN_LAYERS_32 +#undef NUM_FEATURES_32 +#undef NUM_LAYER_0_UNITS_32 +#undef NUM_LOGITS_32 + +#define NUM_HIDDEN_LAYERS_16 1 +#define NUM_FEATURES_16 17 +#define NUM_LAYER_0_UNITS_16 20 +#define NUM_LOGITS_16 1 + +static const float av1_simple_motion_search_split_hiddenlayer_0_kernel_16[] = { + 0.0136957f, 0.182135f, -0.583394f, 0.0556956f, 0.211152f, + 0.168234f, -0.694203f, -0.678216f, 0.289943f, 1.00014f, + -0.0427784f, -0.0427538f, -0.0276009f, -0.00133608f, 0.0901944f, + 0.0674892f, 0.104068f, -0.308582f, -0.43596f, 0.855997f, + -0.223414f, 0.0390026f, 0.366492f, 0.216065f, -0.386863f, + -0.148823f, -0.297022f, 0.0529546f, -0.202885f, 1.26471f, + -0.861163f, -0.0949431f, 0.573627f, -0.00277083f, -0.616063f, + -0.626927f, 0.371583f, -0.411743f, 0.173387f, -0.209734f, + 0.293697f, -0.260714f, 0.442728f, -0.594486f, 1.38987f, + 0.208025f, -0.0433776f, 0.01173f, 0.921766f, -0.168379f, + 0.000697326f, 0.209967f, -0.304577f, 0.149551f, -0.196658f, + 0.389251f, -0.449106f, -0.456329f, 0.669073f, -0.163806f, + 0.083348f, -0.0783998f, 0.0678355f, 0.0510435f, 0.103964f, + 0.104537f, -0.778093f, -1.0641f, -0.626102f, -2.02131f, + 0.159591f, 0.254161f, -0.000362642f, 0.289859f, 0.192713f, + 0.139801f, -0.0251327f, 0.164002f, 1.22892f, -0.0852193f, + 0.0769487f, 0.0296408f, -0.0418688f, 0.0936023f, 0.0448523f, + 0.674015f, -0.0732944f, 0.313575f, -0.593432f, 0.642067f, + -1.06063f, 0.468223f, -0.769085f, -0.173798f, -0.175663f, + 0.692808f, 0.00753295f, -0.123327f, -0.0234937f, -0.0923153f, + 0.0216917f, -0.0690157f, -0.397488f, 0.426628f, 0.264475f, + 0.342074f, -0.139817f, 0.215915f, 0.422544f, -0.321102f, + 0.0355587f, 0.460193f, 0.0315326f, 0.080556f, -0.0256533f, + -0.0857874f, -0.488283f, -0.299653f, -0.245987f, 0.104383f, + 0.203731f, 0.328734f, 0.668104f, -0.586909f, -0.501335f, + -0.661292f, -0.359811f, 0.00951363f, 0.816315f, -0.0124104f, + 0.0545827f, 0.089863f, 0.0125486f, 0.043609f, -0.0259544f, + 0.0123911f, 0.12557f, -0.539875f, -0.0556721f, 0.16532f, + 0.265834f, -0.384171f, 0.646496f, 0.366147f, -0.111272f, + 0.262096f, -0.0845724f, 0.382724f, 0.165783f, 0.1025f, + 0.392988f, 0.290525f, 0.038659f, 0.540269f, -0.485586f, + -0.273065f, -0.154052f, -0.0896895f, -0.35394f, 0.193214f, + -0.423728f, 0.654576f, -0.373321f, 0.814914f, 0.026278f, + -0.0328304f, -0.220913f, -0.0442121f, 0.487545f, -0.509537f, + -0.777581f, -1.23886f, 0.223482f, 0.206009f, 0.20391f, + 0.194628f, 0.226762f, 0.171609f, -0.219037f, 0.557892f, + -0.312011f, 1.27709f, 0.064013f, 0.105384f, 0.0493933f, + 0.074059f, -0.0100078f, -0.0176888f, -0.440005f, 0.302922f, + -0.197456f, 0.296128f, -0.326647f, 0.305323f, -0.30696f, + 0.201951f, -0.15874f, -0.793042f, 0.0197254f, 0.0569867f, + -0.0295468f, -0.0215012f, 0.025855f, -0.0196102f, 0.215558f, + -0.253069f, 0.298469f, 0.261269f, 0.435305f, 0.0120354f, + -0.384789f, -0.2772f, 0.0366613f, -0.494994f, 0.149072f, + 1.32981f, -0.427717f, 0.43938f, -0.16375f, -0.444342f, + 0.548214f, 0.127955f, -1.24387f, 0.0863676f, 0.175071f, + 0.172673f, -0.0906204f, 0.444454f, -0.546669f, 0.215857f, + -0.100621f, 0.200699f, -0.0985915f, 0.134706f, -0.256396f, + 0.393427f, 0.119606f, -0.214278f, -0.0183637f, 0.194266f, + -0.238025f, 0.182203f, 0.599718f, 0.846933f, 0.0607852f, + -0.183434f, -0.723743f, -0.72414f, -0.124701f, 0.0227527f, + -0.0664636f, -0.0385867f, -0.0257377f, -0.149054f, 0.12077f, + 0.678029f, -0.624456f, 0.189644f, -0.518604f, 0.134397f, + -0.189777f, -0.309376f, -0.00377086f, 0.701132f, -0.170915f, + 0.00736111f, -0.121906f, 0.329136f, 0.165514f, 0.0328356f, + 0.171275f, 0.248619f, 0.247704f, -0.449933f, 0.0841684f, + 0.136982f, 0.122703f, -0.0169439f, -0.0726496f, 0.302648f, + -0.128556f, 0.0667425f, -0.289717f, -0.207532f, -1.20269f, + -0.68892f, 0.045259f, 0.0973945f, 0.0988314f, -0.944748f, + -0.180401f, 0.134331f, 0.033834f, 0.109023f, 0.265723f, + 0.38063f, -0.106518f, -0.0686953f, 0.3744f, -1.0957f, + 0.0302782f, 0.0515164f, 0.00188222f, 0.0014413f, -0.0404425f, + 0.0124618f, -0.0828645f, 0.506166f, -0.776352f, -0.405138f, + -0.123887f, 0.0732116f, 0.379928f, 0.604524f, -0.492317f, + 0.439191f, 0.0744193f, 0.389101f, 0.0604518f, 0.0943165f, + 0.0339942f, 0.0917975f, 0.0161988f, 0.512227f, 0.538021f, + -0.411495f, 0.307281f, 0.33746f, -0.218639f, 0.265742f, + 0.39738f, -0.12442f, 0.125236f, -0.0845223f, -0.150396f, + 0.0334878f, -0.00391915f, 0.0406864f, -0.0487059f, 0.0377073f +}; + +static const float av1_simple_motion_search_split_hiddenlayer_0_bias_16[] = { + 0.0535976f, -0.0130279f, 0.150146f, -0.511132f, -0.357698f, + 0.6719f, -1.27877f, -0.0208048f, 0.0961914f, 0.263603f, + 0.704574f, -1.48998f, 0.728063f, 0.941829f, -0.199981f, + 0.797802f, -0.29816f, -0.60894f, -0.116624f, -1.16723f +}; + +static const float av1_simple_motion_search_split_logits_kernel_16[] = { + 0.343153f, -0.2110482f, -0.487199f, 0.3274144f, -2.1975f, + -0.6051438f, 0.1901127f, 0.4741924f, -0.24029f, -0.185018f, + -0.652635f, 2.57714f, -0.31033031f, -0.307222f, 0.329035f, + -0.430181f, 0.3429f, 0.742292f, 0.3269808f, 0.4142165f +}; + +static const float av1_simple_motion_search_split_logits_bias_16[] = { + -0.783658f +}; + +static const NN_CONFIG av1_simple_motion_search_split_nn_config_16 = { + NUM_FEATURES_16, + NUM_LOGITS_16, + NUM_HIDDEN_LAYERS_16, + { + NUM_LAYER_0_UNITS_16, + }, + { + av1_simple_motion_search_split_hiddenlayer_0_kernel_16, + av1_simple_motion_search_split_logits_kernel_16, + }, + { + av1_simple_motion_search_split_hiddenlayer_0_bias_16, + av1_simple_motion_search_split_logits_bias_16, + }, +}; + +#undef NUM_HIDDEN_LAYERS_16 +#undef NUM_FEATURES_16 +#undef NUM_LAYER_0_UNITS_16 +#undef NUM_LOGITS_16 + #define NUM_HIDDEN_LAYERS_8 1 -#define NUM_FEATURES_8 6 -#define NUM_LAYER_0_UNITS_8 16 +#define NUM_FEATURES_8 17 +#define NUM_LAYER_0_UNITS_8 20 #define NUM_LOGITS_8 1 -static const float av1_simple_motion_search_based_split_layer_0_kernel_8[] = { - 0.0370236f, -0.580211f, 2.0134f, 1.69637f, 2.43181f, -0.521648f, - -0.00375187f, 0.122712f, -4.74411f, 7.36187f, 5.42574f, -5.53557f, - 0.0993344f, -0.358843f, 0.0765453f, -0.615987f, -0.754633f, -0.175846f, - 0.714976f, 0.492862f, 0.346604f, -1.23922f, -2.67031f, 2.12749f, - 1.71511f, -1.4239f, 2.09396f, 2.42478f, 2.40151f, 2.90487f, - 0.540813f, -0.0954257f, -4.57571f, -4.88078f, -4.62386f, -5.75167f, - 1.35351f, -1.08114f, 1.43744f, 1.44333f, 0.608153f, 0.193742f, - -0.405512f, -0.155164f, 0.0771456f, -0.473182f, -0.057984f, 0.140435f, - 0.743021f, -0.418589f, -0.377622f, -0.531411f, -0.668025f, -0.826607f, - 1.37834f, -1.07753f, 0.870466f, 0.516756f, 0.708689f, 0.286795f, - -3.97895f, -0.338629f, 2.79427f, 1.80561f, 1.46275f, 1.50438f, - 0.0232533f, -0.43174f, -0.348251f, 0.0863006f, 0.0321103f, 0.129674f, - -1.12024f, -0.0990596f, -0.283472f, -0.238713f, -0.239175f, -0.40816f, - -0.00106566f, 0.0972736f, 5.19284f, -3.70862f, 6.39657f, -5.27588f, - -2.08003f, 0.38825f, 2.38771f, -1.27501f, -2.45619f, 3.07324f, - 0.616966f, -0.451472f, -0.319365f, 0.00807278f, -0.303261f, -0.351679f +static const float av1_simple_motion_search_split_hiddenlayer_0_kernel_8[] = { + 0.079443f, -1.04068f, 0.336819f, -0.20901f, 0.796251f, + 0.181066f, 0.0118876f, -0.207145f, 0.250671f, -0.402119f, + -0.0847227f, 1.88683f, 0.303469f, 0.0718458f, 0.0338589f, + 0.158896f, 0.0540238f, -0.385426f, 0.955925f, 0.424506f, + 0.492584f, -0.795058f, -0.248667f, -0.905349f, -0.316989f, + 0.545471f, 0.63762f, -0.232613f, -0.238947f, -0.395338f, + -0.322673f, -0.0761563f, -0.125357f, 0.0694415f, -0.371599f, + 0.358387f, -0.486841f, 0.403863f, -0.0295666f, 0.283074f, + -0.424396f, 0.156318f, -0.685355f, 0.6663f, 0.337949f, + 0.273198f, 0.517448f, 0.458911f, 0.157252f, 0.692096f, + 0.64965f, -0.23987f, -1.08431f, -0.252475f, -0.332614f, + -0.712291f, -0.380973f, 0.460545f, 0.48936f, 0.337601f, + 0.489223f, 1.65336f, -0.223585f, 0.17367f, -0.235057f, + -0.456773f, 0.327877f, -0.221192f, -0.940151f, -1.06616f, + 0.687084f, -0.109973f, 0.106636f, 0.445895f, 0.163432f, + 0.378306f, 0.201902f, 0.176811f, 0.693082f, 1.62156f, + -0.178346f, 0.455175f, 1.61943f, 0.231376f, 0.0890932f, + -0.889693f, -1.03298f, 0.778196f, -0.0289539f, 0.137848f, + 0.18707f, 0.171889f, 0.119157f, 0.24893f, -0.313628f, + 0.00250735f, -0.0758209f, 0.272974f, -0.229825f, 2.47926f, + -0.0354665f, 0.175366f, 0.0411555f, -1.52149f, -0.0258663f, + 0.253027f, -0.0520839f, -0.0189782f, 0.362387f, -0.371154f, + 0.622929f, 0.0447056f, 0.242529f, -0.168391f, 0.308935f, + -0.117294f, 2.16307f, 0.0673638f, 0.080771f, -0.460779f, + -0.940176f, 0.473266f, -0.0125302f, 0.475145f, -0.218187f, + 0.43258f, -0.0380196f, 0.413607f, -0.110856f, -1.52076f, + 0.0896812f, 0.246636f, -0.0612008f, 0.189583f, 0.0106902f, + -0.158403f, -0.629377f, -0.0634279f, -0.0864584f, -0.226568f, + -0.286234f, -0.0721132f, -0.43702f, 0.113702f, 0.433372f, + 0.743396f, 0.14312f, 0.29914f, 0.801188f, 0.7609f, + 0.385046f, 0.480314f, 0.171119f, -1.59058f, -1.18853f, + 0.150676f, 0.408123f, -0.00677924f, 0.398145f, 0.0914611f, + 0.176945f, 0.0677457f, 0.316478f, 0.998219f, -0.22618f, + 0.0756793f, -0.0156674f, 0.105716f, 0.0496245f, -0.0827133f, + -0.423119f, -0.161033f, 0.212962f, -0.234453f, 0.743366f, + 1.04108f, 0.0597604f, -0.285993f, -0.114829f, -0.557364f, + -0.840051f, 0.326509f, -0.192508f, -0.141769f, 0.370626f, + -0.126353f, 0.00672923f, 0.493623f, -0.852076f, 0.466798f, + -0.226436f, 0.259268f, -0.452662f, 0.0721126f, 0.0198245f, + 0.2048f, 0.02506f, 0.316194f, 0.814651f, 1.01288f, + -0.569607f, -0.0838994f, 1.37146f, -0.613135f, 0.441761f, + -0.643901f, 0.364269f, -0.147177f, 0.338001f, -0.332376f, + 0.518875f, -0.628964f, -0.291889f, -0.050736f, 0.108047f, + 1.05673f, 0.0479492f, 0.466756f, -0.0867334f, -0.0355575f, + 0.57626f, -0.227583f, -0.146421f, 0.0990489f, 0.117351f, + -0.103858f, -0.0336936f, 0.0201903f, -0.0766383f, -0.010211f, + 0.0400779f, 0.0725462f, 0.137142f, 0.478261f, 0.287869f, + 0.0882359f, -0.739754f, -0.853521f, -0.43703f, 0.316856f, + 0.27593f, 0.312149f, 0.175575f, 0.441839f, 0.264325f, + 0.0148051f, -0.005559f, 0.373176f, 0.933701f, -0.0197615f, + 0.0219723f, -0.0559883f, -0.103456f, -0.0323009f, 0.0773202f, + -0.390838f, 0.855488f, -0.596525f, -0.249093f, 0.124262f, + 0.220172f, 0.0552478f, 1.04041f, -0.960992f, -0.495255f, + -0.211612f, 0.350007f, -0.238998f, -0.0265068f, 0.384686f, + -0.0815808f, -0.0570019f, 0.123903f, -0.485114f, -0.00282573f, + -0.0649603f, 0.163719f, -0.469479f, -0.439713f, 0.0602562f, + -0.527993f, -0.111458f, 2.48686f, -0.180723f, 0.0553895f, + 0.0560679f, -0.0978928f, -0.216063f, 0.089457f, -1.5602f, + -1.62332f, -0.147388f, 0.736155f, 0.440409f, 0.243519f, + 0.0622638f, 0.522932f, 0.109686f, 0.422849f, 0.510589f, + 1.01116f, 0.174019f, 0.0191171f, -0.0717751f, -0.0068308f, + 0.172932f, -0.834888f, -0.635788f, 0.32012f, 0.298656f, + 0.274309f, -0.155456f, 0.1755f, -0.175171f, 0.343498f, + -0.122832f, -0.107696f, 0.279924f, -0.797633f, -0.344658f, + 0.162669f, 0.389092f, 0.644479f, -0.635216f, -0.181868f, + 0.0579244f, -0.0568976f, 0.433003f, -0.591067f, 0.71013f, + -0.165515f, 0.225725f, -0.358156f, 0.0541944f, 1.95485f, + -0.315223f, 0.61537f, -0.0401568f, 0.22811f, 0.271147f }; -static const float av1_simple_motion_search_based_split_logits_kernel_8[] = { - -0.625847f, 0.381323f, 0.342475f, 0.526161f, -0.665965f, -0.515317f, - -0.406218f, 0.568007f, 0.479397f, -0.426116f, 0.615638f, 0.338572f, - 0.185583f, 0.308031f, 0.260748f, 0.531619f +static const float av1_simple_motion_search_split_hiddenlayer_0_bias_8[] = { + 1.63441f, -0.616459f, -0.437775f, -0.71669f, 1.56616f, 2.28109f, 1.64054f, + -1.51476f, 0.0274108f, 0.935156f, -0.966329f, 0.906069f, 1.19954f, -1.25867f, + -1.7376f, -0.594211f, 0.322242f, 0.438631f, -1.01682f, 1.30032f }; -static const float av1_simple_motion_search_based_split_layer_0_bias_8[] = { - 4.73775f, -1.12658f, -0.258038f, -6.06696f, 1.79131f, 2.49609f, - 4.28388f, 0.0f, -4.63598f, 3.06034f, 5.31994f, -0.152142f, - 0.514738f, -1.30098f, 3.00296f, -3.83481f +static const float av1_simple_motion_search_split_logits_kernel_8[] = { + -0.463187f, 0.2936127f, 0.16762f, -0.1663271f, -0.292418f, + -0.421457f, -0.378265f, 1.053049f, 0.32432879f, -0.49775575f, + 0.427357f, -0.239251f, -0.1631546f, 0.335468f, 0.255371f, + 0.276901f, -0.665683f, -0.7021493f, 0.381513f, -0.1339761f }; -static const float av1_simple_motion_search_based_split_logits_bias_8[] = { - -3.44508f +static const float av1_simple_motion_search_split_logits_bias_8[] = { + -1.739754f }; -static const NN_CONFIG av1_simple_motion_search_based_split_nn_config_8 = { +static const NN_CONFIG av1_simple_motion_search_split_nn_config_8 = { NUM_FEATURES_8, NUM_LOGITS_8, NUM_HIDDEN_LAYERS_8, @@ -2750,16 +3373,27 @@ static const NN_CONFIG av1_simple_motion_search_based_split_nn_config_8 = { NUM_LAYER_0_UNITS_8, }, { - av1_simple_motion_search_based_split_layer_0_kernel_8, - av1_simple_motion_search_based_split_logits_kernel_8, + av1_simple_motion_search_split_hiddenlayer_0_kernel_8, + av1_simple_motion_search_split_logits_kernel_8, }, { - av1_simple_motion_search_based_split_layer_0_bias_8, - av1_simple_motion_search_based_split_logits_bias_8, + av1_simple_motion_search_split_hiddenlayer_0_bias_8, + av1_simple_motion_search_split_logits_bias_8, }, }; -#endif +#undef NUM_HIDDEN_LAYERS_8 +#undef NUM_FEATURES_8 +#undef NUM_LAYER_0_UNITS_8 +#undef NUM_LOGITS_8 + +static const NN_CONFIG *const av1_simple_motion_search_split_nn_config[5] = { + &av1_simple_motion_search_split_nn_config_128, + &av1_simple_motion_search_split_nn_config_64, + &av1_simple_motion_search_split_nn_config_32, + &av1_simple_motion_search_split_nn_config_16, + &av1_simple_motion_search_split_nn_config_8, +}; // Model based on simple_motion_search @@ -4801,6 +5435,753 @@ static const float av1_fp_simple_motion_search_term_none_thresh_16 = static const float av1_fp_simple_motion_search_term_none_thresh_8 = -3.608804354309157f; +#define FEATURES 31 +#define HIDDEN_NODES 32 +static const float av1_early_term_after_split_nn_weights_64_layer0[] = { + -0.306296f, -0.691664f, 0.335148f, -0.298465f, -0.509241f, -0.632796f, + -0.527979f, -0.009904f, -0.503646f, -0.494002f, -0.575101f, 0.239911f, + -0.413312f, -0.622825f, -0.405448f, -0.419103f, -0.505903f, -0.392550f, + -0.240293f, 0.121749f, -0.489777f, -0.756647f, 0.001047f, -0.016528f, + 0.145714f, 0.172910f, 0.086197f, 0.162882f, -0.070588f, -0.077104f, + 0.502730f, -0.244954f, 0.265605f, -0.323994f, 0.223397f, -1.086453f, + 0.391886f, 0.200343f, 0.253878f, 0.018925f, 0.201819f, -0.205136f, + 0.427314f, 0.041155f, 0.070484f, 0.159925f, -0.057095f, -0.146544f, + -0.073792f, 0.152628f, 0.003986f, -0.515965f, -0.209754f, 0.037457f, + 0.070622f, -0.143571f, -0.059602f, 0.111734f, 0.319674f, 0.149894f, + -0.219883f, 0.206678f, 0.015809f, -0.210549f, 0.130156f, -0.189502f, + -0.850392f, -0.156363f, -0.060354f, 0.189044f, 0.266495f, 0.151305f, + -0.563677f, -0.354896f, 0.300637f, 0.257568f, -0.008359f, -0.535497f, + -0.003127f, 0.293054f, -0.020212f, -0.157278f, 0.229972f, -0.309799f, + -0.329927f, -0.077140f, 0.001177f, -0.024415f, 0.134044f, -0.181587f, + -0.135380f, 0.230989f, -0.281451f, 0.912282f, 0.511562f, -3.900779f, + -0.039917f, 1.956406f, -0.357589f, 0.292998f, -0.950158f, 0.422041f, + 0.526572f, 0.605746f, -0.147110f, 0.256576f, 0.090010f, 0.221641f, + 0.029763f, 0.351592f, 0.458324f, -0.005888f, 0.010521f, -0.389326f, + -0.094006f, -0.171489f, -0.013153f, 0.026333f, -0.454571f, -1.932891f, + -0.168211f, 0.051298f, -0.258061f, -0.028936f, -0.555937f, -0.475566f, + -0.304046f, -0.318113f, 0.099697f, -0.217145f, 0.139433f, -0.203986f, + -0.164012f, 0.051527f, 0.138603f, -0.085100f, -0.082887f, -0.242955f, + -0.663410f, -0.535772f, -0.181665f, -0.197883f, 0.071319f, 0.135086f, + 0.146200f, 0.184827f, -0.199041f, 0.162570f, -0.300167f, 0.017748f, + -0.140111f, 0.103553f, 0.206929f, 0.193446f, 0.123141f, -1.201898f, + -0.052254f, -0.750121f, 0.111741f, 0.204092f, -0.166266f, 0.124008f, + -0.455496f, 0.306035f, 0.275903f, 0.193599f, -0.730011f, 0.126808f, + 0.051059f, 0.103634f, -0.044334f, 0.048889f, 0.405228f, 0.574099f, + 0.061167f, 0.260576f, 0.070032f, -0.038040f, 0.229183f, -0.243269f, + -0.130116f, -0.538563f, -0.070199f, -0.129249f, -0.205153f, -0.268530f, + -0.290828f, -0.233006f, 0.068712f, 0.618085f, -0.407008f, 0.686868f, + 0.172247f, 0.826287f, -0.002672f, 0.239825f, -0.051548f, 0.420773f, + 0.218747f, 0.041057f, -0.071189f, 0.286987f, -0.113915f, 0.122561f, + 0.013979f, -0.049046f, 0.148175f, 0.031313f, -0.248601f, 0.209488f, + 0.069008f, 0.072763f, 0.332475f, 0.079986f, -0.151042f, -0.205110f, + -0.155550f, -0.510408f, 0.330429f, 0.577729f, 0.266524f, -0.378489f, + 0.228204f, 0.055318f, 0.117583f, -0.588557f, -0.778201f, 0.434622f, + -0.227820f, 0.611642f, 0.170548f, 0.817761f, 0.006642f, -1.005794f, + -0.911490f, 1.633684f, -0.290664f, 0.308128f, 0.295986f, 0.243377f, + -0.001275f, -0.131156f, 0.275205f, -0.041865f, -0.201951f, -0.016380f, + 0.336604f, -0.258118f, 0.890810f, 0.441065f, -0.968006f, 0.135989f, + -1.447191f, 0.353426f, -0.343235f, 0.376837f, -0.071602f, -0.319639f, + -0.072347f, 0.547450f, -0.215380f, 0.182141f, -0.066186f, 0.033787f, + 0.257482f, 0.217428f, -0.130249f, 0.057525f, 0.263991f, 0.230664f, + -0.245113f, 0.048610f, -0.079955f, 0.251737f, -0.070368f, -0.017968f, + -0.151815f, 0.025945f, -0.257769f, 0.299735f, 0.077263f, -0.565526f, + 0.326263f, 0.096429f, 0.113414f, 0.092754f, -0.141908f, 0.172060f, + 0.393117f, -0.216755f, 0.331051f, -0.363369f, -0.113363f, -0.095164f, + -0.072784f, 0.214572f, 0.010993f, 0.209456f, 0.260381f, -0.314747f, + -0.422173f, -0.189963f, -0.225130f, 0.339448f, 0.153814f, 0.265616f, + -0.103575f, -0.123841f, -0.106236f, 0.155894f, -0.156264f, -1.361406f, + -0.040736f, -0.614998f, -0.468200f, -0.266505f, -0.342786f, -0.908088f, + 0.105758f, 0.040788f, -0.313589f, -1.359318f, 0.071329f, 0.176404f, + -0.476141f, 0.010108f, -0.201440f, -0.221167f, -0.197448f, -0.013927f, + -0.610270f, -0.607285f, 0.178070f, 0.174320f, 0.313115f, 0.026191f, + -0.112330f, 0.122338f, -0.367751f, 0.196794f, 0.153709f, -0.205454f, + -0.397471f, -1.879336f, -0.030129f, 0.143429f, -0.079832f, 0.435259f, + -1.729539f, 0.518301f, -0.141393f, 0.199399f, -1.914601f, 0.142865f, + -0.219899f, 0.508458f, 0.086365f, -0.220740f, -0.012507f, 1.263320f, + 0.042136f, 0.050922f, -0.329644f, -0.188198f, 0.251522f, 0.394731f, + -0.047866f, -0.260853f, -0.267207f, -0.248489f, 0.146474f, 0.359257f, + -0.427732f, -0.100652f, 0.192129f, 0.075572f, 0.916708f, 0.255747f, + 0.486384f, 0.127989f, -0.556449f, -0.484913f, 0.392298f, 0.045401f, + -0.839551f, -0.703619f, 0.069263f, -0.040720f, 0.542265f, 0.443739f, + 0.862552f, -0.021726f, 0.230858f, -0.261004f, -0.125697f, -0.106435f, + 0.002341f, 0.013904f, 0.011034f, 0.542296f, -0.284325f, 0.135736f, + 0.113882f, 0.040610f, -0.255485f, 0.224061f, -0.087140f, 0.127872f, + -0.002638f, 0.164889f, -0.335958f, -0.031166f, -0.393581f, 0.075455f, + 0.055995f, 0.087934f, -0.133859f, -0.342187f, 0.002492f, -0.340722f, + 0.058304f, 0.104165f, -0.142136f, -0.351111f, -0.158037f, -0.079924f, + -0.253209f, -0.092840f, -0.174646f, -0.202772f, -0.353438f, -0.031111f, + 0.076088f, -0.232091f, -0.070052f, 0.097595f, 0.063173f, -0.211195f, + 0.126478f, -0.178828f, 0.278723f, -0.070807f, -0.179783f, 0.034123f, + 0.035721f, -0.200431f, 0.170640f, 0.107933f, 0.226594f, -0.301499f, + -0.291096f, 0.228076f, -0.272951f, 0.002490f, -0.210707f, -0.128033f, + -0.194009f, -0.011347f, -0.256694f, -0.011841f, -0.005167f, -0.163203f, + -0.253796f, -0.198877f, -0.055827f, -0.882685f, -0.443471f, 0.349601f, + 0.749334f, -1.161845f, 0.505480f, 0.221733f, 0.210490f, -0.234984f, + 0.014183f, -0.510401f, 0.238692f, -0.134111f, 0.083844f, -0.478751f, + -0.088434f, 0.304063f, 0.150336f, -0.749682f, -0.081999f, 0.729739f, + 0.412508f, 0.132571f, 0.058306f, -0.047451f, -0.117435f, -0.445395f, + -0.005182f, -0.025757f, 0.175051f, -0.258194f, -0.150311f, -0.196533f, + -1.314316f, -0.428627f, 0.512451f, 0.045138f, -0.200925f, 0.081538f, + -0.346151f, -0.358197f, -0.422258f, -0.028542f, -0.383534f, -0.026163f, + -0.419858f, -0.154321f, 0.376970f, 0.094017f, 0.783520f, 0.110641f, + 0.077966f, -0.093064f, 0.160522f, -0.863041f, 0.086210f, 0.560764f, + 0.057032f, 0.159224f, 0.323068f, -0.173109f, 0.014042f, -0.126856f, + -0.128237f, -0.245273f, -0.317312f, -0.257597f, -0.181977f, 0.259485f, + -0.215834f, 0.062076f, -0.270596f, 0.271581f, -0.153486f, -0.247165f, + 0.079737f, -0.157049f, -0.027459f, -0.299397f, 0.136729f, -0.334192f, + -0.191722f, 0.145865f, -0.031324f, -0.307165f, -0.244923f, -0.228027f, + 0.063807f, 0.054965f, -0.005709f, -0.041977f, -0.276245f, 0.020003f, + 0.133323f, -0.145992f, -0.951030f, 0.414083f, -1.063323f, 0.137872f, + 0.104732f, -0.123728f, 0.542532f, 0.213654f, 0.542954f, 0.155619f, + 0.543072f, 0.399067f, 0.191402f, -0.102552f, -0.176734f, -0.136776f, + -0.012814f, -0.021298f, -0.802467f, -0.957481f, -0.238787f, -0.138482f, + 0.058331f, 0.126601f, 0.104420f, -0.148684f, 0.343218f, 0.093604f, + -0.055642f, -0.383918f, -0.045250f, -0.090480f, -0.155464f, 0.278299f, + 0.042791f, -0.029084f, -0.373861f, -0.073233f, -0.085172f, 0.186841f, + -0.070898f, -0.156415f, 0.112831f, -0.065931f, -0.353007f, 0.058453f, + -0.136982f, 0.233393f, 0.017240f, -0.018428f, 0.229104f, -0.371440f, + -0.262212f, 0.203075f, -0.263293f, 0.034413f, -0.299354f, 0.227269f, + 0.204977f, -0.118107f, -0.359832f, -0.068252f, 0.480105f, -0.214711f, + -0.614381f, 0.209048f, -0.456014f, -0.188819f, -0.220995f, -0.322104f, + -0.191457f, 0.420874f, -0.454919f, 0.023119f, 0.291700f, -0.532885f, + -0.032642f, 0.043271f, 0.133974f, 0.002399f, -0.179899f, -0.044158f, + -0.027078f, -0.350075f, 0.236766f, 0.346771f, -0.118534f, -0.421221f, + 0.019544f, 0.109349f, 0.141517f, 0.403561f, 0.409102f, 0.054555f, + -0.561751f, 0.577183f, -0.705156f, -0.231188f, -1.969772f, 0.172289f, + -0.048122f, 0.205671f, -0.667130f, -0.066870f, 0.202838f, -0.095538f, + -0.842651f, 0.254170f, 0.046256f, -0.271891f, -0.369254f, 0.492101f, + 0.001189f, -0.186525f, 0.188470f, -0.207072f, 0.030086f, -0.132904f, + 0.127001f, 0.116662f, -0.079246f, 0.227241f, -0.462178f, 0.446304f, + -1.660753f, 0.241832f, -0.288040f, 0.054663f, -0.435804f, 0.296782f, + -0.026421f, -0.115618f, 0.163416f, 0.834001f, 0.008019f, -0.014243f, + 0.524658f, 0.067894f, -0.253936f, -0.100657f, 1.285389f, -0.005952f, + 0.087134f, -0.088375f, -0.121866f, -0.171172f, 0.279463f, -0.598593f, + -0.727761f, 0.189831f, -0.822575f, -0.291141f, -0.012410f, -0.069999f, + 0.098842f, -0.218513f, 0.009494f, 0.100106f, -0.402884f, -0.299236f, + -0.345668f, -0.057739f, -0.213248f, -0.426661f, -0.360268f, -0.349860f, + -0.382177f, -0.357802f, -0.032030f, -0.110597f, -0.155442f, -0.418794f, + -0.012113f, -0.032962f, -0.450648f, 0.129060f, -0.135227f, -0.298593f, + 0.001435f, 0.278790f, -0.272945f, 0.162759f, -0.290208f, 0.058481f, + -0.490971f, 0.019630f, -0.210347f, 0.000520f, -0.340413f, 0.641562f, + 0.023104f, 0.194832f, -0.441894f, -0.253538f, -0.228332f, 0.423264f, + -1.094073f, -0.475657f, -0.238752f, 0.033910f, 0.440425f, 0.036320f, + 0.566989f, -0.065326f, -0.297939f, 0.406098f, 0.529561f, -0.113084f, + 0.141472f, -0.024462f, -0.179212f, 0.187801f, -0.235787f, -0.229624f, + 0.357791f, 0.061110f, -0.607788f, -1.713694f, -0.651041f, 1.734283f, + -0.334701f, 0.161687f, 0.010215f, 0.320708f, 0.169447f, 0.513558f, + 0.488340f, -0.619036f, -0.525441f, -1.144352f, -0.546154f, 0.669973f, + 0.327028f, -0.100539f, 0.012048f, -0.223013f, -0.239680f, 0.323035f, + 0.165950f, -0.155110f, 0.128664f, -0.157378f, -0.124490f, 0.291553f, + 0.055849f, -0.221664f, 0.077770f, -0.350658f, -0.181939f, 0.110230f, + -0.078219f, 0.007472f, -0.031620f, 0.007708f, -0.201794f, 0.017594f, + -0.027480f, 0.058884f, -0.369166f, -0.369770f, 0.181635f, -0.183318f, + -0.389184f, -0.256661f, 0.160107f, 0.037127f, -0.082573f, -0.095815f, + -0.322782f, 0.072528f, -0.348875f, 0.216247f, -0.161757f, -0.385502f, + -0.315738f, 0.020123f, -0.155609f, 0.114403f, -0.383232f, 0.629529f, + 0.066142f, 0.448392f, -0.389557f, -0.083315f, 0.829535f, -0.015531f, + -0.050728f, -0.325127f, 0.812992f, -0.196780f, 0.021060f, -0.952647f, + 0.006687f, -0.512715f, -0.066778f, 0.410067f, -0.116945f, -0.288283f, + 0.189334f, -0.083153f, 0.159980f, -0.068208f, 0.107358f, -0.154411f, + -0.068914f, 0.186816f, 0.032251f, 0.109242f, 0.134825f, 0.035101f, + -0.253175f, 0.157309f, -0.363597f, -0.138176f, -0.334141f, -0.172697f, + 0.045800f, -0.286057f, 0.173403f, -0.172444f, -0.117996f, -0.383848f, + -0.173303f, -0.258482f, -0.021404f, -0.017898f, -0.001970f, 0.003273f, + 0.056121f, 0.155046f, 0.044708f, -0.295609f, -0.211688f, -0.233229f, + -0.264980f, 0.145549f, 0.045323f, -0.027112f, 0.175638f, -0.207251f, + -0.055274f, 0.092706f, 0.086200f, -0.241340f, -0.147416f, 0.024510f, + -0.357194f, -0.181944f, -0.050104f, -0.079024f, -0.290473f, -0.169790f, + -0.277982f, -0.017781f, -0.004854f, -0.094132f, -0.348555f, 0.199291f, + -0.343989f, -0.319299f, -0.268935f, -0.021208f, 0.020938f, -0.090609f, + 0.006595f, -0.200790f, 0.171856f, -0.027766f, -0.032017f, -0.006745f, + 0.566426f, -0.096850f, 0.727633f, -0.408065f, -0.012436f, 0.005646f, + -0.305148f, -0.095075f, -0.391549f, -0.020378f, -0.236498f, -0.252773f, + -0.231385f, -0.203175f, 0.041903f, -0.373694f, 0.058239f, -0.101116f, + 0.183772f, 0.164523f, -0.099046f, -0.201272f, -0.394523f, -0.157517f, + 0.032079f, -0.381173f, -0.238496f, -0.037990f, -0.294553f, 0.141473f, + 0.100268f, -0.023806f, 0.004978f, 0.184916f, 0.142699f, -0.113240f, + -0.213364f, -0.160059f, -0.216263f, -0.406387f, -0.301140f, -0.406355f, + -0.113085f, -0.279699f, -0.267434f, 0.126263f, -0.260527f, -0.153904f, + -0.494653f, -0.355144f, 0.030549f, -0.216400f, -0.123363f, 0.189090f, + 0.219122f, 0.096677f, -0.202037f, -0.014489f, -0.137859f, -0.114184f, + -0.279423f, -0.270683f, +}; + +static const float av1_early_term_after_split_nn_bias_64_layer0[] = { + -0.491455f, 0.464538f, -0.005742f, -0.219951f, -0.073682f, 0.102027f, + 0.567071f, 0.441402f, 0.277521f, 0.314498f, -0.448199f, -0.065032f, + 0.488139f, -0.079632f, 0.000000f, 0.521555f, -0.151950f, -0.034616f, + 0.393438f, -0.072242f, -0.087343f, -0.571308f, 0.017372f, -0.126144f, + 0.372261f, -0.451537f, -0.140238f, -0.092377f, -0.074475f, -0.068879f, + -0.109614f, -0.164492f, +}; + +static const float av1_early_term_after_split_nn_weights_64_layer1[] = { + -0.373195f, -0.283141f, 0.416113f, 0.483659f, 0.230583f, 0.349197f, + -0.168582f, -0.813338f, -0.472369f, -0.173872f, 1.297845f, 0.339355f, + -0.828033f, 0.019617f, 0.118757f, -0.619360f, 0.282295f, -0.054116f, + -0.730596f, 0.068567f, -0.248707f, 0.461225f, 0.330224f, -0.287080f, + -0.458103f, 0.591852f, -0.008491f, 0.632119f, -0.007872f, 0.007869f, + -0.230698f, -0.011437f, +}; + +static const float av1_early_term_after_split_nn_bias_64_layer1[] = { + -0.55403697f, +}; + +static const NN_CONFIG av1_early_term_after_split_nnconfig_64 = { + FEATURES, + 1, + 1, + { + HIDDEN_NODES, + }, + { + av1_early_term_after_split_nn_weights_64_layer0, + av1_early_term_after_split_nn_weights_64_layer1, + }, + { + av1_early_term_after_split_nn_bias_64_layer0, + av1_early_term_after_split_nn_bias_64_layer1, + }, +}; + +static const float av1_early_term_after_split_nn_weights_32_layer0[] = { + 0.026050f, -0.226531f, 0.308107f, -0.083744f, 0.201785f, 0.098562f, + 0.147595f, -0.495771f, -0.245741f, 0.201616f, -0.272070f, -0.579545f, + -0.127261f, -0.229588f, 0.250831f, -0.176929f, -0.031689f, 0.284718f, + 0.085845f, -0.285027f, 0.012304f, 0.382402f, -0.204591f, 0.272514f, + -0.065854f, -0.054228f, -0.231174f, -0.174504f, 0.258287f, 0.195689f, + 0.242530f, 0.023528f, -0.294242f, -0.272132f, 0.460180f, -0.731281f, + -0.208103f, 0.208204f, 0.348250f, 0.016328f, 0.043707f, -0.169551f, + 0.108521f, 0.226895f, -0.020471f, 0.102443f, 0.429640f, -0.252555f, + -0.218434f, -0.163665f, 0.175531f, 0.101588f, -0.135798f, -0.158102f, + 0.142565f, 0.128277f, 0.174985f, -0.100073f, 0.113967f, 0.223682f, + -0.145576f, -0.008443f, 0.112748f, -0.037845f, 0.076954f, -0.287137f, + -0.518185f, -0.106833f, 0.175359f, 0.031408f, 0.219069f, -0.294440f, + 0.007766f, 0.067754f, -0.049168f, -0.212368f, -0.261708f, 0.309252f, + 0.220859f, -0.274852f, -0.653157f, 0.083438f, -0.265386f, 0.174429f, + -0.116931f, -0.091594f, -0.244897f, -0.089015f, 0.274453f, 0.212890f, + 0.272053f, -0.425315f, -0.107726f, 0.294444f, -0.354629f, 0.104402f, + -0.307663f, 0.558430f, 0.140334f, -0.054831f, -0.449456f, 0.058274f, + -0.033768f, -0.354117f, -0.331618f, -0.411772f, 0.232064f, -0.079297f, + -0.638571f, 0.181823f, -0.039611f, 0.206310f, -0.659157f, -0.102930f, + -0.067303f, -0.176881f, -0.001038f, 0.091835f, 0.079739f, -0.121923f, + 0.211070f, 0.362719f, -0.154915f, -0.151876f, -0.165460f, 0.023469f, + -0.251036f, 0.210014f, -0.537125f, 0.156832f, -0.216987f, 0.062975f, + -0.198462f, 0.329123f, 0.125870f, 0.225830f, 0.086377f, -0.128773f, + -0.179673f, -0.074612f, 0.456645f, 0.021905f, -0.243140f, 0.059145f, + -0.273942f, -0.277822f, 0.154556f, -0.025459f, 0.227614f, -0.313076f, + 0.044705f, -0.019017f, 0.108999f, -0.020243f, -0.016373f, 0.560270f, + -0.064818f, 0.050880f, -0.218458f, 0.825699f, -0.534056f, -0.258253f, + 0.222073f, 0.013295f, 0.477870f, -0.386727f, 0.388509f, 0.004128f, + 0.451388f, -0.175788f, 0.264093f, -0.109812f, 0.358132f, 0.500992f, + -0.446933f, -0.222397f, 0.345834f, 0.370943f, -0.233115f, -0.047005f, + -0.111335f, -0.111586f, 0.026975f, -0.052191f, -0.111800f, -0.129782f, + 0.225132f, 0.102524f, 0.544557f, -0.111674f, -0.857884f, 0.133258f, + 0.310001f, 0.043829f, 0.104143f, 0.256493f, 0.242520f, -0.342082f, + 0.421447f, 0.124227f, 0.061542f, -0.090206f, 0.316681f, 0.353452f, + -0.918408f, -0.001903f, -0.052303f, -0.004816f, -0.446393f, -0.053038f, + 0.255725f, -0.126346f, 0.034095f, -0.240276f, -0.135918f, 0.095682f, + -0.147457f, -0.338216f, -0.200426f, 0.010265f, -0.243915f, -0.231375f, + -0.323924f, -0.014353f, 0.150252f, -0.264346f, 0.205303f, -0.194610f, + -0.282527f, 0.180555f, -0.000087f, 0.027240f, -0.000903f, -0.345877f, + -0.353274f, -0.311829f, 0.172985f, -0.111748f, -0.309380f, 0.108110f, + -0.260914f, -0.164990f, 0.183625f, -0.319692f, -0.096988f, 0.094147f, + -0.047062f, -0.080978f, 0.227387f, -0.000450f, -0.220159f, -0.211448f, + -0.020885f, -0.139646f, -0.086721f, 0.067928f, -0.033084f, -0.251996f, + 0.090317f, 0.086313f, -0.228420f, -0.111356f, -0.314304f, -0.223664f, + 0.188176f, -0.002360f, -0.029491f, -0.006000f, -0.075343f, 0.173699f, + -0.272800f, -0.238507f, -0.272071f, -0.015000f, -0.215305f, -0.192943f, + -0.038595f, 0.119537f, 0.260477f, -0.168014f, -0.172751f, 0.532861f, + -0.753250f, -0.017485f, -0.115541f, -0.109291f, -1.098943f, 0.418559f, + -0.532110f, 0.359323f, -0.254786f, 0.471316f, -0.545024f, 0.291912f, + -0.836939f, 0.443427f, -0.441709f, 0.168866f, -0.140372f, 0.546607f, + -0.315465f, 0.023328f, 0.137709f, -0.083492f, -0.049986f, -0.071302f, + -0.293680f, -0.105049f, 0.315317f, 0.279569f, 0.220762f, 0.088161f, + -0.756456f, -0.074512f, 0.958318f, -0.332924f, -0.004906f, -0.629271f, + 0.212050f, 0.279123f, 0.311523f, -0.599580f, 0.516150f, 0.456952f, + 0.020255f, 0.247290f, -0.182670f, -0.335554f, 0.021203f, 0.131081f, + -0.208584f, 0.112530f, -0.198980f, 0.211583f, -0.101271f, -0.206453f, + -0.502688f, -0.294976f, -0.187019f, -0.114473f, 0.282050f, -0.165483f, + 0.094953f, -0.182578f, 0.055068f, 0.135605f, -0.266941f, -0.297556f, + 0.199181f, 0.015979f, -0.158659f, -0.226841f, 0.171306f, 0.013438f, + -0.286309f, -0.071753f, -0.170300f, -0.238188f, 0.093572f, -0.026230f, + -0.254502f, -0.297786f, -0.063480f, -0.300799f, -0.065644f, 0.074710f, + 0.248576f, -0.144425f, -0.113948f, -0.247297f, 0.276682f, 0.010963f, + -0.737786f, 0.026347f, 0.007830f, 0.753543f, 0.371904f, 0.305614f, + 0.105028f, 0.073530f, -0.119137f, 0.102352f, -0.080523f, 0.176366f, + -0.159457f, -0.339948f, 0.360131f, -0.007051f, -0.388378f, -0.101695f, + 0.663041f, -0.234486f, -0.142536f, -0.099931f, 0.041478f, 0.230425f, + 0.005743f, 0.154060f, 0.056233f, -0.080668f, -0.009754f, -0.194356f, + 0.185474f, -0.296474f, 0.192700f, 0.257767f, 0.348529f, 0.458265f, + 0.060276f, -0.130473f, 0.139889f, 0.310073f, -0.306869f, -0.272922f, + -0.259862f, 0.409207f, 0.431991f, -0.100357f, -0.050415f, -0.071830f, + -0.239665f, 0.153399f, 0.177192f, -0.611644f, -0.176114f, -0.022694f, + -0.033701f, -0.345842f, 0.015660f, 0.158931f, -0.097586f, 0.222001f, + 0.257887f, -0.171307f, -0.222607f, -0.245508f, -0.145742f, -0.096461f, + -0.010895f, 0.052815f, -0.265306f, -0.081059f, 0.219162f, -0.256084f, + -0.372676f, 0.148977f, 0.174831f, 0.086980f, 0.108518f, 0.074011f, + 0.038032f, -0.070856f, -0.109407f, 0.126174f, 0.022341f, -0.249786f, + -0.356164f, -0.202841f, -0.087437f, -0.133740f, 0.090956f, -0.017953f, + -0.028353f, 0.233621f, 0.109426f, 0.232798f, -0.104950f, -0.241798f, + -0.018995f, -0.167954f, 0.002473f, 0.060418f, -0.232717f, -0.195980f, + -0.283971f, -0.371881f, 0.219728f, 0.018072f, -0.166694f, -0.083301f, + -0.000616f, -0.212641f, -0.173158f, 0.222739f, -0.235302f, 0.237624f, + 0.222232f, -0.041235f, -0.342411f, 0.121194f, 0.211291f, -0.032237f, + -0.249401f, -0.291668f, 0.206055f, -0.148200f, 0.011824f, -0.272728f, + -0.194854f, 0.367175f, -0.257243f, 0.103433f, -0.231077f, 0.236734f, + 0.135733f, -0.362845f, 0.197147f, 0.242782f, -0.135289f, 0.123311f, + 0.259420f, -0.116278f, 0.127287f, 0.236789f, -0.097438f, 0.118073f, + 0.112796f, -0.035949f, 0.184408f, 0.200948f, -0.008859f, 0.195989f, + 0.161970f, -0.295320f, -0.330389f, 0.141034f, 0.066081f, -0.707857f, + 0.357037f, 0.149633f, 0.679877f, 0.548674f, 0.469076f, 0.194123f, + -0.209872f, -0.071764f, -0.126960f, 0.199420f, 0.327116f, -0.169053f, + -0.429156f, 0.443429f, -0.225530f, -0.130738f, -0.028351f, 0.644393f, + 0.049606f, -0.243602f, -0.409920f, 0.117028f, -0.258557f, 0.073865f, + -0.200454f, -0.139957f, -0.031314f, 0.162325f, 0.247221f, 0.071909f, + -0.336276f, 0.079922f, 0.192780f, -0.148882f, 0.133192f, -0.143177f, + -0.121327f, 0.126221f, -0.089521f, -0.181826f, 0.149923f, -0.280682f, + 0.391572f, 0.108990f, -0.445494f, -0.170787f, 0.225182f, 0.223313f, + -0.234828f, -0.071072f, -0.072673f, -0.093686f, 0.223892f, -0.049377f, + 0.057976f, 0.033558f, 0.068733f, -0.283353f, 0.217877f, 0.158093f, + -0.276761f, -0.097049f, -0.351913f, -0.383604f, 0.002863f, -0.474510f, + -0.096738f, 0.256940f, 0.234203f, -0.226667f, -0.260576f, -0.183403f, + -0.035578f, 0.141570f, 0.078764f, -0.028086f, 0.155800f, -0.251115f, + -0.286703f, -0.014739f, -0.072621f, -0.311506f, -0.048639f, 0.081621f, + 0.043057f, 0.068136f, -0.179903f, 0.143699f, -0.002571f, 0.239012f, + 0.197456f, 0.035745f, -0.311927f, 0.220320f, 0.102687f, -0.294105f, + 0.426740f, 0.209050f, 0.211907f, 0.083453f, 0.006578f, -0.143338f, + 0.003157f, 0.040295f, 0.234497f, 0.035344f, -0.163909f, 0.411115f, + 0.289453f, -0.075357f, -0.008884f, 0.469798f, -0.033304f, -0.153293f, + -0.229322f, -0.004162f, 0.113363f, 0.395381f, 0.067414f, -0.188966f, + -0.117424f, -0.166423f, 0.066839f, 0.595641f, -0.204782f, -0.451727f, + 0.198509f, -0.921583f, -0.246765f, -0.153411f, 0.046491f, 0.365906f, + 0.376710f, -0.017355f, -0.035232f, 0.138785f, -0.163918f, -0.283449f, + -0.094340f, 0.192127f, 0.154815f, 0.035787f, -0.029087f, 0.115649f, + -0.220133f, -0.452741f, 0.311667f, 0.157666f, 0.091401f, 0.236040f, + -0.168523f, 0.122176f, -0.219016f, -0.214856f, 0.172824f, -0.091810f, + 0.031520f, -0.857420f, 0.643446f, -0.017471f, 0.206082f, -0.933517f, + -0.020070f, -0.065091f, -0.117680f, -1.271870f, -0.069177f, -0.149409f, + 0.289970f, -0.889775f, -0.044741f, 0.232647f, -0.319416f, 0.073030f, + 0.278549f, 0.238782f, -0.202206f, 0.272540f, 0.201412f, 0.175574f, + -0.127971f, -0.253164f, -0.086352f, -0.005381f, 0.114714f, 0.505169f, + -0.175049f, -1.534280f, -0.320666f, -2.119298f, -0.023075f, -0.021259f, + -0.161019f, 0.344837f, 0.361958f, -0.097050f, 0.014375f, 0.267110f, + 0.341442f, -0.016688f, 0.073393f, 0.131500f, 0.246331f, 0.011059f, + 0.033597f, 0.014779f, -0.269366f, -0.504788f, 0.048651f, 0.295682f, + 0.237363f, 0.227484f, -0.235814f, -0.160530f, 0.182682f, -0.172999f, + -0.126630f, 0.168357f, -0.078729f, 0.052805f, 0.377021f, -0.004727f, + 0.230415f, -0.876673f, 0.458457f, 0.099401f, -0.019616f, 0.611982f, + -0.231508f, -0.070894f, -0.056142f, 0.548969f, -0.376599f, -0.600428f, + 0.241930f, -0.592893f, 0.189371f, 0.488651f, -0.092446f, -0.272569f, + 0.251643f, 0.315945f, -0.301468f, 0.112961f, 0.052119f, -0.066076f, + -0.082249f, 0.252805f, -0.195539f, 0.150386f, -0.865534f, 0.673447f, + 0.030177f, -0.438528f, -1.006174f, 0.575176f, -0.271656f, 0.035835f, + -1.056916f, 0.495267f, -0.092428f, -0.109511f, -0.192359f, 0.166669f, + -0.624326f, -0.000354f, -0.089075f, 0.176279f, -0.289347f, 0.021346f, + 0.020375f, 0.255282f, -0.045588f, 0.173675f, 0.100957f, -0.294373f, + 0.049303f, -0.134132f, -0.255731f, -0.025559f, -0.307463f, -0.205100f, + 0.079024f, 0.101113f, 0.135742f, -0.348869f, -0.026759f, -0.134155f, + -0.179275f, -0.054297f, -0.054948f, 0.029351f, 0.190560f, 0.102476f, + -0.025785f, 0.169442f, -0.271303f, 0.200667f, 0.099063f, 0.074767f, + -0.326533f, 0.044426f, -0.290251f, -0.082443f, -0.164482f, -0.349412f, + 0.045109f, -0.157330f, 0.165935f, 0.012672f, -0.059818f, 0.399140f, + -0.316620f, 0.386638f, -0.285399f, -0.296777f, -0.200473f, -0.144232f, + 0.251851f, -0.203768f, 0.001071f, -0.179063f, 0.248952f, -0.143029f, + 0.010423f, -0.030293f, -0.046786f, -0.196195f, -0.016845f, 0.295023f, + 0.322825f, 0.133683f, 0.017388f, 0.142467f, 0.221320f, 0.004059f, + -0.115770f, 0.143363f, 0.137972f, -0.272584f, 0.489366f, -0.091828f, + -0.014703f, 0.082332f, -0.476226f, -0.202859f, 0.356094f, -0.283049f, + 0.218086f, 0.202015f, 0.201724f, 0.012617f, 0.050720f, 0.255695f, + 0.244653f, 0.111296f, -0.151450f, -0.056210f, -0.757348f, 0.441724f, + -0.022455f, -0.244662f, 0.296205f, -0.421883f, -0.217386f, -0.254301f, + 0.409105f, -0.031309f, 0.050147f, -0.337170f, -0.106620f, -0.606455f, + 0.308024f, 0.298144f, 0.363993f, 0.704870f, -0.047292f, 0.166901f, + 0.105991f, -0.536757f, -0.424031f, -0.226034f, 0.213635f, -0.526754f, + 0.310990f, -0.116038f, 0.007775f, 0.538330f, -0.177912f, 0.445357f, + -0.290365f, 0.451169f, 0.030931f, 0.033388f, 0.209905f, -0.244492f, + -0.097792f, -0.246042f, 0.132047f, 0.032576f, 0.115516f, 0.022890f, + 0.093508f, -0.071840f, 0.362948f, -0.135245f, 0.659911f, -0.321413f, + 0.193118f, -0.795001f, -0.218311f, 0.024862f, 0.206172f, -0.832878f, + -0.255670f, 0.343402f, -0.275211f, -0.898363f, -0.025172f, 0.158565f, + 0.171347f, -0.127518f, -0.215156f, -0.159198f, 0.250355f, -0.132452f, + 0.061254f, -0.097544f, -0.223246f, 0.013183f, 0.239468f, 0.259017f, + -0.217739f, -0.032263f, 0.123755f, -0.701777f, 0.150049f, -0.555293f, + 0.062430f, -0.260304f, 0.494894f, -0.168702f, -0.134829f, -0.113989f, + 0.150092f, -0.060248f, 0.115711f, -0.277202f, 0.499811f, 0.417116f, + 0.191081f, -0.376432f, -0.321092f, 0.033992f, 0.057193f, 0.127077f, + -0.009042f, 0.014443f, 0.142808f, -0.124349f, 0.213087f, -0.381686f, + 0.129726f, -0.038396f, +}; + +static const float av1_early_term_after_split_nn_bias_32_layer0[] = { + -0.107171f, 0.060848f, -0.069480f, -0.121982f, 0.037637f, -0.291839f, + 0.102257f, -0.065889f, -0.032452f, 0.034171f, -0.073984f, -0.005236f, + 0.218820f, 0.132123f, -0.089621f, -0.067679f, 0.049368f, 0.329444f, + -0.184729f, 0.031702f, 0.009735f, -0.039964f, -0.018024f, -0.073031f, + -0.030166f, -0.191037f, -0.074862f, -0.076548f, 0.076537f, 0.216609f, + -0.078358f, -0.007740f, +}; + +static const float av1_early_term_after_split_nn_weights_32_layer1[] = { + 0.047869f, -0.231773f, -0.185663f, 0.460676f, -0.208182f, 0.590555f, + -0.622627f, 0.279377f, 0.351681f, 0.633504f, 1.069884f, 0.332449f, + -0.457703f, -0.435817f, -0.028853f, 0.327490f, -0.282469f, -0.975792f, + -0.062975f, -0.147187f, 0.348340f, -1.207116f, 0.516159f, -1.509626f, + -0.805072f, 0.522999f, 0.143671f, 0.304246f, -0.360720f, -0.612472f, + 0.260045f, -0.223243f, +}; + +static const float av1_early_term_after_split_nn_bias_32_layer1[] = { + -0.07571174f, +}; + +static const NN_CONFIG av1_early_term_after_split_nnconfig_32 = { + FEATURES, + 1, + 1, + { + HIDDEN_NODES, + }, + { + av1_early_term_after_split_nn_weights_32_layer0, + av1_early_term_after_split_nn_weights_32_layer1, + }, + { + av1_early_term_after_split_nn_bias_32_layer0, + av1_early_term_after_split_nn_bias_32_layer1, + }, +}; + +static const float av1_early_term_after_split_nn_weights_16_layer0[] = { + -0.113798f, 0.053357f, -0.037947f, -0.477171f, 0.276517f, -0.349252f, + -0.177284f, 0.189597f, 0.141744f, 0.230207f, -0.328104f, 0.074328f, + 0.247717f, 0.233533f, 0.145167f, 0.018029f, -0.398725f, -0.226199f, + -0.309724f, 0.125279f, 0.194759f, 0.025531f, 0.349714f, -0.273944f, + 0.186871f, 0.181735f, -0.520614f, -0.264076f, 0.308207f, 0.157438f, + -0.137791f, -0.054582f, 0.125879f, 0.796218f, -0.897562f, 0.885439f, + 0.381640f, 0.106625f, -2.027456f, 0.000874f, 0.179581f, 0.013287f, + -2.329439f, -0.163169f, -0.136191f, 0.320108f, -2.318779f, -0.196722f, + -0.295721f, 0.203658f, -0.182275f, 0.615941f, 0.015762f, 0.257181f, + -0.115297f, 0.295774f, -0.026144f, -0.022686f, -0.219423f, -0.042861f, + 0.207647f, -0.057791f, 0.201671f, -0.169569f, 0.291492f, -0.994991f, + 0.137473f, 0.230948f, 0.505626f, -1.065860f, 0.275225f, -0.250861f, + 0.519466f, -1.217242f, -0.087384f, 0.053441f, 0.030729f, -1.702304f, + -0.034635f, 0.010177f, -0.035422f, -0.749979f, 0.355499f, 0.408166f, + -0.086883f, 0.017203f, 0.195706f, -0.218056f, -0.029153f, 0.367335f, + -0.061732f, -0.241068f, 0.078496f, -0.370346f, -0.124223f, -0.172708f, + 0.037971f, 0.038875f, -0.282489f, -0.266323f, -0.210864f, 0.214714f, + 0.234695f, -0.045625f, 0.015357f, -0.007464f, -0.362003f, -0.113465f, + 0.145141f, 0.238470f, -0.202664f, -0.286587f, -0.347112f, 0.054501f, + -0.190290f, -0.283256f, 0.062179f, 0.041165f, -0.006935f, -0.220351f, + -0.088800f, 0.220924f, -0.200982f, 0.058493f, -0.225175f, 0.057175f, + -0.618187f, 0.761023f, -0.743774f, -0.500599f, -0.584999f, 1.545211f, + 0.123055f, -0.106848f, -0.353057f, 1.552187f, 0.174104f, 0.068060f, + -0.449859f, 1.254299f, -0.161716f, -0.060630f, -0.230721f, 0.165976f, + -0.101582f, -0.422415f, 0.110384f, -0.130098f, 0.104428f, 0.083518f, + 0.031626f, 0.083048f, 0.158877f, 0.173340f, 0.063962f, 0.427845f, + 0.663268f, 0.376996f, 0.146435f, -0.091329f, 0.443447f, 0.518432f, + -0.182777f, -0.091313f, 0.331229f, 0.532604f, -0.187001f, 0.054774f, + 0.298068f, 0.502295f, -0.362378f, 0.054283f, 0.292806f, 0.168901f, + -0.214787f, 0.025637f, 0.458009f, -0.322714f, -0.264059f, 0.140313f, + -0.102696f, -0.431208f, -0.134450f, -0.545415f, 0.253851f, -0.009061f, + -0.050681f, 0.108681f, 0.043272f, -1.073133f, 0.206410f, 0.469576f, + 0.291494f, -2.021244f, -0.001183f, -0.067542f, 0.364907f, -2.470543f, + 0.049147f, -0.018868f, 0.658500f, -2.531048f, 0.275433f, -0.034224f, + -0.171386f, 0.096369f, 0.728069f, 0.272332f, 0.222255f, -0.030426f, + 0.026994f, 0.208928f, -0.173943f, -0.227581f, -0.214798f, 0.079341f, + 0.032344f, -0.253575f, -0.044353f, -0.239265f, -0.055852f, -0.162582f, + -0.086592f, 0.066487f, 0.337353f, -0.168704f, 0.015702f, 0.022607f, + 0.286647f, 0.218106f, 0.193319f, -0.358714f, 0.030796f, 0.007646f, + -0.045617f, 0.165007f, -0.284641f, -0.291812f, 0.207544f, 0.082823f, + -0.141907f, -0.331336f, -0.052908f, 0.120716f, 0.202521f, 0.232782f, + -0.348141f, -0.017332f, 1.191126f, -0.391987f, -0.154537f, -0.206551f, + -2.378690f, 0.057918f, -0.328183f, 2.151556f, 0.238803f, 0.164880f, + -0.480039f, 1.616200f, 0.260243f, 0.083704f, -0.174461f, 1.804634f, + 0.194810f, 0.223837f, 0.550107f, -0.068171f, -0.293435f, -0.186770f, + -0.364846f, 0.127181f, 0.105556f, -0.016202f, 0.278403f, -0.344995f, + -0.009761f, -0.082555f, 0.046731f, -0.301452f, 0.604259f, 0.055895f, + 0.049862f, 0.314249f, -0.305811f, -0.112937f, 0.658787f, -0.549288f, + -0.307567f, -0.460650f, -0.840643f, 0.082576f, 0.373711f, 0.138318f, + 0.336901f, 0.284984f, -0.281400f, 0.408210f, -0.449858f, 0.461054f, + 0.227629f, -0.131705f, 0.301769f, -0.278540f, 0.189290f, -0.269041f, + 0.111350f, -0.300257f, 0.436858f, -0.265920f, -0.211938f, 0.272631f, + 0.206291f, 0.253273f, -0.229776f, -0.031112f, -0.171183f, -0.109676f, + -0.202390f, -0.068857f, 0.182125f, -0.140523f, -0.308742f, -0.045840f, + 0.256545f, -0.262405f, 0.225951f, -0.287463f, -0.189203f, -0.055552f, + -0.052448f, -0.242839f, -0.278877f, 0.140920f, -0.175755f, 0.215402f, + -0.248841f, -0.264080f, -0.178303f, 0.147777f, 0.049460f, -0.279877f, + -0.539725f, -0.004622f, 0.182874f, 0.338814f, 0.265974f, 0.249851f, + -0.141154f, 0.157228f, -0.090972f, 0.179444f, 0.305255f, 0.127788f, + 0.123270f, 0.355320f, 0.076797f, 0.263495f, 0.235965f, -0.133816f, + 0.243624f, 0.227062f, -0.213629f, 0.002075f, 0.061203f, -0.077820f, + -0.008807f, -0.247324f, -0.051464f, -0.191894f, -0.238713f, -0.389526f, + -0.274248f, 0.053950f, -0.225750f, -0.367097f, -0.122391f, 0.181212f, + -0.411824f, -0.084241f, -0.302288f, 0.077860f, -0.187443f, -0.300262f, + 0.083156f, -0.392461f, -0.332320f, -0.346474f, 0.140658f, -0.283656f, + 0.120714f, -0.056577f, -0.280968f, 0.017795f, -0.024686f, 0.073113f, + -0.346637f, 0.082567f, -0.036556f, -0.369730f, 0.081225f, -0.005211f, + 0.144886f, -0.003544f, 0.178307f, -0.366035f, -0.063887f, -0.191767f, + 0.105835f, -0.273978f, -0.266532f, -0.023984f, 0.039166f, 0.065848f, + -0.026802f, -0.268923f, 0.189659f, 0.086300f, 0.030718f, 0.216565f, + -0.130025f, -0.215687f, 0.146341f, -0.286438f, -0.394226f, -0.181509f, + -0.005612f, 0.186040f, 0.133491f, 0.032096f, -0.261609f, 0.074007f, + -0.042929f, -0.234479f, 0.189704f, 0.088395f, -0.003671f, -0.125055f, + -0.252418f, -0.086387f, 0.111197f, -0.297071f, -0.018793f, -0.031902f, + -0.333191f, -0.186279f, 0.039868f, 0.091419f, -0.264438f, -0.216150f, + -0.212550f, 0.203412f, -0.113028f, -0.197169f, -0.346771f, 0.086066f, + 0.091443f, -0.128507f, -0.007281f, -0.118389f, 0.003370f, -0.338661f, + 0.026739f, -0.063571f, -0.281567f, -0.166824f, 0.167455f, 0.216173f, + 0.199163f, 0.256314f, -0.222679f, 0.040282f, -0.154808f, -0.133943f, + -0.270163f, -0.357398f, 0.260373f, 0.176950f, -0.125162f, -0.085050f, + 0.226376f, -0.124585f, -0.324804f, 0.035536f, -0.133600f, 0.173450f, + 0.068107f, -0.337442f, 0.169629f, 0.047223f, 0.057878f, 0.055555f, + -0.317449f, -0.103768f, 0.080899f, -0.194759f, -1.137593f, 0.508999f, + 0.045372f, 1.746454f, 1.250347f, -0.342930f, -0.127821f, -0.220175f, + -0.417649f, -0.480595f, 0.071902f, 0.050231f, -0.562554f, -0.677866f, + -0.121416f, -0.247558f, -0.483876f, -0.504157f, 1.731953f, 0.572936f, + 0.047325f, 0.050619f, 0.112611f, -0.035393f, 0.052585f, -0.071076f, + -0.015798f, -0.050228f, -0.142875f, 0.189329f, 0.048833f, 0.503633f, + 0.249588f, 0.175492f, -0.137664f, -0.018533f, 0.288453f, -0.025644f, + 0.079131f, 0.195096f, -0.154039f, -0.104220f, -0.224072f, 0.095946f, + -0.208424f, 0.214745f, 0.056468f, 0.182603f, 0.341784f, -0.134664f, + -0.194050f, 0.058532f, -0.107336f, -0.087783f, -0.238795f, -0.387212f, + 0.049055f, -0.127417f, -0.299919f, -0.094371f, -0.011735f, -0.264753f, + 0.407375f, -0.462654f, -0.609488f, 0.027742f, -0.985512f, -0.109154f, + -0.423276f, 2.347960f, 0.129240f, 0.187610f, -0.057081f, 2.424892f, + 0.087666f, 0.106716f, -0.039379f, 2.764866f, 0.113309f, 0.028196f, + -0.582789f, 0.335385f, -0.538029f, -0.477337f, -0.114207f, 0.178829f, + 0.006276f, 0.123179f, 0.095101f, 0.139898f, -0.372074f, -0.111010f, + 0.136330f, 0.272900f, 0.126737f, -0.097808f, -0.363697f, 0.108665f, + -0.227749f, -0.083421f, 1.714677f, 0.451943f, 0.107931f, -0.392281f, + 1.615846f, 0.022307f, -0.247011f, 0.257703f, 1.039134f, 0.537789f, + 0.022177f, -0.271532f, 0.351350f, -0.399205f, -0.240534f, -0.315399f, + 0.026928f, -0.005618f, 0.053179f, -0.010277f, 0.000501f, 0.040896f, + -0.109160f, 0.018282f, 0.003887f, 0.199599f, 0.095349f, -0.337284f, + 0.169929f, -0.109409f, -0.166983f, 0.059908f, -0.226574f, -0.120114f, + 0.077329f, -0.333133f, -0.220936f, 0.114309f, -0.233965f, -0.281551f, + 0.042948f, 0.100940f, 0.116037f, -0.313122f, 0.215149f, -0.309057f, + -0.341052f, -0.294417f, -0.179722f, 0.010795f, 0.192053f, -0.275261f, + -0.033077f, 0.117348f, 0.090206f, 0.781573f, 0.602456f, -0.220296f, + 0.172159f, 0.758513f, 0.157910f, -0.217897f, -0.372659f, 0.031935f, + 0.791463f, 0.267195f, 0.931593f, -0.057349f, 0.405512f, -0.058512f, + -0.641663f, -0.076592f, 0.550227f, -0.024094f, 0.048218f, -0.289971f, + 0.180940f, 0.167533f, 0.052711f, -0.360726f, 0.019210f, -0.488879f, + 0.380498f, 0.151608f, -0.276895f, -0.596554f, 0.106076f, -0.245833f, + -0.048783f, 0.073823f, 0.098780f, 0.000211f, 0.113958f, -0.068964f, + -0.265533f, -0.185457f, 0.175586f, -0.163621f, -0.204919f, 0.145802f, + -0.163421f, 0.129576f, -0.153486f, -0.105573f, 0.067289f, -0.213120f, + -0.286103f, 0.249543f, -0.044970f, -0.170464f, -0.105501f, -0.094765f, + -0.050734f, -0.369468f, 0.180020f, -0.363328f, -0.151654f, -0.262550f, + -0.424503f, 0.829032f, -0.559452f, 0.506837f, 0.143823f, 0.276660f, + -1.808608f, -0.259517f, -0.053945f, 0.035676f, -1.842195f, -0.065960f, + -0.069285f, 0.462022f, -2.319453f, -0.370299f, 0.183329f, -0.146412f, + -0.563875f, 0.305068f, 0.480904f, 0.044319f, -0.016098f, 0.168516f, + 0.114874f, -0.097621f, -0.030373f, 0.177700f, 0.181591f, -0.146003f, + -0.330853f, -0.259200f, 0.779319f, -1.517524f, 0.178781f, 0.135451f, + 0.088784f, -2.076089f, 0.628717f, -0.048685f, 0.281327f, -2.341596f, + 0.422171f, 0.006135f, 0.367096f, -1.663118f, 0.365253f, -0.072884f, + -0.197620f, -0.688634f, 0.477354f, 0.395841f, -0.098505f, 0.208709f, + -0.027523f, 0.127119f, 0.106274f, 0.114424f, -0.122877f, -0.087245f, + 0.086923f, -0.527398f, -0.342062f, -0.764662f, 0.713094f, -0.626453f, + -0.081454f, -0.087683f, 0.885047f, 0.323440f, -0.018579f, -0.217166f, + 1.617984f, -0.159038f, 0.265991f, -0.390313f, 1.933182f, -0.032431f, + -0.057513f, -0.300841f, 0.461248f, -0.072147f, -0.287052f, -0.078056f, + 0.011734f, 0.044013f, 0.177174f, 0.093400f, 0.028819f, 0.193686f, + -0.224853f, 0.268321f, -0.075059f, 0.074526f, -0.015618f, 0.165615f, + -0.276780f, -0.063908f, -0.369264f, -0.171497f, -0.173624f, -0.130743f, + -0.224625f, -0.124980f, -0.104482f, 0.076864f, -0.009631f, -0.164682f, + 0.150480f, -0.111880f, -0.260425f, 0.086234f, -0.176936f, -0.136771f, + -0.168867f, -0.405626f, -0.288716f, -0.128950f, -0.207327f, 0.015581f, + -0.109061f, -0.098970f, 0.090792f, -0.109623f, 0.349851f, 0.266341f, + -0.088602f, -0.108071f, 0.082519f, 0.472650f, -1.838758f, 0.456694f, + 0.119927f, 0.461077f, -2.860022f, 0.231495f, 0.235771f, 0.256424f, + -1.938516f, -0.188202f, -0.000832f, -0.518206f, 0.194644f, 0.505510f, + 0.615657f, 0.193760f, 0.224600f, 0.265732f, -0.121553f, -0.354597f, + -0.242414f, -0.276639f, -0.057591f, 0.026369f, -0.261148f, -0.356155f, + -0.149178f, -0.353566f, -0.340835f, -0.141776f, 0.076535f, 0.221299f, + -0.108857f, -0.156514f, 0.050901f, 0.058541f, -0.077141f, 0.071515f, + -0.333283f, -0.181489f, -0.212900f, -0.224698f, -0.174693f, -0.178665f, + -0.143374f, -0.091811f, 0.165161f, 0.060156f, -0.086103f, -0.039031f, + -0.377759f, -0.370533f, 0.074431f, 0.064192f, 0.186576f, 0.447858f, + -0.082260f, -0.020268f, -0.123089f, -0.402017f, 0.080500f, 0.176286f, + 2.850013f, 0.019385f, -0.225361f, -0.235315f, 1.654694f, -0.073978f, + -0.341412f, -1.187575f, 2.815900f, -0.228063f, -0.174547f, 0.623825f, + -0.010676f, 0.157189f, 0.111879f, -0.198965f, 0.051851f, 0.158396f, + 0.045194f, 0.293531f, -0.246714f, -0.351493f, 0.026954f, 0.076233f, + 0.420367f, 0.168154f, -0.131450f, 0.134487f, -0.288851f, -0.134553f, + 0.014902f, 0.756381f, 0.277713f, 0.190080f, -0.020869f, 1.446672f, + 0.029792f, -0.025927f, 0.060640f, 0.559864f, 0.422229f, 0.198459f, + 0.036167f, 0.029432f, 0.001882f, 0.038480f, -0.160528f, -0.288855f, + -0.310886f, 0.291296f, 0.190558f, -0.182816f, -0.002252f, 0.073101f, + -0.172245f, -0.305980f, 0.112492f, -0.422839f, -0.295999f, -0.078160f, + -0.173405f, -0.032819f, 0.373774f, -0.715223f, 0.018911f, 0.131753f, + -0.237364f, -0.128499f, -0.228406f, 0.341619f, 0.343552f, -0.521581f, + -0.263790f, 0.362502f, -0.018450f, 0.054233f, 0.183068f, 0.382772f, + 0.188811f, -0.627287f, 0.040399f, -0.487338f, -0.192591f, 0.247426f, + 0.154372f, -0.483994f, +}; + +static const float av1_early_term_after_split_nn_bias_16_layer0[] = { + -0.173976f, 0.305495f, 0.250981f, -0.067127f, -0.313100f, 0.242464f, + 0.315196f, -0.056052f, -0.241227f, -0.253308f, -0.002697f, 0.003687f, + -0.124421f, -0.090383f, -0.070366f, -0.064074f, -0.056115f, 0.123313f, + -0.239698f, -0.182082f, -0.065296f, 0.021503f, -0.036787f, 0.311861f, + 0.118135f, -0.320456f, -0.110719f, 0.220692f, -0.071727f, -0.088226f, + -0.110874f, -0.111671f, +}; + +static const float av1_early_term_after_split_nn_weights_16_layer1[] = { + -0.338573f, 0.398159f, 0.314774f, -0.037448f, -0.271950f, -0.774991f, + 0.950901f, -0.225380f, -1.841906f, -0.350379f, -0.079350f, 0.383148f, + -0.183676f, -0.313132f, -0.340820f, -0.309401f, -1.050540f, -0.432267f, + -0.657195f, 0.927632f, -0.040150f, 0.578920f, 0.212301f, 0.292495f, + 0.563590f, -0.205735f, 0.195877f, 0.582122f, -0.217860f, 1.613379f, + 0.313278f, -0.555802f, +}; + +static const float av1_early_term_after_split_nn_bias_16_layer1[] = { + 0.16553f, +}; + +static const NN_CONFIG av1_early_term_after_split_nnconfig_16 = { + FEATURES, + 1, + 1, + { + HIDDEN_NODES, + }, + { + av1_early_term_after_split_nn_weights_16_layer0, + av1_early_term_after_split_nn_weights_16_layer1, + }, + { + av1_early_term_after_split_nn_bias_16_layer0, + av1_early_term_after_split_nn_bias_16_layer1, + }, +}; + +static const float av1_early_term_after_split_nn_weights_8_layer0[] = { + -0.719472f, 0.305806f, 0.855829f, 0.100094f, 0.412517f, 1.254673f, + 1.552105f, -5.890773f, -0.089957f, -0.016736f, 1.418074f, -5.393506f, + -0.028214f, 0.117758f, 1.479209f, -5.299794f, 0.171585f, -0.084182f, + -0.162105f, 0.388577f, -0.044319f, -0.025861f, 0.251782f, -0.181462f, + -0.101545f, -0.079999f, -0.033014f, -0.191627f, -0.032802f, -0.053404f, + 0.038038f, -0.119492f, 0.049104f, -0.344384f, -0.354513f, 0.036977f, + 0.017513f, -0.004025f, -0.163212f, -0.261999f, 0.146575f, 0.207541f, + 0.130365f, -0.252127f, 0.097419f, -0.231057f, -0.309421f, 0.347866f, + -0.064670f, -0.283171f, -0.244193f, -0.193323f, -0.226954f, -0.276194f, + -0.233553f, 0.156354f, -0.184009f, 0.344289f, -0.308058f, -0.205202f, + -0.325068f, 0.183820f, -0.361667f, -0.069559f, -0.121834f, -0.038357f, + -0.210043f, -0.266129f, 0.003188f, 0.074902f, -0.328843f, 0.293679f, + -0.234698f, -0.428268f, -0.308772f, -0.136538f, -0.008384f, -0.078227f, + 0.166074f, -0.262899f, 0.102114f, -0.323420f, 0.057064f, -0.203318f, + -0.397413f, -0.317324f, -0.307093f, 0.020574f, -0.188627f, 0.132529f, + 0.118992f, -0.487387f, -0.282975f, 0.573231f, -0.266071f, 0.125140f, + -0.970034f, 1.424008f, -0.487366f, -0.196415f, 3.680273f, -0.008407f, + 0.081109f, -0.187479f, 3.876021f, 0.159168f, 0.111721f, -0.337423f, + 3.901760f, 0.261268f, -0.245555f, -0.187632f, -0.324298f, 0.167234f, + 0.170986f, -0.473055f, 0.087016f, -0.003469f, 0.051035f, 0.251794f, + 0.153549f, 0.217609f, -0.326870f, -0.175511f, 0.637341f, -0.694837f, + -0.873487f, -0.186614f, -1.089884f, -0.607316f, -0.523519f, 5.256331f, + 0.071414f, 0.215265f, -0.835999f, 5.735746f, 0.300101f, 0.089626f, + -0.450261f, 5.608051f, 0.190491f, 0.110220f, -0.595360f, -0.446324f, + 0.311380f, 0.268812f, -0.339656f, -0.008708f, 0.011111f, -0.027557f, + 0.171534f, 0.000676f, 0.227232f, 0.033993f, 0.146684f, 0.094817f, + -0.175381f, -0.211927f, -0.362471f, 0.168834f, 0.264149f, -0.350538f, + -0.463249f, -0.288105f, 0.347155f, 0.183231f, -0.229732f, -0.252202f, + -0.218074f, -0.008769f, -0.156103f, 0.181233f, -0.354736f, 0.263270f, + -0.106636f, 0.081057f, 0.060634f, -0.046887f, 0.050468f, 0.071259f, + 0.221287f, 0.199071f, -0.180185f, -0.406902f, -0.239351f, -0.034957f, + 0.369140f, 0.864600f, 0.233798f, 0.423612f, -0.468918f, 0.976987f, + 0.691198f, -1.597908f, 0.102926f, 0.305546f, 0.391196f, -3.909059f, + 0.333635f, 0.311561f, 0.738886f, -4.002001f, 0.236394f, -0.233141f, + 0.263342f, 0.679898f, 0.136233f, 0.254743f, -0.367571f, 0.066412f, + 0.001606f, -0.059542f, 0.051726f, -0.347145f, -0.045501f, -0.313847f, + -0.021952f, 1.386316f, -0.579139f, -1.275844f, -0.003493f, -1.716577f, + 0.250209f, 0.192086f, 4.177055f, 0.351835f, 0.338177f, 0.140163f, + 4.099592f, 0.321866f, -0.128153f, -0.360414f, 4.350767f, 0.025943f, + -0.116740f, -0.664107f, -0.064558f, -0.039553f, -0.208186f, -0.678774f, + 0.149441f, -0.019823f, 0.012759f, 0.404442f, -0.108881f, 0.067974f, + -0.188278f, 0.136327f, 0.109927f, -0.179270f, -0.272342f, 0.018064f, + -0.304216f, -0.469470f, 0.109310f, -0.326214f, 0.061909f, -0.278997f, + -0.352329f, -0.333770f, -0.186522f, -0.328567f, -0.206211f, -0.008804f, + 0.042441f, -0.126699f, -0.420399f, -0.033842f, 0.016773f, -0.273789f, + 0.081928f, -0.191552f, -0.179533f, -0.263070f, -0.471807f, 0.062601f, + -0.232576f, 0.082955f, -0.490080f, 0.073820f, -0.090384f, 0.035781f, + -0.158880f, -0.506793f, -0.069132f, 0.047602f, -0.349640f, -0.058389f, + -0.017387f, -0.194636f, -0.457227f, -0.143105f, 0.222045f, -0.548909f, + -0.131561f, 0.247196f, -0.207923f, 0.133056f, -0.509854f, -0.193685f, + -0.181327f, -0.242442f, 0.091821f, 0.114430f, -0.375233f, -0.015254f, + -0.336632f, -0.060279f, -0.169169f, -0.429914f, -0.036563f, -0.400560f, + -0.076332f, -0.186232f, -0.268491f, 0.075561f, -0.389082f, -0.077435f, + 0.352562f, -0.020086f, -0.338181f, -0.404629f, 0.254983f, 0.150477f, + -0.265903f, 0.003341f, 0.099969f, -0.211964f, -0.129372f, -0.166366f, + 0.327712f, -0.276234f, 0.140675f, -0.433677f, -0.163050f, -0.143578f, + -0.397840f, -0.422130f, -0.293835f, -0.075362f, -0.468375f, 1.021238f, + 1.394155f, -0.922486f, -1.350222f, 2.030201f, 0.057717f, 0.227650f, + -0.193179f, 0.037224f, 0.065555f, 0.020558f, -0.059205f, -0.023690f, + -0.008718f, 0.095976f, -0.549587f, -0.321164f, -0.243728f, 1.344381f, + -1.254107f, 0.294244f, -0.154737f, -0.152597f, 0.342419f, 0.301883f, + 0.069866f, -0.327766f, 0.209323f, -0.364913f, -0.005530f, -0.558972f, + 0.057684f, -0.309357f, -0.283325f, -0.278445f, -0.420115f, -0.418457f, + -0.391481f, -0.418460f, -0.003897f, -0.023744f, -0.312330f, -0.366213f, + 0.269628f, -0.274877f, -0.189988f, -0.419555f, -0.034033f, 0.192874f, + -0.135487f, -0.326108f, -0.039019f, 0.185029f, -0.264883f, -0.563447f, + -0.163532f, -0.447652f, -0.141851f, 0.001714f, -0.193184f, 0.032609f, + -0.112883f, 0.074599f, 0.490665f, 0.434764f, 0.021652f, -0.219618f, + 0.743267f, 0.147195f, -0.303479f, -0.097674f, 0.195813f, 0.704007f, + -1.290851f, 0.119701f, 0.224065f, 0.260246f, -0.580657f, -0.096201f, + -0.333214f, -0.586689f, 0.567178f, 0.157340f, -0.043184f, 0.194358f, + -0.026506f, -0.339894f, -0.571803f, -0.234828f, 0.147054f, -0.564178f, + -0.156933f, -0.366055f, -0.691687f, -0.187501f, 0.215834f, -0.346106f, + -0.256892f, 0.110915f, -0.337464f, -0.341474f, -0.216113f, 0.249445f, + -0.070175f, -0.412141f, 0.153458f, -0.081280f, 0.164669f, -0.356396f, + -0.294971f, -0.165121f, -0.133585f, -0.071467f, 0.295147f, -0.253233f, + -0.213833f, -0.343416f, -0.474344f, -0.304000f, -0.341379f, -0.331456f, + -0.393952f, -0.508004f, -0.569518f, -0.509864f, 0.121961f, 0.011957f, + 0.000498f, -0.201969f, -0.407195f, -0.414375f, -0.295846f, 0.247492f, + 0.124249f, -0.550804f, -0.420397f, -0.123462f, 0.333292f, -0.240230f, + -0.025604f, 0.337536f, -0.295006f, -0.272614f, -0.496850f, -0.278521f, + 0.234591f, -0.052775f, -0.014052f, -0.260078f, -0.279128f, -0.036385f, + 0.008714f, -0.064018f, -0.124873f, -0.334014f, +}; + +static const float av1_early_term_after_split_nn_bias_8_layer0[] = { + 1.202379f, -0.117005f, -0.135527f, -0.262255f, -0.443658f, -0.078981f, + 0.615653f, -0.124482f, -0.227768f, -0.227014f, -0.135898f, 0.143216f, + -0.225995f, 0.370877f, -0.214821f, -0.227752f, +}; + +static const float av1_early_term_after_split_nn_weights_8_layer1[] = { + 0.376594f, 0.266703f, -0.039847f, 1.680142f, -0.879939f, 0.286806f, + -0.378223f, -0.405295f, -0.021107f, 0.039188f, 0.259308f, 0.193091f, + 0.077994f, -0.269141f, 0.011180f, -0.019262f, +}; + +static const float av1_early_term_after_split_nn_bias_8_layer1[] = { + -1.29585564f, +}; + +static const NN_CONFIG av1_early_term_after_split_nnconfig_8 = { + FEATURES, + 1, + 1, + { + 16, + }, + { + av1_early_term_after_split_nn_weights_8_layer0, + av1_early_term_after_split_nn_weights_8_layer1, + }, + { + av1_early_term_after_split_nn_bias_8_layer0, + av1_early_term_after_split_nn_bias_8_layer1, + }, +}; +#undef FEATURES +#undef HIDDEN_NODES + #ifdef __cplusplus } // extern "C" #endif diff --git a/third_party/aom/av1/encoder/partition_strategy.c b/third_party/aom/av1/encoder/partition_strategy.c index 2dace6c167..3d2a947f0b 100644 --- a/third_party/aom/av1/encoder/partition_strategy.c +++ b/third_party/aom/av1/encoder/partition_strategy.c @@ -21,109 +21,73 @@ #include "av1/encoder/partition_strategy.h" #include "av1/encoder/rdopt.h" -// Performs a simple_motion_search with a single reference frame and extract -// the variance of residues. Here features is assumed to be a length 6 array. -// After this function is called, we will store the following in to features: -// features[0] = log(1 + dc_q**2/256) -// features[1] = log(1 + variance_of_residue) -// for i in [2, 3, 4, 5]: -// features[i] = log(1 + variance_of_residue_in_block[i]/variance_of_residue) -static void get_res_var_features(AV1_COMP *const cpi, MACROBLOCK *x, int mi_row, - int mi_col, BLOCK_SIZE bsize, - float *features) { - // TODO(chiyotsai@google.com): The data this model trained on did not also use - // SIMPLE_TRANSLATION to build the inter_predictor. Retraining and tuning the - // model with the correct data should give better performance. - assert(mi_size_wide[bsize] == mi_size_high[bsize]); +static void simple_motion_search_prune_part_features( + AV1_COMP *const cpi, MACROBLOCK *x, PC_TREE *pc_tree, int mi_row, + int mi_col, BLOCK_SIZE bsize, float *features, int features_to_get); - MACROBLOCKD *xd = &x->e_mbd; - - // Perform a single motion search in Y_PLANE to make a prediction - const int use_subpixel = 0; - - // Start getting the features - int f_idx = 0; - - // Q_INDEX - const int dc_q = av1_dc_quant_QTX(x->qindex, 0, xd->bd) >> (xd->bd - 8); - aom_clear_system_state(); - features[f_idx++] = logf(1.0f + (float)(dc_q * dc_q) / 256.0f); - - // VARIANCE - unsigned int sse = 0; - unsigned int var = 0; - const MV ref_mv_full = { .row = 0, .col = 0 }; - av1_simple_motion_sse_var(cpi, x, mi_row, mi_col, bsize, ref_mv_full, - use_subpixel, &sse, &var); - aom_clear_system_state(); - features[f_idx++] = logf(1.0f + (float)var); - - // Regional - const uint8_t *src = x->plane[0].src.buf; - const int src_stride = x->plane[0].src.stride; - const uint8_t *dst = xd->plane[0].dst.buf; - const int dst_stride = xd->plane[0].dst.stride; - const int bw = block_size_wide[bsize]; - const int bh = block_size_high[bsize]; - const BLOCK_SIZE subsize = get_partition_subsize(bsize, PARTITION_SPLIT); - int r_idx = 0; - for (r_idx = 0; r_idx < 4; r_idx++) { - const int x_idx = (r_idx & 1) * bw / 2; - const int y_idx = (r_idx >> 1) * bh / 2; - const int src_offset = y_idx * src_stride + x_idx; - const int dst_offset = y_idx * dst_stride + x_idx; - const unsigned int sub_var = cpi->fn_ptr[subsize].vf( - src + src_offset, src_stride, dst + dst_offset, dst_stride, &sse); - aom_clear_system_state(); - const float var_ratio = (1.0f + (float)sub_var) / (4.0f + (float)var); - features[f_idx++] = var_ratio; +static INLINE int convert_bsize_to_idx(BLOCK_SIZE bsize) { + switch (bsize) { + case BLOCK_128X128: return 0; + case BLOCK_64X64: return 1; + case BLOCK_32X32: return 2; + case BLOCK_16X16: return 3; + case BLOCK_8X8: return 4; + default: assert(0 && "Invalid bsize"); return -1; } } void av1_simple_motion_search_based_split( - AV1_COMP *const cpi, MACROBLOCK *x, int mi_row, int mi_col, - BLOCK_SIZE bsize, int *partition_none_allowed, int *partition_horz_allowed, - int *partition_vert_allowed, int *do_rectangular_split, - int *do_square_split) { - const NN_CONFIG *nn_config = NULL; - float split_only_thresh = 0.0f; - if (bsize == BLOCK_128X128) { - nn_config = &av1_simple_motion_search_based_split_nn_config_128; - split_only_thresh = av1_simple_motion_search_based_split_thresh_128; - } else if (bsize == BLOCK_64X64) { - nn_config = &av1_simple_motion_search_based_split_nn_config_64; - split_only_thresh = av1_simple_motion_search_based_split_thresh_64; - } else if (bsize == BLOCK_32X32) { - nn_config = &av1_simple_motion_search_based_split_nn_config_32; - split_only_thresh = av1_simple_motion_search_based_split_thresh_32; - } else if (bsize == BLOCK_16X16) { - nn_config = &av1_simple_motion_search_based_split_nn_config_16; - split_only_thresh = av1_simple_motion_search_based_split_thresh_16; - } else if (bsize == BLOCK_8X8) { - // Disable BLOCK_8X8 for now -#if !CONFIG_DISABLE_FULL_PIXEL_SPLIT_8X8 - nn_config = &av1_simple_motion_search_based_split_nn_config_8; - split_only_thresh = av1_simple_motion_search_based_split_thresh_8; -#endif - } else { - assert(0 && "Unexpected block size in simple_motion_based_split"); - } - if (nn_config) { - float features[6] = { 0 }; - float score = 0; - get_res_var_features(cpi, x, mi_row, mi_col, bsize, features); - av1_nn_predict(features, nn_config, &score); + AV1_COMP *const cpi, MACROBLOCK *x, PC_TREE *pc_tree, int mi_row, + int mi_col, BLOCK_SIZE bsize, int *partition_none_allowed, + int *partition_horz_allowed, int *partition_vert_allowed, + int *do_rectangular_split, int *do_square_split) { + aom_clear_system_state(); - if (score > split_only_thresh) { - *partition_none_allowed = 0; - *partition_horz_allowed = 0; - *partition_vert_allowed = 0; - *do_rectangular_split = 0; - } - // TODO(Venkat): Experiment to skip only rectangular/extended parititions - if (cpi->sf.simple_motion_search_split_only >= 2) { - if (score < -split_only_thresh) *do_square_split = 0; - } + const AV1_COMMON *const cm = &cpi->common; + const int is_480p_or_larger = AOMMIN(cm->width, cm->height) >= 480; + const int bsize_idx = convert_bsize_to_idx(bsize); + + assert(bsize_idx >= 0 && bsize_idx <= 4 && + "Invalid bsize in simple_motion_search_based_split"); + + float split_only_thresh = 100.0f, no_split_thresh = -100.0f; + + const float *ml_mean = av1_simple_motion_search_split_mean[bsize_idx]; + const float *ml_std = av1_simple_motion_search_split_std[bsize_idx]; + const NN_CONFIG *nn_config = + av1_simple_motion_search_split_nn_config[bsize_idx]; + if (is_480p_or_larger) { + split_only_thresh = av1_simple_motion_search_split_midres_thresh[bsize_idx]; + no_split_thresh = + av1_simple_motion_search_split_midres_no_thresh[bsize_idx]; + } else { + split_only_thresh = av1_simple_motion_search_split_lowres_thresh[bsize_idx]; + no_split_thresh = + av1_simple_motion_search_split_lowres_no_thresh[bsize_idx]; + } + + float features[FEATURE_SIZE_SMS_SPLIT] = { 0.0f }; + simple_motion_search_prune_part_features(cpi, x, pc_tree, mi_row, mi_col, + bsize, features, + FEATURE_SMS_SPLIT_MODEL_FLAG); + for (int idx = 0; idx < FEATURE_SIZE_SMS_SPLIT; idx++) { + features[idx] = (features[idx] - ml_mean[idx]) / ml_std[idx]; + } + + float score = 0.0f; + + av1_nn_predict(features, nn_config, &score); + aom_clear_system_state(); + + if (score > split_only_thresh) { + *partition_none_allowed = 0; + *partition_horz_allowed = 0; + *partition_vert_allowed = 0; + *do_rectangular_split = 0; + } + + if (cpi->sf.simple_motion_search_split >= 2 && score < no_split_thresh) { + *do_square_split = 0; } } @@ -206,71 +170,94 @@ static int simple_motion_search_get_best_ref( // features[18] = DC q_index static void simple_motion_search_prune_part_features( AV1_COMP *const cpi, MACROBLOCK *x, PC_TREE *pc_tree, int mi_row, - int mi_col, BLOCK_SIZE bsize, float *features) { + int mi_col, BLOCK_SIZE bsize, float *features, int features_to_get) { // TODO(chiyotsai@google.com): Cache the result of the motion search from the // larger bsize. const int w_mi = mi_size_wide[bsize]; const int h_mi = mi_size_high[bsize]; - int f_idx = 0; assert(mi_size_wide[bsize] == mi_size_high[bsize]); assert(cpi->ref_frame_flags & av1_ref_frame_flag_list[LAST_FRAME] || cpi->ref_frame_flags & av1_ref_frame_flag_list[ALTREF_FRAME]); // Setting up motion search - const int ref_list[] = { LAST_FRAME, ALTREF_FRAME }; - const int num_refs = 2; + const int ref_list[] = { cpi->rc.is_src_frame_alt_ref ? ALTREF_FRAME + : LAST_FRAME }; + const int num_refs = 1; const int use_subpixel = 1; - unsigned int int_features[FEATURE_SIZE_SMS_PRUNE_PART - 1]; - // Doing whole block first to update the mv - simple_motion_search_get_best_ref( - cpi, x, pc_tree, mi_row, mi_col, bsize, ref_list, num_refs, use_subpixel, - 4, &int_features[f_idx], &int_features[f_idx + 1]); - f_idx += 2; + if (!pc_tree->sms_none_valid && features_to_get & FEATURE_SMS_NONE_FLAG) { + simple_motion_search_get_best_ref(cpi, x, pc_tree, mi_row, mi_col, bsize, + ref_list, num_refs, use_subpixel, 4, + &pc_tree->sms_none_feat[0], + &pc_tree->sms_none_feat[1]); + pc_tree->sms_none_valid = 1; + } // Split subblocks - BLOCK_SIZE subsize = get_partition_subsize(bsize, PARTITION_SPLIT); - int r_idx = 0; - for (r_idx = 0; r_idx < 4; r_idx++) { - const int sub_mi_col = mi_col + (r_idx & 1) * w_mi / 2; - const int sub_mi_row = mi_row + (r_idx >> 1) * h_mi / 2; + if (!pc_tree->sms_split_valid && features_to_get & FEATURE_SMS_SPLIT_FLAG) { + const BLOCK_SIZE subsize = get_partition_subsize(bsize, PARTITION_SPLIT); + for (int r_idx = 0; r_idx < 4; r_idx++) { + const int sub_mi_col = mi_col + (r_idx & 1) * w_mi / 2; + const int sub_mi_row = mi_row + (r_idx >> 1) * h_mi / 2; - simple_motion_search_get_best_ref( - cpi, x, pc_tree, sub_mi_row, sub_mi_col, subsize, ref_list, num_refs, - use_subpixel, r_idx, &int_features[f_idx], &int_features[f_idx + 1]); - f_idx += 2; + simple_motion_search_get_best_ref( + cpi, x, pc_tree, sub_mi_row, sub_mi_col, subsize, ref_list, num_refs, + use_subpixel, r_idx, &pc_tree->sms_split_feat[2 * r_idx], + &pc_tree->sms_split_feat[2 * r_idx + 1]); + } + + pc_tree->sms_split_valid = 1; } - // Horz subblocks - subsize = get_partition_subsize(bsize, PARTITION_HORZ); - for (r_idx = 0; r_idx < 2; r_idx++) { - const int sub_mi_col = mi_col + 0; - const int sub_mi_row = mi_row + r_idx * h_mi / 2; + // Rectangular subblocks + if (!pc_tree->sms_rect_valid && features_to_get & FEATURE_SMS_RECT_FLAG) { + // Horz subblock + BLOCK_SIZE subsize = get_partition_subsize(bsize, PARTITION_HORZ); + for (int r_idx = 0; r_idx < 2; r_idx++) { + const int sub_mi_col = mi_col + 0; + const int sub_mi_row = mi_row + r_idx * h_mi / 2; - simple_motion_search_get_best_ref( - cpi, x, pc_tree, sub_mi_row, sub_mi_col, subsize, ref_list, num_refs, - use_subpixel, -1, &int_features[f_idx], &int_features[f_idx + 1]); + simple_motion_search_get_best_ref( + cpi, x, pc_tree, sub_mi_row, sub_mi_col, subsize, ref_list, num_refs, + use_subpixel, -1, &pc_tree->sms_rect_feat[2 * r_idx], + &pc_tree->sms_rect_feat[2 * r_idx + 1]); + } - f_idx += 2; + // Vert subblock + subsize = get_partition_subsize(bsize, PARTITION_VERT); + for (int r_idx = 0; r_idx < 2; r_idx++) { + const int sub_mi_col = mi_col + r_idx * w_mi / 2; + const int sub_mi_row = mi_row + 0; + + simple_motion_search_get_best_ref( + cpi, x, pc_tree, sub_mi_row, sub_mi_col, subsize, ref_list, num_refs, + use_subpixel, -1, &pc_tree->sms_rect_feat[4 + 2 * r_idx], + &pc_tree->sms_rect_feat[4 + 2 * r_idx + 1]); + } + pc_tree->sms_rect_valid = 1; } - // Vert subblock - subsize = get_partition_subsize(bsize, PARTITION_VERT); - for (r_idx = 0; r_idx < 2; r_idx++) { - const int sub_mi_col = mi_col + r_idx * w_mi / 2; - const int sub_mi_row = mi_row + 0; - - simple_motion_search_get_best_ref( - cpi, x, pc_tree, sub_mi_row, sub_mi_col, subsize, ref_list, num_refs, - use_subpixel, -1, &int_features[f_idx], &int_features[f_idx + 1]); - - f_idx += 2; - } + if (!features) return; aom_clear_system_state(); - for (int idx = 0; idx < f_idx; idx++) { - features[idx] = logf(1.0f + (float)int_features[idx]); + int f_idx = 0; + if (features_to_get & FEATURE_SMS_NONE_FLAG) { + for (int sub_idx = 0; sub_idx < 2; sub_idx++) { + features[f_idx++] = logf(1.0f + pc_tree->sms_none_feat[sub_idx]); + } + } + + if (features_to_get & FEATURE_SMS_SPLIT_FLAG) { + for (int sub_idx = 0; sub_idx < 8; sub_idx++) { + features[f_idx++] = logf(1.0f + pc_tree->sms_split_feat[sub_idx]); + } + } + + if (features_to_get & FEATURE_SMS_RECT_FLAG) { + for (int sub_idx = 0; sub_idx < 8; sub_idx++) { + features[f_idx++] = logf(1.0f + pc_tree->sms_rect_feat[sub_idx]); + } } const MACROBLOCKD *xd = &x->e_mbd; @@ -291,8 +278,6 @@ static void simple_motion_search_prune_part_features( features[f_idx++] = (float)has_left; features[f_idx++] = (float)mi_size_wide_log2[left_bsize]; features[f_idx++] = (float)mi_size_high_log2[left_bsize]; - - assert(f_idx == FEATURE_SIZE_SMS_PRUNE_PART); } void av1_simple_motion_search_prune_part( @@ -300,13 +285,13 @@ void av1_simple_motion_search_prune_part( int mi_col, BLOCK_SIZE bsize, int *partition_none_allowed, int *partition_horz_allowed, int *partition_vert_allowed, int *do_square_split, int *do_rectangular_split, int *prune_horz, - int *prune_vert, float *features, int *valid) { + int *prune_vert) { const AV1_COMMON *const cm = &cpi->common; // Get model parameters const NN_CONFIG *nn_config = NULL; const float *prune_thresh = NULL, *only_thresh = NULL; const float *ml_mean = NULL, *ml_std = NULL; - float normalized_features[FEATURE_SIZE_SMS_PRUNE_PART] = { 0.0f }; + float features[FEATURE_SIZE_SMS_PRUNE_PART] = { 0.0f }; if (bsize == BLOCK_128X128) { nn_config = &av1_simple_motion_search_prune_part_nn_config_128; @@ -353,11 +338,10 @@ void av1_simple_motion_search_prune_part( // Get features simple_motion_search_prune_part_features(cpi, x, pc_tree, mi_row, mi_col, - bsize, features); - *valid = 1; + bsize, features, + FEATURE_SMS_PRUNE_PART_FLAG); for (int f_idx = 0; f_idx < FEATURE_SIZE_SMS_PRUNE_PART; f_idx++) { - normalized_features[f_idx] = - (features[f_idx] - ml_mean[f_idx]) / ml_std[f_idx]; + features[f_idx] = (features[f_idx] - ml_mean[f_idx]) / ml_std[f_idx]; } // Get probabilities @@ -367,7 +351,7 @@ void av1_simple_motion_search_prune_part( ? PARTITION_TYPES : EXT_PARTITION_TYPES; - av1_nn_predict(normalized_features, nn_config, scores); + av1_nn_predict(features, nn_config, scores); aom_clear_system_state(); av1_nn_softmax(scores, probs, num_classes); @@ -393,24 +377,23 @@ void av1_simple_motion_search_prune_part( // - The frame is not intra only // - The current bsize is > BLOCK_8X8 // - blk_row + blk_height/2 < total_rows and blk_col + blk_width/2 < total_cols -void av1_simple_motion_search_early_term_none( - AV1_COMP *const cpi, MACROBLOCK *x, PC_TREE *pc_tree, int mi_row, - int mi_col, BLOCK_SIZE bsize, const RD_STATS *none_rdc, - int *early_terminate, float *simple_motion_features, - int *simple_motion_features_are_valid) { +void av1_simple_motion_search_early_term_none(AV1_COMP *const cpi, + MACROBLOCK *x, PC_TREE *pc_tree, + int mi_row, int mi_col, + BLOCK_SIZE bsize, + const RD_STATS *none_rdc, + int *early_terminate) { // TODO(chiyotsai@google.com): There are other features we can extract from // PARTITION_NONE. Play with this later. - int f_idx = 0; - if (!*simple_motion_features_are_valid) { - simple_motion_search_prune_part_features(cpi, x, pc_tree, mi_row, mi_col, - bsize, simple_motion_features); - *simple_motion_features_are_valid = 1; - } - f_idx = 25; + float features[FEATURE_SIZE_SMS_TERM_NONE] = { 0.0f }; + simple_motion_search_prune_part_features(cpi, x, pc_tree, mi_row, mi_col, + bsize, features, + FEATURE_SMS_PRUNE_PART_FLAG); + int f_idx = FEATURE_SIZE_SMS_PRUNE_PART; - simple_motion_features[f_idx++] = logf(1.0f + (float)none_rdc->rate); - simple_motion_features[f_idx++] = logf(1.0f + (float)none_rdc->dist); - simple_motion_features[f_idx++] = logf(1.0f + (float)none_rdc->rdcost); + features[f_idx++] = logf(1.0f + (float)none_rdc->rate); + features[f_idx++] = logf(1.0f + (float)none_rdc->dist); + features[f_idx++] = logf(1.0f + (float)none_rdc->rdcost); assert(f_idx == FEATURE_SIZE_SMS_TERM_NONE); @@ -441,8 +424,8 @@ void av1_simple_motion_search_early_term_none( if (ml_model) { float score = 0.0f; for (f_idx = 0; f_idx < FEATURE_SIZE_SMS_TERM_NONE; f_idx++) { - score += ml_model[f_idx] * - (simple_motion_features[f_idx] - ml_mean[f_idx]) / ml_std[f_idx]; + score += + ml_model[f_idx] * (features[f_idx] - ml_mean[f_idx]) / ml_std[f_idx]; } score += ml_model[FEATURE_SIZE_SMS_TERM_NONE]; @@ -720,3 +703,584 @@ BLOCK_SIZE av1_predict_max_partition(AV1_COMP *const cpi, MACROBLOCK *const x, return (BLOCK_SIZE)((result + 2) * 3); } + +// Get the minimum partition block width and height(in log scale) under a +// PC_TREE. +static void get_min_bsize(const PC_TREE *pc_tree, int *min_bw, int *min_bh) { + if (!pc_tree) return; + + const BLOCK_SIZE bsize = pc_tree->block_size; + if (bsize == BLOCK_4X4) { + *min_bw = 0; + *min_bh = 0; + return; + } + + PARTITION_TYPE part_type = pc_tree->partitioning; + if (part_type == PARTITION_INVALID) return; + + if (part_type == PARTITION_SPLIT) { + for (int i = 0; i < 4; ++i) { + get_min_bsize(pc_tree->split[i], min_bw, min_bh); + } + } else { + if (part_type == PARTITION_HORZ_A || part_type == PARTITION_HORZ_B || + part_type == PARTITION_VERT_A || part_type == PARTITION_VERT_B) + part_type = PARTITION_SPLIT; + const BLOCK_SIZE subsize = get_partition_subsize(bsize, part_type); + if (subsize != BLOCK_INVALID) { + *min_bw = AOMMIN(*min_bw, mi_size_wide_log2[subsize]); + *min_bh = AOMMIN(*min_bh, mi_size_high_log2[subsize]); + } + } +} + +static INLINE void add_rd_feature(int64_t rd, int64_t best_rd, float *features, + int *feature_idx) { + const int rd_valid = rd > 0 && rd < INT64_MAX; + const float rd_ratio = rd_valid ? (float)rd / best_rd : 1.0f; + features[(*feature_idx)++] = (float)rd_valid; + features[(*feature_idx)++] = rd_ratio; +} + +#define FEATURES 31 +void av1_ml_early_term_after_split(AV1_COMP *const cpi, MACROBLOCK *const x, + PC_TREE *const pc_tree, BLOCK_SIZE bsize, + int64_t best_rd, int64_t part_none_rd, + int64_t part_split_rd, + int64_t *split_block_rd, int mi_row, + int mi_col, + int *const terminate_partition_search) { + if (best_rd <= 0 || best_rd == INT64_MAX || *terminate_partition_search) + return; + + const AV1_COMMON *const cm = &cpi->common; + const int is_480p_or_larger = AOMMIN(cm->width, cm->height) >= 480; + const NN_CONFIG *nn_config = NULL; + float thresh = -1e6; + switch (bsize) { + case BLOCK_128X128: break; + case BLOCK_64X64: + nn_config = &av1_early_term_after_split_nnconfig_64; + thresh = is_480p_or_larger ? -2.0f : -1.2f; + break; + case BLOCK_32X32: + nn_config = &av1_early_term_after_split_nnconfig_32; + thresh = is_480p_or_larger ? -2.6f : -2.3f; + break; + case BLOCK_16X16: + nn_config = &av1_early_term_after_split_nnconfig_16; + thresh = is_480p_or_larger ? -2.0f : -2.4f; + break; + case BLOCK_8X8: + nn_config = &av1_early_term_after_split_nnconfig_8; + thresh = is_480p_or_larger ? -1.0f : -1.4f; + break; + case BLOCK_4X4: break; + default: + assert(0 && "Invalid block size in av1_ml_early_term_after_split()."); + break; + } + if (!nn_config) return; + + // Use more conservative threshold for level 1. + if (cpi->sf.ml_early_term_after_part_split_level < 2) thresh -= 0.3f; + + const MACROBLOCKD *const xd = &x->e_mbd; + const int dc_q = av1_dc_quant_QTX(x->qindex, 0, xd->bd) >> (xd->bd - 8); + const int bs = block_size_wide[bsize]; + int f_idx = 0; + float features[FEATURES] = { 0.0f }; + + aom_clear_system_state(); + + features[f_idx++] = logf(1.0f + (float)dc_q / 4.0f); + features[f_idx++] = logf(1.0f + (float)best_rd / bs / bs / 1024.0f); + + add_rd_feature(part_none_rd, best_rd, features, &f_idx); + add_rd_feature(part_split_rd, best_rd, features, &f_idx); + + for (int i = 0; i < 4; ++i) { + add_rd_feature(split_block_rd[i], best_rd, features, &f_idx); + int min_bw = MAX_SB_SIZE_LOG2; + int min_bh = MAX_SB_SIZE_LOG2; + get_min_bsize(pc_tree->split[i], &min_bw, &min_bh); + features[f_idx++] = (float)min_bw; + features[f_idx++] = (float)min_bh; + } + + simple_motion_search_prune_part_features(cpi, x, pc_tree, mi_row, mi_col, + bsize, NULL, + FEATURE_SMS_PRUNE_PART_FLAG); + + features[f_idx++] = logf(1.0f + (float)pc_tree->sms_none_feat[1]); + + features[f_idx++] = logf(1.0f + (float)pc_tree->sms_split_feat[1]); + features[f_idx++] = logf(1.0f + (float)pc_tree->sms_split_feat[3]); + features[f_idx++] = logf(1.0f + (float)pc_tree->sms_split_feat[5]); + features[f_idx++] = logf(1.0f + (float)pc_tree->sms_split_feat[7]); + + features[f_idx++] = logf(1.0f + (float)pc_tree->sms_rect_feat[1]); + features[f_idx++] = logf(1.0f + (float)pc_tree->sms_rect_feat[3]); + features[f_idx++] = logf(1.0f + (float)pc_tree->sms_rect_feat[5]); + features[f_idx++] = logf(1.0f + (float)pc_tree->sms_rect_feat[7]); + + assert(f_idx == FEATURES); + + float score = 0.0f; + av1_nn_predict(features, nn_config, &score); + // Score is indicator of confidence that we should NOT terminate. + if (score < thresh) *terminate_partition_search = 1; +} +#undef FEATURES + +#define FEATURE_SIZE 19 +int av1_ml_prune_2pass_split_partition(const PC_TREE_STATS *pc_tree_stats, + BLOCK_SIZE bsize, int *split_score, + int *none_score) { + if (!pc_tree_stats->valid) return 0; + const float *split_weights = NULL; + const float *none_weights = NULL; + switch (bsize) { + case BLOCK_4X4: break; + case BLOCK_8X8: + split_weights = av1_2pass_split_partition_weights_8; + none_weights = av1_2pass_none_partition_weights_8; + break; + case BLOCK_16X16: + split_weights = av1_2pass_split_partition_weights_16; + none_weights = av1_2pass_none_partition_weights_16; + break; + case BLOCK_32X32: + split_weights = av1_2pass_split_partition_weights_32; + none_weights = av1_2pass_none_partition_weights_32; + break; + case BLOCK_64X64: + split_weights = av1_2pass_split_partition_weights_64; + none_weights = av1_2pass_none_partition_weights_64; + break; + case BLOCK_128X128: + split_weights = av1_2pass_split_partition_weights_128; + none_weights = av1_2pass_none_partition_weights_128; + break; + default: assert(0 && "Unexpected bsize."); + } + if (!split_weights || !none_weights) return 0; + + aom_clear_system_state(); + + float features[FEATURE_SIZE]; + int feature_index = 0; + features[feature_index++] = (float)pc_tree_stats->split; + features[feature_index++] = (float)pc_tree_stats->skip; + const int rdcost = (int)AOMMIN(INT_MAX, pc_tree_stats->rdcost); + const int rd_valid = rdcost > 0 && rdcost < 1000000000; + features[feature_index++] = (float)rd_valid; + for (int i = 0; i < 4; ++i) { + features[feature_index++] = (float)pc_tree_stats->sub_block_split[i]; + features[feature_index++] = (float)pc_tree_stats->sub_block_skip[i]; + const int sub_rdcost = + (int)AOMMIN(INT_MAX, pc_tree_stats->sub_block_rdcost[i]); + const int sub_rd_valid = sub_rdcost > 0 && sub_rdcost < 1000000000; + features[feature_index++] = (float)sub_rd_valid; + // Ratio between the sub-block RD and the whole-block RD. + float rd_ratio = 1.0f; + if (rd_valid && sub_rd_valid && sub_rdcost < rdcost) + rd_ratio = (float)sub_rdcost / (float)rdcost; + features[feature_index++] = rd_ratio; + } + assert(feature_index == FEATURE_SIZE); + + float score_1 = split_weights[FEATURE_SIZE]; + float score_2 = none_weights[FEATURE_SIZE]; + for (int i = 0; i < FEATURE_SIZE; ++i) { + score_1 += features[i] * split_weights[i]; + score_2 += features[i] * none_weights[i]; + } + *split_score = (int)(score_1 * 100); + *none_score = (int)(score_2 * 100); + return 1; +} +#undef FEATURE_SIZE + +void av1_ml_prune_rect_partition(const AV1_COMP *const cpi, + const MACROBLOCK *const x, BLOCK_SIZE bsize, + int64_t best_rd, int64_t none_rd, + int64_t *split_rd, int *const dst_prune_horz, + int *const dst_prune_vert) { + if (bsize < BLOCK_8X8 || best_rd >= 1000000000) return; + best_rd = AOMMAX(best_rd, 1); + const NN_CONFIG *nn_config = NULL; + const float prob_thresholds[5] = { 0.01f, 0.01f, 0.004f, 0.002f, 0.002f }; + float cur_thresh = 0.0f; + switch (bsize) { + case BLOCK_8X8: + nn_config = &av1_rect_partition_nnconfig_8; + cur_thresh = prob_thresholds[0]; + break; + case BLOCK_16X16: + nn_config = &av1_rect_partition_nnconfig_16; + cur_thresh = prob_thresholds[1]; + break; + case BLOCK_32X32: + nn_config = &av1_rect_partition_nnconfig_32; + cur_thresh = prob_thresholds[2]; + break; + case BLOCK_64X64: + nn_config = &av1_rect_partition_nnconfig_64; + cur_thresh = prob_thresholds[3]; + break; + case BLOCK_128X128: + nn_config = &av1_rect_partition_nnconfig_128; + cur_thresh = prob_thresholds[4]; + break; + default: assert(0 && "Unexpected bsize."); + } + if (!nn_config) return; + aom_clear_system_state(); + + // 1. Compute input features + float features[9]; + + // RD cost ratios + for (int i = 0; i < 5; i++) features[i] = 1.0f; + if (none_rd > 0 && none_rd < 1000000000) + features[0] = (float)none_rd / (float)best_rd; + for (int i = 0; i < 4; i++) { + if (split_rd[i] > 0 && split_rd[i] < 1000000000) + features[1 + i] = (float)split_rd[i] / (float)best_rd; + } + + // Variance ratios + const MACROBLOCKD *const xd = &x->e_mbd; + int whole_block_variance; + if (is_cur_buf_hbd(xd)) { + whole_block_variance = av1_high_get_sby_perpixel_variance( + cpi, &x->plane[0].src, bsize, xd->bd); + } else { + whole_block_variance = + av1_get_sby_perpixel_variance(cpi, &x->plane[0].src, bsize); + } + whole_block_variance = AOMMAX(whole_block_variance, 1); + + int split_variance[4]; + const BLOCK_SIZE subsize = get_partition_subsize(bsize, PARTITION_SPLIT); + struct buf_2d buf; + buf.stride = x->plane[0].src.stride; + const int bw = block_size_wide[bsize]; + for (int i = 0; i < 4; ++i) { + const int x_idx = (i & 1) * bw / 2; + const int y_idx = (i >> 1) * bw / 2; + buf.buf = x->plane[0].src.buf + x_idx + y_idx * buf.stride; + if (is_cur_buf_hbd(xd)) { + split_variance[i] = + av1_high_get_sby_perpixel_variance(cpi, &buf, subsize, xd->bd); + } else { + split_variance[i] = av1_get_sby_perpixel_variance(cpi, &buf, subsize); + } + } + + for (int i = 0; i < 4; i++) + features[5 + i] = (float)split_variance[i] / (float)whole_block_variance; + + // 2. Do the prediction and prune 0-2 partitions based on their probabilities + float raw_scores[3] = { 0.0f }; + av1_nn_predict(features, nn_config, raw_scores); + aom_clear_system_state(); + float probs[3] = { 0.0f }; + av1_nn_softmax(raw_scores, probs, 3); + + // probs[0] is the probability of the fact that both rectangular partitions + // are worse than current best_rd + if (probs[1] <= cur_thresh) (*dst_prune_horz) = 1; + if (probs[2] <= cur_thresh) (*dst_prune_vert) = 1; +} + +// Use a ML model to predict if horz_a, horz_b, vert_a, and vert_b should be +// considered. +void av1_ml_prune_ab_partition(BLOCK_SIZE bsize, int part_ctx, int var_ctx, + int64_t best_rd, int64_t horz_rd[2], + int64_t vert_rd[2], int64_t split_rd[4], + int *const horza_partition_allowed, + int *const horzb_partition_allowed, + int *const verta_partition_allowed, + int *const vertb_partition_allowed) { + if (bsize < BLOCK_8X8 || best_rd >= 1000000000) return; + const NN_CONFIG *nn_config = NULL; + switch (bsize) { + case BLOCK_8X8: nn_config = NULL; break; + case BLOCK_16X16: nn_config = &av1_ab_partition_nnconfig_16; break; + case BLOCK_32X32: nn_config = &av1_ab_partition_nnconfig_32; break; + case BLOCK_64X64: nn_config = &av1_ab_partition_nnconfig_64; break; + case BLOCK_128X128: nn_config = &av1_ab_partition_nnconfig_128; break; + default: assert(0 && "Unexpected bsize."); + } + if (!nn_config) return; + + aom_clear_system_state(); + + // Generate features. + float features[10]; + int feature_index = 0; + features[feature_index++] = (float)part_ctx; + features[feature_index++] = (float)var_ctx; + const int rdcost = (int)AOMMIN(INT_MAX, best_rd); + int sub_block_rdcost[8] = { 0 }; + int rd_index = 0; + for (int i = 0; i < 2; ++i) { + if (horz_rd[i] > 0 && horz_rd[i] < 1000000000) + sub_block_rdcost[rd_index] = (int)horz_rd[i]; + ++rd_index; + } + for (int i = 0; i < 2; ++i) { + if (vert_rd[i] > 0 && vert_rd[i] < 1000000000) + sub_block_rdcost[rd_index] = (int)vert_rd[i]; + ++rd_index; + } + for (int i = 0; i < 4; ++i) { + if (split_rd[i] > 0 && split_rd[i] < 1000000000) + sub_block_rdcost[rd_index] = (int)split_rd[i]; + ++rd_index; + } + for (int i = 0; i < 8; ++i) { + // Ratio between the sub-block RD and the whole-block RD. + float rd_ratio = 1.0f; + if (sub_block_rdcost[i] > 0 && sub_block_rdcost[i] < rdcost) + rd_ratio = (float)sub_block_rdcost[i] / (float)rdcost; + features[feature_index++] = rd_ratio; + } + assert(feature_index == 10); + + // Calculate scores using the NN model. + float score[16] = { 0.0f }; + av1_nn_predict(features, nn_config, score); + aom_clear_system_state(); + int int_score[16]; + int max_score = -1000; + for (int i = 0; i < 16; ++i) { + int_score[i] = (int)(100 * score[i]); + max_score = AOMMAX(int_score[i], max_score); + } + + // Make decisions based on the model scores. + int thresh = max_score; + switch (bsize) { + case BLOCK_16X16: thresh -= 150; break; + case BLOCK_32X32: thresh -= 100; break; + default: break; + } + *horza_partition_allowed = 0; + *horzb_partition_allowed = 0; + *verta_partition_allowed = 0; + *vertb_partition_allowed = 0; + for (int i = 0; i < 16; ++i) { + if (int_score[i] >= thresh) { + if ((i >> 0) & 1) *horza_partition_allowed = 1; + if ((i >> 1) & 1) *horzb_partition_allowed = 1; + if ((i >> 2) & 1) *verta_partition_allowed = 1; + if ((i >> 3) & 1) *vertb_partition_allowed = 1; + } + } +} + +#define FEATURES 18 +#define LABELS 4 +// Use a ML model to predict if horz4 and vert4 should be considered. +void av1_ml_prune_4_partition(const AV1_COMP *const cpi, MACROBLOCK *const x, + BLOCK_SIZE bsize, int part_ctx, int64_t best_rd, + int64_t horz_rd[2], int64_t vert_rd[2], + int64_t split_rd[4], + int *const partition_horz4_allowed, + int *const partition_vert4_allowed, + unsigned int pb_source_variance, int mi_row, + int mi_col) { + if (best_rd >= 1000000000) return; + const NN_CONFIG *nn_config = NULL; + switch (bsize) { + case BLOCK_16X16: nn_config = &av1_4_partition_nnconfig_16; break; + case BLOCK_32X32: nn_config = &av1_4_partition_nnconfig_32; break; + case BLOCK_64X64: nn_config = &av1_4_partition_nnconfig_64; break; + default: assert(0 && "Unexpected bsize."); + } + if (!nn_config) return; + + aom_clear_system_state(); + + // Generate features. + float features[FEATURES]; + int feature_index = 0; + features[feature_index++] = (float)part_ctx; + features[feature_index++] = (float)get_unsigned_bits(pb_source_variance); + + const int rdcost = (int)AOMMIN(INT_MAX, best_rd); + int sub_block_rdcost[8] = { 0 }; + int rd_index = 0; + for (int i = 0; i < 2; ++i) { + if (horz_rd[i] > 0 && horz_rd[i] < 1000000000) + sub_block_rdcost[rd_index] = (int)horz_rd[i]; + ++rd_index; + } + for (int i = 0; i < 2; ++i) { + if (vert_rd[i] > 0 && vert_rd[i] < 1000000000) + sub_block_rdcost[rd_index] = (int)vert_rd[i]; + ++rd_index; + } + for (int i = 0; i < 4; ++i) { + if (split_rd[i] > 0 && split_rd[i] < 1000000000) + sub_block_rdcost[rd_index] = (int)split_rd[i]; + ++rd_index; + } + for (int i = 0; i < 8; ++i) { + // Ratio between the sub-block RD and the whole-block RD. + float rd_ratio = 1.0f; + if (sub_block_rdcost[i] > 0 && sub_block_rdcost[i] < rdcost) + rd_ratio = (float)sub_block_rdcost[i] / (float)rdcost; + features[feature_index++] = rd_ratio; + } + + // Get variance of the 1:4 and 4:1 sub-blocks. + unsigned int horz_4_source_var[4] = { 0 }; + unsigned int vert_4_source_var[4] = { 0 }; + { + BLOCK_SIZE horz_4_bs = get_partition_subsize(bsize, PARTITION_HORZ_4); + BLOCK_SIZE vert_4_bs = get_partition_subsize(bsize, PARTITION_VERT_4); + av1_setup_src_planes(x, cpi->source, mi_row, mi_col, + av1_num_planes(&cpi->common), bsize); + const int src_stride = x->plane[0].src.stride; + uint8_t *src = x->plane[0].src.buf; + const MACROBLOCKD *const xd = &x->e_mbd; + + struct buf_2d horz_4_src, vert_4_src; + horz_4_src.stride = src_stride; + vert_4_src.stride = src_stride; + + for (int i = 0; i < 4; ++i) { + horz_4_src.buf = src + i * block_size_high[horz_4_bs] * src_stride; + vert_4_src.buf = src + i * block_size_wide[vert_4_bs]; + + if (is_cur_buf_hbd(xd)) { + horz_4_source_var[i] = av1_high_get_sby_perpixel_variance( + cpi, &horz_4_src, horz_4_bs, xd->bd); + vert_4_source_var[i] = av1_high_get_sby_perpixel_variance( + cpi, &vert_4_src, vert_4_bs, xd->bd); + } else { + horz_4_source_var[i] = + av1_get_sby_perpixel_variance(cpi, &horz_4_src, horz_4_bs); + vert_4_source_var[i] = + av1_get_sby_perpixel_variance(cpi, &vert_4_src, vert_4_bs); + } + } + } + + const float denom = (float)(pb_source_variance + 1); + const float low_b = 0.1f; + const float high_b = 10.0f; + for (int i = 0; i < 4; ++i) { + // Ratio between the 4:1 sub-block variance and the whole-block variance. + float var_ratio = (float)(horz_4_source_var[i] + 1) / denom; + if (var_ratio < low_b) var_ratio = low_b; + if (var_ratio > high_b) var_ratio = high_b; + features[feature_index++] = var_ratio; + } + for (int i = 0; i < 4; ++i) { + // Ratio between the 1:4 sub-block RD and the whole-block RD. + float var_ratio = (float)(vert_4_source_var[i] + 1) / denom; + if (var_ratio < low_b) var_ratio = low_b; + if (var_ratio > high_b) var_ratio = high_b; + features[feature_index++] = var_ratio; + } + assert(feature_index == FEATURES); + + // Calculate scores using the NN model. + float score[LABELS] = { 0.0f }; + av1_nn_predict(features, nn_config, score); + aom_clear_system_state(); + int int_score[LABELS]; + int max_score = -1000; + for (int i = 0; i < LABELS; ++i) { + int_score[i] = (int)(100 * score[i]); + max_score = AOMMAX(int_score[i], max_score); + } + + // Make decisions based on the model scores. + int thresh = max_score; + switch (bsize) { + case BLOCK_16X16: thresh -= 500; break; + case BLOCK_32X32: thresh -= 500; break; + case BLOCK_64X64: thresh -= 200; break; + default: break; + } + *partition_horz4_allowed = 0; + *partition_vert4_allowed = 0; + for (int i = 0; i < LABELS; ++i) { + if (int_score[i] >= thresh) { + if ((i >> 0) & 1) *partition_horz4_allowed = 1; + if ((i >> 1) & 1) *partition_vert4_allowed = 1; + } + } +} +#undef FEATURES +#undef LABELS + +#define FEATURES 4 +int av1_ml_predict_breakout(const AV1_COMP *const cpi, BLOCK_SIZE bsize, + const MACROBLOCK *const x, + const RD_STATS *const rd_stats, + unsigned int pb_source_variance) { + const NN_CONFIG *nn_config = NULL; + int thresh = 0; + switch (bsize) { + case BLOCK_8X8: + nn_config = &av1_partition_breakout_nnconfig_8; + thresh = cpi->sf.ml_partition_search_breakout_thresh[0]; + break; + case BLOCK_16X16: + nn_config = &av1_partition_breakout_nnconfig_16; + thresh = cpi->sf.ml_partition_search_breakout_thresh[1]; + break; + case BLOCK_32X32: + nn_config = &av1_partition_breakout_nnconfig_32; + thresh = cpi->sf.ml_partition_search_breakout_thresh[2]; + break; + case BLOCK_64X64: + nn_config = &av1_partition_breakout_nnconfig_64; + thresh = cpi->sf.ml_partition_search_breakout_thresh[3]; + break; + case BLOCK_128X128: + nn_config = &av1_partition_breakout_nnconfig_128; + thresh = cpi->sf.ml_partition_search_breakout_thresh[4]; + break; + default: assert(0 && "Unexpected bsize."); + } + if (!nn_config || thresh < 0) return 0; + + // Generate feature values. + float features[FEATURES]; + int feature_index = 0; + aom_clear_system_state(); + + const int num_pels_log2 = num_pels_log2_lookup[bsize]; + float rate_f = (float)AOMMIN(rd_stats->rate, INT_MAX); + rate_f = ((float)x->rdmult / 128.0f / 512.0f / (float)(1 << num_pels_log2)) * + rate_f; + features[feature_index++] = rate_f; + + const float dist_f = + (float)(AOMMIN(rd_stats->dist, INT_MAX) >> num_pels_log2); + features[feature_index++] = dist_f; + + features[feature_index++] = (float)pb_source_variance; + + const int dc_q = (int)x->plane[0].dequant_QTX[0]; + features[feature_index++] = (float)(dc_q * dc_q) / 256.0f; + assert(feature_index == FEATURES); + + // Calculate score using the NN model. + float score = 0.0f; + av1_nn_predict(features, nn_config, &score); + aom_clear_system_state(); + + // Make decision. + return (int)(score * 100) >= thresh; +} +#undef FEATURES diff --git a/third_party/aom/av1/encoder/partition_strategy.h b/third_party/aom/av1/encoder/partition_strategy.h index 36b1e95c6b..fbe832d27a 100644 --- a/third_party/aom/av1/encoder/partition_strategy.h +++ b/third_party/aom/av1/encoder/partition_strategy.h @@ -16,20 +16,31 @@ #include "av1/encoder/encodemb.h" #include "av1/encoder/encoder.h" +#define FEATURE_SIZE_SMS_SPLIT_FAST 6 +#define FEATURE_SIZE_SMS_SPLIT 17 #define FEATURE_SIZE_SMS_PRUNE_PART 25 #define FEATURE_SIZE_SMS_TERM_NONE 28 #define FEATURE_SIZE_FP_SMS_TERM_NONE 20 #define FEATURE_SIZE_MAX_MIN_PART_PRED 13 #define MAX_NUM_CLASSES_MAX_MIN_PART_PRED 4 +#define FEATURE_SMS_NONE_FLAG 1 +#define FEATURE_SMS_SPLIT_FLAG (1 << 1) +#define FEATURE_SMS_RECT_FLAG (1 << 2) + +#define FEATURE_SMS_PRUNE_PART_FLAG \ + (FEATURE_SMS_NONE_FLAG | FEATURE_SMS_SPLIT_FLAG | FEATURE_SMS_RECT_FLAG) +#define FEATURE_SMS_SPLIT_MODEL_FLAG \ + (FEATURE_SMS_NONE_FLAG | FEATURE_SMS_SPLIT_FLAG) + // Performs a simple_motion_search with a single reference frame and extract // the variance of residues. Then use the features to determine whether we want // to go straight to splitting without trying PARTITION_NONE void av1_simple_motion_search_based_split( - AV1_COMP *const cpi, MACROBLOCK *x, int mi_row, int mi_col, - BLOCK_SIZE bsize, int *partition_none_allowed, int *partition_horz_allowed, - int *partition_vert_allowed, int *do_rectangular_split, - int *do_square_split); + AV1_COMP *const cpi, MACROBLOCK *x, PC_TREE *pc_tree, int mi_row, + int mi_col, BLOCK_SIZE bsize, int *partition_none_allowed, + int *partition_horz_allowed, int *partition_vert_allowed, + int *do_rectangular_split, int *do_square_split); // Performs a simple_motion_search with two reference frames and extract // the variance of residues. Then use the features to determine whether we want @@ -39,7 +50,7 @@ void av1_simple_motion_search_prune_part( int mi_col, BLOCK_SIZE bsize, int *partition_none_allowed, int *partition_horz_allowed, int *partition_vert_allowed, int *do_square_split, int *do_rectangular_split, int *prune_horz, - int *prune_vert, float *features, int *valid); + int *prune_vert); // Early terminates PARTITION_NONE using simple_motion_search features and the // rate, distortion, and rdcost of PARTITION_NONE. This is only called when: @@ -47,11 +58,12 @@ void av1_simple_motion_search_prune_part( // - The frame is not intra only // - The current bsize is > BLOCK_8X8 // - blk_row + blk_height/2 < total_rows and blk_col + blk_width/2 < total_cols -void av1_simple_motion_search_early_term_none( - AV1_COMP *const cpi, MACROBLOCK *x, PC_TREE *pc_tree, int mi_row, - int mi_col, BLOCK_SIZE bsize, const RD_STATS *none_rdc, - int *early_terminate, float *simple_motion_features, - int *simple_motion_features_are_valid); +void av1_simple_motion_search_early_term_none(AV1_COMP *const cpi, + MACROBLOCK *x, PC_TREE *pc_tree, + int mi_row, int mi_col, + BLOCK_SIZE bsize, + const RD_STATS *none_rdc, + int *early_terminate); // Early terminates after PARTITION_NONE in firstpass of two pass partition // search. @@ -73,6 +85,61 @@ void av1_get_max_min_partition_features(AV1_COMP *const cpi, MACROBLOCK *x, BLOCK_SIZE av1_predict_max_partition(AV1_COMP *const cpi, MACROBLOCK *const x, const float *features); +// Attempts an early termination after PARTITION_SPLIT. +void av1_ml_early_term_after_split(AV1_COMP *const cpi, MACROBLOCK *const x, + PC_TREE *const pc_tree, BLOCK_SIZE bsize, + int64_t best_rd, int64_t part_none_rd, + int64_t part_split_rd, + int64_t *split_block_rd, int mi_row, + int mi_col, + int *const terminate_partition_search); + +// Use data from first partition pass to emit split_scores and none_scores. +// Returns 0 if the firstpass data is not valid, 1 otherwise. +// split_score indicates confidence of picking split partition; +// none_score indicates confidence of picking none partition; +int av1_ml_prune_2pass_split_partition(const PC_TREE_STATS *pc_tree_stats, + BLOCK_SIZE bsize, int *split_score, + int *none_score); + +// Use the rdcost ratio and source var ratio to prune PARTITION_HORZ and +// PARTITION_VERT. +// TODO(chiyotsai@google.com): Currently this model does not use q value and has +// no information about rectangular partitions. Preliminary experiments suggest +// that we can get better performance by adding in q_index and rectangular +// sse/var from SMS. We should retrain and tune this model later. +void av1_ml_prune_rect_partition(const AV1_COMP *const cpi, + const MACROBLOCK *const x, BLOCK_SIZE bsize, + int64_t best_rd, int64_t none_rd, + int64_t *split_rd, int *const dst_prune_horz, + int *const dst_prune_vert); + +// Use a ML model to predict if horz_a, horz_b, vert_a, and vert_b should be +// considered. +void av1_ml_prune_ab_partition(BLOCK_SIZE bsize, int part_ctx, int var_ctx, + int64_t best_rd, int64_t horz_rd[2], + int64_t vert_rd[2], int64_t split_rd[4], + int *const horza_partition_allowed, + int *const horzb_partition_allowed, + int *const verta_partition_allowed, + int *const vertb_partition_allowed); + +// Use a ML model to predict if horz4 and vert4 should be considered. +void av1_ml_prune_4_partition(const AV1_COMP *const cpi, MACROBLOCK *const x, + BLOCK_SIZE bsize, int part_ctx, int64_t best_rd, + int64_t horz_rd[2], int64_t vert_rd[2], + int64_t split_rd[4], + int *const partition_horz4_allowed, + int *const partition_vert4_allowed, + unsigned int pb_source_variance, int mi_row, + int mi_col); + +// ML-based partition search breakout after PARTITION_NONE +int av1_ml_predict_breakout(const AV1_COMP *const cpi, BLOCK_SIZE bsize, + const MACROBLOCK *const x, + const RD_STATS *const rd_stats, + unsigned int pb_source_variance); + // A simplified version of set_offsets meant to be used for // simple_motion_search. static INLINE void set_offsets_for_motion_search(const AV1_COMP *const cpi, @@ -116,10 +183,15 @@ static INLINE void set_offsets_for_motion_search(const AV1_COMP *const cpi, } static INLINE void init_simple_motion_search_mvs(PC_TREE *pc_tree) { - for (int idx = 0; idx < REF_FRAMES; idx++) { - pc_tree->mv_ref_fulls[idx].row = 0; - pc_tree->mv_ref_fulls[idx].col = 0; - } + av1_zero(pc_tree->mv_ref_fulls); + + av1_zero(pc_tree->sms_none_feat); + av1_zero(pc_tree->sms_split_feat); + av1_zero(pc_tree->sms_rect_feat); + av1_zero(pc_tree->sms_none_valid); + av1_zero(pc_tree->sms_split_valid); + av1_zero(pc_tree->sms_rect_valid); + if (pc_tree->block_size >= BLOCK_8X8) { init_simple_motion_search_mvs(pc_tree->split[0]); init_simple_motion_search_mvs(pc_tree->split[1]); diff --git a/third_party/aom/av1/encoder/picklpf.c b/third_party/aom/av1/encoder/picklpf.c index aca089c1e4..bc222f1c9a 100644 --- a/third_party/aom/av1/encoder/picklpf.c +++ b/third_party/aom/av1/encoder/picklpf.c @@ -57,7 +57,7 @@ static int64_t try_filter_frame(const YV12_BUFFER_CONFIG *sd, if (plane == 0 && dir == 0) filter_level[1] = cm->lf.filter_level[1]; if (plane == 0 && dir == 1) filter_level[0] = cm->lf.filter_level[0]; - // set base filters for use of get_filter_level when in DELTA_Q_LF mode + // set base filters for use of get_filter_level when in DELTA_LF mode switch (plane) { case 0: cm->lf.filter_level[0] = filter_level[0]; diff --git a/third_party/aom/av1/encoder/pickrst.c b/third_party/aom/av1/encoder/pickrst.c index 1b4f26c57b..b62789e8fb 100644 --- a/third_party/aom/av1/encoder/pickrst.c +++ b/third_party/aom/av1/encoder/pickrst.c @@ -1191,18 +1191,24 @@ static void search_wiener(const RestorationTileLimits *limits, const int wiener_win = (rsc->plane == AOM_PLANE_Y) ? WIENER_WIN : WIENER_WIN_CHROMA; + int reduced_wiener_win = wiener_win; + if (rsc->sf->reduce_wiener_window_size) { + reduced_wiener_win = + (rsc->plane == AOM_PLANE_Y) ? WIENER_WIN_REDUCED : WIENER_WIN_CHROMA; + } + int64_t M[WIENER_WIN2]; int64_t H[WIENER_WIN2 * WIENER_WIN2]; int32_t vfilter[WIENER_WIN], hfilter[WIENER_WIN]; const AV1_COMMON *const cm = rsc->cm; if (cm->seq_params.use_highbitdepth) { - av1_compute_stats_highbd(wiener_win, rsc->dgd_buffer, rsc->src_buffer, - limits->h_start, limits->h_end, limits->v_start, - limits->v_end, rsc->dgd_stride, rsc->src_stride, M, - H, cm->seq_params.bit_depth); + av1_compute_stats_highbd(reduced_wiener_win, rsc->dgd_buffer, + rsc->src_buffer, limits->h_start, limits->h_end, + limits->v_start, limits->v_end, rsc->dgd_stride, + rsc->src_stride, M, H, cm->seq_params.bit_depth); } else { - av1_compute_stats(wiener_win, rsc->dgd_buffer, rsc->src_buffer, + av1_compute_stats(reduced_wiener_win, rsc->dgd_buffer, rsc->src_buffer, limits->h_start, limits->h_end, limits->v_start, limits->v_end, rsc->dgd_stride, rsc->src_stride, M, H); } @@ -1210,7 +1216,7 @@ static void search_wiener(const RestorationTileLimits *limits, const MACROBLOCK *const x = rsc->x; const int64_t bits_none = x->wiener_restore_cost[0]; - if (!wiener_decompose_sep_sym(wiener_win, M, H, vfilter, hfilter)) { + if (!wiener_decompose_sep_sym(reduced_wiener_win, M, H, vfilter, hfilter)) { rsc->bits += bits_none; rsc->sse += rusi->sse[RESTORE_NONE]; rusi->best_rtype[RESTORE_WIENER - 1] = RESTORE_NONE; @@ -1221,13 +1227,13 @@ static void search_wiener(const RestorationTileLimits *limits, RestorationUnitInfo rui; memset(&rui, 0, sizeof(rui)); rui.restoration_type = RESTORE_WIENER; - finalize_sym_filter(wiener_win, vfilter, rui.wiener_info.vfilter); - finalize_sym_filter(wiener_win, hfilter, rui.wiener_info.hfilter); + finalize_sym_filter(reduced_wiener_win, vfilter, rui.wiener_info.vfilter); + finalize_sym_filter(reduced_wiener_win, hfilter, rui.wiener_info.hfilter); // Filter score computes the value of the function x'*A*x - x'*b for the // learned filter and compares it against identity filer. If there is no // reduction in the function, the filter is reverted back to identity - if (compute_score(wiener_win, M, H, rui.wiener_info.vfilter, + if (compute_score(reduced_wiener_win, M, H, rui.wiener_info.vfilter, rui.wiener_info.hfilter) > 0) { rsc->bits += bits_none; rsc->sse += rusi->sse[RESTORE_NONE]; @@ -1238,11 +1244,11 @@ static void search_wiener(const RestorationTileLimits *limits, aom_clear_system_state(); - rusi->sse[RESTORE_WIENER] = - finer_tile_search_wiener(rsc, limits, tile_rect, &rui, wiener_win); + rusi->sse[RESTORE_WIENER] = finer_tile_search_wiener( + rsc, limits, tile_rect, &rui, reduced_wiener_win); rusi->wiener = rui.wiener_info; - if (wiener_win != WIENER_WIN) { + if (reduced_wiener_win != WIENER_WIN) { assert(rui.wiener_info.vfilter[0] == 0 && rui.wiener_info.vfilter[WIENER_WIN - 1] == 0); assert(rui.wiener_info.hfilter[0] == 0 && diff --git a/third_party/aom/av1/encoder/pustats.h b/third_party/aom/av1/encoder/pustats.h index 40dd467689..2e8710108b 100644 --- a/third_party/aom/av1/encoder/pustats.h +++ b/third_party/aom/av1/encoder/pustats.h @@ -43,8 +43,8 @@ static const float -1.0533f, -0.3566f, 0.5294f, -0.4335f, 0.1626f, }; -static const float av1_pustats_rate_hiddenlayer_0_bias[HIDDEN_LAYERS_0_NODES] = - { +static const float + av1_pustats_rate_hiddenlayer_0_bias[HIDDEN_LAYERS_0_NODES] = { 10.5266f, 5.3268f, -1.0678f, 7.7411f, 8.7164f, -0.3235f, 7.3028f, 9.0874f, -6.4594f, -1.0102f, -1.1146f, 10.8419f, }; @@ -72,8 +72,8 @@ static const float -2.7566f, }; -static const float av1_pustats_rate_hiddenlayer_1_bias[HIDDEN_LAYERS_1_NODES] = - { +static const float + av1_pustats_rate_hiddenlayer_1_bias[HIDDEN_LAYERS_1_NODES] = { 13.2435f, -8.5477f, -0.0998f, -1.5131f, -12.0187f, 6.1715f, 0.5094f, 7.6433f, -0.3992f, -1.3555f, }; @@ -124,8 +124,8 @@ static const float -0.0806f, 0.5231f, 0.3928f, 0.4146f, 2.0956f, }; -static const float av1_pustats_dist_hiddenlayer_0_bias[HIDDEN_LAYERS_0_NODES] = - { +static const float + av1_pustats_dist_hiddenlayer_0_bias[HIDDEN_LAYERS_0_NODES] = { 1.1597f, 0.0836f, -0.7471f, -0.2439f, -0.0438f, 2.4626f, 0.f, 1.1485f, 2.7085f, -4.7897f, 1.4093f, -1.657f, }; @@ -153,8 +153,8 @@ static const float -0.4164f, }; -static const float av1_pustats_dist_hiddenlayer_1_bias[HIDDEN_LAYERS_1_NODES] = - { +static const float + av1_pustats_dist_hiddenlayer_1_bias[HIDDEN_LAYERS_1_NODES] = { -2.3014f, -2.4292f, 1.3317f, -3.2361f, -1.918f, 2.7149f, -2.5649f, 2.7765f, 2.9617f, 2.7684f, }; diff --git a/third_party/aom/av1/encoder/ransac.c b/third_party/aom/av1/encoder/ransac.c index 6a8854c58c..b490994567 100644 --- a/third_party/aom/av1/encoder/ransac.c +++ b/third_party/aom/av1/encoder/ransac.c @@ -265,8 +265,10 @@ static int find_rotzoom(int np, double *pts1, double *pts2, double *mat) { } static int find_affine(int np, double *pts1, double *pts2, double *mat) { + assert(np > 0); const int np2 = np * 2; double *a = (double *)aom_malloc(sizeof(*a) * (np2 * 7 + 42)); + if (a == NULL) return 1; double *b = a + np2 * 6; double *temp = b + np2; int i; diff --git a/third_party/aom/av1/encoder/ratectrl.c b/third_party/aom/av1/encoder/ratectrl.c index 861c7373ca..3853f14118 100644 --- a/third_party/aom/av1/encoder/ratectrl.c +++ b/third_party/aom/av1/encoder/ratectrl.c @@ -34,6 +34,8 @@ #include "av1/encoder/random.h" #include "av1/encoder/ratectrl.h" +#define USE_UNRESTRICTED_Q_IN_CQ_MODE 0 + // Max rate target for 1080P and below encodes under normal circumstances // (1920 * 1080 / (16 * 16)) * MAX_MB_RATE bits per MB #define MAX_MB_RATE 250 @@ -1001,6 +1003,28 @@ int av1_frame_type_qdelta(const AV1_COMP *cpi, int q) { cpi->common.seq_params.bit_depth); } +// This unrestricted Q selection on CQ mode is useful when testing new features, +// but may lead to Q being out of range on current RC restrictions +#if USE_UNRESTRICTED_Q_IN_CQ_MODE +static int rc_pick_q_and_bounds_one_pass_cq(const AV1_COMP *cpi, int width, + int height, int *bottom_index, + int *top_index) { + const AV1_COMMON *const cm = &cpi->common; + const RATE_CONTROL *const rc = &cpi->rc; + const AV1EncoderConfig *const oxcf = &cpi->oxcf; + const int cq_level = get_active_cq_level(rc, oxcf, frame_is_intra_only(cm), + cm->superres_scale_denominator); + const int bit_depth = cm->seq_params.bit_depth; + const int q = (int)av1_convert_qindex_to_q(cq_level, bit_depth); + (void)width; + (void)height; + *top_index = q; + *bottom_index = q; + + return q; +} +#endif // USE_UNRESTRICTED_Q_IN_CQ_MODE + #define STATIC_MOTION_THRESH 95 static int rc_pick_q_and_bounds_two_pass(const AV1_COMP *cpi, int width, int height, int *bottom_index, @@ -1089,6 +1113,18 @@ static int rc_pick_q_and_bounds_two_pass(const AV1_COMP *cpi, int width, q_val = av1_convert_qindex_to_q(active_best_quality, bit_depth); active_best_quality += av1_compute_qdelta(rc, q_val, q_val * q_adj_factor, bit_depth); + + // Tweak active_best_quality for AOM_Q mode when superres is on, as this + // will be used directly as 'q' later. + if (oxcf->rc_mode == AOM_Q && + (oxcf->superres_mode == SUPERRES_QTHRESH || + oxcf->superres_mode == SUPERRES_AUTO) && + cm->superres_scale_denominator != SCALE_NUMERATOR) { + active_best_quality = + AOMMAX(active_best_quality - + ((cm->superres_scale_denominator - SCALE_NUMERATOR) * 4), + 0); + } } } else if (!rc->is_src_frame_alt_ref && (cpi->refresh_golden_frame || is_intrl_arf_boost || @@ -1260,6 +1296,11 @@ int av1_rc_pick_q_and_bounds(AV1_COMP *cpi, int width, int height, if (cpi->oxcf.rc_mode == AOM_CBR) q = rc_pick_q_and_bounds_one_pass_cbr(cpi, width, height, bottom_index, top_index); +#if USE_UNRESTRICTED_Q_IN_CQ_MODE + else if (cpi->oxcf.rc_mode == AOM_CQ) + q = rc_pick_q_and_bounds_one_pass_cq(cpi, width, height, bottom_index, + top_index); +#endif // USE_UNRESTRICTED_Q_IN_CQ_MODE else q = rc_pick_q_and_bounds_one_pass_vbr(cpi, width, height, bottom_index, top_index); diff --git a/third_party/aom/av1/encoder/rd.c b/third_party/aom/av1/encoder/rd.c index d78e269e79..7905a3d540 100644 --- a/third_party/aom/av1/encoder/rd.c +++ b/third_party/aom/av1/encoder/rd.c @@ -52,20 +52,20 @@ static const uint8_t rd_thresh_block_size_factor[BLOCK_SIZES_ALL] = { 2, 3, 3, 4, 6, 6, 8, 12, 12, 16, 24, 24, 32, 48, 48, 64, 4, 4, 8, 8, 16, 16 }; -static const int use_intra_ext_tx_for_txsize[EXT_TX_SETS_INTRA][EXT_TX_SIZES] = - { - { 1, 1, 1, 1 }, // unused - { 1, 1, 0, 0 }, - { 0, 0, 1, 0 }, - }; +static const int use_intra_ext_tx_for_txsize[EXT_TX_SETS_INTRA] + [EXT_TX_SIZES] = { + { 1, 1, 1, 1 }, // unused + { 1, 1, 0, 0 }, + { 0, 0, 1, 0 }, + }; -static const int use_inter_ext_tx_for_txsize[EXT_TX_SETS_INTER][EXT_TX_SIZES] = - { - { 1, 1, 1, 1 }, // unused - { 1, 1, 0, 0 }, - { 0, 0, 1, 0 }, - { 0, 0, 0, 1 }, - }; +static const int use_inter_ext_tx_for_txsize[EXT_TX_SETS_INTER] + [EXT_TX_SIZES] = { + { 1, 1, 1, 1 }, // unused + { 1, 1, 0, 0 }, + { 0, 0, 1, 0 }, + { 0, 0, 0, 1 }, + }; static const int av1_ext_tx_set_idx_to_type[2][AOMMAX(EXT_TX_SETS_INTRA, EXT_TX_SETS_INTER)] = { @@ -343,9 +343,9 @@ void av1_init_me_luts(void) { static const int rd_boost_factor[16] = { 64, 32, 32, 32, 24, 16, 12, 12, 8, 8, 4, 4, 2, 2, 1, 0 }; -static const int rd_frame_type_factor[FRAME_UPDATE_TYPES] = { - 128, 144, 128, 128, 144, 144, 128 -}; +static const int rd_frame_type_factor[FRAME_UPDATE_TYPES] = { 128, 144, 128, + 128, 144, 144, + 128 }; int av1_compute_rd_mult_based_on_qindex(const AV1_COMP *cpi, int qindex) { const int q = av1_dc_quant_Q3(qindex, 0, cpi->common.seq_params.bit_depth); @@ -795,9 +795,9 @@ static const double surffit_rate_params[9][4] = { }, }; -static const double surffit_dist_params[7] = { - 1.475844, 4.328362, -5.680233, -0.500994, 0.554585, 4.839478, -0.695837 -}; +static const double surffit_dist_params[7] = { 1.475844, 4.328362, -5.680233, + -0.500994, 0.554585, 4.839478, + -0.695837 }; static void rate_surffit_model_params_lookup(BLOCK_SIZE bsize, double xm, double *rpar) { @@ -958,6 +958,7 @@ void av1_get_entropy_contexts(BLOCK_SIZE bsize, const struct macroblockd_plane *pd, ENTROPY_CONTEXT t_above[MAX_MIB_SIZE], ENTROPY_CONTEXT t_left[MAX_MIB_SIZE]) { + assert(bsize < BLOCK_SIZES_ALL); const BLOCK_SIZE plane_bsize = get_plane_block_size(bsize, pd->subsampling_x, pd->subsampling_y); get_entropy_contexts_plane(plane_bsize, pd, t_above, t_left); diff --git a/third_party/aom/av1/encoder/rd.h b/third_party/aom/av1/encoder/rd.h index ff4608343b..78f0052f57 100644 --- a/third_party/aom/av1/encoder/rd.h +++ b/third_party/aom/av1/encoder/rd.h @@ -78,8 +78,8 @@ enum { THR_GLOBALL3, THR_GLOBALB, THR_GLOBALA2, - THR_GLOBALA, THR_GLOBALG, + THR_GLOBALA, THR_COMP_NEAREST_NEARESTLA, THR_COMP_NEAREST_NEARESTL2A, @@ -303,8 +303,6 @@ static INLINE void av1_init_rd_stats(RD_STATS *rd_stats) { rd_stats->sse = 0; rd_stats->skip = 1; rd_stats->zero_rate = 0; - rd_stats->invalid_rate = 0; - rd_stats->ref_rdcost = INT64_MAX; #if CONFIG_RD_DEBUG // This may run into problems when monochrome video is // encoded, as there will only be 1 plane @@ -330,8 +328,6 @@ static INLINE void av1_invalid_rd_stats(RD_STATS *rd_stats) { rd_stats->sse = INT64_MAX; rd_stats->skip = 0; rd_stats->zero_rate = 0; - rd_stats->invalid_rate = 1; - rd_stats->ref_rdcost = INT64_MAX; #if CONFIG_RD_DEBUG // This may run into problems when monochrome video is // encoded, as there will only be 1 plane @@ -349,20 +345,17 @@ static INLINE void av1_invalid_rd_stats(RD_STATS *rd_stats) { static INLINE void av1_merge_rd_stats(RD_STATS *rd_stats_dst, const RD_STATS *rd_stats_src) { -#if CONFIG_RD_DEBUG - int plane; -#endif + assert(rd_stats_dst->rate != INT_MAX && rd_stats_src->rate != INT_MAX); rd_stats_dst->rate += rd_stats_src->rate; if (!rd_stats_dst->zero_rate) rd_stats_dst->zero_rate = rd_stats_src->zero_rate; rd_stats_dst->dist += rd_stats_src->dist; rd_stats_dst->sse += rd_stats_src->sse; rd_stats_dst->skip &= rd_stats_src->skip; - rd_stats_dst->invalid_rate &= rd_stats_src->invalid_rate; #if CONFIG_RD_DEBUG // This may run into problems when monochrome video is // encoded, as there will only be 1 plane - for (plane = 0; plane < MAX_MB_PLANE; ++plane) { + for (int plane = 0; plane < MAX_MB_PLANE; ++plane) { rd_stats_dst->txb_coeff_cost[plane] += rd_stats_src->txb_coeff_cost[plane]; { // TODO(angiebird): optimize this part diff --git a/third_party/aom/av1/encoder/rdopt.c b/third_party/aom/av1/encoder/rdopt.c index 5e6054e52f..99accb324b 100644 --- a/third_party/aom/av1/encoder/rdopt.c +++ b/third_party/aom/av1/encoder/rdopt.c @@ -2018,11 +2018,14 @@ static void model_rd_for_sb(const AV1_COMP *const cpi, BLOCK_SIZE bsize, int64_t dist_sum = 0; int64_t total_sse = 0; + assert(bsize < BLOCK_SIZES_ALL); + for (plane = plane_from; plane <= plane_to; ++plane) { struct macroblock_plane *const p = &x->plane[plane]; struct macroblockd_plane *const pd = &xd->plane[plane]; const BLOCK_SIZE plane_bsize = get_plane_block_size(bsize, pd->subsampling_x, pd->subsampling_y); + assert(plane_bsize < BLOCK_SIZES_ALL); const int bw = block_size_wide[plane_bsize]; const int bh = block_size_high[plane_bsize]; int64_t sse; @@ -2187,8 +2190,13 @@ static INLINE int64_t pixel_diff_dist(const MACROBLOCK *x, int plane, diff += ((blk_row * diff_stride + blk_col) << tx_size_wide_log2[0]); uint64_t sse = aom_sum_squares_2d_i16(diff, diff_stride, visible_cols, visible_rows); - if (block_mse_q8 != NULL) - *block_mse_q8 = (unsigned int)((256 * sse) / (visible_cols * visible_rows)); + if (block_mse_q8 != NULL) { + if (visible_cols > 0 && visible_rows > 0) + *block_mse_q8 = + (unsigned int)((256 * sse) / (visible_cols * visible_rows)); + else + *block_mse_q8 = UINT_MAX; + } return sse; } @@ -5181,8 +5189,6 @@ static void try_tx_block_no_split( get_txb_ctx(plane_bsize, tx_size, 0, pta, ptl, &txb_ctx); const int zero_blk_rate = x->coeff_costs[txs_ctx][PLANE_TYPE_Y] .txb_skip_cost[txb_ctx.txb_skip_ctx][1]; - - rd_stats->ref_rdcost = ref_best_rd; rd_stats->zero_rate = zero_blk_rate; const int index = av1_get_txb_size_index(plane_bsize, blk_row, blk_col); mbmi->inter_tx_size[index] = tx_size; @@ -5391,6 +5397,7 @@ static int64_t select_tx_size_and_type(const AV1_COMP *cpi, MACROBLOCK *x, TXB_RD_INFO_NODE *rd_info_tree) { MACROBLOCKD *const xd = &x->e_mbd; assert(is_inter_block(xd->mi[0])); + assert(bsize < BLOCK_SIZES_ALL); // TODO(debargha): enable this as a speed feature where the // select_inter_block_yrd() function above will use a simplified search @@ -5410,6 +5417,7 @@ static int64_t select_tx_size_and_type(const AV1_COMP *cpi, MACROBLOCK *x, const struct macroblockd_plane *const pd = &xd->plane[0]; const BLOCK_SIZE plane_bsize = get_plane_block_size(bsize, pd->subsampling_x, pd->subsampling_y); + assert(plane_bsize < BLOCK_SIZES_ALL); const int mi_width = mi_size_wide[plane_bsize]; const int mi_height = mi_size_high[plane_bsize]; ENTROPY_CONTEXT ctxa[MAX_MIB_SIZE]; @@ -5519,7 +5527,6 @@ static void tx_block_yrd(const AV1_COMP *cpi, MACROBLOCK *x, int blk_row, const int zero_blk_rate = x->coeff_costs[txs_ctx][get_plane_type(0)] .txb_skip_cost[txb_ctx.txb_skip_ctx][1]; rd_stats->zero_rate = zero_blk_rate; - rd_stats->ref_rdcost = ref_best_rd; tx_type_rd(cpi, x, tx_size, blk_row, blk_col, 0, block, plane_bsize, &txb_ctx, rd_stats, ftxs_mode, ref_best_rd, NULL); const int mi_width = block_size_wide[plane_bsize] >> tx_size_wide_log2[0]; @@ -6025,12 +6032,14 @@ static void model_rd_for_sb_with_fullrdy( int64_t rate_sum = 0; int64_t dist_sum = 0; int64_t total_sse = 0; + assert(bsize < BLOCK_SIZES_ALL); for (int plane = plane_from; plane <= plane_to; ++plane) { struct macroblock_plane *const p = &x->plane[plane]; struct macroblockd_plane *const pd = &xd->plane[plane]; const BLOCK_SIZE plane_bsize = get_plane_block_size(bsize, pd->subsampling_x, pd->subsampling_y); + assert(plane_bsize < BLOCK_SIZES_ALL); const int bw = block_size_wide[plane_bsize]; const int bh = block_size_high[plane_bsize]; int64_t sse; @@ -6052,7 +6061,7 @@ static void model_rd_for_sb_with_fullrdy( if (plane == 0) { pick_tx_size_type_yrd(cpi, x, &rd_stats, bsize, mi_row, mi_col, INT64_MAX); - if (rd_stats.invalid_rate) { + if (rd_stats.rate == INT_MAX) { rate = 0; dist = sse << 4; } else { @@ -6994,7 +7003,7 @@ static void store_coding_context(MACROBLOCK *x, PICK_MODE_CONTEXT *ctx, // Take a snapshot of the coding context so it can be // restored if we decide to encode this way - ctx->skip = x->skip; + ctx->rd_stats.skip = x->skip; ctx->skippable = skippable; ctx->best_mode_index = mode_index; ctx->mic = *xd->mi[0]; @@ -7032,8 +7041,9 @@ static void setup_buffer_ref_mvs_inter( // Gets an initial list of candidate vectors from neighbours and orders them av1_find_mv_refs(cm, xd, mbmi, ref_frame, mbmi_ext->ref_mv_count, - mbmi_ext->ref_mv_stack, NULL, mbmi_ext->global_mvs, mi_row, - mi_col, mbmi_ext->mode_context); + mbmi_ext->ref_mv_stack, mbmi_ext->weight, NULL, + mbmi_ext->global_mvs, mi_row, mi_col, + mbmi_ext->mode_context); // Further refinement that is encode side only to test the top few candidates // in full and choose the best as the center point for subsequent searches. @@ -8325,6 +8335,23 @@ static INLINE int64_t interpolation_filter_rd( return 0; } +static INLINE int is_pred_filter_search_allowed( + const AV1_COMP *const cpi, BLOCK_SIZE bsize, int mi_row, int mi_col, + InterpFilters af_horiz, InterpFilters af_vert, InterpFilters lf_horiz, + InterpFilters lf_vert) { + const AV1_COMMON *cm = &cpi->common; + const int bsl = mi_size_wide_log2[bsize]; + int pred_filter_search = + cpi->sf.cb_pred_filter_search + ? (((mi_row + mi_col) >> bsl) + + get_chessboard_index(cm->current_frame.frame_number)) & + 0x1 + : 0; + pred_filter_search &= ((af_horiz == lf_horiz) && (af_horiz != SWITCHABLE)) || + ((af_vert == lf_vert) && (af_vert != SWITCHABLE)); + return pred_filter_search; +} + static INLINE void pred_dual_interp_filter_rd( MACROBLOCK *const x, const AV1_COMP *const cpi, const TileDataEnc *tile_data, BLOCK_SIZE bsize, int mi_row, int mi_col, @@ -8385,26 +8412,20 @@ static INLINE void find_best_non_dual_interp_filter( assert(x->e_mbd.mi[0]->interp_filters == filter_sets[0]); assert(filter_set_size == DUAL_FILTER_SET_SIZE); if ((skip_hor & skip_ver) != cpi->default_interp_skip_flags) { - const AV1_COMMON *cm = &cpi->common; - int bsl, pred_filter_search; + int pred_filter_search; InterpFilters af = SWITCHABLE, lf = SWITCHABLE, filter_idx = 0; const MB_MODE_INFO *const above_mbmi = xd->above_mbmi; const MB_MODE_INFO *const left_mbmi = xd->left_mbmi; - bsl = mi_size_wide_log2[bsize]; - pred_filter_search = - cpi->sf.cb_pred_filter_search - ? (((mi_row + mi_col) >> bsl) + - get_chessboard_index(cm->current_frame.frame_number)) & - 0x1 - : 0; if (above_mbmi && is_inter_block(above_mbmi)) { af = above_mbmi->interp_filters; } if (left_mbmi && is_inter_block(left_mbmi)) { lf = left_mbmi->interp_filters; } - pred_filter_search &= ((af == lf) && (af != SWITCHABLE)); + pred_filter_search = is_pred_filter_search_allowed(cpi, bsize, mi_row, + mi_col, af, af, lf, lf); if (pred_filter_search) { + assert(af != SWITCHABLE); filter_idx = SWITCHABLE * (af & 0xf); // This assert tells that (filter_x == filter_y) for non-dual filter case assert((filter_sets[filter_idx] & 0xffff) == @@ -8781,30 +8802,23 @@ static int64_t interpolation_filter_search( const int bw = block_size_wide[bsize]; const int bh = block_size_high[bsize]; int skip_pred; - int bsl, pred_filter_search; + int pred_filter_search = 0; InterpFilters af_horiz = SWITCHABLE, af_vert = SWITCHABLE, lf_horiz = SWITCHABLE, lf_vert = SWITCHABLE, filter_idx = 0; - const MB_MODE_INFO *const above_mbmi = xd->above_mbmi; - const MB_MODE_INFO *const left_mbmi = xd->left_mbmi; - bsl = mi_size_wide_log2[bsize]; - pred_filter_search = - cpi->sf.cb_pred_filter_search - ? (((mi_row + mi_col) >> bsl) + - get_chessboard_index(cm->current_frame.frame_number)) & - 0x1 - : 0; - if (above_mbmi && is_inter_block(above_mbmi)) { - af_horiz = av1_extract_interp_filter(above_mbmi->interp_filters, 1); - af_vert = av1_extract_interp_filter(above_mbmi->interp_filters, 0); + if (!have_newmv_in_inter_mode(mbmi->mode)) { + const MB_MODE_INFO *const above_mbmi = xd->above_mbmi; + const MB_MODE_INFO *const left_mbmi = xd->left_mbmi; + if (above_mbmi && is_inter_block(above_mbmi)) { + af_horiz = av1_extract_interp_filter(above_mbmi->interp_filters, 1); + af_vert = av1_extract_interp_filter(above_mbmi->interp_filters, 0); + } + if (left_mbmi && is_inter_block(left_mbmi)) { + lf_horiz = av1_extract_interp_filter(left_mbmi->interp_filters, 1); + lf_vert = av1_extract_interp_filter(left_mbmi->interp_filters, 0); + } + pred_filter_search = is_pred_filter_search_allowed( + cpi, bsize, mi_row, mi_col, af_horiz, af_vert, lf_horiz, lf_vert); } - if (left_mbmi && is_inter_block(left_mbmi)) { - lf_horiz = av1_extract_interp_filter(left_mbmi->interp_filters, 1); - lf_vert = av1_extract_interp_filter(left_mbmi->interp_filters, 0); - } - pred_filter_search &= !have_newmv_in_inter_mode(mbmi->mode); - pred_filter_search &= - ((af_horiz == lf_horiz) && (af_horiz != SWITCHABLE)) || - ((af_vert == lf_vert) && (af_vert != SWITCHABLE)); if (pred_filter_search) { pred_dual_interp_filter_rd( x, cpi, tile_data, bsize, mi_row, mi_col, orig_dst, rd, @@ -9411,7 +9425,8 @@ static int64_t motion_mode_rd( assert(mbmi->ref_frame[1] != INTRA_FRAME); } - if (cpi->oxcf.enable_obmc == 0 && mbmi->motion_mode == OBMC_CAUSAL) + if ((cpi->oxcf.enable_obmc == 0 || cpi->sf.use_fast_nonrd_pick_mode) && + mbmi->motion_mode == OBMC_CAUSAL) continue; if (identical_obmc_mv_field_detected) { @@ -9576,20 +9591,6 @@ static int64_t motion_mode_rd( } } - if (cpi->sf.model_based_motion_mode_rd_breakout && do_tx_search) { - int model_rate; - int64_t model_dist; - model_rd_sb_fn[MODELRD_TYPE_MOTION_MODE_RD]( - cpi, mbmi->sb_type, x, xd, 0, num_planes - 1, mi_row, mi_col, - &model_rate, &model_dist, NULL, NULL, NULL, NULL, NULL); - const int64_t est_rd = - RDCOST(x->rdmult, rd_stats->rate + model_rate, model_dist); - if ((est_rd >> 3) * 6 > ref_best_rd) { - mbmi->ref_frame[1] = ref_frame_1; - continue; - } - } - if (!do_tx_search) { int64_t curr_sse = -1; int est_residue_cost = 0; @@ -9725,6 +9726,7 @@ static int64_t skip_mode_rd(RD_STATS *rd_stats, const AV1_COMP *const cpi, const AV1_COMMON *cm = &cpi->common; const int num_planes = av1_num_planes(cm); MACROBLOCKD *const xd = &x->e_mbd; + assert(bsize < BLOCK_SIZES_ALL); av1_enc_build_inter_predictor(cm, xd, mi_row, mi_col, orig_dst, bsize, 0, av1_num_planes(cm) - 1); @@ -9825,8 +9827,7 @@ static INLINE int get_drl_cost(const MB_MODE_INFO *mbmi, if (mbmi->mode == NEWMV || mbmi->mode == NEW_NEWMV) { for (int idx = 0; idx < 2; ++idx) { if (mbmi_ext->ref_mv_count[ref_frame_type] > idx + 1) { - uint8_t drl_ctx = - av1_drl_ctx(mbmi_ext->ref_mv_stack[ref_frame_type], idx); + uint8_t drl_ctx = av1_drl_ctx(mbmi_ext->weight[ref_frame_type], idx); cost += drl_mode_cost0[drl_ctx][mbmi->ref_mv_idx != idx]; if (mbmi->ref_mv_idx == idx) return cost; } @@ -9837,8 +9838,7 @@ static INLINE int get_drl_cost(const MB_MODE_INFO *mbmi, if (have_nearmv_in_inter_mode(mbmi->mode)) { for (int idx = 1; idx < 3; ++idx) { if (mbmi_ext->ref_mv_count[ref_frame_type] > idx + 1) { - uint8_t drl_ctx = - av1_drl_ctx(mbmi_ext->ref_mv_stack[ref_frame_type], idx); + uint8_t drl_ctx = av1_drl_ctx(mbmi_ext->weight[ref_frame_type], idx); cost += drl_mode_cost0[drl_ctx][mbmi->ref_mv_idx != (idx - 1)]; if (mbmi->ref_mv_idx == (idx - 1)) return cost; } @@ -10248,8 +10248,8 @@ static int64_t handle_inter_mode( mbmi->ref_frame[0] == LAST3_FRAME || mbmi->ref_frame[1] == LAST2_FRAME || mbmi->ref_frame[1] == LAST3_FRAME) { - if (mbmi_ext->ref_mv_stack[ref_frame_type][ref_mv_idx + has_nearmv] - .weight < REF_CAT_LEVEL) { + if (mbmi_ext->weight[ref_frame_type][ref_mv_idx + has_nearmv] < + REF_CAT_LEVEL) { continue; } } @@ -10642,8 +10642,9 @@ static int64_t rd_pick_intrabc_mode_sb(const AV1_COMP *cpi, MACROBLOCK *x, MB_MODE_INFO_EXT *const mbmi_ext = x->mbmi_ext; MV_REFERENCE_FRAME ref_frame = INTRA_FRAME; av1_find_mv_refs(cm, xd, mbmi, ref_frame, mbmi_ext->ref_mv_count, - mbmi_ext->ref_mv_stack, NULL, mbmi_ext->global_mvs, mi_row, - mi_col, mbmi_ext->mode_context); + mbmi_ext->ref_mv_stack, mbmi_ext->weight, NULL, + mbmi_ext->global_mvs, mi_row, mi_col, + mbmi_ext->mode_context); int_mv nearestmv, nearmv; av1_find_best_ref_mvs_from_stack(0, mbmi_ext, ref_frame, &nearestmv, &nearmv, @@ -10800,7 +10801,7 @@ void av1_rd_pick_intra_mode_sb(const AV1_COMP *cpi, MACROBLOCK *x, int mi_row, int64_t dist_y = 0, dist_uv = 0; TX_SIZE max_uv_tx_size; - ctx->skip = 0; + ctx->rd_stats.skip = 0; mbmi->ref_frame[0] = INTRA_FRAME; mbmi->ref_frame[1] = NONE_FRAME; mbmi->use_intrabc = 0; @@ -10851,7 +10852,7 @@ void av1_rd_pick_intra_mode_sb(const AV1_COMP *cpi, MACROBLOCK *x, int mi_row, if (rd_cost->rate != INT_MAX && rd_cost->rdcost < best_rd) best_rd = rd_cost->rdcost; if (rd_pick_intrabc_mode_sb(cpi, x, rd_cost, bsize, best_rd) < best_rd) { - ctx->skip = x->skip; + ctx->rd_stats.skip = x->skip; memcpy(ctx->blk_skip, x->blk_skip, sizeof(x->blk_skip[0]) * ctx->num_4x4_blk); assert(rd_cost->rate != INT_MAX); @@ -10958,8 +10959,9 @@ static void rd_pick_skip_mode(RD_STATS *rd_cost, } MB_MODE_INFO_EXT *mbmi_ext = x->mbmi_ext; av1_find_mv_refs(cm, xd, mbmi, ref_frame_type, mbmi_ext->ref_mv_count, - mbmi_ext->ref_mv_stack, NULL, mbmi_ext->global_mvs, mi_row, - mi_col, mbmi_ext->mode_context); + mbmi_ext->ref_mv_stack, mbmi_ext->weight, NULL, + mbmi_ext->global_mvs, mi_row, mi_col, + mbmi_ext->mode_context); } assert(this_mode == NEAREST_NEARESTMV); @@ -11431,8 +11433,9 @@ static void set_params_rd_pick_inter_mode( } } av1_find_mv_refs(cm, xd, mbmi, ref_frame, mbmi_ext->ref_mv_count, - mbmi_ext->ref_mv_stack, NULL, mbmi_ext->global_mvs, mi_row, - mi_col, mbmi_ext->mode_context); + mbmi_ext->ref_mv_stack, mbmi_ext->weight, NULL, + mbmi_ext->global_mvs, mi_row, mi_col, + mbmi_ext->mode_context); } av1_count_overlappable_neighbors(cm, xd, mi_row, mi_col); @@ -11472,119 +11475,6 @@ static void set_params_rd_pick_inter_mode( x->comp_rd_stats_idx = 0; } -// TODO(kyslov): now this is very similar to set_params_rd_pick_inter_mode -// (except that doesn't set ALTREF parameters) -// consider passing a flag to select non-rd path (similar to -// encode_sb_row) -static void set_params_nonrd_pick_inter_mode( - const AV1_COMP *cpi, MACROBLOCK *x, HandleInterModeArgs *args, - BLOCK_SIZE bsize, int mi_row, int mi_col, mode_skip_mask_t *mode_skip_mask, - int skip_ref_frame_mask, unsigned int ref_costs_single[REF_FRAMES], - unsigned int ref_costs_comp[REF_FRAMES][REF_FRAMES], - struct buf_2d yv12_mb[REF_FRAMES][MAX_MB_PLANE]) { - const AV1_COMMON *const cm = &cpi->common; - const int num_planes = av1_num_planes(cm); - MACROBLOCKD *const xd = &x->e_mbd; - MB_MODE_INFO *const mbmi = xd->mi[0]; - MB_MODE_INFO_EXT *const mbmi_ext = x->mbmi_ext; - unsigned char segment_id = mbmi->segment_id; - int dst_width1[MAX_MB_PLANE] = { MAX_SB_SIZE, MAX_SB_SIZE, MAX_SB_SIZE }; - int dst_width2[MAX_MB_PLANE] = { MAX_SB_SIZE >> 1, MAX_SB_SIZE >> 1, - MAX_SB_SIZE >> 1 }; - int dst_height1[MAX_MB_PLANE] = { MAX_SB_SIZE >> 1, MAX_SB_SIZE >> 1, - MAX_SB_SIZE >> 1 }; - int dst_height2[MAX_MB_PLANE] = { MAX_SB_SIZE, MAX_SB_SIZE, MAX_SB_SIZE }; - - for (int i = 0; i < MB_MODE_COUNT; ++i) - for (int k = 0; k < REF_FRAMES; ++k) args->single_filter[i][k] = SWITCHABLE; - - if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { - int len = sizeof(uint16_t); - args->above_pred_buf[0] = CONVERT_TO_BYTEPTR(x->above_pred_buf); - args->above_pred_buf[1] = - CONVERT_TO_BYTEPTR(x->above_pred_buf + (MAX_SB_SQUARE >> 1) * len); - args->above_pred_buf[2] = - CONVERT_TO_BYTEPTR(x->above_pred_buf + MAX_SB_SQUARE * len); - args->left_pred_buf[0] = CONVERT_TO_BYTEPTR(x->left_pred_buf); - args->left_pred_buf[1] = - CONVERT_TO_BYTEPTR(x->left_pred_buf + (MAX_SB_SQUARE >> 1) * len); - args->left_pred_buf[2] = - CONVERT_TO_BYTEPTR(x->left_pred_buf + MAX_SB_SQUARE * len); - } else { - args->above_pred_buf[0] = x->above_pred_buf; - args->above_pred_buf[1] = x->above_pred_buf + (MAX_SB_SQUARE >> 1); - args->above_pred_buf[2] = x->above_pred_buf + MAX_SB_SQUARE; - args->left_pred_buf[0] = x->left_pred_buf; - args->left_pred_buf[1] = x->left_pred_buf + (MAX_SB_SQUARE >> 1); - args->left_pred_buf[2] = x->left_pred_buf + MAX_SB_SQUARE; - } - - av1_collect_neighbors_ref_counts(xd); - - estimate_ref_frame_costs(cm, xd, x, segment_id, ref_costs_single, - ref_costs_comp); - - MV_REFERENCE_FRAME ref_frame; - for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) { - x->pred_mv_sad[ref_frame] = INT_MAX; - x->mbmi_ext->mode_context[ref_frame] = 0; - mbmi_ext->ref_mv_count[ref_frame] = UINT8_MAX; - if (cpi->ref_frame_flags & av1_ref_frame_flag_list[ref_frame]) { - if (mbmi->partition != PARTITION_NONE && - mbmi->partition != PARTITION_SPLIT) { - if (skip_ref_frame_mask & (1 << ref_frame)) { - int skip = 1; - for (int r = ALTREF_FRAME + 1; r < MODE_CTX_REF_FRAMES; ++r) { - if (!(skip_ref_frame_mask & (1 << r))) { - const MV_REFERENCE_FRAME *rf = ref_frame_map[r - REF_FRAMES]; - if (rf[0] == ref_frame || rf[1] == ref_frame) { - skip = 0; - break; - } - } - } - if (skip) continue; - } - } - assert(get_ref_frame_yv12_buf(cm, ref_frame) != NULL); - setup_buffer_ref_mvs_inter(cpi, x, ref_frame, bsize, mi_row, mi_col, - yv12_mb); - } - } - av1_count_overlappable_neighbors(cm, xd, mi_row, mi_col); - - if (check_num_overlappable_neighbors(mbmi) && - is_motion_variation_allowed_bsize(bsize)) { - av1_build_prediction_by_above_preds(cm, xd, mi_row, mi_col, - args->above_pred_buf, dst_width1, - dst_height1, args->above_pred_stride); - av1_build_prediction_by_left_preds(cm, xd, mi_row, mi_col, - args->left_pred_buf, dst_width2, - dst_height2, args->left_pred_stride); - av1_setup_dst_planes(xd->plane, bsize, &cm->cur_frame->buf, mi_row, mi_col, - 0, num_planes); - calc_target_weighted_pred( - cm, x, xd, mi_row, mi_col, args->above_pred_buf[0], - args->above_pred_stride[0], args->left_pred_buf[0], - args->left_pred_stride[0]); - } - init_mode_skip_mask(mode_skip_mask, cpi, x, bsize); - - if (cpi->sf.tx_type_search.fast_intra_tx_type_search) - x->use_default_intra_tx_type = 1; - else - x->use_default_intra_tx_type = 0; - - if (cpi->sf.tx_type_search.fast_inter_tx_type_search) - x->use_default_inter_tx_type = 1; - else - x->use_default_inter_tx_type = 0; - if (cpi->sf.skip_repeat_interpolation_filter_search) { - x->interp_filter_stats_idx[0] = 0; - x->interp_filter_stats_idx[1] = 0; - } -} - static void search_palette_mode(const AV1_COMP *cpi, MACROBLOCK *x, int mi_row, int mi_col, RD_STATS *rd_cost, PICK_MODE_CONTEXT *ctx, BLOCK_SIZE bsize, @@ -13207,6 +13097,97 @@ void av1_rd_pick_inter_mode_sb(AV1_COMP *cpi, TileDataEnc *tile_data, } } +// TODO(kyslov): now this is very similar to set_params_rd_pick_inter_mode +// (except that doesn't set ALTREF parameters) +// consider passing a flag to select non-rd path (similar to +// encode_sb_row) +static void set_params_nonrd_pick_inter_mode( + const AV1_COMP *cpi, MACROBLOCK *x, HandleInterModeArgs *args, + BLOCK_SIZE bsize, int mi_row, int mi_col, mode_skip_mask_t *mode_skip_mask, + int skip_ref_frame_mask, unsigned int ref_costs_single[REF_FRAMES], + unsigned int ref_costs_comp[REF_FRAMES][REF_FRAMES], + struct buf_2d yv12_mb[REF_FRAMES][MAX_MB_PLANE]) { + const AV1_COMMON *const cm = &cpi->common; + MACROBLOCKD *const xd = &x->e_mbd; + MB_MODE_INFO *const mbmi = xd->mi[0]; + MB_MODE_INFO_EXT *const mbmi_ext = x->mbmi_ext; + unsigned char segment_id = mbmi->segment_id; + + for (int i = 0; i < MB_MODE_COUNT; ++i) + for (int k = 0; k < REF_FRAMES; ++k) args->single_filter[i][k] = SWITCHABLE; + + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + int len = sizeof(uint16_t); + args->above_pred_buf[0] = CONVERT_TO_BYTEPTR(x->above_pred_buf); + args->above_pred_buf[1] = + CONVERT_TO_BYTEPTR(x->above_pred_buf + (MAX_SB_SQUARE >> 1) * len); + args->above_pred_buf[2] = + CONVERT_TO_BYTEPTR(x->above_pred_buf + MAX_SB_SQUARE * len); + args->left_pred_buf[0] = CONVERT_TO_BYTEPTR(x->left_pred_buf); + args->left_pred_buf[1] = + CONVERT_TO_BYTEPTR(x->left_pred_buf + (MAX_SB_SQUARE >> 1) * len); + args->left_pred_buf[2] = + CONVERT_TO_BYTEPTR(x->left_pred_buf + MAX_SB_SQUARE * len); + } else { + args->above_pred_buf[0] = x->above_pred_buf; + args->above_pred_buf[1] = x->above_pred_buf + (MAX_SB_SQUARE >> 1); + args->above_pred_buf[2] = x->above_pred_buf + MAX_SB_SQUARE; + args->left_pred_buf[0] = x->left_pred_buf; + args->left_pred_buf[1] = x->left_pred_buf + (MAX_SB_SQUARE >> 1); + args->left_pred_buf[2] = x->left_pred_buf + MAX_SB_SQUARE; + } + + av1_collect_neighbors_ref_counts(xd); + + estimate_ref_frame_costs(cm, xd, x, segment_id, ref_costs_single, + ref_costs_comp); + + MV_REFERENCE_FRAME ref_frame; + for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) { + x->pred_mv_sad[ref_frame] = INT_MAX; + x->mbmi_ext->mode_context[ref_frame] = 0; + mbmi_ext->ref_mv_count[ref_frame] = UINT8_MAX; + if (cpi->ref_frame_flags & av1_ref_frame_flag_list[ref_frame]) { + if (mbmi->partition != PARTITION_NONE && + mbmi->partition != PARTITION_SPLIT) { + if (skip_ref_frame_mask & (1 << ref_frame)) { + int skip = 1; + for (int r = ALTREF_FRAME + 1; r < MODE_CTX_REF_FRAMES; ++r) { + if (!(skip_ref_frame_mask & (1 << r))) { + const MV_REFERENCE_FRAME *rf = ref_frame_map[r - REF_FRAMES]; + if (rf[0] == ref_frame || rf[1] == ref_frame) { + skip = 0; + break; + } + } + } + if (skip) continue; + } + } + assert(get_ref_frame_yv12_buf(cm, ref_frame) != NULL); + setup_buffer_ref_mvs_inter(cpi, x, ref_frame, bsize, mi_row, mi_col, + yv12_mb); + } + } + + av1_count_overlappable_neighbors(cm, xd, mi_row, mi_col); + init_mode_skip_mask(mode_skip_mask, cpi, x, bsize); + + if (cpi->sf.tx_type_search.fast_intra_tx_type_search) + x->use_default_intra_tx_type = 1; + else + x->use_default_intra_tx_type = 0; + + if (cpi->sf.tx_type_search.fast_inter_tx_type_search) + x->use_default_inter_tx_type = 1; + else + x->use_default_inter_tx_type = 0; + if (cpi->sf.skip_repeat_interpolation_filter_search) { + x->interp_filter_stats_idx[0] = 0; + x->interp_filter_stats_idx[1] = 0; + } +} + // TODO(kyslov): now this is very similar to av1_rd_pick_inter_mode_sb except: // it only checks non-compound mode and // it doesn't check palette mode @@ -13237,10 +13218,6 @@ void av1_nonrd_pick_inter_mode_sb(AV1_COMP *cpi, TileDataEnc *tile_data, InterModeSearchState search_state; init_inter_mode_search_state(&search_state, cpi, tile_data, x, bsize, best_rd_so_far); - INTERINTRA_MODE interintra_modes[REF_FRAMES] = { - INTERINTRA_MODES, INTERINTRA_MODES, INTERINTRA_MODES, INTERINTRA_MODES, - INTERINTRA_MODES, INTERINTRA_MODES, INTERINTRA_MODES, INTERINTRA_MODES - }; HandleInterModeArgs args = { { NULL }, { MAX_SB_SIZE, MAX_SB_SIZE, MAX_SB_SIZE }, { NULL }, { MAX_SB_SIZE >> 1, MAX_SB_SIZE >> 1, MAX_SB_SIZE >> 1 }, @@ -13248,7 +13225,7 @@ void av1_nonrd_pick_inter_mode_sb(AV1_COMP *cpi, TileDataEnc *tile_data, NULL, search_state.modelled_rd, { { 0 } }, INT_MAX, INT_MAX, search_state.simple_rd, - 0, interintra_modes, + 0, NULL, 1, NULL }; for (i = 0; i < REF_FRAMES; ++i) x->pred_sse[i] = INT_MAX; diff --git a/third_party/aom/av1/encoder/rdopt.h b/third_party/aom/av1/encoder/rdopt.h index 7ba1b18201..3293834804 100644 --- a/third_party/aom/av1/encoder/rdopt.h +++ b/third_party/aom/av1/encoder/rdopt.h @@ -123,6 +123,13 @@ void av1_rd_pick_inter_mode_sb(struct AV1_COMP *cpi, struct RD_STATS *rd_cost, BLOCK_SIZE bsize, PICK_MODE_CONTEXT *ctx, int64_t best_rd_so_far); +void av1_fast_nonrd_pick_inter_mode_sb(struct AV1_COMP *cpi, + struct TileDataEnc *tile_data, + struct macroblock *x, int mi_row, + int mi_col, struct RD_STATS *rd_cost, + BLOCK_SIZE bsize, PICK_MODE_CONTEXT *ctx, + int64_t best_rd_so_far); + void av1_nonrd_pick_inter_mode_sb(struct AV1_COMP *cpi, struct TileDataEnc *tile_data, struct macroblock *x, int mi_row, int mi_col, diff --git a/third_party/aom/av1/encoder/reconinter_enc.c b/third_party/aom/av1/encoder/reconinter_enc.c index 4b477cef62..9198901eff 100644 --- a/third_party/aom/av1/encoder/reconinter_enc.c +++ b/third_party/aom/av1/encoder/reconinter_enc.c @@ -495,6 +495,7 @@ void av1_build_inter_predictors_for_planes_single_buf( MACROBLOCKD *xd, BLOCK_SIZE bsize, int plane_from, int plane_to, int mi_row, int mi_col, int ref, uint8_t *ext_dst[3], int ext_dst_stride[3], int can_use_previous) { + assert(bsize < BLOCK_SIZES_ALL); int plane; const int mi_x = mi_col * MI_SIZE; const int mi_y = mi_row * MI_SIZE; @@ -595,6 +596,7 @@ void av1_build_wedge_inter_predictor_from_buf(MACROBLOCKD *xd, BLOCK_SIZE bsize, uint8_t *ext_dst1[3], int ext_dst_stride1[3]) { int plane; + assert(bsize < BLOCK_SIZES_ALL); for (plane = plane_from; plane <= plane_to; ++plane) { const BLOCK_SIZE plane_bsize = get_plane_block_size( bsize, xd->plane[plane].subsampling_x, xd->plane[plane].subsampling_y); diff --git a/third_party/aom/av1/encoder/segmentation.c b/third_party/aom/av1/encoder/segmentation.c index 6d0c6549df..193c7a77af 100644 --- a/third_party/aom/av1/encoder/segmentation.c +++ b/third_party/aom/av1/encoder/segmentation.c @@ -151,6 +151,7 @@ static void count_segs_sb(const AV1_COMMON *cm, MACROBLOCKD *xd, case PARTITION_SPLIT: { const BLOCK_SIZE subsize = get_partition_subsize(bsize, PARTITION_SPLIT); int n; + assert(subsize < BLOCK_SIZES_ALL); for (n = 0; n < 4; n++) { const int mi_dc = hbs * (n & 1); diff --git a/third_party/aom/av1/encoder/speed_features.c b/third_party/aom/av1/encoder/speed_features.c index 5dfc585a29..062a605092 100644 --- a/third_party/aom/av1/encoder/speed_features.c +++ b/third_party/aom/av1/encoder/speed_features.c @@ -29,9 +29,9 @@ static MESH_PATTERN { { 64, 16 }, { 24, 8 }, { 12, 4 }, { 7, 1 } }, { { 64, 16 }, { 24, 8 }, { 12, 4 }, { 7, 1 } }, }; -static unsigned char good_quality_max_mesh_pct[MAX_MESH_SPEED + 1] = { - 50, 50, 25, 15, 5, 1 -}; +static unsigned char good_quality_max_mesh_pct[MAX_MESH_SPEED + 1] = { 50, 50, + 25, 15, + 5, 1 }; // TODO(huisu@google.com): These settings are pretty relaxed, tune them for // each speed setting @@ -114,8 +114,11 @@ static void set_good_speed_feature_framesize_dependent( sf->ml_partition_search_breakout_thresh[2] = 300; // BLOCK_32X32 sf->ml_partition_search_breakout_thresh[3] = 500; // BLOCK_64X64 sf->ml_partition_search_breakout_thresh[4] = -1; // BLOCK_128X128 + sf->ml_early_term_after_part_split_level = 1; } + // TODO(chiyotsai@google.com): Try to replace two pass partition search with + // other speed features. if (is_720p_or_larger && speed >= CONFIG_2PASS_PARTITION_SEARCH_LVL_START && speed < CONFIG_2PASS_PARTITION_SEARCH_LVL_END) { sf->two_pass_partition_search = 1; @@ -126,8 +129,12 @@ static void set_good_speed_feature_framesize_dependent( sf->use_square_partition_only_threshold = BLOCK_128X128; } else if (is_480p_or_larger) { sf->use_square_partition_only_threshold = BLOCK_64X64; + + sf->simple_motion_search_split = 2; } else { sf->use_square_partition_only_threshold = BLOCK_32X32; + + sf->simple_motion_search_split = 1; } if (!is_720p_or_larger) { @@ -138,6 +145,7 @@ static void set_good_speed_feature_framesize_dependent( sf->ml_partition_search_breakout_thresh[4] = -1; // BLOCK_128X128 sf->firstpass_simple_motion_search_early_term = 1; + sf->ml_early_term_after_part_split_level = 2; } } @@ -165,6 +173,7 @@ static void set_good_speed_feature_framesize_dependent( } if (speed >= 3) { + sf->ml_early_term_after_part_split_level = 0; if (is_720p_or_larger) { sf->partition_search_breakout_dist_thr = (1 << 25); sf->partition_search_breakout_rate_thr = 200; @@ -175,6 +184,14 @@ static void set_good_speed_feature_framesize_dependent( } sf->use_first_partition_pass_interintra_stats = sf->two_pass_partition_search; + + // TODO(Venkat): Clean-up frame type dependency for + // simple_motion_search_split in partition search function and set the + // speed feature accordingly + // TODO(any): The models and thresholds used by simple_motion_split is + // trained and tuned on speed 1 and 2. We might get better performance if we + // readjust them for speed 3 and 4. + sf->simple_motion_search_split = cm->allow_screen_content_tools ? 1 : 2; } if (speed >= 4) { @@ -204,7 +221,6 @@ static void set_good_speed_features_framesize_independent( sf->use_dist_wtd_comp_flag = DIST_WTD_COMP_SKIP_MV_SEARCH; sf->model_based_prune_tx_search_level = 1; sf->model_based_post_interp_filter_breakout = 1; - sf->model_based_motion_mode_rd_breakout = 1; // TODO(debargha): Test, tweak and turn on either 1 or 2 sf->inter_mode_rd_model_estimation = 1; @@ -254,7 +270,7 @@ static void set_good_speed_features_framesize_independent( // speed. sf->prune_single_motion_modes_by_simple_trans = 1; - sf->simple_motion_search_split_only = 1; + sf->simple_motion_search_split = 1; sf->simple_motion_search_early_term_none = 1; sf->disable_wedge_search_var_thresh = 0; @@ -332,12 +348,6 @@ static void set_good_speed_features_framesize_independent( frame_is_intra_only(&cpi->common) ? 0 : (boosted ? 1 : 2); sf->perform_coeff_opt = is_boosted_arf2_bwd_type ? 2 : 3; sf->prune_comp_type_by_model_rd = boosted ? 0 : 1; - // TODO(Venkat): Clean-up frame type dependency for - // simple_motion_search_split_only in partition search function and set the - // speed feature accordingly - // TODO(Venkat): Evaluate this speed feature for speed 1 & 2 - sf->simple_motion_search_split_only = - cm->allow_screen_content_tools ? 1 : 2; sf->disable_smooth_intra = !frame_is_intra_only(&cpi->common) || (cpi->rc.frames_to_key != 1); } @@ -347,6 +357,7 @@ static void set_good_speed_features_framesize_independent( sf->tx_type_search.fast_intra_tx_type_search = 1; sf->disable_loop_restoration_chroma = (boosted || cm->allow_screen_content_tools) ? 0 : 1; + sf->reduce_wiener_window_size = !boosted; sf->mv.subpel_search_method = SUBPEL_TREE_PRUNED; sf->adaptive_pred_interp_filter = 0; sf->cb_pred_filter_search = 1; @@ -433,7 +444,6 @@ static void set_rt_speed_features_framesize_independent(AV1_COMP *cpi, sf->use_dist_wtd_comp_flag = DIST_WTD_COMP_SKIP_MV_SEARCH; sf->model_based_prune_tx_search_level = 1; sf->model_based_post_interp_filter_breakout = 1; - sf->model_based_motion_mode_rd_breakout = 1; // TODO(debargha): Test, tweak and turn on either 1 or 2 sf->inter_mode_rd_model_estimation = 0; @@ -455,6 +465,9 @@ static void set_rt_speed_features_framesize_independent(AV1_COMP *cpi, sf->cb_pred_filter_search = 0; sf->use_nonrd_pick_mode = 0; sf->use_real_time_ref_set = 0; + sf->use_fast_nonrd_pick_mode = 0; + sf->reuse_inter_pred_nonrd = 0; + sf->estimate_motion_for_var_based_partition = 1; if (speed >= 1) { sf->gm_erroradv_type = GM_ERRORADV_TR_1; @@ -593,21 +606,14 @@ static void set_rt_speed_features_framesize_independent(AV1_COMP *cpi, sf->simple_model_rd_from_var = 1; } if (speed >= 7) { - sf->default_max_partition_size = BLOCK_32X32; - sf->default_min_partition_size = BLOCK_8X8; - sf->intra_y_mode_mask[TX_64X64] = INTRA_DC; - sf->intra_y_mode_mask[TX_32X32] = INTRA_DC; - sf->frame_parameter_update = 0; - sf->mv.search_method = FAST_HEX; - sf->partition_search_type = REFERENCE_PARTITION; - sf->mode_search_skip_flags |= FLAG_SKIP_INTRA_DIRMISMATCH; - } - if (speed >= 8) { - sf->mv.search_method = FAST_DIAMOND; sf->lpf_pick = LPF_PICK_FROM_Q; + sf->mv.subpel_force_stop = QUARTER_PEL; sf->default_max_partition_size = BLOCK_128X128; sf->default_min_partition_size = BLOCK_8X8; + sf->frame_parameter_update = 0; + sf->mv.search_method = FAST_DIAMOND; sf->partition_search_type = VAR_BASED_PARTITION; + sf->mode_search_skip_flags |= FLAG_SKIP_INTRA_DIRMISMATCH; sf->use_real_time_ref_set = 1; // Can't use LARGEST TX mode with pre-calculated partition // and disabled TX64 @@ -615,6 +621,12 @@ static void set_rt_speed_features_framesize_independent(AV1_COMP *cpi, sf->use_nonrd_pick_mode = 1; sf->inter_mode_rd_model_estimation = 2; } + if (speed >= 8) { + sf->use_fast_nonrd_pick_mode = 1; + sf->mv.subpel_search_method = SUBPEL_TREE; + sf->tx_size_search_method = USE_FAST_RD; + sf->estimate_motion_for_var_based_partition = 0; + } } void av1_set_speed_features_framesize_dependent(AV1_COMP *cpi, int speed) { @@ -679,7 +691,6 @@ void av1_set_speed_features_framesize_independent(AV1_COMP *cpi, int speed) { sf->tx_size_search_lgr_block = 0; sf->model_based_prune_tx_search_level = 0; sf->model_based_post_interp_filter_breakout = 0; - sf->model_based_motion_mode_rd_breakout = 0; sf->reduce_inter_modes = 0; sf->selective_ref_gm = 1; sf->adaptive_motion_search = 0; @@ -711,6 +722,7 @@ void av1_set_speed_features_framesize_independent(AV1_COMP *cpi, int speed) { sf->use_first_partition_pass_interintra_stats = 0; sf->disable_wedge_search_var_thresh = 0; sf->disable_loop_restoration_chroma = 0; + sf->reduce_wiener_window_size = 0; sf->fast_wedge_sign_estimate = 0; sf->prune_wedge_pred_diff_based = 0; sf->drop_ref = 0; @@ -751,11 +763,12 @@ void av1_set_speed_features_framesize_independent(AV1_COMP *cpi, int speed) { sf->ml_prune_rect_partition = 0; sf->ml_prune_ab_partition = 0; sf->ml_prune_4_partition = 0; + sf->ml_early_term_after_part_split_level = 0; sf->fast_cdef_search = 0; for (i = 0; i < PARTITION_BLOCK_SIZES; ++i) { sf->ml_partition_search_breakout_thresh[i] = -1; // -1 means not enabled. } - sf->simple_motion_search_split_only = 0; + sf->simple_motion_search_split = 0; sf->simple_motion_search_prune_rect = 0; sf->simple_motion_search_early_term_none = 0; @@ -774,9 +787,6 @@ void av1_set_speed_features_framesize_independent(AV1_COMP *cpi, int speed) { // Set decoder side speed feature to use less dual sgr modes sf->dual_sgr_penalty_level = 0; - // TODO(angiebird, debargha): Re-evaluate the impact of - // inter_mode_rd_model_estimation in conjunction with - // model_based_motion_mode_rd_breakout sf->inter_mode_rd_model_estimation = 0; sf->inter_mode_rd_model_estimation_adaptive = 0; diff --git a/third_party/aom/av1/encoder/speed_features.h b/third_party/aom/av1/encoder/speed_features.h index a3211922b7..ef869239c4 100644 --- a/third_party/aom/av1/encoder/speed_features.h +++ b/third_party/aom/av1/encoder/speed_features.h @@ -354,11 +354,6 @@ typedef struct SPEED_FEATURES { // 1: use model based rd breakout int model_based_post_interp_filter_breakout; - // Model based breakout in motion_mode_rd - // 0: no breakout - // 1: use model based rd breakout - int model_based_motion_mode_rd_breakout; - // Used if partition_search_type = FIXED_SIZE_PARTITION BLOCK_SIZE always_this_block_size; @@ -385,6 +380,11 @@ typedef struct SPEED_FEATURES { // Use a ML model to prune horz4 and vert4 partitions. int ml_prune_4_partition; + // Use a ML model to adaptively terminate partition search after trying + // PARTITION_SPLIT. Can take values 0 - 2, 0 meaning not being enabled, and + // 1 - 2 increasing aggressiveness in order. + int ml_early_term_after_part_split_level; + int fast_cdef_search; // 2-pass coding block partition search, and also use the mode decisions made @@ -648,8 +648,11 @@ typedef struct SPEED_FEATURES { int simple_motion_search_prune_rect; // Perform simple motion search before none_partition to decide if we - // want to split directly without trying other partition types. - int simple_motion_search_split_only; + // want to remove all partitions other than PARTITION_SPLIT. If set to 0, this + // model is disabled. If set to 1, the model attempts to perform + // PARTITION_SPLIT only. If set to 2, the model also attempts to prune + // PARTITION_SPLIT. + int simple_motion_search_split; // Use features from simple_motion_search to terminate prediction block // partition after PARTITION_NONE @@ -672,6 +675,9 @@ typedef struct SPEED_FEATURES { // Disable loop restoration for Chroma plane int disable_loop_restoration_chroma; + // Reduce the wiener filter win size for luma + int reduce_wiener_window_size; + // Flag used to control the extent of coeff R-D optimization int perform_coeff_opt; @@ -695,6 +701,16 @@ typedef struct SPEED_FEATURES { // inter-mode RD model for others. Only enabled when // inter_mode_rd_model_estimation != 0 int inter_mode_rd_model_estimation_adaptive; + + // Use very reduced set of inter mode checks and fast non-rd mode cost + // estimation Only enabled when use_nonrd_pick_mode is != 0 + int use_fast_nonrd_pick_mode; + + // Reuse inter prediction in fast non-rd mode. + int reuse_inter_pred_nonrd; + + // Perform croase ME before calculating variance in variance-based partition + int estimate_motion_for_var_based_partition; } SPEED_FEATURES; struct AV1_COMP; diff --git a/third_party/aom/av1/encoder/temporal_filter.c b/third_party/aom/av1/encoder/temporal_filter.c index ba883d7881..9135750cb1 100644 --- a/third_party/aom/av1/encoder/temporal_filter.c +++ b/third_party/aom/av1/encoder/temporal_filter.c @@ -37,9 +37,9 @@ #define EDGE_THRESHOLD 50 #define SQRT_PI_BY_2 1.25331413732 -static unsigned int index_mult[14] = { - 0, 0, 0, 0, 49152, 39322, 32768, 28087, 24576, 21846, 19661, 17874, 0, 15124 -}; +static unsigned int index_mult[14] = { 0, 0, 0, 0, 49152, + 39322, 32768, 28087, 24576, 21846, + 19661, 17874, 0, 15124 }; static int64_t highbd_index_mult[14] = { 0U, 0U, 0U, 0U, 3221225472U, 2576980378U, diff --git a/third_party/aom/av1/encoder/tokenize.c b/third_party/aom/av1/encoder/tokenize.c index ce1a2121a4..dc25ecd479 100644 --- a/third_party/aom/av1/encoder/tokenize.c +++ b/third_party/aom/av1/encoder/tokenize.c @@ -198,6 +198,8 @@ void av1_tokenize_sb_vartx(const AV1_COMP *cpi, ThreadData *td, TOKENEXTRA **t, struct tokenize_b_args arg = { cpi, td, t, 0, allow_update_cdf }; if (mi_row >= cm->mi_rows || mi_col >= cm->mi_cols) return; + assert(bsize < BLOCK_SIZES_ALL); + if (mbmi->skip) { av1_reset_skip_context(xd, mi_row, mi_col, bsize, num_planes); return; @@ -214,6 +216,7 @@ void av1_tokenize_sb_vartx(const AV1_COMP *cpi, ThreadData *td, TOKENEXTRA **t, scale_chroma_bsize(bsize, pd->subsampling_x, pd->subsampling_y); const BLOCK_SIZE plane_bsize = get_plane_block_size(bsizec, pd->subsampling_x, pd->subsampling_y); + assert(plane_bsize < BLOCK_SIZES_ALL); const int mi_width = block_size_wide[plane_bsize] >> tx_size_wide_log2[0]; const int mi_height = block_size_high[plane_bsize] >> tx_size_high_log2[0]; const TX_SIZE max_tx_size = get_vartx_max_txsize(xd, plane_bsize, plane); diff --git a/third_party/aom/av1/encoder/tpl_model.c b/third_party/aom/av1/encoder/tpl_model.c index 79afb6d17b..4b2075cbaa 100644 --- a/third_party/aom/av1/encoder/tpl_model.c +++ b/third_party/aom/av1/encoder/tpl_model.c @@ -459,26 +459,24 @@ static void mc_flow_dispenser(AV1_COMP *cpi, GF_PICTURE *gf_picture, } } -static void init_gop_frames(AV1_COMP *cpi, GF_PICTURE *gf_picture, - const GF_GROUP *gf_group, int *tpl_group_frames, - const EncodeFrameInput *const frame_input) { +static void init_gop_frames_for_tpl(AV1_COMP *cpi, GF_PICTURE *gf_picture, + const GF_GROUP *gf_group, + int *tpl_group_frames, + const EncodeFrameInput *const frame_input) { AV1_COMMON *cm = &cpi->common; const SequenceHeader *const seq_params = &cm->seq_params; int frame_idx = 0; int i; int gld_index = -1; - int alt_index = -1; int lst_index = -1; int extend_frame_count = 0; + int frame_gop_offset = 0; int pframe_qindex = cpi->tpl_stats[2].base_qindex; RefCntBuffer *frame_bufs = cm->buffer_pool->frame_bufs; int recon_frame_index[INTER_REFS_PER_FRAME + 1] = { -1, -1, -1, -1, -1, -1, -1, -1 }; - // TODO(jingning): To be used later for gf frame type parsing. - (void)gf_group; - for (i = 0; i < FRAME_BUFFERS && frame_idx < INTER_REFS_PER_FRAME + 1; ++i) { if (frame_bufs[i].ref_count == 0) { alloc_frame_mvs(cm, &frame_bufs[i]); @@ -503,51 +501,48 @@ static void init_gop_frames(AV1_COMP *cpi, GF_PICTURE *gf_picture, *tpl_group_frames = 0; // Initialize Golden reference frame. - gf_picture[0].frame = NULL; RefCntBuffer *ref_buf = get_ref_frame_buf(cm, GOLDEN_FRAME); - if (ref_buf) gf_picture[0].frame = &ref_buf->buf; + gf_picture[0].frame = &ref_buf->buf; for (i = 0; i < 7; ++i) gf_picture[0].ref_frame[i] = -1; gld_index = 0; ++*tpl_group_frames; - // Initialize ARF frame + // Initialize base layer ARF frame gf_picture[1].frame = frame_input->source; gf_picture[1].ref_frame[0] = gld_index; - gf_picture[1].ref_frame[1] = lst_index; - gf_picture[1].ref_frame[2] = alt_index; // TODO(yuec) Need o figure out full AV1 reference model - for (i = 3; i < 7; ++i) gf_picture[1].ref_frame[i] = -1; - alt_index = 1; + for (i = 1; i < 7; ++i) gf_picture[1].ref_frame[i] = -1; ++*tpl_group_frames; // Initialize P frames for (frame_idx = 2; frame_idx < MAX_LAG_BUFFERS; ++frame_idx) { - struct lookahead_entry *buf = - av1_lookahead_peek(cpi->lookahead, frame_idx - 2); + struct lookahead_entry *buf; + frame_gop_offset = gf_group->frame_disp_idx[frame_idx]; + buf = av1_lookahead_peek(cpi->lookahead, frame_gop_offset - 1); if (buf == NULL) break; gf_picture[frame_idx].frame = &buf->img; gf_picture[frame_idx].ref_frame[0] = gld_index; gf_picture[frame_idx].ref_frame[1] = lst_index; - gf_picture[frame_idx].ref_frame[2] = alt_index; + gf_picture[frame_idx].ref_frame[2] = 1; for (i = 3; i < 7; ++i) gf_picture[frame_idx].ref_frame[i] = -1; ++*tpl_group_frames; lst_index = frame_idx; - if (frame_idx == cpi->rc.baseline_gf_interval + 1) break; + if (frame_idx == gf_group->size) break; } gld_index = frame_idx; lst_index = AOMMAX(0, frame_idx - 1); - alt_index = -1; ++frame_idx; + ++frame_gop_offset; // Extend two frames outside the current gf group. for (; frame_idx < MAX_LAG_BUFFERS && extend_frame_count < 2; ++frame_idx) { struct lookahead_entry *buf = - av1_lookahead_peek(cpi->lookahead, frame_idx - 2); + av1_lookahead_peek(cpi->lookahead, frame_gop_offset - 1); if (buf == NULL) break; @@ -556,11 +551,11 @@ static void init_gop_frames(AV1_COMP *cpi, GF_PICTURE *gf_picture, gf_picture[frame_idx].frame = &buf->img; gf_picture[frame_idx].ref_frame[0] = gld_index; gf_picture[frame_idx].ref_frame[1] = lst_index; - gf_picture[frame_idx].ref_frame[2] = alt_index; - for (i = 3; i < 7; ++i) gf_picture[frame_idx].ref_frame[i] = -1; + for (i = 2; i < 7; ++i) gf_picture[frame_idx].ref_frame[i] = -1; lst_index = frame_idx; ++*tpl_group_frames; ++extend_frame_count; + ++frame_gop_offset; } } @@ -582,7 +577,8 @@ void av1_tpl_setup_stats(AV1_COMP *cpi, int tpl_group_frames = 0; int frame_idx; - init_gop_frames(cpi, gf_picture, gf_group, &tpl_group_frames, frame_input); + init_gop_frames_for_tpl(cpi, gf_picture, gf_group, &tpl_group_frames, + frame_input); init_tpl_stats(cpi); diff --git a/third_party/aom/av1/encoder/var_based_part.c b/third_party/aom/av1/encoder/var_based_part.c index 3cead914ac..461bdd3ec5 100644 --- a/third_party/aom/av1/encoder/var_based_part.c +++ b/third_party/aom/av1/encoder/var_based_part.c @@ -227,36 +227,6 @@ static int set_vt_partitioning(AV1_COMP *cpi, MACROBLOCK *const x, set_block_size(cpi, x, xd, mi_row, mi_col, bsize); return 1; } - - // Check vertical split. - if (mi_row + block_height / 2 < cm->mi_rows) { - BLOCK_SIZE subsize = get_partition_subsize(bsize, PARTITION_VERT); - get_variance(&vt.part_variances->vert[0]); - get_variance(&vt.part_variances->vert[1]); - if (vt.part_variances->vert[0].variance < threshold && - vt.part_variances->vert[1].variance < threshold && - get_plane_block_size(subsize, xd->plane[1].subsampling_x, - xd->plane[1].subsampling_y) < BLOCK_INVALID) { - set_block_size(cpi, x, xd, mi_row, mi_col, subsize); - set_block_size(cpi, x, xd, mi_row, mi_col + block_width / 2, subsize); - return 1; - } - } - // Check horizontal split. - if (mi_col + block_width / 2 < cm->mi_cols) { - BLOCK_SIZE subsize = get_partition_subsize(bsize, PARTITION_HORZ); - get_variance(&vt.part_variances->horz[0]); - get_variance(&vt.part_variances->horz[1]); - if (vt.part_variances->horz[0].variance < threshold && - vt.part_variances->horz[1].variance < threshold && - get_plane_block_size(subsize, xd->plane[1].subsampling_x, - xd->plane[1].subsampling_y) < BLOCK_INVALID) { - set_block_size(cpi, x, xd, mi_row, mi_col, subsize); - set_block_size(cpi, x, xd, mi_row + block_height / 2, mi_col, subsize); - return 1; - } - } - return 0; } return 0; @@ -329,22 +299,15 @@ static void fill_variance_4x4avg(const uint8_t *s, int sp, const uint8_t *d, } } +// TODO(kyslov) Bring back threshold adjustment based on content state static int64_t scale_part_thresh_sumdiff(int64_t threshold_base, int speed, int width, int height, int content_state) { + (void)width; + (void)height; + (void)content_state; if (speed >= 8) { - if (width <= 640 && height <= 480) - return (5 * threshold_base) >> 2; - else if ((content_state == kLowSadLowSumdiff) || - (content_state == kHighSadLowSumdiff) || - (content_state == kLowVarHighSumdiff)) - return (5 * threshold_base) >> 2; - } else if (speed == 7) { - if ((content_state == kLowSadLowSumdiff) || - (content_state == kHighSadLowSumdiff) || - (content_state == kLowVarHighSumdiff)) { - return (5 * threshold_base) >> 2; - } + return (5 * threshold_base) >> 2; } return threshold_base; } @@ -372,7 +335,8 @@ static void set_vbp_thresholds(AV1_COMP *cpi, int64_t thresholds[], int q, threshold_base = scale_part_thresh_sumdiff( threshold_base, cpi->oxcf.speed, cm->width, cm->height, content_state); - thresholds[1] = threshold_base; + thresholds[0] = threshold_base; + thresholds[1] = threshold_base << 1; thresholds[3] = threshold_base << cpi->oxcf.speed; if (cm->width >= 1280 && cm->height >= 720) thresholds[3] = thresholds[3] << 1; @@ -384,7 +348,6 @@ static void set_vbp_thresholds(AV1_COMP *cpi, int64_t thresholds[], int q, thresholds[2] = (5 * threshold_base) >> 2; } else if (cm->width < 1920 && cm->height < 1080) { thresholds[2] = threshold_base << 1; - thresholds[3] <<= 2; } else { thresholds[2] = (5 * threshold_base) >> 1; } @@ -441,22 +404,23 @@ int av1_choose_var_based_partitioning(AV1_COMP *cpi, const TileInfo *const tile, v16x16 *vt2 = NULL; unsigned char force_split[85]; int avg_32x32; - int max_var_32x32 = 0; - int min_var_32x32 = INT_MAX; + int max_var_32x32[4]; + int min_var_32x32[4]; int var_32x32; int var_64x64; int min_var_64x64 = INT_MAX; int max_var_64x64 = 0; - int avg_16x16[4]; - int maxvar_16x16[4]; - int minvar_16x16[4]; + int avg_16x16[4][4]; + int maxvar_16x16[4][4]; + int minvar_16x16[4][4]; int64_t threshold_4x4avg; int content_state = 0; uint8_t *s; const uint8_t *d; int sp; int dp; - int compute_minmax_variance = 1; + // TODO(kyslov) Bring back compute_minmax_variance with content type detection + int compute_minmax_variance = 0; int is_key_frame = frame_is_intra_only(cm); int pixels_wide = 128, pixels_high = 128; assert(cm->seq_params.sb_size == BLOCK_64X64 || @@ -512,10 +476,12 @@ int av1_choose_var_based_partitioning(AV1_COMP *cpi, const TileInfo *const tile, mi->sb_type = cm->seq_params.sb_size; mi->mv[0].as_int = 0; mi->interp_filters = av1_make_interp_filters(BILINEAR, BILINEAR); - if (xd->mb_to_right_edge >= 0 && xd->mb_to_bottom_edge >= 0) { - const MV dummy_mv = { 0, 0 }; - av1_int_pro_motion_estimation(cpi, x, cm->seq_params.sb_size, mi_row, - mi_col, &dummy_mv); + if (cpi->sf.estimate_motion_for_var_based_partition) { + if (xd->mb_to_right_edge >= 0 && xd->mb_to_bottom_edge >= 0) { + const MV dummy_mv = { 0, 0 }; + av1_int_pro_motion_estimation(cpi, x, cm->seq_params.sb_size, mi_row, + mi_col, &dummy_mv); + } } // TODO(kyslov): bring the small SAD functionality back @@ -561,14 +527,16 @@ int av1_choose_var_based_partitioning(AV1_COMP *cpi, const TileInfo *const tile, const int y64_idx = ((m >> 1) << 6); const int m2 = m << 2; force_split[m + 1] = 0; + max_var_32x32[m] = 0; + min_var_32x32[m] = INT_MAX; for (i = 0; i < 4; i++) { const int x32_idx = x64_idx + ((i & 1) << 5); const int y32_idx = y64_idx + ((i >> 1) << 5); const int i2 = (m2 + i) << 2; force_split[5 + m2 + i] = 0; - avg_16x16[i] = 0; - maxvar_16x16[i] = 0; - minvar_16x16[i] = INT_MAX; + avg_16x16[m][i] = 0; + maxvar_16x16[m][i] = 0; + minvar_16x16[m][i] = INT_MAX; for (j = 0; j < 4; j++) { const int x16_idx = x32_idx + ((j & 1) << 4); const int y16_idx = y32_idx + ((j >> 1) << 4); @@ -581,15 +549,15 @@ int av1_choose_var_based_partitioning(AV1_COMP *cpi, const TileInfo *const tile, pixels_high, is_key_frame); fill_variance_tree(&vt->split[m].split[i].split[j], BLOCK_16X16); get_variance(&vt->split[m].split[i].split[j].part_variances.none); - avg_16x16[i] += + avg_16x16[m][i] += vt->split[m].split[i].split[j].part_variances.none.variance; if (vt->split[m].split[i].split[j].part_variances.none.variance < - minvar_16x16[i]) - minvar_16x16[i] = + minvar_16x16[m][i]) + minvar_16x16[m][i] = vt->split[m].split[i].split[j].part_variances.none.variance; if (vt->split[m].split[i].split[j].part_variances.none.variance > - maxvar_16x16[i]) - maxvar_16x16[i] = + maxvar_16x16[m][i]) + maxvar_16x16[m][i] = vt->split[m].split[i].split[j].part_variances.none.variance; if (vt->split[m].split[i].split[j].part_variances.none.variance > thresholds[3]) { @@ -669,21 +637,22 @@ int av1_choose_var_based_partitioning(AV1_COMP *cpi, const TileInfo *const tile, if (!force_split[5 + m2 + i]) { get_variance(&vt->split[m].split[i].part_variances.none); var_32x32 = vt->split[m].split[i].part_variances.none.variance; - max_var_32x32 = AOMMAX(var_32x32, max_var_32x32); - min_var_32x32 = AOMMIN(var_32x32, min_var_32x32); + max_var_32x32[m] = AOMMAX(var_32x32, max_var_32x32[m]); + min_var_32x32[m] = AOMMIN(var_32x32, min_var_32x32[m]); if (vt->split[m].split[i].part_variances.none.variance > thresholds[2] || (!is_key_frame && vt->split[m].split[i].part_variances.none.variance > (thresholds[2] >> 1) && vt->split[m].split[i].part_variances.none.variance > - (avg_16x16[i] >> 1))) { + (avg_16x16[m][i] >> 1))) { force_split[5 + m2 + i] = 1; force_split[m + 1] = 1; force_split[0] = 1; } else if (!is_key_frame && cm->height <= 360 && - (maxvar_16x16[i] - minvar_16x16[i]) > (thresholds[2] >> 1) && - maxvar_16x16[i] > thresholds[2]) { + (maxvar_16x16[m][i] - minvar_16x16[m][i]) > + (thresholds[2] >> 1) && + maxvar_16x16[m][i] > thresholds[2]) { force_split[5 + m2 + i] = 1; force_split[m + 1] = 1; force_split[0] = 1; @@ -702,8 +671,8 @@ int av1_choose_var_based_partitioning(AV1_COMP *cpi, const TileInfo *const tile, // split. Only checking this for noise level >= medium for now. if (!is_key_frame && - (max_var_32x32 - min_var_32x32) > 3 * (thresholds[1] >> 3) && - max_var_32x32 > thresholds[1] >> 1) + (max_var_32x32[m] - min_var_32x32[m]) > 3 * (thresholds[1] >> 3) && + max_var_32x32[m] > thresholds[1] >> 1) force_split[1 + m] = 1; } if (is_small_sb) force_split[0] = 1; diff --git a/third_party/aom/av1/encoder/x86/corner_match_avx2.c b/third_party/aom/av1/encoder/x86/corner_match_avx2.c index 7a3b99970e..554122bfce 100644 --- a/third_party/aom/av1/encoder/x86/corner_match_avx2.c +++ b/third_party/aom/av1/encoder/x86/corner_match_avx2.c @@ -17,9 +17,9 @@ #include "aom_ports/mem.h" #include "av1/encoder/corner_match.h" -DECLARE_ALIGNED(16, static const uint8_t, byte_mask[16]) = { - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 0, 0, 0 -}; +DECLARE_ALIGNED(16, static const uint8_t, + byte_mask[16]) = { 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 0, 0, 0 }; #if MATCH_SZ != 13 #error "Need to change byte_mask in corner_match_sse4.c if MATCH_SZ != 13" #endif diff --git a/third_party/aom/av1/encoder/x86/corner_match_sse4.c b/third_party/aom/av1/encoder/x86/corner_match_sse4.c index 93f37b71d3..1b5f470e08 100644 --- a/third_party/aom/av1/encoder/x86/corner_match_sse4.c +++ b/third_party/aom/av1/encoder/x86/corner_match_sse4.c @@ -21,9 +21,9 @@ #include "aom_ports/mem.h" #include "av1/encoder/corner_match.h" -DECLARE_ALIGNED(16, static const uint8_t, byte_mask[16]) = { - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 0, 0, 0 -}; +DECLARE_ALIGNED(16, static const uint8_t, + byte_mask[16]) = { 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 0, 0, 0 }; #if MATCH_SZ != 13 #error "Need to change byte_mask in corner_match_sse4.c if MATCH_SZ != 13" #endif diff --git a/third_party/aom/av1/encoder/x86/pickrst_avx2.c b/third_party/aom/av1/encoder/x86/pickrst_avx2.c index d00fca0594..5571165f85 100644 --- a/third_party/aom/av1/encoder/x86/pickrst_avx2.c +++ b/third_party/aom/av1/encoder/x86/pickrst_avx2.c @@ -22,9 +22,9 @@ static INLINE void acc_stat_avx2(int32_t *dst, const uint8_t *src, const __m128i *shuffle, const __m256i *kl) { const __m128i s = _mm_shuffle_epi8(xx_loadu_128(src), *shuffle); const __m256i d0 = _mm256_madd_epi16(*kl, _mm256_cvtepu8_epi16(s)); - const __m256i dst0 = yy_loadu_256(dst); + const __m256i dst0 = yy_load_256(dst); const __m256i r0 = _mm256_add_epi32(dst0, d0); - yy_storeu_256(dst, r0); + yy_store_256(dst, r0); } static INLINE void acc_stat_win7_one_line_avx2( @@ -74,7 +74,9 @@ static INLINE void compute_stats_win7_opt_avx2( int32_t M_int32[WIENER_WIN][WIENER_WIN] = { { 0 } }; int64_t M_int64[WIENER_WIN][WIENER_WIN] = { { 0 } }; - int32_t H_int32[WIENER_WIN2][WIENER_WIN * 8] = { { 0 } }; + + DECLARE_ALIGNED(32, int32_t, + H_int32[WIENER_WIN2][WIENER_WIN * 8]) = { { 0 } }; int64_t H_int64[WIENER_WIN2][WIENER_WIN * 8] = { { 0 } }; int32_t sumY[WIENER_WIN][WIENER_WIN] = { { 0 } }; int32_t sumX = 0; @@ -145,14 +147,14 @@ static INLINE void acc_stat_highbd_avx2(int64_t *dst, const uint16_t *dgd, // Take the lower-half of d0, extend to u64, add it on to dst (H) const __m256i d0l = _mm256_cvtepu32_epi64(_mm256_extracti128_si256(d0, 0)); // d0l = [a b] [c d] as u64 - const __m256i dst0 = yy_loadu_256(dst); - yy_storeu_256(dst, _mm256_add_epi64(d0l, dst0)); + const __m256i dst0 = yy_load_256(dst); + yy_store_256(dst, _mm256_add_epi64(d0l, dst0)); // Take the upper-half of d0, extend to u64, add it on to dst (H) const __m256i d0h = _mm256_cvtepu32_epi64(_mm256_extracti128_si256(d0, 1)); // d0h = [e f] [g h] as u64 - const __m256i dst1 = yy_loadu_256(dst + 4); - yy_storeu_256(dst + 4, _mm256_add_epi64(d0h, dst1)); + const __m256i dst1 = yy_load_256(dst + 4); + yy_store_256(dst + 4, _mm256_add_epi64(d0h, dst1)); } static INLINE void acc_stat_highbd_win7_one_line_avx2( @@ -216,7 +218,7 @@ static INLINE void compute_stats_highbd_win7_opt_avx2( find_average_highbd(dgd, h_start, h_end, v_start, v_end, dgd_stride); int64_t M_int[WIENER_WIN][WIENER_WIN] = { { 0 } }; - int64_t H_int[WIENER_WIN2][WIENER_WIN * 8] = { { 0 } }; + DECLARE_ALIGNED(32, int64_t, H_int[WIENER_WIN2][WIENER_WIN * 8]) = { { 0 } }; int32_t sumY[WIENER_WIN][WIENER_WIN] = { { 0 } }; int32_t sumX = 0; const uint16_t *dgd_win = dgd - wiener_halfwin * dgd_stride - wiener_halfwin; @@ -316,7 +318,9 @@ static INLINE void compute_stats_highbd_win5_opt_avx2( find_average_highbd(dgd, h_start, h_end, v_start, v_end, dgd_stride); int64_t M_int64[WIENER_WIN_CHROMA][WIENER_WIN_CHROMA] = { { 0 } }; - int64_t H_int64[WIENER_WIN2_CHROMA][WIENER_WIN_CHROMA * 8] = { { 0 } }; + DECLARE_ALIGNED( + 32, int64_t, + H_int64[WIENER_WIN2_CHROMA][WIENER_WIN_CHROMA * 8]) = { { 0 } }; int32_t sumY[WIENER_WIN_CHROMA][WIENER_WIN_CHROMA] = { { 0 } }; int32_t sumX = 0; const uint16_t *dgd_win = dgd - wiener_halfwin * dgd_stride - wiener_halfwin; @@ -423,7 +427,9 @@ static INLINE void compute_stats_win5_opt_avx2( int32_t M_int32[WIENER_WIN_CHROMA][WIENER_WIN_CHROMA] = { { 0 } }; int64_t M_int64[WIENER_WIN_CHROMA][WIENER_WIN_CHROMA] = { { 0 } }; - int32_t H_int32[WIENER_WIN2_CHROMA][WIENER_WIN_CHROMA * 8] = { { 0 } }; + DECLARE_ALIGNED( + 32, int32_t, + H_int32[WIENER_WIN2_CHROMA][WIENER_WIN_CHROMA * 8]) = { { 0 } }; int64_t H_int64[WIENER_WIN2_CHROMA][WIENER_WIN_CHROMA * 8] = { { 0 } }; int32_t sumY[WIENER_WIN_CHROMA][WIENER_WIN_CHROMA] = { { 0 } }; int32_t sumX = 0; diff --git a/third_party/aom/av1/encoder/x86/temporal_filter_constants.h b/third_party/aom/av1/encoder/x86/temporal_filter_constants.h index b3a10ddddc..7cd61d75ef 100644 --- a/third_party/aom/av1/encoder/x86/temporal_filter_constants.h +++ b/third_party/aom/av1/encoder/x86/temporal_filter_constants.h @@ -373,29 +373,35 @@ static const uint32_t *const HIGHBD_CHROMA_NO_SS_RIGHT_COLUMN_NEIGHBORS[2] = { HIGHBD_RIGHT_CORNER_NEIGHBORS_PLUS_1, HIGHBD_RIGHT_EDGE_NEIGHBORS_PLUS_1 }; -static const uint32_t *const HIGHBD_CHROMA_SINGLE_SS_LEFT_COLUMN_NEIGHBORS[2] = - { HIGHBD_LEFT_CORNER_NEIGHBORS_PLUS_2, HIGHBD_LEFT_EDGE_NEIGHBORS_PLUS_2 }; +static const uint32_t + *const HIGHBD_CHROMA_SINGLE_SS_LEFT_COLUMN_NEIGHBORS[2] = { + HIGHBD_LEFT_CORNER_NEIGHBORS_PLUS_2, HIGHBD_LEFT_EDGE_NEIGHBORS_PLUS_2 + }; static const uint32_t *const HIGHBD_CHROMA_SINGLE_SS_MIDDLE_COLUMN_NEIGHBORS[2] = { HIGHBD_MIDDLE_EDGE_NEIGHBORS_PLUS_2, HIGHBD_MIDDLE_CENTER_NEIGHBORS_PLUS_2 }; -static const uint32_t *const HIGHBD_CHROMA_SINGLE_SS_RIGHT_COLUMN_NEIGHBORS[2] = - { HIGHBD_RIGHT_CORNER_NEIGHBORS_PLUS_2, - HIGHBD_RIGHT_EDGE_NEIGHBORS_PLUS_2 }; +static const uint32_t + *const HIGHBD_CHROMA_SINGLE_SS_RIGHT_COLUMN_NEIGHBORS[2] = { + HIGHBD_RIGHT_CORNER_NEIGHBORS_PLUS_2, HIGHBD_RIGHT_EDGE_NEIGHBORS_PLUS_2 + }; -static const uint32_t *const HIGHBD_CHROMA_DOUBLE_SS_LEFT_COLUMN_NEIGHBORS[2] = - { HIGHBD_LEFT_CORNER_NEIGHBORS_PLUS_4, HIGHBD_LEFT_EDGE_NEIGHBORS_PLUS_4 }; +static const uint32_t + *const HIGHBD_CHROMA_DOUBLE_SS_LEFT_COLUMN_NEIGHBORS[2] = { + HIGHBD_LEFT_CORNER_NEIGHBORS_PLUS_4, HIGHBD_LEFT_EDGE_NEIGHBORS_PLUS_4 + }; static const uint32_t *const HIGHBD_CHROMA_DOUBLE_SS_MIDDLE_COLUMN_NEIGHBORS[2] = { HIGHBD_MIDDLE_EDGE_NEIGHBORS_PLUS_4, HIGHBD_MIDDLE_CENTER_NEIGHBORS_PLUS_4 }; -static const uint32_t *const HIGHBD_CHROMA_DOUBLE_SS_RIGHT_COLUMN_NEIGHBORS[2] = - { HIGHBD_RIGHT_CORNER_NEIGHBORS_PLUS_4, - HIGHBD_RIGHT_EDGE_NEIGHBORS_PLUS_4 }; +static const uint32_t + *const HIGHBD_CHROMA_DOUBLE_SS_RIGHT_COLUMN_NEIGHBORS[2] = { + HIGHBD_RIGHT_CORNER_NEIGHBORS_PLUS_4, HIGHBD_RIGHT_EDGE_NEIGHBORS_PLUS_4 + }; #define DIST_STRIDE ((BW) + 2) #endif // AOM_AV1_ENCODER_X86_TEMPORAL_FILTER_CONSTANTS_H_ diff --git a/third_party/aom/build/cmake/sanitizers.cmake b/third_party/aom/build/cmake/sanitizers.cmake index 77708e101c..a8bd93cb1d 100644 --- a/third_party/aom/build/cmake/sanitizers.cmake +++ b/third_party/aom/build/cmake/sanitizers.cmake @@ -21,13 +21,34 @@ include("${AOM_ROOT}/build/cmake/compiler_flags.cmake") string(TOLOWER ${SANITIZE} SANITIZE) -# Require the sanitizer requested. -require_linker_flag("-fsanitize=${SANITIZE}") -require_compiler_flag("-fsanitize=${SANITIZE}" YES) +# Require the sanitizer requested. cfi sanitizer requires all the flags in order +# for the compiler to accept it. +if("${SANITIZE}" MATCHES "cfi" AND CMAKE_C_COMPILER_ID MATCHES "Clang") + require_linker_flag( + "-fsanitize=${SANITIZE} -flto -fno-sanitize-trap=cfi \ + -fuse-ld=gold" + YES) + require_compiler_flag( + "-fsanitize=${SANITIZE} -flto -fvisibility=hidden \ + -fno-sanitize-trap=cfi" + YES) +else() + require_linker_flag("-fsanitize=${SANITIZE}") + require_compiler_flag("-fsanitize=${SANITIZE}" YES) +endif() # Make callstacks accurate. require_compiler_flag("-fno-omit-frame-pointer -fno-optimize-sibling-calls" YES) +# TODO(http://crbug.com/aomedia/2357): -fsanitize=implicit-integer-truncation +# causes conversion warnings in many of the x86 intrinsics and elsewhere. +if("${SANITIZE}" MATCHES "integer" AND CMAKE_C_COMPILER_ID MATCHES "Clang") + if(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL "7.0.1") + require_compiler_flag("-fno-sanitize=implicit-integer-truncation" YES) + require_linker_flag("-fno-sanitize=implicit-integer-truncation" YES) + endif() +endif() + # Fix link errors due to missing rt compiler lib in 32-bit builds. # http://llvm.org/bugs/show_bug.cgi?id=17693 if(CMAKE_C_COMPILER_ID MATCHES "Clang") diff --git a/third_party/aom/common/av1_config.c b/third_party/aom/common/av1_config.c index 90955fb5b3..9f5b02015b 100644 --- a/third_party/aom/common/av1_config.c +++ b/third_party/aom/common/av1_config.c @@ -237,9 +237,9 @@ static int parse_sequence_header(const uint8_t *const buffer, size_t length, // The reader instance is local to this function, but a pointer to the // reader instance is used within this function and throughout this file to // allow use of the helper macros that reduce parse error checking verbosity. - struct aom_read_bit_buffer reader_instance = { - buffer, buffer + length, 0, &result, bitreader_error_handler - }; + struct aom_read_bit_buffer reader_instance = { buffer, buffer + length, 0, + &result, + bitreader_error_handler }; struct aom_read_bit_buffer *reader = &reader_instance; AV1C_READ_BITS_OR_RETURN_ERROR(seq_profile, 3); @@ -416,9 +416,9 @@ int read_av1config(const uint8_t *buffer, size_t buffer_length, *bytes_read = 0; int result = 0; - struct aom_read_bit_buffer reader_instance = { - buffer, buffer + buffer_length, 0, &result, bitreader_error_handler - }; + struct aom_read_bit_buffer reader_instance = { buffer, buffer + buffer_length, + 0, &result, + bitreader_error_handler }; struct aom_read_bit_buffer *reader = &reader_instance; memset(config, 0, sizeof(*config)); diff --git a/third_party/aom/common/obudec.c b/third_party/aom/common/obudec.c index bd9f98d3ae..1ce218d688 100644 --- a/third_party/aom/common/obudec.c +++ b/third_party/aom/common/obudec.c @@ -438,13 +438,15 @@ int obudec_read_temporal_unit(struct ObuDecInputContext *obu_ctx, return -1; } #endif - uint8_t *new_buffer = (uint8_t *)realloc(*buffer, tu_size); - if (!new_buffer) { - free(*buffer); - fprintf(stderr, "obudec: Out of memory.\n"); - return -1; + if (tu_size > 0) { + uint8_t *new_buffer = (uint8_t *)realloc(*buffer, tu_size); + if (!new_buffer) { + free(*buffer); + fprintf(stderr, "obudec: Out of memory.\n"); + return -1; + } + *buffer = new_buffer; } - *buffer = new_buffer; *bytes_read = tu_size; *buffer_size = tu_size; diff --git a/third_party/aom/examples/inspect.c b/third_party/aom/examples/inspect.c index 9ca2a02c10..9e9e546bd6 100644 --- a/third_party/aom/examples/inspect.c +++ b/third_party/aom/examples/inspect.c @@ -248,9 +248,9 @@ const map_entry uv_prediction_mode_map[] = { const map_entry skip_map[] = { ENUM(SKIP), ENUM(NO_SKIP), LAST_ENUM }; -const map_entry intrabc_map[] = { - { "INTRABC", 1 }, { "NO_INTRABC", 0 }, LAST_ENUM -}; +const map_entry intrabc_map[] = { { "INTRABC", 1 }, + { "NO_INTRABC", 0 }, + LAST_ENUM }; const map_entry palette_map[] = { { "ZERO_COLORS", 0 }, { "TWO_COLORS", 2 }, { "THREE_COLORS", 3 }, diff --git a/third_party/aom/test/aom_integer_test.cc b/third_party/aom/test/aom_integer_test.cc index fe88a54e9e..d5dfad946b 100644 --- a/third_party/aom/test/aom_integer_test.cc +++ b/third_party/aom/test/aom_integer_test.cc @@ -20,9 +20,9 @@ const uint32_t kSizeTestNumValues = 6; const uint32_t kSizeTestExpectedSizes[kSizeTestNumValues] = { 1, 1, 2, 3, 4, 5 }; -const uint64_t kSizeTestInputs[kSizeTestNumValues] = { - 0, 0x7f, 0x3fff, 0x1fffff, 0xffffff, 0x10000000 -}; +const uint64_t kSizeTestInputs[kSizeTestNumValues] = { 0, 0x7f, + 0x3fff, 0x1fffff, + 0xffffff, 0x10000000 }; const uint8_t kOutOfRangeLeb128Value[5] = { 0x80, 0x80, 0x80, 0x80, 0x10 }; // UINT32_MAX + 1 diff --git a/third_party/aom/test/av1_config_test.cc b/third_party/aom/test/av1_config_test.cc index e2f2c53906..fca980f062 100644 --- a/third_party/aom/test/av1_config_test.cc +++ b/third_party/aom/test/av1_config_test.cc @@ -24,20 +24,20 @@ namespace { // Sequence Header OBUs vs Sequence Header OBUs with the // reduced_still_image_flag set). // -const uint8_t kAnnexBFullSequenceHeaderObu[] = { - 0x0c, 0x08, 0x00, 0x00, 0x00, 0x04, 0x45, 0x7e, 0x3e, 0xff, 0xfc, 0xc0, 0x20 -}; +const uint8_t kAnnexBFullSequenceHeaderObu[] = { 0x0c, 0x08, 0x00, 0x00, 0x00, + 0x04, 0x45, 0x7e, 0x3e, 0xff, + 0xfc, 0xc0, 0x20 }; const uint8_t kAnnexBReducedStillImageSequenceHeaderObu[] = { 0x08, 0x08, 0x18, 0x22, 0x2b, 0xf1, 0xfe, 0xc0, 0x20 }; -const uint8_t kLobfFullSequenceHeaderObu[] = { - 0x0a, 0x0b, 0x00, 0x00, 0x00, 0x04, 0x45, 0x7e, 0x3e, 0xff, 0xfc, 0xc0, 0x20 -}; +const uint8_t kLobfFullSequenceHeaderObu[] = { 0x0a, 0x0b, 0x00, 0x00, 0x00, + 0x04, 0x45, 0x7e, 0x3e, 0xff, + 0xfc, 0xc0, 0x20 }; -const uint8_t kLobfReducedStillImageSequenceHeaderObu[] = { - 0x0a, 0x07, 0x18, 0x22, 0x2b, 0xf1, 0xfe, 0xc0, 0x20 -}; +const uint8_t kLobfReducedStillImageSequenceHeaderObu[] = { 0x0a, 0x07, 0x18, + 0x22, 0x2b, 0xf1, + 0xfe, 0xc0, 0x20 }; const uint8_t kAv1cAllZero[] = { 0, 0, 0, 0 }; diff --git a/third_party/aom/test/av1_inv_txfm1d_test.cc b/third_party/aom/test/av1_inv_txfm1d_test.cc index bf3a44ed14..0fc2b03131 100644 --- a/third_party/aom/test/av1_inv_txfm1d_test.cc +++ b/third_party/aom/test/av1_inv_txfm1d_test.cc @@ -16,6 +16,8 @@ #include "av1/common/av1_inv_txfm1d.h" #include "av1/encoder/av1_fwd_txfm1d.h" +typedef TX_SIZE TxSize; + using libaom_test::ACMRandom; using libaom_test::input_base; @@ -77,7 +79,7 @@ TEST(av1_inv_txfm1d, InvAccuracyCheck) { ASSERT_EQ(NELEMENTS(inv_txfm_func_ls), TX_SIZES); for (int k = 0; k < count_test_block; ++k) { // choose a random transform to test - const TX_SIZE tx_size = static_cast(rnd.Rand8() % TX_SIZES); + const TxSize tx_size = static_cast(rnd.Rand8() % TX_SIZES); const int tx_size_pix = txfm_size_ls[tx_size]; const TxfmFunc inv_txfm_func = inv_txfm_func_ls[tx_size][0]; @@ -88,9 +90,11 @@ TEST(av1_inv_txfm1d, InvAccuracyCheck) { memset(input + 32, 0, 32 * sizeof(input[0])); int32_t ref_output[64]; + memset(ref_output, 0, sizeof(ref_output)); reference_idct_1d_int(input, ref_output, tx_size_pix); int32_t output[64]; + memset(output, 0, sizeof(output)); inv_txfm_func(input, output, cos_bit, range_bit); for (int i = 0; i < tx_size_pix; ++i) { diff --git a/third_party/aom/test/av1_inv_txfm2d_test.cc b/third_party/aom/test/av1_inv_txfm2d_test.cc index 5432130d76..df3bafb28a 100644 --- a/third_party/aom/test/av1_inv_txfm2d_test.cc +++ b/third_party/aom/test/av1_inv_txfm2d_test.cc @@ -36,11 +36,33 @@ using ::testing::Values; using std::vector; +typedef TX_TYPE TxType; +typedef TX_SIZE TxSize; + namespace { +static const char *tx_type_name[] = { + "DCT_DCT", + "ADST_DCT", + "DCT_ADST", + "ADST_ADST", + "FLIPADST_DCT", + "DCT_FLIPADST", + "FLIPADST_FLIPADST", + "ADST_FLIPADST", + "FLIPADST_ADST", + "IDTX", + "V_DCT", + "H_DCT", + "V_ADST", + "H_ADST", + "V_FLIPADST", + "H_FLIPADST", +}; + // AV1InvTxfm2dParam argument list: // tx_type_, tx_size_, max_error_, max_avg_error_ -typedef ::testing::tuple AV1InvTxfm2dParam; +typedef ::testing::tuple AV1InvTxfm2dParam; class AV1InvTxfm2d : public ::testing::TestWithParam { public: @@ -86,7 +108,7 @@ class AV1InvTxfm2d : public ::testing::TestWithParam { } double ref_coeffs[64 * 64] = { 0 }; ASSERT_LE(txfm2d_size, NELEMENTS(ref_coeffs)); - ASSERT_EQ(tx_type_, DCT_DCT); + ASSERT_EQ(tx_type_, static_cast(DCT_DCT)); libaom_test::reference_hybrid_2d(ref_input, ref_coeffs, tx_type_, tx_size_); DECLARE_ALIGNED(16, int32_t, ref_coeffs_int[64 * 64]) = { 0 }; @@ -139,8 +161,8 @@ class AV1InvTxfm2d : public ::testing::TestWithParam { int max_error_; double max_avg_error_; - TX_TYPE tx_type_; - TX_SIZE tx_size_; + TxType tx_type_; + TxSize tx_size_; }; static int max_error_ls[TX_SIZES_ALL] = { @@ -193,8 +215,8 @@ vector GetInvTxfm2dParamList() { const int max_error = max_error_ls[s]; const double avg_error = avg_error_ls[s]; for (int t = 0; t < TX_TYPES; ++t) { - const TX_TYPE tx_type = static_cast(t); - const TX_SIZE tx_size = static_cast(s); + const TxType tx_type = static_cast(t); + const TxSize tx_size = static_cast(s); if (libaom_test::IsTxSizeTypeValid(tx_size, tx_type)) { param_list.push_back( AV1InvTxfm2dParam(tx_type, tx_size, max_error, avg_error)); @@ -216,18 +238,18 @@ TEST(AV1InvTxfm2d, CfgTest) { int8_t high_range = libaom_test::high_range_arr[bd_idx]; for (int tx_size = 0; tx_size < TX_SIZES_ALL; ++tx_size) { for (int tx_type = 0; tx_type < TX_TYPES; ++tx_type) { - if (libaom_test::IsTxSizeTypeValid(static_cast(tx_size), - static_cast(tx_type)) == + if (libaom_test::IsTxSizeTypeValid(static_cast(tx_size), + static_cast(tx_type)) == false) { continue; } TXFM_2D_FLIP_CFG cfg; - av1_get_inv_txfm_cfg(static_cast(tx_type), - static_cast(tx_size), &cfg); + av1_get_inv_txfm_cfg(static_cast(tx_type), + static_cast(tx_size), &cfg); int8_t stage_range_col[MAX_TXFM_STAGE_NUM]; int8_t stage_range_row[MAX_TXFM_STAGE_NUM]; av1_gen_inv_stage_range(stage_range_col, stage_range_row, &cfg, - (TX_SIZE)tx_size, bd); + static_cast(tx_size), bd); libaom_test::txfm_stage_range_check(stage_range_col, cfg.stage_num_col, cfg.cos_bit_col, low_range, high_range); @@ -243,14 +265,15 @@ typedef ::testing::tuple AV1LbdInvTxfm2dParam; class AV1LbdInvTxfm2d : public ::testing::TestWithParam { public: virtual void SetUp() { target_func_ = GET_PARAM(0); } - void RunAV1InvTxfm2dTest(TX_TYPE tx_type, TX_SIZE tx_size, int run_times); + void RunAV1InvTxfm2dTest(TxType tx_type, TxSize tx_size, int run_times, + int gt_int16 = 0); private: LbdInvTxfm2dFunc target_func_; }; -void AV1LbdInvTxfm2d::RunAV1InvTxfm2dTest(TX_TYPE tx_type, TX_SIZE tx_size, - int run_times) { +void AV1LbdInvTxfm2d::RunAV1InvTxfm2dTest(TxType tx_type, TxSize tx_size, + int run_times, int gt_int16) { FwdTxfm2dFunc fwd_func_ = libaom_test::fwd_txfm_func_ls[tx_size]; InvTxfm2dFunc ref_func_ = libaom_test::inv_txfm_func_ls[tx_size]; if (fwd_func_ == NULL || ref_func_ == NULL || target_func_ == NULL) { @@ -275,6 +298,7 @@ void AV1LbdInvTxfm2d::RunAV1InvTxfm2dTest(TX_TYPE tx_type, TX_SIZE tx_size, const int16_t eobmax = rows_nonezero * cols_nonezero; ACMRandom rnd(ACMRandom::DeterministicSeed()); int randTimes = run_times == 1 ? (eobmax + 500) : 1; + for (int cnt = 0; cnt < randTimes; ++cnt) { const int16_t max_in = (1 << (bd)) - 1; for (int r = 0; r < BLK_WIDTH; ++r) { @@ -291,7 +315,9 @@ void AV1LbdInvTxfm2d::RunAV1InvTxfm2dTest(TX_TYPE tx_type, TX_SIZE tx_size, for (int i = eob; i < eobmax; i++) { inv_input[scan[i]] = 0; } - + if (gt_int16) { + inv_input[scan[eob - 1]] = ((int32_t)INT16_MAX * 100 / 141); + } aom_usec_timer timer; aom_usec_timer_start(&timer); for (int i = 0; i < run_times; ++i) { @@ -313,10 +339,13 @@ void AV1LbdInvTxfm2d::RunAV1InvTxfm2dTest(TX_TYPE tx_type, TX_SIZE tx_size, for (int r = 0; r < rows; ++r) { for (int c = 0; c < cols; ++c) { uint8_t ref_value = static_cast(ref_output[r * stride + c]); + if (ref_value != output[r * stride + c]) { + printf(" "); + } ASSERT_EQ(ref_value, output[r * stride + c]) << "[" << r << "," << c << "] " << cnt << " tx_size: " << static_cast(tx_size) - << " tx_type: " << tx_type << " eob " << eob; + << " tx_type: " << tx_type_name[tx_type] << " eob " << eob; } } } @@ -325,21 +354,34 @@ void AV1LbdInvTxfm2d::RunAV1InvTxfm2dTest(TX_TYPE tx_type, TX_SIZE tx_size, TEST_P(AV1LbdInvTxfm2d, match) { for (int j = 0; j < (int)(TX_SIZES_ALL); ++j) { for (int i = 0; i < (int)TX_TYPES; ++i) { - if (libaom_test::IsTxSizeTypeValid(static_cast(j), - static_cast(i))) { - RunAV1InvTxfm2dTest(static_cast(i), static_cast(j), - 1); + if (libaom_test::IsTxSizeTypeValid(static_cast(j), + static_cast(i))) { + RunAV1InvTxfm2dTest(static_cast(i), static_cast(j), 1); + } + } + } +} + +TEST_P(AV1LbdInvTxfm2d, gt_int16) { + static const TxType types[] = { DCT_DCT, ADST_DCT, FLIPADST_DCT, IDTX, + V_DCT, H_DCT, H_ADST, H_FLIPADST }; + for (int j = 0; j < (int)(TX_SIZES_ALL); ++j) { + const TxSize sz = static_cast(j); + for (uint8_t i = 0; i < sizeof(types) / sizeof(types[0]); ++i) { + const TxType tp = types[i]; + if (libaom_test::IsTxSizeTypeValid(sz, tp)) { + RunAV1InvTxfm2dTest(tp, sz, 1, 1); } } } } TEST_P(AV1LbdInvTxfm2d, DISABLED_Speed) { - for (int j = 0; j < (int)(TX_SIZES_ALL); ++j) { + for (int j = 1; j < (int)(TX_SIZES_ALL); ++j) { for (int i = 0; i < (int)TX_TYPES; ++i) { - if (libaom_test::IsTxSizeTypeValid(static_cast(j), - static_cast(i))) { - RunAV1InvTxfm2dTest(static_cast(i), static_cast(j), + if (libaom_test::IsTxSizeTypeValid(static_cast(j), + static_cast(i))) { + RunAV1InvTxfm2dTest(static_cast(i), static_cast(j), 10000000); } } @@ -357,22 +399,24 @@ INSTANTIATE_TEST_CASE_P(SSSE3, AV1LbdInvTxfm2d, #if HAVE_AVX2 extern "C" void av1_lowbd_inv_txfm2d_add_avx2(const int32_t *input, uint8_t *output, int stride, - TX_TYPE tx_type, TX_SIZE tx_size, + TxType tx_type, TxSize tx_size, int eob); INSTANTIATE_TEST_CASE_P(AVX2, AV1LbdInvTxfm2d, ::testing::Values(av1_lowbd_inv_txfm2d_add_avx2)); #endif // HAVE_AVX2 -#if HAVE_NEON - -extern "C" void av1_lowbd_inv_txfm2d_add_neon(const int32_t *input, - uint8_t *output, int stride, - TX_TYPE tx_type, TX_SIZE tx_size, - int eob); - -INSTANTIATE_TEST_CASE_P(NEON, AV1LbdInvTxfm2d, - ::testing::Values(av1_lowbd_inv_txfm2d_add_neon)); -#endif // HAVE_NEON +// TODO(yunqing): Re-enable this unit test for NEON version after the functions +// are fixed. +// #if HAVE_NEON +// extern "C" void av1_lowbd_inv_txfm2d_add_neon(const int32_t *input, +// uint8_t *output, int stride, +// TX_TYPE tx_type, +// TX_SIZE tx_size, +// int eob); +// +// INSTANTIATE_TEST_CASE_P(NEON, AV1LbdInvTxfm2d, +// ::testing::Values(av1_lowbd_inv_txfm2d_add_neon)); +// #endif // HAVE_NEON } // namespace diff --git a/third_party/aom/test/cfl_test.cc b/third_party/aom/test/cfl_test.cc index f087dd9fbd..9515b67a49 100644 --- a/third_party/aom/test/cfl_test.cc +++ b/third_party/aom/test/cfl_test.cc @@ -24,30 +24,37 @@ using libaom_test::ACMRandom; #define NUM_ITERATIONS (100) #define NUM_ITERATIONS_SPEED (INT16_MAX) -#define ALL_CFL_TX_SIZES(function) \ - make_tuple(TX_4X4, &function), make_tuple(TX_4X8, &function), \ - make_tuple(TX_4X16, &function), make_tuple(TX_8X4, &function), \ - make_tuple(TX_8X8, &function), make_tuple(TX_8X16, &function), \ - make_tuple(TX_8X32, &function), make_tuple(TX_16X4, &function), \ - make_tuple(TX_16X8, &function), make_tuple(TX_16X16, &function), \ - make_tuple(TX_16X32, &function), make_tuple(TX_32X8, &function), \ - make_tuple(TX_32X16, &function), make_tuple(TX_32X32, &function) +#define ALL_CFL_TX_SIZES(function) \ + make_tuple(static_cast(TX_4X4), &function), \ + make_tuple(static_cast(TX_4X8), &function), \ + make_tuple(static_cast(TX_4X16), &function), \ + make_tuple(static_cast(TX_8X4), &function), \ + make_tuple(static_cast(TX_8X8), &function), \ + make_tuple(static_cast(TX_8X16), &function), \ + make_tuple(static_cast(TX_8X32), &function), \ + make_tuple(static_cast(TX_16X4), &function), \ + make_tuple(static_cast(TX_16X8), &function), \ + make_tuple(static_cast(TX_16X16), &function), \ + make_tuple(static_cast(TX_16X32), &function), \ + make_tuple(static_cast(TX_32X8), &function), \ + make_tuple(static_cast(TX_32X16), &function), \ + make_tuple(static_cast(TX_32X32), &function) -#define ALL_CFL_TX_SIZES_SUBSAMPLE(fun420, fun422, fun444) \ - make_tuple(TX_4X4, &fun420, &fun422, &fun444), \ - make_tuple(TX_4X8, &fun420, &fun422, &fun444), \ - make_tuple(TX_4X16, &fun420, &fun422, &fun444), \ - make_tuple(TX_8X4, &fun420, &fun422, &fun444), \ - make_tuple(TX_8X8, &fun420, &fun422, &fun444), \ - make_tuple(TX_8X16, &fun420, &fun422, &fun444), \ - make_tuple(TX_8X32, &fun420, &fun422, &fun444), \ - make_tuple(TX_16X4, &fun420, &fun422, &fun444), \ - make_tuple(TX_16X8, &fun420, &fun422, &fun444), \ - make_tuple(TX_16X16, &fun420, &fun422, &fun444), \ - make_tuple(TX_16X32, &fun420, &fun422, &fun444), \ - make_tuple(TX_32X8, &fun420, &fun422, &fun444), \ - make_tuple(TX_32X16, &fun420, &fun422, &fun444), \ - make_tuple(TX_32X32, &fun420, &fun422, &fun444) +#define ALL_CFL_TX_SIZES_SUBSAMPLE(fun420, fun422, fun444) \ + make_tuple(static_cast(TX_4X4), &fun420, &fun422, &fun444), \ + make_tuple(static_cast(TX_4X8), &fun420, &fun422, &fun444), \ + make_tuple(static_cast(TX_4X16), &fun420, &fun422, &fun444), \ + make_tuple(static_cast(TX_8X4), &fun420, &fun422, &fun444), \ + make_tuple(static_cast(TX_8X8), &fun420, &fun422, &fun444), \ + make_tuple(static_cast(TX_8X16), &fun420, &fun422, &fun444), \ + make_tuple(static_cast(TX_8X32), &fun420, &fun422, &fun444), \ + make_tuple(static_cast(TX_16X4), &fun420, &fun422, &fun444), \ + make_tuple(static_cast(TX_16X8), &fun420, &fun422, &fun444), \ + make_tuple(static_cast(TX_16X16), &fun420, &fun422, &fun444), \ + make_tuple(static_cast(TX_16X32), &fun420, &fun422, &fun444), \ + make_tuple(static_cast(TX_32X8), &fun420, &fun422, &fun444), \ + make_tuple(static_cast(TX_32X16), &fun420, &fun422, &fun444), \ + make_tuple(static_cast(TX_32X32), &fun420, &fun422, &fun444) namespace { diff --git a/third_party/aom/test/convolve_test.cc b/third_party/aom/test/convolve_test.cc index 760d2c579d..eaa9aebd5d 100644 --- a/third_party/aom/test/convolve_test.cc +++ b/third_party/aom/test/convolve_test.cc @@ -805,9 +805,9 @@ const ConvolveFunctions convolve10_c(wrap_convolve_copy_c_10, const ConvolveFunctions convolve12_c(wrap_convolve_copy_c_12, wrap_convolve8_horiz_c_12, wrap_convolve8_vert_c_12, 12); -const ConvolveParam kArrayConvolve_c[] = { - ALL_SIZES(convolve8_c), ALL_SIZES(convolve10_c), ALL_SIZES(convolve12_c) -}; +const ConvolveParam kArrayConvolve_c[] = { ALL_SIZES(convolve8_c), + ALL_SIZES(convolve10_c), + ALL_SIZES(convolve12_c) }; INSTANTIATE_TEST_CASE_P(C, ConvolveTest, ::testing::ValuesIn(kArrayConvolve_c)); diff --git a/third_party/aom/test/corner_match_test.cc b/third_party/aom/test/corner_match_test.cc index af2baa7b14..0f79b52e65 100644 --- a/third_party/aom/test/corner_match_test.cc +++ b/third_party/aom/test/corner_match_test.cc @@ -127,15 +127,15 @@ TEST_P(AV1CornerMatchTest, DISABLED_Speed) { RunCheckOutput(100000); } #if HAVE_SSE4_1 INSTANTIATE_TEST_CASE_P( SSE4_1, AV1CornerMatchTest, - ::testing::Values(make_tuple(0, compute_cross_correlation_sse4_1), - make_tuple(1, compute_cross_correlation_sse4_1))); + ::testing::Values(make_tuple(0, &compute_cross_correlation_sse4_1), + make_tuple(1, &compute_cross_correlation_sse4_1))); #endif #if HAVE_AVX2 INSTANTIATE_TEST_CASE_P( AVX2, AV1CornerMatchTest, - ::testing::Values(make_tuple(0, compute_cross_correlation_avx2), - make_tuple(1, compute_cross_correlation_avx2))); + ::testing::Values(make_tuple(0, &compute_cross_correlation_avx2), + make_tuple(1, &compute_cross_correlation_avx2))); #endif } // namespace AV1CornerMatch diff --git a/third_party/aom/test/dr_prediction_test.cc b/third_party/aom/test/dr_prediction_test.cc index 4be8489b7d..46bb6494c7 100644 --- a/third_party/aom/test/dr_prediction_test.cc +++ b/third_party/aom/test/dr_prediction_test.cc @@ -365,11 +365,9 @@ INSTANTIATE_TEST_CASE_P( ::testing::Values(DrPredFunc(&z1_wrapper, &z1_wrapper, AOM_BITS_8, kZ1Start), - /* TODO(niva213@gmail.com): Re-enable this test after - fixing valgrind issue: https://crbug.com/aomedia/2316 DrPredFunc(&z2_wrapper, &z2_wrapper, - AOM_BITS_8, kZ2Start), */ + AOM_BITS_8, kZ2Start), DrPredFunc(&z3_wrapper, &z3_wrapper, AOM_BITS_8, kZ3Start))); diff --git a/third_party/aom/test/hiprec_convolve_test_util.cc b/third_party/aom/test/hiprec_convolve_test_util.cc index 2672bcec3d..f5bf56e907 100644 --- a/third_party/aom/test/hiprec_convolve_test_util.cc +++ b/third_party/aom/test/hiprec_convolve_test_util.cc @@ -31,7 +31,7 @@ static void generate_kernels(ACMRandom *rnd, InterpKernel hkernel, hkernel[2] = hkernel[4] = WIENER_FILT_TAP2_MINV + rnd->PseudoUniform(WIENER_FILT_TAP2_MAXV + 1 - WIENER_FILT_TAP2_MINV); - hkernel[3] = -(hkernel[0] + hkernel[1] + hkernel[2]); + hkernel[3] = -2 * (hkernel[0] + hkernel[1] + hkernel[2]); hkernel[7] = 0; vkernel[0] = vkernel[6] = @@ -43,7 +43,7 @@ static void generate_kernels(ACMRandom *rnd, InterpKernel hkernel, vkernel[2] = vkernel[4] = WIENER_FILT_TAP2_MINV + rnd->PseudoUniform(WIENER_FILT_TAP2_MAXV + 1 - WIENER_FILT_TAP2_MINV); - vkernel[3] = -(vkernel[0] + vkernel[1] + vkernel[2]); + vkernel[3] = -2 * (vkernel[0] + vkernel[1] + vkernel[2]); vkernel[7] = 0; } diff --git a/third_party/aom/test/quantize_func_test.cc b/third_party/aom/test/quantize_func_test.cc index 067a981825..1ab1c60403 100644 --- a/third_party/aom/test/quantize_func_test.cc +++ b/third_party/aom/test/quantize_func_test.cc @@ -63,7 +63,7 @@ void highbd_quan64x64_wrapper(QUAN_PARAM_LIST) { HBD_QUAN_FUNC; } -enum { TYPE_B, TYPE_DC, TYPE_FP } UENUM1BYTE(QuantType); +enum QuantType { TYPE_B, TYPE_DC, TYPE_FP }; using ::testing::tuple; typedef tuple @@ -296,6 +296,8 @@ TEST_P(QuantizeTest, DISABLED_Speed) { aom_usec_timer timer, simd_timer; int rows = tx_size_high[tx_size_]; int cols = tx_size_wide[tx_size_]; + rows = AOMMIN(32, rows); + cols = AOMMIN(32, cols); for (int cnt = 0; cnt <= rows; cnt++) { FillCoeffRandomRows(cnt * cols); @@ -325,57 +327,63 @@ using ::testing::make_tuple; #if HAVE_AVX2 const QuantizeParam kQParamArrayAvx2[] = { - make_tuple(&av1_quantize_fp_c, &av1_quantize_fp_avx2, TX_16X16, TYPE_FP, - AOM_BITS_8), - make_tuple(&av1_quantize_fp_c, &av1_quantize_fp_avx2, TX_4X16, TYPE_FP, - AOM_BITS_8), - make_tuple(&av1_quantize_fp_c, &av1_quantize_fp_avx2, TX_16X4, TYPE_FP, - AOM_BITS_8), - make_tuple(&av1_quantize_fp_c, &av1_quantize_fp_avx2, TX_32X8, TYPE_FP, - AOM_BITS_8), - make_tuple(&av1_quantize_fp_c, &av1_quantize_fp_avx2, TX_8X32, TYPE_FP, - AOM_BITS_8), - make_tuple(&av1_quantize_fp_32x32_c, &av1_quantize_fp_32x32_avx2, TX_32X32, - TYPE_FP, AOM_BITS_8), - make_tuple(&av1_quantize_fp_32x32_c, &av1_quantize_fp_32x32_avx2, TX_16X64, - TYPE_FP, AOM_BITS_8), - make_tuple(&av1_quantize_fp_32x32_c, &av1_quantize_fp_32x32_avx2, TX_64X16, - TYPE_FP, AOM_BITS_8), - make_tuple(&av1_quantize_fp_64x64_c, &av1_quantize_fp_64x64_avx2, TX_64X64, - TYPE_FP, AOM_BITS_8), + make_tuple(&av1_quantize_fp_c, &av1_quantize_fp_avx2, + static_cast(TX_16X16), TYPE_FP, AOM_BITS_8), + make_tuple(&av1_quantize_fp_c, &av1_quantize_fp_avx2, + static_cast(TX_4X16), TYPE_FP, AOM_BITS_8), + make_tuple(&av1_quantize_fp_c, &av1_quantize_fp_avx2, + static_cast(TX_16X4), TYPE_FP, AOM_BITS_8), + make_tuple(&av1_quantize_fp_c, &av1_quantize_fp_avx2, + static_cast(TX_32X8), TYPE_FP, AOM_BITS_8), + make_tuple(&av1_quantize_fp_c, &av1_quantize_fp_avx2, + static_cast(TX_8X32), TYPE_FP, AOM_BITS_8), + make_tuple(&av1_quantize_fp_32x32_c, &av1_quantize_fp_32x32_avx2, + static_cast(TX_32X32), TYPE_FP, AOM_BITS_8), + make_tuple(&av1_quantize_fp_32x32_c, &av1_quantize_fp_32x32_avx2, + static_cast(TX_16X64), TYPE_FP, AOM_BITS_8), + make_tuple(&av1_quantize_fp_32x32_c, &av1_quantize_fp_32x32_avx2, + static_cast(TX_64X16), TYPE_FP, AOM_BITS_8), + make_tuple(&av1_quantize_fp_64x64_c, &av1_quantize_fp_64x64_avx2, + static_cast(TX_64X64), TYPE_FP, AOM_BITS_8), make_tuple(&highbd_quan16x16_wrapper, - &highbd_quan16x16_wrapper, TX_16X16, - TYPE_FP, AOM_BITS_8), + &highbd_quan16x16_wrapper, + static_cast(TX_16X16), TYPE_FP, AOM_BITS_8), make_tuple(&highbd_quan16x16_wrapper, - &highbd_quan16x16_wrapper, TX_16X16, - TYPE_FP, AOM_BITS_10), + &highbd_quan16x16_wrapper, + static_cast(TX_16X16), TYPE_FP, AOM_BITS_10), make_tuple(&highbd_quan16x16_wrapper, - &highbd_quan16x16_wrapper, TX_16X16, - TYPE_FP, AOM_BITS_12), + &highbd_quan16x16_wrapper, + static_cast(TX_16X16), TYPE_FP, AOM_BITS_12), make_tuple(&highbd_quan32x32_wrapper, - &highbd_quan32x32_wrapper, TX_32X32, - TYPE_FP, AOM_BITS_8), + &highbd_quan32x32_wrapper, + static_cast(TX_32X32), TYPE_FP, AOM_BITS_8), make_tuple(&highbd_quan32x32_wrapper, - &highbd_quan32x32_wrapper, TX_32X32, - TYPE_FP, AOM_BITS_10), + &highbd_quan32x32_wrapper, + static_cast(TX_32X32), TYPE_FP, AOM_BITS_10), make_tuple(&highbd_quan32x32_wrapper, - &highbd_quan32x32_wrapper, TX_32X32, - TYPE_FP, AOM_BITS_12), + &highbd_quan32x32_wrapper, + static_cast(TX_32X32), TYPE_FP, AOM_BITS_12), make_tuple(&highbd_quan64x64_wrapper, - &highbd_quan64x64_wrapper, TX_64X64, - TYPE_FP, AOM_BITS_8), + &highbd_quan64x64_wrapper, + static_cast(TX_64X64), TYPE_FP, AOM_BITS_8), make_tuple(&highbd_quan64x64_wrapper, - &highbd_quan64x64_wrapper, TX_64X64, - TYPE_FP, AOM_BITS_10), + &highbd_quan64x64_wrapper, + static_cast(TX_64X64), TYPE_FP, AOM_BITS_10), make_tuple(&highbd_quan64x64_wrapper, - &highbd_quan64x64_wrapper, TX_64X64, - TYPE_FP, AOM_BITS_12), - make_tuple(&aom_highbd_quantize_b_c, &aom_highbd_quantize_b_avx2, TX_16X16, - TYPE_B, AOM_BITS_8), - make_tuple(&aom_highbd_quantize_b_c, &aom_highbd_quantize_b_avx2, TX_16X16, - TYPE_B, AOM_BITS_10), - make_tuple(&aom_highbd_quantize_b_c, &aom_highbd_quantize_b_avx2, TX_16X16, - TYPE_B, AOM_BITS_12), + &highbd_quan64x64_wrapper, + static_cast(TX_64X64), TYPE_FP, AOM_BITS_12), + make_tuple(&aom_highbd_quantize_b_c, &aom_highbd_quantize_b_avx2, + static_cast(TX_16X16), TYPE_B, AOM_BITS_8), + make_tuple(&aom_highbd_quantize_b_c, &aom_highbd_quantize_b_avx2, + static_cast(TX_16X16), TYPE_B, AOM_BITS_10), + make_tuple(&aom_highbd_quantize_b_c, &aom_highbd_quantize_b_avx2, + static_cast(TX_16X16), TYPE_B, AOM_BITS_12), + make_tuple(&aom_quantize_b_adaptive_c, &aom_quantize_b_adaptive_avx2, + static_cast(TX_16X16), TYPE_B, AOM_BITS_8), + make_tuple(&aom_quantize_b_adaptive_c, &aom_quantize_b_adaptive_avx2, + static_cast(TX_8X8), TYPE_B, AOM_BITS_8), + make_tuple(&aom_quantize_b_adaptive_c, &aom_quantize_b_adaptive_avx2, + static_cast(TX_4X4), TYPE_B, AOM_BITS_8) }; INSTANTIATE_TEST_CASE_P(AVX2, QuantizeTest, @@ -384,48 +392,60 @@ INSTANTIATE_TEST_CASE_P(AVX2, QuantizeTest, #if HAVE_SSE2 const QuantizeParam kQParamArraySSE2[] = { - make_tuple(&av1_quantize_fp_c, &av1_quantize_fp_sse2, TX_16X16, TYPE_FP, - AOM_BITS_8), - make_tuple(&av1_quantize_fp_c, &av1_quantize_fp_sse2, TX_4X16, TYPE_FP, - AOM_BITS_8), - make_tuple(&av1_quantize_fp_c, &av1_quantize_fp_sse2, TX_16X4, TYPE_FP, - AOM_BITS_8), - make_tuple(&av1_quantize_fp_c, &av1_quantize_fp_sse2, TX_8X32, TYPE_FP, - AOM_BITS_8), - make_tuple(&av1_quantize_fp_c, &av1_quantize_fp_sse2, TX_32X8, TYPE_FP, - AOM_BITS_8), - make_tuple(&aom_quantize_b_c, &aom_quantize_b_sse2, TX_16X16, TYPE_B, - AOM_BITS_8), - make_tuple(&aom_highbd_quantize_b_c, &aom_highbd_quantize_b_sse2, TX_16X16, - TYPE_B, AOM_BITS_8), - make_tuple(&aom_highbd_quantize_b_c, &aom_highbd_quantize_b_sse2, TX_16X16, - TYPE_B, AOM_BITS_10), - make_tuple(&aom_highbd_quantize_b_c, &aom_highbd_quantize_b_sse2, TX_16X16, - TYPE_B, AOM_BITS_12), + make_tuple(&av1_quantize_fp_c, &av1_quantize_fp_sse2, + static_cast(TX_16X16), TYPE_FP, AOM_BITS_8), + make_tuple(&av1_quantize_fp_c, &av1_quantize_fp_sse2, + static_cast(TX_4X16), TYPE_FP, AOM_BITS_8), + make_tuple(&av1_quantize_fp_c, &av1_quantize_fp_sse2, + static_cast(TX_16X4), TYPE_FP, AOM_BITS_8), + make_tuple(&av1_quantize_fp_c, &av1_quantize_fp_sse2, + static_cast(TX_8X32), TYPE_FP, AOM_BITS_8), + make_tuple(&av1_quantize_fp_c, &av1_quantize_fp_sse2, + static_cast(TX_32X8), TYPE_FP, AOM_BITS_8), + make_tuple(&aom_quantize_b_c, &aom_quantize_b_sse2, + static_cast(TX_16X16), TYPE_B, AOM_BITS_8), + make_tuple(&aom_highbd_quantize_b_c, &aom_highbd_quantize_b_sse2, + static_cast(TX_16X16), TYPE_B, AOM_BITS_8), + make_tuple(&aom_highbd_quantize_b_c, &aom_highbd_quantize_b_sse2, + static_cast(TX_16X16), TYPE_B, AOM_BITS_10), + make_tuple(&aom_highbd_quantize_b_c, &aom_highbd_quantize_b_sse2, + static_cast(TX_16X16), TYPE_B, AOM_BITS_12), make_tuple(&aom_highbd_quantize_b_32x32_c, &aom_highbd_quantize_b_32x32_sse2, - TX_32X32, TYPE_B, AOM_BITS_8), + static_cast(TX_32X32), TYPE_B, AOM_BITS_8), make_tuple(&aom_highbd_quantize_b_32x32_c, &aom_highbd_quantize_b_32x32_sse2, - TX_32X32, TYPE_B, AOM_BITS_10), + static_cast(TX_32X32), TYPE_B, AOM_BITS_10), make_tuple(&aom_highbd_quantize_b_32x32_c, &aom_highbd_quantize_b_32x32_sse2, - TX_32X32, TYPE_B, AOM_BITS_12), + static_cast(TX_32X32), TYPE_B, AOM_BITS_12), make_tuple(&aom_highbd_quantize_b_64x64_c, &aom_highbd_quantize_b_64x64_sse2, - TX_64X64, TYPE_B, AOM_BITS_8), + static_cast(TX_64X64), TYPE_B, AOM_BITS_8), make_tuple(&aom_highbd_quantize_b_64x64_c, &aom_highbd_quantize_b_64x64_sse2, - TX_64X64, TYPE_B, AOM_BITS_10), + static_cast(TX_64X64), TYPE_B, AOM_BITS_10), make_tuple(&aom_highbd_quantize_b_64x64_c, &aom_highbd_quantize_b_64x64_sse2, - TX_64X64, TYPE_B, AOM_BITS_12), + static_cast(TX_64X64), TYPE_B, AOM_BITS_12), make_tuple(&aom_quantize_b_adaptive_c, &aom_quantize_b_adaptive_sse2, - TX_16X16, TYPE_B, AOM_BITS_8), - make_tuple(&aom_quantize_b_adaptive_c, &aom_quantize_b_adaptive_sse2, TX_8X8, - TYPE_B, AOM_BITS_8), - make_tuple(&aom_quantize_b_adaptive_c, &aom_quantize_b_adaptive_sse2, TX_4X4, - TYPE_B, AOM_BITS_8), + static_cast(TX_16X16), TYPE_B, AOM_BITS_8), + make_tuple(&aom_quantize_b_adaptive_c, &aom_quantize_b_adaptive_sse2, + static_cast(TX_8X8), TYPE_B, AOM_BITS_8), + make_tuple(&aom_quantize_b_adaptive_c, &aom_quantize_b_adaptive_sse2, + static_cast(TX_4X4), TYPE_B, AOM_BITS_8), make_tuple(&aom_quantize_b_32x32_adaptive_c, - &aom_quantize_b_32x32_adaptive_sse2, TX_32X16, TYPE_B, AOM_BITS_8), + &aom_quantize_b_32x32_adaptive_sse2, + static_cast(TX_32X16), TYPE_B, AOM_BITS_8), make_tuple(&aom_quantize_b_32x32_adaptive_c, - &aom_quantize_b_32x32_adaptive_sse2, TX_16X32, TYPE_B, AOM_BITS_8), + &aom_quantize_b_32x32_adaptive_sse2, + static_cast(TX_16X32), TYPE_B, AOM_BITS_8), make_tuple(&aom_quantize_b_32x32_adaptive_c, - &aom_quantize_b_32x32_adaptive_sse2, TX_32X32, TYPE_B, AOM_BITS_8) + &aom_quantize_b_32x32_adaptive_sse2, + static_cast(TX_32X32), TYPE_B, AOM_BITS_8), + make_tuple(&aom_quantize_b_64x64_adaptive_c, + &aom_quantize_b_64x64_adaptive_sse2, + static_cast(TX_32X64), TYPE_B, AOM_BITS_8), + make_tuple(&aom_quantize_b_64x64_adaptive_c, + &aom_quantize_b_64x64_adaptive_sse2, + static_cast(TX_64X32), TYPE_B, AOM_BITS_8), + make_tuple(&aom_quantize_b_64x64_adaptive_c, + &aom_quantize_b_64x64_adaptive_sse2, + static_cast(TX_64X64), TYPE_B, AOM_BITS_8) }; INSTANTIATE_TEST_CASE_P(SSE2, QuantizeTest, @@ -435,25 +455,24 @@ INSTANTIATE_TEST_CASE_P(SSE2, QuantizeTest, #if HAVE_SSSE3 && ARCH_X86_64 INSTANTIATE_TEST_CASE_P( SSSE3, QuantizeTest, - ::testing::Values(make_tuple(&aom_quantize_b_c, &aom_quantize_b_ssse3, - TX_16X16, TYPE_B, AOM_BITS_8), - make_tuple(&aom_quantize_b_32x32_c, - &aom_quantize_b_32x32_ssse3, TX_32X32, TYPE_B, - AOM_BITS_8), - make_tuple(&aom_quantize_b_64x64_c, - &aom_quantize_b_64x64_ssse3, TX_64X64, TYPE_B, - AOM_BITS_8))); + ::testing::Values( + make_tuple(&aom_quantize_b_c, &aom_quantize_b_ssse3, + static_cast(TX_16X16), TYPE_B, AOM_BITS_8), + make_tuple(&aom_quantize_b_32x32_c, &aom_quantize_b_32x32_ssse3, + static_cast(TX_32X32), TYPE_B, AOM_BITS_8), + make_tuple(&aom_quantize_b_64x64_c, &aom_quantize_b_64x64_ssse3, + static_cast(TX_64X64), TYPE_B, AOM_BITS_8))); #endif // HAVE_SSSE3 && ARCH_X86_64 #if HAVE_AVX && ARCH_X86_64 INSTANTIATE_TEST_CASE_P( AVX, QuantizeTest, - ::testing::Values(make_tuple(&aom_quantize_b_c, &aom_quantize_b_avx, - TX_16X16, TYPE_B, AOM_BITS_8), - make_tuple(&aom_quantize_b_32x32_c, - &aom_quantize_b_32x32_avx, TX_32X32, TYPE_B, - AOM_BITS_8))); + ::testing::Values( + make_tuple(&aom_quantize_b_c, &aom_quantize_b_avx, + static_cast(TX_16X16), TYPE_B, AOM_BITS_8), + make_tuple(&aom_quantize_b_32x32_c, &aom_quantize_b_32x32_avx, + static_cast(TX_32X32), TYPE_B, AOM_BITS_8))); #endif // HAVE_AVX && ARCH_X86_64 } // namespace diff --git a/third_party/aom/test/rt_end_to_end_test.cc b/third_party/aom/test/rt_end_to_end_test.cc index 9c3e96b905..8eeed4bbe9 100644 --- a/third_party/aom/test/rt_end_to_end_test.cc +++ b/third_party/aom/test/rt_end_to_end_test.cc @@ -26,7 +26,7 @@ const int kBitrate = 500; // List of psnr thresholds for speed settings 0-8 const double kPsnrThreshold[9] = { 36.9, 36.9, 36.85, 36.8, 36.6, - 36.4, 36.0, 35.5, 35.0 }; + 36.3, 36.0, 35.3, 35.0 }; typedef struct { const char *filename; diff --git a/third_party/aom/test/simd_cmp_impl.h b/third_party/aom/test/simd_cmp_impl.h index 2aa02c864f..b7bd5e7199 100644 --- a/third_party/aom/test/simd_cmp_impl.h +++ b/third_party/aom/test/simd_cmp_impl.h @@ -469,11 +469,8 @@ typedef struct { fptr simd; } mapping; -#define MAP(name) \ - { \ -#name, reinterpret_cast < fptr > (c_##name), \ - reinterpret_cast < fptr > (name) \ - } +#define MAP(name) \ + { #name, reinterpret_cast < fptr>(c_##name), reinterpret_cast < fptr>(name) } const mapping m[] = { MAP(v64_sad_u8), MAP(v64_ssd_u8), diff --git a/third_party/aom/test/test.cmake b/third_party/aom/test/test.cmake index a44737a456..9e55aea0ec 100644 --- a/third_party/aom/test/test.cmake +++ b/third_party/aom/test/test.cmake @@ -79,7 +79,8 @@ list(APPEND AOM_UNIT_TEST_ENCODER_SOURCES "${AOM_ROOT}/test/scalability_test.cc" "${AOM_ROOT}/test/y4m_test.cc" "${AOM_ROOT}/test/y4m_video_source.h" - "${AOM_ROOT}/test/yuv_video_source.h") + "${AOM_ROOT}/test/yuv_video_source.h" + "${AOM_ROOT}/test/time_stamp_test.cc") list(APPEND AOM_DECODE_PERF_TEST_SOURCES "${AOM_ROOT}/test/decode_perf_test.cc") list(APPEND AOM_ENCODE_PERF_TEST_SOURCES "${AOM_ROOT}/test/encode_perf_test.cc") diff --git a/third_party/aom/test/time_stamp_test.cc b/third_party/aom/test/time_stamp_test.cc new file mode 100644 index 0000000000..679e4da292 --- /dev/null +++ b/third_party/aom/test/time_stamp_test.cc @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2019, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +// Test AOM timestamp handling + +#include "test/codec_factory.h" +#include "test/encode_test_driver.h" +#include "test/util.h" +#include "test/video_source.h" +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" + +namespace { + +const int kVideoSourceWidth = 320; +const int kVideoSourceHeight = 240; +const int kFramesToEncode = 3; + +// A video source that exposes functions to set the timebase, framerate and +// starting pts. +class DummyTimebaseVideoSource : public ::libaom_test::DummyVideoSource { + public: + // Parameters num and den set the timebase for the video source. + DummyTimebaseVideoSource(int num, int den) + : framerate_numerator_(30), framerate_denominator_(1), starting_pts_(0) { + SetSize(kVideoSourceWidth, kVideoSourceHeight); + set_limit(kFramesToEncode); + timebase_.num = num; + timebase_.den = den; + } + + void SetFramerate(int numerator, int denominator) { + framerate_numerator_ = numerator; + framerate_denominator_ = denominator; + } + + // Returns one frames duration in timebase units as a double. + double FrameDuration() const { + return (static_cast(timebase_.den) / timebase_.num) / + (static_cast(framerate_numerator_) / framerate_denominator_); + } + + virtual aom_codec_pts_t pts() const { + return static_cast(frame_ * FrameDuration() + + starting_pts_ + 0.5); + } + + virtual unsigned long duration() const { + return static_cast(FrameDuration() + 0.5); + } + + virtual aom_rational_t timebase() const { return timebase_; } + + void set_starting_pts(int64_t starting_pts) { starting_pts_ = starting_pts; } + + private: + aom_rational_t timebase_; + int framerate_numerator_; + int framerate_denominator_; + int64_t starting_pts_; +}; + +class TimestampTest + : public ::libaom_test::EncoderTest, + public ::libaom_test::CodecTestWithParam { + protected: + TimestampTest() : EncoderTest(GET_PARAM(0)) {} + virtual ~TimestampTest() {} + + virtual void SetUp() { + InitializeConfig(); + SetMode(GET_PARAM(1)); + } +}; + +// Tests encoding in millisecond timebase. +TEST_P(TimestampTest, EncodeFrames) { + DummyTimebaseVideoSource video(1, 1000); + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); +} + +TEST_P(TimestampTest, TestMicrosecondTimebase) { + // Set the timebase to microseconds. + DummyTimebaseVideoSource video(1, 1000000); + video.set_limit(1); + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); +} + +TEST_P(TimestampTest, TestAv1Rollover) { + DummyTimebaseVideoSource video(1, 1000); + video.set_starting_pts(922337170351ll); + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); +} + +AV1_INSTANTIATE_TEST_CASE(TimestampTest, + ::testing::Values(::libaom_test::kTwoPassGood)); + +} // namespace diff --git a/third_party/aom/test/yuv_temporal_filter_test.cc b/third_party/aom/test/yuv_temporal_filter_test.cc index fcaf0df8d0..f5047943e5 100644 --- a/third_party/aom/test/yuv_temporal_filter_test.cc +++ b/third_party/aom/test/yuv_temporal_filter_test.cc @@ -74,9 +74,9 @@ int GetModIndex(int sum_dist, int index, int rounding, int strength, template <> int GetModIndex(int sum_dist, int index, int rounding, int strength, int filter_weight) { - unsigned int index_mult[14] = { - 0, 0, 0, 0, 49152, 39322, 32768, 28087, 24576, 21846, 19661, 17874, 0, 15124 - }; + unsigned int index_mult[14] = { 0, 0, 0, 0, 49152, + 39322, 32768, 28087, 24576, 21846, + 19661, 17874, 0, 15124 }; assert(index >= 0 && index <= 13); assert(index_mult[index] != 0); diff --git a/third_party/aom/third_party/libwebm/README.libaom b/third_party/aom/third_party/libwebm/README.libaom index 17b2f47152..16f17513e7 100644 --- a/third_party/aom/third_party/libwebm/README.libaom +++ b/third_party/aom/third_party/libwebm/README.libaom @@ -1,5 +1,5 @@ URL: https://chromium.googlesource.com/webm/libwebm -Version: 9f23fbc50e7a76c815b1d3f0309abe1066301331 +Version: dbf1d1089756e7cb5b1a04d6752310ef35912347 License: BSD License File: LICENSE.txt diff --git a/third_party/aom/third_party/libwebm/common/file_util.cc b/third_party/aom/third_party/libwebm/common/file_util.cc index e6109d5a0e..6eb6428b98 100644 --- a/third_party/aom/third_party/libwebm/common/file_util.cc +++ b/third_party/aom/third_party/libwebm/common/file_util.cc @@ -24,8 +24,8 @@ namespace libwebm { std::string GetTempFileName() { #if !defined _MSC_VER && !defined __MINGW32__ std::string temp_file_name_template_str = - std::string(std::getenv("TEST_TMPDIR") ? std::getenv("TEST_TMPDIR") : - ".") + + std::string(std::getenv("TEST_TMPDIR") ? std::getenv("TEST_TMPDIR") + : ".") + "/libwebm_temp.XXXXXX"; char* temp_file_name_template = new char[temp_file_name_template_str.length() + 1]; diff --git a/third_party/aom/third_party/libwebm/mkvmuxer/mkvmuxerutil.cc b/third_party/aom/third_party/libwebm/mkvmuxer/mkvmuxerutil.cc index 3bff7cd51a..7636a9f4ef 100644 --- a/third_party/aom/third_party/libwebm/mkvmuxer/mkvmuxerutil.cc +++ b/third_party/aom/third_party/libwebm/mkvmuxer/mkvmuxerutil.cc @@ -562,10 +562,10 @@ uint64 WriteFrame(IMkvWriter* writer, const Frame* const frame, if (relative_timecode < 0 || relative_timecode > kMaxBlockTimecode) return 0; - return frame->CanBeSimpleBlock() ? - WriteSimpleBlock(writer, frame, relative_timecode) : - WriteBlock(writer, frame, relative_timecode, - cluster->timecode_scale()); + return frame->CanBeSimpleBlock() + ? WriteSimpleBlock(writer, frame, relative_timecode) + : WriteBlock(writer, frame, relative_timecode, + cluster->timecode_scale()); } uint64 WriteVoidElement(IMkvWriter* writer, uint64 size) { diff --git a/third_party/aom/third_party/libwebm/mkvmuxer/mkvmuxerutil.h b/third_party/aom/third_party/libwebm/mkvmuxer/mkvmuxerutil.h index 132388da59..3355428bd1 100644 --- a/third_party/aom/third_party/libwebm/mkvmuxer/mkvmuxerutil.h +++ b/third_party/aom/third_party/libwebm/mkvmuxer/mkvmuxerutil.h @@ -31,6 +31,9 @@ const int64 kMaxBlockTimecode = 0x07FFFLL; // Writes out |value| in Big Endian order. Returns 0 on success. int32 SerializeInt(IMkvWriter* writer, int64 value, int32 size); +// Writes out |f| in Big Endian order. Returns 0 on success. +int32 SerializeFloat(IMkvWriter* writer, float f); + // Returns the size in bytes of the element. int32 GetUIntSize(uint64 value); int32 GetIntSize(int64 value); diff --git a/third_party/aom/third_party/libwebm/mkvparser/mkvparser.cc b/third_party/aom/third_party/libwebm/mkvparser/mkvparser.cc index 9c78ead2b0..dcb969dcf5 100644 --- a/third_party/aom/third_party/libwebm/mkvparser/mkvparser.cc +++ b/third_party/aom/third_party/libwebm/mkvparser/mkvparser.cc @@ -5311,7 +5311,7 @@ long VideoTrack::Parse(Segment* pSegment, const Info& info, const long long stop = pos + s.size; - Colour* colour = NULL; + std::unique_ptr colour_ptr; std::unique_ptr projection_ptr; while (pos < stop) { @@ -5361,8 +5361,12 @@ long VideoTrack::Parse(Segment* pSegment, const Info& info, if (rate <= 0) return E_FILE_FORMAT_INVALID; } else if (id == libwebm::kMkvColour) { - if (!Colour::Parse(pReader, pos, size, &colour)) + Colour* colour = NULL; + if (!Colour::Parse(pReader, pos, size, &colour)) { return E_FILE_FORMAT_INVALID; + } else { + colour_ptr.reset(colour); + } } else if (id == libwebm::kMkvProjection) { Projection* projection = NULL; if (!Projection::Parse(pReader, pos, size, &projection)) { @@ -5404,7 +5408,7 @@ long VideoTrack::Parse(Segment* pSegment, const Info& info, pTrack->m_display_unit = display_unit; pTrack->m_stereo_mode = stereo_mode; pTrack->m_rate = rate; - pTrack->m_colour = colour; + pTrack->m_colour = colour_ptr.release(); pTrack->m_colour_space = colour_space; pTrack->m_projection = projection_ptr.release(); diff --git a/third_party/aom/tools/aom_entropy_optimizer.c b/third_party/aom/tools/aom_entropy_optimizer.c index 551adf4f29..d57b8863da 100644 --- a/third_party/aom/tools/aom_entropy_optimizer.c +++ b/third_party/aom/tools/aom_entropy_optimizer.c @@ -323,9 +323,9 @@ int main(int argc, const char **argv) { /* block partition */ cts_each_dim[0] = PARTITION_CONTEXTS; cts_each_dim[1] = EXT_PARTITION_TYPES; - int part_types_each_ctx[PARTITION_CONTEXTS] = { - 4, 4, 4, 4, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 8, 8, 8, 8 - }; + int part_types_each_ctx[PARTITION_CONTEXTS] = { 4, 4, 4, 4, 10, 10, 10, + 10, 10, 10, 10, 10, 10, 10, + 10, 10, 8, 8, 8, 8 }; optimize_cdf_table_var_modes_2d( &fc.partition[0][0], probsfile, 2, cts_each_dim, part_types_each_ctx, "static const aom_cdf_prob default_partition_cdf[PARTITION_CONTEXTS]"