wuggy/Dactyloidae
wuggy/Dactyloidae
1
0
Fork 0
mirror of https://repo.dactyloidae.xyz/Dactyloidae/UXP.git synced 2026-10-31 11:37:30 +09:00
Code Issues Projects Releases Packages Wiki Activity

update libaom to rev 5bdd95475e07ac8641114cb139105b8d734a1b60 (without moz.build and aom_ports/aom_once.h)

Browse source
This commit is contained in:
Roy Tam 2019-03-29 21:53:07 +08:00
commit fb515ea77b
177 changed files with 21088 additions and 11681 deletions
Download patch file Download diff file Expand all files Collapse all files

4
third_party/aom/av1/common/av1_inv_txfm2d.c vendored
View file

@ -228,7 +228,7 @@ void av1_gen_inv_stage_range(int8_t *stage_range_col, int8_t *stage_range_row,
(void)real_range_row;
if (cfg->txfm_type_row == TXFM_TYPE_ADST4 && i == 1) {
// the adst4 may use 1 extra bit on top of opt_range_row at stage 1
// so opt_range_col >= real_range_col will not hold
// so opt_range_row >= real_range_row will not hold
stage_range_row[i] = opt_range_row;
} else {
assert(opt_range_row >= real_range_row);
@ -241,7 +241,7 @@ void av1_gen_inv_stage_range(int8_t *stage_range_col, int8_t *stage_range_row,
cfg->stage_range_col[i] + fwd_shift + shift[0] + bd + 1;
(void)real_range_col;
if (cfg->txfm_type_col == TXFM_TYPE_ADST4 && i == 1) {
// the adst4 may use 1 extra bit on top of opt_range_row at stage 1
// the adst4 may use 1 extra bit on top of opt_range_col at stage 1
// so opt_range_col >= real_range_col will not hold
stage_range_col[i] = opt_range_col;
} else {

23
third_party/aom/av1/common/av1_rtcd_defs.pl vendored
View file

@ -111,10 +111,14 @@ specialize qw/av1_highbd_convolve8_vert/, "$sse2_x86_64";
#inv txfm
add_proto qw/void av1_inv_txfm_add/, "const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param";
specialize qw/av1_inv_txfm_add ssse3 avx2 neon/;
# TODO(http://crbug.com/aomedia/2350): avx2 is disabled due to test vector
# mismatches.
specialize qw/av1_inv_txfm_add ssse3 neon/;
add_proto qw/void av1_highbd_inv_txfm_add/, "const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param";
specialize qw/av1_highbd_inv_txfm_add sse4_1 avx2/;
# TODO(http://crbug.com/aomedia/2350): avx2 is disabled due to test vector
# mismatches.
specialize qw/av1_highbd_inv_txfm_add sse4_1/;
add_proto qw/void av1_highbd_inv_txfm_add_4x4/, "const tran_low_t *dqcoeff, uint8_t *dst, int stride, const TxfmParam *txfm_param";
specialize qw/av1_highbd_inv_txfm_add_4x4 sse4_1/;
@ -160,8 +164,9 @@ add_proto qw/void av1_inv_txfm2d_add_32x8/, "const int32_t *input, uint16_t *out
add_proto qw/void av1_highbd_dr_prediction_z1/, "uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_above, int dx, int dy, int bd";
specialize qw/av1_highbd_dr_prediction_z1 avx2/;
add_proto qw/void av1_highbd_dr_prediction_z2/, "uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_above, int upsample_left, int dx, int dy, int bd";
# TODO(https://crbug.com/aomedia/2288)
#specialize qw/av1_highbd_dr_prediction_z2 avx2/;
# TODO(niva213@gmail.com): Re-enable avx2 after fixing valgrind issue
# https://crbug.com/aomedia/2316
# specialize qw/av1_highbd_dr_prediction_z2 avx2/;
add_proto qw/void av1_highbd_dr_prediction_z3/, "uint16_t *dst, ptrdiff_t stride, int bw, int bh, const uint16_t *above, const uint16_t *left, int upsample_left, int dx, int dy, int bd";
specialize qw/av1_highbd_dr_prediction_z3 avx2/;
@ -255,18 +260,18 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
add_proto qw/int av1_full_range_search/, "const struct macroblock *x, const struct search_site_config *cfg, MV *ref_mv, MV *best_mv, int search_param, int sad_per_bit, int *num00, const struct aom_variance_vtable *fn_ptr, const MV *center_mv";
# TODO(yunqing): Add back the optimizations.
# add_proto qw/void av1_temporal_filter_apply/, "uint8_t *frame1, unsigned int stride, uint8_t *frame2, unsigned int block_width, unsigned int block_height, int strength, int filter_weight, unsigned int *accumulator, uint16_t *count";
# specialize qw/av1_temporal_filter_apply sse2 msa/;
add_proto qw/void av1_apply_temporal_filter/, "const uint8_t *y_frame1, int y_stride, const uint8_t *y_pred, int y_buf_stride, const uint8_t *u_frame1, const uint8_t *v_frame1, int uv_stride, const uint8_t *u_pred, const uint8_t *v_pred, int uv_buf_stride, unsigned int block_width, unsigned int block_height, int ss_x, int ss_y, int strength, const int *blk_fw, int use_32x32, uint32_t *y_accumulator, uint16_t *y_count, uint32_t *u_accumulator, uint16_t *u_count, uint32_t *v_accumulator, uint16_t *v_count";
specialize qw/av1_apply_temporal_filter sse4_1/;
add_proto qw/void av1_quantize_b/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan, const qm_val_t * qm_ptr, const qm_val_t * iqm_ptr, int log_scale";
# ENCODEMB INVOKE
add_proto qw/int64_t av1_highbd_block_error/, "const tran_low_t *coeff, const tran_low_t *dqcoeff, intptr_t block_size, int64_t *ssz, int bd";
specialize qw/av1_highbd_block_error sse2/;
specialize qw/av1_highbd_block_error sse2 avx2/;
# add_proto qw/void av1_highbd_temporal_filter_apply/, "uint8_t *frame1, unsigned int stride, uint8_t *frame2, unsigned int block_width, unsigned int block_height, int strength, int filter_weight, unsigned int *accumulator, uint16_t *count";
add_proto qw/void av1_highbd_apply_temporal_filter/, "const uint8_t *yf, int y_stride, const uint8_t *yp, int y_buf_stride, const uint8_t *uf, const uint8_t *vf, int uv_stride, const uint8_t *up, const uint8_t *vp, int uv_buf_stride, unsigned int block_width, unsigned int block_height, int ss_x, int ss_y, int strength, const int *blk_fw, int use_32x32, uint32_t *y_accumulator, uint16_t *y_count, uint32_t *u_accumulator, uint16_t *u_count, uint32_t *v_accumulator, uint16_t *v_count";
specialize qw/av1_highbd_apply_temporal_filter sse4_1/;
add_proto qw/void av1_highbd_quantize_fp/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan, int log_scale";
specialize qw/av1_highbd_quantize_fp sse4_1 avx2/;

2
third_party/aom/av1/common/av1_txfm.h vendored
View file

@ -59,7 +59,9 @@ static INLINE int32_t range_check_value(int32_t value, int8_t bit) {
const int64_t min_value = -(1LL << (bit - 1));
if (value < min_value || value > max_value) {
fprintf(stderr, "coeff out of bit range, value: %d bit %d\n", value, bit);
#if !CONFIG_AV1_ENCODER
assert(0);
#endif
}
#endif // CONFIG_COEFFICIENT_RANGE_CHECKING
#if DO_RANGE_CHECK_CLAMP

5
third_party/aom/av1/common/blockd.h vendored
View file

@ -190,11 +190,6 @@ typedef struct RD_STATS {
int64_t ref_rdcost;
int zero_rate;
uint8_t invalid_rate;
#if CONFIG_ONE_PASS_SVM
int eob, eob_0, eob_1, eob_2, eob_3;
int64_t rd, rd_0, rd_1, rd_2, rd_3;
int64_t y_sse, sse_0, sse_1, sse_2, sse_3;
#endif
#if CONFIG_RD_DEBUG
int txb_coeff_cost[MAX_MB_PLANE];
int txb_coeff_cost_map[MAX_MB_PLANE][TXB_COEFF_COST_MAP_SIZE]

1
third_party/aom/av1/common/cdef.c vendored
View file

@ -80,7 +80,6 @@ int sb_compute_cdef_list(const AV1_COMMON *const cm, int mi_row, int mi_col,
if (!is_8x8_block_skip(grid, mi_row + r, mi_col + c, cm->mi_stride)) {
dlist[count].by = r >> r_shift;
dlist[count].bx = c >> c_shift;
dlist[count].skip = 0;
count++;
}
}

4
third_party/aom/av1/common/cdef_block.c vendored
View file

@ -232,8 +232,8 @@ void cdef_filter_fb(uint8_t *dst8, uint16_t *dst16, int dstride, uint16_t *in,
}
for (bi = 0; bi < cdef_count; bi++) {
int t = dlist[bi].skip ? 0 : pri_strength;
int s = dlist[bi].skip ? 0 : sec_strength;
int t = pri_strength;
int s = sec_strength;
by = dlist[bi].by;
bx = dlist[bi].bx;
if (dst8)

1
third_party/aom/av1/common/cdef_block.h vendored
View file

@ -38,7 +38,6 @@ DECLARE_ALIGNED(16, extern const int, cdef_directions[8][2]);
typedef struct {
uint8_t by;
uint8_t bx;
uint8_t skip;
} cdef_list;
typedef void (*cdef_filter_block_func)(uint8_t *dst8, uint16_t *dst16,

2
third_party/aom/av1/common/entropy.c vendored
View file

@ -101,7 +101,7 @@ void av1_reset_cdf_symbol_counters(FRAME_CONTEXT *fc) {
RESET_CDF_COUNTER(fc->refmv_cdf, 2);
RESET_CDF_COUNTER(fc->drl_cdf, 2);
RESET_CDF_COUNTER(fc->inter_compound_mode_cdf, INTER_COMPOUND_MODES);
RESET_CDF_COUNTER(fc->compound_type_cdf, COMPOUND_TYPES - 1);
RESET_CDF_COUNTER(fc->compound_type_cdf, MASKED_COMPOUND_TYPES);
RESET_CDF_COUNTER(fc->wedge_idx_cdf, 16);
RESET_CDF_COUNTER(fc->interintra_cdf, 2);
RESET_CDF_COUNTER(fc->wedge_interintra_cdf, 2);

22
third_party/aom/av1/common/entropymode.c vendored
View file

@ -488,17 +488,17 @@ static const aom_cdf_prob
{ AOM_CDF2(16384) }
};
static const aom_cdf_prob
default_compound_type_cdf[BLOCK_SIZES_ALL][CDF_SIZE(COMPOUND_TYPES - 1)] = {
{ AOM_CDF2(16384) }, { AOM_CDF2(16384) }, { AOM_CDF2(16384) },
{ AOM_CDF2(23431) }, { AOM_CDF2(13171) }, { AOM_CDF2(11470) },
{ AOM_CDF2(9770) }, { AOM_CDF2(9100) }, { AOM_CDF2(8233) },
{ AOM_CDF2(6172) }, { AOM_CDF2(16384) }, { AOM_CDF2(16384) },
{ AOM_CDF2(16384) }, { AOM_CDF2(16384) }, { AOM_CDF2(16384) },
{ AOM_CDF2(16384) }, { AOM_CDF2(16384) }, { AOM_CDF2(16384) },
{ AOM_CDF2(11820) }, { AOM_CDF2(7701) }, { AOM_CDF2(16384) },
{ AOM_CDF2(16384) }
};
static const aom_cdf_prob default_compound_type_cdf[BLOCK_SIZES_ALL][CDF_SIZE(
MASKED_COMPOUND_TYPES)] = {
{ AOM_CDF2(16384) }, { AOM_CDF2(16384) }, { AOM_CDF2(16384) },
{ AOM_CDF2(23431) }, { AOM_CDF2(13171) }, { AOM_CDF2(11470) },
{ AOM_CDF2(9770) }, { AOM_CDF2(9100) }, { AOM_CDF2(8233) },
{ AOM_CDF2(6172) }, { AOM_CDF2(16384) }, { AOM_CDF2(16384) },
{ AOM_CDF2(16384) }, { AOM_CDF2(16384) }, { AOM_CDF2(16384) },
{ AOM_CDF2(16384) }, { AOM_CDF2(16384) }, { AOM_CDF2(16384) },
{ AOM_CDF2(11820) }, { AOM_CDF2(7701) }, { AOM_CDF2(16384) },
{ AOM_CDF2(16384) }
};
static const aom_cdf_prob default_wedge_idx_cdf[BLOCK_SIZES_ALL][CDF_SIZE(16)] =
{ { AOM_CDF16(2048, 4096, 6144, 8192, 10240, 12288, 14336, 16384, 18432,

3
third_party/aom/av1/common/entropymode.h vendored
View file

@ -92,7 +92,8 @@ typedef struct frame_contexts {
aom_cdf_prob inter_compound_mode_cdf[INTER_MODE_CONTEXTS]
[CDF_SIZE(INTER_COMPOUND_MODES)];
aom_cdf_prob compound_type_cdf[BLOCK_SIZES_ALL][CDF_SIZE(COMPOUND_TYPES - 1)];
aom_cdf_prob compound_type_cdf[BLOCK_SIZES_ALL]
[CDF_SIZE(MASKED_COMPOUND_TYPES)];
aom_cdf_prob wedge_idx_cdf[BLOCK_SIZES_ALL][CDF_SIZE(16)];
aom_cdf_prob interintra_cdf[BLOCK_SIZE_GROUPS][CDF_SIZE(2)];
aom_cdf_prob wedge_interintra_cdf[BLOCK_SIZES_ALL][CDF_SIZE(2)];

42
third_party/aom/av1/common/enums.h vendored
View file

@ -92,15 +92,6 @@ enum {
MAX_PROFILES,
} SENUM1BYTE(BITSTREAM_PROFILE);
#define LEVEL_MAJOR_BITS 3
#define LEVEL_MINOR_BITS 2
#define LEVEL_BITS (LEVEL_MAJOR_BITS + LEVEL_MINOR_BITS)
#define LEVEL_MAJOR_MIN 2
#define LEVEL_MAJOR_MAX ((1 << LEVEL_MAJOR_BITS) - 1 + LEVEL_MAJOR_MIN)
#define LEVEL_MINOR_MIN 0
#define LEVEL_MINOR_MAX ((1 << LEVEL_MINOR_BITS) - 1)
#define OP_POINTS_CNT_MINUS_1_BITS 5
#define OP_POINTS_IDC_BITS 12
@ -455,9 +446,11 @@ enum {
enum {
COMPOUND_AVERAGE,
COMPOUND_DISTWTD,
COMPOUND_WEDGE,
COMPOUND_DIFFWTD,
COMPOUND_TYPES,
MASKED_COMPOUND_TYPES = 2,
} UENUM1BYTE(COMPOUND_TYPE);
enum {
@ -469,6 +462,37 @@ enum {
FILTER_INTRA_MODES,
} UENUM1BYTE(FILTER_INTRA_MODE);
enum {
SEQ_LEVEL_2_0,
SEQ_LEVEL_2_1,
SEQ_LEVEL_2_2,
SEQ_LEVEL_2_3,
SEQ_LEVEL_3_0,
SEQ_LEVEL_3_1,
SEQ_LEVEL_3_2,
SEQ_LEVEL_3_3,
SEQ_LEVEL_4_0,
SEQ_LEVEL_4_1,
SEQ_LEVEL_4_2,
SEQ_LEVEL_4_3,
SEQ_LEVEL_5_0,
SEQ_LEVEL_5_1,
SEQ_LEVEL_5_2,
SEQ_LEVEL_5_3,
SEQ_LEVEL_6_0,
SEQ_LEVEL_6_1,
SEQ_LEVEL_6_2,
SEQ_LEVEL_6_3,
SEQ_LEVEL_7_0,
SEQ_LEVEL_7_1,
SEQ_LEVEL_7_2,
SEQ_LEVEL_7_3,
SEQ_LEVELS,
SEQ_LEVEL_MAX = 31
} UENUM1BYTE(AV1_LEVEL);
#define LEVEL_BITS 5
#define DIRECTIONAL_MODES 8
#define MAX_ANGLE_DELTA 3
#define ANGLE_STEP 3

28
third_party/aom/av1/common/mvref_common.c vendored
View file

@ -821,7 +821,7 @@ void av1_find_mv_refs(const AV1_COMMON *cm, const MACROBLOCKD *xd,
MV_REFERENCE_FRAME rf[2];
av1_set_ref_frame(rf, ref_frame);
if (ref_frame < REF_FRAMES) {
if (global_mvs != NULL && ref_frame < REF_FRAMES) {
if (ref_frame != INTRA_FRAME) {
global_mvs[ref_frame] = gm_get_motion_vector(
&cm->global_motion[ref_frame], cm->allow_high_precision_mv, bsize,
@ -1362,15 +1362,15 @@ static int compare_ref_frame_info(const void *arg_a, const void *arg_b) {
return info_a->map_idx - info_b->map_idx;
}
static void set_ref_frame_info(AV1_COMMON *const cm, int frame_idx,
static void set_ref_frame_info(int *remapped_ref_idx, int frame_idx,
REF_FRAME_INFO *ref_info) {
assert(frame_idx >= 0 && frame_idx < INTER_REFS_PER_FRAME);
cm->remapped_ref_idx[frame_idx] = ref_info->map_idx;
remapped_ref_idx[frame_idx] = ref_info->map_idx;
}
void av1_set_frame_refs(AV1_COMMON *const cm, int lst_map_idx,
int gld_map_idx) {
void av1_set_frame_refs(AV1_COMMON *const cm, int *remapped_ref_idx,
int lst_map_idx, int gld_map_idx) {
int lst_frame_sort_idx = -1;
int gld_frame_sort_idx = -1;
@ -1395,8 +1395,6 @@ void av1_set_frame_refs(AV1_COMMON *const cm, int lst_map_idx,
if (buf == NULL) continue;
// If this assertion fails, there is a reference leak.
assert(buf->ref_count > 0);
// TODO(wtc@google.com): Remove the checking on ref_count after 2019-03-01.
if (buf->ref_count <= 0) continue;
const int offset = (int)buf->order_hint;
ref_frame_info[i].sort_idx =
@ -1449,7 +1447,7 @@ void av1_set_frame_refs(AV1_COMMON *const cm, int lst_map_idx,
// == ALTREF_FRAME ==
if (bwd_start_idx <= bwd_end_idx) {
set_ref_frame_info(cm, ALTREF_FRAME - LAST_FRAME,
set_ref_frame_info(remapped_ref_idx, ALTREF_FRAME - LAST_FRAME,
&ref_frame_info[bwd_end_idx]);
ref_flag_list[ALTREF_FRAME - LAST_FRAME] = 1;
bwd_end_idx--;
@ -1457,7 +1455,7 @@ void av1_set_frame_refs(AV1_COMMON *const cm, int lst_map_idx,
// == BWDREF_FRAME ==
if (bwd_start_idx <= bwd_end_idx) {
set_ref_frame_info(cm, BWDREF_FRAME - LAST_FRAME,
set_ref_frame_info(remapped_ref_idx, BWDREF_FRAME - LAST_FRAME,
&ref_frame_info[bwd_start_idx]);
ref_flag_list[BWDREF_FRAME - LAST_FRAME] = 1;
bwd_start_idx++;
@ -1465,7 +1463,7 @@ void av1_set_frame_refs(AV1_COMMON *const cm, int lst_map_idx,
// == ALTREF2_FRAME ==
if (bwd_start_idx <= bwd_end_idx) {
set_ref_frame_info(cm, ALTREF2_FRAME - LAST_FRAME,
set_ref_frame_info(remapped_ref_idx, ALTREF2_FRAME - LAST_FRAME,
&ref_frame_info[bwd_start_idx]);
ref_flag_list[ALTREF2_FRAME - LAST_FRAME] = 1;
}
@ -1475,13 +1473,15 @@ void av1_set_frame_refs(AV1_COMMON *const cm, int lst_map_idx,
for (int i = fwd_start_idx; i <= fwd_end_idx; ++i) {
// == LAST_FRAME ==
if (ref_frame_info[i].map_idx == lst_map_idx) {
set_ref_frame_info(cm, LAST_FRAME - LAST_FRAME, &ref_frame_info[i]);
set_ref_frame_info(remapped_ref_idx, LAST_FRAME - LAST_FRAME,
&ref_frame_info[i]);
ref_flag_list[LAST_FRAME - LAST_FRAME] = 1;
}
// == GOLDEN_FRAME ==
if (ref_frame_info[i].map_idx == gld_map_idx) {
set_ref_frame_info(cm, GOLDEN_FRAME - LAST_FRAME, &ref_frame_info[i]);
set_ref_frame_info(remapped_ref_idx, GOLDEN_FRAME - LAST_FRAME,
&ref_frame_info[i]);
ref_flag_list[GOLDEN_FRAME - LAST_FRAME] = 1;
}
}
@ -1513,7 +1513,7 @@ void av1_set_frame_refs(AV1_COMMON *const cm, int lst_map_idx,
}
if (fwd_start_idx > fwd_end_idx) break;
set_ref_frame_info(cm, ref_frame - LAST_FRAME,
set_ref_frame_info(remapped_ref_idx, ref_frame - LAST_FRAME,
&ref_frame_info[fwd_end_idx]);
ref_flag_list[ref_frame - LAST_FRAME] = 1;
@ -1524,7 +1524,7 @@ void av1_set_frame_refs(AV1_COMMON *const cm, int lst_map_idx,
for (; ref_idx < (INTER_REFS_PER_FRAME - 2); ref_idx++) {
const MV_REFERENCE_FRAME ref_frame = ref_frame_list[ref_idx];
if (ref_flag_list[ref_frame - LAST_FRAME] == 1) continue;
set_ref_frame_info(cm, ref_frame - LAST_FRAME,
set_ref_frame_info(remapped_ref_idx, ref_frame - LAST_FRAME,
&ref_frame_info[fwd_start_idx]);
ref_flag_list[ref_frame - LAST_FRAME] = 1;
}

6
third_party/aom/av1/common/mvref_common.h vendored
View file

@ -210,7 +210,8 @@ void av1_setup_frame_buf_refs(AV1_COMMON *cm);
void av1_setup_frame_sign_bias(AV1_COMMON *cm);
void av1_setup_skip_mode_allowed(AV1_COMMON *cm);
void av1_setup_motion_field(AV1_COMMON *cm);
void av1_set_frame_refs(AV1_COMMON *const cm, int lst_map_idx, int gld_map_idx);
void av1_set_frame_refs(AV1_COMMON *const cm, int *remapped_ref_idx,
int lst_map_idx, int gld_map_idx);
static INLINE void av1_collect_neighbors_ref_counts(MACROBLOCKD *const xd) {
av1_zero(xd->neighbors_ref_counts);
@ -243,6 +244,9 @@ void av1_copy_frame_mvs(const AV1_COMMON *const cm,
const MB_MODE_INFO *const mi, int mi_row, int mi_col,
int x_mis, int y_mis);
// The global_mvs output parameter points to an array of REF_FRAMES elements.
// The caller may pass a null global_mvs if it does not need the global_mvs
// output.
void av1_find_mv_refs(const AV1_COMMON *cm, const MACROBLOCKD *xd,
MB_MODE_INFO *mi, MV_REFERENCE_FRAME ref_frame,
uint8_t ref_mv_count[MODE_CTX_REF_FRAMES],

52
third_party/aom/av1/common/onyxc_int.h vendored
View file

@ -109,18 +109,6 @@ typedef struct {
MV_REFERENCE_FRAME ref_frame;
} MV_REF;
// FIXME(jack.haughton@argondesign.com): This enum was originally in
// encoder/ratectrl.h, and is encoder specific. When we move to C++, this
// should go back there and BufferPool should be templatized.
enum {
INTER_NORMAL = 0,
INTER_LOW = 1,
INTER_HIGH = 2,
GF_ARF_LOW = 3,
GF_ARF_STD = 4,
KF_STD = 5,
RATE_FACTOR_LEVELS = 6
} UENUM1BYTE(RATE_FACTOR_LEVEL);
typedef struct RefCntBuffer {
// For a RefCntBuffer, the following are reference-holding variables:
@ -136,8 +124,6 @@ typedef struct RefCntBuffer {
// - Total 'n' of the variables / array elements above have value 'k' (that
// is, they are pointing to buffer at index 'k').
// Then, pool->frame_bufs[k].ref_count = n.
// TODO(david.turner@argondesign.com) Check whether this helpful comment is
// still correct after we finish restructuring
int ref_count;
unsigned int order_hint;
@ -161,6 +147,10 @@ typedef struct RefCntBuffer {
hash_table hash_table;
FRAME_TYPE frame_type;
// This is only used in the encoder but needs to be indexed per ref frame
// so it's extremely convenient to keep it here.
int interp_filter_selected[SWITCHABLE];
// Inter frame reference frame delta for loop filter
int8_t ref_deltas[REF_FRAMES];
@ -168,7 +158,6 @@ typedef struct RefCntBuffer {
int8_t mode_deltas[MAX_MODE_LF_DELTAS];
FRAME_CONTEXT frame_context;
RATE_FACTOR_LEVEL frame_rf_level;
} RefCntBuffer;
typedef struct BufferPool {
@ -193,11 +182,6 @@ typedef struct BufferPool {
InternalFrameBufferList int_frame_buffers;
} BufferPool;
typedef struct BitstreamLevel {
uint8_t major;
uint8_t minor;
} BitstreamLevel;
typedef struct {
int cdef_pri_damping;
int cdef_sec_damping;
@ -279,7 +263,7 @@ typedef struct SequenceHeader {
int operating_point_idc[MAX_NUM_OPERATING_POINTS];
uint8_t display_model_info_present_flag;
uint8_t decoder_model_info_present_flag;
BitstreamLevel level[MAX_NUM_OPERATING_POINTS];
AV1_LEVEL seq_level_idx[MAX_NUM_OPERATING_POINTS];
uint8_t tier[MAX_NUM_OPERATING_POINTS]; // seq_tier in the spec. One bit: 0
// or 1.
@ -314,6 +298,7 @@ typedef struct {
unsigned int frame_number;
SkipModeInfo skip_mode_info;
int refresh_frame_flags; // Which ref frames are overwritten by this frame
int frame_refs_short_signaling;
} CurrentFrame;
typedef struct AV1Common {
@ -343,9 +328,9 @@ typedef struct AV1Common {
// For encoder, we have a two-level mapping from reference frame type to the
// corresponding buffer in the buffer pool:
// * 'remapped_ref_idx[i - 1]' maps reference type ‘i’ (range: LAST_FRAME ...
// EXTREF_FRAME) to a remapped index ‘j’ (in range: 0 ... REF_FRAMES - 1)
// * Later, 'cm->ref_frame_map[j]' maps the remapped index ‘j’ to a pointer to
// * 'remapped_ref_idx[i - 1]' maps reference type 'i' (range: LAST_FRAME ...
// EXTREF_FRAME) to a remapped index 'j' (in range: 0 ... REF_FRAMES - 1)
// * Later, 'cm->ref_frame_map[j]' maps the remapped index 'j' to a pointer to
// the reference counted buffer structure RefCntBuffer, taken from the buffer
// pool cm->buffer_pool->frame_bufs.
//
@ -364,10 +349,10 @@ typedef struct AV1Common {
struct scale_factors ref_scale_factors[REF_FRAMES];
// For decoder, ref_frame_map[i] maps reference type 'i' to a pointer to
// the buffer in the buffer pool ‘cm->buffer_pool.frame_bufs’.
// the buffer in the buffer pool 'cm->buffer_pool.frame_bufs'.
// For encoder, ref_frame_map[j] (where j = remapped_ref_idx[i]) maps
// remapped reference index 'j' (that is, original reference type 'i') to
// a pointer to the buffer in the buffer pool ‘cm->buffer_pool.frame_bufs’.
// a pointer to the buffer in the buffer pool 'cm->buffer_pool.frame_bufs'.
RefCntBuffer *ref_frame_map[REF_FRAMES];
// Prepare ref_frame_map for the next frame.
@ -517,6 +502,7 @@ typedef struct AV1Common {
int tile_col_start_sb[MAX_TILE_COLS + 1]; // valid for 0 <= i <= tile_cols
int tile_row_start_sb[MAX_TILE_ROWS + 1]; // valid for 0 <= i <= tile_rows
int tile_width, tile_height; // In MI units
int min_inner_tile_width; // min width of non-rightmost tile
unsigned int large_scale_tile;
unsigned int single_tile_decoding;
@ -642,6 +628,7 @@ static INLINE RefCntBuffer *assign_cur_frame_new_fb(AV1_COMMON *const cm) {
cm->cur_frame = &cm->buffer_pool->frame_bufs[new_fb_idx];
cm->cur_frame->buf.buf_8bit_valid = 0;
av1_zero(cm->cur_frame->interp_filter_selected);
return cm->cur_frame;
}
@ -1385,17 +1372,8 @@ static INLINE int is_coded_lossless(const AV1_COMMON *cm,
return coded_lossless;
}
static INLINE int is_valid_seq_level_idx(uint8_t seq_level_idx) {
return seq_level_idx < 24 || seq_level_idx == 31;
}
static INLINE uint8_t major_minor_to_seq_level_idx(BitstreamLevel bl) {
assert(bl.major >= LEVEL_MAJOR_MIN && bl.major <= LEVEL_MAJOR_MAX);
// Since bl.minor is unsigned a comparison will return a warning:
// comparison is always true due to limited range of data type
assert(LEVEL_MINOR_MIN == 0);
assert(bl.minor <= LEVEL_MINOR_MAX);
return ((bl.major - LEVEL_MAJOR_MIN) << LEVEL_MINOR_BITS) + bl.minor;
static INLINE int is_valid_seq_level_idx(AV1_LEVEL seq_level_idx) {
return seq_level_idx < SEQ_LEVELS || seq_level_idx == SEQ_LEVEL_MAX;
}
#ifdef __cplusplus

6
third_party/aom/av1/common/pred_common.h vendored
View file

@ -59,9 +59,13 @@ static INLINE int av1_get_spatial_seg_pred(const AV1_COMMON *const cm,
prev_l = get_segment_id(cm, cm->cur_frame->seg_map, BLOCK_4X4, mi_row - 0,
mi_col - 1);
}
// This property follows from the fact that get_segment_id() returns a
// nonnegative value. This allows us to test for all edge cases with a simple
// prev_ul < 0 check.
assert(IMPLIES(prev_ul >= 0, prev_u >= 0 && prev_l >= 0));
// Pick CDF index based on number of matching/out-of-bounds segment IDs.
if (prev_ul < 0 || prev_u < 0 || prev_l < 0) /* Edge case */
if (prev_ul < 0) /* Edge cases */
*cdf_index = 0;
else if ((prev_ul == prev_u) && (prev_ul == prev_l))
*cdf_index = 2;

8
third_party/aom/av1/common/reconinter.h vendored
View file

@ -167,6 +167,7 @@ static INLINE int is_interinter_compound_used(COMPOUND_TYPE type,
const int comp_allowed = is_comp_ref_allowed(sb_type);
switch (type) {
case COMPOUND_AVERAGE:
case COMPOUND_DISTWTD:
case COMPOUND_DIFFWTD: return comp_allowed;
case COMPOUND_WEDGE:
return comp_allowed && wedge_params_lookup[sb_type].bits > 0;
@ -245,13 +246,14 @@ static INLINE MV clamp_mv_to_umv_border_sb(const MACROBLOCKD *xd,
return clamped_mv;
}
static INLINE int scaled_buffer_offset(int x_offset, int y_offset, int stride,
const struct scale_factors *sf) {
static INLINE int64_t scaled_buffer_offset(int x_offset, int y_offset,
int stride,
const struct scale_factors *sf) {
const int x =
sf ? sf->scale_value_x(x_offset, sf) >> SCALE_EXTRA_BITS : x_offset;
const int y =
sf ? sf->scale_value_y(y_offset, sf) >> SCALE_EXTRA_BITS : y_offset;
return y * stride + x;
return (int64_t)y * stride + x;
}
static INLINE void setup_pred_plane(struct buf_2d *dst, BLOCK_SIZE bsize,

2
third_party/aom/av1/common/restoration.h vendored
View file

@ -22,7 +22,7 @@
extern "C" {
#endif
// Boarder for Loop restoration buffer
// Border for Loop restoration buffer
#define AOM_RESTORATION_FRAME_BORDER 32
#define CLIP(x, lo, hi) ((x) < (lo) ? (lo) : (x) > (hi) ? (hi) : (x))
#define RINT(x) ((x) < 0 ? (int)((x)-0.5) : (int)((x) + 0.5))

23
third_party/aom/av1/common/tile_common.c vendored
View file

@ -51,6 +51,10 @@ void av1_calculate_tile_cols(AV1_COMMON *const cm) {
int sb_rows = mi_rows >> cm->seq_params.mib_size_log2;
int i;
// This will be overridden if there is at least two columns of tiles
// (otherwise there is no inner tile width)
cm->min_inner_tile_width = -1;
if (cm->uniform_tile_spacing_flag) {
int start_sb;
int size_sb = ALIGN_POWER_OF_TWO(sb_cols, cm->log2_tile_cols);
@ -67,18 +71,29 @@ void av1_calculate_tile_cols(AV1_COMMON *const cm) {
cm->tile_width = size_sb << cm->seq_params.mib_size_log2;
cm->tile_width = AOMMIN(cm->tile_width, cm->mi_cols);
if (cm->tile_cols > 1) {
cm->min_inner_tile_width = cm->tile_width;
}
} else {
int max_tile_area_sb = (sb_rows * sb_cols);
int widest_tile_sb = 1;
int narrowest_inner_tile_sb = 65536;
cm->log2_tile_cols = tile_log2(1, cm->tile_cols);
for (i = 0; i < cm->tile_cols; i++) {
int size_sb = cm->tile_col_start_sb[i + 1] - cm->tile_col_start_sb[i];
widest_tile_sb = AOMMAX(widest_tile_sb, size_sb);
// ignore the rightmost tile in frame for determining the narrowest
if (i < cm->tile_cols - 1)
narrowest_inner_tile_sb = AOMMIN(narrowest_inner_tile_sb, size_sb);
}
if (cm->min_log2_tiles) {
max_tile_area_sb >>= (cm->min_log2_tiles + 1);
}
cm->max_tile_height_sb = AOMMAX(max_tile_area_sb / widest_tile_sb, 1);
if (cm->tile_cols > 1) {
cm->min_inner_tile_width = narrowest_inner_tile_sb
<< cm->seq_params.mib_size_log2;
}
}
}
@ -204,3 +219,11 @@ void av1_get_uniform_tile_size(const AV1_COMMON *cm, int *w, int *h) {
}
}
}
int is_min_tile_width_satisfied(const AV1_COMMON *cm) {
// Disable check if there is a single tile col in the frame
if (cm->tile_cols == 1) return 1;
return ((cm->min_inner_tile_width << MI_SIZE_LOG2) >=
(64 << av1_superres_scaled(cm)));
}

3
third_party/aom/av1/common/tile_common.h vendored
View file

@ -59,6 +59,9 @@ void av1_get_tile_limits(struct AV1Common *const cm);
void av1_calculate_tile_cols(struct AV1Common *const cm);
void av1_calculate_tile_rows(struct AV1Common *const cm);
// Checks if the minimum tile_width requirement is satisfied
int is_min_tile_width_satisfied(const struct AV1Common *cm);
#ifdef __cplusplus
} // extern "C"
#endif

14
third_party/aom/av1/common/x86/av1_inv_txfm_ssse3.c vendored
View file

@ -2920,8 +2920,18 @@ void av1_inv_txfm_add_ssse3(const tran_low_t *dqcoeff, uint8_t *dst, int stride,
const TxfmParam *txfm_param) {
const TX_TYPE tx_type = txfm_param->tx_type;
if (!txfm_param->lossless) {
av1_lowbd_inv_txfm2d_add_ssse3(dqcoeff, dst, stride, tx_type,
txfm_param->tx_size, txfm_param->eob);
switch (txfm_param->tx_size) {
case TX_4X16:
case TX_16X4:
// TODO(http://crbug.com/aomedia/2350): the ssse3 versions cause test
// vector mismatches.
av1_inv_txfm_add_c(dqcoeff, dst, stride, txfm_param);
break;
default:
av1_lowbd_inv_txfm2d_add_ssse3(dqcoeff, dst, stride, tx_type,
txfm_param->tx_size, txfm_param->eob);
break;
}
} else {
av1_inv_txfm_add_c(dqcoeff, dst, stride, txfm_param);
}

132
third_party/aom/av1/common/x86/convolve_2d_avx2.c vendored
View file

@ -27,31 +27,15 @@ void av1_convolve_2d_sr_avx2(const uint8_t *src, int src_stride, uint8_t *dst,
const int subpel_x_q4, const int subpel_y_q4,
ConvolveParams *conv_params) {
const int bd = 8;
DECLARE_ALIGNED(32, int16_t, im_block[(MAX_SB_SIZE + MAX_FILTER_TAP) * 8]);
int im_h = h + filter_params_y->taps - 1;
int im_stride = 8;
int i, j;
const int fo_vert = filter_params_y->taps / 2 - 1;
const int fo_horiz = filter_params_x->taps / 2 - 1;
const uint8_t *const src_ptr = src - fo_vert * src_stride - fo_horiz;
int i, is_horiz_4tap = 0, is_vert_4tap = 0;
DECLARE_ALIGNED(32, int16_t, im_block[(MAX_SB_SIZE + MAX_FILTER_TAP) * 8]);
const int bits =
FILTER_BITS * 2 - conv_params->round_0 - conv_params->round_1;
const int offset_bits = bd + 2 * FILTER_BITS - conv_params->round_0;
__m256i filt[4], coeffs_h[4], coeffs_v[4];
assert(conv_params->round_0 > 0);
filt[0] = _mm256_load_si256((__m256i const *)filt_global_avx2);
filt[1] = _mm256_load_si256((__m256i const *)(filt_global_avx2 + 32));
filt[2] = _mm256_load_si256((__m256i const *)(filt_global_avx2 + 32 * 2));
filt[3] = _mm256_load_si256((__m256i const *)(filt_global_avx2 + 32 * 3));
prepare_coeffs_lowbd(filter_params_x, subpel_x_q4, coeffs_h);
prepare_coeffs(filter_params_y, subpel_y_q4, coeffs_v);
const __m256i round_const_h = _mm256_set1_epi16(
((1 << (conv_params->round_0 - 1)) >> 1) + (1 << (bd + FILTER_BITS - 2)));
const __m128i round_shift_h = _mm_cvtsi32_si128(conv_params->round_0 - 1);
@ -65,58 +49,96 @@ void av1_convolve_2d_sr_avx2(const uint8_t *src, int src_stride, uint8_t *dst,
((1 << (offset_bits - conv_params->round_1)) >> 1));
const __m128i round_shift_v = _mm_cvtsi32_si128(bits);
for (j = 0; j < w; j += 8) {
for (i = 0; i < im_h; i += 2) {
__m256i data = _mm256_castsi128_si256(
_mm_loadu_si128((__m128i *)&src_ptr[(i * src_stride) + j]));
__m256i filt[4], coeffs_h[4], coeffs_v[4];
// Load the next line
if (i + 1 < im_h)
filt[0] = _mm256_load_si256((__m256i const *)(filt_global_avx2));
filt[1] = _mm256_load_si256((__m256i const *)(filt_global_avx2 + 32));
prepare_coeffs_lowbd(filter_params_x, subpel_x_q4, coeffs_h);
prepare_coeffs(filter_params_y, subpel_y_q4, coeffs_v);
// Condition for checking valid horz_filt taps
if (!(_mm256_extract_epi32(_mm256_or_si256(coeffs_h[0], coeffs_h[3]), 0)))
is_horiz_4tap = 1;
// Condition for checking valid vert_filt taps
if (!(_mm256_extract_epi32(_mm256_or_si256(coeffs_v[0], coeffs_v[3]), 0)))
is_vert_4tap = 1;
// horz_filt as 4 tap and vert_filt as 8 tap
if (is_horiz_4tap) {
int im_h = h + filter_params_y->taps - 1;
const int fo_vert = filter_params_y->taps / 2 - 1;
const int fo_horiz = 1;
const uint8_t *const src_ptr = src - fo_vert * src_stride - fo_horiz;
// horz-filter
for (int j = 0; j < w; j += 8) {
for (i = 0; i < (im_h - 2); i += 2) {
__m256i data = _mm256_castsi128_si256(
_mm_loadu_si128((__m128i *)&src_ptr[(i * src_stride) + j]));
// Load the next line
data = _mm256_inserti128_si256(
data,
_mm_loadu_si128(
(__m128i *)&src_ptr[(i * src_stride) + j + src_stride]),
1);
__m256i res = convolve_lowbd_x_4tap(data, coeffs_h + 1, filt);
__m256i res = convolve_lowbd_x(data, coeffs_h, filt);
res = _mm256_sra_epi16(_mm256_add_epi16(res, round_const_h),
round_shift_h);
_mm256_store_si256((__m256i *)&im_block[i * im_stride], res);
}
__m256i data_1 = _mm256_castsi128_si256(
_mm_loadu_si128((__m128i *)&src_ptr[(i * src_stride) + j]));
__m256i res = convolve_lowbd_x_4tap(data_1, coeffs_h + 1, filt);
res =
_mm256_sra_epi16(_mm256_add_epi16(res, round_const_h), round_shift_h);
_mm256_store_si256((__m256i *)&im_block[i * im_stride], res);
}
/* Vertical filter */
{
// vert filter
CONVOLVE_SR_VERTICAL_FILTER_8TAP;
}
} else if (is_vert_4tap) {
int im_h = h + 3;
const int fo_vert = 1;
const int fo_horiz = filter_params_x->taps / 2 - 1;
const uint8_t *const src_ptr = src - fo_vert * src_stride - fo_horiz;
filt[2] = _mm256_load_si256((__m256i const *)(filt_global_avx2 + 32 * 2));
filt[3] = _mm256_load_si256((__m256i const *)(filt_global_avx2 + 32 * 3));
for (int j = 0; j < w; j += 8) {
// horz_filter
CONVOLVE_SR_HORIZONTAL_FILTER_8TAP;
// vert_filter
__m256i s[6];
__m256i src_0 = _mm256_loadu_si256((__m256i *)(im_block + 0 * im_stride));
__m256i src_1 = _mm256_loadu_si256((__m256i *)(im_block + 1 * im_stride));
__m256i src_2 = _mm256_loadu_si256((__m256i *)(im_block + 2 * im_stride));
__m256i src_3 = _mm256_loadu_si256((__m256i *)(im_block + 3 * im_stride));
__m256i src_4 = _mm256_loadu_si256((__m256i *)(im_block + 4 * im_stride));
__m256i src_5 = _mm256_loadu_si256((__m256i *)(im_block + 5 * im_stride));
__m256i s[8];
s[0] = _mm256_unpacklo_epi16(src_0, src_1);
s[1] = _mm256_unpacklo_epi16(src_2, src_3);
s[2] = _mm256_unpacklo_epi16(src_4, src_5);
s[4] = _mm256_unpackhi_epi16(src_0, src_1);
s[5] = _mm256_unpackhi_epi16(src_2, src_3);
s[6] = _mm256_unpackhi_epi16(src_4, src_5);
s[3] = _mm256_unpackhi_epi16(src_0, src_1);
s[4] = _mm256_unpackhi_epi16(src_2, src_3);
for (i = 0; i < h; i += 2) {
const int16_t *data = &im_block[i * im_stride];
const __m256i s6 =
_mm256_loadu_si256((__m256i *)(data + 6 * im_stride));
const __m256i s7 =
_mm256_loadu_si256((__m256i *)(data + 7 * im_stride));
const __m256i s4 =
_mm256_loadu_si256((__m256i *)(data + 4 * im_stride));
const __m256i s5 =
_mm256_loadu_si256((__m256i *)(data + 5 * im_stride));
s[3] = _mm256_unpacklo_epi16(s6, s7);
s[7] = _mm256_unpackhi_epi16(s6, s7);
s[2] = _mm256_unpacklo_epi16(s4, s5);
s[5] = _mm256_unpackhi_epi16(s4, s5);
__m256i res_a = convolve(s, coeffs_v);
__m256i res_b = convolve(s + 4, coeffs_v);
__m256i res_a = convolve_4tap(s, coeffs_v + 1);
__m256i res_b = convolve_4tap(s + 3, coeffs_v + 1);
// Combine V round and 2F-H-V round into a single rounding
res_a =
@ -154,13 +176,25 @@ void av1_convolve_2d_sr_avx2(const uint8_t *src, int src_stride, uint8_t *dst,
s[0] = s[1];
s[1] = s[2];
s[2] = s[3];
s[3] = s[4];
s[4] = s[5];
s[5] = s[6];
s[6] = s[7];
}
}
} else {
int j;
int im_h = h + filter_params_y->taps - 1;
const int fo_vert = filter_params_y->taps / 2 - 1;
const int fo_horiz = filter_params_x->taps / 2 - 1;
const uint8_t *const src_ptr = src - fo_vert * src_stride - fo_horiz;
filt[2] = _mm256_load_si256((__m256i const *)(filt_global_avx2 + 32 * 2));
filt[3] = _mm256_load_si256((__m256i const *)(filt_global_avx2 + 32 * 3));
for (j = 0; j < w; j += 8) {
CONVOLVE_SR_HORIZONTAL_FILTER_8TAP;
CONVOLVE_SR_VERTICAL_FILTER_8TAP;
}
}
}

484
third_party/aom/av1/common/x86/convolve_avx2.c vendored
View file

@ -23,153 +23,239 @@ void av1_convolve_y_sr_avx2(const uint8_t *src, int src_stride, uint8_t *dst,
const InterpFilterParams *filter_params_y,
const int subpel_x_q4, const int subpel_y_q4,
ConvolveParams *conv_params) {
int i, j;
const int fo_vert = filter_params_y->taps / 2 - 1;
const uint8_t *const src_ptr = src - fo_vert * src_stride;
int i, j, is_vert_4tap = 0;
// right shift is F-1 because we are already dividing
// filter co-efficients by 2
const int right_shift_bits = (FILTER_BITS - 1);
const __m128i right_shift = _mm_cvtsi32_si128(right_shift_bits);
const __m256i right_shift_const =
_mm256_set1_epi16((1 << right_shift_bits) >> 1);
__m256i coeffs[4], s[8];
assert(conv_params->round_0 <= FILTER_BITS);
assert(((conv_params->round_0 + conv_params->round_1) <= (FILTER_BITS + 1)) ||
((conv_params->round_0 + conv_params->round_1) == (2 * FILTER_BITS)));
prepare_coeffs_lowbd(filter_params_y, subpel_y_q4, coeffs);
(void)filter_params_x;
(void)subpel_x_q4;
(void)conv_params;
__m256i coeffs[4], s[8];
__m128i d[6];
for (j = 0; j < w; j += 16) {
const uint8_t *data = &src_ptr[j];
__m256i src6;
prepare_coeffs_lowbd(filter_params_y, subpel_y_q4, coeffs);
// Load lines a and b. Line a to lower 128, line b to upper 128
const __m256i src_01a = _mm256_permute2x128_si256(
_mm256_castsi128_si256(
_mm_loadu_si128((__m128i *)(data + 0 * src_stride))),
_mm256_castsi128_si256(
_mm_loadu_si128((__m128i *)(data + 1 * src_stride))),
0x20);
// Condition for checking valid vert_filt taps
if (!(_mm256_extract_epi32(_mm256_or_si256(coeffs[0], coeffs[3]), 0)))
is_vert_4tap = 1;
const __m256i src_12a = _mm256_permute2x128_si256(
_mm256_castsi128_si256(
_mm_loadu_si128((__m128i *)(data + 1 * src_stride))),
_mm256_castsi128_si256(
_mm_loadu_si128((__m128i *)(data + 2 * src_stride))),
0x20);
// vert_filt as 4 tap
if (is_vert_4tap) {
const int fo_vert = 1;
const uint8_t *const src_ptr = src - fo_vert * src_stride;
for (j = 0; j < w; j += 16) {
const uint8_t *data = &src_ptr[j];
d[0] = _mm_loadu_si128((__m128i *)(data + 0 * src_stride));
d[1] = _mm_loadu_si128((__m128i *)(data + 1 * src_stride));
d[2] = _mm_loadu_si128((__m128i *)(data + 2 * src_stride));
d[3] = _mm_loadu_si128((__m128i *)(data + 3 * src_stride));
d[4] = _mm_loadu_si128((__m128i *)(data + 4 * src_stride));
const __m256i src_23a = _mm256_permute2x128_si256(
_mm256_castsi128_si256(
_mm_loadu_si128((__m128i *)(data + 2 * src_stride))),
_mm256_castsi128_si256(
_mm_loadu_si128((__m128i *)(data + 3 * src_stride))),
0x20);
// Load lines a and b. Line a to lower 128, line b to upper 128
const __m256i src_01a = _mm256_permute2x128_si256(
_mm256_castsi128_si256(d[0]), _mm256_castsi128_si256(d[1]), 0x20);
const __m256i src_34a = _mm256_permute2x128_si256(
_mm256_castsi128_si256(
_mm_loadu_si128((__m128i *)(data + 3 * src_stride))),
_mm256_castsi128_si256(
_mm_loadu_si128((__m128i *)(data + 4 * src_stride))),
0x20);
const __m256i src_12a = _mm256_permute2x128_si256(
_mm256_castsi128_si256(d[1]), _mm256_castsi128_si256(d[2]), 0x20);
const __m256i src_45a = _mm256_permute2x128_si256(
_mm256_castsi128_si256(
_mm_loadu_si128((__m128i *)(data + 4 * src_stride))),
_mm256_castsi128_si256(
_mm_loadu_si128((__m128i *)(data + 5 * src_stride))),
0x20);
const __m256i src_23a = _mm256_permute2x128_si256(
_mm256_castsi128_si256(d[2]), _mm256_castsi128_si256(d[3]), 0x20);
src6 = _mm256_castsi128_si256(
_mm_loadu_si128((__m128i *)(data + 6 * src_stride)));
const __m256i src_56a = _mm256_permute2x128_si256(
_mm256_castsi128_si256(
_mm_loadu_si128((__m128i *)(data + 5 * src_stride))),
src6, 0x20);
const __m256i src_34a = _mm256_permute2x128_si256(
_mm256_castsi128_si256(d[3]), _mm256_castsi128_si256(d[4]), 0x20);
s[0] = _mm256_unpacklo_epi8(src_01a, src_12a);
s[1] = _mm256_unpacklo_epi8(src_23a, src_34a);
s[2] = _mm256_unpacklo_epi8(src_45a, src_56a);
s[0] = _mm256_unpacklo_epi8(src_01a, src_12a);
s[1] = _mm256_unpacklo_epi8(src_23a, src_34a);
s[4] = _mm256_unpackhi_epi8(src_01a, src_12a);
s[5] = _mm256_unpackhi_epi8(src_23a, src_34a);
s[6] = _mm256_unpackhi_epi8(src_45a, src_56a);
s[3] = _mm256_unpackhi_epi8(src_01a, src_12a);
s[4] = _mm256_unpackhi_epi8(src_23a, src_34a);
for (i = 0; i < h; i += 2) {
data = &src_ptr[i * src_stride + j];
const __m256i src_67a = _mm256_permute2x128_si256(
src6,
_mm256_castsi128_si256(
_mm_loadu_si128((__m128i *)(data + 7 * src_stride))),
0x20);
for (i = 0; i < h; i += 2) {
data = &src_ptr[i * src_stride + j];
d[5] = _mm_loadu_si128((__m128i *)(data + 5 * src_stride));
const __m256i src_45a = _mm256_permute2x128_si256(
_mm256_castsi128_si256(d[4]), _mm256_castsi128_si256(d[5]), 0x20);
d[4] = _mm_loadu_si128((__m128i *)(data + 6 * src_stride));
const __m256i src_56a = _mm256_permute2x128_si256(
_mm256_castsi128_si256(d[5]), _mm256_castsi128_si256(d[4]), 0x20);
s[2] = _mm256_unpacklo_epi8(src_45a, src_56a);
s[5] = _mm256_unpackhi_epi8(src_45a, src_56a);
const __m256i res_lo = convolve_lowbd_4tap(s, coeffs + 1);
/* rounding code */
// shift by F - 1
const __m256i res_16b_lo = _mm256_sra_epi16(
_mm256_add_epi16(res_lo, right_shift_const), right_shift);
// 8 bit conversion and saturation to uint8
__m256i res_8b_lo = _mm256_packus_epi16(res_16b_lo, res_16b_lo);
if (w - j > 8) {
const __m256i res_hi = convolve_lowbd_4tap(s + 3, coeffs + 1);
/* rounding code */
// shift by F - 1
const __m256i res_16b_hi = _mm256_sra_epi16(
_mm256_add_epi16(res_hi, right_shift_const), right_shift);
// 8 bit conversion and saturation to uint8
__m256i res_8b_hi = _mm256_packus_epi16(res_16b_hi, res_16b_hi);
__m256i res_a = _mm256_unpacklo_epi64(res_8b_lo, res_8b_hi);
const __m128i res_0 = _mm256_castsi256_si128(res_a);
const __m128i res_1 = _mm256_extracti128_si256(res_a, 1);
_mm_storeu_si128((__m128i *)&dst[i * dst_stride + j], res_0);
_mm_storeu_si128((__m128i *)&dst[i * dst_stride + j + dst_stride],
res_1);
} else {
const __m128i res_0 = _mm256_castsi256_si128(res_8b_lo);
const __m128i res_1 = _mm256_extracti128_si256(res_8b_lo, 1);
if (w - j > 4) {
_mm_storel_epi64((__m128i *)&dst[i * dst_stride + j], res_0);
_mm_storel_epi64((__m128i *)&dst[i * dst_stride + j + dst_stride],
res_1);
} else if (w - j > 2) {
xx_storel_32(&dst[i * dst_stride + j], res_0);
xx_storel_32(&dst[i * dst_stride + j + dst_stride], res_1);
} else {
__m128i *const p_0 = (__m128i *)&dst[i * dst_stride + j];
__m128i *const p_1 =
(__m128i *)&dst[i * dst_stride + j + dst_stride];
*(uint16_t *)p_0 = _mm_cvtsi128_si32(res_0);
*(uint16_t *)p_1 = _mm_cvtsi128_si32(res_1);
}
}
s[0] = s[1];
s[1] = s[2];
s[3] = s[4];
s[4] = s[5];
}
}
} else {
const int fo_vert = filter_params_y->taps / 2 - 1;
const uint8_t *const src_ptr = src - fo_vert * src_stride;
for (j = 0; j < w; j += 16) {
const uint8_t *data = &src_ptr[j];
__m256i src6;
d[0] = _mm_loadu_si128((__m128i *)(data + 0 * src_stride));
d[1] = _mm_loadu_si128((__m128i *)(data + 1 * src_stride));
d[2] = _mm_loadu_si128((__m128i *)(data + 2 * src_stride));
d[3] = _mm_loadu_si128((__m128i *)(data + 3 * src_stride));
d[4] = _mm_loadu_si128((__m128i *)(data + 4 * src_stride));
d[5] = _mm_loadu_si128((__m128i *)(data + 5 * src_stride));
// Load lines a and b. Line a to lower 128, line b to upper 128
const __m256i src_01a = _mm256_permute2x128_si256(
_mm256_castsi128_si256(d[0]), _mm256_castsi128_si256(d[1]), 0x20);
const __m256i src_12a = _mm256_permute2x128_si256(
_mm256_castsi128_si256(d[1]), _mm256_castsi128_si256(d[2]), 0x20);
const __m256i src_23a = _mm256_permute2x128_si256(
_mm256_castsi128_si256(d[2]), _mm256_castsi128_si256(d[3]), 0x20);
const __m256i src_34a = _mm256_permute2x128_si256(
_mm256_castsi128_si256(d[3]), _mm256_castsi128_si256(d[4]), 0x20);
const __m256i src_45a = _mm256_permute2x128_si256(
_mm256_castsi128_si256(d[4]), _mm256_castsi128_si256(d[5]), 0x20);
src6 = _mm256_castsi128_si256(
_mm_loadu_si128((__m128i *)(data + 8 * src_stride)));
const __m256i src_78a = _mm256_permute2x128_si256(
_mm256_castsi128_si256(
_mm_loadu_si128((__m128i *)(data + 7 * src_stride))),
src6, 0x20);
_mm_loadu_si128((__m128i *)(data + 6 * src_stride)));
const __m256i src_56a =
_mm256_permute2x128_si256(_mm256_castsi128_si256(d[5]), src6, 0x20);
s[3] = _mm256_unpacklo_epi8(src_67a, src_78a);
s[7] = _mm256_unpackhi_epi8(src_67a, src_78a);
s[0] = _mm256_unpacklo_epi8(src_01a, src_12a);
s[1] = _mm256_unpacklo_epi8(src_23a, src_34a);
s[2] = _mm256_unpacklo_epi8(src_45a, src_56a);
const __m256i res_lo = convolve_lowbd(s, coeffs);
s[4] = _mm256_unpackhi_epi8(src_01a, src_12a);
s[5] = _mm256_unpackhi_epi8(src_23a, src_34a);
s[6] = _mm256_unpackhi_epi8(src_45a, src_56a);
/* rounding code */
// shift by F - 1
const __m256i res_16b_lo = _mm256_sra_epi16(
_mm256_add_epi16(res_lo, right_shift_const), right_shift);
// 8 bit conversion and saturation to uint8
__m256i res_8b_lo = _mm256_packus_epi16(res_16b_lo, res_16b_lo);
for (i = 0; i < h; i += 2) {
data = &src_ptr[i * src_stride + j];
const __m256i src_67a = _mm256_permute2x128_si256(
src6,
_mm256_castsi128_si256(
_mm_loadu_si128((__m128i *)(data + 7 * src_stride))),
0x20);
if (w - j > 8) {
const __m256i res_hi = convolve_lowbd(s + 4, coeffs);
src6 = _mm256_castsi128_si256(
_mm_loadu_si128((__m128i *)(data + 8 * src_stride)));
const __m256i src_78a = _mm256_permute2x128_si256(
_mm256_castsi128_si256(
_mm_loadu_si128((__m128i *)(data + 7 * src_stride))),
src6, 0x20);
s[3] = _mm256_unpacklo_epi8(src_67a, src_78a);
s[7] = _mm256_unpackhi_epi8(src_67a, src_78a);
const __m256i res_lo = convolve_lowbd(s, coeffs);
/* rounding code */
// shift by F - 1
const __m256i res_16b_hi = _mm256_sra_epi16(
_mm256_add_epi16(res_hi, right_shift_const), right_shift);
const __m256i res_16b_lo = _mm256_sra_epi16(
_mm256_add_epi16(res_lo, right_shift_const), right_shift);
// 8 bit conversion and saturation to uint8
__m256i res_8b_hi = _mm256_packus_epi16(res_16b_hi, res_16b_hi);
__m256i res_8b_lo = _mm256_packus_epi16(res_16b_lo, res_16b_lo);
__m256i res_a = _mm256_unpacklo_epi64(res_8b_lo, res_8b_hi);
if (w - j > 8) {
const __m256i res_hi = convolve_lowbd(s + 4, coeffs);
const __m128i res_0 = _mm256_castsi256_si128(res_a);
const __m128i res_1 = _mm256_extracti128_si256(res_a, 1);
/* rounding code */
// shift by F - 1
const __m256i res_16b_hi = _mm256_sra_epi16(
_mm256_add_epi16(res_hi, right_shift_const), right_shift);
// 8 bit conversion and saturation to uint8
__m256i res_8b_hi = _mm256_packus_epi16(res_16b_hi, res_16b_hi);
_mm_storeu_si128((__m128i *)&dst[i * dst_stride + j], res_0);
_mm_storeu_si128((__m128i *)&dst[i * dst_stride + j + dst_stride],
res_1);
} else {
const __m128i res_0 = _mm256_castsi256_si128(res_8b_lo);
const __m128i res_1 = _mm256_extracti128_si256(res_8b_lo, 1);
if (w - j > 4) {
_mm_storel_epi64((__m128i *)&dst[i * dst_stride + j], res_0);
_mm_storel_epi64((__m128i *)&dst[i * dst_stride + j + dst_stride],
__m256i res_a = _mm256_unpacklo_epi64(res_8b_lo, res_8b_hi);
const __m128i res_0 = _mm256_castsi256_si128(res_a);
const __m128i res_1 = _mm256_extracti128_si256(res_a, 1);
_mm_storeu_si128((__m128i *)&dst[i * dst_stride + j], res_0);
_mm_storeu_si128((__m128i *)&dst[i * dst_stride + j + dst_stride],
res_1);
} else if (w - j > 2) {
xx_storel_32(&dst[i * dst_stride + j], res_0);
xx_storel_32(&dst[i * dst_stride + j + dst_stride], res_1);
} else {
__m128i *const p_0 = (__m128i *)&dst[i * dst_stride + j];
__m128i *const p_1 = (__m128i *)&dst[i * dst_stride + j + dst_stride];
*(uint16_t *)p_0 = _mm_cvtsi128_si32(res_0);
*(uint16_t *)p_1 = _mm_cvtsi128_si32(res_1);
const __m128i res_0 = _mm256_castsi256_si128(res_8b_lo);
const __m128i res_1 = _mm256_extracti128_si256(res_8b_lo, 1);
if (w - j > 4) {
_mm_storel_epi64((__m128i *)&dst[i * dst_stride + j], res_0);
_mm_storel_epi64((__m128i *)&dst[i * dst_stride + j + dst_stride],
res_1);
} else if (w - j > 2) {
xx_storel_32(&dst[i * dst_stride + j], res_0);
xx_storel_32(&dst[i * dst_stride + j + dst_stride], res_1);
} else {
__m128i *const p_0 = (__m128i *)&dst[i * dst_stride + j];
__m128i *const p_1 =
(__m128i *)&dst[i * dst_stride + j + dst_stride];
*(uint16_t *)p_0 = _mm_cvtsi128_si32(res_0);
*(uint16_t *)p_1 = _mm_cvtsi128_si32(res_1);
}
}
s[0] = s[1];
s[1] = s[2];
s[2] = s[3];
s[4] = s[5];
s[5] = s[6];
s[6] = s[7];
}
s[0] = s[1];
s[1] = s[2];
s[2] = s[3];
s[4] = s[5];
s[5] = s[6];
s[6] = s[7];
}
}
}
@ -180,26 +266,14 @@ void av1_convolve_x_sr_avx2(const uint8_t *src, int src_stride, uint8_t *dst,
const InterpFilterParams *filter_params_y,
const int subpel_x_q4, const int subpel_y_q4,
ConvolveParams *conv_params) {
int i, j;
const int fo_horiz = filter_params_x->taps / 2 - 1;
const uint8_t *const src_ptr = src - fo_horiz;
const int bits = FILTER_BITS - conv_params->round_0;
__m256i filt[4], coeffs[4];
filt[0] = _mm256_load_si256((__m256i const *)filt_global_avx2);
filt[1] = _mm256_load_si256((__m256i const *)(filt_global_avx2 + 32));
filt[2] = _mm256_load_si256((__m256i const *)(filt_global_avx2 + 32 * 2));
filt[3] = _mm256_load_si256((__m256i const *)(filt_global_avx2 + 32 * 3));
prepare_coeffs_lowbd(filter_params_x, subpel_x_q4, coeffs);
const __m256i round_0_const =
_mm256_set1_epi16((1 << (conv_params->round_0 - 1)) >> 1);
const __m128i round_0_shift = _mm_cvtsi32_si128(conv_params->round_0 - 1);
const __m256i round_const = _mm256_set1_epi16((1 << bits) >> 1);
const __m128i round_shift = _mm_cvtsi32_si128(bits);
int i, is_horiz_4tap = 0;
(void)filter_params_y;
(void)subpel_y_q4;
@ -208,51 +282,101 @@ void av1_convolve_x_sr_avx2(const uint8_t *src, int src_stride, uint8_t *dst,
((conv_params->round_0 + conv_params->round_1) == 2 * FILTER_BITS));
assert(conv_params->round_0 > 0);
if (w <= 8) {
for (i = 0; i < h; i += 2) {
const __m256i data = _mm256_permute2x128_si256(
_mm256_castsi128_si256(
_mm_loadu_si128((__m128i *)(&src_ptr[i * src_stride]))),
_mm256_castsi128_si256(_mm_loadu_si128(
(__m128i *)(&src_ptr[i * src_stride + src_stride]))),
0x20);
__m256i coeffs[4], filt[4];
filt[0] = _mm256_load_si256((__m256i const *)(filt_global_avx2));
filt[1] = _mm256_load_si256((__m256i const *)(filt_global_avx2 + 32));
__m256i res_16b = convolve_lowbd_x(data, coeffs, filt);
prepare_coeffs_lowbd(filter_params_x, subpel_x_q4, coeffs);
res_16b = _mm256_sra_epi16(_mm256_add_epi16(res_16b, round_0_const),
round_0_shift);
// Condition for checking valid horz_filt taps
if (!(_mm256_extract_epi32(_mm256_or_si256(coeffs[0], coeffs[3]), 0)))
is_horiz_4tap = 1;
res_16b =
_mm256_sra_epi16(_mm256_add_epi16(res_16b, round_const), round_shift);
// horz_filt as 4 tap
if (is_horiz_4tap) {
const int fo_horiz = 1;
const uint8_t *const src_ptr = src - fo_horiz;
if (w <= 8) {
for (i = 0; i < h; i += 2) {
const __m256i data = _mm256_permute2x128_si256(
_mm256_castsi128_si256(
_mm_loadu_si128((__m128i *)(&src_ptr[i * src_stride]))),
_mm256_castsi128_si256(_mm_loadu_si128(
(__m128i *)(&src_ptr[i * src_stride + src_stride]))),
0x20);
/* rounding code */
// 8 bit conversion and saturation to uint8
__m256i res_8b = _mm256_packus_epi16(res_16b, res_16b);
__m256i res_16b = convolve_lowbd_x_4tap(data, coeffs + 1, filt);
const __m128i res_0 = _mm256_castsi256_si128(res_8b);
const __m128i res_1 = _mm256_extracti128_si256(res_8b, 1);
if (w > 4) {
_mm_storel_epi64((__m128i *)&dst[i * dst_stride], res_0);
_mm_storel_epi64((__m128i *)&dst[i * dst_stride + dst_stride], res_1);
} else if (w > 2) {
xx_storel_32(&dst[i * dst_stride], res_0);
xx_storel_32(&dst[i * dst_stride + dst_stride], res_1);
} else {
__m128i *const p_0 = (__m128i *)&dst[i * dst_stride];
__m128i *const p_1 = (__m128i *)&dst[i * dst_stride + dst_stride];
*(uint16_t *)p_0 = _mm_cvtsi128_si32(res_0);
*(uint16_t *)p_1 = _mm_cvtsi128_si32(res_1);
res_16b = _mm256_sra_epi16(_mm256_add_epi16(res_16b, round_0_const),
round_0_shift);
res_16b = _mm256_sra_epi16(_mm256_add_epi16(res_16b, round_const),
round_shift);
/* rounding code */
// 8 bit conversion and saturation to uint8
__m256i res_8b = _mm256_packus_epi16(res_16b, res_16b);
const __m128i res_0 = _mm256_castsi256_si128(res_8b);
const __m128i res_1 = _mm256_extracti128_si256(res_8b, 1);
if (w > 4) {
_mm_storel_epi64((__m128i *)&dst[i * dst_stride], res_0);
_mm_storel_epi64((__m128i *)&dst[i * dst_stride + dst_stride], res_1);
} else if (w > 2) {
xx_storel_32(&dst[i * dst_stride], res_0);
xx_storel_32(&dst[i * dst_stride + dst_stride], res_1);
} else {
__m128i *const p_0 = (__m128i *)&dst[i * dst_stride];
__m128i *const p_1 = (__m128i *)&dst[i * dst_stride + dst_stride];
*(uint16_t *)p_0 = _mm_cvtsi128_si32(res_0);
*(uint16_t *)p_1 = _mm_cvtsi128_si32(res_1);
}
}
} else {
for (i = 0; i < h; ++i) {
for (int j = 0; j < w; j += 16) {
// 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 8 9 10 11 12 13 14 15 16 17
// 18 19 20 21 22 23
const __m256i data = _mm256_inserti128_si256(
_mm256_loadu_si256((__m256i *)&src_ptr[(i * src_stride) + j]),
_mm_loadu_si128((__m128i *)&src_ptr[(i * src_stride) + (j + 8)]),
1);
__m256i res_16b = convolve_lowbd_x_4tap(data, coeffs + 1, filt);
res_16b = _mm256_sra_epi16(_mm256_add_epi16(res_16b, round_0_const),
round_0_shift);
res_16b = _mm256_sra_epi16(_mm256_add_epi16(res_16b, round_const),
round_shift);
/* rounding code */
// 8 bit conversion and saturation to uint8
__m256i res_8b = _mm256_packus_epi16(res_16b, res_16b);
// Store values into the destination buffer
// 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
res_8b = _mm256_permute4x64_epi64(res_8b, 216);
__m128i res = _mm256_castsi256_si128(res_8b);
_mm_storeu_si128((__m128i *)&dst[i * dst_stride + j], res);
}
}
}
} else {
for (i = 0; i < h; ++i) {
for (j = 0; j < w; j += 16) {
// 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 8 9 10 11 12 13 14 15 16 17 18
// 19 20 21 22 23
const __m256i data = _mm256_inserti128_si256(
_mm256_loadu_si256((__m256i *)&src_ptr[(i * src_stride) + j]),
_mm_loadu_si128((__m128i *)&src_ptr[(i * src_stride) + (j + 8)]),
1);
const int fo_horiz = filter_params_x->taps / 2 - 1;
const uint8_t *const src_ptr = src - fo_horiz;
filt[2] = _mm256_load_si256((__m256i const *)(filt_global_avx2 + 32 * 2));
filt[3] = _mm256_load_si256((__m256i const *)(filt_global_avx2 + 32 * 3));
if (w <= 8) {
for (i = 0; i < h; i += 2) {
const __m256i data = _mm256_permute2x128_si256(
_mm256_castsi128_si256(
_mm_loadu_si128((__m128i *)(&src_ptr[i * src_stride]))),
_mm256_castsi128_si256(_mm_loadu_si128(
(__m128i *)(&src_ptr[i * src_stride + src_stride]))),
0x20);
__m256i res_16b = convolve_lowbd_x(data, coeffs, filt);
@ -266,11 +390,49 @@ void av1_convolve_x_sr_avx2(const uint8_t *src, int src_stride, uint8_t *dst,
// 8 bit conversion and saturation to uint8
__m256i res_8b = _mm256_packus_epi16(res_16b, res_16b);
// Store values into the destination buffer
// 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
res_8b = _mm256_permute4x64_epi64(res_8b, 216);
__m128i res = _mm256_castsi256_si128(res_8b);
_mm_storeu_si128((__m128i *)&dst[i * dst_stride + j], res);
const __m128i res_0 = _mm256_castsi256_si128(res_8b);
const __m128i res_1 = _mm256_extracti128_si256(res_8b, 1);
if (w > 4) {
_mm_storel_epi64((__m128i *)&dst[i * dst_stride], res_0);
_mm_storel_epi64((__m128i *)&dst[i * dst_stride + dst_stride], res_1);
} else if (w > 2) {
xx_storel_32(&dst[i * dst_stride], res_0);
xx_storel_32(&dst[i * dst_stride + dst_stride], res_1);
} else {
__m128i *const p_0 = (__m128i *)&dst[i * dst_stride];
__m128i *const p_1 = (__m128i *)&dst[i * dst_stride + dst_stride];
*(uint16_t *)p_0 = _mm_cvtsi128_si32(res_0);
*(uint16_t *)p_1 = _mm_cvtsi128_si32(res_1);
}
}
} else {
for (i = 0; i < h; ++i) {
for (int j = 0; j < w; j += 16) {
// 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 8 9 10 11 12 13 14 15 16 17
// 18 19 20 21 22 23
const __m256i data = _mm256_inserti128_si256(
_mm256_loadu_si256((__m256i *)&src_ptr[(i * src_stride) + j]),
_mm_loadu_si128((__m128i *)&src_ptr[(i * src_stride) + (j + 8)]),
1);
__m256i res_16b = convolve_lowbd_x(data, coeffs, filt);
res_16b = _mm256_sra_epi16(_mm256_add_epi16(res_16b, round_0_const),
round_0_shift);
res_16b = _mm256_sra_epi16(_mm256_add_epi16(res_16b, round_const),
round_shift);
/* rounding code */
// 8 bit conversion and saturation to uint8
__m256i res_8b = _mm256_packus_epi16(res_16b, res_16b);
// Store values into the destination buffer
// 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
res_8b = _mm256_permute4x64_epi64(res_8b, 216);
__m128i res = _mm256_castsi256_si128(res_8b);
_mm_storeu_si128((__m128i *)&dst[i * dst_stride + j], res);
}
}
}
}

18
third_party/aom/av1/common/x86/highbd_inv_txfm_sse4.c vendored
View file

@ -5926,28 +5926,16 @@ void av1_highbd_inv_txfm_add_sse4_1(const tran_low_t *input, uint8_t *dest,
assert(av1_ext_tx_used[txfm_param->tx_set_type][txfm_param->tx_type]);
const TX_SIZE tx_size = txfm_param->tx_size;
switch (tx_size) {
case TX_8X8:
av1_highbd_inv_txfm_add_8x8_sse4_1(input, dest, stride, txfm_param);
break;
case TX_4X8:
av1_highbd_inv_txfm_add_4x8_sse4_1(input, dest, stride, txfm_param);
break;
case TX_8X4:
av1_highbd_inv_txfm_add_8x4_sse4_1(input, dest, stride, txfm_param);
break;
case TX_4X4:
av1_highbd_inv_txfm_add_4x4_sse4_1(input, dest, stride, txfm_param);
break;
case TX_16X4:
av1_highbd_inv_txfm_add_16x4_sse4_1(input, dest, stride, txfm_param);
break;
case TX_4X16:
av1_highbd_inv_txfm_add_4x16_sse4_1(input, dest, stride, txfm_param);
break;
default:
av1_highbd_inv_txfm2d_add_universe_sse4_1(
input, dest, stride, txfm_param->tx_type, tx_size, txfm_param->eob,
txfm_param->bd);
// TODO(http://crbug.com/aomedia/2350): the remaining sse4_1 versions
// cause test vector mismatches.
av1_highbd_inv_txfm_add_c(input, dest, stride, txfm_param);
break;
}
}

747
third_party/aom/av1/common/x86/jnt_convolve_avx2.c vendored
View file

@ -44,9 +44,7 @@ void av1_dist_wtd_convolve_x_avx2(const uint8_t *src, int src_stride,
CONV_BUF_TYPE *dst = conv_params->dst;
int dst_stride = conv_params->dst_stride;
const int bd = 8;
int i, j;
const int fo_horiz = filter_params_x->taps / 2 - 1;
const uint8_t *const src_ptr = src - fo_horiz;
int i, j, is_horiz_4tap = 0;
const int bits = FILTER_BITS - conv_params->round_1;
const __m256i wt = unpack_weights_avx2(conv_params);
const int do_average = conv_params->do_average;
@ -58,18 +56,10 @@ void av1_dist_wtd_convolve_x_avx2(const uint8_t *src, int src_stride,
const int rounding_shift =
2 * FILTER_BITS - conv_params->round_0 - conv_params->round_1;
const __m256i rounding_const = _mm256_set1_epi16((1 << rounding_shift) >> 1);
__m256i filt[4], coeffs[4];
assert(bits >= 0);
assert(conv_params->round_0 > 0);
filt[0] = _mm256_load_si256((__m256i const *)filt_global_avx2);
filt[1] = _mm256_load_si256((__m256i const *)(filt_global_avx2 + 32));
filt[2] = _mm256_load_si256((__m256i const *)(filt_global_avx2 + 32 * 2));
filt[3] = _mm256_load_si256((__m256i const *)(filt_global_avx2 + 32 * 3));
prepare_coeffs_lowbd(filter_params_x, subpel_x_q4, coeffs);
const __m256i round_const =
_mm256_set1_epi16((1 << (conv_params->round_0 - 1)) >> 1);
const __m128i round_shift = _mm_cvtsi32_si128(conv_params->round_0 - 1);
@ -77,51 +67,121 @@ void av1_dist_wtd_convolve_x_avx2(const uint8_t *src, int src_stride,
(void)filter_params_y;
(void)subpel_y_q4;
for (i = 0; i < h; i += 2) {
const uint8_t *src_data = src_ptr + i * src_stride;
CONV_BUF_TYPE *dst_data = dst + i * dst_stride;
for (j = 0; j < w; j += 8) {
const __m256i data =
load_line2_avx2(&src_data[j], &src_data[j + src_stride]);
__m256i filt[4], coeffs[4];
__m256i res = convolve_lowbd_x(data, coeffs, filt);
filt[0] = _mm256_load_si256((__m256i const *)filt_global_avx2);
filt[1] = _mm256_load_si256((__m256i const *)(filt_global_avx2 + 32));
res = _mm256_sra_epi16(_mm256_add_epi16(res, round_const), round_shift);
prepare_coeffs_lowbd(filter_params_x, subpel_x_q4, coeffs);
res = _mm256_slli_epi16(res, bits);
// Condition for checking valid horz_filt taps
if (!(_mm256_extract_epi32(_mm256_or_si256(coeffs[0], coeffs[3]), 0)))
is_horiz_4tap = 1;
const __m256i res_unsigned = _mm256_add_epi16(res, offset_const);
// horz_filt as 4 tap
if (is_horiz_4tap) {
const int fo_horiz = 1;
const uint8_t *const src_ptr = src - fo_horiz;
for (i = 0; i < h; i += 2) {
const uint8_t *src_data = src_ptr + i * src_stride;
CONV_BUF_TYPE *dst_data = dst + i * dst_stride;
for (j = 0; j < w; j += 8) {
const __m256i data =
load_line2_avx2(&src_data[j], &src_data[j + src_stride]);
// Accumulate values into the destination buffer
if (do_average) {
const __m256i data_ref_0 =
load_line2_avx2(&dst_data[j], &dst_data[j + dst_stride]);
const __m256i comp_avg_res =
comp_avg(&data_ref_0, &res_unsigned, &wt, use_dist_wtd_comp_avg);
__m256i res = convolve_lowbd_x_4tap(data, coeffs + 1, filt);
res = _mm256_sra_epi16(_mm256_add_epi16(res, round_const), round_shift);
res = _mm256_slli_epi16(res, bits);
const __m256i round_result = convolve_rounding(
&comp_avg_res, &offset_const, &rounding_const, rounding_shift);
const __m256i res_unsigned = _mm256_add_epi16(res, offset_const);
const __m256i res_8 = _mm256_packus_epi16(round_result, round_result);
const __m128i res_0 = _mm256_castsi256_si128(res_8);
const __m128i res_1 = _mm256_extracti128_si256(res_8, 1);
// Accumulate values into the destination buffer
if (do_average) {
const __m256i data_ref_0 =
load_line2_avx2(&dst_data[j], &dst_data[j + dst_stride]);
const __m256i comp_avg_res =
comp_avg(&data_ref_0, &res_unsigned, &wt, use_dist_wtd_comp_avg);
if (w > 4) {
_mm_storel_epi64((__m128i *)(&dst0[i * dst_stride0 + j]), res_0);
_mm_storel_epi64(
(__m128i *)((&dst0[i * dst_stride0 + j + dst_stride0])), res_1);
const __m256i round_result = convolve_rounding(
&comp_avg_res, &offset_const, &rounding_const, rounding_shift);
const __m256i res_8 = _mm256_packus_epi16(round_result, round_result);
const __m128i res_0 = _mm256_castsi256_si128(res_8);
const __m128i res_1 = _mm256_extracti128_si256(res_8, 1);
if (w > 4) {
_mm_storel_epi64((__m128i *)(&dst0[i * dst_stride0 + j]), res_0);
_mm_storel_epi64(
(__m128i *)((&dst0[i * dst_stride0 + j + dst_stride0])), res_1);
} else {
*(uint32_t *)(&dst0[i * dst_stride0 + j]) =
_mm_cvtsi128_si32(res_0);
*(uint32_t *)(&dst0[i * dst_stride0 + j + dst_stride0]) =
_mm_cvtsi128_si32(res_1);
}
} else {
*(uint32_t *)(&dst0[i * dst_stride0 + j]) = _mm_cvtsi128_si32(res_0);
*(uint32_t *)(&dst0[i * dst_stride0 + j + dst_stride0]) =
_mm_cvtsi128_si32(res_1);
}
} else {
const __m128i res_0 = _mm256_castsi256_si128(res_unsigned);
_mm_store_si128((__m128i *)(&dst[i * dst_stride + j]), res_0);
const __m128i res_0 = _mm256_castsi256_si128(res_unsigned);
_mm_store_si128((__m128i *)(&dst[i * dst_stride + j]), res_0);
const __m128i res_1 = _mm256_extracti128_si256(res_unsigned, 1);
_mm_store_si128((__m128i *)(&dst[i * dst_stride + j + dst_stride]),
res_1);
const __m128i res_1 = _mm256_extracti128_si256(res_unsigned, 1);
_mm_store_si128((__m128i *)(&dst[i * dst_stride + j + dst_stride]),
res_1);
}
}
}
} else {
const int fo_horiz = filter_params_x->taps / 2 - 1;
const uint8_t *const src_ptr = src - fo_horiz;
filt[2] = _mm256_load_si256((__m256i const *)(filt_global_avx2 + 32 * 2));
filt[3] = _mm256_load_si256((__m256i const *)(filt_global_avx2 + 32 * 3));
for (i = 0; i < h; i += 2) {
const uint8_t *src_data = src_ptr + i * src_stride;
CONV_BUF_TYPE *dst_data = dst + i * dst_stride;
for (j = 0; j < w; j += 8) {
const __m256i data =
load_line2_avx2(&src_data[j], &src_data[j + src_stride]);
__m256i res = convolve_lowbd_x(data, coeffs, filt);
res = _mm256_sra_epi16(_mm256_add_epi16(res, round_const), round_shift);
res = _mm256_slli_epi16(res, bits);
const __m256i res_unsigned = _mm256_add_epi16(res, offset_const);
// Accumulate values into the destination buffer
if (do_average) {
const __m256i data_ref_0 =
load_line2_avx2(&dst_data[j], &dst_data[j + dst_stride]);
const __m256i comp_avg_res =
comp_avg(&data_ref_0, &res_unsigned, &wt, use_dist_wtd_comp_avg);
const __m256i round_result = convolve_rounding(
&comp_avg_res, &offset_const, &rounding_const, rounding_shift);
const __m256i res_8 = _mm256_packus_epi16(round_result, round_result);
const __m128i res_0 = _mm256_castsi256_si128(res_8);
const __m128i res_1 = _mm256_extracti128_si256(res_8, 1);
if (w > 4) {
_mm_storel_epi64((__m128i *)(&dst0[i * dst_stride0 + j]), res_0);
_mm_storel_epi64(
(__m128i *)((&dst0[i * dst_stride0 + j + dst_stride0])), res_1);
} else {
*(uint32_t *)(&dst0[i * dst_stride0 + j]) =
_mm_cvtsi128_si32(res_0);
*(uint32_t *)(&dst0[i * dst_stride0 + j + dst_stride0]) =
_mm_cvtsi128_si32(res_1);
}
} else {
const __m128i res_0 = _mm256_castsi256_si128(res_unsigned);
_mm_store_si128((__m128i *)(&dst[i * dst_stride + j]), res_0);
const __m128i res_1 = _mm256_extracti128_si256(res_unsigned, 1);
_mm_store_si128((__m128i *)(&dst[i * dst_stride + j + dst_stride]),
res_1);
}
}
}
}
@ -136,9 +196,7 @@ void av1_dist_wtd_convolve_y_avx2(const uint8_t *src, int src_stride,
CONV_BUF_TYPE *dst = conv_params->dst;
int dst_stride = conv_params->dst_stride;
const int bd = 8;
int i, j;
const int fo_vert = filter_params_y->taps / 2 - 1;
const uint8_t *const src_ptr = src - fo_vert * src_stride;
int i, j, is_vert_4tap = 0;
// +1 to compensate for dividing the filter coeffs by 2
const int left_shift = FILTER_BITS - conv_params->round_0 + 1;
const __m256i round_const =
@ -168,171 +226,368 @@ void av1_dist_wtd_convolve_y_avx2(const uint8_t *src, int src_stride,
(void)filter_params_x;
(void)subpel_x_q4;
for (j = 0; j < w; j += 16) {
const uint8_t *data = &src_ptr[j];
__m256i src6;
// Load lines a and b. Line a to lower 128, line b to upper 128
{
__m256i src_ab[7];
__m256i src_a[7];
src_a[0] = _mm256_castsi128_si256(_mm_loadu_si128((__m128i *)data));
for (int kk = 0; kk < 6; ++kk) {
data += src_stride;
src_a[kk + 1] =
_mm256_castsi128_si256(_mm_loadu_si128((__m128i *)data));
src_ab[kk] = _mm256_permute2x128_si256(src_a[kk], src_a[kk + 1], 0x20);
// Condition for checking valid vert_filt taps
if (!(_mm256_extract_epi32(_mm256_or_si256(coeffs[0], coeffs[3]), 0)))
is_vert_4tap = 1;
if (is_vert_4tap) {
const int fo_vert = 1;
const uint8_t *const src_ptr = src - fo_vert * src_stride;
for (j = 0; j < w; j += 16) {
const uint8_t *data = &src_ptr[j];
__m256i src4;
// Load lines a and b. Line a to lower 128, line b to upper 128
{
__m256i src_ab[4];
__m256i src_a[5];
src_a[0] = _mm256_castsi128_si256(_mm_loadu_si128((__m128i *)data));
for (int kk = 0; kk < 4; ++kk) {
data += src_stride;
src_a[kk + 1] =
_mm256_castsi128_si256(_mm_loadu_si128((__m128i *)data));
src_ab[kk] =
_mm256_permute2x128_si256(src_a[kk], src_a[kk + 1], 0x20);
}
src4 = src_a[4];
s[0] = _mm256_unpacklo_epi8(src_ab[0], src_ab[1]);
s[1] = _mm256_unpacklo_epi8(src_ab[2], src_ab[3]);
s[3] = _mm256_unpackhi_epi8(src_ab[0], src_ab[1]);
s[4] = _mm256_unpackhi_epi8(src_ab[2], src_ab[3]);
}
src6 = src_a[6];
s[0] = _mm256_unpacklo_epi8(src_ab[0], src_ab[1]);
s[1] = _mm256_unpacklo_epi8(src_ab[2], src_ab[3]);
s[2] = _mm256_unpacklo_epi8(src_ab[4], src_ab[5]);
s[4] = _mm256_unpackhi_epi8(src_ab[0], src_ab[1]);
s[5] = _mm256_unpackhi_epi8(src_ab[2], src_ab[3]);
s[6] = _mm256_unpackhi_epi8(src_ab[4], src_ab[5]);
}
for (i = 0; i < h; i += 2) {
data = &src_ptr[(i + 7) * src_stride + j];
const __m256i src7 =
_mm256_castsi128_si256(_mm_loadu_si128((__m128i *)data));
const __m256i src_67a = _mm256_permute2x128_si256(src6, src7, 0x20);
for (i = 0; i < h; i += 2) {
data = &src_ptr[(i + 5) * src_stride + j];
const __m256i src5 =
_mm256_castsi128_si256(_mm_loadu_si128((__m128i *)data));
const __m256i src_45a = _mm256_permute2x128_si256(src4, src5, 0x20);
src6 = _mm256_castsi128_si256(
_mm_loadu_si128((__m128i *)(data + src_stride)));
const __m256i src_78a = _mm256_permute2x128_si256(src7, src6, 0x20);
src4 = _mm256_castsi128_si256(
_mm_loadu_si128((__m128i *)(data + src_stride)));
const __m256i src_56a = _mm256_permute2x128_si256(src5, src4, 0x20);
s[3] = _mm256_unpacklo_epi8(src_67a, src_78a);
s[7] = _mm256_unpackhi_epi8(src_67a, src_78a);
s[2] = _mm256_unpacklo_epi8(src_45a, src_56a);
s[5] = _mm256_unpackhi_epi8(src_45a, src_56a);
__m256i res_lo = convolve_lowbd(s, coeffs);
__m256i res_lo = convolve_lowbd_4tap(s, coeffs + 1);
res_lo = _mm256_add_epi16(res_lo, offset_const_1);
res_lo = _mm256_add_epi16(res_lo, offset_const_1);
const __m256i res_lo_0_32b = _mm256_unpacklo_epi16(res_lo, zero);
const __m256i res_lo_0_shift =
_mm256_slli_epi32(res_lo_0_32b, left_shift);
const __m256i res_lo_0_round = _mm256_sra_epi32(
_mm256_add_epi32(res_lo_0_shift, round_const), round_shift);
const __m256i res_lo_0_32b = _mm256_unpacklo_epi16(res_lo, zero);
const __m256i res_lo_0_shift =
_mm256_slli_epi32(res_lo_0_32b, left_shift);
const __m256i res_lo_0_round = _mm256_sra_epi32(
_mm256_add_epi32(res_lo_0_shift, round_const), round_shift);
const __m256i res_lo_1_32b = _mm256_unpackhi_epi16(res_lo, zero);
const __m256i res_lo_1_shift =
_mm256_slli_epi32(res_lo_1_32b, left_shift);
const __m256i res_lo_1_round = _mm256_sra_epi32(
_mm256_add_epi32(res_lo_1_shift, round_const), round_shift);
const __m256i res_lo_1_32b = _mm256_unpackhi_epi16(res_lo, zero);
const __m256i res_lo_1_shift =
_mm256_slli_epi32(res_lo_1_32b, left_shift);
const __m256i res_lo_1_round = _mm256_sra_epi32(
_mm256_add_epi32(res_lo_1_shift, round_const), round_shift);
const __m256i res_lo_round =
_mm256_packs_epi32(res_lo_0_round, res_lo_1_round);
const __m256i res_lo_round =
_mm256_packs_epi32(res_lo_0_round, res_lo_1_round);
const __m256i res_lo_unsigned =
_mm256_add_epi16(res_lo_round, offset_const_2);
const __m256i res_lo_unsigned =
_mm256_add_epi16(res_lo_round, offset_const_2);
if (w - j < 16) {
if (do_average) {
const __m256i data_ref_0 = load_line2_avx2(
&dst[i * dst_stride + j], &dst[i * dst_stride + j + dst_stride]);
const __m256i comp_avg_res = comp_avg(&data_ref_0, &res_lo_unsigned,
&wt, use_dist_wtd_comp_avg);
if (w - j < 16) {
if (do_average) {
const __m256i data_ref_0 =
load_line2_avx2(&dst[i * dst_stride + j],
&dst[i * dst_stride + j + dst_stride]);
const __m256i comp_avg_res = comp_avg(&data_ref_0, &res_lo_unsigned,
&wt, use_dist_wtd_comp_avg);
const __m256i round_result = convolve_rounding(
&comp_avg_res, &offset_const, &rounding_const, rounding_shift);
const __m256i round_result = convolve_rounding(
&comp_avg_res, &offset_const, &rounding_const, rounding_shift);
const __m256i res_8 = _mm256_packus_epi16(round_result, round_result);
const __m128i res_0 = _mm256_castsi256_si128(res_8);
const __m128i res_1 = _mm256_extracti128_si256(res_8, 1);
const __m256i res_8 =
_mm256_packus_epi16(round_result, round_result);
const __m128i res_0 = _mm256_castsi256_si128(res_8);
const __m128i res_1 = _mm256_extracti128_si256(res_8, 1);
if (w - j > 4) {
_mm_storel_epi64((__m128i *)(&dst0[i * dst_stride0 + j]), res_0);
_mm_storel_epi64(
(__m128i *)((&dst0[i * dst_stride0 + j + dst_stride0])), res_1);
if (w - j > 4) {
_mm_storel_epi64((__m128i *)(&dst0[i * dst_stride0 + j]), res_0);
_mm_storel_epi64(
(__m128i *)((&dst0[i * dst_stride0 + j + dst_stride0])),
res_1);
} else {
*(uint32_t *)(&dst0[i * dst_stride0 + j]) =
_mm_cvtsi128_si32(res_0);
*(uint32_t *)(&dst0[i * dst_stride0 + j + dst_stride0]) =
_mm_cvtsi128_si32(res_1);
}
} else {
*(uint32_t *)(&dst0[i * dst_stride0 + j]) =
_mm_cvtsi128_si32(res_0);
*(uint32_t *)(&dst0[i * dst_stride0 + j + dst_stride0]) =
_mm_cvtsi128_si32(res_1);
const __m128i res_0 = _mm256_castsi256_si128(res_lo_unsigned);
_mm_store_si128((__m128i *)(&dst[i * dst_stride + j]), res_0);
const __m128i res_1 = _mm256_extracti128_si256(res_lo_unsigned, 1);
_mm_store_si128((__m128i *)(&dst[i * dst_stride + j + dst_stride]),
res_1);
}
} else {
const __m128i res_0 = _mm256_castsi256_si128(res_lo_unsigned);
_mm_store_si128((__m128i *)(&dst[i * dst_stride + j]), res_0);
__m256i res_hi = convolve_lowbd_4tap(s + 3, coeffs + 1);
const __m128i res_1 = _mm256_extracti128_si256(res_lo_unsigned, 1);
_mm_store_si128((__m128i *)(&dst[i * dst_stride + j + dst_stride]),
res_1);
res_hi = _mm256_add_epi16(res_hi, offset_const_1);
const __m256i res_hi_0_32b = _mm256_unpacklo_epi16(res_hi, zero);
const __m256i res_hi_0_shift =
_mm256_slli_epi32(res_hi_0_32b, left_shift);
const __m256i res_hi_0_round = _mm256_sra_epi32(
_mm256_add_epi32(res_hi_0_shift, round_const), round_shift);
const __m256i res_hi_1_32b = _mm256_unpackhi_epi16(res_hi, zero);
const __m256i res_hi_1_shift =
_mm256_slli_epi32(res_hi_1_32b, left_shift);
const __m256i res_hi_1_round = _mm256_sra_epi32(
_mm256_add_epi32(res_hi_1_shift, round_const), round_shift);
const __m256i res_hi_round =
_mm256_packs_epi32(res_hi_0_round, res_hi_1_round);
const __m256i res_hi_unsigned =
_mm256_add_epi16(res_hi_round, offset_const_2);
if (do_average) {
const __m256i data_ref_0_lo =
load_line2_avx2(&dst[i * dst_stride + j],
&dst[i * dst_stride + j + dst_stride]);
const __m256i data_ref_0_hi =
load_line2_avx2(&dst[i * dst_stride + j + 8],
&dst[i * dst_stride + j + 8 + dst_stride]);
const __m256i comp_avg_res_lo = comp_avg(
&data_ref_0_lo, &res_lo_unsigned, &wt, use_dist_wtd_comp_avg);
const __m256i comp_avg_res_hi = comp_avg(
&data_ref_0_hi, &res_hi_unsigned, &wt, use_dist_wtd_comp_avg);
const __m256i round_result_lo =
convolve_rounding(&comp_avg_res_lo, &offset_const,
&rounding_const, rounding_shift);
const __m256i round_result_hi =
convolve_rounding(&comp_avg_res_hi, &offset_const,
&rounding_const, rounding_shift);
const __m256i res_8 =
_mm256_packus_epi16(round_result_lo, round_result_hi);
const __m128i res_0 = _mm256_castsi256_si128(res_8);
const __m128i res_1 = _mm256_extracti128_si256(res_8, 1);
_mm_store_si128((__m128i *)(&dst0[i * dst_stride0 + j]), res_0);
_mm_store_si128(
(__m128i *)((&dst0[i * dst_stride0 + j + dst_stride0])), res_1);
} else {
const __m128i res_lo_0 = _mm256_castsi256_si128(res_lo_unsigned);
_mm_store_si128((__m128i *)(&dst[i * dst_stride + j]), res_lo_0);
const __m128i res_lo_1 =
_mm256_extracti128_si256(res_lo_unsigned, 1);
_mm_store_si128((__m128i *)(&dst[i * dst_stride + j + dst_stride]),
res_lo_1);
const __m128i res_hi_0 = _mm256_castsi256_si128(res_hi_unsigned);
_mm_store_si128((__m128i *)(&dst[i * dst_stride + j + 8]),
res_hi_0);
const __m128i res_hi_1 =
_mm256_extracti128_si256(res_hi_unsigned, 1);
_mm_store_si128(
(__m128i *)(&dst[i * dst_stride + j + 8 + dst_stride]),
res_hi_1);
}
}
} else {
__m256i res_hi = convolve_lowbd(s + 4, coeffs);
s[0] = s[1];
s[1] = s[2];
res_hi = _mm256_add_epi16(res_hi, offset_const_1);
const __m256i res_hi_0_32b = _mm256_unpacklo_epi16(res_hi, zero);
const __m256i res_hi_0_shift =
_mm256_slli_epi32(res_hi_0_32b, left_shift);
const __m256i res_hi_0_round = _mm256_sra_epi32(
_mm256_add_epi32(res_hi_0_shift, round_const), round_shift);
const __m256i res_hi_1_32b = _mm256_unpackhi_epi16(res_hi, zero);
const __m256i res_hi_1_shift =
_mm256_slli_epi32(res_hi_1_32b, left_shift);
const __m256i res_hi_1_round = _mm256_sra_epi32(
_mm256_add_epi32(res_hi_1_shift, round_const), round_shift);
const __m256i res_hi_round =
_mm256_packs_epi32(res_hi_0_round, res_hi_1_round);
const __m256i res_hi_unsigned =
_mm256_add_epi16(res_hi_round, offset_const_2);
if (do_average) {
const __m256i data_ref_0_lo = load_line2_avx2(
&dst[i * dst_stride + j], &dst[i * dst_stride + j + dst_stride]);
const __m256i data_ref_0_hi =
load_line2_avx2(&dst[i * dst_stride + j + 8],
&dst[i * dst_stride + j + 8 + dst_stride]);
const __m256i comp_avg_res_lo = comp_avg(
&data_ref_0_lo, &res_lo_unsigned, &wt, use_dist_wtd_comp_avg);
const __m256i comp_avg_res_hi = comp_avg(
&data_ref_0_hi, &res_hi_unsigned, &wt, use_dist_wtd_comp_avg);
const __m256i round_result_lo = convolve_rounding(
&comp_avg_res_lo, &offset_const, &rounding_const, rounding_shift);
const __m256i round_result_hi = convolve_rounding(
&comp_avg_res_hi, &offset_const, &rounding_const, rounding_shift);
const __m256i res_8 =
_mm256_packus_epi16(round_result_lo, round_result_hi);
const __m128i res_0 = _mm256_castsi256_si128(res_8);
const __m128i res_1 = _mm256_extracti128_si256(res_8, 1);
_mm_store_si128((__m128i *)(&dst0[i * dst_stride0 + j]), res_0);
_mm_store_si128(
(__m128i *)((&dst0[i * dst_stride0 + j + dst_stride0])), res_1);
} else {
const __m128i res_lo_0 = _mm256_castsi256_si128(res_lo_unsigned);
_mm_store_si128((__m128i *)(&dst[i * dst_stride + j]), res_lo_0);
const __m128i res_lo_1 = _mm256_extracti128_si256(res_lo_unsigned, 1);
_mm_store_si128((__m128i *)(&dst[i * dst_stride + j + dst_stride]),
res_lo_1);
const __m128i res_hi_0 = _mm256_castsi256_si128(res_hi_unsigned);
_mm_store_si128((__m128i *)(&dst[i * dst_stride + j + 8]), res_hi_0);
const __m128i res_hi_1 = _mm256_extracti128_si256(res_hi_unsigned, 1);
_mm_store_si128(
(__m128i *)(&dst[i * dst_stride + j + 8 + dst_stride]), res_hi_1);
}
s[3] = s[4];
s[4] = s[5];
}
}
} else {
const int fo_vert = filter_params_y->taps / 2 - 1;
const uint8_t *const src_ptr = src - fo_vert * src_stride;
for (j = 0; j < w; j += 16) {
const uint8_t *data = &src_ptr[j];
__m256i src6;
// Load lines a and b. Line a to lower 128, line b to upper 128
{
__m256i src_ab[7];
__m256i src_a[7];
src_a[0] = _mm256_castsi128_si256(_mm_loadu_si128((__m128i *)data));
for (int kk = 0; kk < 6; ++kk) {
data += src_stride;
src_a[kk + 1] =
_mm256_castsi128_si256(_mm_loadu_si128((__m128i *)data));
src_ab[kk] =
_mm256_permute2x128_si256(src_a[kk], src_a[kk + 1], 0x20);
}
src6 = src_a[6];
s[0] = _mm256_unpacklo_epi8(src_ab[0], src_ab[1]);
s[1] = _mm256_unpacklo_epi8(src_ab[2], src_ab[3]);
s[2] = _mm256_unpacklo_epi8(src_ab[4], src_ab[5]);
s[4] = _mm256_unpackhi_epi8(src_ab[0], src_ab[1]);
s[5] = _mm256_unpackhi_epi8(src_ab[2], src_ab[3]);
s[6] = _mm256_unpackhi_epi8(src_ab[4], src_ab[5]);
}
s[0] = s[1];
s[1] = s[2];
s[2] = s[3];
s[4] = s[5];
s[5] = s[6];
s[6] = s[7];
for (i = 0; i < h; i += 2) {
data = &src_ptr[(i + 7) * src_stride + j];
const __m256i src7 =
_mm256_castsi128_si256(_mm_loadu_si128((__m128i *)data));
const __m256i src_67a = _mm256_permute2x128_si256(src6, src7, 0x20);
src6 = _mm256_castsi128_si256(
_mm_loadu_si128((__m128i *)(data + src_stride)));
const __m256i src_78a = _mm256_permute2x128_si256(src7, src6, 0x20);
s[3] = _mm256_unpacklo_epi8(src_67a, src_78a);
s[7] = _mm256_unpackhi_epi8(src_67a, src_78a);
__m256i res_lo = convolve_lowbd(s, coeffs);
res_lo = _mm256_add_epi16(res_lo, offset_const_1);
const __m256i res_lo_0_32b = _mm256_unpacklo_epi16(res_lo, zero);
const __m256i res_lo_0_shift =
_mm256_slli_epi32(res_lo_0_32b, left_shift);
const __m256i res_lo_0_round = _mm256_sra_epi32(
_mm256_add_epi32(res_lo_0_shift, round_const), round_shift);
const __m256i res_lo_1_32b = _mm256_unpackhi_epi16(res_lo, zero);
const __m256i res_lo_1_shift =
_mm256_slli_epi32(res_lo_1_32b, left_shift);
const __m256i res_lo_1_round = _mm256_sra_epi32(
_mm256_add_epi32(res_lo_1_shift, round_const), round_shift);
const __m256i res_lo_round =
_mm256_packs_epi32(res_lo_0_round, res_lo_1_round);
const __m256i res_lo_unsigned =
_mm256_add_epi16(res_lo_round, offset_const_2);
if (w - j < 16) {
if (do_average) {
const __m256i data_ref_0 =
load_line2_avx2(&dst[i * dst_stride + j],
&dst[i * dst_stride + j + dst_stride]);
const __m256i comp_avg_res = comp_avg(&data_ref_0, &res_lo_unsigned,
&wt, use_dist_wtd_comp_avg);
const __m256i round_result = convolve_rounding(
&comp_avg_res, &offset_const, &rounding_const, rounding_shift);
const __m256i res_8 =
_mm256_packus_epi16(round_result, round_result);
const __m128i res_0 = _mm256_castsi256_si128(res_8);
const __m128i res_1 = _mm256_extracti128_si256(res_8, 1);
if (w - j > 4) {
_mm_storel_epi64((__m128i *)(&dst0[i * dst_stride0 + j]), res_0);
_mm_storel_epi64(
(__m128i *)((&dst0[i * dst_stride0 + j + dst_stride0])),
res_1);
} else {
*(uint32_t *)(&dst0[i * dst_stride0 + j]) =
_mm_cvtsi128_si32(res_0);
*(uint32_t *)(&dst0[i * dst_stride0 + j + dst_stride0]) =
_mm_cvtsi128_si32(res_1);
}
} else {
const __m128i res_0 = _mm256_castsi256_si128(res_lo_unsigned);
_mm_store_si128((__m128i *)(&dst[i * dst_stride + j]), res_0);
const __m128i res_1 = _mm256_extracti128_si256(res_lo_unsigned, 1);
_mm_store_si128((__m128i *)(&dst[i * dst_stride + j + dst_stride]),
res_1);
}
} else {
__m256i res_hi = convolve_lowbd(s + 4, coeffs);
res_hi = _mm256_add_epi16(res_hi, offset_const_1);
const __m256i res_hi_0_32b = _mm256_unpacklo_epi16(res_hi, zero);
const __m256i res_hi_0_shift =
_mm256_slli_epi32(res_hi_0_32b, left_shift);
const __m256i res_hi_0_round = _mm256_sra_epi32(
_mm256_add_epi32(res_hi_0_shift, round_const), round_shift);
const __m256i res_hi_1_32b = _mm256_unpackhi_epi16(res_hi, zero);
const __m256i res_hi_1_shift =
_mm256_slli_epi32(res_hi_1_32b, left_shift);
const __m256i res_hi_1_round = _mm256_sra_epi32(
_mm256_add_epi32(res_hi_1_shift, round_const), round_shift);
const __m256i res_hi_round =
_mm256_packs_epi32(res_hi_0_round, res_hi_1_round);
const __m256i res_hi_unsigned =
_mm256_add_epi16(res_hi_round, offset_const_2);
if (do_average) {
const __m256i data_ref_0_lo =
load_line2_avx2(&dst[i * dst_stride + j],
&dst[i * dst_stride + j + dst_stride]);
const __m256i data_ref_0_hi =
load_line2_avx2(&dst[i * dst_stride + j + 8],
&dst[i * dst_stride + j + 8 + dst_stride]);
const __m256i comp_avg_res_lo = comp_avg(
&data_ref_0_lo, &res_lo_unsigned, &wt, use_dist_wtd_comp_avg);
const __m256i comp_avg_res_hi = comp_avg(
&data_ref_0_hi, &res_hi_unsigned, &wt, use_dist_wtd_comp_avg);
const __m256i round_result_lo =
convolve_rounding(&comp_avg_res_lo, &offset_const,
&rounding_const, rounding_shift);
const __m256i round_result_hi =
convolve_rounding(&comp_avg_res_hi, &offset_const,
&rounding_const, rounding_shift);
const __m256i res_8 =
_mm256_packus_epi16(round_result_lo, round_result_hi);
const __m128i res_0 = _mm256_castsi256_si128(res_8);
const __m128i res_1 = _mm256_extracti128_si256(res_8, 1);
_mm_store_si128((__m128i *)(&dst0[i * dst_stride0 + j]), res_0);
_mm_store_si128(
(__m128i *)((&dst0[i * dst_stride0 + j + dst_stride0])), res_1);
} else {
const __m128i res_lo_0 = _mm256_castsi256_si128(res_lo_unsigned);
_mm_store_si128((__m128i *)(&dst[i * dst_stride + j]), res_lo_0);
const __m128i res_lo_1 =
_mm256_extracti128_si256(res_lo_unsigned, 1);
_mm_store_si128((__m128i *)(&dst[i * dst_stride + j + dst_stride]),
res_lo_1);
const __m128i res_hi_0 = _mm256_castsi256_si128(res_hi_unsigned);
_mm_store_si128((__m128i *)(&dst[i * dst_stride + j + 8]),
res_hi_0);
const __m128i res_hi_1 =
_mm256_extracti128_si256(res_hi_unsigned, 1);
_mm_store_si128(
(__m128i *)(&dst[i * dst_stride + j + 8 + dst_stride]),
res_hi_1);
}
}
s[0] = s[1];
s[1] = s[2];
s[2] = s[3];
s[4] = s[5];
s[5] = s[6];
s[6] = s[7];
}
}
}
}
@ -348,12 +603,9 @@ void av1_dist_wtd_convolve_2d_avx2(const uint8_t *src, int src_stride,
const int bd = 8;
DECLARE_ALIGNED(32, int16_t, im_block[(MAX_SB_SIZE + MAX_FILTER_TAP) * 8]);
int im_h = h + filter_params_y->taps - 1;
int im_stride = 8;
int i, j;
const int fo_vert = filter_params_y->taps / 2 - 1;
const int fo_horiz = filter_params_x->taps / 2 - 1;
const uint8_t *const src_ptr = src - fo_vert * src_stride - fo_horiz;
int i, is_horiz_4tap = 0, is_vert_4tap = 0;
const __m256i wt = unpack_weights_avx2(conv_params);
const int do_average = conv_params->do_average;
const int use_dist_wtd_comp_avg = conv_params->use_dist_wtd_comp_avg;
@ -364,18 +616,9 @@ void av1_dist_wtd_convolve_2d_avx2(const uint8_t *src, int src_stride,
const int rounding_shift =
2 * FILTER_BITS - conv_params->round_0 - conv_params->round_1;
const __m256i rounding_const = _mm256_set1_epi16((1 << rounding_shift) >> 1);
__m256i filt[4], s[8], coeffs_x[4], coeffs_y[4];
assert(conv_params->round_0 > 0);
filt[0] = _mm256_load_si256((__m256i const *)filt_global_avx2);
filt[1] = _mm256_load_si256((__m256i const *)(filt_global_avx2 + 32));
filt[2] = _mm256_load_si256((__m256i const *)(filt_global_avx2 + 32 * 2));
filt[3] = _mm256_load_si256((__m256i const *)(filt_global_avx2 + 32 * 3));
prepare_coeffs_lowbd(filter_params_x, subpel_x_q4, coeffs_x);
prepare_coeffs(filter_params_y, subpel_y_q4, coeffs_y);
const __m256i round_const_h = _mm256_set1_epi16(
((1 << (conv_params->round_0 - 1)) >> 1) + (1 << (bd + FILTER_BITS - 2)));
const __m128i round_shift_h = _mm_cvtsi32_si128(conv_params->round_0 - 1);
@ -385,9 +628,29 @@ void av1_dist_wtd_convolve_2d_avx2(const uint8_t *src, int src_stride,
(1 << (bd + 2 * FILTER_BITS - conv_params->round_0 - 1)));
const __m128i round_shift_v = _mm_cvtsi32_si128(conv_params->round_1);
for (j = 0; j < w; j += 8) {
/* Horizontal filter */
{
__m256i filt[4], coeffs_x[4], coeffs_y[4];
filt[0] = _mm256_load_si256((__m256i const *)filt_global_avx2);
filt[1] = _mm256_load_si256((__m256i const *)(filt_global_avx2 + 32));
prepare_coeffs_lowbd(filter_params_x, subpel_x_q4, coeffs_x);
prepare_coeffs(filter_params_y, subpel_y_q4, coeffs_y);
// Condition for checking valid horz_filt taps
if (!(_mm256_extract_epi32(_mm256_or_si256(coeffs_x[0], coeffs_x[3]), 0)))
is_horiz_4tap = 1;
// Condition for checking valid vert_filt taps
if (!(_mm256_extract_epi32(_mm256_or_si256(coeffs_y[0], coeffs_y[3]), 0)))
is_vert_4tap = 1;
if (is_horiz_4tap) {
int im_h = h + filter_params_y->taps - 1;
const int fo_vert = filter_params_y->taps / 2 - 1;
const int fo_horiz = 1;
const uint8_t *const src_ptr = src - fo_vert * src_stride - fo_horiz;
for (int j = 0; j < w; j += 8) {
/* Horizontal filter */
const uint8_t *src_h = src_ptr + j;
for (i = 0; i < im_h; i += 2) {
__m256i data =
@ -396,49 +659,59 @@ void av1_dist_wtd_convolve_2d_avx2(const uint8_t *src, int src_stride,
data = _mm256_inserti128_si256(
data, _mm_loadu_si128((__m128i *)(src_h + src_stride)), 1);
src_h += (src_stride << 1);
__m256i res = convolve_lowbd_x(data, coeffs_x, filt);
__m256i res = convolve_lowbd_x_4tap(data, coeffs_x + 1, filt);
res = _mm256_sra_epi16(_mm256_add_epi16(res, round_const_h),
round_shift_h);
_mm256_store_si256((__m256i *)&im_block[i * im_stride], res);
}
DIST_WTD_CONVOLVE_VERTICAL_FILTER_8TAP;
}
} else if (is_vert_4tap) {
int im_h = h + 3;
const int fo_vert = 1;
const int fo_horiz = filter_params_x->taps / 2 - 1;
const uint8_t *const src_ptr = src - fo_vert * src_stride - fo_horiz;
/* Vertical filter */
{
filt[2] = _mm256_load_si256((__m256i const *)(filt_global_avx2 + 32 * 2));
filt[3] = _mm256_load_si256((__m256i const *)(filt_global_avx2 + 32 * 3));
for (int j = 0; j < w; j += 8) {
/* Horizontal filter */
const uint8_t *src_h = src_ptr + j;
DIST_WTD_CONVOLVE_HORIZONTAL_FILTER_8TAP;
/* Vertical filter */
__m256i s[6];
__m256i s0 = _mm256_loadu_si256((__m256i *)(im_block + 0 * im_stride));
__m256i s1 = _mm256_loadu_si256((__m256i *)(im_block + 1 * im_stride));
__m256i s2 = _mm256_loadu_si256((__m256i *)(im_block + 2 * im_stride));
__m256i s3 = _mm256_loadu_si256((__m256i *)(im_block + 3 * im_stride));
__m256i s4 = _mm256_loadu_si256((__m256i *)(im_block + 4 * im_stride));
__m256i s5 = _mm256_loadu_si256((__m256i *)(im_block + 5 * im_stride));
s[0] = _mm256_unpacklo_epi16(s0, s1);
s[1] = _mm256_unpacklo_epi16(s2, s3);
s[2] = _mm256_unpacklo_epi16(s4, s5);
s[4] = _mm256_unpackhi_epi16(s0, s1);
s[5] = _mm256_unpackhi_epi16(s2, s3);
s[6] = _mm256_unpackhi_epi16(s4, s5);
s[3] = _mm256_unpackhi_epi16(s0, s1);
s[4] = _mm256_unpackhi_epi16(s2, s3);
for (i = 0; i < h; i += 2) {
const int16_t *data = &im_block[i * im_stride];
const __m256i s6 =
_mm256_loadu_si256((__m256i *)(data + 6 * im_stride));
const __m256i s7 =
_mm256_loadu_si256((__m256i *)(data + 7 * im_stride));
const __m256i s4 =
_mm256_loadu_si256((__m256i *)(data + 4 * im_stride));
const __m256i s5 =
_mm256_loadu_si256((__m256i *)(data + 5 * im_stride));
s[3] = _mm256_unpacklo_epi16(s6, s7);
s[7] = _mm256_unpackhi_epi16(s6, s7);
s[2] = _mm256_unpacklo_epi16(s4, s5);
s[5] = _mm256_unpackhi_epi16(s4, s5);
const __m256i res_a = convolve(s, coeffs_y);
const __m256i res_a = convolve_4tap(s, coeffs_y + 1);
const __m256i res_a_round = _mm256_sra_epi32(
_mm256_add_epi32(res_a, round_const_v), round_shift_v);
if (w - j > 4) {
const __m256i res_b = convolve(s + 4, coeffs_y);
const __m256i res_b = convolve_4tap(s + 3, coeffs_y + 1);
const __m256i res_b_round = _mm256_sra_epi32(
_mm256_add_epi32(res_b, round_const_v), round_shift_v);
const __m256i res_16b = _mm256_packs_epi32(res_a_round, res_b_round);
@ -504,16 +777,28 @@ void av1_dist_wtd_convolve_2d_avx2(const uint8_t *src, int src_stride,
res_1);
}
}
s[0] = s[1];
s[1] = s[2];
s[2] = s[3];
s[3] = s[4];
s[4] = s[5];
s[5] = s[6];
s[6] = s[7];
}
}
} else {
int im_h = h + filter_params_y->taps - 1;
const int fo_vert = filter_params_y->taps / 2 - 1;
const int fo_horiz = filter_params_x->taps / 2 - 1;
const uint8_t *const src_ptr = src - fo_vert * src_stride - fo_horiz;
filt[2] = _mm256_load_si256((__m256i const *)(filt_global_avx2 + 32 * 2));
filt[3] = _mm256_load_si256((__m256i const *)(filt_global_avx2 + 32 * 3));
for (int j = 0; j < w; j += 8) {
/* Horizontal filter */
const uint8_t *src_h = src_ptr + j;
DIST_WTD_CONVOLVE_HORIZONTAL_FILTER_8TAP;
DIST_WTD_CONVOLVE_VERTICAL_FILTER_8TAP;
}
}
}

360
third_party/aom/av1/common/x86/wiener_convolve_avx2.c vendored
View file

@ -17,7 +17,6 @@
#include "av1/common/convolve.h"
#include "aom_dsp/aom_dsp_common.h"
#include "aom_dsp/aom_filter.h"
#include "aom_dsp/x86/convolve_avx2.h"
#include "aom_dsp/x86/synonyms.h"
#include "aom_dsp/x86/synonyms_avx2.h"
@ -26,207 +25,236 @@
// on the left.
// A row of, say, 8-bit pixels with values p0, p1, p2, ..., p30, p31 will be
// loaded and stored as [ p31 ... p17 p16 ][ p15 ... p1 p0 ].
// Exploiting the range of wiener filter coefficients,
// horizontal filtering can be done in 16 bit intermediate precision.
// The details are as follows :
// Consider the horizontal wiener filter coefficients of the following form :
// [C0, C1, C2, 2^(FILTER_BITS) -2 * (C0 + C1 + C2), C2, C1, C0]
// Subtracting 2^(FILTER_BITS) from the centre tap we get the following :
// [C0, C1, C2, -2 * (C0 + C1 + C2), C2, C1, C0]
// The sum of the product "C0 * p0 + C1 * p1 + C2 * p2 -2 * (C0 + C1 + C2) * p3
// + C2 * p4 + C1 * p5 + C0 * p6" would be in the range of signed 16 bit
// precision. Finally, after rounding the above result by round_0, we multiply
// the centre pixel by 2^(FILTER_BITS - round_0) and add it to get the
// horizontal filter output.
void av1_wiener_convolve_add_src_avx2(const uint8_t *src, ptrdiff_t src_stride,
uint8_t *dst, ptrdiff_t dst_stride,
const int16_t *filter_x, int x_step_q4,
const int16_t *filter_y, int y_step_q4,
int w, int h,
const ConvolveParams *conv_params) {
const int bd = 8;
assert(x_step_q4 == 16 && y_step_q4 == 16);
assert(!(w & 7));
(void)x_step_q4;
(void)y_step_q4;
DECLARE_ALIGNED(32, int16_t, im_block[(MAX_SB_SIZE + SUBPEL_TAPS) * 8]);
int im_h = h + SUBPEL_TAPS - 2;
int im_stride = 8;
memset(im_block + (im_h * im_stride), 0, MAX_SB_SIZE);
int i, j;
const int center_tap = (SUBPEL_TAPS - 1) / 2;
DECLARE_ALIGNED(32, uint16_t,
temp[(MAX_SB_SIZE + SUBPEL_TAPS - 1) * MAX_SB_SIZE]);
int intermediate_height = h + SUBPEL_TAPS - 2;
memset(temp + (intermediate_height * MAX_SB_SIZE), 0, MAX_SB_SIZE);
const int center_tap = ((SUBPEL_TAPS - 1) / 2);
const uint8_t *const src_ptr = src - center_tap * src_stride - center_tap;
__m256i filt[4], coeffs_h[4], coeffs_v[4], filt_center;
assert(conv_params->round_0 > 0);
filt[0] = _mm256_load_si256((__m256i const *)filt1_global_avx2);
filt[1] = _mm256_load_si256((__m256i const *)filt2_global_avx2);
filt[2] = _mm256_load_si256((__m256i const *)filt3_global_avx2);
filt[3] = _mm256_load_si256((__m256i const *)filt4_global_avx2);
filt_center = _mm256_load_si256((__m256i const *)filt_center_global_avx2);
const __m128i coeffs_x = _mm_loadu_si128((__m128i *)filter_x);
const __m256i filter_coeffs_x = _mm256_broadcastsi128_si256(coeffs_x);
// coeffs 0 1 0 1 0 1 0 1
coeffs_h[0] =
_mm256_shuffle_epi8(filter_coeffs_x, _mm256_set1_epi16(0x0200u));
// coeffs 2 3 2 3 2 3 2 3
coeffs_h[1] =
_mm256_shuffle_epi8(filter_coeffs_x, _mm256_set1_epi16(0x0604u));
// coeffs 4 5 4 5 4 5 4 5
coeffs_h[2] =
_mm256_shuffle_epi8(filter_coeffs_x, _mm256_set1_epi16(0x0a08u));
// coeffs 6 7 6 7 6 7 6 7
coeffs_h[3] =
_mm256_shuffle_epi8(filter_coeffs_x, _mm256_set1_epi16(0x0e0cu));
const __m256i round_const_h =
_mm256_set1_epi16((1 << (conv_params->round_0 - 1)));
const __m128i round_shift_h = _mm_cvtsi32_si128(conv_params->round_0);
const __m128i zero_128 = _mm_setzero_si128();
const __m256i zero_256 = _mm256_setzero_si256();
// Add an offset to account for the "add_src" part of the convolve function.
const __m128i zero_128 = _mm_setzero_si128();
const __m128i offset_0 = _mm_insert_epi16(zero_128, 1 << FILTER_BITS, 3);
const __m128i coeffs_y = _mm_add_epi16(xx_loadu_128(filter_y), offset_0);
const __m128i offset = _mm_insert_epi16(zero_128, 1 << FILTER_BITS, 3);
const __m256i filter_coeffs_y = _mm256_broadcastsi128_si256(coeffs_y);
const __m256i clamp_low = zero_256;
const __m256i clamp_high =
_mm256_set1_epi16(WIENER_CLAMP_LIMIT(conv_params->round_0, bd) - 1);
// coeffs 0 1 0 1 0 1 0 1
coeffs_v[0] = _mm256_shuffle_epi32(filter_coeffs_y, 0x00);
// coeffs 2 3 2 3 2 3 2 3
coeffs_v[1] = _mm256_shuffle_epi32(filter_coeffs_y, 0x55);
// coeffs 4 5 4 5 4 5 4 5
coeffs_v[2] = _mm256_shuffle_epi32(filter_coeffs_y, 0xaa);
// coeffs 6 7 6 7 6 7 6 7
coeffs_v[3] = _mm256_shuffle_epi32(filter_coeffs_y, 0xff);
/* Horizontal filter */
{
// coeffs [ f7 f6 f5 f4 f3 f2 f1 f0 ]
const __m128i coeffs_x = _mm_add_epi16(xx_loadu_128(filter_x), offset);
const __m256i round_const_v =
_mm256_set1_epi32((1 << (conv_params->round_1 - 1)));
const __m128i round_shift_v = _mm_cvtsi32_si128(conv_params->round_1);
// coeffs [ f3 f2 f3 f2 f1 f0 f1 f0 ]
const __m128i coeffs_0123 = _mm_unpacklo_epi32(coeffs_x, coeffs_x);
// coeffs [ f7 f6 f7 f6 f5 f4 f5 f4 ]
const __m128i coeffs_4567 = _mm_unpackhi_epi32(coeffs_x, coeffs_x);
for (j = 0; j < w; j += 8) {
for (i = 0; i < im_h; i += 2) {
__m256i data = _mm256_castsi128_si256(
_mm_loadu_si128((__m128i *)&src_ptr[(i * src_stride) + j]));
// coeffs [ f1 f0 f1 f0 f1 f0 f1 f0 ]
const __m128i coeffs_01_128 = _mm_unpacklo_epi64(coeffs_0123, coeffs_0123);
// coeffs [ f3 f2 f3 f2 f3 f2 f3 f2 ]
const __m128i coeffs_23_128 = _mm_unpackhi_epi64(coeffs_0123, coeffs_0123);
// coeffs [ f5 f4 f5 f4 f5 f4 f5 f4 ]
const __m128i coeffs_45_128 = _mm_unpacklo_epi64(coeffs_4567, coeffs_4567);
// coeffs [ f7 f6 f7 f6 f7 f6 f7 f6 ]
const __m128i coeffs_67_128 = _mm_unpackhi_epi64(coeffs_4567, coeffs_4567);
// Load the next line
if (i + 1 < im_h)
data = _mm256_inserti128_si256(
data,
_mm_loadu_si128(
(__m128i *)&src_ptr[(i * src_stride) + j + src_stride]),
1);
// coeffs [ f1 f0 f1 f0 f1 f0 f1 f0 ][ f1 f0 f1 f0 f1 f0 f1 f0 ]
const __m256i coeffs_01 = yy_set_m128i(coeffs_01_128, coeffs_01_128);
// coeffs [ f3 f2 f3 f2 f3 f2 f3 f2 ][ f3 f2 f3 f2 f3 f2 f3 f2 ]
const __m256i coeffs_23 = yy_set_m128i(coeffs_23_128, coeffs_23_128);
// coeffs [ f5 f4 f5 f4 f5 f4 f5 f4 ][ f5 f4 f5 f4 f5 f4 f5 f4 ]
const __m256i coeffs_45 = yy_set_m128i(coeffs_45_128, coeffs_45_128);
// coeffs [ f7 f6 f7 f6 f7 f6 f7 f6 ][ f7 f6 f7 f6 f7 f6 f7 f6 ]
const __m256i coeffs_67 = yy_set_m128i(coeffs_67_128, coeffs_67_128);
__m256i res = convolve_lowbd_x(data, coeffs_h, filt);
const __m256i round_const = _mm256_set1_epi32(
(1 << (conv_params->round_0 - 1)) + (1 << (bd + FILTER_BITS - 1)));
res =
_mm256_sra_epi16(_mm256_add_epi16(res, round_const_h), round_shift_h);
for (int i = 0; i < intermediate_height; ++i) {
for (int j = 0; j < w; j += 16) {
const uint8_t *data_ij = src_ptr + i * src_stride + j;
__m256i data_0 = _mm256_shuffle_epi8(data, filt_center);
// Load 8-bit src data
const __m128i data_0 = xx_loadu_128(data_ij + 0);
const __m128i data_1 = xx_loadu_128(data_ij + 1);
const __m128i data_2 = xx_loadu_128(data_ij + 2);
const __m128i data_3 = xx_loadu_128(data_ij + 3);
const __m128i data_4 = xx_loadu_128(data_ij + 4);
const __m128i data_5 = xx_loadu_128(data_ij + 5);
const __m128i data_6 = xx_loadu_128(data_ij + 6);
const __m128i data_7 = xx_loadu_128(data_ij + 7);
// multiply the center pixel by 2^(FILTER_BITS - round_0) and add it to
// the result
data_0 = _mm256_slli_epi16(data_0, FILTER_BITS - conv_params->round_0);
res = _mm256_add_epi16(res, data_0);
// (Zero-)Extend 8-bit data to 16-bit data
const __m256i src_0 = _mm256_cvtepu8_epi16(data_0);
const __m256i src_1 = _mm256_cvtepu8_epi16(data_1);
const __m256i src_2 = _mm256_cvtepu8_epi16(data_2);
const __m256i src_3 = _mm256_cvtepu8_epi16(data_3);
const __m256i src_4 = _mm256_cvtepu8_epi16(data_4);
const __m256i src_5 = _mm256_cvtepu8_epi16(data_5);
const __m256i src_6 = _mm256_cvtepu8_epi16(data_6);
const __m256i src_7 = _mm256_cvtepu8_epi16(data_7);
_mm256_store_si256((__m256i *)&im_block[i * im_stride], res);
}
// Multiply src data by filter coeffs and sum pairs
const __m256i res_0 = _mm256_madd_epi16(src_0, coeffs_01);
const __m256i res_1 = _mm256_madd_epi16(src_1, coeffs_01);
const __m256i res_2 = _mm256_madd_epi16(src_2, coeffs_23);
const __m256i res_3 = _mm256_madd_epi16(src_3, coeffs_23);
const __m256i res_4 = _mm256_madd_epi16(src_4, coeffs_45);
const __m256i res_5 = _mm256_madd_epi16(src_5, coeffs_45);
const __m256i res_6 = _mm256_madd_epi16(src_6, coeffs_67);
const __m256i res_7 = _mm256_madd_epi16(src_7, coeffs_67);
/* Vertical filter */
{
__m256i src_0 = _mm256_loadu_si256((__m256i *)(im_block + 0 * im_stride));
__m256i src_1 = _mm256_loadu_si256((__m256i *)(im_block + 1 * im_stride));
__m256i src_2 = _mm256_loadu_si256((__m256i *)(im_block + 2 * im_stride));
__m256i src_3 = _mm256_loadu_si256((__m256i *)(im_block + 3 * im_stride));
__m256i src_4 = _mm256_loadu_si256((__m256i *)(im_block + 4 * im_stride));
__m256i src_5 = _mm256_loadu_si256((__m256i *)(im_block + 5 * im_stride));
// Calculate scalar product for even- and odd-indices separately,
// increasing to 32-bit precision
const __m256i res_even_sum = _mm256_add_epi32(
_mm256_add_epi32(res_0, res_4), _mm256_add_epi32(res_2, res_6));
const __m256i res_odd_sum = _mm256_add_epi32(
_mm256_add_epi32(res_1, res_5), _mm256_add_epi32(res_3, res_7));
__m256i s[8];
s[0] = _mm256_unpacklo_epi16(src_0, src_1);
s[1] = _mm256_unpacklo_epi16(src_2, src_3);
s[2] = _mm256_unpacklo_epi16(src_4, src_5);
const __m256i res_even = _mm256_srai_epi32(
_mm256_add_epi32(res_even_sum, round_const), conv_params->round_0);
const __m256i res_odd = _mm256_srai_epi32(
_mm256_add_epi32(res_odd_sum, round_const), conv_params->round_0);
s[4] = _mm256_unpackhi_epi16(src_0, src_1);
s[5] = _mm256_unpackhi_epi16(src_2, src_3);
s[6] = _mm256_unpackhi_epi16(src_4, src_5);
// Reduce to 16-bit precision and pack even- and odd-index results
// back into one register. The _mm256_packs_epi32 intrinsic returns
// a register with the pixels ordered as follows:
// [ 15 13 11 9 14 12 10 8 ] [ 7 5 3 1 6 4 2 0 ]
const __m256i res = _mm256_packs_epi32(res_even, res_odd);
const __m256i res_clamped =
_mm256_min_epi16(_mm256_max_epi16(res, clamp_low), clamp_high);
for (i = 0; i < h - 1; i += 2) {
const int16_t *data = &im_block[i * im_stride];
const __m256i s6 =
_mm256_loadu_si256((__m256i *)(data + 6 * im_stride));
const __m256i s7 =
_mm256_loadu_si256((__m256i *)(data + 7 * im_stride));
s[3] = _mm256_unpacklo_epi16(s6, s7);
s[7] = _mm256_unpackhi_epi16(s6, s7);
__m256i res_a = convolve(s, coeffs_v);
__m256i res_b = convolve(s + 4, coeffs_v);
const __m256i res_a_round = _mm256_sra_epi32(
_mm256_add_epi32(res_a, round_const_v), round_shift_v);
const __m256i res_b_round = _mm256_sra_epi32(
_mm256_add_epi32(res_b, round_const_v), round_shift_v);
/* rounding code */
// 16 bit conversion
const __m256i res_16bit = _mm256_packs_epi32(res_a_round, res_b_round);
// 8 bit conversion and saturation to uint8
const __m256i res_8b = _mm256_packus_epi16(res_16bit, res_16bit);
const __m128i res_0 = _mm256_castsi256_si128(res_8b);
const __m128i res_1 = _mm256_extracti128_si256(res_8b, 1);
// Store values into the destination buffer
__m128i *const p_0 = (__m128i *)&dst[i * dst_stride + j];
__m128i *const p_1 = (__m128i *)&dst[i * dst_stride + j + dst_stride];
_mm_storel_epi64(p_0, res_0);
_mm_storel_epi64(p_1, res_1);
s[0] = s[1];
s[1] = s[2];
s[2] = s[3];
s[4] = s[5];
s[5] = s[6];
s[6] = s[7];
// Store in a temporary array
yy_storeu_256(temp + i * MAX_SB_SIZE + j, res_clamped);
}
if (h - i) {
s[0] = _mm256_permute2x128_si256(s[0], s[4], 0x20);
s[1] = _mm256_permute2x128_si256(s[1], s[5], 0x20);
s[2] = _mm256_permute2x128_si256(s[2], s[6], 0x20);
}
}
const int16_t *data = &im_block[i * im_stride];
const __m128i s6_ = _mm_loadu_si128((__m128i *)(data + 6 * im_stride));
const __m128i s7_ = _mm_loadu_si128((__m128i *)(data + 7 * im_stride));
/* Vertical filter */
{
// coeffs [ g7 g6 g5 g4 g3 g2 g1 g0 ]
const __m128i coeffs_y = _mm_add_epi16(xx_loadu_128(filter_y), offset);
__m128i s3 = _mm_unpacklo_epi16(s6_, s7_);
__m128i s7 = _mm_unpackhi_epi16(s6_, s7_);
// coeffs [ g3 g2 g3 g2 g1 g0 g1 g0 ]
const __m128i coeffs_0123 = _mm_unpacklo_epi32(coeffs_y, coeffs_y);
// coeffs [ g7 g6 g7 g6 g5 g4 g5 g4 ]
const __m128i coeffs_4567 = _mm_unpackhi_epi32(coeffs_y, coeffs_y);
s[3] = _mm256_inserti128_si256(_mm256_castsi128_si256(s3), s7, 1);
__m256i convolveres = convolve(s, coeffs_v);
// coeffs [ g1 g0 g1 g0 g1 g0 g1 g0 ]
const __m128i coeffs_01_128 = _mm_unpacklo_epi64(coeffs_0123, coeffs_0123);
// coeffs [ g3 g2 g3 g2 g3 g2 g3 g2 ]
const __m128i coeffs_23_128 = _mm_unpackhi_epi64(coeffs_0123, coeffs_0123);
// coeffs [ g5 g4 g5 g4 g5 g4 g5 g4 ]
const __m128i coeffs_45_128 = _mm_unpacklo_epi64(coeffs_4567, coeffs_4567);
// coeffs [ g7 g6 g7 g6 g7 g6 g7 g6 ]
const __m128i coeffs_67_128 = _mm_unpackhi_epi64(coeffs_4567, coeffs_4567);
const __m256i res_round = _mm256_sra_epi32(
_mm256_add_epi32(convolveres, round_const_v), round_shift_v);
// coeffs [ g1 g0 g1 g0 g1 g0 g1 g0 ][ g1 g0 g1 g0 g1 g0 g1 g0 ]
const __m256i coeffs_01 = yy_set_m128i(coeffs_01_128, coeffs_01_128);
// coeffs [ g3 g2 g3 g2 g3 g2 g3 g2 ][ g3 g2 g3 g2 g3 g2 g3 g2 ]
const __m256i coeffs_23 = yy_set_m128i(coeffs_23_128, coeffs_23_128);
// coeffs [ g5 g4 g5 g4 g5 g4 g5 g4 ][ g5 g4 g5 g4 g5 g4 g5 g4 ]
const __m256i coeffs_45 = yy_set_m128i(coeffs_45_128, coeffs_45_128);
// coeffs [ g7 g6 g7 g6 g7 g6 g7 g6 ][ g7 g6 g7 g6 g7 g6 g7 g6 ]
const __m256i coeffs_67 = yy_set_m128i(coeffs_67_128, coeffs_67_128);
/* rounding code */
// 16 bit conversion
__m128i reslo = _mm256_castsi256_si128(res_round);
__m128i reshi = _mm256_extracti128_si256(res_round, 1);
const __m128i res_16bit = _mm_packus_epi32(reslo, reshi);
const __m256i round_const =
_mm256_set1_epi32((1 << (conv_params->round_1 - 1)) -
(1 << (bd + conv_params->round_1 - 1)));
// 8 bit conversion and saturation to uint8
const __m128i res_8b = _mm_packus_epi16(res_16bit, res_16bit);
__m128i *const p_0 = (__m128i *)&dst[i * dst_stride + j];
_mm_storel_epi64(p_0, res_8b);
for (int i = 0; i < h; ++i) {
for (int j = 0; j < w; j += 16) {
const uint16_t *data_ij = temp + i * MAX_SB_SIZE + j;
// Load 16-bit data from the output of the horizontal filter in
// which the pixels are ordered as follows:
// [ 15 13 11 9 14 12 10 8 ] [ 7 5 3 1 6 4 2 0 ]
const __m256i data_0 = yy_loadu_256(data_ij + 0 * MAX_SB_SIZE);
const __m256i data_1 = yy_loadu_256(data_ij + 1 * MAX_SB_SIZE);
const __m256i data_2 = yy_loadu_256(data_ij + 2 * MAX_SB_SIZE);
const __m256i data_3 = yy_loadu_256(data_ij + 3 * MAX_SB_SIZE);
const __m256i data_4 = yy_loadu_256(data_ij + 4 * MAX_SB_SIZE);
const __m256i data_5 = yy_loadu_256(data_ij + 5 * MAX_SB_SIZE);
const __m256i data_6 = yy_loadu_256(data_ij + 6 * MAX_SB_SIZE);
const __m256i data_7 = yy_loadu_256(data_ij + 7 * MAX_SB_SIZE);
// Filter the even-indices, increasing to 32-bit precision
const __m256i src_0 = _mm256_unpacklo_epi16(data_0, data_1);
const __m256i src_2 = _mm256_unpacklo_epi16(data_2, data_3);
const __m256i src_4 = _mm256_unpacklo_epi16(data_4, data_5);
const __m256i src_6 = _mm256_unpacklo_epi16(data_6, data_7);
const __m256i res_0 = _mm256_madd_epi16(src_0, coeffs_01);
const __m256i res_2 = _mm256_madd_epi16(src_2, coeffs_23);
const __m256i res_4 = _mm256_madd_epi16(src_4, coeffs_45);
const __m256i res_6 = _mm256_madd_epi16(src_6, coeffs_67);
const __m256i res_even = _mm256_add_epi32(
_mm256_add_epi32(res_0, res_2), _mm256_add_epi32(res_4, res_6));
// Filter the odd-indices, increasing to 32-bit precision
const __m256i src_1 = _mm256_unpackhi_epi16(data_0, data_1);
const __m256i src_3 = _mm256_unpackhi_epi16(data_2, data_3);
const __m256i src_5 = _mm256_unpackhi_epi16(data_4, data_5);
const __m256i src_7 = _mm256_unpackhi_epi16(data_6, data_7);
const __m256i res_1 = _mm256_madd_epi16(src_1, coeffs_01);
const __m256i res_3 = _mm256_madd_epi16(src_3, coeffs_23);
const __m256i res_5 = _mm256_madd_epi16(src_5, coeffs_45);
const __m256i res_7 = _mm256_madd_epi16(src_7, coeffs_67);
const __m256i res_odd = _mm256_add_epi32(
_mm256_add_epi32(res_1, res_3), _mm256_add_epi32(res_5, res_7));
// Pixels are currently in the following order:
// res_even order: [ 14 12 10 8 ] [ 6 4 2 0 ]
// res_odd order: [ 15 13 11 9 ] [ 7 5 3 1 ]
//
// Rearrange the pixels into the following order:
// res_lo order: [ 11 10 9 8 ] [ 3 2 1 0 ]
// res_hi order: [ 15 14 13 12 ] [ 7 6 5 4 ]
const __m256i res_lo = _mm256_unpacklo_epi32(res_even, res_odd);
const __m256i res_hi = _mm256_unpackhi_epi32(res_even, res_odd);
const __m256i res_lo_round = _mm256_srai_epi32(
_mm256_add_epi32(res_lo, round_const), conv_params->round_1);
const __m256i res_hi_round = _mm256_srai_epi32(
_mm256_add_epi32(res_hi, round_const), conv_params->round_1);
// Reduce to 16-bit precision and pack into the correct order:
// [ 15 14 13 12 11 10 9 8 ][ 7 6 5 4 3 2 1 0 ]
const __m256i res_16bit =
_mm256_packs_epi32(res_lo_round, res_hi_round);
// Reduce to 8-bit precision. This messes up the order:
// [ - - - - - - - - 15 14 13 12 11 10 9 8 ]
// [ - - - - - - - - 7 6 5 4 3 2 1 0 ]
const __m256i res_8bit =
_mm256_packus_epi16(res_16bit, zero_256 /* don't care value */);
// Swap the two central 32-bit values to get the order:
// [ - - - - - - - - - - - - - - - - ]
// [ 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1 0 ]
const __m256i res_8bit2 = _mm256_permute4x64_epi64(res_8bit, 0xd8);
// Store the lower 128-bit lane in the dst array
xx_storeu_128(dst + i * dst_stride + j,
_mm256_castsi256_si128(res_8bit2));
}
}
}