wuggy/Dactyloidae
wuggy/Dactyloidae
1
0
Fork 0
mirror of https://repo.dactyloidae.xyz/Dactyloidae/UXP.git synced 2026-10-08 16:27:31 +09:00
Code Issues Projects Releases Packages Wiki Activity

Import aom library

Browse source 
This is the reference implementation for the Alliance for Open Media's av1 video code.

The commit used was 4d668d7feb1f8abd809d1bca0418570a7f142a36.
This commit is contained in:
trav90 2018-10-15 21:45:30 -05:00 • committed by Roy Tam
commit edc8d83307
989 changed files with 470949 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

1029
third_party/aom/av1/common/x86/av1_convolve_ssse3.c vendored Normal file
View file

File diff suppressed because it is too large Load diff

839
third_party/aom/av1/common/x86/av1_fwd_txfm1d_sse4.c vendored Normal file
View file

@ -0,0 +1,839 @@
#include "av1/common/x86/av1_txfm1d_sse4.h"
void av1_fdct32_new_sse4_1(const __m128i *input, __m128i *output,
const int8_t *cos_bit, const int8_t *stage_range) {
const int txfm_size = 32;
const int num_per_128 = 4;
const int32_t *cospi;
__m128i buf0[32];
__m128i buf1[32];
int col_num = txfm_size / num_per_128;
int bit;
int col;
(void)stage_range;
for (col = 0; col < col_num; col++) {
// stage 0;
int32_t stage_idx = 0;
int j;
for (j = 0; j < 32; ++j) {
buf0[j] = input[j * col_num + col];
}
// stage 1
stage_idx++;
buf1[0] = _mm_add_epi32(buf0[0], buf0[31]);
buf1[31] = _mm_sub_epi32(buf0[0], buf0[31]);
buf1[1] = _mm_add_epi32(buf0[1], buf0[30]);
buf1[30] = _mm_sub_epi32(buf0[1], buf0[30]);
buf1[2] = _mm_add_epi32(buf0[2], buf0[29]);
buf1[29] = _mm_sub_epi32(buf0[2], buf0[29]);
buf1[3] = _mm_add_epi32(buf0[3], buf0[28]);
buf1[28] = _mm_sub_epi32(buf0[3], buf0[28]);
buf1[4] = _mm_add_epi32(buf0[4], buf0[27]);
buf1[27] = _mm_sub_epi32(buf0[4], buf0[27]);
buf1[5] = _mm_add_epi32(buf0[5], buf0[26]);
buf1[26] = _mm_sub_epi32(buf0[5], buf0[26]);
buf1[6] = _mm_add_epi32(buf0[6], buf0[25]);
buf1[25] = _mm_sub_epi32(buf0[6], buf0[25]);
buf1[7] = _mm_add_epi32(buf0[7], buf0[24]);
buf1[24] = _mm_sub_epi32(buf0[7], buf0[24]);
buf1[8] = _mm_add_epi32(buf0[8], buf0[23]);
buf1[23] = _mm_sub_epi32(buf0[8], buf0[23]);
buf1[9] = _mm_add_epi32(buf0[9], buf0[22]);
buf1[22] = _mm_sub_epi32(buf0[9], buf0[22]);
buf1[10] = _mm_add_epi32(buf0[10], buf0[21]);
buf1[21] = _mm_sub_epi32(buf0[10], buf0[21]);
buf1[11] = _mm_add_epi32(buf0[11], buf0[20]);
buf1[20] = _mm_sub_epi32(buf0[11], buf0[20]);
buf1[12] = _mm_add_epi32(buf0[12], buf0[19]);
buf1[19] = _mm_sub_epi32(buf0[12], buf0[19]);
buf1[13] = _mm_add_epi32(buf0[13], buf0[18]);
buf1[18] = _mm_sub_epi32(buf0[13], buf0[18]);
buf1[14] = _mm_add_epi32(buf0[14], buf0[17]);
buf1[17] = _mm_sub_epi32(buf0[14], buf0[17]);
buf1[15] = _mm_add_epi32(buf0[15], buf0[16]);
buf1[16] = _mm_sub_epi32(buf0[15], buf0[16]);
// stage 2
stage_idx++;
bit = cos_bit[stage_idx];
cospi = cospi_arr[bit - cos_bit_min];
buf0[0] = _mm_add_epi32(buf1[0], buf1[15]);
buf0[15] = _mm_sub_epi32(buf1[0], buf1[15]);
buf0[1] = _mm_add_epi32(buf1[1], buf1[14]);
buf0[14] = _mm_sub_epi32(buf1[1], buf1[14]);
buf0[2] = _mm_add_epi32(buf1[2], buf1[13]);
buf0[13] = _mm_sub_epi32(buf1[2], buf1[13]);
buf0[3] = _mm_add_epi32(buf1[3], buf1[12]);
buf0[12] = _mm_sub_epi32(buf1[3], buf1[12]);
buf0[4] = _mm_add_epi32(buf1[4], buf1[11]);
buf0[11] = _mm_sub_epi32(buf1[4], buf1[11]);
buf0[5] = _mm_add_epi32(buf1[5], buf1[10]);
buf0[10] = _mm_sub_epi32(buf1[5], buf1[10]);
buf0[6] = _mm_add_epi32(buf1[6], buf1[9]);
buf0[9] = _mm_sub_epi32(buf1[6], buf1[9]);
buf0[7] = _mm_add_epi32(buf1[7], buf1[8]);
buf0[8] = _mm_sub_epi32(buf1[7], buf1[8]);
buf0[16] = buf1[16];
buf0[17] = buf1[17];
buf0[18] = buf1[18];
buf0[19] = buf1[19];
btf_32_sse4_1_type0(-cospi[32], cospi[32], buf1[20], buf1[27], buf0[20],
buf0[27], bit);
btf_32_sse4_1_type0(-cospi[32], cospi[32], buf1[21], buf1[26], buf0[21],
buf0[26], bit);
btf_32_sse4_1_type0(-cospi[32], cospi[32], buf1[22], buf1[25], buf0[22],
buf0[25], bit);
btf_32_sse4_1_type0(-cospi[32], cospi[32], buf1[23], buf1[24], buf0[23],
buf0[24], bit);
buf0[28] = buf1[28];
buf0[29] = buf1[29];
buf0[30] = buf1[30];
buf0[31] = buf1[31];
// stage 3
stage_idx++;
bit = cos_bit[stage_idx];
cospi = cospi_arr[bit - cos_bit_min];
buf1[0] = _mm_add_epi32(buf0[0], buf0[7]);
buf1[7] = _mm_sub_epi32(buf0[0], buf0[7]);
buf1[1] = _mm_add_epi32(buf0[1], buf0[6]);
buf1[6] = _mm_sub_epi32(buf0[1], buf0[6]);
buf1[2] = _mm_add_epi32(buf0[2], buf0[5]);
buf1[5] = _mm_sub_epi32(buf0[2], buf0[5]);
buf1[3] = _mm_add_epi32(buf0[3], buf0[4]);
buf1[4] = _mm_sub_epi32(buf0[3], buf0[4]);
buf1[8] = buf0[8];
buf1[9] = buf0[9];
btf_32_sse4_1_type0(-cospi[32], cospi[32], buf0[10], buf0[13], buf1[10],
buf1[13], bit);
btf_32_sse4_1_type0(-cospi[32], cospi[32], buf0[11], buf0[12], buf1[11],
buf1[12], bit);
buf1[14] = buf0[14];
buf1[15] = buf0[15];
buf1[16] = _mm_add_epi32(buf0[16], buf0[23]);
buf1[23] = _mm_sub_epi32(buf0[16], buf0[23]);
buf1[17] = _mm_add_epi32(buf0[17], buf0[22]);
buf1[22] = _mm_sub_epi32(buf0[17], buf0[22]);
buf1[18] = _mm_add_epi32(buf0[18], buf0[21]);
buf1[21] = _mm_sub_epi32(buf0[18], buf0[21]);
buf1[19] = _mm_add_epi32(buf0[19], buf0[20]);
buf1[20] = _mm_sub_epi32(buf0[19], buf0[20]);
buf1[24] = _mm_sub_epi32(buf0[31], buf0[24]);
buf1[31] = _mm_add_epi32(buf0[31], buf0[24]);
buf1[25] = _mm_sub_epi32(buf0[30], buf0[25]);
buf1[30] = _mm_add_epi32(buf0[30], buf0[25]);
buf1[26] = _mm_sub_epi32(buf0[29], buf0[26]);
buf1[29] = _mm_add_epi32(buf0[29], buf0[26]);
buf1[27] = _mm_sub_epi32(buf0[28], buf0[27]);
buf1[28] = _mm_add_epi32(buf0[28], buf0[27]);
// stage 4
stage_idx++;
bit = cos_bit[stage_idx];
cospi = cospi_arr[bit - cos_bit_min];
buf0[0] = _mm_add_epi32(buf1[0], buf1[3]);
buf0[3] = _mm_sub_epi32(buf1[0], buf1[3]);
buf0[1] = _mm_add_epi32(buf1[1], buf1[2]);
buf0[2] = _mm_sub_epi32(buf1[1], buf1[2]);
buf0[4] = buf1[4];
btf_32_sse4_1_type0(-cospi[32], cospi[32], buf1[5], buf1[6], buf0[5],
buf0[6], bit);
buf0[7] = buf1[7];
buf0[8] = _mm_add_epi32(buf1[8], buf1[11]);
buf0[11] = _mm_sub_epi32(buf1[8], buf1[11]);
buf0[9] = _mm_add_epi32(buf1[9], buf1[10]);
buf0[10] = _mm_sub_epi32(buf1[9], buf1[10]);
buf0[12] = _mm_sub_epi32(buf1[15], buf1[12]);
buf0[15] = _mm_add_epi32(buf1[15], buf1[12]);
buf0[13] = _mm_sub_epi32(buf1[14], buf1[13]);
buf0[14] = _mm_add_epi32(buf1[14], buf1[13]);
buf0[16] = buf1[16];
buf0[17] = buf1[17];
btf_32_sse4_1_type0(-cospi[16], cospi[48], buf1[18], buf1[29], buf0[18],
buf0[29], bit);
btf_32_sse4_1_type0(-cospi[16], cospi[48], buf1[19], buf1[28], buf0[19],
buf0[28], bit);
btf_32_sse4_1_type0(-cospi[48], -cospi[16], buf1[20], buf1[27], buf0[20],
buf0[27], bit);
btf_32_sse4_1_type0(-cospi[48], -cospi[16], buf1[21], buf1[26], buf0[21],
buf0[26], bit);
buf0[22] = buf1[22];
buf0[23] = buf1[23];
buf0[24] = buf1[24];
buf0[25] = buf1[25];
buf0[30] = buf1[30];
buf0[31] = buf1[31];
// stage 5
stage_idx++;
bit = cos_bit[stage_idx];
cospi = cospi_arr[bit - cos_bit_min];
btf_32_sse4_1_type0(cospi[32], cospi[32], buf0[0], buf0[1], buf1[0],
buf1[1], bit);
btf_32_sse4_1_type1(cospi[48], cospi[16], buf0[2], buf0[3], buf1[2],
buf1[3], bit);
buf1[4] = _mm_add_epi32(buf0[4], buf0[5]);
buf1[5] = _mm_sub_epi32(buf0[4], buf0[5]);
buf1[6] = _mm_sub_epi32(buf0[7], buf0[6]);
buf1[7] = _mm_add_epi32(buf0[7], buf0[6]);
buf1[8] = buf0[8];
btf_32_sse4_1_type0(-cospi[16], cospi[48], buf0[9], buf0[14], buf1[9],
buf1[14], bit);
btf_32_sse4_1_type0(-cospi[48], -cospi[16], buf0[10], buf0[13], buf1[10],
buf1[13], bit);
buf1[11] = buf0[11];
buf1[12] = buf0[12];
buf1[15] = buf0[15];
buf1[16] = _mm_add_epi32(buf0[16], buf0[19]);
buf1[19] = _mm_sub_epi32(buf0[16], buf0[19]);
buf1[17] = _mm_add_epi32(buf0[17], buf0[18]);
buf1[18] = _mm_sub_epi32(buf0[17], buf0[18]);
buf1[20] = _mm_sub_epi32(buf0[23], buf0[20]);
buf1[23] = _mm_add_epi32(buf0[23], buf0[20]);
buf1[21] = _mm_sub_epi32(buf0[22], buf0[21]);
buf1[22] = _mm_add_epi32(buf0[22], buf0[21]);
buf1[24] = _mm_add_epi32(buf0[24], buf0[27]);
buf1[27] = _mm_sub_epi32(buf0[24], buf0[27]);
buf1[25] = _mm_add_epi32(buf0[25], buf0[26]);
buf1[26] = _mm_sub_epi32(buf0[25], buf0[26]);
buf1[28] = _mm_sub_epi32(buf0[31], buf0[28]);
buf1[31] = _mm_add_epi32(buf0[31], buf0[28]);
buf1[29] = _mm_sub_epi32(buf0[30], buf0[29]);
buf1[30] = _mm_add_epi32(buf0[30], buf0[29]);
// stage 6
stage_idx++;
bit = cos_bit[stage_idx];
cospi = cospi_arr[bit - cos_bit_min];
buf0[0] = buf1[0];
buf0[1] = buf1[1];
buf0[2] = buf1[2];
buf0[3] = buf1[3];
btf_32_sse4_1_type1(cospi[56], cospi[8], buf1[4], buf1[7], buf0[4], buf0[7],
bit);
btf_32_sse4_1_type1(cospi[24], cospi[40], buf1[5], buf1[6], buf0[5],
buf0[6], bit);
buf0[8] = _mm_add_epi32(buf1[8], buf1[9]);
buf0[9] = _mm_sub_epi32(buf1[8], buf1[9]);
buf0[10] = _mm_sub_epi32(buf1[11], buf1[10]);
buf0[11] = _mm_add_epi32(buf1[11], buf1[10]);
buf0[12] = _mm_add_epi32(buf1[12], buf1[13]);
buf0[13] = _mm_sub_epi32(buf1[12], buf1[13]);
buf0[14] = _mm_sub_epi32(buf1[15], buf1[14]);
buf0[15] = _mm_add_epi32(buf1[15], buf1[14]);
buf0[16] = buf1[16];
btf_32_sse4_1_type0(-cospi[8], cospi[56], buf1[17], buf1[30], buf0[17],
buf0[30], bit);
btf_32_sse4_1_type0(-cospi[56], -cospi[8], buf1[18], buf1[29], buf0[18],
buf0[29], bit);
buf0[19] = buf1[19];
buf0[20] = buf1[20];
btf_32_sse4_1_type0(-cospi[40], cospi[24], buf1[21], buf1[26], buf0[21],
buf0[26], bit);
btf_32_sse4_1_type0(-cospi[24], -cospi[40], buf1[22], buf1[25], buf0[22],
buf0[25], bit);
buf0[23] = buf1[23];
buf0[24] = buf1[24];
buf0[27] = buf1[27];
buf0[28] = buf1[28];
buf0[31] = buf1[31];
// stage 7
stage_idx++;
bit = cos_bit[stage_idx];
cospi = cospi_arr[bit - cos_bit_min];
buf1[0] = buf0[0];
buf1[1] = buf0[1];
buf1[2] = buf0[2];
buf1[3] = buf0[3];
buf1[4] = buf0[4];
buf1[5] = buf0[5];
buf1[6] = buf0[6];
buf1[7] = buf0[7];
btf_32_sse4_1_type1(cospi[60], cospi[4], buf0[8], buf0[15], buf1[8],
buf1[15], bit);
btf_32_sse4_1_type1(cospi[28], cospi[36], buf0[9], buf0[14], buf1[9],
buf1[14], bit);
btf_32_sse4_1_type1(cospi[44], cospi[20], buf0[10], buf0[13], buf1[10],
buf1[13], bit);
btf_32_sse4_1_type1(cospi[12], cospi[52], buf0[11], buf0[12], buf1[11],
buf1[12], bit);
buf1[16] = _mm_add_epi32(buf0[16], buf0[17]);
buf1[17] = _mm_sub_epi32(buf0[16], buf0[17]);
buf1[18] = _mm_sub_epi32(buf0[19], buf0[18]);
buf1[19] = _mm_add_epi32(buf0[19], buf0[18]);
buf1[20] = _mm_add_epi32(buf0[20], buf0[21]);
buf1[21] = _mm_sub_epi32(buf0[20], buf0[21]);
buf1[22] = _mm_sub_epi32(buf0[23], buf0[22]);
buf1[23] = _mm_add_epi32(buf0[23], buf0[22]);
buf1[24] = _mm_add_epi32(buf0[24], buf0[25]);
buf1[25] = _mm_sub_epi32(buf0[24], buf0[25]);
buf1[26] = _mm_sub_epi32(buf0[27], buf0[26]);
buf1[27] = _mm_add_epi32(buf0[27], buf0[26]);
buf1[28] = _mm_add_epi32(buf0[28], buf0[29]);
buf1[29] = _mm_sub_epi32(buf0[28], buf0[29]);
buf1[30] = _mm_sub_epi32(buf0[31], buf0[30]);
buf1[31] = _mm_add_epi32(buf0[31], buf0[30]);
// stage 8
stage_idx++;
bit = cos_bit[stage_idx];
cospi = cospi_arr[bit - cos_bit_min];
buf0[0] = buf1[0];
buf0[1] = buf1[1];
buf0[2] = buf1[2];
buf0[3] = buf1[3];
buf0[4] = buf1[4];
buf0[5] = buf1[5];
buf0[6] = buf1[6];
buf0[7] = buf1[7];
buf0[8] = buf1[8];
buf0[9] = buf1[9];
buf0[10] = buf1[10];
buf0[11] = buf1[11];
buf0[12] = buf1[12];
buf0[13] = buf1[13];
buf0[14] = buf1[14];
buf0[15] = buf1[15];
btf_32_sse4_1_type1(cospi[62], cospi[2], buf1[16], buf1[31], buf0[16],
buf0[31], bit);
btf_32_sse4_1_type1(cospi[30], cospi[34], buf1[17], buf1[30], buf0[17],
buf0[30], bit);
btf_32_sse4_1_type1(cospi[46], cospi[18], buf1[18], buf1[29], buf0[18],
buf0[29], bit);
btf_32_sse4_1_type1(cospi[14], cospi[50], buf1[19], buf1[28], buf0[19],
buf0[28], bit);
btf_32_sse4_1_type1(cospi[54], cospi[10], buf1[20], buf1[27], buf0[20],
buf0[27], bit);
btf_32_sse4_1_type1(cospi[22], cospi[42], buf1[21], buf1[26], buf0[21],
buf0[26], bit);
btf_32_sse4_1_type1(cospi[38], cospi[26], buf1[22], buf1[25], buf0[22],
buf0[25], bit);
btf_32_sse4_1_type1(cospi[6], cospi[58], buf1[23], buf1[24], buf0[23],
buf0[24], bit);
// stage 9
stage_idx++;
buf1[0] = buf0[0];
buf1[1] = buf0[16];
buf1[2] = buf0[8];
buf1[3] = buf0[24];
buf1[4] = buf0[4];
buf1[5] = buf0[20];
buf1[6] = buf0[12];
buf1[7] = buf0[28];
buf1[8] = buf0[2];
buf1[9] = buf0[18];
buf1[10] = buf0[10];
buf1[11] = buf0[26];
buf1[12] = buf0[6];
buf1[13] = buf0[22];
buf1[14] = buf0[14];
buf1[15] = buf0[30];
buf1[16] = buf0[1];
buf1[17] = buf0[17];
buf1[18] = buf0[9];
buf1[19] = buf0[25];
buf1[20] = buf0[5];
buf1[21] = buf0[21];
buf1[22] = buf0[13];
buf1[23] = buf0[29];
buf1[24] = buf0[3];
buf1[25] = buf0[19];
buf1[26] = buf0[11];
buf1[27] = buf0[27];
buf1[28] = buf0[7];
buf1[29] = buf0[23];
buf1[30] = buf0[15];
buf1[31] = buf0[31];
for (j = 0; j < 32; ++j) {
output[j * col_num + col] = buf1[j];
}
}
}
void av1_fadst4_new_sse4_1(const __m128i *input, __m128i *output,
const int8_t *cos_bit, const int8_t *stage_range) {
const int txfm_size = 4;
const int num_per_128 = 4;
const int32_t *cospi;
__m128i buf0[4];
__m128i buf1[4];
int col_num = txfm_size / num_per_128;
int bit;
int col;
(void)stage_range;
for (col = 0; col < col_num; col++) {
// stage 0;
int32_t stage_idx = 0;
int j;
for (j = 0; j < 4; ++j) {
buf0[j] = input[j * col_num + col];
}
// stage 1
stage_idx++;
buf1[0] = buf0[3];
buf1[1] = buf0[0];
buf1[2] = buf0[1];
buf1[3] = buf0[2];
// stage 2
stage_idx++;
bit = cos_bit[stage_idx];
cospi = cospi_arr[bit - cos_bit_min];
btf_32_sse4_1_type0(cospi[8], cospi[56], buf1[0], buf1[1], buf0[0], buf0[1],
bit);
btf_32_sse4_1_type0(cospi[40], cospi[24], buf1[2], buf1[3], buf0[2],
buf0[3], bit);
// stage 3
stage_idx++;
buf1[0] = _mm_add_epi32(buf0[0], buf0[2]);
buf1[2] = _mm_sub_epi32(buf0[0], buf0[2]);
buf1[1] = _mm_add_epi32(buf0[1], buf0[3]);
buf1[3] = _mm_sub_epi32(buf0[1], buf0[3]);
// stage 4
stage_idx++;
bit = cos_bit[stage_idx];
cospi = cospi_arr[bit - cos_bit_min];
buf0[0] = buf1[0];
buf0[1] = buf1[1];
btf_32_sse4_1_type0(cospi[32], cospi[32], buf1[2], buf1[3], buf0[2],
buf0[3], bit);
// stage 5
stage_idx++;
buf1[0] = buf0[0];
buf1[1] = _mm_sub_epi32(_mm_set1_epi32(0), buf0[2]);
buf1[2] = buf0[3];
buf1[3] = _mm_sub_epi32(_mm_set1_epi32(0), buf0[1]);
for (j = 0; j < 4; ++j) {
output[j * col_num + col] = buf1[j];
}
}
}
void av1_fadst32_new_sse4_1(const __m128i *input, __m128i *output,
const int8_t *cos_bit, const int8_t *stage_range) {
const int txfm_size = 32;
const int num_per_128 = 4;
const int32_t *cospi;
__m128i buf0[32];
__m128i buf1[32];
int col_num = txfm_size / num_per_128;
int bit;
int col;
(void)stage_range;
for (col = 0; col < col_num; col++) {
// stage 0;
int32_t stage_idx = 0;
int j;
for (j = 0; j < 32; ++j) {
buf0[j] = input[j * col_num + col];
}
// stage 1
stage_idx++;
buf1[0] = buf0[31];
buf1[1] = buf0[0];
buf1[2] = buf0[29];
buf1[3] = buf0[2];
buf1[4] = buf0[27];
buf1[5] = buf0[4];
buf1[6] = buf0[25];
buf1[7] = buf0[6];
buf1[8] = buf0[23];
buf1[9] = buf0[8];
buf1[10] = buf0[21];
buf1[11] = buf0[10];
buf1[12] = buf0[19];
buf1[13] = buf0[12];
buf1[14] = buf0[17];
buf1[15] = buf0[14];
buf1[16] = buf0[15];
buf1[17] = buf0[16];
buf1[18] = buf0[13];
buf1[19] = buf0[18];
buf1[20] = buf0[11];
buf1[21] = buf0[20];
buf1[22] = buf0[9];
buf1[23] = buf0[22];
buf1[24] = buf0[7];
buf1[25] = buf0[24];
buf1[26] = buf0[5];
buf1[27] = buf0[26];
buf1[28] = buf0[3];
buf1[29] = buf0[28];
buf1[30] = buf0[1];
buf1[31] = buf0[30];
// stage 2
stage_idx++;
bit = cos_bit[stage_idx];
cospi = cospi_arr[bit - cos_bit_min];
btf_32_sse4_1_type0(cospi[1], cospi[63], buf1[0], buf1[1], buf0[0], buf0[1],
bit);
btf_32_sse4_1_type0(cospi[5], cospi[59], buf1[2], buf1[3], buf0[2], buf0[3],
bit);
btf_32_sse4_1_type0(cospi[9], cospi[55], buf1[4], buf1[5], buf0[4], buf0[5],
bit);
btf_32_sse4_1_type0(cospi[13], cospi[51], buf1[6], buf1[7], buf0[6],
buf0[7], bit);
btf_32_sse4_1_type0(cospi[17], cospi[47], buf1[8], buf1[9], buf0[8],
buf0[9], bit);
btf_32_sse4_1_type0(cospi[21], cospi[43], buf1[10], buf1[11], buf0[10],
buf0[11], bit);
btf_32_sse4_1_type0(cospi[25], cospi[39], buf1[12], buf1[13], buf0[12],
buf0[13], bit);
btf_32_sse4_1_type0(cospi[29], cospi[35], buf1[14], buf1[15], buf0[14],
buf0[15], bit);
btf_32_sse4_1_type0(cospi[33], cospi[31], buf1[16], buf1[17], buf0[16],
buf0[17], bit);
btf_32_sse4_1_type0(cospi[37], cospi[27], buf1[18], buf1[19], buf0[18],
buf0[19], bit);
btf_32_sse4_1_type0(cospi[41], cospi[23], buf1[20], buf1[21], buf0[20],
buf0[21], bit);
btf_32_sse4_1_type0(cospi[45], cospi[19], buf1[22], buf1[23], buf0[22],
buf0[23], bit);
btf_32_sse4_1_type0(cospi[49], cospi[15], buf1[24], buf1[25], buf0[24],
buf0[25], bit);
btf_32_sse4_1_type0(cospi[53], cospi[11], buf1[26], buf1[27], buf0[26],
buf0[27], bit);
btf_32_sse4_1_type0(cospi[57], cospi[7], buf1[28], buf1[29], buf0[28],
buf0[29], bit);
btf_32_sse4_1_type0(cospi[61], cospi[3], buf1[30], buf1[31], buf0[30],
buf0[31], bit);
// stage 3
stage_idx++;
buf1[0] = _mm_add_epi32(buf0[0], buf0[16]);
buf1[16] = _mm_sub_epi32(buf0[0], buf0[16]);
buf1[1] = _mm_add_epi32(buf0[1], buf0[17]);
buf1[17] = _mm_sub_epi32(buf0[1], buf0[17]);
buf1[2] = _mm_add_epi32(buf0[2], buf0[18]);
buf1[18] = _mm_sub_epi32(buf0[2], buf0[18]);
buf1[3] = _mm_add_epi32(buf0[3], buf0[19]);
buf1[19] = _mm_sub_epi32(buf0[3], buf0[19]);
buf1[4] = _mm_add_epi32(buf0[4], buf0[20]);
buf1[20] = _mm_sub_epi32(buf0[4], buf0[20]);
buf1[5] = _mm_add_epi32(buf0[5], buf0[21]);
buf1[21] = _mm_sub_epi32(buf0[5], buf0[21]);
buf1[6] = _mm_add_epi32(buf0[6], buf0[22]);
buf1[22] = _mm_sub_epi32(buf0[6], buf0[22]);
buf1[7] = _mm_add_epi32(buf0[7], buf0[23]);
buf1[23] = _mm_sub_epi32(buf0[7], buf0[23]);
buf1[8] = _mm_add_epi32(buf0[8], buf0[24]);
buf1[24] = _mm_sub_epi32(buf0[8], buf0[24]);
buf1[9] = _mm_add_epi32(buf0[9], buf0[25]);
buf1[25] = _mm_sub_epi32(buf0[9], buf0[25]);
buf1[10] = _mm_add_epi32(buf0[10], buf0[26]);
buf1[26] = _mm_sub_epi32(buf0[10], buf0[26]);
buf1[11] = _mm_add_epi32(buf0[11], buf0[27]);
buf1[27] = _mm_sub_epi32(buf0[11], buf0[27]);
buf1[12] = _mm_add_epi32(buf0[12], buf0[28]);
buf1[28] = _mm_sub_epi32(buf0[12], buf0[28]);
buf1[13] = _mm_add_epi32(buf0[13], buf0[29]);
buf1[29] = _mm_sub_epi32(buf0[13], buf0[29]);
buf1[14] = _mm_add_epi32(buf0[14], buf0[30]);
buf1[30] = _mm_sub_epi32(buf0[14], buf0[30]);
buf1[15] = _mm_add_epi32(buf0[15], buf0[31]);
buf1[31] = _mm_sub_epi32(buf0[15], buf0[31]);
// stage 4
stage_idx++;
bit = cos_bit[stage_idx];
cospi = cospi_arr[bit - cos_bit_min];
buf0[0] = buf1[0];
buf0[1] = buf1[1];
buf0[2] = buf1[2];
buf0[3] = buf1[3];
buf0[4] = buf1[4];
buf0[5] = buf1[5];
buf0[6] = buf1[6];
buf0[7] = buf1[7];
buf0[8] = buf1[8];
buf0[9] = buf1[9];
buf0[10] = buf1[10];
buf0[11] = buf1[11];
buf0[12] = buf1[12];
buf0[13] = buf1[13];
buf0[14] = buf1[14];
buf0[15] = buf1[15];
btf_32_sse4_1_type0(cospi[4], cospi[60], buf1[16], buf1[17], buf0[16],
buf0[17], bit);
btf_32_sse4_1_type0(cospi[20], cospi[44], buf1[18], buf1[19], buf0[18],
buf0[19], bit);
btf_32_sse4_1_type0(cospi[36], cospi[28], buf1[20], buf1[21], buf0[20],
buf0[21], bit);
btf_32_sse4_1_type0(cospi[52], cospi[12], buf1[22], buf1[23], buf0[22],
buf0[23], bit);
btf_32_sse4_1_type0(-cospi[60], cospi[4], buf1[24], buf1[25], buf0[24],
buf0[25], bit);
btf_32_sse4_1_type0(-cospi[44], cospi[20], buf1[26], buf1[27], buf0[26],
buf0[27], bit);
btf_32_sse4_1_type0(-cospi[28], cospi[36], buf1[28], buf1[29], buf0[28],
buf0[29], bit);
btf_32_sse4_1_type0(-cospi[12], cospi[52], buf1[30], buf1[31], buf0[30],
buf0[31], bit);
// stage 5
stage_idx++;
buf1[0] = _mm_add_epi32(buf0[0], buf0[8]);
buf1[8] = _mm_sub_epi32(buf0[0], buf0[8]);
buf1[1] = _mm_add_epi32(buf0[1], buf0[9]);
buf1[9] = _mm_sub_epi32(buf0[1], buf0[9]);
buf1[2] = _mm_add_epi32(buf0[2], buf0[10]);
buf1[10] = _mm_sub_epi32(buf0[2], buf0[10]);
buf1[3] = _mm_add_epi32(buf0[3], buf0[11]);
buf1[11] = _mm_sub_epi32(buf0[3], buf0[11]);
buf1[4] = _mm_add_epi32(buf0[4], buf0[12]);
buf1[12] = _mm_sub_epi32(buf0[4], buf0[12]);
buf1[5] = _mm_add_epi32(buf0[5], buf0[13]);
buf1[13] = _mm_sub_epi32(buf0[5], buf0[13]);
buf1[6] = _mm_add_epi32(buf0[6], buf0[14]);
buf1[14] = _mm_sub_epi32(buf0[6], buf0[14]);
buf1[7] = _mm_add_epi32(buf0[7], buf0[15]);
buf1[15] = _mm_sub_epi32(buf0[7], buf0[15]);
buf1[16] = _mm_add_epi32(buf0[16], buf0[24]);
buf1[24] = _mm_sub_epi32(buf0[16], buf0[24]);
buf1[17] = _mm_add_epi32(buf0[17], buf0[25]);
buf1[25] = _mm_sub_epi32(buf0[17], buf0[25]);
buf1[18] = _mm_add_epi32(buf0[18], buf0[26]);
buf1[26] = _mm_sub_epi32(buf0[18], buf0[26]);
buf1[19] = _mm_add_epi32(buf0[19], buf0[27]);
buf1[27] = _mm_sub_epi32(buf0[19], buf0[27]);
buf1[20] = _mm_add_epi32(buf0[20], buf0[28]);
buf1[28] = _mm_sub_epi32(buf0[20], buf0[28]);
buf1[21] = _mm_add_epi32(buf0[21], buf0[29]);
buf1[29] = _mm_sub_epi32(buf0[21], buf0[29]);
buf1[22] = _mm_add_epi32(buf0[22], buf0[30]);
buf1[30] = _mm_sub_epi32(buf0[22], buf0[30]);
buf1[23] = _mm_add_epi32(buf0[23], buf0[31]);
buf1[31] = _mm_sub_epi32(buf0[23], buf0[31]);
// stage 6
stage_idx++;
bit = cos_bit[stage_idx];
cospi = cospi_arr[bit - cos_bit_min];
buf0[0] = buf1[0];
buf0[1] = buf1[1];
buf0[2] = buf1[2];
buf0[3] = buf1[3];
buf0[4] = buf1[4];
buf0[5] = buf1[5];
buf0[6] = buf1[6];
buf0[7] = buf1[7];
btf_32_sse4_1_type0(cospi[8], cospi[56], buf1[8], buf1[9], buf0[8], buf0[9],
bit);
btf_32_sse4_1_type0(cospi[40], cospi[24], buf1[10], buf1[11], buf0[10],
buf0[11], bit);
btf_32_sse4_1_type0(-cospi[56], cospi[8], buf1[12], buf1[13], buf0[12],
buf0[13], bit);
btf_32_sse4_1_type0(-cospi[24], cospi[40], buf1[14], buf1[15], buf0[14],
buf0[15], bit);
buf0[16] = buf1[16];
buf0[17] = buf1[17];
buf0[18] = buf1[18];
buf0[19] = buf1[19];
buf0[20] = buf1[20];
buf0[21] = buf1[21];
buf0[22] = buf1[22];
buf0[23] = buf1[23];
btf_32_sse4_1_type0(cospi[8], cospi[56], buf1[24], buf1[25], buf0[24],
buf0[25], bit);
btf_32_sse4_1_type0(cospi[40], cospi[24], buf1[26], buf1[27], buf0[26],
buf0[27], bit);
btf_32_sse4_1_type0(-cospi[56], cospi[8], buf1[28], buf1[29], buf0[28],
buf0[29], bit);
btf_32_sse4_1_type0(-cospi[24], cospi[40], buf1[30], buf1[31], buf0[30],
buf0[31], bit);
// stage 7
stage_idx++;
buf1[0] = _mm_add_epi32(buf0[0], buf0[4]);
buf1[4] = _mm_sub_epi32(buf0[0], buf0[4]);
buf1[1] = _mm_add_epi32(buf0[1], buf0[5]);
buf1[5] = _mm_sub_epi32(buf0[1], buf0[5]);
buf1[2] = _mm_add_epi32(buf0[2], buf0[6]);
buf1[6] = _mm_sub_epi32(buf0[2], buf0[6]);
buf1[3] = _mm_add_epi32(buf0[3], buf0[7]);
buf1[7] = _mm_sub_epi32(buf0[3], buf0[7]);
buf1[8] = _mm_add_epi32(buf0[8], buf0[12]);
buf1[12] = _mm_sub_epi32(buf0[8], buf0[12]);
buf1[9] = _mm_add_epi32(buf0[9], buf0[13]);
buf1[13] = _mm_sub_epi32(buf0[9], buf0[13]);
buf1[10] = _mm_add_epi32(buf0[10], buf0[14]);
buf1[14] = _mm_sub_epi32(buf0[10], buf0[14]);
buf1[11] = _mm_add_epi32(buf0[11], buf0[15]);
buf1[15] = _mm_sub_epi32(buf0[11], buf0[15]);
buf1[16] = _mm_add_epi32(buf0[16], buf0[20]);
buf1[20] = _mm_sub_epi32(buf0[16], buf0[20]);
buf1[17] = _mm_add_epi32(buf0[17], buf0[21]);
buf1[21] = _mm_sub_epi32(buf0[17], buf0[21]);
buf1[18] = _mm_add_epi32(buf0[18], buf0[22]);
buf1[22] = _mm_sub_epi32(buf0[18], buf0[22]);
buf1[19] = _mm_add_epi32(buf0[19], buf0[23]);
buf1[23] = _mm_sub_epi32(buf0[19], buf0[23]);
buf1[24] = _mm_add_epi32(buf0[24], buf0[28]);
buf1[28] = _mm_sub_epi32(buf0[24], buf0[28]);
buf1[25] = _mm_add_epi32(buf0[25], buf0[29]);
buf1[29] = _mm_sub_epi32(buf0[25], buf0[29]);
buf1[26] = _mm_add_epi32(buf0[26], buf0[30]);
buf1[30] = _mm_sub_epi32(buf0[26], buf0[30]);
buf1[27] = _mm_add_epi32(buf0[27], buf0[31]);
buf1[31] = _mm_sub_epi32(buf0[27], buf0[31]);
// stage 8
stage_idx++;
bit = cos_bit[stage_idx];
cospi = cospi_arr[bit - cos_bit_min];
buf0[0] = buf1[0];
buf0[1] = buf1[1];
buf0[2] = buf1[2];
buf0[3] = buf1[3];
btf_32_sse4_1_type0(cospi[16], cospi[48], buf1[4], buf1[5], buf0[4],
buf0[5], bit);
btf_32_sse4_1_type0(-cospi[48], cospi[16], buf1[6], buf1[7], buf0[6],
buf0[7], bit);
buf0[8] = buf1[8];
buf0[9] = buf1[9];
buf0[10] = buf1[10];
buf0[11] = buf1[11];
btf_32_sse4_1_type0(cospi[16], cospi[48], buf1[12], buf1[13], buf0[12],
buf0[13], bit);
btf_32_sse4_1_type0(-cospi[48], cospi[16], buf1[14], buf1[15], buf0[14],
buf0[15], bit);
buf0[16] = buf1[16];
buf0[17] = buf1[17];
buf0[18] = buf1[18];
buf0[19] = buf1[19];
btf_32_sse4_1_type0(cospi[16], cospi[48], buf1[20], buf1[21], buf0[20],
buf0[21], bit);
btf_32_sse4_1_type0(-cospi[48], cospi[16], buf1[22], buf1[23], buf0[22],
buf0[23], bit);
buf0[24] = buf1[24];
buf0[25] = buf1[25];
buf0[26] = buf1[26];
buf0[27] = buf1[27];
btf_32_sse4_1_type0(cospi[16], cospi[48], buf1[28], buf1[29], buf0[28],
buf0[29], bit);
btf_32_sse4_1_type0(-cospi[48], cospi[16], buf1[30], buf1[31], buf0[30],
buf0[31], bit);
// stage 9
stage_idx++;
buf1[0] = _mm_add_epi32(buf0[0], buf0[2]);
buf1[2] = _mm_sub_epi32(buf0[0], buf0[2]);
buf1[1] = _mm_add_epi32(buf0[1], buf0[3]);
buf1[3] = _mm_sub_epi32(buf0[1], buf0[3]);
buf1[4] = _mm_add_epi32(buf0[4], buf0[6]);
buf1[6] = _mm_sub_epi32(buf0[4], buf0[6]);
buf1[5] = _mm_add_epi32(buf0[5], buf0[7]);
buf1[7] = _mm_sub_epi32(buf0[5], buf0[7]);
buf1[8] = _mm_add_epi32(buf0[8], buf0[10]);
buf1[10] = _mm_sub_epi32(buf0[8], buf0[10]);
buf1[9] = _mm_add_epi32(buf0[9], buf0[11]);
buf1[11] = _mm_sub_epi32(buf0[9], buf0[11]);
buf1[12] = _mm_add_epi32(buf0[12], buf0[14]);
buf1[14] = _mm_sub_epi32(buf0[12], buf0[14]);
buf1[13] = _mm_add_epi32(buf0[13], buf0[15]);
buf1[15] = _mm_sub_epi32(buf0[13], buf0[15]);
buf1[16] = _mm_add_epi32(buf0[16], buf0[18]);
buf1[18] = _mm_sub_epi32(buf0[16], buf0[18]);
buf1[17] = _mm_add_epi32(buf0[17], buf0[19]);
buf1[19] = _mm_sub_epi32(buf0[17], buf0[19]);
buf1[20] = _mm_add_epi32(buf0[20], buf0[22]);
buf1[22] = _mm_sub_epi32(buf0[20], buf0[22]);
buf1[21] = _mm_add_epi32(buf0[21], buf0[23]);
buf1[23] = _mm_sub_epi32(buf0[21], buf0[23]);
buf1[24] = _mm_add_epi32(buf0[24], buf0[26]);
buf1[26] = _mm_sub_epi32(buf0[24], buf0[26]);
buf1[25] = _mm_add_epi32(buf0[25], buf0[27]);
buf1[27] = _mm_sub_epi32(buf0[25], buf0[27]);
buf1[28] = _mm_add_epi32(buf0[28], buf0[30]);
buf1[30] = _mm_sub_epi32(buf0[28], buf0[30]);
buf1[29] = _mm_add_epi32(buf0[29], buf0[31]);
buf1[31] = _mm_sub_epi32(buf0[29], buf0[31]);
// stage 10
stage_idx++;
bit = cos_bit[stage_idx];
cospi = cospi_arr[bit - cos_bit_min];
buf0[0] = buf1[0];
buf0[1] = buf1[1];
btf_32_sse4_1_type0(cospi[32], cospi[32], buf1[2], buf1[3], buf0[2],
buf0[3], bit);
buf0[4] = buf1[4];
buf0[5] = buf1[5];
btf_32_sse4_1_type0(cospi[32], cospi[32], buf1[6], buf1[7], buf0[6],
buf0[7], bit);
buf0[8] = buf1[8];
buf0[9] = buf1[9];
btf_32_sse4_1_type0(cospi[32], cospi[32], buf1[10], buf1[11], buf0[10],
buf0[11], bit);
buf0[12] = buf1[12];
buf0[13] = buf1[13];
btf_32_sse4_1_type0(cospi[32], cospi[32], buf1[14], buf1[15], buf0[14],
buf0[15], bit);
buf0[16] = buf1[16];
buf0[17] = buf1[17];
btf_32_sse4_1_type0(cospi[32], cospi[32], buf1[18], buf1[19], buf0[18],
buf0[19], bit);
buf0[20] = buf1[20];
buf0[21] = buf1[21];
btf_32_sse4_1_type0(cospi[32], cospi[32], buf1[22], buf1[23], buf0[22],
buf0[23], bit);
buf0[24] = buf1[24];
buf0[25] = buf1[25];
btf_32_sse4_1_type0(cospi[32], cospi[32], buf1[26], buf1[27], buf0[26],
buf0[27], bit);
buf0[28] = buf1[28];
buf0[29] = buf1[29];
btf_32_sse4_1_type0(cospi[32], cospi[32], buf1[30], buf1[31], buf0[30],
buf0[31], bit);
// stage 11
stage_idx++;
buf1[0] = buf0[0];
buf1[1] = _mm_sub_epi32(_mm_set1_epi32(0), buf0[16]);
buf1[2] = buf0[24];
buf1[3] = _mm_sub_epi32(_mm_set1_epi32(0), buf0[8]);
buf1[4] = buf0[12];
buf1[5] = _mm_sub_epi32(_mm_set1_epi32(0), buf0[28]);
buf1[6] = buf0[20];
buf1[7] = _mm_sub_epi32(_mm_set1_epi32(0), buf0[4]);
buf1[8] = buf0[6];
buf1[9] = _mm_sub_epi32(_mm_set1_epi32(0), buf0[22]);
buf1[10] = buf0[30];
buf1[11] = _mm_sub_epi32(_mm_set1_epi32(0), buf0[14]);
buf1[12] = buf0[10];
buf1[13] = _mm_sub_epi32(_mm_set1_epi32(0), buf0[26]);
buf1[14] = buf0[18];
buf1[15] = _mm_sub_epi32(_mm_set1_epi32(0), buf0[2]);
buf1[16] = buf0[3];
buf1[17] = _mm_sub_epi32(_mm_set1_epi32(0), buf0[19]);
buf1[18] = buf0[27];
buf1[19] = _mm_sub_epi32(_mm_set1_epi32(0), buf0[11]);
buf1[20] = buf0[15];
buf1[21] = _mm_sub_epi32(_mm_set1_epi32(0), buf0[31]);
buf1[22] = buf0[23];
buf1[23] = _mm_sub_epi32(_mm_set1_epi32(0), buf0[7]);
buf1[24] = buf0[5];
buf1[25] = _mm_sub_epi32(_mm_set1_epi32(0), buf0[21]);
buf1[26] = buf0[29];
buf1[27] = _mm_sub_epi32(_mm_set1_epi32(0), buf0[13]);
buf1[28] = buf0[9];
buf1[29] = _mm_sub_epi32(_mm_set1_epi32(0), buf0[25]);
buf1[30] = buf0[17];
buf1[31] = _mm_sub_epi32(_mm_set1_epi32(0), buf0[1]);
for (j = 0; j < 32; ++j) {
output[j * col_num + col] = buf1[j];
}
}
}

81
third_party/aom/av1/common/x86/av1_fwd_txfm2d_sse4.c vendored Normal file
View file

@ -0,0 +1,81 @@
/*
* Copyright (c) 2016, Alliance for Open Media. All rights reserved
*
* This source code is subject to the terms of the BSD 2 Clause License and
* the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
* was not distributed with this source code in the LICENSE file, you can
* obtain it at www.aomedia.org/license/software. If the Alliance for Open
* Media Patent License 1.0 was not distributed with this source code in the
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
*/
#include "./av1_rtcd.h"
#include "av1/common/enums.h"
#include "av1/common/av1_txfm.h"
#include "av1/common/x86/av1_txfm1d_sse4.h"
static INLINE void int16_array_with_stride_to_int32_array_without_stride(
const int16_t *input, int stride, int32_t *output, int txfm1d_size) {
int r, c;
for (r = 0; r < txfm1d_size; r++) {
for (c = 0; c < txfm1d_size; c++) {
output[r * txfm1d_size + c] = (int32_t)input[r * stride + c];
}
}
}
typedef void (*TxfmFuncSSE2)(const __m128i *input, __m128i *output,
const int8_t *cos_bit, const int8_t *stage_range);
static INLINE TxfmFuncSSE2 fwd_txfm_type_to_func(TXFM_TYPE txfm_type) {
switch (txfm_type) {
case TXFM_TYPE_DCT32: return av1_fdct32_new_sse4_1; break;
case TXFM_TYPE_ADST32: return av1_fadst32_new_sse4_1; break;
default: assert(0);
}
return NULL;
}
static INLINE void fwd_txfm2d_sse4_1(const int16_t *input, int32_t *output,
const int stride, const TXFM_2D_CFG *cfg,
int32_t *txfm_buf) {
const int txfm_size = cfg->txfm_size;
const int8_t *shift = cfg->shift;
const int8_t *stage_range_col = cfg->stage_range_col;
const int8_t *stage_range_row = cfg->stage_range_row;
const int8_t *cos_bit_col = cfg->cos_bit_col;
const int8_t *cos_bit_row = cfg->cos_bit_row;
const TxfmFuncSSE2 txfm_func_col = fwd_txfm_type_to_func(cfg->txfm_type_col);
const TxfmFuncSSE2 txfm_func_row = fwd_txfm_type_to_func(cfg->txfm_type_row);
__m128i *buf_128 = (__m128i *)txfm_buf;
__m128i *out_128 = (__m128i *)output;
int num_per_128 = 4;
int txfm2d_size_128 = txfm_size * txfm_size / num_per_128;
int16_array_with_stride_to_int32_array_without_stride(input, stride, txfm_buf,
txfm_size);
round_shift_array_32_sse4_1(buf_128, out_128, txfm2d_size_128, -shift[0]);
txfm_func_col(out_128, buf_128, cos_bit_col, stage_range_col);
round_shift_array_32_sse4_1(buf_128, out_128, txfm2d_size_128, -shift[1]);
transpose_32(txfm_size, out_128, buf_128);
txfm_func_row(buf_128, out_128, cos_bit_row, stage_range_row);
round_shift_array_32_sse4_1(out_128, buf_128, txfm2d_size_128, -shift[2]);
transpose_32(txfm_size, buf_128, out_128);
}
void av1_fwd_txfm2d_32x32_sse4_1(const int16_t *input, int32_t *output,
int stride, int tx_type, int bd) {
DECLARE_ALIGNED(16, int32_t, txfm_buf[1024]);
TXFM_2D_FLIP_CFG cfg = av1_get_fwd_txfm_cfg(tx_type, TX_32X32);
(void)bd;
fwd_txfm2d_sse4_1(input, output, stride, cfg.cfg, txfm_buf);
}
void av1_fwd_txfm2d_64x64_sse4_1(const int16_t *input, int32_t *output,
int stride, int tx_type, int bd) {
DECLARE_ALIGNED(16, int32_t, txfm_buf[4096]);
TXFM_2D_FLIP_CFG cfg = av1_get_fwd_txfm_64x64_cfg(tx_type);
(void)bd;
fwd_txfm2d_sse4_1(input, output, stride, cfg.cfg, txfm_buf);
}

533
third_party/aom/av1/common/x86/av1_highbd_convolve_sse4.c vendored Normal file
View file

@ -0,0 +1,533 @@
/*
* Copyright (c) 2016, Alliance for Open Media. All rights reserved
*
* This source code is subject to the terms of the BSD 2 Clause License and
* the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
* was not distributed with this source code in the LICENSE file, you can
* obtain it at www.aomedia.org/license/software. If the Alliance for Open
* Media Patent License 1.0 was not distributed with this source code in the
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
*/
#include <assert.h>
#include <smmintrin.h>
#include "./av1_rtcd.h"
#include "av1/common/filter.h"
#if CONFIG_DUAL_FILTER
DECLARE_ALIGNED(16, static int16_t, subpel_filters_sharp[15][6][8]);
#endif
#if USE_TEMPORALFILTER_12TAP
DECLARE_ALIGNED(16, static int16_t, subpel_temporalfilter[15][6][8]);
#endif
typedef int16_t (*HbdSubpelFilterCoeffs)[8];
typedef void (*TransposeSave)(int width, int pixelsNum, uint32_t *src,
int src_stride, uint16_t *dst, int dst_stride,
int bd);
static INLINE HbdSubpelFilterCoeffs
hbd_get_subpel_filter_ver_signal_dir(const InterpFilterParams p, int index) {
#if CONFIG_DUAL_FILTER
if (p.interp_filter == MULTITAP_SHARP) {
return &subpel_filters_sharp[index][0];
}
#endif
#if USE_TEMPORALFILTER_12TAP
if (p.interp_filter == TEMPORALFILTER_12TAP) {
return &subpel_temporalfilter[index][0];
}
#endif
(void)p;
(void)index;
return NULL;
}
static void init_simd_filter(const int16_t *filter_ptr, int taps,
int16_t (*simd_filter)[6][8]) {
int shift;
int offset = (12 - taps) / 2;
for (shift = 1; shift < SUBPEL_SHIFTS; ++shift) {
const int16_t *filter_row = filter_ptr + shift * taps;
int i, j;
for (i = 0; i < 12; ++i) {
for (j = 0; j < 4; ++j) {
int r = i / 2;
int c = j * 2 + (i % 2);
if (i - offset >= 0 && i - offset < taps)
simd_filter[shift - 1][r][c] = filter_row[i - offset];
else
simd_filter[shift - 1][r][c] = 0;
}
}
}
}
void av1_highbd_convolve_init_sse4_1(void) {
#if USE_TEMPORALFILTER_12TAP
{
InterpFilterParams filter_params =
av1_get_interp_filter_params(TEMPORALFILTER_12TAP);
int taps = filter_params.taps;
const int16_t *filter_ptr = filter_params.filter_ptr;
init_simd_filter(filter_ptr, taps, subpel_temporalfilter);
}
#endif
#if CONFIG_DUAL_FILTER
{
InterpFilterParams filter_params =
av1_get_interp_filter_params(MULTITAP_SHARP);
int taps = filter_params.taps;
const int16_t *filter_ptr = filter_params.filter_ptr;
init_simd_filter(filter_ptr, taps, subpel_filters_sharp);
}
#endif
}
// pixelsNum 0: write all 4 pixels
// 1/2/3: residual pixels 1/2/3
static void writePixel(__m128i *u, int width, int pixelsNum, uint16_t *dst,
int dst_stride) {
if (2 == width) {
if (0 == pixelsNum) {
*(int *)dst = _mm_cvtsi128_si32(u[0]);
*(int *)(dst + dst_stride) = _mm_cvtsi128_si32(u[1]);
*(int *)(dst + 2 * dst_stride) = _mm_cvtsi128_si32(u[2]);
*(int *)(dst + 3 * dst_stride) = _mm_cvtsi128_si32(u[3]);
} else if (1 == pixelsNum) {
*(int *)dst = _mm_cvtsi128_si32(u[0]);
} else if (2 == pixelsNum) {
*(int *)dst = _mm_cvtsi128_si32(u[0]);
*(int *)(dst + dst_stride) = _mm_cvtsi128_si32(u[1]);
} else if (3 == pixelsNum) {
*(int *)dst = _mm_cvtsi128_si32(u[0]);
*(int *)(dst + dst_stride) = _mm_cvtsi128_si32(u[1]);
*(int *)(dst + 2 * dst_stride) = _mm_cvtsi128_si32(u[2]);
}
} else {
if (0 == pixelsNum) {
_mm_storel_epi64((__m128i *)dst, u[0]);
_mm_storel_epi64((__m128i *)(dst + dst_stride), u[1]);
_mm_storel_epi64((__m128i *)(dst + 2 * dst_stride), u[2]);
_mm_storel_epi64((__m128i *)(dst + 3 * dst_stride), u[3]);
} else if (1 == pixelsNum) {
_mm_storel_epi64((__m128i *)dst, u[0]);
} else if (2 == pixelsNum) {
_mm_storel_epi64((__m128i *)dst, u[0]);
_mm_storel_epi64((__m128i *)(dst + dst_stride), u[1]);
} else if (3 == pixelsNum) {
_mm_storel_epi64((__m128i *)dst, u[0]);
_mm_storel_epi64((__m128i *)(dst + dst_stride), u[1]);
_mm_storel_epi64((__m128i *)(dst + 2 * dst_stride), u[2]);
}
}
}
// 16-bit pixels clip with bd (10/12)
static void highbd_clip(__m128i *p, int numVecs, int bd) {
const __m128i zero = _mm_setzero_si128();
const __m128i one = _mm_set1_epi16(1);
const __m128i max = _mm_sub_epi16(_mm_slli_epi16(one, bd), one);
__m128i clamped, mask;
int i;
for (i = 0; i < numVecs; i++) {
mask = _mm_cmpgt_epi16(p[i], max);
clamped = _mm_andnot_si128(mask, p[i]);
mask = _mm_and_si128(mask, max);
clamped = _mm_or_si128(mask, clamped);
mask = _mm_cmpgt_epi16(clamped, zero);
p[i] = _mm_and_si128(clamped, mask);
}
}
static void transClipPixel(uint32_t *src, int src_stride, __m128i *u, int bd) {
__m128i v0, v1;
__m128i rnd = _mm_set1_epi32(1 << (FILTER_BITS - 1));
u[0] = _mm_loadu_si128((__m128i const *)src);
u[1] = _mm_loadu_si128((__m128i const *)(src + src_stride));
u[2] = _mm_loadu_si128((__m128i const *)(src + 2 * src_stride));
u[3] = _mm_loadu_si128((__m128i const *)(src + 3 * src_stride));
u[0] = _mm_add_epi32(u[0], rnd);
u[1] = _mm_add_epi32(u[1], rnd);
u[2] = _mm_add_epi32(u[2], rnd);
u[3] = _mm_add_epi32(u[3], rnd);
u[0] = _mm_srai_epi32(u[0], FILTER_BITS);
u[1] = _mm_srai_epi32(u[1], FILTER_BITS);
u[2] = _mm_srai_epi32(u[2], FILTER_BITS);
u[3] = _mm_srai_epi32(u[3], FILTER_BITS);
u[0] = _mm_packus_epi32(u[0], u[1]);
u[1] = _mm_packus_epi32(u[2], u[3]);
highbd_clip(u, 2, bd);
v0 = _mm_unpacklo_epi16(u[0], u[1]);
v1 = _mm_unpackhi_epi16(u[0], u[1]);
u[0] = _mm_unpacklo_epi16(v0, v1);
u[2] = _mm_unpackhi_epi16(v0, v1);
u[1] = _mm_srli_si128(u[0], 8);
u[3] = _mm_srli_si128(u[2], 8);
}
// pixelsNum = 0 : all 4 rows of pixels will be saved.
// pixelsNum = 1/2/3 : residual 1/2/4 rows of pixels will be saved.
void trans_save_4x4(int width, int pixelsNum, uint32_t *src, int src_stride,
uint16_t *dst, int dst_stride, int bd) {
__m128i u[4];
transClipPixel(src, src_stride, u, bd);
writePixel(u, width, pixelsNum, dst, dst_stride);
}
void trans_accum_save_4x4(int width, int pixelsNum, uint32_t *src,
int src_stride, uint16_t *dst, int dst_stride,
int bd) {
__m128i u[4], v[4];
const __m128i ones = _mm_set1_epi16(1);
transClipPixel(src, src_stride, u, bd);
v[0] = _mm_loadl_epi64((__m128i const *)dst);
v[1] = _mm_loadl_epi64((__m128i const *)(dst + dst_stride));
v[2] = _mm_loadl_epi64((__m128i const *)(dst + 2 * dst_stride));
v[3] = _mm_loadl_epi64((__m128i const *)(dst + 3 * dst_stride));
u[0] = _mm_add_epi16(u[0], v[0]);
u[1] = _mm_add_epi16(u[1], v[1]);
u[2] = _mm_add_epi16(u[2], v[2]);
u[3] = _mm_add_epi16(u[3], v[3]);
u[0] = _mm_add_epi16(u[0], ones);
u[1] = _mm_add_epi16(u[1], ones);
u[2] = _mm_add_epi16(u[2], ones);
u[3] = _mm_add_epi16(u[3], ones);
u[0] = _mm_srai_epi16(u[0], 1);
u[1] = _mm_srai_epi16(u[1], 1);
u[2] = _mm_srai_epi16(u[2], 1);
u[3] = _mm_srai_epi16(u[3], 1);
writePixel(u, width, pixelsNum, dst, dst_stride);
}
static TransposeSave transSaveTab[2] = { trans_save_4x4, trans_accum_save_4x4 };
static INLINE void transpose_pair(__m128i *in, __m128i *out) {
__m128i x0, x1;
x0 = _mm_unpacklo_epi32(in[0], in[1]);
x1 = _mm_unpacklo_epi32(in[2], in[3]);
out[0] = _mm_unpacklo_epi64(x0, x1);
out[1] = _mm_unpackhi_epi64(x0, x1);
x0 = _mm_unpackhi_epi32(in[0], in[1]);
x1 = _mm_unpackhi_epi32(in[2], in[3]);
out[2] = _mm_unpacklo_epi64(x0, x1);
out[3] = _mm_unpackhi_epi64(x0, x1);
x0 = _mm_unpacklo_epi32(in[4], in[5]);
x1 = _mm_unpacklo_epi32(in[6], in[7]);
out[4] = _mm_unpacklo_epi64(x0, x1);
out[5] = _mm_unpackhi_epi64(x0, x1);
}
static void highbd_filter_horiz(const uint16_t *src, int src_stride, __m128i *f,
int tapsNum, uint32_t *buf) {
__m128i u[8], v[6];
if (tapsNum == 10) {
src -= 1;
}
u[0] = _mm_loadu_si128((__m128i const *)src);
u[1] = _mm_loadu_si128((__m128i const *)(src + src_stride));
u[2] = _mm_loadu_si128((__m128i const *)(src + 2 * src_stride));
u[3] = _mm_loadu_si128((__m128i const *)(src + 3 * src_stride));
u[4] = _mm_loadu_si128((__m128i const *)(src + 8));
u[5] = _mm_loadu_si128((__m128i const *)(src + src_stride + 8));
u[6] = _mm_loadu_si128((__m128i const *)(src + 2 * src_stride + 8));
u[7] = _mm_loadu_si128((__m128i const *)(src + 3 * src_stride + 8));
transpose_pair(u, v);
u[0] = _mm_madd_epi16(v[0], f[0]);
u[1] = _mm_madd_epi16(v[1], f[1]);
u[2] = _mm_madd_epi16(v[2], f[2]);
u[3] = _mm_madd_epi16(v[3], f[3]);
u[4] = _mm_madd_epi16(v[4], f[4]);
u[5] = _mm_madd_epi16(v[5], f[5]);
u[6] = _mm_min_epi32(u[2], u[3]);
u[7] = _mm_max_epi32(u[2], u[3]);
u[0] = _mm_add_epi32(u[0], u[1]);
u[0] = _mm_add_epi32(u[0], u[5]);
u[0] = _mm_add_epi32(u[0], u[4]);
u[0] = _mm_add_epi32(u[0], u[6]);
u[0] = _mm_add_epi32(u[0], u[7]);
_mm_storeu_si128((__m128i *)buf, u[0]);
}
void av1_highbd_convolve_horiz_sse4_1(const uint16_t *src, int src_stride,
uint16_t *dst, int dst_stride, int w,
int h,
const InterpFilterParams filter_params,
const int subpel_x_q4, int x_step_q4,
int avg, int bd) {
DECLARE_ALIGNED(16, uint32_t, temp[4 * 4]);
__m128i verf[6];
HbdSubpelFilterCoeffs vCoeffs;
const uint16_t *srcPtr;
const int tapsNum = filter_params.taps;
int i, col, count, blkResidu, blkHeight;
TransposeSave transSave = transSaveTab[avg];
(void)x_step_q4;
if (0 == subpel_x_q4 || 16 != x_step_q4) {
av1_highbd_convolve_horiz_c(src, src_stride, dst, dst_stride, w, h,
filter_params, subpel_x_q4, x_step_q4, avg, bd);
return;
}
vCoeffs =
hbd_get_subpel_filter_ver_signal_dir(filter_params, subpel_x_q4 - 1);
if (!vCoeffs) {
av1_highbd_convolve_horiz_c(src, src_stride, dst, dst_stride, w, h,
filter_params, subpel_x_q4, x_step_q4, avg, bd);
return;
}
verf[0] = *((const __m128i *)(vCoeffs));
verf[1] = *((const __m128i *)(vCoeffs + 1));
verf[2] = *((const __m128i *)(vCoeffs + 2));
verf[3] = *((const __m128i *)(vCoeffs + 3));
verf[4] = *((const __m128i *)(vCoeffs + 4));
verf[5] = *((const __m128i *)(vCoeffs + 5));
src -= (tapsNum >> 1) - 1;
srcPtr = src;
count = 0;
blkHeight = h >> 2;
blkResidu = h & 3;
while (blkHeight != 0) {
for (col = 0; col < w; col += 4) {
for (i = 0; i < 4; ++i) {
highbd_filter_horiz(srcPtr, src_stride, verf, tapsNum, temp + (i * 4));
srcPtr += 1;
}
transSave(w, 0, temp, 4, dst + col, dst_stride, bd);
}
count++;
srcPtr = src + count * src_stride * 4;
dst += dst_stride * 4;
blkHeight--;
}
if (blkResidu == 0) return;
for (col = 0; col < w; col += 4) {
for (i = 0; i < 4; ++i) {
highbd_filter_horiz(srcPtr, src_stride, verf, tapsNum, temp + (i * 4));
srcPtr += 1;
}
transSave(w, blkResidu, temp, 4, dst + col, dst_stride, bd);
}
}
// Vertical convolutional filter
typedef void (*WritePixels)(__m128i *u, int bd, uint16_t *dst);
static void highbdRndingPacks(__m128i *u) {
__m128i rnd = _mm_set1_epi32(1 << (FILTER_BITS - 1));
u[0] = _mm_add_epi32(u[0], rnd);
u[0] = _mm_srai_epi32(u[0], FILTER_BITS);
u[0] = _mm_packus_epi32(u[0], u[0]);
}
static void write2pixelsOnly(__m128i *u, int bd, uint16_t *dst) {
highbdRndingPacks(u);
highbd_clip(u, 1, bd);
*(uint32_t *)dst = _mm_cvtsi128_si32(u[0]);
}
static void write2pixelsAccum(__m128i *u, int bd, uint16_t *dst) {
__m128i v = _mm_loadl_epi64((__m128i const *)dst);
const __m128i ones = _mm_set1_epi16(1);
highbdRndingPacks(u);
highbd_clip(u, 1, bd);
v = _mm_add_epi16(v, u[0]);
v = _mm_add_epi16(v, ones);
v = _mm_srai_epi16(v, 1);
*(uint32_t *)dst = _mm_cvtsi128_si32(v);
}
WritePixels write2pixelsTab[2] = { write2pixelsOnly, write2pixelsAccum };
static void write4pixelsOnly(__m128i *u, int bd, uint16_t *dst) {
highbdRndingPacks(u);
highbd_clip(u, 1, bd);
_mm_storel_epi64((__m128i *)dst, u[0]);
}
static void write4pixelsAccum(__m128i *u, int bd, uint16_t *dst) {
__m128i v = _mm_loadl_epi64((__m128i const *)dst);
const __m128i ones = _mm_set1_epi16(1);
highbdRndingPacks(u);
highbd_clip(u, 1, bd);
v = _mm_add_epi16(v, u[0]);
v = _mm_add_epi16(v, ones);
v = _mm_srai_epi16(v, 1);
_mm_storel_epi64((__m128i *)dst, v);
}
WritePixels write4pixelsTab[2] = { write4pixelsOnly, write4pixelsAccum };
static void filter_vert_horiz_parallel(const uint16_t *src, int src_stride,
const __m128i *f, int taps,
uint16_t *dst, WritePixels saveFunc,
int bd) {
__m128i s[12];
__m128i zero = _mm_setzero_si128();
int i = 0;
int r = 0;
// TODO(luoyi) treat s[12] as a circular buffer in width = 2 case
if (10 == taps) {
i += 1;
s[0] = zero;
}
while (i < 12) {
s[i] = _mm_loadu_si128((__m128i const *)(src + r * src_stride));
i += 1;
r += 1;
}
s[0] = _mm_unpacklo_epi16(s[0], s[1]);
s[2] = _mm_unpacklo_epi16(s[2], s[3]);
s[4] = _mm_unpacklo_epi16(s[4], s[5]);
s[6] = _mm_unpacklo_epi16(s[6], s[7]);
s[8] = _mm_unpacklo_epi16(s[8], s[9]);
s[10] = _mm_unpacklo_epi16(s[10], s[11]);
s[0] = _mm_madd_epi16(s[0], f[0]);
s[2] = _mm_madd_epi16(s[2], f[1]);
s[4] = _mm_madd_epi16(s[4], f[2]);
s[6] = _mm_madd_epi16(s[6], f[3]);
s[8] = _mm_madd_epi16(s[8], f[4]);
s[10] = _mm_madd_epi16(s[10], f[5]);
s[1] = _mm_min_epi32(s[4], s[6]);
s[3] = _mm_max_epi32(s[4], s[6]);
s[0] = _mm_add_epi32(s[0], s[2]);
s[0] = _mm_add_epi32(s[0], s[10]);
s[0] = _mm_add_epi32(s[0], s[8]);
s[0] = _mm_add_epi32(s[0], s[1]);
s[0] = _mm_add_epi32(s[0], s[3]);
saveFunc(s, bd, dst);
}
static void highbd_filter_vert_compute_large(const uint16_t *src,
int src_stride, const __m128i *f,
int taps, int w, int h,
uint16_t *dst, int dst_stride,
int avg, int bd) {
int col;
int rowIndex = 0;
const uint16_t *src_ptr = src;
uint16_t *dst_ptr = dst;
const int step = 4;
WritePixels write4pixels = write4pixelsTab[avg];
do {
for (col = 0; col < w; col += step) {
filter_vert_horiz_parallel(src_ptr, src_stride, f, taps, dst_ptr,
write4pixels, bd);
src_ptr += step;
dst_ptr += step;
}
rowIndex++;
src_ptr = src + rowIndex * src_stride;
dst_ptr = dst + rowIndex * dst_stride;
} while (rowIndex < h);
}
static void highbd_filter_vert_compute_small(const uint16_t *src,
int src_stride, const __m128i *f,
int taps, int w, int h,
uint16_t *dst, int dst_stride,
int avg, int bd) {
int rowIndex = 0;
WritePixels write2pixels = write2pixelsTab[avg];
(void)w;
do {
filter_vert_horiz_parallel(src, src_stride, f, taps, dst, write2pixels, bd);
rowIndex++;
src += src_stride;
dst += dst_stride;
} while (rowIndex < h);
}
void av1_highbd_convolve_vert_sse4_1(const uint16_t *src, int src_stride,
uint16_t *dst, int dst_stride, int w,
int h,
const InterpFilterParams filter_params,
const int subpel_y_q4, int y_step_q4,
int avg, int bd) {
__m128i verf[6];
HbdSubpelFilterCoeffs vCoeffs;
const int tapsNum = filter_params.taps;
if (0 == subpel_y_q4 || 16 != y_step_q4) {
av1_highbd_convolve_vert_c(src, src_stride, dst, dst_stride, w, h,
filter_params, subpel_y_q4, y_step_q4, avg, bd);
return;
}
vCoeffs =
hbd_get_subpel_filter_ver_signal_dir(filter_params, subpel_y_q4 - 1);
if (!vCoeffs) {
av1_highbd_convolve_vert_c(src, src_stride, dst, dst_stride, w, h,
filter_params, subpel_y_q4, y_step_q4, avg, bd);
return;
}
verf[0] = *((const __m128i *)(vCoeffs));
verf[1] = *((const __m128i *)(vCoeffs + 1));
verf[2] = *((const __m128i *)(vCoeffs + 2));
verf[3] = *((const __m128i *)(vCoeffs + 3));
verf[4] = *((const __m128i *)(vCoeffs + 4));
verf[5] = *((const __m128i *)(vCoeffs + 5));
src -= src_stride * ((tapsNum >> 1) - 1);
if (w > 2) {
highbd_filter_vert_compute_large(src, src_stride, verf, tapsNum, w, h, dst,
dst_stride, avg, bd);
} else {
highbd_filter_vert_compute_small(src, src_stride, verf, tapsNum, w, h, dst,
dst_stride, avg, bd);
}
}

144
third_party/aom/av1/common/x86/av1_txfm1d_sse4.h vendored Normal file
View file

@ -0,0 +1,144 @@
#ifndef AV1_TXMF1D_SSE2_H_
#define AV1_TXMF1D_SSE2_H_
#include <smmintrin.h>
#include "av1/common/av1_txfm.h"
#ifdef __cplusplus
extern "C" {
#endif
void av1_fdct4_new_sse4_1(const __m128i *input, __m128i *output,
const int8_t *cos_bit, const int8_t *stage_range);
void av1_fdct8_new_sse4_1(const __m128i *input, __m128i *output,
const int8_t *cos_bit, const int8_t *stage_range);
void av1_fdct16_new_sse4_1(const __m128i *input, __m128i *output,
const int8_t *cos_bit, const int8_t *stage_range);
void av1_fdct32_new_sse4_1(const __m128i *input, __m128i *output,
const int8_t *cos_bit, const int8_t *stage_range);
void av1_fdct64_new_sse4_1(const __m128i *input, __m128i *output,
const int8_t *cos_bit, const int8_t *stage_range);
void av1_fadst4_new_sse4_1(const __m128i *input, __m128i *output,
const int8_t *cos_bit, const int8_t *stage_range);
void av1_fadst8_new_sse4_1(const __m128i *input, __m128i *output,
const int8_t *cos_bit, const int8_t *stage_range);
void av1_fadst16_new_sse4_1(const __m128i *input, __m128i *output,
const int8_t *cos_bit, const int8_t *stage_range);
void av1_fadst32_new_sse4_1(const __m128i *input, __m128i *output,
const int8_t *cos_bit, const int8_t *stage_range);
void av1_idct4_new_sse4_1(const __m128i *input, __m128i *output,
const int8_t *cos_bit, const int8_t *stage_range);
void av1_idct8_new_sse4_1(const __m128i *input, __m128i *output,
const int8_t *cos_bit, const int8_t *stage_range);
void av1_idct16_new_sse4_1(const __m128i *input, __m128i *output,
const int8_t *cos_bit, const int8_t *stage_range);
void av1_idct32_new_sse4_1(const __m128i *input, __m128i *output,
const int8_t *cos_bit, const int8_t *stage_range);
void av1_idct64_new_sse4_1(const __m128i *input, __m128i *output,
const int8_t *cos_bit, const int8_t *stage_range);
void av1_iadst4_new_sse4_1(const __m128i *input, __m128i *output,
const int8_t *cos_bit, const int8_t *stage_range);
void av1_iadst8_new_sse4_1(const __m128i *input, __m128i *output,
const int8_t *cos_bit, const int8_t *stage_range);
void av1_iadst16_new_sse4_1(const __m128i *input, __m128i *output,
const int8_t *cos_bit, const int8_t *stage_range);
void av1_iadst32_new_sse4_1(const __m128i *input, __m128i *output,
const int8_t *cos_bit, const int8_t *stage_range);
static INLINE void transpose_32_4x4(int stride, const __m128i *input,
__m128i *output) {
__m128i temp0 = _mm_unpacklo_epi32(input[0 * stride], input[2 * stride]);
__m128i temp1 = _mm_unpackhi_epi32(input[0 * stride], input[2 * stride]);
__m128i temp2 = _mm_unpacklo_epi32(input[1 * stride], input[3 * stride]);
__m128i temp3 = _mm_unpackhi_epi32(input[1 * stride], input[3 * stride]);
output[0 * stride] = _mm_unpacklo_epi32(temp0, temp2);
output[1 * stride] = _mm_unpackhi_epi32(temp0, temp2);
output[2 * stride] = _mm_unpacklo_epi32(temp1, temp3);
output[3 * stride] = _mm_unpackhi_epi32(temp1, temp3);
}
// the entire input block can be represent by a grid of 4x4 blocks
// each 4x4 blocks can be represent by 4 vertical __m128i
// we first transpose each 4x4 block internally
// than transpose the grid
static INLINE void transpose_32(int txfm_size, const __m128i *input,
__m128i *output) {
const int num_per_128 = 4;
const int row_size = txfm_size;
const int col_size = txfm_size / num_per_128;
int r, c;
// transpose each 4x4 block internally
for (r = 0; r < row_size; r += 4) {
for (c = 0; c < col_size; c++) {
transpose_32_4x4(col_size, &input[r * col_size + c],
&output[c * 4 * col_size + r / 4]);
}
}
}
static INLINE __m128i round_shift_32_sse4_1(__m128i vec, int bit) {
__m128i tmp, round;
round = _mm_set1_epi32(1 << (bit - 1));
tmp = _mm_add_epi32(vec, round);
return _mm_srai_epi32(tmp, bit);
}
static INLINE void round_shift_array_32_sse4_1(__m128i *input, __m128i *output,
const int size, const int bit) {
if (bit > 0) {
int i;
for (i = 0; i < size; i++) {
output[i] = round_shift_32_sse4_1(input[i], bit);
}
} else {
int i;
for (i = 0; i < size; i++) {
output[i] = _mm_slli_epi32(input[i], -bit);
}
}
}
// out0 = in0*w0 + in1*w1
// out1 = -in1*w0 + in0*w1
#define btf_32_sse4_1_type0(w0, w1, in0, in1, out0, out1, bit) \
do { \
__m128i ww0, ww1, in0_w0, in1_w1, in0_w1, in1_w0; \
ww0 = _mm_set1_epi32(w0); \
ww1 = _mm_set1_epi32(w1); \
in0_w0 = _mm_mullo_epi32(in0, ww0); \
in1_w1 = _mm_mullo_epi32(in1, ww1); \
out0 = _mm_add_epi32(in0_w0, in1_w1); \
out0 = round_shift_32_sse4_1(out0, bit); \
in0_w1 = _mm_mullo_epi32(in0, ww1); \
in1_w0 = _mm_mullo_epi32(in1, ww0); \
out1 = _mm_sub_epi32(in0_w1, in1_w0); \
out1 = round_shift_32_sse4_1(out1, bit); \
} while (0)
// out0 = in0*w0 + in1*w1
// out1 = in1*w0 - in0*w1
#define btf_32_sse4_1_type1(w0, w1, in0, in1, out0, out1, bit) \
do { \
__m128i ww0, ww1, in0_w0, in1_w1, in0_w1, in1_w0; \
ww0 = _mm_set1_epi32(w0); \
ww1 = _mm_set1_epi32(w1); \
in0_w0 = _mm_mullo_epi32(in0, ww0); \
in1_w1 = _mm_mullo_epi32(in1, ww1); \
out0 = _mm_add_epi32(in0_w0, in1_w1); \
out0 = round_shift_32_sse4_1(out0, bit); \
in0_w1 = _mm_mullo_epi32(in0, ww1); \
in1_w0 = _mm_mullo_epi32(in1, ww0); \
out1 = _mm_sub_epi32(in1_w0, in0_w1); \
out1 = round_shift_32_sse4_1(out1, bit); \
} while (0)
#ifdef __cplusplus
}
#endif
#endif // AV1_TXMF1D_SSE2_H_

898
third_party/aom/av1/common/x86/filterintra_sse4.c vendored Normal file
View file

@ -0,0 +1,898 @@
/*
* Copyright (c) 2016, Alliance for Open Media. All rights reserved
*
* This source code is subject to the terms of the BSD 2 Clause License and
* the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
* was not distributed with this source code in the LICENSE file, you can
* obtain it at www.aomedia.org/license/software. If the Alliance for Open
* Media Patent License 1.0 was not distributed with this source code in the
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
*/
#include <smmintrin.h>
#include "./av1_rtcd.h"
#include "aom_ports/mem.h"
#include "av1/common/enums.h"
#include "av1/common/reconintra.h"
#if USE_3TAP_INTRA_FILTER
void filterintra_sse4_3tap_dummy_func(void);
void filterintra_sse4_3tap_dummy_func(void) {}
#else
static INLINE void AddPixelsSmall(const uint8_t *above, const uint8_t *left,
__m128i *sum) {
const __m128i a = _mm_loadu_si128((const __m128i *)above);
const __m128i l = _mm_loadu_si128((const __m128i *)left);
const __m128i zero = _mm_setzero_si128();
__m128i u0 = _mm_unpacklo_epi8(a, zero);
__m128i u1 = _mm_unpacklo_epi8(l, zero);
sum[0] = _mm_add_epi16(u0, u1);
}
static INLINE int GetMeanValue4x4(const uint8_t *above, const uint8_t *left,
__m128i *params) {
const __m128i zero = _mm_setzero_si128();
__m128i sum_vector, u;
uint16_t sum_value;
AddPixelsSmall(above, left, &sum_vector);
sum_vector = _mm_hadd_epi16(sum_vector, zero); // still has 2 values
u = _mm_srli_si128(sum_vector, 2);
sum_vector = _mm_add_epi16(sum_vector, u);
sum_value = _mm_extract_epi16(sum_vector, 0);
sum_value += 4;
sum_value >>= 3;
*params = _mm_set1_epi32(sum_value);
return sum_value;
}
static INLINE int GetMeanValue8x8(const uint8_t *above, const uint8_t *left,
__m128i *params) {
const __m128i zero = _mm_setzero_si128();
__m128i sum_vector, u;
uint16_t sum_value;
AddPixelsSmall(above, left, &sum_vector);
sum_vector = _mm_hadd_epi16(sum_vector, zero); // still has 4 values
sum_vector = _mm_hadd_epi16(sum_vector, zero); // still has 2 values
u = _mm_srli_si128(sum_vector, 2);
sum_vector = _mm_add_epi16(sum_vector, u);
sum_value = _mm_extract_epi16(sum_vector, 0);
sum_value += 8;
sum_value >>= 4;
*params = _mm_set1_epi32(sum_value);
return sum_value;
}
static INLINE void AddPixelsLarge(const uint8_t *above, const uint8_t *left,
__m128i *sum) {
const __m128i a = _mm_loadu_si128((const __m128i *)above);
const __m128i l = _mm_loadu_si128((const __m128i *)left);
const __m128i zero = _mm_setzero_si128();
__m128i u0 = _mm_unpacklo_epi8(a, zero);
__m128i u1 = _mm_unpacklo_epi8(l, zero);
sum[0] = _mm_add_epi16(u0, u1);
u0 = _mm_unpackhi_epi8(a, zero);
u1 = _mm_unpackhi_epi8(l, zero);
sum[0] = _mm_add_epi16(sum[0], u0);
sum[0] = _mm_add_epi16(sum[0], u1);
}
static INLINE int GetMeanValue16x16(const uint8_t *above, const uint8_t *left,
__m128i *params) {
const __m128i zero = _mm_setzero_si128();
__m128i sum_vector, u;
uint16_t sum_value;
AddPixelsLarge(above, left, &sum_vector);
sum_vector = _mm_hadd_epi16(sum_vector, zero); // still has 4 values
sum_vector = _mm_hadd_epi16(sum_vector, zero); // still has 2 values
u = _mm_srli_si128(sum_vector, 2);
sum_vector = _mm_add_epi16(sum_vector, u);
sum_value = _mm_extract_epi16(sum_vector, 0);
sum_value += 16;
sum_value >>= 5;
*params = _mm_set1_epi32(sum_value);
return sum_value;
}
static INLINE int GetMeanValue32x32(const uint8_t *above, const uint8_t *left,
__m128i *params) {
const __m128i zero = _mm_setzero_si128();
__m128i sum_vector[2], u;
uint16_t sum_value;
AddPixelsLarge(above, left, &sum_vector[0]);
AddPixelsLarge(above + 16, left + 16, &sum_vector[1]);
sum_vector[0] = _mm_add_epi16(sum_vector[0], sum_vector[1]);
sum_vector[0] = _mm_hadd_epi16(sum_vector[0], zero); // still has 4 values
sum_vector[0] = _mm_hadd_epi16(sum_vector[0], zero); // still has 2 values
u = _mm_srli_si128(sum_vector[0], 2);
sum_vector[0] = _mm_add_epi16(sum_vector[0], u);
sum_value = _mm_extract_epi16(sum_vector[0], 0);
sum_value += 32;
sum_value >>= 6;
*params = _mm_set1_epi32(sum_value);
return sum_value;
}
// Note:
// params[4] : mean value, 4 int32_t repetition
//
static INLINE int CalcRefPixelsMeanValue(const uint8_t *above,
const uint8_t *left, int bs,
__m128i *params) {
int meanValue = 0;
switch (bs) {
case 4: meanValue = GetMeanValue4x4(above, left, params); break;
case 8: meanValue = GetMeanValue8x8(above, left, params); break;
case 16: meanValue = GetMeanValue16x16(above, left, params); break;
case 32: meanValue = GetMeanValue32x32(above, left, params); break;
default: assert(0);
}
return meanValue;
}
// Note:
// params[0-3] : 4-tap filter coefficients (int32_t per coefficient)
//
static INLINE void GetIntraFilterParams(int bs, int mode, __m128i *params) {
const TX_SIZE tx_size =
(bs == 32) ? TX_32X32
: ((bs == 16) ? TX_16X16 : ((bs == 8) ? TX_8X8 : (TX_4X4)));
// c0
params[0] = _mm_set_epi32(av1_filter_intra_taps_4[tx_size][mode][0],
av1_filter_intra_taps_4[tx_size][mode][0],
av1_filter_intra_taps_4[tx_size][mode][0],
av1_filter_intra_taps_4[tx_size][mode][0]);
// c1
params[1] = _mm_set_epi32(av1_filter_intra_taps_4[tx_size][mode][1],
av1_filter_intra_taps_4[tx_size][mode][1],
av1_filter_intra_taps_4[tx_size][mode][1],
av1_filter_intra_taps_4[tx_size][mode][1]);
// c2
params[2] = _mm_set_epi32(av1_filter_intra_taps_4[tx_size][mode][2],
av1_filter_intra_taps_4[tx_size][mode][2],
av1_filter_intra_taps_4[tx_size][mode][2],
av1_filter_intra_taps_4[tx_size][mode][2]);
// c3
params[3] = _mm_set_epi32(av1_filter_intra_taps_4[tx_size][mode][3],
av1_filter_intra_taps_4[tx_size][mode][3],
av1_filter_intra_taps_4[tx_size][mode][3],
av1_filter_intra_taps_4[tx_size][mode][3]);
}
static const int maxBlkSize = 32;
static INLINE void SavePred4x4(int *pred, const __m128i *mean, uint8_t *dst,
ptrdiff_t stride) {
const int predStride = (maxBlkSize << 1) + 1;
__m128i p0 = _mm_loadu_si128((const __m128i *)pred);
__m128i p1 = _mm_loadu_si128((const __m128i *)(pred + predStride));
__m128i p2 = _mm_loadu_si128((const __m128i *)(pred + 2 * predStride));
__m128i p3 = _mm_loadu_si128((const __m128i *)(pred + 3 * predStride));
p0 = _mm_add_epi32(p0, mean[0]);
p1 = _mm_add_epi32(p1, mean[0]);
p2 = _mm_add_epi32(p2, mean[0]);
p3 = _mm_add_epi32(p3, mean[0]);
p0 = _mm_packus_epi32(p0, p1);
p1 = _mm_packus_epi32(p2, p3);
p0 = _mm_packus_epi16(p0, p1);
*((int *)dst) = _mm_cvtsi128_si32(p0);
p0 = _mm_srli_si128(p0, 4);
*((int *)(dst + stride)) = _mm_cvtsi128_si32(p0);
p0 = _mm_srli_si128(p0, 4);
*((int *)(dst + 2 * stride)) = _mm_cvtsi128_si32(p0);
p0 = _mm_srli_si128(p0, 4);
*((int *)(dst + 3 * stride)) = _mm_cvtsi128_si32(p0);
}
static void SavePred8x8(int *pred, const __m128i *mean, uint8_t *dst,
ptrdiff_t stride) {
const int predStride = (maxBlkSize << 1) + 1;
__m128i p0, p1, p2, p3;
int r = 0;
while (r < 8) {
p0 = _mm_loadu_si128((const __m128i *)(pred + r * predStride));
p1 = _mm_loadu_si128((const __m128i *)(pred + r * predStride + 4));
r += 1;
p2 = _mm_loadu_si128((const __m128i *)(pred + r * predStride));
p3 = _mm_loadu_si128((const __m128i *)(pred + r * predStride + 4));
p0 = _mm_add_epi32(p0, mean[0]);
p1 = _mm_add_epi32(p1, mean[0]);
p2 = _mm_add_epi32(p2, mean[0]);
p3 = _mm_add_epi32(p3, mean[0]);
p0 = _mm_packus_epi32(p0, p1);
p1 = _mm_packus_epi32(p2, p3);
p0 = _mm_packus_epi16(p0, p1);
_mm_storel_epi64((__m128i *)dst, p0);
dst += stride;
p0 = _mm_srli_si128(p0, 8);
_mm_storel_epi64((__m128i *)dst, p0);
dst += stride;
r += 1;
}
}
static void SavePred16x16(int *pred, const __m128i *mean, uint8_t *dst,
ptrdiff_t stride) {
const int predStride = (maxBlkSize << 1) + 1;
__m128i p0, p1, p2, p3;
int r = 0;
while (r < 16) {
p0 = _mm_loadu_si128((const __m128i *)(pred + r * predStride));
p1 = _mm_loadu_si128((const __m128i *)(pred + r * predStride + 4));
p2 = _mm_loadu_si128((const __m128i *)(pred + r * predStride + 8));
p3 = _mm_loadu_si128((const __m128i *)(pred + r * predStride + 12));
p0 = _mm_add_epi32(p0, mean[0]);
p1 = _mm_add_epi32(p1, mean[0]);
p2 = _mm_add_epi32(p2, mean[0]);
p3 = _mm_add_epi32(p3, mean[0]);
p0 = _mm_packus_epi32(p0, p1);
p1 = _mm_packus_epi32(p2, p3);
p0 = _mm_packus_epi16(p0, p1);
_mm_storel_epi64((__m128i *)dst, p0);
p0 = _mm_srli_si128(p0, 8);
_mm_storel_epi64((__m128i *)(dst + 8), p0);
dst += stride;
r += 1;
}
}
static void SavePred32x32(int *pred, const __m128i *mean, uint8_t *dst,
ptrdiff_t stride) {
const int predStride = (maxBlkSize << 1) + 1;
__m128i p0, p1, p2, p3, p4, p5, p6, p7;
int r = 0;
while (r < 32) {
p0 = _mm_loadu_si128((const __m128i *)(pred + r * predStride));
p1 = _mm_loadu_si128((const __m128i *)(pred + r * predStride + 4));
p2 = _mm_loadu_si128((const __m128i *)(pred + r * predStride + 8));
p3 = _mm_loadu_si128((const __m128i *)(pred + r * predStride + 12));
p4 = _mm_loadu_si128((const __m128i *)(pred + r * predStride + 16));
p5 = _mm_loadu_si128((const __m128i *)(pred + r * predStride + 20));
p6 = _mm_loadu_si128((const __m128i *)(pred + r * predStride + 24));
p7 = _mm_loadu_si128((const __m128i *)(pred + r * predStride + 28));
p0 = _mm_add_epi32(p0, mean[0]);
p1 = _mm_add_epi32(p1, mean[0]);
p2 = _mm_add_epi32(p2, mean[0]);
p3 = _mm_add_epi32(p3, mean[0]);
p4 = _mm_add_epi32(p4, mean[0]);
p5 = _mm_add_epi32(p5, mean[0]);
p6 = _mm_add_epi32(p6, mean[0]);
p7 = _mm_add_epi32(p7, mean[0]);
p0 = _mm_packus_epi32(p0, p1);
p1 = _mm_packus_epi32(p2, p3);
p0 = _mm_packus_epi16(p0, p1);
p4 = _mm_packus_epi32(p4, p5);
p5 = _mm_packus_epi32(p6, p7);
p4 = _mm_packus_epi16(p4, p5);
_mm_storel_epi64((__m128i *)dst, p0);
p0 = _mm_srli_si128(p0, 8);
_mm_storel_epi64((__m128i *)(dst + 8), p0);
_mm_storel_epi64((__m128i *)(dst + 16), p4);
p4 = _mm_srli_si128(p4, 8);
_mm_storel_epi64((__m128i *)(dst + 24), p4);
dst += stride;
r += 1;
}
}
static void SavePrediction(int *pred, const __m128i *mean, int bs, uint8_t *dst,
ptrdiff_t stride) {
switch (bs) {
case 4: SavePred4x4(pred, mean, dst, stride); break;
case 8: SavePred8x8(pred, mean, dst, stride); break;
case 16: SavePred16x16(pred, mean, dst, stride); break;
case 32: SavePred32x32(pred, mean, dst, stride); break;
default: assert(0);
}
}
typedef void (*ProducePixelsFunc)(__m128i *p, const __m128i *prm, int *pred,
const int predStride);
static void ProduceFourPixels(__m128i *p, const __m128i *prm, int *pred,
const int predStride) {
__m128i u0, u1, u2;
int c0 = _mm_extract_epi32(prm[1], 0);
int x = *(pred + predStride);
int sum;
u0 = _mm_mullo_epi32(p[0], prm[2]);
u1 = _mm_mullo_epi32(p[1], prm[0]);
u2 = _mm_mullo_epi32(p[2], prm[3]);
u0 = _mm_add_epi32(u0, u1);
u0 = _mm_add_epi32(u0, u2);
sum = _mm_extract_epi32(u0, 0);
sum += c0 * x;
x = ROUND_POWER_OF_TWO_SIGNED(sum, FILTER_INTRA_PREC_BITS);
*(pred + predStride + 1) = x;
sum = _mm_extract_epi32(u0, 1);
sum += c0 * x;
x = ROUND_POWER_OF_TWO_SIGNED(sum, FILTER_INTRA_PREC_BITS);
*(pred + predStride + 2) = x;
sum = _mm_extract_epi32(u0, 2);
sum += c0 * x;
x = ROUND_POWER_OF_TWO_SIGNED(sum, FILTER_INTRA_PREC_BITS);
*(pred + predStride + 3) = x;
sum = _mm_extract_epi32(u0, 3);
sum += c0 * x;
x = ROUND_POWER_OF_TWO_SIGNED(sum, FILTER_INTRA_PREC_BITS);
*(pred + predStride + 4) = x;
}
static void ProduceThreePixels(__m128i *p, const __m128i *prm, int *pred,
const int predStride) {
__m128i u0, u1, u2;
int c0 = _mm_extract_epi32(prm[1], 0);
int x = *(pred + predStride);
int sum;
u0 = _mm_mullo_epi32(p[0], prm[2]);
u1 = _mm_mullo_epi32(p[1], prm[0]);
u2 = _mm_mullo_epi32(p[2], prm[3]);
u0 = _mm_add_epi32(u0, u1);
u0 = _mm_add_epi32(u0, u2);
sum = _mm_extract_epi32(u0, 0);
sum += c0 * x;
x = ROUND_POWER_OF_TWO_SIGNED(sum, FILTER_INTRA_PREC_BITS);
*(pred + predStride + 1) = x;
sum = _mm_extract_epi32(u0, 1);
sum += c0 * x;
x = ROUND_POWER_OF_TWO_SIGNED(sum, FILTER_INTRA_PREC_BITS);
*(pred + predStride + 2) = x;
sum = _mm_extract_epi32(u0, 2);
sum += c0 * x;
x = ROUND_POWER_OF_TWO_SIGNED(sum, FILTER_INTRA_PREC_BITS);
*(pred + predStride + 3) = x;
}
static void ProduceTwoPixels(__m128i *p, const __m128i *prm, int *pred,
const int predStride) {
__m128i u0, u1, u2;
int c0 = _mm_extract_epi32(prm[1], 0);
int x = *(pred + predStride);
int sum;
u0 = _mm_mullo_epi32(p[0], prm[2]);
u1 = _mm_mullo_epi32(p[1], prm[0]);
u2 = _mm_mullo_epi32(p[2], prm[3]);
u0 = _mm_add_epi32(u0, u1);
u0 = _mm_add_epi32(u0, u2);
sum = _mm_extract_epi32(u0, 0);
sum += c0 * x;
x = ROUND_POWER_OF_TWO_SIGNED(sum, FILTER_INTRA_PREC_BITS);
*(pred + predStride + 1) = x;
sum = _mm_extract_epi32(u0, 1);
sum += c0 * x;
x = ROUND_POWER_OF_TWO_SIGNED(sum, FILTER_INTRA_PREC_BITS);
*(pred + predStride + 2) = x;
}
static void ProduceOnePixels(__m128i *p, const __m128i *prm, int *pred,
const int predStride) {
__m128i u0, u1, u2;
int c0 = _mm_extract_epi32(prm[1], 0);
int x = *(pred + predStride);
int sum;
u0 = _mm_mullo_epi32(p[0], prm[2]);
u1 = _mm_mullo_epi32(p[1], prm[0]);
u2 = _mm_mullo_epi32(p[2], prm[3]);
u0 = _mm_add_epi32(u0, u1);
u0 = _mm_add_epi32(u0, u2);
sum = _mm_extract_epi32(u0, 0);
sum += c0 * x;
x = ROUND_POWER_OF_TWO_SIGNED(sum, FILTER_INTRA_PREC_BITS);
*(pred + predStride + 1) = x;
}
static ProducePixelsFunc prodPixelsFuncTab[4] = {
ProduceOnePixels, ProduceTwoPixels, ProduceThreePixels, ProduceFourPixels
};
static void ProducePixels(int *pred, const __m128i *prm, int remain) {
__m128i p[3];
const int predStride = (maxBlkSize << 1) + 1;
int index;
p[0] = _mm_loadu_si128((const __m128i *)pred);
p[1] = _mm_loadu_si128((const __m128i *)(pred + 1));
p[2] = _mm_loadu_si128((const __m128i *)(pred + 2));
if (remain <= 2) {
return;
}
if (remain > 5) {
index = 3;
} else {
index = remain - 3;
}
prodPixelsFuncTab[index](p, prm, pred, predStride);
}
// Note:
// At column index c, the remaining pixels are R = 2 * bs + 1 - r - c
// the number of pixels to produce is R - 2 = 2 * bs - r - c - 1
static void GeneratePrediction(const uint8_t *above, const uint8_t *left,
const int bs, const __m128i *prm, int meanValue,
uint8_t *dst, ptrdiff_t stride) {
int pred[33][65];
int r, c, colBound;
int remainings;
for (r = 0; r < bs; ++r) {
pred[r + 1][0] = (int)left[r] - meanValue;
}
above -= 1;
for (c = 0; c < 2 * bs + 1; ++c) {
pred[0][c] = (int)above[c] - meanValue;
}
r = 0;
c = 0;
while (r < bs) {
colBound = (bs << 1) - r;
for (c = 0; c < colBound; c += 4) {
remainings = colBound - c + 1;
ProducePixels(&pred[r][c], prm, remainings);
}
r += 1;
}
SavePrediction(&pred[1][1], &prm[4], bs, dst, stride);
}
static void FilterPrediction(const uint8_t *above, const uint8_t *left, int bs,
__m128i *prm, uint8_t *dst, ptrdiff_t stride) {
int meanValue = 0;
meanValue = CalcRefPixelsMeanValue(above, left, bs, &prm[4]);
GeneratePrediction(above, left, bs, prm, meanValue, dst, stride);
}
void av1_dc_filter_predictor_sse4_1(uint8_t *dst, ptrdiff_t stride, int bs,
const uint8_t *above, const uint8_t *left) {
__m128i prm[5];
GetIntraFilterParams(bs, DC_PRED, &prm[0]);
FilterPrediction(above, left, bs, prm, dst, stride);
}
void av1_v_filter_predictor_sse4_1(uint8_t *dst, ptrdiff_t stride, int bs,
const uint8_t *above, const uint8_t *left) {
__m128i prm[5];
GetIntraFilterParams(bs, V_PRED, &prm[0]);
FilterPrediction(above, left, bs, prm, dst, stride);
}
void av1_h_filter_predictor_sse4_1(uint8_t *dst, ptrdiff_t stride, int bs,
const uint8_t *above, const uint8_t *left) {
__m128i prm[5];
GetIntraFilterParams(bs, H_PRED, &prm[0]);
FilterPrediction(above, left, bs, prm, dst, stride);
}
void av1_d45_filter_predictor_sse4_1(uint8_t *dst, ptrdiff_t stride, int bs,
const uint8_t *above,
const uint8_t *left) {
__m128i prm[5];
GetIntraFilterParams(bs, D45_PRED, &prm[0]);
FilterPrediction(above, left, bs, prm, dst, stride);
}
void av1_d135_filter_predictor_sse4_1(uint8_t *dst, ptrdiff_t stride, int bs,
const uint8_t *above,
const uint8_t *left) {
__m128i prm[5];
GetIntraFilterParams(bs, D135_PRED, &prm[0]);
FilterPrediction(above, left, bs, prm, dst, stride);
}
void av1_d117_filter_predictor_sse4_1(uint8_t *dst, ptrdiff_t stride, int bs,
const uint8_t *above,
const uint8_t *left) {
__m128i prm[5];
GetIntraFilterParams(bs, D117_PRED, &prm[0]);
FilterPrediction(above, left, bs, prm, dst, stride);
}
void av1_d153_filter_predictor_sse4_1(uint8_t *dst, ptrdiff_t stride, int bs,
const uint8_t *above,
const uint8_t *left) {
__m128i prm[5];
GetIntraFilterParams(bs, D153_PRED, &prm[0]);
FilterPrediction(above, left, bs, prm, dst, stride);
}
void av1_d207_filter_predictor_sse4_1(uint8_t *dst, ptrdiff_t stride, int bs,
const uint8_t *above,
const uint8_t *left) {
__m128i prm[5];
GetIntraFilterParams(bs, D207_PRED, &prm[0]);
FilterPrediction(above, left, bs, prm, dst, stride);
}
void av1_d63_filter_predictor_sse4_1(uint8_t *dst, ptrdiff_t stride, int bs,
const uint8_t *above,
const uint8_t *left) {
__m128i prm[5];
GetIntraFilterParams(bs, D63_PRED, &prm[0]);
FilterPrediction(above, left, bs, prm, dst, stride);
}
void av1_tm_filter_predictor_sse4_1(uint8_t *dst, ptrdiff_t stride, int bs,
const uint8_t *above, const uint8_t *left) {
__m128i prm[5];
GetIntraFilterParams(bs, TM_PRED, &prm[0]);
FilterPrediction(above, left, bs, prm, dst, stride);
}
// ============== High Bit Depth ==============
#if CONFIG_HIGHBITDEPTH
static INLINE int HighbdGetMeanValue4x4(const uint16_t *above,
const uint16_t *left, const int bd,
__m128i *params) {
const __m128i a = _mm_loadu_si128((const __m128i *)above);
const __m128i l = _mm_loadu_si128((const __m128i *)left);
const __m128i zero = _mm_setzero_si128();
__m128i sum_vector, u;
uint16_t sum_value;
(void)bd;
sum_vector = _mm_add_epi16(a, l);
sum_vector = _mm_hadd_epi16(sum_vector, zero); // still has 2 values
u = _mm_srli_si128(sum_vector, 2);
sum_vector = _mm_add_epi16(sum_vector, u);
sum_value = _mm_extract_epi16(sum_vector, 0);
sum_value += 4;
sum_value >>= 3;
*params = _mm_set1_epi32(sum_value);
return sum_value;
}
static INLINE int HighbdGetMeanValue8x8(const uint16_t *above,
const uint16_t *left, const int bd,
__m128i *params) {
const __m128i a = _mm_loadu_si128((const __m128i *)above);
const __m128i l = _mm_loadu_si128((const __m128i *)left);
const __m128i zero = _mm_setzero_si128();
__m128i sum_vector, u;
uint16_t sum_value;
(void)bd;
sum_vector = _mm_add_epi16(a, l);
sum_vector = _mm_hadd_epi16(sum_vector, zero); // still has 4 values
sum_vector = _mm_hadd_epi16(sum_vector, zero); // still has 2 values
u = _mm_srli_si128(sum_vector, 2);
sum_vector = _mm_add_epi16(sum_vector, u);
sum_value = _mm_extract_epi16(sum_vector, 0);
sum_value += 8;
sum_value >>= 4;
*params = _mm_set1_epi32(sum_value);
return sum_value;
}
// Note:
// Process 16 pixels above and left, 10-bit depth
// Add to the last 8 pixels sum
static INLINE void AddPixels10bit(const uint16_t *above, const uint16_t *left,
__m128i *sum) {
__m128i a = _mm_loadu_si128((const __m128i *)above);
__m128i l = _mm_loadu_si128((const __m128i *)left);
sum[0] = _mm_add_epi16(a, l);
a = _mm_loadu_si128((const __m128i *)(above + 8));
l = _mm_loadu_si128((const __m128i *)(left + 8));
sum[0] = _mm_add_epi16(sum[0], a);
sum[0] = _mm_add_epi16(sum[0], l);
}
// Note:
// Process 16 pixels above and left, 12-bit depth
// Add to the last 8 pixels sum
static INLINE void AddPixels12bit(const uint16_t *above, const uint16_t *left,
__m128i *sum) {
__m128i a = _mm_loadu_si128((const __m128i *)above);
__m128i l = _mm_loadu_si128((const __m128i *)left);
const __m128i zero = _mm_setzero_si128();
__m128i v0, v1;
v0 = _mm_unpacklo_epi16(a, zero);
v1 = _mm_unpacklo_epi16(l, zero);
sum[0] = _mm_add_epi32(v0, v1);
v0 = _mm_unpackhi_epi16(a, zero);
v1 = _mm_unpackhi_epi16(l, zero);
sum[0] = _mm_add_epi32(sum[0], v0);
sum[0] = _mm_add_epi32(sum[0], v1);
a = _mm_loadu_si128((const __m128i *)(above + 8));
l = _mm_loadu_si128((const __m128i *)(left + 8));
v0 = _mm_unpacklo_epi16(a, zero);
v1 = _mm_unpacklo_epi16(l, zero);
sum[0] = _mm_add_epi32(sum[0], v0);
sum[0] = _mm_add_epi32(sum[0], v1);
v0 = _mm_unpackhi_epi16(a, zero);
v1 = _mm_unpackhi_epi16(l, zero);
sum[0] = _mm_add_epi32(sum[0], v0);
sum[0] = _mm_add_epi32(sum[0], v1);
}
static INLINE int HighbdGetMeanValue16x16(const uint16_t *above,
const uint16_t *left, const int bd,
__m128i *params) {
const __m128i zero = _mm_setzero_si128();
__m128i sum_vector, u;
uint32_t sum_value = 0;
if (10 == bd) {
AddPixels10bit(above, left, &sum_vector);
sum_vector = _mm_hadd_epi16(sum_vector, zero); // still has 4 values
sum_vector = _mm_hadd_epi16(sum_vector, zero); // still has 2 values
u = _mm_srli_si128(sum_vector, 2);
sum_vector = _mm_add_epi16(sum_vector, u);
sum_value = _mm_extract_epi16(sum_vector, 0);
} else if (12 == bd) {
AddPixels12bit(above, left, &sum_vector);
sum_vector = _mm_hadd_epi32(sum_vector, zero);
u = _mm_srli_si128(sum_vector, 4);
sum_vector = _mm_add_epi32(u, sum_vector);
sum_value = _mm_extract_epi32(sum_vector, 0);
}
sum_value += 16;
sum_value >>= 5;
*params = _mm_set1_epi32(sum_value);
return sum_value;
}
static INLINE int HighbdGetMeanValue32x32(const uint16_t *above,
const uint16_t *left, const int bd,
__m128i *params) {
const __m128i zero = _mm_setzero_si128();
__m128i sum_vector[2], u;
uint32_t sum_value = 0;
if (10 == bd) {
AddPixels10bit(above, left, &sum_vector[0]);
AddPixels10bit(above + 16, left + 16, &sum_vector[1]);
sum_vector[0] = _mm_add_epi16(sum_vector[0], sum_vector[1]);
sum_vector[0] = _mm_hadd_epi16(sum_vector[0], zero); // still has 4 values
sum_vector[0] = _mm_hadd_epi16(sum_vector[0], zero); // still has 2 values
u = _mm_srli_si128(sum_vector[0], 2);
sum_vector[0] = _mm_add_epi16(sum_vector[0], u);
sum_value = _mm_extract_epi16(sum_vector[0], 0);
} else if (12 == bd) {
AddPixels12bit(above, left, &sum_vector[0]);
AddPixels12bit(above + 16, left + 16, &sum_vector[1]);
sum_vector[0] = _mm_add_epi32(sum_vector[0], sum_vector[1]);
sum_vector[0] = _mm_hadd_epi32(sum_vector[0], zero);
u = _mm_srli_si128(sum_vector[0], 4);
sum_vector[0] = _mm_add_epi32(u, sum_vector[0]);
sum_value = _mm_extract_epi32(sum_vector[0], 0);
}
sum_value += 32;
sum_value >>= 6;
*params = _mm_set1_epi32(sum_value);
return sum_value;
}
// Note:
// params[4] : mean value, 4 int32_t repetition
//
static INLINE int HighbdCalcRefPixelsMeanValue(const uint16_t *above,
const uint16_t *left, int bs,
const int bd, __m128i *params) {
int meanValue = 0;
switch (bs) {
case 4: meanValue = HighbdGetMeanValue4x4(above, left, bd, params); break;
case 8: meanValue = HighbdGetMeanValue8x8(above, left, bd, params); break;
case 16:
meanValue = HighbdGetMeanValue16x16(above, left, bd, params);
break;
case 32:
meanValue = HighbdGetMeanValue32x32(above, left, bd, params);
break;
default: assert(0);
}
return meanValue;
}
// Note:
// At column index c, the remaining pixels are R = 2 * bs + 1 - r - c
// the number of pixels to produce is R - 2 = 2 * bs - r - c - 1
static void HighbdGeneratePrediction(const uint16_t *above,
const uint16_t *left, const int bs,
const int bd, const __m128i *prm,
int meanValue, uint16_t *dst,
ptrdiff_t stride) {
int pred[33][65];
int r, c, colBound;
int remainings;
int ipred;
for (r = 0; r < bs; ++r) {
pred[r + 1][0] = (int)left[r] - meanValue;
}
above -= 1;
for (c = 0; c < 2 * bs + 1; ++c) {
pred[0][c] = (int)above[c] - meanValue;
}
r = 0;
c = 0;
while (r < bs) {
colBound = (bs << 1) - r;
for (c = 0; c < colBound; c += 4) {
remainings = colBound - c + 1;
ProducePixels(&pred[r][c], prm, remainings);
}
r += 1;
}
for (r = 0; r < bs; ++r) {
for (c = 0; c < bs; ++c) {
ipred = pred[r + 1][c + 1] + meanValue;
dst[c] = clip_pixel_highbd(ipred, bd);
}
dst += stride;
}
}
static void HighbdFilterPrediction(const uint16_t *above, const uint16_t *left,
int bs, const int bd, __m128i *prm,
uint16_t *dst, ptrdiff_t stride) {
int meanValue = 0;
meanValue = HighbdCalcRefPixelsMeanValue(above, left, bs, bd, &prm[4]);
HighbdGeneratePrediction(above, left, bs, bd, prm, meanValue, dst, stride);
}
void av1_highbd_dc_filter_predictor_sse4_1(uint16_t *dst, ptrdiff_t stride,
int bs, const uint16_t *above,
const uint16_t *left, int bd) {
__m128i prm[5];
GetIntraFilterParams(bs, DC_PRED, &prm[0]);
HighbdFilterPrediction(above, left, bs, bd, prm, dst, stride);
}
void av1_highbd_v_filter_predictor_sse4_1(uint16_t *dst, ptrdiff_t stride,
int bs, const uint16_t *above,
const uint16_t *left, int bd) {
__m128i prm[5];
GetIntraFilterParams(bs, V_PRED, &prm[0]);
HighbdFilterPrediction(above, left, bs, bd, prm, dst, stride);
}
void av1_highbd_h_filter_predictor_sse4_1(uint16_t *dst, ptrdiff_t stride,
int bs, const uint16_t *above,
const uint16_t *left, int bd) {
__m128i prm[5];
GetIntraFilterParams(bs, H_PRED, &prm[0]);
HighbdFilterPrediction(above, left, bs, bd, prm, dst, stride);
}
void av1_highbd_d45_filter_predictor_sse4_1(uint16_t *dst, ptrdiff_t stride,
int bs, const uint16_t *above,
const uint16_t *left, int bd) {
__m128i prm[5];
GetIntraFilterParams(bs, D45_PRED, &prm[0]);
HighbdFilterPrediction(above, left, bs, bd, prm, dst, stride);
}
void av1_highbd_d135_filter_predictor_sse4_1(uint16_t *dst, ptrdiff_t stride,
int bs, const uint16_t *above,
const uint16_t *left, int bd) {
__m128i prm[5];
GetIntraFilterParams(bs, D135_PRED, &prm[0]);
HighbdFilterPrediction(above, left, bs, bd, prm, dst, stride);
}
void av1_highbd_d117_filter_predictor_sse4_1(uint16_t *dst, ptrdiff_t stride,
int bs, const uint16_t *above,
const uint16_t *left, int bd) {
__m128i prm[5];
GetIntraFilterParams(bs, D117_PRED, &prm[0]);
HighbdFilterPrediction(above, left, bs, bd, prm, dst, stride);
}
void av1_highbd_d153_filter_predictor_sse4_1(uint16_t *dst, ptrdiff_t stride,
int bs, const uint16_t *above,
const uint16_t *left, int bd) {
__m128i prm[5];
GetIntraFilterParams(bs, D153_PRED, &prm[0]);
HighbdFilterPrediction(above, left, bs, bd, prm, dst, stride);
}
void av1_highbd_d207_filter_predictor_sse4_1(uint16_t *dst, ptrdiff_t stride,
int bs, const uint16_t *above,
const uint16_t *left, int bd) {
__m128i prm[5];
GetIntraFilterParams(bs, D207_PRED, &prm[0]);
HighbdFilterPrediction(above, left, bs, bd, prm, dst, stride);
}
void av1_highbd_d63_filter_predictor_sse4_1(uint16_t *dst, ptrdiff_t stride,
int bs, const uint16_t *above,
const uint16_t *left, int bd) {
__m128i prm[5];
GetIntraFilterParams(bs, D63_PRED, &prm[0]);
HighbdFilterPrediction(above, left, bs, bd, prm, dst, stride);
}
void av1_highbd_tm_filter_predictor_sse4_1(uint16_t *dst, ptrdiff_t stride,
int bs, const uint16_t *above,
const uint16_t *left, int bd) {
__m128i prm[5];
GetIntraFilterParams(bs, TM_PRED, &prm[0]);
HighbdFilterPrediction(above, left, bs, bd, prm, dst, stride);
}
#endif // CONFIG_HIGHBITDEPTH
#endif // USE_3TAP_INTRA_FILTER

557
third_party/aom/av1/common/x86/highbd_inv_txfm_avx2.c vendored Normal file
View file

@ -0,0 +1,557 @@
/*
* Copyright (c) 2016, Alliance for Open Media. All rights reserved
*
* This source code is subject to the terms of the BSD 2 Clause License and
* the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
* was not distributed with this source code in the LICENSE file, you can
* obtain it at www.aomedia.org/license/software. If the Alliance for Open
* Media Patent License 1.0 was not distributed with this source code in the
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
*/
#include <assert.h>
#include <immintrin.h>
#include "./av1_rtcd.h"
#include "./aom_config.h"
#include "av1/common/av1_inv_txfm2d_cfg.h"
// Note:
// Total 32x4 registers to represent 32x32 block coefficients.
// For high bit depth, each coefficient is 4-byte.
// Each __m256i register holds 8 coefficients.
// So each "row" we needs 4 register. Totally 32 rows
// Register layout:
// v0, v1, v2, v3,
// v4, v5, v6, v7,
// ... ...
// v124, v125, v126, v127
static void transpose_32x32_8x8(const __m256i *in, __m256i *out) {
__m256i u0, u1, u2, u3, u4, u5, u6, u7;
__m256i x0, x1;
u0 = _mm256_unpacklo_epi32(in[0], in[4]);
u1 = _mm256_unpackhi_epi32(in[0], in[4]);
u2 = _mm256_unpacklo_epi32(in[8], in[12]);
u3 = _mm256_unpackhi_epi32(in[8], in[12]);
u4 = _mm256_unpacklo_epi32(in[16], in[20]);
u5 = _mm256_unpackhi_epi32(in[16], in[20]);
u6 = _mm256_unpacklo_epi32(in[24], in[28]);
u7 = _mm256_unpackhi_epi32(in[24], in[28]);
x0 = _mm256_unpacklo_epi64(u0, u2);
x1 = _mm256_unpacklo_epi64(u4, u6);
out[0] = _mm256_permute2f128_si256(x0, x1, 0x20);
out[16] = _mm256_permute2f128_si256(x0, x1, 0x31);
x0 = _mm256_unpackhi_epi64(u0, u2);
x1 = _mm256_unpackhi_epi64(u4, u6);
out[4] = _mm256_permute2f128_si256(x0, x1, 0x20);
out[20] = _mm256_permute2f128_si256(x0, x1, 0x31);
x0 = _mm256_unpacklo_epi64(u1, u3);
x1 = _mm256_unpacklo_epi64(u5, u7);
out[8] = _mm256_permute2f128_si256(x0, x1, 0x20);
out[24] = _mm256_permute2f128_si256(x0, x1, 0x31);
x0 = _mm256_unpackhi_epi64(u1, u3);
x1 = _mm256_unpackhi_epi64(u5, u7);
out[12] = _mm256_permute2f128_si256(x0, x1, 0x20);
out[28] = _mm256_permute2f128_si256(x0, x1, 0x31);
}
static void transpose_32x32_16x16(const __m256i *in, __m256i *out) {
transpose_32x32_8x8(&in[0], &out[0]);
transpose_32x32_8x8(&in[1], &out[32]);
transpose_32x32_8x8(&in[32], &out[1]);
transpose_32x32_8x8(&in[33], &out[33]);
}
static void transpose_32x32(const __m256i *in, __m256i *out) {
transpose_32x32_16x16(&in[0], &out[0]);
transpose_32x32_16x16(&in[2], &out[64]);
transpose_32x32_16x16(&in[64], &out[2]);
transpose_32x32_16x16(&in[66], &out[66]);
}
static void load_buffer_32x32(const int32_t *coeff, __m256i *in) {
int i;
for (i = 0; i < 128; ++i) {
in[i] = _mm256_loadu_si256((const __m256i *)coeff);
coeff += 8;
}
}
static void round_shift_32x32(__m256i *in, int shift) {
__m256i rnding = _mm256_set1_epi32(1 << (shift - 1));
int i = 0;
while (i < 128) {
in[i] = _mm256_add_epi32(in[i], rnding);
in[i] = _mm256_srai_epi32(in[i], shift);
i++;
}
}
static __m256i highbd_clamp_epi32(__m256i x, int bd) {
const __m256i zero = _mm256_setzero_si256();
const __m256i one = _mm256_set1_epi16(1);
const __m256i max = _mm256_sub_epi16(_mm256_slli_epi16(one, bd), one);
__m256i clamped, mask;
mask = _mm256_cmpgt_epi16(x, max);
clamped = _mm256_andnot_si256(mask, x);
mask = _mm256_and_si256(mask, max);
clamped = _mm256_or_si256(mask, clamped);
mask = _mm256_cmpgt_epi16(clamped, zero);
clamped = _mm256_and_si256(clamped, mask);
return clamped;
}
static void write_buffer_32x32(__m256i *in, uint16_t *output, int stride,
int fliplr, int flipud, int shift, int bd) {
__m256i u0, u1, x0, x1, x2, x3, v0, v1, v2, v3;
const __m256i zero = _mm256_setzero_si256();
int i = 0;
(void)fliplr;
(void)flipud;
round_shift_32x32(in, shift);
while (i < 128) {
u0 = _mm256_loadu_si256((const __m256i *)output);
u1 = _mm256_loadu_si256((const __m256i *)(output + 16));
x0 = _mm256_unpacklo_epi16(u0, zero);
x1 = _mm256_unpackhi_epi16(u0, zero);
x2 = _mm256_unpacklo_epi16(u1, zero);
x3 = _mm256_unpackhi_epi16(u1, zero);
v0 = _mm256_permute2f128_si256(in[i], in[i + 1], 0x20);
v1 = _mm256_permute2f128_si256(in[i], in[i + 1], 0x31);
v2 = _mm256_permute2f128_si256(in[i + 2], in[i + 3], 0x20);
v3 = _mm256_permute2f128_si256(in[i + 2], in[i + 3], 0x31);
v0 = _mm256_add_epi32(v0, x0);
v1 = _mm256_add_epi32(v1, x1);
v2 = _mm256_add_epi32(v2, x2);
v3 = _mm256_add_epi32(v3, x3);
v0 = _mm256_packus_epi32(v0, v1);
v2 = _mm256_packus_epi32(v2, v3);
v0 = highbd_clamp_epi32(v0, bd);
v2 = highbd_clamp_epi32(v2, bd);
_mm256_storeu_si256((__m256i *)output, v0);
_mm256_storeu_si256((__m256i *)(output + 16), v2);
output += stride;
i += 4;
}
}
static INLINE __m256i half_btf_avx2(__m256i w0, __m256i n0, __m256i w1,
__m256i n1, __m256i rounding, int bit) {
__m256i x, y;
x = _mm256_mullo_epi32(w0, n0);
y = _mm256_mullo_epi32(w1, n1);
x = _mm256_add_epi32(x, y);
x = _mm256_add_epi32(x, rounding);
x = _mm256_srai_epi32(x, bit);
return x;
}
static void idct32_avx2(__m256i *in, __m256i *out, int bit) {
const int32_t *cospi = cospi_arr[bit - cos_bit_min];
const __m256i cospi62 = _mm256_set1_epi32(cospi[62]);
const __m256i cospi30 = _mm256_set1_epi32(cospi[30]);
const __m256i cospi46 = _mm256_set1_epi32(cospi[46]);
const __m256i cospi14 = _mm256_set1_epi32(cospi[14]);
const __m256i cospi54 = _mm256_set1_epi32(cospi[54]);
const __m256i cospi22 = _mm256_set1_epi32(cospi[22]);
const __m256i cospi38 = _mm256_set1_epi32(cospi[38]);
const __m256i cospi6 = _mm256_set1_epi32(cospi[6]);
const __m256i cospi58 = _mm256_set1_epi32(cospi[58]);
const __m256i cospi26 = _mm256_set1_epi32(cospi[26]);
const __m256i cospi42 = _mm256_set1_epi32(cospi[42]);
const __m256i cospi10 = _mm256_set1_epi32(cospi[10]);
const __m256i cospi50 = _mm256_set1_epi32(cospi[50]);
const __m256i cospi18 = _mm256_set1_epi32(cospi[18]);
const __m256i cospi34 = _mm256_set1_epi32(cospi[34]);
const __m256i cospi2 = _mm256_set1_epi32(cospi[2]);
const __m256i cospim58 = _mm256_set1_epi32(-cospi[58]);
const __m256i cospim26 = _mm256_set1_epi32(-cospi[26]);
const __m256i cospim42 = _mm256_set1_epi32(-cospi[42]);
const __m256i cospim10 = _mm256_set1_epi32(-cospi[10]);
const __m256i cospim50 = _mm256_set1_epi32(-cospi[50]);
const __m256i cospim18 = _mm256_set1_epi32(-cospi[18]);
const __m256i cospim34 = _mm256_set1_epi32(-cospi[34]);
const __m256i cospim2 = _mm256_set1_epi32(-cospi[2]);
const __m256i cospi60 = _mm256_set1_epi32(cospi[60]);
const __m256i cospi28 = _mm256_set1_epi32(cospi[28]);
const __m256i cospi44 = _mm256_set1_epi32(cospi[44]);
const __m256i cospi12 = _mm256_set1_epi32(cospi[12]);
const __m256i cospi52 = _mm256_set1_epi32(cospi[52]);
const __m256i cospi20 = _mm256_set1_epi32(cospi[20]);
const __m256i cospi36 = _mm256_set1_epi32(cospi[36]);
const __m256i cospi4 = _mm256_set1_epi32(cospi[4]);
const __m256i cospim52 = _mm256_set1_epi32(-cospi[52]);
const __m256i cospim20 = _mm256_set1_epi32(-cospi[20]);
const __m256i cospim36 = _mm256_set1_epi32(-cospi[36]);
const __m256i cospim4 = _mm256_set1_epi32(-cospi[4]);
const __m256i cospi56 = _mm256_set1_epi32(cospi[56]);
const __m256i cospi24 = _mm256_set1_epi32(cospi[24]);
const __m256i cospi40 = _mm256_set1_epi32(cospi[40]);
const __m256i cospi8 = _mm256_set1_epi32(cospi[8]);
const __m256i cospim40 = _mm256_set1_epi32(-cospi[40]);
const __m256i cospim8 = _mm256_set1_epi32(-cospi[8]);
const __m256i cospim56 = _mm256_set1_epi32(-cospi[56]);
const __m256i cospim24 = _mm256_set1_epi32(-cospi[24]);
const __m256i cospi32 = _mm256_set1_epi32(cospi[32]);
const __m256i cospim32 = _mm256_set1_epi32(-cospi[32]);
const __m256i cospi48 = _mm256_set1_epi32(cospi[48]);
const __m256i cospim48 = _mm256_set1_epi32(-cospi[48]);
const __m256i cospi16 = _mm256_set1_epi32(cospi[16]);
const __m256i cospim16 = _mm256_set1_epi32(-cospi[16]);
const __m256i rounding = _mm256_set1_epi32(1 << (bit - 1));
__m256i bf1[32], bf0[32];
int col;
for (col = 0; col < 4; ++col) {
// stage 0
// stage 1
bf1[0] = in[0 * 4 + col];
bf1[1] = in[16 * 4 + col];
bf1[2] = in[8 * 4 + col];
bf1[3] = in[24 * 4 + col];
bf1[4] = in[4 * 4 + col];
bf1[5] = in[20 * 4 + col];
bf1[6] = in[12 * 4 + col];
bf1[7] = in[28 * 4 + col];
bf1[8] = in[2 * 4 + col];
bf1[9] = in[18 * 4 + col];
bf1[10] = in[10 * 4 + col];
bf1[11] = in[26 * 4 + col];
bf1[12] = in[6 * 4 + col];
bf1[13] = in[22 * 4 + col];
bf1[14] = in[14 * 4 + col];
bf1[15] = in[30 * 4 + col];
bf1[16] = in[1 * 4 + col];
bf1[17] = in[17 * 4 + col];
bf1[18] = in[9 * 4 + col];
bf1[19] = in[25 * 4 + col];
bf1[20] = in[5 * 4 + col];
bf1[21] = in[21 * 4 + col];
bf1[22] = in[13 * 4 + col];
bf1[23] = in[29 * 4 + col];
bf1[24] = in[3 * 4 + col];
bf1[25] = in[19 * 4 + col];
bf1[26] = in[11 * 4 + col];
bf1[27] = in[27 * 4 + col];
bf1[28] = in[7 * 4 + col];
bf1[29] = in[23 * 4 + col];
bf1[30] = in[15 * 4 + col];
bf1[31] = in[31 * 4 + col];
// stage 2
bf0[0] = bf1[0];
bf0[1] = bf1[1];
bf0[2] = bf1[2];
bf0[3] = bf1[3];
bf0[4] = bf1[4];
bf0[5] = bf1[5];
bf0[6] = bf1[6];
bf0[7] = bf1[7];
bf0[8] = bf1[8];
bf0[9] = bf1[9];
bf0[10] = bf1[10];
bf0[11] = bf1[11];
bf0[12] = bf1[12];
bf0[13] = bf1[13];
bf0[14] = bf1[14];
bf0[15] = bf1[15];
bf0[16] = half_btf_avx2(cospi62, bf1[16], cospim2, bf1[31], rounding, bit);
bf0[17] = half_btf_avx2(cospi30, bf1[17], cospim34, bf1[30], rounding, bit);
bf0[18] = half_btf_avx2(cospi46, bf1[18], cospim18, bf1[29], rounding, bit);
bf0[19] = half_btf_avx2(cospi14, bf1[19], cospim50, bf1[28], rounding, bit);
bf0[20] = half_btf_avx2(cospi54, bf1[20], cospim10, bf1[27], rounding, bit);
bf0[21] = half_btf_avx2(cospi22, bf1[21], cospim42, bf1[26], rounding, bit);
bf0[22] = half_btf_avx2(cospi38, bf1[22], cospim26, bf1[25], rounding, bit);
bf0[23] = half_btf_avx2(cospi6, bf1[23], cospim58, bf1[24], rounding, bit);
bf0[24] = half_btf_avx2(cospi58, bf1[23], cospi6, bf1[24], rounding, bit);
bf0[25] = half_btf_avx2(cospi26, bf1[22], cospi38, bf1[25], rounding, bit);
bf0[26] = half_btf_avx2(cospi42, bf1[21], cospi22, bf1[26], rounding, bit);
bf0[27] = half_btf_avx2(cospi10, bf1[20], cospi54, bf1[27], rounding, bit);
bf0[28] = half_btf_avx2(cospi50, bf1[19], cospi14, bf1[28], rounding, bit);
bf0[29] = half_btf_avx2(cospi18, bf1[18], cospi46, bf1[29], rounding, bit);
bf0[30] = half_btf_avx2(cospi34, bf1[17], cospi30, bf1[30], rounding, bit);
bf0[31] = half_btf_avx2(cospi2, bf1[16], cospi62, bf1[31], rounding, bit);
// stage 3
bf1[0] = bf0[0];
bf1[1] = bf0[1];
bf1[2] = bf0[2];
bf1[3] = bf0[3];
bf1[4] = bf0[4];
bf1[5] = bf0[5];
bf1[6] = bf0[6];
bf1[7] = bf0[7];
bf1[8] = half_btf_avx2(cospi60, bf0[8], cospim4, bf0[15], rounding, bit);
bf1[9] = half_btf_avx2(cospi28, bf0[9], cospim36, bf0[14], rounding, bit);
bf1[10] = half_btf_avx2(cospi44, bf0[10], cospim20, bf0[13], rounding, bit);
bf1[11] = half_btf_avx2(cospi12, bf0[11], cospim52, bf0[12], rounding, bit);
bf1[12] = half_btf_avx2(cospi52, bf0[11], cospi12, bf0[12], rounding, bit);
bf1[13] = half_btf_avx2(cospi20, bf0[10], cospi44, bf0[13], rounding, bit);
bf1[14] = half_btf_avx2(cospi36, bf0[9], cospi28, bf0[14], rounding, bit);
bf1[15] = half_btf_avx2(cospi4, bf0[8], cospi60, bf0[15], rounding, bit);
bf1[16] = _mm256_add_epi32(bf0[16], bf0[17]);
bf1[17] = _mm256_sub_epi32(bf0[16], bf0[17]);
bf1[18] = _mm256_sub_epi32(bf0[19], bf0[18]);
bf1[19] = _mm256_add_epi32(bf0[18], bf0[19]);
bf1[20] = _mm256_add_epi32(bf0[20], bf0[21]);
bf1[21] = _mm256_sub_epi32(bf0[20], bf0[21]);
bf1[22] = _mm256_sub_epi32(bf0[23], bf0[22]);
bf1[23] = _mm256_add_epi32(bf0[22], bf0[23]);
bf1[24] = _mm256_add_epi32(bf0[24], bf0[25]);
bf1[25] = _mm256_sub_epi32(bf0[24], bf0[25]);
bf1[26] = _mm256_sub_epi32(bf0[27], bf0[26]);
bf1[27] = _mm256_add_epi32(bf0[26], bf0[27]);
bf1[28] = _mm256_add_epi32(bf0[28], bf0[29]);
bf1[29] = _mm256_sub_epi32(bf0[28], bf0[29]);
bf1[30] = _mm256_sub_epi32(bf0[31], bf0[30]);
bf1[31] = _mm256_add_epi32(bf0[30], bf0[31]);
// stage 4
bf0[0] = bf1[0];
bf0[1] = bf1[1];
bf0[2] = bf1[2];
bf0[3] = bf1[3];
bf0[4] = half_btf_avx2(cospi56, bf1[4], cospim8, bf1[7], rounding, bit);
bf0[5] = half_btf_avx2(cospi24, bf1[5], cospim40, bf1[6], rounding, bit);
bf0[6] = half_btf_avx2(cospi40, bf1[5], cospi24, bf1[6], rounding, bit);
bf0[7] = half_btf_avx2(cospi8, bf1[4], cospi56, bf1[7], rounding, bit);
bf0[8] = _mm256_add_epi32(bf1[8], bf1[9]);
bf0[9] = _mm256_sub_epi32(bf1[8], bf1[9]);
bf0[10] = _mm256_sub_epi32(bf1[11], bf1[10]);
bf0[11] = _mm256_add_epi32(bf1[10], bf1[11]);
bf0[12] = _mm256_add_epi32(bf1[12], bf1[13]);
bf0[13] = _mm256_sub_epi32(bf1[12], bf1[13]);
bf0[14] = _mm256_sub_epi32(bf1[15], bf1[14]);
bf0[15] = _mm256_add_epi32(bf1[14], bf1[15]);
bf0[16] = bf1[16];
bf0[17] = half_btf_avx2(cospim8, bf1[17], cospi56, bf1[30], rounding, bit);
bf0[18] = half_btf_avx2(cospim56, bf1[18], cospim8, bf1[29], rounding, bit);
bf0[19] = bf1[19];
bf0[20] = bf1[20];
bf0[21] = half_btf_avx2(cospim40, bf1[21], cospi24, bf1[26], rounding, bit);
bf0[22] =
half_btf_avx2(cospim24, bf1[22], cospim40, bf1[25], rounding, bit);
bf0[23] = bf1[23];
bf0[24] = bf1[24];
bf0[25] = half_btf_avx2(cospim40, bf1[22], cospi24, bf1[25], rounding, bit);
bf0[26] = half_btf_avx2(cospi24, bf1[21], cospi40, bf1[26], rounding, bit);
bf0[27] = bf1[27];
bf0[28] = bf1[28];
bf0[29] = half_btf_avx2(cospim8, bf1[18], cospi56, bf1[29], rounding, bit);
bf0[30] = half_btf_avx2(cospi56, bf1[17], cospi8, bf1[30], rounding, bit);
bf0[31] = bf1[31];
// stage 5
bf1[0] = half_btf_avx2(cospi32, bf0[0], cospi32, bf0[1], rounding, bit);
bf1[1] = half_btf_avx2(cospi32, bf0[0], cospim32, bf0[1], rounding, bit);
bf1[2] = half_btf_avx2(cospi48, bf0[2], cospim16, bf0[3], rounding, bit);
bf1[3] = half_btf_avx2(cospi16, bf0[2], cospi48, bf0[3], rounding, bit);
bf1[4] = _mm256_add_epi32(bf0[4], bf0[5]);
bf1[5] = _mm256_sub_epi32(bf0[4], bf0[5]);
bf1[6] = _mm256_sub_epi32(bf0[7], bf0[6]);
bf1[7] = _mm256_add_epi32(bf0[6], bf0[7]);
bf1[8] = bf0[8];
bf1[9] = half_btf_avx2(cospim16, bf0[9], cospi48, bf0[14], rounding, bit);
bf1[10] =
half_btf_avx2(cospim48, bf0[10], cospim16, bf0[13], rounding, bit);
bf1[11] = bf0[11];
bf1[12] = bf0[12];
bf1[13] = half_btf_avx2(cospim16, bf0[10], cospi48, bf0[13], rounding, bit);
bf1[14] = half_btf_avx2(cospi48, bf0[9], cospi16, bf0[14], rounding, bit);
bf1[15] = bf0[15];
bf1[16] = _mm256_add_epi32(bf0[16], bf0[19]);
bf1[17] = _mm256_add_epi32(bf0[17], bf0[18]);
bf1[18] = _mm256_sub_epi32(bf0[17], bf0[18]);
bf1[19] = _mm256_sub_epi32(bf0[16], bf0[19]);
bf1[20] = _mm256_sub_epi32(bf0[23], bf0[20]);
bf1[21] = _mm256_sub_epi32(bf0[22], bf0[21]);
bf1[22] = _mm256_add_epi32(bf0[21], bf0[22]);
bf1[23] = _mm256_add_epi32(bf0[20], bf0[23]);
bf1[24] = _mm256_add_epi32(bf0[24], bf0[27]);
bf1[25] = _mm256_add_epi32(bf0[25], bf0[26]);
bf1[26] = _mm256_sub_epi32(bf0[25], bf0[26]);
bf1[27] = _mm256_sub_epi32(bf0[24], bf0[27]);
bf1[28] = _mm256_sub_epi32(bf0[31], bf0[28]);
bf1[29] = _mm256_sub_epi32(bf0[30], bf0[29]);
bf1[30] = _mm256_add_epi32(bf0[29], bf0[30]);
bf1[31] = _mm256_add_epi32(bf0[28], bf0[31]);
// stage 6
bf0[0] = _mm256_add_epi32(bf1[0], bf1[3]);
bf0[1] = _mm256_add_epi32(bf1[1], bf1[2]);
bf0[2] = _mm256_sub_epi32(bf1[1], bf1[2]);
bf0[3] = _mm256_sub_epi32(bf1[0], bf1[3]);
bf0[4] = bf1[4];
bf0[5] = half_btf_avx2(cospim32, bf1[5], cospi32, bf1[6], rounding, bit);
bf0[6] = half_btf_avx2(cospi32, bf1[5], cospi32, bf1[6], rounding, bit);
bf0[7] = bf1[7];
bf0[8] = _mm256_add_epi32(bf1[8], bf1[11]);
bf0[9] = _mm256_add_epi32(bf1[9], bf1[10]);
bf0[10] = _mm256_sub_epi32(bf1[9], bf1[10]);
bf0[11] = _mm256_sub_epi32(bf1[8], bf1[11]);
bf0[12] = _mm256_sub_epi32(bf1[15], bf1[12]);
bf0[13] = _mm256_sub_epi32(bf1[14], bf1[13]);
bf0[14] = _mm256_add_epi32(bf1[13], bf1[14]);
bf0[15] = _mm256_add_epi32(bf1[12], bf1[15]);
bf0[16] = bf1[16];
bf0[17] = bf1[17];
bf0[18] = half_btf_avx2(cospim16, bf1[18], cospi48, bf1[29], rounding, bit);
bf0[19] = half_btf_avx2(cospim16, bf1[19], cospi48, bf1[28], rounding, bit);
bf0[20] =
half_btf_avx2(cospim48, bf1[20], cospim16, bf1[27], rounding, bit);
bf0[21] =
half_btf_avx2(cospim48, bf1[21], cospim16, bf1[26], rounding, bit);
bf0[22] = bf1[22];
bf0[23] = bf1[23];
bf0[24] = bf1[24];
bf0[25] = bf1[25];
bf0[26] = half_btf_avx2(cospim16, bf1[21], cospi48, bf1[26], rounding, bit);
bf0[27] = half_btf_avx2(cospim16, bf1[20], cospi48, bf1[27], rounding, bit);
bf0[28] = half_btf_avx2(cospi48, bf1[19], cospi16, bf1[28], rounding, bit);
bf0[29] = half_btf_avx2(cospi48, bf1[18], cospi16, bf1[29], rounding, bit);
bf0[30] = bf1[30];
bf0[31] = bf1[31];
// stage 7
bf1[0] = _mm256_add_epi32(bf0[0], bf0[7]);
bf1[1] = _mm256_add_epi32(bf0[1], bf0[6]);
bf1[2] = _mm256_add_epi32(bf0[2], bf0[5]);
bf1[3] = _mm256_add_epi32(bf0[3], bf0[4]);
bf1[4] = _mm256_sub_epi32(bf0[3], bf0[4]);
bf1[5] = _mm256_sub_epi32(bf0[2], bf0[5]);
bf1[6] = _mm256_sub_epi32(bf0[1], bf0[6]);
bf1[7] = _mm256_sub_epi32(bf0[0], bf0[7]);
bf1[8] = bf0[8];
bf1[9] = bf0[9];
bf1[10] = half_btf_avx2(cospim32, bf0[10], cospi32, bf0[13], rounding, bit);
bf1[11] = half_btf_avx2(cospim32, bf0[11], cospi32, bf0[12], rounding, bit);
bf1[12] = half_btf_avx2(cospi32, bf0[11], cospi32, bf0[12], rounding, bit);
bf1[13] = half_btf_avx2(cospi32, bf0[10], cospi32, bf0[13], rounding, bit);
bf1[14] = bf0[14];
bf1[15] = bf0[15];
bf1[16] = _mm256_add_epi32(bf0[16], bf0[23]);
bf1[17] = _mm256_add_epi32(bf0[17], bf0[22]);
bf1[18] = _mm256_add_epi32(bf0[18], bf0[21]);
bf1[19] = _mm256_add_epi32(bf0[19], bf0[20]);
bf1[20] = _mm256_sub_epi32(bf0[19], bf0[20]);
bf1[21] = _mm256_sub_epi32(bf0[18], bf0[21]);
bf1[22] = _mm256_sub_epi32(bf0[17], bf0[22]);
bf1[23] = _mm256_sub_epi32(bf0[16], bf0[23]);
bf1[24] = _mm256_sub_epi32(bf0[31], bf0[24]);
bf1[25] = _mm256_sub_epi32(bf0[30], bf0[25]);
bf1[26] = _mm256_sub_epi32(bf0[29], bf0[26]);
bf1[27] = _mm256_sub_epi32(bf0[28], bf0[27]);
bf1[28] = _mm256_add_epi32(bf0[27], bf0[28]);
bf1[29] = _mm256_add_epi32(bf0[26], bf0[29]);
bf1[30] = _mm256_add_epi32(bf0[25], bf0[30]);
bf1[31] = _mm256_add_epi32(bf0[24], bf0[31]);
// stage 8
bf0[0] = _mm256_add_epi32(bf1[0], bf1[15]);
bf0[1] = _mm256_add_epi32(bf1[1], bf1[14]);
bf0[2] = _mm256_add_epi32(bf1[2], bf1[13]);
bf0[3] = _mm256_add_epi32(bf1[3], bf1[12]);
bf0[4] = _mm256_add_epi32(bf1[4], bf1[11]);
bf0[5] = _mm256_add_epi32(bf1[5], bf1[10]);
bf0[6] = _mm256_add_epi32(bf1[6], bf1[9]);
bf0[7] = _mm256_add_epi32(bf1[7], bf1[8]);
bf0[8] = _mm256_sub_epi32(bf1[7], bf1[8]);
bf0[9] = _mm256_sub_epi32(bf1[6], bf1[9]);
bf0[10] = _mm256_sub_epi32(bf1[5], bf1[10]);
bf0[11] = _mm256_sub_epi32(bf1[4], bf1[11]);
bf0[12] = _mm256_sub_epi32(bf1[3], bf1[12]);
bf0[13] = _mm256_sub_epi32(bf1[2], bf1[13]);
bf0[14] = _mm256_sub_epi32(bf1[1], bf1[14]);
bf0[15] = _mm256_sub_epi32(bf1[0], bf1[15]);
bf0[16] = bf1[16];
bf0[17] = bf1[17];
bf0[18] = bf1[18];
bf0[19] = bf1[19];
bf0[20] = half_btf_avx2(cospim32, bf1[20], cospi32, bf1[27], rounding, bit);
bf0[21] = half_btf_avx2(cospim32, bf1[21], cospi32, bf1[26], rounding, bit);
bf0[22] = half_btf_avx2(cospim32, bf1[22], cospi32, bf1[25], rounding, bit);
bf0[23] = half_btf_avx2(cospim32, bf1[23], cospi32, bf1[24], rounding, bit);
bf0[24] = half_btf_avx2(cospi32, bf1[23], cospi32, bf1[24], rounding, bit);
bf0[25] = half_btf_avx2(cospi32, bf1[22], cospi32, bf1[25], rounding, bit);
bf0[26] = half_btf_avx2(cospi32, bf1[21], cospi32, bf1[26], rounding, bit);
bf0[27] = half_btf_avx2(cospi32, bf1[20], cospi32, bf1[27], rounding, bit);
bf0[28] = bf1[28];
bf0[29] = bf1[29];
bf0[30] = bf1[30];
bf0[31] = bf1[31];
// stage 9
out[0 * 4 + col] = _mm256_add_epi32(bf0[0], bf0[31]);
out[1 * 4 + col] = _mm256_add_epi32(bf0[1], bf0[30]);
out[2 * 4 + col] = _mm256_add_epi32(bf0[2], bf0[29]);
out[3 * 4 + col] = _mm256_add_epi32(bf0[3], bf0[28]);
out[4 * 4 + col] = _mm256_add_epi32(bf0[4], bf0[27]);
out[5 * 4 + col] = _mm256_add_epi32(bf0[5], bf0[26]);
out[6 * 4 + col] = _mm256_add_epi32(bf0[6], bf0[25]);
out[7 * 4 + col] = _mm256_add_epi32(bf0[7], bf0[24]);
out[8 * 4 + col] = _mm256_add_epi32(bf0[8], bf0[23]);
out[9 * 4 + col] = _mm256_add_epi32(bf0[9], bf0[22]);
out[10 * 4 + col] = _mm256_add_epi32(bf0[10], bf0[21]);
out[11 * 4 + col] = _mm256_add_epi32(bf0[11], bf0[20]);
out[12 * 4 + col] = _mm256_add_epi32(bf0[12], bf0[19]);
out[13 * 4 + col] = _mm256_add_epi32(bf0[13], bf0[18]);
out[14 * 4 + col] = _mm256_add_epi32(bf0[14], bf0[17]);
out[15 * 4 + col] = _mm256_add_epi32(bf0[15], bf0[16]);
out[16 * 4 + col] = _mm256_sub_epi32(bf0[15], bf0[16]);
out[17 * 4 + col] = _mm256_sub_epi32(bf0[14], bf0[17]);
out[18 * 4 + col] = _mm256_sub_epi32(bf0[13], bf0[18]);
out[19 * 4 + col] = _mm256_sub_epi32(bf0[12], bf0[19]);
out[20 * 4 + col] = _mm256_sub_epi32(bf0[11], bf0[20]);
out[21 * 4 + col] = _mm256_sub_epi32(bf0[10], bf0[21]);
out[22 * 4 + col] = _mm256_sub_epi32(bf0[9], bf0[22]);
out[23 * 4 + col] = _mm256_sub_epi32(bf0[8], bf0[23]);
out[24 * 4 + col] = _mm256_sub_epi32(bf0[7], bf0[24]);
out[25 * 4 + col] = _mm256_sub_epi32(bf0[6], bf0[25]);
out[26 * 4 + col] = _mm256_sub_epi32(bf0[5], bf0[26]);
out[27 * 4 + col] = _mm256_sub_epi32(bf0[4], bf0[27]);
out[28 * 4 + col] = _mm256_sub_epi32(bf0[3], bf0[28]);
out[29 * 4 + col] = _mm256_sub_epi32(bf0[2], bf0[29]);
out[30 * 4 + col] = _mm256_sub_epi32(bf0[1], bf0[30]);
out[31 * 4 + col] = _mm256_sub_epi32(bf0[0], bf0[31]);
}
}
void av1_inv_txfm2d_add_32x32_avx2(const int32_t *coeff, uint16_t *output,
int stride, int tx_type, int bd) {
__m256i in[128], out[128];
const TXFM_2D_CFG *cfg = NULL;
switch (tx_type) {
case DCT_DCT:
cfg = &inv_txfm_2d_cfg_dct_dct_32;
load_buffer_32x32(coeff, in);
transpose_32x32(in, out);
idct32_avx2(out, in, cfg->cos_bit_row[2]);
round_shift_32x32(in, -cfg->shift[0]);
transpose_32x32(in, out);
idct32_avx2(out, in, cfg->cos_bit_col[2]);
write_buffer_32x32(in, output, stride, 0, 0, -cfg->shift[1], bd);
break;
default: assert(0);
}
}

1398
third_party/aom/av1/common/x86/highbd_inv_txfm_sse4.c vendored Normal file
View file

File diff suppressed because it is too large Load diff

92
third_party/aom/av1/common/x86/highbd_txfm_utility_sse4.h vendored Normal file
View file

@ -0,0 +1,92 @@
/*
* Copyright (c) 2016, Alliance for Open Media. All rights reserved
*
* This source code is subject to the terms of the BSD 2 Clause License and
* the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
* was not distributed with this source code in the LICENSE file, you can
* obtain it at www.aomedia.org/license/software. If the Alliance for Open
* Media Patent License 1.0 was not distributed with this source code in the
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
*/
#ifndef _HIGHBD_TXFM_UTILITY_SSE4_H
#define _HIGHBD_TXFM_UTILITY_SSE4_H
#include <smmintrin.h> /* SSE4.1 */
#define TRANSPOSE_4X4(x0, x1, x2, x3, y0, y1, y2, y3) \
do { \
__m128i u0, u1, u2, u3; \
u0 = _mm_unpacklo_epi32(x0, x1); \
u1 = _mm_unpackhi_epi32(x0, x1); \
u2 = _mm_unpacklo_epi32(x2, x3); \
u3 = _mm_unpackhi_epi32(x2, x3); \
y0 = _mm_unpacklo_epi64(u0, u2); \
y1 = _mm_unpackhi_epi64(u0, u2); \
y2 = _mm_unpacklo_epi64(u1, u3); \
y3 = _mm_unpackhi_epi64(u1, u3); \
} while (0)
static INLINE void transpose_8x8(const __m128i *in, __m128i *out) {
TRANSPOSE_4X4(in[0], in[2], in[4], in[6], out[0], out[2], out[4], out[6]);
TRANSPOSE_4X4(in[1], in[3], in[5], in[7], out[8], out[10], out[12], out[14]);
TRANSPOSE_4X4(in[8], in[10], in[12], in[14], out[1], out[3], out[5], out[7]);
TRANSPOSE_4X4(in[9], in[11], in[13], in[15], out[9], out[11], out[13],
out[15]);
}
static INLINE void transpose_16x16(const __m128i *in, __m128i *out) {
// Upper left 8x8
TRANSPOSE_4X4(in[0], in[4], in[8], in[12], out[0], out[4], out[8], out[12]);
TRANSPOSE_4X4(in[1], in[5], in[9], in[13], out[16], out[20], out[24],
out[28]);
TRANSPOSE_4X4(in[16], in[20], in[24], in[28], out[1], out[5], out[9],
out[13]);
TRANSPOSE_4X4(in[17], in[21], in[25], in[29], out[17], out[21], out[25],
out[29]);
// Upper right 8x8
TRANSPOSE_4X4(in[2], in[6], in[10], in[14], out[32], out[36], out[40],
out[44]);
TRANSPOSE_4X4(in[3], in[7], in[11], in[15], out[48], out[52], out[56],
out[60]);
TRANSPOSE_4X4(in[18], in[22], in[26], in[30], out[33], out[37], out[41],
out[45]);
TRANSPOSE_4X4(in[19], in[23], in[27], in[31], out[49], out[53], out[57],
out[61]);
// Lower left 8x8
TRANSPOSE_4X4(in[32], in[36], in[40], in[44], out[2], out[6], out[10],
out[14]);
TRANSPOSE_4X4(in[33], in[37], in[41], in[45], out[18], out[22], out[26],
out[30]);
TRANSPOSE_4X4(in[48], in[52], in[56], in[60], out[3], out[7], out[11],
out[15]);
TRANSPOSE_4X4(in[49], in[53], in[57], in[61], out[19], out[23], out[27],
out[31]);
// Lower right 8x8
TRANSPOSE_4X4(in[34], in[38], in[42], in[46], out[34], out[38], out[42],
out[46]);
TRANSPOSE_4X4(in[35], in[39], in[43], in[47], out[50], out[54], out[58],
out[62]);
TRANSPOSE_4X4(in[50], in[54], in[58], in[62], out[35], out[39], out[43],
out[47]);
TRANSPOSE_4X4(in[51], in[55], in[59], in[63], out[51], out[55], out[59],
out[63]);
}
// Note:
// rounding = 1 << (bit - 1)
static INLINE __m128i half_btf_sse4_1(__m128i w0, __m128i n0, __m128i w1,
__m128i n1, __m128i rounding, int bit) {
__m128i x, y;
x = _mm_mullo_epi32(w0, n0);
y = _mm_mullo_epi32(w1, n1);
x = _mm_add_epi32(x, y);
x = _mm_add_epi32(x, rounding);
x = _mm_srai_epi32(x, bit);
return x;
}
#endif // _HIGHBD_TXFM_UTILITY_SSE4_H

286
third_party/aom/av1/common/x86/highbd_warp_plane_ssse3.c vendored Normal file
View file

@ -0,0 +1,286 @@
/*
* Copyright (c) 2016, Alliance for Open Media. All rights reserved
*
* This source code is subject to the terms of the BSD 2 Clause License and
* the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
* was not distributed with this source code in the LICENSE file, you can
* obtain it at www.aomedia.org/license/software. If the Alliance for Open
* Media Patent License 1.0 was not distributed with this source code in the
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
*/
#include <tmmintrin.h>
#include "./av1_rtcd.h"
#include "av1/common/warped_motion.h"
static const __m128i *const filter = (const __m128i *const)warped_filter;
/* SSE2 version of the rotzoom/affine warp filter */
void av1_highbd_warp_affine_ssse3(int32_t *mat, uint16_t *ref, int width,
int height, int stride, uint16_t *pred,
int p_col, int p_row, int p_width,
int p_height, int p_stride, int subsampling_x,
int subsampling_y, int bd, int ref_frm,
int16_t alpha, int16_t beta, int16_t gamma,
int16_t delta) {
#if HORSHEAR_REDUCE_PREC_BITS >= 5
__m128i tmp[15];
#else
#error "HORSHEAR_REDUCE_PREC_BITS < 5 not currently supported by SSSE3 filter"
#endif
int i, j, k;
/* Note: For this code to work, the left/right frame borders need to be
extended by at least 13 pixels each. By the time we get here, other
code will have set up this border, but we allow an explicit check
for debugging purposes.
*/
/*for (i = 0; i < height; ++i) {
for (j = 0; j < 13; ++j) {
assert(ref[i * stride - 13 + j] == ref[i * stride]);
assert(ref[i * stride + width + j] == ref[i * stride + (width - 1)]);
}
}*/
for (i = 0; i < p_height; i += 8) {
for (j = 0; j < p_width; j += 8) {
// (x, y) coordinates of the center of this block in the destination
// image
int32_t dst_x = p_col + j + 4;
int32_t dst_y = p_row + i + 4;
int32_t x4, y4, ix4, sx4, iy4, sy4;
if (subsampling_x)
x4 = ROUND_POWER_OF_TWO_SIGNED(
mat[2] * 2 * dst_x + mat[3] * 2 * dst_y + mat[0] +
(mat[2] + mat[3] - (1 << WARPEDMODEL_PREC_BITS)) / 2,
1);
else
x4 = mat[2] * dst_x + mat[3] * dst_y + mat[0];
if (subsampling_y)
y4 = ROUND_POWER_OF_TWO_SIGNED(
mat[4] * 2 * dst_x + mat[5] * 2 * dst_y + mat[1] +
(mat[4] + mat[5] - (1 << WARPEDMODEL_PREC_BITS)) / 2,
1);
else
y4 = mat[4] * dst_x + mat[5] * dst_y + mat[1];
ix4 = x4 >> WARPEDMODEL_PREC_BITS;
sx4 = x4 & ((1 << WARPEDMODEL_PREC_BITS) - 1);
iy4 = y4 >> WARPEDMODEL_PREC_BITS;
sy4 = y4 & ((1 << WARPEDMODEL_PREC_BITS) - 1);
// Horizontal filter
for (k = -7; k < AOMMIN(8, p_height - i); ++k) {
int iy = iy4 + k;
if (iy < 0)
iy = 0;
else if (iy > height - 1)
iy = height - 1;
// If the block is aligned such that, after clamping, every sample
// would be taken from the leftmost/rightmost column, then we can
// skip the expensive horizontal filter.
if (ix4 <= -7) {
tmp[k + 7] = _mm_set1_epi16(
ref[iy * stride] *
(1 << (WARPEDPIXEL_FILTER_BITS - HORSHEAR_REDUCE_PREC_BITS)));
} else if (ix4 >= width + 6) {
tmp[k + 7] = _mm_set1_epi16(
ref[iy * stride + (width - 1)] *
(1 << (WARPEDPIXEL_FILTER_BITS - HORSHEAR_REDUCE_PREC_BITS)));
} else {
int sx = sx4 + alpha * (-4) + beta * k +
// Include rounding and offset here
(1 << (WARPEDDIFF_PREC_BITS - 1)) +
(WARPEDPIXEL_PREC_SHIFTS << WARPEDDIFF_PREC_BITS);
// Load source pixels
__m128i src =
_mm_loadu_si128((__m128i *)(ref + iy * stride + ix4 - 7));
__m128i src2 =
_mm_loadu_si128((__m128i *)(ref + iy * stride + ix4 + 1));
// Filter even-index pixels
__m128i tmp_0 = filter[(sx + 0 * alpha) >> WARPEDDIFF_PREC_BITS];
__m128i tmp_2 = filter[(sx + 2 * alpha) >> WARPEDDIFF_PREC_BITS];
__m128i tmp_4 = filter[(sx + 4 * alpha) >> WARPEDDIFF_PREC_BITS];
__m128i tmp_6 = filter[(sx + 6 * alpha) >> WARPEDDIFF_PREC_BITS];
// coeffs 0 1 0 1 2 3 2 3 for pixels 0, 2
__m128i tmp_8 = _mm_unpacklo_epi32(tmp_0, tmp_2);
// coeffs 0 1 0 1 2 3 2 3 for pixels 4, 6
__m128i tmp_10 = _mm_unpacklo_epi32(tmp_4, tmp_6);
// coeffs 4 5 4 5 6 7 6 7 for pixels 0, 2
__m128i tmp_12 = _mm_unpackhi_epi32(tmp_0, tmp_2);
// coeffs 4 5 4 5 6 7 6 7 for pixels 4, 6
__m128i tmp_14 = _mm_unpackhi_epi32(tmp_4, tmp_6);
// coeffs 0 1 0 1 0 1 0 1 for pixels 0, 2, 4, 6
__m128i coeff_0 = _mm_unpacklo_epi64(tmp_8, tmp_10);
// coeffs 2 3 2 3 2 3 2 3 for pixels 0, 2, 4, 6
__m128i coeff_2 = _mm_unpackhi_epi64(tmp_8, tmp_10);
// coeffs 4 5 4 5 4 5 4 5 for pixels 0, 2, 4, 6
__m128i coeff_4 = _mm_unpacklo_epi64(tmp_12, tmp_14);
// coeffs 6 7 6 7 6 7 6 7 for pixels 0, 2, 4, 6
__m128i coeff_6 = _mm_unpackhi_epi64(tmp_12, tmp_14);
__m128i round_const =
_mm_set1_epi32((1 << HORSHEAR_REDUCE_PREC_BITS) >> 1);
// Calculate filtered results
__m128i res_0 = _mm_madd_epi16(src, coeff_0);
__m128i res_2 =
_mm_madd_epi16(_mm_alignr_epi8(src2, src, 4), coeff_2);
__m128i res_4 =
_mm_madd_epi16(_mm_alignr_epi8(src2, src, 8), coeff_4);
__m128i res_6 =
_mm_madd_epi16(_mm_alignr_epi8(src2, src, 12), coeff_6);
__m128i res_even = _mm_add_epi32(_mm_add_epi32(res_0, res_4),
_mm_add_epi32(res_2, res_6));
res_even = _mm_srai_epi32(_mm_add_epi32(res_even, round_const),
HORSHEAR_REDUCE_PREC_BITS);
// Filter odd-index pixels
__m128i tmp_1 = filter[(sx + 1 * alpha) >> WARPEDDIFF_PREC_BITS];
__m128i tmp_3 = filter[(sx + 3 * alpha) >> WARPEDDIFF_PREC_BITS];
__m128i tmp_5 = filter[(sx + 5 * alpha) >> WARPEDDIFF_PREC_BITS];
__m128i tmp_7 = filter[(sx + 7 * alpha) >> WARPEDDIFF_PREC_BITS];
__m128i tmp_9 = _mm_unpacklo_epi32(tmp_1, tmp_3);
__m128i tmp_11 = _mm_unpacklo_epi32(tmp_5, tmp_7);
__m128i tmp_13 = _mm_unpackhi_epi32(tmp_1, tmp_3);
__m128i tmp_15 = _mm_unpackhi_epi32(tmp_5, tmp_7);
__m128i coeff_1 = _mm_unpacklo_epi64(tmp_9, tmp_11);
__m128i coeff_3 = _mm_unpackhi_epi64(tmp_9, tmp_11);
__m128i coeff_5 = _mm_unpacklo_epi64(tmp_13, tmp_15);
__m128i coeff_7 = _mm_unpackhi_epi64(tmp_13, tmp_15);
__m128i res_1 =
_mm_madd_epi16(_mm_alignr_epi8(src2, src, 2), coeff_1);
__m128i res_3 =
_mm_madd_epi16(_mm_alignr_epi8(src2, src, 6), coeff_3);
__m128i res_5 =
_mm_madd_epi16(_mm_alignr_epi8(src2, src, 10), coeff_5);
__m128i res_7 =
_mm_madd_epi16(_mm_alignr_epi8(src2, src, 14), coeff_7);
__m128i res_odd = _mm_add_epi32(_mm_add_epi32(res_1, res_5),
_mm_add_epi32(res_3, res_7));
res_odd = _mm_srai_epi32(_mm_add_epi32(res_odd, round_const),
HORSHEAR_REDUCE_PREC_BITS);
// Combine results into one register.
// We store the columns in the order 0, 2, 4, 6, 1, 3, 5, 7
// as this order helps with the vertical filter.
tmp[k + 7] = _mm_packs_epi32(res_even, res_odd);
}
}
// Vertical filter
for (k = -4; k < AOMMIN(4, p_height - i - 4); ++k) {
int sy = sy4 + gamma * (-4) + delta * k +
(1 << (WARPEDDIFF_PREC_BITS - 1)) +
(WARPEDPIXEL_PREC_SHIFTS << WARPEDDIFF_PREC_BITS);
// Load from tmp and rearrange pairs of consecutive rows into the
// column order 0 0 2 2 4 4 6 6; 1 1 3 3 5 5 7 7
__m128i *src = tmp + (k + 4);
__m128i src_0 = _mm_unpacklo_epi16(src[0], src[1]);
__m128i src_2 = _mm_unpacklo_epi16(src[2], src[3]);
__m128i src_4 = _mm_unpacklo_epi16(src[4], src[5]);
__m128i src_6 = _mm_unpacklo_epi16(src[6], src[7]);
// Filter even-index pixels
__m128i tmp_0 = filter[(sy + 0 * gamma) >> WARPEDDIFF_PREC_BITS];
__m128i tmp_2 = filter[(sy + 2 * gamma) >> WARPEDDIFF_PREC_BITS];
__m128i tmp_4 = filter[(sy + 4 * gamma) >> WARPEDDIFF_PREC_BITS];
__m128i tmp_6 = filter[(sy + 6 * gamma) >> WARPEDDIFF_PREC_BITS];
__m128i tmp_8 = _mm_unpacklo_epi32(tmp_0, tmp_2);
__m128i tmp_10 = _mm_unpacklo_epi32(tmp_4, tmp_6);
__m128i tmp_12 = _mm_unpackhi_epi32(tmp_0, tmp_2);
__m128i tmp_14 = _mm_unpackhi_epi32(tmp_4, tmp_6);
__m128i coeff_0 = _mm_unpacklo_epi64(tmp_8, tmp_10);
__m128i coeff_2 = _mm_unpackhi_epi64(tmp_8, tmp_10);
__m128i coeff_4 = _mm_unpacklo_epi64(tmp_12, tmp_14);
__m128i coeff_6 = _mm_unpackhi_epi64(tmp_12, tmp_14);
__m128i res_0 = _mm_madd_epi16(src_0, coeff_0);
__m128i res_2 = _mm_madd_epi16(src_2, coeff_2);
__m128i res_4 = _mm_madd_epi16(src_4, coeff_4);
__m128i res_6 = _mm_madd_epi16(src_6, coeff_6);
__m128i res_even = _mm_add_epi32(_mm_add_epi32(res_0, res_2),
_mm_add_epi32(res_4, res_6));
// Filter odd-index pixels
__m128i src_1 = _mm_unpackhi_epi16(src[0], src[1]);
__m128i src_3 = _mm_unpackhi_epi16(src[2], src[3]);
__m128i src_5 = _mm_unpackhi_epi16(src[4], src[5]);
__m128i src_7 = _mm_unpackhi_epi16(src[6], src[7]);
__m128i tmp_1 = filter[(sy + 1 * gamma) >> WARPEDDIFF_PREC_BITS];
__m128i tmp_3 = filter[(sy + 3 * gamma) >> WARPEDDIFF_PREC_BITS];
__m128i tmp_5 = filter[(sy + 5 * gamma) >> WARPEDDIFF_PREC_BITS];
__m128i tmp_7 = filter[(sy + 7 * gamma) >> WARPEDDIFF_PREC_BITS];
__m128i tmp_9 = _mm_unpacklo_epi32(tmp_1, tmp_3);
__m128i tmp_11 = _mm_unpacklo_epi32(tmp_5, tmp_7);
__m128i tmp_13 = _mm_unpackhi_epi32(tmp_1, tmp_3);
__m128i tmp_15 = _mm_unpackhi_epi32(tmp_5, tmp_7);
__m128i coeff_1 = _mm_unpacklo_epi64(tmp_9, tmp_11);
__m128i coeff_3 = _mm_unpackhi_epi64(tmp_9, tmp_11);
__m128i coeff_5 = _mm_unpacklo_epi64(tmp_13, tmp_15);
__m128i coeff_7 = _mm_unpackhi_epi64(tmp_13, tmp_15);
__m128i res_1 = _mm_madd_epi16(src_1, coeff_1);
__m128i res_3 = _mm_madd_epi16(src_3, coeff_3);
__m128i res_5 = _mm_madd_epi16(src_5, coeff_5);
__m128i res_7 = _mm_madd_epi16(src_7, coeff_7);
__m128i res_odd = _mm_add_epi32(_mm_add_epi32(res_1, res_3),
_mm_add_epi32(res_5, res_7));
// Rearrange pixels back into the order 0 ... 7
__m128i res_lo = _mm_unpacklo_epi32(res_even, res_odd);
__m128i res_hi = _mm_unpackhi_epi32(res_even, res_odd);
// Round and pack into 8 bits
__m128i round_const =
_mm_set1_epi32((1 << VERSHEAR_REDUCE_PREC_BITS) >> 1);
__m128i res_lo_round = _mm_srai_epi32(
_mm_add_epi32(res_lo, round_const), VERSHEAR_REDUCE_PREC_BITS);
__m128i res_hi_round = _mm_srai_epi32(
_mm_add_epi32(res_hi, round_const), VERSHEAR_REDUCE_PREC_BITS);
__m128i res_16bit = _mm_packs_epi32(res_lo_round, res_hi_round);
// Clamp res_16bit to the range [0, 2^bd - 1]
__m128i max_val = _mm_set1_epi16((1 << bd) - 1);
__m128i zero = _mm_setzero_si128();
res_16bit = _mm_max_epi16(_mm_min_epi16(res_16bit, max_val), zero);
// Store, blending with 'pred' if needed
__m128i *p = (__m128i *)&pred[(i + k + 4) * p_stride + j];
// Note: If we're outputting a 4x4 block, we need to be very careful
// to only output 4 pixels at this point, to avoid encode/decode
// mismatches when encoding with multiple threads.
if (p_width == 4) {
if (ref_frm) res_16bit = _mm_avg_epu16(res_16bit, _mm_loadl_epi64(p));
_mm_storel_epi64(p, res_16bit);
} else {
if (ref_frm) res_16bit = _mm_avg_epu16(res_16bit, _mm_loadu_si128(p));
_mm_storeu_si128(p, res_16bit);
}
}
}
}
}

507
third_party/aom/av1/common/x86/hybrid_inv_txfm_avx2.c vendored Normal file
View file

@ -0,0 +1,507 @@
/*
* Copyright (c) 2016, Alliance for Open Media. All rights reserved
*
* This source code is subject to the terms of the BSD 2 Clause License and
* the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
* was not distributed with this source code in the LICENSE file, you can
* obtain it at www.aomedia.org/license/software. If the Alliance for Open
* Media Patent License 1.0 was not distributed with this source code in the
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
*/
#include <immintrin.h> // avx2
#include "./aom_config.h"
#include "./av1_rtcd.h"
#include "aom_dsp/x86/txfm_common_avx2.h"
static INLINE void load_coeff(const tran_low_t *coeff, __m256i *in) {
#if CONFIG_HIGHBITDEPTH
*in = _mm256_setr_epi16(
(int16_t)coeff[0], (int16_t)coeff[1], (int16_t)coeff[2],
(int16_t)coeff[3], (int16_t)coeff[4], (int16_t)coeff[5],
(int16_t)coeff[6], (int16_t)coeff[7], (int16_t)coeff[8],
(int16_t)coeff[9], (int16_t)coeff[10], (int16_t)coeff[11],
(int16_t)coeff[12], (int16_t)coeff[13], (int16_t)coeff[14],
(int16_t)coeff[15]);
#else
*in = _mm256_loadu_si256((const __m256i *)coeff);
#endif
}
static void load_buffer_16x16(const tran_low_t *coeff, __m256i *in) {
int i = 0;
while (i < 16) {
load_coeff(coeff + (i << 4), &in[i]);
i += 1;
}
}
static void recon_and_store(const __m256i *res, uint8_t *output) {
const __m128i zero = _mm_setzero_si128();
__m128i x = _mm_loadu_si128((__m128i const *)output);
__m128i p0 = _mm_unpacklo_epi8(x, zero);
__m128i p1 = _mm_unpackhi_epi8(x, zero);
p0 = _mm_add_epi16(p0, _mm256_castsi256_si128(*res));
p1 = _mm_add_epi16(p1, _mm256_extractf128_si256(*res, 1));
x = _mm_packus_epi16(p0, p1);
_mm_storeu_si128((__m128i *)output, x);
}
#define IDCT_ROUNDING_POS (6)
static void write_buffer_16x16(__m256i *in, const int stride, uint8_t *output) {
const __m256i rounding = _mm256_set1_epi16(1 << (IDCT_ROUNDING_POS - 1));
int i = 0;
while (i < 16) {
in[i] = _mm256_add_epi16(in[i], rounding);
in[i] = _mm256_srai_epi16(in[i], IDCT_ROUNDING_POS);
recon_and_store(&in[i], output + i * stride);
i += 1;
}
}
static INLINE void unpack_butter_fly(const __m256i *a0, const __m256i *a1,
const __m256i *c0, const __m256i *c1,
__m256i *b0, __m256i *b1) {
__m256i x0, x1;
x0 = _mm256_unpacklo_epi16(*a0, *a1);
x1 = _mm256_unpackhi_epi16(*a0, *a1);
*b0 = butter_fly(x0, x1, *c0);
*b1 = butter_fly(x0, x1, *c1);
}
static void idct16_avx2(__m256i *in) {
const __m256i cospi_p30_m02 = pair256_set_epi16(cospi_30_64, -cospi_2_64);
const __m256i cospi_p02_p30 = pair256_set_epi16(cospi_2_64, cospi_30_64);
const __m256i cospi_p14_m18 = pair256_set_epi16(cospi_14_64, -cospi_18_64);
const __m256i cospi_p18_p14 = pair256_set_epi16(cospi_18_64, cospi_14_64);
const __m256i cospi_p22_m10 = pair256_set_epi16(cospi_22_64, -cospi_10_64);
const __m256i cospi_p10_p22 = pair256_set_epi16(cospi_10_64, cospi_22_64);
const __m256i cospi_p06_m26 = pair256_set_epi16(cospi_6_64, -cospi_26_64);
const __m256i cospi_p26_p06 = pair256_set_epi16(cospi_26_64, cospi_6_64);
const __m256i cospi_p28_m04 = pair256_set_epi16(cospi_28_64, -cospi_4_64);
const __m256i cospi_p04_p28 = pair256_set_epi16(cospi_4_64, cospi_28_64);
const __m256i cospi_p12_m20 = pair256_set_epi16(cospi_12_64, -cospi_20_64);
const __m256i cospi_p20_p12 = pair256_set_epi16(cospi_20_64, cospi_12_64);
const __m256i cospi_p16_p16 = _mm256_set1_epi16((int16_t)cospi_16_64);
const __m256i cospi_p16_m16 = pair256_set_epi16(cospi_16_64, -cospi_16_64);
const __m256i cospi_p24_m08 = pair256_set_epi16(cospi_24_64, -cospi_8_64);
const __m256i cospi_p08_p24 = pair256_set_epi16(cospi_8_64, cospi_24_64);
const __m256i cospi_m08_p24 = pair256_set_epi16(-cospi_8_64, cospi_24_64);
const __m256i cospi_p24_p08 = pair256_set_epi16(cospi_24_64, cospi_8_64);
const __m256i cospi_m24_m08 = pair256_set_epi16(-cospi_24_64, -cospi_8_64);
__m256i u0, u1, u2, u3, u4, u5, u6, u7;
__m256i v0, v1, v2, v3, v4, v5, v6, v7;
__m256i t0, t1, t2, t3, t4, t5, t6, t7;
// stage 1, (0-7)
u0 = in[0];
u1 = in[8];
u2 = in[4];
u3 = in[12];
u4 = in[2];
u5 = in[10];
u6 = in[6];
u7 = in[14];
// stage 2, (0-7)
// stage 3, (0-7)
t0 = u0;
t1 = u1;
t2 = u2;
t3 = u3;
unpack_butter_fly(&u4, &u7, &cospi_p28_m04, &cospi_p04_p28, &t4, &t7);
unpack_butter_fly(&u5, &u6, &cospi_p12_m20, &cospi_p20_p12, &t5, &t6);
// stage 4, (0-7)
unpack_butter_fly(&t0, &t1, &cospi_p16_p16, &cospi_p16_m16, &u0, &u1);
unpack_butter_fly(&t2, &t3, &cospi_p24_m08, &cospi_p08_p24, &u2, &u3);
u4 = _mm256_add_epi16(t4, t5);
u5 = _mm256_sub_epi16(t4, t5);
u6 = _mm256_sub_epi16(t7, t6);
u7 = _mm256_add_epi16(t7, t6);
// stage 5, (0-7)
t0 = _mm256_add_epi16(u0, u3);
t1 = _mm256_add_epi16(u1, u2);
t2 = _mm256_sub_epi16(u1, u2);
t3 = _mm256_sub_epi16(u0, u3);
t4 = u4;
t7 = u7;
unpack_butter_fly(&u6, &u5, &cospi_p16_m16, &cospi_p16_p16, &t5, &t6);
// stage 6, (0-7)
u0 = _mm256_add_epi16(t0, t7);
u1 = _mm256_add_epi16(t1, t6);
u2 = _mm256_add_epi16(t2, t5);
u3 = _mm256_add_epi16(t3, t4);
u4 = _mm256_sub_epi16(t3, t4);
u5 = _mm256_sub_epi16(t2, t5);
u6 = _mm256_sub_epi16(t1, t6);
u7 = _mm256_sub_epi16(t0, t7);
// stage 1, (8-15)
v0 = in[1];
v1 = in[9];
v2 = in[5];
v3 = in[13];
v4 = in[3];
v5 = in[11];
v6 = in[7];
v7 = in[15];
// stage 2, (8-15)
unpack_butter_fly(&v0, &v7, &cospi_p30_m02, &cospi_p02_p30, &t0, &t7);
unpack_butter_fly(&v1, &v6, &cospi_p14_m18, &cospi_p18_p14, &t1, &t6);
unpack_butter_fly(&v2, &v5, &cospi_p22_m10, &cospi_p10_p22, &t2, &t5);
unpack_butter_fly(&v3, &v4, &cospi_p06_m26, &cospi_p26_p06, &t3, &t4);
// stage 3, (8-15)
v0 = _mm256_add_epi16(t0, t1);
v1 = _mm256_sub_epi16(t0, t1);
v2 = _mm256_sub_epi16(t3, t2);
v3 = _mm256_add_epi16(t2, t3);
v4 = _mm256_add_epi16(t4, t5);
v5 = _mm256_sub_epi16(t4, t5);
v6 = _mm256_sub_epi16(t7, t6);
v7 = _mm256_add_epi16(t6, t7);
// stage 4, (8-15)
t0 = v0;
t7 = v7;
t3 = v3;
t4 = v4;
unpack_butter_fly(&v1, &v6, &cospi_m08_p24, &cospi_p24_p08, &t1, &t6);
unpack_butter_fly(&v2, &v5, &cospi_m24_m08, &cospi_m08_p24, &t2, &t5);
// stage 5, (8-15)
v0 = _mm256_add_epi16(t0, t3);
v1 = _mm256_add_epi16(t1, t2);
v2 = _mm256_sub_epi16(t1, t2);
v3 = _mm256_sub_epi16(t0, t3);
v4 = _mm256_sub_epi16(t7, t4);
v5 = _mm256_sub_epi16(t6, t5);
v6 = _mm256_add_epi16(t6, t5);
v7 = _mm256_add_epi16(t7, t4);
// stage 6, (8-15)
t0 = v0;
t1 = v1;
t6 = v6;
t7 = v7;
unpack_butter_fly(&v5, &v2, &cospi_p16_m16, &cospi_p16_p16, &t2, &t5);
unpack_butter_fly(&v4, &v3, &cospi_p16_m16, &cospi_p16_p16, &t3, &t4);
// stage 7
in[0] = _mm256_add_epi16(u0, t7);
in[1] = _mm256_add_epi16(u1, t6);
in[2] = _mm256_add_epi16(u2, t5);
in[3] = _mm256_add_epi16(u3, t4);
in[4] = _mm256_add_epi16(u4, t3);
in[5] = _mm256_add_epi16(u5, t2);
in[6] = _mm256_add_epi16(u6, t1);
in[7] = _mm256_add_epi16(u7, t0);
in[8] = _mm256_sub_epi16(u7, t0);
in[9] = _mm256_sub_epi16(u6, t1);
in[10] = _mm256_sub_epi16(u5, t2);
in[11] = _mm256_sub_epi16(u4, t3);
in[12] = _mm256_sub_epi16(u3, t4);
in[13] = _mm256_sub_epi16(u2, t5);
in[14] = _mm256_sub_epi16(u1, t6);
in[15] = _mm256_sub_epi16(u0, t7);
}
static void idct16(__m256i *in) {
mm256_transpose_16x16(in);
idct16_avx2(in);
}
static INLINE void butterfly_32b(const __m256i *a0, const __m256i *a1,
const __m256i *c0, const __m256i *c1,
__m256i *b) {
__m256i x0, x1;
x0 = _mm256_unpacklo_epi16(*a0, *a1);
x1 = _mm256_unpackhi_epi16(*a0, *a1);
b[0] = _mm256_madd_epi16(x0, *c0);
b[1] = _mm256_madd_epi16(x1, *c0);
b[2] = _mm256_madd_epi16(x0, *c1);
b[3] = _mm256_madd_epi16(x1, *c1);
}
static INLINE void group_rounding(__m256i *a, int num) {
const __m256i dct_rounding = _mm256_set1_epi32(DCT_CONST_ROUNDING);
int i;
for (i = 0; i < num; ++i) {
a[i] = _mm256_add_epi32(a[i], dct_rounding);
a[i] = _mm256_srai_epi32(a[i], DCT_CONST_BITS);
}
}
static INLINE void add_rnd(const __m256i *a, const __m256i *b, __m256i *out) {
__m256i x[4];
x[0] = _mm256_add_epi32(a[0], b[0]);
x[1] = _mm256_add_epi32(a[1], b[1]);
x[2] = _mm256_add_epi32(a[2], b[2]);
x[3] = _mm256_add_epi32(a[3], b[3]);
group_rounding(x, 4);
out[0] = _mm256_packs_epi32(x[0], x[1]);
out[1] = _mm256_packs_epi32(x[2], x[3]);
}
static INLINE void sub_rnd(const __m256i *a, const __m256i *b, __m256i *out) {
__m256i x[4];
x[0] = _mm256_sub_epi32(a[0], b[0]);
x[1] = _mm256_sub_epi32(a[1], b[1]);
x[2] = _mm256_sub_epi32(a[2], b[2]);
x[3] = _mm256_sub_epi32(a[3], b[3]);
group_rounding(x, 4);
out[0] = _mm256_packs_epi32(x[0], x[1]);
out[1] = _mm256_packs_epi32(x[2], x[3]);
}
static INLINE void butterfly_rnd(__m256i *a, __m256i *out) {
group_rounding(a, 4);
out[0] = _mm256_packs_epi32(a[0], a[1]);
out[1] = _mm256_packs_epi32(a[2], a[3]);
}
static void iadst16_avx2(__m256i *in) {
const __m256i cospi_p01_p31 = pair256_set_epi16(cospi_1_64, cospi_31_64);
const __m256i cospi_p31_m01 = pair256_set_epi16(cospi_31_64, -cospi_1_64);
const __m256i cospi_p05_p27 = pair256_set_epi16(cospi_5_64, cospi_27_64);
const __m256i cospi_p27_m05 = pair256_set_epi16(cospi_27_64, -cospi_5_64);
const __m256i cospi_p09_p23 = pair256_set_epi16(cospi_9_64, cospi_23_64);
const __m256i cospi_p23_m09 = pair256_set_epi16(cospi_23_64, -cospi_9_64);
const __m256i cospi_p13_p19 = pair256_set_epi16(cospi_13_64, cospi_19_64);
const __m256i cospi_p19_m13 = pair256_set_epi16(cospi_19_64, -cospi_13_64);
const __m256i cospi_p17_p15 = pair256_set_epi16(cospi_17_64, cospi_15_64);
const __m256i cospi_p15_m17 = pair256_set_epi16(cospi_15_64, -cospi_17_64);
const __m256i cospi_p21_p11 = pair256_set_epi16(cospi_21_64, cospi_11_64);
const __m256i cospi_p11_m21 = pair256_set_epi16(cospi_11_64, -cospi_21_64);
const __m256i cospi_p25_p07 = pair256_set_epi16(cospi_25_64, cospi_7_64);
const __m256i cospi_p07_m25 = pair256_set_epi16(cospi_7_64, -cospi_25_64);
const __m256i cospi_p29_p03 = pair256_set_epi16(cospi_29_64, cospi_3_64);
const __m256i cospi_p03_m29 = pair256_set_epi16(cospi_3_64, -cospi_29_64);
const __m256i cospi_p04_p28 = pair256_set_epi16(cospi_4_64, cospi_28_64);
const __m256i cospi_p28_m04 = pair256_set_epi16(cospi_28_64, -cospi_4_64);
const __m256i cospi_p20_p12 = pair256_set_epi16(cospi_20_64, cospi_12_64);
const __m256i cospi_p12_m20 = pair256_set_epi16(cospi_12_64, -cospi_20_64);
const __m256i cospi_m28_p04 = pair256_set_epi16(-cospi_28_64, cospi_4_64);
const __m256i cospi_m12_p20 = pair256_set_epi16(-cospi_12_64, cospi_20_64);
const __m256i cospi_p08_p24 = pair256_set_epi16(cospi_8_64, cospi_24_64);
const __m256i cospi_p24_m08 = pair256_set_epi16(cospi_24_64, -cospi_8_64);
const __m256i cospi_m24_p08 = pair256_set_epi16(-cospi_24_64, cospi_8_64);
const __m256i cospi_m16_m16 = _mm256_set1_epi16((int16_t)-cospi_16_64);
const __m256i cospi_p16_p16 = _mm256_set1_epi16((int16_t)cospi_16_64);
const __m256i cospi_p16_m16 = pair256_set_epi16(cospi_16_64, -cospi_16_64);
const __m256i cospi_m16_p16 = pair256_set_epi16(-cospi_16_64, cospi_16_64);
const __m256i zero = _mm256_setzero_si256();
__m256i x[16], s[16];
__m256i u[4], v[4];
// stage 1
butterfly_32b(&in[15], &in[0], &cospi_p01_p31, &cospi_p31_m01, u);
butterfly_32b(&in[7], &in[8], &cospi_p17_p15, &cospi_p15_m17, v);
add_rnd(u, v, &x[0]);
sub_rnd(u, v, &x[8]);
butterfly_32b(&in[13], &in[2], &cospi_p05_p27, &cospi_p27_m05, u);
butterfly_32b(&in[5], &in[10], &cospi_p21_p11, &cospi_p11_m21, v);
add_rnd(u, v, &x[2]);
sub_rnd(u, v, &x[10]);
butterfly_32b(&in[11], &in[4], &cospi_p09_p23, &cospi_p23_m09, u);
butterfly_32b(&in[3], &in[12], &cospi_p25_p07, &cospi_p07_m25, v);
add_rnd(u, v, &x[4]);
sub_rnd(u, v, &x[12]);
butterfly_32b(&in[9], &in[6], &cospi_p13_p19, &cospi_p19_m13, u);
butterfly_32b(&in[1], &in[14], &cospi_p29_p03, &cospi_p03_m29, v);
add_rnd(u, v, &x[6]);
sub_rnd(u, v, &x[14]);
// stage 2
s[0] = _mm256_add_epi16(x[0], x[4]);
s[1] = _mm256_add_epi16(x[1], x[5]);
s[2] = _mm256_add_epi16(x[2], x[6]);
s[3] = _mm256_add_epi16(x[3], x[7]);
s[4] = _mm256_sub_epi16(x[0], x[4]);
s[5] = _mm256_sub_epi16(x[1], x[5]);
s[6] = _mm256_sub_epi16(x[2], x[6]);
s[7] = _mm256_sub_epi16(x[3], x[7]);
butterfly_32b(&x[8], &x[9], &cospi_p04_p28, &cospi_p28_m04, u);
butterfly_32b(&x[12], &x[13], &cospi_m28_p04, &cospi_p04_p28, v);
add_rnd(u, v, &s[8]);
sub_rnd(u, v, &s[12]);
butterfly_32b(&x[10], &x[11], &cospi_p20_p12, &cospi_p12_m20, u);
butterfly_32b(&x[14], &x[15], &cospi_m12_p20, &cospi_p20_p12, v);
add_rnd(u, v, &s[10]);
sub_rnd(u, v, &s[14]);
// stage 3
x[0] = _mm256_add_epi16(s[0], s[2]);
x[1] = _mm256_add_epi16(s[1], s[3]);
x[2] = _mm256_sub_epi16(s[0], s[2]);
x[3] = _mm256_sub_epi16(s[1], s[3]);
x[8] = _mm256_add_epi16(s[8], s[10]);
x[9] = _mm256_add_epi16(s[9], s[11]);
x[10] = _mm256_sub_epi16(s[8], s[10]);
x[11] = _mm256_sub_epi16(s[9], s[11]);
butterfly_32b(&s[4], &s[5], &cospi_p08_p24, &cospi_p24_m08, u);
butterfly_32b(&s[6], &s[7], &cospi_m24_p08, &cospi_p08_p24, v);
add_rnd(u, v, &x[4]);
sub_rnd(u, v, &x[6]);
butterfly_32b(&s[12], &s[13], &cospi_p08_p24, &cospi_p24_m08, u);
butterfly_32b(&s[14], &s[15], &cospi_m24_p08, &cospi_p08_p24, v);
add_rnd(u, v, &x[12]);
sub_rnd(u, v, &x[14]);
// stage 4
butterfly_32b(&x[2], &x[3], &cospi_m16_m16, &cospi_p16_m16, u);
butterfly_32b(&x[6], &x[7], &cospi_p16_p16, &cospi_m16_p16, v);
butterfly_rnd(u, &x[2]);
butterfly_rnd(v, &x[6]);
butterfly_32b(&x[10], &x[11], &cospi_p16_p16, &cospi_m16_p16, u);
butterfly_32b(&x[14], &x[15], &cospi_m16_m16, &cospi_p16_m16, v);
butterfly_rnd(u, &x[10]);
butterfly_rnd(v, &x[14]);
in[0] = x[0];
in[1] = _mm256_sub_epi16(zero, x[8]);
in[2] = x[12];
in[3] = _mm256_sub_epi16(zero, x[4]);
in[4] = x[6];
in[5] = x[14];
in[6] = x[10];
in[7] = x[2];
in[8] = x[3];
in[9] = x[11];
in[10] = x[15];
in[11] = x[7];
in[12] = x[5];
in[13] = _mm256_sub_epi16(zero, x[13]);
in[14] = x[9];
in[15] = _mm256_sub_epi16(zero, x[1]);
}
static void iadst16(__m256i *in) {
mm256_transpose_16x16(in);
iadst16_avx2(in);
}
#if CONFIG_EXT_TX
static void flip_row(__m256i *in, int rows) {
int i;
for (i = 0; i < rows; ++i) {
mm256_reverse_epi16(&in[i]);
}
}
static void flip_col(uint8_t **dest, int *stride, int rows) {
*dest = *dest + (rows - 1) * (*stride);
*stride = -*stride;
}
static void iidtx16(__m256i *in) {
mm256_transpose_16x16(in);
txfm_scaling16_avx2(Sqrt2, in);
}
#endif
void av1_iht16x16_256_add_avx2(const tran_low_t *input, uint8_t *dest,
int stride, int tx_type) {
__m256i in[16];
load_buffer_16x16(input, in);
switch (tx_type) {
case DCT_DCT:
idct16(in);
idct16(in);
break;
case ADST_DCT:
idct16(in);
iadst16(in);
break;
case DCT_ADST:
iadst16(in);
idct16(in);
break;
case ADST_ADST:
iadst16(in);
iadst16(in);
break;
#if CONFIG_EXT_TX
case FLIPADST_DCT:
idct16(in);
iadst16(in);
flip_col(&dest, &stride, 16);
break;
case DCT_FLIPADST:
iadst16(in);
idct16(in);
flip_row(in, 16);
break;
case FLIPADST_FLIPADST:
iadst16(in);
iadst16(in);
flip_row(in, 16);
flip_col(&dest, &stride, 16);
break;
case ADST_FLIPADST:
iadst16(in);
iadst16(in);
flip_row(in, 16);
break;
case FLIPADST_ADST:
iadst16(in);
iadst16(in);
flip_col(&dest, &stride, 16);
break;
case IDTX:
iidtx16(in);
iidtx16(in);
break;
case V_DCT:
iidtx16(in);
idct16(in);
break;
case H_DCT:
idct16(in);
iidtx16(in);
break;
case V_ADST:
iidtx16(in);
iadst16(in);
break;
case H_ADST:
iadst16(in);
iidtx16(in);
break;
case V_FLIPADST:
iidtx16(in);
iadst16(in);
flip_col(&dest, &stride, 16);
break;
case H_FLIPADST:
iadst16(in);
iidtx16(in);
flip_row(in, 16);
break;
#endif // CONFIG_EXT_TX
default: assert(0); break;
}
write_buffer_16x16(in, stride, dest);
}

1402
third_party/aom/av1/common/x86/idct_intrin_sse2.c vendored Normal file
View file

File diff suppressed because it is too large Load diff

252
third_party/aom/av1/common/x86/pvq_sse4.c vendored Normal file
View file

@ -0,0 +1,252 @@
/*
* Copyright (c) 2016, Alliance for Open Media. All rights reserved
*
* This source code is subject to the terms of the BSD 2 Clause License and
* the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
* was not distributed with this source code in the LICENSE file, you can
* obtain it at www.aomedia.org/license/software. If the Alliance for Open
* Media Patent License 1.0 was not distributed with this source code in the
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
*/
#include <smmintrin.h>
#include <emmintrin.h>
#include <tmmintrin.h>
#include <float.h>
#include "./av1_rtcd.h"
#include "av1/common/x86/pvq_sse4.h"
#include "../odintrin.h"
#include "av1/common/pvq.h"
#define EPSILON 1e-15f
static __m128 horizontal_sum_ps(__m128 x) {
x = _mm_add_ps(x, _mm_shuffle_ps(x, x, _MM_SHUFFLE(1, 0, 3, 2)));
x = _mm_add_ps(x, _mm_shuffle_ps(x, x, _MM_SHUFFLE(2, 3, 0, 1)));
return x;
}
static __m128i horizontal_sum_epi32(__m128i x) {
x = _mm_add_epi32(x, _mm_shuffle_epi32(x, _MM_SHUFFLE(1, 0, 3, 2)));
x = _mm_add_epi32(x, _mm_shuffle_epi32(x, _MM_SHUFFLE(2, 3, 0, 1)));
return x;
}
static INLINE float rsqrtf(float x) {
float y;
_mm_store_ss(&y, _mm_rsqrt_ss(_mm_load_ss(&x)));
return y;
}
/** Find the codepoint on the given PSphere closest to the desired
* vector. This is a float-precision PVQ search just to make sure
* our tests aren't limited by numerical accuracy. It's close to the
* pvq_search_rdo_double_c implementation, but is not bit accurate and
* it performs slightly worse on PSNR. One reason is that this code runs
* more RDO iterations than the C code. It also uses single precision
* floating point math, whereas the C version uses double precision.
*
* @param [in] xcoeff input vector to quantize (x in the math doc)
* @param [in] n number of dimensions
* @param [in] k number of pulses
* @param [out] ypulse optimal codevector found (y in the math doc)
* @param [in] g2 multiplier for the distortion (typically squared
* gain units)
* @param [in] pvq_norm_lambda enc->pvq_norm_lambda for quantized RDO
* @param [in] prev_k number of pulses already in ypulse that we should
* reuse for the search (or 0 for a new search)
* @return cosine distance between x and y (between 0 and 1)
*/
double pvq_search_rdo_double_sse4_1(const od_val16 *xcoeff, int n, int k,
int *ypulse, double g2,
double pvq_norm_lambda, int prev_k) {
int i, j;
int reuse_pulses = prev_k > 0 && prev_k <= k;
/* TODO - This blows our 8kB stack space budget and should be fixed when
converting PVQ to fixed point. */
float xx = 0, xy = 0, yy = 0;
float x[MAXN + 3];
float y[MAXN + 3];
float sign_y[MAXN + 3];
for (i = 0; i < n; i++) {
float tmp = (float)xcoeff[i];
xx += tmp * tmp;
x[i] = xcoeff[i];
}
x[n] = x[n + 1] = x[n + 2] = 0;
ypulse[n] = ypulse[n + 1] = ypulse[n + 2] = 0;
__m128 sums = _mm_setzero_ps();
for (i = 0; i < n; i += 4) {
__m128 x4 = _mm_loadu_ps(&x[i]);
__m128 s4 = _mm_cmplt_ps(x4, _mm_setzero_ps());
/* Save the sign, we'll put it back later. */
_mm_storeu_ps(&sign_y[i], s4);
/* Get rid of the sign. */
x4 = _mm_andnot_ps(_mm_set_ps1(-0.f), x4);
sums = _mm_add_ps(sums, x4);
if (!reuse_pulses) {
/* Clear y and ypulse in case we don't do the projection. */
_mm_storeu_ps(&y[i], _mm_setzero_ps());
_mm_storeu_si128((__m128i *)&ypulse[i], _mm_setzero_si128());
}
_mm_storeu_ps(&x[i], x4);
}
sums = horizontal_sum_ps(sums);
int pulses_left = k;
{
__m128i pulses_sum;
__m128 yy4, xy4;
xy4 = yy4 = _mm_setzero_ps();
pulses_sum = _mm_setzero_si128();
if (reuse_pulses) {
/* We reuse pulses from a previous search so we don't have to search them
again. */
for (j = 0; j < n; j += 4) {
__m128 x4, y4;
__m128i iy4;
iy4 = _mm_abs_epi32(_mm_loadu_si128((__m128i *)&ypulse[j]));
pulses_sum = _mm_add_epi32(pulses_sum, iy4);
_mm_storeu_si128((__m128i *)&ypulse[j], iy4);
y4 = _mm_cvtepi32_ps(iy4);
x4 = _mm_loadu_ps(&x[j]);
xy4 = _mm_add_ps(xy4, _mm_mul_ps(x4, y4));
yy4 = _mm_add_ps(yy4, _mm_mul_ps(y4, y4));
/* Double the y[] vector so we don't have to do it in the search loop.
*/
_mm_storeu_ps(&y[j], _mm_add_ps(y4, y4));
}
pulses_left -= _mm_cvtsi128_si32(horizontal_sum_epi32(pulses_sum));
xy4 = horizontal_sum_ps(xy4);
xy = _mm_cvtss_f32(xy4);
yy4 = horizontal_sum_ps(yy4);
yy = _mm_cvtss_f32(yy4);
} else if (k > (n >> 1)) {
/* Do a pre-search by projecting on the pyramid. */
__m128 rcp4;
float sum = _mm_cvtss_f32(sums);
/* If x is too small, just replace it with a pulse at 0. This prevents
infinities and NaNs from causing too many pulses to be allocated. Here,
64 is an
approximation of infinity. */
if (sum <= EPSILON) {
x[0] = 1.f;
for (i = 1; i < n; i++) {
x[i] = 0;
}
sums = _mm_set_ps1(1.f);
}
/* Using k + e with e < 1 guarantees we cannot get more than k pulses. */
rcp4 = _mm_mul_ps(_mm_set_ps1((float)k + .8f), _mm_rcp_ps(sums));
xy4 = yy4 = _mm_setzero_ps();
pulses_sum = _mm_setzero_si128();
for (j = 0; j < n; j += 4) {
__m128 rx4, x4, y4;
__m128i iy4;
x4 = _mm_loadu_ps(&x[j]);
rx4 = _mm_mul_ps(x4, rcp4);
iy4 = _mm_cvttps_epi32(rx4);
pulses_sum = _mm_add_epi32(pulses_sum, iy4);
_mm_storeu_si128((__m128i *)&ypulse[j], iy4);
y4 = _mm_cvtepi32_ps(iy4);
xy4 = _mm_add_ps(xy4, _mm_mul_ps(x4, y4));
yy4 = _mm_add_ps(yy4, _mm_mul_ps(y4, y4));
/* Double the y[] vector so we don't have to do it in the search loop.
*/
_mm_storeu_ps(&y[j], _mm_add_ps(y4, y4));
}
pulses_left -= _mm_cvtsi128_si32(horizontal_sum_epi32(pulses_sum));
xy = _mm_cvtss_f32(horizontal_sum_ps(xy4));
yy = _mm_cvtss_f32(horizontal_sum_ps(yy4));
}
x[n] = x[n + 1] = x[n + 2] = -100;
y[n] = y[n + 1] = y[n + 2] = 100;
}
/* This should never happen. */
OD_ASSERT(pulses_left <= n + 3);
float lambda_delta_rate[MAXN + 3];
if (pulses_left) {
/* Hoist lambda to avoid the multiply in the loop. */
float lambda =
0.5f * sqrtf(xx) * (float)pvq_norm_lambda / (FLT_MIN + (float)g2);
float delta_rate = 3.f / n;
__m128 count = _mm_set_ps(3, 2, 1, 0);
for (i = 0; i < n; i += 4) {
_mm_storeu_ps(&lambda_delta_rate[i],
_mm_mul_ps(count, _mm_set_ps1(lambda * delta_rate)));
count = _mm_add_ps(count, _mm_set_ps(4, 4, 4, 4));
}
}
lambda_delta_rate[n] = lambda_delta_rate[n + 1] = lambda_delta_rate[n + 2] =
1e30f;
for (i = 0; i < pulses_left; i++) {
int best_id = 0;
__m128 xy4, yy4;
__m128 max, max2;
__m128i count;
__m128i pos;
/* The squared magnitude term gets added anyway, so we might as well
add it outside the loop. */
yy = yy + 1;
xy4 = _mm_load1_ps(&xy);
yy4 = _mm_load1_ps(&yy);
max = _mm_setzero_ps();
pos = _mm_setzero_si128();
count = _mm_set_epi32(3, 2, 1, 0);
for (j = 0; j < n; j += 4) {
__m128 x4, y4, r4;
x4 = _mm_loadu_ps(&x[j]);
y4 = _mm_loadu_ps(&y[j]);
x4 = _mm_add_ps(x4, xy4);
y4 = _mm_add_ps(y4, yy4);
y4 = _mm_rsqrt_ps(y4);
r4 = _mm_mul_ps(x4, y4);
/* Subtract lambda. */
r4 = _mm_sub_ps(r4, _mm_loadu_ps(&lambda_delta_rate[j]));
/* Update the index of the max. */
pos = _mm_max_epi16(
pos, _mm_and_si128(count, _mm_castps_si128(_mm_cmpgt_ps(r4, max))));
/* Update the max. */
max = _mm_max_ps(max, r4);
/* Update the indices (+4) */
count = _mm_add_epi32(count, _mm_set_epi32(4, 4, 4, 4));
}
/* Horizontal max. */
max2 = _mm_max_ps(max, _mm_shuffle_ps(max, max, _MM_SHUFFLE(1, 0, 3, 2)));
max2 =
_mm_max_ps(max2, _mm_shuffle_ps(max2, max2, _MM_SHUFFLE(2, 3, 0, 1)));
/* Now that max2 contains the max at all positions, look at which value(s)
of the
partial max is equal to the global max. */
pos = _mm_and_si128(pos, _mm_castps_si128(_mm_cmpeq_ps(max, max2)));
pos = _mm_max_epi16(pos, _mm_unpackhi_epi64(pos, pos));
pos = _mm_max_epi16(pos, _mm_shufflelo_epi16(pos, _MM_SHUFFLE(1, 0, 3, 2)));
best_id = _mm_cvtsi128_si32(pos);
OD_ASSERT(best_id < n);
/* Updating the sums of the new pulse(s) */
xy = xy + x[best_id];
/* We're multiplying y[j] by two so we don't have to do it here. */
yy = yy + y[best_id];
/* Only now that we've made the final choice, update y/ypulse. */
/* Multiplying y[j] by 2 so we don't have to do it everywhere else. */
y[best_id] += 2;
ypulse[best_id]++;
}
/* Put the original sign back. */
for (i = 0; i < n; i += 4) {
__m128i y4;
__m128i s4;
y4 = _mm_loadu_si128((__m128i *)&ypulse[i]);
s4 = _mm_castps_si128(_mm_loadu_ps(&sign_y[i]));
y4 = _mm_xor_si128(_mm_add_epi32(y4, s4), s4);
_mm_storeu_si128((__m128i *)&ypulse[i], y4);
}
return xy * rsqrtf(xx * yy + FLT_MIN);
}

13
third_party/aom/av1/common/x86/pvq_sse4.h vendored Normal file
View file

@ -0,0 +1,13 @@
/*
* Copyright (c) 2016, Alliance for Open Media. All rights reserved
*
* This source code is subject to the terms of the BSD 2 Clause License and
* the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
* was not distributed with this source code in the LICENSE file, you can
* obtain it at www.aomedia.org/license/software. If the Alliance for Open
* Media Patent License 1.0 was not distributed with this source code in the
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
*/
#ifndef AOM_COMMON_PVQ_X86_SSE4_H_
#define AOM_COMMON_PVQ_X86_SSE4_H_
#endif // AOM_COMMON_PVQ_X86_SSE4_H_

1805
third_party/aom/av1/common/x86/selfguided_sse4.c vendored Normal file
View file

File diff suppressed because it is too large Load diff

297
third_party/aom/av1/common/x86/warp_plane_sse2.c vendored Normal file
View file

@ -0,0 +1,297 @@
/*
* Copyright (c) 2016, Alliance for Open Media. All rights reserved
*
* This source code is subject to the terms of the BSD 2 Clause License and
* the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
* was not distributed with this source code in the LICENSE file, you can
* obtain it at www.aomedia.org/license/software. If the Alliance for Open
* Media Patent License 1.0 was not distributed with this source code in the
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
*/
#include <emmintrin.h>
#include "./av1_rtcd.h"
#include "av1/common/warped_motion.h"
static const __m128i *const filter = (const __m128i *const)warped_filter;
/* SSE2 version of the rotzoom/affine warp filter */
void av1_warp_affine_sse2(int32_t *mat, uint8_t *ref, int width, int height,
int stride, uint8_t *pred, int p_col, int p_row,
int p_width, int p_height, int p_stride,
int subsampling_x, int subsampling_y, int ref_frm,
int16_t alpha, int16_t beta, int16_t gamma,
int16_t delta) {
__m128i tmp[15];
int i, j, k;
/* Note: For this code to work, the left/right frame borders need to be
extended by at least 13 pixels each. By the time we get here, other
code will have set up this border, but we allow an explicit check
for debugging purposes.
*/
/*for (i = 0; i < height; ++i) {
for (j = 0; j < 13; ++j) {
assert(ref[i * stride - 13 + j] == ref[i * stride]);
assert(ref[i * stride + width + j] == ref[i * stride + (width - 1)]);
}
}*/
for (i = 0; i < p_height; i += 8) {
for (j = 0; j < p_width; j += 8) {
// (x, y) coordinates of the center of this block in the destination
// image
int32_t dst_x = p_col + j + 4;
int32_t dst_y = p_row + i + 4;
int32_t x4, y4, ix4, sx4, iy4, sy4;
if (subsampling_x)
x4 = ROUND_POWER_OF_TWO_SIGNED(
mat[2] * 2 * dst_x + mat[3] * 2 * dst_y + mat[0] +
(mat[2] + mat[3] - (1 << WARPEDMODEL_PREC_BITS)) / 2,
1);
else
x4 = mat[2] * dst_x + mat[3] * dst_y + mat[0];
if (subsampling_y)
y4 = ROUND_POWER_OF_TWO_SIGNED(
mat[4] * 2 * dst_x + mat[5] * 2 * dst_y + mat[1] +
(mat[4] + mat[5] - (1 << WARPEDMODEL_PREC_BITS)) / 2,
1);
else
y4 = mat[4] * dst_x + mat[5] * dst_y + mat[1];
ix4 = x4 >> WARPEDMODEL_PREC_BITS;
sx4 = x4 & ((1 << WARPEDMODEL_PREC_BITS) - 1);
iy4 = y4 >> WARPEDMODEL_PREC_BITS;
sy4 = y4 & ((1 << WARPEDMODEL_PREC_BITS) - 1);
// Horizontal filter
for (k = -7; k < AOMMIN(8, p_height - i); ++k) {
int iy = iy4 + k;
if (iy < 0)
iy = 0;
else if (iy > height - 1)
iy = height - 1;
// If the block is aligned such that, after clamping, every sample
// would be taken from the leftmost/rightmost column, then we can
// skip the expensive horizontal filter.
if (ix4 <= -7) {
tmp[k + 7] = _mm_set1_epi16(
ref[iy * stride] *
(1 << (WARPEDPIXEL_FILTER_BITS - HORSHEAR_REDUCE_PREC_BITS)));
} else if (ix4 >= width + 6) {
tmp[k + 7] = _mm_set1_epi16(
ref[iy * stride + (width - 1)] *
(1 << (WARPEDPIXEL_FILTER_BITS - HORSHEAR_REDUCE_PREC_BITS)));
} else {
int sx = sx4 + alpha * (-4) + beta * k +
// Include rounding and offset here
(1 << (WARPEDDIFF_PREC_BITS - 1)) +
(WARPEDPIXEL_PREC_SHIFTS << WARPEDDIFF_PREC_BITS);
// Load source pixels
__m128i zero = _mm_setzero_si128();
__m128i src =
_mm_loadu_si128((__m128i *)(ref + iy * stride + ix4 - 7));
// Filter even-index pixels
__m128i tmp_0 = _mm_loadu_si128(
(__m128i *)(filter + ((sx + 0 * alpha) >> WARPEDDIFF_PREC_BITS)));
__m128i tmp_2 = _mm_loadu_si128(
(__m128i *)(filter + ((sx + 2 * alpha) >> WARPEDDIFF_PREC_BITS)));
__m128i tmp_4 = _mm_loadu_si128(
(__m128i *)(filter + ((sx + 4 * alpha) >> WARPEDDIFF_PREC_BITS)));
__m128i tmp_6 = _mm_loadu_si128(
(__m128i *)(filter + ((sx + 6 * alpha) >> WARPEDDIFF_PREC_BITS)));
// coeffs 0 1 0 1 2 3 2 3 for pixels 0, 2
__m128i tmp_8 = _mm_unpacklo_epi32(tmp_0, tmp_2);
// coeffs 0 1 0 1 2 3 2 3 for pixels 4, 6
__m128i tmp_10 = _mm_unpacklo_epi32(tmp_4, tmp_6);
// coeffs 4 5 4 5 6 7 6 7 for pixels 0, 2
__m128i tmp_12 = _mm_unpackhi_epi32(tmp_0, tmp_2);
// coeffs 4 5 4 5 6 7 6 7 for pixels 4, 6
__m128i tmp_14 = _mm_unpackhi_epi32(tmp_4, tmp_6);
// coeffs 0 1 0 1 0 1 0 1 for pixels 0, 2, 4, 6
__m128i coeff_0 = _mm_unpacklo_epi64(tmp_8, tmp_10);
// coeffs 2 3 2 3 2 3 2 3 for pixels 0, 2, 4, 6
__m128i coeff_2 = _mm_unpackhi_epi64(tmp_8, tmp_10);
// coeffs 4 5 4 5 4 5 4 5 for pixels 0, 2, 4, 6
__m128i coeff_4 = _mm_unpacklo_epi64(tmp_12, tmp_14);
// coeffs 6 7 6 7 6 7 6 7 for pixels 0, 2, 4, 6
__m128i coeff_6 = _mm_unpackhi_epi64(tmp_12, tmp_14);
__m128i round_const =
_mm_set1_epi32((1 << HORSHEAR_REDUCE_PREC_BITS) >> 1);
// Calculate filtered results
__m128i src_0 = _mm_unpacklo_epi8(src, zero);
__m128i res_0 = _mm_madd_epi16(src_0, coeff_0);
__m128i src_2 = _mm_unpacklo_epi8(_mm_srli_si128(src, 2), zero);
__m128i res_2 = _mm_madd_epi16(src_2, coeff_2);
__m128i src_4 = _mm_unpacklo_epi8(_mm_srli_si128(src, 4), zero);
__m128i res_4 = _mm_madd_epi16(src_4, coeff_4);
__m128i src_6 = _mm_unpacklo_epi8(_mm_srli_si128(src, 6), zero);
__m128i res_6 = _mm_madd_epi16(src_6, coeff_6);
__m128i res_even = _mm_add_epi32(_mm_add_epi32(res_0, res_4),
_mm_add_epi32(res_2, res_6));
res_even = _mm_srai_epi32(_mm_add_epi32(res_even, round_const),
HORSHEAR_REDUCE_PREC_BITS);
// Filter odd-index pixels
__m128i tmp_1 = _mm_loadu_si128(
(__m128i *)(filter + ((sx + 1 * alpha) >> WARPEDDIFF_PREC_BITS)));
__m128i tmp_3 = _mm_loadu_si128(
(__m128i *)(filter + ((sx + 3 * alpha) >> WARPEDDIFF_PREC_BITS)));
__m128i tmp_5 = _mm_loadu_si128(
(__m128i *)(filter + ((sx + 5 * alpha) >> WARPEDDIFF_PREC_BITS)));
__m128i tmp_7 = _mm_loadu_si128(
(__m128i *)(filter + ((sx + 7 * alpha) >> WARPEDDIFF_PREC_BITS)));
__m128i tmp_9 = _mm_unpacklo_epi32(tmp_1, tmp_3);
__m128i tmp_11 = _mm_unpacklo_epi32(tmp_5, tmp_7);
__m128i tmp_13 = _mm_unpackhi_epi32(tmp_1, tmp_3);
__m128i tmp_15 = _mm_unpackhi_epi32(tmp_5, tmp_7);
__m128i coeff_1 = _mm_unpacklo_epi64(tmp_9, tmp_11);
__m128i coeff_3 = _mm_unpackhi_epi64(tmp_9, tmp_11);
__m128i coeff_5 = _mm_unpacklo_epi64(tmp_13, tmp_15);
__m128i coeff_7 = _mm_unpackhi_epi64(tmp_13, tmp_15);
__m128i src_1 = _mm_unpacklo_epi8(_mm_srli_si128(src, 1), zero);
__m128i res_1 = _mm_madd_epi16(src_1, coeff_1);
__m128i src_3 = _mm_unpacklo_epi8(_mm_srli_si128(src, 3), zero);
__m128i res_3 = _mm_madd_epi16(src_3, coeff_3);
__m128i src_5 = _mm_unpacklo_epi8(_mm_srli_si128(src, 5), zero);
__m128i res_5 = _mm_madd_epi16(src_5, coeff_5);
__m128i src_7 = _mm_unpacklo_epi8(_mm_srli_si128(src, 7), zero);
__m128i res_7 = _mm_madd_epi16(src_7, coeff_7);
__m128i res_odd = _mm_add_epi32(_mm_add_epi32(res_1, res_5),
_mm_add_epi32(res_3, res_7));
res_odd = _mm_srai_epi32(_mm_add_epi32(res_odd, round_const),
HORSHEAR_REDUCE_PREC_BITS);
// Combine results into one register.
// We store the columns in the order 0, 2, 4, 6, 1, 3, 5, 7
// as this order helps with the vertical filter.
tmp[k + 7] = _mm_packs_epi32(res_even, res_odd);
}
}
// Vertical filter
for (k = -4; k < AOMMIN(4, p_height - i - 4); ++k) {
int sy = sy4 + gamma * (-4) + delta * k +
(1 << (WARPEDDIFF_PREC_BITS - 1)) +
(WARPEDPIXEL_PREC_SHIFTS << WARPEDDIFF_PREC_BITS);
// Load from tmp and rearrange pairs of consecutive rows into the
// column order 0 0 2 2 4 4 6 6; 1 1 3 3 5 5 7 7
__m128i *src = tmp + (k + 4);
__m128i src_0 = _mm_unpacklo_epi16(src[0], src[1]);
__m128i src_2 = _mm_unpacklo_epi16(src[2], src[3]);
__m128i src_4 = _mm_unpacklo_epi16(src[4], src[5]);
__m128i src_6 = _mm_unpacklo_epi16(src[6], src[7]);
// Filter even-index pixels
__m128i tmp_0 = _mm_loadu_si128(
(__m128i *)(filter + ((sy + 0 * gamma) >> WARPEDDIFF_PREC_BITS)));
__m128i tmp_2 = _mm_loadu_si128(
(__m128i *)(filter + ((sy + 2 * gamma) >> WARPEDDIFF_PREC_BITS)));
__m128i tmp_4 = _mm_loadu_si128(
(__m128i *)(filter + ((sy + 4 * gamma) >> WARPEDDIFF_PREC_BITS)));
__m128i tmp_6 = _mm_loadu_si128(
(__m128i *)(filter + ((sy + 6 * gamma) >> WARPEDDIFF_PREC_BITS)));
__m128i tmp_8 = _mm_unpacklo_epi32(tmp_0, tmp_2);
__m128i tmp_10 = _mm_unpacklo_epi32(tmp_4, tmp_6);
__m128i tmp_12 = _mm_unpackhi_epi32(tmp_0, tmp_2);
__m128i tmp_14 = _mm_unpackhi_epi32(tmp_4, tmp_6);
__m128i coeff_0 = _mm_unpacklo_epi64(tmp_8, tmp_10);
__m128i coeff_2 = _mm_unpackhi_epi64(tmp_8, tmp_10);
__m128i coeff_4 = _mm_unpacklo_epi64(tmp_12, tmp_14);
__m128i coeff_6 = _mm_unpackhi_epi64(tmp_12, tmp_14);
__m128i res_0 = _mm_madd_epi16(src_0, coeff_0);
__m128i res_2 = _mm_madd_epi16(src_2, coeff_2);
__m128i res_4 = _mm_madd_epi16(src_4, coeff_4);
__m128i res_6 = _mm_madd_epi16(src_6, coeff_6);
__m128i res_even = _mm_add_epi32(_mm_add_epi32(res_0, res_2),
_mm_add_epi32(res_4, res_6));
// Filter odd-index pixels
__m128i src_1 = _mm_unpackhi_epi16(src[0], src[1]);
__m128i src_3 = _mm_unpackhi_epi16(src[2], src[3]);
__m128i src_5 = _mm_unpackhi_epi16(src[4], src[5]);
__m128i src_7 = _mm_unpackhi_epi16(src[6], src[7]);
__m128i tmp_1 = _mm_loadu_si128(
(__m128i *)(filter + ((sy + 1 * gamma) >> WARPEDDIFF_PREC_BITS)));
__m128i tmp_3 = _mm_loadu_si128(
(__m128i *)(filter + ((sy + 3 * gamma) >> WARPEDDIFF_PREC_BITS)));
__m128i tmp_5 = _mm_loadu_si128(
(__m128i *)(filter + ((sy + 5 * gamma) >> WARPEDDIFF_PREC_BITS)));
__m128i tmp_7 = _mm_loadu_si128(
(__m128i *)(filter + ((sy + 7 * gamma) >> WARPEDDIFF_PREC_BITS)));
__m128i tmp_9 = _mm_unpacklo_epi32(tmp_1, tmp_3);
__m128i tmp_11 = _mm_unpacklo_epi32(tmp_5, tmp_7);
__m128i tmp_13 = _mm_unpackhi_epi32(tmp_1, tmp_3);
__m128i tmp_15 = _mm_unpackhi_epi32(tmp_5, tmp_7);
__m128i coeff_1 = _mm_unpacklo_epi64(tmp_9, tmp_11);
__m128i coeff_3 = _mm_unpackhi_epi64(tmp_9, tmp_11);
__m128i coeff_5 = _mm_unpacklo_epi64(tmp_13, tmp_15);
__m128i coeff_7 = _mm_unpackhi_epi64(tmp_13, tmp_15);
__m128i res_1 = _mm_madd_epi16(src_1, coeff_1);
__m128i res_3 = _mm_madd_epi16(src_3, coeff_3);
__m128i res_5 = _mm_madd_epi16(src_5, coeff_5);
__m128i res_7 = _mm_madd_epi16(src_7, coeff_7);
__m128i res_odd = _mm_add_epi32(_mm_add_epi32(res_1, res_3),
_mm_add_epi32(res_5, res_7));
// Rearrange pixels back into the order 0 ... 7
__m128i res_lo = _mm_unpacklo_epi32(res_even, res_odd);
__m128i res_hi = _mm_unpackhi_epi32(res_even, res_odd);
// Round and pack into 8 bits
__m128i round_const =
_mm_set1_epi32((1 << VERSHEAR_REDUCE_PREC_BITS) >> 1);
__m128i res_lo_round = _mm_srai_epi32(
_mm_add_epi32(res_lo, round_const), VERSHEAR_REDUCE_PREC_BITS);
__m128i res_hi_round = _mm_srai_epi32(
_mm_add_epi32(res_hi, round_const), VERSHEAR_REDUCE_PREC_BITS);
__m128i res_16bit = _mm_packs_epi32(res_lo_round, res_hi_round);
__m128i res_8bit = _mm_packus_epi16(res_16bit, res_16bit);
// Store, blending with 'pred' if needed
__m128i *p = (__m128i *)&pred[(i + k + 4) * p_stride + j];
// Note: If we're outputting a 4x4 block, we need to be very careful
// to only output 4 pixels at this point, to avoid encode/decode
// mismatches when encoding with multiple threads.
if (p_width == 4) {
if (ref_frm) {
const __m128i orig = _mm_cvtsi32_si128(*(uint32_t *)p);
res_8bit = _mm_avg_epu8(res_8bit, orig);
}
*(uint32_t *)p = _mm_cvtsi128_si32(res_8bit);
} else {
if (ref_frm) res_8bit = _mm_avg_epu8(res_8bit, _mm_loadl_epi64(p));
_mm_storel_epi64(p, res_8bit);
}
}
}
}
}