wuggy/Dactyloidae
wuggy/Dactyloidae
1
0
Fork 0
mirror of https://repo.dactyloidae.xyz/Dactyloidae/UXP.git synced 2026-09-26 10:27:32 +09:00
Code Issues Projects Releases Packages Wiki Activity

aom: update libaom to 0ec86ac7ae1e32a7e70410fa4972a655ec3670a4

Browse source
This commit is contained in:
Roy Tam 2019-02-22 21:52:01 +08:00
commit eb361970c5
438 changed files with 52661 additions and 21905 deletions
Download patch file Download diff file Expand all files Collapse all files

79
third_party/aom/CMakeLists.txt vendored
View file

@ -22,10 +22,6 @@ project(AOM C CXX)
set(AOM_ROOT "${CMAKE_CURRENT_SOURCE_DIR}")
set(AOM_CONFIG_DIR "${CMAKE_CURRENT_BINARY_DIR}")
set(INCLUDE_INSTALL_DIR "${CMAKE_INSTALL_PREFIX}/include"
CACHE PATH "Installation path of includes")
set(LIB_INSTALL_DIR "${CMAKE_INSTALL_PREFIX}/lib"
CACHE PATH "Installation path of libraries")
if("${AOM_ROOT}" STREQUAL "${AOM_CONFIG_DIR}")
message(FATAL_ERROR
@ -42,9 +38,10 @@ include("${AOM_ROOT}/aom_ports/aom_ports.cmake")
include("${AOM_ROOT}/aom_scale/aom_scale.cmake")
include("${AOM_ROOT}/aom_util/aom_util.cmake")
include("${AOM_ROOT}/av1/av1.cmake")
include("${AOM_ROOT}/test/test.cmake")
include("${AOM_ROOT}/build/cmake/aom_install.cmake")
include("${AOM_ROOT}/build/cmake/sanitizers.cmake")
include("${AOM_ROOT}/build/cmake/util.cmake")
include("${AOM_ROOT}/test/test.cmake")
list(APPEND AOM_RTCD_SOURCES
"${AOM_CONFIG_DIR}/config/aom_dsp_rtcd.h"
@ -158,8 +155,6 @@ list(APPEND AOM_ENCODER_STATS_SOURCES "${AOM_ROOT}/stats/aomstats.c"
"${AOM_ROOT}/stats/aomstats.h" "${AOM_ROOT}/stats/rate_hist.c"
"${AOM_ROOT}/stats/rate_hist.h")
list(APPEND AOM_PKG_CONFIG_SOURCES "${AOM_CONFIG_DIR}/aom.pc")
list(APPEND AOM_VERSION_SOURCES "${AOM_CONFIG_DIR}/config/aom_version.h")
list(APPEND AOM_WEBM_DECODER_SOURCES "${AOM_ROOT}/common/webmdec.cc"
@ -192,22 +187,6 @@ add_custom_target(aom_version_check
COMMENT "Updating version info if necessary." VERBATIM)
add_dependencies(aom_version aom_version_check)
if(NOT MSVC)
add_library(aom_pc ${AOM_PKG_CONFIG_SOURCES})
add_dummy_source_file_to_target(aom_pc c)
add_custom_command(OUTPUT "${AOM_CONFIG_DIR}/aom.pc"
COMMAND ${CMAKE_COMMAND} ARGS
-DAOM_CONFIG_DIR=${AOM_CONFIG_DIR}
-DAOM_ROOT=${AOM_ROOT}
-DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}
-DCMAKE_PROJECT_NAME=${CMAKE_PROJECT_NAME}
-DCONFIG_MULTITHREAD=${CONFIG_MULTITHREAD}
-DHAVE_PTHREAD_H=${HAVE_PTHREAD_H} -P
"${AOM_ROOT}/build/cmake/pkg_config.cmake"
COMMENT "Writing aom.pc" VERBATIM)
add_dependencies(aom_pc aom_version)
endif()
# TODO(tomfinegan): Move rtcd target setup where it belongs for each rtcd
# source.
add_rtcd_build_step("${AOM_ROOT}/aom_dsp/aom_dsp_rtcd_defs.pl"
@ -234,8 +213,7 @@ if(NOT MSVC AND NOT APPLE)
endif()
# List of object and static library targets.
set(AOM_LIB_TARGETS ${AOM_LIB_TARGETS} aom_rtcd aom_encoder_stats aom_mem
aom_scale aom)
set(AOM_LIB_TARGETS ${AOM_LIB_TARGETS} aom_rtcd aom_mem aom_scale aom)
# Setup dependencies.
setup_aom_dsp_targets()
@ -403,7 +381,6 @@ endif()
if(ENABLE_TOOLS)
if(CONFIG_AV1_DECODER)
require_cxx_flag_nomsvc("-std=c++11" NO)
add_executable(dump_obu "${AOM_GEN_SRC_DIR}/usage_exit.cc"
"${AOM_ROOT}/tools/dump_obu.cc"
"${AOM_ROOT}/tools/obu_parser.cc"
@ -414,10 +391,6 @@ if(ENABLE_TOOLS)
list(APPEND AOM_TOOL_TARGETS dump_obu)
list(APPEND AOM_APP_TARGETS dump_obu)
if(NOT MSVC)
target_compile_options(dump_obu PUBLIC -std=c++11)
endif()
# Maintain a separate variable listing only the examples to facilitate
# installation of example programs into an tools sub directory of
# $AOM_DIST_DIR/bin when building the dist target.
@ -489,8 +462,6 @@ if(ENABLE_EXAMPLES OR ENABLE_TESTS OR ENABLE_TOOLS)
endif()
if(CONFIG_WEBM_IO)
require_cxx_flag_nomsvc("-std=c++11" NO)
add_library(webm OBJECT ${AOM_LIBWEBM_SOURCES})
include_directories("${AOM_ROOT}/third_party/libwebm")
target_compile_definitions(webm PRIVATE __STDC_CONSTANT_MACROS)
@ -498,7 +469,6 @@ if(ENABLE_EXAMPLES OR ENABLE_TESTS OR ENABLE_TOOLS)
if(NOT MSVC)
target_compile_options(webm PRIVATE -Wno-shadow)
target_compile_options(webm PUBLIC -std=c++11)
endif()
# Add to existing targets.
@ -593,46 +563,6 @@ if(ENABLE_DOCS)
endif()
endif()
if(NOT (MSVC OR XCODE))
# Aomedia install rule.
list(APPEND AOM_INSTALL_INCS "${AOM_ROOT}/aom/aom.h"
"${AOM_ROOT}/aom/aom_codec.h"
"${AOM_ROOT}/aom/aom_frame_buffer.h"
"${AOM_ROOT}/aom/aom_image.h" "${AOM_ROOT}/aom/aom_integer.h"
"${AOM_ROOT}/aom/aom.h")
if(CONFIG_AV1_DECODER)
if(ENABLE_EXAMPLES)
list(APPEND AOM_INSTALL_BINS aomdec)
endif()
list(APPEND AOM_INSTALL_INCS "${AOM_ROOT}/aom/aom_decoder.h"
"${AOM_ROOT}/aom/aomdx.h")
endif()
if(CONFIG_AV1_ENCODER)
if(ENABLE_EXAMPLES)
list(APPEND AOM_INSTALL_BINS aomenc)
endif()
list(APPEND AOM_INSTALL_INCS "${AOM_ROOT}/aom/aomcx.h"
"${AOM_ROOT}/aom/aom_encoder.h")
endif()
set(AOM_INSTALL_LIBS aom)
install(FILES ${AOM_INSTALL_INCS} DESTINATION "${INCLUDE_INSTALL_DIR}/aom")
install(FILES "${AOM_CONFIG_DIR}/aom.pc" DESTINATION
"${LIB_INSTALL_DIR}/pkgconfig")
install(TARGETS ${AOM_INSTALL_LIBS} DESTINATION "${LIB_INSTALL_DIR}")
if(ENABLE_EXAMPLES)
install(TARGETS ${AOM_INSTALL_BINS} DESTINATION
"${CMAKE_INSTALL_PREFIX}/bin")
endif()
endif()
# Aomedia dist rule.
if(CONFIG_AV1_DECODER AND ENABLE_EXAMPLES)
list(APPEND AOM_DIST_APPS $<TARGET_FILE:aomdec>)
@ -756,3 +686,6 @@ foreach(aom_source_var ${aom_source_vars})
file(APPEND "${libaom_srcs_gni_file}" "]\n")
endif()
endforeach()
# Generate aom.pc and setup install rule.
setup_aom_install_targets()

14
third_party/aom/PATENTS vendored
View file

@ -57,10 +57,10 @@ Alliance for Open Media Patent License 1.0
2. Definitions.
2.1. Affiliate. <EFBFBD>Affiliate<EFBFBD> means an entity that directly or indirectly
2.1. Affiliate. "Affiliate" means an entity that directly or indirectly
Controls, is Controlled by, or is under common Control of that party.
2.2. Control. <EFBFBD>Control<EFBFBD> means direct or indirect control of more than 50% of
2.2. Control. "Control" means direct or indirect control of more than 50% of
the voting power to elect directors of that corporation, or for any other
entity, the power to direct management of such entity.
@ -70,7 +70,7 @@ Alliance for Open Media Patent License 1.0
2.4. Encoder. "Encoder" means any encoder that produces a bitstream that can
be decoded by a Decoder only to the extent it produces such a bitstream.
2.5. Final Deliverable. <EFBFBD>Final Deliverable<6C> means the final version of a
2.5. Final Deliverable. "Final Deliverable" means the final version of a
deliverable approved by the Alliance for Open Media as a Final
Deliverable.
@ -79,9 +79,9 @@ Alliance for Open Media Patent License 1.0
Implementation also includes components of an Implementation only to the
extent they are used as part of an Implementation.
2.7. License. <EFBFBD>License<EFBFBD> means this license.
2.7. License. "License" means this license.
2.8. Licensee. <EFBFBD>Licensee<EFBFBD> means any person or entity who exercises patent
2.8. Licensee. "Licensee" means any person or entity who exercises patent
rights granted under this License.
2.9. Licensor. "Licensor" means (i) any Licensee that makes, sells, offers
@ -98,11 +98,11 @@ Alliance for Open Media Patent License 1.0
as if the Specification was a W3C Recommendation; or (ii) are infringed
by the Reference Implementation.
2.11. Reference Implementation. <EFBFBD>Reference Implementation<6F> means an Encoder
2.11. Reference Implementation. "Reference Implementation" means an Encoder
and/or Decoder released by the Alliance for Open Media as a Final
Deliverable.
2.12. Specification. <EFBFBD>Specification<EFBFBD> means the specification designated by
2.12. Specification. "Specification" means the specification designated by
the Alliance for Open Media as a Final Deliverable for which this
License was issued.

9
third_party/aom/README.md vendored
View file

@ -210,15 +210,18 @@ compiler documentation to determine which, if any, are available.
### Microsoft Visual Studio builds
Building the AV1 codec library in Microsoft Visual Studio is supported. The
following example demonstrates generating projects and a solution for the
Microsoft IDE:
Building the AV1 codec library in Microsoft Visual Studio is supported. Visual
Studio 2015 (14.0) or later is required. The following example demonstrates
generating projects and a solution for the Microsoft IDE:
~~~
# This does not require a bash shell; command.exe is fine.
$ cmake path/to/aom -G "Visual Studio 15 2017"
~~~
NOTE: The build system targets Windows 7 or later by compiling files with
`-D_WIN32_WINNT=0x0601`.
### Xcode builds
Building the AV1 codec library in Xcode is supported. The following example

16
third_party/aom/aom/aom_decoder.h vendored
View file

@ -55,8 +55,6 @@ extern "C" {
#define AOM_CODEC_CAP_PUT_SLICE 0x10000 /**< Will issue put_slice callbacks */
#define AOM_CODEC_CAP_PUT_FRAME 0x20000 /**< Will issue put_frame callbacks */
#define AOM_CODEC_CAP_POSTPROC 0x40000 /**< Can postprocess decoded frame */
/*!\brief Can receive encoded frames one fragment at a time */
#define AOM_CODEC_CAP_INPUT_FRAGMENTS 0x100000
/*! \brief Initialization-time Feature Enabling
*
@ -65,15 +63,10 @@ extern "C" {
*
* The available flags are specified by AOM_CODEC_USE_* defines.
*/
/*!\brief Can support frame-based multi-threading */
#define AOM_CODEC_CAP_FRAME_THREADING 0x200000
/*!brief Can support external frame buffers */
#define AOM_CODEC_CAP_EXTERNAL_FRAME_BUFFER 0x400000
#define AOM_CODEC_CAP_EXTERNAL_FRAME_BUFFER 0x200000
#define AOM_CODEC_USE_POSTPROC 0x10000 /**< Postprocess decoded frame */
/*!\brief The input frame should be passed to the decoder one fragment at a
* time */
#define AOM_CODEC_USE_INPUT_FRAGMENTS 0x40000
/*!\brief Stream properties
*
@ -191,13 +184,6 @@ aom_codec_err_t aom_codec_get_stream_info(aom_codec_ctx_t *ctx,
* generated, as appropriate. Encoded data \ref MUST be passed in DTS (decode
* time stamp) order. Frames produced will always be in PTS (presentation
* time stamp) order.
* If the decoder is configured with AOM_CODEC_USE_INPUT_FRAGMENTS enabled,
* data and data_sz can contain a fragment of the encoded frame. Fragment
* \#n must contain at least partition \#n, but can also contain subsequent
* partitions (\#n+1 - \#n+i), and if so, fragments \#n+1, .., \#n+i must
* be empty. When no more data is available, this function should be called
* with NULL as data and 0 as data_sz. The memory passed to this function
* must be available until the frame has been decoded.
*
* \param[in] ctx Pointer to this instance's context
* \param[in] data Pointer to this block of new coded data. If

12
third_party/aom/aom/aom_encoder.h vendored
View file

@ -97,10 +97,14 @@ typedef uint32_t aom_codec_frame_flags_t;
/*!\brief frame can be dropped without affecting the stream (no future frame
* depends on this one) */
#define AOM_FRAME_IS_DROPPABLE 0x2
/*!\brief frame should be decoded but will not be shown */
#define AOM_FRAME_IS_INVISIBLE 0x4
/*!\brief this is a fragment of the encoded frame */
#define AOM_FRAME_IS_FRAGMENT 0x8
/*!\brief this is an INTRA_ONLY frame */
#define AOM_FRAME_IS_INTRAONLY 0x10
/*!\brief this is an S-frame */
#define AOM_FRAME_IS_SWITCH 0x20
/*!\brief this is an error-resilient frame */
#define AOM_FRAME_IS_ERROR_RESILIENT 0x40
/*!\brief this is a key-frame dependent recovery-point frame */
#define AOM_FRAME_IS_DELAYED_RANDOM_ACCESS_POINT 0x80
/*!\brief Error Resilient flags
*

6
third_party/aom/aom/aom_frame_buffer.h vendored
View file

@ -53,9 +53,9 @@ typedef struct aom_codec_frame_buffer {
* data. The callback is triggered when the decoder needs a frame buffer to
* decode a compressed image into. This function may be called more than once
* for every call to aom_codec_decode. The application may set fb->priv to
* some data which will be passed back in the ximage and the release function
* call. |fb| is guaranteed to not be NULL. On success the callback must
* return 0. Any failure the callback must return a value less than 0.
* some data which will be passed back in the aom_image_t and the release
* function call. |fb| is guaranteed to not be NULL. On success the callback
* must return 0. Any failure the callback must return a value less than 0.
*
* \param[in] priv Callback's private data
* \param[in] new_size Size in bytes needed by the buffer

15
third_party/aom/aom/aom_image.h vendored
View file

@ -30,11 +30,11 @@ extern "C" {
* types, removing or reassigning enums, adding/removing/rearranging
* fields to structures
*/
#define AOM_IMAGE_ABI_VERSION (4) /**<\hideinitializer*/
#define AOM_IMAGE_ABI_VERSION (5) /**<\hideinitializer*/
#define AOM_IMG_FMT_PLANAR 0x100 /**< Image is a planar format. */
#define AOM_IMG_FMT_UV_FLIP 0x200 /**< V plane precedes U in memory. */
#define AOM_IMG_FMT_HAS_ALPHA 0x400 /**< Image has an alpha channel. */
#define AOM_IMG_FMT_PLANAR 0x100 /**< Image is a planar format. */
#define AOM_IMG_FMT_UV_FLIP 0x200 /**< V plane precedes U in memory. */
/** 0x400 used to signal alpha channel, skipping for backwards compatibility. */
#define AOM_IMG_FMT_HIGHBITDEPTH 0x800 /**< Image uses 16bit framebuffer. */
/*!\brief List of supported image formats */
@ -48,8 +48,8 @@ typedef enum aom_img_fmt {
AOM_IMG_FMT_AOMI420 = AOM_IMG_FMT_PLANAR | 4,
AOM_IMG_FMT_I422 = AOM_IMG_FMT_PLANAR | 5,
AOM_IMG_FMT_I444 = AOM_IMG_FMT_PLANAR | 6,
AOM_IMG_FMT_444A = AOM_IMG_FMT_PLANAR | AOM_IMG_FMT_HAS_ALPHA | 6,
AOM_IMG_FMT_I42016 = AOM_IMG_FMT_I420 | AOM_IMG_FMT_HIGHBITDEPTH,
AOM_IMG_FMT_YV1216 = AOM_IMG_FMT_YV12 | AOM_IMG_FMT_HIGHBITDEPTH,
AOM_IMG_FMT_I42216 = AOM_IMG_FMT_I422 | AOM_IMG_FMT_HIGHBITDEPTH,
AOM_IMG_FMT_I44416 = AOM_IMG_FMT_I444 | AOM_IMG_FMT_HIGHBITDEPTH,
} aom_img_fmt_t; /**< alias for enum aom_img_fmt */
@ -169,9 +169,8 @@ typedef struct aom_image {
#define AOM_PLANE_Y 0 /**< Y (Luminance) plane */
#define AOM_PLANE_U 1 /**< U (Chroma) plane */
#define AOM_PLANE_V 2 /**< V (Chroma) plane */
#define AOM_PLANE_ALPHA 3 /**< A (Transparency) plane */
unsigned char *planes[4]; /**< pointer to the top left pixel for each plane */
int stride[4]; /**< stride between rows for each plane */
unsigned char *planes[3]; /**< pointer to the top left pixel for each plane */
int stride[3]; /**< stride between rows for each plane */
size_t sz; /**< data size */
int bps; /**< bits per sample (for packed formats) */

319
third_party/aom/aom/aomcx.h vendored
View file

@ -300,9 +300,6 @@ enum aome_enc_control_id {
* 2 = 4 tile columns
* .....
* n = 2**n tile columns
* The requested tile columns will be capped by encoder based on image size
* limitation (The minimum width of a tile column is 256 pixel, the maximum
* is 4096).
*
* By default, the value is 0, i.e. one single column tile for entire image.
*/
@ -324,6 +321,13 @@ enum aome_enc_control_id {
*/
AV1E_SET_TILE_ROWS,
/*!\brief Codec control function to enable RDO modulated by frame temporal
* dependency.
*
* By default, this feature is off.
*/
AV1E_SET_ENABLE_TPL_MODEL,
/*!\brief Codec control function to enable frame parallel decoding feature.
*
* AV1 has a bitstream feature to reduce decoding dependency between frames
@ -557,6 +561,16 @@ enum aome_enc_control_id {
*/
AV1E_SET_ENABLE_RESTORATION,
/*!\brief Codec control function to predict with OBMC mode.
*
* 0 = do not allow OBMC mode
* 1 = allow OBMC mode
*
* By default, the encoder allows OBMC prediction mode.
*
*/
AV1E_SET_ENABLE_OBMC,
/*!\brief Codec control function to encode without trellis quantization.
*
* 0 = apply trellis quantization
@ -684,18 +698,6 @@ enum aome_enc_control_id {
*/
AV1E_SET_MTU,
/*!\brief Codec control function to set dependent_horz_tiles.
*
* In encoding and decoding, AV1 allows enabling dependent horizontal tile
* The parameter for this control describes the value of this flag,
* which has a valid range [0, 1]:
* 0 = disable dependent horizontal tile
* 1 = enable dependent horizontal tile,
*
* By default, the value is 0, i.e. disable dependent horizontal tile.
*/
AV1E_SET_TILE_DEPENDENT_ROWS,
/*!\brief Codec control function to set the number of symbols in an ANS data
* window.
*
@ -708,13 +710,22 @@ enum aome_enc_control_id {
*/
AV1E_SET_ANS_WINDOW_SIZE_LOG2,
/*!\brief Codec control function to turn on / off dual filter
* enabling/disabling.
/*!\brief Codec control function to enable/disable rectangular partitions.
*
* This will enable or disable dual filter. The default value is 1
* This will enable or disable usage of rectangular partitions. The default
* value is 1.
*
*/
AV1E_SET_ENABLE_DF,
AV1E_SET_ENABLE_RECT_PARTITIONS,
/*!\brief Codec control function to turn on / off intra edge filter
* at sequence level.
*
* This will enable or disable usage of intra-edge filtering. The default
* value is 1.
*
*/
AV1E_SET_ENABLE_INTRA_EDGE_FILTER,
/*!\brief Codec control function to turn on / off frame order hint for a
* few tools:
@ -728,14 +739,23 @@ enum aome_enc_control_id {
*/
AV1E_SET_ENABLE_ORDER_HINT,
/*!\brief Codec control function to turn on / off joint compound mode
* at sequence level.
/*!\brief Codec control function to turn on / off 64-length transforms.
*
* This will enable or disable joint compound mode. The default value is 1.
* If AV1E_SET_ENABLE_ORDER_HINT is 0, then this flag is forced to 0.
* This will enable or disable usage of length 64 transforms in any
* direction. The default value is 1.
*
*/
AV1E_SET_ENABLE_JNT_COMP,
AV1E_SET_ENABLE_TX64,
/*!\brief Codec control function to turn on / off dist-wtd compound mode
* at sequence level.
*
* This will enable or disable distance-weighted compound mode. The default
* value is 1. If AV1E_SET_ENABLE_ORDER_HINT is 0, then this flag is forced
* to 0.
*
*/
AV1E_SET_ENABLE_DIST_WTD_COMP,
/*!\brief Codec control function to turn on / off ref frame mvs (mfmv) usage
* at sequence level.
@ -755,6 +775,77 @@ enum aome_enc_control_id {
*/
AV1E_SET_ALLOW_REF_FRAME_MVS,
/*!\brief Codec control function to turn on / off dual filter usage
* for a sequence.
*
* This will enable or disable use of dual interpolation filter.
* The default value is 1.
*
*/
AV1E_SET_ENABLE_DUAL_FILTER,
/*!\brief Codec control function to turn on / off masked compound usage
* for a sequence.
*
* This will enable or disable usage of wedge and diff-wtd compound
* modes. The default value is 1.
*
*/
AV1E_SET_ENABLE_MASKED_COMP,
/*!\brief Codec control function to turn on / off interintra compound
* for a sequence.
*
* This will enable or disable usage of inter-intra compound modes.
* The default value is 1.
*
*/
AV1E_SET_ENABLE_INTERINTRA_COMP,
/*!\brief Codec control function to turn on / off smooth inter-intra
* mode for a sequence.
*
* This will enable or disable usage of smooth inter-intra mode.
* The default value is 1.
*
*/
AV1E_SET_ENABLE_SMOOTH_INTERINTRA,
/*!\brief Codec control function to turn on / off difference weighted
* compound.
*
* This will enable or disable usage of difference weighted compound.
* The default value is 1.
*
*/
AV1E_SET_ENABLE_DIFF_WTD_COMP,
/*!\brief Codec control function to turn on / off interinter wedge
* compound.
*
* This will enable or disable usage of interinter wedge compound.
* The default value is 1.
*
*/
AV1E_SET_ENABLE_INTERINTER_WEDGE,
/*!\brief Codec control function to turn on / off interintra wedge
* compound.
*
* This will enable or disable usage of interintra wedge compound.
* The default value is 1.
*
*/
AV1E_SET_ENABLE_INTERINTRA_WEDGE,
/*!\brief Codec control function to turn on / off global motion usage
* for a sequence.
*
* This will enable or disable usage of global motion. The default value is 1.
*
*/
AV1E_SET_ENABLE_GLOBAL_MOTION,
/*!\brief Codec control function to turn on / off warped motion usage
* at sequence level.
*
@ -772,6 +863,39 @@ enum aome_enc_control_id {
*/
AV1E_SET_ALLOW_WARPED_MOTION,
/*!\brief Codec control function to turn on / off filter intra usage at
* sequence level.
*
* This will enable or disable usage of filter intra. The default value is 1.
* If AV1E_SET_ENABLE_FILTER_INTRA is 0, then this flag is forced to 0.
*
*/
AV1E_SET_ENABLE_FILTER_INTRA,
/*!\brief Codec control function to turn on / off smooth intra modes usage.
*
* This will enable or disable usage of smooth, smooth_h and smooth_v intra
* modes. The default value is 1.
*
*/
AV1E_SET_ENABLE_SMOOTH_INTRA,
/*!\brief Codec control function to turn on / off Paeth intra mode usage.
*
* This will enable or disable usage of Paeth intra mode. The default value
* is 1.
*
*/
AV1E_SET_ENABLE_PAETH_INTRA,
/*!\brief Codec control function to turn on / off CFL uv intra mode usage.
*
* This will enable or disable usage of chroma-from-luma intra mode. The
* default value is 1.
*
*/
AV1E_SET_ENABLE_CFL_INTRA,
/*!\brief Codec control function to turn on / off frame superresolution.
*
* This will enable or disable frame superresolution. The default value is 1
@ -779,34 +903,14 @@ enum aome_enc_control_id {
*/
AV1E_SET_ENABLE_SUPERRES,
/*!\brief Codec control function to set loop_filter_across_tiles_v_enabled
* and loop_filter_across_tiles_h_enabled.
* In encoding and decoding, AV1 allows disabling loop filter across tile
* boundary The parameter for this control describes the value of this flag,
* which has a valid range [0, 1]:
* 0 = disable loop filter across tile boundary
* 1 = enable loop filter across tile boundary
*
* By default, the value is 1, i.e. enable loop filter across tile boundary.
*
* Experiment: LOOPFILTERING_ACROSS_TILES_EXT
*/
AV1E_SET_TILE_LOOPFILTER_V,
AV1E_SET_TILE_LOOPFILTER_H,
/*!\brief Codec control function to turn on/off palette mode */
AV1E_SET_ENABLE_PALETTE,
/*!\brief Codec control function to set loop_filter_across_tiles_enabled.
*
* In encoding and decoding, AV1 allows disabling loop filter across tile
* boundary The parameter for this control describes the value of this flag,
* which has a valid range [0, 1]:
* 0 = disable loop filter across tile boundary
* 1 = enable loop filter across tile boundary
*
* By default, the value is 1, i.e. enable loop filter across tile boundary.
*
* Experiment: LOOPFILTERING_ACROSS_TILES
*/
AV1E_SET_TILE_LOOPFILTER,
/*!\brief Codec control function to turn on/off intra block copy mode */
AV1E_SET_ENABLE_INTRABC,
/*!\brief Codec control function to turn on/off intra angle delta */
AV1E_SET_ENABLE_ANGLE_DELTA,
/*!\brief Codec control function to set the delta q mode
*
@ -865,6 +969,26 @@ enum aome_enc_control_id {
/*!\brief Sets the chroma subsampling y value */
AV1E_SET_CHROMA_SUBSAMPLING_Y,
/*!\brief Control to use a reduced tx type set */
AV1E_SET_REDUCED_TX_TYPE_SET,
/*!\brief Control to use dct only for intra modes */
AV1E_SET_INTRA_DCT_ONLY,
/*!\brief Control to use dct only for inter modes */
AV1E_SET_INTER_DCT_ONLY,
/*!\brief Control to use adaptive quantize_b */
AV1E_SET_QUANT_B_ADAPT,
/*!\brief Control to select maximum height for the GF group pyramid structure
* (valid values: 1 - 4) */
AV1E_SET_GF_MAX_PYRAMID_HEIGHT,
/*!\brief Control to select maximum reference frames allowed per frame
* (valid values: 3 - 7) */
AV1E_SET_MAX_REFERENCE_FRAMES,
};
/*!\brief aom 1-D scaling mode
@ -1006,15 +1130,8 @@ AOM_CTRL_USE_TYPE(AV1E_SET_TILE_COLUMNS, int)
AOM_CTRL_USE_TYPE(AV1E_SET_TILE_ROWS, int)
#define AOM_CTRL_AV1E_SET_TILE_ROWS
AOM_CTRL_USE_TYPE(AV1E_SET_TILE_DEPENDENT_ROWS, int)
#define AOM_CTRL_AV1E_SET_TILE_DEPENDENT_ROWS
AOM_CTRL_USE_TYPE(AV1E_SET_TILE_LOOPFILTER_V, int)
#define AOM_CTRL_AV1E_SET_TILE_LOOPFILTER_V
AOM_CTRL_USE_TYPE(AV1E_SET_TILE_LOOPFILTER_H, int)
#define AOM_CTRL_AV1E_SET_TILE_LOOPFILTER_H
AOM_CTRL_USE_TYPE(AV1E_SET_TILE_LOOPFILTER, int)
#define AOM_CTRL_AV1E_SET_TILE_LOOPFILTER
AOM_CTRL_USE_TYPE(AV1E_SET_ENABLE_TPL_MODEL, unsigned int)
#define AOM_CTRL_AV1E_SET_ENABLE_TPL_MODEL
AOM_CTRL_USE_TYPE(AOME_GET_LAST_QUANTIZER, int *)
#define AOM_CTRL_AOME_GET_LAST_QUANTIZER
@ -1041,6 +1158,9 @@ AOM_CTRL_USE_TYPE(AV1E_SET_ENABLE_CDEF, unsigned int)
AOM_CTRL_USE_TYPE(AV1E_SET_ENABLE_RESTORATION, unsigned int)
#define AOM_CTRL_AV1E_SET_ENABLE_RESTORATION
AOM_CTRL_USE_TYPE(AV1E_SET_ENABLE_OBMC, unsigned int)
#define AOM_CTRL_AV1E_SET_ENABLE_OBMC
AOM_CTRL_USE_TYPE(AV1E_SET_DISABLE_TRELLIS_QUANT, unsigned int)
#define AOM_CTRL_AV1E_SET_DISABLE_TRELLIS_QUANT
@ -1070,17 +1190,23 @@ AOM_CTRL_USE_TYPE(AV1E_SET_NUM_TG, unsigned int)
AOM_CTRL_USE_TYPE(AV1E_SET_MTU, unsigned int)
#define AOM_CTRL_AV1E_SET_MTU
AOM_CTRL_USE_TYPE(AV1E_SET_TIMING_INFO_TYPE, aom_timing_info_type_t)
AOM_CTRL_USE_TYPE(AV1E_SET_TIMING_INFO_TYPE, int) /* aom_timing_info_type_t */
#define AOM_CTRL_AV1E_SET_TIMING_INFO_TYPE
AOM_CTRL_USE_TYPE(AV1E_SET_ENABLE_DF, unsigned int)
#define AOM_CTRL_AV1E_SET_ENABLE_DF
AOM_CTRL_USE_TYPE(AV1E_SET_ENABLE_RECT_PARTITIONS, unsigned int)
#define AOM_CTRL_AV1E_SET_ENABLE_RECT_PARTITIONS
AOM_CTRL_USE_TYPE(AV1E_SET_ENABLE_INTRA_EDGE_FILTER, unsigned int)
#define AOM_CTRL_AV1E_SET_ENABLE_INTRA_EDGE_FILTER
AOM_CTRL_USE_TYPE(AV1E_SET_ENABLE_ORDER_HINT, unsigned int)
#define AOM_CTRL_AV1E_SET_ENABLE_ORDER_HINT
AOM_CTRL_USE_TYPE(AV1E_SET_ENABLE_JNT_COMP, unsigned int)
#define AOM_CTRL_AV1E_SET_ENABLE_JNT_COMP
AOM_CTRL_USE_TYPE(AV1E_SET_ENABLE_TX64, unsigned int)
#define AOM_CTRL_AV1E_SET_ENABLE_TX64
AOM_CTRL_USE_TYPE(AV1E_SET_ENABLE_DIST_WTD_COMP, unsigned int)
#define AOM_CTRL_AV1E_SET_ENABLE_DIST_WTD_COMP
AOM_CTRL_USE_TYPE(AV1E_SET_ENABLE_REF_FRAME_MVS, unsigned int)
#define AOM_CTRL_AV1E_SET_ENABLE_REF_FRAME_MVS
@ -1088,15 +1214,60 @@ AOM_CTRL_USE_TYPE(AV1E_SET_ENABLE_REF_FRAME_MVS, unsigned int)
AOM_CTRL_USE_TYPE(AV1E_SET_ALLOW_REF_FRAME_MVS, unsigned int)
#define AOM_CTRL_AV1E_SET_ALLOW_REF_FRAME_MVS
AOM_CTRL_USE_TYPE(AV1E_SET_ENABLE_DUAL_FILTER, unsigned int)
#define AOM_CTRL_AV1E_SET_ENABLE_DUAL_FILTER
AOM_CTRL_USE_TYPE(AV1E_SET_ENABLE_MASKED_COMP, unsigned int)
#define AOM_CTRL_AV1E_SET_ENABLE_MASKED_COMP
AOM_CTRL_USE_TYPE(AV1E_SET_ENABLE_INTERINTRA_COMP, unsigned int)
#define AOM_CTRL_AV1E_SET_ENABLE_INTERINTRA_COMP
AOM_CTRL_USE_TYPE(AV1E_SET_ENABLE_SMOOTH_INTERINTRA, unsigned int)
#define AOM_CTRL_AV1E_SET_ENABLE_SMOOTH_INTERINTRA
AOM_CTRL_USE_TYPE(AV1E_SET_ENABLE_DIFF_WTD_COMP, unsigned int)
#define AOM_CTRL_AV1E_SET_ENABLE_DIFF_WTD_COMP
AOM_CTRL_USE_TYPE(AV1E_SET_ENABLE_INTERINTER_WEDGE, unsigned int)
#define AOM_CTRL_AV1E_SET_ENABLE_INTERINTER_WEDGE
AOM_CTRL_USE_TYPE(AV1E_SET_ENABLE_INTERINTRA_WEDGE, unsigned int)
#define AOM_CTRL_AV1E_SET_ENABLE_INTERINTRA_WEDGE
AOM_CTRL_USE_TYPE(AV1E_SET_ENABLE_GLOBAL_MOTION, unsigned int)
#define AOM_CTRL_AV1E_SET_ENABLE_GLOBAL_MOTION
AOM_CTRL_USE_TYPE(AV1E_SET_ENABLE_WARPED_MOTION, unsigned int)
#define AOM_CTRL_AV1E_SET_ENABLE_WARPED_MOTION
AOM_CTRL_USE_TYPE(AV1E_SET_ALLOW_WARPED_MOTION, unsigned int)
#define AOM_CTRL_AV1E_SET_ALLOW_WARPED_MOTION
AOM_CTRL_USE_TYPE(AV1E_SET_ENABLE_FILTER_INTRA, unsigned int)
#define AOM_CTRL_AV1E_SET_ENABLE_FILTER_INTRA
AOM_CTRL_USE_TYPE(AV1E_SET_ENABLE_SMOOTH_INTRA, unsigned int)
#define AOM_CTRL_AV1E_SET_ENABLE_SMOOTH_INTRA
AOM_CTRL_USE_TYPE(AV1E_SET_ENABLE_PAETH_INTRA, unsigned int)
#define AOM_CTRL_AV1E_SET_ENABLE_PAETH_INTRA
AOM_CTRL_USE_TYPE(AV1E_SET_ENABLE_CFL_INTRA, unsigned int)
#define AOM_CTRL_AV1E_SET_ENABLE_CFL_INTRA
AOM_CTRL_USE_TYPE(AV1E_SET_ENABLE_SUPERRES, unsigned int)
#define AOM_CTRL_AV1E_SET_ENABLE_SUPERRES
AOM_CTRL_USE_TYPE(AV1E_SET_ENABLE_PALETTE, unsigned int)
#define AOM_CTRL_AV1E_SET_ENABLE_PALETTE
AOM_CTRL_USE_TYPE(AV1E_SET_ENABLE_INTRABC, unsigned int)
#define AOM_CTRL_AV1E_SET_ENABLE_INTRABC
AOM_CTRL_USE_TYPE(AV1E_SET_ENABLE_ANGLE_DELTA, unsigned int)
#define AOM_CTRL_AV1E_SET_ENABLE_ANGLE_DELTA
AOM_CTRL_USE_TYPE(AV1E_SET_FRAME_PARALLEL_DECODING, unsigned int)
#define AOM_CTRL_AV1E_SET_FRAME_PARALLEL_DECODING
@ -1189,6 +1360,24 @@ AOM_CTRL_USE_TYPE(AV1E_SET_CHROMA_SUBSAMPLING_X, unsigned int)
AOM_CTRL_USE_TYPE(AV1E_SET_CHROMA_SUBSAMPLING_Y, unsigned int)
#define AOM_CTRL_AV1E_SET_CHROMA_SUBSAMPLING_Y
AOM_CTRL_USE_TYPE(AV1E_SET_REDUCED_TX_TYPE_SET, unsigned int)
#define AOM_CTRL_AV1E_SET_REDUCED_TX_TYPE_SET
AOM_CTRL_USE_TYPE(AV1E_SET_INTRA_DCT_ONLY, unsigned int)
#define AOM_CTRL_AV1E_SET_INTRA_DCT_ONLY
AOM_CTRL_USE_TYPE(AV1E_SET_INTER_DCT_ONLY, unsigned int)
#define AOM_CTRL_AV1E_SET_INTER_DCT_ONLY
AOM_CTRL_USE_TYPE(AV1E_SET_QUANT_B_ADAPT, unsigned int)
#define AOM_CTRL_AV1E_SET_QUANT_B_ADAPT
AOM_CTRL_USE_TYPE(AV1E_SET_GF_MAX_PYRAMID_HEIGHT, unsigned int)
#define AOM_CTRL_AV1E_SET_GF_MAX_PYRAMID_HEIGHT
AOM_CTRL_USE_TYPE(AV1E_SET_MAX_REFERENCE_FRAMES, unsigned int)
#define AOM_CTRL_AV1E_SET_MAX_REFERENCE_FRAMES
/*!\endcond */
/*! @} - end defgroup aom_encoder */
#ifdef __cplusplus

21
third_party/aom/aom/aomdx.h vendored
View file

@ -45,6 +45,7 @@ typedef struct Accounting Accounting;
/** Callback that inspects decoder frame data.
*/
typedef void (*aom_inspect_cb)(void *decoder, void *ctx);
#endif
/*!\brief Structure to hold inspection callback and context.
@ -60,6 +61,21 @@ typedef struct aom_inspect_init {
void *inspect_ctx;
} aom_inspect_init;
/*!\brief Structure to collect a buffer index when inspecting.
*
* Defines a structure to hold the buffer and return an index
* when calling decode from inspect. This enables us to decode
* non showable sub frames.
*/
typedef struct {
/*! Pointer for new position in compressed buffer after decoding 1 OBU. */
const unsigned char *buf;
/*! Index into reference buffer array to see result of decoding 1 OBU. */
int idx;
/*! Is a show existing frame. */
int show_existing;
} Av1DecodeReturn;
/*!\brief Structure to hold a tile's start address and size in the bitstream.
*
* Defines a structure to hold a tile's start address and size in the bitstream.
@ -125,6 +141,9 @@ enum aom_dec_control_id {
/** control function to get the size of the tile. */
AV1D_GET_TILE_SIZE,
/** control function to get the tile count in a tile list. */
AV1D_GET_TILE_COUNT,
/** control function to set the byte alignment of the planes in the reference
* buffers. Valid values are power of 2, from 32 to 1024. A value of 0 sets
* legacy alignment. I.e. Y plane is aligned to 32 bytes, U plane directly
@ -260,6 +279,8 @@ AOM_CTRL_USE_TYPE(AV1D_GET_IMG_FORMAT, aom_img_fmt_t *)
#define AOM_CTRL_AV1D_GET_IMG_FORMAT
AOM_CTRL_USE_TYPE(AV1D_GET_TILE_SIZE, unsigned int *)
#define AOM_CTRL_AV1D_GET_TILE_SIZE
AOM_CTRL_USE_TYPE(AV1D_GET_TILE_COUNT, unsigned int *)
#define AOM_CTRL_AV1D_GET_TILE_COUNT
AOM_CTRL_USE_TYPE(AV1D_GET_FRAME_SIZE, int *)
#define AOM_CTRL_AV1D_GET_FRAME_SIZE
AOM_CTRL_USE_TYPE(AV1_INVERT_TILE_DECODE_ORDER, int)

1
third_party/aom/aom/exports_com vendored
View file

@ -9,6 +9,7 @@ text aom_codec_iface_name
text aom_codec_version
text aom_codec_version_extra_str
text aom_codec_version_str
text aom_free
text aom_img_alloc
text aom_img_alloc_with_border
text aom_img_flip

3
third_party/aom/aom/src/aom_decoder.c vendored
View file

@ -37,9 +37,6 @@ aom_codec_err_t aom_codec_dec_init_ver(aom_codec_ctx_t *ctx,
else if ((flags & AOM_CODEC_USE_POSTPROC) &&
!(iface->caps & AOM_CODEC_CAP_POSTPROC))
res = AOM_CODEC_INCAPABLE;
else if ((flags & AOM_CODEC_USE_INPUT_FRAGMENTS) &&
!(iface->caps & AOM_CODEC_CAP_INPUT_FRAGMENTS))
res = AOM_CODEC_INCAPABLE;
else if (!(iface->caps & AOM_CODEC_CAP_DECODER))
res = AOM_CODEC_INCAPABLE;
else {

16
third_party/aom/aom/src/aom_image.c vendored
View file

@ -59,6 +59,7 @@ static aom_image_t *img_alloc_helper(
case AOM_IMG_FMT_AOMYV12: bps = 12; break;
case AOM_IMG_FMT_I422:
case AOM_IMG_FMT_I444: bps = 24; break;
case AOM_IMG_FMT_YV1216:
case AOM_IMG_FMT_I42016: bps = 24; break;
case AOM_IMG_FMT_I42216:
case AOM_IMG_FMT_I44416: bps = 48; break;
@ -73,6 +74,7 @@ static aom_image_t *img_alloc_helper(
case AOM_IMG_FMT_AOMYV12:
case AOM_IMG_FMT_I422:
case AOM_IMG_FMT_I42016:
case AOM_IMG_FMT_YV1216:
case AOM_IMG_FMT_I42216: xcs = 1; break;
default: xcs = 0; break;
}
@ -82,6 +84,7 @@ static aom_image_t *img_alloc_helper(
case AOM_IMG_FMT_YV12:
case AOM_IMG_FMT_AOMI420:
case AOM_IMG_FMT_AOMYV12:
case AOM_IMG_FMT_YV1216:
case AOM_IMG_FMT_I42016: ycs = 1; break;
default: ycs = 0; break;
}
@ -117,6 +120,7 @@ static aom_image_t *img_alloc_helper(
img->img_data = (uint8_t *)aom_memalign(buf_align, (size_t)alloc_size);
img->img_data_owner = 1;
img->sz = (size_t)alloc_size;
}
if (!img->img_data) goto fail;
@ -131,7 +135,7 @@ static aom_image_t *img_alloc_helper(
img->bps = bps;
/* Calculate strides */
img->stride[AOM_PLANE_Y] = img->stride[AOM_PLANE_ALPHA] = stride_in_bytes;
img->stride[AOM_PLANE_Y] = stride_in_bytes;
img->stride[AOM_PLANE_U] = img->stride[AOM_PLANE_V] = stride_in_bytes >> xcs;
/* Default viewport to entire image */
@ -185,12 +189,6 @@ int aom_img_set_rect(aom_image_t *img, unsigned int x, unsigned int y,
(img->fmt & AOM_IMG_FMT_HIGHBITDEPTH) ? 2 : 1;
data = img->img_data;
if (img->fmt & AOM_IMG_FMT_HAS_ALPHA) {
img->planes[AOM_PLANE_ALPHA] =
data + x * bytes_per_sample + y * img->stride[AOM_PLANE_ALPHA];
data += (img->h + 2 * border) * img->stride[AOM_PLANE_ALPHA];
}
img->planes[AOM_PLANE_Y] =
data + x * bytes_per_sample + y * img->stride[AOM_PLANE_Y];
data += (img->h + 2 * border) * img->stride[AOM_PLANE_Y];
@ -236,10 +234,6 @@ void aom_img_flip(aom_image_t *img) {
img->planes[AOM_PLANE_V] += (signed)((img->d_h >> img->y_chroma_shift) - 1) *
img->stride[AOM_PLANE_V];
img->stride[AOM_PLANE_V] = -img->stride[AOM_PLANE_V];
img->planes[AOM_PLANE_ALPHA] +=
(signed)(img->d_h - 1) * img->stride[AOM_PLANE_ALPHA];
img->stride[AOM_PLANE_ALPHA] = -img->stride[AOM_PLANE_ALPHA];
}
void aom_img_free(aom_image_t *img) {

2
third_party/aom/aom_dsp/add_noise.c vendored
View file

@ -40,7 +40,7 @@ void aom_plane_add_noise_c(uint8_t *start, char *noise, char blackclamp[16],
}
static double gaussian(double sigma, double mu, double x) {
return 1 / (sigma * sqrt(2.0 * 3.14159265)) *
return 1 / (sigma * sqrt(2.0 * PI)) *
(exp(-(x - mu) * (x - mu) / (2 * sigma * sigma)));
}

30
third_party/aom/aom_dsp/aom_dsp.cmake vendored
View file

@ -35,6 +35,7 @@ list(APPEND AOM_DSP_COMMON_SOURCES
"${AOM_ROOT}/aom_dsp/intrapred_common.h"
"${AOM_ROOT}/aom_dsp/loopfilter.c"
"${AOM_ROOT}/aom_dsp/prob.h"
"${AOM_ROOT}/aom_dsp/recenter.h"
"${AOM_ROOT}/aom_dsp/simd/v128_intrinsics.h"
"${AOM_ROOT}/aom_dsp/simd/v128_intrinsics_c.h"
"${AOM_ROOT}/aom_dsp/simd/v256_intrinsics.h"
@ -43,7 +44,8 @@ list(APPEND AOM_DSP_COMMON_SOURCES
"${AOM_ROOT}/aom_dsp/simd/v64_intrinsics_c.h"
"${AOM_ROOT}/aom_dsp/subtract.c"
"${AOM_ROOT}/aom_dsp/txfm_common.h"
"${AOM_ROOT}/aom_dsp/x86/convolve_common_intrin.h")
"${AOM_ROOT}/aom_dsp/x86/convolve_common_intrin.h"
"${AOM_ROOT}/aom_dsp/avg.c")
list(APPEND AOM_DSP_COMMON_ASM_SSE2
"${AOM_ROOT}/aom_dsp/x86/aom_convolve_copy_sse2.asm"
@ -51,15 +53,17 @@ list(APPEND AOM_DSP_COMMON_ASM_SSE2
"${AOM_ROOT}/aom_dsp/x86/aom_high_subpixel_bilinear_sse2.asm"
"${AOM_ROOT}/aom_dsp/x86/aom_subpixel_8t_sse2.asm"
"${AOM_ROOT}/aom_dsp/x86/aom_subpixel_bilinear_sse2.asm"
"${AOM_ROOT}/aom_dsp/x86/highbd_intrapred_sse2.asm"
"${AOM_ROOT}/aom_dsp/x86/intrapred_sse2.asm"
"${AOM_ROOT}/aom_dsp/x86/highbd_intrapred_asm_sse2.asm"
"${AOM_ROOT}/aom_dsp/x86/intrapred_asm_sse2.asm"
"${AOM_ROOT}/aom_dsp/x86/inv_wht_sse2.asm")
list(APPEND AOM_DSP_COMMON_INTRIN_SSE2
"${AOM_ROOT}/aom_dsp/x86/aom_subpixel_8t_intrin_sse2.c"
"${AOM_ROOT}/aom_dsp/x86/aom_asm_stubs.c"
"${AOM_ROOT}/aom_dsp/x86/convolve.h"
"${AOM_ROOT}/aom_dsp/x86/convolve_sse2.h"
"${AOM_ROOT}/aom_dsp/x86/fft_sse2.c"
"${AOM_ROOT}/aom_dsp/x86/highbd_convolve_sse2.c"
"${AOM_ROOT}/aom_dsp/x86/highbd_intrapred_sse2.c"
"${AOM_ROOT}/aom_dsp/x86/highbd_loopfilter_sse2.c"
"${AOM_ROOT}/aom_dsp/x86/intrapred_sse2.c"
@ -68,7 +72,9 @@ list(APPEND AOM_DSP_COMMON_INTRIN_SSE2
"${AOM_ROOT}/aom_dsp/x86/mem_sse2.h"
"${AOM_ROOT}/aom_dsp/x86/transpose_sse2.h"
"${AOM_ROOT}/aom_dsp/x86/txfm_common_sse2.h"
"${AOM_ROOT}/aom_dsp/x86/sum_squares_sse2.h")
"${AOM_ROOT}/aom_dsp/x86/sum_squares_sse2.h"
"${AOM_ROOT}/aom_dsp/x86/avg_intrin_sse2.c"
"${AOM_ROOT}/aom_dsp/x86/bitdepth_conversion_sse2.h")
list(APPEND AOM_DSP_COMMON_ASM_SSSE3
"${AOM_ROOT}/aom_dsp/x86/aom_subpixel_8t_ssse3.asm"
@ -94,7 +100,9 @@ list(APPEND AOM_DSP_COMMON_INTRIN_AVX2
"${AOM_ROOT}/aom_dsp/x86/highbd_convolve_avx2.c"
"${AOM_ROOT}/aom_dsp/x86/highbd_loopfilter_avx2.c"
"${AOM_ROOT}/aom_dsp/x86/intrapred_avx2.c"
"${AOM_ROOT}/aom_dsp/x86/blend_a64_mask_avx2.c")
"${AOM_ROOT}/aom_dsp/x86/blend_a64_mask_avx2.c"
"${AOM_ROOT}/aom_dsp/x86/avg_intrin_avx2.c"
"${AOM_ROOT}/aom_dsp/x86/bitdepth_conversion_avx2.h")
list(APPEND AOM_DSP_COMMON_INTRIN_NEON
"${AOM_ROOT}/aom_dsp/arm/fwd_txfm_neon.c"
@ -175,6 +183,9 @@ if(CONFIG_AV1_ENCODER)
"${AOM_ROOT}/aom_dsp/x86/subpel_variance_sse2.asm"
"${AOM_ROOT}/aom_dsp/x86/subtract_sse2.asm")
list(APPEND AOM_DSP_ENCODER_ASM_SSE2_X86_64
"${AOM_ROOT}/aom_dsp/x86/ssim_sse2_x86_64.asm")
list(APPEND AOM_DSP_ENCODER_INTRIN_SSE2
"${AOM_ROOT}/aom_dsp/x86/fwd_txfm_impl_sse2.h"
"${AOM_ROOT}/aom_dsp/x86/fwd_txfm_sse2.c"
@ -189,7 +200,7 @@ if(CONFIG_AV1_ENCODER)
list(APPEND AOM_DSP_ENCODER_ASM_SSSE3_X86_64
"${AOM_ROOT}/aom_dsp/x86/fwd_txfm_ssse3_x86_64.asm"
"${AOM_ROOT}/aom_dsp/x86/ssim_opt_x86_64.asm")
"${AOM_ROOT}/aom_dsp/x86/quantize_ssse3_x86_64.asm")
list(APPEND AOM_DSP_ENCODER_INTRIN_AVX2
"${AOM_ROOT}/aom_dsp/x86/masked_sad_intrin_avx2.c"
@ -207,9 +218,6 @@ if(CONFIG_AV1_ENCODER)
"${AOM_ROOT}/aom_dsp/x86/obmc_variance_avx2.c"
"${AOM_ROOT}/aom_dsp/x86/sum_squares_avx2.c")
list(APPEND AOM_DSP_ENCODER_ASM_SSSE3_X86_64
"${AOM_ROOT}/aom_dsp/x86/quantize_ssse3_x86_64.asm")
list(APPEND AOM_DSP_ENCODER_AVX_ASM_X86_64
"${AOM_ROOT}/aom_dsp/x86/quantize_avx_x86_64.asm")
@ -218,6 +226,7 @@ if(CONFIG_AV1_ENCODER)
"${AOM_ROOT}/aom_dsp/x86/masked_sad_intrin_ssse3.c"
"${AOM_ROOT}/aom_dsp/x86/masked_variance_intrin_ssse3.h"
"${AOM_ROOT}/aom_dsp/x86/masked_variance_intrin_ssse3.c"
"${AOM_ROOT}/aom_dsp/x86/quantize_ssse3.c"
"${AOM_ROOT}/aom_dsp/x86/variance_impl_ssse3.c"
"${AOM_ROOT}/aom_dsp/x86/jnt_variance_ssse3.c"
"${AOM_ROOT}/aom_dsp/x86/jnt_sad_ssse3.c")
@ -278,6 +287,9 @@ function(setup_aom_dsp_targets)
"AOM_DSP_COMMON_INTRIN_SSE2" "aom")
if(CONFIG_AV1_ENCODER)
if("${AOM_TARGET_CPU}" STREQUAL "x86_64")
list(APPEND AOM_DSP_ENCODER_ASM_SSE2 ${AOM_DSP_ENCODER_ASM_SSE2_X86_64})
endif()
add_asm_library("aom_dsp_encoder_sse2" "AOM_DSP_ENCODER_ASM_SSE2" "aom")
add_intrinsics_object_library("-msse2" "sse2" "aom_dsp_encoder"
"AOM_DSP_ENCODER_INTRIN_SSE2" "aom")

144
third_party/aom/aom_dsp/aom_dsp_rtcd_defs.pl vendored
View file

@ -362,10 +362,10 @@ add_proto qw/void aom_highbd_convolve_copy/, "const uint8_t *src, ptrdiff_t src_
specialize qw/aom_highbd_convolve_copy sse2 avx2/;
add_proto qw/void aom_highbd_convolve8_horiz/, "const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, const int16_t *filter_y, int y_step_q4, int w, int h, int bps";
specialize qw/aom_highbd_convolve8_horiz avx2/, "$sse2_x86_64";
specialize qw/aom_highbd_convolve8_horiz sse2 avx2/;
add_proto qw/void aom_highbd_convolve8_vert/, "const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, const int16_t *filter_y, int y_step_q4, int w, int h, int bps";
specialize qw/aom_highbd_convolve8_vert avx2/, "$sse2_x86_64";
specialize qw/aom_highbd_convolve8_vert sse2 avx2/;
#
# Loopfilter
@ -466,10 +466,6 @@ specialize qw/aom_highbd_lpf_horizontal_4 sse2/;
add_proto qw/void aom_highbd_lpf_horizontal_4_dual/, "uint16_t *s, int pitch, const uint8_t *blimit0, const uint8_t *limit0, const uint8_t *thresh0, const uint8_t *blimit1, const uint8_t *limit1, const uint8_t *thresh1, int bd";
specialize qw/aom_highbd_lpf_horizontal_4_dual sse2 avx2/;
# Helper functions.
add_proto qw/void av1_round_shift_array/, "int32_t *arr, int size, int bit";
specialize "av1_round_shift_array", qw/sse4_1 neon/;
#
# Encoder functions.
#
@ -526,6 +522,7 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
specialize qw/aom_quantize_b_32x32/, "$ssse3_x86_64", "$avx_x86_64";
add_proto qw/void aom_quantize_b_64x64/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan";
specialize qw/aom_quantize_b_64x64 ssse3/;
} # CONFIG_AV1_ENCODER
if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
@ -544,7 +541,6 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
#
add_proto qw/void aom_lowbd_blend_a64_d16_mask/, "uint8_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0, uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride, const uint8_t *mask, uint32_t mask_stride, int w, int h, int subx, int suby, ConvolveParams *conv_params";
specialize qw/aom_lowbd_blend_a64_d16_mask sse4_1 avx2 neon/;
add_proto qw/void aom_highbd_blend_a64_d16_mask/, "uint8_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0, uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride, const uint8_t *mask, uint32_t mask_stride, int w, int h, int subx, int suby, ConvolveParams *conv_params, const int bd";
add_proto qw/void aom_blend_a64_mask/, "uint8_t *dst, uint32_t dst_stride, const uint8_t *src0, uint32_t src0_stride, const uint8_t *src1, uint32_t src1_stride, const uint8_t *mask, uint32_t mask_stride, int w, int h, int subx, int suby";
add_proto qw/void aom_blend_a64_hmask/, "uint8_t *dst, uint32_t dst_stride, const uint8_t *src0, uint32_t src0_stride, const uint8_t *src1, uint32_t src1_stride, const uint8_t *mask, int w, int h";
add_proto qw/void aom_blend_a64_vmask/, "uint8_t *dst, uint32_t dst_stride, const uint8_t *src0, uint32_t src0_stride, const uint8_t *src1, uint32_t src1_stride, const uint8_t *mask, int w, int h";
@ -555,9 +551,11 @@ specialize "aom_blend_a64_vmask", qw/sse4_1 neon/;
add_proto qw/void aom_highbd_blend_a64_mask/, "uint8_t *dst, uint32_t dst_stride, const uint8_t *src0, uint32_t src0_stride, const uint8_t *src1, uint32_t src1_stride, const uint8_t *mask, uint32_t mask_stride, int w, int h, int subx, int suby, int bd";
add_proto qw/void aom_highbd_blend_a64_hmask/, "uint8_t *dst, uint32_t dst_stride, const uint8_t *src0, uint32_t src0_stride, const uint8_t *src1, uint32_t src1_stride, const uint8_t *mask, int w, int h, int bd";
add_proto qw/void aom_highbd_blend_a64_vmask/, "uint8_t *dst, uint32_t dst_stride, const uint8_t *src0, uint32_t src0_stride, const uint8_t *src1, uint32_t src1_stride, const uint8_t *mask, int w, int h, int bd";
add_proto qw/void aom_highbd_blend_a64_d16_mask/, "uint8_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0, uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride, const uint8_t *mask, uint32_t mask_stride, int w, int h, int subx, int suby, ConvolveParams *conv_params, const int bd";
specialize "aom_highbd_blend_a64_mask", qw/sse4_1/;
specialize "aom_highbd_blend_a64_hmask", qw/sse4_1/;
specialize "aom_highbd_blend_a64_vmask", qw/sse4_1/;
specialize "aom_highbd_blend_a64_d16_mask", qw/sse4_1 avx2/;
if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
#
@ -595,7 +593,7 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
($w, $h) = @$_;
add_proto qw/unsigned int/, "aom_sad${w}x${h}", "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride";
add_proto qw/unsigned int/, "aom_sad${w}x${h}_avg", "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred";
add_proto qw/unsigned int/, "aom_jnt_sad${w}x${h}_avg", "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred, const JNT_COMP_PARAMS *jcp_param";
add_proto qw/unsigned int/, "aom_dist_wtd_sad${w}x${h}_avg", "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred, const DIST_WTD_COMP_PARAMS *jcp_param";
}
specialize qw/aom_sad128x128 avx2 sse2/;
@ -646,29 +644,29 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
specialize qw/aom_sad16x64_avg sse2/;
specialize qw/aom_sad64x16_avg sse2/;
specialize qw/aom_jnt_sad128x128_avg ssse3/;
specialize qw/aom_jnt_sad128x64_avg ssse3/;
specialize qw/aom_jnt_sad64x128_avg ssse3/;
specialize qw/aom_jnt_sad64x64_avg ssse3/;
specialize qw/aom_jnt_sad64x32_avg ssse3/;
specialize qw/aom_jnt_sad32x64_avg ssse3/;
specialize qw/aom_jnt_sad32x32_avg ssse3/;
specialize qw/aom_jnt_sad32x16_avg ssse3/;
specialize qw/aom_jnt_sad16x32_avg ssse3/;
specialize qw/aom_jnt_sad16x16_avg ssse3/;
specialize qw/aom_jnt_sad16x8_avg ssse3/;
specialize qw/aom_jnt_sad8x16_avg ssse3/;
specialize qw/aom_jnt_sad8x8_avg ssse3/;
specialize qw/aom_jnt_sad8x4_avg ssse3/;
specialize qw/aom_jnt_sad4x8_avg ssse3/;
specialize qw/aom_jnt_sad4x4_avg ssse3/;
specialize qw/aom_dist_wtd_sad128x128_avg ssse3/;
specialize qw/aom_dist_wtd_sad128x64_avg ssse3/;
specialize qw/aom_dist_wtd_sad64x128_avg ssse3/;
specialize qw/aom_dist_wtd_sad64x64_avg ssse3/;
specialize qw/aom_dist_wtd_sad64x32_avg ssse3/;
specialize qw/aom_dist_wtd_sad32x64_avg ssse3/;
specialize qw/aom_dist_wtd_sad32x32_avg ssse3/;
specialize qw/aom_dist_wtd_sad32x16_avg ssse3/;
specialize qw/aom_dist_wtd_sad16x32_avg ssse3/;
specialize qw/aom_dist_wtd_sad16x16_avg ssse3/;
specialize qw/aom_dist_wtd_sad16x8_avg ssse3/;
specialize qw/aom_dist_wtd_sad8x16_avg ssse3/;
specialize qw/aom_dist_wtd_sad8x8_avg ssse3/;
specialize qw/aom_dist_wtd_sad8x4_avg ssse3/;
specialize qw/aom_dist_wtd_sad4x8_avg ssse3/;
specialize qw/aom_dist_wtd_sad4x4_avg ssse3/;
specialize qw/aom_jnt_sad4x16_avg ssse3/;
specialize qw/aom_jnt_sad16x4_avg ssse3/;
specialize qw/aom_jnt_sad8x32_avg ssse3/;
specialize qw/aom_jnt_sad32x8_avg ssse3/;
specialize qw/aom_jnt_sad16x64_avg ssse3/;
specialize qw/aom_jnt_sad64x16_avg ssse3/;
specialize qw/aom_dist_wtd_sad4x16_avg ssse3/;
specialize qw/aom_dist_wtd_sad16x4_avg ssse3/;
specialize qw/aom_dist_wtd_sad8x32_avg ssse3/;
specialize qw/aom_dist_wtd_sad32x8_avg ssse3/;
specialize qw/aom_dist_wtd_sad16x64_avg ssse3/;
specialize qw/aom_dist_wtd_sad64x16_avg ssse3/;
add_proto qw/unsigned int/, "aom_sad4xh", "const uint8_t *a, int a_stride, const uint8_t *b, int b_stride, int width, int height";
add_proto qw/unsigned int/, "aom_sad8xh", "const uint8_t *a, int a_stride, const uint8_t *b, int b_stride, int width, int height";
@ -693,7 +691,7 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
specialize "aom_highbd_sad${w}x${h}", qw/sse2/;
specialize "aom_highbd_sad${w}x${h}_avg", qw/sse2/;
}
add_proto qw/unsigned int/, "aom_highbd_jnt_sad${w}x${h}_avg", "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred, const JNT_COMP_PARAMS* jcp_param";
add_proto qw/unsigned int/, "aom_highbd_dist_wtd_sad${w}x${h}_avg", "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred, const DIST_WTD_COMP_PARAMS* jcp_param";
}
specialize qw/aom_highbd_sad128x128 avx2/;
specialize qw/aom_highbd_sad128x64 avx2/;
@ -837,6 +835,21 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
specialize qw/aom_highbd_sad16x64x4d sse2/;
specialize qw/aom_highbd_sad64x16x4d sse2/;
#
# hamadard transform and satd for implmenting temporal dependency model
#
add_proto qw/void aom_hadamard_8x8/, "const int16_t *src_diff, ptrdiff_t src_stride, tran_low_t *coeff";
specialize qw/aom_hadamard_8x8 sse2/;
add_proto qw/void aom_hadamard_16x16/, "const int16_t *src_diff, ptrdiff_t src_stride, tran_low_t *coeff";
specialize qw/aom_hadamard_16x16 avx2 sse2/;
add_proto qw/void aom_hadamard_32x32/, "const int16_t *src_diff, ptrdiff_t src_stride, tran_low_t *coeff";
specialize qw/aom_hadamard_32x32 avx2 sse2/;
add_proto qw/int aom_satd/, "const tran_low_t *coeff, int length";
specialize qw/aom_satd avx2 sse2/;
#
# Structured Similarity (SSIM)
#
@ -903,11 +916,11 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
int ref_stride, int subpel_search";
specialize qw/aom_comp_avg_upsampled_pred sse2/;
add_proto qw/void aom_jnt_comp_avg_upsampled_pred/, "MACROBLOCKD *xd, const struct AV1Common *const cm, int mi_row, int mi_col,
add_proto qw/void aom_dist_wtd_comp_avg_upsampled_pred/, "MACROBLOCKD *xd, const struct AV1Common *const cm, int mi_row, int mi_col,
const MV *const mv, uint8_t *comp_pred, const uint8_t *pred, int width,
int height, int subpel_x_q3, int subpel_y_q3, const uint8_t *ref,
int ref_stride, const JNT_COMP_PARAMS *jcp_param, int subpel_search";
specialize qw/aom_jnt_comp_avg_upsampled_pred ssse3/;
int ref_stride, const DIST_WTD_COMP_PARAMS *jcp_param, int subpel_search";
specialize qw/aom_dist_wtd_comp_avg_upsampled_pred ssse3/;
add_proto qw/void aom_comp_mask_upsampled_pred/, "MACROBLOCKD *xd, const struct AV1Common *const cm, int mi_row, int mi_col,
const MV *const mv, uint8_t *comp_pred, const uint8_t *pred, int width,
@ -916,7 +929,6 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
int subpel_search";
specialize qw/aom_comp_mask_upsampled_pred sse2/;
add_proto qw/void aom_highbd_upsampled_pred/, "MACROBLOCKD *xd, const struct AV1Common *const cm, int mi_row, int mi_col,
const MV *const mv, uint8_t *comp_pred8, int width, int height, int subpel_x_q3,
int subpel_y_q3, const uint8_t *ref8, int ref_stride, int bd, int subpel_search";
@ -927,11 +939,11 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
int height, int subpel_x_q3, int subpel_y_q3, const uint8_t *ref8, int ref_stride, int bd, int subpel_search";
specialize qw/aom_highbd_comp_avg_upsampled_pred sse2/;
add_proto qw/void aom_highbd_jnt_comp_avg_upsampled_pred/, "MACROBLOCKD *xd, const struct AV1Common *const cm, int mi_row, int mi_col,
add_proto qw/void aom_highbd_dist_wtd_comp_avg_upsampled_pred/, "MACROBLOCKD *xd, const struct AV1Common *const cm, int mi_row, int mi_col,
const MV *const mv, uint8_t *comp_pred8, const uint8_t *pred8, int width,
int height, int subpel_x_q3, int subpel_y_q3, const uint8_t *ref8,
int ref_stride, int bd, const JNT_COMP_PARAMS *jcp_param, int subpel_search";
specialize qw/aom_highbd_jnt_comp_avg_upsampled_pred sse2/;
int ref_stride, int bd, const DIST_WTD_COMP_PARAMS *jcp_param, int subpel_search";
specialize qw/aom_highbd_dist_wtd_comp_avg_upsampled_pred sse2/;
#
@ -957,7 +969,7 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
add_proto qw/unsigned int/, "aom_variance${w}x${h}", "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
add_proto qw/uint32_t/, "aom_sub_pixel_variance${w}x${h}", "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse";
add_proto qw/uint32_t/, "aom_sub_pixel_avg_variance${w}x${h}", "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred";
add_proto qw/uint32_t/, "aom_jnt_sub_pixel_avg_variance${w}x${h}", "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred, const JNT_COMP_PARAMS *jcp_param";
add_proto qw/uint32_t/, "aom_dist_wtd_sub_pixel_avg_variance${w}x${h}", "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred, const DIST_WTD_COMP_PARAMS *jcp_param";
}
specialize qw/aom_variance128x128 sse2 avx2 /;
specialize qw/aom_variance128x64 sse2 avx2 /;
@ -1029,30 +1041,30 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
specialize qw/aom_sub_pixel_avg_variance16x64 sse2 ssse3/;
specialize qw/aom_sub_pixel_avg_variance64x16 sse2 ssse3/;
specialize qw/aom_jnt_sub_pixel_avg_variance64x64 ssse3/;
specialize qw/aom_jnt_sub_pixel_avg_variance64x32 ssse3/;
specialize qw/aom_jnt_sub_pixel_avg_variance32x64 ssse3/;
specialize qw/aom_jnt_sub_pixel_avg_variance32x32 ssse3/;
specialize qw/aom_jnt_sub_pixel_avg_variance32x16 ssse3/;
specialize qw/aom_jnt_sub_pixel_avg_variance16x32 ssse3/;
specialize qw/aom_jnt_sub_pixel_avg_variance16x16 ssse3/;
specialize qw/aom_jnt_sub_pixel_avg_variance16x8 ssse3/;
specialize qw/aom_jnt_sub_pixel_avg_variance8x16 ssse3/;
specialize qw/aom_jnt_sub_pixel_avg_variance8x8 ssse3/;
specialize qw/aom_jnt_sub_pixel_avg_variance8x4 ssse3/;
specialize qw/aom_jnt_sub_pixel_avg_variance4x8 ssse3/;
specialize qw/aom_jnt_sub_pixel_avg_variance4x4 ssse3/;
specialize qw/aom_dist_wtd_sub_pixel_avg_variance64x64 ssse3/;
specialize qw/aom_dist_wtd_sub_pixel_avg_variance64x32 ssse3/;
specialize qw/aom_dist_wtd_sub_pixel_avg_variance32x64 ssse3/;
specialize qw/aom_dist_wtd_sub_pixel_avg_variance32x32 ssse3/;
specialize qw/aom_dist_wtd_sub_pixel_avg_variance32x16 ssse3/;
specialize qw/aom_dist_wtd_sub_pixel_avg_variance16x32 ssse3/;
specialize qw/aom_dist_wtd_sub_pixel_avg_variance16x16 ssse3/;
specialize qw/aom_dist_wtd_sub_pixel_avg_variance16x8 ssse3/;
specialize qw/aom_dist_wtd_sub_pixel_avg_variance8x16 ssse3/;
specialize qw/aom_dist_wtd_sub_pixel_avg_variance8x8 ssse3/;
specialize qw/aom_dist_wtd_sub_pixel_avg_variance8x4 ssse3/;
specialize qw/aom_dist_wtd_sub_pixel_avg_variance4x8 ssse3/;
specialize qw/aom_dist_wtd_sub_pixel_avg_variance4x4 ssse3/;
specialize qw/aom_jnt_sub_pixel_avg_variance4x16 ssse3/;
specialize qw/aom_jnt_sub_pixel_avg_variance16x4 ssse3/;
specialize qw/aom_jnt_sub_pixel_avg_variance8x32 ssse3/;
specialize qw/aom_jnt_sub_pixel_avg_variance32x8 ssse3/;
specialize qw/aom_jnt_sub_pixel_avg_variance16x64 ssse3/;
specialize qw/aom_jnt_sub_pixel_avg_variance64x16 ssse3/;
specialize qw/aom_dist_wtd_sub_pixel_avg_variance4x16 ssse3/;
specialize qw/aom_dist_wtd_sub_pixel_avg_variance16x4 ssse3/;
specialize qw/aom_dist_wtd_sub_pixel_avg_variance8x32 ssse3/;
specialize qw/aom_dist_wtd_sub_pixel_avg_variance32x8 ssse3/;
specialize qw/aom_dist_wtd_sub_pixel_avg_variance16x64 ssse3/;
specialize qw/aom_dist_wtd_sub_pixel_avg_variance64x16 ssse3/;
specialize qw/aom_jnt_sub_pixel_avg_variance128x128 ssse3/;
specialize qw/aom_jnt_sub_pixel_avg_variance128x64 ssse3/;
specialize qw/aom_jnt_sub_pixel_avg_variance64x128 ssse3/;
specialize qw/aom_dist_wtd_sub_pixel_avg_variance128x128 ssse3/;
specialize qw/aom_dist_wtd_sub_pixel_avg_variance128x64 ssse3/;
specialize qw/aom_dist_wtd_sub_pixel_avg_variance64x128 ssse3/;
foreach $bd (8, 10, 12) {
@ -1084,7 +1096,7 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
specialize "aom_highbd_${bd}_sub_pixel_avg_variance${w}x${h}", "sse4_1";
}
add_proto qw/uint32_t/, "aom_highbd_${bd}_jnt_sub_pixel_avg_variance${w}x${h}", "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred, const JNT_COMP_PARAMS* jcp_param";
add_proto qw/uint32_t/, "aom_highbd_${bd}_dist_wtd_sub_pixel_avg_variance${w}x${h}", "const uint8_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred, const DIST_WTD_COMP_PARAMS* jcp_param";
}
}
@ -1173,8 +1185,8 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
#
add_proto qw/void aom_comp_avg_pred/, "uint8_t *comp_pred, const uint8_t *pred, int width, int height, const uint8_t *ref, int ref_stride";
add_proto qw/void aom_jnt_comp_avg_pred/, "uint8_t *comp_pred, const uint8_t *pred, int width, int height, const uint8_t *ref, int ref_stride, const JNT_COMP_PARAMS *jcp_param";
specialize qw/aom_jnt_comp_avg_pred ssse3/;
add_proto qw/void aom_dist_wtd_comp_avg_pred/, "uint8_t *comp_pred, const uint8_t *pred, int width, int height, const uint8_t *ref, int ref_stride, const DIST_WTD_COMP_PARAMS *jcp_param";
specialize qw/aom_dist_wtd_comp_avg_pred ssse3/;
add_proto qw/unsigned int aom_highbd_12_variance128x128/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
specialize qw/aom_highbd_12_variance128x128 sse2/;
@ -1340,8 +1352,8 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
add_proto qw/void aom_highbd_comp_avg_pred/, "uint8_t *comp_pred8, const uint8_t *pred8, int width, int height, const uint8_t *ref8, int ref_stride";
add_proto qw/void aom_highbd_jnt_comp_avg_pred/, "uint8_t *comp_pred8, const uint8_t *pred8, int width, int height, const uint8_t *ref8, int ref_stride, const JNT_COMP_PARAMS *jcp_param";
specialize qw/aom_highbd_jnt_comp_avg_pred sse2/;
add_proto qw/void aom_highbd_dist_wtd_comp_avg_pred/, "uint8_t *comp_pred8, const uint8_t *pred8, int width, int height, const uint8_t *ref8, int ref_stride, const DIST_WTD_COMP_PARAMS *jcp_param";
specialize qw/aom_highbd_dist_wtd_comp_avg_pred sse2/;
#
# Subpixel Variance

17
third_party/aom/aom_dsp/arm/loopfilter_neon.c vendored
View file

@ -785,16 +785,15 @@ void aom_lpf_vertical_4_neon(uint8_t *src, int stride, const uint8_t *blimit,
const uint8_t *limit, const uint8_t *thresh) {
uint32x2x2_t p1q0_p0q1, p1q1_p0q0, p1p0_q1q0;
uint32x2_t pq_rev;
uint8x8_t UNINITIALIZED_IS_SAFE(p1p0), q0q1, p0q0, p1q1;
uint8x8_t UNINITIALIZED_IS_SAFE(p1p0), UNINITIALIZED_IS_SAFE(q0q1);
uint8x8_t p0q0, p1q1;
// row0: p1 p0 | q0 q1
// row1: p1 p0 | q0 q1
// row2: p1 p0 | q0 q1
// row3: p1 p0 | q0 q1
load_u8_4x1(src - 2, &p1p0, 0);
load_u8_4x1((src - 2) + 1 * stride, &p1p0, 1);
load_u8_4x1((src - 2) + 2 * stride, &q0q1, 0);
load_u8_4x1((src - 2) + 3 * stride, &q0q1, 1);
load_unaligned_u8_4x4(src - 2, stride, (uint32x2_t *)&p1p0,
(uint32x2_t *)&q0q1);
transpose_u8_4x4(&p1p0, &q0q1);
@ -815,10 +814,10 @@ void aom_lpf_vertical_4_neon(uint8_t *src, int stride, const uint8_t *blimit,
transpose_u8_4x4(&p1p0, &q0q1);
store_u8_4x1(src - 2, p1p0, 0);
store_u8_4x1((src - 2) + 1 * stride, q0q1, 0);
store_u8_4x1((src - 2) + 2 * stride, p1p0, 1);
store_u8_4x1((src - 2) + 3 * stride, q0q1, 1);
store_unaligned_u8_4x1(src - 2, p1p0, 0);
store_unaligned_u8_4x1((src - 2) + 1 * stride, q0q1, 0);
store_unaligned_u8_4x1((src - 2) + 2 * stride, p1p0, 1);
store_unaligned_u8_4x1((src - 2) + 3 * stride, q0q1, 1);
}
void aom_lpf_horizontal_14_neon(uint8_t *src, int stride, const uint8_t *blimit,

148
third_party/aom/aom_dsp/avg.c vendored Normal file
View file

@ -0,0 +1,148 @@
/*
* Copyright (c) 2016, Alliance for Open Media. All rights reserved
*
* This source code is subject to the terms of the BSD 2 Clause License and
* the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
* was not distributed with this source code in the LICENSE file, you can
* obtain it at www.aomedia.org/license/software. If the Alliance for Open
* Media Patent License 1.0 was not distributed with this source code in the
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
*/
#include <stdlib.h>
#include "config/aom_dsp_rtcd.h"
#include "aom_ports/mem.h"
// src_diff: first pass, 9 bit, dynamic range [-255, 255]
// second pass, 12 bit, dynamic range [-2040, 2040]
static void hadamard_col8(const int16_t *src_diff, ptrdiff_t src_stride,
int16_t *coeff) {
int16_t b0 = src_diff[0 * src_stride] + src_diff[1 * src_stride];
int16_t b1 = src_diff[0 * src_stride] - src_diff[1 * src_stride];
int16_t b2 = src_diff[2 * src_stride] + src_diff[3 * src_stride];
int16_t b3 = src_diff[2 * src_stride] - src_diff[3 * src_stride];
int16_t b4 = src_diff[4 * src_stride] + src_diff[5 * src_stride];
int16_t b5 = src_diff[4 * src_stride] - src_diff[5 * src_stride];
int16_t b6 = src_diff[6 * src_stride] + src_diff[7 * src_stride];
int16_t b7 = src_diff[6 * src_stride] - src_diff[7 * src_stride];
int16_t c0 = b0 + b2;
int16_t c1 = b1 + b3;
int16_t c2 = b0 - b2;
int16_t c3 = b1 - b3;
int16_t c4 = b4 + b6;
int16_t c5 = b5 + b7;
int16_t c6 = b4 - b6;
int16_t c7 = b5 - b7;
coeff[0] = c0 + c4;
coeff[7] = c1 + c5;
coeff[3] = c2 + c6;
coeff[4] = c3 + c7;
coeff[2] = c0 - c4;
coeff[6] = c1 - c5;
coeff[1] = c2 - c6;
coeff[5] = c3 - c7;
}
// The order of the output coeff of the hadamard is not important. For
// optimization purposes the final transpose may be skipped.
void aom_hadamard_8x8_c(const int16_t *src_diff, ptrdiff_t src_stride,
tran_low_t *coeff) {
int idx;
int16_t buffer[64];
int16_t buffer2[64];
int16_t *tmp_buf = &buffer[0];
for (idx = 0; idx < 8; ++idx) {
hadamard_col8(src_diff, src_stride, tmp_buf); // src_diff: 9 bit
// dynamic range [-255, 255]
tmp_buf += 8;
++src_diff;
}
tmp_buf = &buffer[0];
for (idx = 0; idx < 8; ++idx) {
hadamard_col8(tmp_buf, 8, buffer2 + 8 * idx); // tmp_buf: 12 bit
// dynamic range [-2040, 2040]
// buffer2: 15 bit
// dynamic range [-16320, 16320]
++tmp_buf;
}
for (idx = 0; idx < 64; ++idx) coeff[idx] = (tran_low_t)buffer2[idx];
}
// In place 16x16 2D Hadamard transform
void aom_hadamard_16x16_c(const int16_t *src_diff, ptrdiff_t src_stride,
tran_low_t *coeff) {
int idx;
for (idx = 0; idx < 4; ++idx) {
// src_diff: 9 bit, dynamic range [-255, 255]
const int16_t *src_ptr =
src_diff + (idx >> 1) * 8 * src_stride + (idx & 0x01) * 8;
aom_hadamard_8x8_c(src_ptr, src_stride, coeff + idx * 64);
}
// coeff: 15 bit, dynamic range [-16320, 16320]
for (idx = 0; idx < 64; ++idx) {
tran_low_t a0 = coeff[0];
tran_low_t a1 = coeff[64];
tran_low_t a2 = coeff[128];
tran_low_t a3 = coeff[192];
tran_low_t b0 = (a0 + a1) >> 1; // (a0 + a1): 16 bit, [-32640, 32640]
tran_low_t b1 = (a0 - a1) >> 1; // b0-b3: 15 bit, dynamic range
tran_low_t b2 = (a2 + a3) >> 1; // [-16320, 16320]
tran_low_t b3 = (a2 - a3) >> 1;
coeff[0] = b0 + b2; // 16 bit, [-32640, 32640]
coeff[64] = b1 + b3;
coeff[128] = b0 - b2;
coeff[192] = b1 - b3;
++coeff;
}
}
void aom_hadamard_32x32_c(const int16_t *src_diff, ptrdiff_t src_stride,
tran_low_t *coeff) {
int idx;
for (idx = 0; idx < 4; ++idx) {
// src_diff: 9 bit, dynamic range [-255, 255]
const int16_t *src_ptr =
src_diff + (idx >> 1) * 16 * src_stride + (idx & 0x01) * 16;
aom_hadamard_16x16_c(src_ptr, src_stride, coeff + idx * 256);
}
// coeff: 15 bit, dynamic range [-16320, 16320]
for (idx = 0; idx < 256; ++idx) {
tran_low_t a0 = coeff[0];
tran_low_t a1 = coeff[256];
tran_low_t a2 = coeff[512];
tran_low_t a3 = coeff[768];
tran_low_t b0 = (a0 + a1) >> 2; // (a0 + a1): 16 bit, [-32640, 32640]
tran_low_t b1 = (a0 - a1) >> 2; // b0-b3: 15 bit, dynamic range
tran_low_t b2 = (a2 + a3) >> 2; // [-16320, 16320]
tran_low_t b3 = (a2 - a3) >> 2;
coeff[0] = b0 + b2; // 16 bit, [-32640, 32640]
coeff[256] = b1 + b3;
coeff[512] = b0 - b2;
coeff[768] = b1 - b3;
++coeff;
}
}
// coeff: 16 bits, dynamic range [-32640, 32640].
// length: value range {16, 64, 256, 1024}.
int aom_satd_c(const tran_low_t *coeff, int length) {
int i;
int satd = 0;
for (i = 0; i < length; ++i) satd += abs(coeff[i]);
// satd: 26 bits, dynamic range [-32640 * 1024, 32640 * 1024]
return satd;
}

69
third_party/aom/aom_dsp/binary_codes_reader.c vendored
View file

@ -10,29 +10,9 @@
*/
#include "aom_dsp/binary_codes_reader.h"
#include "aom_dsp/recenter.h"
#include "av1/common/common.h"
// Inverse recenters a non-negative literal v around a reference r
static uint16_t inv_recenter_nonneg(uint16_t r, uint16_t v) {
if (v > (r << 1))
return v;
else if ((v & 1) == 0)
return (v >> 1) + r;
else
return r - ((v + 1) >> 1);
}
// Inverse recenters a non-negative literal v in [0, n-1] around a
// reference r also in [0, n-1]
static uint16_t inv_recenter_finite_nonneg(uint16_t n, uint16_t r, uint16_t v) {
if ((r << 1) <= n) {
return inv_recenter_nonneg(r, v);
} else {
return n - 1 - inv_recenter_nonneg(n - 1 - r, v);
}
}
uint16_t aom_read_primitive_quniform_(aom_reader *r,
uint16_t n ACCT_STR_PARAM) {
if (n <= 1) return 0;
@ -42,15 +22,6 @@ uint16_t aom_read_primitive_quniform_(aom_reader *r,
return v < m ? v : (v << 1) - m + aom_read_bit(r, ACCT_STR_NAME);
}
static uint16_t aom_rb_read_primitive_quniform(struct aom_read_bit_buffer *rb,
uint16_t n) {
if (n <= 1) return 0;
const int l = get_msb(n) + 1;
const int m = (1 << l) - n;
const int v = aom_rb_read_literal(rb, l - 1);
return v < m ? v : (v << 1) - m + aom_rb_read_bit(rb);
}
// Decode finite subexponential code that for a symbol v in [0, n-1] with
// parameter k
uint16_t aom_read_primitive_subexpfin_(aom_reader *r, uint16_t n,
@ -78,46 +49,8 @@ uint16_t aom_read_primitive_subexpfin_(aom_reader *r, uint16_t n,
return 0;
}
static uint16_t aom_rb_read_primitive_subexpfin(struct aom_read_bit_buffer *rb,
uint16_t n, uint16_t k) {
int i = 0;
int mk = 0;
while (1) {
int b = (i ? k + i - 1 : k);
int a = (1 << b);
if (n <= mk + 3 * a) {
return aom_rb_read_primitive_quniform(rb, n - mk) + mk;
}
if (!aom_rb_read_bit(rb)) {
return aom_rb_read_literal(rb, b) + mk;
}
i = i + 1;
mk += a;
}
assert(0);
return 0;
}
uint16_t aom_read_primitive_refsubexpfin_(aom_reader *r, uint16_t n, uint16_t k,
uint16_t ref ACCT_STR_PARAM) {
return inv_recenter_finite_nonneg(
n, ref, aom_read_primitive_subexpfin(r, n, k, ACCT_STR_NAME));
}
static uint16_t aom_rb_read_primitive_refsubexpfin(
struct aom_read_bit_buffer *rb, uint16_t n, uint16_t k, uint16_t ref) {
return inv_recenter_finite_nonneg(n, ref,
aom_rb_read_primitive_subexpfin(rb, n, k));
}
int16_t aom_rb_read_signed_primitive_refsubexpfin(
struct aom_read_bit_buffer *rb, uint16_t n, uint16_t k, int16_t ref) {
ref += n - 1;
const uint16_t scaled_n = (n << 1) - 1;
return aom_rb_read_primitive_refsubexpfin(rb, scaled_n, k, ref) - n + 1;
}

3
third_party/aom/aom_dsp/binary_codes_reader.h vendored
View file

@ -37,9 +37,6 @@ uint16_t aom_read_primitive_subexpfin_(aom_reader *r, uint16_t n,
uint16_t aom_read_primitive_refsubexpfin_(aom_reader *r, uint16_t n, uint16_t k,
uint16_t ref ACCT_STR_PARAM);
int16_t aom_rb_read_signed_primitive_refsubexpfin(
struct aom_read_bit_buffer *rb, uint16_t n, uint16_t k, int16_t ref);
#ifdef __cplusplus
} // extern "C"
#endif

76
third_party/aom/aom_dsp/binary_codes_writer.c vendored
View file

@ -11,29 +11,10 @@
#include "aom_dsp/bitwriter.h"
#include "aom_dsp/binary_codes_writer.h"
#include "aom_dsp/recenter.h"
#include "aom_ports/bitops.h"
#include "av1/common/common.h"
// Recenters a non-negative literal v around a reference r
static uint16_t recenter_nonneg(uint16_t r, uint16_t v) {
if (v > (r << 1))
return v;
else if (v >= r)
return ((v - r) << 1);
else
return ((r - v) << 1) - 1;
}
// Recenters a non-negative literal v in [0, n-1] around a
// reference r also in [0, n-1]
static uint16_t recenter_finite_nonneg(uint16_t n, uint16_t r, uint16_t v) {
if ((r << 1) <= n) {
return recenter_nonneg(r, v);
} else {
return recenter_nonneg(n - 1 - r, n - 1 - v);
}
}
// Codes a symbol v in [-2^mag_bits, 2^mag_bits].
// mag_bits is number of bits for magnitude. The alphabet is of size
// 2 * 2^mag_bits + 1, symmetric around 0, where one bit is used to
@ -69,19 +50,6 @@ void aom_write_primitive_quniform(aom_writer *w, uint16_t n, uint16_t v) {
}
}
static void aom_wb_write_primitive_quniform(struct aom_write_bit_buffer *wb,
uint16_t n, uint16_t v) {
if (n <= 1) return;
const int l = get_msb(n) + 1;
const int m = (1 << l) - n;
if (v < m) {
aom_wb_write_literal(wb, v, l - 1);
} else {
aom_wb_write_literal(wb, m + ((v - m) >> 1), l - 1);
aom_wb_write_bit(wb, (v - m) & 1);
}
}
int aom_count_primitive_quniform(uint16_t n, uint16_t v) {
if (n <= 1) return 0;
const int l = get_msb(n) + 1;
@ -114,31 +82,6 @@ void aom_write_primitive_subexpfin(aom_writer *w, uint16_t n, uint16_t k,
}
}
static void aom_wb_write_primitive_subexpfin(struct aom_write_bit_buffer *wb,
uint16_t n, uint16_t k,
uint16_t v) {
int i = 0;
int mk = 0;
while (1) {
int b = (i ? k + i - 1 : k);
int a = (1 << b);
if (n <= mk + 3 * a) {
aom_wb_write_primitive_quniform(wb, n - mk, v - mk);
break;
} else {
int t = (v >= mk + a);
aom_wb_write_bit(wb, t);
if (t) {
i = i + 1;
mk += a;
} else {
aom_wb_write_literal(wb, v - mk, b);
break;
}
}
}
}
int aom_count_primitive_subexpfin(uint16_t n, uint16_t k, uint16_t v) {
int count = 0;
int i = 0;
@ -172,12 +115,6 @@ void aom_write_primitive_refsubexpfin(aom_writer *w, uint16_t n, uint16_t k,
aom_write_primitive_subexpfin(w, n, k, recenter_finite_nonneg(n, ref, v));
}
static void aom_wb_write_primitive_refsubexpfin(struct aom_write_bit_buffer *wb,
uint16_t n, uint16_t k,
uint16_t ref, uint16_t v) {
aom_wb_write_primitive_subexpfin(wb, n, k, recenter_finite_nonneg(n, ref, v));
}
void aom_write_signed_primitive_refsubexpfin(aom_writer *w, uint16_t n,
uint16_t k, int16_t ref,
int16_t v) {
@ -187,15 +124,6 @@ void aom_write_signed_primitive_refsubexpfin(aom_writer *w, uint16_t n,
aom_write_primitive_refsubexpfin(w, scaled_n, k, ref, v);
}
void aom_wb_write_signed_primitive_refsubexpfin(struct aom_write_bit_buffer *wb,
uint16_t n, uint16_t k,
int16_t ref, int16_t v) {
ref += n - 1;
v += n - 1;
const uint16_t scaled_n = (n << 1) - 1;
aom_wb_write_primitive_refsubexpfin(wb, scaled_n, k, ref, v);
}
int aom_count_primitive_refsubexpfin(uint16_t n, uint16_t k, uint16_t ref,
uint16_t v) {
return aom_count_primitive_subexpfin(n, k, recenter_finite_nonneg(n, ref, v));

5
third_party/aom/aom_dsp/binary_codes_writer.h vendored
View file

@ -49,10 +49,6 @@ void aom_write_signed_primitive_refsubexpfin(aom_writer *w, uint16_t n,
uint16_t k, int16_t ref,
int16_t v);
void aom_wb_write_signed_primitive_refsubexpfin(struct aom_write_bit_buffer *wb,
uint16_t n, uint16_t k,
int16_t ref, int16_t v);
// Functions that counts bits for the above primitives
int aom_count_primitive_symmetric(int16_t v, unsigned int mag_bits);
int aom_count_primitive_quniform(uint16_t n, uint16_t v);
@ -61,6 +57,7 @@ int aom_count_primitive_refsubexpfin(uint16_t n, uint16_t k, uint16_t ref,
uint16_t v);
int aom_count_signed_primitive_refsubexpfin(uint16_t n, uint16_t k, int16_t ref,
int16_t v);
#ifdef __cplusplus
} // extern "C"
#endif

4
third_party/aom/aom_dsp/bitreader.h vendored
View file

@ -65,10 +65,6 @@ static INLINE const uint8_t *aom_reader_find_end(aom_reader *r) {
return aom_daala_reader_find_end(r);
}
static INLINE int aom_reader_has_error(aom_reader *r) {
return aom_daala_reader_has_error(r);
}
// Returns true if the bit reader has tried to decode more data from the buffer
// than was actually provided.
static INLINE int aom_reader_has_overflowed(const aom_reader *r) {

49
third_party/aom/aom_dsp/bitreader_buffer.c vendored
View file

@ -14,6 +14,8 @@
#include "config/aom_config.h"
#include "aom_dsp/bitreader_buffer.h"
#include "aom_dsp/recenter.h"
#include "aom_ports/bitops.h"
size_t aom_rb_bytes_read(const struct aom_read_bit_buffer *rb) {
return (rb->bit_offset + 7) >> 3;
@ -65,3 +67,50 @@ uint32_t aom_rb_read_uvlc(struct aom_read_bit_buffer *rb) {
const uint32_t value = aom_rb_read_literal(rb, leading_zeros);
return base + value;
}
static uint16_t aom_rb_read_primitive_quniform(struct aom_read_bit_buffer *rb,
uint16_t n) {
if (n <= 1) return 0;
const int l = get_msb(n) + 1;
const int m = (1 << l) - n;
const int v = aom_rb_read_literal(rb, l - 1);
return v < m ? v : (v << 1) - m + aom_rb_read_bit(rb);
}
static uint16_t aom_rb_read_primitive_subexpfin(struct aom_read_bit_buffer *rb,
uint16_t n, uint16_t k) {
int i = 0;
int mk = 0;
while (1) {
int b = (i ? k + i - 1 : k);
int a = (1 << b);
if (n <= mk + 3 * a) {
return aom_rb_read_primitive_quniform(rb, n - mk) + mk;
}
if (!aom_rb_read_bit(rb)) {
return aom_rb_read_literal(rb, b) + mk;
}
i = i + 1;
mk += a;
}
assert(0);
return 0;
}
static uint16_t aom_rb_read_primitive_refsubexpfin(
struct aom_read_bit_buffer *rb, uint16_t n, uint16_t k, uint16_t ref) {
return inv_recenter_finite_nonneg(n, ref,
aom_rb_read_primitive_subexpfin(rb, n, k));
}
int16_t aom_rb_read_signed_primitive_refsubexpfin(
struct aom_read_bit_buffer *rb, uint16_t n, uint16_t k, int16_t ref) {
ref += n - 1;
const uint16_t scaled_n = (n << 1) - 1;
return aom_rb_read_primitive_refsubexpfin(rb, scaled_n, k, ref) - n + 1;
}

3
third_party/aom/aom_dsp/bitreader_buffer.h vendored
View file

@ -43,6 +43,9 @@ int aom_rb_read_inv_signed_literal(struct aom_read_bit_buffer *rb, int bits);
uint32_t aom_rb_read_uvlc(struct aom_read_bit_buffer *rb);
int16_t aom_rb_read_signed_primitive_refsubexpfin(
struct aom_read_bit_buffer *rb, uint16_t n, uint16_t k, int16_t ref);
#ifdef __cplusplus
} // extern "C"
#endif

55
third_party/aom/aom_dsp/bitwriter_buffer.c vendored
View file

@ -16,6 +16,8 @@
#include "config/aom_config.h"
#include "aom_dsp/bitwriter_buffer.h"
#include "aom_dsp/recenter.h"
#include "aom_ports/bitops.h"
int aom_wb_is_byte_aligned(const struct aom_write_bit_buffer *wb) {
return (wb->bit_offset % CHAR_BIT == 0);
@ -85,3 +87,56 @@ void aom_wb_write_uvlc(struct aom_write_bit_buffer *wb, uint32_t v) {
aom_wb_write_literal(wb, 0, leading_zeroes >> 1);
aom_wb_write_unsigned_literal(wb, v, (leading_zeroes + 1) >> 1);
}
static void aom_wb_write_primitive_quniform(struct aom_write_bit_buffer *wb,
uint16_t n, uint16_t v) {
if (n <= 1) return;
const int l = get_msb(n) + 1;
const int m = (1 << l) - n;
if (v < m) {
aom_wb_write_literal(wb, v, l - 1);
} else {
aom_wb_write_literal(wb, m + ((v - m) >> 1), l - 1);
aom_wb_write_bit(wb, (v - m) & 1);
}
}
static void aom_wb_write_primitive_subexpfin(struct aom_write_bit_buffer *wb,
uint16_t n, uint16_t k,
uint16_t v) {
int i = 0;
int mk = 0;
while (1) {
int b = (i ? k + i - 1 : k);
int a = (1 << b);
if (n <= mk + 3 * a) {
aom_wb_write_primitive_quniform(wb, n - mk, v - mk);
break;
} else {
int t = (v >= mk + a);
aom_wb_write_bit(wb, t);
if (t) {
i = i + 1;
mk += a;
} else {
aom_wb_write_literal(wb, v - mk, b);
break;
}
}
}
}
static void aom_wb_write_primitive_refsubexpfin(struct aom_write_bit_buffer *wb,
uint16_t n, uint16_t k,
uint16_t ref, uint16_t v) {
aom_wb_write_primitive_subexpfin(wb, n, k, recenter_finite_nonneg(n, ref, v));
}
void aom_wb_write_signed_primitive_refsubexpfin(struct aom_write_bit_buffer *wb,
uint16_t n, uint16_t k,
int16_t ref, int16_t v) {
ref += n - 1;
v += n - 1;
const uint16_t scaled_n = (n << 1) - 1;
aom_wb_write_primitive_refsubexpfin(wb, scaled_n, k, ref, v);
}

4
third_party/aom/aom_dsp/bitwriter_buffer.h vendored
View file

@ -44,6 +44,10 @@ void aom_wb_write_inv_signed_literal(struct aom_write_bit_buffer *wb, int data,
void aom_wb_write_uvlc(struct aom_write_bit_buffer *wb, uint32_t v);
void aom_wb_write_signed_primitive_refsubexpfin(struct aom_write_bit_buffer *wb,
uint16_t n, uint16_t k,
int16_t ref, int16_t v);
#ifdef __cplusplus
} // extern "C"
#endif

4
third_party/aom/aom_dsp/blend_a64_mask.c vendored
View file

@ -22,7 +22,7 @@
// as described for AOM_BLEND_A64 in aom_dsp/blend.h. src0 or src1 can
// be the same as dst, or dst can be different from both sources.
// NOTE(david.barker): The input and output of aom_blend_a64_d32_mask_c() are
// NOTE(david.barker): The input and output of aom_blend_a64_d16_mask_c() are
// in a higher intermediate precision, and will later be rounded down to pixel
// precision.
// Thus, in order to avoid double-rounding, we want to use normal right shifts
@ -30,7 +30,7 @@
// This works because of the identity:
// ROUND_POWER_OF_TWO(x >> y, z) == ROUND_POWER_OF_TWO(x, y+z)
//
// In contrast, the output of the non-d32 functions will not be further rounded,
// In contrast, the output of the non-d16 functions will not be further rounded,
// so we *should* use ROUND_POWER_OF_TWO there.
void aom_lowbd_blend_a64_d16_mask_c(

4
third_party/aom/aom_dsp/daalaboolreader.h vendored
View file

@ -101,10 +101,6 @@ static INLINE int aom_daala_read(daala_reader *r, int prob) {
return bit;
}
static INLINE int aom_daala_reader_has_error(daala_reader *r) {
return r->ec.error;
}
static INLINE int daala_read_symbol(daala_reader *r, const aom_cdf_prob *cdf,
int nsymbs) {
int symb;

1
third_party/aom/aom_dsp/entcode.h vendored
View file

@ -24,6 +24,7 @@
on a larger type, you can speed up the decoder by using it here.*/
typedef uint32_t od_ec_window;
/*The size in bits of od_ec_window.*/
#define OD_EC_WINDOW_SIZE ((int)sizeof(od_ec_window) * CHAR_BIT)
/*The resolution of fractional-precision bit usage measurements, i.e.,

24
third_party/aom/aom_dsp/entdec.c vendored
View file

@ -87,11 +87,25 @@ static void od_ec_dec_refill(od_ec_dec *dec) {
end = dec->end;
s = OD_EC_WINDOW_SIZE - 9 - (cnt + 15);
for (; s >= 0 && bptr < end; s -= 8, bptr++) {
/*Each time a byte is inserted into the window (dif), bptr advances and cnt
is incremented by 8, so the total number of consumed bits (the return
value of od_ec_dec_tell) does not change.*/
assert(s <= OD_EC_WINDOW_SIZE - 8);
dif ^= (od_ec_window)bptr[0] << s;
cnt += 8;
}
if (bptr >= end) {
/*We've reached the end of the buffer. It is perfectly valid for us to need
to fill the window with additional bits past the end of the buffer (and
this happens in normal operation). These bits should all just be taken
as zero. But we cannot increment bptr past 'end' (this is undefined
behavior), so we start to increment dec->tell_offs. We also don't want
to keep testing bptr against 'end', so we set cnt to OD_EC_LOTS_OF_BITS
and adjust dec->tell_offs so that the total number of unconsumed bits in
the window (dec->cnt - dec->tell_offs) does not change. This effectively
puts lots of zero bits into the window, and means we won't try to refill
it from the buffer for a very long time (at which point we'll put lots
of zero bits into the window again).*/
dec->tell_offs += OD_EC_LOTS_OF_BITS - cnt;
cnt = OD_EC_LOTS_OF_BITS;
}
@ -112,8 +126,9 @@ static int od_ec_dec_normalize(od_ec_dec *dec, od_ec_window dif, unsigned rng,
int ret) {
int d;
assert(rng <= 65535U);
// The number of leading zeros in the 16-bit binary representation of rng.
/*The number of leading zeros in the 16-bit binary representation of rng.*/
d = 16 - OD_ILOG_NZ(rng);
/*d bits in dec->dif are consumed.*/
dec->cnt -= d;
/*This is equivalent to shifting in 1's instead of 0's.*/
dec->dif = ((dif + 1) << d) - 1;
@ -124,7 +139,7 @@ static int od_ec_dec_normalize(od_ec_dec *dec, od_ec_window dif, unsigned rng,
/*Initializes the decoder.
buf: The input buffer to use.
Return: 0 on success, or a negative value on error.*/
storage: The size in bytes of the input buffer.*/
void od_ec_dec_init(od_ec_dec *dec, const unsigned char *buf,
uint32_t storage) {
dec->buf = buf;
@ -134,7 +149,6 @@ void od_ec_dec_init(od_ec_dec *dec, const unsigned char *buf,
dec->dif = ((od_ec_window)1 << (OD_EC_WINDOW_SIZE - 1)) - 1;
dec->rng = 0x8000;
dec->cnt = -15;
dec->error = 0;
od_ec_dec_refill(dec);
}
@ -215,6 +229,10 @@ int od_ec_decode_cdf_q15(od_ec_dec *dec, const uint16_t *icdf, int nsyms) {
This will always be slightly larger than the exact value (e.g., all
rounding error is in the positive direction).*/
int od_ec_dec_tell(const od_ec_dec *dec) {
/*There is a window of bits stored in dec->dif. The difference
(dec->bptr - dec->buf) tells us how many bytes have been read into this
window. The difference (dec->cnt - dec->tell_offs) tells us how many of
the bits in that window remain unconsumed.*/
return (int)((dec->bptr - dec->buf) * 8 - dec->cnt + dec->tell_offs);
}

4
third_party/aom/aom_dsp/entdec.h vendored
View file

@ -34,7 +34,7 @@ struct od_ec_dec {
const unsigned char *buf;
/*An offset used to keep track of tell after reaching the end of the stream.
This is constant throughout most of the decoding process, but becomes
important once we hit the end of the buffer and stop incrementing pointers
important once we hit the end of the buffer and stop incrementing bptr
(and instead pretend cnt has lots of bits).*/
int32_t tell_offs;
/*The end of the current input buffer.*/
@ -53,8 +53,6 @@ struct od_ec_dec {
uint16_t rng;
/*The number of bits of data in the current value.*/
int16_t cnt;
/*Nonzero if an error occurred.*/
int error;
};
/*See entdec.c for further documentation.*/

2
third_party/aom/aom_dsp/entenc.c vendored
View file

@ -60,7 +60,7 @@ static void od_ec_enc_normalize(od_ec_enc *enc, od_ec_window low,
int s;
c = enc->cnt;
assert(rng <= 65535U);
// The number of leading zeros in the 16-bit binary representation of rng.
/*The number of leading zeros in the 16-bit binary representation of rng.*/
d = 16 - OD_ILOG_NZ(rng);
s = c + d;
/*TODO: Right now we flush every time we have at least one byte available.

70
third_party/aom/aom_dsp/grain_synthesis.h vendored
View file

@ -20,6 +20,8 @@
extern "C" {
#endif
#include <string.h>
#include "aom_dsp/aom_dsp_common.h"
#include "aom/aom_image.h"
@ -28,6 +30,9 @@ extern "C" {
* This structure contains input parameters for film grain synthesis
*/
typedef struct {
// This structure is compared element-by-element in the function
// av1_check_grain_params_equiv: this function must be updated if any changes
// are made to this structure.
int apply_grain;
int update_parameters;
@ -79,8 +84,73 @@ typedef struct {
int grain_scale_shift;
uint16_t random_seed;
// This structure is compared element-by-element in the function
// av1_check_grain_params_equiv: this function must be updated if any changes
// are made to this structure.
} aom_film_grain_t;
/*!\brief Check if two film grain parameters structs are equivalent
*
* Check if two film grain parameters are equal, except for the
* update_parameters and random_seed elements which are ignored.
*
* \param[in] pa The first set of parameters to compare
* \param[in] pb The second set of parameters to compare
* \return Returns 1 if the params are equivalent, 0 otherwise
*/
static INLINE int av1_check_grain_params_equiv(
const aom_film_grain_t *const pa, const aom_film_grain_t *const pb) {
if (pa->apply_grain != pb->apply_grain) return 0;
// Don't compare update_parameters
if (pa->num_y_points != pb->num_y_points) return 0;
if (memcmp(pa->scaling_points_y, pb->scaling_points_y,
pa->num_y_points * 2 * sizeof(*pa->scaling_points_y)) != 0)
return 0;
if (pa->num_cb_points != pb->num_cb_points) return 0;
if (memcmp(pa->scaling_points_cb, pb->scaling_points_cb,
pa->num_cb_points * 2 * sizeof(*pa->scaling_points_cb)) != 0)
return 0;
if (pa->num_cr_points != pb->num_cr_points) return 0;
if (memcmp(pa->scaling_points_cr, pb->scaling_points_cr,
pa->num_cr_points * 2 * sizeof(*pa->scaling_points_cr)) != 0)
return 0;
if (pa->scaling_shift != pb->scaling_shift) return 0;
if (pa->ar_coeff_lag != pb->ar_coeff_lag) return 0;
const int num_pos = 2 * pa->ar_coeff_lag * (pa->ar_coeff_lag + 1);
if (memcmp(pa->ar_coeffs_y, pb->ar_coeffs_y,
num_pos * sizeof(*pa->ar_coeffs_y)) != 0)
return 0;
if (memcmp(pa->ar_coeffs_cb, pb->ar_coeffs_cb,
num_pos * sizeof(*pa->ar_coeffs_cb)) != 0)
return 0;
if (memcmp(pa->ar_coeffs_cr, pb->ar_coeffs_cr,
num_pos * sizeof(*pa->ar_coeffs_cr)) != 0)
return 0;
if (pa->ar_coeff_shift != pb->ar_coeff_shift) return 0;
if (pa->cb_mult != pb->cb_mult) return 0;
if (pa->cb_luma_mult != pb->cb_luma_mult) return 0;
if (pa->cb_offset != pb->cb_offset) return 0;
if (pa->cr_mult != pb->cr_mult) return 0;
if (pa->cr_luma_mult != pb->cr_luma_mult) return 0;
if (pa->cr_offset != pb->cr_offset) return 0;
if (pa->overlap_flag != pb->overlap_flag) return 0;
if (pa->clip_to_restricted_range != pb->clip_to_restricted_range) return 0;
if (pa->bit_depth != pb->bit_depth) return 0;
if (pa->chroma_scaling_from_luma != pb->chroma_scaling_from_luma) return 0;
if (pa->grain_scale_shift != pb->grain_scale_shift) return 0;
return 1;
}
/*!\brief Add film grain
*
* Add film grain to an image

3
third_party/aom/aom_dsp/grain_table.c vendored
View file

@ -134,7 +134,8 @@ static void grain_table_entry_read(FILE *file,
}
}
void grain_table_entry_write(FILE *file, aom_film_grain_table_entry_t *entry) {
static void grain_table_entry_write(FILE *file,
aom_film_grain_table_entry_t *entry) {
const aom_film_grain_t *pars = &entry->params;
fprintf(file, "E %" PRId64 " %" PRId64 " %d %d %d\n", entry->start_time,
entry->end_time, pars->apply_grain, pars->random_seed,

32
third_party/aom/aom_dsp/intrapred.c vendored
View file

@ -239,23 +239,23 @@ static INLINE int divide_using_multiply_shift(int num, int shift1,
return interm * multiplier >> shift2;
}
// The constants (multiplier and shifts) for a given block size are obtained
// as follows:
// - Let sum_w_h = block width + block height.
// - Shift 'sum_w_h' right until we reach an odd number. Let the number of
// shifts for that block size be called 'shift1' (see the parameter in
// dc_predictor_rect() function), and let the odd number be 'd'. [d has only 2
// possible values: d = 3 for a 1:2 rect block and d = 5 for a 1:4 rect
// block].
// - Find multipliers for (i) dividing by 3, and (ii) dividing by 5,
// using the "Algorithm 1" in:
// http://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=1467632
// by ensuring that m + n = 16 (in that algorithm). This ensures that our 2nd
// shift will be 16, regardless of the block size.
// The constants (multiplier and shifts) for a given block size are obtained
// as follows:
// - Let sum_w_h = block width + block height.
// - Shift 'sum_w_h' right until we reach an odd number. Let the number of
// shifts for that block size be called 'shift1' (see the parameter in
// dc_predictor_rect() function), and let the odd number be 'd'. [d has only 2
// possible values: d = 3 for a 1:2 rect block and d = 5 for a 1:4 rect
// block].
// - Find multipliers for (i) dividing by 3, and (ii) dividing by 5,
// using the "Algorithm 1" in:
// http://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=1467632
// by ensuring that m + n = 16 (in that algorithm). This ensures that our 2nd
// shift will be 16, regardless of the block size.
// Note: For low bitdepth, assembly code may be optimized by using smaller
// constants for smaller block sizes, where the range of the 'sum' is
// restricted to fewer bits.
// Note: For low bitdepth, assembly code may be optimized by using smaller
// constants for smaller block sizes, where the range of the 'sum' is
// restricted to fewer bits.
#define DC_MULTIPLIER_1X2 0x5556
#define DC_MULTIPLIER_1X4 0x3334

8
third_party/aom/aom_dsp/noise_model.h vendored
View file

@ -158,10 +158,10 @@ int aom_flat_block_finder_run(const aom_flat_block_finder_t *block_finder,
int stride, uint8_t *flat_blocks);
// The noise shape indicates the allowed coefficients in the AR model.
typedef enum {
enum {
AOM_NOISE_SHAPE_DIAMOND = 0,
AOM_NOISE_SHAPE_SQUARE = 1
} aom_noise_shape;
} UENUM1BYTE(aom_noise_shape);
// The parameters of the noise model include the shape type, lag, the
// bit depth of the input images provided, and whether the input images
@ -202,13 +202,13 @@ typedef struct {
} aom_noise_model_t;
/*!\brief Result of a noise model update. */
typedef enum {
enum {
AOM_NOISE_STATUS_OK = 0,
AOM_NOISE_STATUS_INVALID_ARGUMENT,
AOM_NOISE_STATUS_INSUFFICIENT_FLAT_BLOCKS,
AOM_NOISE_STATUS_DIFFERENT_NOISE_TYPE,
AOM_NOISE_STATUS_INTERNAL_ERROR,
} aom_noise_status_t;
} UENUM1BYTE(aom_noise_status_t);
/*!\brief Initializes a noise model with the given parameters.
*

2
third_party/aom/aom_dsp/prob.h vendored
View file

@ -641,7 +641,7 @@ static INLINE uint8_t get_prob(unsigned int num, unsigned int den) {
}
}
static INLINE void update_cdf(aom_cdf_prob *cdf, int val, int nsymbs) {
static INLINE void update_cdf(aom_cdf_prob *cdf, int8_t val, int nsymbs) {
int rate;
int i, tmp;

194
third_party/aom/aom_dsp/quantize.c vendored
View file

@ -12,6 +12,68 @@
#include "aom_dsp/quantize.h"
#include "aom_mem/aom_mem.h"
void quantize_b_adaptive_helper_c(
const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr,
const int16_t *round_ptr, const int16_t *quant_ptr,
const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr,
tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr,
const int16_t *scan, const int16_t *iscan, const qm_val_t *qm_ptr,
const qm_val_t *iqm_ptr, const int log_scale) {
const int zbins[2] = { ROUND_POWER_OF_TWO(zbin_ptr[0], log_scale),
ROUND_POWER_OF_TWO(zbin_ptr[1], log_scale) };
const int nzbins[2] = { zbins[0] * -1, zbins[1] * -1 };
int i, non_zero_count = (int)n_coeffs, eob = -1;
(void)iscan;
memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr));
memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr));
// Pre-scan pass
for (i = (int)n_coeffs - 1; i >= 0; i--) {
const int rc = scan[i];
const qm_val_t wt = qm_ptr != NULL ? qm_ptr[rc] : (1 << AOM_QM_BITS);
const int coeff = coeff_ptr[rc] * wt;
int prescan_add = ROUND_POWER_OF_TWO(dequant_ptr[rc != 0] * 325, 7);
if (coeff < (zbins[rc != 0] * (1 << AOM_QM_BITS) + prescan_add) &&
coeff > (nzbins[rc != 0] * (1 << AOM_QM_BITS) - prescan_add))
non_zero_count--;
else
break;
}
// Quantization pass: All coefficients with index >= zero_flag are
// skippable. Note: zero_flag can be zero.
for (i = 0; i < non_zero_count; i++) {
const int rc = scan[i];
const int coeff = coeff_ptr[rc];
const int coeff_sign = (coeff >> 31);
const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign;
int tmp32;
const qm_val_t wt = qm_ptr != NULL ? qm_ptr[rc] : (1 << AOM_QM_BITS);
if (abs_coeff * wt >= (zbins[rc != 0] << AOM_QM_BITS)) {
int64_t tmp =
clamp(abs_coeff + ROUND_POWER_OF_TWO(round_ptr[rc != 0], log_scale),
INT16_MIN, INT16_MAX);
tmp *= wt;
tmp32 = (int)(((((tmp * quant_ptr[rc != 0]) >> 16) + tmp) *
quant_shift_ptr[rc != 0]) >>
(16 - log_scale + AOM_QM_BITS)); // quantization
qcoeff_ptr[rc] = (tmp32 ^ coeff_sign) - coeff_sign;
const int iwt = iqm_ptr != NULL ? iqm_ptr[rc] : (1 << AOM_QM_BITS);
const int dequant =
(dequant_ptr[rc != 0] * iwt + (1 << (AOM_QM_BITS - 1))) >>
AOM_QM_BITS;
const tran_low_t abs_dqcoeff = (tmp32 * dequant) >> log_scale;
dqcoeff_ptr[rc] = (tran_low_t)((abs_dqcoeff ^ coeff_sign) - coeff_sign);
if (tmp32) eob = i;
}
}
*eob_ptr = eob + 1;
}
void quantize_b_helper_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
const int16_t *zbin_ptr, const int16_t *round_ptr,
const int16_t *quant_ptr,
@ -74,6 +136,64 @@ void quantize_b_helper_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
*eob_ptr = eob + 1;
}
void highbd_quantize_b_adaptive_helper_c(
const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr,
const int16_t *round_ptr, const int16_t *quant_ptr,
const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr,
tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr,
const int16_t *scan, const int16_t *iscan, const qm_val_t *qm_ptr,
const qm_val_t *iqm_ptr, const int log_scale) {
int i, eob = -1;
const int zbins[2] = { ROUND_POWER_OF_TWO(zbin_ptr[0], log_scale),
ROUND_POWER_OF_TWO(zbin_ptr[1], log_scale) };
const int nzbins[2] = { zbins[0] * -1, zbins[1] * -1 };
int dequant;
int idx_arr[4096];
(void)iscan;
int idx = 0;
memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr));
memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr));
// Pre-scan pass
for (i = 0; i < n_coeffs; i++) {
const int rc = scan[i];
const qm_val_t wt = qm_ptr != NULL ? qm_ptr[rc] : (1 << AOM_QM_BITS);
const int coeff = coeff_ptr[rc] * wt;
// If the coefficient is out of the base ZBIN range, keep it for
// quantization.
int prescan_add = ROUND_POWER_OF_TWO(dequant_ptr[rc != 0] * 325, 7);
if (coeff >= (zbins[rc != 0] * (1 << AOM_QM_BITS) + prescan_add) &&
coeff <= (nzbins[rc != 0] * (1 << AOM_QM_BITS) - prescan_add))
idx_arr[idx++] = i;
}
// Quantization pass: only process the coefficients selected in
// pre-scan pass. Note: idx can be zero.
for (i = 0; i < idx; i++) {
const int rc = scan[idx_arr[i]];
const int coeff = coeff_ptr[rc];
const int coeff_sign = (coeff >> 31);
const qm_val_t wt = qm_ptr != NULL ? qm_ptr[rc] : (1 << AOM_QM_BITS);
const qm_val_t iwt = iqm_ptr != NULL ? iqm_ptr[rc] : (1 << AOM_QM_BITS);
const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign;
const int64_t tmp1 =
abs_coeff + ROUND_POWER_OF_TWO(round_ptr[rc != 0], log_scale);
const int64_t tmpw = tmp1 * wt;
const int64_t tmp2 = ((tmpw * quant_ptr[rc != 0]) >> 16) + tmpw;
const int abs_qcoeff = (int)((tmp2 * quant_shift_ptr[rc != 0]) >>
(16 - log_scale + AOM_QM_BITS));
qcoeff_ptr[rc] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign);
dequant =
(dequant_ptr[rc != 0] * iwt + (1 << (AOM_QM_BITS - 1))) >> AOM_QM_BITS;
const tran_low_t abs_dqcoeff = (abs_qcoeff * dequant) >> log_scale;
dqcoeff_ptr[rc] = (tran_low_t)((abs_dqcoeff ^ coeff_sign) - coeff_sign);
if (abs_qcoeff) eob = idx_arr[i];
}
*eob_ptr = eob + 1;
}
void highbd_quantize_b_helper_c(
const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr,
const int16_t *round_ptr, const int16_t *quant_ptr,
@ -133,6 +253,80 @@ void highbd_quantize_b_helper_c(
/* These functions should only be called when quantisation matrices
are not used. */
void aom_quantize_b_adaptive_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
const int16_t *zbin_ptr,
const int16_t *round_ptr,
const int16_t *quant_ptr,
const int16_t *quant_shift_ptr,
tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr,
const int16_t *dequant_ptr, uint16_t *eob_ptr,
const int16_t *scan, const int16_t *iscan) {
quantize_b_adaptive_helper_c(coeff_ptr, n_coeffs, zbin_ptr, round_ptr,
quant_ptr, quant_shift_ptr, qcoeff_ptr,
dqcoeff_ptr, dequant_ptr, eob_ptr, scan, iscan,
NULL, NULL, 0);
}
void aom_quantize_b_32x32_adaptive_c(
const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr,
const int16_t *round_ptr, const int16_t *quant_ptr,
const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr,
tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr,
const int16_t *scan, const int16_t *iscan) {
quantize_b_adaptive_helper_c(coeff_ptr, n_coeffs, zbin_ptr, round_ptr,
quant_ptr, quant_shift_ptr, qcoeff_ptr,
dqcoeff_ptr, dequant_ptr, eob_ptr, scan, iscan,
NULL, NULL, 1);
}
void aom_quantize_b_64x64_adaptive_c(
const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr,
const int16_t *round_ptr, const int16_t *quant_ptr,
const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr,
tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr,
const int16_t *scan, const int16_t *iscan) {
quantize_b_adaptive_helper_c(coeff_ptr, n_coeffs, zbin_ptr, round_ptr,
quant_ptr, quant_shift_ptr, qcoeff_ptr,
dqcoeff_ptr, dequant_ptr, eob_ptr, scan, iscan,
NULL, NULL, 2);
}
void aom_highbd_quantize_b_adaptive_c(
const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr,
const int16_t *round_ptr, const int16_t *quant_ptr,
const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr,
tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr,
const int16_t *scan, const int16_t *iscan) {
highbd_quantize_b_adaptive_helper_c(coeff_ptr, n_coeffs, zbin_ptr, round_ptr,
quant_ptr, quant_shift_ptr, qcoeff_ptr,
dqcoeff_ptr, dequant_ptr, eob_ptr, scan,
iscan, NULL, NULL, 0);
}
void aom_highbd_quantize_b_32x32_adaptive_c(
const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr,
const int16_t *round_ptr, const int16_t *quant_ptr,
const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr,
tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr,
const int16_t *scan, const int16_t *iscan) {
highbd_quantize_b_adaptive_helper_c(coeff_ptr, n_coeffs, zbin_ptr, round_ptr,
quant_ptr, quant_shift_ptr, qcoeff_ptr,
dqcoeff_ptr, dequant_ptr, eob_ptr, scan,
iscan, NULL, NULL, 1);
}
void aom_highbd_quantize_b_64x64_adaptive_c(
const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr,
const int16_t *round_ptr, const int16_t *quant_ptr,
const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr,
tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr,
const int16_t *scan, const int16_t *iscan) {
highbd_quantize_b_adaptive_helper_c(coeff_ptr, n_coeffs, zbin_ptr, round_ptr,
quant_ptr, quant_shift_ptr, qcoeff_ptr,
dqcoeff_ptr, dequant_ptr, eob_ptr, scan,
iscan, NULL, NULL, 2);
}
void aom_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
const int16_t *zbin_ptr, const int16_t *round_ptr,
const int16_t *quant_ptr, const int16_t *quant_shift_ptr,

60
third_party/aom/aom_dsp/quantize.h vendored
View file

@ -20,6 +20,66 @@
extern "C" {
#endif
void quantize_b_adaptive_helper_c(
const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr,
const int16_t *round_ptr, const int16_t *quant_ptr,
const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr,
tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr,
const int16_t *scan, const int16_t *iscan, const qm_val_t *qm_ptr,
const qm_val_t *iqm_ptr, const int log_scale);
void aom_quantize_b_adaptive_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
const int16_t *zbin_ptr,
const int16_t *round_ptr,
const int16_t *quant_ptr,
const int16_t *quant_shift_ptr,
tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr,
const int16_t *dequant_ptr, uint16_t *eob_ptr,
const int16_t *scan, const int16_t *iscan);
void aom_quantize_b_32x32_adaptive_c(
const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr,
const int16_t *round_ptr, const int16_t *quant_ptr,
const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr,
tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr,
const int16_t *scan, const int16_t *iscan);
void aom_quantize_b_64x64_adaptive_c(
const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr,
const int16_t *round_ptr, const int16_t *quant_ptr,
const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr,
tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr,
const int16_t *scan, const int16_t *iscan);
void highbd_quantize_b_adaptive_helper_c(
const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr,
const int16_t *round_ptr, const int16_t *quant_ptr,
const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr,
tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr,
const int16_t *scan, const int16_t *iscan, const qm_val_t *qm_ptr,
const qm_val_t *iqm_ptr, const int log_scale);
void aom_highbd_quantize_b_adaptive_c(
const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr,
const int16_t *round_ptr, const int16_t *quant_ptr,
const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr,
tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr,
const int16_t *scan, const int16_t *iscan);
void aom_highbd_quantize_b_32x32_adaptive_c(
const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr,
const int16_t *round_ptr, const int16_t *quant_ptr,
const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr,
tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr,
const int16_t *scan, const int16_t *iscan);
void aom_highbd_quantize_b_64x64_adaptive_c(
const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr,
const int16_t *round_ptr, const int16_t *quant_ptr,
const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr,
tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr,
const int16_t *scan, const int16_t *iscan);
void quantize_b_helper_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
const int16_t *zbin_ptr, const int16_t *round_ptr,
const int16_t *quant_ptr,

61
third_party/aom/aom_dsp/recenter.h vendored Normal file
View file

@ -0,0 +1,61 @@
/*
* Copyright (c) 2018, Alliance for Open Media. All rights reserved
*
* This source code is subject to the terms of the BSD 2 Clause License and
* the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
* was not distributed with this source code in the LICENSE file, you can
* obtain it at www.aomedia.org/license/software. If the Alliance for Open
* Media Patent License 1.0 was not distributed with this source code in the
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
*/
#ifndef AOM_AOM_DSP_RECENTER_H_
#define AOM_AOM_DSP_RECENTER_H_
#include "config/aom_config.h"
#include "aom/aom_integer.h"
// Inverse recenters a non-negative literal v around a reference r
static INLINE uint16_t inv_recenter_nonneg(uint16_t r, uint16_t v) {
if (v > (r << 1))
return v;
else if ((v & 1) == 0)
return (v >> 1) + r;
else
return r - ((v + 1) >> 1);
}
// Inverse recenters a non-negative literal v in [0, n-1] around a
// reference r also in [0, n-1]
static INLINE uint16_t inv_recenter_finite_nonneg(uint16_t n, uint16_t r,
uint16_t v) {
if ((r << 1) <= n) {
return inv_recenter_nonneg(r, v);
} else {
return n - 1 - inv_recenter_nonneg(n - 1 - r, v);
}
}
// Recenters a non-negative literal v around a reference r
static INLINE uint16_t recenter_nonneg(uint16_t r, uint16_t v) {
if (v > (r << 1))
return v;
else if (v >= r)
return ((v - r) << 1);
else
return ((r - v) << 1) - 1;
}
// Recenters a non-negative literal v in [0, n-1] around a
// reference r also in [0, n-1]
static INLINE uint16_t recenter_finite_nonneg(uint16_t n, uint16_t r,
uint16_t v) {
if ((r << 1) <= n) {
return recenter_nonneg(r, v);
} else {
return recenter_nonneg(n - 1 - r, n - 1 - v);
}
}
#endif // AOM_AOM_DSP_RECENTER_H_

219
third_party/aom/aom_dsp/sad.c vendored
View file

@ -25,7 +25,9 @@ static INLINE unsigned int sad(const uint8_t *a, int a_stride, const uint8_t *b,
unsigned int sad = 0;
for (y = 0; y < height; y++) {
for (x = 0; x < width; x++) sad += abs(a[x] - b[x]);
for (x = 0; x < width; x++) {
sad += abs(a[x] - b[x]);
}
a += a_stride;
b += b_stride;
@ -52,12 +54,12 @@ static INLINE unsigned int sad(const uint8_t *a, int a_stride, const uint8_t *b,
aom_comp_avg_pred(comp_pred, second_pred, m, n, ref, ref_stride); \
return sad(src, src_stride, comp_pred, m, m, n); \
} \
unsigned int aom_jnt_sad##m##x##n##_avg_c( \
unsigned int aom_dist_wtd_sad##m##x##n##_avg_c( \
const uint8_t *src, int src_stride, const uint8_t *ref, int ref_stride, \
const uint8_t *second_pred, const JNT_COMP_PARAMS *jcp_param) { \
const uint8_t *second_pred, const DIST_WTD_COMP_PARAMS *jcp_param) { \
uint8_t comp_pred[m * n]; \
aom_jnt_comp_avg_pred_c(comp_pred, second_pred, m, n, ref, ref_stride, \
jcp_param); \
aom_dist_wtd_comp_avg_pred_c(comp_pred, second_pred, m, n, ref, \
ref_stride, jcp_param); \
return sad(src, src_stride, comp_pred, m, m, n); \
}
@ -67,75 +69,75 @@ static INLINE unsigned int sad(const uint8_t *a, int a_stride, const uint8_t *b,
const uint8_t *const ref_array[], \
int ref_stride, uint32_t *sad_array) { \
int i; \
for (i = 0; i < 4; ++i) \
for (i = 0; i < 4; ++i) { \
sad_array[i] = \
aom_sad##m##x##n##_c(src, src_stride, ref_array[i], ref_stride); \
} \
}
/* clang-format off */
// 128x128
sadMxN(128, 128)
sadMxNx4D(128, 128)
sadMxN(128, 128);
sadMxNx4D(128, 128);
// 128x64
sadMxN(128, 64)
sadMxNx4D(128, 64)
sadMxN(128, 64);
sadMxNx4D(128, 64);
// 64x128
sadMxN(64, 128)
sadMxNx4D(64, 128)
sadMxN(64, 128);
sadMxNx4D(64, 128);
// 64x64
sadMxN(64, 64)
sadMxNx4D(64, 64)
sadMxN(64, 64);
sadMxNx4D(64, 64);
// 64x32
sadMxN(64, 32)
sadMxNx4D(64, 32)
sadMxN(64, 32);
sadMxNx4D(64, 32);
// 32x64
sadMxN(32, 64)
sadMxNx4D(32, 64)
sadMxN(32, 64);
sadMxNx4D(32, 64);
// 32x32
sadMxN(32, 32)
sadMxNx4D(32, 32)
sadMxN(32, 32);
sadMxNx4D(32, 32);
// 32x16
sadMxN(32, 16)
sadMxNx4D(32, 16)
sadMxN(32, 16);
sadMxNx4D(32, 16);
// 16x32
sadMxN(16, 32)
sadMxNx4D(16, 32)
sadMxN(16, 32);
sadMxNx4D(16, 32);
// 16x16
sadMxN(16, 16)
sadMxNx4D(16, 16)
sadMxN(16, 16);
sadMxNx4D(16, 16);
// 16x8
sadMxN(16, 8)
sadMxNx4D(16, 8)
sadMxN(16, 8);
sadMxNx4D(16, 8);
// 8x16
sadMxN(8, 16)
sadMxNx4D(8, 16)
sadMxN(8, 16);
sadMxNx4D(8, 16);
// 8x8
sadMxN(8, 8)
sadMxNx4D(8, 8)
sadMxN(8, 8);
sadMxNx4D(8, 8);
// 8x4
sadMxN(8, 4)
sadMxNx4D(8, 4)
sadMxN(8, 4);
sadMxNx4D(8, 4);
// 4x8
sadMxN(4, 8)
sadMxNx4D(4, 8)
sadMxN(4, 8);
sadMxNx4D(4, 8);
// 4x4
sadMxN(4, 4)
sadMxNx4D(4, 4)
sadMxN(4, 4);
sadMxNx4D(4, 4);
sadMxh(128);
sadMxh(64);
@ -144,30 +146,30 @@ sadMxh(16);
sadMxh(8);
sadMxh(4);
sadMxN(4, 16)
sadMxNx4D(4, 16)
sadMxN(16, 4)
sadMxNx4D(16, 4)
sadMxN(8, 32)
sadMxNx4D(8, 32)
sadMxN(32, 8)
sadMxNx4D(32, 8)
sadMxN(16, 64)
sadMxNx4D(16, 64)
sadMxN(64, 16)
sadMxNx4D(64, 16)
sadMxN(4, 16);
sadMxNx4D(4, 16);
sadMxN(16, 4);
sadMxNx4D(16, 4);
sadMxN(8, 32);
sadMxNx4D(8, 32);
sadMxN(32, 8);
sadMxNx4D(32, 8);
sadMxN(16, 64);
sadMxNx4D(16, 64);
sadMxN(64, 16);
sadMxNx4D(64, 16);
/* clang-format on */
static INLINE
unsigned int highbd_sad(const uint8_t *a8, int a_stride, const uint8_t *b8,
int b_stride, int width, int height) {
static INLINE unsigned int highbd_sad(const uint8_t *a8, int a_stride,
const uint8_t *b8, int b_stride,
int width, int height) {
int y, x;
unsigned int sad = 0;
const uint16_t *a = CONVERT_TO_SHORTPTR(a8);
const uint16_t *b = CONVERT_TO_SHORTPTR(b8);
for (y = 0; y < height; y++) {
for (x = 0; x < width; x++) sad += abs(a[x] - b[x]);
for (x = 0; x < width; x++) {
sad += abs(a[x] - b[x]);
}
a += a_stride;
b += b_stride;
@ -182,7 +184,9 @@ static INLINE unsigned int highbd_sadb(const uint8_t *a8, int a_stride,
unsigned int sad = 0;
const uint16_t *a = CONVERT_TO_SHORTPTR(a8);
for (y = 0; y < height; y++) {
for (x = 0; x < width; x++) sad += abs(a[x] - b[x]);
for (x = 0; x < width; x++) {
sad += abs(a[x] - b[x]);
}
a += a_stride;
b += b_stride;
@ -204,12 +208,13 @@ static INLINE unsigned int highbd_sadb(const uint8_t *a8, int a_stride,
ref, ref_stride); \
return highbd_sadb(src, src_stride, comp_pred, m, m, n); \
} \
unsigned int aom_highbd_jnt_sad##m##x##n##_avg_c( \
unsigned int aom_highbd_dist_wtd_sad##m##x##n##_avg_c( \
const uint8_t *src, int src_stride, const uint8_t *ref, int ref_stride, \
const uint8_t *second_pred, const JNT_COMP_PARAMS *jcp_param) { \
const uint8_t *second_pred, const DIST_WTD_COMP_PARAMS *jcp_param) { \
uint16_t comp_pred[m * n]; \
aom_highbd_jnt_comp_avg_pred(CONVERT_TO_BYTEPTR(comp_pred), second_pred, \
m, n, ref, ref_stride, jcp_param); \
aom_highbd_dist_wtd_comp_avg_pred(CONVERT_TO_BYTEPTR(comp_pred), \
second_pred, m, n, ref, ref_stride, \
jcp_param); \
return highbd_sadb(src, src_stride, comp_pred, m, m, n); \
}
@ -224,81 +229,79 @@ static INLINE unsigned int highbd_sadb(const uint8_t *a8, int a_stride,
} \
}
/* clang-format off */
// 128x128
highbd_sadMxN(128, 128)
highbd_sadMxNx4D(128, 128)
highbd_sadMxN(128, 128);
highbd_sadMxNx4D(128, 128);
// 128x64
highbd_sadMxN(128, 64)
highbd_sadMxNx4D(128, 64)
highbd_sadMxN(128, 64);
highbd_sadMxNx4D(128, 64);
// 64x128
highbd_sadMxN(64, 128)
highbd_sadMxNx4D(64, 128)
highbd_sadMxN(64, 128);
highbd_sadMxNx4D(64, 128);
// 64x64
highbd_sadMxN(64, 64)
highbd_sadMxNx4D(64, 64)
highbd_sadMxN(64, 64);
highbd_sadMxNx4D(64, 64);
// 64x32
highbd_sadMxN(64, 32)
highbd_sadMxNx4D(64, 32)
highbd_sadMxN(64, 32);
highbd_sadMxNx4D(64, 32);
// 32x64
highbd_sadMxN(32, 64)
highbd_sadMxNx4D(32, 64)
highbd_sadMxN(32, 64);
highbd_sadMxNx4D(32, 64);
// 32x32
highbd_sadMxN(32, 32)
highbd_sadMxNx4D(32, 32)
highbd_sadMxN(32, 32);
highbd_sadMxNx4D(32, 32);
// 32x16
highbd_sadMxN(32, 16)
highbd_sadMxNx4D(32, 16)
highbd_sadMxN(32, 16);
highbd_sadMxNx4D(32, 16);
// 16x32
highbd_sadMxN(16, 32)
highbd_sadMxNx4D(16, 32)
highbd_sadMxN(16, 32);
highbd_sadMxNx4D(16, 32);
// 16x16
highbd_sadMxN(16, 16)
highbd_sadMxNx4D(16, 16)
highbd_sadMxN(16, 16);
highbd_sadMxNx4D(16, 16);
// 16x8
highbd_sadMxN(16, 8)
highbd_sadMxNx4D(16, 8)
highbd_sadMxN(16, 8);
highbd_sadMxNx4D(16, 8);
// 8x16
highbd_sadMxN(8, 16)
highbd_sadMxNx4D(8, 16)
highbd_sadMxN(8, 16);
highbd_sadMxNx4D(8, 16);
// 8x8
highbd_sadMxN(8, 8)
highbd_sadMxNx4D(8, 8)
highbd_sadMxN(8, 8);
highbd_sadMxNx4D(8, 8);
// 8x4
highbd_sadMxN(8, 4)
highbd_sadMxNx4D(8, 4)
highbd_sadMxN(8, 4);
highbd_sadMxNx4D(8, 4);
// 4x8
highbd_sadMxN(4, 8)
highbd_sadMxNx4D(4, 8)
highbd_sadMxN(4, 8);
highbd_sadMxNx4D(4, 8);
// 4x4
highbd_sadMxN(4, 4)
highbd_sadMxNx4D(4, 4)
highbd_sadMxN(4, 4);
highbd_sadMxNx4D(4, 4);
highbd_sadMxN(4, 16)
highbd_sadMxNx4D(4, 16)
highbd_sadMxN(16, 4)
highbd_sadMxNx4D(16, 4)
highbd_sadMxN(8, 32)
highbd_sadMxNx4D(8, 32)
highbd_sadMxN(32, 8)
highbd_sadMxNx4D(32, 8)
highbd_sadMxN(16, 64)
highbd_sadMxNx4D(16, 64)
highbd_sadMxN(64, 16)
highbd_sadMxNx4D(64, 16)
/* clang-format on */
highbd_sadMxN(4, 16);
highbd_sadMxNx4D(4, 16);
highbd_sadMxN(16, 4);
highbd_sadMxNx4D(16, 4);
highbd_sadMxN(8, 32);
highbd_sadMxNx4D(8, 32);
highbd_sadMxN(32, 8);
highbd_sadMxNx4D(32, 8);
highbd_sadMxN(16, 64);
highbd_sadMxNx4D(16, 64);
highbd_sadMxN(64, 16);
highbd_sadMxNx4D(64, 16);

2
third_party/aom/aom_dsp/txfm_common.h vendored
View file

@ -39,7 +39,7 @@ typedef struct txfm_param {
// Constants:
// for (int i = 1; i< 32; ++i)
// printf("static const int cospi_%d_64 = %.0f;\n", i,
// round(16384 * cos(i*M_PI/64)));
// round(16384 * cos(i*PI/64)));
// Note: sin(k*Pi/64) = cos((32-k)*Pi/64)
static const tran_high_t cospi_1_64 = 16364;
static const tran_high_t cospi_2_64 = 16305;

370
third_party/aom/aom_dsp/variance.c vendored
View file

@ -164,40 +164,40 @@ void aom_var_filter_block2d_bil_second_pass_c(const uint16_t *a, uint8_t *b,
return aom_variance##W##x##H##_c(temp2, W, b, b_stride, sse); \
}
#define SUBPIX_AVG_VAR(W, H) \
uint32_t aom_sub_pixel_avg_variance##W##x##H##_c( \
const uint8_t *a, int a_stride, int xoffset, int yoffset, \
const uint8_t *b, int b_stride, uint32_t *sse, \
const uint8_t *second_pred) { \
uint16_t fdata3[(H + 1) * W]; \
uint8_t temp2[H * W]; \
DECLARE_ALIGNED(16, uint8_t, temp3[H * W]); \
\
aom_var_filter_block2d_bil_first_pass_c(a, fdata3, a_stride, 1, H + 1, W, \
bilinear_filters_2t[xoffset]); \
aom_var_filter_block2d_bil_second_pass_c(fdata3, temp2, W, W, H, W, \
bilinear_filters_2t[yoffset]); \
\
aom_comp_avg_pred(temp3, second_pred, W, H, temp2, W); \
\
return aom_variance##W##x##H##_c(temp3, W, b, b_stride, sse); \
} \
uint32_t aom_jnt_sub_pixel_avg_variance##W##x##H##_c( \
const uint8_t *a, int a_stride, int xoffset, int yoffset, \
const uint8_t *b, int b_stride, uint32_t *sse, \
const uint8_t *second_pred, const JNT_COMP_PARAMS *jcp_param) { \
uint16_t fdata3[(H + 1) * W]; \
uint8_t temp2[H * W]; \
DECLARE_ALIGNED(16, uint8_t, temp3[H * W]); \
\
aom_var_filter_block2d_bil_first_pass_c(a, fdata3, a_stride, 1, H + 1, W, \
bilinear_filters_2t[xoffset]); \
aom_var_filter_block2d_bil_second_pass_c(fdata3, temp2, W, W, H, W, \
bilinear_filters_2t[yoffset]); \
\
aom_jnt_comp_avg_pred(temp3, second_pred, W, H, temp2, W, jcp_param); \
\
return aom_variance##W##x##H(temp3, W, b, b_stride, sse); \
#define SUBPIX_AVG_VAR(W, H) \
uint32_t aom_sub_pixel_avg_variance##W##x##H##_c( \
const uint8_t *a, int a_stride, int xoffset, int yoffset, \
const uint8_t *b, int b_stride, uint32_t *sse, \
const uint8_t *second_pred) { \
uint16_t fdata3[(H + 1) * W]; \
uint8_t temp2[H * W]; \
DECLARE_ALIGNED(16, uint8_t, temp3[H * W]); \
\
aom_var_filter_block2d_bil_first_pass_c(a, fdata3, a_stride, 1, H + 1, W, \
bilinear_filters_2t[xoffset]); \
aom_var_filter_block2d_bil_second_pass_c(fdata3, temp2, W, W, H, W, \
bilinear_filters_2t[yoffset]); \
\
aom_comp_avg_pred(temp3, second_pred, W, H, temp2, W); \
\
return aom_variance##W##x##H##_c(temp3, W, b, b_stride, sse); \
} \
uint32_t aom_dist_wtd_sub_pixel_avg_variance##W##x##H##_c( \
const uint8_t *a, int a_stride, int xoffset, int yoffset, \
const uint8_t *b, int b_stride, uint32_t *sse, \
const uint8_t *second_pred, const DIST_WTD_COMP_PARAMS *jcp_param) { \
uint16_t fdata3[(H + 1) * W]; \
uint8_t temp2[H * W]; \
DECLARE_ALIGNED(16, uint8_t, temp3[H * W]); \
\
aom_var_filter_block2d_bil_first_pass_c(a, fdata3, a_stride, 1, H + 1, W, \
bilinear_filters_2t[xoffset]); \
aom_var_filter_block2d_bil_second_pass_c(fdata3, temp2, W, W, H, W, \
bilinear_filters_2t[yoffset]); \
\
aom_dist_wtd_comp_avg_pred(temp3, second_pred, W, H, temp2, W, jcp_param); \
\
return aom_variance##W##x##H(temp3, W, b, b_stride, sse); \
}
/* Identical to the variance call except it takes an additional parameter, sum,
@ -291,7 +291,7 @@ void aom_upsampled_pred_c(MACROBLOCKD *xd, const AV1_COMMON *const cm,
const int ref_num = 0;
const int is_intrabc = is_intrabc_block(mi);
const struct scale_factors *const sf =
is_intrabc ? &cm->sf_identity : &xd->block_refs[ref_num]->sf;
is_intrabc ? &cm->sf_identity : xd->block_ref_scale_factors[ref_num];
const int is_scaled = av1_is_scaled(sf);
if (is_scaled) {
@ -368,10 +368,7 @@ void aom_upsampled_pred_c(MACROBLOCKD *xd, const AV1_COMMON *const cm,
}
}
const InterpFilterParams *filter =
(subpel_search == 1)
? av1_get_4tap_interp_filter_params(EIGHTTAP_REGULAR)
: av1_get_interp_filter_params_with_block_size(EIGHTTAP_REGULAR, 8);
const InterpFilterParams *filter = av1_get_filter(subpel_search);
if (!subpel_x_q3 && !subpel_y_q3) {
for (int i = 0; i < height; i++) {
@ -427,9 +424,10 @@ void aom_comp_avg_upsampled_pred_c(MACROBLOCKD *xd, const AV1_COMMON *const cm,
}
}
void aom_jnt_comp_avg_pred_c(uint8_t *comp_pred, const uint8_t *pred, int width,
int height, const uint8_t *ref, int ref_stride,
const JNT_COMP_PARAMS *jcp_param) {
void aom_dist_wtd_comp_avg_pred_c(uint8_t *comp_pred, const uint8_t *pred,
int width, int height, const uint8_t *ref,
int ref_stride,
const DIST_WTD_COMP_PARAMS *jcp_param) {
int i, j;
const int fwd_offset = jcp_param->fwd_offset;
const int bck_offset = jcp_param->bck_offset;
@ -446,11 +444,11 @@ void aom_jnt_comp_avg_pred_c(uint8_t *comp_pred, const uint8_t *pred, int width,
}
}
void aom_jnt_comp_avg_upsampled_pred_c(
void aom_dist_wtd_comp_avg_upsampled_pred_c(
MACROBLOCKD *xd, const AV1_COMMON *const cm, int mi_row, int mi_col,
const MV *const mv, uint8_t *comp_pred, const uint8_t *pred, int width,
int height, int subpel_x_q3, int subpel_y_q3, const uint8_t *ref,
int ref_stride, const JNT_COMP_PARAMS *jcp_param, int subpel_search) {
int ref_stride, const DIST_WTD_COMP_PARAMS *jcp_param, int subpel_search) {
int i, j;
const int fwd_offset = jcp_param->fwd_offset;
const int bck_offset = jcp_param->bck_offset;
@ -691,125 +689,128 @@ void aom_highbd_var_filter_block2d_bil_second_pass(
dst, dst_stride, sse); \
}
#define HIGHBD_SUBPIX_AVG_VAR(W, H) \
uint32_t aom_highbd_8_sub_pixel_avg_variance##W##x##H##_c( \
const uint8_t *src, int src_stride, int xoffset, int yoffset, \
const uint8_t *dst, int dst_stride, uint32_t *sse, \
const uint8_t *second_pred) { \
uint16_t fdata3[(H + 1) * W]; \
uint16_t temp2[H * W]; \
DECLARE_ALIGNED(16, uint16_t, temp3[H * W]); \
\
aom_highbd_var_filter_block2d_bil_first_pass( \
src, fdata3, src_stride, 1, H + 1, W, bilinear_filters_2t[xoffset]); \
aom_highbd_var_filter_block2d_bil_second_pass( \
fdata3, temp2, W, W, H, W, bilinear_filters_2t[yoffset]); \
\
aom_highbd_comp_avg_pred_c(CONVERT_TO_BYTEPTR(temp3), second_pred, W, H, \
CONVERT_TO_BYTEPTR(temp2), W); \
\
return aom_highbd_8_variance##W##x##H##_c(CONVERT_TO_BYTEPTR(temp3), W, \
dst, dst_stride, sse); \
} \
\
uint32_t aom_highbd_10_sub_pixel_avg_variance##W##x##H##_c( \
const uint8_t *src, int src_stride, int xoffset, int yoffset, \
const uint8_t *dst, int dst_stride, uint32_t *sse, \
const uint8_t *second_pred) { \
uint16_t fdata3[(H + 1) * W]; \
uint16_t temp2[H * W]; \
DECLARE_ALIGNED(16, uint16_t, temp3[H * W]); \
\
aom_highbd_var_filter_block2d_bil_first_pass( \
src, fdata3, src_stride, 1, H + 1, W, bilinear_filters_2t[xoffset]); \
aom_highbd_var_filter_block2d_bil_second_pass( \
fdata3, temp2, W, W, H, W, bilinear_filters_2t[yoffset]); \
\
aom_highbd_comp_avg_pred_c(CONVERT_TO_BYTEPTR(temp3), second_pred, W, H, \
CONVERT_TO_BYTEPTR(temp2), W); \
\
return aom_highbd_10_variance##W##x##H##_c(CONVERT_TO_BYTEPTR(temp3), W, \
dst, dst_stride, sse); \
} \
\
uint32_t aom_highbd_12_sub_pixel_avg_variance##W##x##H##_c( \
const uint8_t *src, int src_stride, int xoffset, int yoffset, \
const uint8_t *dst, int dst_stride, uint32_t *sse, \
const uint8_t *second_pred) { \
uint16_t fdata3[(H + 1) * W]; \
uint16_t temp2[H * W]; \
DECLARE_ALIGNED(16, uint16_t, temp3[H * W]); \
\
aom_highbd_var_filter_block2d_bil_first_pass( \
src, fdata3, src_stride, 1, H + 1, W, bilinear_filters_2t[xoffset]); \
aom_highbd_var_filter_block2d_bil_second_pass( \
fdata3, temp2, W, W, H, W, bilinear_filters_2t[yoffset]); \
\
aom_highbd_comp_avg_pred_c(CONVERT_TO_BYTEPTR(temp3), second_pred, W, H, \
CONVERT_TO_BYTEPTR(temp2), W); \
\
return aom_highbd_12_variance##W##x##H##_c(CONVERT_TO_BYTEPTR(temp3), W, \
dst, dst_stride, sse); \
} \
\
uint32_t aom_highbd_8_jnt_sub_pixel_avg_variance##W##x##H##_c( \
const uint8_t *src, int src_stride, int xoffset, int yoffset, \
const uint8_t *dst, int dst_stride, uint32_t *sse, \
const uint8_t *second_pred, const JNT_COMP_PARAMS *jcp_param) { \
uint16_t fdata3[(H + 1) * W]; \
uint16_t temp2[H * W]; \
DECLARE_ALIGNED(16, uint16_t, temp3[H * W]); \
\
aom_highbd_var_filter_block2d_bil_first_pass( \
src, fdata3, src_stride, 1, H + 1, W, bilinear_filters_2t[xoffset]); \
aom_highbd_var_filter_block2d_bil_second_pass( \
fdata3, temp2, W, W, H, W, bilinear_filters_2t[yoffset]); \
\
aom_highbd_jnt_comp_avg_pred(CONVERT_TO_BYTEPTR(temp3), second_pred, W, H, \
CONVERT_TO_BYTEPTR(temp2), W, jcp_param); \
\
return aom_highbd_8_variance##W##x##H(CONVERT_TO_BYTEPTR(temp3), W, dst, \
dst_stride, sse); \
} \
\
uint32_t aom_highbd_10_jnt_sub_pixel_avg_variance##W##x##H##_c( \
const uint8_t *src, int src_stride, int xoffset, int yoffset, \
const uint8_t *dst, int dst_stride, uint32_t *sse, \
const uint8_t *second_pred, const JNT_COMP_PARAMS *jcp_param) { \
uint16_t fdata3[(H + 1) * W]; \
uint16_t temp2[H * W]; \
DECLARE_ALIGNED(16, uint16_t, temp3[H * W]); \
\
aom_highbd_var_filter_block2d_bil_first_pass( \
src, fdata3, src_stride, 1, H + 1, W, bilinear_filters_2t[xoffset]); \
aom_highbd_var_filter_block2d_bil_second_pass( \
fdata3, temp2, W, W, H, W, bilinear_filters_2t[yoffset]); \
\
aom_highbd_jnt_comp_avg_pred(CONVERT_TO_BYTEPTR(temp3), second_pred, W, H, \
CONVERT_TO_BYTEPTR(temp2), W, jcp_param); \
\
return aom_highbd_10_variance##W##x##H(CONVERT_TO_BYTEPTR(temp3), W, dst, \
dst_stride, sse); \
} \
\
uint32_t aom_highbd_12_jnt_sub_pixel_avg_variance##W##x##H##_c( \
const uint8_t *src, int src_stride, int xoffset, int yoffset, \
const uint8_t *dst, int dst_stride, uint32_t *sse, \
const uint8_t *second_pred, const JNT_COMP_PARAMS *jcp_param) { \
uint16_t fdata3[(H + 1) * W]; \
uint16_t temp2[H * W]; \
DECLARE_ALIGNED(16, uint16_t, temp3[H * W]); \
\
aom_highbd_var_filter_block2d_bil_first_pass( \
src, fdata3, src_stride, 1, H + 1, W, bilinear_filters_2t[xoffset]); \
aom_highbd_var_filter_block2d_bil_second_pass( \
fdata3, temp2, W, W, H, W, bilinear_filters_2t[yoffset]); \
\
aom_highbd_jnt_comp_avg_pred(CONVERT_TO_BYTEPTR(temp3), second_pred, W, H, \
CONVERT_TO_BYTEPTR(temp2), W, jcp_param); \
\
return aom_highbd_12_variance##W##x##H(CONVERT_TO_BYTEPTR(temp3), W, dst, \
dst_stride, sse); \
#define HIGHBD_SUBPIX_AVG_VAR(W, H) \
uint32_t aom_highbd_8_sub_pixel_avg_variance##W##x##H##_c( \
const uint8_t *src, int src_stride, int xoffset, int yoffset, \
const uint8_t *dst, int dst_stride, uint32_t *sse, \
const uint8_t *second_pred) { \
uint16_t fdata3[(H + 1) * W]; \
uint16_t temp2[H * W]; \
DECLARE_ALIGNED(16, uint16_t, temp3[H * W]); \
\
aom_highbd_var_filter_block2d_bil_first_pass( \
src, fdata3, src_stride, 1, H + 1, W, bilinear_filters_2t[xoffset]); \
aom_highbd_var_filter_block2d_bil_second_pass( \
fdata3, temp2, W, W, H, W, bilinear_filters_2t[yoffset]); \
\
aom_highbd_comp_avg_pred_c(CONVERT_TO_BYTEPTR(temp3), second_pred, W, H, \
CONVERT_TO_BYTEPTR(temp2), W); \
\
return aom_highbd_8_variance##W##x##H##_c(CONVERT_TO_BYTEPTR(temp3), W, \
dst, dst_stride, sse); \
} \
\
uint32_t aom_highbd_10_sub_pixel_avg_variance##W##x##H##_c( \
const uint8_t *src, int src_stride, int xoffset, int yoffset, \
const uint8_t *dst, int dst_stride, uint32_t *sse, \
const uint8_t *second_pred) { \
uint16_t fdata3[(H + 1) * W]; \
uint16_t temp2[H * W]; \
DECLARE_ALIGNED(16, uint16_t, temp3[H * W]); \
\
aom_highbd_var_filter_block2d_bil_first_pass( \
src, fdata3, src_stride, 1, H + 1, W, bilinear_filters_2t[xoffset]); \
aom_highbd_var_filter_block2d_bil_second_pass( \
fdata3, temp2, W, W, H, W, bilinear_filters_2t[yoffset]); \
\
aom_highbd_comp_avg_pred_c(CONVERT_TO_BYTEPTR(temp3), second_pred, W, H, \
CONVERT_TO_BYTEPTR(temp2), W); \
\
return aom_highbd_10_variance##W##x##H##_c(CONVERT_TO_BYTEPTR(temp3), W, \
dst, dst_stride, sse); \
} \
\
uint32_t aom_highbd_12_sub_pixel_avg_variance##W##x##H##_c( \
const uint8_t *src, int src_stride, int xoffset, int yoffset, \
const uint8_t *dst, int dst_stride, uint32_t *sse, \
const uint8_t *second_pred) { \
uint16_t fdata3[(H + 1) * W]; \
uint16_t temp2[H * W]; \
DECLARE_ALIGNED(16, uint16_t, temp3[H * W]); \
\
aom_highbd_var_filter_block2d_bil_first_pass( \
src, fdata3, src_stride, 1, H + 1, W, bilinear_filters_2t[xoffset]); \
aom_highbd_var_filter_block2d_bil_second_pass( \
fdata3, temp2, W, W, H, W, bilinear_filters_2t[yoffset]); \
\
aom_highbd_comp_avg_pred_c(CONVERT_TO_BYTEPTR(temp3), second_pred, W, H, \
CONVERT_TO_BYTEPTR(temp2), W); \
\
return aom_highbd_12_variance##W##x##H##_c(CONVERT_TO_BYTEPTR(temp3), W, \
dst, dst_stride, sse); \
} \
\
uint32_t aom_highbd_8_dist_wtd_sub_pixel_avg_variance##W##x##H##_c( \
const uint8_t *src, int src_stride, int xoffset, int yoffset, \
const uint8_t *dst, int dst_stride, uint32_t *sse, \
const uint8_t *second_pred, const DIST_WTD_COMP_PARAMS *jcp_param) { \
uint16_t fdata3[(H + 1) * W]; \
uint16_t temp2[H * W]; \
DECLARE_ALIGNED(16, uint16_t, temp3[H * W]); \
\
aom_highbd_var_filter_block2d_bil_first_pass( \
src, fdata3, src_stride, 1, H + 1, W, bilinear_filters_2t[xoffset]); \
aom_highbd_var_filter_block2d_bil_second_pass( \
fdata3, temp2, W, W, H, W, bilinear_filters_2t[yoffset]); \
\
aom_highbd_dist_wtd_comp_avg_pred(CONVERT_TO_BYTEPTR(temp3), second_pred, \
W, H, CONVERT_TO_BYTEPTR(temp2), W, \
jcp_param); \
\
return aom_highbd_8_variance##W##x##H(CONVERT_TO_BYTEPTR(temp3), W, dst, \
dst_stride, sse); \
} \
\
uint32_t aom_highbd_10_dist_wtd_sub_pixel_avg_variance##W##x##H##_c( \
const uint8_t *src, int src_stride, int xoffset, int yoffset, \
const uint8_t *dst, int dst_stride, uint32_t *sse, \
const uint8_t *second_pred, const DIST_WTD_COMP_PARAMS *jcp_param) { \
uint16_t fdata3[(H + 1) * W]; \
uint16_t temp2[H * W]; \
DECLARE_ALIGNED(16, uint16_t, temp3[H * W]); \
\
aom_highbd_var_filter_block2d_bil_first_pass( \
src, fdata3, src_stride, 1, H + 1, W, bilinear_filters_2t[xoffset]); \
aom_highbd_var_filter_block2d_bil_second_pass( \
fdata3, temp2, W, W, H, W, bilinear_filters_2t[yoffset]); \
\
aom_highbd_dist_wtd_comp_avg_pred(CONVERT_TO_BYTEPTR(temp3), second_pred, \
W, H, CONVERT_TO_BYTEPTR(temp2), W, \
jcp_param); \
\
return aom_highbd_10_variance##W##x##H(CONVERT_TO_BYTEPTR(temp3), W, dst, \
dst_stride, sse); \
} \
\
uint32_t aom_highbd_12_dist_wtd_sub_pixel_avg_variance##W##x##H##_c( \
const uint8_t *src, int src_stride, int xoffset, int yoffset, \
const uint8_t *dst, int dst_stride, uint32_t *sse, \
const uint8_t *second_pred, const DIST_WTD_COMP_PARAMS *jcp_param) { \
uint16_t fdata3[(H + 1) * W]; \
uint16_t temp2[H * W]; \
DECLARE_ALIGNED(16, uint16_t, temp3[H * W]); \
\
aom_highbd_var_filter_block2d_bil_first_pass( \
src, fdata3, src_stride, 1, H + 1, W, bilinear_filters_2t[xoffset]); \
aom_highbd_var_filter_block2d_bil_second_pass( \
fdata3, temp2, W, W, H, W, bilinear_filters_2t[yoffset]); \
\
aom_highbd_dist_wtd_comp_avg_pred(CONVERT_TO_BYTEPTR(temp3), second_pred, \
W, H, CONVERT_TO_BYTEPTR(temp2), W, \
jcp_param); \
\
return aom_highbd_12_variance##W##x##H(CONVERT_TO_BYTEPTR(temp3), W, dst, \
dst_stride, sse); \
}
/* All three forms of the variance are available in the same sizes. */
@ -883,7 +884,7 @@ void aom_highbd_upsampled_pred_c(MACROBLOCKD *xd,
const int ref_num = 0;
const int is_intrabc = is_intrabc_block(mi);
const struct scale_factors *const sf =
is_intrabc ? &cm->sf_identity : &xd->block_refs[ref_num]->sf;
is_intrabc ? &cm->sf_identity : xd->block_ref_scale_factors[ref_num];
const int is_scaled = av1_is_scaled(sf);
if (is_scaled) {
@ -959,10 +960,7 @@ void aom_highbd_upsampled_pred_c(MACROBLOCKD *xd,
}
}
const InterpFilterParams *filter =
(subpel_search == 1)
? av1_get_4tap_interp_filter_params(EIGHTTAP_REGULAR)
: av1_get_interp_filter_params_with_block_size(EIGHTTAP_REGULAR, 8);
const InterpFilterParams *filter = av1_get_filter(subpel_search);
if (!subpel_x_q3 && !subpel_y_q3) {
const uint16_t *ref = CONVERT_TO_SHORTPTR(ref8);
@ -975,13 +973,13 @@ void aom_highbd_upsampled_pred_c(MACROBLOCKD *xd,
} else if (!subpel_y_q3) {
const int16_t *const kernel =
av1_get_interp_filter_subpel_kernel(filter, subpel_x_q3 << 1);
aom_highbd_convolve8_horiz(ref8, ref_stride, comp_pred8, width, kernel, 16,
NULL, -1, width, height, bd);
aom_highbd_convolve8_horiz_c(ref8, ref_stride, comp_pred8, width, kernel,
16, NULL, -1, width, height, bd);
} else if (!subpel_x_q3) {
const int16_t *const kernel =
av1_get_interp_filter_subpel_kernel(filter, subpel_y_q3 << 1);
aom_highbd_convolve8_vert(ref8, ref_stride, comp_pred8, width, NULL, -1,
kernel, 16, width, height, bd);
aom_highbd_convolve8_vert_c(ref8, ref_stride, comp_pred8, width, NULL, -1,
kernel, 16, width, height, bd);
} else {
DECLARE_ALIGNED(16, uint16_t,
temp[((MAX_SB_SIZE + 16) + 16) * MAX_SB_SIZE]);
@ -992,11 +990,11 @@ void aom_highbd_upsampled_pred_c(MACROBLOCKD *xd,
const int intermediate_height =
(((height - 1) * 8 + subpel_y_q3) >> 3) + filter->taps;
assert(intermediate_height <= (MAX_SB_SIZE * 2 + 16) + 16);
aom_highbd_convolve8_horiz(ref8 - ref_stride * ((filter->taps >> 1) - 1),
ref_stride, CONVERT_TO_BYTEPTR(temp),
MAX_SB_SIZE, kernel_x, 16, NULL, -1, width,
intermediate_height, bd);
aom_highbd_convolve8_vert(
aom_highbd_convolve8_horiz_c(ref8 - ref_stride * ((filter->taps >> 1) - 1),
ref_stride, CONVERT_TO_BYTEPTR(temp),
MAX_SB_SIZE, kernel_x, 16, NULL, -1, width,
intermediate_height, bd);
aom_highbd_convolve8_vert_c(
CONVERT_TO_BYTEPTR(temp + MAX_SB_SIZE * ((filter->taps >> 1) - 1)),
MAX_SB_SIZE, comp_pred8, width, NULL, -1, kernel_y, 16, width, height,
bd);
@ -1024,10 +1022,10 @@ void aom_highbd_comp_avg_upsampled_pred_c(
}
}
void aom_highbd_jnt_comp_avg_pred_c(uint8_t *comp_pred8, const uint8_t *pred8,
int width, int height, const uint8_t *ref8,
int ref_stride,
const JNT_COMP_PARAMS *jcp_param) {
void aom_highbd_dist_wtd_comp_avg_pred_c(
uint8_t *comp_pred8, const uint8_t *pred8, int width, int height,
const uint8_t *ref8, int ref_stride,
const DIST_WTD_COMP_PARAMS *jcp_param) {
int i, j;
const int fwd_offset = jcp_param->fwd_offset;
const int bck_offset = jcp_param->bck_offset;
@ -1047,20 +1045,20 @@ void aom_highbd_jnt_comp_avg_pred_c(uint8_t *comp_pred8, const uint8_t *pred8,
}
}
void aom_highbd_jnt_comp_avg_upsampled_pred_c(
void aom_highbd_dist_wtd_comp_avg_upsampled_pred_c(
MACROBLOCKD *xd, const struct AV1Common *const cm, int mi_row, int mi_col,
const MV *const mv, uint8_t *comp_pred8, const uint8_t *pred8, int width,
int height, int subpel_x_q3, int subpel_y_q3, const uint8_t *ref8,
int ref_stride, int bd, const JNT_COMP_PARAMS *jcp_param,
int ref_stride, int bd, const DIST_WTD_COMP_PARAMS *jcp_param,
int subpel_search) {
int i, j;
const int fwd_offset = jcp_param->fwd_offset;
const int bck_offset = jcp_param->bck_offset;
const uint16_t *pred = CONVERT_TO_SHORTPTR(pred8);
uint16_t *comp_pred = CONVERT_TO_SHORTPTR(comp_pred8);
aom_highbd_upsampled_pred(xd, cm, mi_row, mi_col, mv, comp_pred8, width,
height, subpel_x_q3, subpel_y_q3, ref8, ref_stride,
bd, subpel_search);
aom_highbd_upsampled_pred_c(xd, cm, mi_row, mi_col, mv, comp_pred8, width,
height, subpel_x_q3, subpel_y_q3, ref8,
ref_stride, bd, subpel_search);
for (i = 0; i < height; i++) {
for (j = 0; j < width; j++) {

15
third_party/aom/aom_dsp/variance.h vendored
View file

@ -50,15 +50,14 @@ typedef unsigned int (*aom_subp_avg_variance_fn_t)(
const uint8_t *a, int a_stride, int xoffset, int yoffset, const uint8_t *b,
int b_stride, unsigned int *sse, const uint8_t *second_pred);
typedef unsigned int (*aom_jnt_sad_avg_fn_t)(const uint8_t *a, int a_stride,
const uint8_t *b, int b_stride,
const uint8_t *second_pred,
const JNT_COMP_PARAMS *jcp_param);
typedef unsigned int (*aom_dist_wtd_sad_avg_fn_t)(
const uint8_t *a, int a_stride, const uint8_t *b, int b_stride,
const uint8_t *second_pred, const DIST_WTD_COMP_PARAMS *jcp_param);
typedef unsigned int (*aom_jnt_subp_avg_variance_fn_t)(
typedef unsigned int (*aom_dist_wtd_subp_avg_variance_fn_t)(
const uint8_t *a, int a_stride, int xoffset, int yoffset, const uint8_t *b,
int b_stride, unsigned int *sse, const uint8_t *second_pred,
const JNT_COMP_PARAMS *jcp_param);
const DIST_WTD_COMP_PARAMS *jcp_param);
typedef unsigned int (*aom_masked_sad_fn_t)(const uint8_t *src, int src_stride,
const uint8_t *ref, int ref_stride,
@ -101,8 +100,8 @@ typedef struct aom_variance_vtable {
aom_obmc_sad_fn_t osdf;
aom_obmc_variance_fn_t ovf;
aom_obmc_subpixvariance_fn_t osvf;
aom_jnt_sad_avg_fn_t jsdaf;
aom_jnt_subp_avg_variance_fn_t jsvaf;
aom_dist_wtd_sad_avg_fn_t jsdaf;
aom_dist_wtd_subp_avg_variance_fn_t jsvaf;
} aom_variance_fn_ptr_t;
void aom_highbd_var_filter_block2d_bil_first_pass(

21
third_party/aom/aom_dsp/x86/aom_asm_stubs.c vendored
View file

@ -21,13 +21,13 @@ filter8_1dfunction aom_filter_block1d8_v8_sse2;
filter8_1dfunction aom_filter_block1d8_h8_sse2;
filter8_1dfunction aom_filter_block1d4_v8_sse2;
filter8_1dfunction aom_filter_block1d4_h8_sse2;
filter8_1dfunction aom_filter_block1d16_v4_sse2;
filter8_1dfunction aom_filter_block1d16_h4_sse2;
#define aom_filter_block1d16_h4_sse2 aom_filter_block1d16_h8_sse2
#define aom_filter_block1d16_v4_sse2 aom_filter_block1d16_v8_sse2
#define aom_filter_block1d8_h4_sse2 aom_filter_block1d8_h8_sse2
#define aom_filter_block1d8_v4_sse2 aom_filter_block1d8_v8_sse2
#define aom_filter_block1d4_h4_sse2 aom_filter_block1d4_h8_sse2
#define aom_filter_block1d4_v4_sse2 aom_filter_block1d4_v8_sse2
filter8_1dfunction aom_filter_block1d8_h4_sse2;
filter8_1dfunction aom_filter_block1d8_v4_sse2;
filter8_1dfunction aom_filter_block1d4_h4_sse2;
filter8_1dfunction aom_filter_block1d4_v4_sse2;
filter8_1dfunction aom_filter_block1d16_v2_sse2;
filter8_1dfunction aom_filter_block1d16_h2_sse2;
@ -49,7 +49,6 @@ filter8_1dfunction aom_filter_block1d4_h2_sse2;
FUN_CONV_1D(horiz, x_step_q4, filter_x, h, src, , sse2);
FUN_CONV_1D(vert, y_step_q4, filter_y, v, src - src_stride * 3, , sse2);
#if ARCH_X86_64
highbd_filter8_1dfunction aom_highbd_filter_block1d16_v8_sse2;
highbd_filter8_1dfunction aom_highbd_filter_block1d16_h8_sse2;
highbd_filter8_1dfunction aom_highbd_filter_block1d8_v8_sse2;
@ -57,6 +56,13 @@ highbd_filter8_1dfunction aom_highbd_filter_block1d8_h8_sse2;
highbd_filter8_1dfunction aom_highbd_filter_block1d4_v8_sse2;
highbd_filter8_1dfunction aom_highbd_filter_block1d4_h8_sse2;
highbd_filter8_1dfunction aom_highbd_filter_block1d16_v4_sse2;
highbd_filter8_1dfunction aom_highbd_filter_block1d16_h4_sse2;
highbd_filter8_1dfunction aom_highbd_filter_block1d8_v4_sse2;
highbd_filter8_1dfunction aom_highbd_filter_block1d8_h4_sse2;
highbd_filter8_1dfunction aom_highbd_filter_block1d4_v4_sse2;
highbd_filter8_1dfunction aom_highbd_filter_block1d4_h4_sse2;
highbd_filter8_1dfunction aom_highbd_filter_block1d16_v2_sse2;
highbd_filter8_1dfunction aom_highbd_filter_block1d16_h2_sse2;
highbd_filter8_1dfunction aom_highbd_filter_block1d8_v2_sse2;
@ -85,5 +91,4 @@ highbd_filter8_1dfunction aom_highbd_filter_block1d4_h2_sse2;
HIGH_FUN_CONV_1D(horiz, x_step_q4, filter_x, h, src, , sse2);
HIGH_FUN_CONV_1D(vert, y_step_q4, filter_y, v, src - src_stride * 3, , sse2);
#endif // ARCH_X86_64
#endif // HAVE_SSE2

75
third_party/aom/aom_dsp/x86/aom_high_subpixel_bilinear_sse2.asm vendored
View file

@ -67,7 +67,6 @@
dec rcx
%endm
%if ARCH_X86_64
%macro HIGH_GET_PARAM 0
mov rdx, arg(5) ;filter ptr
mov rsi, arg(0) ;src_ptr
@ -86,14 +85,17 @@
mov rdx, 0x00010001
movsxd rcx, DWORD PTR arg(6) ;bps
movq xmm8, rdx
movq xmm3, rdx
movq xmm5, rcx
pshufd xmm8, xmm8, 0b
movdqa xmm1, xmm8
psllw xmm8, xmm5
psubw xmm8, xmm1 ;max value (for clamping)
pshufd xmm3, xmm3, 0b
movdqa xmm1, xmm3
psllw xmm3, xmm5
psubw xmm3, xmm1 ;max value (for clamping)
pxor xmm5, xmm5 ;min value (for clamping)
movdqa max, xmm3
movdqa min, xmm5
movsxd rax, DWORD PTR arg(1) ;pixels_per_line
movsxd rdx, DWORD PTR arg(3) ;out_pitch
movsxd rcx, DWORD PTR arg(4) ;output_height
@ -113,8 +115,8 @@
packssdw xmm0, xmm6 ;pack back to word
;clamp the values
pminsw xmm0, xmm8
pmaxsw xmm0, xmm5
pminsw xmm0, max
pmaxsw xmm0, min
%if %1
movdqu xmm1, [rdi]
@ -128,36 +130,36 @@
%endm
%macro HIGH_APPLY_FILTER_16 1
movdqa xmm9, xmm0
movdqa xmm5, xmm0
movdqa xmm6, xmm2
punpckhwd xmm9, xmm1
punpckhwd xmm5, xmm1
punpckhwd xmm6, xmm3
punpcklwd xmm0, xmm1
punpcklwd xmm2, xmm3
pmaddwd xmm9, xmm7
pmaddwd xmm5, xmm7
pmaddwd xmm6, xmm7
pmaddwd xmm0, xmm7
pmaddwd xmm2, xmm7
paddd xmm9, xmm4 ;rounding
paddd xmm5, xmm4 ;rounding
paddd xmm6, xmm4
paddd xmm0, xmm4
paddd xmm2, xmm4
psrad xmm9, 7 ;shift
psrad xmm5, 7 ;shift
psrad xmm6, 7
psrad xmm0, 7
psrad xmm2, 7
packssdw xmm0, xmm9 ;pack back to word
packssdw xmm0, xmm5 ;pack back to word
packssdw xmm2, xmm6 ;pack back to word
;clamp the values
pminsw xmm0, xmm8
pmaxsw xmm0, xmm5
pminsw xmm2, xmm8
pmaxsw xmm2, xmm5
pminsw xmm0, max
pmaxsw xmm0, min
pminsw xmm2, max
pmaxsw xmm2, min
%if %1
movdqu xmm1, [rdi]
@ -172,7 +174,6 @@
lea rdi, [rdi + 2*rdx]
dec rcx
%endm
%endif
SECTION .text
@ -200,7 +201,6 @@ sym(aom_highbd_filter_block1d4_v2_sse2):
pop rbp
ret
%if ARCH_X86_64
global sym(aom_highbd_filter_block1d8_v2_sse2) PRIVATE
sym(aom_highbd_filter_block1d8_v2_sse2):
push rbp
@ -211,6 +211,11 @@ sym(aom_highbd_filter_block1d8_v2_sse2):
push rdi
; end prolog
ALIGN_STACK 16, rax
sub rsp, 16 * 2
%define max [rsp + 16 * 0]
%define min [rsp + 16 * 1]
HIGH_GET_PARAM
.loop:
movdqu xmm0, [rsi] ;0
@ -219,6 +224,9 @@ sym(aom_highbd_filter_block1d8_v2_sse2):
HIGH_APPLY_FILTER_8 0
jnz .loop
add rsp, 16 * 2
pop rsp
; begin epilog
pop rdi
pop rsi
@ -237,6 +245,11 @@ sym(aom_highbd_filter_block1d16_v2_sse2):
push rdi
; end prolog
ALIGN_STACK 16, rax
sub rsp, 16 * 2
%define max [rsp + 16 * 0]
%define min [rsp + 16 * 1]
HIGH_GET_PARAM
.loop:
movdqu xmm0, [rsi] ;0
@ -247,6 +260,9 @@ sym(aom_highbd_filter_block1d16_v2_sse2):
HIGH_APPLY_FILTER_16 0
jnz .loop
add rsp, 16 * 2
pop rsp
; begin epilog
pop rdi
pop rsi
@ -254,7 +270,6 @@ sym(aom_highbd_filter_block1d16_v2_sse2):
UNSHADOW_ARGS
pop rbp
ret
%endif
global sym(aom_highbd_filter_block1d4_h2_sse2) PRIVATE
sym(aom_highbd_filter_block1d4_h2_sse2):
@ -281,7 +296,6 @@ sym(aom_highbd_filter_block1d4_h2_sse2):
pop rbp
ret
%if ARCH_X86_64
global sym(aom_highbd_filter_block1d8_h2_sse2) PRIVATE
sym(aom_highbd_filter_block1d8_h2_sse2):
push rbp
@ -292,6 +306,11 @@ sym(aom_highbd_filter_block1d8_h2_sse2):
push rdi
; end prolog
ALIGN_STACK 16, rax
sub rsp, 16 * 2
%define max [rsp + 16 * 0]
%define min [rsp + 16 * 1]
HIGH_GET_PARAM
.loop:
movdqu xmm0, [rsi] ;load src
@ -300,6 +319,9 @@ sym(aom_highbd_filter_block1d8_h2_sse2):
HIGH_APPLY_FILTER_8 0
jnz .loop
add rsp, 16 * 2
pop rsp
; begin epilog
pop rdi
pop rsi
@ -318,6 +340,11 @@ sym(aom_highbd_filter_block1d16_h2_sse2):
push rdi
; end prolog
ALIGN_STACK 16, rax
sub rsp, 16 * 2
%define max [rsp + 16 * 0]
%define min [rsp + 16 * 1]
HIGH_GET_PARAM
.loop:
movdqu xmm0, [rsi] ;load src
@ -328,6 +355,9 @@ sym(aom_highbd_filter_block1d16_h2_sse2):
HIGH_APPLY_FILTER_16 0
jnz .loop
add rsp, 16 * 2
pop rsp
; begin epilog
pop rdi
pop rsi
@ -335,4 +365,3 @@ sym(aom_highbd_filter_block1d16_h2_sse2):
UNSHADOW_ARGS
pop rbp
ret
%endif

569
third_party/aom/aom_dsp/x86/aom_subpixel_8t_intrin_sse2.c vendored Normal file
View file

@ -0,0 +1,569 @@
/*
* Copyright (c) 2018, Alliance for Open Media. All rights reserved
*
* This source code is subject to the terms of the BSD 2 Clause License and
* the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
* was not distributed with this source code in the LICENSE file, you can
* obtain it at www.aomedia.org/license/software. If the Alliance for Open
* Media Patent License 1.0 was not distributed with this source code in the
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
*/
#include <emmintrin.h> // SSE2
#include "config/aom_dsp_rtcd.h"
#include "aom_dsp/x86/convolve.h"
#include "aom_ports/mem.h"
void aom_filter_block1d16_h4_sse2(const uint8_t *src_ptr,
ptrdiff_t src_pixels_per_line,
uint8_t *output_ptr, ptrdiff_t output_pitch,
uint32_t output_height,
const int16_t *filter) {
__m128i filtersReg;
__m128i addFilterReg32;
__m128i secondFilters, thirdFilters;
__m128i srcRegFilt32b1_1, srcRegFilt32b1_2, srcRegFilt32b2_1,
srcRegFilt32b2_2;
__m128i srcReg32b1, srcReg32b2;
unsigned int i;
src_ptr -= 3;
addFilterReg32 = _mm_set1_epi16(32);
filtersReg = _mm_loadu_si128((const __m128i *)filter);
filtersReg = _mm_srai_epi16(filtersReg, 1);
// coeffs 0 1 0 1 2 3 2 3
const __m128i tmp_0 = _mm_unpacklo_epi32(filtersReg, filtersReg);
// coeffs 4 5 4 5 6 7 6 7
const __m128i tmp_1 = _mm_unpackhi_epi32(filtersReg, filtersReg);
secondFilters = _mm_unpackhi_epi64(tmp_0, tmp_0); // coeffs 2 3 2 3 2 3 2 3
thirdFilters = _mm_unpacklo_epi64(tmp_1, tmp_1); // coeffs 4 5 4 5 4 5 4 5
for (i = output_height; i > 0; i -= 1) {
srcReg32b1 = _mm_loadu_si128((const __m128i *)src_ptr);
__m128i ss_2 = _mm_srli_si128(srcReg32b1, 2);
__m128i ss_4 = _mm_srli_si128(srcReg32b1, 4);
__m128i ss_1_1 = _mm_unpacklo_epi8(ss_2, _mm_setzero_si128());
__m128i ss_2_1 = _mm_unpacklo_epi8(ss_4, _mm_setzero_si128());
__m128i d1 = _mm_madd_epi16(ss_1_1, secondFilters);
__m128i d2 = _mm_madd_epi16(ss_2_1, thirdFilters);
srcRegFilt32b1_1 = _mm_add_epi32(d1, d2);
__m128i ss_1 = _mm_srli_si128(srcReg32b1, 3);
__m128i ss_3 = _mm_srli_si128(srcReg32b1, 5);
__m128i ss_1_2 = _mm_unpacklo_epi8(ss_1, _mm_setzero_si128());
__m128i ss_2_2 = _mm_unpacklo_epi8(ss_3, _mm_setzero_si128());
d1 = _mm_madd_epi16(ss_1_2, secondFilters);
d2 = _mm_madd_epi16(ss_2_2, thirdFilters);
srcRegFilt32b1_2 = _mm_add_epi32(d1, d2);
__m128i res_lo = _mm_unpacklo_epi32(srcRegFilt32b1_1, srcRegFilt32b1_2);
__m128i res_hi = _mm_unpackhi_epi32(srcRegFilt32b1_1, srcRegFilt32b1_2);
srcRegFilt32b1_1 = _mm_packs_epi32(res_lo, res_hi);
// reading stride of the next 16 bytes
// (part of it was being read by earlier read)
srcReg32b2 = _mm_loadu_si128((const __m128i *)(src_ptr + 8));
ss_2 = _mm_srli_si128(srcReg32b2, 2);
ss_4 = _mm_srli_si128(srcReg32b2, 4);
ss_1_1 = _mm_unpacklo_epi8(ss_2, _mm_setzero_si128());
ss_2_1 = _mm_unpacklo_epi8(ss_4, _mm_setzero_si128());
d1 = _mm_madd_epi16(ss_1_1, secondFilters);
d2 = _mm_madd_epi16(ss_2_1, thirdFilters);
srcRegFilt32b2_1 = _mm_add_epi32(d1, d2);
ss_1 = _mm_srli_si128(srcReg32b2, 3);
ss_3 = _mm_srli_si128(srcReg32b2, 5);
ss_1_2 = _mm_unpacklo_epi8(ss_1, _mm_setzero_si128());
ss_2_2 = _mm_unpacklo_epi8(ss_3, _mm_setzero_si128());
d1 = _mm_madd_epi16(ss_1_2, secondFilters);
d2 = _mm_madd_epi16(ss_2_2, thirdFilters);
srcRegFilt32b2_2 = _mm_add_epi32(d1, d2);
res_lo = _mm_unpacklo_epi32(srcRegFilt32b2_1, srcRegFilt32b2_2);
res_hi = _mm_unpackhi_epi32(srcRegFilt32b2_1, srcRegFilt32b2_2);
srcRegFilt32b2_1 = _mm_packs_epi32(res_lo, res_hi);
// shift by 6 bit each 16 bit
srcRegFilt32b1_1 = _mm_adds_epi16(srcRegFilt32b1_1, addFilterReg32);
srcRegFilt32b2_1 = _mm_adds_epi16(srcRegFilt32b2_1, addFilterReg32);
srcRegFilt32b1_1 = _mm_srai_epi16(srcRegFilt32b1_1, 6);
srcRegFilt32b2_1 = _mm_srai_epi16(srcRegFilt32b2_1, 6);
// shrink to 8 bit each 16 bits, the first lane contain the first
// convolve result and the second lane contain the second convolve result
srcRegFilt32b1_1 = _mm_packus_epi16(srcRegFilt32b1_1, srcRegFilt32b2_1);
src_ptr += src_pixels_per_line;
_mm_store_si128((__m128i *)output_ptr, srcRegFilt32b1_1);
output_ptr += output_pitch;
}
}
void aom_filter_block1d16_v4_sse2(const uint8_t *src_ptr, ptrdiff_t src_pitch,
uint8_t *output_ptr, ptrdiff_t out_pitch,
uint32_t output_height,
const int16_t *filter) {
__m128i filtersReg;
__m128i srcReg2, srcReg3, srcReg4, srcReg5, srcReg6;
__m128i srcReg23_lo, srcReg23_hi, srcReg34_lo, srcReg34_hi;
__m128i srcReg45_lo, srcReg45_hi, srcReg56_lo, srcReg56_hi;
__m128i resReg23_lo, resReg34_lo, resReg45_lo, resReg56_lo;
__m128i resReg23_hi, resReg34_hi, resReg45_hi, resReg56_hi;
__m128i resReg23_45_lo, resReg34_56_lo, resReg23_45_hi, resReg34_56_hi;
__m128i resReg23_45, resReg34_56;
__m128i addFilterReg32, secondFilters, thirdFilters;
__m128i tmp_0, tmp_1;
unsigned int i;
ptrdiff_t src_stride, dst_stride;
addFilterReg32 = _mm_set1_epi16(32);
filtersReg = _mm_loadu_si128((const __m128i *)filter);
filtersReg = _mm_srai_epi16(filtersReg, 1);
// coeffs 0 1 0 1 2 3 2 3
const __m128i tmp0 = _mm_unpacklo_epi32(filtersReg, filtersReg);
// coeffs 4 5 4 5 6 7 6 7
const __m128i tmp1 = _mm_unpackhi_epi32(filtersReg, filtersReg);
secondFilters = _mm_unpackhi_epi64(tmp0, tmp0); // coeffs 2 3 2 3 2 3 2 3
thirdFilters = _mm_unpacklo_epi64(tmp1, tmp1); // coeffs 4 5 4 5 4 5 4 5
// multiply the size of the source and destination stride by two
src_stride = src_pitch << 1;
dst_stride = out_pitch << 1;
srcReg2 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 2));
srcReg3 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 3));
srcReg23_lo = _mm_unpacklo_epi8(srcReg2, srcReg3);
srcReg23_hi = _mm_unpackhi_epi8(srcReg2, srcReg3);
__m128i resReg23_lo_1 = _mm_unpacklo_epi8(srcReg23_lo, _mm_setzero_si128());
__m128i resReg23_lo_2 = _mm_unpackhi_epi8(srcReg23_lo, _mm_setzero_si128());
__m128i resReg23_hi_1 = _mm_unpacklo_epi8(srcReg23_hi, _mm_setzero_si128());
__m128i resReg23_hi_2 = _mm_unpackhi_epi8(srcReg23_hi, _mm_setzero_si128());
srcReg4 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 4));
srcReg34_lo = _mm_unpacklo_epi8(srcReg3, srcReg4);
srcReg34_hi = _mm_unpackhi_epi8(srcReg3, srcReg4);
__m128i resReg34_lo_1 = _mm_unpacklo_epi8(srcReg34_lo, _mm_setzero_si128());
__m128i resReg34_lo_2 = _mm_unpackhi_epi8(srcReg34_lo, _mm_setzero_si128());
__m128i resReg34_hi_1 = _mm_unpacklo_epi8(srcReg34_hi, _mm_setzero_si128());
__m128i resReg34_hi_2 = _mm_unpackhi_epi8(srcReg34_hi, _mm_setzero_si128());
for (i = output_height; i > 1; i -= 2) {
srcReg5 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 5));
srcReg45_lo = _mm_unpacklo_epi8(srcReg4, srcReg5);
srcReg45_hi = _mm_unpackhi_epi8(srcReg4, srcReg5);
srcReg6 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 6));
srcReg56_lo = _mm_unpacklo_epi8(srcReg5, srcReg6);
srcReg56_hi = _mm_unpackhi_epi8(srcReg5, srcReg6);
// multiply 2 adjacent elements with the filter and add the result
tmp_0 = _mm_madd_epi16(resReg23_lo_1, secondFilters);
tmp_1 = _mm_madd_epi16(resReg23_lo_2, secondFilters);
resReg23_lo = _mm_packs_epi32(tmp_0, tmp_1);
tmp_0 = _mm_madd_epi16(resReg34_lo_1, secondFilters);
tmp_1 = _mm_madd_epi16(resReg34_lo_2, secondFilters);
resReg34_lo = _mm_packs_epi32(tmp_0, tmp_1);
__m128i resReg45_lo_1 = _mm_unpacklo_epi8(srcReg45_lo, _mm_setzero_si128());
__m128i resReg45_lo_2 = _mm_unpackhi_epi8(srcReg45_lo, _mm_setzero_si128());
tmp_0 = _mm_madd_epi16(resReg45_lo_1, thirdFilters);
tmp_1 = _mm_madd_epi16(resReg45_lo_2, thirdFilters);
resReg45_lo = _mm_packs_epi32(tmp_0, tmp_1);
__m128i resReg56_lo_1 = _mm_unpacklo_epi8(srcReg56_lo, _mm_setzero_si128());
__m128i resReg56_lo_2 = _mm_unpackhi_epi8(srcReg56_lo, _mm_setzero_si128());
tmp_0 = _mm_madd_epi16(resReg56_lo_1, thirdFilters);
tmp_1 = _mm_madd_epi16(resReg56_lo_2, thirdFilters);
resReg56_lo = _mm_packs_epi32(tmp_0, tmp_1);
// add and saturate the results together
resReg23_45_lo = _mm_adds_epi16(resReg23_lo, resReg45_lo);
resReg34_56_lo = _mm_adds_epi16(resReg34_lo, resReg56_lo);
// multiply 2 adjacent elements with the filter and add the result
tmp_0 = _mm_madd_epi16(resReg23_hi_1, secondFilters);
tmp_1 = _mm_madd_epi16(resReg23_hi_2, secondFilters);
resReg23_hi = _mm_packs_epi32(tmp_0, tmp_1);
tmp_0 = _mm_madd_epi16(resReg34_hi_1, secondFilters);
tmp_1 = _mm_madd_epi16(resReg34_hi_2, secondFilters);
resReg34_hi = _mm_packs_epi32(tmp_0, tmp_1);
__m128i resReg45_hi_1 = _mm_unpacklo_epi8(srcReg45_hi, _mm_setzero_si128());
__m128i resReg45_hi_2 = _mm_unpackhi_epi8(srcReg45_hi, _mm_setzero_si128());
tmp_0 = _mm_madd_epi16(resReg45_hi_1, thirdFilters);
tmp_1 = _mm_madd_epi16(resReg45_hi_2, thirdFilters);
resReg45_hi = _mm_packs_epi32(tmp_0, tmp_1);
__m128i resReg56_hi_1 = _mm_unpacklo_epi8(srcReg56_hi, _mm_setzero_si128());
__m128i resReg56_hi_2 = _mm_unpackhi_epi8(srcReg56_hi, _mm_setzero_si128());
tmp_0 = _mm_madd_epi16(resReg56_hi_1, thirdFilters);
tmp_1 = _mm_madd_epi16(resReg56_hi_2, thirdFilters);
resReg56_hi = _mm_packs_epi32(tmp_0, tmp_1);
// add and saturate the results together
resReg23_45_hi = _mm_adds_epi16(resReg23_hi, resReg45_hi);
resReg34_56_hi = _mm_adds_epi16(resReg34_hi, resReg56_hi);
// shift by 6 bit each 16 bit
resReg23_45_lo = _mm_adds_epi16(resReg23_45_lo, addFilterReg32);
resReg34_56_lo = _mm_adds_epi16(resReg34_56_lo, addFilterReg32);
resReg23_45_hi = _mm_adds_epi16(resReg23_45_hi, addFilterReg32);
resReg34_56_hi = _mm_adds_epi16(resReg34_56_hi, addFilterReg32);
resReg23_45_lo = _mm_srai_epi16(resReg23_45_lo, 6);
resReg34_56_lo = _mm_srai_epi16(resReg34_56_lo, 6);
resReg23_45_hi = _mm_srai_epi16(resReg23_45_hi, 6);
resReg34_56_hi = _mm_srai_epi16(resReg34_56_hi, 6);
// shrink to 8 bit each 16 bits, the first lane contain the first
// convolve result and the second lane contain the second convolve
// result
resReg23_45 = _mm_packus_epi16(resReg23_45_lo, resReg23_45_hi);
resReg34_56 = _mm_packus_epi16(resReg34_56_lo, resReg34_56_hi);
src_ptr += src_stride;
_mm_store_si128((__m128i *)output_ptr, (resReg23_45));
_mm_store_si128((__m128i *)(output_ptr + out_pitch), (resReg34_56));
output_ptr += dst_stride;
// save part of the registers for next strides
resReg23_lo_1 = resReg45_lo_1;
resReg23_lo_2 = resReg45_lo_2;
resReg23_hi_1 = resReg45_hi_1;
resReg23_hi_2 = resReg45_hi_2;
resReg34_lo_1 = resReg56_lo_1;
resReg34_lo_2 = resReg56_lo_2;
resReg34_hi_1 = resReg56_hi_1;
resReg34_hi_2 = resReg56_hi_2;
srcReg4 = srcReg6;
}
}
void aom_filter_block1d8_h4_sse2(const uint8_t *src_ptr,
ptrdiff_t src_pixels_per_line,
uint8_t *output_ptr, ptrdiff_t output_pitch,
uint32_t output_height,
const int16_t *filter) {
__m128i filtersReg;
__m128i addFilterReg32;
__m128i secondFilters, thirdFilters;
__m128i srcRegFilt32b1_1, srcRegFilt32b1_2;
__m128i srcReg32b1;
unsigned int i;
src_ptr -= 3;
addFilterReg32 = _mm_set1_epi16(32);
filtersReg = _mm_loadu_si128((const __m128i *)filter);
filtersReg = _mm_srai_epi16(filtersReg, 1);
// coeffs 0 1 0 1 2 3 2 3
const __m128i tmp_0 = _mm_unpacklo_epi32(filtersReg, filtersReg);
// coeffs 4 5 4 5 6 7 6 7
const __m128i tmp_1 = _mm_unpackhi_epi32(filtersReg, filtersReg);
secondFilters = _mm_unpackhi_epi64(tmp_0, tmp_0); // coeffs 2 3 2 3 2 3 2 3
thirdFilters = _mm_unpacklo_epi64(tmp_1, tmp_1); // coeffs 4 5 4 5 4 5 4 5
for (i = output_height; i > 0; i -= 1) {
srcReg32b1 = _mm_loadu_si128((const __m128i *)src_ptr);
__m128i ss_2 = _mm_srli_si128(srcReg32b1, 2);
__m128i ss_4 = _mm_srli_si128(srcReg32b1, 4);
ss_2 = _mm_unpacklo_epi8(ss_2, _mm_setzero_si128());
ss_4 = _mm_unpacklo_epi8(ss_4, _mm_setzero_si128());
__m128i d1 = _mm_madd_epi16(ss_2, secondFilters);
__m128i d2 = _mm_madd_epi16(ss_4, thirdFilters);
srcRegFilt32b1_1 = _mm_add_epi32(d1, d2);
__m128i ss_3 = _mm_srli_si128(srcReg32b1, 3);
__m128i ss_5 = _mm_srli_si128(srcReg32b1, 5);
ss_3 = _mm_unpacklo_epi8(ss_3, _mm_setzero_si128());
ss_5 = _mm_unpacklo_epi8(ss_5, _mm_setzero_si128());
d1 = _mm_madd_epi16(ss_3, secondFilters);
d2 = _mm_madd_epi16(ss_5, thirdFilters);
srcRegFilt32b1_2 = _mm_add_epi32(d1, d2);
__m128i res_lo = _mm_unpacklo_epi32(srcRegFilt32b1_1, srcRegFilt32b1_2);
__m128i res_hi = _mm_unpackhi_epi32(srcRegFilt32b1_1, srcRegFilt32b1_2);
srcRegFilt32b1_1 = _mm_packs_epi32(res_lo, res_hi);
// shift by 6 bit each 16 bit
srcRegFilt32b1_1 = _mm_adds_epi16(srcRegFilt32b1_1, addFilterReg32);
srcRegFilt32b1_1 = _mm_srai_epi16(srcRegFilt32b1_1, 6);
// shrink to 8 bit each 16 bits, the first lane contain the first
// convolve result and the second lane contain the second convolve result
srcRegFilt32b1_1 = _mm_packus_epi16(srcRegFilt32b1_1, _mm_setzero_si128());
src_ptr += src_pixels_per_line;
_mm_storel_epi64((__m128i *)output_ptr, srcRegFilt32b1_1);
output_ptr += output_pitch;
}
}
void aom_filter_block1d8_v4_sse2(const uint8_t *src_ptr, ptrdiff_t src_pitch,
uint8_t *output_ptr, ptrdiff_t out_pitch,
uint32_t output_height,
const int16_t *filter) {
__m128i filtersReg;
__m128i srcReg2, srcReg3, srcReg4, srcReg5, srcReg6;
__m128i srcReg23_lo, srcReg34_lo;
__m128i srcReg45_lo, srcReg56_lo;
__m128i resReg23_lo, resReg34_lo, resReg45_lo, resReg56_lo;
__m128i resReg23_45_lo, resReg34_56_lo;
__m128i resReg23_45, resReg34_56;
__m128i addFilterReg32, secondFilters, thirdFilters;
__m128i tmp_0, tmp_1;
unsigned int i;
ptrdiff_t src_stride, dst_stride;
addFilterReg32 = _mm_set1_epi16(32);
filtersReg = _mm_loadu_si128((const __m128i *)filter);
filtersReg = _mm_srai_epi16(filtersReg, 1);
// coeffs 0 1 0 1 2 3 2 3
const __m128i tmp0 = _mm_unpacklo_epi32(filtersReg, filtersReg);
// coeffs 4 5 4 5 6 7 6 7
const __m128i tmp1 = _mm_unpackhi_epi32(filtersReg, filtersReg);
secondFilters = _mm_unpackhi_epi64(tmp0, tmp0); // coeffs 2 3 2 3 2 3 2 3
thirdFilters = _mm_unpacklo_epi64(tmp1, tmp1); // coeffs 4 5 4 5 4 5 4 5
// multiply the size of the source and destination stride by two
src_stride = src_pitch << 1;
dst_stride = out_pitch << 1;
srcReg2 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 2));
srcReg3 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 3));
srcReg23_lo = _mm_unpacklo_epi8(srcReg2, srcReg3);
__m128i resReg23_lo_1 = _mm_unpacklo_epi8(srcReg23_lo, _mm_setzero_si128());
__m128i resReg23_lo_2 = _mm_unpackhi_epi8(srcReg23_lo, _mm_setzero_si128());
srcReg4 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 4));
srcReg34_lo = _mm_unpacklo_epi8(srcReg3, srcReg4);
__m128i resReg34_lo_1 = _mm_unpacklo_epi8(srcReg34_lo, _mm_setzero_si128());
__m128i resReg34_lo_2 = _mm_unpackhi_epi8(srcReg34_lo, _mm_setzero_si128());
for (i = output_height; i > 1; i -= 2) {
srcReg5 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 5));
srcReg45_lo = _mm_unpacklo_epi8(srcReg4, srcReg5);
srcReg6 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 6));
srcReg56_lo = _mm_unpacklo_epi8(srcReg5, srcReg6);
// multiply 2 adjacent elements with the filter and add the result
tmp_0 = _mm_madd_epi16(resReg23_lo_1, secondFilters);
tmp_1 = _mm_madd_epi16(resReg23_lo_2, secondFilters);
resReg23_lo = _mm_packs_epi32(tmp_0, tmp_1);
tmp_0 = _mm_madd_epi16(resReg34_lo_1, secondFilters);
tmp_1 = _mm_madd_epi16(resReg34_lo_2, secondFilters);
resReg34_lo = _mm_packs_epi32(tmp_0, tmp_1);
__m128i resReg45_lo_1 = _mm_unpacklo_epi8(srcReg45_lo, _mm_setzero_si128());
__m128i resReg45_lo_2 = _mm_unpackhi_epi8(srcReg45_lo, _mm_setzero_si128());
tmp_0 = _mm_madd_epi16(resReg45_lo_1, thirdFilters);
tmp_1 = _mm_madd_epi16(resReg45_lo_2, thirdFilters);
resReg45_lo = _mm_packs_epi32(tmp_0, tmp_1);
__m128i resReg56_lo_1 = _mm_unpacklo_epi8(srcReg56_lo, _mm_setzero_si128());
__m128i resReg56_lo_2 = _mm_unpackhi_epi8(srcReg56_lo, _mm_setzero_si128());
tmp_0 = _mm_madd_epi16(resReg56_lo_1, thirdFilters);
tmp_1 = _mm_madd_epi16(resReg56_lo_2, thirdFilters);
resReg56_lo = _mm_packs_epi32(tmp_0, tmp_1);
// add and saturate the results together
resReg23_45_lo = _mm_adds_epi16(resReg23_lo, resReg45_lo);
resReg34_56_lo = _mm_adds_epi16(resReg34_lo, resReg56_lo);
// shift by 6 bit each 16 bit
resReg23_45_lo = _mm_adds_epi16(resReg23_45_lo, addFilterReg32);
resReg34_56_lo = _mm_adds_epi16(resReg34_56_lo, addFilterReg32);
resReg23_45_lo = _mm_srai_epi16(resReg23_45_lo, 6);
resReg34_56_lo = _mm_srai_epi16(resReg34_56_lo, 6);
// shrink to 8 bit each 16 bits, the first lane contain the first
// convolve result and the second lane contain the second convolve
// result
resReg23_45 = _mm_packus_epi16(resReg23_45_lo, _mm_setzero_si128());
resReg34_56 = _mm_packus_epi16(resReg34_56_lo, _mm_setzero_si128());
src_ptr += src_stride;
_mm_storel_epi64((__m128i *)output_ptr, (resReg23_45));
_mm_storel_epi64((__m128i *)(output_ptr + out_pitch), (resReg34_56));
output_ptr += dst_stride;
// save part of the registers for next strides
resReg23_lo_1 = resReg45_lo_1;
resReg23_lo_2 = resReg45_lo_2;
resReg34_lo_1 = resReg56_lo_1;
resReg34_lo_2 = resReg56_lo_2;
srcReg4 = srcReg6;
}
}
void aom_filter_block1d4_h4_sse2(const uint8_t *src_ptr,
ptrdiff_t src_pixels_per_line,
uint8_t *output_ptr, ptrdiff_t output_pitch,
uint32_t output_height,
const int16_t *filter) {
__m128i filtersReg;
__m128i addFilterReg32;
__m128i secondFilters, thirdFilters;
__m128i srcRegFilt32b1_1;
__m128i srcReg32b1;
unsigned int i;
src_ptr -= 3;
addFilterReg32 = _mm_set1_epi16(32);
filtersReg = _mm_loadu_si128((const __m128i *)filter);
filtersReg = _mm_srai_epi16(filtersReg, 1);
// coeffs 0 1 0 1 2 3 2 3
const __m128i tmp_0 = _mm_unpacklo_epi32(filtersReg, filtersReg);
// coeffs 4 5 4 5 6 7 6 7
const __m128i tmp_1 = _mm_unpackhi_epi32(filtersReg, filtersReg);
secondFilters = _mm_unpackhi_epi64(tmp_0, tmp_0); // coeffs 2 3 2 3 2 3 2 3
thirdFilters = _mm_unpacklo_epi64(tmp_1, tmp_1); // coeffs 4 5 4 5 4 5 4 5
for (i = output_height; i > 0; i -= 1) {
srcReg32b1 = _mm_loadu_si128((const __m128i *)src_ptr);
__m128i ss_2 = _mm_srli_si128(srcReg32b1, 2);
__m128i ss_3 = _mm_srli_si128(srcReg32b1, 3);
__m128i ss_4 = _mm_srli_si128(srcReg32b1, 4);
__m128i ss_5 = _mm_srli_si128(srcReg32b1, 5);
ss_2 = _mm_unpacklo_epi8(ss_2, _mm_setzero_si128());
ss_3 = _mm_unpacklo_epi8(ss_3, _mm_setzero_si128());
ss_4 = _mm_unpacklo_epi8(ss_4, _mm_setzero_si128());
ss_5 = _mm_unpacklo_epi8(ss_5, _mm_setzero_si128());
__m128i ss_1_1 = _mm_unpacklo_epi32(ss_2, ss_3);
__m128i ss_1_2 = _mm_unpacklo_epi32(ss_4, ss_5);
__m128i d1 = _mm_madd_epi16(ss_1_1, secondFilters);
__m128i d2 = _mm_madd_epi16(ss_1_2, thirdFilters);
srcRegFilt32b1_1 = _mm_add_epi32(d1, d2);
srcRegFilt32b1_1 = _mm_packs_epi32(srcRegFilt32b1_1, _mm_setzero_si128());
// shift by 6 bit each 16 bit
srcRegFilt32b1_1 = _mm_adds_epi16(srcRegFilt32b1_1, addFilterReg32);
srcRegFilt32b1_1 = _mm_srai_epi16(srcRegFilt32b1_1, 6);
// shrink to 8 bit each 16 bits, the first lane contain the first
// convolve result and the second lane contain the second convolve result
srcRegFilt32b1_1 = _mm_packus_epi16(srcRegFilt32b1_1, _mm_setzero_si128());
src_ptr += src_pixels_per_line;
*((uint32_t *)(output_ptr)) = _mm_cvtsi128_si32(srcRegFilt32b1_1);
output_ptr += output_pitch;
}
}
void aom_filter_block1d4_v4_sse2(const uint8_t *src_ptr, ptrdiff_t src_pitch,
uint8_t *output_ptr, ptrdiff_t out_pitch,
uint32_t output_height,
const int16_t *filter) {
__m128i filtersReg;
__m128i srcReg2, srcReg3, srcReg4, srcReg5, srcReg6;
__m128i srcReg23, srcReg34, srcReg45, srcReg56;
__m128i resReg23_34, resReg45_56;
__m128i resReg23_34_45_56;
__m128i addFilterReg32, secondFilters, thirdFilters;
__m128i tmp_0, tmp_1;
unsigned int i;
ptrdiff_t src_stride, dst_stride;
addFilterReg32 = _mm_set1_epi16(32);
filtersReg = _mm_loadu_si128((const __m128i *)filter);
filtersReg = _mm_srai_epi16(filtersReg, 1);
// coeffs 0 1 0 1 2 3 2 3
const __m128i tmp0 = _mm_unpacklo_epi32(filtersReg, filtersReg);
// coeffs 4 5 4 5 6 7 6 7
const __m128i tmp1 = _mm_unpackhi_epi32(filtersReg, filtersReg);
secondFilters = _mm_unpackhi_epi64(tmp0, tmp0); // coeffs 2 3 2 3 2 3 2 3
thirdFilters = _mm_unpacklo_epi64(tmp1, tmp1); // coeffs 4 5 4 5 4 5 4 5
// multiply the size of the source and destination stride by two
src_stride = src_pitch << 1;
dst_stride = out_pitch << 1;
srcReg2 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 2));
srcReg3 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 3));
srcReg23 = _mm_unpacklo_epi8(srcReg2, srcReg3);
__m128i resReg23 = _mm_unpacklo_epi8(srcReg23, _mm_setzero_si128());
srcReg4 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 4));
srcReg34 = _mm_unpacklo_epi8(srcReg3, srcReg4);
__m128i resReg34 = _mm_unpacklo_epi8(srcReg34, _mm_setzero_si128());
for (i = output_height; i > 1; i -= 2) {
srcReg5 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 5));
srcReg45 = _mm_unpacklo_epi8(srcReg4, srcReg5);
srcReg6 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 6));
srcReg56 = _mm_unpacklo_epi8(srcReg5, srcReg6);
// multiply 2 adjacent elements with the filter and add the result
tmp_0 = _mm_madd_epi16(resReg23, secondFilters);
tmp_1 = _mm_madd_epi16(resReg34, secondFilters);
resReg23_34 = _mm_packs_epi32(tmp_0, tmp_1);
__m128i resReg45 = _mm_unpacklo_epi8(srcReg45, _mm_setzero_si128());
__m128i resReg56 = _mm_unpacklo_epi8(srcReg56, _mm_setzero_si128());
tmp_0 = _mm_madd_epi16(resReg45, thirdFilters);
tmp_1 = _mm_madd_epi16(resReg56, thirdFilters);
resReg45_56 = _mm_packs_epi32(tmp_0, tmp_1);
// add and saturate the results together
resReg23_34_45_56 = _mm_adds_epi16(resReg23_34, resReg45_56);
// shift by 6 bit each 16 bit
resReg23_34_45_56 = _mm_adds_epi16(resReg23_34_45_56, addFilterReg32);
resReg23_34_45_56 = _mm_srai_epi16(resReg23_34_45_56, 6);
// shrink to 8 bit each 16 bits, the first lane contain the first
// convolve result and the second lane contain the second convolve
// result
resReg23_34_45_56 =
_mm_packus_epi16(resReg23_34_45_56, _mm_setzero_si128());
src_ptr += src_stride;
*((uint32_t *)(output_ptr)) = _mm_cvtsi128_si32(resReg23_34_45_56);
*((uint32_t *)(output_ptr + out_pitch)) =
_mm_cvtsi128_si32(_mm_srli_si128(resReg23_34_45_56, 4));
output_ptr += dst_stride;
// save part of the registers for next strides
resReg23 = resReg45;
resReg34 = resReg56;
srcReg4 = srcReg6;
}
}

469
third_party/aom/aom_dsp/x86/aom_subpixel_8t_intrin_ssse3.c vendored
View file

@ -45,11 +45,153 @@ DECLARE_ALIGNED(16, static const uint8_t, filt4_global[16]) = {
6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14
};
DECLARE_ALIGNED(32, static const uint8_t, filt_h4[]) = {
0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 0, 1, 1,
2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 2, 3, 3, 4, 4, 5,
5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 2, 3, 3, 4, 4, 5, 5, 6, 6,
7, 7, 8, 8, 9, 9, 10, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10,
10, 11, 11, 12, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11,
12, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 6, 7,
7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14
};
DECLARE_ALIGNED(32, static const uint8_t, filtd4[]) = {
2, 3, 4, 5, 3, 4, 5, 6, 4, 5, 6, 7, 5, 6, 7, 8,
2, 3, 4, 5, 3, 4, 5, 6, 4, 5, 6, 7, 5, 6, 7, 8,
};
// These are reused by the avx2 intrinsics.
filter8_1dfunction aom_filter_block1d8_v8_intrin_ssse3;
filter8_1dfunction aom_filter_block1d8_h8_intrin_ssse3;
filter8_1dfunction aom_filter_block1d4_h8_intrin_ssse3;
static void aom_filter_block1d4_h4_ssse3(
const uint8_t *src_ptr, ptrdiff_t src_pixels_per_line, uint8_t *output_ptr,
ptrdiff_t output_pitch, uint32_t output_height, const int16_t *filter) {
__m128i filtersReg;
__m128i addFilterReg32, filt1Reg, firstFilters, srcReg32b1, srcRegFilt32b1_1;
unsigned int i;
src_ptr -= 3;
addFilterReg32 = _mm_set1_epi16(32);
filtersReg = _mm_loadu_si128((const __m128i *)filter);
filtersReg = _mm_srai_epi16(filtersReg, 1);
// converting the 16 bit (short) to 8 bit (byte) and have the same data
// in both lanes of 128 bit register.
filtersReg = _mm_packs_epi16(filtersReg, filtersReg);
firstFilters = _mm_shuffle_epi8(filtersReg, _mm_set1_epi32(0x5040302u));
filt1Reg = _mm_load_si128((__m128i const *)(filtd4));
for (i = output_height; i > 0; i -= 1) {
// load the 2 strides of source
srcReg32b1 = _mm_loadu_si128((const __m128i *)src_ptr);
// filter the source buffer
srcRegFilt32b1_1 = _mm_shuffle_epi8(srcReg32b1, filt1Reg);
// multiply 4 adjacent elements with the filter and add the result
srcRegFilt32b1_1 = _mm_maddubs_epi16(srcRegFilt32b1_1, firstFilters);
srcRegFilt32b1_1 = _mm_hadds_epi16(srcRegFilt32b1_1, _mm_setzero_si128());
// shift by 6 bit each 16 bit
srcRegFilt32b1_1 = _mm_adds_epi16(srcRegFilt32b1_1, addFilterReg32);
srcRegFilt32b1_1 = _mm_srai_epi16(srcRegFilt32b1_1, 6);
// shrink to 8 bit each 16 bits, the first lane contain the first
// convolve result and the second lane contain the second convolve result
srcRegFilt32b1_1 = _mm_packus_epi16(srcRegFilt32b1_1, _mm_setzero_si128());
src_ptr += src_pixels_per_line;
*((uint32_t *)(output_ptr)) = _mm_cvtsi128_si32(srcRegFilt32b1_1);
output_ptr += output_pitch;
}
}
static void aom_filter_block1d4_v4_ssse3(
const uint8_t *src_ptr, ptrdiff_t src_pitch, uint8_t *output_ptr,
ptrdiff_t out_pitch, uint32_t output_height, const int16_t *filter) {
__m128i filtersReg;
__m128i addFilterReg32;
__m128i srcReg2, srcReg3, srcReg23, srcReg4, srcReg34, srcReg5, srcReg45,
srcReg6, srcReg56;
__m128i srcReg23_34_lo, srcReg45_56_lo;
__m128i srcReg2345_3456_lo, srcReg2345_3456_hi;
__m128i resReglo, resReghi;
__m128i firstFilters;
unsigned int i;
ptrdiff_t src_stride, dst_stride;
addFilterReg32 = _mm_set1_epi16(32);
filtersReg = _mm_loadu_si128((const __m128i *)filter);
// converting the 16 bit (short) to 8 bit (byte) and have the
// same data in both lanes of 128 bit register.
filtersReg = _mm_srai_epi16(filtersReg, 1);
filtersReg = _mm_packs_epi16(filtersReg, filtersReg);
firstFilters = _mm_shuffle_epi8(filtersReg, _mm_set1_epi32(0x5040302u));
// multiple the size of the source and destination stride by two
src_stride = src_pitch << 1;
dst_stride = out_pitch << 1;
srcReg2 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 2));
srcReg3 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 3));
srcReg23 = _mm_unpacklo_epi32(srcReg2, srcReg3);
srcReg4 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 4));
// have consecutive loads on the same 256 register
srcReg34 = _mm_unpacklo_epi32(srcReg3, srcReg4);
srcReg23_34_lo = _mm_unpacklo_epi8(srcReg23, srcReg34);
for (i = output_height; i > 1; i -= 2) {
srcReg5 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 5));
srcReg45 = _mm_unpacklo_epi32(srcReg4, srcReg5);
srcReg6 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 6));
srcReg56 = _mm_unpacklo_epi32(srcReg5, srcReg6);
// merge every two consecutive registers
srcReg45_56_lo = _mm_unpacklo_epi8(srcReg45, srcReg56);
srcReg2345_3456_lo = _mm_unpacklo_epi16(srcReg23_34_lo, srcReg45_56_lo);
srcReg2345_3456_hi = _mm_unpackhi_epi16(srcReg23_34_lo, srcReg45_56_lo);
// multiply 2 adjacent elements with the filter and add the result
resReglo = _mm_maddubs_epi16(srcReg2345_3456_lo, firstFilters);
resReghi = _mm_maddubs_epi16(srcReg2345_3456_hi, firstFilters);
resReglo = _mm_hadds_epi16(resReglo, _mm_setzero_si128());
resReghi = _mm_hadds_epi16(resReghi, _mm_setzero_si128());
// shift by 6 bit each 16 bit
resReglo = _mm_adds_epi16(resReglo, addFilterReg32);
resReghi = _mm_adds_epi16(resReghi, addFilterReg32);
resReglo = _mm_srai_epi16(resReglo, 6);
resReghi = _mm_srai_epi16(resReghi, 6);
// shrink to 8 bit each 16 bits, the first lane contain the first
// convolve result and the second lane contain the second convolve
// result
resReglo = _mm_packus_epi16(resReglo, resReglo);
resReghi = _mm_packus_epi16(resReghi, resReghi);
src_ptr += src_stride;
*((uint32_t *)(output_ptr)) = _mm_cvtsi128_si32(resReglo);
*((uint32_t *)(output_ptr + out_pitch)) = _mm_cvtsi128_si32(resReghi);
output_ptr += dst_stride;
// save part of the registers for next strides
srcReg23_34_lo = srcReg45_56_lo;
srcReg4 = srcReg6;
}
}
void aom_filter_block1d4_h8_intrin_ssse3(
const uint8_t *src_ptr, ptrdiff_t src_pixels_per_line, uint8_t *output_ptr,
ptrdiff_t output_pitch, uint32_t output_height, const int16_t *filter) {
@ -118,6 +260,145 @@ void aom_filter_block1d4_h8_intrin_ssse3(
}
}
static void aom_filter_block1d8_h4_ssse3(
const uint8_t *src_ptr, ptrdiff_t src_pixels_per_line, uint8_t *output_ptr,
ptrdiff_t output_pitch, uint32_t output_height, const int16_t *filter) {
__m128i filtersReg;
__m128i addFilterReg32, filt2Reg, filt3Reg;
__m128i secondFilters, thirdFilters;
__m128i srcRegFilt32b1_1, srcRegFilt32b2, srcRegFilt32b3;
__m128i srcReg32b1;
unsigned int i;
src_ptr -= 3;
addFilterReg32 = _mm_set1_epi16(32);
filtersReg = _mm_loadu_si128((const __m128i *)filter);
filtersReg = _mm_srai_epi16(filtersReg, 1);
// converting the 16 bit (short) to 8 bit (byte) and have the same data
// in both lanes of 128 bit register.
filtersReg = _mm_packs_epi16(filtersReg, filtersReg);
// duplicate only the second 16 bits (third and forth byte)
// across 256 bit register
secondFilters = _mm_shuffle_epi8(filtersReg, _mm_set1_epi16(0x302u));
// duplicate only the third 16 bits (fifth and sixth byte)
// across 256 bit register
thirdFilters = _mm_shuffle_epi8(filtersReg, _mm_set1_epi16(0x504u));
filt2Reg = _mm_load_si128((__m128i const *)(filt_h4 + 32));
filt3Reg = _mm_load_si128((__m128i const *)(filt_h4 + 32 * 2));
for (i = output_height; i > 0; i -= 1) {
srcReg32b1 = _mm_loadu_si128((const __m128i *)src_ptr);
// filter the source buffer
srcRegFilt32b3 = _mm_shuffle_epi8(srcReg32b1, filt2Reg);
srcRegFilt32b2 = _mm_shuffle_epi8(srcReg32b1, filt3Reg);
// multiply 2 adjacent elements with the filter and add the result
srcRegFilt32b3 = _mm_maddubs_epi16(srcRegFilt32b3, secondFilters);
srcRegFilt32b2 = _mm_maddubs_epi16(srcRegFilt32b2, thirdFilters);
srcRegFilt32b1_1 = _mm_adds_epi16(srcRegFilt32b3, srcRegFilt32b2);
// shift by 6 bit each 16 bit
srcRegFilt32b1_1 = _mm_adds_epi16(srcRegFilt32b1_1, addFilterReg32);
srcRegFilt32b1_1 = _mm_srai_epi16(srcRegFilt32b1_1, 6);
// shrink to 8 bit each 16 bits
srcRegFilt32b1_1 = _mm_packus_epi16(srcRegFilt32b1_1, _mm_setzero_si128());
src_ptr += src_pixels_per_line;
_mm_storel_epi64((__m128i *)output_ptr, srcRegFilt32b1_1);
output_ptr += output_pitch;
}
}
static void aom_filter_block1d8_v4_ssse3(
const uint8_t *src_ptr, ptrdiff_t src_pitch, uint8_t *output_ptr,
ptrdiff_t out_pitch, uint32_t output_height, const int16_t *filter) {
__m128i filtersReg;
__m128i srcReg2, srcReg3, srcReg4, srcReg5, srcReg6;
__m128i srcReg23, srcReg34, srcReg45, srcReg56;
__m128i resReg23, resReg34, resReg45, resReg56;
__m128i resReg23_45, resReg34_56;
__m128i addFilterReg32, secondFilters, thirdFilters;
unsigned int i;
ptrdiff_t src_stride, dst_stride;
addFilterReg32 = _mm_set1_epi16(32);
filtersReg = _mm_loadu_si128((const __m128i *)filter);
// converting the 16 bit (short) to 8 bit (byte) and have the
// same data in both lanes of 128 bit register.
filtersReg = _mm_srai_epi16(filtersReg, 1);
filtersReg = _mm_packs_epi16(filtersReg, filtersReg);
// duplicate only the second 16 bits (third and forth byte)
// across 128 bit register
secondFilters = _mm_shuffle_epi8(filtersReg, _mm_set1_epi16(0x302u));
// duplicate only the third 16 bits (fifth and sixth byte)
// across 128 bit register
thirdFilters = _mm_shuffle_epi8(filtersReg, _mm_set1_epi16(0x504u));
// multiple the size of the source and destination stride by two
src_stride = src_pitch << 1;
dst_stride = out_pitch << 1;
srcReg2 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 2));
srcReg3 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 3));
srcReg23 = _mm_unpacklo_epi8(srcReg2, srcReg3);
srcReg4 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 4));
// have consecutive loads on the same 256 register
srcReg34 = _mm_unpacklo_epi8(srcReg3, srcReg4);
for (i = output_height; i > 1; i -= 2) {
srcReg5 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 5));
srcReg45 = _mm_unpacklo_epi8(srcReg4, srcReg5);
srcReg6 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 6));
srcReg56 = _mm_unpacklo_epi8(srcReg5, srcReg6);
// multiply 2 adjacent elements with the filter and add the result
resReg23 = _mm_maddubs_epi16(srcReg23, secondFilters);
resReg34 = _mm_maddubs_epi16(srcReg34, secondFilters);
resReg45 = _mm_maddubs_epi16(srcReg45, thirdFilters);
resReg56 = _mm_maddubs_epi16(srcReg56, thirdFilters);
// add and saturate the results together
resReg23_45 = _mm_adds_epi16(resReg23, resReg45);
resReg34_56 = _mm_adds_epi16(resReg34, resReg56);
// shift by 6 bit each 16 bit
resReg23_45 = _mm_adds_epi16(resReg23_45, addFilterReg32);
resReg34_56 = _mm_adds_epi16(resReg34_56, addFilterReg32);
resReg23_45 = _mm_srai_epi16(resReg23_45, 6);
resReg34_56 = _mm_srai_epi16(resReg34_56, 6);
// shrink to 8 bit each 16 bits, the first lane contain the first
// convolve result and the second lane contain the second convolve
// result
resReg23_45 = _mm_packus_epi16(resReg23_45, _mm_setzero_si128());
resReg34_56 = _mm_packus_epi16(resReg34_56, _mm_setzero_si128());
src_ptr += src_stride;
_mm_storel_epi64((__m128i *)output_ptr, (resReg23_45));
_mm_storel_epi64((__m128i *)(output_ptr + out_pitch), (resReg34_56));
output_ptr += dst_stride;
// save part of the registers for next strides
srcReg23 = srcReg45;
srcReg34 = srcReg56;
srcReg4 = srcReg6;
}
}
void aom_filter_block1d8_h8_intrin_ssse3(
const uint8_t *src_ptr, ptrdiff_t src_pixels_per_line, uint8_t *output_ptr,
ptrdiff_t output_pitch, uint32_t output_height, const int16_t *filter) {
@ -280,6 +561,187 @@ void aom_filter_block1d8_v8_intrin_ssse3(
}
}
static void aom_filter_block1d16_h4_ssse3(
const uint8_t *src_ptr, ptrdiff_t src_pixels_per_line, uint8_t *output_ptr,
ptrdiff_t output_pitch, uint32_t output_height, const int16_t *filter) {
__m128i filtersReg;
__m128i addFilterReg32, filt2Reg, filt3Reg;
__m128i secondFilters, thirdFilters;
__m128i srcRegFilt32b1_1, srcRegFilt32b2_1, srcRegFilt32b2, srcRegFilt32b3;
__m128i srcReg32b1, srcReg32b2;
unsigned int i;
src_ptr -= 3;
addFilterReg32 = _mm_set1_epi16(32);
filtersReg = _mm_loadu_si128((const __m128i *)filter);
filtersReg = _mm_srai_epi16(filtersReg, 1);
// converting the 16 bit (short) to 8 bit (byte) and have the same data
// in both lanes of 128 bit register.
filtersReg = _mm_packs_epi16(filtersReg, filtersReg);
// duplicate only the second 16 bits (third and forth byte)
// across 256 bit register
secondFilters = _mm_shuffle_epi8(filtersReg, _mm_set1_epi16(0x302u));
// duplicate only the third 16 bits (fifth and sixth byte)
// across 256 bit register
thirdFilters = _mm_shuffle_epi8(filtersReg, _mm_set1_epi16(0x504u));
filt2Reg = _mm_load_si128((__m128i const *)(filt_h4 + 32));
filt3Reg = _mm_load_si128((__m128i const *)(filt_h4 + 32 * 2));
for (i = output_height; i > 0; i -= 1) {
srcReg32b1 = _mm_loadu_si128((const __m128i *)src_ptr);
// filter the source buffer
srcRegFilt32b3 = _mm_shuffle_epi8(srcReg32b1, filt2Reg);
srcRegFilt32b2 = _mm_shuffle_epi8(srcReg32b1, filt3Reg);
// multiply 2 adjacent elements with the filter and add the result
srcRegFilt32b3 = _mm_maddubs_epi16(srcRegFilt32b3, secondFilters);
srcRegFilt32b2 = _mm_maddubs_epi16(srcRegFilt32b2, thirdFilters);
srcRegFilt32b1_1 = _mm_adds_epi16(srcRegFilt32b3, srcRegFilt32b2);
// reading stride of the next 16 bytes
// (part of it was being read by earlier read)
srcReg32b2 = _mm_loadu_si128((const __m128i *)(src_ptr + 8));
// filter the source buffer
srcRegFilt32b3 = _mm_shuffle_epi8(srcReg32b2, filt2Reg);
srcRegFilt32b2 = _mm_shuffle_epi8(srcReg32b2, filt3Reg);
// multiply 2 adjacent elements with the filter and add the result
srcRegFilt32b3 = _mm_maddubs_epi16(srcRegFilt32b3, secondFilters);
srcRegFilt32b2 = _mm_maddubs_epi16(srcRegFilt32b2, thirdFilters);
// add and saturate the results together
srcRegFilt32b2_1 = _mm_adds_epi16(srcRegFilt32b3, srcRegFilt32b2);
// shift by 6 bit each 16 bit
srcRegFilt32b1_1 = _mm_adds_epi16(srcRegFilt32b1_1, addFilterReg32);
srcRegFilt32b2_1 = _mm_adds_epi16(srcRegFilt32b2_1, addFilterReg32);
srcRegFilt32b1_1 = _mm_srai_epi16(srcRegFilt32b1_1, 6);
srcRegFilt32b2_1 = _mm_srai_epi16(srcRegFilt32b2_1, 6);
// shrink to 8 bit each 16 bits, the first lane contain the first
// convolve result and the second lane contain the second convolve result
srcRegFilt32b1_1 = _mm_packus_epi16(srcRegFilt32b1_1, srcRegFilt32b2_1);
src_ptr += src_pixels_per_line;
_mm_store_si128((__m128i *)output_ptr, srcRegFilt32b1_1);
output_ptr += output_pitch;
}
}
static void aom_filter_block1d16_v4_ssse3(
const uint8_t *src_ptr, ptrdiff_t src_pitch, uint8_t *output_ptr,
ptrdiff_t out_pitch, uint32_t output_height, const int16_t *filter) {
__m128i filtersReg;
__m128i srcReg2, srcReg3, srcReg4, srcReg5, srcReg6;
__m128i srcReg23_lo, srcReg23_hi, srcReg34_lo, srcReg34_hi;
__m128i srcReg45_lo, srcReg45_hi, srcReg56_lo, srcReg56_hi;
__m128i resReg23_lo, resReg34_lo, resReg45_lo, resReg56_lo;
__m128i resReg23_hi, resReg34_hi, resReg45_hi, resReg56_hi;
__m128i resReg23_45_lo, resReg34_56_lo, resReg23_45_hi, resReg34_56_hi;
__m128i resReg23_45, resReg34_56;
__m128i addFilterReg32, secondFilters, thirdFilters;
unsigned int i;
ptrdiff_t src_stride, dst_stride;
addFilterReg32 = _mm_set1_epi16(32);
filtersReg = _mm_loadu_si128((const __m128i *)filter);
// converting the 16 bit (short) to 8 bit (byte) and have the
// same data in both lanes of 128 bit register.
filtersReg = _mm_srai_epi16(filtersReg, 1);
filtersReg = _mm_packs_epi16(filtersReg, filtersReg);
// duplicate only the second 16 bits (third and forth byte)
// across 128 bit register
secondFilters = _mm_shuffle_epi8(filtersReg, _mm_set1_epi16(0x302u));
// duplicate only the third 16 bits (fifth and sixth byte)
// across 128 bit register
thirdFilters = _mm_shuffle_epi8(filtersReg, _mm_set1_epi16(0x504u));
// multiple the size of the source and destination stride by two
src_stride = src_pitch << 1;
dst_stride = out_pitch << 1;
srcReg2 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 2));
srcReg3 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 3));
srcReg23_lo = _mm_unpacklo_epi8(srcReg2, srcReg3);
srcReg23_hi = _mm_unpackhi_epi8(srcReg2, srcReg3);
srcReg4 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 4));
// have consecutive loads on the same 256 register
srcReg34_lo = _mm_unpacklo_epi8(srcReg3, srcReg4);
srcReg34_hi = _mm_unpackhi_epi8(srcReg3, srcReg4);
for (i = output_height; i > 1; i -= 2) {
srcReg5 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 5));
srcReg45_lo = _mm_unpacklo_epi8(srcReg4, srcReg5);
srcReg45_hi = _mm_unpackhi_epi8(srcReg4, srcReg5);
srcReg6 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 6));
srcReg56_lo = _mm_unpacklo_epi8(srcReg5, srcReg6);
srcReg56_hi = _mm_unpackhi_epi8(srcReg5, srcReg6);
// multiply 2 adjacent elements with the filter and add the result
resReg23_lo = _mm_maddubs_epi16(srcReg23_lo, secondFilters);
resReg34_lo = _mm_maddubs_epi16(srcReg34_lo, secondFilters);
resReg45_lo = _mm_maddubs_epi16(srcReg45_lo, thirdFilters);
resReg56_lo = _mm_maddubs_epi16(srcReg56_lo, thirdFilters);
// add and saturate the results together
resReg23_45_lo = _mm_adds_epi16(resReg23_lo, resReg45_lo);
resReg34_56_lo = _mm_adds_epi16(resReg34_lo, resReg56_lo);
// multiply 2 adjacent elements with the filter and add the result
resReg23_hi = _mm_maddubs_epi16(srcReg23_hi, secondFilters);
resReg34_hi = _mm_maddubs_epi16(srcReg34_hi, secondFilters);
resReg45_hi = _mm_maddubs_epi16(srcReg45_hi, thirdFilters);
resReg56_hi = _mm_maddubs_epi16(srcReg56_hi, thirdFilters);
// add and saturate the results together
resReg23_45_hi = _mm_adds_epi16(resReg23_hi, resReg45_hi);
resReg34_56_hi = _mm_adds_epi16(resReg34_hi, resReg56_hi);
// shift by 6 bit each 16 bit
resReg23_45_lo = _mm_adds_epi16(resReg23_45_lo, addFilterReg32);
resReg34_56_lo = _mm_adds_epi16(resReg34_56_lo, addFilterReg32);
resReg23_45_hi = _mm_adds_epi16(resReg23_45_hi, addFilterReg32);
resReg34_56_hi = _mm_adds_epi16(resReg34_56_hi, addFilterReg32);
resReg23_45_lo = _mm_srai_epi16(resReg23_45_lo, 6);
resReg34_56_lo = _mm_srai_epi16(resReg34_56_lo, 6);
resReg23_45_hi = _mm_srai_epi16(resReg23_45_hi, 6);
resReg34_56_hi = _mm_srai_epi16(resReg34_56_hi, 6);
// shrink to 8 bit each 16 bits, the first lane contain the first
// convolve result and the second lane contain the second convolve
// result
resReg23_45 = _mm_packus_epi16(resReg23_45_lo, resReg23_45_hi);
resReg34_56 = _mm_packus_epi16(resReg34_56_lo, resReg34_56_hi);
src_ptr += src_stride;
_mm_store_si128((__m128i *)output_ptr, (resReg23_45));
_mm_store_si128((__m128i *)(output_ptr + out_pitch), (resReg34_56));
output_ptr += dst_stride;
// save part of the registers for next strides
srcReg23_lo = srcReg45_lo;
srcReg34_lo = srcReg56_lo;
srcReg23_hi = srcReg45_hi;
srcReg34_hi = srcReg56_hi;
srcReg4 = srcReg6;
}
}
filter8_1dfunction aom_filter_block1d16_v8_ssse3;
filter8_1dfunction aom_filter_block1d16_h8_ssse3;
filter8_1dfunction aom_filter_block1d8_v8_ssse3;
@ -287,13 +749,6 @@ filter8_1dfunction aom_filter_block1d8_h8_ssse3;
filter8_1dfunction aom_filter_block1d4_v8_ssse3;
filter8_1dfunction aom_filter_block1d4_h8_ssse3;
#define aom_filter_block1d16_h4_ssse3 aom_filter_block1d16_h8_ssse3
#define aom_filter_block1d16_v4_ssse3 aom_filter_block1d16_v8_ssse3
#define aom_filter_block1d8_h4_ssse3 aom_filter_block1d8_h8_ssse3
#define aom_filter_block1d8_v4_ssse3 aom_filter_block1d8_v8_ssse3
#define aom_filter_block1d4_h4_ssse3 aom_filter_block1d4_h8_ssse3
#define aom_filter_block1d4_v4_ssse3 aom_filter_block1d4_v8_ssse3
filter8_1dfunction aom_filter_block1d16_v2_ssse3;
filter8_1dfunction aom_filter_block1d16_h2_ssse3;
filter8_1dfunction aom_filter_block1d8_v2_ssse3;

249
third_party/aom/aom_dsp/x86/avg_intrin_avx2.c vendored Normal file
View file

@ -0,0 +1,249 @@
/*
* Copyright (c) 2016, Alliance for Open Media. All rights reserved
*
* This source code is subject to the terms of the BSD 2 Clause License and
* the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
* was not distributed with this source code in the LICENSE file, you can
* obtain it at www.aomedia.org/license/software. If the Alliance for Open
* Media Patent License 1.0 was not distributed with this source code in the
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
*/
#include <immintrin.h>
#include "config/aom_dsp_rtcd.h"
#include "aom/aom_integer.h"
#include "aom_dsp/x86/bitdepth_conversion_avx2.h"
#include "aom_ports/mem.h"
static void hadamard_col8x2_avx2(__m256i *in, int iter) {
__m256i a0 = in[0];
__m256i a1 = in[1];
__m256i a2 = in[2];
__m256i a3 = in[3];
__m256i a4 = in[4];
__m256i a5 = in[5];
__m256i a6 = in[6];
__m256i a7 = in[7];
__m256i b0 = _mm256_add_epi16(a0, a1);
__m256i b1 = _mm256_sub_epi16(a0, a1);
__m256i b2 = _mm256_add_epi16(a2, a3);
__m256i b3 = _mm256_sub_epi16(a2, a3);
__m256i b4 = _mm256_add_epi16(a4, a5);
__m256i b5 = _mm256_sub_epi16(a4, a5);
__m256i b6 = _mm256_add_epi16(a6, a7);
__m256i b7 = _mm256_sub_epi16(a6, a7);
a0 = _mm256_add_epi16(b0, b2);
a1 = _mm256_add_epi16(b1, b3);
a2 = _mm256_sub_epi16(b0, b2);
a3 = _mm256_sub_epi16(b1, b3);
a4 = _mm256_add_epi16(b4, b6);
a5 = _mm256_add_epi16(b5, b7);
a6 = _mm256_sub_epi16(b4, b6);
a7 = _mm256_sub_epi16(b5, b7);
if (iter == 0) {
b0 = _mm256_add_epi16(a0, a4);
b7 = _mm256_add_epi16(a1, a5);
b3 = _mm256_add_epi16(a2, a6);
b4 = _mm256_add_epi16(a3, a7);
b2 = _mm256_sub_epi16(a0, a4);
b6 = _mm256_sub_epi16(a1, a5);
b1 = _mm256_sub_epi16(a2, a6);
b5 = _mm256_sub_epi16(a3, a7);
a0 = _mm256_unpacklo_epi16(b0, b1);
a1 = _mm256_unpacklo_epi16(b2, b3);
a2 = _mm256_unpackhi_epi16(b0, b1);
a3 = _mm256_unpackhi_epi16(b2, b3);
a4 = _mm256_unpacklo_epi16(b4, b5);
a5 = _mm256_unpacklo_epi16(b6, b7);
a6 = _mm256_unpackhi_epi16(b4, b5);
a7 = _mm256_unpackhi_epi16(b6, b7);
b0 = _mm256_unpacklo_epi32(a0, a1);
b1 = _mm256_unpacklo_epi32(a4, a5);
b2 = _mm256_unpackhi_epi32(a0, a1);
b3 = _mm256_unpackhi_epi32(a4, a5);
b4 = _mm256_unpacklo_epi32(a2, a3);
b5 = _mm256_unpacklo_epi32(a6, a7);
b6 = _mm256_unpackhi_epi32(a2, a3);
b7 = _mm256_unpackhi_epi32(a6, a7);
in[0] = _mm256_unpacklo_epi64(b0, b1);
in[1] = _mm256_unpackhi_epi64(b0, b1);
in[2] = _mm256_unpacklo_epi64(b2, b3);
in[3] = _mm256_unpackhi_epi64(b2, b3);
in[4] = _mm256_unpacklo_epi64(b4, b5);
in[5] = _mm256_unpackhi_epi64(b4, b5);
in[6] = _mm256_unpacklo_epi64(b6, b7);
in[7] = _mm256_unpackhi_epi64(b6, b7);
} else {
in[0] = _mm256_add_epi16(a0, a4);
in[7] = _mm256_add_epi16(a1, a5);
in[3] = _mm256_add_epi16(a2, a6);
in[4] = _mm256_add_epi16(a3, a7);
in[2] = _mm256_sub_epi16(a0, a4);
in[6] = _mm256_sub_epi16(a1, a5);
in[1] = _mm256_sub_epi16(a2, a6);
in[5] = _mm256_sub_epi16(a3, a7);
}
}
static void hadamard_8x8x2_avx2(const int16_t *src_diff, ptrdiff_t src_stride,
int16_t *coeff) {
__m256i src[8];
src[0] = _mm256_loadu_si256((const __m256i *)src_diff);
src[1] = _mm256_loadu_si256((const __m256i *)(src_diff += src_stride));
src[2] = _mm256_loadu_si256((const __m256i *)(src_diff += src_stride));
src[3] = _mm256_loadu_si256((const __m256i *)(src_diff += src_stride));
src[4] = _mm256_loadu_si256((const __m256i *)(src_diff += src_stride));
src[5] = _mm256_loadu_si256((const __m256i *)(src_diff += src_stride));
src[6] = _mm256_loadu_si256((const __m256i *)(src_diff += src_stride));
src[7] = _mm256_loadu_si256((const __m256i *)(src_diff += src_stride));
hadamard_col8x2_avx2(src, 0);
hadamard_col8x2_avx2(src, 1);
_mm256_storeu_si256((__m256i *)coeff,
_mm256_permute2x128_si256(src[0], src[1], 0x20));
coeff += 16;
_mm256_storeu_si256((__m256i *)coeff,
_mm256_permute2x128_si256(src[2], src[3], 0x20));
coeff += 16;
_mm256_storeu_si256((__m256i *)coeff,
_mm256_permute2x128_si256(src[4], src[5], 0x20));
coeff += 16;
_mm256_storeu_si256((__m256i *)coeff,
_mm256_permute2x128_si256(src[6], src[7], 0x20));
coeff += 16;
_mm256_storeu_si256((__m256i *)coeff,
_mm256_permute2x128_si256(src[0], src[1], 0x31));
coeff += 16;
_mm256_storeu_si256((__m256i *)coeff,
_mm256_permute2x128_si256(src[2], src[3], 0x31));
coeff += 16;
_mm256_storeu_si256((__m256i *)coeff,
_mm256_permute2x128_si256(src[4], src[5], 0x31));
coeff += 16;
_mm256_storeu_si256((__m256i *)coeff,
_mm256_permute2x128_si256(src[6], src[7], 0x31));
}
static INLINE void hadamard_16x16_avx2(const int16_t *src_diff,
ptrdiff_t src_stride, tran_low_t *coeff,
int is_final) {
DECLARE_ALIGNED(32, int16_t, temp_coeff[16 * 16]);
int16_t *t_coeff = temp_coeff;
int16_t *coeff16 = (int16_t *)coeff;
int idx;
for (idx = 0; idx < 2; ++idx) {
const int16_t *src_ptr = src_diff + idx * 8 * src_stride;
hadamard_8x8x2_avx2(src_ptr, src_stride, t_coeff + (idx * 64 * 2));
}
for (idx = 0; idx < 64; idx += 16) {
const __m256i coeff0 = _mm256_loadu_si256((const __m256i *)t_coeff);
const __m256i coeff1 = _mm256_loadu_si256((const __m256i *)(t_coeff + 64));
const __m256i coeff2 = _mm256_loadu_si256((const __m256i *)(t_coeff + 128));
const __m256i coeff3 = _mm256_loadu_si256((const __m256i *)(t_coeff + 192));
__m256i b0 = _mm256_add_epi16(coeff0, coeff1);
__m256i b1 = _mm256_sub_epi16(coeff0, coeff1);
__m256i b2 = _mm256_add_epi16(coeff2, coeff3);
__m256i b3 = _mm256_sub_epi16(coeff2, coeff3);
b0 = _mm256_srai_epi16(b0, 1);
b1 = _mm256_srai_epi16(b1, 1);
b2 = _mm256_srai_epi16(b2, 1);
b3 = _mm256_srai_epi16(b3, 1);
if (is_final) {
store_tran_low(_mm256_add_epi16(b0, b2), coeff);
store_tran_low(_mm256_add_epi16(b1, b3), coeff + 64);
store_tran_low(_mm256_sub_epi16(b0, b2), coeff + 128);
store_tran_low(_mm256_sub_epi16(b1, b3), coeff + 192);
coeff += 16;
} else {
_mm256_storeu_si256((__m256i *)coeff16, _mm256_add_epi16(b0, b2));
_mm256_storeu_si256((__m256i *)(coeff16 + 64), _mm256_add_epi16(b1, b3));
_mm256_storeu_si256((__m256i *)(coeff16 + 128), _mm256_sub_epi16(b0, b2));
_mm256_storeu_si256((__m256i *)(coeff16 + 192), _mm256_sub_epi16(b1, b3));
coeff16 += 16;
}
t_coeff += 16;
}
}
void aom_hadamard_16x16_avx2(const int16_t *src_diff, ptrdiff_t src_stride,
tran_low_t *coeff) {
hadamard_16x16_avx2(src_diff, src_stride, coeff, 1);
}
void aom_hadamard_32x32_avx2(const int16_t *src_diff, ptrdiff_t src_stride,
tran_low_t *coeff) {
// For high bitdepths, it is unnecessary to store_tran_low
// (mult/unpack/store), then load_tran_low (load/pack) the same memory in the
// next stage. Output to an intermediate buffer first, then store_tran_low()
// in the final stage.
DECLARE_ALIGNED(32, int16_t, temp_coeff[32 * 32]);
int16_t *t_coeff = temp_coeff;
int idx;
for (idx = 0; idx < 4; ++idx) {
// src_diff: 9 bit, dynamic range [-255, 255]
const int16_t *src_ptr =
src_diff + (idx >> 1) * 16 * src_stride + (idx & 0x01) * 16;
hadamard_16x16_avx2(src_ptr, src_stride,
(tran_low_t *)(t_coeff + idx * 256), 0);
}
for (idx = 0; idx < 256; idx += 16) {
const __m256i coeff0 = _mm256_loadu_si256((const __m256i *)t_coeff);
const __m256i coeff1 = _mm256_loadu_si256((const __m256i *)(t_coeff + 256));
const __m256i coeff2 = _mm256_loadu_si256((const __m256i *)(t_coeff + 512));
const __m256i coeff3 = _mm256_loadu_si256((const __m256i *)(t_coeff + 768));
__m256i b0 = _mm256_add_epi16(coeff0, coeff1);
__m256i b1 = _mm256_sub_epi16(coeff0, coeff1);
__m256i b2 = _mm256_add_epi16(coeff2, coeff3);
__m256i b3 = _mm256_sub_epi16(coeff2, coeff3);
b0 = _mm256_srai_epi16(b0, 2);
b1 = _mm256_srai_epi16(b1, 2);
b2 = _mm256_srai_epi16(b2, 2);
b3 = _mm256_srai_epi16(b3, 2);
store_tran_low(_mm256_add_epi16(b0, b2), coeff);
store_tran_low(_mm256_add_epi16(b1, b3), coeff + 256);
store_tran_low(_mm256_sub_epi16(b0, b2), coeff + 512);
store_tran_low(_mm256_sub_epi16(b1, b3), coeff + 768);
coeff += 16;
t_coeff += 16;
}
}
int aom_satd_avx2(const tran_low_t *coeff, int length) {
const __m256i one = _mm256_set1_epi16(1);
__m256i accum = _mm256_setzero_si256();
int i;
for (i = 0; i < length; i += 16) {
const __m256i src_line = load_tran_low(coeff);
const __m256i abs = _mm256_abs_epi16(src_line);
const __m256i sum = _mm256_madd_epi16(abs, one);
accum = _mm256_add_epi32(accum, sum);
coeff += 16;
}
{ // 32 bit horizontal add
const __m256i a = _mm256_srli_si256(accum, 8);
const __m256i b = _mm256_add_epi32(accum, a);
const __m256i c = _mm256_srli_epi64(b, 32);
const __m256i d = _mm256_add_epi32(b, c);
const __m128i accum_128 = _mm_add_epi32(_mm256_castsi256_si128(d),
_mm256_extractf128_si256(d, 1));
return _mm_cvtsi128_si32(accum_128);
}
}

285
third_party/aom/aom_dsp/x86/avg_intrin_sse2.c vendored Normal file
View file

@ -0,0 +1,285 @@
/*
* Copyright (c) 2016, Alliance for Open Media. All rights reserved
*
* This source code is subject to the terms of the BSD 2 Clause License and
* the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
* was not distributed with this source code in the LICENSE file, you can
* obtain it at www.aomedia.org/license/software. If the Alliance for Open
* Media Patent License 1.0 was not distributed with this source code in the
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
*/
#include <immintrin.h>
#include "config/aom_dsp_rtcd.h"
#include "aom/aom_integer.h"
#include "aom_dsp/x86/bitdepth_conversion_sse2.h"
#include "aom_ports/mem.h"
static void hadamard_col8_sse2(__m128i *in, int iter) {
__m128i a0 = in[0];
__m128i a1 = in[1];
__m128i a2 = in[2];
__m128i a3 = in[3];
__m128i a4 = in[4];
__m128i a5 = in[5];
__m128i a6 = in[6];
__m128i a7 = in[7];
__m128i b0 = _mm_add_epi16(a0, a1);
__m128i b1 = _mm_sub_epi16(a0, a1);
__m128i b2 = _mm_add_epi16(a2, a3);
__m128i b3 = _mm_sub_epi16(a2, a3);
__m128i b4 = _mm_add_epi16(a4, a5);
__m128i b5 = _mm_sub_epi16(a4, a5);
__m128i b6 = _mm_add_epi16(a6, a7);
__m128i b7 = _mm_sub_epi16(a6, a7);
a0 = _mm_add_epi16(b0, b2);
a1 = _mm_add_epi16(b1, b3);
a2 = _mm_sub_epi16(b0, b2);
a3 = _mm_sub_epi16(b1, b3);
a4 = _mm_add_epi16(b4, b6);
a5 = _mm_add_epi16(b5, b7);
a6 = _mm_sub_epi16(b4, b6);
a7 = _mm_sub_epi16(b5, b7);
if (iter == 0) {
b0 = _mm_add_epi16(a0, a4);
b7 = _mm_add_epi16(a1, a5);
b3 = _mm_add_epi16(a2, a6);
b4 = _mm_add_epi16(a3, a7);
b2 = _mm_sub_epi16(a0, a4);
b6 = _mm_sub_epi16(a1, a5);
b1 = _mm_sub_epi16(a2, a6);
b5 = _mm_sub_epi16(a3, a7);
a0 = _mm_unpacklo_epi16(b0, b1);
a1 = _mm_unpacklo_epi16(b2, b3);
a2 = _mm_unpackhi_epi16(b0, b1);
a3 = _mm_unpackhi_epi16(b2, b3);
a4 = _mm_unpacklo_epi16(b4, b5);
a5 = _mm_unpacklo_epi16(b6, b7);
a6 = _mm_unpackhi_epi16(b4, b5);
a7 = _mm_unpackhi_epi16(b6, b7);
b0 = _mm_unpacklo_epi32(a0, a1);
b1 = _mm_unpacklo_epi32(a4, a5);
b2 = _mm_unpackhi_epi32(a0, a1);
b3 = _mm_unpackhi_epi32(a4, a5);
b4 = _mm_unpacklo_epi32(a2, a3);
b5 = _mm_unpacklo_epi32(a6, a7);
b6 = _mm_unpackhi_epi32(a2, a3);
b7 = _mm_unpackhi_epi32(a6, a7);
in[0] = _mm_unpacklo_epi64(b0, b1);
in[1] = _mm_unpackhi_epi64(b0, b1);
in[2] = _mm_unpacklo_epi64(b2, b3);
in[3] = _mm_unpackhi_epi64(b2, b3);
in[4] = _mm_unpacklo_epi64(b4, b5);
in[5] = _mm_unpackhi_epi64(b4, b5);
in[6] = _mm_unpacklo_epi64(b6, b7);
in[7] = _mm_unpackhi_epi64(b6, b7);
} else {
in[0] = _mm_add_epi16(a0, a4);
in[7] = _mm_add_epi16(a1, a5);
in[3] = _mm_add_epi16(a2, a6);
in[4] = _mm_add_epi16(a3, a7);
in[2] = _mm_sub_epi16(a0, a4);
in[6] = _mm_sub_epi16(a1, a5);
in[1] = _mm_sub_epi16(a2, a6);
in[5] = _mm_sub_epi16(a3, a7);
}
}
static INLINE void hadamard_8x8_sse2(const int16_t *src_diff,
ptrdiff_t src_stride, tran_low_t *coeff,
int is_final) {
__m128i src[8];
src[0] = _mm_load_si128((const __m128i *)src_diff);
src[1] = _mm_load_si128((const __m128i *)(src_diff += src_stride));
src[2] = _mm_load_si128((const __m128i *)(src_diff += src_stride));
src[3] = _mm_load_si128((const __m128i *)(src_diff += src_stride));
src[4] = _mm_load_si128((const __m128i *)(src_diff += src_stride));
src[5] = _mm_load_si128((const __m128i *)(src_diff += src_stride));
src[6] = _mm_load_si128((const __m128i *)(src_diff += src_stride));
src[7] = _mm_load_si128((const __m128i *)(src_diff += src_stride));
hadamard_col8_sse2(src, 0);
hadamard_col8_sse2(src, 1);
if (is_final) {
store_tran_low(src[0], coeff);
coeff += 8;
store_tran_low(src[1], coeff);
coeff += 8;
store_tran_low(src[2], coeff);
coeff += 8;
store_tran_low(src[3], coeff);
coeff += 8;
store_tran_low(src[4], coeff);
coeff += 8;
store_tran_low(src[5], coeff);
coeff += 8;
store_tran_low(src[6], coeff);
coeff += 8;
store_tran_low(src[7], coeff);
} else {
int16_t *coeff16 = (int16_t *)coeff;
_mm_store_si128((__m128i *)coeff16, src[0]);
coeff16 += 8;
_mm_store_si128((__m128i *)coeff16, src[1]);
coeff16 += 8;
_mm_store_si128((__m128i *)coeff16, src[2]);
coeff16 += 8;
_mm_store_si128((__m128i *)coeff16, src[3]);
coeff16 += 8;
_mm_store_si128((__m128i *)coeff16, src[4]);
coeff16 += 8;
_mm_store_si128((__m128i *)coeff16, src[5]);
coeff16 += 8;
_mm_store_si128((__m128i *)coeff16, src[6]);
coeff16 += 8;
_mm_store_si128((__m128i *)coeff16, src[7]);
}
}
void aom_hadamard_8x8_sse2(const int16_t *src_diff, ptrdiff_t src_stride,
tran_low_t *coeff) {
hadamard_8x8_sse2(src_diff, src_stride, coeff, 1);
}
static INLINE void hadamard_16x16_sse2(const int16_t *src_diff,
ptrdiff_t src_stride, tran_low_t *coeff,
int is_final) {
// For high bitdepths, it is unnecessary to store_tran_low
// (mult/unpack/store), then load_tran_low (load/pack) the same memory in the
// next stage. Output to an intermediate buffer first, then store_tran_low()
// in the final stage.
DECLARE_ALIGNED(32, int16_t, temp_coeff[16 * 16]);
int16_t *t_coeff = temp_coeff;
int16_t *coeff16 = (int16_t *)coeff;
int idx;
for (idx = 0; idx < 4; ++idx) {
const int16_t *src_ptr =
src_diff + (idx >> 1) * 8 * src_stride + (idx & 0x01) * 8;
hadamard_8x8_sse2(src_ptr, src_stride, (tran_low_t *)(t_coeff + idx * 64),
0);
}
for (idx = 0; idx < 64; idx += 8) {
__m128i coeff0 = _mm_load_si128((const __m128i *)t_coeff);
__m128i coeff1 = _mm_load_si128((const __m128i *)(t_coeff + 64));
__m128i coeff2 = _mm_load_si128((const __m128i *)(t_coeff + 128));
__m128i coeff3 = _mm_load_si128((const __m128i *)(t_coeff + 192));
__m128i b0 = _mm_add_epi16(coeff0, coeff1);
__m128i b1 = _mm_sub_epi16(coeff0, coeff1);
__m128i b2 = _mm_add_epi16(coeff2, coeff3);
__m128i b3 = _mm_sub_epi16(coeff2, coeff3);
b0 = _mm_srai_epi16(b0, 1);
b1 = _mm_srai_epi16(b1, 1);
b2 = _mm_srai_epi16(b2, 1);
b3 = _mm_srai_epi16(b3, 1);
coeff0 = _mm_add_epi16(b0, b2);
coeff1 = _mm_add_epi16(b1, b3);
coeff2 = _mm_sub_epi16(b0, b2);
coeff3 = _mm_sub_epi16(b1, b3);
if (is_final) {
store_tran_low(coeff0, coeff);
store_tran_low(coeff1, coeff + 64);
store_tran_low(coeff2, coeff + 128);
store_tran_low(coeff3, coeff + 192);
coeff += 8;
} else {
_mm_store_si128((__m128i *)coeff16, coeff0);
_mm_store_si128((__m128i *)(coeff16 + 64), coeff1);
_mm_store_si128((__m128i *)(coeff16 + 128), coeff2);
_mm_store_si128((__m128i *)(coeff16 + 192), coeff3);
coeff16 += 8;
}
t_coeff += 8;
}
}
void aom_hadamard_16x16_sse2(const int16_t *src_diff, ptrdiff_t src_stride,
tran_low_t *coeff) {
hadamard_16x16_sse2(src_diff, src_stride, coeff, 1);
}
void aom_hadamard_32x32_sse2(const int16_t *src_diff, ptrdiff_t src_stride,
tran_low_t *coeff) {
// For high bitdepths, it is unnecessary to store_tran_low
// (mult/unpack/store), then load_tran_low (load/pack) the same memory in the
// next stage. Output to an intermediate buffer first, then store_tran_low()
// in the final stage.
DECLARE_ALIGNED(32, int16_t, temp_coeff[32 * 32]);
int16_t *t_coeff = temp_coeff;
int idx;
for (idx = 0; idx < 4; ++idx) {
const int16_t *src_ptr =
src_diff + (idx >> 1) * 16 * src_stride + (idx & 0x01) * 16;
hadamard_16x16_sse2(src_ptr, src_stride,
(tran_low_t *)(t_coeff + idx * 256), 0);
}
for (idx = 0; idx < 256; idx += 8) {
__m128i coeff0 = _mm_load_si128((const __m128i *)t_coeff);
__m128i coeff1 = _mm_load_si128((const __m128i *)(t_coeff + 256));
__m128i coeff2 = _mm_load_si128((const __m128i *)(t_coeff + 512));
__m128i coeff3 = _mm_load_si128((const __m128i *)(t_coeff + 768));
__m128i b0 = _mm_add_epi16(coeff0, coeff1);
__m128i b1 = _mm_sub_epi16(coeff0, coeff1);
__m128i b2 = _mm_add_epi16(coeff2, coeff3);
__m128i b3 = _mm_sub_epi16(coeff2, coeff3);
b0 = _mm_srai_epi16(b0, 2);
b1 = _mm_srai_epi16(b1, 2);
b2 = _mm_srai_epi16(b2, 2);
b3 = _mm_srai_epi16(b3, 2);
coeff0 = _mm_add_epi16(b0, b2);
coeff1 = _mm_add_epi16(b1, b3);
store_tran_low(coeff0, coeff);
store_tran_low(coeff1, coeff + 256);
coeff2 = _mm_sub_epi16(b0, b2);
coeff3 = _mm_sub_epi16(b1, b3);
store_tran_low(coeff2, coeff + 512);
store_tran_low(coeff3, coeff + 768);
coeff += 8;
t_coeff += 8;
}
}
int aom_satd_sse2(const tran_low_t *coeff, int length) {
int i;
const __m128i zero = _mm_setzero_si128();
__m128i accum = zero;
for (i = 0; i < length; i += 8) {
const __m128i src_line = load_tran_low(coeff);
const __m128i inv = _mm_sub_epi16(zero, src_line);
const __m128i abs = _mm_max_epi16(src_line, inv); // abs(src_line)
const __m128i abs_lo = _mm_unpacklo_epi16(abs, zero);
const __m128i abs_hi = _mm_unpackhi_epi16(abs, zero);
const __m128i sum = _mm_add_epi32(abs_lo, abs_hi);
accum = _mm_add_epi32(accum, sum);
coeff += 8;
}
{ // cascading summation of accum
__m128i hi = _mm_srli_si128(accum, 8);
accum = _mm_add_epi32(accum, hi);
hi = _mm_srli_epi64(accum, 32);
accum = _mm_add_epi32(accum, hi);
}
return _mm_cvtsi128_si32(accum);
}

32
third_party/aom/aom_dsp/x86/bitdepth_conversion_avx2.h vendored Normal file
View file

@ -0,0 +1,32 @@
/*
* Copyright (c) 2016, Alliance for Open Media. All rights reserved
*
* This source code is subject to the terms of the BSD 2 Clause License and
* the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
* was not distributed with this source code in the LICENSE file, you can
* obtain it at www.aomedia.org/license/software. If the Alliance for Open
* Media Patent License 1.0 was not distributed with this source code in the
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
*/
#include <immintrin.h>
#include "config/aom_config.h"
#include "aom/aom_integer.h"
#include "aom_dsp/aom_dsp_common.h"
static INLINE __m256i load_tran_low(const tran_low_t *a) {
const __m256i a_low = _mm256_loadu_si256((const __m256i *)a);
const __m256i a_high = _mm256_loadu_si256((const __m256i *)(a + 8));
return _mm256_packs_epi32(a_low, a_high);
}
static INLINE void store_tran_low(__m256i a, tran_low_t *b) {
const __m256i one = _mm256_set1_epi16(1);
const __m256i a_hi = _mm256_mulhi_epi16(a, one);
const __m256i a_lo = _mm256_mullo_epi16(a, one);
const __m256i a_1 = _mm256_unpacklo_epi16(a_lo, a_hi);
const __m256i a_2 = _mm256_unpackhi_epi16(a_lo, a_hi);
_mm256_storeu_si256((__m256i *)b, a_1);
_mm256_storeu_si256((__m256i *)(b + 8), a_2);
}

35
third_party/aom/aom_dsp/x86/bitdepth_conversion_sse2.h vendored Normal file
View file

@ -0,0 +1,35 @@
/*
* Copyright (c) 2016, Alliance for Open Media. All rights reserved
*
* This source code is subject to the terms of the BSD 2 Clause License and
* the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
* was not distributed with this source code in the LICENSE file, you can
* obtain it at www.aomedia.org/license/software. If the Alliance for Open
* Media Patent License 1.0 was not distributed with this source code in the
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
*/
#include <xmmintrin.h>
#include "config/aom_config.h"
#include "aom/aom_integer.h"
#include "aom_dsp/aom_dsp_common.h"
// Load 8 16 bit values. If the source is 32 bits then pack down with
// saturation.
static INLINE __m128i load_tran_low(const tran_low_t *a) {
const __m128i a_low = _mm_load_si128((const __m128i *)a);
return _mm_packs_epi32(a_low, *(const __m128i *)(a + 4));
}
// Store 8 16 bit values. If the destination is 32 bits then sign extend the
// values by multiplying by 1.
static INLINE void store_tran_low(__m128i a, tran_low_t *b) {
const __m128i one = _mm_set1_epi16(1);
const __m128i a_hi = _mm_mulhi_epi16(a, one);
const __m128i a_lo = _mm_mullo_epi16(a, one);
const __m128i a_1 = _mm_unpacklo_epi16(a_lo, a_hi);
const __m128i a_2 = _mm_unpackhi_epi16(a_lo, a_hi);
_mm_store_si128((__m128i *)(b), a_1);
_mm_store_si128((__m128i *)(b + 4), a_2);
}

472
third_party/aom/aom_dsp/x86/blend_a64_mask_avx2.c vendored
View file

@ -898,3 +898,475 @@ void aom_blend_a64_mask_avx2(uint8_t *dst, uint32_t dst_stride,
}
}
}
//////////////////////////////////////////////////////////////////////////////
// aom_highbd_blend_a64_d16_mask_avx2()
//////////////////////////////////////////////////////////////////////////////
static INLINE void highbd_blend_a64_d16_mask_w4_avx2(
uint16_t *dst, int dst_stride, const CONV_BUF_TYPE *src0, int src0_stride,
const CONV_BUF_TYPE *src1, int src1_stride, const __m256i *mask0,
const __m256i *round_offset, int shift, const __m256i *clip_low,
const __m256i *clip_high, const __m256i *mask_max) {
// Load 4x u16 pixels from each of 4 rows from each source
const __m256i s0 = _mm256_set_epi64x(*(uint64_t *)(src0 + 3 * src0_stride),
*(uint64_t *)(src0 + 2 * src0_stride),
*(uint64_t *)(src0 + 1 * src0_stride),
*(uint64_t *)(src0 + 0 * src0_stride));
const __m256i s1 = _mm256_set_epi64x(*(uint64_t *)(src1 + 3 * src1_stride),
*(uint64_t *)(src1 + 2 * src1_stride),
*(uint64_t *)(src1 + 1 * src1_stride),
*(uint64_t *)(src1 + 0 * src1_stride));
// Generate the inverse mask
const __m256i mask1 = _mm256_sub_epi16(*mask_max, *mask0);
// Multiply each mask by the respective source
const __m256i mul0_highs = _mm256_mulhi_epu16(*mask0, s0);
const __m256i mul0_lows = _mm256_mullo_epi16(*mask0, s0);
const __m256i mul0h = _mm256_unpackhi_epi16(mul0_lows, mul0_highs);
const __m256i mul0l = _mm256_unpacklo_epi16(mul0_lows, mul0_highs);
// Note that AVX2 unpack orders 64-bit words as [3 1] [2 0] to keep within
// lanes Later, packs does the same again which cancels this out with no need
// for a permute. The intermediate values being reordered makes no difference
const __m256i mul1_highs = _mm256_mulhi_epu16(mask1, s1);
const __m256i mul1_lows = _mm256_mullo_epi16(mask1, s1);
const __m256i mul1h = _mm256_unpackhi_epi16(mul1_lows, mul1_highs);
const __m256i mul1l = _mm256_unpacklo_epi16(mul1_lows, mul1_highs);
const __m256i sumh = _mm256_add_epi32(mul0h, mul1h);
const __m256i suml = _mm256_add_epi32(mul0l, mul1l);
const __m256i roundh =
_mm256_srai_epi32(_mm256_sub_epi32(sumh, *round_offset), shift);
const __m256i roundl =
_mm256_srai_epi32(_mm256_sub_epi32(suml, *round_offset), shift);
const __m256i pack = _mm256_packs_epi32(roundl, roundh);
const __m256i clip =
_mm256_min_epi16(_mm256_max_epi16(pack, *clip_low), *clip_high);
// _mm256_extract_epi64 doesn't exist on x86, so do it the old-fashioned way:
const __m128i cliph = _mm256_extracti128_si256(clip, 1);
xx_storel_64(dst + 3 * dst_stride, _mm_srli_si128(cliph, 8));
xx_storel_64(dst + 2 * dst_stride, cliph);
const __m128i clipl = _mm256_castsi256_si128(clip);
xx_storel_64(dst + 1 * dst_stride, _mm_srli_si128(clipl, 8));
xx_storel_64(dst + 0 * dst_stride, clipl);
}
static INLINE void highbd_blend_a64_d16_mask_subw0_subh0_w4_avx2(
uint16_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0,
uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride,
const uint8_t *mask, uint32_t mask_stride, int h,
const __m256i *round_offset, int shift, const __m256i *clip_low,
const __m256i *clip_high, const __m256i *mask_max) {
do {
// Load 8x u8 pixels from each of 4 rows of the mask, pad each to u16
const __m128i mask08 = _mm_set_epi32(*(uint32_t *)(mask + 3 * mask_stride),
*(uint32_t *)(mask + 2 * mask_stride),
*(uint32_t *)(mask + 1 * mask_stride),
*(uint32_t *)(mask + 0 * mask_stride));
const __m256i mask0 = _mm256_cvtepu8_epi16(mask08);
highbd_blend_a64_d16_mask_w4_avx2(dst, dst_stride, src0, src0_stride, src1,
src1_stride, &mask0, round_offset, shift,
clip_low, clip_high, mask_max);
dst += dst_stride * 4;
src0 += src0_stride * 4;
src1 += src1_stride * 4;
mask += mask_stride * 4;
} while (h -= 4);
}
static INLINE void highbd_blend_a64_d16_mask_subw1_subh1_w4_avx2(
uint16_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0,
uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride,
const uint8_t *mask, uint32_t mask_stride, int h,
const __m256i *round_offset, int shift, const __m256i *clip_low,
const __m256i *clip_high, const __m256i *mask_max) {
const __m256i one_b = _mm256_set1_epi8(1);
const __m256i two_w = _mm256_set1_epi16(2);
do {
// Load 8 pixels from each of 8 rows of mask,
// (saturating) add together rows then use madd to add adjacent pixels
// Finally, divide each value by 4 (with rounding)
const __m256i m0246 =
_mm256_set_epi64x(*(uint64_t *)(mask + 6 * mask_stride),
*(uint64_t *)(mask + 4 * mask_stride),
*(uint64_t *)(mask + 2 * mask_stride),
*(uint64_t *)(mask + 0 * mask_stride));
const __m256i m1357 =
_mm256_set_epi64x(*(uint64_t *)(mask + 7 * mask_stride),
*(uint64_t *)(mask + 5 * mask_stride),
*(uint64_t *)(mask + 3 * mask_stride),
*(uint64_t *)(mask + 1 * mask_stride));
const __m256i addrows = _mm256_adds_epu8(m0246, m1357);
const __m256i adjacent = _mm256_maddubs_epi16(addrows, one_b);
const __m256i mask0 =
_mm256_srli_epi16(_mm256_add_epi16(adjacent, two_w), 2);
highbd_blend_a64_d16_mask_w4_avx2(dst, dst_stride, src0, src0_stride, src1,
src1_stride, &mask0, round_offset, shift,
clip_low, clip_high, mask_max);
dst += dst_stride * 4;
src0 += src0_stride * 4;
src1 += src1_stride * 4;
mask += mask_stride * 8;
} while (h -= 4);
}
static INLINE void highbd_blend_a64_d16_mask_w8_avx2(
uint16_t *dst, int dst_stride, const CONV_BUF_TYPE *src0, int src0_stride,
const CONV_BUF_TYPE *src1, int src1_stride, const __m256i *mask0a,
const __m256i *mask0b, const __m256i *round_offset, int shift,
const __m256i *clip_low, const __m256i *clip_high,
const __m256i *mask_max) {
// Load 8x u16 pixels from each of 4 rows from each source
const __m256i s0a =
yy_loadu2_128(src0 + 0 * src0_stride, src0 + 1 * src0_stride);
const __m256i s0b =
yy_loadu2_128(src0 + 2 * src0_stride, src0 + 3 * src0_stride);
const __m256i s1a =
yy_loadu2_128(src1 + 0 * src1_stride, src1 + 1 * src1_stride);
const __m256i s1b =
yy_loadu2_128(src1 + 2 * src1_stride, src1 + 3 * src1_stride);
// Generate inverse masks
const __m256i mask1a = _mm256_sub_epi16(*mask_max, *mask0a);
const __m256i mask1b = _mm256_sub_epi16(*mask_max, *mask0b);
// Multiply sources by respective masks
const __m256i mul0a_highs = _mm256_mulhi_epu16(*mask0a, s0a);
const __m256i mul0a_lows = _mm256_mullo_epi16(*mask0a, s0a);
const __m256i mul0ah = _mm256_unpackhi_epi16(mul0a_lows, mul0a_highs);
const __m256i mul0al = _mm256_unpacklo_epi16(mul0a_lows, mul0a_highs);
// Note that AVX2 unpack orders 64-bit words as [3 1] [2 0] to keep within
// lanes Later, packs does the same again which cancels this out with no need
// for a permute. The intermediate values being reordered makes no difference
const __m256i mul1a_highs = _mm256_mulhi_epu16(mask1a, s1a);
const __m256i mul1a_lows = _mm256_mullo_epi16(mask1a, s1a);
const __m256i mul1ah = _mm256_unpackhi_epi16(mul1a_lows, mul1a_highs);
const __m256i mul1al = _mm256_unpacklo_epi16(mul1a_lows, mul1a_highs);
const __m256i sumah = _mm256_add_epi32(mul0ah, mul1ah);
const __m256i sumal = _mm256_add_epi32(mul0al, mul1al);
const __m256i mul0b_highs = _mm256_mulhi_epu16(*mask0b, s0b);
const __m256i mul0b_lows = _mm256_mullo_epi16(*mask0b, s0b);
const __m256i mul0bh = _mm256_unpackhi_epi16(mul0b_lows, mul0b_highs);
const __m256i mul0bl = _mm256_unpacklo_epi16(mul0b_lows, mul0b_highs);
const __m256i mul1b_highs = _mm256_mulhi_epu16(mask1b, s1b);
const __m256i mul1b_lows = _mm256_mullo_epi16(mask1b, s1b);
const __m256i mul1bh = _mm256_unpackhi_epi16(mul1b_lows, mul1b_highs);
const __m256i mul1bl = _mm256_unpacklo_epi16(mul1b_lows, mul1b_highs);
const __m256i sumbh = _mm256_add_epi32(mul0bh, mul1bh);
const __m256i sumbl = _mm256_add_epi32(mul0bl, mul1bl);
// Divide down each result, with rounding
const __m256i roundah =
_mm256_srai_epi32(_mm256_sub_epi32(sumah, *round_offset), shift);
const __m256i roundal =
_mm256_srai_epi32(_mm256_sub_epi32(sumal, *round_offset), shift);
const __m256i roundbh =
_mm256_srai_epi32(_mm256_sub_epi32(sumbh, *round_offset), shift);
const __m256i roundbl =
_mm256_srai_epi32(_mm256_sub_epi32(sumbl, *round_offset), shift);
// Pack each i32 down to an i16 with saturation, then clip to valid range
const __m256i packa = _mm256_packs_epi32(roundal, roundah);
const __m256i clipa =
_mm256_min_epi16(_mm256_max_epi16(packa, *clip_low), *clip_high);
const __m256i packb = _mm256_packs_epi32(roundbl, roundbh);
const __m256i clipb =
_mm256_min_epi16(_mm256_max_epi16(packb, *clip_low), *clip_high);
// Store 8x u16 pixels to each of 4 rows in the destination
yy_storeu2_128(dst + 0 * dst_stride, dst + 1 * dst_stride, clipa);
yy_storeu2_128(dst + 2 * dst_stride, dst + 3 * dst_stride, clipb);
}
static INLINE void highbd_blend_a64_d16_mask_subw0_subh0_w8_avx2(
uint16_t *dst, int dst_stride, const CONV_BUF_TYPE *src0, int src0_stride,
const CONV_BUF_TYPE *src1, int src1_stride, const uint8_t *mask,
int mask_stride, int h, const __m256i *round_offset, int shift,
const __m256i *clip_low, const __m256i *clip_high,
const __m256i *mask_max) {
do {
// Load 8x u8 pixels from each of 4 rows in the mask
const __m128i mask0a8 =
_mm_set_epi64x(*(uint64_t *)mask, *(uint64_t *)(mask + mask_stride));
const __m128i mask0b8 =
_mm_set_epi64x(*(uint64_t *)(mask + 2 * mask_stride),
*(uint64_t *)(mask + 3 * mask_stride));
const __m256i mask0a = _mm256_cvtepu8_epi16(mask0a8);
const __m256i mask0b = _mm256_cvtepu8_epi16(mask0b8);
highbd_blend_a64_d16_mask_w8_avx2(
dst, dst_stride, src0, src0_stride, src1, src1_stride, &mask0a, &mask0b,
round_offset, shift, clip_low, clip_high, mask_max);
dst += dst_stride * 4;
src0 += src0_stride * 4;
src1 += src1_stride * 4;
mask += mask_stride * 4;
} while (h -= 4);
}
static INLINE void highbd_blend_a64_d16_mask_subw1_subh1_w8_avx2(
uint16_t *dst, int dst_stride, const CONV_BUF_TYPE *src0, int src0_stride,
const CONV_BUF_TYPE *src1, int src1_stride, const uint8_t *mask,
int mask_stride, int h, const __m256i *round_offset, int shift,
const __m256i *clip_low, const __m256i *clip_high,
const __m256i *mask_max) {
const __m256i one_b = _mm256_set1_epi8(1);
const __m256i two_w = _mm256_set1_epi16(2);
do {
// Load 16x u8 pixels from each of 8 rows in the mask,
// (saturating) add together rows then use madd to add adjacent pixels
// Finally, divide each value by 4 (with rounding)
const __m256i m02 =
yy_loadu2_128(mask + 0 * mask_stride, mask + 2 * mask_stride);
const __m256i m13 =
yy_loadu2_128(mask + 1 * mask_stride, mask + 3 * mask_stride);
const __m256i m0123 =
_mm256_maddubs_epi16(_mm256_adds_epu8(m02, m13), one_b);
const __m256i mask_0a =
_mm256_srli_epi16(_mm256_add_epi16(m0123, two_w), 2);
const __m256i m46 =
yy_loadu2_128(mask + 4 * mask_stride, mask + 6 * mask_stride);
const __m256i m57 =
yy_loadu2_128(mask + 5 * mask_stride, mask + 7 * mask_stride);
const __m256i m4567 =
_mm256_maddubs_epi16(_mm256_adds_epu8(m46, m57), one_b);
const __m256i mask_0b =
_mm256_srli_epi16(_mm256_add_epi16(m4567, two_w), 2);
highbd_blend_a64_d16_mask_w8_avx2(
dst, dst_stride, src0, src0_stride, src1, src1_stride, &mask_0a,
&mask_0b, round_offset, shift, clip_low, clip_high, mask_max);
dst += dst_stride * 4;
src0 += src0_stride * 4;
src1 += src1_stride * 4;
mask += mask_stride * 8;
} while (h -= 4);
}
static INLINE void highbd_blend_a64_d16_mask_w16_avx2(
uint16_t *dst, int dst_stride, const CONV_BUF_TYPE *src0, int src0_stride,
const CONV_BUF_TYPE *src1, int src1_stride, const __m256i *mask0a,
const __m256i *mask0b, const __m256i *round_offset, int shift,
const __m256i *clip_low, const __m256i *clip_high,
const __m256i *mask_max) {
// Load 16x pixels from each of 2 rows from each source
const __m256i s0a = yy_loadu_256(src0);
const __m256i s0b = yy_loadu_256(src0 + src0_stride);
const __m256i s1a = yy_loadu_256(src1);
const __m256i s1b = yy_loadu_256(src1 + src1_stride);
// Calculate inverse masks
const __m256i mask1a = _mm256_sub_epi16(*mask_max, *mask0a);
const __m256i mask1b = _mm256_sub_epi16(*mask_max, *mask0b);
// Multiply each source by appropriate mask
const __m256i mul0a_highs = _mm256_mulhi_epu16(*mask0a, s0a);
const __m256i mul0a_lows = _mm256_mullo_epi16(*mask0a, s0a);
const __m256i mul0ah = _mm256_unpackhi_epi16(mul0a_lows, mul0a_highs);
const __m256i mul0al = _mm256_unpacklo_epi16(mul0a_lows, mul0a_highs);
// Note that AVX2 unpack orders 64-bit words as [3 1] [2 0] to keep within
// lanes Later, packs does the same again which cancels this out with no need
// for a permute. The intermediate values being reordered makes no difference
const __m256i mul1a_highs = _mm256_mulhi_epu16(mask1a, s1a);
const __m256i mul1a_lows = _mm256_mullo_epi16(mask1a, s1a);
const __m256i mul1ah = _mm256_unpackhi_epi16(mul1a_lows, mul1a_highs);
const __m256i mul1al = _mm256_unpacklo_epi16(mul1a_lows, mul1a_highs);
const __m256i mulah = _mm256_add_epi32(mul0ah, mul1ah);
const __m256i mulal = _mm256_add_epi32(mul0al, mul1al);
const __m256i mul0b_highs = _mm256_mulhi_epu16(*mask0b, s0b);
const __m256i mul0b_lows = _mm256_mullo_epi16(*mask0b, s0b);
const __m256i mul0bh = _mm256_unpackhi_epi16(mul0b_lows, mul0b_highs);
const __m256i mul0bl = _mm256_unpacklo_epi16(mul0b_lows, mul0b_highs);
const __m256i mul1b_highs = _mm256_mulhi_epu16(mask1b, s1b);
const __m256i mul1b_lows = _mm256_mullo_epi16(mask1b, s1b);
const __m256i mul1bh = _mm256_unpackhi_epi16(mul1b_lows, mul1b_highs);
const __m256i mul1bl = _mm256_unpacklo_epi16(mul1b_lows, mul1b_highs);
const __m256i mulbh = _mm256_add_epi32(mul0bh, mul1bh);
const __m256i mulbl = _mm256_add_epi32(mul0bl, mul1bl);
const __m256i resah =
_mm256_srai_epi32(_mm256_sub_epi32(mulah, *round_offset), shift);
const __m256i resal =
_mm256_srai_epi32(_mm256_sub_epi32(mulal, *round_offset), shift);
const __m256i resbh =
_mm256_srai_epi32(_mm256_sub_epi32(mulbh, *round_offset), shift);
const __m256i resbl =
_mm256_srai_epi32(_mm256_sub_epi32(mulbl, *round_offset), shift);
// Signed saturating pack from i32 to i16:
const __m256i packa = _mm256_packs_epi32(resal, resah);
const __m256i packb = _mm256_packs_epi32(resbl, resbh);
// Clip the values to the valid range
const __m256i clipa =
_mm256_min_epi16(_mm256_max_epi16(packa, *clip_low), *clip_high);
const __m256i clipb =
_mm256_min_epi16(_mm256_max_epi16(packb, *clip_low), *clip_high);
// Store 16 pixels
yy_storeu_256(dst, clipa);
yy_storeu_256(dst + dst_stride, clipb);
}
static INLINE void highbd_blend_a64_d16_mask_subw0_subh0_w16_avx2(
uint16_t *dst, int dst_stride, const CONV_BUF_TYPE *src0, int src0_stride,
const CONV_BUF_TYPE *src1, int src1_stride, const uint8_t *mask,
int mask_stride, int h, int w, const __m256i *round_offset, int shift,
const __m256i *clip_low, const __m256i *clip_high,
const __m256i *mask_max) {
for (int i = 0; i < h; i += 2) {
for (int j = 0; j < w; j += 16) {
// Load 16x u8 alpha-mask values from each of two rows and pad to u16
const __m128i masks_a8 = xx_loadu_128(mask + j);
const __m128i masks_b8 = xx_loadu_128(mask + mask_stride + j);
const __m256i mask0a = _mm256_cvtepu8_epi16(masks_a8);
const __m256i mask0b = _mm256_cvtepu8_epi16(masks_b8);
highbd_blend_a64_d16_mask_w16_avx2(
dst + j, dst_stride, src0 + j, src0_stride, src1 + j, src1_stride,
&mask0a, &mask0b, round_offset, shift, clip_low, clip_high, mask_max);
}
dst += dst_stride * 2;
src0 += src0_stride * 2;
src1 += src1_stride * 2;
mask += mask_stride * 2;
}
}
static INLINE void highbd_blend_a64_d16_mask_subw1_subh1_w16_avx2(
uint16_t *dst, int dst_stride, const CONV_BUF_TYPE *src0, int src0_stride,
const CONV_BUF_TYPE *src1, int src1_stride, const uint8_t *mask,
int mask_stride, int h, int w, const __m256i *round_offset, int shift,
const __m256i *clip_low, const __m256i *clip_high,
const __m256i *mask_max) {
const __m256i one_b = _mm256_set1_epi8(1);
const __m256i two_w = _mm256_set1_epi16(2);
for (int i = 0; i < h; i += 2) {
for (int j = 0; j < w; j += 16) {
// Load 32x u8 alpha-mask values from each of four rows
// (saturating) add pairs of rows, then use madd to add adjacent values
// Finally, divide down each result with rounding
const __m256i m0 = yy_loadu_256(mask + 0 * mask_stride + 2 * j);
const __m256i m1 = yy_loadu_256(mask + 1 * mask_stride + 2 * j);
const __m256i m2 = yy_loadu_256(mask + 2 * mask_stride + 2 * j);
const __m256i m3 = yy_loadu_256(mask + 3 * mask_stride + 2 * j);
const __m256i m01_8 = _mm256_adds_epu8(m0, m1);
const __m256i m23_8 = _mm256_adds_epu8(m2, m3);
const __m256i m01 = _mm256_maddubs_epi16(m01_8, one_b);
const __m256i m23 = _mm256_maddubs_epi16(m23_8, one_b);
const __m256i mask0a = _mm256_srli_epi16(_mm256_add_epi16(m01, two_w), 2);
const __m256i mask0b = _mm256_srli_epi16(_mm256_add_epi16(m23, two_w), 2);
highbd_blend_a64_d16_mask_w16_avx2(
dst + j, dst_stride, src0 + j, src0_stride, src1 + j, src1_stride,
&mask0a, &mask0b, round_offset, shift, clip_low, clip_high, mask_max);
}
dst += dst_stride * 2;
src0 += src0_stride * 2;
src1 += src1_stride * 2;
mask += mask_stride * 4;
}
}
void aom_highbd_blend_a64_d16_mask_avx2(
uint8_t *dst8, uint32_t dst_stride, const CONV_BUF_TYPE *src0,
uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride,
const uint8_t *mask, uint32_t mask_stride, int w, int h, int subw, int subh,
ConvolveParams *conv_params, const int bd) {
uint16_t *dst = CONVERT_TO_SHORTPTR(dst8);
const int round_bits =
2 * FILTER_BITS - conv_params->round_0 - conv_params->round_1;
const int32_t round_offset =
((1 << (round_bits + bd)) + (1 << (round_bits + bd - 1)) -
(1 << (round_bits - 1)))
<< AOM_BLEND_A64_ROUND_BITS;
const __m256i v_round_offset = _mm256_set1_epi32(round_offset);
const int shift = round_bits + AOM_BLEND_A64_ROUND_BITS;
const __m256i clip_low = _mm256_set1_epi16(0);
const __m256i clip_high = _mm256_set1_epi16((1 << bd) - 1);
const __m256i mask_max = _mm256_set1_epi16(AOM_BLEND_A64_MAX_ALPHA);
assert(IMPLIES((void *)src0 == dst, src0_stride == dst_stride));
assert(IMPLIES((void *)src1 == dst, src1_stride == dst_stride));
assert(h >= 4);
assert(w >= 4);
assert(IS_POWER_OF_TWO(h));
assert(IS_POWER_OF_TWO(w));
if (subw == 0 && subh == 0) {
switch (w) {
case 4:
highbd_blend_a64_d16_mask_subw0_subh0_w4_avx2(
dst, dst_stride, src0, src0_stride, src1, src1_stride, mask,
mask_stride, h, &v_round_offset, shift, &clip_low, &clip_high,
&mask_max);
break;
case 8:
highbd_blend_a64_d16_mask_subw0_subh0_w8_avx2(
dst, dst_stride, src0, src0_stride, src1, src1_stride, mask,
mask_stride, h, &v_round_offset, shift, &clip_low, &clip_high,
&mask_max);
break;
default: // >= 16
highbd_blend_a64_d16_mask_subw0_subh0_w16_avx2(
dst, dst_stride, src0, src0_stride, src1, src1_stride, mask,
mask_stride, h, w, &v_round_offset, shift, &clip_low, &clip_high,
&mask_max);
break;
}
} else if (subw == 1 && subh == 1) {
switch (w) {
case 4:
highbd_blend_a64_d16_mask_subw1_subh1_w4_avx2(
dst, dst_stride, src0, src0_stride, src1, src1_stride, mask,
mask_stride, h, &v_round_offset, shift, &clip_low, &clip_high,
&mask_max);
break;
case 8:
highbd_blend_a64_d16_mask_subw1_subh1_w8_avx2(
dst, dst_stride, src0, src0_stride, src1, src1_stride, mask,
mask_stride, h, &v_round_offset, shift, &clip_low, &clip_high,
&mask_max);
break;
default: // >= 16
highbd_blend_a64_d16_mask_subw1_subh1_w16_avx2(
dst, dst_stride, src0, src0_stride, src1, src1_stride, mask,
mask_stride, h, w, &v_round_offset, shift, &clip_low, &clip_high,
&mask_max);
break;
}
} else {
// Sub-sampling in only one axis doesn't seem to happen very much, so fall
// back to the vanilla C implementation instead of having all the optimised
// code for these.
aom_highbd_blend_a64_d16_mask_c(dst8, dst_stride, src0, src0_stride, src1,
src1_stride, mask, mask_stride, w, h, subw,
subh, conv_params, bd);
}
}

449
third_party/aom/aom_dsp/x86/blend_a64_mask_sse4.c vendored
View file

@ -1107,3 +1107,452 @@ void aom_lowbd_blend_a64_d16_mask_sse4_1(
}
}
}
//////////////////////////////////////////////////////////////////////////////
// aom_highbd_blend_a64_d16_mask_sse4_1()
//////////////////////////////////////////////////////////////////////////////
static INLINE void highbd_blend_a64_d16_mask_w4_sse4_1(
uint16_t *dst, int dst_stride, const CONV_BUF_TYPE *src0, int src0_stride,
const CONV_BUF_TYPE *src1, int src1_stride, const __m128i *mask0a,
const __m128i *mask0b, const __m128i *round_offset, int shift,
const __m128i *clip_low, const __m128i *clip_high,
const __m128i *mask_max) {
// Load 4 pixels from each of 4 rows from each source
const __m128i s0a =
_mm_set_epi64x(*(uint64_t *)src0, *(uint64_t *)(src0 + src0_stride));
const __m128i s0b = _mm_set_epi64x(*(uint64_t *)(src0 + 2 * src0_stride),
*(uint64_t *)(src0 + 3 * src0_stride));
const __m128i s1a =
_mm_set_epi64x(*(uint64_t *)(src1), *(uint64_t *)(src1 + src1_stride));
const __m128i s1b = _mm_set_epi64x(*(uint64_t *)(src1 + 2 * src1_stride),
*(uint64_t *)(src1 + 3 * src1_stride));
// Generate the inverse masks
const __m128i mask1a = _mm_sub_epi16(*mask_max, *mask0a);
const __m128i mask1b = _mm_sub_epi16(*mask_max, *mask0b);
// Multiply each mask by the respective source
const __m128i mul0a_highs = _mm_mulhi_epu16(*mask0a, s0a);
const __m128i mul0a_lows = _mm_mullo_epi16(*mask0a, s0a);
const __m128i mul0ah = _mm_unpackhi_epi16(mul0a_lows, mul0a_highs);
const __m128i mul0al = _mm_unpacklo_epi16(mul0a_lows, mul0a_highs);
const __m128i mul1a_highs = _mm_mulhi_epu16(mask1a, s1a);
const __m128i mul1a_lows = _mm_mullo_epi16(mask1a, s1a);
const __m128i mul1ah = _mm_unpackhi_epi16(mul1a_lows, mul1a_highs);
const __m128i mul1al = _mm_unpacklo_epi16(mul1a_lows, mul1a_highs);
const __m128i mul0b_highs = _mm_mulhi_epu16(*mask0b, s0b);
const __m128i mul0b_lows = _mm_mullo_epi16(*mask0b, s0b);
const __m128i mul0bh = _mm_unpackhi_epi16(mul0b_lows, mul0b_highs);
const __m128i mul0bl = _mm_unpacklo_epi16(mul0b_lows, mul0b_highs);
const __m128i mul1b_highs = _mm_mulhi_epu16(mask1b, s1b);
const __m128i mul1b_lows = _mm_mullo_epi16(mask1b, s1b);
const __m128i mul1bh = _mm_unpackhi_epi16(mul1b_lows, mul1b_highs);
const __m128i mul1bl = _mm_unpacklo_epi16(mul1b_lows, mul1b_highs);
const __m128i sumah = _mm_add_epi32(mul0ah, mul1ah);
const __m128i sumal = _mm_add_epi32(mul0al, mul1al);
const __m128i sumbh = _mm_add_epi32(mul0bh, mul1bh);
const __m128i sumbl = _mm_add_epi32(mul0bl, mul1bl);
const __m128i roundah =
_mm_srai_epi32(_mm_sub_epi32(sumah, *round_offset), shift);
const __m128i roundbh =
_mm_srai_epi32(_mm_sub_epi32(sumbh, *round_offset), shift);
const __m128i roundal =
_mm_srai_epi32(_mm_sub_epi32(sumal, *round_offset), shift);
const __m128i roundbl =
_mm_srai_epi32(_mm_sub_epi32(sumbl, *round_offset), shift);
const __m128i packa = _mm_packs_epi32(roundal, roundah);
const __m128i packb = _mm_packs_epi32(roundbl, roundbh);
const __m128i clipa =
_mm_min_epi16(_mm_max_epi16(packa, *clip_low), *clip_high);
const __m128i clipb =
_mm_min_epi16(_mm_max_epi16(packb, *clip_low), *clip_high);
xx_storel_64(dst, _mm_srli_si128(clipa, 8));
xx_storel_64(dst + dst_stride, clipa);
xx_storel_64(dst + 2 * dst_stride, _mm_srli_si128(clipb, 8));
xx_storel_64(dst + 3 * dst_stride, clipb);
}
static INLINE void highbd_blend_a64_d16_mask_subw0_subh0_w4_sse4_1(
uint16_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0,
uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride,
const uint8_t *mask, uint32_t mask_stride, int h,
const __m128i *round_offset, int shift, const __m128i *clip_low,
const __m128i *clip_high, const __m128i *mask_max) {
do {
const __m128i mask0a8 = _mm_set_epi32(0, 0, *(uint32_t *)mask,
*(uint32_t *)(mask + mask_stride));
const __m128i mask0b8 =
_mm_set_epi32(0, 0, *(uint32_t *)(mask + 2 * mask_stride),
*(uint32_t *)(mask + 3 * mask_stride));
const __m128i mask0a = _mm_cvtepu8_epi16(mask0a8);
const __m128i mask0b = _mm_cvtepu8_epi16(mask0b8);
highbd_blend_a64_d16_mask_w4_sse4_1(
dst, dst_stride, src0, src0_stride, src1, src1_stride, &mask0a, &mask0b,
round_offset, shift, clip_low, clip_high, mask_max);
dst += dst_stride * 4;
src0 += src0_stride * 4;
src1 += src1_stride * 4;
mask += mask_stride * 4;
} while (h -= 4);
}
static INLINE void highbd_blend_a64_d16_mask_subw1_subh1_w4_sse4_1(
uint16_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0,
uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride,
const uint8_t *mask, uint32_t mask_stride, int h,
const __m128i *round_offset, int shift, const __m128i *clip_low,
const __m128i *clip_high, const __m128i *mask_max) {
const __m128i one_b = _mm_set1_epi8(1);
const __m128i two_w = _mm_set1_epi16(2);
do {
// Load 8 pixels from each of 8 rows of mask,
// (saturating) add together rows then use madd to add adjacent pixels
// Finally, divide each value by 4 (with rounding)
const __m128i m02 = _mm_set_epi64x(*(uint64_t *)(mask),
*(uint64_t *)(mask + 2 * mask_stride));
const __m128i m13 = _mm_set_epi64x(*(uint64_t *)(mask + mask_stride),
*(uint64_t *)(mask + 3 * mask_stride));
const __m128i m0123 = _mm_maddubs_epi16(_mm_adds_epu8(m02, m13), one_b);
const __m128i mask_0a = _mm_srli_epi16(_mm_add_epi16(m0123, two_w), 2);
const __m128i m46 = _mm_set_epi64x(*(uint64_t *)(mask + 4 * mask_stride),
*(uint64_t *)(mask + 6 * mask_stride));
const __m128i m57 = _mm_set_epi64x(*(uint64_t *)(mask + 5 * mask_stride),
*(uint64_t *)(mask + 7 * mask_stride));
const __m128i m4567 = _mm_maddubs_epi16(_mm_adds_epu8(m46, m57), one_b);
const __m128i mask_0b = _mm_srli_epi16(_mm_add_epi16(m4567, two_w), 2);
highbd_blend_a64_d16_mask_w4_sse4_1(
dst, dst_stride, src0, src0_stride, src1, src1_stride, &mask_0a,
&mask_0b, round_offset, shift, clip_low, clip_high, mask_max);
dst += dst_stride * 4;
src0 += src0_stride * 4;
src1 += src1_stride * 4;
mask += mask_stride * 8;
} while (h -= 4);
}
static INLINE void highbd_blend_a64_d16_mask_w8_sse4_1(
uint16_t *dst, int dst_stride, const CONV_BUF_TYPE *src0, int src0_stride,
const CONV_BUF_TYPE *src1, int src1_stride, const __m128i *mask0a,
const __m128i *mask0b, const __m128i *round_offset, int shift,
const __m128i *clip_low, const __m128i *clip_high,
const __m128i *max_mask) {
// Load 8x pixels from each of 2 rows from each source
const __m128i s0a = xx_loadu_128(src0);
const __m128i s0b = xx_loadu_128(src0 + src0_stride);
const __m128i s1a = xx_loadu_128(src1);
const __m128i s1b = xx_loadu_128(src1 + src1_stride);
// Generate inverse masks
const __m128i mask1a = _mm_sub_epi16(*max_mask, *mask0a);
const __m128i mask1b = _mm_sub_epi16(*max_mask, *mask0b);
// Multiply sources by respective masks
const __m128i mul0a_highs = _mm_mulhi_epu16(*mask0a, s0a);
const __m128i mul0a_lows = _mm_mullo_epi16(*mask0a, s0a);
const __m128i mul0ah = _mm_unpackhi_epi16(mul0a_lows, mul0a_highs);
const __m128i mul0al = _mm_unpacklo_epi16(mul0a_lows, mul0a_highs);
const __m128i mul1a_highs = _mm_mulhi_epu16(mask1a, s1a);
const __m128i mul1a_lows = _mm_mullo_epi16(mask1a, s1a);
const __m128i mul1ah = _mm_unpackhi_epi16(mul1a_lows, mul1a_highs);
const __m128i mul1al = _mm_unpacklo_epi16(mul1a_lows, mul1a_highs);
const __m128i sumah = _mm_add_epi32(mul0ah, mul1ah);
const __m128i sumal = _mm_add_epi32(mul0al, mul1al);
const __m128i mul0b_highs = _mm_mulhi_epu16(*mask0b, s0b);
const __m128i mul0b_lows = _mm_mullo_epi16(*mask0b, s0b);
const __m128i mul0bh = _mm_unpackhi_epi16(mul0b_lows, mul0b_highs);
const __m128i mul0bl = _mm_unpacklo_epi16(mul0b_lows, mul0b_highs);
const __m128i mul1b_highs = _mm_mulhi_epu16(mask1b, s1b);
const __m128i mul1b_lows = _mm_mullo_epi16(mask1b, s1b);
const __m128i mul1bh = _mm_unpackhi_epi16(mul1b_lows, mul1b_highs);
const __m128i mul1bl = _mm_unpacklo_epi16(mul1b_lows, mul1b_highs);
const __m128i sumbh = _mm_add_epi32(mul0bh, mul1bh);
const __m128i sumbl = _mm_add_epi32(mul0bl, mul1bl);
const __m128i roundah =
_mm_srai_epi32(_mm_sub_epi32(sumah, *round_offset), shift);
const __m128i roundal =
_mm_srai_epi32(_mm_sub_epi32(sumal, *round_offset), shift);
const __m128i roundbh =
_mm_srai_epi32(_mm_sub_epi32(sumbh, *round_offset), shift);
const __m128i roundbl =
_mm_srai_epi32(_mm_sub_epi32(sumbl, *round_offset), shift);
const __m128i packa = _mm_packs_epi32(roundal, roundah);
const __m128i clipa =
_mm_min_epi16(_mm_max_epi16(packa, *clip_low), *clip_high);
const __m128i packb = _mm_packs_epi32(roundbl, roundbh);
const __m128i clipb =
_mm_min_epi16(_mm_max_epi16(packb, *clip_low), *clip_high);
xx_storeu_128(dst, clipa);
xx_storeu_128(dst + dst_stride, clipb);
}
static INLINE void highbd_blend_a64_d16_mask_subw0_subh0_w8_sse4_1(
uint16_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0,
uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride,
const uint8_t *mask, uint32_t mask_stride, int h,
const __m128i *round_offset, int shift, const __m128i *clip_low,
const __m128i *clip_high, const __m128i *max_mask) {
do {
const __m128i mask0a = _mm_cvtepu8_epi16(xx_loadl_64(mask));
const __m128i mask0b = _mm_cvtepu8_epi16(xx_loadl_64(mask + mask_stride));
highbd_blend_a64_d16_mask_w8_sse4_1(
dst, dst_stride, src0, src0_stride, src1, src1_stride, &mask0a, &mask0b,
round_offset, shift, clip_low, clip_high, max_mask);
dst += dst_stride * 2;
src0 += src0_stride * 2;
src1 += src1_stride * 2;
mask += mask_stride * 2;
} while (h -= 2);
}
static INLINE void highbd_blend_a64_d16_mask_subw1_subh1_w8_sse4_1(
uint16_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0,
uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride,
const uint8_t *mask, uint32_t mask_stride, int h,
const __m128i *round_offset, int shift, const __m128i *clip_low,
const __m128i *clip_high, const __m128i *max_mask) {
const __m128i one_b = _mm_set1_epi8(1);
const __m128i two_w = _mm_set1_epi16(2);
do {
const __m128i mask_thisrowa = xx_loadu_128(mask);
const __m128i mask_nextrowa = xx_loadu_128(mask + mask_stride);
const __m128i mask_thisrowb = xx_loadu_128(mask + 2 * mask_stride);
const __m128i mask_nextrowb = xx_loadu_128(mask + 3 * mask_stride);
const __m128i mask_bothrowsa = _mm_adds_epu8(mask_thisrowa, mask_nextrowa);
const __m128i mask_bothrowsb = _mm_adds_epu8(mask_thisrowb, mask_nextrowb);
const __m128i mask_16a = _mm_maddubs_epi16(mask_bothrowsa, one_b);
const __m128i mask_16b = _mm_maddubs_epi16(mask_bothrowsb, one_b);
const __m128i mask_sa = _mm_srli_epi16(_mm_add_epi16(mask_16a, two_w), 2);
const __m128i mask_sb = _mm_srli_epi16(_mm_add_epi16(mask_16b, two_w), 2);
highbd_blend_a64_d16_mask_w8_sse4_1(
dst, dst_stride, src0, src0_stride, src1, src1_stride, &mask_sa,
&mask_sb, round_offset, shift, clip_low, clip_high, max_mask);
dst += dst_stride * 2;
src0 += src0_stride * 2;
src1 += src1_stride * 2;
mask += mask_stride * 4;
} while (h -= 2);
}
static INLINE void highbd_blend_a64_d16_mask_w16_sse4_1(
uint16_t *dst, const CONV_BUF_TYPE *src0, const CONV_BUF_TYPE *src1,
const __m128i *round_offset, int shift, const __m128i *mask0l,
const __m128i *mask0h, const __m128i *clip_low, const __m128i *clip_high,
const __m128i *mask_max) {
// Load 16x u16 pixels for this row from each src
const __m128i s0l = xx_loadu_128(src0);
const __m128i s0h = xx_loadu_128(src0 + 8);
const __m128i s1l = xx_loadu_128(src1);
const __m128i s1h = xx_loadu_128(src1 + 8);
// Calculate inverse masks
const __m128i mask1h = _mm_sub_epi16(*mask_max, *mask0h);
const __m128i mask1l = _mm_sub_epi16(*mask_max, *mask0l);
const __m128i mul0_highs = _mm_mulhi_epu16(*mask0h, s0h);
const __m128i mul0_lows = _mm_mullo_epi16(*mask0h, s0h);
const __m128i mul0h = _mm_unpackhi_epi16(mul0_lows, mul0_highs);
const __m128i mul0l = _mm_unpacklo_epi16(mul0_lows, mul0_highs);
const __m128i mul1_highs = _mm_mulhi_epu16(mask1h, s1h);
const __m128i mul1_lows = _mm_mullo_epi16(mask1h, s1h);
const __m128i mul1h = _mm_unpackhi_epi16(mul1_lows, mul1_highs);
const __m128i mul1l = _mm_unpacklo_epi16(mul1_lows, mul1_highs);
const __m128i mulhh = _mm_add_epi32(mul0h, mul1h);
const __m128i mulhl = _mm_add_epi32(mul0l, mul1l);
const __m128i mul2_highs = _mm_mulhi_epu16(*mask0l, s0l);
const __m128i mul2_lows = _mm_mullo_epi16(*mask0l, s0l);
const __m128i mul2h = _mm_unpackhi_epi16(mul2_lows, mul2_highs);
const __m128i mul2l = _mm_unpacklo_epi16(mul2_lows, mul2_highs);
const __m128i mul3_highs = _mm_mulhi_epu16(mask1l, s1l);
const __m128i mul3_lows = _mm_mullo_epi16(mask1l, s1l);
const __m128i mul3h = _mm_unpackhi_epi16(mul3_lows, mul3_highs);
const __m128i mul3l = _mm_unpacklo_epi16(mul3_lows, mul3_highs);
const __m128i mullh = _mm_add_epi32(mul2h, mul3h);
const __m128i mulll = _mm_add_epi32(mul2l, mul3l);
const __m128i reshh =
_mm_srai_epi32(_mm_sub_epi32(mulhh, *round_offset), shift);
const __m128i reshl =
_mm_srai_epi32(_mm_sub_epi32(mulhl, *round_offset), shift);
const __m128i reslh =
_mm_srai_epi32(_mm_sub_epi32(mullh, *round_offset), shift);
const __m128i resll =
_mm_srai_epi32(_mm_sub_epi32(mulll, *round_offset), shift);
// Signed saturating pack from i32 to i16:
const __m128i packh = _mm_packs_epi32(reshl, reshh);
const __m128i packl = _mm_packs_epi32(resll, reslh);
// Clip the values to the valid range
const __m128i cliph =
_mm_min_epi16(_mm_max_epi16(packh, *clip_low), *clip_high);
const __m128i clipl =
_mm_min_epi16(_mm_max_epi16(packl, *clip_low), *clip_high);
// Store 16 pixels
xx_storeu_128(dst, clipl);
xx_storeu_128(dst + 8, cliph);
}
static INLINE void highbd_blend_a64_d16_mask_subw0_subh0_w16_sse4_1(
uint16_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0,
uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride,
const uint8_t *mask, uint32_t mask_stride, int h, int w,
const __m128i *round_offset, int shift, const __m128i *clip_low,
const __m128i *clip_high, const __m128i *mask_max) {
for (int i = 0; i < h; i++) {
for (int j = 0; j < w; j += 16) {
// Load 16x u8 alpha-mask values and pad to u16
const __m128i masks_u8 = xx_loadu_128(mask + j);
const __m128i mask0l = _mm_cvtepu8_epi16(masks_u8);
const __m128i mask0h = _mm_cvtepu8_epi16(_mm_srli_si128(masks_u8, 8));
highbd_blend_a64_d16_mask_w16_sse4_1(
dst + j, src0 + j, src1 + j, round_offset, shift, &mask0l, &mask0h,
clip_low, clip_high, mask_max);
}
dst += dst_stride;
src0 += src0_stride;
src1 += src1_stride;
mask += mask_stride;
}
}
static INLINE void highbd_blend_a64_d16_mask_subw1_subh1_w16_sse4_1(
uint16_t *dst, uint32_t dst_stride, const CONV_BUF_TYPE *src0,
uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride,
const uint8_t *mask, uint32_t mask_stride, int h, int w,
const __m128i *round_offset, int shift, const __m128i *clip_low,
const __m128i *clip_high, const __m128i *mask_max) {
const __m128i one_b = _mm_set1_epi8(1);
const __m128i two_w = _mm_set1_epi16(2);
for (int i = 0; i < h; i++) {
for (int j = 0; j < w; j += 16) {
const __m128i m_i00 = xx_loadu_128(mask + 2 * j);
const __m128i m_i01 = xx_loadu_128(mask + 2 * j + 16);
const __m128i m_i10 = xx_loadu_128(mask + mask_stride + 2 * j);
const __m128i m_i11 = xx_loadu_128(mask + mask_stride + 2 * j + 16);
const __m128i m0_ac = _mm_adds_epu8(m_i00, m_i10);
const __m128i m1_ac = _mm_adds_epu8(m_i01, m_i11);
const __m128i m0_acbd = _mm_maddubs_epi16(m0_ac, one_b);
const __m128i m1_acbd = _mm_maddubs_epi16(m1_ac, one_b);
const __m128i mask_l = _mm_srli_epi16(_mm_add_epi16(m0_acbd, two_w), 2);
const __m128i mask_h = _mm_srli_epi16(_mm_add_epi16(m1_acbd, two_w), 2);
highbd_blend_a64_d16_mask_w16_sse4_1(
dst + j, src0 + j, src1 + j, round_offset, shift, &mask_l, &mask_h,
clip_low, clip_high, mask_max);
}
dst += dst_stride;
src0 += src0_stride;
src1 += src1_stride;
mask += mask_stride * 2;
}
}
void aom_highbd_blend_a64_d16_mask_sse4_1(
uint8_t *dst8, uint32_t dst_stride, const CONV_BUF_TYPE *src0,
uint32_t src0_stride, const CONV_BUF_TYPE *src1, uint32_t src1_stride,
const uint8_t *mask, uint32_t mask_stride, int w, int h, int subw, int subh,
ConvolveParams *conv_params, const int bd) {
uint16_t *dst = CONVERT_TO_SHORTPTR(dst8);
const int round_bits =
2 * FILTER_BITS - conv_params->round_0 - conv_params->round_1;
const int32_t round_offset =
((1 << (round_bits + bd)) + (1 << (round_bits + bd - 1)) -
(1 << (round_bits - 1)))
<< AOM_BLEND_A64_ROUND_BITS;
const __m128i v_round_offset = _mm_set1_epi32(round_offset);
const int shift = round_bits + AOM_BLEND_A64_ROUND_BITS;
const __m128i clip_low = _mm_set1_epi16(0);
const __m128i clip_high = _mm_set1_epi16((1 << bd) - 1);
const __m128i mask_max = _mm_set1_epi16(AOM_BLEND_A64_MAX_ALPHA);
assert(IMPLIES((void *)src0 == dst, src0_stride == dst_stride));
assert(IMPLIES((void *)src1 == dst, src1_stride == dst_stride));
assert(h >= 4);
assert(w >= 4);
assert(IS_POWER_OF_TWO(h));
assert(IS_POWER_OF_TWO(w));
if (subw == 0 && subh == 0) {
switch (w) {
case 4:
highbd_blend_a64_d16_mask_subw0_subh0_w4_sse4_1(
dst, dst_stride, src0, src0_stride, src1, src1_stride, mask,
mask_stride, h, &v_round_offset, shift, &clip_low, &clip_high,
&mask_max);
break;
case 8:
highbd_blend_a64_d16_mask_subw0_subh0_w8_sse4_1(
dst, dst_stride, src0, src0_stride, src1, src1_stride, mask,
mask_stride, h, &v_round_offset, shift, &clip_low, &clip_high,
&mask_max);
break;
default: // >=16
highbd_blend_a64_d16_mask_subw0_subh0_w16_sse4_1(
dst, dst_stride, src0, src0_stride, src1, src1_stride, mask,
mask_stride, h, w, &v_round_offset, shift, &clip_low, &clip_high,
&mask_max);
break;
}
} else if (subw == 1 && subh == 1) {
switch (w) {
case 4:
highbd_blend_a64_d16_mask_subw1_subh1_w4_sse4_1(
dst, dst_stride, src0, src0_stride, src1, src1_stride, mask,
mask_stride, h, &v_round_offset, shift, &clip_low, &clip_high,
&mask_max);
break;
case 8:
highbd_blend_a64_d16_mask_subw1_subh1_w8_sse4_1(
dst, dst_stride, src0, src0_stride, src1, src1_stride, mask,
mask_stride, h, &v_round_offset, shift, &clip_low, &clip_high,
&mask_max);
break;
default: // >=16
highbd_blend_a64_d16_mask_subw1_subh1_w16_sse4_1(
dst, dst_stride, src0, src0_stride, src1, src1_stride, mask,
mask_stride, h, w, &v_round_offset, shift, &clip_low, &clip_high,
&mask_max);
break;
}
} else {
// Sub-sampling in only one axis doesn't seem to happen very much, so fall
// back to the vanilla C implementation instead of having all the optimised
// code for these.
aom_highbd_blend_a64_d16_mask_c(dst8, dst_stride, src0, src0_stride, src1,
src1_stride, mask, mask_stride, w, h, subw,
subh, conv_params, bd);
}
}

25
third_party/aom/aom_dsp/x86/convolve.h vendored
View file

@ -122,7 +122,30 @@ typedef void highbd_filter8_1dfunction(const uint16_t *src_ptr,
uint16_t *src = CONVERT_TO_SHORTPTR(src8); \
uint16_t *dst = CONVERT_TO_SHORTPTR(dst8); \
if (step_q4 == 16 && filter[3] != 128) { \
if (filter[0] | filter[1] | filter[2]) { \
if (((filter[0] | filter[1] | filter[6] | filter[7]) == 0) && \
(filter[2] | filter[5])) { \
while (w >= 16) { \
aom_highbd_filter_block1d16_##dir##4_##avg##opt( \
src_start, src_stride, dst, dst_stride, h, filter, bd); \
src += 16; \
dst += 16; \
w -= 16; \
} \
while (w >= 8) { \
aom_highbd_filter_block1d8_##dir##4_##avg##opt( \
src_start, src_stride, dst, dst_stride, h, filter, bd); \
src += 8; \
dst += 8; \
w -= 8; \
} \
while (w >= 4) { \
aom_highbd_filter_block1d4_##dir##4_##avg##opt( \
src_start, src_stride, dst, dst_stride, h, filter, bd); \
src += 4; \
dst += 4; \
w -= 4; \
} \
} else if (filter[0] | filter[1] | filter[2]) { \
while (w >= 16) { \
aom_highbd_filter_block1d16_##dir##8_##avg##opt( \
src_start, src_stride, dst, dst_stride, h, filter, bd); \

42
third_party/aom/aom_dsp/x86/convolve_avx2.h vendored
View file

@ -34,6 +34,31 @@ DECLARE_ALIGNED(32, static const uint8_t, filt4_d4_global_avx2[]) = {
2, 3, 4, 5, 3, 4, 5, 6, 4, 5, 6, 7, 5, 6, 7, 8,
};
DECLARE_ALIGNED(32, static const uint8_t, filt_center_global_avx2[32]) = {
3, 255, 4, 255, 5, 255, 6, 255, 7, 255, 8, 255, 9, 255, 10, 255,
3, 255, 4, 255, 5, 255, 6, 255, 7, 255, 8, 255, 9, 255, 10, 255
};
DECLARE_ALIGNED(32, static const uint8_t, filt1_global_avx2[32]) = {
0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8,
0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8
};
DECLARE_ALIGNED(32, static const uint8_t, filt2_global_avx2[32]) = {
2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10,
2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10
};
DECLARE_ALIGNED(32, static const uint8_t, filt3_global_avx2[32]) = {
4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12,
4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12
};
DECLARE_ALIGNED(32, static const uint8_t, filt4_global_avx2[32]) = {
6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14,
6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14
};
static INLINE void prepare_coeffs_lowbd(
const InterpFilterParams *const filter_params, const int subpel_q4,
__m256i *const coeffs /* [4] */) {
@ -108,6 +133,15 @@ static INLINE __m256i convolve(const __m256i *const s,
return res;
}
static INLINE __m256i convolve_4tap(const __m256i *const s,
const __m256i *const coeffs) {
const __m256i res_1 = _mm256_madd_epi16(s[0], coeffs[0]);
const __m256i res_2 = _mm256_madd_epi16(s[1], coeffs[1]);
const __m256i res = _mm256_add_epi32(res_1, res_2);
return res;
}
static INLINE __m256i convolve_lowbd_x(const __m256i data,
const __m256i *const coeffs,
const __m256i *const filt) {
@ -138,9 +172,9 @@ static INLINE void add_store_aligned_256(CONV_BUF_TYPE *const dst,
static INLINE __m256i comp_avg(const __m256i *const data_ref_0,
const __m256i *const res_unsigned,
const __m256i *const wt,
const int use_jnt_comp_avg) {
const int use_dist_wtd_comp_avg) {
__m256i res;
if (use_jnt_comp_avg) {
if (use_dist_wtd_comp_avg) {
const __m256i data_lo = _mm256_unpacklo_epi16(*data_ref_0, *res_unsigned);
const __m256i data_hi = _mm256_unpackhi_epi16(*data_ref_0, *res_unsigned);
@ -172,9 +206,9 @@ static INLINE __m256i highbd_comp_avg(const __m256i *const data_ref_0,
const __m256i *const res_unsigned,
const __m256i *const wt0,
const __m256i *const wt1,
const int use_jnt_comp_avg) {
const int use_dist_wtd_comp_avg) {
__m256i res;
if (use_jnt_comp_avg) {
if (use_dist_wtd_comp_avg) {
const __m256i wt0_res = _mm256_mullo_epi32(*data_ref_0, *wt0);
const __m256i wt1_res = _mm256_mullo_epi32(*res_unsigned, *wt1);
const __m256i wt_res = _mm256_add_epi32(wt0_res, wt1_res);

4
third_party/aom/aom_dsp/x86/convolve_sse2.h vendored
View file

@ -78,9 +78,9 @@ static INLINE __m128i convolve_hi_y(const __m128i *const s,
static INLINE __m128i comp_avg(const __m128i *const data_ref_0,
const __m128i *const res_unsigned,
const __m128i *const wt,
const int use_jnt_comp_avg) {
const int use_dist_wtd_avg) {
__m128i res;
if (use_jnt_comp_avg) {
if (use_dist_wtd_avg) {
const __m128i data_lo = _mm_unpacklo_epi16(*data_ref_0, *res_unsigned);
const __m128i data_hi = _mm_unpackhi_epi16(*data_ref_0, *res_unsigned);

4
third_party/aom/aom_dsp/x86/convolve_sse4_1.h vendored
View file

@ -35,9 +35,9 @@ static INLINE __m128i highbd_comp_avg_sse4_1(const __m128i *const data_ref_0,
const __m128i *const res_unsigned,
const __m128i *const wt0,
const __m128i *const wt1,
const int use_jnt_comp_avg) {
const int use_dist_wtd_avg) {
__m128i res;
if (use_jnt_comp_avg) {
if (use_dist_wtd_avg) {
const __m128i wt0_res = _mm_mullo_epi32(*data_ref_0, *wt0);
const __m128i wt1_res = _mm_mullo_epi32(*res_unsigned, *wt1);

1
third_party/aom/aom_dsp/x86/fft_avx2.c vendored
View file

@ -11,6 +11,7 @@
#include <immintrin.h>
#include "config/aom_dsp_rtcd.h"
#include "aom_dsp/aom_dsp_common.h"
#include "aom_dsp/fft_common.h"

1
third_party/aom/aom_dsp/x86/fft_sse2.c vendored
View file

@ -11,6 +11,7 @@ s * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
#include <xmmintrin.h>
#include "config/aom_dsp_rtcd.h"
#include "aom_dsp/aom_dsp_common.h"
#include "aom_dsp/fft_common.h"

40
third_party/aom/aom_dsp/x86/fwd_txfm_sse2.c vendored
View file

@ -17,46 +17,6 @@
#include "aom_dsp/aom_dsp_common.h"
#include "aom_dsp/x86/fwd_txfm_sse2.h"
void aom_fdct8x8_1_sse2(const int16_t *input, tran_low_t *output, int stride) {
__m128i in0 = _mm_load_si128((const __m128i *)(input + 0 * stride));
__m128i in1 = _mm_load_si128((const __m128i *)(input + 1 * stride));
__m128i in2 = _mm_load_si128((const __m128i *)(input + 2 * stride));
__m128i in3 = _mm_load_si128((const __m128i *)(input + 3 * stride));
__m128i u0, u1, sum;
u0 = _mm_add_epi16(in0, in1);
u1 = _mm_add_epi16(in2, in3);
in0 = _mm_load_si128((const __m128i *)(input + 4 * stride));
in1 = _mm_load_si128((const __m128i *)(input + 5 * stride));
in2 = _mm_load_si128((const __m128i *)(input + 6 * stride));
in3 = _mm_load_si128((const __m128i *)(input + 7 * stride));
sum = _mm_add_epi16(u0, u1);
in0 = _mm_add_epi16(in0, in1);
in2 = _mm_add_epi16(in2, in3);
sum = _mm_add_epi16(sum, in0);
u0 = _mm_setzero_si128();
sum = _mm_add_epi16(sum, in2);
in0 = _mm_unpacklo_epi16(u0, sum);
in1 = _mm_unpackhi_epi16(u0, sum);
in0 = _mm_srai_epi32(in0, 16);
in1 = _mm_srai_epi32(in1, 16);
sum = _mm_add_epi32(in0, in1);
in0 = _mm_unpacklo_epi32(sum, u0);
in1 = _mm_unpackhi_epi32(sum, u0);
sum = _mm_add_epi32(in0, in1);
in0 = _mm_srli_si128(sum, 8);
in1 = _mm_add_epi32(sum, in0);
output[0] = (tran_low_t)_mm_cvtsi128_si32(in1);
}
#define DCT_HIGH_BIT_DEPTH 0
#define FDCT8x8_2D aom_fdct8x8_sse2
#include "aom_dsp/x86/fwd_txfm_impl_sse2.h"

325
third_party/aom/aom_dsp/x86/highbd_convolve_avx2.c vendored
View file

@ -20,6 +20,14 @@
// -----------------------------------------------------------------------------
// Copy and average
static const uint8_t ip_shuffle_f2f3[32] = { 0, 1, 2, 3, 2, 3, 4, 5, 4, 5, 6,
7, 6, 7, 8, 9, 0, 1, 2, 3, 2, 3,
4, 5, 4, 5, 6, 7, 6, 7, 8, 9 };
static const uint8_t ip_shuffle_f4f5[32] = { 4, 5, 6, 7, 6, 7, 8, 9,
8, 9, 10, 11, 10, 11, 12, 13,
4, 5, 6, 7, 6, 7, 8, 9,
8, 9, 10, 11, 10, 11, 12, 13 };
void aom_highbd_convolve_copy_avx2(const uint8_t *src8, ptrdiff_t src_stride,
uint8_t *dst8, ptrdiff_t dst_stride,
const int16_t *filter_x, int filter_x_stride,
@ -444,6 +452,17 @@ static INLINE void pack_filters(const int16_t *filter, __m256i *f /*f[4]*/) {
f[3] = _mm256_shuffle_epi8(hh, p3);
}
static INLINE void pack_filters_4tap(const int16_t *filter,
__m256i *f /*f[4]*/) {
const __m128i h = _mm_loadu_si128((const __m128i *)filter);
const __m256i coeff = _mm256_broadcastsi128_si256(h);
// coeffs 2 3 2 3 2 3 2 3
f[0] = _mm256_shuffle_epi32(coeff, 0x55);
// coeffs 4 5 4 5 4 5 4 5
f[1] = _mm256_shuffle_epi32(coeff, 0xaa);
}
static INLINE void filter_8x1_pixels(const __m256i *sig /*sig[4]*/,
const __m256i *fil /*fil[4]*/,
__m256i *y) {
@ -544,6 +563,176 @@ static void aom_highbd_filter_block1d16_h8_avx2(
} while (height > 0);
}
static void aom_highbd_filter_block1d4_h4_avx2(
const uint16_t *src_ptr, ptrdiff_t src_pitch, uint16_t *dst_ptr,
ptrdiff_t dst_pitch, uint32_t height, const int16_t *filter, int bd) {
const __m256i rounding = _mm256_set1_epi32(1 << (CONV8_ROUNDING_BITS - 1));
__m256i ff[2], s[2];
uint32_t i;
const __m256i clip_pixel =
_mm256_set1_epi16(bd == 10 ? 1023 : (bd == 12 ? 4095 : 255));
const __m256i zero = _mm256_setzero_si256();
static const uint8_t shuffle_mask[32] = { 0, 1, 2, 3, 2, 3, 4, 5, 4, 5, 6,
7, 6, 7, 8, 9, 0, 1, 2, 3, 2, 3,
4, 5, 4, 5, 6, 7, 6, 7, 8, 9 };
__m256i mask = _mm256_loadu_si256((__m256i *)shuffle_mask);
__m256i ip_mask_f2f3 = _mm256_loadu_si256((__m256i *)ip_shuffle_f2f3);
__m256i ip_mask_f4f5 = _mm256_loadu_si256((__m256i *)ip_shuffle_f4f5);
pack_filters_4tap(filter, ff);
src_ptr -= 3;
for (i = 0; i <= (height - 2); i += 2) {
__m256i row0 = _mm256_castsi128_si256(
_mm_loadu_si128((__m128i *)&src_ptr[i * src_pitch + 2]));
__m256i row1 = _mm256_castsi128_si256(
_mm_loadu_si128((__m128i *)&src_ptr[(i + 1) * src_pitch + 2]));
s[0] = _mm256_inserti128_si256(row0, _mm256_castsi256_si128(row1), 1);
s[1] = _mm256_alignr_epi8(s[0], s[0], 4);
s[0] = _mm256_shuffle_epi8(s[0], mask);
s[1] = _mm256_shuffle_epi8(s[1], mask);
__m256i res = convolve_4tap(s, ff);
res =
_mm256_srai_epi32(_mm256_add_epi32(res, rounding), CONV8_ROUNDING_BITS);
res = _mm256_packs_epi32(res, res);
res = _mm256_min_epi16(res, clip_pixel);
res = _mm256_max_epi16(res, zero);
_mm_storel_epi64((__m128i *)&dst_ptr[i * dst_pitch],
_mm256_castsi256_si128(res));
_mm_storel_epi64((__m128i *)&dst_ptr[(i + 1) * dst_pitch],
_mm256_extracti128_si256(res, 1));
}
if (height % 2 != 0) {
i = height - 1;
const __m256i row0_0 = _mm256_castsi128_si256(
_mm_loadu_si128((__m128i *)&src_ptr[i * src_pitch + 2]));
const __m256i row0_1 = _mm256_castsi128_si256(
_mm_loadu_si128((__m128i *)&src_ptr[i * src_pitch + 6]));
const __m256i r0 =
_mm256_inserti128_si256(row0_0, _mm256_castsi256_si128(row0_1), 1);
s[0] = _mm256_shuffle_epi8(r0, ip_mask_f2f3);
s[1] = _mm256_shuffle_epi8(r0, ip_mask_f4f5);
__m256i res = convolve_4tap(s, ff);
res =
_mm256_srai_epi32(_mm256_add_epi32(res, rounding), CONV8_ROUNDING_BITS);
res = _mm256_packs_epi32(res, res);
res = _mm256_min_epi16(res, clip_pixel);
res = _mm256_max_epi16(res, zero);
_mm_storel_epi64((__m128i *)&dst_ptr[i * dst_pitch],
_mm256_castsi256_si128(res));
}
}
static void aom_highbd_filter_block1d8_h4_avx2(
const uint16_t *src_ptr, ptrdiff_t src_pitch, uint16_t *dst_ptr,
ptrdiff_t dst_pitch, uint32_t height, const int16_t *filter, int bd) {
const __m256i rounding = _mm256_set1_epi32(1 << (CONV8_ROUNDING_BITS - 1));
__m256i ff[2], s[2];
uint32_t i = 0;
const __m256i clip_pixel =
_mm256_set1_epi16(bd == 10 ? 1023 : (bd == 12 ? 4095 : 255));
const __m256i zero = _mm256_setzero_si256();
static const uint8_t shuffle_mask[32] = { 0, 1, 8, 9, 2, 3, 10, 11,
4, 5, 12, 13, 6, 7, 14, 15,
0, 1, 8, 9, 2, 3, 10, 11,
4, 5, 12, 13, 6, 7, 14, 15 };
__m256i mask = _mm256_loadu_si256((__m256i *)shuffle_mask);
__m256i ip_mask_f2f3 = _mm256_loadu_si256((__m256i *)ip_shuffle_f2f3);
__m256i ip_mask_f4f5 = _mm256_loadu_si256((__m256i *)ip_shuffle_f4f5);
pack_filters_4tap(filter, ff);
src_ptr -= 3;
/* Horizontal filter */
for (i = 0; i <= (height - 2); i += 2) {
const __m256i row0 =
_mm256_loadu_si256((__m256i *)&src_ptr[i * src_pitch + 2]);
__m256i row1 =
_mm256_loadu_si256((__m256i *)&src_ptr[(i + 1) * src_pitch + 2]);
const __m256i r0 =
_mm256_inserti128_si256(row0, _mm256_castsi256_si128(row1), 1);
const __m256i r1 = _mm256_permute2x128_si256(row0, row1, 0x31);
// even pixels
s[0] = r0;
s[1] = _mm256_alignr_epi8(r1, r0, 4);
__m256i res_even = convolve_4tap(s, ff);
res_even = _mm256_srai_epi32(_mm256_add_epi32(res_even, rounding),
CONV8_ROUNDING_BITS);
// odd pixels
s[0] = _mm256_alignr_epi8(r1, r0, 2);
s[1] = _mm256_alignr_epi8(r1, r0, 6);
__m256i res_odd = convolve_4tap(s, ff);
res_odd = _mm256_srai_epi32(_mm256_add_epi32(res_odd, rounding),
CONV8_ROUNDING_BITS);
__m256i res = _mm256_packs_epi32(res_even, res_odd);
res = _mm256_shuffle_epi8(res, mask);
res = _mm256_min_epi16(res, clip_pixel);
res = _mm256_max_epi16(res, zero);
_mm_storeu_si128((__m128i *)&dst_ptr[i * dst_pitch],
_mm256_castsi256_si128(res));
_mm_storeu_si128((__m128i *)&dst_ptr[i * dst_pitch + dst_pitch],
_mm256_extracti128_si256(res, 1));
}
if (height % 2 != 0) {
i = height - 1;
const __m256i row0_0 =
_mm256_loadu_si256((__m256i *)&src_ptr[i * src_pitch + 2]);
const __m256i row0_1 =
_mm256_loadu_si256((__m256i *)&src_ptr[i * src_pitch + 6]);
const __m256i r0 =
_mm256_inserti128_si256(row0_0, _mm256_castsi256_si128(row0_1), 1);
s[0] = _mm256_shuffle_epi8(r0, ip_mask_f2f3);
s[1] = _mm256_shuffle_epi8(r0, ip_mask_f4f5);
__m256i res = convolve_4tap(s, ff);
res =
_mm256_srai_epi32(_mm256_add_epi32(res, rounding), CONV8_ROUNDING_BITS);
res = _mm256_packs_epi32(res, res);
res = _mm256_min_epi16(res, clip_pixel);
res = _mm256_max_epi16(res, zero);
_mm_storel_epi64((__m128i *)&dst_ptr[i * dst_pitch],
_mm256_castsi256_si128(res));
_mm_storel_epi64((__m128i *)&dst_ptr[i * dst_pitch + 4],
_mm256_extracti128_si256(res, 1));
}
}
static void aom_highbd_filter_block1d16_h4_avx2(
const uint16_t *src_ptr, ptrdiff_t src_pitch, uint16_t *dst_ptr,
ptrdiff_t dst_pitch, uint32_t height, const int16_t *filter, int bd) {
aom_highbd_filter_block1d8_h4_avx2(src_ptr, src_pitch, dst_ptr, dst_pitch,
height, filter, bd);
aom_highbd_filter_block1d8_h4_avx2(src_ptr + 8, src_pitch, dst_ptr + 8,
dst_pitch, height, filter, bd);
}
// -----------------------------------------------------------------------------
// 2-tap horizontal filtering
@ -875,6 +1064,142 @@ static void aom_highbd_filter_block1d16_v8_avx2(
} while (height > 0);
}
static void aom_highbd_filter_block1d4_v4_avx2(
const uint16_t *src_ptr, ptrdiff_t src_pitch, uint16_t *dst_ptr,
ptrdiff_t dst_pitch, uint32_t height, const int16_t *filter, int bd) {
const int bits = FILTER_BITS;
const __m128i round_shift_bits = _mm_cvtsi32_si128(bits);
const __m256i round_const_bits = _mm256_set1_epi32((1 << bits) >> 1);
const __m256i clip_pixel =
_mm256_set1_epi32(bd == 10 ? 1023 : (bd == 12 ? 4095 : 255));
const __m256i zero = _mm256_setzero_si256();
uint32_t i;
__m256i s[2], ff[2];
pack_filters_4tap(filter, ff);
const uint16_t *data = src_ptr;
/* Vertical filter */
{
__m128i s2 = _mm_loadl_epi64((__m128i *)(data + 2 * src_pitch));
__m128i s3 = _mm_loadl_epi64((__m128i *)(data + 3 * src_pitch));
__m256i s23 = _mm256_inserti128_si256(_mm256_castsi128_si256(s2), s3, 1);
__m128i s4 = _mm_loadl_epi64((__m128i *)(data + 4 * src_pitch));
__m256i s34 = _mm256_inserti128_si256(_mm256_castsi128_si256(s3), s4, 1);
s[0] = _mm256_unpacklo_epi16(s23, s34);
for (i = 0; i < height; i += 2) {
data = &src_ptr[i * src_pitch];
__m128i s5 = _mm_loadl_epi64((__m128i *)(data + 5 * src_pitch));
__m128i s6 = _mm_loadl_epi64((__m128i *)(data + 6 * src_pitch));
__m256i s45 = _mm256_inserti128_si256(_mm256_castsi128_si256(s4), s5, 1);
__m256i s56 = _mm256_inserti128_si256(_mm256_castsi128_si256(s5), s6, 1);
s[1] = _mm256_unpacklo_epi16(s45, s56);
const __m256i res_a = convolve_4tap(s, ff);
__m256i res_a_round = _mm256_sra_epi32(
_mm256_add_epi32(res_a, round_const_bits), round_shift_bits);
__m256i res_16bit = _mm256_min_epi32(res_a_round, clip_pixel);
res_16bit = _mm256_max_epi32(res_16bit, zero);
res_16bit = _mm256_packs_epi32(res_16bit, res_16bit);
_mm_storel_epi64((__m128i *)&dst_ptr[i * dst_pitch],
_mm256_castsi256_si128(res_16bit));
_mm_storel_epi64((__m128i *)&dst_ptr[i * dst_pitch + dst_pitch],
_mm256_extracti128_si256(res_16bit, 1));
s[0] = s[1];
s4 = s6;
}
}
}
static void aom_highbd_filter_block1d8_v4_avx2(
const uint16_t *src_ptr, ptrdiff_t src_pitch, uint16_t *dst_ptr,
ptrdiff_t dst_pitch, uint32_t height, const int16_t *filter, int bd) {
const int bits = FILTER_BITS;
const __m128i round_shift_bits = _mm_cvtsi32_si128(bits);
const __m256i round_const_bits = _mm256_set1_epi32((1 << bits) >> 1);
const __m256i clip_pixel =
_mm256_set1_epi16(bd == 10 ? 1023 : (bd == 12 ? 4095 : 255));
const __m256i zero = _mm256_setzero_si256();
__m256i s[4], ff[2];
uint32_t i;
pack_filters_4tap(filter, ff);
const uint16_t *data = src_ptr;
/* Vertical filter */
{
__m128i s2 = _mm_loadu_si128((__m128i *)(data + 2 * src_pitch));
__m128i s3 = _mm_loadu_si128((__m128i *)(data + 3 * src_pitch));
__m256i s23 = _mm256_inserti128_si256(_mm256_castsi128_si256(s2), s3, 1);
__m128i s4 = _mm_loadu_si128((__m128i *)(data + 4 * src_pitch));
__m256i s34 = _mm256_inserti128_si256(_mm256_castsi128_si256(s3), s4, 1);
s[0] = _mm256_unpacklo_epi16(s23, s34);
s[2] = _mm256_unpackhi_epi16(s23, s34);
for (i = 0; i < height; i += 2) {
data = &src_ptr[i * src_pitch];
__m128i s5 = _mm_loadu_si128((__m128i *)(data + 5 * src_pitch));
__m128i s6 = _mm_loadu_si128((__m128i *)(data + 6 * src_pitch));
__m256i s45 = _mm256_inserti128_si256(_mm256_castsi128_si256(s4), s5, 1);
__m256i s56 = _mm256_inserti128_si256(_mm256_castsi128_si256(s5), s6, 1);
s[1] = _mm256_unpacklo_epi16(s45, s56);
s[3] = _mm256_unpackhi_epi16(s45, s56);
const __m256i res_a = convolve_4tap(s, ff);
__m256i res_a_round = _mm256_sra_epi32(
_mm256_add_epi32(res_a, round_const_bits), round_shift_bits);
const __m256i res_b = convolve_4tap(s + 2, ff);
__m256i res_b_round = _mm256_sra_epi32(
_mm256_add_epi32(res_b, round_const_bits), round_shift_bits);
__m256i res_16bit = _mm256_packs_epi32(res_a_round, res_b_round);
res_16bit = _mm256_min_epi16(res_16bit, clip_pixel);
res_16bit = _mm256_max_epi16(res_16bit, zero);
_mm_storeu_si128((__m128i *)&dst_ptr[i * dst_pitch],
_mm256_castsi256_si128(res_16bit));
_mm_storeu_si128((__m128i *)&dst_ptr[i * dst_pitch + dst_pitch],
_mm256_extracti128_si256(res_16bit, 1));
s[0] = s[1];
s[2] = s[3];
s4 = s6;
}
}
}
static void aom_highbd_filter_block1d16_v4_avx2(
const uint16_t *src_ptr, ptrdiff_t src_pitch, uint16_t *dst_ptr,
ptrdiff_t dst_pitch, uint32_t height, const int16_t *filter, int bd) {
aom_highbd_filter_block1d8_v4_avx2(src_ptr, src_pitch, dst_ptr, dst_pitch,
height, filter, bd);
aom_highbd_filter_block1d8_v4_avx2(src_ptr + 8, src_pitch, dst_ptr + 8,
dst_pitch, height, filter, bd);
}
// -----------------------------------------------------------------------------
// 2-tap vertical filtering

351
third_party/aom/aom_dsp/x86/highbd_convolve_sse2.c vendored Normal file
View file

@ -0,0 +1,351 @@
/*
* Copyright (c) 2018, Alliance for Open Media. All rights reserved
*
* This source code is subject to the terms of the BSD 2 Clause License and
* the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
* was not distributed with this source code in the LICENSE file, you can
* obtain it at www.aomedia.org/license/software. If the Alliance for Open
* Media Patent License 1.0 was not distributed with this source code in the
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
*/
#include <emmintrin.h>
#include "config/aom_dsp_rtcd.h"
#include "aom_dsp/x86/convolve.h"
// -----------------------------------------------------------------------------
void aom_highbd_filter_block1d4_v4_sse2(const uint16_t *src_ptr,
ptrdiff_t src_pitch, uint16_t *dst_ptr,
ptrdiff_t dst_pitch, uint32_t height,
const int16_t *filter, int bd) {
__m128i filtersReg;
__m128i srcReg2, srcReg3, srcReg4, srcReg5, srcReg6;
__m128i srcReg23_lo, srcReg34_lo;
__m128i srcReg45_lo, srcReg56_lo;
__m128i resReg23_lo, resReg34_lo, resReg45_lo, resReg56_lo;
__m128i resReg23_45_lo, resReg34_56_lo;
__m128i resReg23_45, resReg34_56;
__m128i addFilterReg64, secondFilters, thirdFilters;
unsigned int i;
ptrdiff_t src_stride, dst_stride;
const __m128i max = _mm_set1_epi16((1 << bd) - 1);
addFilterReg64 = _mm_set1_epi32(64);
filtersReg = _mm_loadu_si128((const __m128i *)filter);
// coeffs 0 1 0 1 2 3 2 3
const __m128i tmp0 = _mm_unpacklo_epi32(filtersReg, filtersReg);
// coeffs 4 5 4 5 6 7 6 7
const __m128i tmp1 = _mm_unpackhi_epi32(filtersReg, filtersReg);
secondFilters = _mm_unpackhi_epi64(tmp0, tmp0); // coeffs 2 3 2 3 2 3 2 3
thirdFilters = _mm_unpacklo_epi64(tmp1, tmp1); // coeffs 4 5 4 5 4 5 4 5
// multiply the size of the source and destination stride by two
src_stride = src_pitch << 1;
dst_stride = dst_pitch << 1;
srcReg2 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 2));
srcReg3 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 3));
srcReg23_lo = _mm_unpacklo_epi16(srcReg2, srcReg3);
srcReg4 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 4));
srcReg34_lo = _mm_unpacklo_epi16(srcReg3, srcReg4);
for (i = height; i > 1; i -= 2) {
srcReg5 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 5));
srcReg45_lo = _mm_unpacklo_epi16(srcReg4, srcReg5);
srcReg6 = _mm_loadl_epi64((const __m128i *)(src_ptr + src_pitch * 6));
srcReg56_lo = _mm_unpacklo_epi16(srcReg5, srcReg6);
// multiply 2 adjacent elements with the filter and add the result
resReg23_lo = _mm_madd_epi16(srcReg23_lo, secondFilters);
resReg34_lo = _mm_madd_epi16(srcReg34_lo, secondFilters);
resReg45_lo = _mm_madd_epi16(srcReg45_lo, thirdFilters);
resReg56_lo = _mm_madd_epi16(srcReg56_lo, thirdFilters);
resReg23_45_lo = _mm_add_epi32(resReg23_lo, resReg45_lo);
resReg34_56_lo = _mm_add_epi32(resReg34_lo, resReg56_lo);
// shift by 7 bit each 32 bit
resReg23_45_lo = _mm_add_epi32(resReg23_45_lo, addFilterReg64);
resReg34_56_lo = _mm_add_epi32(resReg34_56_lo, addFilterReg64);
resReg23_45_lo = _mm_srai_epi32(resReg23_45_lo, 7);
resReg34_56_lo = _mm_srai_epi32(resReg34_56_lo, 7);
// shrink to 16 bit each 32 bits, the first lane contain the first
// convolve result and the second lane contain the second convolve
// result
resReg23_45 = _mm_packs_epi32(resReg23_45_lo, _mm_setzero_si128());
resReg34_56 = _mm_packs_epi32(resReg34_56_lo, _mm_setzero_si128());
resReg23_45 = _mm_max_epi16(resReg23_45, _mm_setzero_si128());
resReg23_45 = _mm_min_epi16(resReg23_45, max);
resReg34_56 = _mm_max_epi16(resReg34_56, _mm_setzero_si128());
resReg34_56 = _mm_min_epi16(resReg34_56, max);
src_ptr += src_stride;
_mm_storel_epi64((__m128i *)dst_ptr, (resReg23_45));
_mm_storel_epi64((__m128i *)(dst_ptr + dst_pitch), (resReg34_56));
dst_ptr += dst_stride;
// save part of the registers for next strides
srcReg23_lo = srcReg45_lo;
srcReg34_lo = srcReg56_lo;
srcReg4 = srcReg6;
}
}
void aom_highbd_filter_block1d4_h4_sse2(const uint16_t *src_ptr,
ptrdiff_t src_pitch, uint16_t *dst_ptr,
ptrdiff_t dst_pitch, uint32_t height,
const int16_t *filter, int bd) {
__m128i filtersReg;
__m128i addFilterReg64;
__m128i secondFilters, thirdFilters;
__m128i srcRegFilt32b1_1;
__m128i srcReg32b1;
unsigned int i;
src_ptr -= 3;
addFilterReg64 = _mm_set1_epi32(64);
filtersReg = _mm_loadu_si128((const __m128i *)filter);
const __m128i max = _mm_set1_epi16((1 << bd) - 1);
// coeffs 0 1 0 1 2 3 2 3
const __m128i tmp_0 = _mm_unpacklo_epi32(filtersReg, filtersReg);
// coeffs 4 5 4 5 6 7 6 7
const __m128i tmp_1 = _mm_unpackhi_epi32(filtersReg, filtersReg);
secondFilters = _mm_unpackhi_epi64(tmp_0, tmp_0); // coeffs 2 3 2 3 2 3 2 3
thirdFilters = _mm_unpacklo_epi64(tmp_1, tmp_1); // coeffs 4 5 4 5 4 5 4 5
for (i = height; i > 0; i -= 1) {
srcReg32b1 = _mm_loadu_si128((const __m128i *)(src_ptr + 2));
__m128i ss_3_1 = _mm_srli_si128(srcReg32b1, 2);
__m128i ss_4_1 = _mm_srli_si128(srcReg32b1, 4);
__m128i ss_5_1 = _mm_srli_si128(srcReg32b1, 6);
__m128i ss_23 = _mm_unpacklo_epi32(srcReg32b1, ss_3_1);
__m128i ss_45 = _mm_unpacklo_epi32(ss_4_1, ss_5_1);
ss_23 = _mm_madd_epi16(ss_23, secondFilters);
ss_45 = _mm_madd_epi16(ss_45, thirdFilters);
srcRegFilt32b1_1 = _mm_add_epi32(ss_23, ss_45);
// shift by 7 bit each 32 bit
srcRegFilt32b1_1 = _mm_add_epi32(srcRegFilt32b1_1, addFilterReg64);
srcRegFilt32b1_1 = _mm_srai_epi32(srcRegFilt32b1_1, 7);
srcRegFilt32b1_1 = _mm_packs_epi32(srcRegFilt32b1_1, _mm_setzero_si128());
srcRegFilt32b1_1 = _mm_max_epi16(srcRegFilt32b1_1, _mm_setzero_si128());
srcRegFilt32b1_1 = _mm_min_epi16(srcRegFilt32b1_1, max);
src_ptr += src_pitch;
_mm_storel_epi64((__m128i *)dst_ptr, srcRegFilt32b1_1);
dst_ptr += dst_pitch;
}
}
void aom_highbd_filter_block1d8_v4_sse2(const uint16_t *src_ptr,
ptrdiff_t src_pitch, uint16_t *dst_ptr,
ptrdiff_t dst_pitch, uint32_t height,
const int16_t *filter, int bd) {
__m128i filtersReg;
__m128i srcReg2, srcReg3, srcReg4, srcReg5, srcReg6;
__m128i srcReg23_lo, srcReg23_hi, srcReg34_lo, srcReg34_hi;
__m128i srcReg45_lo, srcReg45_hi, srcReg56_lo, srcReg56_hi;
__m128i resReg23_lo, resReg34_lo, resReg45_lo, resReg56_lo;
__m128i resReg23_hi, resReg34_hi, resReg45_hi, resReg56_hi;
__m128i resReg23_45_lo, resReg34_56_lo, resReg23_45_hi, resReg34_56_hi;
__m128i resReg23_45, resReg34_56;
__m128i addFilterReg64, secondFilters, thirdFilters;
unsigned int i;
ptrdiff_t src_stride, dst_stride;
const __m128i max = _mm_set1_epi16((1 << bd) - 1);
addFilterReg64 = _mm_set1_epi32(64);
filtersReg = _mm_loadu_si128((const __m128i *)filter);
// coeffs 0 1 0 1 2 3 2 3
const __m128i tmp0 = _mm_unpacklo_epi32(filtersReg, filtersReg);
// coeffs 4 5 4 5 6 7 6 7
const __m128i tmp1 = _mm_unpackhi_epi32(filtersReg, filtersReg);
secondFilters = _mm_unpackhi_epi64(tmp0, tmp0); // coeffs 2 3 2 3 2 3 2 3
thirdFilters = _mm_unpacklo_epi64(tmp1, tmp1); // coeffs 4 5 4 5 4 5 4 5
// multiple the size of the source and destination stride by two
src_stride = src_pitch << 1;
dst_stride = dst_pitch << 1;
srcReg2 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 2));
srcReg3 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 3));
srcReg23_lo = _mm_unpacklo_epi16(srcReg2, srcReg3);
srcReg23_hi = _mm_unpackhi_epi16(srcReg2, srcReg3);
srcReg4 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 4));
srcReg34_lo = _mm_unpacklo_epi16(srcReg3, srcReg4);
srcReg34_hi = _mm_unpackhi_epi16(srcReg3, srcReg4);
for (i = height; i > 1; i -= 2) {
srcReg5 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 5));
srcReg45_lo = _mm_unpacklo_epi16(srcReg4, srcReg5);
srcReg45_hi = _mm_unpackhi_epi16(srcReg4, srcReg5);
srcReg6 = _mm_loadu_si128((const __m128i *)(src_ptr + src_pitch * 6));
srcReg56_lo = _mm_unpacklo_epi16(srcReg5, srcReg6);
srcReg56_hi = _mm_unpackhi_epi16(srcReg5, srcReg6);
// multiply 2 adjacent elements with the filter and add the result
resReg23_lo = _mm_madd_epi16(srcReg23_lo, secondFilters);
resReg34_lo = _mm_madd_epi16(srcReg34_lo, secondFilters);
resReg45_lo = _mm_madd_epi16(srcReg45_lo, thirdFilters);
resReg56_lo = _mm_madd_epi16(srcReg56_lo, thirdFilters);
resReg23_45_lo = _mm_add_epi32(resReg23_lo, resReg45_lo);
resReg34_56_lo = _mm_add_epi32(resReg34_lo, resReg56_lo);
// multiply 2 adjacent elements with the filter and add the result
resReg23_hi = _mm_madd_epi16(srcReg23_hi, secondFilters);
resReg34_hi = _mm_madd_epi16(srcReg34_hi, secondFilters);
resReg45_hi = _mm_madd_epi16(srcReg45_hi, thirdFilters);
resReg56_hi = _mm_madd_epi16(srcReg56_hi, thirdFilters);
resReg23_45_hi = _mm_add_epi32(resReg23_hi, resReg45_hi);
resReg34_56_hi = _mm_add_epi32(resReg34_hi, resReg56_hi);
// shift by 7 bit each 32 bit
resReg23_45_lo = _mm_add_epi32(resReg23_45_lo, addFilterReg64);
resReg34_56_lo = _mm_add_epi32(resReg34_56_lo, addFilterReg64);
resReg23_45_hi = _mm_add_epi32(resReg23_45_hi, addFilterReg64);
resReg34_56_hi = _mm_add_epi32(resReg34_56_hi, addFilterReg64);
resReg23_45_lo = _mm_srai_epi32(resReg23_45_lo, 7);
resReg34_56_lo = _mm_srai_epi32(resReg34_56_lo, 7);
resReg23_45_hi = _mm_srai_epi32(resReg23_45_hi, 7);
resReg34_56_hi = _mm_srai_epi32(resReg34_56_hi, 7);
// shrink to 16 bit each 32 bits, the first lane contain the first
// convolve result and the second lane contain the second convolve
// result
resReg23_45 = _mm_packs_epi32(resReg23_45_lo, resReg23_45_hi);
resReg34_56 = _mm_packs_epi32(resReg34_56_lo, resReg34_56_hi);
resReg23_45 = _mm_max_epi16(resReg23_45, _mm_setzero_si128());
resReg23_45 = _mm_min_epi16(resReg23_45, max);
resReg34_56 = _mm_max_epi16(resReg34_56, _mm_setzero_si128());
resReg34_56 = _mm_min_epi16(resReg34_56, max);
src_ptr += src_stride;
_mm_store_si128((__m128i *)dst_ptr, (resReg23_45));
_mm_store_si128((__m128i *)(dst_ptr + dst_pitch), (resReg34_56));
dst_ptr += dst_stride;
// save part of the registers for next strides
srcReg23_lo = srcReg45_lo;
srcReg23_hi = srcReg45_hi;
srcReg34_lo = srcReg56_lo;
srcReg34_hi = srcReg56_hi;
srcReg4 = srcReg6;
}
}
void aom_highbd_filter_block1d8_h4_sse2(const uint16_t *src_ptr,
ptrdiff_t src_pitch, uint16_t *dst_ptr,
ptrdiff_t dst_pitch, uint32_t height,
const int16_t *filter, int bd) {
__m128i filtersReg;
__m128i addFilterReg64;
__m128i secondFilters, thirdFilters;
__m128i srcRegFilt32b1_1, srcRegFilt32b1_2;
__m128i srcReg32b1, srcReg32b2;
unsigned int i;
src_ptr -= 3;
addFilterReg64 = _mm_set1_epi32(64);
filtersReg = _mm_loadu_si128((const __m128i *)filter);
const __m128i max = _mm_set1_epi16((1 << bd) - 1);
// coeffs 0 1 0 1 2 3 2 3
const __m128i tmp_0 = _mm_unpacklo_epi32(filtersReg, filtersReg);
// coeffs 4 5 4 5 6 7 6 7
const __m128i tmp_1 = _mm_unpackhi_epi32(filtersReg, filtersReg);
secondFilters = _mm_unpackhi_epi64(tmp_0, tmp_0); // coeffs 2 3 2 3 2 3 2 3
thirdFilters = _mm_unpacklo_epi64(tmp_1, tmp_1); // coeffs 4 5 4 5 4 5 4 5
for (i = height; i > 0; i -= 1) {
srcReg32b1 = _mm_loadu_si128((const __m128i *)(src_ptr + 2));
srcReg32b2 = _mm_loadu_si128((const __m128i *)(src_ptr + 6));
__m128i ss_4_1 = _mm_srli_si128(srcReg32b1, 4);
__m128i ss_4_2 = _mm_srli_si128(srcReg32b2, 4);
__m128i ss_4 = _mm_unpacklo_epi64(ss_4_1, ss_4_2);
__m128i d1 = _mm_madd_epi16(srcReg32b1, secondFilters);
__m128i d2 = _mm_madd_epi16(ss_4, thirdFilters);
srcRegFilt32b1_1 = _mm_add_epi32(d1, d2);
__m128i ss_3_1 = _mm_srli_si128(srcReg32b1, 2);
__m128i ss_5_1 = _mm_srli_si128(srcReg32b1, 6);
__m128i ss_3_2 = _mm_srli_si128(srcReg32b2, 2);
__m128i ss_5_2 = _mm_srli_si128(srcReg32b2, 6);
__m128i ss_3 = _mm_unpacklo_epi64(ss_3_1, ss_3_2);
__m128i ss_5 = _mm_unpacklo_epi64(ss_5_1, ss_5_2);
d1 = _mm_madd_epi16(ss_3, secondFilters);
d2 = _mm_madd_epi16(ss_5, thirdFilters);
srcRegFilt32b1_2 = _mm_add_epi32(d1, d2);
__m128i res_lo_1 = _mm_unpacklo_epi32(srcRegFilt32b1_1, srcRegFilt32b1_2);
__m128i res_hi_1 = _mm_unpackhi_epi32(srcRegFilt32b1_1, srcRegFilt32b1_2);
// shift by 7 bit each 32 bit
res_lo_1 = _mm_add_epi32(res_lo_1, addFilterReg64);
res_hi_1 = _mm_add_epi32(res_hi_1, addFilterReg64);
res_lo_1 = _mm_srai_epi32(res_lo_1, 7);
res_hi_1 = _mm_srai_epi32(res_hi_1, 7);
srcRegFilt32b1_1 = _mm_packs_epi32(res_lo_1, res_hi_1);
srcRegFilt32b1_1 = _mm_max_epi16(srcRegFilt32b1_1, _mm_setzero_si128());
srcRegFilt32b1_1 = _mm_min_epi16(srcRegFilt32b1_1, max);
src_ptr += src_pitch;
_mm_store_si128((__m128i *)dst_ptr, srcRegFilt32b1_1);
dst_ptr += dst_pitch;
}
}
void aom_highbd_filter_block1d16_v4_sse2(const uint16_t *src_ptr,
ptrdiff_t src_pitch, uint16_t *dst_ptr,
ptrdiff_t dst_pitch, uint32_t height,
const int16_t *filter, int bd) {
aom_highbd_filter_block1d8_v4_sse2(src_ptr, src_pitch, dst_ptr, dst_pitch,
height, filter, bd);
aom_highbd_filter_block1d8_v4_sse2((src_ptr + 8), src_pitch, (dst_ptr + 8),
dst_pitch, height, filter, bd);
}
void aom_highbd_filter_block1d16_h4_sse2(const uint16_t *src_ptr,
ptrdiff_t src_pitch, uint16_t *dst_ptr,
ptrdiff_t dst_pitch, uint32_t height,
const int16_t *filter, int bd) {
aom_highbd_filter_block1d8_h4_sse2(src_ptr, src_pitch, dst_ptr, dst_pitch,
height, filter, bd);
aom_highbd_filter_block1d8_h4_sse2((src_ptr + 8), src_pitch, (dst_ptr + 8),
dst_pitch, height, filter, bd);
}

259
third_party/aom/aom_dsp/x86/highbd_intrapred_asm_sse2.asm vendored Normal file
View file

@ -0,0 +1,259 @@
;
; Copyright (c) 2016, Alliance for Open Media. All rights reserved
;
; This source code is subject to the terms of the BSD 2 Clause License and
; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
; was not distributed with this source code in the LICENSE file, you can
; obtain it at www.aomedia.org/license/software. If the Alliance for Open
; Media Patent License 1.0 was not distributed with this source code in the
; PATENTS file, you can obtain it at www.aomedia.org/license/patent.
;
;
%include "third_party/x86inc/x86inc.asm"
SECTION_RODATA
pw_4: times 8 dw 4
pw_8: times 8 dw 8
pw_16: times 4 dd 16
pw_32: times 4 dd 32
SECTION .text
INIT_XMM sse2
cglobal highbd_dc_predictor_4x4, 4, 5, 4, dst, stride, above, left, goffset
GET_GOT goffsetq
movq m0, [aboveq]
movq m2, [leftq]
paddw m0, m2
pshuflw m1, m0, 0xe
paddw m0, m1
pshuflw m1, m0, 0x1
paddw m0, m1
paddw m0, [GLOBAL(pw_4)]
psraw m0, 3
pshuflw m0, m0, 0x0
movq [dstq ], m0
movq [dstq+strideq*2], m0
lea dstq, [dstq+strideq*4]
movq [dstq ], m0
movq [dstq+strideq*2], m0
RESTORE_GOT
RET
INIT_XMM sse2
cglobal highbd_dc_predictor_8x8, 4, 5, 4, dst, stride, above, left, goffset
GET_GOT goffsetq
pxor m1, m1
mova m0, [aboveq]
mova m2, [leftq]
DEFINE_ARGS dst, stride, stride3, one
mov oned, 0x00010001
lea stride3q, [strideq*3]
movd m3, oned
pshufd m3, m3, 0x0
paddw m0, m2
pmaddwd m0, m3
packssdw m0, m1
pmaddwd m0, m3
packssdw m0, m1
pmaddwd m0, m3
paddw m0, [GLOBAL(pw_8)]
psrlw m0, 4
pshuflw m0, m0, 0x0
punpcklqdq m0, m0
mova [dstq ], m0
mova [dstq+strideq*2 ], m0
mova [dstq+strideq*4 ], m0
mova [dstq+stride3q*2], m0
lea dstq, [dstq+strideq*8]
mova [dstq ], m0
mova [dstq+strideq*2 ], m0
mova [dstq+strideq*4 ], m0
mova [dstq+stride3q*2], m0
RESTORE_GOT
RET
INIT_XMM sse2
cglobal highbd_dc_predictor_16x16, 4, 5, 5, dst, stride, above, left, goffset
GET_GOT goffsetq
pxor m1, m1
mova m0, [aboveq]
mova m3, [aboveq+16]
mova m2, [leftq]
mova m4, [leftq+16]
DEFINE_ARGS dst, stride, stride3, lines4
lea stride3q, [strideq*3]
mov lines4d, 4
paddw m0, m2
paddw m0, m3
paddw m0, m4
movhlps m2, m0
paddw m0, m2
punpcklwd m0, m1
movhlps m2, m0
paddd m0, m2
punpckldq m0, m1
movhlps m2, m0
paddd m0, m2
paddd m0, [GLOBAL(pw_16)]
psrad m0, 5
pshuflw m0, m0, 0x0
punpcklqdq m0, m0
.loop:
mova [dstq ], m0
mova [dstq +16], m0
mova [dstq+strideq*2 ], m0
mova [dstq+strideq*2 +16], m0
mova [dstq+strideq*4 ], m0
mova [dstq+strideq*4 +16], m0
mova [dstq+stride3q*2 ], m0
mova [dstq+stride3q*2+16], m0
lea dstq, [dstq+strideq*8]
dec lines4d
jnz .loop
RESTORE_GOT
REP_RET
INIT_XMM sse2
cglobal highbd_dc_predictor_32x32, 4, 5, 7, dst, stride, above, left, goffset
GET_GOT goffsetq
mova m0, [aboveq]
mova m2, [aboveq+16]
mova m3, [aboveq+32]
mova m4, [aboveq+48]
paddw m0, m2
paddw m3, m4
mova m2, [leftq]
mova m4, [leftq+16]
mova m5, [leftq+32]
mova m6, [leftq+48]
paddw m2, m4
paddw m5, m6
paddw m0, m3
paddw m2, m5
pxor m1, m1
paddw m0, m2
DEFINE_ARGS dst, stride, stride3, lines4
lea stride3q, [strideq*3]
mov lines4d, 8
movhlps m2, m0
paddw m0, m2
punpcklwd m0, m1
movhlps m2, m0
paddd m0, m2
punpckldq m0, m1
movhlps m2, m0
paddd m0, m2
paddd m0, [GLOBAL(pw_32)]
psrad m0, 6
pshuflw m0, m0, 0x0
punpcklqdq m0, m0
.loop:
mova [dstq ], m0
mova [dstq +16 ], m0
mova [dstq +32 ], m0
mova [dstq +48 ], m0
mova [dstq+strideq*2 ], m0
mova [dstq+strideq*2+16 ], m0
mova [dstq+strideq*2+32 ], m0
mova [dstq+strideq*2+48 ], m0
mova [dstq+strideq*4 ], m0
mova [dstq+strideq*4+16 ], m0
mova [dstq+strideq*4+32 ], m0
mova [dstq+strideq*4+48 ], m0
mova [dstq+stride3q*2 ], m0
mova [dstq+stride3q*2 +16], m0
mova [dstq+stride3q*2 +32], m0
mova [dstq+stride3q*2 +48], m0
lea dstq, [dstq+strideq*8]
dec lines4d
jnz .loop
RESTORE_GOT
REP_RET
INIT_XMM sse2
cglobal highbd_v_predictor_4x4, 3, 3, 1, dst, stride, above
movq m0, [aboveq]
movq [dstq ], m0
movq [dstq+strideq*2], m0
lea dstq, [dstq+strideq*4]
movq [dstq ], m0
movq [dstq+strideq*2], m0
RET
INIT_XMM sse2
cglobal highbd_v_predictor_8x8, 3, 3, 1, dst, stride, above
mova m0, [aboveq]
DEFINE_ARGS dst, stride, stride3
lea stride3q, [strideq*3]
mova [dstq ], m0
mova [dstq+strideq*2 ], m0
mova [dstq+strideq*4 ], m0
mova [dstq+stride3q*2], m0
lea dstq, [dstq+strideq*8]
mova [dstq ], m0
mova [dstq+strideq*2 ], m0
mova [dstq+strideq*4 ], m0
mova [dstq+stride3q*2], m0
RET
INIT_XMM sse2
cglobal highbd_v_predictor_16x16, 3, 4, 2, dst, stride, above
mova m0, [aboveq]
mova m1, [aboveq+16]
DEFINE_ARGS dst, stride, stride3, nlines4
lea stride3q, [strideq*3]
mov nlines4d, 4
.loop:
mova [dstq ], m0
mova [dstq +16], m1
mova [dstq+strideq*2 ], m0
mova [dstq+strideq*2 +16], m1
mova [dstq+strideq*4 ], m0
mova [dstq+strideq*4 +16], m1
mova [dstq+stride3q*2 ], m0
mova [dstq+stride3q*2+16], m1
lea dstq, [dstq+strideq*8]
dec nlines4d
jnz .loop
REP_RET
INIT_XMM sse2
cglobal highbd_v_predictor_32x32, 3, 4, 4, dst, stride, above
mova m0, [aboveq]
mova m1, [aboveq+16]
mova m2, [aboveq+32]
mova m3, [aboveq+48]
DEFINE_ARGS dst, stride, stride3, nlines4
lea stride3q, [strideq*3]
mov nlines4d, 8
.loop:
mova [dstq ], m0
mova [dstq +16], m1
mova [dstq +32], m2
mova [dstq +48], m3
mova [dstq+strideq*2 ], m0
mova [dstq+strideq*2 +16], m1
mova [dstq+strideq*2 +32], m2
mova [dstq+strideq*2 +48], m3
mova [dstq+strideq*4 ], m0
mova [dstq+strideq*4 +16], m1
mova [dstq+strideq*4 +32], m2
mova [dstq+strideq*4 +48], m3
mova [dstq+stride3q*2 ], m0
mova [dstq+stride3q*2 +16], m1
mova [dstq+stride3q*2 +32], m2
mova [dstq+stride3q*2 +48], m3
lea dstq, [dstq+strideq*8]
dec nlines4d
jnz .loop
REP_RET

4
third_party/aom/aom_dsp/x86/highbd_loopfilter_sse2.c vendored
View file

@ -727,8 +727,8 @@ void aom_highbd_lpf_horizontal_14_dual_sse2(
_limit1, _thresh1, bd);
for (i = 0; i < 6; i++) {
_mm_store_si128((__m128i *)(s - (i + 1) * pitch), p[i]);
_mm_store_si128((__m128i *)(s + i * pitch), q[i]);
_mm_storeu_si128((__m128i *)(s - (i + 1) * pitch), p[i]);
_mm_storeu_si128((__m128i *)(s + i * pitch), q[i]);
}
}

52
third_party/aom/aom_dsp/x86/highbd_subpel_variance_impl_sse2.asm vendored
View file

@ -114,45 +114,33 @@ SECTION .text
cglobal highbd_sub_pixel_avg_variance%1xh, 7, 7, 13, src, src_stride, \
x_offset, y_offset, \
dst, dst_stride, \
sec, sec_stride, height, sse, \
g_bilin_filter, g_pw_8
sec, sec_stride, height, sse
%define block_height dword heightm
%define sec_str sec_stridemp
; Store bilin_filter and pw_8 location in stack
%if GET_GOT_DEFINED == 1
GET_GOT eax
add esp, 4 ; restore esp
%endif
lea ecx, [GLOBAL(bilin_filter_m)]
mov g_bilin_filterm, ecx
lea ecx, [GLOBAL(pw_8)]
mov g_pw_8m, ecx
LOAD_IF_USED 0, 1 ; load eax, ecx back
%else
cglobal highbd_sub_pixel_variance%1xh, 7, 7, 13, src, src_stride, \
x_offset, y_offset, \
dst, dst_stride, height, sse, \
g_bilin_filter, g_pw_8
dst, dst_stride, height, sse
%define block_height heightd
; Store bilin_filter and pw_8 location in stack
%if GET_GOT_DEFINED == 1
GET_GOT eax
add esp, 4 ; restore esp
%endif
lea ecx, [GLOBAL(bilin_filter_m)]
mov g_bilin_filterm, ecx
lea ecx, [GLOBAL(pw_8)]
mov g_pw_8m, ecx
LOAD_IF_USED 0, 1 ; load eax, ecx back
%endif
; reuse argument stack space
%define g_bilin_filterm x_offsetm
%define g_pw_8m y_offsetm
; Store bilin_filter and pw_8 location in stack
%if GET_GOT_DEFINED == 1
GET_GOT eax
add esp, 4 ; restore esp
%endif
lea ecx, [GLOBAL(bilin_filter_m)]
mov g_bilin_filterm, ecx
lea ecx, [GLOBAL(pw_8)]
mov g_pw_8m, ecx
LOAD_IF_USED 0, 1 ; load eax, ecx back
%else
%if %2 == 1 ; avg
cglobal highbd_sub_pixel_avg_variance%1xh, 7, 7, 13, src, src_stride, \

48
third_party/aom/aom_dsp/x86/highbd_subtract_sse2.c vendored
View file

@ -29,15 +29,15 @@ static void subtract_4x4(int16_t *diff, ptrdiff_t diff_stride,
__m128i x0, x1, x2, x3;
int64_t *store_diff = (int64_t *)(diff + 0 * diff_stride);
u0 = _mm_loadu_si128((__m128i const *)(src + 0 * src_stride));
u1 = _mm_loadu_si128((__m128i const *)(src + 1 * src_stride));
u2 = _mm_loadu_si128((__m128i const *)(src + 2 * src_stride));
u3 = _mm_loadu_si128((__m128i const *)(src + 3 * src_stride));
u0 = _mm_loadl_epi64((__m128i const *)(src + 0 * src_stride));
u1 = _mm_loadl_epi64((__m128i const *)(src + 1 * src_stride));
u2 = _mm_loadl_epi64((__m128i const *)(src + 2 * src_stride));
u3 = _mm_loadl_epi64((__m128i const *)(src + 3 * src_stride));
v0 = _mm_loadu_si128((__m128i const *)(pred + 0 * pred_stride));
v1 = _mm_loadu_si128((__m128i const *)(pred + 1 * pred_stride));
v2 = _mm_loadu_si128((__m128i const *)(pred + 2 * pred_stride));
v3 = _mm_loadu_si128((__m128i const *)(pred + 3 * pred_stride));
v0 = _mm_loadl_epi64((__m128i const *)(pred + 0 * pred_stride));
v1 = _mm_loadl_epi64((__m128i const *)(pred + 1 * pred_stride));
v2 = _mm_loadl_epi64((__m128i const *)(pred + 2 * pred_stride));
v3 = _mm_loadl_epi64((__m128i const *)(pred + 3 * pred_stride));
x0 = _mm_sub_epi16(u0, v0);
x1 = _mm_sub_epi16(u1, v1);
@ -61,23 +61,23 @@ static void subtract_4x8(int16_t *diff, ptrdiff_t diff_stride,
__m128i x0, x1, x2, x3, x4, x5, x6, x7;
int64_t *store_diff = (int64_t *)(diff + 0 * diff_stride);
u0 = _mm_loadu_si128((__m128i const *)(src + 0 * src_stride));
u1 = _mm_loadu_si128((__m128i const *)(src + 1 * src_stride));
u2 = _mm_loadu_si128((__m128i const *)(src + 2 * src_stride));
u3 = _mm_loadu_si128((__m128i const *)(src + 3 * src_stride));
u4 = _mm_loadu_si128((__m128i const *)(src + 4 * src_stride));
u5 = _mm_loadu_si128((__m128i const *)(src + 5 * src_stride));
u6 = _mm_loadu_si128((__m128i const *)(src + 6 * src_stride));
u7 = _mm_loadu_si128((__m128i const *)(src + 7 * src_stride));
u0 = _mm_loadl_epi64((__m128i const *)(src + 0 * src_stride));
u1 = _mm_loadl_epi64((__m128i const *)(src + 1 * src_stride));
u2 = _mm_loadl_epi64((__m128i const *)(src + 2 * src_stride));
u3 = _mm_loadl_epi64((__m128i const *)(src + 3 * src_stride));
u4 = _mm_loadl_epi64((__m128i const *)(src + 4 * src_stride));
u5 = _mm_loadl_epi64((__m128i const *)(src + 5 * src_stride));
u6 = _mm_loadl_epi64((__m128i const *)(src + 6 * src_stride));
u7 = _mm_loadl_epi64((__m128i const *)(src + 7 * src_stride));
v0 = _mm_loadu_si128((__m128i const *)(pred + 0 * pred_stride));
v1 = _mm_loadu_si128((__m128i const *)(pred + 1 * pred_stride));
v2 = _mm_loadu_si128((__m128i const *)(pred + 2 * pred_stride));
v3 = _mm_loadu_si128((__m128i const *)(pred + 3 * pred_stride));
v4 = _mm_loadu_si128((__m128i const *)(pred + 4 * pred_stride));
v5 = _mm_loadu_si128((__m128i const *)(pred + 5 * pred_stride));
v6 = _mm_loadu_si128((__m128i const *)(pred + 6 * pred_stride));
v7 = _mm_loadu_si128((__m128i const *)(pred + 7 * pred_stride));
v0 = _mm_loadl_epi64((__m128i const *)(pred + 0 * pred_stride));
v1 = _mm_loadl_epi64((__m128i const *)(pred + 1 * pred_stride));
v2 = _mm_loadl_epi64((__m128i const *)(pred + 2 * pred_stride));
v3 = _mm_loadl_epi64((__m128i const *)(pred + 3 * pred_stride));
v4 = _mm_loadl_epi64((__m128i const *)(pred + 4 * pred_stride));
v5 = _mm_loadl_epi64((__m128i const *)(pred + 5 * pred_stride));
v6 = _mm_loadl_epi64((__m128i const *)(pred + 6 * pred_stride));
v7 = _mm_loadl_epi64((__m128i const *)(pred + 7 * pred_stride));
x0 = _mm_sub_epi16(u0, v0);
x1 = _mm_sub_epi16(u1, v1);

59
third_party/aom/aom_dsp/x86/highbd_variance_sse2.c vendored
View file

@ -603,7 +603,7 @@ void aom_highbd_upsampled_pred_sse2(MACROBLOCKD *xd,
const int ref_num = 0;
const int is_intrabc = is_intrabc_block(mi);
const struct scale_factors *const sf =
is_intrabc ? &cm->sf_identity : &xd->block_refs[ref_num]->sf;
is_intrabc ? &cm->sf_identity : xd->block_ref_scale_factors[ref_num];
const int is_scaled = av1_is_scaled(sf);
if (is_scaled) {
@ -677,11 +677,8 @@ void aom_highbd_upsampled_pred_sse2(MACROBLOCKD *xd,
}
}
const InterpFilterParams *filter =
(subpel_search == 1)
? av1_get_4tap_interp_filter_params(EIGHTTAP_REGULAR)
: av1_get_interp_filter_params_with_block_size(EIGHTTAP_REGULAR, 8);
const InterpFilterParams *filter = av1_get_filter(subpel_search);
int filter_taps = (subpel_search <= USE_4_TAPS) ? 4 : SUBPEL_TAPS;
if (!subpel_x_q3 && !subpel_y_q3) {
uint16_t *ref = CONVERT_TO_SHORTPTR(ref8);
uint16_t *comp_pred = CONVERT_TO_SHORTPTR(comp_pred8);
@ -729,17 +726,20 @@ void aom_highbd_upsampled_pred_sse2(MACROBLOCKD *xd,
av1_get_interp_filter_subpel_kernel(filter, subpel_x_q3 << 1);
const int16_t *const kernel_y =
av1_get_interp_filter_subpel_kernel(filter, subpel_y_q3 << 1);
const uint8_t *ref_start = ref8 - ref_stride * ((filter_taps >> 1) - 1);
uint16_t *temp_start_horiz = (subpel_search <= USE_4_TAPS)
? temp + (filter_taps >> 1) * MAX_SB_SIZE
: temp;
uint16_t *temp_start_vert = temp + MAX_SB_SIZE * ((filter->taps >> 1) - 1);
const int intermediate_height =
(((height - 1) * 8 + subpel_y_q3) >> 3) + filter->taps;
(((height - 1) * 8 + subpel_y_q3) >> 3) + filter_taps;
assert(intermediate_height <= (MAX_SB_SIZE * 2 + 16) + 16);
aom_highbd_convolve8_horiz(ref8 - ref_stride * ((filter->taps >> 1) - 1),
ref_stride, CONVERT_TO_BYTEPTR(temp),
MAX_SB_SIZE, kernel_x, 16, NULL, -1, width,
intermediate_height, bd);
aom_highbd_convolve8_vert(
CONVERT_TO_BYTEPTR(temp + MAX_SB_SIZE * ((filter->taps >> 1) - 1)),
MAX_SB_SIZE, comp_pred8, width, NULL, -1, kernel_y, 16, width, height,
bd);
aom_highbd_convolve8_horiz(
ref_start, ref_stride, CONVERT_TO_BYTEPTR(temp_start_horiz),
MAX_SB_SIZE, kernel_x, 16, NULL, -1, width, intermediate_height, bd);
aom_highbd_convolve8_vert(CONVERT_TO_BYTEPTR(temp_start_vert), MAX_SB_SIZE,
comp_pred8, width, NULL, -1, kernel_y, 16, width,
height, bd);
}
}
@ -765,11 +765,11 @@ void aom_highbd_comp_avg_upsampled_pred_sse2(
}
}
static INLINE void highbd_compute_jnt_comp_avg(__m128i *p0, __m128i *p1,
const __m128i *w0,
const __m128i *w1,
const __m128i *r,
void *const result) {
static INLINE void highbd_compute_dist_wtd_comp_avg(__m128i *p0, __m128i *p1,
const __m128i *w0,
const __m128i *w1,
const __m128i *r,
void *const result) {
assert(DIST_PRECISION_BITS <= 4);
__m128i mult0 = _mm_mullo_epi16(*p0, *w0);
__m128i mult1 = _mm_mullo_epi16(*p1, *w1);
@ -780,11 +780,10 @@ static INLINE void highbd_compute_jnt_comp_avg(__m128i *p0, __m128i *p1,
xx_storeu_128(result, shift);
}
void aom_highbd_jnt_comp_avg_pred_sse2(uint8_t *comp_pred8,
const uint8_t *pred8, int width,
int height, const uint8_t *ref8,
int ref_stride,
const JNT_COMP_PARAMS *jcp_param) {
void aom_highbd_dist_wtd_comp_avg_pred_sse2(
uint8_t *comp_pred8, const uint8_t *pred8, int width, int height,
const uint8_t *ref8, int ref_stride,
const DIST_WTD_COMP_PARAMS *jcp_param) {
int i;
const uint16_t wt0 = (uint16_t)jcp_param->fwd_offset;
const uint16_t wt1 = (uint16_t)jcp_param->bck_offset;
@ -806,7 +805,7 @@ void aom_highbd_jnt_comp_avg_pred_sse2(uint8_t *comp_pred8,
__m128i p0 = xx_loadu_128(ref);
__m128i p1 = xx_loadu_128(pred);
highbd_compute_jnt_comp_avg(&p0, &p1, &w0, &w1, &r, comp_pred);
highbd_compute_dist_wtd_comp_avg(&p0, &p1, &w0, &w1, &r, comp_pred);
comp_pred += 8;
pred += 8;
@ -823,7 +822,7 @@ void aom_highbd_jnt_comp_avg_pred_sse2(uint8_t *comp_pred8,
__m128i p0 = _mm_unpacklo_epi64(p0_0, p0_1);
__m128i p1 = xx_loadu_128(pred);
highbd_compute_jnt_comp_avg(&p0, &p1, &w0, &w1, &r, comp_pred);
highbd_compute_dist_wtd_comp_avg(&p0, &p1, &w0, &w1, &r, comp_pred);
comp_pred += 8;
pred += 8;
@ -832,11 +831,11 @@ void aom_highbd_jnt_comp_avg_pred_sse2(uint8_t *comp_pred8,
}
}
void aom_highbd_jnt_comp_avg_upsampled_pred_sse2(
void aom_highbd_dist_wtd_comp_avg_upsampled_pred_sse2(
MACROBLOCKD *xd, const struct AV1Common *const cm, int mi_row, int mi_col,
const MV *const mv, uint8_t *comp_pred8, const uint8_t *pred8, int width,
int height, int subpel_x_q3, int subpel_y_q3, const uint8_t *ref8,
int ref_stride, int bd, const JNT_COMP_PARAMS *jcp_param,
int ref_stride, int bd, const DIST_WTD_COMP_PARAMS *jcp_param,
int subpel_search) {
uint16_t *pred = CONVERT_TO_SHORTPTR(pred8);
int n;
@ -860,7 +859,7 @@ void aom_highbd_jnt_comp_avg_upsampled_pred_sse2(
__m128i p0 = xx_loadu_128(comp_pred16);
__m128i p1 = xx_loadu_128(pred);
highbd_compute_jnt_comp_avg(&p0, &p1, &w0, &w1, &r, comp_pred16);
highbd_compute_dist_wtd_comp_avg(&p0, &p1, &w0, &w1, &r, comp_pred16);
comp_pred16 += 8;
pred += 8;

608
third_party/aom/aom_dsp/x86/intrapred_asm_sse2.asm vendored Normal file
View file

@ -0,0 +1,608 @@
;
; Copyright (c) 2016, Alliance for Open Media. All rights reserved
;
; This source code is subject to the terms of the BSD 2 Clause License and
; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
; was not distributed with this source code in the LICENSE file, you can
; obtain it at www.aomedia.org/license/software. If the Alliance for Open
; Media Patent License 1.0 was not distributed with this source code in the
; PATENTS file, you can obtain it at www.aomedia.org/license/patent.
;
;
%include "third_party/x86inc/x86inc.asm"
SECTION_RODATA
pb_1: times 16 db 1
pw_4: times 8 dw 4
pw_8: times 8 dw 8
pw_16: times 8 dw 16
pw_32: times 8 dw 32
dc_128: times 16 db 128
pw2_4: times 8 dw 2
pw2_8: times 8 dw 4
pw2_16: times 8 dw 8
pw2_32: times 8 dw 16
SECTION .text
INIT_XMM sse2
cglobal dc_predictor_4x4, 4, 5, 3, dst, stride, above, left, goffset
GET_GOT goffsetq
movd m2, [leftq]
movd m0, [aboveq]
pxor m1, m1
punpckldq m0, m2
psadbw m0, m1
paddw m0, [GLOBAL(pw_4)]
psraw m0, 3
pshuflw m0, m0, 0x0
packuswb m0, m0
movd [dstq ], m0
movd [dstq+strideq], m0
lea dstq, [dstq+strideq*2]
movd [dstq ], m0
movd [dstq+strideq], m0
RESTORE_GOT
RET
INIT_XMM sse2
cglobal dc_left_predictor_4x4, 2, 5, 2, dst, stride, above, left, goffset
movifnidn leftq, leftmp
GET_GOT goffsetq
pxor m1, m1
movd m0, [leftq]
psadbw m0, m1
paddw m0, [GLOBAL(pw2_4)]
psraw m0, 2
pshuflw m0, m0, 0x0
packuswb m0, m0
movd [dstq ], m0
movd [dstq+strideq], m0
lea dstq, [dstq+strideq*2]
movd [dstq ], m0
movd [dstq+strideq], m0
RESTORE_GOT
RET
INIT_XMM sse2
cglobal dc_top_predictor_4x4, 3, 5, 2, dst, stride, above, left, goffset
GET_GOT goffsetq
pxor m1, m1
movd m0, [aboveq]
psadbw m0, m1
paddw m0, [GLOBAL(pw2_4)]
psraw m0, 2
pshuflw m0, m0, 0x0
packuswb m0, m0
movd [dstq ], m0
movd [dstq+strideq], m0
lea dstq, [dstq+strideq*2]
movd [dstq ], m0
movd [dstq+strideq], m0
RESTORE_GOT
RET
INIT_XMM sse2
cglobal dc_predictor_8x8, 4, 5, 3, dst, stride, above, left, goffset
GET_GOT goffsetq
pxor m1, m1
movq m0, [aboveq]
movq m2, [leftq]
DEFINE_ARGS dst, stride, stride3
lea stride3q, [strideq*3]
psadbw m0, m1
psadbw m2, m1
paddw m0, m2
paddw m0, [GLOBAL(pw_8)]
psraw m0, 4
punpcklbw m0, m0
pshuflw m0, m0, 0x0
movq [dstq ], m0
movq [dstq+strideq ], m0
movq [dstq+strideq*2], m0
movq [dstq+stride3q ], m0
lea dstq, [dstq+strideq*4]
movq [dstq ], m0
movq [dstq+strideq ], m0
movq [dstq+strideq*2], m0
movq [dstq+stride3q ], m0
RESTORE_GOT
RET
INIT_XMM sse2
cglobal dc_top_predictor_8x8, 3, 5, 2, dst, stride, above, left, goffset
GET_GOT goffsetq
pxor m1, m1
movq m0, [aboveq]
DEFINE_ARGS dst, stride, stride3
lea stride3q, [strideq*3]
psadbw m0, m1
paddw m0, [GLOBAL(pw2_8)]
psraw m0, 3
punpcklbw m0, m0
pshuflw m0, m0, 0x0
movq [dstq ], m0
movq [dstq+strideq ], m0
movq [dstq+strideq*2], m0
movq [dstq+stride3q ], m0
lea dstq, [dstq+strideq*4]
movq [dstq ], m0
movq [dstq+strideq ], m0
movq [dstq+strideq*2], m0
movq [dstq+stride3q ], m0
RESTORE_GOT
RET
INIT_XMM sse2
cglobal dc_left_predictor_8x8, 2, 5, 2, dst, stride, above, left, goffset
movifnidn leftq, leftmp
GET_GOT goffsetq
pxor m1, m1
movq m0, [leftq]
DEFINE_ARGS dst, stride, stride3
lea stride3q, [strideq*3]
psadbw m0, m1
paddw m0, [GLOBAL(pw2_8)]
psraw m0, 3
punpcklbw m0, m0
pshuflw m0, m0, 0x0
movq [dstq ], m0
movq [dstq+strideq ], m0
movq [dstq+strideq*2], m0
movq [dstq+stride3q ], m0
lea dstq, [dstq+strideq*4]
movq [dstq ], m0
movq [dstq+strideq ], m0
movq [dstq+strideq*2], m0
movq [dstq+stride3q ], m0
RESTORE_GOT
RET
INIT_XMM sse2
cglobal dc_128_predictor_4x4, 2, 5, 1, dst, stride, above, left, goffset
GET_GOT goffsetq
DEFINE_ARGS dst, stride, stride3
lea stride3q, [strideq*3]
movd m0, [GLOBAL(dc_128)]
movd [dstq ], m0
movd [dstq+strideq ], m0
movd [dstq+strideq*2], m0
movd [dstq+stride3q ], m0
RESTORE_GOT
RET
INIT_XMM sse2
cglobal dc_128_predictor_8x8, 2, 5, 1, dst, stride, above, left, goffset
GET_GOT goffsetq
DEFINE_ARGS dst, stride, stride3
lea stride3q, [strideq*3]
movq m0, [GLOBAL(dc_128)]
movq [dstq ], m0
movq [dstq+strideq ], m0
movq [dstq+strideq*2], m0
movq [dstq+stride3q ], m0
lea dstq, [dstq+strideq*4]
movq [dstq ], m0
movq [dstq+strideq ], m0
movq [dstq+strideq*2], m0
movq [dstq+stride3q ], m0
RESTORE_GOT
RET
INIT_XMM sse2
cglobal dc_predictor_16x16, 4, 5, 3, dst, stride, above, left, goffset
GET_GOT goffsetq
pxor m1, m1
mova m0, [aboveq]
mova m2, [leftq]
DEFINE_ARGS dst, stride, stride3, lines4
lea stride3q, [strideq*3]
mov lines4d, 4
psadbw m0, m1
psadbw m2, m1
paddw m0, m2
movhlps m2, m0
paddw m0, m2
paddw m0, [GLOBAL(pw_16)]
psraw m0, 5
pshuflw m0, m0, 0x0
punpcklqdq m0, m0
packuswb m0, m0
.loop:
mova [dstq ], m0
mova [dstq+strideq ], m0
mova [dstq+strideq*2], m0
mova [dstq+stride3q ], m0
lea dstq, [dstq+strideq*4]
dec lines4d
jnz .loop
RESTORE_GOT
REP_RET
INIT_XMM sse2
cglobal dc_top_predictor_16x16, 4, 5, 3, dst, stride, above, left, goffset
GET_GOT goffsetq
pxor m1, m1
mova m0, [aboveq]
DEFINE_ARGS dst, stride, stride3, lines4
lea stride3q, [strideq*3]
mov lines4d, 4
psadbw m0, m1
movhlps m2, m0
paddw m0, m2
paddw m0, [GLOBAL(pw2_16)]
psraw m0, 4
pshuflw m0, m0, 0x0
punpcklqdq m0, m0
packuswb m0, m0
.loop:
mova [dstq ], m0
mova [dstq+strideq ], m0
mova [dstq+strideq*2], m0
mova [dstq+stride3q ], m0
lea dstq, [dstq+strideq*4]
dec lines4d
jnz .loop
RESTORE_GOT
REP_RET
INIT_XMM sse2
cglobal dc_left_predictor_16x16, 4, 5, 3, dst, stride, above, left, goffset
GET_GOT goffsetq
pxor m1, m1
mova m0, [leftq]
DEFINE_ARGS dst, stride, stride3, lines4
lea stride3q, [strideq*3]
mov lines4d, 4
psadbw m0, m1
movhlps m2, m0
paddw m0, m2
paddw m0, [GLOBAL(pw2_16)]
psraw m0, 4
pshuflw m0, m0, 0x0
punpcklqdq m0, m0
packuswb m0, m0
.loop:
mova [dstq ], m0
mova [dstq+strideq ], m0
mova [dstq+strideq*2], m0
mova [dstq+stride3q ], m0
lea dstq, [dstq+strideq*4]
dec lines4d
jnz .loop
RESTORE_GOT
REP_RET
INIT_XMM sse2
cglobal dc_128_predictor_16x16, 4, 5, 3, dst, stride, above, left, goffset
GET_GOT goffsetq
DEFINE_ARGS dst, stride, stride3, lines4
lea stride3q, [strideq*3]
mov lines4d, 4
mova m0, [GLOBAL(dc_128)]
.loop:
mova [dstq ], m0
mova [dstq+strideq ], m0
mova [dstq+strideq*2], m0
mova [dstq+stride3q ], m0
lea dstq, [dstq+strideq*4]
dec lines4d
jnz .loop
RESTORE_GOT
RET
INIT_XMM sse2
cglobal dc_predictor_32x32, 4, 5, 5, dst, stride, above, left, goffset
GET_GOT goffsetq
pxor m1, m1
mova m0, [aboveq]
mova m2, [aboveq+16]
mova m3, [leftq]
mova m4, [leftq+16]
DEFINE_ARGS dst, stride, stride3, lines4
lea stride3q, [strideq*3]
mov lines4d, 8
psadbw m0, m1
psadbw m2, m1
psadbw m3, m1
psadbw m4, m1
paddw m0, m2
paddw m0, m3
paddw m0, m4
movhlps m2, m0
paddw m0, m2
paddw m0, [GLOBAL(pw_32)]
psraw m0, 6
pshuflw m0, m0, 0x0
punpcklqdq m0, m0
packuswb m0, m0
.loop:
mova [dstq ], m0
mova [dstq +16], m0
mova [dstq+strideq ], m0
mova [dstq+strideq +16], m0
mova [dstq+strideq*2 ], m0
mova [dstq+strideq*2+16], m0
mova [dstq+stride3q ], m0
mova [dstq+stride3q +16], m0
lea dstq, [dstq+strideq*4]
dec lines4d
jnz .loop
RESTORE_GOT
REP_RET
INIT_XMM sse2
cglobal dc_top_predictor_32x32, 4, 5, 5, dst, stride, above, left, goffset
GET_GOT goffsetq
pxor m1, m1
mova m0, [aboveq]
mova m2, [aboveq+16]
DEFINE_ARGS dst, stride, stride3, lines4
lea stride3q, [strideq*3]
mov lines4d, 8
psadbw m0, m1
psadbw m2, m1
paddw m0, m2
movhlps m2, m0
paddw m0, m2
paddw m0, [GLOBAL(pw2_32)]
psraw m0, 5
pshuflw m0, m0, 0x0
punpcklqdq m0, m0
packuswb m0, m0
.loop:
mova [dstq ], m0
mova [dstq +16], m0
mova [dstq+strideq ], m0
mova [dstq+strideq +16], m0
mova [dstq+strideq*2 ], m0
mova [dstq+strideq*2+16], m0
mova [dstq+stride3q ], m0
mova [dstq+stride3q +16], m0
lea dstq, [dstq+strideq*4]
dec lines4d
jnz .loop
RESTORE_GOT
REP_RET
INIT_XMM sse2
cglobal dc_left_predictor_32x32, 4, 5, 5, dst, stride, above, left, goffset
GET_GOT goffsetq
pxor m1, m1
mova m0, [leftq]
mova m2, [leftq+16]
DEFINE_ARGS dst, stride, stride3, lines4
lea stride3q, [strideq*3]
mov lines4d, 8
psadbw m0, m1
psadbw m2, m1
paddw m0, m2
movhlps m2, m0
paddw m0, m2
paddw m0, [GLOBAL(pw2_32)]
psraw m0, 5
pshuflw m0, m0, 0x0
punpcklqdq m0, m0
packuswb m0, m0
.loop:
mova [dstq ], m0
mova [dstq +16], m0
mova [dstq+strideq ], m0
mova [dstq+strideq +16], m0
mova [dstq+strideq*2 ], m0
mova [dstq+strideq*2+16], m0
mova [dstq+stride3q ], m0
mova [dstq+stride3q +16], m0
lea dstq, [dstq+strideq*4]
dec lines4d
jnz .loop
RESTORE_GOT
REP_RET
INIT_XMM sse2
cglobal dc_128_predictor_32x32, 4, 5, 3, dst, stride, above, left, goffset
GET_GOT goffsetq
DEFINE_ARGS dst, stride, stride3, lines4
lea stride3q, [strideq*3]
mov lines4d, 8
mova m0, [GLOBAL(dc_128)]
.loop:
mova [dstq ], m0
mova [dstq +16], m0
mova [dstq+strideq ], m0
mova [dstq+strideq +16], m0
mova [dstq+strideq*2 ], m0
mova [dstq+strideq*2+16], m0
mova [dstq+stride3q ], m0
mova [dstq+stride3q +16], m0
lea dstq, [dstq+strideq*4]
dec lines4d
jnz .loop
RESTORE_GOT
RET
INIT_XMM sse2
cglobal v_predictor_4x4, 3, 3, 1, dst, stride, above
movd m0, [aboveq]
movd [dstq ], m0
movd [dstq+strideq], m0
lea dstq, [dstq+strideq*2]
movd [dstq ], m0
movd [dstq+strideq], m0
RET
INIT_XMM sse2
cglobal v_predictor_8x8, 3, 3, 1, dst, stride, above
movq m0, [aboveq]
DEFINE_ARGS dst, stride, stride3
lea stride3q, [strideq*3]
movq [dstq ], m0
movq [dstq+strideq ], m0
movq [dstq+strideq*2], m0
movq [dstq+stride3q ], m0
lea dstq, [dstq+strideq*4]
movq [dstq ], m0
movq [dstq+strideq ], m0
movq [dstq+strideq*2], m0
movq [dstq+stride3q ], m0
RET
INIT_XMM sse2
cglobal v_predictor_16x16, 3, 4, 1, dst, stride, above
mova m0, [aboveq]
DEFINE_ARGS dst, stride, stride3, nlines4
lea stride3q, [strideq*3]
mov nlines4d, 4
.loop:
mova [dstq ], m0
mova [dstq+strideq ], m0
mova [dstq+strideq*2], m0
mova [dstq+stride3q ], m0
lea dstq, [dstq+strideq*4]
dec nlines4d
jnz .loop
REP_RET
INIT_XMM sse2
cglobal v_predictor_32x32, 3, 4, 2, dst, stride, above
mova m0, [aboveq]
mova m1, [aboveq+16]
DEFINE_ARGS dst, stride, stride3, nlines4
lea stride3q, [strideq*3]
mov nlines4d, 8
.loop:
mova [dstq ], m0
mova [dstq +16], m1
mova [dstq+strideq ], m0
mova [dstq+strideq +16], m1
mova [dstq+strideq*2 ], m0
mova [dstq+strideq*2+16], m1
mova [dstq+stride3q ], m0
mova [dstq+stride3q +16], m1
lea dstq, [dstq+strideq*4]
dec nlines4d
jnz .loop
REP_RET
INIT_XMM sse2
cglobal h_predictor_4x4, 2, 4, 4, dst, stride, line, left
movifnidn leftq, leftmp
movd m0, [leftq]
punpcklbw m0, m0
punpcklbw m0, m0
pshufd m1, m0, 0x1
movd [dstq ], m0
movd [dstq+strideq], m1
pshufd m2, m0, 0x2
lea dstq, [dstq+strideq*2]
pshufd m3, m0, 0x3
movd [dstq ], m2
movd [dstq+strideq], m3
RET
INIT_XMM sse2
cglobal h_predictor_8x8, 2, 5, 3, dst, stride, line, left
movifnidn leftq, leftmp
mov lineq, -2
DEFINE_ARGS dst, stride, line, left, stride3
lea stride3q, [strideq*3]
movq m0, [leftq ]
punpcklbw m0, m0 ; l1 l1 l2 l2 ... l8 l8
.loop:
pshuflw m1, m0, 0x0 ; l1 l1 l1 l1 l1 l1 l1 l1
pshuflw m2, m0, 0x55 ; l2 l2 l2 l2 l2 l2 l2 l2
movq [dstq ], m1
movq [dstq+strideq], m2
pshuflw m1, m0, 0xaa
pshuflw m2, m0, 0xff
movq [dstq+strideq*2], m1
movq [dstq+stride3q ], m2
pshufd m0, m0, 0xe ; [63:0] l5 l5 l6 l6 l7 l7 l8 l8
inc lineq
lea dstq, [dstq+strideq*4]
jnz .loop
REP_RET
INIT_XMM sse2
cglobal h_predictor_16x16, 2, 5, 3, dst, stride, line, left
movifnidn leftq, leftmp
mov lineq, -4
DEFINE_ARGS dst, stride, line, left, stride3
lea stride3q, [strideq*3]
.loop:
movd m0, [leftq]
punpcklbw m0, m0
punpcklbw m0, m0 ; l1 to l4 each repeated 4 times
pshufd m1, m0, 0x0 ; l1 repeated 16 times
pshufd m2, m0, 0x55 ; l2 repeated 16 times
mova [dstq ], m1
mova [dstq+strideq ], m2
pshufd m1, m0, 0xaa
pshufd m2, m0, 0xff
mova [dstq+strideq*2], m1
mova [dstq+stride3q ], m2
inc lineq
lea leftq, [leftq+4 ]
lea dstq, [dstq+strideq*4]
jnz .loop
REP_RET
INIT_XMM sse2
cglobal h_predictor_32x32, 2, 5, 3, dst, stride, line, left
movifnidn leftq, leftmp
mov lineq, -8
DEFINE_ARGS dst, stride, line, left, stride3
lea stride3q, [strideq*3]
.loop:
movd m0, [leftq]
punpcklbw m0, m0
punpcklbw m0, m0 ; l1 to l4 each repeated 4 times
pshufd m1, m0, 0x0 ; l1 repeated 16 times
pshufd m2, m0, 0x55 ; l2 repeated 16 times
mova [dstq ], m1
mova [dstq+16 ], m1
mova [dstq+strideq ], m2
mova [dstq+strideq+16 ], m2
pshufd m1, m0, 0xaa
pshufd m2, m0, 0xff
mova [dstq+strideq*2 ], m1
mova [dstq+strideq*2+16], m1
mova [dstq+stride3q ], m2
mova [dstq+stride3q+16 ], m2
inc lineq
lea leftq, [leftq+4 ]
lea dstq, [dstq+strideq*4]
jnz .loop
REP_RET

3573
third_party/aom/aom_dsp/x86/intrapred_avx2.c vendored
View file

File diff suppressed because it is too large Load diff

64
third_party/aom/aom_dsp/x86/jnt_sad_ssse3.c vendored
View file

@ -192,47 +192,47 @@ unsigned int aom_sad128xh_sse2(const uint8_t *a, int a_stride, const uint8_t *b,
return res;
}
#define jnt_sadMxN_sse2(m, n) \
unsigned int aom_jnt_sad##m##x##n##_avg_ssse3( \
#define dist_wtd_sadMxN_sse2(m, n) \
unsigned int aom_dist_wtd_sad##m##x##n##_avg_ssse3( \
const uint8_t *src, int src_stride, const uint8_t *ref, int ref_stride, \
const uint8_t *second_pred, const JNT_COMP_PARAMS *jcp_param) { \
const uint8_t *second_pred, const DIST_WTD_COMP_PARAMS *jcp_param) { \
uint8_t comp_pred[m * n]; \
aom_jnt_comp_avg_pred(comp_pred, second_pred, m, n, ref, ref_stride, \
jcp_param); \
aom_dist_wtd_comp_avg_pred(comp_pred, second_pred, m, n, ref, ref_stride, \
jcp_param); \
return aom_sad##m##xh_sse2(src, src_stride, comp_pred, m, m, n); \
}
#define jnt_sadMxN_avx2(m, n) \
unsigned int aom_jnt_sad##m##x##n##_avg_avx2( \
#define dist_wtd_sadMxN_avx2(m, n) \
unsigned int aom_dist_wtd_sad##m##x##n##_avg_avx2( \
const uint8_t *src, int src_stride, const uint8_t *ref, int ref_stride, \
const uint8_t *second_pred, const JNT_COMP_PARAMS *jcp_param) { \
const uint8_t *second_pred, const DIST_WTD_COMP_PARAMS *jcp_param) { \
uint8_t comp_pred[m * n]; \
aom_jnt_comp_avg_pred(comp_pred, second_pred, m, n, ref, ref_stride, \
jcp_param); \
aom_dist_wtd_comp_avg_pred(comp_pred, second_pred, m, n, ref, ref_stride, \
jcp_param); \
return aom_sad##m##xh_avx2(src, src_stride, comp_pred, m, m, n); \
}
/* clang-format off */
jnt_sadMxN_sse2(128, 128)
jnt_sadMxN_sse2(128, 64)
jnt_sadMxN_sse2(64, 128)
jnt_sadMxN_sse2(64, 64)
jnt_sadMxN_sse2(64, 32)
jnt_sadMxN_sse2(32, 64)
jnt_sadMxN_sse2(32, 32)
jnt_sadMxN_sse2(32, 16)
jnt_sadMxN_sse2(16, 32)
jnt_sadMxN_sse2(16, 16)
jnt_sadMxN_sse2(16, 8)
jnt_sadMxN_sse2(8, 16)
jnt_sadMxN_sse2(8, 8)
jnt_sadMxN_sse2(8, 4)
jnt_sadMxN_sse2(4, 8)
jnt_sadMxN_sse2(4, 4)
jnt_sadMxN_sse2(4, 16)
jnt_sadMxN_sse2(16, 4)
jnt_sadMxN_sse2(8, 32)
jnt_sadMxN_sse2(32, 8)
jnt_sadMxN_sse2(16, 64)
jnt_sadMxN_sse2(64, 16)
dist_wtd_sadMxN_sse2(128, 128)
dist_wtd_sadMxN_sse2(128, 64)
dist_wtd_sadMxN_sse2(64, 128)
dist_wtd_sadMxN_sse2(64, 64)
dist_wtd_sadMxN_sse2(64, 32)
dist_wtd_sadMxN_sse2(32, 64)
dist_wtd_sadMxN_sse2(32, 32)
dist_wtd_sadMxN_sse2(32, 16)
dist_wtd_sadMxN_sse2(16, 32)
dist_wtd_sadMxN_sse2(16, 16)
dist_wtd_sadMxN_sse2(16, 8)
dist_wtd_sadMxN_sse2(8, 16)
dist_wtd_sadMxN_sse2(8, 8)
dist_wtd_sadMxN_sse2(8, 4)
dist_wtd_sadMxN_sse2(4, 8)
dist_wtd_sadMxN_sse2(4, 4)
dist_wtd_sadMxN_sse2(4, 16)
dist_wtd_sadMxN_sse2(16, 4)
dist_wtd_sadMxN_sse2(8, 32)
dist_wtd_sadMxN_sse2(32, 8)
dist_wtd_sadMxN_sse2(16, 64)
dist_wtd_sadMxN_sse2(64, 16)
/* clang-format on */

102
third_party/aom/aom_dsp/x86/jnt_variance_ssse3.c vendored
View file

@ -29,7 +29,7 @@ void aom_var_filter_block2d_bil_second_pass_ssse3(
unsigned int pixel_step, unsigned int output_height,
unsigned int output_width, const uint8_t *filter);
static INLINE void compute_jnt_comp_avg(__m128i *p0, __m128i *p1,
static INLINE void compute_dist_wtd_avg(__m128i *p0, __m128i *p1,
const __m128i *w, const __m128i *r,
void *const result) {
__m128i p_lo = _mm_unpacklo_epi8(*p0, *p1);
@ -45,10 +45,10 @@ static INLINE void compute_jnt_comp_avg(__m128i *p0, __m128i *p1,
xx_storeu_128(result, _mm_packus_epi16(shift_lo, shift_hi));
}
void aom_jnt_comp_avg_pred_ssse3(uint8_t *comp_pred, const uint8_t *pred,
int width, int height, const uint8_t *ref,
int ref_stride,
const JNT_COMP_PARAMS *jcp_param) {
void aom_dist_wtd_comp_avg_pred_ssse3(uint8_t *comp_pred, const uint8_t *pred,
int width, int height, const uint8_t *ref,
int ref_stride,
const DIST_WTD_COMP_PARAMS *jcp_param) {
int i;
const uint8_t w0 = (uint8_t)jcp_param->fwd_offset;
const uint8_t w1 = (uint8_t)jcp_param->bck_offset;
@ -67,7 +67,7 @@ void aom_jnt_comp_avg_pred_ssse3(uint8_t *comp_pred, const uint8_t *pred,
__m128i p0 = xx_loadu_128(ref);
__m128i p1 = xx_loadu_128(pred);
compute_jnt_comp_avg(&p0, &p1, &w, &r, comp_pred);
compute_dist_wtd_avg(&p0, &p1, &w, &r, comp_pred);
comp_pred += 16;
pred += 16;
@ -85,7 +85,7 @@ void aom_jnt_comp_avg_pred_ssse3(uint8_t *comp_pred, const uint8_t *pred,
__m128i p0 = _mm_unpacklo_epi64(p0_0, p0_1);
__m128i p1 = xx_loadu_128(pred);
compute_jnt_comp_avg(&p0, &p1, &w, &r, comp_pred);
compute_dist_wtd_avg(&p0, &p1, &w, &r, comp_pred);
comp_pred += 16;
pred += 16;
@ -107,7 +107,7 @@ void aom_jnt_comp_avg_pred_ssse3(uint8_t *comp_pred, const uint8_t *pred,
row3[0], row3[1], row3[2], row3[3]);
__m128i p1 = xx_loadu_128(pred);
compute_jnt_comp_avg(&p0, &p1, &w, &r, comp_pred);
compute_dist_wtd_avg(&p0, &p1, &w, &r, comp_pred);
comp_pred += 16;
pred += 16;
@ -116,11 +116,11 @@ void aom_jnt_comp_avg_pred_ssse3(uint8_t *comp_pred, const uint8_t *pred,
}
}
void aom_jnt_comp_avg_upsampled_pred_ssse3(
void aom_dist_wtd_comp_avg_upsampled_pred_ssse3(
MACROBLOCKD *xd, const struct AV1Common *const cm, int mi_row, int mi_col,
const MV *const mv, uint8_t *comp_pred, const uint8_t *pred, int width,
int height, int subpel_x_q3, int subpel_y_q3, const uint8_t *ref,
int ref_stride, const JNT_COMP_PARAMS *jcp_param, int subpel_search) {
int ref_stride, const DIST_WTD_COMP_PARAMS *jcp_param, int subpel_search) {
int n;
int i;
aom_upsampled_pred(xd, cm, mi_row, mi_col, mv, comp_pred, width, height,
@ -141,52 +141,52 @@ void aom_jnt_comp_avg_upsampled_pred_ssse3(
__m128i p0 = xx_loadu_128(comp_pred);
__m128i p1 = xx_loadu_128(pred);
compute_jnt_comp_avg(&p0, &p1, &w, &r, comp_pred);
compute_dist_wtd_avg(&p0, &p1, &w, &r, comp_pred);
comp_pred += 16;
pred += 16;
}
}
#define JNT_SUBPIX_AVG_VAR(W, H) \
uint32_t aom_jnt_sub_pixel_avg_variance##W##x##H##_ssse3( \
const uint8_t *a, int a_stride, int xoffset, int yoffset, \
const uint8_t *b, int b_stride, uint32_t *sse, \
const uint8_t *second_pred, const JNT_COMP_PARAMS *jcp_param) { \
uint16_t fdata3[(H + 1) * W]; \
uint8_t temp2[H * W]; \
DECLARE_ALIGNED(16, uint8_t, temp3[H * W]); \
\
aom_var_filter_block2d_bil_first_pass_ssse3( \
a, fdata3, a_stride, 1, H + 1, W, bilinear_filters_2t[xoffset]); \
aom_var_filter_block2d_bil_second_pass_ssse3( \
fdata3, temp2, W, W, H, W, bilinear_filters_2t[yoffset]); \
\
aom_jnt_comp_avg_pred_ssse3(temp3, second_pred, W, H, temp2, W, \
jcp_param); \
\
return aom_variance##W##x##H(temp3, W, b, b_stride, sse); \
#define DIST_WTD_SUBPIX_AVG_VAR(W, H) \
uint32_t aom_dist_wtd_sub_pixel_avg_variance##W##x##H##_ssse3( \
const uint8_t *a, int a_stride, int xoffset, int yoffset, \
const uint8_t *b, int b_stride, uint32_t *sse, \
const uint8_t *second_pred, const DIST_WTD_COMP_PARAMS *jcp_param) { \
uint16_t fdata3[(H + 1) * W]; \
uint8_t temp2[H * W]; \
DECLARE_ALIGNED(16, uint8_t, temp3[H * W]); \
\
aom_var_filter_block2d_bil_first_pass_ssse3( \
a, fdata3, a_stride, 1, H + 1, W, bilinear_filters_2t[xoffset]); \
aom_var_filter_block2d_bil_second_pass_ssse3( \
fdata3, temp2, W, W, H, W, bilinear_filters_2t[yoffset]); \
\
aom_dist_wtd_comp_avg_pred_ssse3(temp3, second_pred, W, H, temp2, W, \
jcp_param); \
\
return aom_variance##W##x##H(temp3, W, b, b_stride, sse); \
}
JNT_SUBPIX_AVG_VAR(128, 128)
JNT_SUBPIX_AVG_VAR(128, 64)
JNT_SUBPIX_AVG_VAR(64, 128)
JNT_SUBPIX_AVG_VAR(64, 64)
JNT_SUBPIX_AVG_VAR(64, 32)
JNT_SUBPIX_AVG_VAR(32, 64)
JNT_SUBPIX_AVG_VAR(32, 32)
JNT_SUBPIX_AVG_VAR(32, 16)
JNT_SUBPIX_AVG_VAR(16, 32)
JNT_SUBPIX_AVG_VAR(16, 16)
JNT_SUBPIX_AVG_VAR(16, 8)
JNT_SUBPIX_AVG_VAR(8, 16)
JNT_SUBPIX_AVG_VAR(8, 8)
JNT_SUBPIX_AVG_VAR(8, 4)
JNT_SUBPIX_AVG_VAR(4, 8)
JNT_SUBPIX_AVG_VAR(4, 4)
JNT_SUBPIX_AVG_VAR(4, 16)
JNT_SUBPIX_AVG_VAR(16, 4)
JNT_SUBPIX_AVG_VAR(8, 32)
JNT_SUBPIX_AVG_VAR(32, 8)
JNT_SUBPIX_AVG_VAR(16, 64)
JNT_SUBPIX_AVG_VAR(64, 16)
DIST_WTD_SUBPIX_AVG_VAR(128, 128)
DIST_WTD_SUBPIX_AVG_VAR(128, 64)
DIST_WTD_SUBPIX_AVG_VAR(64, 128)
DIST_WTD_SUBPIX_AVG_VAR(64, 64)
DIST_WTD_SUBPIX_AVG_VAR(64, 32)
DIST_WTD_SUBPIX_AVG_VAR(32, 64)
DIST_WTD_SUBPIX_AVG_VAR(32, 32)
DIST_WTD_SUBPIX_AVG_VAR(32, 16)
DIST_WTD_SUBPIX_AVG_VAR(16, 32)
DIST_WTD_SUBPIX_AVG_VAR(16, 16)
DIST_WTD_SUBPIX_AVG_VAR(16, 8)
DIST_WTD_SUBPIX_AVG_VAR(8, 16)
DIST_WTD_SUBPIX_AVG_VAR(8, 8)
DIST_WTD_SUBPIX_AVG_VAR(8, 4)
DIST_WTD_SUBPIX_AVG_VAR(4, 8)
DIST_WTD_SUBPIX_AVG_VAR(4, 4)
DIST_WTD_SUBPIX_AVG_VAR(4, 16)
DIST_WTD_SUBPIX_AVG_VAR(16, 4)
DIST_WTD_SUBPIX_AVG_VAR(8, 32)
DIST_WTD_SUBPIX_AVG_VAR(32, 8)
DIST_WTD_SUBPIX_AVG_VAR(16, 64)
DIST_WTD_SUBPIX_AVG_VAR(64, 16)

453
third_party/aom/aom_dsp/x86/loopfilter_sse2.c vendored
View file

@ -16,347 +16,12 @@
#include "aom_dsp/x86/synonyms.h"
#include "aom_ports/mem.h"
#include "aom_ports/emmintrin_compat.h"
#include "aom_dsp/x86/lpf_common_sse2.h"
static INLINE __m128i abs_diff(__m128i a, __m128i b) {
return _mm_or_si128(_mm_subs_epu8(a, b), _mm_subs_epu8(b, a));
}
static INLINE void transpose4x8_8x4_low_sse2(__m128i *x0, __m128i *x1,
__m128i *x2, __m128i *x3,
__m128i *d0, __m128i *d1,
__m128i *d2, __m128i *d3) {
// input
// x0 00 01 02 03 04 05 06 07 xx xx xx xx xx xx xx xx
// x1 10 11 12 13 14 15 16 17 xx xx xx xx xx xx xx xx
// x2 20 21 22 23 24 25 26 27 xx xx xx xx xx xx xx xx
// x3 30 31 32 33 34 35 36 37 xx xx xx xx xx xx xx xx
// output
// 00 10 20 30 xx xx xx xx xx xx xx xx xx xx xx xx
// 01 11 21 31 xx xx xx xx xx xx xx xx xx xx xx xx
// 02 12 22 32 xx xx xx xx xx xx xx xx xx xx xx xx
// 03 13 23 33 xx xx xx xx xx xx xx xx xx xx xx xx
__m128i w0, w1;
w0 = _mm_unpacklo_epi8(
*x0, *x1); // 00 10 01 11 02 12 03 13 04 14 05 15 06 16 07 17
w1 = _mm_unpacklo_epi8(
*x2, *x3); // 20 30 21 31 22 32 23 33 24 34 25 35 26 36 27 37
*d0 = _mm_unpacklo_epi16(
w0, w1); // 00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33
*d1 = _mm_srli_si128(*d0,
4); // 01 11 21 31 xx xx xx xx xx xx xx xx xx xx xx xx
*d2 = _mm_srli_si128(*d0,
8); // 02 12 22 32 xx xx xx xx xx xx xx xx xx xx xx xx
*d3 = _mm_srli_si128(*d0,
12); // 03 13 23 33 xx xx xx xx xx xx xx xx xx xx xx xx
}
static INLINE void transpose4x8_8x4_sse2(__m128i *x0, __m128i *x1, __m128i *x2,
__m128i *x3, __m128i *d0, __m128i *d1,
__m128i *d2, __m128i *d3, __m128i *d4,
__m128i *d5, __m128i *d6,
__m128i *d7) {
// input
// x0 00 01 02 03 04 05 06 07 xx xx xx xx xx xx xx xx
// x1 10 11 12 13 14 15 16 17 xx xx xx xx xx xx xx xx
// x2 20 21 22 23 24 25 26 27 xx xx xx xx xx xx xx xx
// x3 30 31 32 33 34 35 36 37 xx xx xx xx xx xx xx xx
// output
// 00 10 20 30 xx xx xx xx xx xx xx xx xx xx xx xx
// 01 11 21 31 xx xx xx xx xx xx xx xx xx xx xx xx
// 02 12 22 32 xx xx xx xx xx xx xx xx xx xx xx xx
// 03 13 23 33 xx xx xx xx xx xx xx xx xx xx xx xx
// 04 14 24 34 xx xx xx xx xx xx xx xx xx xx xx xx
// 05 15 25 35 xx xx xx xx xx xx xx xx xx xx xx xx
// 06 16 26 36 xx xx xx xx xx xx xx xx xx xx xx xx
// 07 17 27 37 xx xx xx xx xx xx xx xx xx xx xx xx
__m128i w0, w1, ww0, ww1;
w0 = _mm_unpacklo_epi8(
*x0, *x1); // 00 10 01 11 02 12 03 13 04 14 05 15 06 16 07 17
w1 = _mm_unpacklo_epi8(
*x2, *x3); // 20 30 21 31 22 32 23 33 24 34 25 35 26 36 27 37
ww0 = _mm_unpacklo_epi16(
w0, w1); // 00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33
ww1 = _mm_unpackhi_epi16(
w0, w1); // 04 14 24 34 05 15 25 35 06 16 26 36 07 17 27 37
*d0 = ww0; // 00 10 20 30 xx xx xx xx xx xx xx xx xx xx xx xx
*d1 = _mm_srli_si128(ww0,
4); // 01 11 21 31 xx xx xx xx xx xx xx xx xx xx xx xx
*d2 = _mm_srli_si128(ww0,
8); // 02 12 22 32 xx xx xx xx xx xx xx xx xx xx xx xx
*d3 = _mm_srli_si128(ww0,
12); // 03 13 23 33 xx xx xx xx xx xx xx xx xx xx xx xx
*d4 = ww1; // 04 14 24 34 xx xx xx xx xx xx xx xx xx xx xx xx
*d5 = _mm_srli_si128(ww1,
4); // 05 15 25 35 xx xx xx xx xx xx xx xx xx xx xx xx
*d6 = _mm_srli_si128(ww1,
8); // 06 16 26 36 xx xx xx xx xx xx xx xx xx xx xx xx
*d7 = _mm_srli_si128(ww1,
12); // 07 17 27 37 xx xx xx xx xx xx xx xx xx xx xx xx
}
static INLINE void transpose8x8_low_sse2(__m128i *x0, __m128i *x1, __m128i *x2,
__m128i *x3, __m128i *x4, __m128i *x5,
__m128i *x6, __m128i *x7, __m128i *d0,
__m128i *d1, __m128i *d2,
__m128i *d3) {
// input
// x0 00 01 02 03 04 05 06 07
// x1 10 11 12 13 14 15 16 17
// x2 20 21 22 23 24 25 26 27
// x3 30 31 32 33 34 35 36 37
// x4 40 41 42 43 44 45 46 47
// x5 50 51 52 53 54 55 56 57
// x6 60 61 62 63 64 65 66 67
// x7 70 71 72 73 74 75 76 77
// output
// d0 00 10 20 30 40 50 60 70 xx xx xx xx xx xx xx
// d1 01 11 21 31 41 51 61 71 xx xx xx xx xx xx xx xx
// d2 02 12 22 32 42 52 62 72 xx xx xx xx xx xx xx xx
// d3 03 13 23 33 43 53 63 73 xx xx xx xx xx xx xx xx
__m128i w0, w1, w2, w3, w4, w5;
w0 = _mm_unpacklo_epi8(
*x0, *x1); // 00 10 01 11 02 12 03 13 04 14 05 15 06 16 07 17
w1 = _mm_unpacklo_epi8(
*x2, *x3); // 20 30 21 31 22 32 23 33 24 34 25 35 26 36 27 37
w2 = _mm_unpacklo_epi8(
*x4, *x5); // 40 50 41 51 42 52 43 53 44 54 45 55 46 56 47 57
w3 = _mm_unpacklo_epi8(
*x6, *x7); // 60 70 61 71 62 72 63 73 64 74 65 75 66 76 67 77
w4 = _mm_unpacklo_epi16(
w0, w1); // 00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33
w5 = _mm_unpacklo_epi16(
w2, w3); // 40 50 60 70 41 51 61 71 42 52 62 72 43 53 63 73
*d0 = _mm_unpacklo_epi32(
w4, w5); // 00 10 20 30 40 50 60 70 01 11 21 31 41 51 61 71
*d1 = _mm_srli_si128(*d0, 8);
*d2 = _mm_unpackhi_epi32(
w4, w5); // 02 12 22 32 42 52 62 72 03 13 23 33 43 53 63 73
*d3 = _mm_srli_si128(*d2, 8);
}
static INLINE void transpose8x8_sse2(__m128i *x0, __m128i *x1, __m128i *x2,
__m128i *x3, __m128i *x4, __m128i *x5,
__m128i *x6, __m128i *x7, __m128i *d0d1,
__m128i *d2d3, __m128i *d4d5,
__m128i *d6d7) {
__m128i w0, w1, w2, w3, w4, w5, w6, w7;
// x0 00 01 02 03 04 05 06 07
// x1 10 11 12 13 14 15 16 17
w0 = _mm_unpacklo_epi8(
*x0, *x1); // 00 10 01 11 02 12 03 13 04 14 05 15 06 16 07 17
// x2 20 21 22 23 24 25 26 27
// x3 30 31 32 33 34 35 36 37
w1 = _mm_unpacklo_epi8(
*x2, *x3); // 20 30 21 31 22 32 23 33 24 34 25 35 26 36 27 37
// x4 40 41 42 43 44 45 46 47
// x5 50 51 52 53 54 55 56 57
w2 = _mm_unpacklo_epi8(
*x4, *x5); // 40 50 41 51 42 52 43 53 44 54 45 55 46 56 47 57
// x6 60 61 62 63 64 65 66 67
// x7 70 71 72 73 74 75 76 77
w3 = _mm_unpacklo_epi8(
*x6, *x7); // 60 70 61 71 62 72 63 73 64 74 65 75 66 76 67 77
w4 = _mm_unpacklo_epi16(
w0, w1); // 00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33
w5 = _mm_unpacklo_epi16(
w2, w3); // 40 50 60 70 41 51 61 71 42 52 62 72 43 53 63 73
*d0d1 = _mm_unpacklo_epi32(
w4, w5); // 00 10 20 30 40 50 60 70 01 11 21 31 41 51 61 71
*d2d3 = _mm_unpackhi_epi32(
w4, w5); // 02 12 22 32 42 52 62 72 03 13 23 33 43 53 63 73
w6 = _mm_unpackhi_epi16(
w0, w1); // 04 14 24 34 05 15 25 35 06 16 26 36 07 17 27 37
w7 = _mm_unpackhi_epi16(
w2, w3); // 44 54 64 74 45 55 65 75 46 56 66 76 47 57 67 77
*d4d5 = _mm_unpacklo_epi32(
w6, w7); // 04 14 24 34 44 54 64 74 05 15 25 35 45 55 65 75
*d6d7 = _mm_unpackhi_epi32(
w6, w7); // 06 16 26 36 46 56 66 76 07 17 27 37 47 57 67 77
}
static INLINE void transpose16x8_8x16_sse2(
__m128i *x0, __m128i *x1, __m128i *x2, __m128i *x3, __m128i *x4,
__m128i *x5, __m128i *x6, __m128i *x7, __m128i *x8, __m128i *x9,
__m128i *x10, __m128i *x11, __m128i *x12, __m128i *x13, __m128i *x14,
__m128i *x15, __m128i *d0, __m128i *d1, __m128i *d2, __m128i *d3,
__m128i *d4, __m128i *d5, __m128i *d6, __m128i *d7) {
__m128i w0, w1, w2, w3, w4, w5, w6, w7, w8, w9;
__m128i w10, w11, w12, w13, w14, w15;
w0 = _mm_unpacklo_epi8(*x0, *x1);
w1 = _mm_unpacklo_epi8(*x2, *x3);
w2 = _mm_unpacklo_epi8(*x4, *x5);
w3 = _mm_unpacklo_epi8(*x6, *x7);
w8 = _mm_unpacklo_epi8(*x8, *x9);
w9 = _mm_unpacklo_epi8(*x10, *x11);
w10 = _mm_unpacklo_epi8(*x12, *x13);
w11 = _mm_unpacklo_epi8(*x14, *x15);
w4 = _mm_unpacklo_epi16(w0, w1);
w5 = _mm_unpacklo_epi16(w2, w3);
w12 = _mm_unpacklo_epi16(w8, w9);
w13 = _mm_unpacklo_epi16(w10, w11);
w6 = _mm_unpacklo_epi32(w4, w5);
w7 = _mm_unpackhi_epi32(w4, w5);
w14 = _mm_unpacklo_epi32(w12, w13);
w15 = _mm_unpackhi_epi32(w12, w13);
// Store first 4-line result
*d0 = _mm_unpacklo_epi64(w6, w14);
*d1 = _mm_unpackhi_epi64(w6, w14);
*d2 = _mm_unpacklo_epi64(w7, w15);
*d3 = _mm_unpackhi_epi64(w7, w15);
w4 = _mm_unpackhi_epi16(w0, w1);
w5 = _mm_unpackhi_epi16(w2, w3);
w12 = _mm_unpackhi_epi16(w8, w9);
w13 = _mm_unpackhi_epi16(w10, w11);
w6 = _mm_unpacklo_epi32(w4, w5);
w7 = _mm_unpackhi_epi32(w4, w5);
w14 = _mm_unpacklo_epi32(w12, w13);
w15 = _mm_unpackhi_epi32(w12, w13);
// Store second 4-line result
*d4 = _mm_unpacklo_epi64(w6, w14);
*d5 = _mm_unpackhi_epi64(w6, w14);
*d6 = _mm_unpacklo_epi64(w7, w15);
*d7 = _mm_unpackhi_epi64(w7, w15);
}
// this function treats its input as 2 parallel 8x4 matrices, transposes each of
// them independently while flipping the second matrix horizontaly Used for 14
// taps filter pq pairs inverse
static INLINE void transpose_pq_14_inv_sse2(__m128i *x0, __m128i *x1,
__m128i *x2, __m128i *x3,
__m128i *x4, __m128i *x5,
__m128i *x6, __m128i *x7,
__m128i *pq0, __m128i *pq1,
__m128i *pq2, __m128i *pq3) {
__m128i w10, w11, w12, w13;
__m128i w0, w1, w2, w3, w4, w5;
__m128i d0, d1, d2, d3;
w0 = _mm_unpacklo_epi8(
*x0, *x1); // p 00 10 01 11 02 12 03 13 04 14 05 15 06 16 07 17
w1 = _mm_unpacklo_epi8(
*x2, *x3); // p 20 30 21 31 22 32 23 33 24 34 25 35 26 36 27 37
w2 = _mm_unpacklo_epi8(
*x4, *x5); // p 40 50 41 51 42 52 43 53 44 54 45 55 46 56 47 57
w3 = _mm_unpacklo_epi8(
*x6, *x7); // p 60 70 61 71 62 72 63 73 64 74 65 75 66 76 67 77
w4 = _mm_unpacklo_epi16(
w0, w1); // 00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33
w5 = _mm_unpacklo_epi16(
w2, w3); // 40 50 60 70 41 51 61 71 42 52 62 72 43 53 63 73
d0 = _mm_unpacklo_epi32(
w4, w5); // 00 10 20 30 40 50 60 70 01 11 21 31 41 51 61 71
d2 = _mm_unpackhi_epi32(
w4, w5); // 02 12 22 32 42 52 62 72 03 13 23 33 43 53 63 73
w10 = _mm_unpacklo_epi8(
*x7, *x6); // q xx xx xx xx xx xx xx xx 00 10 01 11 02 12 03 13
w11 = _mm_unpacklo_epi8(
*x5, *x4); // q xx xx xx xx xx xx xx xx 20 30 21 31 22 32 23 33
w12 = _mm_unpacklo_epi8(
*x3, *x2); // q xx xx xx xx xx xx xx xx 40 50 41 51 42 52 43 53
w13 = _mm_unpacklo_epi8(
*x1, *x0); // q xx xx xx xx xx xx xx xx 60 70 61 71 62 72 63 73
w4 = _mm_unpackhi_epi16(
w10, w11); // 00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33
w5 = _mm_unpackhi_epi16(
w12, w13); // 40 50 60 70 41 51 61 71 42 52 62 72 43 53 63 73
d1 = _mm_unpacklo_epi32(
w4, w5); // 00 10 20 30 40 50 60 70 01 11 21 31 41 51 61 71
d3 = _mm_unpackhi_epi32(
w4, w5); // 02 12 22 32 42 52 62 72 03 13 23 33 43 53 63 73
*pq0 = _mm_unpacklo_epi64(d0, d1); // pq
*pq1 = _mm_unpackhi_epi64(d0, d1); // pq
*pq2 = _mm_unpacklo_epi64(d2, d3); // pq
*pq3 = _mm_unpackhi_epi64(d2, d3); // pq
}
static INLINE void transpose8x16_16x8_sse2(
__m128i *x0, __m128i *x1, __m128i *x2, __m128i *x3, __m128i *x4,
__m128i *x5, __m128i *x6, __m128i *x7, __m128i *d0d1, __m128i *d2d3,
__m128i *d4d5, __m128i *d6d7, __m128i *d8d9, __m128i *d10d11,
__m128i *d12d13, __m128i *d14d15) {
__m128i w0, w1, w2, w3, w4, w5, w6, w7, w8, w9;
__m128i w10, w11, w12, w13, w14, w15;
w0 = _mm_unpacklo_epi8(*x0, *x1);
w1 = _mm_unpacklo_epi8(*x2, *x3);
w2 = _mm_unpacklo_epi8(*x4, *x5);
w3 = _mm_unpacklo_epi8(*x6, *x7);
w8 = _mm_unpackhi_epi8(*x0, *x1);
w9 = _mm_unpackhi_epi8(*x2, *x3);
w10 = _mm_unpackhi_epi8(*x4, *x5);
w11 = _mm_unpackhi_epi8(*x6, *x7);
w4 = _mm_unpacklo_epi16(w0, w1);
w5 = _mm_unpacklo_epi16(w2, w3);
w12 = _mm_unpacklo_epi16(w8, w9);
w13 = _mm_unpacklo_epi16(w10, w11);
w6 = _mm_unpacklo_epi32(w4, w5);
w7 = _mm_unpackhi_epi32(w4, w5);
w14 = _mm_unpacklo_epi32(w12, w13);
w15 = _mm_unpackhi_epi32(w12, w13);
// Store first 4-line result
*d0d1 = _mm_unpacklo_epi64(w6, w14);
*d2d3 = _mm_unpackhi_epi64(w6, w14);
*d4d5 = _mm_unpacklo_epi64(w7, w15);
*d6d7 = _mm_unpackhi_epi64(w7, w15);
w4 = _mm_unpackhi_epi16(w0, w1);
w5 = _mm_unpackhi_epi16(w2, w3);
w12 = _mm_unpackhi_epi16(w8, w9);
w13 = _mm_unpackhi_epi16(w10, w11);
w6 = _mm_unpacklo_epi32(w4, w5);
w7 = _mm_unpackhi_epi32(w4, w5);
w14 = _mm_unpacklo_epi32(w12, w13);
w15 = _mm_unpackhi_epi32(w12, w13);
// Store second 4-line result
*d8d9 = _mm_unpacklo_epi64(w6, w14);
*d10d11 = _mm_unpackhi_epi64(w6, w14);
*d12d13 = _mm_unpacklo_epi64(w7, w15);
*d14d15 = _mm_unpackhi_epi64(w7, w15);
}
// this function treats its input as 2 parallel 8x4 matrices, transposes each of
// them to 4x8 independently while flipping the second matrix horizontaly. Used
// for 14 taps pq pairs creation
@ -416,6 +81,63 @@ static INLINE void transpose_pq_14_sse2(__m128i *x0, __m128i *x1, __m128i *x2,
ww2); // 07 17 27 37 08 18 28 38 xx xx xx xx xx xx xx xx
}
// this function treats its input as 2 parallel 8x4 matrices, transposes each of
// them independently while flipping the second matrix horizontaly Used for 14
// taps filter pq pairs inverse
static INLINE void transpose_pq_14_inv_sse2(__m128i *x0, __m128i *x1,
__m128i *x2, __m128i *x3,
__m128i *x4, __m128i *x5,
__m128i *x6, __m128i *x7,
__m128i *pq0, __m128i *pq1,
__m128i *pq2, __m128i *pq3) {
__m128i w10, w11, w12, w13;
__m128i w0, w1, w2, w3, w4, w5;
__m128i d0, d1, d2, d3;
w0 = _mm_unpacklo_epi8(
*x0, *x1); // p 00 10 01 11 02 12 03 13 04 14 05 15 06 16 07 17
w1 = _mm_unpacklo_epi8(
*x2, *x3); // p 20 30 21 31 22 32 23 33 24 34 25 35 26 36 27 37
w2 = _mm_unpacklo_epi8(
*x4, *x5); // p 40 50 41 51 42 52 43 53 44 54 45 55 46 56 47 57
w3 = _mm_unpacklo_epi8(
*x6, *x7); // p 60 70 61 71 62 72 63 73 64 74 65 75 66 76 67 77
w4 = _mm_unpacklo_epi16(
w0, w1); // 00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33
w5 = _mm_unpacklo_epi16(
w2, w3); // 40 50 60 70 41 51 61 71 42 52 62 72 43 53 63 73
d0 = _mm_unpacklo_epi32(
w4, w5); // 00 10 20 30 40 50 60 70 01 11 21 31 41 51 61 71
d2 = _mm_unpackhi_epi32(
w4, w5); // 02 12 22 32 42 52 62 72 03 13 23 33 43 53 63 73
w10 = _mm_unpacklo_epi8(
*x7, *x6); // q xx xx xx xx xx xx xx xx 00 10 01 11 02 12 03 13
w11 = _mm_unpacklo_epi8(
*x5, *x4); // q xx xx xx xx xx xx xx xx 20 30 21 31 22 32 23 33
w12 = _mm_unpacklo_epi8(
*x3, *x2); // q xx xx xx xx xx xx xx xx 40 50 41 51 42 52 43 53
w13 = _mm_unpacklo_epi8(
*x1, *x0); // q xx xx xx xx xx xx xx xx 60 70 61 71 62 72 63 73
w4 = _mm_unpackhi_epi16(
w10, w11); // 00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33
w5 = _mm_unpackhi_epi16(
w12, w13); // 40 50 60 70 41 51 61 71 42 52 62 72 43 53 63 73
d1 = _mm_unpacklo_epi32(
w4, w5); // 00 10 20 30 40 50 60 70 01 11 21 31 41 51 61 71
d3 = _mm_unpackhi_epi32(
w4, w5); // 02 12 22 32 42 52 62 72 03 13 23 33 43 53 63 73
*pq0 = _mm_unpacklo_epi64(d0, d1); // pq
*pq1 = _mm_unpackhi_epi64(d0, d1); // pq
*pq2 = _mm_unpacklo_epi64(d2, d3); // pq
*pq3 = _mm_unpackhi_epi64(d2, d3); // pq
}
static AOM_FORCE_INLINE void filter4_sse2(__m128i *p1p0, __m128i *q1q0,
__m128i *hev, __m128i *mask,
__m128i *qs1qs0, __m128i *ps1ps0) {
@ -616,10 +338,10 @@ void aom_lpf_horizontal_4_sse2(uint8_t *s, int p /* pitch */,
__m128i qs1qs0, ps1ps0;
__m128i p1, p0, q0, q1;
p1 = _mm_cvtsi32_si128(*(int *)(s - 2 * p));
p0 = _mm_cvtsi32_si128(*(int *)(s - 1 * p));
q0 = _mm_cvtsi32_si128(*(int *)(s + 0 * p));
q1 = _mm_cvtsi32_si128(*(int *)(s + 1 * p));
p1 = xx_loadl_32(s - 2 * p);
p0 = xx_loadl_32(s - 1 * p);
q0 = xx_loadl_32(s - 0 * p);
q1 = xx_loadl_32(s + 1 * p);
lpf_internal_4_sse2(&p1, &p0, &q0, &q1, &limit, &thresh, &qs1qs0, &ps1ps0);
@ -1241,23 +963,16 @@ void aom_lpf_horizontal_14_sse2(unsigned char *s, int p,
__m128i limit = _mm_load_si128((const __m128i *)_limit);
__m128i thresh = _mm_load_si128((const __m128i *)_thresh);
q4p4 = _mm_unpacklo_epi32(_mm_cvtsi32_si128(*(int *)(s - 5 * p)),
_mm_cvtsi32_si128(*(int *)(s + 4 * p)));
q3p3 = _mm_unpacklo_epi32(_mm_cvtsi32_si128(*(int *)(s - 4 * p)),
_mm_cvtsi32_si128(*(int *)(s + 3 * p)));
q2p2 = _mm_unpacklo_epi32(_mm_cvtsi32_si128(*(int *)(s - 3 * p)),
_mm_cvtsi32_si128(*(int *)(s + 2 * p)));
q1p1 = _mm_unpacklo_epi32(_mm_cvtsi32_si128(*(int *)(s - 2 * p)),
_mm_cvtsi32_si128(*(int *)(s + 1 * p)));
q4p4 = _mm_unpacklo_epi32(xx_loadl_32(s - 5 * p), xx_loadl_32(s + 4 * p));
q3p3 = _mm_unpacklo_epi32(xx_loadl_32(s - 4 * p), xx_loadl_32(s + 3 * p));
q2p2 = _mm_unpacklo_epi32(xx_loadl_32(s - 3 * p), xx_loadl_32(s + 2 * p));
q1p1 = _mm_unpacklo_epi32(xx_loadl_32(s - 2 * p), xx_loadl_32(s + 1 * p));
q0p0 = _mm_unpacklo_epi32(_mm_cvtsi32_si128(*(int *)(s - 1 * p)),
_mm_cvtsi32_si128(*(int *)(s - 0 * p)));
q0p0 = _mm_unpacklo_epi32(xx_loadl_32(s - 1 * p), xx_loadl_32(s - 0 * p));
q5p5 = _mm_unpacklo_epi32(_mm_cvtsi32_si128(*(int *)(s - 6 * p)),
_mm_cvtsi32_si128(*(int *)(s + 5 * p)));
q5p5 = _mm_unpacklo_epi32(xx_loadl_32(s - 6 * p), xx_loadl_32(s + 5 * p));
q6p6 = _mm_unpacklo_epi32(_mm_cvtsi32_si128(*(int *)(s - 7 * p)),
_mm_cvtsi32_si128(*(int *)(s + 6 * p)));
q6p6 = _mm_unpacklo_epi32(xx_loadl_32(s - 7 * p), xx_loadl_32(s + 6 * p));
lpf_internal_14_sse2(&q6p6, &q5p5, &q4p4, &q3p3, &q2p2, &q1p1, &q0p0, &blimit,
&limit, &thresh);
@ -1543,12 +1258,12 @@ void aom_lpf_horizontal_6_sse2(unsigned char *s, int p,
__m128i limit = _mm_load_si128((__m128i *)_limit);
__m128i thresh = _mm_load_si128((__m128i *)_thresh);
p2 = _mm_cvtsi32_si128(*(int *)(s - 3 * p));
p1 = _mm_cvtsi32_si128(*(int *)(s - 2 * p));
p0 = _mm_cvtsi32_si128(*(int *)(s - 1 * p));
q0 = _mm_cvtsi32_si128(*(int *)(s - 0 * p));
q1 = _mm_cvtsi32_si128(*(int *)(s + 1 * p));
q2 = _mm_cvtsi32_si128(*(int *)(s + 2 * p));
p2 = xx_loadl_32(s - 3 * p);
p1 = xx_loadl_32(s - 2 * p);
p0 = xx_loadl_32(s - 1 * p);
q0 = xx_loadl_32(s - 0 * p);
q1 = xx_loadl_32(s + 1 * p);
q2 = xx_loadl_32(s + 2 * p);
lpf_internal_6_sse2(&p2, &q2, &p1, &q1, &p0, &q0, &q1q0, &p1p0, &blimit,
&limit, &thresh);
@ -1895,20 +1610,20 @@ void aom_lpf_horizontal_8_sse2(unsigned char *s, int p,
const unsigned char *_blimit,
const unsigned char *_limit,
const unsigned char *_thresh) {
__m128i p2, p1, p0, q0, q1, q2, p3, q3;
__m128i p3, p2, p1, p0, q0, q1, q2, q3;
__m128i q1q0, p1p0;
__m128i blimit = _mm_load_si128((const __m128i *)_blimit);
__m128i limit = _mm_load_si128((const __m128i *)_limit);
__m128i thresh = _mm_load_si128((const __m128i *)_thresh);
p3 = _mm_cvtsi32_si128(*(int *)(s - 4 * p));
p2 = _mm_cvtsi32_si128(*(int *)(s - 3 * p));
p1 = _mm_cvtsi32_si128(*(int *)(s - 2 * p));
p0 = _mm_cvtsi32_si128(*(int *)(s - 1 * p));
q0 = _mm_cvtsi32_si128(*(int *)(s - 0 * p));
q1 = _mm_cvtsi32_si128(*(int *)(s + 1 * p));
q2 = _mm_cvtsi32_si128(*(int *)(s + 2 * p));
q3 = _mm_cvtsi32_si128(*(int *)(s + 3 * p));
p3 = xx_loadl_32(s - 4 * p);
p2 = xx_loadl_32(s - 3 * p);
p1 = xx_loadl_32(s - 2 * p);
p0 = xx_loadl_32(s - 1 * p);
q0 = xx_loadl_32(s - 0 * p);
q1 = xx_loadl_32(s + 1 * p);
q2 = xx_loadl_32(s + 2 * p);
q3 = xx_loadl_32(s + 3 * p);
lpf_internal_8_sse2(&p3, &q3, &p2, &q2, &p1, &q1, &p0, &q0, &q1q0, &p1p0,
&blimit, &limit, &thresh);

280
third_party/aom/aom_dsp/x86/lpf_common_sse2.h vendored
View file

@ -212,4 +212,284 @@ static INLINE void highbd_transpose8x16_sse2(
d4 + 1, d5 + 1, d6 + 1, d7 + 1);
}
// Low bit depth functions
static INLINE void transpose4x8_8x4_low_sse2(__m128i *x0, __m128i *x1,
__m128i *x2, __m128i *x3,
__m128i *d0, __m128i *d1,
__m128i *d2, __m128i *d3) {
// input
// x0 00 01 02 03 04 05 06 07 xx xx xx xx xx xx xx xx
// x1 10 11 12 13 14 15 16 17 xx xx xx xx xx xx xx xx
// x2 20 21 22 23 24 25 26 27 xx xx xx xx xx xx xx xx
// x3 30 31 32 33 34 35 36 37 xx xx xx xx xx xx xx xx
// output
// 00 10 20 30 xx xx xx xx xx xx xx xx xx xx xx xx
// 01 11 21 31 xx xx xx xx xx xx xx xx xx xx xx xx
// 02 12 22 32 xx xx xx xx xx xx xx xx xx xx xx xx
// 03 13 23 33 xx xx xx xx xx xx xx xx xx xx xx xx
__m128i w0, w1;
w0 = _mm_unpacklo_epi8(
*x0, *x1); // 00 10 01 11 02 12 03 13 04 14 05 15 06 16 07 17
w1 = _mm_unpacklo_epi8(
*x2, *x3); // 20 30 21 31 22 32 23 33 24 34 25 35 26 36 27 37
*d0 = _mm_unpacklo_epi16(
w0, w1); // 00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33
*d1 = _mm_srli_si128(*d0,
4); // 01 11 21 31 xx xx xx xx xx xx xx xx xx xx xx xx
*d2 = _mm_srli_si128(*d0,
8); // 02 12 22 32 xx xx xx xx xx xx xx xx xx xx xx xx
*d3 = _mm_srli_si128(*d0,
12); // 03 13 23 33 xx xx xx xx xx xx xx xx xx xx xx xx
}
static INLINE void transpose4x8_8x4_sse2(__m128i *x0, __m128i *x1, __m128i *x2,
__m128i *x3, __m128i *d0, __m128i *d1,
__m128i *d2, __m128i *d3, __m128i *d4,
__m128i *d5, __m128i *d6,
__m128i *d7) {
// input
// x0 00 01 02 03 04 05 06 07 xx xx xx xx xx xx xx xx
// x1 10 11 12 13 14 15 16 17 xx xx xx xx xx xx xx xx
// x2 20 21 22 23 24 25 26 27 xx xx xx xx xx xx xx xx
// x3 30 31 32 33 34 35 36 37 xx xx xx xx xx xx xx xx
// output
// 00 10 20 30 xx xx xx xx xx xx xx xx xx xx xx xx
// 01 11 21 31 xx xx xx xx xx xx xx xx xx xx xx xx
// 02 12 22 32 xx xx xx xx xx xx xx xx xx xx xx xx
// 03 13 23 33 xx xx xx xx xx xx xx xx xx xx xx xx
// 04 14 24 34 xx xx xx xx xx xx xx xx xx xx xx xx
// 05 15 25 35 xx xx xx xx xx xx xx xx xx xx xx xx
// 06 16 26 36 xx xx xx xx xx xx xx xx xx xx xx xx
// 07 17 27 37 xx xx xx xx xx xx xx xx xx xx xx xx
__m128i w0, w1, ww0, ww1;
w0 = _mm_unpacklo_epi8(
*x0, *x1); // 00 10 01 11 02 12 03 13 04 14 05 15 06 16 07 17
w1 = _mm_unpacklo_epi8(
*x2, *x3); // 20 30 21 31 22 32 23 33 24 34 25 35 26 36 27 37
ww0 = _mm_unpacklo_epi16(
w0, w1); // 00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33
ww1 = _mm_unpackhi_epi16(
w0, w1); // 04 14 24 34 05 15 25 35 06 16 26 36 07 17 27 37
*d0 = ww0; // 00 10 20 30 xx xx xx xx xx xx xx xx xx xx xx xx
*d1 = _mm_srli_si128(ww0,
4); // 01 11 21 31 xx xx xx xx xx xx xx xx xx xx xx xx
*d2 = _mm_srli_si128(ww0,
8); // 02 12 22 32 xx xx xx xx xx xx xx xx xx xx xx xx
*d3 = _mm_srli_si128(ww0,
12); // 03 13 23 33 xx xx xx xx xx xx xx xx xx xx xx xx
*d4 = ww1; // 04 14 24 34 xx xx xx xx xx xx xx xx xx xx xx xx
*d5 = _mm_srli_si128(ww1,
4); // 05 15 25 35 xx xx xx xx xx xx xx xx xx xx xx xx
*d6 = _mm_srli_si128(ww1,
8); // 06 16 26 36 xx xx xx xx xx xx xx xx xx xx xx xx
*d7 = _mm_srli_si128(ww1,
12); // 07 17 27 37 xx xx xx xx xx xx xx xx xx xx xx xx
}
static INLINE void transpose8x8_low_sse2(__m128i *x0, __m128i *x1, __m128i *x2,
__m128i *x3, __m128i *x4, __m128i *x5,
__m128i *x6, __m128i *x7, __m128i *d0,
__m128i *d1, __m128i *d2,
__m128i *d3) {
// input
// x0 00 01 02 03 04 05 06 07
// x1 10 11 12 13 14 15 16 17
// x2 20 21 22 23 24 25 26 27
// x3 30 31 32 33 34 35 36 37
// x4 40 41 42 43 44 45 46 47
// x5 50 51 52 53 54 55 56 57
// x6 60 61 62 63 64 65 66 67
// x7 70 71 72 73 74 75 76 77
// output
// d0 00 10 20 30 40 50 60 70 xx xx xx xx xx xx xx
// d1 01 11 21 31 41 51 61 71 xx xx xx xx xx xx xx xx
// d2 02 12 22 32 42 52 62 72 xx xx xx xx xx xx xx xx
// d3 03 13 23 33 43 53 63 73 xx xx xx xx xx xx xx xx
__m128i w0, w1, w2, w3, w4, w5;
w0 = _mm_unpacklo_epi8(
*x0, *x1); // 00 10 01 11 02 12 03 13 04 14 05 15 06 16 07 17
w1 = _mm_unpacklo_epi8(
*x2, *x3); // 20 30 21 31 22 32 23 33 24 34 25 35 26 36 27 37
w2 = _mm_unpacklo_epi8(
*x4, *x5); // 40 50 41 51 42 52 43 53 44 54 45 55 46 56 47 57
w3 = _mm_unpacklo_epi8(
*x6, *x7); // 60 70 61 71 62 72 63 73 64 74 65 75 66 76 67 77
w4 = _mm_unpacklo_epi16(
w0, w1); // 00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33
w5 = _mm_unpacklo_epi16(
w2, w3); // 40 50 60 70 41 51 61 71 42 52 62 72 43 53 63 73
*d0 = _mm_unpacklo_epi32(
w4, w5); // 00 10 20 30 40 50 60 70 01 11 21 31 41 51 61 71
*d1 = _mm_srli_si128(*d0, 8);
*d2 = _mm_unpackhi_epi32(
w4, w5); // 02 12 22 32 42 52 62 72 03 13 23 33 43 53 63 73
*d3 = _mm_srli_si128(*d2, 8);
}
static INLINE void transpose8x8_sse2(__m128i *x0, __m128i *x1, __m128i *x2,
__m128i *x3, __m128i *x4, __m128i *x5,
__m128i *x6, __m128i *x7, __m128i *d0d1,
__m128i *d2d3, __m128i *d4d5,
__m128i *d6d7) {
__m128i w0, w1, w2, w3, w4, w5, w6, w7;
// x0 00 01 02 03 04 05 06 07
// x1 10 11 12 13 14 15 16 17
w0 = _mm_unpacklo_epi8(
*x0, *x1); // 00 10 01 11 02 12 03 13 04 14 05 15 06 16 07 17
// x2 20 21 22 23 24 25 26 27
// x3 30 31 32 33 34 35 36 37
w1 = _mm_unpacklo_epi8(
*x2, *x3); // 20 30 21 31 22 32 23 33 24 34 25 35 26 36 27 37
// x4 40 41 42 43 44 45 46 47
// x5 50 51 52 53 54 55 56 57
w2 = _mm_unpacklo_epi8(
*x4, *x5); // 40 50 41 51 42 52 43 53 44 54 45 55 46 56 47 57
// x6 60 61 62 63 64 65 66 67
// x7 70 71 72 73 74 75 76 77
w3 = _mm_unpacklo_epi8(
*x6, *x7); // 60 70 61 71 62 72 63 73 64 74 65 75 66 76 67 77
w4 = _mm_unpacklo_epi16(
w0, w1); // 00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33
w5 = _mm_unpacklo_epi16(
w2, w3); // 40 50 60 70 41 51 61 71 42 52 62 72 43 53 63 73
*d0d1 = _mm_unpacklo_epi32(
w4, w5); // 00 10 20 30 40 50 60 70 01 11 21 31 41 51 61 71
*d2d3 = _mm_unpackhi_epi32(
w4, w5); // 02 12 22 32 42 52 62 72 03 13 23 33 43 53 63 73
w6 = _mm_unpackhi_epi16(
w0, w1); // 04 14 24 34 05 15 25 35 06 16 26 36 07 17 27 37
w7 = _mm_unpackhi_epi16(
w2, w3); // 44 54 64 74 45 55 65 75 46 56 66 76 47 57 67 77
*d4d5 = _mm_unpacklo_epi32(
w6, w7); // 04 14 24 34 44 54 64 74 05 15 25 35 45 55 65 75
*d6d7 = _mm_unpackhi_epi32(
w6, w7); // 06 16 26 36 46 56 66 76 07 17 27 37 47 57 67 77
}
static INLINE void transpose16x8_8x16_sse2(
__m128i *x0, __m128i *x1, __m128i *x2, __m128i *x3, __m128i *x4,
__m128i *x5, __m128i *x6, __m128i *x7, __m128i *x8, __m128i *x9,
__m128i *x10, __m128i *x11, __m128i *x12, __m128i *x13, __m128i *x14,
__m128i *x15, __m128i *d0, __m128i *d1, __m128i *d2, __m128i *d3,
__m128i *d4, __m128i *d5, __m128i *d6, __m128i *d7) {
__m128i w0, w1, w2, w3, w4, w5, w6, w7, w8, w9;
__m128i w10, w11, w12, w13, w14, w15;
w0 = _mm_unpacklo_epi8(*x0, *x1);
w1 = _mm_unpacklo_epi8(*x2, *x3);
w2 = _mm_unpacklo_epi8(*x4, *x5);
w3 = _mm_unpacklo_epi8(*x6, *x7);
w8 = _mm_unpacklo_epi8(*x8, *x9);
w9 = _mm_unpacklo_epi8(*x10, *x11);
w10 = _mm_unpacklo_epi8(*x12, *x13);
w11 = _mm_unpacklo_epi8(*x14, *x15);
w4 = _mm_unpacklo_epi16(w0, w1);
w5 = _mm_unpacklo_epi16(w2, w3);
w12 = _mm_unpacklo_epi16(w8, w9);
w13 = _mm_unpacklo_epi16(w10, w11);
w6 = _mm_unpacklo_epi32(w4, w5);
w7 = _mm_unpackhi_epi32(w4, w5);
w14 = _mm_unpacklo_epi32(w12, w13);
w15 = _mm_unpackhi_epi32(w12, w13);
// Store first 4-line result
*d0 = _mm_unpacklo_epi64(w6, w14);
*d1 = _mm_unpackhi_epi64(w6, w14);
*d2 = _mm_unpacklo_epi64(w7, w15);
*d3 = _mm_unpackhi_epi64(w7, w15);
w4 = _mm_unpackhi_epi16(w0, w1);
w5 = _mm_unpackhi_epi16(w2, w3);
w12 = _mm_unpackhi_epi16(w8, w9);
w13 = _mm_unpackhi_epi16(w10, w11);
w6 = _mm_unpacklo_epi32(w4, w5);
w7 = _mm_unpackhi_epi32(w4, w5);
w14 = _mm_unpacklo_epi32(w12, w13);
w15 = _mm_unpackhi_epi32(w12, w13);
// Store second 4-line result
*d4 = _mm_unpacklo_epi64(w6, w14);
*d5 = _mm_unpackhi_epi64(w6, w14);
*d6 = _mm_unpacklo_epi64(w7, w15);
*d7 = _mm_unpackhi_epi64(w7, w15);
}
static INLINE void transpose8x16_16x8_sse2(
__m128i *x0, __m128i *x1, __m128i *x2, __m128i *x3, __m128i *x4,
__m128i *x5, __m128i *x6, __m128i *x7, __m128i *d0d1, __m128i *d2d3,
__m128i *d4d5, __m128i *d6d7, __m128i *d8d9, __m128i *d10d11,
__m128i *d12d13, __m128i *d14d15) {
__m128i w0, w1, w2, w3, w4, w5, w6, w7, w8, w9;
__m128i w10, w11, w12, w13, w14, w15;
w0 = _mm_unpacklo_epi8(*x0, *x1);
w1 = _mm_unpacklo_epi8(*x2, *x3);
w2 = _mm_unpacklo_epi8(*x4, *x5);
w3 = _mm_unpacklo_epi8(*x6, *x7);
w8 = _mm_unpackhi_epi8(*x0, *x1);
w9 = _mm_unpackhi_epi8(*x2, *x3);
w10 = _mm_unpackhi_epi8(*x4, *x5);
w11 = _mm_unpackhi_epi8(*x6, *x7);
w4 = _mm_unpacklo_epi16(w0, w1);
w5 = _mm_unpacklo_epi16(w2, w3);
w12 = _mm_unpacklo_epi16(w8, w9);
w13 = _mm_unpacklo_epi16(w10, w11);
w6 = _mm_unpacklo_epi32(w4, w5);
w7 = _mm_unpackhi_epi32(w4, w5);
w14 = _mm_unpacklo_epi32(w12, w13);
w15 = _mm_unpackhi_epi32(w12, w13);
// Store first 4-line result
*d0d1 = _mm_unpacklo_epi64(w6, w14);
*d2d3 = _mm_unpackhi_epi64(w6, w14);
*d4d5 = _mm_unpacklo_epi64(w7, w15);
*d6d7 = _mm_unpackhi_epi64(w7, w15);
w4 = _mm_unpackhi_epi16(w0, w1);
w5 = _mm_unpackhi_epi16(w2, w3);
w12 = _mm_unpackhi_epi16(w8, w9);
w13 = _mm_unpackhi_epi16(w10, w11);
w6 = _mm_unpacklo_epi32(w4, w5);
w7 = _mm_unpackhi_epi32(w4, w5);
w14 = _mm_unpacklo_epi32(w12, w13);
w15 = _mm_unpackhi_epi32(w12, w13);
// Store second 4-line result
*d8d9 = _mm_unpacklo_epi64(w6, w14);
*d10d11 = _mm_unpackhi_epi64(w6, w14);
*d12d13 = _mm_unpacklo_epi64(w7, w15);
*d14d15 = _mm_unpackhi_epi64(w7, w15);
}
#endif // AOM_AOM_DSP_X86_LPF_COMMON_SSE2_H_

69
third_party/aom/aom_dsp/x86/masked_variance_intrin_ssse3.c vendored
View file

@ -218,15 +218,15 @@ static void bilinear_filter(const uint8_t *src, int src_stride, int xoffset,
}
}
static INLINE __m128i filter_block_2rows(const __m128i a0, const __m128i b0,
const __m128i a1, const __m128i b1,
const __m128i filter) {
__m128i v0 = _mm_unpacklo_epi8(a0, b0);
v0 = _mm_maddubs_epi16(v0, filter);
static INLINE __m128i filter_block_2rows(const __m128i *a0, const __m128i *b0,
const __m128i *a1, const __m128i *b1,
const __m128i *filter) {
__m128i v0 = _mm_unpacklo_epi8(*a0, *b0);
v0 = _mm_maddubs_epi16(v0, *filter);
v0 = xx_roundn_epu16(v0, FILTER_BITS);
__m128i v1 = _mm_unpacklo_epi8(a1, b1);
v1 = _mm_maddubs_epi16(v1, filter);
__m128i v1 = _mm_unpacklo_epi8(*a1, *b1);
v1 = _mm_maddubs_epi16(v1, *filter);
v1 = xx_roundn_epu16(v1, FILTER_BITS);
return _mm_packus_epi16(v0, v1);
@ -262,7 +262,7 @@ static void bilinear_filter8xh(const uint8_t *src, int src_stride, int xoffset,
const __m128i z0 = _mm_srli_si128(x0, 1);
const __m128i x1 = _mm_loadu_si128((__m128i *)&src[src_stride]);
const __m128i z1 = _mm_srli_si128(x1, 1);
const __m128i res = filter_block_2rows(x0, z0, x1, z1, hfilter_vec);
const __m128i res = filter_block_2rows(&x0, &z0, &x1, &z1, &hfilter_vec);
_mm_storeu_si128((__m128i *)b, res);
src += src_stride * 2;
@ -296,7 +296,7 @@ static void bilinear_filter8xh(const uint8_t *src, int src_stride, int xoffset,
const __m128i x = _mm_loadl_epi64((__m128i *)dst);
const __m128i y = _mm_loadl_epi64((__m128i *)&dst[8]);
const __m128i z = _mm_loadl_epi64((__m128i *)&dst[16]);
const __m128i res = filter_block_2rows(x, y, y, z, vfilter_vec);
const __m128i res = filter_block_2rows(&x, &y, &y, &z, &vfilter_vec);
_mm_storeu_si128((__m128i *)dst, res);
dst += 16;
@ -343,7 +343,7 @@ static void bilinear_filter4xh(const uint8_t *src, int src_stride, int xoffset,
const __m128i b0 = _mm_unpacklo_epi32(z0, z1);
const __m128i a1 = _mm_unpacklo_epi32(x2, x3);
const __m128i b1 = _mm_unpacklo_epi32(z2, z3);
const __m128i res = filter_block_2rows(a0, b0, a1, b1, hfilter_vec);
const __m128i res = filter_block_2rows(&a0, &b0, &a1, &b1, &hfilter_vec);
_mm_storeu_si128((__m128i *)b, res);
src += src_stride * 4;
@ -384,7 +384,7 @@ static void bilinear_filter4xh(const uint8_t *src, int src_stride, int xoffset,
const __m128i b0 = _mm_unpacklo_epi32(b, c);
const __m128i a1 = _mm_unpacklo_epi32(c, d);
const __m128i b1 = _mm_unpacklo_epi32(d, e);
const __m128i res = filter_block_2rows(a0, b0, a1, b1, vfilter_vec);
const __m128i res = filter_block_2rows(&a0, &b0, &a1, &b1, &vfilter_vec);
_mm_storeu_si128((__m128i *)dst, res);
dst += 16;
@ -392,29 +392,29 @@ static void bilinear_filter4xh(const uint8_t *src, int src_stride, int xoffset,
}
}
static INLINE void accumulate_block(const __m128i src, const __m128i a,
const __m128i b, const __m128i m,
static INLINE void accumulate_block(const __m128i *src, const __m128i *a,
const __m128i *b, const __m128i *m,
__m128i *sum, __m128i *sum_sq) {
const __m128i zero = _mm_setzero_si128();
const __m128i one = _mm_set1_epi16(1);
const __m128i mask_max = _mm_set1_epi8((1 << AOM_BLEND_A64_ROUND_BITS));
const __m128i m_inv = _mm_sub_epi8(mask_max, m);
const __m128i m_inv = _mm_sub_epi8(mask_max, *m);
// Calculate 16 predicted pixels.
// Note that the maximum value of any entry of 'pred_l' or 'pred_r'
// is 64 * 255, so we have plenty of space to add rounding constants.
const __m128i data_l = _mm_unpacklo_epi8(a, b);
const __m128i mask_l = _mm_unpacklo_epi8(m, m_inv);
const __m128i data_l = _mm_unpacklo_epi8(*a, *b);
const __m128i mask_l = _mm_unpacklo_epi8(*m, m_inv);
__m128i pred_l = _mm_maddubs_epi16(data_l, mask_l);
pred_l = xx_roundn_epu16(pred_l, AOM_BLEND_A64_ROUND_BITS);
const __m128i data_r = _mm_unpackhi_epi8(a, b);
const __m128i mask_r = _mm_unpackhi_epi8(m, m_inv);
const __m128i data_r = _mm_unpackhi_epi8(*a, *b);
const __m128i mask_r = _mm_unpackhi_epi8(*m, m_inv);
__m128i pred_r = _mm_maddubs_epi16(data_r, mask_r);
pred_r = xx_roundn_epu16(pred_r, AOM_BLEND_A64_ROUND_BITS);
const __m128i src_l = _mm_unpacklo_epi8(src, zero);
const __m128i src_r = _mm_unpackhi_epi8(src, zero);
const __m128i src_l = _mm_unpacklo_epi8(*src, zero);
const __m128i src_r = _mm_unpackhi_epi8(*src, zero);
const __m128i diff_l = _mm_sub_epi16(pred_l, src_l);
const __m128i diff_r = _mm_sub_epi16(pred_r, src_r);
@ -440,7 +440,7 @@ static void masked_variance(const uint8_t *src_ptr, int src_stride,
const __m128i a = _mm_loadu_si128((const __m128i *)&a_ptr[x]);
const __m128i b = _mm_loadu_si128((const __m128i *)&b_ptr[x]);
const __m128i m = _mm_loadu_si128((const __m128i *)&m_ptr[x]);
accumulate_block(src, a, b, m, &sum, &sum_sq);
accumulate_block(&src, &a, &b, &m, &sum, &sum_sq);
}
src_ptr += src_stride;
@ -471,7 +471,7 @@ static void masked_variance8xh(const uint8_t *src_ptr, int src_stride,
const __m128i m =
_mm_unpacklo_epi64(_mm_loadl_epi64((const __m128i *)m_ptr),
_mm_loadl_epi64((const __m128i *)&m_ptr[m_stride]));
accumulate_block(src, a, b, m, &sum, &sum_sq);
accumulate_block(&src, &a, &b, &m, &sum, &sum_sq);
src_ptr += src_stride * 2;
a_ptr += 16;
@ -503,7 +503,7 @@ static void masked_variance4xh(const uint8_t *src_ptr, int src_stride,
const __m128i m = _mm_setr_epi32(
*(uint32_t *)m_ptr, *(uint32_t *)&m_ptr[m_stride],
*(uint32_t *)&m_ptr[m_stride * 2], *(uint32_t *)&m_ptr[m_stride * 3]);
accumulate_block(src, a, b, m, &sum, &sum_sq);
accumulate_block(&src, &a, &b, &m, &sum, &sum_sq);
src_ptr += src_stride * 4;
a_ptr += 16;
@ -797,17 +797,17 @@ static void highbd_bilinear_filter(const uint16_t *src, int src_stride,
}
}
static INLINE __m128i highbd_filter_block_2rows(const __m128i a0,
const __m128i b0,
const __m128i a1,
const __m128i b1,
const __m128i filter) {
__m128i v0 = _mm_unpacklo_epi16(a0, b0);
v0 = _mm_madd_epi16(v0, filter);
static INLINE __m128i highbd_filter_block_2rows(const __m128i *a0,
const __m128i *b0,
const __m128i *a1,
const __m128i *b1,
const __m128i *filter) {
__m128i v0 = _mm_unpacklo_epi16(*a0, *b0);
v0 = _mm_madd_epi16(v0, *filter);
v0 = xx_roundn_epu32(v0, FILTER_BITS);
__m128i v1 = _mm_unpacklo_epi16(a1, b1);
v1 = _mm_madd_epi16(v1, filter);
__m128i v1 = _mm_unpacklo_epi16(*a1, *b1);
v1 = _mm_madd_epi16(v1, *filter);
v1 = xx_roundn_epu32(v1, FILTER_BITS);
return _mm_packs_epi32(v0, v1);
@ -845,7 +845,7 @@ static void highbd_bilinear_filter4xh(const uint16_t *src, int src_stride,
const __m128i x1 = _mm_loadu_si128((__m128i *)&src[src_stride]);
const __m128i z1 = _mm_srli_si128(x1, 2);
const __m128i res =
highbd_filter_block_2rows(x0, z0, x1, z1, hfilter_vec);
highbd_filter_block_2rows(&x0, &z0, &x1, &z1, &hfilter_vec);
_mm_storeu_si128((__m128i *)b, res);
src += src_stride * 2;
@ -879,7 +879,8 @@ static void highbd_bilinear_filter4xh(const uint16_t *src, int src_stride,
const __m128i x = _mm_loadl_epi64((__m128i *)dst);
const __m128i y = _mm_loadl_epi64((__m128i *)&dst[4]);
const __m128i z = _mm_loadl_epi64((__m128i *)&dst[8]);
const __m128i res = highbd_filter_block_2rows(x, y, y, z, vfilter_vec);
const __m128i res =
highbd_filter_block_2rows(&x, &y, &y, &z, &vfilter_vec);
_mm_storeu_si128((__m128i *)dst, res);
dst += 8;

43
third_party/aom/aom_dsp/x86/quantize_avx_x86_64.asm vendored
View file

@ -126,7 +126,7 @@ cglobal quantize_%1, 0, %2, 15, coeff, ncoeff, zbin, round, quant, \
punpckhqdq m3, m3
pmullw m13, m3 ; dqc[i] = qc[i] * q
; Store 16bit numbers as 32bit numbers in array pointed to by qcoeff
; Store 16bit numbers as 32bit numbers in array pointed to by dqcoeff
pcmpgtw m6, m5, m8
punpckhwd m6, m8, m6
pmovsxwd m11, m8
@ -198,10 +198,7 @@ DEFINE_ARGS coeff, ncoeff, zbin, round, quant, shift, \
mova m4, [r2] ; m4 = shift
mov r4, dqcoeffmp
mov r5, iscanmp
%ifidn %1, b_32x32
psllw m4, 1
%endif
pxor m5, m5 ; m5 = dedicated zero
pxor m5, m5 ; m5 = dedicated zero
DEFINE_ARGS coeff, ncoeff, d1, qcoeff, dqcoeff, iscan, d2, d3, d4, eob
@ -255,9 +252,26 @@ DEFINE_ARGS coeff, ncoeff, zbin, round, quant, shift, \
pmulhw m13, m11, m2 ; m13 = m11*q>>16
paddw m8, m6 ; m8 += m6
paddw m13, m11 ; m13 += m11
%ifidn %1, b_32x32
pmullw m5, m8, m4 ; store the lower 16 bits of m8*qsh
%endif
pmulhw m8, m4 ; m8 = m8*qsh>>16
%ifidn %1, b_32x32
psllw m8, 1
psrlw m5, 15
por m8, m5
%endif
punpckhqdq m4, m4
%ifidn %1, b_32x32
pmullw m5, m13, m4 ; store the lower 16 bits of m13*qsh
%endif
pmulhw m13, m4 ; m13 = m13*qsh>>16
%ifidn %1, b_32x32
psllw m13, 1
psrlw m5, 15
por m13, m5
pxor m5, m5 ; reset m5 to zero register
%endif
psignw m8, m9 ; m8 = reinsert sign
psignw m13, m10 ; m13 = reinsert sign
pand m8, m7
@ -289,7 +303,7 @@ DEFINE_ARGS coeff, ncoeff, zbin, round, quant, shift, \
psignw m13, m10
%endif
; store 16bit numbers as 32bit numbers in array pointed to by qcoeff
; store 16bit numbers as 32bit numbers in array pointed to by dqcoeff
pcmpgtw m6, m5, m8
punpckhwd m6, m8, m6
pmovsxwd m11, m8
@ -359,8 +373,23 @@ DEFINE_ARGS coeff, ncoeff, zbin, round, quant, shift, \
pmulhw m13, m11, m2 ; m13 = m11*q>>16
paddw m14, m6 ; m14 += m6
paddw m13, m11 ; m13 += m11
%ifidn %1, b_32x32
pmullw m5, m14, m4 ; store the lower 16 bits of m14*qsh
%endif
pmulhw m14, m4 ; m14 = m14*qsh>>16
%ifidn %1, b_32x32
psllw m14, 1
psrlw m5, 15
por m14, m5
pmullw m5, m13, m4 ; store the lower 16 bits of m13*qsh
%endif
pmulhw m13, m4 ; m13 = m13*qsh>>16
%ifidn %1, b_32x32
psllw m13, 1
psrlw m5, 15
por m13, m5
pxor m5, m5 ; reset m5 to zero register
%endif
psignw m14, m9 ; m14 = reinsert sign
psignw m13, m10 ; m13 = reinsert sign
pand m14, m7
@ -391,7 +420,7 @@ DEFINE_ARGS coeff, ncoeff, zbin, round, quant, shift, \
psignw m13, m10
%endif
; store 16bit numbers as 32bit numbers in array pointed to by qcoeff
; store 16bit numbers as 32bit numbers in array pointed to by dqcoeff
pcmpgtw m6, m5, m14
punpckhwd m6, m14, m6
pmovsxwd m11, m14

22
third_party/aom/aom_dsp/x86/quantize_sse2.c vendored
View file

@ -18,28 +18,6 @@
#include "aom/aom_integer.h"
#include "aom_dsp/x86/quantize_x86.h"
static INLINE __m128i load_coefficients(const tran_low_t *coeff_ptr) {
assert(sizeof(tran_low_t) == 4);
return _mm_setr_epi16((int16_t)coeff_ptr[0], (int16_t)coeff_ptr[1],
(int16_t)coeff_ptr[2], (int16_t)coeff_ptr[3],
(int16_t)coeff_ptr[4], (int16_t)coeff_ptr[5],
(int16_t)coeff_ptr[6], (int16_t)coeff_ptr[7]);
}
static INLINE void store_coefficients(__m128i coeff_vals,
tran_low_t *coeff_ptr) {
assert(sizeof(tran_low_t) == 4);
__m128i one = _mm_set1_epi16(1);
__m128i coeff_vals_hi = _mm_mulhi_epi16(coeff_vals, one);
__m128i coeff_vals_lo = _mm_mullo_epi16(coeff_vals, one);
__m128i coeff_vals_1 = _mm_unpacklo_epi16(coeff_vals_lo, coeff_vals_hi);
__m128i coeff_vals_2 = _mm_unpackhi_epi16(coeff_vals_lo, coeff_vals_hi);
_mm_store_si128((__m128i *)(coeff_ptr), coeff_vals_1);
_mm_store_si128((__m128i *)(coeff_ptr + 4), coeff_vals_2);
}
void aom_quantize_b_sse2(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
const int16_t *zbin_ptr, const int16_t *round_ptr,
const int16_t *quant_ptr,

192
third_party/aom/aom_dsp/x86/quantize_ssse3.c vendored Normal file
View file

@ -0,0 +1,192 @@
/*
* Copyright (c) 2019, Alliance for Open Media. All rights reserved
*
* This source code is subject to the terms of the BSD 2 Clause License and
* the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
* was not distributed with this source code in the LICENSE file, you can
* obtain it at www.aomedia.org/license/software. If the Alliance for Open
* Media Patent License 1.0 was not distributed with this source code in the
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
*/
#include <assert.h>
#include <tmmintrin.h>
#include <emmintrin.h>
#include <xmmintrin.h>
#include "config/aom_dsp_rtcd.h"
#include "aom/aom_integer.h"
#include "aom_dsp/x86/quantize_x86.h"
static INLINE void calculate_qcoeff_64x64(__m128i *coeff, const __m128i round,
const __m128i quant,
const __m128i *shift) {
__m128i tmp, qcoeff, tmp1;
qcoeff = _mm_adds_epi16(*coeff, round);
tmp = _mm_mulhi_epi16(qcoeff, quant);
qcoeff = _mm_add_epi16(tmp, qcoeff);
tmp = _mm_mullo_epi16(qcoeff, *shift);
tmp = _mm_srli_epi16(tmp, 14);
tmp1 = _mm_mulhi_epi16(qcoeff, *shift);
tmp1 = _mm_slli_epi16(tmp1, 2);
*coeff = _mm_or_si128(tmp, tmp1);
}
static INLINE void calculate_dqcoeff_and_store_64x64(const __m128i qcoeff,
const __m128i dequant,
const __m128i zero,
tran_low_t *dqcoeff) {
// Un-sign to bias rounding like C.
const __m128i coeff = _mm_abs_epi16(qcoeff);
const __m128i sign_0 = _mm_unpacklo_epi16(zero, qcoeff);
const __m128i sign_1 = _mm_unpackhi_epi16(zero, qcoeff);
const __m128i low = _mm_mullo_epi16(coeff, dequant);
const __m128i high = _mm_mulhi_epi16(coeff, dequant);
__m128i dqcoeff32_0 = _mm_unpacklo_epi16(low, high);
__m128i dqcoeff32_1 = _mm_unpackhi_epi16(low, high);
// "Divide" by 4.
dqcoeff32_0 = _mm_srli_epi32(dqcoeff32_0, 2);
dqcoeff32_1 = _mm_srli_epi32(dqcoeff32_1, 2);
dqcoeff32_0 = _mm_sign_epi32(dqcoeff32_0, sign_0);
dqcoeff32_1 = _mm_sign_epi32(dqcoeff32_1, sign_1);
_mm_store_si128((__m128i *)(dqcoeff), dqcoeff32_0);
_mm_store_si128((__m128i *)(dqcoeff + 4), dqcoeff32_1);
}
void aom_quantize_b_64x64_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
const int16_t *zbin_ptr,
const int16_t *round_ptr,
const int16_t *quant_ptr,
const int16_t *quant_shift_ptr,
tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr,
const int16_t *dequant_ptr, uint16_t *eob_ptr,
const int16_t *scan, const int16_t *iscan) {
const __m128i zero = _mm_setzero_si128();
const __m128i one = _mm_set1_epi16(1);
const __m128i two = _mm_set1_epi16(2);
int index;
__m128i zbin, round, quant, dequant, shift;
__m128i coeff0, coeff1, qcoeff0, qcoeff1;
__m128i cmp_mask0, cmp_mask1, all_zero;
__m128i eob = zero, eob0;
(void)scan;
(void)n_coeffs;
// Setup global values.
zbin = _mm_load_si128((const __m128i *)zbin_ptr);
round = _mm_load_si128((const __m128i *)round_ptr);
quant = _mm_load_si128((const __m128i *)quant_ptr);
dequant = _mm_load_si128((const __m128i *)dequant_ptr);
shift = _mm_load_si128((const __m128i *)quant_shift_ptr);
// Shift with rounding.
zbin = _mm_add_epi16(zbin, two);
round = _mm_add_epi16(round, two);
zbin = _mm_srli_epi16(zbin, 2);
round = _mm_srli_epi16(round, 2);
zbin = _mm_sub_epi16(zbin, one);
// Do DC and first 15 AC.
coeff0 = load_coefficients(coeff_ptr);
coeff1 = load_coefficients(coeff_ptr + 8);
qcoeff0 = _mm_abs_epi16(coeff0);
qcoeff1 = _mm_abs_epi16(coeff1);
cmp_mask0 = _mm_cmpgt_epi16(qcoeff0, zbin);
zbin = _mm_unpackhi_epi64(zbin, zbin);
cmp_mask1 = _mm_cmpgt_epi16(qcoeff1, zbin);
all_zero = _mm_or_si128(cmp_mask0, cmp_mask1);
if (_mm_movemask_epi8(all_zero) == 0) {
_mm_store_si128((__m128i *)(qcoeff_ptr), zero);
_mm_store_si128((__m128i *)(qcoeff_ptr + 4), zero);
_mm_store_si128((__m128i *)(qcoeff_ptr + 8), zero);
_mm_store_si128((__m128i *)(qcoeff_ptr + 12), zero);
_mm_store_si128((__m128i *)(dqcoeff_ptr), zero);
_mm_store_si128((__m128i *)(dqcoeff_ptr + 4), zero);
_mm_store_si128((__m128i *)(dqcoeff_ptr + 8), zero);
_mm_store_si128((__m128i *)(dqcoeff_ptr + 12), zero);
round = _mm_unpackhi_epi64(round, round);
quant = _mm_unpackhi_epi64(quant, quant);
shift = _mm_unpackhi_epi64(shift, shift);
dequant = _mm_unpackhi_epi64(dequant, dequant);
} else {
calculate_qcoeff_64x64(&qcoeff0, round, quant, &shift);
round = _mm_unpackhi_epi64(round, round);
quant = _mm_unpackhi_epi64(quant, quant);
shift = _mm_unpackhi_epi64(shift, shift);
calculate_qcoeff_64x64(&qcoeff1, round, quant, &shift);
// Reinsert signs.
qcoeff0 = _mm_sign_epi16(qcoeff0, coeff0);
qcoeff1 = _mm_sign_epi16(qcoeff1, coeff1);
// Mask out zbin threshold coeffs.
qcoeff0 = _mm_and_si128(qcoeff0, cmp_mask0);
qcoeff1 = _mm_and_si128(qcoeff1, cmp_mask1);
store_coefficients(qcoeff0, qcoeff_ptr);
store_coefficients(qcoeff1, qcoeff_ptr + 8);
calculate_dqcoeff_and_store_64x64(qcoeff0, dequant, zero, dqcoeff_ptr);
dequant = _mm_unpackhi_epi64(dequant, dequant);
calculate_dqcoeff_and_store_64x64(qcoeff1, dequant, zero, dqcoeff_ptr + 8);
eob =
scan_for_eob(&qcoeff0, &qcoeff1, cmp_mask0, cmp_mask1, iscan, 0, zero);
}
// AC only loop.
for (index = 16; index < 1024; index += 16) {
coeff0 = load_coefficients(coeff_ptr + index);
coeff1 = load_coefficients(coeff_ptr + index + 8);
qcoeff0 = _mm_abs_epi16(coeff0);
qcoeff1 = _mm_abs_epi16(coeff1);
cmp_mask0 = _mm_cmpgt_epi16(qcoeff0, zbin);
cmp_mask1 = _mm_cmpgt_epi16(qcoeff1, zbin);
all_zero = _mm_or_si128(cmp_mask0, cmp_mask1);
if (_mm_movemask_epi8(all_zero) == 0) {
_mm_store_si128((__m128i *)(qcoeff_ptr + index), zero);
_mm_store_si128((__m128i *)(qcoeff_ptr + index + 4), zero);
_mm_store_si128((__m128i *)(qcoeff_ptr + index + 8), zero);
_mm_store_si128((__m128i *)(qcoeff_ptr + index + 12), zero);
_mm_store_si128((__m128i *)(dqcoeff_ptr + index), zero);
_mm_store_si128((__m128i *)(dqcoeff_ptr + index + 4), zero);
_mm_store_si128((__m128i *)(dqcoeff_ptr + index + 8), zero);
_mm_store_si128((__m128i *)(dqcoeff_ptr + index + 12), zero);
continue;
}
calculate_qcoeff_64x64(&qcoeff0, round, quant, &shift);
calculate_qcoeff_64x64(&qcoeff1, round, quant, &shift);
qcoeff0 = _mm_sign_epi16(qcoeff0, coeff0);
qcoeff1 = _mm_sign_epi16(qcoeff1, coeff1);
qcoeff0 = _mm_and_si128(qcoeff0, cmp_mask0);
qcoeff1 = _mm_and_si128(qcoeff1, cmp_mask1);
store_coefficients(qcoeff0, qcoeff_ptr + index);
store_coefficients(qcoeff1, qcoeff_ptr + index + 8);
calculate_dqcoeff_and_store_64x64(qcoeff0, dequant, zero,
dqcoeff_ptr + index);
calculate_dqcoeff_and_store_64x64(qcoeff1, dequant, zero,
dqcoeff_ptr + 8 + index);
eob0 = scan_for_eob(&qcoeff0, &qcoeff1, cmp_mask0, cmp_mask1, iscan, index,
zero);
eob = _mm_max_epi16(eob, eob0);
}
*eob_ptr = accumulate_eob(eob);
}

40
third_party/aom/aom_dsp/x86/quantize_ssse3_x86_64.asm vendored
View file

@ -48,9 +48,6 @@ cglobal quantize_%1, 0, %2, 15, coeff, ncoeff, zbin, round, quant, \
mov r3, qcoeffmp
mov r4, dqcoeffmp
mov r5, iscanmp
%ifidn %1, b_32x32
psllw m4, 1
%endif
pxor m5, m5 ; m5 = dedicated zero
DEFINE_ARGS coeff, ncoeff, d1, qcoeff, dqcoeff, iscan, d2, d3, d4, eob
lea coeffq, [ coeffq+ncoeffq*4]
@ -78,9 +75,26 @@ cglobal quantize_%1, 0, %2, 15, coeff, ncoeff, zbin, round, quant, \
pmulhw m13, m11, m2 ; m13 = m11*q>>16
paddw m8, m6 ; m8 += m6
paddw m13, m11 ; m13 += m11
%ifidn %1, b_32x32
pmullw m5, m8, m4 ; store the lower 16 bits of m8*qsh
%endif
pmulhw m8, m4 ; m8 = m8*qsh>>16
%ifidn %1, b_32x32
psllw m8, 1
psrlw m5, 15
por m8, m5
%endif
punpckhqdq m4, m4
%ifidn %1, b_32x32
pmullw m5, m13, m4 ; store the lower 16 bits of m13*qsh
%endif
pmulhw m13, m4 ; m13 = m13*qsh>>16
%ifidn %1, b_32x32
psllw m13, 1
psrlw m5, 15
por m13, m5
pxor m5, m5 ; reset m5 to zero register
%endif
psignw m8, m9 ; m8 = reinsert sign
psignw m13, m10 ; m13 = reinsert sign
pand m8, m7
@ -117,7 +131,7 @@ cglobal quantize_%1, 0, %2, 15, coeff, ncoeff, zbin, round, quant, \
psignw m8, m9
psignw m13, m10
%endif
; store 16bit numbers as 32bit numbers in array pointed to by qcoeff
; store 16bit numbers as 32bit numbers in array pointed to by dqcoeff
mova m11, m8
mova m6, m8
pcmpgtw m5, m8
@ -169,12 +183,28 @@ cglobal quantize_%1, 0, %2, 15, coeff, ncoeff, zbin, round, quant, \
pmulhw m13, m11, m2 ; m13 = m11*q>>16
paddw m14, m6 ; m14 += m6
paddw m13, m11 ; m13 += m11
%ifidn %1, b_32x32
pmullw m5, m14, m4 ; store the lower 16 bits of m14*qsh
%endif
pmulhw m14, m4 ; m14 = m14*qsh>>16
%ifidn %1, b_32x32
psllw m14, 1
psrlw m5, 15
por m14, m5
pmullw m5, m13, m4 ; store the lower 16 bits of m13*qsh
%endif
pmulhw m13, m4 ; m13 = m13*qsh>>16
%ifidn %1, b_32x32
psllw m13, 1
psrlw m5, 15
por m13, m5
pxor m5, m5 ; reset m5 to zero register
%endif
psignw m14, m9 ; m14 = reinsert sign
psignw m13, m10 ; m13 = reinsert sign
pand m14, m7
pand m13, m12
; store 16bit numbers as 32bit numbers in array pointed to by qcoeff
pxor m11, m11
mova m11, m14
@ -207,7 +237,7 @@ cglobal quantize_%1, 0, %2, 15, coeff, ncoeff, zbin, round, quant, \
psignw m13, m10
%endif
; store 16bit numbers as 32bit numbers in array pointed to by qcoeff
; store 16bit numbers as 32bit numbers in array pointed to by dqcoeff
mova m11, m14
mova m6, m14
pcmpgtw m5, m14

20
third_party/aom/aom_dsp/x86/quantize_x86.h vendored
View file

@ -75,3 +75,23 @@ static INLINE int16_t accumulate_eob(__m128i eob) {
eob = _mm_max_epi16(eob, eob_shuffled);
return _mm_extract_epi16(eob, 1);
}
static INLINE __m128i load_coefficients(const tran_low_t *coeff_ptr) {
assert(sizeof(tran_low_t) == 4);
const __m128i coeff1 = _mm_load_si128((__m128i *)(coeff_ptr));
const __m128i coeff2 = _mm_load_si128((__m128i *)(coeff_ptr + 4));
return _mm_packs_epi32(coeff1, coeff2);
}
static INLINE void store_coefficients(__m128i coeff_vals,
tran_low_t *coeff_ptr) {
assert(sizeof(tran_low_t) == 4);
__m128i one = _mm_set1_epi16(1);
__m128i coeff_vals_hi = _mm_mulhi_epi16(coeff_vals, one);
__m128i coeff_vals_lo = _mm_mullo_epi16(coeff_vals, one);
__m128i coeff_vals_1 = _mm_unpacklo_epi16(coeff_vals_lo, coeff_vals_hi);
__m128i coeff_vals_2 = _mm_unpackhi_epi16(coeff_vals_lo, coeff_vals_hi);
_mm_store_si128((__m128i *)(coeff_ptr), coeff_vals_1);
_mm_store_si128((__m128i *)(coeff_ptr + 4), coeff_vals_2);
}

308
third_party/aom/aom_dsp/x86/sse_avx2.c vendored
View file

@ -21,12 +21,11 @@ static INLINE void sse_w32_avx2(__m256i *sum, const uint8_t *a,
const uint8_t *b) {
const __m256i v_a0 = yy_loadu_256(a);
const __m256i v_b0 = yy_loadu_256(b);
const __m256i v_a00_w = _mm256_cvtepu8_epi16(_mm256_castsi256_si128(v_a0));
const __m256i v_a01_w =
_mm256_cvtepu8_epi16(_mm256_extracti128_si256(v_a0, 1));
const __m256i v_b00_w = _mm256_cvtepu8_epi16(_mm256_castsi256_si128(v_b0));
const __m256i v_b01_w =
_mm256_cvtepu8_epi16(_mm256_extracti128_si256(v_b0, 1));
const __m256i zero = _mm256_setzero_si256();
const __m256i v_a00_w = _mm256_unpacklo_epi8(v_a0, zero);
const __m256i v_a01_w = _mm256_unpackhi_epi8(v_a0, zero);
const __m256i v_b00_w = _mm256_unpacklo_epi8(v_b0, zero);
const __m256i v_b01_w = _mm256_unpackhi_epi8(v_b0, zero);
const __m256i v_d00_w = _mm256_sub_epi16(v_a00_w, v_b00_w);
const __m256i v_d01_w = _mm256_sub_epi16(v_a01_w, v_b01_w);
*sum = _mm256_add_epi32(*sum, _mm256_madd_epi16(v_d00_w, v_d00_w));
@ -35,43 +34,76 @@ static INLINE void sse_w32_avx2(__m256i *sum, const uint8_t *a,
static INLINE int64_t summary_all_avx2(const __m256i *sum_all) {
int64_t sum;
const __m256i sum0_4x64 =
_mm256_cvtepu32_epi64(_mm256_castsi256_si128(*sum_all));
const __m256i sum1_4x64 =
_mm256_cvtepu32_epi64(_mm256_extracti128_si256(*sum_all, 1));
__m256i zero = _mm256_setzero_si256();
const __m256i sum0_4x64 = _mm256_unpacklo_epi32(*sum_all, zero);
const __m256i sum1_4x64 = _mm256_unpackhi_epi32(*sum_all, zero);
const __m256i sum_4x64 = _mm256_add_epi64(sum0_4x64, sum1_4x64);
const __m128i sum_2x64 = _mm_add_epi64(_mm256_castsi256_si128(sum_4x64),
_mm256_extracti128_si256(sum_4x64, 1));
const __m128i sum_1x64 = _mm_add_epi64(sum_2x64, _mm_srli_si128(sum_2x64, 8));
xx_storel_64(&sum, sum_1x64);
return sum;
}
static INLINE void summary_32_avx2(const __m256i *sum32, __m256i *sum) {
const __m256i sum0_4x64 =
_mm256_cvtepu32_epi64(_mm256_castsi256_si128(*sum32));
const __m256i sum1_4x64 =
_mm256_cvtepu32_epi64(_mm256_extracti128_si256(*sum32, 1));
const __m256i sum_4x64 = _mm256_add_epi64(sum0_4x64, sum1_4x64);
*sum = _mm256_add_epi64(*sum, sum_4x64);
}
static INLINE int64_t summary_4x64_avx2(const __m256i sum_4x64) {
int64_t sum;
const __m128i sum_2x64 = _mm_add_epi64(_mm256_castsi256_si128(sum_4x64),
_mm256_extracti128_si256(sum_4x64, 1));
const __m128i sum_1x64 = _mm_add_epi64(sum_2x64, _mm_srli_si128(sum_2x64, 8));
xx_storel_64(&sum, sum_1x64);
return sum;
}
static INLINE void sse_w4x4_avx2(const uint8_t *a, int a_stride,
const uint8_t *b, int b_stride, __m256i *sum) {
const __m128i v_a0 = xx_loadl_32(a);
const __m128i v_a1 = xx_loadl_32(a + a_stride);
const __m128i v_a2 = xx_loadl_32(a + a_stride * 2);
const __m128i v_a3 = xx_loadl_32(a + a_stride * 3);
const __m128i v_b0 = xx_loadl_32(b);
const __m128i v_b1 = xx_loadl_32(b + b_stride);
const __m128i v_b2 = xx_loadl_32(b + b_stride * 2);
const __m128i v_b3 = xx_loadl_32(b + b_stride * 3);
const __m128i v_a0123 = _mm_unpacklo_epi64(_mm_unpacklo_epi32(v_a0, v_a1),
_mm_unpacklo_epi32(v_a2, v_a3));
const __m128i v_b0123 = _mm_unpacklo_epi64(_mm_unpacklo_epi32(v_b0, v_b1),
_mm_unpacklo_epi32(v_b2, v_b3));
const __m256i v_a_w = _mm256_cvtepu8_epi16(v_a0123);
const __m256i v_b_w = _mm256_cvtepu8_epi16(v_b0123);
const __m256i v_d_w = _mm256_sub_epi16(v_a_w, v_b_w);
*sum = _mm256_add_epi32(*sum, _mm256_madd_epi16(v_d_w, v_d_w));
}
static INLINE void sse_w8x2_avx2(const uint8_t *a, int a_stride,
const uint8_t *b, int b_stride, __m256i *sum) {
const __m128i v_a0 = xx_loadl_64(a);
const __m128i v_a1 = xx_loadl_64(a + a_stride);
const __m128i v_b0 = xx_loadl_64(b);
const __m128i v_b1 = xx_loadl_64(b + b_stride);
const __m256i v_a_w = _mm256_cvtepu8_epi16(_mm_unpacklo_epi64(v_a0, v_a1));
const __m256i v_b_w = _mm256_cvtepu8_epi16(_mm_unpacklo_epi64(v_b0, v_b1));
const __m256i v_d_w = _mm256_sub_epi16(v_a_w, v_b_w);
*sum = _mm256_add_epi32(*sum, _mm256_madd_epi16(v_d_w, v_d_w));
}
int64_t aom_sse_avx2(const uint8_t *a, int a_stride, const uint8_t *b,
int b_stride, int width, int height) {
int32_t y = 0;
int64_t sse = 0;
__m256i sum = _mm256_setzero_si256();
__m256i zero = _mm256_setzero_si256();
switch (width) {
case 4:
do {
const __m128i v_a0 = xx_loadl_32(a);
const __m128i v_a1 = xx_loadl_32(a + a_stride);
const __m128i v_a2 = xx_loadl_32(a + a_stride * 2);
const __m128i v_a3 = xx_loadl_32(a + a_stride * 3);
const __m128i v_b0 = xx_loadl_32(b);
const __m128i v_b1 = xx_loadl_32(b + b_stride);
const __m128i v_b2 = xx_loadl_32(b + b_stride * 2);
const __m128i v_b3 = xx_loadl_32(b + b_stride * 3);
const __m128i v_a0123 = _mm_unpacklo_epi64(
_mm_unpacklo_epi32(v_a0, v_a1), _mm_unpacklo_epi32(v_a2, v_a3));
const __m128i v_b0123 = _mm_unpacklo_epi64(
_mm_unpacklo_epi32(v_b0, v_b1), _mm_unpacklo_epi32(v_b2, v_b3));
const __m256i v_a_w = _mm256_cvtepu8_epi16(v_a0123);
const __m256i v_b_w = _mm256_cvtepu8_epi16(v_b0123);
const __m256i v_d_w = _mm256_sub_epi16(v_a_w, v_b_w);
sum = _mm256_add_epi32(sum, _mm256_madd_epi16(v_d_w, v_d_w));
sse_w4x4_avx2(a, a_stride, b, b_stride, &sum);
a += a_stride << 2;
b += b_stride << 2;
y += 4;
@ -80,16 +112,7 @@ int64_t aom_sse_avx2(const uint8_t *a, int a_stride, const uint8_t *b,
break;
case 8:
do {
const __m128i v_a0 = xx_loadl_64(a);
const __m128i v_a1 = xx_loadl_64(a + a_stride);
const __m128i v_b0 = xx_loadl_64(b);
const __m128i v_b1 = xx_loadl_64(b + b_stride);
const __m256i v_a_w =
_mm256_cvtepu8_epi16(_mm_unpacklo_epi64(v_a0, v_a1));
const __m256i v_b_w =
_mm256_cvtepu8_epi16(_mm_unpacklo_epi64(v_b0, v_b1));
const __m256i v_d_w = _mm256_sub_epi16(v_a_w, v_b_w);
sum = _mm256_add_epi32(sum, _mm256_madd_epi16(v_d_w, v_d_w));
sse_w8x2_avx2(a, a_stride, b, b_stride, &sum);
a += a_stride << 1;
b += b_stride << 1;
y += 2;
@ -99,14 +122,26 @@ int64_t aom_sse_avx2(const uint8_t *a, int a_stride, const uint8_t *b,
case 16:
do {
const __m128i v_a0 = xx_loadu_128(a);
const __m128i v_a1 = xx_loadu_128(a + a_stride);
const __m128i v_b0 = xx_loadu_128(b);
const __m256i v_a_w = _mm256_cvtepu8_epi16(v_a0);
const __m256i v_b_w = _mm256_cvtepu8_epi16(v_b0);
const __m256i v_d_w = _mm256_sub_epi16(v_a_w, v_b_w);
sum = _mm256_add_epi32(sum, _mm256_madd_epi16(v_d_w, v_d_w));
a += a_stride;
b += b_stride;
y += 1;
const __m128i v_b1 = xx_loadu_128(b + b_stride);
const __m256i v_a =
_mm256_insertf128_si256(_mm256_castsi128_si256(v_a0), v_a1, 0x01);
const __m256i v_b =
_mm256_insertf128_si256(_mm256_castsi128_si256(v_b0), v_b1, 0x01);
const __m256i v_al = _mm256_unpacklo_epi8(v_a, zero);
const __m256i v_au = _mm256_unpackhi_epi8(v_a, zero);
const __m256i v_bl = _mm256_unpacklo_epi8(v_b, zero);
const __m256i v_bu = _mm256_unpackhi_epi8(v_b, zero);
const __m256i v_asub = _mm256_sub_epi16(v_al, v_bl);
const __m256i v_bsub = _mm256_sub_epi16(v_au, v_bu);
const __m256i temp =
_mm256_add_epi32(_mm256_madd_epi16(v_asub, v_asub),
_mm256_madd_epi16(v_bsub, v_bsub));
sum = _mm256_add_epi32(sum, temp);
a += a_stride << 1;
b += b_stride << 1;
y += 2;
} while (y < height);
sse = summary_all_avx2(&sum);
break;
@ -141,7 +176,36 @@ int64_t aom_sse_avx2(const uint8_t *a, int a_stride, const uint8_t *b,
} while (y < height);
sse = summary_all_avx2(&sum);
break;
default: break;
default:
if ((width & 0x07) == 0) {
do {
int i = 0;
do {
sse_w8x2_avx2(a + i, a_stride, b + i, b_stride, &sum);
i += 8;
} while (i < width);
a += a_stride << 1;
b += b_stride << 1;
y += 2;
} while (y < height);
} else {
do {
int i = 0;
do {
sse_w8x2_avx2(a + i, a_stride, b + i, b_stride, &sum);
const uint8_t *a2 = a + i + (a_stride << 1);
const uint8_t *b2 = b + i + (b_stride << 1);
sse_w8x2_avx2(a2, a_stride, b2, b_stride, &sum);
i += 8;
} while (i + 4 < width);
sse_w4x4_avx2(a + i, a_stride, b + i, b_stride, &sum);
a += a_stride << 2;
b += b_stride << 2;
y += 4;
} while (y < height);
}
sse = summary_all_avx2(&sum);
break;
}
return sse;
@ -155,6 +219,33 @@ static INLINE void highbd_sse_w16_avx2(__m256i *sum, const uint16_t *a,
*sum = _mm256_add_epi32(*sum, _mm256_madd_epi16(v_d_w, v_d_w));
}
static INLINE void highbd_sse_w4x4_avx2(__m256i *sum, const uint16_t *a,
int a_stride, const uint16_t *b,
int b_stride) {
const __m128i v_a0 = xx_loadl_64(a);
const __m128i v_a1 = xx_loadl_64(a + a_stride);
const __m128i v_a2 = xx_loadl_64(a + a_stride * 2);
const __m128i v_a3 = xx_loadl_64(a + a_stride * 3);
const __m128i v_b0 = xx_loadl_64(b);
const __m128i v_b1 = xx_loadl_64(b + b_stride);
const __m128i v_b2 = xx_loadl_64(b + b_stride * 2);
const __m128i v_b3 = xx_loadl_64(b + b_stride * 3);
const __m256i v_a_w = yy_set_m128i(_mm_unpacklo_epi64(v_a0, v_a1),
_mm_unpacklo_epi64(v_a2, v_a3));
const __m256i v_b_w = yy_set_m128i(_mm_unpacklo_epi64(v_b0, v_b1),
_mm_unpacklo_epi64(v_b2, v_b3));
const __m256i v_d_w = _mm256_sub_epi16(v_a_w, v_b_w);
*sum = _mm256_add_epi32(*sum, _mm256_madd_epi16(v_d_w, v_d_w));
}
static INLINE void highbd_sse_w8x2_avx2(__m256i *sum, const uint16_t *a,
int a_stride, const uint16_t *b,
int b_stride) {
const __m256i v_a_w = yy_loadu2_128(a + a_stride, a);
const __m256i v_b_w = yy_loadu2_128(b + b_stride, b);
const __m256i v_d_w = _mm256_sub_epi16(v_a_w, v_b_w);
*sum = _mm256_add_epi32(*sum, _mm256_madd_epi16(v_d_w, v_d_w));
}
int64_t aom_highbd_sse_avx2(const uint8_t *a8, int a_stride, const uint8_t *b8,
int b_stride, int width, int height) {
int32_t y = 0;
@ -165,20 +256,7 @@ int64_t aom_highbd_sse_avx2(const uint8_t *a8, int a_stride, const uint8_t *b8,
switch (width) {
case 4:
do {
const __m128i v_a0 = xx_loadl_64(a);
const __m128i v_a1 = xx_loadl_64(a + a_stride);
const __m128i v_a2 = xx_loadl_64(a + a_stride * 2);
const __m128i v_a3 = xx_loadl_64(a + a_stride * 3);
const __m128i v_b0 = xx_loadl_64(b);
const __m128i v_b1 = xx_loadl_64(b + b_stride);
const __m128i v_b2 = xx_loadl_64(b + b_stride * 2);
const __m128i v_b3 = xx_loadl_64(b + b_stride * 3);
const __m256i v_a_w = yy_set_m128i(_mm_unpacklo_epi64(v_a0, v_a1),
_mm_unpacklo_epi64(v_a2, v_a3));
const __m256i v_b_w = yy_set_m128i(_mm_unpacklo_epi64(v_b0, v_b1),
_mm_unpacklo_epi64(v_b2, v_b3));
const __m256i v_d_w = _mm256_sub_epi16(v_a_w, v_b_w);
sum = _mm256_add_epi32(sum, _mm256_madd_epi16(v_d_w, v_d_w));
highbd_sse_w4x4_avx2(&sum, a, a_stride, b, b_stride);
a += a_stride << 2;
b += b_stride << 2;
y += 4;
@ -187,10 +265,7 @@ int64_t aom_highbd_sse_avx2(const uint8_t *a8, int a_stride, const uint8_t *b8,
break;
case 8:
do {
const __m256i v_a_w = yy_loadu2_128(a + a_stride, a);
const __m256i v_b_w = yy_loadu2_128(b + b_stride, b);
const __m256i v_d_w = _mm256_sub_epi16(v_a_w, v_b_w);
sum = _mm256_add_epi32(sum, _mm256_madd_epi16(v_d_w, v_d_w));
highbd_sse_w8x2_avx2(&sum, a, a_stride, b, b_stride);
a += a_stride << 1;
b += b_stride << 1;
y += 2;
@ -208,43 +283,98 @@ int64_t aom_highbd_sse_avx2(const uint8_t *a8, int a_stride, const uint8_t *b8,
break;
case 32:
do {
highbd_sse_w16_avx2(&sum, a, b);
highbd_sse_w16_avx2(&sum, a + 16, b + 16);
a += a_stride;
b += b_stride;
y += 1;
int l = 0;
__m256i sum32 = _mm256_setzero_si256();
do {
highbd_sse_w16_avx2(&sum32, a, b);
highbd_sse_w16_avx2(&sum32, a + 16, b + 16);
a += a_stride;
b += b_stride;
l += 1;
} while (l < 64 && l < (height - y));
summary_32_avx2(&sum32, &sum);
y += 64;
} while (y < height);
sse = summary_all_avx2(&sum);
sse = summary_4x64_avx2(sum);
break;
case 64:
do {
highbd_sse_w16_avx2(&sum, a, b);
highbd_sse_w16_avx2(&sum, a + 16 * 1, b + 16 * 1);
highbd_sse_w16_avx2(&sum, a + 16 * 2, b + 16 * 2);
highbd_sse_w16_avx2(&sum, a + 16 * 3, b + 16 * 3);
a += a_stride;
b += b_stride;
y += 1;
int l = 0;
__m256i sum32 = _mm256_setzero_si256();
do {
highbd_sse_w16_avx2(&sum32, a, b);
highbd_sse_w16_avx2(&sum32, a + 16 * 1, b + 16 * 1);
highbd_sse_w16_avx2(&sum32, a + 16 * 2, b + 16 * 2);
highbd_sse_w16_avx2(&sum32, a + 16 * 3, b + 16 * 3);
a += a_stride;
b += b_stride;
l += 1;
} while (l < 32 && l < (height - y));
summary_32_avx2(&sum32, &sum);
y += 32;
} while (y < height);
sse = summary_all_avx2(&sum);
sse = summary_4x64_avx2(sum);
break;
case 128:
do {
highbd_sse_w16_avx2(&sum, a, b);
highbd_sse_w16_avx2(&sum, a + 16 * 1, b + 16 * 1);
highbd_sse_w16_avx2(&sum, a + 16 * 2, b + 16 * 2);
highbd_sse_w16_avx2(&sum, a + 16 * 3, b + 16 * 3);
highbd_sse_w16_avx2(&sum, a + 16 * 4, b + 16 * 4);
highbd_sse_w16_avx2(&sum, a + 16 * 5, b + 16 * 5);
highbd_sse_w16_avx2(&sum, a + 16 * 6, b + 16 * 6);
highbd_sse_w16_avx2(&sum, a + 16 * 7, b + 16 * 7);
a += a_stride;
b += b_stride;
y += 1;
int l = 0;
__m256i sum32 = _mm256_setzero_si256();
do {
highbd_sse_w16_avx2(&sum32, a, b);
highbd_sse_w16_avx2(&sum32, a + 16 * 1, b + 16 * 1);
highbd_sse_w16_avx2(&sum32, a + 16 * 2, b + 16 * 2);
highbd_sse_w16_avx2(&sum32, a + 16 * 3, b + 16 * 3);
highbd_sse_w16_avx2(&sum32, a + 16 * 4, b + 16 * 4);
highbd_sse_w16_avx2(&sum32, a + 16 * 5, b + 16 * 5);
highbd_sse_w16_avx2(&sum32, a + 16 * 6, b + 16 * 6);
highbd_sse_w16_avx2(&sum32, a + 16 * 7, b + 16 * 7);
a += a_stride;
b += b_stride;
l += 1;
} while (l < 16 && l < (height - y));
summary_32_avx2(&sum32, &sum);
y += 16;
} while (y < height);
sse = summary_all_avx2(&sum);
sse = summary_4x64_avx2(sum);
break;
default:
if (width & 0x7) {
do {
int i = 0;
__m256i sum32 = _mm256_setzero_si256();
do {
highbd_sse_w8x2_avx2(&sum32, a + i, a_stride, b + i, b_stride);
const uint16_t *a2 = a + i + (a_stride << 1);
const uint16_t *b2 = b + i + (b_stride << 1);
highbd_sse_w8x2_avx2(&sum32, a2, a_stride, b2, b_stride);
i += 8;
} while (i + 4 < width);
highbd_sse_w4x4_avx2(&sum32, a + i, a_stride, b + i, b_stride);
summary_32_avx2(&sum32, &sum);
a += a_stride << 2;
b += b_stride << 2;
y += 4;
} while (y < height);
} else {
do {
int l = 0;
__m256i sum32 = _mm256_setzero_si256();
do {
int i = 0;
do {
highbd_sse_w8x2_avx2(&sum32, a + i, a_stride, b + i, b_stride);
i += 8;
} while (i < width);
a += a_stride << 1;
b += b_stride << 1;
l += 2;
} while (l < 8 && l < (height - y));
summary_32_avx2(&sum32, &sum);
y += 8;
} while (y < height);
}
sse = summary_4x64_avx2(sum);
break;
default: break;
}
return sse;
}

249
third_party/aom/aom_dsp/x86/sse_sse4.c vendored
View file

@ -28,6 +28,13 @@ static INLINE int64_t summary_all_sse4(const __m128i *sum_all) {
return sum;
}
static INLINE void summary_32_sse4(const __m128i *sum32, __m128i *sum64) {
const __m128i sum0 = _mm_cvtepu32_epi64(*sum32);
const __m128i sum1 = _mm_cvtepu32_epi64(_mm_srli_si128(*sum32, 8));
*sum64 = _mm_add_epi64(sum0, *sum64);
*sum64 = _mm_add_epi64(sum1, *sum64);
}
static INLINE void sse_w16_sse4_1(__m128i *sum, const uint8_t *a,
const uint8_t *b) {
const __m128i v_a0 = xx_loadu_128(a);
@ -42,6 +49,28 @@ static INLINE void sse_w16_sse4_1(__m128i *sum, const uint8_t *a,
*sum = _mm_add_epi32(*sum, _mm_madd_epi16(v_d01_w, v_d01_w));
}
static INLINE void aom_sse4x2_sse4_1(const uint8_t *a, int a_stride,
const uint8_t *b, int b_stride,
__m128i *sum) {
const __m128i v_a0 = xx_loadl_32(a);
const __m128i v_a1 = xx_loadl_32(a + a_stride);
const __m128i v_b0 = xx_loadl_32(b);
const __m128i v_b1 = xx_loadl_32(b + b_stride);
const __m128i v_a_w = _mm_cvtepu8_epi16(_mm_unpacklo_epi32(v_a0, v_a1));
const __m128i v_b_w = _mm_cvtepu8_epi16(_mm_unpacklo_epi32(v_b0, v_b1));
const __m128i v_d_w = _mm_sub_epi16(v_a_w, v_b_w);
*sum = _mm_add_epi32(*sum, _mm_madd_epi16(v_d_w, v_d_w));
}
static INLINE void aom_sse8_sse4_1(const uint8_t *a, const uint8_t *b,
__m128i *sum) {
const __m128i v_a0 = xx_loadl_64(a);
const __m128i v_b0 = xx_loadl_64(b);
const __m128i v_a_w = _mm_cvtepu8_epi16(v_a0);
const __m128i v_b_w = _mm_cvtepu8_epi16(v_b0);
const __m128i v_d_w = _mm_sub_epi16(v_a_w, v_b_w);
*sum = _mm_add_epi32(*sum, _mm_madd_epi16(v_d_w, v_d_w));
}
int64_t aom_sse_sse4_1(const uint8_t *a, int a_stride, const uint8_t *b,
int b_stride, int width, int height) {
int y = 0;
@ -50,14 +79,7 @@ int64_t aom_sse_sse4_1(const uint8_t *a, int a_stride, const uint8_t *b,
switch (width) {
case 4:
do {
const __m128i v_a0 = xx_loadl_32(a);
const __m128i v_a1 = xx_loadl_32(a + a_stride);
const __m128i v_b0 = xx_loadl_32(b);
const __m128i v_b1 = xx_loadl_32(b + b_stride);
const __m128i v_a_w = _mm_cvtepu8_epi16(_mm_unpacklo_epi32(v_a0, v_a1));
const __m128i v_b_w = _mm_cvtepu8_epi16(_mm_unpacklo_epi32(v_b0, v_b1));
const __m128i v_d_w = _mm_sub_epi16(v_a_w, v_b_w);
sum = _mm_add_epi32(sum, _mm_madd_epi16(v_d_w, v_d_w));
aom_sse4x2_sse4_1(a, a_stride, b, b_stride, &sum);
a += a_stride << 1;
b += b_stride << 1;
y += 2;
@ -66,12 +88,7 @@ int64_t aom_sse_sse4_1(const uint8_t *a, int a_stride, const uint8_t *b,
break;
case 8:
do {
const __m128i v_a0 = xx_loadl_64(a);
const __m128i v_b0 = xx_loadl_64(b);
const __m128i v_a_w = _mm_cvtepu8_epi16(v_a0);
const __m128i v_b_w = _mm_cvtepu8_epi16(v_b0);
const __m128i v_d_w = _mm_sub_epi16(v_a_w, v_b_w);
sum = _mm_add_epi32(sum, _mm_madd_epi16(v_d_w, v_d_w));
aom_sse8_sse4_1(a, b, &sum);
a += a_stride;
b += b_stride;
y += 1;
@ -125,12 +142,52 @@ int64_t aom_sse_sse4_1(const uint8_t *a, int a_stride, const uint8_t *b,
} while (y < height);
sse = summary_all_sse4(&sum);
break;
default: break;
default:
if (width & 0x07) {
do {
int i = 0;
do {
aom_sse8_sse4_1(a + i, b + i, &sum);
aom_sse8_sse4_1(a + i + a_stride, b + i + b_stride, &sum);
i += 8;
} while (i + 4 < width);
aom_sse4x2_sse4_1(a + i, a_stride, b + i, b_stride, &sum);
a += (a_stride << 1);
b += (b_stride << 1);
y += 2;
} while (y < height);
} else {
do {
int i = 0;
do {
aom_sse8_sse4_1(a + i, b + i, &sum);
i += 8;
} while (i < width);
a += a_stride;
b += b_stride;
y += 1;
} while (y < height);
}
sse = summary_all_sse4(&sum);
break;
}
return sse;
}
static INLINE void highbd_sse_w4x2_sse4_1(__m128i *sum, const uint16_t *a,
int a_stride, const uint16_t *b,
int b_stride) {
const __m128i v_a0 = xx_loadl_64(a);
const __m128i v_a1 = xx_loadl_64(a + a_stride);
const __m128i v_b0 = xx_loadl_64(b);
const __m128i v_b1 = xx_loadl_64(b + b_stride);
const __m128i v_a_w = _mm_unpacklo_epi64(v_a0, v_a1);
const __m128i v_b_w = _mm_unpacklo_epi64(v_b0, v_b1);
const __m128i v_d_w = _mm_sub_epi16(v_a_w, v_b_w);
*sum = _mm_add_epi32(*sum, _mm_madd_epi16(v_d_w, v_d_w));
}
static INLINE void highbd_sse_w8_sse4_1(__m128i *sum, const uint16_t *a,
const uint16_t *b) {
const __m128i v_a_w = xx_loadu_128(a);
@ -150,14 +207,7 @@ int64_t aom_highbd_sse_sse4_1(const uint8_t *a8, int a_stride,
switch (width) {
case 4:
do {
const __m128i v_a0 = xx_loadl_64(a);
const __m128i v_a1 = xx_loadl_64(a + a_stride);
const __m128i v_b0 = xx_loadl_64(b);
const __m128i v_b1 = xx_loadl_64(b + b_stride);
const __m128i v_a_w = _mm_unpacklo_epi64(v_a0, v_a1);
const __m128i v_b_w = _mm_unpacklo_epi64(v_b0, v_b1);
const __m128i v_d_w = _mm_sub_epi16(v_a_w, v_b_w);
sum = _mm_add_epi32(sum, _mm_madd_epi16(v_d_w, v_d_w));
highbd_sse_w4x2_sse4_1(&sum, a, a_stride, b, b_stride);
a += a_stride << 1;
b += b_stride << 1;
y += 2;
@ -175,67 +225,126 @@ int64_t aom_highbd_sse_sse4_1(const uint8_t *a8, int a_stride,
break;
case 16:
do {
highbd_sse_w8_sse4_1(&sum, a, b);
highbd_sse_w8_sse4_1(&sum, a + 8, b + 8);
a += a_stride;
b += b_stride;
y += 1;
int l = 0;
__m128i sum32 = _mm_setzero_si128();
do {
highbd_sse_w8_sse4_1(&sum32, a, b);
highbd_sse_w8_sse4_1(&sum32, a + 8, b + 8);
a += a_stride;
b += b_stride;
l += 1;
} while (l < 64 && l < (height - y));
summary_32_sse4(&sum32, &sum);
y += 64;
} while (y < height);
sse = summary_all_sse4(&sum);
xx_storel_64(&sse, _mm_add_epi64(sum, _mm_srli_si128(sum, 8)));
break;
case 32:
do {
highbd_sse_w8_sse4_1(&sum, a, b);
highbd_sse_w8_sse4_1(&sum, a + 8 * 1, b + 8 * 1);
highbd_sse_w8_sse4_1(&sum, a + 8 * 2, b + 8 * 2);
highbd_sse_w8_sse4_1(&sum, a + 8 * 3, b + 8 * 3);
a += a_stride;
b += b_stride;
y += 1;
int l = 0;
__m128i sum32 = _mm_setzero_si128();
do {
highbd_sse_w8_sse4_1(&sum32, a, b);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 1, b + 8 * 1);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 2, b + 8 * 2);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 3, b + 8 * 3);
a += a_stride;
b += b_stride;
l += 1;
} while (l < 32 && l < (height - y));
summary_32_sse4(&sum32, &sum);
y += 32;
} while (y < height);
sse = summary_all_sse4(&sum);
xx_storel_64(&sse, _mm_add_epi64(sum, _mm_srli_si128(sum, 8)));
break;
case 64:
do {
highbd_sse_w8_sse4_1(&sum, a, b);
highbd_sse_w8_sse4_1(&sum, a + 8 * 1, b + 8 * 1);
highbd_sse_w8_sse4_1(&sum, a + 8 * 2, b + 8 * 2);
highbd_sse_w8_sse4_1(&sum, a + 8 * 3, b + 8 * 3);
highbd_sse_w8_sse4_1(&sum, a + 8 * 4, b + 8 * 4);
highbd_sse_w8_sse4_1(&sum, a + 8 * 5, b + 8 * 5);
highbd_sse_w8_sse4_1(&sum, a + 8 * 6, b + 8 * 6);
highbd_sse_w8_sse4_1(&sum, a + 8 * 7, b + 8 * 7);
a += a_stride;
b += b_stride;
y += 1;
int l = 0;
__m128i sum32 = _mm_setzero_si128();
do {
highbd_sse_w8_sse4_1(&sum32, a, b);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 1, b + 8 * 1);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 2, b + 8 * 2);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 3, b + 8 * 3);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 4, b + 8 * 4);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 5, b + 8 * 5);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 6, b + 8 * 6);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 7, b + 8 * 7);
a += a_stride;
b += b_stride;
l += 1;
} while (l < 16 && l < (height - y));
summary_32_sse4(&sum32, &sum);
y += 16;
} while (y < height);
sse = summary_all_sse4(&sum);
xx_storel_64(&sse, _mm_add_epi64(sum, _mm_srli_si128(sum, 8)));
break;
case 128:
do {
highbd_sse_w8_sse4_1(&sum, a, b);
highbd_sse_w8_sse4_1(&sum, a + 8 * 1, b + 8 * 1);
highbd_sse_w8_sse4_1(&sum, a + 8 * 2, b + 8 * 2);
highbd_sse_w8_sse4_1(&sum, a + 8 * 3, b + 8 * 3);
highbd_sse_w8_sse4_1(&sum, a + 8 * 4, b + 8 * 4);
highbd_sse_w8_sse4_1(&sum, a + 8 * 5, b + 8 * 5);
highbd_sse_w8_sse4_1(&sum, a + 8 * 6, b + 8 * 6);
highbd_sse_w8_sse4_1(&sum, a + 8 * 7, b + 8 * 7);
highbd_sse_w8_sse4_1(&sum, a + 8 * 8, b + 8 * 8);
highbd_sse_w8_sse4_1(&sum, a + 8 * 9, b + 8 * 9);
highbd_sse_w8_sse4_1(&sum, a + 8 * 10, b + 8 * 10);
highbd_sse_w8_sse4_1(&sum, a + 8 * 11, b + 8 * 11);
highbd_sse_w8_sse4_1(&sum, a + 8 * 12, b + 8 * 12);
highbd_sse_w8_sse4_1(&sum, a + 8 * 13, b + 8 * 13);
highbd_sse_w8_sse4_1(&sum, a + 8 * 14, b + 8 * 14);
highbd_sse_w8_sse4_1(&sum, a + 8 * 15, b + 8 * 15);
a += a_stride;
b += b_stride;
y += 1;
int l = 0;
__m128i sum32 = _mm_setzero_si128();
do {
highbd_sse_w8_sse4_1(&sum32, a, b);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 1, b + 8 * 1);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 2, b + 8 * 2);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 3, b + 8 * 3);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 4, b + 8 * 4);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 5, b + 8 * 5);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 6, b + 8 * 6);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 7, b + 8 * 7);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 8, b + 8 * 8);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 9, b + 8 * 9);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 10, b + 8 * 10);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 11, b + 8 * 11);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 12, b + 8 * 12);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 13, b + 8 * 13);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 14, b + 8 * 14);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 15, b + 8 * 15);
a += a_stride;
b += b_stride;
l += 1;
} while (l < 8 && l < (height - y));
summary_32_sse4(&sum32, &sum);
y += 8;
} while (y < height);
sse = summary_all_sse4(&sum);
xx_storel_64(&sse, _mm_add_epi64(sum, _mm_srli_si128(sum, 8)));
break;
default:
if (width & 0x7) {
do {
__m128i sum32 = _mm_setzero_si128();
int i = 0;
do {
highbd_sse_w8_sse4_1(&sum32, a + i, b + i);
highbd_sse_w8_sse4_1(&sum32, a + i + a_stride, b + i + b_stride);
i += 8;
} while (i + 4 < width);
highbd_sse_w4x2_sse4_1(&sum32, a + i, a_stride, b + i, b_stride);
a += (a_stride << 1);
b += (b_stride << 1);
y += 2;
summary_32_sse4(&sum32, &sum);
} while (y < height);
} else {
do {
int l = 0;
__m128i sum32 = _mm_setzero_si128();
do {
int i = 0;
do {
highbd_sse_w8_sse4_1(&sum32, a + i, b + i);
i += 8;
} while (i < width);
a += a_stride;
b += b_stride;
l += 1;
} while (l < 8 && l < (height - y));
summary_32_sse4(&sum32, &sum);
y += 8;
} while (y < height);
}
xx_storel_64(&sse, _mm_add_epi64(sum, _mm_srli_si128(sum, 8)));
break;
default: break;
}
return sse;
}

222
third_party/aom/aom_dsp/x86/ssim_sse2_x86_64.asm vendored Normal file
View file

@ -0,0 +1,222 @@
;
; Copyright (c) 2016, Alliance for Open Media. All rights reserved
;
; This source code is subject to the terms of the BSD 2 Clause License and
; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
; was not distributed with this source code in the LICENSE file, you can
; obtain it at www.aomedia.org/license/software. If the Alliance for Open
; Media Patent License 1.0 was not distributed with this source code in the
; PATENTS file, you can obtain it at www.aomedia.org/license/patent.
;
;
%include "aom_ports/x86_abi_support.asm"
; tabulate_ssim - sums sum_s,sum_r,sum_sq_s,sum_sq_r, sum_sxr
%macro TABULATE_SSIM 0
paddusw xmm15, xmm3 ; sum_s
paddusw xmm14, xmm4 ; sum_r
movdqa xmm1, xmm3
pmaddwd xmm1, xmm1
paddd xmm13, xmm1 ; sum_sq_s
movdqa xmm2, xmm4
pmaddwd xmm2, xmm2
paddd xmm12, xmm2 ; sum_sq_r
pmaddwd xmm3, xmm4
paddd xmm11, xmm3 ; sum_sxr
%endmacro
; Sum across the register %1 starting with q words
%macro SUM_ACROSS_Q 1
movdqa xmm2,%1
punpckldq %1,xmm0
punpckhdq xmm2,xmm0
paddq %1,xmm2
movdqa xmm2,%1
punpcklqdq %1,xmm0
punpckhqdq xmm2,xmm0
paddq %1,xmm2
%endmacro
; Sum across the register %1 starting with q words
%macro SUM_ACROSS_W 1
movdqa xmm1, %1
punpcklwd %1,xmm0
punpckhwd xmm1,xmm0
paddd %1, xmm1
SUM_ACROSS_Q %1
%endmacro
SECTION .text
;void ssim_parms_sse2(
; unsigned char *s,
; int sp,
; unsigned char *r,
; int rp
; uint32_t *sum_s,
; uint32_t *sum_r,
; uint32_t *sum_sq_s,
; uint32_t *sum_sq_r,
; uint32_t *sum_sxr);
;
; TODO: Use parm passing through structure, probably don't need the pxors
; ( calling app will initialize to 0 ) could easily fit everything in sse2
; without too much hastle, and can probably do better estimates with psadw
; or pavgb At this point this is just meant to be first pass for calculating
; all the parms needed for 16x16 ssim so we can play with dssim as distortion
; in mode selection code.
global sym(aom_ssim_parms_16x16_sse2) PRIVATE
sym(aom_ssim_parms_16x16_sse2):
push rbp
mov rbp, rsp
SHADOW_ARGS_TO_STACK 9
SAVE_XMM 15
push rsi
push rdi
; end prolog
mov rsi, arg(0) ;s
mov rcx, arg(1) ;sp
mov rdi, arg(2) ;r
mov rax, arg(3) ;rp
pxor xmm0, xmm0
pxor xmm15,xmm15 ;sum_s
pxor xmm14,xmm14 ;sum_r
pxor xmm13,xmm13 ;sum_sq_s
pxor xmm12,xmm12 ;sum_sq_r
pxor xmm11,xmm11 ;sum_sxr
mov rdx, 16 ;row counter
.NextRow:
;grab source and reference pixels
movdqu xmm5, [rsi]
movdqu xmm6, [rdi]
movdqa xmm3, xmm5
movdqa xmm4, xmm6
punpckhbw xmm3, xmm0 ; high_s
punpckhbw xmm4, xmm0 ; high_r
TABULATE_SSIM
movdqa xmm3, xmm5
movdqa xmm4, xmm6
punpcklbw xmm3, xmm0 ; low_s
punpcklbw xmm4, xmm0 ; low_r
TABULATE_SSIM
add rsi, rcx ; next s row
add rdi, rax ; next r row
dec rdx ; counter
jnz .NextRow
SUM_ACROSS_W xmm15
SUM_ACROSS_W xmm14
SUM_ACROSS_Q xmm13
SUM_ACROSS_Q xmm12
SUM_ACROSS_Q xmm11
mov rdi,arg(4)
movd [rdi], xmm15;
mov rdi,arg(5)
movd [rdi], xmm14;
mov rdi,arg(6)
movd [rdi], xmm13;
mov rdi,arg(7)
movd [rdi], xmm12;
mov rdi,arg(8)
movd [rdi], xmm11;
; begin epilog
pop rdi
pop rsi
RESTORE_XMM
UNSHADOW_ARGS
pop rbp
ret
;void ssim_parms_sse2(
; unsigned char *s,
; int sp,
; unsigned char *r,
; int rp
; uint32_t *sum_s,
; uint32_t *sum_r,
; uint32_t *sum_sq_s,
; uint32_t *sum_sq_r,
; uint32_t *sum_sxr);
;
; TODO: Use parm passing through structure, probably don't need the pxors
; ( calling app will initialize to 0 ) could easily fit everything in sse2
; without too much hastle, and can probably do better estimates with psadw
; or pavgb At this point this is just meant to be first pass for calculating
; all the parms needed for 16x16 ssim so we can play with dssim as distortion
; in mode selection code.
global sym(aom_ssim_parms_8x8_sse2) PRIVATE
sym(aom_ssim_parms_8x8_sse2):
push rbp
mov rbp, rsp
SHADOW_ARGS_TO_STACK 9
SAVE_XMM 15
push rsi
push rdi
; end prolog
mov rsi, arg(0) ;s
mov rcx, arg(1) ;sp
mov rdi, arg(2) ;r
mov rax, arg(3) ;rp
pxor xmm0, xmm0
pxor xmm15,xmm15 ;sum_s
pxor xmm14,xmm14 ;sum_r
pxor xmm13,xmm13 ;sum_sq_s
pxor xmm12,xmm12 ;sum_sq_r
pxor xmm11,xmm11 ;sum_sxr
mov rdx, 8 ;row counter
.NextRow:
;grab source and reference pixels
movq xmm3, [rsi]
movq xmm4, [rdi]
punpcklbw xmm3, xmm0 ; low_s
punpcklbw xmm4, xmm0 ; low_r
TABULATE_SSIM
add rsi, rcx ; next s row
add rdi, rax ; next r row
dec rdx ; counter
jnz .NextRow
SUM_ACROSS_W xmm15
SUM_ACROSS_W xmm14
SUM_ACROSS_Q xmm13
SUM_ACROSS_Q xmm12
SUM_ACROSS_Q xmm11
mov rdi,arg(4)
movd [rdi], xmm15;
mov rdi,arg(5)
movd [rdi], xmm14;
mov rdi,arg(6)
movd [rdi], xmm13;
mov rdi,arg(7)
movd [rdi], xmm12;
mov rdi,arg(8)
movd [rdi], xmm11;
; begin epilog
pop rdi
pop rsi
RESTORE_XMM
UNSHADOW_ARGS
pop rbp
ret

51
third_party/aom/aom_dsp/x86/subpel_variance_sse2.asm vendored
View file

@ -135,44 +135,33 @@ SECTION .text
%if %2 == 1 ; avg
cglobal sub_pixel_avg_variance%1xh, 7, 7, 13, src, src_stride, \
x_offset, y_offset, dst, dst_stride, \
sec, sec_stride, height, sse, \
g_bilin_filter, g_pw_8
sec, sec_stride, height, sse
%define block_height dword heightm
%define sec_str sec_stridemp
;Store bilin_filter and pw_8 location in stack
%if GET_GOT_DEFINED == 1
GET_GOT eax
add esp, 4 ; restore esp
%endif
lea ecx, [GLOBAL(bilin_filter_m)]
mov g_bilin_filterm, ecx
lea ecx, [GLOBAL(pw_8)]
mov g_pw_8m, ecx
LOAD_IF_USED 0, 1 ; load eax, ecx back
%else
cglobal sub_pixel_variance%1xh, 7, 7, 13, src, src_stride, \
x_offset, y_offset, dst, dst_stride, \
height, sse, g_bilin_filter, g_pw_8
height, sse
%define block_height heightd
;Store bilin_filter and pw_8 location in stack
%if GET_GOT_DEFINED == 1
GET_GOT eax
add esp, 4 ; restore esp
%endif
lea ecx, [GLOBAL(bilin_filter_m)]
mov g_bilin_filterm, ecx
lea ecx, [GLOBAL(pw_8)]
mov g_pw_8m, ecx
LOAD_IF_USED 0, 1 ; load eax, ecx back
%endif
; reuse argument stack space
%define g_bilin_filterm x_offsetm
%define g_pw_8m y_offsetm
;Store bilin_filter and pw_8 location in stack
%if GET_GOT_DEFINED == 1
GET_GOT eax
add esp, 4 ; restore esp
%endif
lea ecx, [GLOBAL(bilin_filter_m)]
mov g_bilin_filterm, ecx
lea ecx, [GLOBAL(pw_8)]
mov g_pw_8m, ecx
LOAD_IF_USED 0, 1 ; load eax, ecx back
%else
%if %2 == 1 ; avg
cglobal sub_pixel_avg_variance%1xh, 7, 7, 13, src, src_stride, \

8
third_party/aom/aom_dsp/x86/synonyms.h vendored
View file

@ -13,6 +13,7 @@
#define AOM_AOM_DSP_X86_SYNONYMS_H_
#include <immintrin.h>
#include <string.h>
#include "config/aom_config.h"
@ -28,7 +29,9 @@
// Loads and stores to do away with the tedium of casting the address
// to the right type.
static INLINE __m128i xx_loadl_32(const void *a) {
return _mm_cvtsi32_si128(*(const uint32_t *)a);
int val;
memcpy(&val, a, sizeof(val));
return _mm_cvtsi32_si128(val);
}
static INLINE __m128i xx_loadl_64(const void *a) {
@ -44,7 +47,8 @@ static INLINE __m128i xx_loadu_128(const void *a) {
}
static INLINE void xx_storel_32(void *const a, const __m128i v) {
*(uint32_t *)a = _mm_cvtsi128_si32(v);
const int val = _mm_cvtsi128_si32(v);
memcpy(a, &val, sizeof(val));
}
static INLINE void xx_storel_64(void *const a, const __m128i v) {

5
third_party/aom/aom_dsp/x86/synonyms_avx2.h vendored
View file

@ -67,6 +67,11 @@ static INLINE __m256i yy_loadu2_128(const void *hi, const void *lo) {
return yy_set_m128i(mhi, mlo);
}
static INLINE void yy_storeu2_128(void *hi, void *lo, const __m256i a) {
_mm_storeu_si128((__m128i *)hi, _mm256_extracti128_si256(a, 1));
_mm_storeu_si128((__m128i *)lo, _mm256_castsi256_si128(a));
}
static INLINE __m256i yy_roundn_epu16(__m256i v_val_w, int bits) {
const __m256i v_s_w = _mm256_srli_epi16(v_val_w, bits - 1);
return _mm256_avg_epu16(v_s_w, _mm256_setzero_si256());

50
third_party/aom/aom_dsp/x86/txfm_common_avx2.h vendored
View file

@ -20,9 +20,6 @@
extern "C" {
#endif
typedef void (*transform_1d_avx2)(const __m256i *input, __m256i *output,
int8_t cos_bit);
static INLINE __m256i pair_set_w16_epi16(int16_t a, int16_t b) {
return _mm256_set1_epi32(
(int32_t)(((uint16_t)(a)) | (((uint32_t)(b)) << 16)));
@ -192,6 +189,53 @@ static INLINE void round_shift_16bit_w16_avx2(__m256i *in, int size, int bit) {
}
}
static INLINE __m256i av1_round_shift_32_avx2(__m256i vec, int bit) {
__m256i tmp, round;
round = _mm256_set1_epi32(1 << (bit - 1));
tmp = _mm256_add_epi32(vec, round);
return _mm256_srai_epi32(tmp, bit);
}
static INLINE void av1_round_shift_array_32_avx2(__m256i *input,
__m256i *output,
const int size,
const int bit) {
if (bit > 0) {
int i;
for (i = 0; i < size; i++) {
output[i] = av1_round_shift_32_avx2(input[i], bit);
}
} else {
int i;
for (i = 0; i < size; i++) {
output[i] = _mm256_slli_epi32(input[i], -bit);
}
}
}
static INLINE void av1_round_shift_rect_array_32_avx2(__m256i *input,
__m256i *output,
const int size,
const int bit,
const int val) {
const __m256i sqrt2 = _mm256_set1_epi32(val);
if (bit > 0) {
int i;
for (i = 0; i < size; i++) {
const __m256i r0 = av1_round_shift_32_avx2(input[i], bit);
const __m256i r1 = _mm256_mullo_epi32(sqrt2, r0);
output[i] = av1_round_shift_32_avx2(r1, NewSqrt2Bits);
}
} else {
int i;
for (i = 0; i < size; i++) {
const __m256i r0 = _mm256_slli_epi32(input[i], -bit);
const __m256i r1 = _mm256_mullo_epi32(sqrt2, r0);
output[i] = av1_round_shift_32_avx2(r1, NewSqrt2Bits);
}
}
}
#ifdef __cplusplus
}
#endif

24
third_party/aom/aom_dsp/x86/variance_sse2.c vendored
View file

@ -494,7 +494,7 @@ void aom_upsampled_pred_sse2(MACROBLOCKD *xd, const struct AV1Common *const cm,
const int ref_num = 0;
const int is_intrabc = is_intrabc_block(mi);
const struct scale_factors *const sf =
is_intrabc ? &cm->sf_identity : &xd->block_refs[ref_num]->sf;
is_intrabc ? &cm->sf_identity : xd->block_ref_scale_factors[ref_num];
const int is_scaled = av1_is_scaled(sf);
if (is_scaled) {
@ -571,11 +571,10 @@ void aom_upsampled_pred_sse2(MACROBLOCKD *xd, const struct AV1Common *const cm,
}
}
const InterpFilterParams *filter =
(subpel_search == 1)
? av1_get_4tap_interp_filter_params(EIGHTTAP_REGULAR)
: av1_get_interp_filter_params_with_block_size(EIGHTTAP_REGULAR, 8);
int filter_taps = (subpel_search == 1) ? 4 : SUBPEL_TAPS;
const InterpFilterParams *filter = av1_get_filter(subpel_search);
// (TODO:yunqing) 2-tap case uses 4-tap functions since there is no SIMD for
// 2-tap yet.
int filter_taps = (subpel_search <= USE_4_TAPS) ? 4 : SUBPEL_TAPS;
if (!subpel_x_q3 && !subpel_y_q3) {
if (width >= 16) {
@ -638,20 +637,13 @@ void aom_upsampled_pred_sse2(MACROBLOCKD *xd, const struct AV1Common *const cm,
const int16_t *const kernel_y =
av1_get_interp_filter_subpel_kernel(filter, subpel_y_q3 << 1);
const uint8_t *ref_start = ref - ref_stride * ((filter_taps >> 1) - 1);
uint8_t *temp_start_horiz =
(subpel_search == 1) ? temp + (filter_taps >> 1) * MAX_SB_SIZE : temp;
uint8_t *temp_start_horiz = (subpel_search <= USE_4_TAPS)
? temp + (filter_taps >> 1) * MAX_SB_SIZE
: temp;
uint8_t *temp_start_vert = temp + MAX_SB_SIZE * ((filter->taps >> 1) - 1);
int intermediate_height =
(((height - 1) * 8 + subpel_y_q3) >> 3) + filter_taps;
assert(intermediate_height <= (MAX_SB_SIZE * 2 + 16) + 16);
// TODO(Deepa): Remove the memset below when we have
// 4 tap simd for sse2 and ssse3.
if (subpel_search == 1) {
memset(temp_start_vert - 3 * MAX_SB_SIZE, 0, width);
memset(temp_start_vert - 2 * MAX_SB_SIZE, 0, width);
memset(temp_start_vert + (height + 2) * MAX_SB_SIZE, 0, width);
memset(temp_start_vert + (height + 3) * MAX_SB_SIZE, 0, width);
}
aom_convolve8_horiz(ref_start, ref_stride, temp_start_horiz, MAX_SB_SIZE,
kernel_x, 16, NULL, -1, width, intermediate_height);
aom_convolve8_vert(temp_start_vert, MAX_SB_SIZE, comp_pred, width, NULL, -1,

30
third_party/aom/aom_ports/mem.h vendored
View file

@ -66,4 +66,34 @@
#define CONVERT_TO_SHORTPTR(x) ((uint16_t *)(((uintptr_t)(x)) << 1))
#define CONVERT_TO_BYTEPTR(x) ((uint8_t *)(((uintptr_t)(x)) >> 1))
/*!\brief force enum to be unsigned 1 byte*/
#define UENUM1BYTE(enumvar) \
; \
typedef uint8_t enumvar
/*!\brief force enum to be signed 1 byte*/
#define SENUM1BYTE(enumvar) \
; \
typedef int8_t enumvar
/*!\brief force enum to be unsigned 2 byte*/
#define UENUM2BYTE(enumvar) \
; \
typedef uint16_t enumvar
/*!\brief force enum to be signed 2 byte*/
#define SENUM2BYTE(enumvar) \
; \
typedef int16_t enumvar
/*!\brief force enum to be unsigned 4 byte*/
#define UENUM4BYTE(enumvar) \
; \
typedef uint32_t enumvar
/*!\brief force enum to be unsigned 4 byte*/
#define SENUM4BYTE(enumvar) \
; \
typedef int32_t enumvar
#endif // AOM_AOM_PORTS_MEM_H_

60
third_party/aom/aom_ports/x86.h vendored
View file

@ -222,11 +222,26 @@ static INLINE int x86_simd_caps(void) {
return flags & mask;
}
// Note:
// 32-bit CPU cycle counter is light-weighted for most function performance
// measurement. For large function (CPU time > a couple of seconds), 64-bit
// counter should be used.
// 32-bit CPU cycle counter
// Fine-Grain Measurement Functions
//
// If you are a timing a small region of code, access the timestamp counter
// (TSC) via:
//
// unsigned int start = x86_tsc_start();
// ...
// unsigned int end = x86_tsc_end();
// unsigned int diff = end - start;
//
// The start/end functions introduce a few more instructions than using
// x86_readtsc directly, but prevent the CPU's out-of-order execution from
// affecting the measurement (by having earlier/later instructions be evaluated
// in the time interval). See the white paper, "How to Benchmark Code
// Execution Times on Intel® IA-32 and IA-64 Instruction Set Architectures" by
// Gabriele Paoloni for more information.
//
// If you are timing a large function (CPU time > a couple of seconds), use
// x86_readtsc64 to read the timestamp counter in a 64-bit integer. The
// out-of-order leakage that can occur is minimal compared to total runtime.
static INLINE unsigned int x86_readtsc(void) {
#if defined(__GNUC__) && __GNUC__
unsigned int tsc;
@ -263,6 +278,41 @@ static INLINE uint64_t x86_readtsc64(void) {
#endif
}
// 32-bit CPU cycle counter with a partial fence against out-of-order execution.
static INLINE unsigned int x86_readtscp(void) {
#if defined(__GNUC__) && __GNUC__
unsigned int tscp;
__asm__ __volatile__("rdtscp\n\t" : "=a"(tscp) :);
return tscp;
#elif defined(__SUNPRO_C) || defined(__SUNPRO_CC)
unsigned int tscp;
asm volatile("rdtscp\n\t" : "=a"(tscp) :);
return tscp;
#elif defined(_MSC_VER)
unsigned int ui;
return (unsigned int)__rdtscp(&ui);
#else
#if ARCH_X86_64
return (unsigned int)__rdtscp();
#else
__asm rdtscp;
#endif
#endif
}
static INLINE unsigned int x86_tsc_start(void) {
unsigned int reg_eax, reg_ebx, reg_ecx, reg_edx;
cpuid(0, 0, reg_eax, reg_ebx, reg_ecx, reg_edx);
return x86_readtsc();
}
static INLINE unsigned int x86_tsc_end(void) {
uint32_t v = x86_readtscp();
unsigned int reg_eax, reg_ebx, reg_ecx, reg_edx;
cpuid(0, 0, reg_eax, reg_ebx, reg_ecx, reg_edx);
return v;
}
#if defined(__GNUC__) && __GNUC__
#define x86_pause_hint() __asm__ __volatile__("pause \n\t")
#elif defined(__SUNPRO_C) || defined(__SUNPRO_CC)

11
third_party/aom/aom_scale/aom_scale_rtcd.pl vendored
View file

@ -36,11 +36,12 @@ add_proto qw/void aom_yv12_copy_u/, "const struct yv12_buffer_config *src_bc, st
add_proto qw/void aom_yv12_copy_v/, "const struct yv12_buffer_config *src_bc, struct yv12_buffer_config *dst_bc";
add_proto qw/void aom_yv12_partial_copy_y/, "const struct yv12_buffer_config *src_ybc, struct yv12_buffer_config *dst_ybc, int hstart, int hend, int vstart, int vend";
add_proto qw/void aom_yv12_partial_copy_u/, "const struct yv12_buffer_config *src_bc, struct yv12_buffer_config *dst_bc, int hstart, int hend, int vstart, int vend";
add_proto qw/void aom_yv12_partial_copy_v/, "const struct yv12_buffer_config *src_bc, struct yv12_buffer_config *dst_bc, int hstart, int hend, int vstart, int vend";
add_proto qw/void aom_yv12_partial_copy_y/, "const struct yv12_buffer_config *src_ybc, int hstart1, int hend1, int vstart1, int vend1, struct yv12_buffer_config *dst_ybc, int hstart2, int vstart2";
add_proto qw/void aom_yv12_partial_coloc_copy_y/, "const struct yv12_buffer_config *src_ybc, struct yv12_buffer_config *dst_ybc, int hstart, int hend, int vstart, int vend";
add_proto qw/void aom_yv12_partial_copy_u/, "const struct yv12_buffer_config *src_bc, int hstart1, int hend1, int vstart1, int vend1, struct yv12_buffer_config *dst_bc, int hstart2, int vstart2";
add_proto qw/void aom_yv12_partial_coloc_copy_u/, "const struct yv12_buffer_config *src_bc, struct yv12_buffer_config *dst_bc, int hstart, int hend, int vstart, int vend";
add_proto qw/void aom_yv12_partial_copy_v/, "const struct yv12_buffer_config *src_bc, int hstart1, int hend1, int vstart1, int vend1, struct yv12_buffer_config *dst_bc, int hstart2, int vstart2";
add_proto qw/void aom_yv12_partial_coloc_copy_v/, "const struct yv12_buffer_config *src_bc, struct yv12_buffer_config *dst_bc, int hstart, int hend, int vstart, int vend";
add_proto qw/void aom_extend_frame_borders/, "struct yv12_buffer_config *ybf, const int num_planes";
specialize qw/aom_extend_frame_borders dspr2/;

17
third_party/aom/aom_scale/generic/yv12config.c vendored
View file

@ -55,6 +55,13 @@ int aom_realloc_frame_buffer(YV12_BUFFER_CONFIG *ybf, int width, int height,
if (width > DECODE_WIDTH_LIMIT || height > DECODE_HEIGHT_LIMIT) return -1;
#endif
/* Only support allocating buffers that have a border that's a multiple
* of 32. The border restriction is required to get 16-byte alignment of
* the start of the chroma rows without introducing an arbitrary gap
* between planes, which would break the semantics of things like
* aom_img_set_rect(). */
if (border & 0x1f) return -3;
if (ybf) {
const int aom_byte_align = (byte_alignment == 0) ? 1 : byte_alignment;
const int aligned_width = (width + 7) & ~7;
@ -109,10 +116,11 @@ int aom_realloc_frame_buffer(YV12_BUFFER_CONFIG *ybf, int width, int height,
memset(ybf->buffer_alloc, 0, (size_t)frame_size);
#endif
#endif
} else if (frame_size > (size_t)ybf->buffer_alloc_sz) {
} else if (frame_size > ybf->buffer_alloc_sz) {
// Allocation to hold larger frame, or first allocation.
aom_free(ybf->buffer_alloc);
ybf->buffer_alloc = NULL;
ybf->buffer_alloc_sz = 0;
if (frame_size != (size_t)frame_size) return -1;
@ -127,13 +135,6 @@ int aom_realloc_frame_buffer(YV12_BUFFER_CONFIG *ybf, int width, int height,
memset(ybf->buffer_alloc, 0, ybf->buffer_alloc_sz);
}
/* Only support allocating buffers that have a border that's a multiple
* of 32. The border restriction is required to get 16-byte alignment of
* the start of the chroma rows without introducing an arbitrary gap
* between planes, which would break the semantics of things like
* aom_img_set_rect(). */
if (border & 0x1f) return -3;
ybf->y_crop_width = width;
ybf->y_crop_height = height;
ybf->y_width = aligned_width;

91
third_party/aom/aom_scale/generic/yv12extend.c vendored
View file

@ -321,91 +321,116 @@ void aom_yv12_copy_v_c(const YV12_BUFFER_CONFIG *src_bc,
}
}
void aom_yv12_partial_copy_y_c(const YV12_BUFFER_CONFIG *src_ybc,
YV12_BUFFER_CONFIG *dst_ybc, int hstart,
int hend, int vstart, int vend) {
void aom_yv12_partial_copy_y_c(const YV12_BUFFER_CONFIG *src_ybc, int hstart1,
int hend1, int vstart1, int vend1,
YV12_BUFFER_CONFIG *dst_ybc, int hstart2,
int vstart2) {
int row;
const uint8_t *src = src_ybc->y_buffer;
uint8_t *dst = dst_ybc->y_buffer;
if (src_ybc->flags & YV12_FLAG_HIGHBITDEPTH) {
const uint16_t *src16 =
CONVERT_TO_SHORTPTR(src + vstart * src_ybc->y_stride + hstart);
CONVERT_TO_SHORTPTR(src + vstart1 * src_ybc->y_stride + hstart1);
uint16_t *dst16 =
CONVERT_TO_SHORTPTR(dst + vstart * dst_ybc->y_stride + hstart);
for (row = vstart; row < vend; ++row) {
memcpy(dst16, src16, (hend - hstart) * sizeof(uint16_t));
CONVERT_TO_SHORTPTR(dst + vstart2 * dst_ybc->y_stride + hstart2);
for (row = vstart1; row < vend1; ++row) {
memcpy(dst16, src16, (hend1 - hstart1) * sizeof(uint16_t));
src16 += src_ybc->y_stride;
dst16 += dst_ybc->y_stride;
}
return;
}
src = (src + vstart * src_ybc->y_stride + hstart);
dst = (dst + vstart * dst_ybc->y_stride + hstart);
src = (src + vstart1 * src_ybc->y_stride + hstart1);
dst = (dst + vstart2 * dst_ybc->y_stride + hstart2);
for (row = vstart; row < vend; ++row) {
memcpy(dst, src, (hend - hstart));
for (row = vstart1; row < vend1; ++row) {
memcpy(dst, src, (hend1 - hstart1));
src += src_ybc->y_stride;
dst += dst_ybc->y_stride;
}
}
void aom_yv12_partial_copy_u_c(const YV12_BUFFER_CONFIG *src_bc,
YV12_BUFFER_CONFIG *dst_bc, int hstart, int hend,
int vstart, int vend) {
void aom_yv12_partial_coloc_copy_y_c(const YV12_BUFFER_CONFIG *src_ybc,
YV12_BUFFER_CONFIG *dst_ybc, int hstart,
int hend, int vstart, int vend) {
aom_yv12_partial_copy_y_c(src_ybc, hstart, hend, vstart, vend, dst_ybc,
hstart, vstart);
}
void aom_yv12_partial_copy_u_c(const YV12_BUFFER_CONFIG *src_bc, int hstart1,
int hend1, int vstart1, int vend1,
YV12_BUFFER_CONFIG *dst_bc, int hstart2,
int vstart2) {
int row;
const uint8_t *src = src_bc->u_buffer;
uint8_t *dst = dst_bc->u_buffer;
if (src_bc->flags & YV12_FLAG_HIGHBITDEPTH) {
const uint16_t *src16 =
CONVERT_TO_SHORTPTR(src + vstart * src_bc->uv_stride + hstart);
CONVERT_TO_SHORTPTR(src + vstart1 * src_bc->uv_stride + hstart1);
uint16_t *dst16 =
CONVERT_TO_SHORTPTR(dst + vstart * dst_bc->uv_stride + hstart);
for (row = vstart; row < vend; ++row) {
memcpy(dst16, src16, (hend - hstart) * sizeof(uint16_t));
CONVERT_TO_SHORTPTR(dst + vstart2 * dst_bc->uv_stride + hstart2);
for (row = vstart1; row < vend1; ++row) {
memcpy(dst16, src16, (hend1 - hstart1) * sizeof(uint16_t));
src16 += src_bc->uv_stride;
dst16 += dst_bc->uv_stride;
}
return;
}
src = (src + vstart * src_bc->uv_stride + hstart);
dst = (dst + vstart * dst_bc->uv_stride + hstart);
src = (src + vstart1 * src_bc->uv_stride + hstart1);
dst = (dst + vstart2 * dst_bc->uv_stride + hstart2);
for (row = vstart; row < vend; ++row) {
memcpy(dst, src, (hend - hstart));
for (row = vstart1; row < vend1; ++row) {
memcpy(dst, src, (hend1 - hstart1));
src += src_bc->uv_stride;
dst += dst_bc->uv_stride;
}
}
void aom_yv12_partial_copy_v_c(const YV12_BUFFER_CONFIG *src_bc,
YV12_BUFFER_CONFIG *dst_bc, int hstart, int hend,
int vstart, int vend) {
void aom_yv12_partial_coloc_copy_u_c(const YV12_BUFFER_CONFIG *src_bc,
YV12_BUFFER_CONFIG *dst_bc, int hstart,
int hend, int vstart, int vend) {
aom_yv12_partial_copy_u_c(src_bc, hstart, hend, vstart, vend, dst_bc, hstart,
vstart);
}
void aom_yv12_partial_copy_v_c(const YV12_BUFFER_CONFIG *src_bc, int hstart1,
int hend1, int vstart1, int vend1,
YV12_BUFFER_CONFIG *dst_bc, int hstart2,
int vstart2) {
int row;
const uint8_t *src = src_bc->v_buffer;
uint8_t *dst = dst_bc->v_buffer;
if (src_bc->flags & YV12_FLAG_HIGHBITDEPTH) {
const uint16_t *src16 =
CONVERT_TO_SHORTPTR(src + vstart * src_bc->uv_stride + hstart);
CONVERT_TO_SHORTPTR(src + vstart1 * src_bc->uv_stride + hstart1);
uint16_t *dst16 =
CONVERT_TO_SHORTPTR(dst + vstart * dst_bc->uv_stride + hstart);
for (row = vstart; row < vend; ++row) {
memcpy(dst16, src16, (hend - hstart) * sizeof(uint16_t));
CONVERT_TO_SHORTPTR(dst + vstart2 * dst_bc->uv_stride + hstart2);
for (row = vstart1; row < vend1; ++row) {
memcpy(dst16, src16, (hend1 - hstart1) * sizeof(uint16_t));
src16 += src_bc->uv_stride;
dst16 += dst_bc->uv_stride;
}
return;
}
src = (src + vstart * src_bc->uv_stride + hstart);
dst = (dst + vstart * dst_bc->uv_stride + hstart);
src = (src + vstart1 * src_bc->uv_stride + hstart1);
dst = (dst + vstart2 * dst_bc->uv_stride + hstart2);
for (row = vstart; row < vend; ++row) {
memcpy(dst, src, (hend - hstart));
for (row = vstart1; row < vend1; ++row) {
memcpy(dst, src, (hend1 - hstart1));
src += src_bc->uv_stride;
dst += dst_bc->uv_stride;
}
}
void aom_yv12_partial_coloc_copy_v_c(const YV12_BUFFER_CONFIG *src_bc,
YV12_BUFFER_CONFIG *dst_bc, int hstart,
int hend, int vstart, int vend) {
aom_yv12_partial_copy_v_c(src_bc, hstart, hend, vstart, vend, dst_bc, hstart,
vstart);
}

24
third_party/aom/aom_scale/yv12config.h vendored
View file

@ -24,32 +24,24 @@ extern "C" {
#define AOMINNERBORDERINPIXELS 160
#define AOM_INTERP_EXTEND 4
// TODO(jingning): Use unified inter predictor for encoder and
// decoder during the development process. Revisit the frame border
// to improve the decoder performance.
#if CONFIG_REDUCED_ENCODER_BORDER
#define AOM_BORDER_IN_PIXELS 160
#else
#define AOM_BORDER_IN_PIXELS 288
#endif // CONFIG_REDUCED_ENCODER_BORDER
#define AOM_ENC_NO_SCALE_BORDER 160
#define AOM_DEC_BORDER_IN_PIXELS 64
typedef struct yv12_buffer_config {
union {
struct {
int y_width;
int uv_width;
int alpha_width;
};
int widths[3];
int widths[2];
};
union {
struct {
int y_height;
int uv_height;
int alpha_height;
};
int heights[3];
int heights[2];
};
union {
struct {
@ -69,18 +61,16 @@ typedef struct yv12_buffer_config {
struct {
int y_stride;
int uv_stride;
int alpha_stride;
};
int strides[3];
int strides[2];
};
union {
struct {
uint8_t *y_buffer;
uint8_t *u_buffer;
uint8_t *v_buffer;
uint8_t *alpha_buffer;
};
uint8_t *buffers[4];
uint8_t *buffers[3];
};
// Indicate whether y_buffer, u_buffer, and v_buffer points to the internally
@ -106,7 +96,7 @@ typedef struct yv12_buffer_config {
aom_color_primaries_t color_primaries;
aom_transfer_characteristics_t transfer_characteristics;
aom_matrix_coefficients_t matrix_coefficients;
int monochrome;
uint8_t monochrome;
aom_chroma_sample_position_t chroma_sample_position;
aom_color_range_t color_range;
int render_width;

28
third_party/aom/aom_util/aom_thread.c vendored
View file

@ -14,6 +14,12 @@
// Original source:
// https://chromium.googlesource.com/webm/libwebp
// Enable GNU extensions in glibc so that we can call pthread_setname_np().
// This must be before any #include statements.
#ifndef _GNU_SOURCE
#define _GNU_SOURCE
#endif
#include <assert.h>
#include <string.h> // for memset()
@ -34,6 +40,28 @@ static void execute(AVxWorker *const worker); // Forward declaration.
static THREADFN thread_loop(void *ptr) {
AVxWorker *const worker = (AVxWorker *)ptr;
#ifdef __APPLE__
if (worker->thread_name != NULL) {
// Apple's version of pthread_setname_np takes one argument and operates on
// the current thread only. The maximum size of the thread_name buffer was
// noted in the Chromium source code and was confirmed by experiments. If
// thread_name is too long, pthread_setname_np returns -1 with errno
// ENAMETOOLONG (63).
char thread_name[64];
strncpy(thread_name, worker->thread_name, sizeof(thread_name));
thread_name[sizeof(thread_name) - 1] = '\0';
pthread_setname_np(thread_name);
}
#elif defined(__GLIBC__) || defined(__BIONIC__)
if (worker->thread_name != NULL) {
// Linux and Android require names (with nul) fit in 16 chars, otherwise
// pthread_setname_np() returns ERANGE (34).
char thread_name[16];
strncpy(thread_name, worker->thread_name, sizeof(thread_name));
thread_name[sizeof(thread_name) - 1] = '\0';
pthread_setname_np(pthread_self(), thread_name);
}
#endif
int done = 0;
while (!done) {
pthread_mutex_lock(&worker->impl_->mutex_);

5
third_party/aom/aom_util/aom_thread.h vendored
View file

@ -26,6 +26,7 @@ extern "C" {
// Set maximum decode threads to be 8 due to the limit of frame buffers
// and not enough semaphores in the emulation layer on windows.
#define MAX_DECODE_THREADS 8
#define MAX_NUM_THREADS 64
#if CONFIG_MULTITHREAD
@ -380,6 +381,10 @@ typedef struct AVxWorkerImpl AVxWorkerImpl;
typedef struct {
AVxWorkerImpl *impl_;
AVxWorkerStatus status_;
// Thread name for the debugger. If not NULL, must point to a string that
// outlives the worker thread. For portability, use a name <= 15 characters
// long (not including the terminating NUL character).
const char *thread_name;
AVxWorkerHook hook; // hook to call
void *data1; // first argument passed to 'hook'
void *data2; // second argument passed to 'hook'

67
third_party/aom/apps/aomdec.c vendored
View file

@ -143,7 +143,7 @@ static INLINE int libyuv_scale(aom_image_t *src, aom_image_t *dst,
}
#endif
void show_help(FILE *fout, int shorthelp) {
static void show_help(FILE *fout, int shorthelp) {
fprintf(fout, "Usage: %s <options> filename\n\n", exec_name);
if (shorthelp) {
@ -427,13 +427,6 @@ static FILE *open_outfile(const char *name) {
}
}
static int img_shifted_realloc_required(const aom_image_t *img,
const aom_image_t *shifted,
aom_img_fmt_t required_fmt) {
return img->d_w != shifted->d_w || img->d_h != shifted->d_h ||
required_fmt != shifted->fmt;
}
static int main_loop(int argc, const char **argv_) {
aom_codec_ctx_t decoder;
char *fn = NULL;
@ -459,7 +452,7 @@ static int main_loop(int argc, const char **argv_) {
int opt_i420 = 0;
int opt_raw = 0;
aom_codec_dec_cfg_t cfg = { 0, 0, 0, CONFIG_LOWBITDEPTH, { 1 } };
unsigned int output_bit_depth = 0;
unsigned int fixed_output_bit_depth = 0;
unsigned int is_annexb = 0;
int frames_corrupted = 0;
int dec_flags = 0;
@ -491,6 +484,7 @@ static int main_loop(int argc, const char **argv_) {
input.webm_ctx = &webm_ctx;
#endif
struct ObuDecInputContext obu_ctx = { NULL, NULL, 0, 0, 0 };
int is_ivf = 0;
obu_ctx.avx_ctx = &aom_input_ctx;
input.obu_ctx = &obu_ctx;
@ -572,7 +566,7 @@ static int main_loop(int argc, const char **argv_) {
} else if (arg_match(&arg, &continuearg, argi)) {
keep_going = 1;
} else if (arg_match(&arg, &outbitdeptharg, argi)) {
output_bit_depth = arg_parse_uint(&arg);
fixed_output_bit_depth = arg_parse_uint(&arg);
} else if (arg_match(&arg, &isannexb, argi)) {
is_annexb = 1;
input.obu_ctx->is_annexb = 1;
@ -617,8 +611,10 @@ static int main_loop(int argc, const char **argv_) {
#endif
input.aom_input_ctx->filename = fn;
input.aom_input_ctx->file = infile;
if (file_is_ivf(input.aom_input_ctx))
if (file_is_ivf(input.aom_input_ctx)) {
input.aom_input_ctx->file_type = FILE_TYPE_IVF;
is_ivf = 1;
}
#if CONFIG_WEBM_IO
else if (file_is_webm(input.webm_ctx, input.aom_input_ctx))
input.aom_input_ctx->file_type = FILE_TYPE_WEBM;
@ -668,6 +664,10 @@ static int main_loop(int argc, const char **argv_) {
}
fourcc_interface = get_aom_decoder_by_fourcc(aom_input_ctx.fourcc);
if (is_ivf && !fourcc_interface)
fatal("Unsupported fourcc: %x\n", aom_input_ctx.fourcc);
if (interface && fourcc_interface && interface != fourcc_interface)
warn("Header indicates codec: %s\n", fourcc_interface->name);
else
@ -779,7 +779,7 @@ static int main_loop(int argc, const char **argv_) {
aom_usec_timer_start(&timer);
if (flush_decoder) {
// Flush the decoder in frame parallel decode.
// Flush the decoder.
if (aom_codec_decode(&decoder, NULL, 0, NULL)) {
warn("Failed to flush decoder: %s", aom_codec_error(&decoder));
}
@ -850,41 +850,15 @@ static int main_loop(int argc, const char **argv_) {
}
}
// Default to codec bit depth if output bit depth not set
if (!output_bit_depth && single_file && !do_md5) {
unsigned int output_bit_depth;
if (!fixed_output_bit_depth && single_file && !do_md5) {
output_bit_depth = img->bit_depth;
} else {
output_bit_depth = fixed_output_bit_depth;
}
// Shift up or down if necessary
if (output_bit_depth != 0) {
const aom_img_fmt_t shifted_fmt =
output_bit_depth == 8 ? img->fmt & ~AOM_IMG_FMT_HIGHBITDEPTH
: img->fmt | AOM_IMG_FMT_HIGHBITDEPTH;
if (shifted_fmt != img->fmt || output_bit_depth != img->bit_depth) {
if (img_shifted &&
img_shifted_realloc_required(img, img_shifted, shifted_fmt)) {
aom_img_free(img_shifted);
img_shifted = NULL;
}
if (img_shifted) {
img_shifted->monochrome = img->monochrome;
}
if (!img_shifted) {
img_shifted =
aom_img_alloc(NULL, shifted_fmt, img->d_w, img->d_h, 16);
img_shifted->bit_depth = output_bit_depth;
img_shifted->monochrome = img->monochrome;
img_shifted->csp = img->csp;
}
if (output_bit_depth > img->bit_depth) {
aom_img_upshift(img_shifted, img,
output_bit_depth - img->bit_depth);
} else {
aom_img_downshift(img_shifted, img,
img->bit_depth - output_bit_depth);
}
img = img_shifted;
}
}
if (output_bit_depth != 0)
aom_shift_img(output_bit_depth, &img, &img_shifted);
aom_input_ctx.width = img->d_w;
aom_input_ctx.height = img->d_h;
@ -900,6 +874,11 @@ static int main_loop(int argc, const char **argv_) {
y4m_buf, sizeof(y4m_buf), aom_input_ctx.width,
aom_input_ctx.height, &aom_input_ctx.framerate,
img->monochrome, img->csp, img->fmt, img->bit_depth);
if (img->csp == AOM_CSP_COLOCATED) {
fprintf(stderr,
"Warning: Y4M lacks a colorspace for colocated "
"chroma. Using a placeholder.\n");
}
if (do_md5) {
MD5Update(&md5_ctx, (md5byte *)y4m_buf, (unsigned int)len);
} else {

258
third_party/aom/apps/aomenc.c vendored
View file

@ -263,9 +263,9 @@ static const arg_def_t global_error_resilient =
"Enable global error resiliency features");
static const arg_def_t lag_in_frames =
ARG_DEF(NULL, "lag-in-frames", 1, "Max number of frames to lag");
static const arg_def_t large_scale_tile =
ARG_DEF(NULL, "large-scale-tile", 1,
"Large scale tile coding (0: off (default), 1: on)");
static const arg_def_t large_scale_tile = ARG_DEF(
NULL, "large-scale-tile", 1,
"Large scale tile coding (0: off (default), 1: on (ivf output only))");
static const arg_def_t monochrome =
ARG_DEF(NULL, "monochrome", 0, "Monochrome video (no chroma planes)");
static const arg_def_t full_still_picture_hdr = ARG_DEF(
@ -415,11 +415,14 @@ static const arg_def_t cpu_used_av1 =
ARG_DEF(NULL, "cpu-used", 1, "CPU Used (0..8)");
static const arg_def_t rowmtarg =
ARG_DEF(NULL, "row-mt", 1,
"Enable row based multi-threading (0: off (default), 1: on)");
"Enable row based multi-threading (0: off, 1: on (default))");
static const arg_def_t tile_cols =
ARG_DEF(NULL, "tile-columns", 1, "Number of tile columns to use, log2");
static const arg_def_t tile_rows =
ARG_DEF(NULL, "tile-rows", 1, "Number of tile rows to use, log2");
static const arg_def_t enable_tpl_model =
ARG_DEF(NULL, "enable-tpl-model", 1,
"RDO modulation based on frame temporal dependency");
static const arg_def_t tile_width =
ARG_DEF(NULL, "tile-width", 1, "Tile widths (comma separated)");
static const arg_def_t tile_height =
@ -434,6 +437,88 @@ static const arg_def_t enable_restoration =
ARG_DEF(NULL, "enable-restoration", 1,
"Enable the loop restoration filter (0: false, "
"1: true (default))");
static const arg_def_t enable_rect_partitions =
ARG_DEF(NULL, "enable-rect-partitions", 1,
"Enable rectangular partitions "
"(0: false, 1: true (default))");
static const arg_def_t enable_dual_filter =
ARG_DEF(NULL, "enable-dual-filter", 1,
"Enable dual filter "
"(0: false, 1: true (default))");
static const arg_def_t enable_intra_edge_filter =
ARG_DEF(NULL, "enable-intra-edge-filter", 1,
"Enable intra edge filtering "
"(0: false, 1: true (default))");
static const arg_def_t enable_order_hint =
ARG_DEF(NULL, "enable-order-hint", 1,
"Enable order hint "
"(0: false, 1: true (default))");
static const arg_def_t enable_tx64 =
ARG_DEF(NULL, "enable-tx64", 1,
"Enable 64-pt transform (0: false, 1: true (default))");
static const arg_def_t enable_dist_wtd_comp =
ARG_DEF(NULL, "enable-dist-wtd-comp", 1,
"Enable distance-weighted compound "
"(0: false, 1: true (default))");
static const arg_def_t enable_masked_comp =
ARG_DEF(NULL, "enable-masked-comp", 1,
"Enable masked (wedge/diff-wtd) compound "
"(0: false, 1: true (default))");
static const arg_def_t enable_interintra_comp =
ARG_DEF(NULL, "enable-interintra-comp", 1,
"Enable interintra compound "
"(0: false, 1: true (default))");
static const arg_def_t enable_smooth_interintra =
ARG_DEF(NULL, "enable-smooth-interintra", 1,
"Enable smooth interintra mode "
"(0: false, 1: true (default))");
static const arg_def_t enable_diff_wtd_comp =
ARG_DEF(NULL, "enable-diff-wtd-comp", 1,
"Enable difference-weighted compound "
"(0: false, 1: true (default))");
static const arg_def_t enable_interinter_wedge =
ARG_DEF(NULL, "enable-interinter-wedge", 1,
"Enable interinter wedge compound "
"(0: false, 1: true (default))");
static const arg_def_t enable_interintra_wedge =
ARG_DEF(NULL, "enable-interintra-wedge", 1,
"Enable interintra wedge compound "
"(0: false, 1: true (default))");
static const arg_def_t enable_global_motion =
ARG_DEF(NULL, "enable-global-motion", 1,
"Enable global motion "
"(0: false, 1: true (default))");
static const arg_def_t enable_warped_motion =
ARG_DEF(NULL, "enable-warped-motion", 1,
"Enable local warped motion "
"(0: false, 1: true (default))");
static const arg_def_t enable_filter_intra =
ARG_DEF(NULL, "enable-filter-intra", 1,
"Enable filter intra prediction mode "
"(0: false, 1: true (default))");
static const arg_def_t enable_smooth_intra =
ARG_DEF(NULL, "enable-smooth-intra", 1,
"Enable smooth intra prediction modes "
"(0: false, 1: true (default))");
static const arg_def_t enable_paeth_intra =
ARG_DEF(NULL, "enable-paeth-intra", 1,
"Enable Paeth intra prediction mode (0: false, 1: true (default))");
static const arg_def_t enable_cfl_intra =
ARG_DEF(NULL, "enable-cfl-intra", 1,
"Enable chroma from luma intra prediction mode "
"(0: false, 1: true (default))");
static const arg_def_t enable_obmc = ARG_DEF(
NULL, "enable-obmc", 1, "Enable OBMC (0: false, 1: true (default))");
static const arg_def_t enable_palette =
ARG_DEF(NULL, "enable-palette", 1,
"Enable palette prediction mode (0: false, 1: true (default))");
static const arg_def_t enable_intrabc =
ARG_DEF(NULL, "enable-intrabc", 1,
"Enable intra block copy prediction mode "
"(0: false, 1: true (default))");
static const arg_def_t enable_angle_delta =
ARG_DEF(NULL, "enable-angle-delta", 1,
"Enable intra angle delta (0: false, 1: true (default))");
static const arg_def_t disable_trellis_quant =
ARG_DEF(NULL, "disable-trellis-quant", 1,
"Disable trellis optimization of quantized coefficients (0: false ("
@ -445,6 +530,14 @@ static const arg_def_t qm_min = ARG_DEF(
NULL, "qm-min", 1, "Min quant matrix flatness (0..15), default is 8");
static const arg_def_t qm_max = ARG_DEF(
NULL, "qm-max", 1, "Max quant matrix flatness (0..15), default is 15");
static const arg_def_t reduced_tx_type_set = ARG_DEF(
NULL, "reduced-tx-type-set", 1, "Use reduced set of transform types");
static const arg_def_t use_intra_dct_only =
ARG_DEF(NULL, "use-intra-dct-only", 1, "Use DCT only for INTRA modes");
static const arg_def_t use_inter_dct_only =
ARG_DEF(NULL, "use-inter-dct-only", 1, "Use DCT only for INTER modes");
static const arg_def_t quant_b_adapt =
ARG_DEF(NULL, "quant-b-adapt", 1, "Use adaptive quantize_b");
#if CONFIG_DIST_8X8
static const arg_def_t enable_dist_8x8 =
ARG_DEF(NULL, "enable-dist-8x8", 1,
@ -512,6 +605,12 @@ static const arg_def_t min_gf_interval = ARG_DEF(
static const arg_def_t max_gf_interval = ARG_DEF(
NULL, "max-gf-interval", 1,
"max gf/arf frame interval (default 0, indicating in-built behavior)");
static const arg_def_t gf_max_pyr_height =
ARG_DEF(NULL, "gf-max-pyr-height", 1,
"maximum height for GF group pyramid structure (1 to 4 (default))");
static const arg_def_t max_reference_frames = ARG_DEF(
NULL, "max-reference-frames", 1,
"maximum number of reference frames allowed per frame (3 to 7 (default))");
static const struct arg_enum_list color_primaries_enum[] = {
{ "bt709", AOM_CICP_CP_BT_709 },
@ -624,6 +723,7 @@ static const arg_def_t *av1_args[] = { &cpu_used_av1,
&rowmtarg,
&tile_cols,
&tile_rows,
&enable_tpl_model,
&arnr_maxframes,
&arnr_strength,
&tune_metric,
@ -634,10 +734,36 @@ static const arg_def_t *av1_args[] = { &cpu_used_av1,
&lossless,
&enable_cdef,
&enable_restoration,
&enable_rect_partitions,
&enable_dual_filter,
&enable_intra_edge_filter,
&enable_order_hint,
&enable_tx64,
&enable_dist_wtd_comp,
&enable_masked_comp,
&enable_interintra_comp,
&enable_smooth_interintra,
&enable_diff_wtd_comp,
&enable_interinter_wedge,
&enable_interintra_wedge,
&enable_global_motion,
&enable_warped_motion,
&enable_filter_intra,
&enable_smooth_intra,
&enable_paeth_intra,
&enable_cfl_intra,
&enable_obmc,
&enable_palette,
&enable_intrabc,
&enable_angle_delta,
&disable_trellis_quant,
&enable_qm,
&qm_min,
&qm_max,
&reduced_tx_type_set,
&use_intra_dct_only,
&use_inter_dct_only,
&quant_b_adapt,
#if CONFIG_DIST_8X8
&enable_dist_8x8,
#endif
@ -655,6 +781,7 @@ static const arg_def_t *av1_args[] = { &cpu_used_av1,
&input_chroma_sample_position,
&min_gf_interval,
&max_gf_interval,
&gf_max_pyr_height,
&superblock_size,
&num_tg,
&mtu_size,
@ -664,7 +791,8 @@ static const arg_def_t *av1_args[] = { &cpu_used_av1,
#if CONFIG_DENOISE
&denoise_noise_level,
&denoise_block_size,
#endif
#endif // CONFIG_DENOISE
&max_reference_frames,
&enable_ref_frame_mvs,
&bitdeptharg,
&inbitdeptharg,
@ -681,6 +809,7 @@ static const int av1_arg_ctrl_map[] = { AOME_SET_CPUUSED,
AV1E_SET_ROW_MT,
AV1E_SET_TILE_COLUMNS,
AV1E_SET_TILE_ROWS,
AV1E_SET_ENABLE_TPL_MODEL,
AOME_SET_ARNR_MAXFRAMES,
AOME_SET_ARNR_STRENGTH,
AOME_SET_TUNING,
@ -691,10 +820,36 @@ static const int av1_arg_ctrl_map[] = { AOME_SET_CPUUSED,
AV1E_SET_LOSSLESS,
AV1E_SET_ENABLE_CDEF,
AV1E_SET_ENABLE_RESTORATION,
AV1E_SET_ENABLE_RECT_PARTITIONS,
AV1E_SET_ENABLE_DUAL_FILTER,
AV1E_SET_ENABLE_INTRA_EDGE_FILTER,
AV1E_SET_ENABLE_ORDER_HINT,
AV1E_SET_ENABLE_TX64,
AV1E_SET_ENABLE_DIST_WTD_COMP,
AV1E_SET_ENABLE_MASKED_COMP,
AV1E_SET_ENABLE_INTERINTRA_COMP,
AV1E_SET_ENABLE_SMOOTH_INTERINTRA,
AV1E_SET_ENABLE_DIFF_WTD_COMP,
AV1E_SET_ENABLE_INTERINTER_WEDGE,
AV1E_SET_ENABLE_INTERINTRA_WEDGE,
AV1E_SET_ENABLE_GLOBAL_MOTION,
AV1E_SET_ENABLE_WARPED_MOTION,
AV1E_SET_ENABLE_FILTER_INTRA,
AV1E_SET_ENABLE_SMOOTH_INTRA,
AV1E_SET_ENABLE_PAETH_INTRA,
AV1E_SET_ENABLE_CFL_INTRA,
AV1E_SET_ENABLE_OBMC,
AV1E_SET_ENABLE_PALETTE,
AV1E_SET_ENABLE_INTRABC,
AV1E_SET_ENABLE_ANGLE_DELTA,
AV1E_SET_DISABLE_TRELLIS_QUANT,
AV1E_SET_ENABLE_QM,
AV1E_SET_QM_MIN,
AV1E_SET_QM_MAX,
AV1E_SET_REDUCED_TX_TYPE_SET,
AV1E_SET_INTRA_DCT_ONLY,
AV1E_SET_INTER_DCT_ONLY,
AV1E_SET_QUANT_B_ADAPT,
#if CONFIG_DIST_8X8
AV1E_SET_ENABLE_DIST_8X8,
#endif
@ -712,6 +867,7 @@ static const int av1_arg_ctrl_map[] = { AOME_SET_CPUUSED,
AV1E_SET_CHROMA_SAMPLE_POSITION,
AV1E_SET_MIN_GF_INTERVAL,
AV1E_SET_MAX_GF_INTERVAL,
AV1E_SET_GF_MAX_PYRAMID_HEIGHT,
AV1E_SET_SUPERBLOCK_SIZE,
AV1E_SET_NUM_TG,
AV1E_SET_MTU,
@ -721,18 +877,15 @@ static const int av1_arg_ctrl_map[] = { AOME_SET_CPUUSED,
#if CONFIG_DENOISE
AV1E_SET_DENOISE_NOISE_LEVEL,
AV1E_SET_DENOISE_BLOCK_SIZE,
#endif
#endif // CONFIG_DENOISE
AV1E_SET_MAX_REFERENCE_FRAMES,
AV1E_SET_ENABLE_REF_FRAME_MVS,
AV1E_SET_ENABLE_DF,
AV1E_SET_ENABLE_ORDER_HINT,
AV1E_SET_ENABLE_JNT_COMP,
AV1E_SET_ENABLE_SUPERRES,
0 };
#endif // CONFIG_AV1_ENCODER
static const arg_def_t *no_args[] = { NULL };
void show_help(FILE *fout, int shorthelp) {
static void show_help(FILE *fout, int shorthelp) {
fprintf(fout, "Usage: %s <options> -o dst_filename src_filename \n",
exec_name);
@ -845,14 +998,17 @@ static void validate_positive_rational(const char *msg,
if (!rat->den) die("Error: %s has zero denominator\n", msg);
}
static void parse_global_config(struct AvxEncoderConfig *global, int *argc,
/* Parses global config arguments into the AvxEncoderConfig. Note that
* argv is modified and overwrites all parsed arguments.
*/
static void parse_global_config(struct AvxEncoderConfig *global, int argc,
char ***argv) {
char **argi, **argj;
struct arg arg;
const int num_encoder = get_aom_encoder_count();
char **argv_local = (char **)*argv;
#if CONFIG_FILEOPTIONS
int argc_local = *argc;
int argc_local = argc;
#endif
if (num_encoder < 1) die("Error: no valid encoder available\n");
@ -861,6 +1017,7 @@ static void parse_global_config(struct AvxEncoderConfig *global, int *argc,
global->codec = get_aom_encoder_by_index(num_encoder - 1);
global->passes = 0;
global->color_type = I420;
global->csp = AOM_CSP_UNKNOWN;
#if CONFIG_FILEOPTIONS
const char *cfg = NULL;
@ -900,6 +1057,10 @@ static void parse_global_config(struct AvxEncoderConfig *global, int *argc,
if (global->pass < 1 || global->pass > 2)
die("Error: Invalid pass selected (%d)\n", global->pass);
} else if (arg_match(&arg, &input_chroma_sample_position, argi)) {
global->csp = arg_parse_enum(&arg);
/* Flag is used by later code as well, preserve it. */
argj++;
} else if (arg_match(&arg, &usage, argi))
global->usage = arg_parse_uint(&arg);
else if (arg_match(&arg, &good_dl, argi))
@ -963,7 +1124,8 @@ static void parse_global_config(struct AvxEncoderConfig *global, int *argc,
}
}
static void open_input_file(struct AvxInputContext *input) {
static void open_input_file(struct AvxInputContext *input,
aom_chroma_sample_position_t csp) {
/* Parse certain options from the input file, if possible */
input->file = strcmp(input->filename, "-") ? fopen(input->filename, "rb")
: set_binary_mode(stdin);
@ -989,7 +1151,7 @@ static void open_input_file(struct AvxInputContext *input) {
input->detect.position = 0;
if (input->detect.buf_read == 4 && file_is_y4m(input->detect.buf)) {
if (y4m_input_open(&input->y4m, input->file, input->detect.buf, 4,
if (y4m_input_open(&input->y4m, input->file, input->detect.buf, 4, csp,
input->only_i420) >= 0) {
input->file_type = FILE_TYPE_Y4M;
input->width = input->y4m.pic_w;
@ -1195,6 +1357,7 @@ static int parse_stream_params(struct AvxEncoderConfig *global,
config->cfg.g_lag_in_frames = arg_parse_uint(&arg);
} else if (arg_match(&arg, &large_scale_tile, argi)) {
config->cfg.large_scale_tile = arg_parse_uint(&arg);
if (config->cfg.large_scale_tile) global->codec = get_aom_lst_encoder();
} else if (arg_match(&arg, &monochrome, argi)) {
config->cfg.monochrome = 1;
} else if (arg_match(&arg, &full_still_picture_hdr, argi)) {
@ -1306,13 +1469,11 @@ static void validate_stream_config(const struct stream_state *stream,
" and --height (-h)",
stream->index);
// Check that the codec bit depth is greater than the input bit depth.
if (stream->config.cfg.g_input_bit_depth >
(unsigned int)stream->config.cfg.g_bit_depth) {
fatal("Stream %d: codec bit depth (%d) less than input bit depth (%d)",
stream->index, (int)stream->config.cfg.g_bit_depth,
stream->config.cfg.g_input_bit_depth);
}
/* Even if bit depth is set on the command line flag to be lower,
* it is upgraded to at least match the input bit depth.
*/
assert(stream->config.cfg.g_input_bit_depth <=
(unsigned int)stream->config.cfg.g_bit_depth);
for (streami = stream; streami; streami = streami->next) {
/* All streams require output files */
@ -1378,6 +1539,7 @@ static const char *image_format_to_string(aom_img_fmt_t f) {
case AOM_IMG_FMT_I422: return "I422";
case AOM_IMG_FMT_I444: return "I444";
case AOM_IMG_FMT_YV12: return "YV12";
case AOM_IMG_FMT_YV1216: return "YV1216";
case AOM_IMG_FMT_I42016: return "I42016";
case AOM_IMG_FMT_I42216: return "I42216";
case AOM_IMG_FMT_I44416: return "I44416";
@ -1463,8 +1625,11 @@ static void open_output_file(struct stream_state *stream,
#if CONFIG_WEBM_IO
if (stream->config.write_webm) {
stream->webm_ctx.stream = stream->file;
write_webm_file_header(&stream->webm_ctx, cfg, stream->config.stereo_fmt,
global->codec->fourcc, pixel_aspect_ratio);
if (write_webm_file_header(&stream->webm_ctx, &stream->encoder, cfg,
stream->config.stereo_fmt, global->codec->fourcc,
pixel_aspect_ratio) != 0) {
fatal("WebM writer initialization failed.");
}
}
#else
(void)pixel_aspect_ratio;
@ -1483,7 +1648,9 @@ static void close_output_file(struct stream_state *stream,
#if CONFIG_WEBM_IO
if (stream->config.write_webm) {
write_webm_file_footer(&stream->webm_ctx);
if (write_webm_file_footer(&stream->webm_ctx) != 0) {
fatal("WebM writer finalization failed.");
}
}
#endif
@ -1699,16 +1866,16 @@ static void get_cx_data(struct stream_state *stream,
switch (pkt->kind) {
case AOM_CODEC_CX_FRAME_PKT:
if (!(pkt->data.frame.flags & AOM_FRAME_IS_FRAGMENT)) {
stream->frames_out++;
}
++stream->frames_out;
if (!global->quiet)
fprintf(stderr, " %6luF", (unsigned long)pkt->data.frame.sz);
update_rate_histogram(stream->rate_hist, cfg, pkt);
#if CONFIG_WEBM_IO
if (stream->config.write_webm) {
write_webm_block(&stream->webm_ctx, cfg, pkt);
if (write_webm_block(&stream->webm_ctx, cfg, pkt) != 0) {
fatal("WebM writer failed.");
}
}
#endif
if (!stream->config.write_webm) {
@ -1721,12 +1888,10 @@ static void get_cx_data(struct stream_state *stream,
} else {
fsize += pkt->data.frame.sz;
if (!(pkt->data.frame.flags & AOM_FRAME_IS_FRAGMENT)) {
const FileOffset currpos = ftello(stream->file);
fseeko(stream->file, ivf_header_pos, SEEK_SET);
ivf_write_frame_size(stream->file, fsize);
fseeko(stream->file, currpos, SEEK_SET);
}
const FileOffset currpos = ftello(stream->file);
fseeko(stream->file, ivf_header_pos, SEEK_SET);
ivf_write_frame_size(stream->file, fsize);
fseeko(stream->file, currpos, SEEK_SET);
}
}
@ -1913,7 +2078,7 @@ int main(int argc, const char **argv_) {
* codec.
*/
argv = argv_dup(argc - 1, argv_ + 1);
parse_global_config(&global, &argc, &argv);
parse_global_config(&global, argc, &argv);
#if CONFIG_FILEOPTIONS
if (argc < 2) usage_exit();
@ -1950,6 +2115,10 @@ int main(int argc, const char **argv_) {
FOREACH_STREAM(stream, streams) {
check_encoder_config(global.disable_warning_prompt, &global,
&stream->config.cfg);
// If large_scale_tile = 1, only support to output to ivf format.
if (stream->config.cfg.large_scale_tile && !stream->config.write_ivf)
die("only support ivf output format while large-scale-tile=1\n");
}
/* Handle non-option arguments */
@ -1969,7 +2138,7 @@ int main(int argc, const char **argv_) {
int64_t average_rate = -1;
int64_t lagged_count = 0;
open_input_file(&input);
open_input_file(&input, global.csp);
/* If the input file doesn't specify its w/h (raw files), try to get
* the data from the first stream's configuration.
@ -2069,6 +2238,18 @@ int main(int argc, const char **argv_) {
default: break;
}
}
/* Automatically set the codec bit depth to match the input bit depth.
* Upgrade the profile if required. */
if (stream->config.cfg.g_input_bit_depth >
(unsigned int)stream->config.cfg.g_bit_depth) {
stream->config.cfg.g_bit_depth = stream->config.cfg.g_input_bit_depth;
if (!global.quiet) {
fprintf(stderr,
"Warning: automatically updating bit depth to %d to "
"match input format.\n",
stream->config.cfg.g_input_bit_depth);
}
}
if (stream->config.cfg.g_bit_depth > 10) {
switch (stream->config.cfg.g_profile) {
case 0:
@ -2153,10 +2334,11 @@ int main(int argc, const char **argv_) {
}
FOREACH_STREAM(stream, streams) { setup_pass(stream, &global, pass); }
FOREACH_STREAM(stream, streams) { initialize_encoder(stream, &global); }
FOREACH_STREAM(stream, streams) {
open_output_file(stream, &global, &input.pixel_aspect_ratio);
}
FOREACH_STREAM(stream, streams) { initialize_encoder(stream, &global); }
if (strcmp(global.codec->name, "av1") == 0 ||
strcmp(global.codec->name, "av1") == 0) {
// Check to see if at least one stream uses 16 bit internal.

1
third_party/aom/apps/aomenc.h vendored
View file

@ -53,6 +53,7 @@ struct AvxEncoderConfig {
int disable_warnings;
int disable_warning_prompt;
int experimental_bitstream;
aom_chroma_sample_position_t csp;
};
#ifdef __cplusplus

21
third_party/aom/av1/av1.cmake vendored
View file

@ -100,7 +100,6 @@ list(APPEND AOM_AV1_DECODER_SOURCES
"${AOM_ROOT}/av1/decoder/decodetxb.h"
"${AOM_ROOT}/av1/decoder/detokenize.c"
"${AOM_ROOT}/av1/decoder/detokenize.h"
"${AOM_ROOT}/av1/decoder/dthread.c"
"${AOM_ROOT}/av1/decoder/dthread.h"
"${AOM_ROOT}/av1/decoder/obu.h"
"${AOM_ROOT}/av1/decoder/obu.c")
@ -117,6 +116,8 @@ list(APPEND AOM_AV1_ENCODER_SOURCES
"${AOM_ROOT}/av1/encoder/av1_fwd_txfm1d.h"
"${AOM_ROOT}/av1/encoder/av1_fwd_txfm1d_cfg.h"
"${AOM_ROOT}/av1/encoder/av1_fwd_txfm2d.c"
"${AOM_ROOT}/av1/encoder/av1_multi_thread.c"
"${AOM_ROOT}/av1/encoder/av1_multi_thread.h"
"${AOM_ROOT}/av1/encoder/av1_quantize.c"
"${AOM_ROOT}/av1/encoder/av1_quantize.h"
"${AOM_ROOT}/av1/encoder/bitstream.c"
@ -136,6 +137,8 @@ list(APPEND AOM_AV1_ENCODER_SOURCES
"${AOM_ROOT}/av1/encoder/encodemb.h"
"${AOM_ROOT}/av1/encoder/encodemv.c"
"${AOM_ROOT}/av1/encoder/encodemv.h"
"${AOM_ROOT}/av1/encoder/encode_strategy.c"
"${AOM_ROOT}/av1/encoder/encode_strategy.h"
"${AOM_ROOT}/av1/encoder/encoder.c"
"${AOM_ROOT}/av1/encoder/encoder.h"
"${AOM_ROOT}/av1/encoder/encodetxb.c"
@ -188,6 +191,8 @@ list(APPEND AOM_AV1_ENCODER_SOURCES
"${AOM_ROOT}/av1/encoder/temporal_filter.h"
"${AOM_ROOT}/av1/encoder/tokenize.c"
"${AOM_ROOT}/av1/encoder/tokenize.h"
"${AOM_ROOT}/av1/encoder/tpl_model.c"
"${AOM_ROOT}/av1/encoder/tpl_model.h"
"${AOM_ROOT}/av1/encoder/wedge_utils.c"
"${AOM_ROOT}/third_party/fastfeat/fast.c"
"${AOM_ROOT}/third_party/fastfeat/fast.h"
@ -263,6 +268,8 @@ list(APPEND AOM_AV1_ENCODER_INTRIN_SSE2
"${AOM_ROOT}/av1/encoder/x86/highbd_block_error_intrin_sse2.c"
"${AOM_ROOT}/av1/encoder/x86/wedge_utils_sse2.c")
list(APPEND AOM_AV1_ENCODER_INTRIN_SSE3 "${AOM_ROOT}/av1/encoder/x86/ml_sse3.c")
list(APPEND AOM_AV1_ENCODER_ASM_SSSE3_X86_64
"${AOM_ROOT}/av1/encoder/x86/av1_quantize_ssse3_x86_64.asm")
@ -273,16 +280,20 @@ list(APPEND AOM_AV1_ENCODER_INTRIN_SSE4_1
"${AOM_ROOT}/av1/encoder/x86/corner_match_sse4.c"
"${AOM_ROOT}/av1/encoder/x86/encodetxb_sse4.c"
"${AOM_ROOT}/av1/encoder/x86/highbd_fwd_txfm_sse4.c"
"${AOM_ROOT}/av1/encoder/x86/rdopt_sse4.c"
"${AOM_ROOT}/av1/encoder/x86/pickrst_sse4.c")
list(APPEND AOM_AV1_ENCODER_INTRIN_AVX2
"${AOM_ROOT}/av1/encoder/x86/av1_quantize_avx2.c"
"${AOM_ROOT}/av1/encoder/x86/av1_highbd_quantize_avx2.c"
"${AOM_ROOT}/av1/encoder/x86/corner_match_avx2.c"
"${AOM_ROOT}/av1/encoder/x86/error_intrin_avx2.c"
"${AOM_ROOT}/av1/encoder/x86/av1_fwd_txfm_avx2.h"
"${AOM_ROOT}/av1/encoder/x86/av1_fwd_txfm2d_avx2.c"
"${AOM_ROOT}/av1/encoder/x86/highbd_fwd_txfm_avx2.c"
"${AOM_ROOT}/av1/encoder/x86/wedge_utils_avx2.c"
"${AOM_ROOT}/av1/encoder/x86/encodetxb_avx2.c"
"${AOM_ROOT}/av1/encoder/x86/rdopt_avx2.c"
"${AOM_ROOT}/av1/encoder/x86/pickrst_avx2.c")
list(APPEND AOM_AV1_ENCODER_INTRIN_NEON
@ -379,6 +390,14 @@ function(setup_av1_targets)
endif()
endif()
if(HAVE_SSE3)
require_compiler_flag_nomsvc("-msse3" NO)
if(CONFIG_AV1_ENCODER)
add_intrinsics_object_library("-msse3" "sse3" "aom_av1_encoder"
"AOM_AV1_ENCODER_INTRIN_SSE3" "aom")
endif()
endif()
if(HAVE_SSSE3)
require_compiler_flag_nomsvc("-mssse3" NO)
add_intrinsics_object_library("-mssse3" "ssse3" "aom_av1_common"

Some files were not shown because too many files have changed in this diff Show more